Add makedirs_p to cross-compatible API (counsyl#60)

This is a no-op directory methods for swift paths

Closes counsyl#37

docs/release_notes.rst

@@ -1,14 +1,17 @@
 Release Notes
 =============
 
-v1.5.3
+v1.6.0
 ------
 
-* Hoist ``stor.utils.is_obs_path`` --> ``stor.is_obs_path``
+* Add ``to_url()`` method on Path and ``url`` cli method to translate swift and s3 paths to HTTP paths.
+* Add ``stor.makedirs_p(path, mode=0o777)`` to cross-compatible API. This does
+  nothing on OBS-paths (just there for convenience).
 
 v1.5.2
 ------
 
+* Hoist ``stor.utils.is_obs_path`` --> ``stor.is_obs_path``
 * Build universal wheels for both Python 2 and Python 3.
   (no actual code changes)
 

stor/__init__.py

@@ -75,6 +75,7 @@ def wrapper(path, *args, **kwargs):
 islink = _delegate_to_path('islink')
 ismount = _delegate_to_path('ismount')
 getsize = _delegate_to_path('getsize')
+makedirs_p = _delegate_to_path('makedirs_p')
 remove = _delegate_to_path('remove')
 rmtree = _delegate_to_path('rmtree')
 walkfiles = _delegate_to_path('walkfiles')

stor/base.py

@@ -342,6 +342,11 @@ def rmtree(self):
         See shutil.rmtree"""
         raise NotImplementedError
 
+    def makedirs_p(self, mode=0o777):
+        """ Like :func:`os.makedirs`, but does not raise an exception if the
+        directory already exists. """
+        raise NotImplementedError
+
     def walkfiles(self, pattern=None):
         """Iterate over files recursively.
 

stor/cli.py

@@ -92,7 +92,7 @@
 from stor import Path
 from stor import utils
 
-PRINT_CMDS = ('list', 'listdir', 'ls', 'cat', 'pwd', 'walkfiles')
+PRINT_CMDS = ('list', 'listdir', 'ls', 'cat', 'pwd', 'walkfiles', 'url')
 SERVICES = ('s3', 'swift')
 
 ENV_FILE = os.path.expanduser('~/.stor-cli.env')
@@ -268,6 +268,12 @@ def get_path(pth, mode=None):
     return prefix / path_part.split(rel_part, depth)[depth].lstrip('/')
 
 
+def _to_url(path):
+    if stor.is_filesystem_path(path):
+        raise ValueError('must be swift or s3 path')
+    return stor.Path(path).to_url()
+
+
 def create_parser():
     parser = argparse.ArgumentParser(description='A command line interface for stor.')
 
@@ -370,6 +376,9 @@ def create_parser():
                                          ' will be cleared if SERVICE is omitted.' % clear_msg)
     parser_clear.add_argument('service', nargs='?', type=str, metavar='SERVICE')
     parser_clear.set_defaults(func=_clear_env)
+    url_parser = subparsers.add_parser('url', help='generate URL for swift or s3 path')
+    url_parser.add_argument('path')
+    url_parser.set_defaults(func=_to_url)
 
     return parser
 

stor/obs.py

@@ -186,6 +186,12 @@ def islink(self):
     def ismount(self):
         return True
 
+    # NOTE: we only have makedirs_p() because the other mkdir/mkdir_p/makedirs methods are expected
+    # to raise errors if directories exist or intermediate directories don't exist.
+    def makedirs_p(self, mode=0o777):
+        """No-op (no directories on OBS)"""
+        return
+
     def getsize(self):
         """Returns size, in bytes of path."""
         raise NotImplementedError

stor/s3.py

@@ -757,3 +757,8 @@ def upload(self, source, condition=None, use_manifest=False, headers=None, **kwa
 
         utils.check_condition(condition, [r['dest'] for r in uploaded['completed']])
         return uploaded
+
+    def to_url(self):
+        """Returns HTTPS path for object (virtual host-style)"""
+        return u'https://{bucket}.s3.amazonaws.com/{key}'.format(bucket=self.bucket,
+                                                                 key=self.resource)

stor/swift.py

@@ -1563,3 +1563,15 @@ def walkfiles(self, pattern=None):
             for f in self.list(ignore_dir_markers=True):
                 if pattern is None or f.fnmatch(pattern):
                     yield f
+
+    def to_url(self):
+        """Returns path for object (based on storage URL)
+
+        Returns:
+            str: the http path
+        Raises:
+            UnauthorizedError: if we cannot authenticate to get a storage URL"""
+        storage_url = _get_or_create_auth_credentials(self.tenant)['os_storage_url']
+        return u'{storage_url}/{container}/{resource}'.format(storage_url=storage_url,
+                                                              container=self.container,
+                                                              resource=self.resource)

stor/tests/test_cli.py

@@ -455,6 +455,12 @@ def test_walkfiles_no_pattern(self, mock_walkfiles):
         mock_walkfiles.assert_called_once_with(PosixPath('.'))
 
 
+class TestToUrl(BaseCliTest):
+    def test_url(self):
+        self.parse_args('stor url s3://test/file')
+        self.assertEquals(sys.stdout.getvalue(), 'https://test.s3.amazonaws.com/file\n')
+
+
 class TestCat(BaseCliTest):
     @mock.patch.object(S3Path, 'read_object', autospec=True)
     def test_cat_s3(self, mock_read):

stor/tests/test_s3.py

@@ -50,6 +50,11 @@ def test_basename(self):
         p = Path('s3://bucket/path/to/resource')
         self.assertEquals(p.basename(), 'resource')
 
+    def test_to_url(self):
+        p = Path('s3://mybucket/path/to/resource')
+        self.assertEquals(p.to_url(),
+                          'https://mybucket.s3.amazonaws.com/path/to/resource')
+
 
 class TestValidation(unittest.TestCase):
     def test_new_failed(self):

stor/tests/test_swift.py

@@ -83,6 +83,15 @@ def test_basename(self):
         p = Path('swift://tenant/container/path/to/resource')
         self.assertEquals(p.basename(), 'resource')
 
+    def test_to_url(self):
+        p = Path('swift://tenant/container/path/to/resource')
+        with mock.patch.object('stor.swift._get_or_create_auth_credentials',
+                               # storage url may be an opaque value... ensure we roll with that
+                               {'os_storage_url': 'https://example.com/v1.0/othertenant',
+                                'os_auth_token': 'sometoken'}):
+            self.assertEquals(p.to_url(),
+                              'https://example.com/v1.0/othertenant/container/path/to/resource')
+
 
 class TestManifestUtilities(unittest.TestCase):
     def test_generate_save_and_read_manifest(self):
@@ -231,6 +240,10 @@ def setUp(self):
         super(TestSwiftFile, self).setUp()
         settings.update({'swift': {'num_retries': 5}})
 
+    def test_makedirs_p_does_nothing(self):
+        # dumb test... but why not?
+        SwiftPath('swift://tenant/container/obj').makedirs_p()
+
     def test_invalid_buffer_mode(self):
         swift_f = SwiftPath('swift://tenant/container/obj').open()
         swift_f.mode = 'invalid'