From 827f4b027c8feeef09d73608bd7a05d21cf55703 Mon Sep 17 00:00:00 2001 From: Jeff Tratner Date: Fri, 19 Jan 2018 16:07:17 -0800 Subject: [PATCH] Add makedirs_p to cross-compatible API (#60) This is a no-op directory methods for swift paths Closes #37 --- docs/release_notes.rst | 7 +++++-- stor/__init__.py | 1 + stor/base.py | 5 +++++ stor/cli.py | 11 ++++++++++- stor/obs.py | 6 ++++++ stor/s3.py | 5 +++++ stor/swift.py | 12 ++++++++++++ stor/tests/test_cli.py | 6 ++++++ stor/tests/test_s3.py | 5 +++++ stor/tests/test_swift.py | 13 +++++++++++++ 10 files changed, 68 insertions(+), 3 deletions(-) diff --git a/docs/release_notes.rst b/docs/release_notes.rst index 0462e62a..c83976be 100644 --- a/docs/release_notes.rst +++ b/docs/release_notes.rst @@ -1,14 +1,17 @@ Release Notes ============= -v1.5.3 +v1.6.0 ------ -* Hoist ``stor.utils.is_obs_path`` --> ``stor.is_obs_path`` +* Add ``to_url()`` method on Path and ``url`` cli method to translate swift and s3 paths to HTTP paths. +* Add ``stor.makedirs_p(path, mode=0o777)`` to cross-compatible API. This does + nothing on OBS-paths (just there for convenience). v1.5.2 ------ +* Hoist ``stor.utils.is_obs_path`` --> ``stor.is_obs_path`` * Build universal wheels for both Python 2 and Python 3. (no actual code changes) diff --git a/stor/__init__.py b/stor/__init__.py index 050cfc11..63f8ebaa 100644 --- a/stor/__init__.py +++ b/stor/__init__.py @@ -75,6 +75,7 @@ def wrapper(path, *args, **kwargs): islink = _delegate_to_path('islink') ismount = _delegate_to_path('ismount') getsize = _delegate_to_path('getsize') +makedirs_p = _delegate_to_path('makedirs_p') remove = _delegate_to_path('remove') rmtree = _delegate_to_path('rmtree') walkfiles = _delegate_to_path('walkfiles') diff --git a/stor/base.py b/stor/base.py index 6768091a..bae3dca5 100644 --- a/stor/base.py +++ b/stor/base.py @@ -342,6 +342,11 @@ def rmtree(self): See shutil.rmtree""" raise NotImplementedError + def makedirs_p(self, mode=0o777): + """ Like :func:`os.makedirs`, but does not raise an exception if the + directory already exists. """ + raise NotImplementedError + def walkfiles(self, pattern=None): """Iterate over files recursively. diff --git a/stor/cli.py b/stor/cli.py index 86f1d61a..5f9c606f 100644 --- a/stor/cli.py +++ b/stor/cli.py @@ -92,7 +92,7 @@ from stor import Path from stor import utils -PRINT_CMDS = ('list', 'listdir', 'ls', 'cat', 'pwd', 'walkfiles') +PRINT_CMDS = ('list', 'listdir', 'ls', 'cat', 'pwd', 'walkfiles', 'url') SERVICES = ('s3', 'swift') ENV_FILE = os.path.expanduser('~/.stor-cli.env') @@ -268,6 +268,12 @@ def get_path(pth, mode=None): return prefix / path_part.split(rel_part, depth)[depth].lstrip('/') +def _to_url(path): + if stor.is_filesystem_path(path): + raise ValueError('must be swift or s3 path') + return stor.Path(path).to_url() + + def create_parser(): parser = argparse.ArgumentParser(description='A command line interface for stor.') @@ -370,6 +376,9 @@ def create_parser(): ' will be cleared if SERVICE is omitted.' % clear_msg) parser_clear.add_argument('service', nargs='?', type=str, metavar='SERVICE') parser_clear.set_defaults(func=_clear_env) + url_parser = subparsers.add_parser('url', help='generate URL for swift or s3 path') + url_parser.add_argument('path') + url_parser.set_defaults(func=_to_url) return parser diff --git a/stor/obs.py b/stor/obs.py index 51b9a3ad..5d79bb13 100644 --- a/stor/obs.py +++ b/stor/obs.py @@ -186,6 +186,12 @@ def islink(self): def ismount(self): return True + # NOTE: we only have makedirs_p() because the other mkdir/mkdir_p/makedirs methods are expected + # to raise errors if directories exist or intermediate directories don't exist. + def makedirs_p(self, mode=0o777): + """No-op (no directories on OBS)""" + return + def getsize(self): """Returns size, in bytes of path.""" raise NotImplementedError diff --git a/stor/s3.py b/stor/s3.py index af890a39..7bb37396 100644 --- a/stor/s3.py +++ b/stor/s3.py @@ -757,3 +757,8 @@ def upload(self, source, condition=None, use_manifest=False, headers=None, **kwa utils.check_condition(condition, [r['dest'] for r in uploaded['completed']]) return uploaded + + def to_url(self): + """Returns HTTPS path for object (virtual host-style)""" + return u'https://{bucket}.s3.amazonaws.com/{key}'.format(bucket=self.bucket, + key=self.resource) diff --git a/stor/swift.py b/stor/swift.py index 51c4aacd..b20df1f1 100644 --- a/stor/swift.py +++ b/stor/swift.py @@ -1563,3 +1563,15 @@ def walkfiles(self, pattern=None): for f in self.list(ignore_dir_markers=True): if pattern is None or f.fnmatch(pattern): yield f + + def to_url(self): + """Returns path for object (based on storage URL) + + Returns: + str: the http path + Raises: + UnauthorizedError: if we cannot authenticate to get a storage URL""" + storage_url = _get_or_create_auth_credentials(self.tenant)['os_storage_url'] + return u'{storage_url}/{container}/{resource}'.format(storage_url=storage_url, + container=self.container, + resource=self.resource) diff --git a/stor/tests/test_cli.py b/stor/tests/test_cli.py index 9228f7c7..3d165692 100644 --- a/stor/tests/test_cli.py +++ b/stor/tests/test_cli.py @@ -455,6 +455,12 @@ def test_walkfiles_no_pattern(self, mock_walkfiles): mock_walkfiles.assert_called_once_with(PosixPath('.')) +class TestToUrl(BaseCliTest): + def test_url(self): + self.parse_args('stor url s3://test/file') + self.assertEquals(sys.stdout.getvalue(), 'https://test.s3.amazonaws.com/file\n') + + class TestCat(BaseCliTest): @mock.patch.object(S3Path, 'read_object', autospec=True) def test_cat_s3(self, mock_read): diff --git a/stor/tests/test_s3.py b/stor/tests/test_s3.py index 59cf303c..1ebdeb68 100644 --- a/stor/tests/test_s3.py +++ b/stor/tests/test_s3.py @@ -50,6 +50,11 @@ def test_basename(self): p = Path('s3://bucket/path/to/resource') self.assertEquals(p.basename(), 'resource') + def test_to_url(self): + p = Path('s3://mybucket/path/to/resource') + self.assertEquals(p.to_url(), + 'https://mybucket.s3.amazonaws.com/path/to/resource') + class TestValidation(unittest.TestCase): def test_new_failed(self): diff --git a/stor/tests/test_swift.py b/stor/tests/test_swift.py index e00dd40b..01b251a0 100644 --- a/stor/tests/test_swift.py +++ b/stor/tests/test_swift.py @@ -83,6 +83,15 @@ def test_basename(self): p = Path('swift://tenant/container/path/to/resource') self.assertEquals(p.basename(), 'resource') + def test_to_url(self): + p = Path('swift://tenant/container/path/to/resource') + with mock.patch.object('stor.swift._get_or_create_auth_credentials', + # storage url may be an opaque value... ensure we roll with that + {'os_storage_url': 'https://example.com/v1.0/othertenant', + 'os_auth_token': 'sometoken'}): + self.assertEquals(p.to_url(), + 'https://example.com/v1.0/othertenant/container/path/to/resource') + class TestManifestUtilities(unittest.TestCase): def test_generate_save_and_read_manifest(self): @@ -231,6 +240,10 @@ def setUp(self): super(TestSwiftFile, self).setUp() settings.update({'swift': {'num_retries': 5}}) + def test_makedirs_p_does_nothing(self): + # dumb test... but why not? + SwiftPath('swift://tenant/container/obj').makedirs_p() + def test_invalid_buffer_mode(self): swift_f = SwiftPath('swift://tenant/container/obj').open() swift_f.mode = 'invalid'