From 291eb4adbcccd42e259ba8842fd039aabf0aaffb Mon Sep 17 00:00:00 2001 From: John Tordoff Date: Thu, 12 Oct 2017 15:39:44 -0400 Subject: [PATCH 01/13] Expand cloudfiles provider to preform all major functions, handle revisions etc. --- tests/providers/cloudfiles/fixtures.py | 243 ++++++++ .../cloudfiles/fixtures/fixtures.json | 181 ++++++ tests/providers/cloudfiles/test_metadata.py | 48 +- tests/providers/cloudfiles/test_provider.py | 568 +++++++----------- waterbutler/providers/cloudfiles/metadata.py | 35 +- waterbutler/providers/cloudfiles/provider.py | 181 ++++-- 6 files changed, 818 insertions(+), 438 deletions(-) create mode 100644 tests/providers/cloudfiles/fixtures.py create mode 100644 tests/providers/cloudfiles/fixtures/fixtures.json diff --git a/tests/providers/cloudfiles/fixtures.py b/tests/providers/cloudfiles/fixtures.py new file mode 100644 index 000000000..c7704198d --- /dev/null +++ b/tests/providers/cloudfiles/fixtures.py @@ -0,0 +1,243 @@ +import os +import io +import time +import json +import pytest +import aiohttp +import aiohttpretty + +from unittest import mock + + +from waterbutler.core import streams +from waterbutler.providers.cloudfiles import CloudFilesProvider + + + +@pytest.fixture +def auth(): + return { + 'name': 'cat', + 'email': 'cat@cat.com', + } + + +@pytest.fixture +def credentials(): + return { + 'username': 'prince', + 'token': 'revolutionary', + 'region': 'iad', + } + + +@pytest.fixture +def settings(): + return {'container': 'purple rain'} + + +@pytest.fixture +def provider(auth, credentials, settings): + return CloudFilesProvider(auth, credentials, settings) + + + +@pytest.fixture +def token(auth_json): + return auth_json['access']['token']['id'] + + +@pytest.fixture +def endpoint(auth_json): + return auth_json['access']['serviceCatalog'][0]['endpoints'][0]['publicURL'] + + +@pytest.fixture +def temp_url_key(): + return 'temporary beret' + + +@pytest.fixture +def mock_auth(auth_json): + aiohttpretty.register_json_uri( + 'POST', + settings.AUTH_URL, + body=auth_json, + ) + + +@pytest.fixture +def mock_temp_key(endpoint, temp_url_key): + aiohttpretty.register_uri( + 'HEAD', + endpoint, + status=204, + headers={'X-Account-Meta-Temp-URL-Key': temp_url_key}, + ) + + +@pytest.fixture +def mock_time(monkeypatch): + mock_time = mock.Mock() + mock_time.return_value = 10 + monkeypatch.setattr(time, 'time', mock_time) + + +@pytest.fixture +def connected_provider(provider, token, endpoint, temp_url_key, mock_time): + provider.token = token + provider.endpoint = endpoint + provider.temp_url_key = temp_url_key.encode() + return provider + + +@pytest.fixture +def file_content(): + return b'sleepy' + + +@pytest.fixture +def file_like(file_content): + return io.BytesIO(file_content) + + +@pytest.fixture +def file_stream(file_like): + return streams.FileStreamReader(file_like) + + + +@pytest.fixture +def folder_root_empty(): + return [] + +@pytest.fixture +def container_header_metadata_with_verision_location(): + with open(os.path.join(os.path.dirname(__file__), 'fixtures/fixtures.json'), 'r') as fp: + return json.load(fp)['container_header_metadata_with_verision_location'] + + +@pytest.fixture +def container_header_metadata_without_verision_location(): + with open(os.path.join(os.path.dirname(__file__), 'fixtures/fixtures.json'), 'r') as fp: + return json.load(fp)['container_header_metadata_without_verision_location'] + +@pytest.fixture +def file_metadata(): + with open(os.path.join(os.path.dirname(__file__), 'fixtures/fixtures.json'), 'r') as fp: + return json.load(fp)['file_metadata'] + +@pytest.fixture +def folder_root(): + with open(os.path.join(os.path.dirname(__file__), 'fixtures/fixtures.json'), 'r') as fp: + return json.load(fp)['folder_root'] + + +@pytest.fixture +def folder_root_level1_level2(): + with open(os.path.join(os.path.dirname(__file__), 'fixtures/fixtures.json'), 'r') as fp: + return json.load(fp)['folder_root_level1_level2'] + + +@pytest.fixture +def folder_root_level1(): + with open(os.path.join(os.path.dirname(__file__), 'fixtures/fixtures.json'), 'r') as fp: + return json.load(fp)['folder_root_level1'] + + +@pytest.fixture +def file_header_metadata(): + with open(os.path.join(os.path.dirname(__file__), 'fixtures/fixtures.json'), 'r') as fp: + return json.load(fp)['file_header_metadata'] + + +@pytest.fixture +def auth_json(): + with open(os.path.join(os.path.dirname(__file__), 'fixtures/fixtures.json'), 'r') as fp: + return json.load(fp)['auth_json'] + + +@pytest.fixture +def folder_root(): + with open(os.path.join(os.path.dirname(__file__), 'fixtures/fixtures.json'), 'r') as fp: + return json.load(fp)['folder_root'] + +@pytest.fixture +def revision_list(): + with open(os.path.join(os.path.dirname(__file__), 'fixtures/fixtures.json'), 'r') as fp: + return json.load(fp)['revision_list'] + +@pytest.fixture +def file_root_level1_level2_file2_txt(): + return aiohttp.multidict.CIMultiDict([ + ('ORIGIN', 'https://mycloud.rackspace.com'), + ('CONTENT-LENGTH', '216945'), + ('ACCEPT-RANGES', 'bytes'), + ('LAST-MODIFIED', 'Mon, 22 Dec 2014 19:01:02 GMT'), + ('ETAG', '44325d4f13b09f3769ede09d7c20a82c'), + ('X-TIMESTAMP', '1419274861.04433'), + ('CONTENT-TYPE', 'text/plain'), + ('X-TRANS-ID', 'tx836375d817a34b558756a-0054987deeiad3'), + ('DATE', 'Mon, 22 Dec 2014 20:24:14 GMT') + ]) + + +@pytest.fixture +def folder_root_level1_empty(): + return aiohttp.multidict.CIMultiDict([ + ('ORIGIN', 'https://mycloud.rackspace.com'), + ('CONTENT-LENGTH', '0'), + ('ACCEPT-RANGES', 'bytes'), + ('LAST-MODIFIED', 'Mon, 22 Dec 2014 18:58:56 GMT'), + ('ETAG', 'd41d8cd98f00b204e9800998ecf8427e'), + ('X-TIMESTAMP', '1419274735.03160'), + ('CONTENT-TYPE', 'application/directory'), + ('X-TRANS-ID', 'txd78273e328fc4ba3a98e3-0054987eeeiad3'), + ('DATE', 'Mon, 22 Dec 2014 20:28:30 GMT') + ]) + + +@pytest.fixture +def file_root_similar(): + return aiohttp.multidict.CIMultiDict([ + ('ORIGIN', 'https://mycloud.rackspace.com'), + ('CONTENT-LENGTH', '190'), + ('ACCEPT-RANGES', 'bytes'), + ('LAST-MODIFIED', 'Fri, 19 Dec 2014 23:22:24 GMT'), + ('ETAG', 'edfa12d00b779b4b37b81fe5b61b2b3f'), + ('X-TIMESTAMP', '1419031343.23224'), + ('CONTENT-TYPE', 'application/x-www-form-urlencoded;charset=utf-8'), + ('X-TRANS-ID', 'tx7cfeef941f244807aec37-005498754diad3'), + ('DATE', 'Mon, 22 Dec 2014 19:47:25 GMT') + ]) + + +@pytest.fixture +def file_root_similar_name(): + return aiohttp.multidict.CIMultiDict([ + ('ORIGIN', 'https://mycloud.rackspace.com'), + ('CONTENT-LENGTH', '190'), + ('ACCEPT-RANGES', 'bytes'), + ('LAST-MODIFIED', 'Mon, 22 Dec 2014 19:07:12 GMT'), + ('ETAG', 'edfa12d00b779b4b37b81fe5b61b2b3f'), + ('X-TIMESTAMP', '1419275231.66160'), + ('CONTENT-TYPE', 'application/x-www-form-urlencoded;charset=utf-8'), + ('X-TRANS-ID', 'tx438cbb32b5344d63b267c-0054987f3biad3'), + ('DATE', 'Mon, 22 Dec 2014 20:29:47 GMT') + ]) + + + +@pytest.fixture +def file_header_metadata_txt(): + return aiohttp.multidict.CIMultiDict([ + ('ORIGIN', 'https://mycloud.rackspace.com'), + ('CONTENT-LENGTH', '216945'), + ('ACCEPT-RANGES', 'bytes'), + ('LAST-MODIFIED', 'Mon, 22 Dec 2014 19:01:02 GMT'), + ('ETAG', '44325d4f13b09f3769ede09d7c20a82c'), + ('X-TIMESTAMP', '1419274861.04433'), + ('CONTENT-TYPE', 'text/plain'), + ('X-TRANS-ID', 'tx836375d817a34b558756a-0054987deeiad3'), + ('DATE', 'Mon, 22 Dec 2014 20:24:14 GMT') + ]) diff --git a/tests/providers/cloudfiles/fixtures/fixtures.json b/tests/providers/cloudfiles/fixtures/fixtures.json new file mode 100644 index 000000000..2e313777b --- /dev/null +++ b/tests/providers/cloudfiles/fixtures/fixtures.json @@ -0,0 +1,181 @@ +{ + "container_header_metadata_with_verision_location":{ + "ACCEPT-RANGES":"bytes", + "CONTENT-LENGTH":"0", + "CONTENT-TYPE":"application/json; charset=utf-8", + "DATE":"Thu, 12 Oct 2017 16:13:04 GMT", + "X-CONTAINER-BYTES-USED":"90", + "X-CONTAINER-META-ACCESS-CONTROL-EXPOSE-HEADERS":"etag location x-timestamp x-trans-id", + "X-CONTAINER-META-ACCESS-LOG-DELIVERY":"false", + "X-CONTAINER-OBJECT-COUNT":"2", + "X-STORAGE-POLICY":"Policy-0", + "X-TIMESTAMP":"1506696708.03681", + "X-TRANS-ID":"txffaa4b0a06984dd0bbf27-0059df9490iad3", + "X-VERSIONS-LOCATION":"versions-container" + }, + "container_header_metadata_without_verision_location":{ + "ACCEPT-RANGES":"bytes", + "CONTENT-LENGTH":"0", + "CONTENT-TYPE":"application/json; charset=utf-8", + "DATE":"Thu, 12 Oct 2017 16:13:04 GMT", + "X-CONTAINER-BYTES-USED":"90", + "X-CONTAINER-META-ACCESS-CONTROL-EXPOSE-HEADERS":"etag location x-timestamp x-trans-id", + "X-CONTAINER-META-ACCESS-LOG-DELIVERY":"false", + "X-CONTAINER-OBJECT-COUNT":"2", + "X-STORAGE-POLICY":"Policy-0", + "X-TIMESTAMP":"1506696708.03681", + "X-TRANS-ID":"txffaa4b0a06984dd0bbf27-0059df9490iad3" + }, + "file_metadata":{ + "last_modified":"2014-12-19T23:22:14.728640", + "content_type":"application/x-www-form-urlencoded;charset=utf-8", + "hash":"edfa12d00b779b4b37b81fe5b61b2b3f", + "name":"similar.file", + "bytes":190 + }, + "file_header_metadata":{ + "CONTENT-LENGTH":"90", + "ACCEPT-RANGES":"bytes", + "LAST-MODIFIED":"Wed, 11 Oct 2017 21:37:55 GMT", + "ETAG":"8a839ea73aaa78718e27e025bdc2c767", + "X-TIMESTAMP":"1507757874.70544", + "CONTENT-TYPE":"application/octet-stream", + "X-TRANS-ID":"txae77ecf20a83452ebe2c0-0059dfa57aiad3", + "DATE":"Thu, 12 Oct 2017 17:25:14 GMT" + }, + "folder_root":[ + { + "last_modified":"2014-12-19T22:08:23.006360", + "content_type":"application/directory", + "hash":"d41d8cd98f00b204e9800998ecf8427e", + "name":"level1", + "bytes":0 + }, + { + "subdir":"level1/" + }, + { + "last_modified":"2014-12-19T23:22:23.232240", + "content_type":"application/x-www-form-urlencoded;charset=utf-8", + "hash":"edfa12d00b779b4b37b81fe5b61b2b3f", + "name":"similar", + "bytes":190 + }, + { + "last_modified":"2014-12-19T23:22:14.728640", + "content_type":"application/x-www-form-urlencoded;charset=utf-8", + "hash":"edfa12d00b779b4b37b81fe5b61b2b3f", + "name":"similar.file", + "bytes":190 + }, + { + "last_modified":"2014-12-19T23:20:16.718860", + "content_type":"application/directory", + "hash":"d41d8cd98f00b204e9800998ecf8427e", + "name":"level1_empty", + "bytes":0 + } + ], + "folder_root_level1":[ + { + "last_modified":"2014-12-19T22:08:26.958830", + "content_type":"application/directory", + "hash":"d41d8cd98f00b204e9800998ecf8427e", + "name":"level1/level2", + "bytes":0 + }, + { + "subdir":"level1/level2/" + } + ], + "folder_root_level1_level2":[ + { + "name":"level1/level2/file2.txt", + "content_type":"application/x-www-form-urlencoded;charset=utf-8", + "last_modified":"2014-12-19T23:25:22.497420", + "bytes":1365336, + "hash":"ebc8cdd3f712fd39476fb921d43aca1a" + } + ], + "auth_json":{ + "access":{ + "serviceCatalog":[ + { + "name":"cloudFiles", + "type":"object-store", + "endpoints":[ + { + "publicURL":"https://fakestorage", + "internalURL":"https://internal_fake_storage", + "region":"IAD", + "tenantId":"someid_123456" + } + ] + } + ], + "token":{ + "RAX-AUTH:authenticatedBy":[ + "APIKEY" + ], + "tenant":{ + "name":"12345", + "id":"12345" + }, + "id":"2322f6b2322f4dbfa69802baf50b0832", + "expires":"2014-12-17T09:12:26.069Z" + }, + "user":{ + "name":"osf-production", + "roles":[ + { + "name":"object-store:admin", + "id":"10000256", + "description":"Object Store Admin Role for Account User" + }, + { + "name":"compute:default", + "description":"A Role that allows a user access to keystone Service methods", + "id":"6", + "tenantId":"12345" + }, + { + "name":"object-store:default", + "description":"A Role that allows a user access to keystone Service methods", + "id":"5", + "tenantId":"some_id_12345" + }, + { + "name":"identity:default", + "id":"2", + "description":"Default Role." + } + ], + "id":"secret", + "RAX-AUTH:defaultRegion":"IAD" + } + } + }, + "revision_list":[ + { + "hash":"8a839ea73aaa78718e27e025bdc2c767", + "bytes":90, + "name":"007123.csv/1507756317.92019", + "content_type":"application/octet-stream", + "last_modified":"2017-10-11T21:24:43.459520" + }, + { + "hash":"cacef99009078d6fbf994dd18aac5658", + "bytes":90, + "name":"007123.csv/1507757083.60055", + "content_type":"application/octet-stream", + "last_modified":"2017-10-11T21:31:34.386170" + }, + { + "hash":"63e4d56ff3b8a3bf4981f071dac1522e", + "bytes":90, + "name":"007123.csv/1507757494.53144", + "content_type":"application/octet-stream", + "last_modified":"2017-10-11T21:37:54.645380" + } + ] +} \ No newline at end of file diff --git a/tests/providers/cloudfiles/test_metadata.py b/tests/providers/cloudfiles/test_metadata.py index b69e44cc3..e304b3dce 100644 --- a/tests/providers/cloudfiles/test_metadata.py +++ b/tests/providers/cloudfiles/test_metadata.py @@ -1,36 +1,16 @@ import pytest -import aiohttp from waterbutler.core.path import WaterButlerPath from waterbutler.providers.cloudfiles.metadata import (CloudFilesFileMetadata, CloudFilesHeaderMetadata, - CloudFilesFolderMetadata) - - -@pytest.fixture -def file_header_metadata_txt(): - return aiohttp.multidict.CIMultiDict([ - ('ORIGIN', 'https://mycloud.rackspace.com'), - ('CONTENT-LENGTH', '216945'), - ('ACCEPT-RANGES', 'bytes'), - ('LAST-MODIFIED', 'Mon, 22 Dec 2014 19:01:02 GMT'), - ('ETAG', '44325d4f13b09f3769ede09d7c20a82c'), - ('X-TIMESTAMP', '1419274861.04433'), - ('CONTENT-TYPE', 'text/plain'), - ('X-TRANS-ID', 'tx836375d817a34b558756a-0054987deeiad3'), - ('DATE', 'Mon, 22 Dec 2014 20:24:14 GMT') - ]) - - -@pytest.fixture -def file_metadata(): - return { - 'last_modified': '2014-12-19T23:22:14.728640', - 'content_type': 'application/x-www-form-urlencoded;charset=utf-8', - 'hash': 'edfa12d00b779b4b37b81fe5b61b2b3f', - 'name': 'similar.file', - 'bytes': 190 - } + CloudFilesFolderMetadata, + CloudFilesRevisonMetadata) + +from tests.providers.cloudfiles.fixtures import ( + file_header_metadata_txt, + file_metadata, + revision_list +) class TestCloudfilesMetadata: @@ -38,7 +18,7 @@ class TestCloudfilesMetadata: def test_header_metadata(self, file_header_metadata_txt): path = WaterButlerPath('/file.txt') - data = CloudFilesHeaderMetadata(file_header_metadata_txt, path.path) + data = CloudFilesHeaderMetadata(file_header_metadata_txt, path) assert data.name == 'file.txt' assert data.path == '/file.txt' assert data.provider == 'cloudfiles' @@ -242,3 +222,13 @@ def test_folder_metadata(self): 'new_folder': ('http://localhost:7777/v1/resources/' 'cn42d/providers/cloudfiles/level1/?kind=folder') } + + def test_revision_metadata(self, revision_list): + data = CloudFilesRevisonMetadata(revision_list[0]) + + assert data.version_identifier == 'revision' + assert data.name == '007123.csv/1507756317.92019' + assert data.version == '007123.csv/1507756317.92019' + assert data.size == 90 + assert data.content_type == 'application/octet-stream' + assert data.modified == '2017-10-11T21:24:43.459520' diff --git a/tests/providers/cloudfiles/test_provider.py b/tests/providers/cloudfiles/test_provider.py index da8bb55a5..a03e021b8 100644 --- a/tests/providers/cloudfiles/test_provider.py +++ b/tests/providers/cloudfiles/test_provider.py @@ -1,331 +1,48 @@ -import io -import os import json -import time import hashlib -import functools -from unittest import mock import furl import pytest import aiohttp import aiohttpretty -import aiohttp.multidict -from waterbutler.core import streams +from tests.utils import MockCoroutine from waterbutler.core import exceptions from waterbutler.core.path import WaterButlerPath -from waterbutler.providers.cloudfiles import CloudFilesProvider from waterbutler.providers.cloudfiles import settings as cloud_settings - -@pytest.fixture -def auth(): - return { - 'name': 'cat', - 'email': 'cat@cat.com', - } - - -@pytest.fixture -def credentials(): - return { - 'username': 'prince', - 'token': 'revolutionary', - 'region': 'iad', - } - - -@pytest.fixture -def settings(): - return {'container': 'purple rain'} - - -@pytest.fixture -def provider(auth, credentials, settings): - return CloudFilesProvider(auth, credentials, settings) - - -@pytest.fixture -def auth_json(): - return { - "access": { - "serviceCatalog": [ - { - "name": "cloudFiles", - "type": "object-store", - "endpoints": [ - { - "publicURL": "https://fakestorage", - "internalURL": "https://internal_fake_storage", - "region": "IAD", - "tenantId": "someid_123456" - }, - ] - } - ], - "token": { - "RAX-AUTH:authenticatedBy": [ - "APIKEY" - ], - "tenant": { - "name": "12345", - "id": "12345" - }, - "id": "2322f6b2322f4dbfa69802baf50b0832", - "expires": "2014-12-17T09:12:26.069Z" - }, - "user": { - "name": "osf-production", - "roles": [ - { - "name": "object-store:admin", - "id": "10000256", - "description": "Object Store Admin Role for Account User" - }, - { - "name": "compute:default", - "description": "A Role that allows a user access to keystone Service methods", - "id": "6", - "tenantId": "12345" - }, - { - "name": "object-store:default", - "description": "A Role that allows a user access to keystone Service methods", - "id": "5", - "tenantId": "some_id_12345" - }, - { - "name": "identity:default", - "id": "2", - "description": "Default Role." - } - ], - "id": "secret", - "RAX-AUTH:defaultRegion": "IAD" - } - } - } - - -@pytest.fixture -def token(auth_json): - return auth_json['access']['token']['id'] - - -@pytest.fixture -def endpoint(auth_json): - return auth_json['access']['serviceCatalog'][0]['endpoints'][0]['publicURL'] - - -@pytest.fixture -def temp_url_key(): - return 'temporary beret' - - -@pytest.fixture -def mock_auth(auth_json): - aiohttpretty.register_json_uri( - 'POST', - settings.AUTH_URL, - body=auth_json, - ) - - -@pytest.fixture -def mock_temp_key(endpoint, temp_url_key): - aiohttpretty.register_uri( - 'HEAD', - endpoint, - status=204, - headers={'X-Account-Meta-Temp-URL-Key': temp_url_key}, - ) - - -@pytest.fixture -def mock_time(monkeypatch): - mock_time = mock.Mock() - mock_time.return_value = 10 - monkeypatch.setattr(time, 'time', mock_time) - - -@pytest.fixture -def connected_provider(provider, token, endpoint, temp_url_key, mock_time): - provider.token = token - provider.endpoint = endpoint - provider.temp_url_key = temp_url_key.encode() - return provider - - -@pytest.fixture -def file_content(): - return b'sleepy' - - -@pytest.fixture -def file_like(file_content): - return io.BytesIO(file_content) - - -@pytest.fixture -def file_stream(file_like): - return streams.FileStreamReader(file_like) - - -@pytest.fixture -def file_metadata(): - return aiohttp.multidict.CIMultiDict([ - ('LAST-MODIFIED', 'Thu, 25 Dec 2014 02:54:35 GMT'), - ('CONTENT-LENGTH', '0'), - ('ETAG', 'edfa12d00b779b4b37b81fe5b61b2b3f'), - ('CONTENT-TYPE', 'text/html; charset=UTF-8'), - ('X-TRANS-ID', 'txf876a4b088e3451d94442-00549b7c6aiad3'), - ('DATE', 'Thu, 25 Dec 2014 02:54:34 GMT') - ]) - - -# Metadata Test Scenarios -# / (folder_root_empty) -# / (folder_root) -# /level1/ (folder_root_level1) -# /level1/level2/ (folder_root_level1_level2) -# /level1/level2/file2.file - (file_root_level1_level2_file2_txt) -# /level1_empty/ (folder_root_level1_empty) -# /similar (file_similar) -# /similar.name (file_similar_name) -# /does_not_exist (404) -# /does_not_exist/ (404) - - -@pytest.fixture -def folder_root_empty(): - return [] - - -@pytest.fixture -def folder_root(): - return [ - { - 'last_modified': '2014-12-19T22:08:23.006360', - 'content_type': 'application/directory', - 'hash': 'd41d8cd98f00b204e9800998ecf8427e', - 'name': 'level1', - 'bytes': 0 - }, - { - 'subdir': 'level1/' - }, - { - 'last_modified': '2014-12-19T23:22:23.232240', - 'content_type': 'application/x-www-form-urlencoded;charset=utf-8', - 'hash': 'edfa12d00b779b4b37b81fe5b61b2b3f', - 'name': 'similar', - 'bytes': 190 - }, - { - 'last_modified': '2014-12-19T23:22:14.728640', - 'content_type': 'application/x-www-form-urlencoded;charset=utf-8', - 'hash': 'edfa12d00b779b4b37b81fe5b61b2b3f', - 'name': 'similar.file', - 'bytes': 190 - }, - { - 'last_modified': '2014-12-19T23:20:16.718860', - 'content_type': 'application/directory', - 'hash': 'd41d8cd98f00b204e9800998ecf8427e', - 'name': 'level1_empty', - 'bytes': 0 - } - ] - - -@pytest.fixture -def folder_root_level1(): - return [ - { - 'last_modified': '2014-12-19T22:08:26.958830', - 'content_type': 'application/directory', - 'hash': 'd41d8cd98f00b204e9800998ecf8427e', - 'name': 'level1/level2', - 'bytes': 0 - }, - { - 'subdir': 'level1/level2/' - } - ] - - -@pytest.fixture -def folder_root_level1_level2(): - return [ - { - 'name': 'level1/level2/file2.txt', - 'content_type': 'application/x-www-form-urlencoded;charset=utf-8', - 'last_modified': '2014-12-19T23:25:22.497420', - 'bytes': 1365336, - 'hash': 'ebc8cdd3f712fd39476fb921d43aca1a' - } - ] - - -@pytest.fixture -def file_root_level1_level2_file2_txt(): - return aiohttp.multidict.CIMultiDict([ - ('ORIGIN', 'https://mycloud.rackspace.com'), - ('CONTENT-LENGTH', '216945'), - ('ACCEPT-RANGES', 'bytes'), - ('LAST-MODIFIED', 'Mon, 22 Dec 2014 19:01:02 GMT'), - ('ETAG', '44325d4f13b09f3769ede09d7c20a82c'), - ('X-TIMESTAMP', '1419274861.04433'), - ('CONTENT-TYPE', 'text/plain'), - ('X-TRANS-ID', 'tx836375d817a34b558756a-0054987deeiad3'), - ('DATE', 'Mon, 22 Dec 2014 20:24:14 GMT') - ]) - - -@pytest.fixture -def folder_root_level1_empty(): - return aiohttp.multidict.CIMultiDict([ - ('ORIGIN', 'https://mycloud.rackspace.com'), - ('CONTENT-LENGTH', '0'), - ('ACCEPT-RANGES', 'bytes'), - ('LAST-MODIFIED', 'Mon, 22 Dec 2014 18:58:56 GMT'), - ('ETAG', 'd41d8cd98f00b204e9800998ecf8427e'), - ('X-TIMESTAMP', '1419274735.03160'), - ('CONTENT-TYPE', 'application/directory'), - ('X-TRANS-ID', 'txd78273e328fc4ba3a98e3-0054987eeeiad3'), - ('DATE', 'Mon, 22 Dec 2014 20:28:30 GMT') - ]) - - -@pytest.fixture -def file_root_similar(): - return aiohttp.multidict.CIMultiDict([ - ('ORIGIN', 'https://mycloud.rackspace.com'), - ('CONTENT-LENGTH', '190'), - ('ACCEPT-RANGES', 'bytes'), - ('LAST-MODIFIED', 'Fri, 19 Dec 2014 23:22:24 GMT'), - ('ETAG', 'edfa12d00b779b4b37b81fe5b61b2b3f'), - ('X-TIMESTAMP', '1419031343.23224'), - ('CONTENT-TYPE', 'application/x-www-form-urlencoded;charset=utf-8'), - ('X-TRANS-ID', 'tx7cfeef941f244807aec37-005498754diad3'), - ('DATE', 'Mon, 22 Dec 2014 19:47:25 GMT') - ]) - - -@pytest.fixture -def file_root_similar_name(): - return aiohttp.multidict.CIMultiDict([ - ('ORIGIN', 'https://mycloud.rackspace.com'), - ('CONTENT-LENGTH', '190'), - ('ACCEPT-RANGES', 'bytes'), - ('LAST-MODIFIED', 'Mon, 22 Dec 2014 19:07:12 GMT'), - ('ETAG', 'edfa12d00b779b4b37b81fe5b61b2b3f'), - ('X-TIMESTAMP', '1419275231.66160'), - ('CONTENT-TYPE', 'application/x-www-form-urlencoded;charset=utf-8'), - ('X-TRANS-ID', 'tx438cbb32b5344d63b267c-0054987f3biad3'), - ('DATE', 'Mon, 22 Dec 2014 20:29:47 GMT') - ]) +from waterbutler.providers.cloudfiles.metadata import (CloudFilesHeaderMetadata, + CloudFilesRevisonMetadata) + +from tests.providers.cloudfiles.fixtures import ( + auth, + settings, + credentials, + token, + endpoint, + temp_url_key, + mock_time, + file_content, + file_stream, + file_like, + mock_temp_key, + provider, + connected_provider, + file_metadata, + auth_json, + folder_root, + folder_root_empty, + folder_root_level1_empty, + file_root_similar_name, + file_root_similar, + file_header_metadata, + folder_root_level1, + folder_root_level1_level2, + file_root_level1_level2_file2_txt, + container_header_metadata_with_verision_location, + container_header_metadata_without_verision_location, + revision_list +) class TestCRUD: @@ -344,6 +61,31 @@ async def test_download(self, connected_provider): assert content == body + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_download_revision(self, + connected_provider, + container_header_metadata_with_verision_location): + body = b'dearly-beloved' + path = WaterButlerPath('/lets-go-crazy') + url = connected_provider.sign_url(path) + aiohttpretty.register_uri('GET', url, body=body, auto_length=True) + + container_url = connected_provider.build_url('') + aiohttpretty.register_uri('HEAD', + container_url, + headers=container_header_metadata_with_verision_location) + + version_name = '{:03x}'.format(len(path.name)) + path.name + '/' + revision_url = connected_provider.build_url(version_name, container='versions-container', ) + aiohttpretty.register_uri('GET', revision_url, body=body, auto_length=True) + + + result = await connected_provider.download(path, version=version_name) + content = await result.read() + + assert content == body + @pytest.mark.asyncio @pytest.mark.aiohttpretty async def test_download_accept_url(self, connected_provider): @@ -373,21 +115,26 @@ async def test_download_not_found(self, connected_provider): @pytest.mark.asyncio @pytest.mark.aiohttpretty - async def test_upload(self, connected_provider, file_content, file_stream, file_metadata): - path = WaterButlerPath('/foo.bar') + async def test_upload(self, + connected_provider, + file_content, + file_stream, + file_header_metadata): + path = WaterButlerPath('/similar.file') content_md5 = hashlib.md5(file_content).hexdigest() metadata_url = connected_provider.build_url(path.path) url = connected_provider.sign_url(path, 'PUT') - aiohttpretty.register_uri( - 'HEAD', - metadata_url, - responses=[ - {'status': 404}, - {'headers': file_metadata}, - ] - ) + aiohttpretty.register_uri('HEAD', + metadata_url, + responses=[ + {'status': 404}, + {'headers': file_header_metadata} + ] + ) + aiohttpretty.register_uri('PUT', url, status=200, headers={'ETag': '"{}"'.format(content_md5)}) + metadata, created = await connected_provider.upload(file_stream, path) assert created is True @@ -395,22 +142,31 @@ async def test_upload(self, connected_provider, file_content, file_stream, file_ assert aiohttpretty.has_call(method='PUT', uri=url) assert aiohttpretty.has_call(method='HEAD', uri=metadata_url) + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_create_folder(self, connected_provider, file_header_metadata): + path = WaterButlerPath('/foo/', folder=True) + metadata_url = connected_provider.build_url(path.path) + url = connected_provider.sign_url(path, 'PUT') + print(metadata_url) + aiohttpretty.register_uri('PUT', url, status=200) + aiohttpretty.register_uri('HEAD', metadata_url, headers=file_header_metadata) + + metadata = await connected_provider.create_folder(path) + + assert metadata.kind == 'folder' + assert aiohttpretty.has_call(method='PUT', uri=url) + assert aiohttpretty.has_call(method='HEAD', uri=metadata_url) + @pytest.mark.asyncio @pytest.mark.aiohttpretty async def test_upload_check_none(self, connected_provider, - file_content, file_stream, file_metadata): - path = WaterButlerPath('/foo.bar') + file_content, file_stream, file_header_metadata): + path = WaterButlerPath('/similar.file') content_md5 = hashlib.md5(file_content).hexdigest() metadata_url = connected_provider.build_url(path.path) url = connected_provider.sign_url(path, 'PUT') - aiohttpretty.register_uri( - 'HEAD', - metadata_url, - responses=[ - {'status': 404}, - {'headers': file_metadata}, - ] - ) + aiohttpretty.register_uri('HEAD', metadata_url, status=404, headers=file_header_metadata) aiohttpretty.register_uri('PUT', url, status=200, headers={'ETag': '"{}"'.format(content_md5)}) metadata, created = await connected_provider.upload( @@ -422,18 +178,15 @@ async def test_upload_check_none(self, connected_provider, @pytest.mark.asyncio @pytest.mark.aiohttpretty - async def test_upload_checksum_mismatch(self, connected_provider, file_stream, file_metadata): - path = WaterButlerPath('/foo.bar') + async def test_upload_checksum_mismatch(self, + connected_provider, + file_stream, + file_header_metadata): + path = WaterButlerPath('/similar.file') metadata_url = connected_provider.build_url(path.path) url = connected_provider.sign_url(path, 'PUT') - aiohttpretty.register_uri( - 'HEAD', - metadata_url, - responses=[ - {'status': 404}, - {'headers': file_metadata}, - ] - ) + aiohttpretty.register_uri('HEAD', metadata_url, status=404, headers=file_header_metadata) + aiohttpretty.register_uri('PUT', url, status=200, headers={'ETag': '"Bogus MD5"'}) with pytest.raises(exceptions.UploadChecksumMismatchError): @@ -442,30 +195,30 @@ async def test_upload_checksum_mismatch(self, connected_provider, file_stream, f assert aiohttpretty.has_call(method='PUT', uri=url) assert aiohttpretty.has_call(method='HEAD', uri=metadata_url) - # @pytest.mark.asyncio - # @pytest.mark.aiohttpretty - # async def test_delete_folder(self, connected_provider, folder_root_empty, file_metadata): - # # This test will probably fail on a live - # # version of the provider because build_url is called wrong. - # # Will comment out parts of this test till that is fixed. - # path = WaterButlerPath('/delete/') - # query = {'prefix': path.path} - # url = connected_provider.build_url('', **query) - # body = json.dumps(folder_root_empty).encode('utf-8') + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_delete_folder(self, connected_provider, folder_root_empty, file_header_metadata): - # delete_query = {'bulk-delete': ''} - # delete_url = connected_provider.build_url('', **delete_query) + path = WaterButlerPath('/delete/') + query = {'prefix': path.path} + url = connected_provider.build_url('', **query) + body = json.dumps(folder_root_empty).encode('utf-8') - # file_url = connected_provider.build_url(path.path) + delete_query = {'bulk-delete': ''} + delete_url_folder = connected_provider.build_url(path.name, **delete_query) + delete_url_content = connected_provider.build_url('', **delete_query) - # aiohttpretty.register_uri('GET', url, body=body) - # aiohttpretty.register_uri('HEAD', file_url, headers=file_metadata) + file_url = connected_provider.build_url(path.path) + aiohttpretty.register_uri('GET', url, body=body) + aiohttpretty.register_uri('HEAD', file_url, headers=file_header_metadata) - # aiohttpretty.register_uri('DELETE', delete_url) + aiohttpretty.register_uri('DELETE', delete_url_content, status=200) + aiohttpretty.register_uri('DELETE', delete_url_folder, status=204) - # await connected_provider.delete(path) + await connected_provider.delete(path) - # assert aiohttpretty.has_call(method='DELETE', uri=delete_url) + assert aiohttpretty.has_call(method='DELETE', uri=delete_url_content) + assert aiohttpretty.has_call(method='DELETE', uri=delete_url_folder) @pytest.mark.asyncio @pytest.mark.aiohttpretty @@ -479,21 +232,74 @@ async def test_delete_file(self, connected_provider): @pytest.mark.asyncio @pytest.mark.aiohttpretty - async def test_intra_copy(self, connected_provider, file_metadata): + async def test_intra_copy(self, connected_provider, file_header_metadata): src_path = WaterButlerPath('/delete.file') dest_path = WaterButlerPath('/folder1/delete.file') dest_url = connected_provider.build_url(dest_path.path) - aiohttpretty.register_uri('HEAD', dest_url, headers=file_metadata) + aiohttpretty.register_uri('HEAD', dest_url, headers=file_header_metadata) aiohttpretty.register_uri('PUT', dest_url, status=201) result = await connected_provider.intra_copy(connected_provider, src_path, dest_path) assert result[0].path == '/folder1/delete.file' assert result[0].name == 'delete.file' - assert result[0].etag == 'edfa12d00b779b4b37b81fe5b61b2b3f' + assert result[0].etag == '8a839ea73aaa78718e27e025bdc2c767' + + +class TestRevisions: + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_revisions(self, + connected_provider, + container_header_metadata_with_verision_location, + revision_list): + + path = WaterButlerPath('/file.txt') + + container_url = connected_provider.build_url('') + aiohttpretty.register_uri('HEAD', + container_url, + headers=container_header_metadata_with_verision_location) + + query = {'prefix': '{:03x}'.format(len(path.name)) + path.name + '/'} + revision_url = connected_provider.build_url('', container='versions-container', **query) + aiohttpretty.register_json_uri('GET', revision_url, body=revision_list) + + result = await connected_provider.revisions(path) + + assert type(result) == list + assert len(result) == 3 + assert type(result[0]) == CloudFilesRevisonMetadata + assert result[0].name == '007123.csv/1507756317.92019' + + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_revision_metadata(self, + connected_provider, + container_header_metadata_with_verision_location, + file_header_metadata): + + path = WaterButlerPath('/file.txt') + container_url = connected_provider.build_url('') + aiohttpretty.register_uri('HEAD', container_url, + headers=container_header_metadata_with_verision_location) + + version_name = '{:03x}'.format(len(path.name)) + path.name + '/' + query = {'prefix' : version_name} + revision_url = connected_provider.build_url(version_name + '1507756317.92019', + container='versions-container') + print(revision_url) + aiohttpretty.register_json_uri('HEAD', revision_url, body=file_header_metadata) + + result = await connected_provider.metadata(path, version=version_name + '1507756317.92019') + + assert type(result) == CloudFilesHeaderMetadata + assert result.name == 'file.txt' + assert result.path == '/file.txt' + assert result.kind == 'file' class TestMetadata: @@ -532,6 +338,18 @@ async def test_metadata_folder_root(self, connected_provider, folder_root): assert result[3].path == '/level1_empty/' assert result[3].kind == 'folder' + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_metadata_404(self, connected_provider, folder_root_level1): + path = WaterButlerPath('/level1/') + body = json.dumps(folder_root_level1).encode('utf-8') + url = connected_provider.build_url('', prefix=path.path, delimiter='/') + aiohttpretty.register_uri('GET', url, status=200, body=b'') + connected_provider._metadata_item = MockCoroutine(return_value=None) + + with pytest.raises(exceptions.MetadataError): + await connected_provider.metadata(path) + @pytest.mark.asyncio @pytest.mark.aiohttpretty async def test_metadata_folder_root_level1(self, connected_provider, folder_root_level1): @@ -674,6 +492,22 @@ async def test_ensure_connection(self, provider, auth_json, mock_temp_key): await provider._ensure_connection() assert aiohttpretty.has_call(method='POST', uri=token_url) + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_no_version_location(self, + connected_provider, + container_header_metadata_without_verision_location): + + path = WaterButlerPath('/file.txt') + + container_url = connected_provider.build_url('') + aiohttpretty.register_uri('HEAD', + container_url, + headers=container_header_metadata_without_verision_location) + + with pytest.raises(exceptions.MetadataError): + await connected_provider.revisions(path) + @pytest.mark.asyncio @pytest.mark.aiohttpretty async def test_ensure_connection_not_public(self, provider, auth_json, temp_url_key): diff --git a/waterbutler/providers/cloudfiles/metadata.py b/waterbutler/providers/cloudfiles/metadata.py index aa13c1ed9..f13f192ca 100644 --- a/waterbutler/providers/cloudfiles/metadata.py +++ b/waterbutler/providers/cloudfiles/metadata.py @@ -55,13 +55,17 @@ def __init__(self, raw, path): super().__init__(raw) self._path = path + @property + def kind(self): + return 'folder' if self._path.is_dir else 'file' + @property def name(self): - return os.path.split(self._path)[1] + return self._path.name @property def path(self): - return self.build_path(self._path) + return self._path.materialized_path @property def size(self): @@ -101,3 +105,30 @@ def name(self): @property def path(self): return self.build_path(self.raw['subdir']) + + +class CloudFilesRevisonMetadata(metadata.BaseFileRevisionMetadata): + + @property + def version_identifier(self): + return 'revision' + + @property + def version(self): + return self.raw['name'] + + @property + def modified(self): + return self.raw['last_modified'] + + @property + def size(self): + return self.raw['bytes'] + + @property + def name(self): + return self.raw['name'] + + @property + def content_type(self): + return self.raw['content_type'] diff --git a/waterbutler/providers/cloudfiles/provider.py b/waterbutler/providers/cloudfiles/provider.py index fa0248571..4b47fd455 100644 --- a/waterbutler/providers/cloudfiles/provider.py +++ b/waterbutler/providers/cloudfiles/provider.py @@ -14,9 +14,12 @@ from waterbutler.core.path import WaterButlerPath from waterbutler.providers.cloudfiles import settings -from waterbutler.providers.cloudfiles.metadata import CloudFilesFileMetadata -from waterbutler.providers.cloudfiles.metadata import CloudFilesFolderMetadata -from waterbutler.providers.cloudfiles.metadata import CloudFilesHeaderMetadata +from waterbutler.providers.cloudfiles.metadata import ( + CloudFilesFileMetadata, + CloudFilesFolderMetadata, + CloudFilesHeaderMetadata, + CloudFilesRevisonMetadata +) def ensure_connection(func): @@ -32,7 +35,8 @@ async def wrapped(self, *args, **kwargs): class CloudFilesProvider(provider.BaseProvider): """Provider for Rackspace CloudFiles. - API Docs: https://developer.rackspace.com/docs/cloud-files/v1/developer-guide/#document-developer-guide + API Docs: https://developer.rackspace.com/docs/cloud-files/v1/developer-guide/ + #document-developer-guide """ NAME = 'cloudfiles' @@ -79,7 +83,14 @@ async def intra_copy(self, dest_provider, source_path, dest_path): return (await dest_provider.metadata(dest_path)), not exists @ensure_connection - async def download(self, path, accept_url=False, range=None, **kwargs): + async def download(self, + path, + accept_url=False, + range=None, + version=None, + revision=None, + displayName=None, + **kwargs): """Returns a ResponseStreamReader (Stream) for the specified path :param str path: Path to the object you want to download :param dict \*\*kwargs: Additional arguments that are ignored @@ -90,9 +101,13 @@ async def download(self, path, accept_url=False, range=None, **kwargs): self.metrics.add('download.accept_url', accept_url) if accept_url: parsed_url = furl.furl(self.sign_url(path, endpoint=self.public_endpoint)) - parsed_url.args['filename'] = kwargs.get('displayName') or path.name + parsed_url.args['filename'] = displayName or path.name return parsed_url.url + version = revision or version + if version: + return await self._download_revision(range, version) + resp = await self.make_request( 'GET', functools.partial(self.sign_url, path), @@ -103,11 +118,13 @@ async def download(self, path, accept_url=False, range=None, **kwargs): return streams.ResponseStreamReader(resp) @ensure_connection - async def upload(self, stream, path, check_created=True, fetch_metadata=True, **kwargs): + async def upload(self, stream, path, check_created=True, fetch_metadata=True): """Uploads the given stream to CloudFiles :param ResponseStreamReader stream: The stream to put to CloudFiles :param str path: The full path of the object to upload to/into - :rtype ResponseStreamReader: + :param bool check_created: This checks if uploaded file already exists + :param bool fetch_metadata: If true upload will return metadata + :rtype (dict/None, bool): """ if check_created: created = not (await self.exists(path)) @@ -138,25 +155,25 @@ async def upload(self, stream, path, check_created=True, fetch_metadata=True, ** return metadata, created @ensure_connection - async def delete(self, path, **kwargs): + async def delete(self, path, confirm_delete=False): """Deletes the key at the specified path :param str path: The path of the key to delete - :rtype ResponseStreamReader: + :param bool confirm_delete: not used in this provider + :rtype None: """ if path.is_dir: - metadata = await self.metadata(path, recursive=True) + metadata = await self._metadata_folder(path, recursive=True) delete_files = [ - os.path.join('/', self.container, path.child(item['name']).path) + os.path.join('/', self.container, path.child(item.name).path) for item in metadata ] delete_files.append(os.path.join('/', self.container, path.path)) - query = {'bulk-delete': ''} resp = await self.make_request( 'DELETE', - functools.partial(self.build_url, **query), + functools.partial(self.build_url, '', **query), data='\n'.join(delete_files), expects=(200, ), throws=exceptions.DeleteError, @@ -164,35 +181,39 @@ async def delete(self, path, **kwargs): 'Content-Type': 'text/plain', }, ) - else: - resp = await self.make_request( - 'DELETE', - functools.partial(self.build_url, path.path), - expects=(204, ), - throws=exceptions.DeleteError, - ) + await resp.release() + + resp = await self.make_request( + 'DELETE', + functools.partial(self.build_url, path.path), + expects=(204, ), + throws=exceptions.DeleteError, + ) await resp.release() @ensure_connection - async def metadata(self, path, recursive=False, **kwargs): + async def metadata(self, path, recursive=False, version=None, revision=None): """Get Metadata about the requested file or folder :param str path: The path to a key or folder :rtype dict: :rtype list: """ if path.is_dir: - return (await self._metadata_folder(path, recursive=recursive, **kwargs)) + return (await self._metadata_folder(path, recursive=recursive)) + elif version or revision: + return (await self.get_metadata_revision(path, version, revision)) else: - return (await self._metadata_file(path, **kwargs)) + return (await self._metadata_item(path)) - def build_url(self, path, _endpoint=None, **query): + def build_url(self, path, _endpoint=None, container=None, **query): """Build the url for the specified object :param args segments: URI segments :param kwargs query: Query parameters :rtype str: """ endpoint = _endpoint or self.endpoint - return provider.build_url(endpoint, self.container, *path.split('/'), **query) + container = container or self.container + return provider.build_url(endpoint, container, *path.split('/'), **query) def can_duplicate_names(self): return False @@ -211,18 +232,23 @@ def sign_url(self, path, method='GET', endpoint=None, seconds=settings.TEMP_URL_ :param int seconds: Time for the url to live :rtype str: """ + from urllib.parse import unquote + method = method.upper() expires = str(int(time.time() + seconds)) - url = furl.furl(self.build_url(path.path, _endpoint=endpoint)) + path_str = path.path + url = furl.furl(self.build_url(path_str, _endpoint=endpoint)) - body = '\n'.join([method, expires, str(url.path)]).encode() + body = '\n'.join([method, expires]) + body += '\n' + str(url.path) + body = unquote(body).encode() signature = hmac.new(self.temp_url_key, body, hashlib.sha1).hexdigest() - url.args.update({ 'temp_url_sig': signature, 'temp_url_expires': expires, }) - return url.url + + return unquote(str(url.url)) async def make_request(self, *args, **kwargs): try: @@ -297,7 +323,7 @@ async def _get_token(self): data = await resp.json() return data - async def _metadata_file(self, path, is_folder=False, **kwargs): + async def _metadata_item(self, path): """Get Metadata about the requested file :param str path: The path to a key :rtype dict: @@ -309,18 +335,17 @@ async def _metadata_file(self, path, is_folder=False, **kwargs): expects=(200, ), throws=exceptions.MetadataError, ) - await resp.release() - if (resp.headers['Content-Type'] == 'application/directory' and not is_folder): + if resp.headers['Content-Type'] == 'application/directory' and path.is_file: raise exceptions.MetadataError( 'Could not retrieve file \'{0}\''.format(str(path)), code=404, ) - return CloudFilesHeaderMetadata(resp.headers, path.path) + return CloudFilesHeaderMetadata(resp.headers, path) - async def _metadata_folder(self, path, recursive=False, **kwargs): + async def _metadata_folder(self, path, recursive=False): """Get Metadata about the requested folder :param str path: The path to a folder :rtype dict: @@ -339,11 +364,12 @@ async def _metadata_folder(self, path, recursive=False, **kwargs): ) data = await resp.json() - # no data and the provider path is not root, we are left with either a file or a directory marker + # no data and the provider path is not root, we are left with either a file or a directory + # marker if not data and not path.is_root: # Convert the parent path into a directory marker (file) and check for an empty folder - dir_marker = path.parent.child(path.name, folder=False) - metadata = await self._metadata_file(dir_marker, is_folder=True, **kwargs) + dir_marker = path.parent.child(path.name, folder=path.is_dir) + metadata = await self._metadata_item(dir_marker) if not metadata: raise exceptions.MetadataError( 'Could not retrieve folder \'{0}\''.format(str(path)), @@ -361,13 +387,88 @@ async def _metadata_folder(self, path, recursive=False, **kwargs): break return [ - self._serialize_folder_metadata(item) + self._serialize_metadata(item) for item in data ] - def _serialize_folder_metadata(self, data): + def _serialize_metadata(self, data): if data.get('subdir'): return CloudFilesFolderMetadata(data) elif data['content_type'] == 'application/directory': return CloudFilesFolderMetadata({'subdir': data['name'] + '/'}) return CloudFilesFileMetadata(data) + + @ensure_connection + async def create_folder(self, path): + + resp = await self.make_request( + 'PUT', + functools.partial(self.sign_url, path, 'PUT'), + expects=(200, 201), + throws=exceptions.UploadError, + headers={'Content-Type': 'application/directory'} + ) + await resp.release() + return await self._metadata_item(path) + + @ensure_connection + async def _get_version_location(self): + resp = await self.make_request( + 'HEAD', + functools.partial(self.build_url, ''), + expects=(200, 204), + throws=exceptions.MetadataError, + ) + await resp.release() + + try: + return resp.headers['X-VERSIONS-LOCATION'] + except KeyError: + raise exceptions.MetadataError('The your container does not have a defined version' + ' location. To set a version location and store file ' + 'versions follow the instructions here: ' + 'https://developer.rackspace.com/docs/cloud-files/v1/' + 'use-cases/additional-object-services-information/' + '#object-versioning') + + @ensure_connection + async def revisions(self, path): + version_location = await self._get_version_location() + + query = {'prefix': '{:03x}'.format(len(path.name)) + path.name + '/'} + resp = await self.make_request( + 'GET', + functools.partial(self.build_url, '', container=version_location, **query), + expects=(200, 204), + throws=exceptions.MetadataError, + ) + json_resp = await resp.json() + return [CloudFilesRevisonMetadata(revision_data) for revision_data in json_resp] + + @ensure_connection + async def get_metadata_revision(self, path, version=None, revision=None): + version_location = await self._get_version_location() + + resp = await self.make_request( + 'HEAD', + functools.partial(self.build_url, version or revision, container=version_location), + expects=(200, ), + throws=exceptions.MetadataError, + ) + await resp.release() + + return CloudFilesHeaderMetadata(resp.headers, path) + + async def _download_revision(self, range, version): + + version_location = await self._get_version_location() + + resp = await self.make_request( + 'GET', + functools.partial(self.build_url, version, container=version_location), + range=range, + expects=(200, 206), + throws=exceptions.DownloadError + ) + + return streams.ResponseStreamReader(resp) From 7a759d03f14830816f708195c8541eeb1d611741 Mon Sep 17 00:00:00 2001 From: John Tordoff Date: Fri, 13 Oct 2017 12:20:23 -0400 Subject: [PATCH 02/13] Add new revisions behavior and clean up metadata --- tests/providers/cloudfiles/test_provider.py | 15 +++++++++------ waterbutler/core/provider.py | 1 - waterbutler/providers/cloudfiles/metadata.py | 18 +++++++++++++----- waterbutler/providers/cloudfiles/provider.py | 7 ++++--- 4 files changed, 26 insertions(+), 15 deletions(-) diff --git a/tests/providers/cloudfiles/test_provider.py b/tests/providers/cloudfiles/test_provider.py index a03e021b8..eaad6a6fa 100644 --- a/tests/providers/cloudfiles/test_provider.py +++ b/tests/providers/cloudfiles/test_provider.py @@ -148,7 +148,6 @@ async def test_create_folder(self, connected_provider, file_header_metadata): path = WaterButlerPath('/foo/', folder=True) metadata_url = connected_provider.build_url(path.path) url = connected_provider.sign_url(path, 'PUT') - print(metadata_url) aiohttpretty.register_uri('PUT', url, status=200) aiohttpretty.register_uri('HEAD', metadata_url, headers=file_header_metadata) @@ -255,7 +254,8 @@ class TestRevisions: async def test_revisions(self, connected_provider, container_header_metadata_with_verision_location, - revision_list): + revision_list, + file_header_metadata): path = WaterButlerPath('/file.txt') @@ -268,12 +268,17 @@ async def test_revisions(self, revision_url = connected_provider.build_url('', container='versions-container', **query) aiohttpretty.register_json_uri('GET', revision_url, body=revision_list) + metadata_url = connected_provider.build_url(path.path) + aiohttpretty.register_uri('HEAD', metadata_url, status=200, headers=file_header_metadata) + + result = await connected_provider.revisions(path) assert type(result) == list - assert len(result) == 3 + assert len(result) == 4 assert type(result[0]) == CloudFilesRevisonMetadata - assert result[0].name == '007123.csv/1507756317.92019' + assert result[0].name == 'file.txt' + assert result[1].name == '007123.csv/1507756317.92019' @pytest.mark.asyncio @pytest.mark.aiohttpretty @@ -288,10 +293,8 @@ async def test_revision_metadata(self, headers=container_header_metadata_with_verision_location) version_name = '{:03x}'.format(len(path.name)) + path.name + '/' - query = {'prefix' : version_name} revision_url = connected_provider.build_url(version_name + '1507756317.92019', container='versions-container') - print(revision_url) aiohttpretty.register_json_uri('HEAD', revision_url, body=file_header_metadata) result = await connected_provider.metadata(path, version=version_name + '1507756317.92019') diff --git a/waterbutler/core/provider.py b/waterbutler/core/provider.py index ba443e33f..559f9a46a 100644 --- a/waterbutler/core/provider.py +++ b/waterbutler/core/provider.py @@ -323,7 +323,6 @@ async def _folder_file_op(self, """ assert src_path.is_dir, 'src_path must be a directory' assert asyncio.iscoroutinefunction(func), 'func must be a coroutine' - try: await dest_provider.delete(dest_path) created = False diff --git a/waterbutler/providers/cloudfiles/metadata.py b/waterbutler/providers/cloudfiles/metadata.py index f13f192ca..5e44720b1 100644 --- a/waterbutler/providers/cloudfiles/metadata.py +++ b/waterbutler/providers/cloudfiles/metadata.py @@ -55,6 +55,14 @@ def __init__(self, raw, path): super().__init__(raw) self._path = path + def to_revision(self): + revison_dict = {'bytes': self.size, + 'name': self.name, + 'last_modified': self.modified, + 'content_type': self.content_type} + + return CloudFilesRevisonMetadata(revison_dict) + @property def kind(self): return 'folder' if self._path.is_dir else 'file' @@ -69,11 +77,11 @@ def path(self): @property def size(self): - return int(self.raw['Content-Length']) + return int(self.raw['CONTENT-LENGTH']) @property def modified(self): - return self.raw['Last-Modified'] + return self.raw['LAST-MODIFIED'] @property def created_utc(self): @@ -81,17 +89,17 @@ def created_utc(self): @property def content_type(self): - return self.raw['Content-Type'] + return self.raw['CONTENT-TYPE'] @property def etag(self): - return self.raw['etag'] + return self.raw['ETAG'] @property def extra(self): return { 'hashes': { - 'md5': self.raw['etag'].replace('"', ''), + 'md5': self.raw['ETAG'], }, } diff --git a/waterbutler/providers/cloudfiles/provider.py b/waterbutler/providers/cloudfiles/provider.py index 4b47fd455..412468172 100644 --- a/waterbutler/providers/cloudfiles/provider.py +++ b/waterbutler/providers/cloudfiles/provider.py @@ -343,7 +343,7 @@ async def _metadata_item(self, path): code=404, ) - return CloudFilesHeaderMetadata(resp.headers, path) + return CloudFilesHeaderMetadata(dict(resp.headers), path) async def _metadata_folder(self, path, recursive=False): """Get Metadata about the requested folder @@ -399,7 +399,7 @@ def _serialize_metadata(self, data): return CloudFilesFileMetadata(data) @ensure_connection - async def create_folder(self, path): + async def create_folder(self, path, folder_precheck=None): resp = await self.make_request( 'PUT', @@ -443,7 +443,8 @@ async def revisions(self, path): throws=exceptions.MetadataError, ) json_resp = await resp.json() - return [CloudFilesRevisonMetadata(revision_data) for revision_data in json_resp] + current = (await self.metadata(path)).to_revision() + return [current] + [CloudFilesRevisonMetadata(revision_data) for revision_data in json_resp] @ensure_connection async def get_metadata_revision(self, path, version=None, revision=None): From 9fef6c683d5c8e8f315b3d8af7c9046a94549a84 Mon Sep 17 00:00:00 2001 From: John Tordoff Date: Tue, 17 Oct 2017 14:22:04 -0400 Subject: [PATCH 03/13] Allow all kwargs for registrations. --- waterbutler/providers/cloudfiles/provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/waterbutler/providers/cloudfiles/provider.py b/waterbutler/providers/cloudfiles/provider.py index 412468172..d189e3c55 100644 --- a/waterbutler/providers/cloudfiles/provider.py +++ b/waterbutler/providers/cloudfiles/provider.py @@ -192,7 +192,7 @@ async def delete(self, path, confirm_delete=False): await resp.release() @ensure_connection - async def metadata(self, path, recursive=False, version=None, revision=None): + async def metadata(self, path, recursive=False, version=None, revision=None, **kwargs): """Get Metadata about the requested file or folder :param str path: The path to a key or folder :rtype dict: From 03442e3eaecb10cb5155fe6a65fae6ea72af9f80 Mon Sep 17 00:00:00 2001 From: John Tordoff Date: Fri, 20 Oct 2017 15:05:20 -0400 Subject: [PATCH 04/13] fix import order --- tests/providers/cloudfiles/fixtures.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/providers/cloudfiles/fixtures.py b/tests/providers/cloudfiles/fixtures.py index c7704198d..deb6148f3 100644 --- a/tests/providers/cloudfiles/fixtures.py +++ b/tests/providers/cloudfiles/fixtures.py @@ -2,13 +2,12 @@ import io import time import json +from unittest import mock + import pytest import aiohttp import aiohttpretty -from unittest import mock - - from waterbutler.core import streams from waterbutler.providers.cloudfiles import CloudFilesProvider From a596fa59b2f1df821c30c7385368b51b9a97ae97 Mon Sep 17 00:00:00 2001 From: John Tordoff Date: Fri, 20 Oct 2017 15:37:35 -0400 Subject: [PATCH 05/13] Add doc strings and clean up --- waterbutler/providers/cloudfiles/provider.py | 41 +++++++++++++++++--- 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/waterbutler/providers/cloudfiles/provider.py b/waterbutler/providers/cloudfiles/provider.py index d189e3c55..3720c191f 100644 --- a/waterbutler/providers/cloudfiles/provider.py +++ b/waterbutler/providers/cloudfiles/provider.py @@ -5,6 +5,7 @@ import asyncio import hashlib import functools +from urllib.parse import unquote import furl @@ -35,8 +36,7 @@ async def wrapped(self, *args, **kwargs): class CloudFilesProvider(provider.BaseProvider): """Provider for Rackspace CloudFiles. - API Docs: https://developer.rackspace.com/docs/cloud-files/v1/developer-guide/ - #document-developer-guide + API Docs: https://developer.rackspace.com/docs/cloud-files/v1/developer-guide/document-developer-guide """ NAME = 'cloudfiles' @@ -158,9 +158,18 @@ async def upload(self, stream, path, check_created=True, fetch_metadata=True): async def delete(self, path, confirm_delete=False): """Deletes the key at the specified path :param str path: The path of the key to delete - :param bool confirm_delete: not used in this provider + :param int confirm_delete: Must be 1 to confirm root folder delete, this deletes entire + container object. :rtype None: """ + + if path.is_root and not confirm_delete: + raise exceptions.DeleteError( + 'query arguement confirm_delete=1 is required for deleting the entire container.', + code=400 + ) + + if path.is_dir: metadata = await self._metadata_folder(path, recursive=True) @@ -232,7 +241,6 @@ def sign_url(self, path, method='GET', endpoint=None, seconds=settings.TEMP_URL_ :param int seconds: Time for the url to live :rtype str: """ - from urllib.parse import unquote method = method.upper() expires = str(int(time.time() + seconds)) @@ -399,13 +407,22 @@ def _serialize_metadata(self, data): return CloudFilesFileMetadata(data) @ensure_connection - async def create_folder(self, path, folder_precheck=None): + async def create_folder(self, path, **kwargs): + """Create a folder in the current provider at `path`. Returns a `BaseFolderMetadata` object + if successful. May throw a 409 Conflict if a directory with the same name already exists. + Enpoint information can be found here: + https://developer.rackspace.com/docs/cloud-files/v1/general-api-info/pseudo-hierarchical-folders-and-directories/ + :param path: ( :class:`.WaterButlerPath` )User-supplied path to create. Must be a directory. + :param kwargs: dict unused params + :rtype: :class:`.BaseFileMetadata` + :raises: :class:`.CreateFolderError` + """ resp = await self.make_request( 'PUT', functools.partial(self.sign_url, path, 'PUT'), expects=(200, 201), - throws=exceptions.UploadError, + throws=exceptions.CreateFolderError, headers={'Content-Type': 'application/directory'} ) await resp.release() @@ -433,6 +450,18 @@ async def _get_version_location(self): @ensure_connection async def revisions(self, path): + """Get past versions of the request file from special user designated version container, + if the user hasn't designated a version_location container in raises an infomative error + message. The revision endpoint also doesn't return the current version so that is added to + the revision list after other revisions are returned. More info about versioning with Cloud + Files here: + + https://developer.rackspace.com/docs/cloud-files/v1/use-cases/additional-object-services-information/#object-versioning + + :param str path: The path to a key + :rtype list: + """ + version_location = await self._get_version_location() query = {'prefix': '{:03x}'.format(len(path.name)) + path.name + '/'} From d27bd747984b4dd76f4de61c22479a0253d3638c Mon Sep 17 00:00:00 2001 From: John Tordoff Date: Fri, 20 Oct 2017 15:42:36 -0400 Subject: [PATCH 06/13] flake fix --- waterbutler/providers/cloudfiles/provider.py | 1 - 1 file changed, 1 deletion(-) diff --git a/waterbutler/providers/cloudfiles/provider.py b/waterbutler/providers/cloudfiles/provider.py index 3720c191f..e0bbb2e22 100644 --- a/waterbutler/providers/cloudfiles/provider.py +++ b/waterbutler/providers/cloudfiles/provider.py @@ -169,7 +169,6 @@ async def delete(self, path, confirm_delete=False): code=400 ) - if path.is_dir: metadata = await self._metadata_folder(path, recursive=True) From 87e5fca24cc7d454b123175482a541e943b21e7c Mon Sep 17 00:00:00 2001 From: John Tordoff Date: Fri, 20 Oct 2017 15:52:50 -0400 Subject: [PATCH 07/13] fix misspelling --- waterbutler/providers/cloudfiles/provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/waterbutler/providers/cloudfiles/provider.py b/waterbutler/providers/cloudfiles/provider.py index e0bbb2e22..0e669d25f 100644 --- a/waterbutler/providers/cloudfiles/provider.py +++ b/waterbutler/providers/cloudfiles/provider.py @@ -165,7 +165,7 @@ async def delete(self, path, confirm_delete=False): if path.is_root and not confirm_delete: raise exceptions.DeleteError( - 'query arguement confirm_delete=1 is required for deleting the entire container.', + 'query argument confirm_delete=1 is required for deleting the entire container.', code=400 ) From 34860a2bc35b7f90b410f9811892bdb69aa965d3 Mon Sep 17 00:00:00 2001 From: John Tordoff Date: Tue, 24 Oct 2017 14:32:26 -0400 Subject: [PATCH 08/13] add chunked uploads for cloudfiles --- tests/providers/cloudfiles/fixtures.py | 8 +++ tests/providers/cloudfiles/test_provider.py | 59 ++++++++++++++++++++ waterbutler/providers/cloudfiles/provider.py | 42 +++++++++++++- waterbutler/server/settings.py | 2 +- 4 files changed, 108 insertions(+), 3 deletions(-) diff --git a/tests/providers/cloudfiles/fixtures.py b/tests/providers/cloudfiles/fixtures.py index deb6148f3..ffbca4244 100644 --- a/tests/providers/cloudfiles/fixtures.py +++ b/tests/providers/cloudfiles/fixtures.py @@ -94,6 +94,10 @@ def connected_provider(provider, token, endpoint, temp_url_key, mock_time): def file_content(): return b'sleepy' +@pytest.fixture +def file_content_100_bytes(): + return os.urandom(100) + @pytest.fixture def file_like(file_content): @@ -105,6 +109,10 @@ def file_stream(file_like): return streams.FileStreamReader(file_like) +@pytest.fixture +def file_stream_100_bytes(file_content_100_bytes): + return streams.FileStreamReader(io.BytesIO(file_content_100_bytes)) + @pytest.fixture def folder_root_empty(): diff --git a/tests/providers/cloudfiles/test_provider.py b/tests/providers/cloudfiles/test_provider.py index eaad6a6fa..2f55bb932 100644 --- a/tests/providers/cloudfiles/test_provider.py +++ b/tests/providers/cloudfiles/test_provider.py @@ -23,7 +23,9 @@ temp_url_key, mock_time, file_content, + file_content_100_bytes, file_stream, + file_stream_100_bytes, file_like, mock_temp_key, provider, @@ -142,6 +144,55 @@ async def test_upload(self, assert aiohttpretty.has_call(method='PUT', uri=url) assert aiohttpretty.has_call(method='HEAD', uri=metadata_url) + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_chunked_upload(self, + connected_provider, + file_stream_100_bytes, + file_header_metadata): + path = WaterButlerPath('/similar.file') + metadata_url = connected_provider.build_url(path.path) + aiohttpretty.register_uri('HEAD', + metadata_url, + responses=[ + {'status': 404}, + {'headers': file_header_metadata} + ] + ) + + for i in range(0, 10): + url = connected_provider.sign_url(path, 'PUT', segment_num=str(i).zfill(5)) + aiohttpretty.register_uri('PUT', url, status=200) + + url = connected_provider.build_url(path.path) + aiohttpretty.register_uri('PUT', url, status=200) + + connected_provider.SEGMENT_SIZE = 10 # for testing + metadata, created = await connected_provider.chunked_upload(file_stream_100_bytes, path) + + assert created is True + assert metadata.kind == 'file' + assert aiohttpretty.has_call(method='HEAD', uri=metadata_url) # check metadata was called + assert aiohttpretty.has_call(method='PUT', uri=url) # check manifest was uploaded + for i in range(0, 10): # check all 10 segments were uploaded + url = connected_provider.sign_url(path, 'PUT', segment_num=str(i).zfill(5)) + assert aiohttpretty.has_call(method='PUT', uri=url) + + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_chunked_upload_segment_size(self, + connected_provider, + file_stream_100_bytes, + file_header_metadata): + + path = WaterButlerPath('/similar.file') + + connected_provider.chunked_upload = MockCoroutine() + connected_provider.SEGMENT_SIZE = 10 # for test, we dont want to load a 5GB fixture. + await connected_provider.upload(file_stream_100_bytes, path) + + assert connected_provider.chunked_upload.called_with(file_stream_100_bytes, path) + @pytest.mark.asyncio @pytest.mark.aiohttpretty async def test_create_folder(self, connected_provider, file_header_metadata): @@ -194,6 +245,14 @@ async def test_upload_checksum_mismatch(self, assert aiohttpretty.has_call(method='PUT', uri=url) assert aiohttpretty.has_call(method='HEAD', uri=metadata_url) + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_delete_root_without_confirm(self, connected_provider, folder_root_empty, file_header_metadata): + path = WaterButlerPath('/') + + with pytest.raises(exceptions.DeleteError): + await connected_provider.delete(path) + @pytest.mark.asyncio @pytest.mark.aiohttpretty async def test_delete_folder(self, connected_provider, folder_root_empty, file_header_metadata): diff --git a/waterbutler/providers/cloudfiles/provider.py b/waterbutler/providers/cloudfiles/provider.py index 0e669d25f..69780e4c1 100644 --- a/waterbutler/providers/cloudfiles/provider.py +++ b/waterbutler/providers/cloudfiles/provider.py @@ -40,6 +40,8 @@ class CloudFilesProvider(provider.BaseProvider): """ NAME = 'cloudfiles' + SEGMENT_SIZE = 50000 # 5 GB + def __init__(self, auth, credentials, settings): super().__init__(auth, credentials, settings) self.token = None @@ -126,6 +128,11 @@ async def upload(self, stream, path, check_created=True, fetch_metadata=True): :param bool fetch_metadata: If true upload will return metadata :rtype (dict/None, bool): """ + if stream.size > self.SEGMENT_SIZE: + return await self.chunked_upload(stream, path, + check_created=check_created, + fetch_metadata=fetch_metadata) + if check_created: created = not (await self.exists(path)) else: @@ -154,6 +161,36 @@ async def upload(self, stream, path, check_created=True, fetch_metadata=True): return metadata, created + @ensure_connection + async def chunked_upload(self, stream, path, check_created=True, fetch_metadata=True): + + created = not (await self.exists(path)) if check_created else None + + for i, _ in enumerate(range(0, stream.size, self.SEGMENT_SIZE)): + data = await stream.read(self.SEGMENT_SIZE) + resp = await self.make_request( + 'PUT', + functools.partial(self.sign_url, path, 'PUT', segment_num=str(i).zfill(5)), + data=data, + headers={'Content-Length': str(len(bytearray(data)))}, + expects=(200, 201), + throws=exceptions.UploadError, + ) + await resp.release() + + resp = await self.make_request( + 'PUT', + functools.partial(self.build_url, path.path), + headers={'X-Object-Manifest': '{}/{}'.format(self.container, path.path)}, + expects=(200, 201), + throws=exceptions.UploadError, + ) + await resp.release() + + metadata = await self.metadata(path) if fetch_metadata else None + + return metadata, created + @ensure_connection async def delete(self, path, confirm_delete=False): """Deletes the key at the specified path @@ -232,7 +269,7 @@ def can_intra_copy(self, dest_provider, path=None): def can_intra_move(self, dest_provider, path=None): return type(self) == type(dest_provider) and not getattr(path, 'is_dir', False) - def sign_url(self, path, method='GET', endpoint=None, seconds=settings.TEMP_URL_SECS): + def sign_url(self, path, method='GET', endpoint=None, segment_num=None, seconds=settings.TEMP_URL_SECS): """Sign a temp url for the specified stream :param str stream: The requested stream's path :param CloudFilesPath path: A path to a file/folder @@ -243,7 +280,8 @@ def sign_url(self, path, method='GET', endpoint=None, seconds=settings.TEMP_URL_ method = method.upper() expires = str(int(time.time() + seconds)) - path_str = path.path + path_str = path.path + '/' + segment_num if segment_num else path.path + url = furl.furl(self.build_url(path_str, _endpoint=endpoint)) body = '\n'.join([method, expires]) diff --git a/waterbutler/server/settings.py b/waterbutler/server/settings.py index 7dee586e0..9676b39d4 100644 --- a/waterbutler/server/settings.py +++ b/waterbutler/server/settings.py @@ -18,7 +18,7 @@ CORS_ALLOW_ORIGIN = config.get('CORS_ALLOW_ORIGIN', '*') CHUNK_SIZE = int(config.get('CHUNK_SIZE', 65536)) # 64KB -MAX_BODY_SIZE = int(config.get('MAX_BODY_SIZE', int(4.9 * (1024 ** 3)))) # 4.9 GB +MAX_BODY_SIZE = int(config.get('MAX_BODY_SIZE', 10 ** 12)) # 1 TB AUTH_HANDLERS = config.get('AUTH_HANDLERS', [ 'osf', From 0757cc61a334d840c8d9d04e2c4c4509955e81d5 Mon Sep 17 00:00:00 2001 From: John Tordoff Date: Mon, 6 Nov 2017 15:18:12 -0500 Subject: [PATCH 09/13] Add conflict handling and clean up tests --- tests/providers/cloudfiles/test_provider.py | 5 +---- waterbutler/providers/cloudfiles/provider.py | 9 ++++++++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/providers/cloudfiles/test_provider.py b/tests/providers/cloudfiles/test_provider.py index 2f55bb932..6df07bcc4 100644 --- a/tests/providers/cloudfiles/test_provider.py +++ b/tests/providers/cloudfiles/test_provider.py @@ -180,10 +180,7 @@ async def test_chunked_upload(self, @pytest.mark.asyncio @pytest.mark.aiohttpretty - async def test_chunked_upload_segment_size(self, - connected_provider, - file_stream_100_bytes, - file_header_metadata): + async def test_chunked_upload_segment(self, connected_provider, file_stream_100_bytes): path = WaterButlerPath('/similar.file') diff --git a/waterbutler/providers/cloudfiles/provider.py b/waterbutler/providers/cloudfiles/provider.py index 69780e4c1..95cc0c74d 100644 --- a/waterbutler/providers/cloudfiles/provider.py +++ b/waterbutler/providers/cloudfiles/provider.py @@ -120,7 +120,11 @@ async def download(self, return streams.ResponseStreamReader(resp) @ensure_connection - async def upload(self, stream, path, check_created=True, fetch_metadata=True): + async def upload(self, stream, + path, + check_created=True, + fetch_metadata=True, + conflict='replace'): """Uploads the given stream to CloudFiles :param ResponseStreamReader stream: The stream to put to CloudFiles :param str path: The full path of the object to upload to/into @@ -128,6 +132,9 @@ async def upload(self, stream, path, check_created=True, fetch_metadata=True): :param bool fetch_metadata: If true upload will return metadata :rtype (dict/None, bool): """ + + path, _ = await self.handle_name_conflict(path, conflict=conflict, kind=path.kind) + if stream.size > self.SEGMENT_SIZE: return await self.chunked_upload(stream, path, check_created=check_created, From d1a7a2b484dd9aaa98ba042e76868ceb4f7ab6b7 Mon Sep 17 00:00:00 2001 From: John Tordoff Date: Mon, 6 Nov 2017 15:19:44 -0500 Subject: [PATCH 10/13] remove extra blank line --- tests/providers/cloudfiles/test_provider.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/providers/cloudfiles/test_provider.py b/tests/providers/cloudfiles/test_provider.py index 6df07bcc4..2212a5588 100644 --- a/tests/providers/cloudfiles/test_provider.py +++ b/tests/providers/cloudfiles/test_provider.py @@ -327,7 +327,6 @@ async def test_revisions(self, metadata_url = connected_provider.build_url(path.path) aiohttpretty.register_uri('HEAD', metadata_url, status=200, headers=file_header_metadata) - result = await connected_provider.revisions(path) assert type(result) == list From 71ddc861554451f031a3a286306f49f7aaa760e2 Mon Sep 17 00:00:00 2001 From: John Tordoff Date: Mon, 6 Nov 2017 15:38:25 -0500 Subject: [PATCH 11/13] handle name conflict handling. --- waterbutler/providers/cloudfiles/provider.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/waterbutler/providers/cloudfiles/provider.py b/waterbutler/providers/cloudfiles/provider.py index 95cc0c74d..d96308d2a 100644 --- a/waterbutler/providers/cloudfiles/provider.py +++ b/waterbutler/providers/cloudfiles/provider.py @@ -133,7 +133,9 @@ async def upload(self, stream, :rtype (dict/None, bool): """ - path, _ = await self.handle_name_conflict(path, conflict=conflict, kind=path.kind) + if path.identifier and conflict == 'keep': + path, _ = await self.handle_name_conflict(path, conflict=conflict, kind='folder') + path._parts[-1]._id = None if stream.size > self.SEGMENT_SIZE: return await self.chunked_upload(stream, path, From 2bfa324e28a92bfd59489f5d4a736a6487b83109 Mon Sep 17 00:00:00 2001 From: John Tordoff Date: Tue, 7 Nov 2017 09:36:24 -0500 Subject: [PATCH 12/13] fix segment size --- waterbutler/providers/cloudfiles/provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/waterbutler/providers/cloudfiles/provider.py b/waterbutler/providers/cloudfiles/provider.py index d96308d2a..1f0c79a00 100644 --- a/waterbutler/providers/cloudfiles/provider.py +++ b/waterbutler/providers/cloudfiles/provider.py @@ -40,7 +40,7 @@ class CloudFilesProvider(provider.BaseProvider): """ NAME = 'cloudfiles' - SEGMENT_SIZE = 50000 # 5 GB + SEGMENT_SIZE = 5000000000 # 5 GB def __init__(self, auth, credentials, settings): super().__init__(auth, credentials, settings) From 02f4cac362696c1a377ff9bb7fe5bda3b3d4d3a9 Mon Sep 17 00:00:00 2001 From: John Tordoff Date: Thu, 16 Nov 2017 13:30:03 -0500 Subject: [PATCH 13/13] Update provider.py --- waterbutler/core/provider.py | 1 + 1 file changed, 1 insertion(+) diff --git a/waterbutler/core/provider.py b/waterbutler/core/provider.py index 559f9a46a..ba443e33f 100644 --- a/waterbutler/core/provider.py +++ b/waterbutler/core/provider.py @@ -323,6 +323,7 @@ async def _folder_file_op(self, """ assert src_path.is_dir, 'src_path must be a directory' assert asyncio.iscoroutinefunction(func), 'func must be a coroutine' + try: await dest_provider.delete(dest_path) created = False