Merge branch 'develop'

.gitignore

@@ -11,4 +11,4 @@ build/
 
 venv
 /env
-/.DS_Store
+.DS_Store

arcsecond/__init__.py

@@ -7,4 +7,4 @@
            "ArcsecondConnectionError",
            "ArcsecondInvalidEndpointError"]
 
-__version__ = '0.7.4'
+__version__ = '0.7.5'

arcsecond/api/endpoints/_base.py

@@ -17,7 +17,7 @@
     API_AUTH_PATH_REGISTER)
 
 from arcsecond.api.error import ArcsecondConnectionError, ArcsecondError
-from arcsecond.api.helpers import transform_payload_for_multipart_encoder_fields
+from arcsecond.api.helpers import extract_multipart_encoder_file_fields
 from arcsecond.config import config_file_read_api_key, config_file_read_organisation_memberships
 from arcsecond.options import State
 
@@ -30,6 +30,61 @@
 EVENT_METHOD_PROGRESS_PERCENT = 'EVENT_METHOD_PROGRESS_PERCENT'
 
 
+class AsyncFileUploader(object):
+    """AsyncFileUploader is a helper class used when uploading files to the cloud.
+
+    Technically speaking, it can handle any http request in a background thread.
+    It is however named like this because it is returned in place of a standard
+    response payload when a file is to be uploaded.
+    """
+
+    def __init__(self, url, method, data=None, payload=None, **headers):
+        self.url = url
+        self.method = method
+        self.payload = payload
+        self.data = data
+        self.headers = headers
+        self._storage = {}
+        self._thread = None
+
+    def start(self):
+        if self._thread is None:
+            args = (self.url, self.method, self.data, self.payload, self.headers)
+            self._thread = threading.Thread(target=self._target, args=args)
+        self._thread.start()
+
+    def _target(self, url, method, data, payload, headers):
+        try:
+            self._storage['response'] = method(url, data=data, json=payload, headers=headers)
+        except requests.exceptions.ConnectionError:
+            self._storage['error'] = ArcsecondConnectionError(url)
+        except Exception as e:
+            self._storage['error'] = ArcsecondError(str(e))
+
+    def finish(self):
+        self.join()
+        return self.get_results()
+
+    def join(self):
+        self._thread.join()
+
+    def is_alive(self):
+        return self._thread.is_alive()
+
+    def get_results(self):
+        response = self._storage.get('response')
+        if isinstance(response, dict):
+            # Responses of standard JSON payload requests are dict
+            return response
+        elif response is not None:
+            if 200 <= response.status_code < 300:
+                return response.json() if response.text else {}, None
+            else:
+                return None, response.text
+        else:
+            return None, self._storage.get('error')
+
+
 class APIEndPoint(object):
     name = None
 
@@ -72,62 +127,50 @@ def _check_uuid(self, uuid_str):
         except ValueError:
             raise ArcsecondError('Invalid UUID {}.'.format(uuid_str))
 
-    def _check_and_set_api_key(self, headers, url):
-        if API_AUTH_PATH_REGISTER in url or API_AUTH_PATH_LOGIN in url or 'Authorization' in headers.keys():
-            return headers
-
-        if self.state.verbose:
-            click.echo('Checking local API key... ', nl=False)
-
-        api_key = config_file_read_api_key(self.state.config_section())
-        if not api_key:
-            raise ArcsecondError('Missing API key. You must login first: $ arcsecond login')
-
-        headers['X-Arcsecond-API-Authorization'] = 'Key ' + api_key
+    def list(self, name='', **headers):
+        return self._perform_request(self._list_url(name), 'get', None, None, **headers)
 
-        if self.state.verbose:
-            click.echo('OK')
-        return headers
+    def create(self, payload, callback=None, **headers):
+        # If a file is provided as part of the payload, a instance of AsyncFileUploader is returned
+        # in place of a standard JSON body response.
+        return self._perform_request(self._list_url(), 'post', payload, callback, **headers)
 
-    def _check_organisation_membership_and_permission(self, method_name, organisation):
-        memberships = config_file_read_organisation_memberships(self.state.config_section())
-        if self.state.organisation not in memberships.keys():
-            raise ArcsecondError('No membership found for organisation {}'.format(organisation))
+    def read(self, id_name_uuid, **headers):
+        return self._perform_request(self._detail_url(id_name_uuid), 'get', None, None, **headers)
 
-        membership = memberships[self.state.organisation]
-        if method_name not in SAFE_METHODS and membership not in WRITABLE_MEMBERSHIPS:
-            raise ArcsecondError('Membership for organisation {} has no write permission'.format(organisation))
+    def update(self, id_name_uuid, payload, **headers):
+        return self._perform_request(self._detail_url(id_name_uuid), 'put', payload, None, **headers)
 
-    def _async_perform_request(self, url, method, payload=None, **headers):
-        def _async_perform_request_store_response(storage, method, url, payload, headers):
-            try:
-                storage['response'] = method(url, json=payload, headers=headers)
-            except requests.exceptions.ConnectionError:
-                storage['error'] = ArcsecondConnectionError(self._get_base_url())
-            except Exception as e:
-                storage['error'] = ArcsecondError(str(e))
+    def delete(self, id_name_uuid, **headers):
+        return self._perform_request(self._detail_url(id_name_uuid), 'delete', None, None, **headers)
 
-        storage = {}
-        thread = threading.Thread(target=_async_perform_request_store_response,
-                                  args=(storage, method, url, payload, headers))
-        thread.start()
+    def _perform_request(self, url, method, payload, callback=None, **headers):
+        method_name, method, payload, headers = self._prepare_request(url, method, payload, **headers)
 
-        spinner = Spinner()
-        while thread.is_alive():
-            if self.state.verbose:
-                spinner.next()
-        thread.join()
-        if self.state.verbose:
-            click.echo()
+        payload, fields = extract_multipart_encoder_file_fields(payload)
+        if fields is None:
+            # Standard JSON sync request
+            return self._perform_spinner_request(url, method, method_name, None, payload, **headers)
+        else:
+            # Process payload synchronously nonetheless
+            if payload:
+                self._perform_spinner_request(url, method, method_name, None, payload, **headers)
 
-        if 'error' in storage.keys():
-            raise storage.get('error')
+            # File upload
+            upload_monitor = self._build_dynamic_upload_data(fields, callback)
+            headers.update(**{'Content-Type': upload_monitor.content_type})
 
-        return storage.get('response', None)
+            if self.state.is_using_cli:
+                return self._perform_spinner_request(url, method, method_name, upload_monitor, None, **headers)
+            else:
+                return AsyncFileUploader(url, method, data=upload_monitor, payload=None, **headers)
 
     def _prepare_request(self, url, method, payload, **headers):
         assert (url and method)
 
+        if self.state.verbose:
+            click.echo('Preparing request...')
+
         if not isinstance(method, str) or callable(method):
             raise ArcsecondError('Invalid HTTP request method {}. '.format(str(method)))
 
@@ -145,62 +188,73 @@ def _prepare_request(self, url, method, payload, **headers):
             # Filtering None values out of payload.
             payload = {k: v for k, v in payload.items() if v is not None}
 
-        return url, method_name, method, payload, headers
+        return method_name, method, payload, headers
 
-    def _perform_request(self, url, method, payload, callback=None, **headers):
-        if self.state.verbose:
-            click.echo('Preparing request...')
+    def _build_dynamic_upload_data(self, fields, callback=None):
+        # The monitor is the data!
+        encoded_data = encoder.MultipartEncoder(fields=fields)
 
-        url, method_name, method, payload, headers = self._prepare_request(url, method, payload, **headers)
+        if self.state.is_using_cli is True and self.state.verbose:
+            bar = Bar('Uploading ' + fields['file'][0], suffix='%(percent)d%%')
+            return encoder.MultipartEncoderMonitor(encoded_data, lambda m: bar.goto(m.bytes_read / m.len * 100))
+        elif self.state.is_using_cli is False and callback:
+            return encoder.MultipartEncoderMonitor(encoded_data, lambda m: callback(EVENT_METHOD_PROGRESS_PERCENT,
+                                                                                    m.bytes_read / m.len * 100))
+        else:
+            return encoder.MultipartEncoderMonitor(encoded_data, None)
 
+    def _perform_spinner_request(self, url, method, method_name, data=None, payload=None, **headers):
         if self.state.verbose:
             click.echo('Sending {} request to {}'.format(method_name, url))
+            click.echo('Payload: {}'.format(payload))
 
-        payload, fields = transform_payload_for_multipart_encoder_fields(payload)
-        if fields:
-            encoded_data = encoder.MultipartEncoder(fields=fields)
-            bar, upload_callback = None, None
+        performer = AsyncFileUploader(url, method, data=data, payload=payload, **headers)
+        performer.start()
 
-            if self.state.is_using_cli is False and callback:
-                upload_callback = lambda m: callback(EVENT_METHOD_PROGRESS_PERCENT, m.bytes_read / m.len * 100)
-            elif self.state.verbose:
-                bar = Bar('Uploading ' + fields['file'][0], suffix='%(percent)d%%')
-                upload_callback = lambda m: bar.goto(m.bytes_read / m.len * 100)
-
-            upload_monitor = encoder.MultipartEncoderMonitor(encoded_data, upload_callback)
-            headers.update(**{'Content-Type': upload_monitor.content_type})
-            response = method(url, data=upload_monitor, headers=headers)
-
-            if self.state.verbose:
-                bar.finish()
-        else:
+        spinner = Spinner()
+        while performer.is_alive():
             if self.state.verbose:
-                click.echo('Payload: {}'.format(payload))
+                spinner.next()
+
+        response, error = performer.finish()
 
-            response = self._async_perform_request(url, method, payload, **headers)
+        # If we have an error and it is an ArcsecondError, raise it.
+        # As for now, only ArcsecondError could be returned, and there is no
+        # real point of returning both response and error below. But
+        # methods in main.py expect them both.
 
-        if response is None:
-            raise ArcsecondConnectionError(url)
+        if error and isinstance(error, ArcsecondError):
+            raise error
 
         if self.state.verbose:
+            click.echo()
             click.echo('Request status code ' + str(response.status_code))
 
-        if 200 <= response.status_code < 300:
-            return response.json() if response.text else {}, None
-        else:
-            return None, response.text
+        return response, error
 
-    def list(self, name='', **headers):
-        return self._perform_request(self._list_url(name), 'get', None, None, **headers)
+    def _check_and_set_api_key(self, headers, url):
+        if API_AUTH_PATH_REGISTER in url or API_AUTH_PATH_LOGIN in url or 'Authorization' in headers.keys():
+            return headers
 
-    def create(self, payload, callback=None, **headers):
-        return self._perform_request(self._list_url(), 'post', payload, callback, **headers)
+        if self.state.verbose:
+            click.echo('Checking local API key... ', nl=False)
 
-    def read(self, id_name_uuid, **headers):
-        return self._perform_request(self._detail_url(id_name_uuid), 'get', None, None, **headers)
+        api_key = config_file_read_api_key(self.state.config_section())
+        if not api_key:
+            raise ArcsecondError('Missing API key. You must login first: $ arcsecond login')
 
-    def update(self, id_name_uuid, payload, **headers):
-        return self._perform_request(self._detail_url(id_name_uuid), 'put', payload, None, **headers)
+        headers['X-Arcsecond-API-Authorization'] = 'Key ' + api_key
 
-    def delete(self, id_name_uuid, **headers):
-        return self._perform_request(self._detail_url(id_name_uuid), 'delete', None, None, **headers)
+        if self.state.verbose:
+            click.echo('OK')
+
+        return headers
+
+    def _check_organisation_membership_and_permission(self, method_name, organisation):
+        memberships = config_file_read_organisation_memberships(self.state.config_section())
+        if self.state.organisation not in memberships.keys():
+            raise ArcsecondError('No membership found for organisation {}'.format(organisation))
+
+        membership = memberships[self.state.organisation]
+        if method_name not in SAFE_METHODS and membership not in WRITABLE_MEMBERSHIPS:
+            raise ArcsecondError('Membership for organisation {} has no write permission'.format(organisation))

arcsecond/api/error.py

@@ -23,7 +23,7 @@ def __init__(self, endpoint, endpoints):
 class ArcsecondConnectionError(ArcsecondError):
     def __init__(self, url):
         msg = "Unable to connect to API server {}.\n".format(url)
-        msg += "Suggestion: Test whether it's reachable, by typing for instance: 'ping {}'".format(url)
+        msg += "Suggestion: Test whether it's reachable, by typing for instance: 'curl -Is {} | head -n 1'".format(url)
         super(ArcsecondConnectionError, self).__init__(msg)
 
 

arcsecond/api/helpers.py

@@ -7,7 +7,7 @@ def make_file_upload_multipart_dict(filepath):
     return {'fields': {'file': (os.path.basename(filepath), open(os.path.abspath(filepath), 'rb'))}}
 
 
-def transform_payload_for_multipart_encoder_fields(payload):
+def extract_multipart_encoder_file_fields(payload):
     if isinstance(payload, str) and os.path.exists(payload) and os.path.isfile(payload):
         payload = make_file_upload_multipart_dict(payload)  # transform a str into a dict
 

arcsecond/api/main.py

@@ -27,11 +27,13 @@
                         PersonalProfileAPIEndPoint, ProfileAPIEndPoint, ProfileAPIKeyAPIEndPoint, SatellitesAPIEndPoint,
                         StandardStarsAPIEndPoint, TelegramsATelAPIEndPoint, TelescopesAPIEndPoint)
 
+from .endpoints._base import AsyncFileUploader
+
 pp = pprint.PrettyPrinter(indent=4, depth=5)
 ECHO_PREFIX = u' • '
 ECHO_ERROR_PREFIX = u' • [error] '
 
-__all__ = ["ArcsecondAPI"]
+__all__ = ["ArcsecondAPI", "AsyncFileUploader"]
 
 ENDPOINTS = [ActivitiesAPIEndPoint,
              CataloguesAPIEndPoint,
@@ -156,6 +158,8 @@ def _echo_error(cls, state, error):
                 click.echo(ECHO_PREFIX + str(error))
 
     def _echo_response(self, response):
+        if isinstance(response, AsyncFileUploader):
+            return response
         result, error = response
         if result is not None:  # check against None, to avoid skipping empty lists.
             return ArcsecondAPI._echo_result(self.state, result)

tests/test_activities.py → tests/cli/test_activities.py

@@ -7,7 +7,7 @@
 from arcsecond import cli
 from arcsecond.api.constants import ARCSECOND_API_URL_DEV
 from arcsecond.api.error import ArcsecondInputValueError
-from .utils import register_successful_personal_login
+from tests.utils import register_successful_personal_login
 
 
 @httpretty.activate

tests/test_auth.py → tests/cli/test_auth.py

@@ -2,7 +2,7 @@
 import httpretty
 from click.testing import CliRunner
 from arcsecond import cli
-from arcsecond.api.constants import API_AUTH_PATH_LOGIN, ARCSECOND_API_URL_DEV, API_AUTH_PATH_REGISTER
+from arcsecond.api.constants import API_AUTH_PATH_LOGIN, ARCSECOND_API_URL_DEV
 
 python_version = sys.version_info.major
 

tests/test_datasets_organisations.py → tests/cli/test_datasets_organisations.py

@@ -6,10 +6,10 @@
 from arcsecond.api.error import ArcsecondError
 from arcsecond.config import config_file_clear_section
 
-from .utils import (register_successful_personal_login,
-                    register_successful_organisation_login,
-                    mock_http_get,
-                    mock_http_post)
+from tests.utils import (register_successful_personal_login,
+                         register_successful_organisation_login,
+                         mock_http_get,
+                         mock_http_post)
 
 
 class DatasetsInOrganisationsTestCase(TestCase):

tests/test_datasets_personal.py → tests/cli/test_datasets_personal.py

@@ -7,7 +7,7 @@
 
 from arcsecond import cli
 from arcsecond.api.constants import ARCSECOND_API_URL_DEV
-from .utils import register_successful_personal_login
+from tests.utils import register_successful_personal_login
 
 
 @httpretty.activate

tests/test_arcsecond_root.py → tests/module/test_arcsecond_root.py

@@ -1,5 +1,5 @@
 from arcsecond import Arcsecond
-from .utils import save_test_credentials, clear_test_credentials
+from tests.utils import save_test_credentials, clear_test_credentials
 
 
 def test_default_empty_state():