From 57b44255738f4484f6c4e8b44a2a9ea67578967d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ce=CC=81dric=20Foellmi?= Date: Sun, 19 Jan 2020 10:15:00 +0100 Subject: [PATCH 01/14] =?UTF-8?q?Fixed=20wrong=20suggestion=20(ping=20does?= =?UTF-8?q?n=E2=80=99t=20work=20with=20http=20urls)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- arcsecond/api/error.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arcsecond/api/error.py b/arcsecond/api/error.py index 19e2b58..6f4387c 100644 --- a/arcsecond/api/error.py +++ b/arcsecond/api/error.py @@ -23,7 +23,7 @@ def __init__(self, endpoint, endpoints): class ArcsecondConnectionError(ArcsecondError): def __init__(self, url): msg = "Unable to connect to API server {}.\n".format(url) - msg += "Suggestion: Test whether it's reachable, by typing for instance: 'ping {}'".format(url) + msg += "Suggestion: Test whether it's reachable, by typing for instance: 'curl -Is {} | head -n 1'".format(url) super(ArcsecondConnectionError, self).__init__(msg) From a430882b1fac01a41e70e7a80b4b11adc2348995 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ce=CC=81dric=20Foellmi?= Date: Sun, 19 Jan 2020 10:15:16 +0100 Subject: [PATCH 02/14] Small refactor of utility method --- arcsecond/api/endpoints/_base.py | 2 +- arcsecond/api/helpers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/arcsecond/api/endpoints/_base.py b/arcsecond/api/endpoints/_base.py index 3e75700..cbebcb5 100644 --- a/arcsecond/api/endpoints/_base.py +++ b/arcsecond/api/endpoints/_base.py @@ -17,7 +17,7 @@ API_AUTH_PATH_REGISTER) from arcsecond.api.error import ArcsecondConnectionError, ArcsecondError -from arcsecond.api.helpers import transform_payload_for_multipart_encoder_fields +from arcsecond.api.helpers import extract_multipart_encoder_file_fields from arcsecond.config import config_file_read_api_key, config_file_read_organisation_memberships from arcsecond.options import State diff --git a/arcsecond/api/helpers.py b/arcsecond/api/helpers.py index 1905d2d..bd5827e 100644 --- a/arcsecond/api/helpers.py +++ b/arcsecond/api/helpers.py @@ -7,7 +7,7 @@ def make_file_upload_multipart_dict(filepath): return {'fields': {'file': (os.path.basename(filepath), open(os.path.abspath(filepath), 'rb'))}} -def transform_payload_for_multipart_encoder_fields(payload): +def extract_multipart_encoder_file_fields(payload): if isinstance(payload, str) and os.path.exists(payload) and os.path.isfile(payload): payload = make_file_upload_multipart_dict(payload) # transform a str into a dict From a69b9545515214e17ab7054f1aee5240c8db557d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ce=CC=81dric=20Foellmi?= Date: Sun, 19 Jan 2020 10:15:39 +0100 Subject: [PATCH 03/14] Complete rewrite of performing the request to allow a true threaded request. --- arcsecond/api/endpoints/_base.py | 190 ++++++++++++++++++------------- 1 file changed, 110 insertions(+), 80 deletions(-) diff --git a/arcsecond/api/endpoints/_base.py b/arcsecond/api/endpoints/_base.py index cbebcb5..a57d9c3 100644 --- a/arcsecond/api/endpoints/_base.py +++ b/arcsecond/api/endpoints/_base.py @@ -1,5 +1,6 @@ import threading import uuid +from typing import Tuple import click import requests @@ -30,6 +31,41 @@ EVENT_METHOD_PROGRESS_PERCENT = 'EVENT_METHOD_PROGRESS_PERCENT' +class AsyncRequestPerformer(object): + def __init__(self, url, method, data=None, payload=None, **headers): + self.url = url + self.method = method + self.payload = payload + self.data = data + self.headers = headers + + self._storage = {} + self._thread = None + + def start(self): + if self._thread is None: + args = (self.url, self.method, self.data, self.payload, self.headers) + self._thread = threading.Thread(target=self._target, args=args) + self._thread.start() + + def _target(self, url, method, data, payload, headers): + try: + self._storage['response'] = method(url, data=data, json=payload, headers=headers) + except requests.exceptions.ConnectionError: + self._storage['error'] = ArcsecondConnectionError(url) + except Exception as e: + self._storage['error'] = ArcsecondError(str(e)) + + def join(self): + self._thread.join() + + def is_alive(self): + return self._thread.is_alive() + + def get_results(self): + return self._storage.get('response'), self._storage.get('error') + + class APIEndPoint(object): name = None @@ -72,62 +108,44 @@ def _check_uuid(self, uuid_str): except ValueError: raise ArcsecondError('Invalid UUID {}.'.format(uuid_str)) - def _check_and_set_api_key(self, headers, url): - if API_AUTH_PATH_REGISTER in url or API_AUTH_PATH_LOGIN in url or 'Authorization' in headers.keys(): - return headers - - if self.state.verbose: - click.echo('Checking local API key... ', nl=False) - - api_key = config_file_read_api_key(self.state.config_section()) - if not api_key: - raise ArcsecondError('Missing API key. You must login first: $ arcsecond login') - - headers['X-Arcsecond-API-Authorization'] = 'Key ' + api_key - - if self.state.verbose: - click.echo('OK') - return headers + def list(self, name='', **headers): + return self._perform_request(self._list_url(name), 'get', None, None, **headers) - def _check_organisation_membership_and_permission(self, method_name, organisation): - memberships = config_file_read_organisation_memberships(self.state.config_section()) - if self.state.organisation not in memberships.keys(): - raise ArcsecondError('No membership found for organisation {}'.format(organisation)) + def create(self, payload, callback=None, **headers): + return self._perform_request(self._list_url(), 'post', payload, callback, **headers) - membership = memberships[self.state.organisation] - if method_name not in SAFE_METHODS and membership not in WRITABLE_MEMBERSHIPS: - raise ArcsecondError('Membership for organisation {} has no write permission'.format(organisation)) + def read(self, id_name_uuid, **headers): + return self._perform_request(self._detail_url(id_name_uuid), 'get', None, None, **headers) - def _async_perform_request(self, url, method, payload=None, **headers): - def _async_perform_request_store_response(storage, method, url, payload, headers): - try: - storage['response'] = method(url, json=payload, headers=headers) - except requests.exceptions.ConnectionError: - storage['error'] = ArcsecondConnectionError(self._get_base_url()) - except Exception as e: - storage['error'] = ArcsecondError(str(e)) + def update(self, id_name_uuid, payload, **headers): + return self._perform_request(self._detail_url(id_name_uuid), 'put', payload, None, **headers) - storage = {} - thread = threading.Thread(target=_async_perform_request_store_response, - args=(storage, method, url, payload, headers)) - thread.start() + def delete(self, id_name_uuid, **headers): + return self._perform_request(self._detail_url(id_name_uuid), 'delete', None, None, **headers) - spinner = Spinner() - while thread.is_alive(): - if self.state.verbose: - spinner.next() - thread.join() - if self.state.verbose: - click.echo() + def _perform_request(self, url, method, payload, callback=None, **headers) -> (Tuple, AsyncRequestPerformer): + method_name, method, payload, headers = self._prepare_request(url, method, payload, **headers) - if 'error' in storage.keys(): - raise storage.get('error') + payload, fields = extract_multipart_encoder_file_fields(payload) + if fields is None: + # Standard JSON sync request + return self._perform_spinner_request(url, method, method_name, None, payload, **headers) + else: + # File upload + upload_monitor = self._build_dynamic_upload_data(fields, callback) + headers.update(**{'Content-Type': upload_monitor.content_type}) - return storage.get('response', None) + if self.state.is_using_cli: + return self._perform_spinner_request(url, method, method_name, upload_monitor, None, **headers) + else: + return AsyncRequestPerformer(url, method, data=upload_monitor, payload=None, **headers) def _prepare_request(self, url, method, payload, **headers): assert (url and method) + if self.state.verbose: + click.echo('Preparing request...') + if not isinstance(method, str) or callable(method): raise ArcsecondError('Invalid HTTP request method {}. '.format(str(method))) @@ -145,44 +163,44 @@ def _prepare_request(self, url, method, payload, **headers): # Filtering None values out of payload. payload = {k: v for k, v in payload.items() if v is not None} - return url, method_name, method, payload, headers + return method_name, method, payload, headers - def _perform_request(self, url, method, payload, callback=None, **headers): - if self.state.verbose: - click.echo('Preparing request...') + def _build_dynamic_upload_data(self, fields, callback=None): + encoded_data = encoder.MultipartEncoder(fields=fields) + upload_callback = None - url, method_name, method, payload, headers = self._prepare_request(url, method, payload, **headers) + if self.state.is_using_cli is True and self.state.verbose: + bar = Bar('Uploading ' + fields['file'][0], suffix='%(percent)d%%') + upload_callback = lambda m: bar.goto(m.bytes_read / m.len * 100) + elif self.state.is_using_cli is False: + upload_callback = lambda m: callback(EVENT_METHOD_PROGRESS_PERCENT, m.bytes_read / m.len * 100) + # The monitor is the data! + return encoder.MultipartEncoderMonitor(encoded_data, upload_callback) + + def _perform_spinner_request(self, url, method, method_name, data=None, payload=None, **headers): if self.state.verbose: click.echo('Sending {} request to {}'.format(method_name, url)) + click.echo('Payload: {}'.format(payload)) - payload, fields = transform_payload_for_multipart_encoder_fields(payload) - if fields: - encoded_data = encoder.MultipartEncoder(fields=fields) - bar, upload_callback = None, None - - if self.state.is_using_cli is False and callback: - upload_callback = lambda m: callback(EVENT_METHOD_PROGRESS_PERCENT, m.bytes_read / m.len * 100) - elif self.state.verbose: - bar = Bar('Uploading ' + fields['file'][0], suffix='%(percent)d%%') - upload_callback = lambda m: bar.goto(m.bytes_read / m.len * 100) - - upload_monitor = encoder.MultipartEncoderMonitor(encoded_data, upload_callback) - headers.update(**{'Content-Type': upload_monitor.content_type}) - response = method(url, data=upload_monitor, headers=headers) + performer = AsyncRequestPerformer(url, method, data=data, payload=payload, **headers) + performer.start() + spinner = Spinner() + while performer.is_alive(): if self.state.verbose: - bar.finish() - else: - if self.state.verbose: - click.echo('Payload: {}'.format(payload)) + spinner.next() - response = self._async_perform_request(url, method, payload, **headers) + performer.join() + response, error = performer.get_results() - if response is None: + if error: + raise error + elif response is None: raise ArcsecondConnectionError(url) if self.state.verbose: + click.echo() click.echo('Request status code ' + str(response.status_code)) if 200 <= response.status_code < 300: @@ -190,17 +208,29 @@ def _perform_request(self, url, method, payload, callback=None, **headers): else: return None, response.text - def list(self, name='', **headers): - return self._perform_request(self._list_url(name), 'get', None, None, **headers) + def _check_and_set_api_key(self, headers, url): + if API_AUTH_PATH_REGISTER in url or API_AUTH_PATH_LOGIN in url or 'Authorization' in headers.keys(): + return headers - def create(self, payload, callback=None, **headers): - return self._perform_request(self._list_url(), 'post', payload, callback, **headers) + if self.state.verbose: + click.echo('Checking local API key... ', nl=False) - def read(self, id_name_uuid, **headers): - return self._perform_request(self._detail_url(id_name_uuid), 'get', None, None, **headers) + api_key = config_file_read_api_key(self.state.config_section()) + if not api_key: + raise ArcsecondError('Missing API key. You must login first: $ arcsecond login') - def update(self, id_name_uuid, payload, **headers): - return self._perform_request(self._detail_url(id_name_uuid), 'put', payload, None, **headers) + headers['X-Arcsecond-API-Authorization'] = 'Key ' + api_key - def delete(self, id_name_uuid, **headers): - return self._perform_request(self._detail_url(id_name_uuid), 'delete', None, None, **headers) + if self.state.verbose: + click.echo('OK') + + return headers + + def _check_organisation_membership_and_permission(self, method_name, organisation): + memberships = config_file_read_organisation_memberships(self.state.config_section()) + if self.state.organisation not in memberships.keys(): + raise ArcsecondError('No membership found for organisation {}'.format(organisation)) + + membership = memberships[self.state.organisation] + if method_name not in SAFE_METHODS and membership not in WRITABLE_MEMBERSHIPS: + raise ArcsecondError('Membership for organisation {} has no write permission'.format(organisation)) From b0f1766ebe97bbf1c8febd099614b1eb87201c72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ce=CC=81dric=20Foellmi?= Date: Sun, 19 Jan 2020 10:20:54 +0100 Subject: [PATCH 04/14] Bumpung version number for future release --- arcsecond/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arcsecond/__init__.py b/arcsecond/__init__.py index 0d5965b..707c0bd 100644 --- a/arcsecond/__init__.py +++ b/arcsecond/__init__.py @@ -7,4 +7,4 @@ "ArcsecondConnectionError", "ArcsecondInvalidEndpointError"] -__version__ = '0.7.4' +__version__ = '0.7.5' From a4c664e547a0e3b830b080b8ff6192d962d9ce18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ce=CC=81dric=20Foellmi?= Date: Sun, 19 Jan 2020 10:21:59 +0100 Subject: [PATCH 05/14] Removing typing hint for Python2 --- arcsecond/api/endpoints/_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arcsecond/api/endpoints/_base.py b/arcsecond/api/endpoints/_base.py index a57d9c3..4d9cc04 100644 --- a/arcsecond/api/endpoints/_base.py +++ b/arcsecond/api/endpoints/_base.py @@ -123,7 +123,7 @@ def update(self, id_name_uuid, payload, **headers): def delete(self, id_name_uuid, **headers): return self._perform_request(self._detail_url(id_name_uuid), 'delete', None, None, **headers) - def _perform_request(self, url, method, payload, callback=None, **headers) -> (Tuple, AsyncRequestPerformer): + def _perform_request(self, url, method, payload, callback=None, **headers): method_name, method, payload, headers = self._prepare_request(url, method, payload, **headers) payload, fields = extract_multipart_encoder_file_fields(payload) From 54f81ab4fc6956f6efb56aadbe670c2d5c9c78d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ce=CC=81dric=20Foellmi?= Date: Sun, 19 Jan 2020 10:26:02 +0100 Subject: [PATCH 06/14] Small reorg of tests files --- tests/cli/__init__.py | 0 tests/{ => cli}/test_activities.py | 2 +- tests/{ => cli}/test_auth.py | 0 tests/{test_cli.py => cli/test_cli_options.py} | 0 tests/{ => cli}/test_datasets_organisations.py | 8 ++++---- tests/{ => cli}/test_datasets_personal.py | 2 +- tests/module/__init__.py | 0 tests/{ => module}/test_arcsecond_root.py | 2 +- 8 files changed, 7 insertions(+), 7 deletions(-) create mode 100644 tests/cli/__init__.py rename tests/{ => cli}/test_activities.py (97%) rename tests/{ => cli}/test_auth.py (100%) rename tests/{test_cli.py => cli/test_cli_options.py} (100%) rename tests/{ => cli}/test_datasets_organisations.py (94%) rename tests/{ => cli}/test_datasets_personal.py (96%) create mode 100644 tests/module/__init__.py rename tests/{ => module}/test_arcsecond_root.py (91%) diff --git a/tests/cli/__init__.py b/tests/cli/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_activities.py b/tests/cli/test_activities.py similarity index 97% rename from tests/test_activities.py rename to tests/cli/test_activities.py index 4110dee..a0862c8 100644 --- a/tests/test_activities.py +++ b/tests/cli/test_activities.py @@ -7,7 +7,7 @@ from arcsecond import cli from arcsecond.api.constants import ARCSECOND_API_URL_DEV from arcsecond.api.error import ArcsecondInputValueError -from .utils import register_successful_personal_login +from tests.utils import register_successful_personal_login @httpretty.activate diff --git a/tests/test_auth.py b/tests/cli/test_auth.py similarity index 100% rename from tests/test_auth.py rename to tests/cli/test_auth.py diff --git a/tests/test_cli.py b/tests/cli/test_cli_options.py similarity index 100% rename from tests/test_cli.py rename to tests/cli/test_cli_options.py diff --git a/tests/test_datasets_organisations.py b/tests/cli/test_datasets_organisations.py similarity index 94% rename from tests/test_datasets_organisations.py rename to tests/cli/test_datasets_organisations.py index 90b3a69..c03293f 100644 --- a/tests/test_datasets_organisations.py +++ b/tests/cli/test_datasets_organisations.py @@ -6,10 +6,10 @@ from arcsecond.api.error import ArcsecondError from arcsecond.config import config_file_clear_section -from .utils import (register_successful_personal_login, - register_successful_organisation_login, - mock_http_get, - mock_http_post) +from tests.utils import (register_successful_personal_login, + register_successful_organisation_login, + mock_http_get, + mock_http_post) class DatasetsInOrganisationsTestCase(TestCase): diff --git a/tests/test_datasets_personal.py b/tests/cli/test_datasets_personal.py similarity index 96% rename from tests/test_datasets_personal.py rename to tests/cli/test_datasets_personal.py index 82eeae0..1e59be2 100644 --- a/tests/test_datasets_personal.py +++ b/tests/cli/test_datasets_personal.py @@ -7,7 +7,7 @@ from arcsecond import cli from arcsecond.api.constants import ARCSECOND_API_URL_DEV -from .utils import register_successful_personal_login +from tests.utils import register_successful_personal_login @httpretty.activate diff --git a/tests/module/__init__.py b/tests/module/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_arcsecond_root.py b/tests/module/test_arcsecond_root.py similarity index 91% rename from tests/test_arcsecond_root.py rename to tests/module/test_arcsecond_root.py index acdb5aa..c4f0cdb 100644 --- a/tests/test_arcsecond_root.py +++ b/tests/module/test_arcsecond_root.py @@ -1,5 +1,5 @@ from arcsecond import Arcsecond -from .utils import save_test_credentials, clear_test_credentials +from tests.utils import save_test_credentials, clear_test_credentials def test_default_empty_state(): From 212e149beed7638e915d7e5d3adda1a6e29b16cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ce=CC=81dric=20Foellmi?= Date: Sun, 19 Jan 2020 10:26:07 +0100 Subject: [PATCH 07/14] Better gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 2c9f16f..18515a2 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,4 @@ build/ venv /env -/.DS_Store +.DS_Store From 9660cc0f7fc34c533f8ef87b7c7af9b10b37fe8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ce=CC=81dric=20Foellmi?= Date: Sun, 19 Jan 2020 19:20:16 +0100 Subject: [PATCH 08/14] Creating an AsyncFileUploader class --- arcsecond/api/endpoints/_base.py | 26 +++++++++++++------- arcsecond/api/main.py | 6 ++++- tests/module/test_datafiles_upload.py | 35 +++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 10 deletions(-) create mode 100644 tests/module/test_datafiles_upload.py diff --git a/arcsecond/api/endpoints/_base.py b/arcsecond/api/endpoints/_base.py index 4d9cc04..7a695c3 100644 --- a/arcsecond/api/endpoints/_base.py +++ b/arcsecond/api/endpoints/_base.py @@ -1,6 +1,5 @@ import threading import uuid -from typing import Tuple import click import requests @@ -31,14 +30,13 @@ EVENT_METHOD_PROGRESS_PERCENT = 'EVENT_METHOD_PROGRESS_PERCENT' -class AsyncRequestPerformer(object): +class AsyncFileUploader(object): def __init__(self, url, method, data=None, payload=None, **headers): self.url = url self.method = method self.payload = payload self.data = data self.headers = headers - self._storage = {} self._thread = None @@ -56,6 +54,10 @@ def _target(self, url, method, data, payload, headers): except Exception as e: self._storage['error'] = ArcsecondError(str(e)) + def finish(self): + self.join() + return self.get_results() + def join(self): self._thread.join() @@ -63,7 +65,14 @@ def is_alive(self): return self._thread.is_alive() def get_results(self): - return self._storage.get('response'), self._storage.get('error') + response = self._storage.get('response') + if response is not None: + if 200 <= response.status_code < 300: + return response.json() if response.text else {}, None + else: + return None, response.text + else: + return None, self._storage.get('error') class APIEndPoint(object): @@ -138,7 +147,7 @@ def _perform_request(self, url, method, payload, callback=None, **headers): if self.state.is_using_cli: return self._perform_spinner_request(url, method, method_name, upload_monitor, None, **headers) else: - return AsyncRequestPerformer(url, method, data=upload_monitor, payload=None, **headers) + return AsyncFileUploader(url, method, data=upload_monitor, payload=None, **headers) def _prepare_request(self, url, method, payload, **headers): assert (url and method) @@ -172,7 +181,7 @@ def _build_dynamic_upload_data(self, fields, callback=None): if self.state.is_using_cli is True and self.state.verbose: bar = Bar('Uploading ' + fields['file'][0], suffix='%(percent)d%%') upload_callback = lambda m: bar.goto(m.bytes_read / m.len * 100) - elif self.state.is_using_cli is False: + elif self.state.is_using_cli is False and callback: upload_callback = lambda m: callback(EVENT_METHOD_PROGRESS_PERCENT, m.bytes_read / m.len * 100) # The monitor is the data! @@ -183,7 +192,7 @@ def _perform_spinner_request(self, url, method, method_name, data=None, payload= click.echo('Sending {} request to {}'.format(method_name, url)) click.echo('Payload: {}'.format(payload)) - performer = AsyncRequestPerformer(url, method, data=data, payload=payload, **headers) + performer = AsyncFileUploader(url, method, data=data, payload=payload, **headers) performer.start() spinner = Spinner() @@ -191,8 +200,7 @@ def _perform_spinner_request(self, url, method, method_name, data=None, payload= if self.state.verbose: spinner.next() - performer.join() - response, error = performer.get_results() + response, error = performer.finish() if error: raise error diff --git a/arcsecond/api/main.py b/arcsecond/api/main.py index 77052eb..a2c44bb 100644 --- a/arcsecond/api/main.py +++ b/arcsecond/api/main.py @@ -27,11 +27,13 @@ PersonalProfileAPIEndPoint, ProfileAPIEndPoint, ProfileAPIKeyAPIEndPoint, SatellitesAPIEndPoint, StandardStarsAPIEndPoint, TelegramsATelAPIEndPoint, TelescopesAPIEndPoint) +from .endpoints._base import AsyncFileUploader + pp = pprint.PrettyPrinter(indent=4, depth=5) ECHO_PREFIX = u' • ' ECHO_ERROR_PREFIX = u' • [error] ' -__all__ = ["ArcsecondAPI"] +__all__ = ["ArcsecondAPI", "AsyncFileUploader"] ENDPOINTS = [ActivitiesAPIEndPoint, CataloguesAPIEndPoint, @@ -156,6 +158,8 @@ def _echo_error(cls, state, error): click.echo(ECHO_PREFIX + str(error)) def _echo_response(self, response): + if isinstance(response, AsyncFileUploader): + return response result, error = response if result is not None: # check against None, to avoid skipping empty lists. return ArcsecondAPI._echo_result(self.state, result) diff --git a/tests/module/test_datafiles_upload.py b/tests/module/test_datafiles_upload.py new file mode 100644 index 0000000..6357308 --- /dev/null +++ b/tests/module/test_datafiles_upload.py @@ -0,0 +1,35 @@ +import os +import time +import uuid + +import httpretty +from click.testing import CliRunner + +from arcsecond import Arcsecond +from arcsecond.api.constants import ARCSECOND_API_URL_DEV +from tests.utils import register_successful_personal_login + + +@httpretty.activate +def test_datafiles_upload_file_threaded_no_callback(): + # Using standard CLI runner to make sure we login successfuly as in other tests. + runner = CliRunner() + register_successful_personal_login(runner) + + dataset_uuid = uuid.uuid4() + httpretty.register_uri( + httpretty.POST, + ARCSECOND_API_URL_DEV + '/datasets/' + str(dataset_uuid) + '/datafiles/', + status=201, + body='{"result": "OK"}' + ) + + # Go for Python module tests + datafiles_api = Arcsecond.create_datafiles_api(debug=True, dataset=str(dataset_uuid)) + uploader = datafiles_api.create({'file': os.path.abspath(__file__)}) + uploader.start() + time.sleep(0.1) + results, error = uploader.finish() + + assert results is not None + assert error is None From eb426d16a03b98fa57c64bbcd1e9030b98d25a50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ce=CC=81dric=20Foellmi?= Date: Sun, 19 Jan 2020 19:31:38 +0100 Subject: [PATCH 09/14] Doc --- arcsecond/api/endpoints/_base.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/arcsecond/api/endpoints/_base.py b/arcsecond/api/endpoints/_base.py index 7a695c3..7209c88 100644 --- a/arcsecond/api/endpoints/_base.py +++ b/arcsecond/api/endpoints/_base.py @@ -31,6 +31,13 @@ class AsyncFileUploader(object): + """AsyncFileUploader is a helper class used when uploading files to the cloud. + + Technically speaking, it can handle any http request in a background thread. + It is however named like this because it is returned in place of a standard + response payload when a file is to be uploaded. + """ + def __init__(self, url, method, data=None, payload=None, **headers): self.url = url self.method = method @@ -121,6 +128,8 @@ def list(self, name='', **headers): return self._perform_request(self._list_url(name), 'get', None, None, **headers) def create(self, payload, callback=None, **headers): + # If a file is provided as part of the payload, a instance of AsyncFileUploader is returned + # in place of a standard JSON body response. return self._perform_request(self._list_url(), 'post', payload, callback, **headers) def read(self, id_name_uuid, **headers): From 529df7a61c19b21482302622322833dd1ea8687d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ce=CC=81dric=20Foellmi?= Date: Sun, 19 Jan 2020 19:31:47 +0100 Subject: [PATCH 10/14] Also making sync request for payload --- arcsecond/api/endpoints/_base.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arcsecond/api/endpoints/_base.py b/arcsecond/api/endpoints/_base.py index 7209c88..dbd4ce4 100644 --- a/arcsecond/api/endpoints/_base.py +++ b/arcsecond/api/endpoints/_base.py @@ -149,6 +149,10 @@ def _perform_request(self, url, method, payload, callback=None, **headers): # Standard JSON sync request return self._perform_spinner_request(url, method, method_name, None, payload, **headers) else: + # Process payload synchronously nonetheless + if payload: + self._perform_spinner_request(url, method, method_name, None, payload, **headers) + # File upload upload_monitor = self._build_dynamic_upload_data(fields, callback) headers.update(**{'Content-Type': upload_monitor.content_type}) From a7e20c753ff4249300db0d1b31b738a6661d0122 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ce=CC=81dric=20Foellmi?= Date: Sun, 19 Jan 2020 19:31:55 +0100 Subject: [PATCH 11/14] Adding test with callback! --- tests/module/test_datafiles_upload.py | 32 +++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/module/test_datafiles_upload.py b/tests/module/test_datafiles_upload.py index 6357308..4c315d1 100644 --- a/tests/module/test_datafiles_upload.py +++ b/tests/module/test_datafiles_upload.py @@ -9,6 +9,8 @@ from arcsecond.api.constants import ARCSECOND_API_URL_DEV from tests.utils import register_successful_personal_login +has_callback_been_called = False + @httpretty.activate def test_datafiles_upload_file_threaded_no_callback(): @@ -33,3 +35,33 @@ def test_datafiles_upload_file_threaded_no_callback(): assert results is not None assert error is None + + +@httpretty.activate +def test_datafiles_upload_file_threaded_with_callback(): + # Using standard CLI runner to make sure we login successfuly as in other tests. + runner = CliRunner() + register_successful_personal_login(runner) + + dataset_uuid = uuid.uuid4() + httpretty.register_uri( + httpretty.POST, + ARCSECOND_API_URL_DEV + '/datasets/' + str(dataset_uuid) + '/datafiles/', + status=201, + body='{"result": "OK"}' + ) + + def upload_callback(eventName, progress): + global has_callback_been_called + has_callback_been_called = True + + # Go for Python module tests + datafiles_api = Arcsecond.create_datafiles_api(debug=True, dataset=str(dataset_uuid)) + uploader = datafiles_api.create({'file': os.path.abspath(__file__)}, callback=upload_callback) + uploader.start() + time.sleep(0.1) + results, error = uploader.finish() + + assert results is not None + assert error is None + assert has_callback_been_called is True From 87832a38d697c5b762215300dda5709b79f72fe3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ce=CC=81dric=20Foellmi?= Date: Sun, 19 Jan 2020 19:36:56 +0100 Subject: [PATCH 12/14] Correcting flake8 issues --- arcsecond/api/endpoints/_base.py | 12 ++++++------ tests/cli/test_auth.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/arcsecond/api/endpoints/_base.py b/arcsecond/api/endpoints/_base.py index dbd4ce4..4aa59e9 100644 --- a/arcsecond/api/endpoints/_base.py +++ b/arcsecond/api/endpoints/_base.py @@ -188,17 +188,17 @@ def _prepare_request(self, url, method, payload, **headers): return method_name, method, payload, headers def _build_dynamic_upload_data(self, fields, callback=None): + # The monitor is the data! encoded_data = encoder.MultipartEncoder(fields=fields) - upload_callback = None if self.state.is_using_cli is True and self.state.verbose: bar = Bar('Uploading ' + fields['file'][0], suffix='%(percent)d%%') - upload_callback = lambda m: bar.goto(m.bytes_read / m.len * 100) + return encoder.MultipartEncoderMonitor(encoded_data, lambda m: bar.goto(m.bytes_read / m.len * 100)) elif self.state.is_using_cli is False and callback: - upload_callback = lambda m: callback(EVENT_METHOD_PROGRESS_PERCENT, m.bytes_read / m.len * 100) - - # The monitor is the data! - return encoder.MultipartEncoderMonitor(encoded_data, upload_callback) + return encoder.MultipartEncoderMonitor(encoded_data, lambda m: callback(EVENT_METHOD_PROGRESS_PERCENT, + m.bytes_read / m.len * 100)) + else: + return encoder.MultipartEncoderMonitor(encoded_data, None) def _perform_spinner_request(self, url, method, method_name, data=None, payload=None, **headers): if self.state.verbose: diff --git a/tests/cli/test_auth.py b/tests/cli/test_auth.py index ae5df13..44b6fb3 100644 --- a/tests/cli/test_auth.py +++ b/tests/cli/test_auth.py @@ -2,7 +2,7 @@ import httpretty from click.testing import CliRunner from arcsecond import cli -from arcsecond.api.constants import API_AUTH_PATH_LOGIN, ARCSECOND_API_URL_DEV, API_AUTH_PATH_REGISTER +from arcsecond.api.constants import API_AUTH_PATH_LOGIN, ARCSECOND_API_URL_DEV python_version = sys.version_info.major From 3fda670e614bda201794a5ebcc3e19bc0bc66e8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ce=CC=81dric=20Foellmi?= Date: Sun, 19 Jan 2020 19:49:12 +0100 Subject: [PATCH 13/14] Fixing handling of response and errors --- arcsecond/api/endpoints/_base.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/arcsecond/api/endpoints/_base.py b/arcsecond/api/endpoints/_base.py index 4aa59e9..84aa68a 100644 --- a/arcsecond/api/endpoints/_base.py +++ b/arcsecond/api/endpoints/_base.py @@ -73,7 +73,10 @@ def is_alive(self): def get_results(self): response = self._storage.get('response') - if response is not None: + if isinstance(response, dict): + # Responses of standard JSON payload requests are dict + return response + elif response is not None: if 200 <= response.status_code < 300: return response.json() if response.text else {}, None else: @@ -215,19 +218,11 @@ def _perform_spinner_request(self, url, method, method_name, data=None, payload= response, error = performer.finish() - if error: - raise error - elif response is None: - raise ArcsecondConnectionError(url) - if self.state.verbose: click.echo() click.echo('Request status code ' + str(response.status_code)) - if 200 <= response.status_code < 300: - return response.json() if response.text else {}, None - else: - return None, response.text + return response, error def _check_and_set_api_key(self, headers, url): if API_AUTH_PATH_REGISTER in url or API_AUTH_PATH_LOGIN in url or 'Authorization' in headers.keys(): From 960515518e4e8d385d9d0957890e8efb598451c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ce=CC=81dric=20Foellmi?= Date: Sun, 19 Jan 2020 19:57:58 +0100 Subject: [PATCH 14/14] Fixing test + doc --- arcsecond/api/endpoints/_base.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/arcsecond/api/endpoints/_base.py b/arcsecond/api/endpoints/_base.py index 84aa68a..b8de655 100644 --- a/arcsecond/api/endpoints/_base.py +++ b/arcsecond/api/endpoints/_base.py @@ -218,6 +218,14 @@ def _perform_spinner_request(self, url, method, method_name, data=None, payload= response, error = performer.finish() + # If we have an error and it is an ArcsecondError, raise it. + # As for now, only ArcsecondError could be returned, and there is no + # real point of returning both response and error below. But + # methods in main.py expect them both. + + if error and isinstance(error, ArcsecondError): + raise error + if self.state.verbose: click.echo() click.echo('Request status code ' + str(response.status_code))