From bf21b56d0be88d7b09bf8ac117bc15d41bf20407 Mon Sep 17 00:00:00 2001 From: Justin Martin Date: Tue, 27 Nov 2018 09:11:15 -0500 Subject: [PATCH] Use POST request for some datatables routes if request length is long (#126) * Initial work on getting post requests working for datatable route * Initial work on getting post requests working for datatable export route * Additional work regarding post request body * Begin work on formatting post request arguments correctly * Format dictionary params correctly for post request * Update failing connection tests * Add .zip files to gitignore * Fix some failing tests due to httppretty * Add sanity tests for function determining request type * Add additional tests for modifying arguments to get/post request params * Add venv to flake8 ignore list * Run existing connection tests for both get and post requests * Fix some flake8 warnings * Update some datatable tests to account for get and post requests * Update some datatable data tests for get/post requests * Add config to always use post request for testing purposes * Add some more tests to ensure request made with correct params * Fix incorrect params format being passed to mocked tests * Add comment regarding 8000 character get request limit * Fix line being over 100 characters in length * Update version.py and changelog --- .gitignore | 1 + CHANGELOG.md | 4 ++ quandl/model/datatable.py | 16 +++-- quandl/operations/list.py | 9 ++- quandl/util.py | 33 ++++++++- quandl/utils/request_type_util.py | 24 +++++++ quandl/version.py | 2 +- setup.cfg | 2 +- setup.py | 3 +- test/helpers/random_data_helper.py | 13 ++++ test/test_connection.py | 39 +++++----- test/test_datatable.py | 90 ++++++++++++++++++++--- test/test_datatable_data.py | 67 +++++++++++++---- test/test_get_table.py | 61 +++++++++++++++- test/test_request_type_util.py | 27 +++++++ test/test_retries.py | 6 +- test/test_util.py | 112 ++++++++++++++++++++++++----- tox.ini | 1 + 18 files changed, 437 insertions(+), 73 deletions(-) create mode 100644 quandl/utils/request_type_util.py create mode 100644 test/helpers/random_data_helper.py create mode 100644 test/test_request_type_util.py diff --git a/.gitignore b/.gitignore index 5e59685..f93d797 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ *.pyc +*.zip /.tox/ /.eggs diff --git a/CHANGELOG.md b/CHANGELOG.md index 8bf4daa..c900c3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ ### unreleased * Remove dependency on unittest2, use unittest instead (#113) +### 3.4.5 - 2018-11-21 + +* Use POST requests for some datatable calls https://github.com/quandl/quandl-python/pull/126 + ### 3.4.4 - 2018-10-24 * Add functionality to automatically retry failed API calls https://github.com/quandl/quandl-python/pull/124 diff --git a/quandl/model/datatable.py b/quandl/model/datatable.py index 558c53d..39d863a 100644 --- a/quandl/model/datatable.py +++ b/quandl/model/datatable.py @@ -11,6 +11,7 @@ from quandl.errors.quandl_error import QuandlError from quandl.operations.get import GetOperation from quandl.operations.list import ListOperation +from quandl.utils.request_type_util import RequestType from .model_base import ModelBase from quandl.message import Message @@ -26,8 +27,9 @@ def get_path(cls): return "%s/metadata" % cls.default_path() def data(self, **options): - updated_options = Util.convert_options(**options) - return Data.page(self, **updated_options) + if not options: + options = {'params': {}} + return Data.page(self, **options) def download_file(self, file_or_folder_path, **options): if not isinstance(file_or_folder_path, str): @@ -36,19 +38,21 @@ def download_file(self, file_or_folder_path, **options): file_is_ready = False while not file_is_ready: - file_is_ready = self._request_file_info(file_or_folder_path, **options) + file_is_ready = self._request_file_info(file_or_folder_path, params=options) if not file_is_ready: print(Message.LONG_GENERATION_TIME) sleep(self.WAIT_GENERATION_INTERVAL) def _request_file_info(self, file_or_folder_path, **options): url = self._download_request_path() - updated_options = Util.convert_options(params=options) code_name = self.code + options['params']['qopts.export'] = 'true' - updated_options['params']['qopts.export'] = 'true' + request_type = RequestType.get_request_type(url, **options) - r = Connection.request('get', url, **updated_options) + updated_options = Util.convert_options(request_type=request_type, **options) + + r = Connection.request(request_type, url, **updated_options) response_data = r.json() diff --git a/quandl/operations/list.py b/quandl/operations/list.py index b6d3dd4..27d2328 100644 --- a/quandl/operations/list.py +++ b/quandl/operations/list.py @@ -2,6 +2,7 @@ from quandl.connection import Connection from quandl.util import Util from quandl.model.paginated_list import PaginatedList +from quandl.utils.request_type_util import RequestType class ListOperation(Operation): @@ -21,7 +22,13 @@ def all(cls, **options): def page(cls, datatable, **options): params = {'id': str(datatable.code)} path = Util.constructed_path(datatable.default_path(), params) - r = Connection.request('get', path, **options) + + request_type = RequestType.get_request_type(path, **options) + + updated_options = Util.convert_options(request_type=request_type, **options) + + r = Connection.request(request_type, path, **updated_options) + response_data = r.json() Util.convert_to_dates(response_data) resource = cls.create_datatable_list_from_response(response_data) diff --git a/quandl/util.py b/quandl/util.py index 8be96f1..af326fe 100644 --- a/quandl/util.py +++ b/quandl/util.py @@ -60,7 +60,16 @@ def convert_to_date(value): return value @staticmethod - def convert_options(**options): + def convert_options(request_type, **options): + if request_type == 'get': + return Util._convert_options_for_get_request(**options) + elif request_type == 'post': + return Util._convert_options_for_post_request(**options) + else: + raise Exception('Can only convert options for get or post requests') + + @staticmethod + def _convert_options_for_get_request(**options): new_options = dict() if 'params' in options.keys(): for key, value in options['params'].items(): @@ -85,6 +94,28 @@ def convert_options(**options): new_options[key] = value return {'params': new_options} + @staticmethod + def _convert_options_for_post_request(**options): + new_options = dict() + if 'params' in options.keys(): + for key, value in options['params'].items(): + if isinstance(value, dict) and value != {}: + new_value = dict() + is_dict = True + old_key = key + for k, v in value.items(): + key = key + '.' + k + new_value[key] = v + key = old_key + else: + is_dict = False + + if is_dict: + new_options.update(new_value) + else: + new_options[key] = value + return {'json': new_options} + @staticmethod def convert_to_columns_list(meta, type): columns = [] diff --git a/quandl/utils/request_type_util.py b/quandl/utils/request_type_util.py new file mode 100644 index 0000000..d5f502c --- /dev/null +++ b/quandl/utils/request_type_util.py @@ -0,0 +1,24 @@ +try: + from urllib import urlencode +except ImportError: + from urllib.parse import urlencode + +from quandl.api_config import ApiConfig + + +class RequestType(object): + """ Determines whether a request should be made using a GET or a POST request. + Default limit of 8000 is set here as it appears to be the maximum for many + webservers. + """ + MAX_URL_LENGTH_FOR_GET = 8000 + USE_GET_REQUEST = True # This is used to simplify testing code + + @classmethod + def get_request_type(cls, url, **params): + query_string = urlencode(params['params']) + request_url = '%s/%s/%s' % (ApiConfig.api_base, url, query_string) + if RequestType.USE_GET_REQUEST and (len(request_url) < cls.MAX_URL_LENGTH_FOR_GET): + return 'get' + else: + return 'post' diff --git a/quandl/version.py b/quandl/version.py index ed072bd..9254ef5 100644 --- a/quandl/version.py +++ b/quandl/version.py @@ -1 +1 @@ -VERSION = '3.4.4' +VERSION = '3.4.5' diff --git a/setup.cfg b/setup.cfg index dc58f35..a599762 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,4 +3,4 @@ universal = 1 [flake8] max-line-length = 100 -exclude = .git,__init__.py,tmp,__pycache__,.eggs,Quandl.egg-info,build,dist,.tox +exclude = .git,__init__.py,tmp,__pycache__,.eggs,Quandl.egg-info,build,dist,.tox,venv diff --git a/setup.py b/setup.py index da77b57..9d7ff82 100644 --- a/setup.py +++ b/setup.py @@ -61,7 +61,8 @@ 'httpretty', 'mock', 'factory_boy', - 'jsondate' + 'jsondate', + 'parameterized' ], test_suite="nose.collector", packages=packages diff --git a/test/helpers/random_data_helper.py b/test/helpers/random_data_helper.py new file mode 100644 index 0000000..0926507 --- /dev/null +++ b/test/helpers/random_data_helper.py @@ -0,0 +1,13 @@ +import random +import string + + +def generate_random_string(n=10): + return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(n)) + + +def generate_random_dictionary(n): + random_dictionary = dict() + for _ in range(n): + random_dictionary[generate_random_string()] = generate_random_string() + return random_dictionary diff --git a/test/test_connection.py b/test/test_connection.py index 76b00a3..491d640 100644 --- a/test/test_connection.py +++ b/test/test_connection.py @@ -9,12 +9,19 @@ import json from mock import patch, call from quandl.version import VERSION +from parameterized import parameterized class ConnectionTest(ModifyRetrySettingsTestCase): - @httpretty.activate - def test_quandl_exceptions_no_retries(self): + def setUp(self): + httpretty.enable() + + def tearDown(self): + httpretty.disable() + + @parameterized.expand(['GET', 'POST']) + def test_quandl_exceptions_no_retries(self, request_method): ApiConfig.use_retries = False quandl_errors = [('QELx04', 429, LimitExceededError), ('QEMx01', 500, InternalServerError), @@ -25,7 +32,7 @@ def test_quandl_exceptions_no_retries(self): ('QEXx01', 503, ServiceUnavailableError), ('QEZx02', 400, QuandlError)] - httpretty.register_uri(httpretty.GET, + httpretty.register_uri(getattr(httpretty, request_method), "https://www.quandl.com/api/v3/databases", responses=[httpretty.Response(body=json.dumps( {'quandl_error': @@ -35,37 +42,37 @@ def test_quandl_exceptions_no_retries(self): for expected_error in quandl_errors: self.assertRaises( - expected_error[2], lambda: Connection.request('get', 'databases')) + expected_error[2], lambda: Connection.request(request_method, 'databases')) - @httpretty.activate - def test_parse_error(self): + @parameterized.expand(['GET', 'POST']) + def test_parse_error(self, request_method): ApiConfig.retry_backoff_factor = 0 - httpretty.register_uri(httpretty.GET, + httpretty.register_uri(getattr(httpretty, request_method), "https://www.quandl.com/api/v3/databases", body="not json", status=500) self.assertRaises( - QuandlError, lambda: Connection.request('get', 'databases')) + QuandlError, lambda: Connection.request(request_method, 'databases')) - @httpretty.activate - def test_non_quandl_error(self): + @parameterized.expand(['GET', 'POST']) + def test_non_quandl_error(self, request_method): ApiConfig.retry_backoff_factor = 0 - httpretty.register_uri(httpretty.GET, + httpretty.register_uri(getattr(httpretty, request_method), "https://www.quandl.com/api/v3/databases", body=json.dumps( {'foobar': {'code': 'blah', 'message': 'something went wrong'}}), status=500) self.assertRaises( - QuandlError, lambda: Connection.request('get', 'databases')) + QuandlError, lambda: Connection.request(request_method, 'databases')) - @httpretty.activate + @parameterized.expand(['GET', 'POST']) @patch('quandl.connection.Connection.execute_request') - def test_build_request(self, mock): + def test_build_request(self, request_method, mock): ApiConfig.api_key = 'api_token' ApiConfig.api_version = '2015-04-09' params = {'per_page': 10, 'page': 2} headers = {'x-custom-header': 'header value'} - Connection.request('get', 'databases', headers=headers, params=params) - expected = call('get', 'https://www.quandl.com/api/v3/databases', + Connection.request(request_method, 'databases', headers=headers, params=params) + expected = call(request_method, 'https://www.quandl.com/api/v3/databases', headers={'x-custom-header': 'header value', 'x-api-token': 'api_token', 'accept': ('application/json, ' diff --git a/test/test_datatable.py b/test/test_datatable.py index fc08772..d3c1947 100644 --- a/test/test_datatable.py +++ b/test/test_datatable.py @@ -13,7 +13,9 @@ from test.factories.datatable import DatatableFactory from test.test_retries import ModifyRetrySettingsTestCase from quandl.api_config import ApiConfig +from quandl.utils.request_type_util import RequestType from quandl.errors.quandl_error import (InternalServerError, QuandlError) +from parameterized import parameterized class GetDatatableDatasetTest(ModifyRetrySettingsTestCase): @@ -34,18 +36,69 @@ def tearDownClass(cls): httpretty.disable() httpretty.reset() + def tearDown(self): + RequestType.USE_GET_REQUEST = True + @patch('quandl.connection.Connection.request') - def test_datatable_meatadata_calls_connection(self, mock): + def test_datatable_metadata_calls_connection(self, mock): Datatable('ZACKS/FC').data_fields() expected = call('get', 'datatables/ZACKS/FC/metadata', params={}) self.assertEqual(mock.call_args, expected) @patch('quandl.connection.Connection.request') - def test_datatable_data_calls_connection(self, mock): + def test_datatable_data_calls_connection_with_no_params_for_get_request(self, mock): Datatable('ZACKS/FC').data() expected = call('get', 'datatables/ZACKS/FC', params={}) self.assertEqual(mock.call_args, expected) + @patch('quandl.connection.Connection.request') + def test_datatable_data_calls_connection_with_no_params_for_post_request(self, mock): + RequestType.USE_GET_REQUEST = False + Datatable('ZACKS/FC').data() + expected = call('post', 'datatables/ZACKS/FC', json={}) + self.assertEqual(mock.call_args, expected) + + @patch('quandl.connection.Connection.request') + def test_datatable_calls_connection_with_params_for_get_request(self, mock): + params = {'ticker': ['AAPL', 'MSFT'], + 'per_end_date': {'gte': '2015-01-01'}, + 'qopts': {'columns': ['ticker', 'per_end_date']}, + 'foo': 'bar', + 'baz': 4 + } + + expected_params = {'ticker[]': ['AAPL', 'MSFT'], + 'per_end_date.gte': '2015-01-01', + 'qopts.columns[]': ['ticker', 'per_end_date'], + 'foo': 'bar', + 'baz': 4 + } + + Datatable('ZACKS/FC').data(params=params) + expected = call('get', 'datatables/ZACKS/FC', params=expected_params) + self.assertEqual(mock.call_args, expected) + + @patch('quandl.connection.Connection.request') + def test_datatable_calls_connection_with_params_for_post_request(self, mock): + RequestType.USE_GET_REQUEST = False + params = {'ticker': ['AAPL', 'MSFT'], + 'per_end_date': {'gte': '2015-01-01'}, + 'qopts': {'columns': ['ticker', 'per_end_date']}, + 'foo': 'bar', + 'baz': 4 + } + + expected_params = {'ticker': ['AAPL', 'MSFT'], + 'per_end_date.gte': '2015-01-01', + 'qopts.columns': ['ticker', 'per_end_date'], + 'foo': 'bar', + 'baz': 4 + } + + Datatable('ZACKS/FC').data(params=params) + expected = call('post', 'datatables/ZACKS/FC', json=expected_params) + self.assertEqual(mock.call_args, expected) + def test_datatable_returns_datatable_object(self): datatable = Datatable('ZACKS/FC') self.assertIsInstance(datatable, Datatable) @@ -69,6 +122,11 @@ def setUpClass(cls): re.compile( 'https://www.quandl.com/api/v3/datatables/*'), body=json.dumps(datatable)) + + httpretty.register_uri(httpretty.POST, + re.compile( + 'https://www.quandl.com/api/v3/datatables/*'), + body=json.dumps(datatable)) cls.datatable_instance = Datatable(datatable['datatable']) @classmethod @@ -84,15 +142,19 @@ def setUp(self): ApiConfig.api_key = 'api_token' ApiConfig.api_version = '2015-04-09' + def tearDown(self): + RequestType.USE_GET_REQUEST = True + def test_download_get_file_info(self): url = self.datatable._download_request_path() parsed_url = urlparse(url) self.assertEqual(parsed_url.path, 'datatables/AUSBS/D.json') - def test_download_generated_file(self): + @parameterized.expand(['GET', 'POST']) + def test_download_generated_file(self, request_method): m = mock_open() - httpretty.register_uri(httpretty.GET, + httpretty.register_uri(getattr(httpretty, request_method), re.compile( 'https://www.quandl.com/api/v3/datatables/*'), body=json.dumps({ @@ -106,15 +168,21 @@ def test_download_generated_file(self): status=200) with patch('quandl.model.datatable.urlopen', m, create=True): - self.datatable.download_file('.') + self.datatable.download_file('.', params={}) self.assertEqual(m.call_count, 1) - def test_bulk_download_raises_exception_when_no_path(self): - self.assertRaises( - QuandlError, lambda: self.datatable.download_file(None)) + @parameterized.expand(['GET', 'POST']) + def test_bulk_download_raises_exception_when_no_path(self, request_method): + if request_method == 'POST': + RequestType.USE_GET_REQUEST = False + self.assertRaises( + QuandlError, lambda: self.datatable.download_file(None, params={})) - def test_bulk_download_table_raises_exception_when_error_response(self): + @parameterized.expand(['GET', 'POST']) + def test_bulk_download_table_raises_exception_when_error_response(self, request_method): + if request_method == 'POST': + RequestType.USE_GET_REQUEST = False httpretty.reset() ApiConfig.number_of_retries = 2 error_responses = [httpretty.Response( @@ -122,10 +190,10 @@ def test_bulk_download_table_raises_exception_when_error_response(self): 'message': 'something went wrong'}}), status=500)] - httpretty.register_uri(httpretty.GET, + httpretty.register_uri(getattr(httpretty, request_method), re.compile( 'https://www.quandl.com/api/v3/datatables/*'), responses=error_responses) self.assertRaises( - InternalServerError, lambda: self.datatable.download_file('.')) + InternalServerError, lambda: self.datatable.download_file('.', params={})) diff --git a/test/test_datatable_data.py b/test/test_datatable_data.py index 4f0306e..c16f651 100644 --- a/test/test_datatable_data.py +++ b/test/test_datatable_data.py @@ -7,9 +7,11 @@ import six from quandl.model.data import Data from quandl.model.datatable import Datatable +from quandl.utils.request_type_util import RequestType from mock import patch, call from test.factories.datatable_data import DatatableDataFactory from test.factories.datatable_meta import DatatableMetaFactory +from parameterized import parameterized class DatatableDataTest(unittest.TestCase): @@ -64,6 +66,11 @@ def setUpClass(cls): re.compile( 'https://www.quandl.com/api/v3/datatables/*'), body=json.dumps(datatable_data)) + + httpretty.register_uri(httpretty.POST, + re.compile( + 'https://www.quandl.com/api/v3/datatables/*'), + body=json.dumps(datatable_data)) cls.expected_raw_data = [] cls.expected_list_values = [] @@ -72,41 +79,67 @@ def tearDownClass(cls): httpretty.disable() httpretty.reset() + def tearDown(self): + RequestType.USE_GET_REQUEST = True + @patch('quandl.connection.Connection.request') - def test_data_calls_connection(self, mock): + def test_data_calls_connection_get(self, mock): datatable = Datatable('ZACKS/FC') Data.page(datatable, params={'ticker': ['AAPL', 'MSFT'], - 'per_end_date': {'gte': {'2015-01-01'}}, + 'per_end_date': {'gte': '2015-01-01'}, 'qopts': {'columns': ['ticker', 'per_end_date']}}) - expected = call('get', 'datatables/ZACKS/FC', params={'ticker': ['AAPL', 'MSFT'], - 'per_end_date': - {'gte': {'2015-01-01'}}, - 'qopts': {'columns': - ['ticker', - 'per_end_date']}}) + expected = call('get', 'datatables/ZACKS/FC', + params={'ticker[]': ['AAPL', 'MSFT'], + 'per_end_date.gte': '2015-01-01', + 'qopts.columns[]': ['ticker', 'per_end_date']}) self.assertEqual(mock.call_args, expected) - def test_values_and_meta_exist(self): + @patch('quandl.connection.Connection.request') + def test_data_calls_connection_post(self, mock): + RequestType.USE_GET_REQUEST = False + datatable = Datatable('ZACKS/FC') + Data.page(datatable, params={'ticker': ['AAPL', 'MSFT'], + 'per_end_date': {'gte': '2015-01-01'}, + 'qopts': {'columns': ['ticker', 'per_end_date']}}) + expected = call('post', 'datatables/ZACKS/FC', + json={'ticker': ['AAPL', 'MSFT'], + 'per_end_date.gte': '2015-01-01', + 'qopts.columns': ['ticker', 'per_end_date']}) + self.assertEqual(mock.call_args, expected) + + @parameterized.expand(['GET', 'POST']) + def test_values_and_meta_exist(self, request_method): + if request_method == 'POST': + RequestType.USE_GET_REQUEST = False datatable = Datatable('ZACKS/FC') results = Data.page(datatable, params={}) self.assertIsNotNone(results.values) self.assertIsNotNone(results.meta) - def test_to_pandas_returns_pandas_dataframe_object(self): + @parameterized.expand(['GET', 'POST']) + def test_to_pandas_returns_pandas_dataframe_object(self, request_method): + if request_method == 'POST': + RequestType.USE_GET_REQUEST = False datatable = Datatable('ZACKS/FC') results = Data.page(datatable, params={}) df = results.to_pandas() self.assertIsInstance(df, pandas.core.frame.DataFrame) # no index is set for datatable.to_pandas - def test_pandas_dataframe_index_is_none(self): + @parameterized.expand(['GET', 'POST']) + def test_pandas_dataframe_index_is_none(self, request_method): + if request_method == 'POST': + RequestType.USE_GET_REQUEST = False datatable = Datatable('ZACKS/FC') results = Data.page(datatable, params={}) df = results.to_pandas() self.assertEqual(df.index.name, 'None') # if datatable has Date field then it should be convert to pandas datetime - def test_pandas_dataframe_date_field_is_datetime(self): + @parameterized.expand(['GET', 'POST']) + def test_pandas_dataframe_date_field_is_datetime(self, request_method): + if request_method == 'POST': + RequestType.USE_GET_REQUEST = False datatable = Datatable('ZACKS/FC') results = Data.page(datatable, params={}) df = results.to_pandas() @@ -115,13 +148,19 @@ def test_pandas_dataframe_date_field_is_datetime(self): self.assertIsInstance(df['per_end_date'][2], pandas.datetime) self.assertIsInstance(df['per_end_date'][3], pandas.datetime) - def test_to_numpy_returns_numpy_object(self): + @parameterized.expand(['GET', 'POST']) + def test_to_numpy_returns_numpy_object(self, request_method): + if request_method == 'POST': + RequestType.USE_GET_REQUEST = False datatable = Datatable('ZACKS/FC') results = Data.page(datatable, params={}) data = results.to_numpy() self.assertIsInstance(data, numpy.core.records.recarray) - def test_to_csv_returns_expected_csv(self): + @parameterized.expand(['GET', 'POST']) + def test_to_csv_returns_expected_csv(self, request_method): + if request_method == 'POST': + RequestType.USE_GET_REQUEST = False datatable = Datatable('ZACKS/FC') results = Data.page(datatable, params={}) data = results.to_csv() diff --git a/test/test_get_table.py b/test/test_get_table.py index 774f839..4804ba3 100644 --- a/test/test_get_table.py +++ b/test/test_get_table.py @@ -4,11 +4,12 @@ import json from quandl.model.datatable import Datatable import pandas -from mock import patch +from mock import patch, call from test.factories.datatable import DatatableFactory from test.factories.datatable_data import DatatableDataFactory from test.factories.datatable_meta import DatatableMetaFactory import quandl +from quandl.utils.request_type_util import RequestType class GetDataTableTest(unittest.TestCase): @@ -33,6 +34,9 @@ def tearDownClass(cls): httpretty.disable() httpretty.reset() + def tearDown(self): + RequestType.USE_GET_REQUEST = True + @patch('quandl.connection.Connection.request') def test_datatable_returns_datatable_object(self, mock): df = quandl.get_table('ZACKS/FC', params={}) @@ -42,3 +46,58 @@ def test_datatable_returns_datatable_object(self, mock): def test_datatable_with_code_returns_datatable_object(self, mock): df = quandl.get_table('AR/MWCF', code="ICEP_WAC_Z2017_S") self.assertIsInstance(df, pandas.core.frame.DataFrame) + + @patch('quandl.connection.Connection.request') + def test_get_table_calls_connection_with_no_params_for_get_request(self, mock): + quandl.get_table('ZACKS/FC') + expected = call('get', 'datatables/ZACKS/FC', params={}) + self.assertEqual(mock.call_args, expected) + + @patch('quandl.connection.Connection.request') + def test_get_table_calls_connection_with_no_params_for_post_request(self, mock): + RequestType.USE_GET_REQUEST = False + + quandl.get_table('ZACKS/FC') + expected = call('post', 'datatables/ZACKS/FC', json={}) + self.assertEqual(mock.call_args, expected) + + @patch('quandl.connection.Connection.request') + def test_get_table_calls_connection_with_params_for_get_request(self, mock): + params = {'ticker': ['AAPL', 'MSFT'], + 'per_end_date': {'gte': '2015-01-01'}, + 'qopts': {'columns': ['ticker', 'per_end_date']}, + 'foo': 'bar', + 'baz': 4 + } + + expected_params = {'ticker[]': ['AAPL', 'MSFT'], + 'per_end_date.gte': '2015-01-01', + 'qopts.columns[]': ['ticker', 'per_end_date'], + 'foo': 'bar', + 'baz': 4 + } + + quandl.get_table('ZACKS/FC', **params) + expected = call('get', 'datatables/ZACKS/FC', params=expected_params) + self.assertEqual(mock.call_args, expected) + + @patch('quandl.connection.Connection.request') + def test_get_table_calls_connection_with_params_for_post_request(self, mock): + RequestType.USE_GET_REQUEST = False + params = {'ticker': ['AAPL', 'MSFT'], + 'per_end_date': {'gte': '2015-01-01'}, + 'qopts': {'columns': ['ticker', 'per_end_date']}, + 'foo': 'bar', + 'baz': 4 + } + + expected_params = {'ticker': ['AAPL', 'MSFT'], + 'per_end_date.gte': '2015-01-01', + 'qopts.columns': ['ticker', 'per_end_date'], + 'foo': 'bar', + 'baz': 4 + } + + quandl.get_table('ZACKS/FC', **params) + expected = call('post', 'datatables/ZACKS/FC', json=expected_params) + self.assertEqual(mock.call_args, expected) diff --git a/test/test_request_type_util.py b/test/test_request_type_util.py new file mode 100644 index 0000000..70111f8 --- /dev/null +++ b/test/test_request_type_util.py @@ -0,0 +1,27 @@ +import unittest +from quandl.utils.request_type_util import RequestType +from test.helpers.random_data_helper import generate_random_dictionary + + +class RequestTypeUtilTest(unittest.TestCase): + + def setUp(self): + self.test_url = '/datables/WIKI/PRICES.json' + RequestType.MAX_URL_LENGTH_FOR_GET = 200 + + def tearDown(self): + RequestType.MAX_URL_LENGTH_FOR_GET = 8000 + + def test_no_params(self): + request_type = RequestType.get_request_type(self.test_url, params={}) + self.assertEqual(request_type, 'get') + + def test_small_params(self): + params = {'foo': 'bar', 'qopts': {'columns': 'date'}} + request_type = RequestType.get_request_type(self.test_url, params=params) + self.assertEqual(request_type, 'get') + + def test_long_params(self): + params = generate_random_dictionary(20) + request_type = RequestType.get_request_type(self.test_url, params=params) + self.assertEqual(request_type, 'post') diff --git a/test/test_retries.py b/test/test_retries.py index a4ba4c9..7e6ff04 100644 --- a/test/test_retries.py +++ b/test/test_retries.py @@ -76,7 +76,7 @@ def test_modifying_max_wait_between_retries(self): retries = Connection.get_session().get_adapter(ApiConfig.api_protocol).max_retries self.assertEqual(retries.BACKOFF_MAX, ApiConfig.max_wait_between_retries) - @httpretty.activate + @httpretty.enabled def test_correct_response_returned_if_retries_succeed(self): ApiConfig.number_of_retries = 3 ApiConfig.retry_status_codes = [self.error_response.status] @@ -90,7 +90,7 @@ def test_correct_response_returned_if_retries_succeed(self): self.assertEqual(response.json(), self.datatable) self.assertEqual(response.status_code, self.success_response.status) - @httpretty.activate + @httpretty.enabled def test_correct_response_exception_raised_if_retries_fail(self): ApiConfig.number_of_retries = 2 ApiConfig.retry_status_codes = [self.error_response.status] @@ -101,7 +101,7 @@ def test_correct_response_exception_raised_if_retries_fail(self): self.assertRaises(InternalServerError, Connection.request, 'get', 'databases') - @httpretty.activate + @httpretty.enabled def test_correct_response_exception_raised_for_errors_not_in_retry_status_codes(self): ApiConfig.retry_status_codes = [] mock_responses = [self.error_response] diff --git a/test/test_util.py b/test/test_util.py index 6f5a67a..357b011 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -47,22 +47,100 @@ def test_constructed_path(self): self.assertEqual(result, '/hello/bar/world/1') self.assertDictEqual(params, {'another': 'a'}) - def test_convert_options(self): - options = {'params': {'ticker': ['AAPL', 'MSFT'], - 'per_end_date': {'gte': {'2015-01-01'}}, + def test_convert_options_get_request_with_empty_params(self): + options = {'params': {}} + expected_result = options + + result = Util.convert_options(request_type='get', **options) + self.assertEqual(cmp(result, expected_result), 0) + + def test_convert_options_get_request_with_simple_params(self): + options = {'params': {'foo': 'bar'}} + expected_result = options + + result = Util.convert_options(request_type='get', **options) + self.assertEqual(cmp(result, expected_result), 0) + + def test_convert_options_get_request_with_array_params(self): + options = {'params': {'foo': ['bar', 'baz']}} + expected_result = {'params': {'foo[]': ['bar', 'baz']}} + + result = Util.convert_options(request_type='get', **options) + self.assertEqual(cmp(result, expected_result), 0) + + def test_convert_options_get_request_with_dictionary_params(self): + options = {'params': {'foo': {'bar': 'baz'}}} + expected_result = {'params': {'foo.bar': 'baz'}} + + result = Util.convert_options(request_type='get', **options) + self.assertEqual(cmp(result, expected_result), 0) + + def test_convert_options_get_request_with_dictionary_params_and_array_values(self): + options = {'params': {'foo': {'bar': ['baz', 'bax']}}} + expected_result = {'params': {'foo.bar[]': ['baz', 'bax']}} + + result = Util.convert_options(request_type='get', **options) + self.assertEqual(cmp(result, expected_result), 0) + + def test_convert_options_get_request_all_param_types(self): + options = {'params': {'foo': 'bar', + 'ticker': ['AAPL', 'MSFT'], + 'per_end_date': {'gte': '2015-01-01'}, + 'qopts': {'columns': ['ticker', 'per_end_date'], + 'per_page': 5}}} + expected_result = {'params': {'foo': 'bar', + 'qopts.per_page': 5, + 'per_end_date.gte': '2015-01-01', + 'ticker[]': ['AAPL', 'MSFT'], + 'qopts.columns[]': ['ticker', 'per_end_date']}} + result = Util.convert_options(request_type='get', **options) + self.assertEqual(cmp(result, expected_result), 0) + + def test_convert_options_post_request_with_empty_params(self): + options = {'params': {}} + expected_result = {'json': {}} + + result = Util.convert_options(request_type='post', **options) + self.assertEqual(cmp(result, expected_result), 0) + + def test_convert_options_post_request_with_simple_params(self): + options = {'params': {'foo': 'bar'}} + expected_result = {'json': options['params']} + + result = Util.convert_options(request_type='post', **options) + self.assertEqual(cmp(result, expected_result), 0) + + def test_convert_options_post_request_with_array_params(self): + options = {'params': {'foo': ['bar', 'baz']}} + expected_result = {'json': options['params']} + + result = Util.convert_options(request_type='post', **options) + self.assertEqual(cmp(result, expected_result), 0) + + def test_convert_options_post_request_with_dictionary_params(self): + options = {'params': {'foo': {'bar': 'baz'}}} + expected_result = {'json': {'foo.bar': 'baz'}} + + result = Util.convert_options(request_type='post', **options) + self.assertEqual(cmp(result, expected_result), 0) + + def test_convert_options_post_request_with_dictionary_params_and_array_values(self): + options = {'params': {'foo': {'bar': ['baz', 'bax']}}} + expected_result = {'json': {'foo.bar': ['baz', 'bax']}} + + result = Util.convert_options(request_type='post', **options) + self.assertEqual(cmp(result, expected_result), 0) + + def test_convert_options_post_request_all_param_types(self): + options = {'params': {'foo': 'bar', + 'ticker': ['AAPL', 'MSFT'], + 'per_end_date': {'gte': '2015-01-01'}, 'qopts': {'columns': ['ticker', 'per_end_date'], 'per_page': 5}}} - expect_result = {'params': {'qopts.per_page': 5, - 'per_end_date.gte': set(['2015-01-01']), - 'ticker[]': ['AAPL', 'MSFT'], - 'qopts.columns[]': ['ticker', 'per_end_date']}} - result = Util.convert_options(**options) - self.assertEqual(cmp(result, expect_result), 0) - - options = {'params': {'ticker': 'AAPL', 'per_end_date': {'gte': {'2015-01-01'}}, - 'qopts': {'columns': ['ticker', 'per_end_date']}}} - expect_result = {'params': {'per_end_date.gte': set(['2015-01-01']), - 'ticker': 'AAPL', - 'qopts.columns[]': ['ticker', 'per_end_date']}} - result = Util.convert_options(**options) - self.assertEqual(cmp(result, expect_result), 0) + expected_result = {'json': {'foo': 'bar', + 'qopts.per_page': 5, + 'per_end_date.gte': '2015-01-01', + 'ticker': ['AAPL', 'MSFT'], + 'qopts.columns': ['ticker', 'per_end_date']}} + result = Util.convert_options(request_type='post', **options) + self.assertEqual(cmp(result, expected_result), 0) diff --git a/tox.ini b/tox.ini index b63c85a..772f442 100644 --- a/tox.ini +++ b/tox.ini @@ -18,3 +18,4 @@ deps = mock ndg-httpsclient jsondate + parameterized \ No newline at end of file