From e59aeac2969081e13c4f71c853745a80ee9c0b82 Mon Sep 17 00:00:00 2001 From: Rostyslav Zatserkovnyi Date: Tue, 19 Sep 2023 15:32:08 +0300 Subject: [PATCH 01/15] Remove usage of PHP alias in the Python client --- .../covid_hosp/facility/test_scenarios.py | 2 +- .../covid_hosp/state_daily/test_scenarios.py | 2 +- .../state_timeseries/test_scenarios.py | 2 +- .../covidcast/test_covidcast_meta_caching.py | 6 +- .../covidcast/test_csv_uploading.py | 17 ++- .../covidcast_nowcast/test_csv_uploading.py | 2 +- integrations/client/test_delphi_epidata.py | 41 ++++--- integrations/client/test_nowcast.py | 4 +- integrations/server/test_covid_hosp.py | 2 +- integrations/server/test_covidcast.py | 35 +++--- integrations/server/test_covidcast_meta.py | 5 +- integrations/server/test_covidcast_nowcast.py | 5 +- integrations/server/test_fluview.py | 2 +- integrations/server/test_fluview_meta.py | 2 +- src/client/delphi_epidata.py | 109 +++++++----------- 15 files changed, 108 insertions(+), 128 deletions(-) diff --git a/integrations/acquisition/covid_hosp/facility/test_scenarios.py b/integrations/acquisition/covid_hosp/facility/test_scenarios.py index c6c51e2f5..44ee3572d 100644 --- a/integrations/acquisition/covid_hosp/facility/test_scenarios.py +++ b/integrations/acquisition/covid_hosp/facility/test_scenarios.py @@ -28,7 +28,7 @@ def setUp(self): self.test_utils = UnitTestUtils(__file__) # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' Epidata.auth = ('epidata', 'key') # use the local instance of the epidata database diff --git a/integrations/acquisition/covid_hosp/state_daily/test_scenarios.py b/integrations/acquisition/covid_hosp/state_daily/test_scenarios.py index 2054d19c8..8636295bc 100644 --- a/integrations/acquisition/covid_hosp/state_daily/test_scenarios.py +++ b/integrations/acquisition/covid_hosp/state_daily/test_scenarios.py @@ -32,7 +32,7 @@ def setUp(self): self.test_utils = UnitTestUtils(__file__) # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' Epidata.auth = ('epidata', 'key') # use the local instance of the epidata database diff --git a/integrations/acquisition/covid_hosp/state_timeseries/test_scenarios.py b/integrations/acquisition/covid_hosp/state_timeseries/test_scenarios.py index 8565b8e7f..46bdeebcd 100644 --- a/integrations/acquisition/covid_hosp/state_timeseries/test_scenarios.py +++ b/integrations/acquisition/covid_hosp/state_timeseries/test_scenarios.py @@ -28,7 +28,7 @@ def setUp(self): self.test_utils = UnitTestUtils(__file__) # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' Epidata.auth = ('epidata', 'key') # use the local instance of the epidata database diff --git a/integrations/acquisition/covidcast/test_covidcast_meta_caching.py b/integrations/acquisition/covidcast/test_covidcast_meta_caching.py index 6e4c6378f..f0a31518d 100644 --- a/integrations/acquisition/covidcast/test_covidcast_meta_caching.py +++ b/integrations/acquisition/covidcast/test_covidcast_meta_caching.py @@ -22,7 +22,7 @@ ) # use the local instance of the Epidata API -BASE_URL = 'http://delphi_web_epidata/epidata/api.php' +BASE_URL = 'http://delphi_web_epidata/epidata' class CovidcastMetaCacheTests(unittest.TestCase): @@ -69,8 +69,8 @@ def tearDown(self): @staticmethod def _make_request(): - params = {'endpoint': 'covidcast_meta', 'cached': 'true'} - response = requests.get(Epidata.BASE_URL, params=params, auth=Epidata.auth) + params = {'cached': 'true'} + response = requests.get(f"{Epidata.BASE_URL}/covidcast_meta", params=params, auth=Epidata.auth) response.raise_for_status() return response.json() diff --git a/integrations/acquisition/covidcast/test_csv_uploading.py b/integrations/acquisition/covidcast/test_csv_uploading.py index e4c9d881e..9d9157d21 100644 --- a/integrations/acquisition/covidcast/test_csv_uploading.py +++ b/integrations/acquisition/covidcast/test_csv_uploading.py @@ -56,7 +56,7 @@ def setUp(self): secrets.db.epi = ('user', 'pass') # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' Epidata.auth = ('epidata', 'key') def tearDown(self): @@ -119,7 +119,7 @@ def test_uploading(self): main(args) response = Epidata.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*') - expected_values = pd.concat([values, pd.DataFrame({ "time_value": [20200419] * 3, "signal": [signal_name] * 3, "direction": [None] * 3})], axis=1).rename(columns=uploader_column_rename).to_dict(orient="records") + expected_values = pd.concat([values, pd.DataFrame({ "geo_type": "state", "source": "src-name", "time_type": "day", "time_value": [20200419] * 3, "signal": [signal_name] * 3, "direction": [None] * 3})], axis=1).rename(columns=uploader_column_rename).to_dict(orient="records") expected_response = {'result': 1, 'epidata': self.apply_lag(expected_values), 'message': 'success'} self.assertEqual(response, expected_response) @@ -148,6 +148,9 @@ def test_uploading(self): response = Epidata.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*') expected_values = pd.concat([values, pd.DataFrame({ + "geo_type": "state", + "source": "src-name", + "time_type": "day", "time_value": [20200419] * 3, "signal": [signal_name] * 3, "direction": [None] * 3, @@ -181,7 +184,7 @@ def test_uploading(self): main(args) response = Epidata.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*') - expected_response = {'result': -2, 'message': 'no results'} + expected_response = {'epidata': [], 'result': -2, 'message': 'no results'} self.assertEqual(response, expected_response) self.verify_timestamps_and_defaults() @@ -207,6 +210,9 @@ def test_uploading(self): response = Epidata.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*') expected_values_df = pd.concat([values, pd.DataFrame({ + "geo_type": "state", + "source": "src-name", + "time_type": "day", "time_value": [20200419], "signal": [signal_name], "direction": [None]})], axis=1).rename(columns=uploader_column_rename) @@ -240,6 +246,9 @@ def test_uploading(self): response = Epidata.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*') expected_values = pd.concat([values, pd.DataFrame({ + "geo_type": "state", + "source": "src-name", + "time_type": "day", "time_value": [20200419], "signal": [signal_name], "direction": [None] @@ -270,7 +279,7 @@ def test_uploading(self): main(args) response = Epidata.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*') - expected_response = {'result': -2, 'message': 'no results'} + expected_response = {'epidata': [], 'result': -2, 'message': 'no results'} self.assertEqual(response, expected_response) self.verify_timestamps_and_defaults() diff --git a/integrations/acquisition/covidcast_nowcast/test_csv_uploading.py b/integrations/acquisition/covidcast_nowcast/test_csv_uploading.py index 1299c6144..bf1a0f9a0 100644 --- a/integrations/acquisition/covidcast_nowcast/test_csv_uploading.py +++ b/integrations/acquisition/covidcast_nowcast/test_csv_uploading.py @@ -55,7 +55,7 @@ def setUp(self): secrets.db.epi = ('user', 'pass') # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' Epidata.auth = ('epidata', 'key') def tearDown(self): diff --git a/integrations/client/test_delphi_epidata.py b/integrations/client/test_delphi_epidata.py index 4ef1fa6a3..7497890c2 100644 --- a/integrations/client/test_delphi_epidata.py +++ b/integrations/client/test_delphi_epidata.py @@ -24,9 +24,9 @@ def fake_epidata_endpoint(func): """This can be used as a decorator to enable a bogus Epidata endpoint to return 404 responses.""" def wrapper(*args): - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/fake_api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/fake_epidata' func(*args) - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' return wrapper class DelphiEpidataPythonClientTests(CovidcastBase): @@ -39,7 +39,7 @@ def localSetUp(self): self._db._cursor.execute('update covidcast_meta_cache set timestamp = 0, epidata = "[]"') # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' Epidata.auth = ('epidata', 'key') # use the local instance of the epidata database @@ -65,8 +65,8 @@ def test_covidcast(self): ) expected = [ - row_latest_issue.as_api_compatibility_row_dict(), - rows[-1].as_api_compatibility_row_dict() + row_latest_issue.as_api_row_dict(), + rows[-1].as_api_row_dict() ] self.assertEqual(response['epidata'], expected) @@ -85,10 +85,10 @@ def test_covidcast(self): expected = [{ rows[0].signal: [ - row_latest_issue.as_api_compatibility_row_dict(ignore_fields=['signal']), + row_latest_issue.as_api_row_dict(ignore_fields=['signal']), ], rows[-1].signal: [ - rows[-1].as_api_compatibility_row_dict(ignore_fields=['signal']), + rows[-1].as_api_row_dict(ignore_fields=['signal']), ], }] @@ -105,7 +105,7 @@ def test_covidcast(self): **self.params_from_row(rows[0]) ) - expected = [row_latest_issue.as_api_compatibility_row_dict()] + expected = [row_latest_issue.as_api_row_dict()] # check result self.assertEqual(response_1, { @@ -120,7 +120,7 @@ def test_covidcast(self): **self.params_from_row(rows[0], as_of=rows[1].issue) ) - expected = [rows[1].as_api_compatibility_row_dict()] + expected = [rows[1].as_api_row_dict()] # check result self.maxDiff=None @@ -137,8 +137,8 @@ def test_covidcast(self): ) expected = [ - rows[0].as_api_compatibility_row_dict(), - rows[1].as_api_compatibility_row_dict() + rows[0].as_api_row_dict(), + rows[1].as_api_row_dict() ] # check result @@ -154,7 +154,7 @@ def test_covidcast(self): **self.params_from_row(rows[0], lag=2) ) - expected = [row_latest_issue.as_api_compatibility_row_dict()] + expected = [row_latest_issue.as_api_row_dict()] # check result self.assertDictEqual(response_3, { @@ -170,7 +170,7 @@ def test_covidcast(self): ) # check result - self.assertEqual(response_1, {'message': 'no results', 'result': -2}) + self.assertEqual(response_1, {'epidata': [], 'message': 'no results', 'result': -2}) @patch('requests.post') @patch('requests.get') @@ -196,7 +196,7 @@ def test_retry_request(self, get): mock_response = MagicMock() mock_response.status_code = 200 get.side_effect = [JSONDecodeError('Expecting value', "", 0), mock_response] - response = Epidata._request(None) + response = Epidata._request(None, "") self.assertEqual(get.call_count, 2) self.assertEqual(response, mock_response.json()) @@ -207,7 +207,7 @@ def test_retry_request(self, get): get.side_effect = [JSONDecodeError('Expecting value', "", 0), JSONDecodeError('Expecting value', "", 0), mock_response] - response = Epidata._request(None) + response = Epidata._request(None, "") self.assertEqual(get.call_count, 2) # 2 from previous test + 2 from this one self.assertEqual(response, {'result': 0, 'message': 'error: Expecting value: line 1 column 1 (char 0)'} @@ -228,7 +228,7 @@ def test_geo_value(self): self._insert_rows(rows) counties = [ - rows[i].as_api_compatibility_row_dict() for i in range(N) + rows[i].as_api_row_dict() for i in range(N) ] def fetch(geo): @@ -336,9 +336,9 @@ def test_async_epidata(self): self._insert_rows(rows) test_output = Epidata.async_epidata([ - self.params_from_row(rows[0], source='covidcast'), - self.params_from_row(rows[1], source='covidcast') - ]*12, batch_size=10) + self.params_from_row(rows[0]), + self.params_from_row(rows[1]) + ]*12, 'covidcast', batch_size=10) responses = [i[0] for i in test_output] # check response is same as standard covidcast call, using 24 calls to test batch sizing self.assertEqual( @@ -354,7 +354,6 @@ def test_async_epidata_fail(self): with pytest.raises(ClientResponseError, match="404, message='NOT FOUND'"): Epidata.async_epidata([ { - 'source': 'covidcast', 'data_source': 'src', 'signals': 'sig', 'time_type': 'day', @@ -362,4 +361,4 @@ def test_async_epidata_fail(self): 'geo_value': '11111', 'time_values': '20200414' } - ]) + ], 'covidcast') diff --git a/integrations/client/test_nowcast.py b/integrations/client/test_nowcast.py index f5124e021..84fc0e080 100644 --- a/integrations/client/test_nowcast.py +++ b/integrations/client/test_nowcast.py @@ -39,7 +39,7 @@ def setUp(self): self.cur = cnx.cursor() # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' Epidata.auth = ('epidata', 'key') # use the local instance of the epidata database @@ -133,4 +133,4 @@ def test_covidcast_nowcast(self): response = Epidata.covidcast_nowcast( 'src', 'sig1', 'sensor', 'day', 'county', 22222222, '01001') - self.assertEqual(response, {'result': -2, 'message': 'no results'}) + self.assertEqual(response, {'epidata': [], 'result': -2, 'message': 'no results'}) diff --git a/integrations/server/test_covid_hosp.py b/integrations/server/test_covid_hosp.py index 37aa77363..100d961c4 100644 --- a/integrations/server/test_covid_hosp.py +++ b/integrations/server/test_covid_hosp.py @@ -16,7 +16,7 @@ def setUp(self): """Perform per-test setup.""" # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' Epidata.auth = ('epidata', 'key') # use the local instance of the epidata database diff --git a/integrations/server/test_covidcast.py b/integrations/server/test_covidcast.py index 73787d664..935d7badb 100644 --- a/integrations/server/test_covidcast.py +++ b/integrations/server/test_covidcast.py @@ -23,7 +23,7 @@ def localSetUp(self): def request_based_on_row(self, row: CovidcastTestRow, **kwargs): params = self.params_from_row(row, endpoint='covidcast', **kwargs) # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' Epidata.auth = ('epidata', 'key') response = Epidata.covidcast(**params) @@ -90,7 +90,7 @@ def test_round_trip(self): # make the request response = self.request_based_on_row(row) - expected = [row.as_api_compatibility_row_dict()] + expected = [row.as_api_row_dict()] self.assertEqual(response, { 'result': 1, @@ -158,11 +158,12 @@ def test_csv_format(self): # This is a hardcoded mess because of api.php. column_order = [ - "geo_value", "signal", "time_value", "direction", "issue", "lag", "missing_value", + "geo_value", "signal", "source", "geo_type", "time_type", + "time_value", "direction", "issue", "lag", "missing_value", "missing_stderr", "missing_sample_size", "value", "stderr", "sample_size" ] expected = ( - row.as_api_compatibility_row_df() + row.as_api_row_df() .assign(direction = None) .to_csv(columns=column_order, index=False) ) @@ -179,7 +180,7 @@ def test_raw_json_format(self): # make the request response = self.request_based_on_row(row, **{'format':'json'}) - expected = [row.as_api_compatibility_row_dict()] + expected = [row.as_api_row_dict()] # assert that the right data came back self.assertEqual(response, expected) @@ -191,13 +192,13 @@ def test_fields(self): row = self._insert_placeholder_set_one() # limit fields - response = self.request_based_on_row(row, **{"fields":"time_value,geo_value"}) + response = self.request_based_on_row(row, **{"fields":"time_value,geo_value,geo_type,source,time_type"}) - expected = row.as_api_compatibility_row_dict() + expected = row.as_api_row_dict() expected_all = { 'result': 1, 'epidata': [{ - k: expected[k] for k in ['time_value', 'geo_value'] + k: expected[k] for k in ['time_value', 'geo_value', 'geo_type', 'source', 'time_type'] }], 'message': 'success', } @@ -206,7 +207,7 @@ def test_fields(self): self.assertEqual(response, expected_all) # limit using invalid fields - response = self.request_based_on_row(row, fields='time_value,geo_value,doesnt_exist') + response = self.request_based_on_row(row, fields='time_value,geo_value,,geo_type,source,time_type,doesnt_exist') # assert that the right data came back (only valid fields) self.assertEqual(response, expected_all) @@ -226,7 +227,7 @@ def test_location_wildcard(self): # insert placeholder data rows = self._insert_placeholder_set_two() - expected = [row.as_api_compatibility_row_dict() for row in rows[:3]] + expected = [row.as_api_row_dict() for row in rows[:3]] # make the request response = self.request_based_on_row(rows[0], geo_value="*") @@ -243,7 +244,7 @@ def test_time_values_wildcard(self): # insert placeholder data rows = self._insert_placeholder_set_three() - expected = [row.as_api_compatibility_row_dict() for row in rows[:3]] + expected = [row.as_api_row_dict() for row in rows[:3]] # make the request response = self.request_based_on_row(rows[0], time_values="*") @@ -261,7 +262,7 @@ def test_issues_wildcard(self): # insert placeholder data rows = self._insert_placeholder_set_five() - expected = [row.as_api_compatibility_row_dict() for row in rows[:3]] + expected = [row.as_api_row_dict() for row in rows[:3]] # make the request response = self.request_based_on_row(rows[0], issues="*") @@ -279,7 +280,7 @@ def test_signal_wildcard(self): # insert placeholder data rows = self._insert_placeholder_set_four() - expected_signals = [row.as_api_compatibility_row_dict() for row in rows[:3]] + expected_signals = [row.as_api_row_dict() for row in rows[:3]] # make the request response = self.request_based_on_row(rows[0], signals="*") @@ -297,7 +298,7 @@ def test_geo_value(self): # insert placeholder data rows = self._insert_placeholder_set_two() - expected = [row.as_api_compatibility_row_dict() for row in rows[:3]] + expected = [row.as_api_row_dict() for row in rows[:3]] def fetch(geo_value): # make the request @@ -335,7 +336,7 @@ def test_location_timeline(self): # insert placeholder data rows = self._insert_placeholder_set_three() - expected_timeseries = [row.as_api_compatibility_row_dict() for row in rows[:3]] + expected_timeseries = [row.as_api_row_dict() for row in rows[:3]] # make the request response = self.request_based_on_row(rows[0], time_values='20000101-20000105') @@ -372,7 +373,7 @@ def test_nullable_columns(self): # make the request response = self.request_based_on_row(row) - expected = row.as_api_compatibility_row_dict() + expected = row.as_api_row_dict() # assert that the right data came back self.assertEqual(response, { @@ -393,7 +394,7 @@ def test_temporal_partitioning(self): # make the request response = self.request_based_on_row(rows[1], time_values="*") - expected = [rows[1].as_api_compatibility_row_dict()] + expected = [rows[1].as_api_row_dict()] # assert that the right data came back self.assertEqual(response, { diff --git a/integrations/server/test_covidcast_meta.py b/integrations/server/test_covidcast_meta.py index d03317c98..857422a41 100644 --- a/integrations/server/test_covidcast_meta.py +++ b/integrations/server/test_covidcast_meta.py @@ -14,7 +14,7 @@ import delphi.operations.secrets as secrets # use the local instance of the Epidata API -BASE_URL = 'http://delphi_web_epidata/epidata/api.php' +BASE_URL = 'http://delphi_web_epidata/epidata' AUTH = ('epidata', 'key') @@ -152,8 +152,7 @@ def _get_id(self): @staticmethod def _fetch(auth=AUTH, **kwargs): params = kwargs.copy() - params['endpoint'] = 'covidcast_meta' - response = requests.get(BASE_URL, params=params, auth=auth) + response = requests.get(f"{BASE_URL}/covidcast_meta", params=params, auth=auth) response.raise_for_status() return response.json() diff --git a/integrations/server/test_covidcast_nowcast.py b/integrations/server/test_covidcast_nowcast.py index 889d962dd..32445afdf 100644 --- a/integrations/server/test_covidcast_nowcast.py +++ b/integrations/server/test_covidcast_nowcast.py @@ -9,7 +9,7 @@ # use the local instance of the Epidata API -BASE_URL = 'http://delphi_web_epidata/epidata/api.php' +BASE_URL = 'http://delphi_web_epidata/epidata' AUTH = ('epidata', 'key') @@ -43,7 +43,7 @@ def tearDown(self): @staticmethod def _make_request(params: dict): - response = requests.get(BASE_URL, params=params, auth=AUTH) + response = requests.get(f"{BASE_URL}/covidcast_nowcast", params=params, auth=AUTH) response.raise_for_status() return response.json() @@ -59,7 +59,6 @@ def test_query(self): self.cnx.commit() # make the request with specified issue date params={ - 'source': 'covidcast_nowcast', 'data_source': 'src', 'signals': 'sig', 'sensor_names': 'sensor', diff --git a/integrations/server/test_fluview.py b/integrations/server/test_fluview.py index c192da637..48d9585fd 100644 --- a/integrations/server/test_fluview.py +++ b/integrations/server/test_fluview.py @@ -18,7 +18,7 @@ def setUpClass(cls): """Perform one-time setup.""" # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' Epidata.auth = ('epidata', 'key') def setUp(self): diff --git a/integrations/server/test_fluview_meta.py b/integrations/server/test_fluview_meta.py index 1e2cf73e3..6f81c1859 100644 --- a/integrations/server/test_fluview_meta.py +++ b/integrations/server/test_fluview_meta.py @@ -18,7 +18,7 @@ def setUpClass(cls): """Perform one-time setup.""" # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' Epidata.auth = ('epidata', 'key') def setUp(self): diff --git a/src/client/delphi_epidata.py b/src/client/delphi_epidata.py index fe8dbe51d..ae618e2b1 100644 --- a/src/client/delphi_epidata.py +++ b/src/client/delphi_epidata.py @@ -44,7 +44,7 @@ class Epidata: """An interface to DELPHI's Epidata API.""" # API base url - BASE_URL = "https://api.delphi.cmu.edu/epidata/api.php" + BASE_URL = "https://api.delphi.cmu.edu/epidata" auth = None client_version = _version @@ -68,17 +68,18 @@ def _list(values): @staticmethod @retry(reraise=True, stop=stop_after_attempt(2)) - def _request_with_retry(params): + def _request_with_retry(params, endpoint): """Make request with a retry if an exception is thrown.""" - req = requests.get(Epidata.BASE_URL, params, auth=Epidata.auth, headers=_HEADERS) + request_url = f"{Epidata.BASE_URL}/{endpoint}" + req = requests.get(request_url, params, auth=Epidata.auth, headers=_HEADERS) if req.status_code == 414: - req = requests.post(Epidata.BASE_URL, params, auth=Epidata.auth, headers=_HEADERS) + req = requests.post(request_url, params, auth=Epidata.auth, headers=_HEADERS) # handle 401 and 429 req.raise_for_status() return req @staticmethod - def _request(params): + def _request(params, endpoint): """Request and parse epidata. We default to GET since it has better caching and logging @@ -86,7 +87,7 @@ def _request(params): long and returns a 414. """ try: - result = Epidata._request_with_retry(params) + result = Epidata._request_with_retry(params, endpoint) except Exception as e: return {"result": 0, "message": "error: " + str(e)} if params is not None and "format" in params and params["format"] == "csv": @@ -125,7 +126,6 @@ def fluview(regions, epiweeks, issues=None, lag=None, auth=None): raise EpidataBadRequestException(ISSUES_LAG_EXCLUSIVE) # Set up request params = { - "endpoint": "fluview", "regions": Epidata._list(regions), "epiweeks": Epidata._list(epiweeks), } @@ -136,18 +136,13 @@ def fluview(regions, epiweeks, issues=None, lag=None, auth=None): if auth is not None: params["auth"] = auth # Make the API call - return Epidata._request(params) + return Epidata._request(params, "fluview") # Fetch FluView metadata @staticmethod def fluview_meta(): """Fetch FluView metadata.""" - # Set up request - params = { - "endpoint": "fluview_meta", - } - # Make the API call - return Epidata._request(params) + return Epidata._request({}, "fluview_meta") # Fetch FluView clinical data @staticmethod @@ -160,7 +155,6 @@ def fluview_clinical(regions, epiweeks, issues=None, lag=None): raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) # Set up request params = { - "endpoint": "fluview_clinical", "regions": Epidata._list(regions), "epiweeks": Epidata._list(epiweeks), } @@ -169,7 +163,7 @@ def fluview_clinical(regions, epiweeks, issues=None, lag=None): if lag is not None: params["lag"] = lag # Make the API call - return Epidata._request(params) + return Epidata._request(params, "fluview_clinical") # Fetch FluSurv data @staticmethod @@ -182,7 +176,6 @@ def flusurv(locations, epiweeks, issues=None, lag=None): raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) # Set up request params = { - "endpoint": "flusurv", "locations": Epidata._list(locations), "epiweeks": Epidata._list(epiweeks), } @@ -191,7 +184,7 @@ def flusurv(locations, epiweeks, issues=None, lag=None): if lag is not None: params["lag"] = lag # Make the API call - return Epidata._request(params) + return Epidata._request(params, "flusurv") # Fetch PAHO Dengue data @staticmethod @@ -204,7 +197,6 @@ def paho_dengue(regions, epiweeks, issues=None, lag=None): raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) # Set up request params = { - "endpoint": "paho_dengue", "regions": Epidata._list(regions), "epiweeks": Epidata._list(epiweeks), } @@ -213,7 +205,7 @@ def paho_dengue(regions, epiweeks, issues=None, lag=None): if lag is not None: params["lag"] = lag # Make the API call - return Epidata._request(params) + return Epidata._request(params, "paho_dengue") # Fetch ECDC ILI data @staticmethod @@ -226,7 +218,6 @@ def ecdc_ili(regions, epiweeks, issues=None, lag=None): raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) # Set up request params = { - "endpoint": "ecdc_ili", "regions": Epidata._list(regions), "epiweeks": Epidata._list(epiweeks), } @@ -235,7 +226,7 @@ def ecdc_ili(regions, epiweeks, issues=None, lag=None): if lag is not None: params["lag"] = lag # Make the API call - return Epidata._request(params) + return Epidata._request(params, "ecdc_ili") # Fetch KCDC ILI data @staticmethod @@ -248,7 +239,6 @@ def kcdc_ili(regions, epiweeks, issues=None, lag=None): raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) # Set up request params = { - "endpoint": "kcdc_ili", "regions": Epidata._list(regions), "epiweeks": Epidata._list(epiweeks), } @@ -257,7 +247,7 @@ def kcdc_ili(regions, epiweeks, issues=None, lag=None): if lag is not None: params["lag"] = lag # Make the API call - return Epidata._request(params) + return Epidata._request(params, "kcdc_ili") # Fetch Google Flu Trends data @staticmethod @@ -268,12 +258,11 @@ def gft(locations, epiweeks): raise EpidataBadRequestException(LOCATIONS_EPIWEEKS_REQUIRED) # Set up request params = { - "endpoint": "gft", "locations": Epidata._list(locations), "epiweeks": Epidata._list(epiweeks), } # Make the API call - return Epidata._request(params) + return Epidata._request(params, "gft") # Fetch Google Health Trends data @staticmethod @@ -286,14 +275,13 @@ def ght(auth, locations, epiweeks, query): ) # Set up request params = { - "endpoint": "ght", "auth": auth, "locations": Epidata._list(locations), "epiweeks": Epidata._list(epiweeks), "query": query, } # Make the API call - return Epidata._request(params) + return Epidata._request(params, "ght") # Fetch HealthTweets data @staticmethod @@ -306,7 +294,6 @@ def twitter(auth, locations, dates=None, epiweeks=None): raise EpidataBadRequestException("exactly one of `dates` and `epiweeks` is required") # Set up request params = { - "endpoint": "twitter", "auth": auth, "locations": Epidata._list(locations), } @@ -315,7 +302,7 @@ def twitter(auth, locations, dates=None, epiweeks=None): if epiweeks is not None: params["epiweeks"] = Epidata._list(epiweeks) # Make the API call - return Epidata._request(params) + return Epidata._request(params, "twitter") # Fetch Wikipedia access data @staticmethod @@ -328,7 +315,6 @@ def wiki(articles, dates=None, epiweeks=None, hours=None, language="en"): raise EpidataBadRequestException("exactly one of `dates` and `epiweeks` is required") # Set up request params = { - "endpoint": "wiki", "articles": Epidata._list(articles), "language": language, } @@ -339,7 +325,7 @@ def wiki(articles, dates=None, epiweeks=None, hours=None, language="en"): if hours is not None: params["hours"] = Epidata._list(hours) # Make the API call - return Epidata._request(params) + return Epidata._request(params, "wiki") # Fetch CDC page hits @staticmethod @@ -350,13 +336,12 @@ def cdc(auth, epiweeks, locations): raise EpidataBadRequestException("`auth`, `epiweeks`, and `locations` are all required") # Set up request params = { - "endpoint": "cdc", "auth": auth, "epiweeks": Epidata._list(epiweeks), "locations": Epidata._list(locations), } # Make the API call - return Epidata._request(params) + return Epidata._request(params, "cdc") # Fetch Quidel data @staticmethod @@ -367,13 +352,12 @@ def quidel(auth, epiweeks, locations): raise EpidataBadRequestException("`auth`, `epiweeks`, and `locations` are all required") # Set up request params = { - "endpoint": "quidel", "auth": auth, "epiweeks": Epidata._list(epiweeks), "locations": Epidata._list(locations), } # Make the API call - return Epidata._request(params) + return Epidata._request(params, "quidel") # Fetch NoroSTAT data (point data, no min/max) @staticmethod @@ -384,13 +368,12 @@ def norostat(auth, location, epiweeks): raise EpidataBadRequestException("`auth`, `location`, and `epiweeks` are all required") # Set up request params = { - "endpoint": "norostat", "auth": auth, "location": location, "epiweeks": Epidata._list(epiweeks), } # Make the API call - return Epidata._request(params) + return Epidata._request(params, "norostat") # Fetch NoroSTAT metadata @staticmethod @@ -401,11 +384,10 @@ def meta_norostat(auth): raise EpidataBadRequestException("`auth` is required") # Set up request params = { - "endpoint": "meta_norostat", "auth": auth, } # Make the API call - return Epidata._request(params) + return Epidata._request(params, "meta_norostat") # Fetch NIDSS flu data @staticmethod @@ -418,7 +400,6 @@ def nidss_flu(regions, epiweeks, issues=None, lag=None): raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) # Set up request params = { - "endpoint": "nidss_flu", "regions": Epidata._list(regions), "epiweeks": Epidata._list(epiweeks), } @@ -427,7 +408,7 @@ def nidss_flu(regions, epiweeks, issues=None, lag=None): if lag is not None: params["lag"] = lag # Make the API call - return Epidata._request(params) + return Epidata._request(params, "nidss_flu") # Fetch NIDSS dengue data @staticmethod @@ -438,12 +419,11 @@ def nidss_dengue(locations, epiweeks): raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) # Set up request params = { - "endpoint": "nidss_dengue", "locations": Epidata._list(locations), "epiweeks": Epidata._list(epiweeks), } # Make the API call - return Epidata._request(params) + return Epidata._request(params, "nidss_dengue") # Fetch Delphi's forecast @staticmethod @@ -454,12 +434,11 @@ def delphi(system, epiweek): raise EpidataBadRequestException("`system` and `epiweek` are both required") # Set up request params = { - "endpoint": "delphi", "system": system, "epiweek": epiweek, } # Make the API call - return Epidata._request(params) + return Epidata._request(params, "delphi") # Fetch Delphi's digital surveillance sensors @staticmethod @@ -472,7 +451,6 @@ def sensors(auth, names, locations, epiweeks): ) # Set up request params = { - "endpoint": "sensors", "names": Epidata._list(names), "locations": Epidata._list(locations), "epiweeks": Epidata._list(epiweeks), @@ -480,7 +458,7 @@ def sensors(auth, names, locations, epiweeks): if auth is not None: params["auth"] = auth # Make the API call - return Epidata._request(params) + return Epidata._request(params, "sensors") # Fetch Delphi's dengue digital surveillance sensors @staticmethod @@ -493,14 +471,13 @@ def dengue_sensors(auth, names, locations, epiweeks): ) # Set up request params = { - "endpoint": "dengue_sensors", "auth": auth, "names": Epidata._list(names), "locations": Epidata._list(locations), "epiweeks": Epidata._list(epiweeks), } # Make the API call - return Epidata._request(params) + return Epidata._request(params, "dengue_sensors") # Fetch Delphi's wILI nowcast @staticmethod @@ -511,12 +488,11 @@ def nowcast(locations, epiweeks): raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) # Set up request params = { - "endpoint": "nowcast", "locations": Epidata._list(locations), "epiweeks": Epidata._list(epiweeks), } # Make the API call - return Epidata._request(params) + return Epidata._request(params, "nowcast") # Fetch Delphi's dengue nowcast @staticmethod @@ -527,18 +503,17 @@ def dengue_nowcast(locations, epiweeks): raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) # Set up request params = { - "endpoint": "dengue_nowcast", "locations": Epidata._list(locations), "epiweeks": Epidata._list(epiweeks), } # Make the API call - return Epidata._request(params) + return Epidata._request(params, "dengue_nowcast") # Fetch API metadata @staticmethod def meta(): """Fetch API metadata.""" - return Epidata._request({"endpoint": "meta"}) + return Epidata._request({}, "meta") # Fetch Delphi's COVID-19 Surveillance Streams @staticmethod @@ -568,7 +543,6 @@ def covidcast( raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) # Set up request params = { - "endpoint": "covidcast", "data_source": data_source, "signals": Epidata._list(signals), "time_type": time_type, @@ -594,13 +568,13 @@ def covidcast( params["fields"] = kwargs["fields"] # Make the API call - return Epidata._request(params) + return Epidata._request(params, "covidcast") # Fetch Delphi's COVID-19 Surveillance Streams metadata @staticmethod def covidcast_meta(): """Fetch Delphi's COVID-19 Surveillance Streams metadata""" - return Epidata._request({"endpoint": "covidcast_meta"}) + return Epidata._request({}, "covidcast_meta") # Fetch COVID hospitalization data @staticmethod @@ -611,7 +585,6 @@ def covid_hosp(states, dates, issues=None, as_of=None): raise EpidataBadRequestException("`states` and `dates` are both required") # Set up request params = { - "endpoint": "covid_hosp", "states": Epidata._list(states), "dates": Epidata._list(dates), } @@ -620,7 +593,7 @@ def covid_hosp(states, dates, issues=None, as_of=None): if as_of is not None: params["as_of"] = as_of # Make the API call - return Epidata._request(params) + return Epidata._request(params, "covid_hosp_state_timeseries") # Fetch COVID hospitalization data for specific facilities @staticmethod @@ -633,21 +606,20 @@ def covid_hosp_facility(hospital_pks, collection_weeks, publication_dates=None): ) # Set up request params = { - "source": "covid_hosp_facility", "hospital_pks": Epidata._list(hospital_pks), "collection_weeks": Epidata._list(collection_weeks), } if publication_dates is not None: params["publication_dates"] = Epidata._list(publication_dates) # Make the API call - return Epidata._request(params) + return Epidata._request(params, "covid_hosp_facility") # Lookup COVID hospitalization facility identifiers @staticmethod def covid_hosp_facility_lookup(state=None, ccn=None, city=None, zip=None, fips_code=None): """Lookup COVID hospitalization facility identifiers.""" # Set up request - params = {"source": "covid_hosp_facility_lookup"} + params = {} if state is not None: params["state"] = state elif ccn is not None: @@ -663,7 +635,7 @@ def covid_hosp_facility_lookup(state=None, ccn=None, city=None, zip=None, fips_c "one of `state`, `ccn`, `city`, `zip`, or `fips_code` is required" ) # Make the API call - return Epidata._request(params) + return Epidata._request(params, "covid_hosp_facility_lookup") # Fetch Delphi's COVID-19 Nowcast sensors @staticmethod @@ -693,7 +665,6 @@ def covidcast_nowcast( raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) # Set up request params = { - "source": "covidcast_nowcast", "data_source": data_source, "signals": Epidata._list(signals), "sensor_names": Epidata._list(sensor_names), @@ -717,15 +688,17 @@ def covidcast_nowcast( params["format"] = kwargs["format"] # Make the API call - return Epidata._request(params) + return Epidata._request(params, "covidcast_nowcast") @staticmethod - def async_epidata(param_list, batch_size=50): + def async_epidata(param_list, endpoint, batch_size=50): """Make asynchronous Epidata calls for a list of parameters.""" + request_url = f"{Epidata.BASE_URL}/{endpoint}" + async def async_get(params, session): """Helper function to make Epidata GET requests.""" - async with session.get(Epidata.BASE_URL, params=params) as response: + async with session.get(request_url, params=params) as response: response.raise_for_status() return await response.json(), params From b5128f09779de310ccd775a8a83edb5205372291 Mon Sep 17 00:00:00 2001 From: Rostyslav Zatserkovnyi Date: Fri, 22 Sep 2023 11:54:16 +0300 Subject: [PATCH 02/15] Post review tweaks --- docs/new_endpoint_tutorial.md | 6 +- integrations/client/test_delphi_epidata.py | 12 ++-- integrations/server/test_covidcast.py | 5 +- .../server/test_covidcast_endpoints.py | 20 +++++- src/client/delphi_epidata.py | 66 +++++++++---------- src/maintenance/signal_dash_data_generator.py | 2 +- 6 files changed, 64 insertions(+), 47 deletions(-) diff --git a/docs/new_endpoint_tutorial.md b/docs/new_endpoint_tutorial.md index 6e6094161..abf8a5fa1 100644 --- a/docs/new_endpoint_tutorial.md +++ b/docs/new_endpoint_tutorial.md @@ -123,11 +123,9 @@ Here's what we add to each client: def fluview_meta(): """Fetch FluView metadata.""" # Set up request - params = { - 'endpoint': 'fluview_meta', - } + params = {} # Make the API call - return Epidata._request(params) + return Epidata._request("fluview_meta", params) ``` - [`delphi_epidata.R`](https://github.com/cmu-delphi/delphi-epidata/blob/dev/src/client/delphi_epidata.R) diff --git a/integrations/client/test_delphi_epidata.py b/integrations/client/test_delphi_epidata.py index 7497890c2..99473f0ac 100644 --- a/integrations/client/test_delphi_epidata.py +++ b/integrations/client/test_delphi_epidata.py @@ -196,7 +196,7 @@ def test_retry_request(self, get): mock_response = MagicMock() mock_response.status_code = 200 get.side_effect = [JSONDecodeError('Expecting value', "", 0), mock_response] - response = Epidata._request(None, "") + response = Epidata._request("") self.assertEqual(get.call_count, 2) self.assertEqual(response, mock_response.json()) @@ -207,7 +207,7 @@ def test_retry_request(self, get): get.side_effect = [JSONDecodeError('Expecting value', "", 0), JSONDecodeError('Expecting value', "", 0), mock_response] - response = Epidata._request(None, "") + response = Epidata._request("") self.assertEqual(get.call_count, 2) # 2 from previous test + 2 from this one self.assertEqual(response, {'result': 0, 'message': 'error: Expecting value: line 1 column 1 (char 0)'} @@ -335,10 +335,10 @@ def test_async_epidata(self): ] self._insert_rows(rows) - test_output = Epidata.async_epidata([ + test_output = Epidata.async_epidata('covidcast', [ self.params_from_row(rows[0]), self.params_from_row(rows[1]) - ]*12, 'covidcast', batch_size=10) + ]*12, batch_size=10) responses = [i[0] for i in test_output] # check response is same as standard covidcast call, using 24 calls to test batch sizing self.assertEqual( @@ -352,7 +352,7 @@ def test_async_epidata(self): @fake_epidata_endpoint def test_async_epidata_fail(self): with pytest.raises(ClientResponseError, match="404, message='NOT FOUND'"): - Epidata.async_epidata([ + Epidata.async_epidata('covidcast', [ { 'data_source': 'src', 'signals': 'sig', @@ -361,4 +361,4 @@ def test_async_epidata_fail(self): 'geo_value': '11111', 'time_values': '20200414' } - ], 'covidcast') + ]) diff --git a/integrations/server/test_covidcast.py b/integrations/server/test_covidcast.py index 935d7badb..4963ca9a8 100644 --- a/integrations/server/test_covidcast.py +++ b/integrations/server/test_covidcast.py @@ -156,7 +156,8 @@ def test_csv_format(self): **{'format':'csv'} ) - # This is a hardcoded mess because of api.php. + # This is a hardcoded mess because of the field ordering constructed here: + # https://github.com/cmu-delphi/delphi-epidata/blob/f7da6598a810be8df5374e3a71512c631c3a14f1/src/server/endpoints/covidcast.py#L83-L93 column_order = [ "geo_value", "signal", "source", "geo_type", "time_type", "time_value", "direction", "issue", "lag", "missing_value", @@ -207,7 +208,7 @@ def test_fields(self): self.assertEqual(response, expected_all) # limit using invalid fields - response = self.request_based_on_row(row, fields='time_value,geo_value,,geo_type,source,time_type,doesnt_exist') + response = self.request_based_on_row(row, fields='time_value,geo_value,geo_type,source,time_type,doesnt_exist') # assert that the right data came back (only valid fields) self.assertEqual(response, expected_all) diff --git a/integrations/server/test_covidcast_endpoints.py b/integrations/server/test_covidcast_endpoints.py index 3ba0af039..e79b6f32f 100644 --- a/integrations/server/test_covidcast_endpoints.py +++ b/integrations/server/test_covidcast_endpoints.py @@ -113,7 +113,25 @@ def test_compatibility(self): with self.subTest("simple"): out = self._fetch("/", signal=first.signal_pair(), geo=first.geo_pair(), time="day:*", is_compatibility=True) - self.assertEqual(len(out["epidata"]), len(rows)) + self.assertEqual(out["epidata"], [row.as_api_compatibility_row_dict() for row in rows]) + + def test_compatibility_restricted_source(self): + """Restricted request at the /api.php endpoint.""" + rows = [CovidcastTestRow.make_default_row(time_value=2020_04_01 + i, value=i, source="quidel") for i in range(10)] + first = rows[0] + self._insert_rows(rows) + + with self.subTest("no_roles"): + out = self._fetch("/", signal=first.signal_pair(), geo=first.geo_pair(), time="day:*", is_compatibility=True) + self.assertTrue("epidata" not in out) + + with self.subTest("no_api_key"): + out = self._fetch("/", auth=None, signal=first.signal_pair(), geo=first.geo_pair(), time="day:*", is_compatibility=True) + self.assertTrue("epidata" not in out) + + with self.subTest("quidel_role"): + out = self._fetch("/", auth=("epidata", "quidel_key"), signal=first.signal_pair(), geo=first.geo_pair(), time="day:*", is_compatibility=True) + self.assertEqual(out["epidata"], [row.as_api_compatibility_row_dict() for row in rows]) def test_trend(self): """Request a signal from the /trend endpoint.""" diff --git a/src/client/delphi_epidata.py b/src/client/delphi_epidata.py index ae618e2b1..e773bdc50 100644 --- a/src/client/delphi_epidata.py +++ b/src/client/delphi_epidata.py @@ -68,7 +68,7 @@ def _list(values): @staticmethod @retry(reraise=True, stop=stop_after_attempt(2)) - def _request_with_retry(params, endpoint): + def _request_with_retry(endpoint, params={}): """Make request with a retry if an exception is thrown.""" request_url = f"{Epidata.BASE_URL}/{endpoint}" req = requests.get(request_url, params, auth=Epidata.auth, headers=_HEADERS) @@ -79,7 +79,7 @@ def _request_with_retry(params, endpoint): return req @staticmethod - def _request(params, endpoint): + def _request(endpoint, params={}): """Request and parse epidata. We default to GET since it has better caching and logging @@ -87,7 +87,7 @@ def _request(params, endpoint): long and returns a 414. """ try: - result = Epidata._request_with_retry(params, endpoint) + result = Epidata._request_with_retry(endpoint, params) except Exception as e: return {"result": 0, "message": "error: " + str(e)} if params is not None and "format" in params and params["format"] == "csv": @@ -136,13 +136,13 @@ def fluview(regions, epiweeks, issues=None, lag=None, auth=None): if auth is not None: params["auth"] = auth # Make the API call - return Epidata._request(params, "fluview") + return Epidata._request("fluview", params) # Fetch FluView metadata @staticmethod def fluview_meta(): """Fetch FluView metadata.""" - return Epidata._request({}, "fluview_meta") + return Epidata._request("fluview_meta") # Fetch FluView clinical data @staticmethod @@ -163,7 +163,7 @@ def fluview_clinical(regions, epiweeks, issues=None, lag=None): if lag is not None: params["lag"] = lag # Make the API call - return Epidata._request(params, "fluview_clinical") + return Epidata._request("fluview_clinical", params) # Fetch FluSurv data @staticmethod @@ -184,7 +184,7 @@ def flusurv(locations, epiweeks, issues=None, lag=None): if lag is not None: params["lag"] = lag # Make the API call - return Epidata._request(params, "flusurv") + return Epidata._request("flusurv", params) # Fetch PAHO Dengue data @staticmethod @@ -205,7 +205,7 @@ def paho_dengue(regions, epiweeks, issues=None, lag=None): if lag is not None: params["lag"] = lag # Make the API call - return Epidata._request(params, "paho_dengue") + return Epidata._request("paho_dengue", params) # Fetch ECDC ILI data @staticmethod @@ -226,7 +226,7 @@ def ecdc_ili(regions, epiweeks, issues=None, lag=None): if lag is not None: params["lag"] = lag # Make the API call - return Epidata._request(params, "ecdc_ili") + return Epidata._request("ecdc_ili", params) # Fetch KCDC ILI data @staticmethod @@ -247,7 +247,7 @@ def kcdc_ili(regions, epiweeks, issues=None, lag=None): if lag is not None: params["lag"] = lag # Make the API call - return Epidata._request(params, "kcdc_ili") + return Epidata._request("kcdc_ili", params) # Fetch Google Flu Trends data @staticmethod @@ -262,7 +262,7 @@ def gft(locations, epiweeks): "epiweeks": Epidata._list(epiweeks), } # Make the API call - return Epidata._request(params, "gft") + return Epidata._request("gft", params) # Fetch Google Health Trends data @staticmethod @@ -281,7 +281,7 @@ def ght(auth, locations, epiweeks, query): "query": query, } # Make the API call - return Epidata._request(params, "ght") + return Epidata._request("ght", params) # Fetch HealthTweets data @staticmethod @@ -302,7 +302,7 @@ def twitter(auth, locations, dates=None, epiweeks=None): if epiweeks is not None: params["epiweeks"] = Epidata._list(epiweeks) # Make the API call - return Epidata._request(params, "twitter") + return Epidata._request("twitter", params) # Fetch Wikipedia access data @staticmethod @@ -325,7 +325,7 @@ def wiki(articles, dates=None, epiweeks=None, hours=None, language="en"): if hours is not None: params["hours"] = Epidata._list(hours) # Make the API call - return Epidata._request(params, "wiki") + return Epidata._request("wiki", params) # Fetch CDC page hits @staticmethod @@ -341,7 +341,7 @@ def cdc(auth, epiweeks, locations): "locations": Epidata._list(locations), } # Make the API call - return Epidata._request(params, "cdc") + return Epidata._request("cdc", params) # Fetch Quidel data @staticmethod @@ -357,7 +357,7 @@ def quidel(auth, epiweeks, locations): "locations": Epidata._list(locations), } # Make the API call - return Epidata._request(params, "quidel") + return Epidata._request("quidel", params) # Fetch NoroSTAT data (point data, no min/max) @staticmethod @@ -373,7 +373,7 @@ def norostat(auth, location, epiweeks): "epiweeks": Epidata._list(epiweeks), } # Make the API call - return Epidata._request(params, "norostat") + return Epidata._request("norostat", params) # Fetch NoroSTAT metadata @staticmethod @@ -387,7 +387,7 @@ def meta_norostat(auth): "auth": auth, } # Make the API call - return Epidata._request(params, "meta_norostat") + return Epidata._request("meta_norostat", params) # Fetch NIDSS flu data @staticmethod @@ -408,7 +408,7 @@ def nidss_flu(regions, epiweeks, issues=None, lag=None): if lag is not None: params["lag"] = lag # Make the API call - return Epidata._request(params, "nidss_flu") + return Epidata._request("nidss_flu", params) # Fetch NIDSS dengue data @staticmethod @@ -423,7 +423,7 @@ def nidss_dengue(locations, epiweeks): "epiweeks": Epidata._list(epiweeks), } # Make the API call - return Epidata._request(params, "nidss_dengue") + return Epidata._request("nidss_dengue", params) # Fetch Delphi's forecast @staticmethod @@ -438,7 +438,7 @@ def delphi(system, epiweek): "epiweek": epiweek, } # Make the API call - return Epidata._request(params, "delphi") + return Epidata._request("delphi", params) # Fetch Delphi's digital surveillance sensors @staticmethod @@ -458,7 +458,7 @@ def sensors(auth, names, locations, epiweeks): if auth is not None: params["auth"] = auth # Make the API call - return Epidata._request(params, "sensors") + return Epidata._request("sensors", params) # Fetch Delphi's dengue digital surveillance sensors @staticmethod @@ -477,7 +477,7 @@ def dengue_sensors(auth, names, locations, epiweeks): "epiweeks": Epidata._list(epiweeks), } # Make the API call - return Epidata._request(params, "dengue_sensors") + return Epidata._request("dengue_sensors", params) # Fetch Delphi's wILI nowcast @staticmethod @@ -492,7 +492,7 @@ def nowcast(locations, epiweeks): "epiweeks": Epidata._list(epiweeks), } # Make the API call - return Epidata._request(params, "nowcast") + return Epidata._request("nowcast", params) # Fetch Delphi's dengue nowcast @staticmethod @@ -507,13 +507,13 @@ def dengue_nowcast(locations, epiweeks): "epiweeks": Epidata._list(epiweeks), } # Make the API call - return Epidata._request(params, "dengue_nowcast") + return Epidata._request("dengue_nowcast", params) # Fetch API metadata @staticmethod def meta(): """Fetch API metadata.""" - return Epidata._request({}, "meta") + return Epidata._request("meta") # Fetch Delphi's COVID-19 Surveillance Streams @staticmethod @@ -568,13 +568,13 @@ def covidcast( params["fields"] = kwargs["fields"] # Make the API call - return Epidata._request(params, "covidcast") + return Epidata._request("covidcast", params) # Fetch Delphi's COVID-19 Surveillance Streams metadata @staticmethod def covidcast_meta(): """Fetch Delphi's COVID-19 Surveillance Streams metadata""" - return Epidata._request({}, "covidcast_meta") + return Epidata._request("covidcast_meta") # Fetch COVID hospitalization data @staticmethod @@ -593,7 +593,7 @@ def covid_hosp(states, dates, issues=None, as_of=None): if as_of is not None: params["as_of"] = as_of # Make the API call - return Epidata._request(params, "covid_hosp_state_timeseries") + return Epidata._request("covid_hosp_state_timeseries", params) # Fetch COVID hospitalization data for specific facilities @staticmethod @@ -612,7 +612,7 @@ def covid_hosp_facility(hospital_pks, collection_weeks, publication_dates=None): if publication_dates is not None: params["publication_dates"] = Epidata._list(publication_dates) # Make the API call - return Epidata._request(params, "covid_hosp_facility") + return Epidata._request("covid_hosp_facility", params) # Lookup COVID hospitalization facility identifiers @staticmethod @@ -635,7 +635,7 @@ def covid_hosp_facility_lookup(state=None, ccn=None, city=None, zip=None, fips_c "one of `state`, `ccn`, `city`, `zip`, or `fips_code` is required" ) # Make the API call - return Epidata._request(params, "covid_hosp_facility_lookup") + return Epidata._request("covid_hosp_facility_lookup", params) # Fetch Delphi's COVID-19 Nowcast sensors @staticmethod @@ -688,10 +688,10 @@ def covidcast_nowcast( params["format"] = kwargs["format"] # Make the API call - return Epidata._request(params, "covidcast_nowcast") + return Epidata._request("covidcast_nowcast", params) @staticmethod - def async_epidata(param_list, endpoint, batch_size=50): + def async_epidata(endpoint, param_list, batch_size=50): """Make asynchronous Epidata calls for a list of parameters.""" request_url = f"{Epidata.BASE_URL}/{endpoint}" diff --git a/src/maintenance/signal_dash_data_generator.py b/src/maintenance/signal_dash_data_generator.py index 6eea06579..b7f1048f5 100644 --- a/src/maintenance/signal_dash_data_generator.py +++ b/src/maintenance/signal_dash_data_generator.py @@ -19,7 +19,7 @@ LOOKBACK_DAYS_FOR_COVERAGE = 56 -BASE_COVIDCAST = covidcast.covidcast.Epidata.BASE_URL[:-len("api.php")] + "covidcast" +BASE_COVIDCAST = covidcast.covidcast.Epidata.BASE_URL + "/covidcast" COVERAGE_URL = f"{BASE_COVIDCAST}/coverage?format=csv&signal={{source}}:{{signal}}&days={LOOKBACK_DAYS_FOR_COVERAGE}" @dataclass From c65095e43bf521ae37936c5bae8863e8c7a02ced Mon Sep 17 00:00:00 2001 From: george haff Date: Fri, 22 Sep 2023 21:15:55 -0400 Subject: [PATCH 03/15] removed 'api.php' from comments where not appropriate --- tests/server/dev_test_granular_sensor_authentication.py | 2 +- tests/server/endpoints/test_nidss_flu.py | 2 -- tests/server/test_exceptions.py | 2 -- tests/server/test_query.py | 2 -- tests/server/test_security.py | 2 +- tests/server/test_validate.py | 2 -- 6 files changed, 2 insertions(+), 10 deletions(-) diff --git a/tests/server/dev_test_granular_sensor_authentication.py b/tests/server/dev_test_granular_sensor_authentication.py index bc742392a..2309c626e 100644 --- a/tests/server/dev_test_granular_sensor_authentication.py +++ b/tests/server/dev_test_granular_sensor_authentication.py @@ -1,4 +1,4 @@ -"""Unit tests for granular sensor authentication in api.php.""" +"""Unit tests for granular sensor authentication.""" # standard library import unittest diff --git a/tests/server/endpoints/test_nidss_flu.py b/tests/server/endpoints/test_nidss_flu.py index bc0723bf3..7ec50a6d2 100644 --- a/tests/server/endpoints/test_nidss_flu.py +++ b/tests/server/endpoints/test_nidss_flu.py @@ -1,5 +1,3 @@ -"""Unit tests for granular sensor authentication in api.php.""" - # standard library import unittest import base64 diff --git a/tests/server/test_exceptions.py b/tests/server/test_exceptions.py index 94cdc34f1..b07fe3ac7 100644 --- a/tests/server/test_exceptions.py +++ b/tests/server/test_exceptions.py @@ -1,5 +1,3 @@ -"""Unit tests for granular sensor authentication in api.php.""" - # standard library import unittest diff --git a/tests/server/test_query.py b/tests/server/test_query.py index ec07d3e8b..95b21a55a 100644 --- a/tests/server/test_query.py +++ b/tests/server/test_query.py @@ -1,5 +1,3 @@ -"""Unit tests for granular sensor authentication in api.php.""" - # standard library import unittest import base64 diff --git a/tests/server/test_security.py b/tests/server/test_security.py index e209d9342..eb8e32700 100644 --- a/tests/server/test_security.py +++ b/tests/server/test_security.py @@ -1,4 +1,4 @@ -"""Unit tests for granular sensor authentication in api.php.""" +"""Unit tests for granular sensor authentication.""" # standard library import unittest diff --git a/tests/server/test_validate.py b/tests/server/test_validate.py index 27ce28672..f06e9e997 100644 --- a/tests/server/test_validate.py +++ b/tests/server/test_validate.py @@ -1,5 +1,3 @@ -"""Unit tests for granular sensor authentication in api.php.""" - # standard library import unittest From 05e2b843c168bb52b1363ab0221b76b173436723 Mon Sep 17 00:00:00 2001 From: Rostyslav Zatserkovnyi Date: Wed, 27 Sep 2023 18:00:16 +0300 Subject: [PATCH 04/15] Refactor? --- integrations/server/test_api_keys.py | 53 +++++++------------- integrations/server/test_signal_dashboard.py | 10 +--- src/common/integration_test_base_class.py | 2 +- 3 files changed, 22 insertions(+), 43 deletions(-) diff --git a/integrations/server/test_api_keys.py b/integrations/server/test_api_keys.py index 418e265ac..0c5be4c0b 100644 --- a/integrations/server/test_api_keys.py +++ b/integrations/server/test_api_keys.py @@ -11,9 +11,8 @@ class APIKeysTets(DelphiTestBase): def localSetUp(self): self.role_name = "cdc" - def _make_request(self, url: str = None, params: dict = {}, auth: tuple = None): - if not url: - url = self.epidata_client.BASE_URL + def _make_request(self, endpoint: str, params: dict = {}, auth: tuple = None): + url = f"{self.epidata_client.BASE_URL}/{endpoint}" response = requests.get(url, params=params, auth=auth) return response @@ -22,13 +21,12 @@ def test_public_route(self): public_route = "http://delphi_web_epidata/epidata/version" status_codes = set() for _ in range(10): - status_codes.add(self._make_request(public_route).status_code) + status_codes.add(self._make_request("version", public_route).status_code) self.assertEqual(status_codes, {200}) def test_no_multiples_data_source(self): """Test requests with no multiples and with provided `data_source` and `signal` as a separate query params.""" params = { - "source": "covidcast", "data_source": "fb-survey", "signal": "smoothed_wcli", "time_type": "day", @@ -38,13 +36,12 @@ def test_no_multiples_data_source(self): } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params).status_code) + status_codes.add(self._make_request("covidcast", params=params).status_code) self.assertEqual(status_codes, {200}) def test_no_multiples_source_signal(self): """Test requests with colon-delimited source-signal param presentation.""" params = { - "source": "covidcast", "signal": "fb-survey:smoothed_wcli", "time_type": "day", "geo_type": "state", @@ -53,13 +50,12 @@ def test_no_multiples_source_signal(self): } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params).status_code) + status_codes.add(self._make_request("covidcast", params=params).status_code) self.assertEqual(status_codes, {200}) def test_multiples_allowed_signal_two_multiples(self): """Test requests with 2 multiples and allowed dashboard signal""" params = { - "source": "covidcast", "signal": "fb-survey:smoothed_wcli", "time_type": "day", "geo_type": "state", @@ -68,13 +64,12 @@ def test_multiples_allowed_signal_two_multiples(self): } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params).status_code) + status_codes.add(self._make_request("covidcast", params=params).status_code) self.assertEqual(status_codes, {200}) def test_multiples_non_allowed_signal(self): """Test requests with 2 multiples and non-allowed dashboard signal""" params = { - "source": "covidcast", "signal": "hospital-admissions:smoothed_adj_covid19_from_claims", "time_type": "day", "geo_type": "state", @@ -83,13 +78,12 @@ def test_multiples_non_allowed_signal(self): } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params).status_code) + status_codes.add(self._make_request("covidcast", params=params).status_code) self.assertEqual(status_codes, {200, 429}) def test_multiples_mixed_allowed_signal_two_multiples(self): """Test requests with 2 multiples and mixed-allowed dashboard signal""" params = { - "source": "covidcast", "signal": "fb-survey:smoothed_wcli,hospital-admissions:smoothed_adj_covid19_from_claims", "time_type": "day", "geo_type": "state", @@ -98,13 +92,12 @@ def test_multiples_mixed_allowed_signal_two_multiples(self): } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params).status_code) + status_codes.add(self._make_request("covidcast", params=params).status_code) self.assertEqual(status_codes, {200, 429}) def test_multiples_allowed_signal_three_multiples(self): """Test requests with 3 multiples and allowed dashboard signal""" params = { - "source": "covidcast", "signal": "fb-survey:smoothed_wcli,fb-survey:smoothed_wcli", "time_type": "day", "geo_type": "state", @@ -113,13 +106,12 @@ def test_multiples_allowed_signal_three_multiples(self): } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params).status_code) + status_codes.add(self._make_request("covidcast", params=params).status_code) self.assertEqual(status_codes, {401}) def test_multiples_mixed_allowed_signal_three_multiples(self): """Test requests with 3 multiples and mixed-allowed dashboard signal""" params = { - "source": "covidcast", "signal": "fb-survey:smoothed_wcli,fb-survey:smoothed_wcli1", "time_type": "day", "geo_type": "state", @@ -128,13 +120,12 @@ def test_multiples_mixed_allowed_signal_three_multiples(self): } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params).status_code) + status_codes.add(self._make_request("covidcast", params=params).status_code) self.assertEqual(status_codes, {401}) def test_multiples_mixed_allowed_signal_api_key(self): """Test requests with 3 multiples and mixed-allowed dashboard signal + valid API Key""" params = { - "source": "covidcast", "signal": "fb-survey:smoothed_wcli,fb-survey:smoothed_wcli1", "time_type": "day", "geo_type": "state", @@ -144,14 +135,13 @@ def test_multiples_mixed_allowed_signal_api_key(self): status_codes = set() for _ in range(10): status_codes.add( - self._make_request(params=params, auth=self.epidata_client.auth).status_code + self._make_request("covidcast", params=params, auth=self.epidata_client.auth).status_code ) self.assertEqual(status_codes, {200}) def test_multiples_allowed_signal_api_key(self): """Test requests with 3 multiples and allowed dashboard signal + valid API Key""" params = { - "source": "covidcast", "signal": "fb-survey:smoothed_wcli,fb-survey:smoothed_wcli", "time_type": "day", "geo_type": "state", @@ -161,14 +151,13 @@ def test_multiples_allowed_signal_api_key(self): status_codes = set() for _ in range(10): status_codes.add( - self._make_request(params=params, auth=self.epidata_client.auth).status_code + self._make_request("covidcast", params=params, auth=self.epidata_client.auth).status_code ) self.assertEqual(status_codes, {200}) def test_no_multiples_allowed_signal_api_key(self): """Test requests with no multiples and allowed dashboard signal + valid API Key""" params = { - "source": "covidcast", "signal": "fb-survey:smoothed_wcli", "time_type": "day", "geo_type": "state", @@ -178,14 +167,13 @@ def test_no_multiples_allowed_signal_api_key(self): status_codes = set() for _ in range(10): status_codes.add( - self._make_request(params=params, auth=self.epidata_client.auth).status_code + self._make_request("covidcast", params=params, auth=self.epidata_client.auth).status_code ) self.assertEqual(status_codes, {200}) def test_no_multiples_allowed_signal_bad_api_key(self): """Test requests with no multiples and allowed dashboard signal + bad API Key""" params = { - "source": "covidcast", "signal": "fb-survey:smoothed_wcli", "time_type": "day", "geo_type": "state", @@ -196,54 +184,51 @@ def test_no_multiples_allowed_signal_bad_api_key(self): for _ in range(10): status_codes.add( self._make_request( - params=params, auth=("bad_key", "bad_email") + "covidcast", params=params, auth=("bad_key", "bad_email") ).status_code ) self.assertEqual(status_codes, {200}) def test_restricted_endpoint_no_key(self): """Test restricted endpoint with no auth key""" - params = {"source": "cdc", "regions": "1as", "epiweeks": "202020"} + params = {"regions": "1as", "epiweeks": "202020"} status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params).status_code) + status_codes.add(self._make_request("cdc", params=params).status_code) self.assertEqual(status_codes, {401}) def test_restricted_endpoint_invalid_key(self): """Test restricted endpoint with invalid auth key""" params = { - "source": "cdc", "regions": "1as", "epiweeks": "202020", "auth": "invalid_key", } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params).status_code) + status_codes.add(self._make_request("cdc", params=params).status_code) self.assertEqual(status_codes, {401}) def test_restricted_endpoint_no_roles_key(self): """Test restricted endpoint with no roles key""" params = { - "source": "cdc", "regions": "1as", "epiweeks": "202020", "auth": "key", } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params).status_code) + status_codes.add(self._make_request("cdc", params=params).status_code) self.assertEqual(status_codes, {401}) def test_restricted_endpoint_valid_roles_key(self): """Test restricted endpoint with valid auth key with required role""" params = { - "source": "cdc", "regions": "1as", "epiweeks": "202020", "auth": "cdc_key", } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params).status_code) + status_codes.add(self._make_request("cdc", params=params).status_code) self.assertEqual(status_codes, {200}) diff --git a/integrations/server/test_signal_dashboard.py b/integrations/server/test_signal_dashboard.py index e99a464fd..1aed3eec6 100644 --- a/integrations/server/test_signal_dashboard.py +++ b/integrations/server/test_signal_dashboard.py @@ -25,10 +25,7 @@ def setUp(self) -> None: def test_signal_dashboard_coverage(self): """Basic integration test for signal_dashboard_coverage endpoint""" - params = { - "endpoint": "signal_dashboard_coverage", - } - response = self.epidata_client._request(params=params) + response = self.epidata_client._request("signal_dashboard_coverage") self.assertEqual( response, { @@ -41,10 +38,7 @@ def test_signal_dashboard_coverage(self): def test_signal_dashboard_status(self): """Basic integration test for signal_dashboard_status endpoint""" - params = { - "endpoint": "signal_dashboard_status", - } - response = self.epidata_client._request(params=params) + response = self.epidata_client._request("signal_dashboard_status") self.assertEqual( response, { diff --git a/src/common/integration_test_base_class.py b/src/common/integration_test_base_class.py index 387c0f617..47c9f68e4 100644 --- a/src/common/integration_test_base_class.py +++ b/src/common/integration_test_base_class.py @@ -19,7 +19,7 @@ def __init__(self, methodName: str = "runTest") -> None: self.create_tables_list = [] self.role_name = None self.epidata_client = Epidata - self.epidata_client.BASE_URL = "http://delphi_web_epidata/epidata/api.php" + self.epidata_client.BASE_URL = "http://delphi_web_epidata/epidata" self.epidata_client.auth = ("epidata", "key") def create_key_with_role(self, cur, role_name: str): From 6e7190f96bbf02b7fcd4aca0687fd8466a934840 Mon Sep 17 00:00:00 2001 From: Rostyslav Zatserkovnyi Date: Wed, 27 Sep 2023 19:03:29 +0300 Subject: [PATCH 05/15] Add pypi changelog --- src/client/packaging/pypi/CHANGELOG.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 src/client/packaging/pypi/CHANGELOG.md diff --git a/src/client/packaging/pypi/CHANGELOG.md b/src/client/packaging/pypi/CHANGELOG.md new file mode 100644 index 000000000..c9853bba8 --- /dev/null +++ b/src/client/packaging/pypi/CHANGELOG.md @@ -0,0 +1,19 @@ +# Change Log +All notable changes to this client will be documented in this file. + +The format is based on [Keep a Changelog](http://keepachangelog.com/) +and this project adheres to [Semantic Versioning](http://semver.org/). + +## [4.2.0] - 2023-09-27 + +### Added + +### Changed + +- Modify the signatures for several methods: endpoint no longer needs to be specified in the params dict under the "source" key, but becomes a mandatory parameter for the method: + - `_request(params)` → `_request(endpoint, params={})` + - `_request_with_retry(params)` → `_request_with_retry(endpoint, params={})` + - `async_epidata(param_list, batch_size=50)` → `async_epidata(endpoint, param_list, batch_size=50)` + +### Fixed + \ No newline at end of file From 4513ced99f69557fc6227e24e87a9fa25a74fac4 Mon Sep 17 00:00:00 2001 From: Rostyslav Zatserkovnyi Date: Wed, 27 Sep 2023 19:03:40 +0300 Subject: [PATCH 06/15] Extra test for compat endpoint --- integrations/server/test_covidcast_endpoints.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/integrations/server/test_covidcast_endpoints.py b/integrations/server/test_covidcast_endpoints.py index e79b6f32f..2b74004bf 100644 --- a/integrations/server/test_covidcast_endpoints.py +++ b/integrations/server/test_covidcast_endpoints.py @@ -115,6 +115,12 @@ def test_compatibility(self): out = self._fetch("/", signal=first.signal_pair(), geo=first.geo_pair(), time="day:*", is_compatibility=True) self.assertEqual(out["epidata"], [row.as_api_compatibility_row_dict() for row in rows]) + with self.subTest("same contents sans excluded columns"): + compat = self._fetch("/", signal=first.signal_pair(), geo=first.geo_pair(), time="day:*", is_compatibility=True) + regular = self._fetch("/", signal=first.signal_pair(), geo=first.geo_pair(), time="day:*") + regular_sans_excluded = [{k: v for k, v in row.items() if k not in ["source", "geo_type", "time_type"]} for row in regular["epidata"]] + self.assertEqual(compat["epidata"], regular_sans_excluded) + def test_compatibility_restricted_source(self): """Restricted request at the /api.php endpoint.""" rows = [CovidcastTestRow.make_default_row(time_value=2020_04_01 + i, value=i, source="quidel") for i in range(10)] From 832b25ccaec86650fae5954acbea4b7a73945a9f Mon Sep 17 00:00:00 2001 From: Rostyslav Zatserkovnyi Date: Wed, 27 Sep 2023 19:07:08 +0300 Subject: [PATCH 07/15] Use del --- integrations/server/test_covidcast_endpoints.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/integrations/server/test_covidcast_endpoints.py b/integrations/server/test_covidcast_endpoints.py index 2b74004bf..206ed3c54 100644 --- a/integrations/server/test_covidcast_endpoints.py +++ b/integrations/server/test_covidcast_endpoints.py @@ -118,8 +118,11 @@ def test_compatibility(self): with self.subTest("same contents sans excluded columns"): compat = self._fetch("/", signal=first.signal_pair(), geo=first.geo_pair(), time="day:*", is_compatibility=True) regular = self._fetch("/", signal=first.signal_pair(), geo=first.geo_pair(), time="day:*") - regular_sans_excluded = [{k: v for k, v in row.items() if k not in ["source", "geo_type", "time_type"]} for row in regular["epidata"]] - self.assertEqual(compat["epidata"], regular_sans_excluded) + # Remove keys from the regular row which are excluded in the compat rows + for row in regular['epidata']: + for key in ['source', 'geo_type', 'time_type']: + del row[key] + self.assertEqual(compat, regular) def test_compatibility_restricted_source(self): """Restricted request at the /api.php endpoint.""" From 9eb559cfcbaa5a8c7bc8de625fcc97a7b8ba66c5 Mon Sep 17 00:00:00 2001 From: george haff Date: Mon, 2 Oct 2023 14:02:35 -0400 Subject: [PATCH 08/15] revert to previous behavior, and mark deprecated ( not fully tested) --- src/client/delphi_epidata.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/client/delphi_epidata.py b/src/client/delphi_epidata.py index e773bdc50..99d8f4abc 100644 --- a/src/client/delphi_epidata.py +++ b/src/client/delphi_epidata.py @@ -691,14 +691,17 @@ def covidcast_nowcast( return Epidata._request("covidcast_nowcast", params) @staticmethod - def async_epidata(endpoint, param_list, batch_size=50): - """Make asynchronous Epidata calls for a list of parameters.""" + def async_epidata(param_list, batch_size=50): + """[DEPRECATED] Make asynchronous Epidata calls for a list of parameters.""" - request_url = f"{Epidata.BASE_URL}/{endpoint}" + import warnings + warnings.filterwarnings("once", category=DeprecationWarning, module="delphi_epidata") + warnings.warn("Method `Epidata.async_epidata()` is deprecated and will be removed in a future version.", + category=DeprecationWarning) async def async_get(params, session): """Helper function to make Epidata GET requests.""" - async with session.get(request_url, params=params) as response: + async with session.get(f"{Epidata.BASE_URL}/api.php", params=params) as response: response.raise_for_status() return await response.json(), params From 3f5e1a6fc8ceeddb4904dea5e433189924df82e1 Mon Sep 17 00:00:00 2001 From: george haff Date: Mon, 2 Oct 2023 14:13:30 -0400 Subject: [PATCH 09/15] also revert tests --- integrations/client/test_delphi_epidata.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/integrations/client/test_delphi_epidata.py b/integrations/client/test_delphi_epidata.py index 99473f0ac..361105b22 100644 --- a/integrations/client/test_delphi_epidata.py +++ b/integrations/client/test_delphi_epidata.py @@ -335,9 +335,9 @@ def test_async_epidata(self): ] self._insert_rows(rows) - test_output = Epidata.async_epidata('covidcast', [ - self.params_from_row(rows[0]), - self.params_from_row(rows[1]) + test_output = Epidata.async_epidata([ + self.params_from_row(rows[0], source='covidcast'), + self.params_from_row(rows[1], source='covidcast') ]*12, batch_size=10) responses = [i[0] for i in test_output] # check response is same as standard covidcast call, using 24 calls to test batch sizing @@ -352,8 +352,9 @@ def test_async_epidata(self): @fake_epidata_endpoint def test_async_epidata_fail(self): with pytest.raises(ClientResponseError, match="404, message='NOT FOUND'"): - Epidata.async_epidata('covidcast', [ + Epidata.async_epidata([ { + 'source': 'covidcast', 'data_source': 'src', 'signals': 'sig', 'time_type': 'day', From 5f0b5211522d9d7dc08227d12344e754703358d4 Mon Sep 17 00:00:00 2001 From: Rostyslav Zatserkovnyi Date: Tue, 3 Oct 2023 18:57:30 +0300 Subject: [PATCH 10/15] test_api_keys tweak --- integrations/server/test_api_keys.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integrations/server/test_api_keys.py b/integrations/server/test_api_keys.py index 0c5be4c0b..d5c742d3d 100644 --- a/integrations/server/test_api_keys.py +++ b/integrations/server/test_api_keys.py @@ -21,7 +21,7 @@ def test_public_route(self): public_route = "http://delphi_web_epidata/epidata/version" status_codes = set() for _ in range(10): - status_codes.add(self._make_request("version", public_route).status_code) + status_codes.add(self._make_request("version").status_code) self.assertEqual(status_codes, {200}) def test_no_multiples_data_source(self): From cd665bd39ee93d4b13df1aa0e8cb2e76a2a263a9 Mon Sep 17 00:00:00 2001 From: Rostyslav Zatserkovnyi Date: Tue, 3 Oct 2023 19:42:56 +0300 Subject: [PATCH 11/15] Fix failing test --- integrations/client/test_delphi_epidata.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/integrations/client/test_delphi_epidata.py b/integrations/client/test_delphi_epidata.py index 361105b22..ff32d1ac6 100644 --- a/integrations/client/test_delphi_epidata.py +++ b/integrations/client/test_delphi_epidata.py @@ -339,13 +339,15 @@ def test_async_epidata(self): self.params_from_row(rows[0], source='covidcast'), self.params_from_row(rows[1], source='covidcast') ]*12, batch_size=10) - responses = [i[0] for i in test_output] - # check response is same as standard covidcast call, using 24 calls to test batch sizing + responses = [i[0]["epidata"] for i in test_output] + # check response is same as standard covidcast call (minus fields omitted by the api.php endpoint), + # using 24 calls to test batch sizing + ignore_fields = ["source", "time_type", "geo_type"] self.assertEqual( responses, [ - Epidata.covidcast(**self.params_from_row(rows[0])), - Epidata.covidcast(**self.params_from_row(rows[1])), + [{k: v for k, v in row.items() if k not in ignore_fields} for row in Epidata.covidcast(**self.params_from_row(rows[0]))["epidata"]], + [{k: v for k, v in row.items() if k not in ignore_fields} for row in Epidata.covidcast(**self.params_from_row(rows[1]))["epidata"]], ]*12 ) From 65d1587e154931bc29e8278336ea15ded4c76fdd Mon Sep 17 00:00:00 2001 From: Rostyslav Zatserkovnyi Date: Wed, 4 Oct 2023 18:18:42 +0300 Subject: [PATCH 12/15] More test tweaks --- integrations/client/test_delphi_epidata.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integrations/client/test_delphi_epidata.py b/integrations/client/test_delphi_epidata.py index ff32d1ac6..8c2f26803 100644 --- a/integrations/client/test_delphi_epidata.py +++ b/integrations/client/test_delphi_epidata.py @@ -342,12 +342,12 @@ def test_async_epidata(self): responses = [i[0]["epidata"] for i in test_output] # check response is same as standard covidcast call (minus fields omitted by the api.php endpoint), # using 24 calls to test batch sizing - ignore_fields = ["source", "time_type", "geo_type"] + ignore_fields = CovidcastTestRow._api_row_compatibility_ignore_fields self.assertEqual( responses, [ - [{k: v for k, v in row.items() if k not in ignore_fields} for row in Epidata.covidcast(**self.params_from_row(rows[0]))["epidata"]], - [{k: v for k, v in row.items() if k not in ignore_fields} for row in Epidata.covidcast(**self.params_from_row(rows[1]))["epidata"]], + [{k: row[k] for k in row.keys() - ignore_fields} for row in Epidata.covidcast(**self.params_from_row(rows[0]))["epidata"]], + [{k: row[k] for k in row.keys() - ignore_fields} for row in Epidata.covidcast(**self.params_from_row(rows[1]))["epidata"]], ]*12 ) From e701cb6d8d81687f0be58e4e065b8981e3bc9f54 Mon Sep 17 00:00:00 2001 From: george haff Date: Thu, 5 Oct 2023 14:55:31 -0400 Subject: [PATCH 13/15] changelog update --- src/client/packaging/pypi/CHANGELOG.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/client/packaging/pypi/CHANGELOG.md b/src/client/packaging/pypi/CHANGELOG.md index ec322b6ae..f138604ad 100644 --- a/src/client/packaging/pypi/CHANGELOG.md +++ b/src/client/packaging/pypi/CHANGELOG.md @@ -3,12 +3,16 @@ All notable future changes to the `delphi_epidata` python client will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/). -## [4.2.0] - 2023-09-27 +## [4.1.11] - 2023-10-12 + +https://github.com/cmu-delphi/delphi-epidata/pull/1288 ### Changed - -- Modify the signatures for several methods: endpoint no longer needs to be specified in the params dict under the "source" key, but becomes a mandatory parameter for the method: - - `async_epidata(param_list, batch_size=50)` → `async_epidata(endpoint, param_list, batch_size=50)` +- Internally uses newer endpoint-specific URLs instead of an older compatibility alias URL. +- Upper limit on the number of rows returned per request has been increased to 1M (was 3650). +- Method `Epidata.async_epidata()` is now deprecated and will be removed in a future version. +- The dict object returned from data request methods will now always have an entry for the "`epidata`" key, potentially with an empty list as its value. previously, if there were no results for the request, the "`epidata`" entry was not present. +- Results from the `Epidata.covidcast()` method (or potentially the `covidcast` endpoint via the `async_epidata()` method) will now include the keys "`source`", "`geo_type`", and "`time_type`" (and their associated values). ## [4.1.10] - 2023-09-28 From fefc0889b97e7c97397e4cde73662d9caa8db94f Mon Sep 17 00:00:00 2001 From: george haff Date: Thu, 5 Oct 2023 14:58:31 -0400 Subject: [PATCH 14/15] changelog update (formatting) --- src/client/packaging/pypi/CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/client/packaging/pypi/CHANGELOG.md b/src/client/packaging/pypi/CHANGELOG.md index f138604ad..ee95128f7 100644 --- a/src/client/packaging/pypi/CHANGELOG.md +++ b/src/client/packaging/pypi/CHANGELOG.md @@ -5,7 +5,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/). ## [4.1.11] - 2023-10-12 -https://github.com/cmu-delphi/delphi-epidata/pull/1288 +### Includes +- https://github.com/cmu-delphi/delphi-epidata/pull/1288 ### Changed - Internally uses newer endpoint-specific URLs instead of an older compatibility alias URL. From b5273472774f286f7b2c2ec738dd50b018c36fb9 Mon Sep 17 00:00:00 2001 From: Rostyslav Zatserkovnyi Date: Thu, 5 Oct 2023 22:15:58 +0300 Subject: [PATCH 15/15] Remove public_route --- integrations/server/test_api_keys.py | 1 - 1 file changed, 1 deletion(-) diff --git a/integrations/server/test_api_keys.py b/integrations/server/test_api_keys.py index d5c742d3d..8cb50016a 100644 --- a/integrations/server/test_api_keys.py +++ b/integrations/server/test_api_keys.py @@ -18,7 +18,6 @@ def _make_request(self, endpoint: str, params: dict = {}, auth: tuple = None): def test_public_route(self): """Test public route""" - public_route = "http://delphi_web_epidata/epidata/version" status_codes = set() for _ in range(10): status_codes.add(self._make_request("version").status_code)