diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2dca44222..fd2d5fb4b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -75,7 +75,7 @@ jobs: - name: Start delphi_web_epidata run: | - docker run --rm -d -p 10080:80 --env "MODULE_NAME=delphi.epidata.server.main" --env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" --env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" --env "REDIS_HOST=delphi_redis" --env "REDIS_PASSWORD=1234" --env "API_KEY_REGISTER_WEBHOOK_TOKEN=abc" --env "API_KEY_ADMIN_PASSWORD=test_admin_password" --network delphi-net --name delphi_web_epidata delphi_web_epidata + docker run --rm -d -p 10080:80 --env "MODULE_NAME=delphi.epidata.server.main" --env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" --env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" --env "REDIS_HOST=delphi_redis" --env "REDIS_PASSWORD=1234" --env "API_KEY_REGISTER_WEBHOOK_TOKEN=abc" --env "API_KEY_ADMIN_PASSWORD=test_admin_password" --env "TESTING_MODE=True" --network delphi-net --name delphi_web_epidata delphi_web_epidata docker ps - name: Run Unit Tests diff --git a/dev/local/Makefile b/dev/local/Makefile index e7e896aa6..fb02668ee 100644 --- a/dev/local/Makefile +++ b/dev/local/Makefile @@ -104,6 +104,7 @@ web: --env "REDIS_PASSWORD=1234" \ --env "API_KEY_ADMIN_PASSWORD=test_admin_password" \ --env "API_KEY_REGISTER_WEBHOOK_TOKEN=abc" \ + --env "TESTING_MODE=True" \ --network delphi-net --name delphi_web_epidata \ delphi_web_epidata >$(LOG_WEB) 2>&1 & diff --git a/integrations/acquisition/covidcast/test_covidcast_meta_caching.py b/integrations/acquisition/covidcast/test_covidcast_meta_caching.py index 6e4c6378f..16e2506fc 100644 --- a/integrations/acquisition/covidcast/test_covidcast_meta_caching.py +++ b/integrations/acquisition/covidcast/test_covidcast_meta_caching.py @@ -58,6 +58,19 @@ def setUp(self): secrets.db.host = 'delphi_database_epidata' secrets.db.epi = ('user', 'pass') + epidata_cnx = mysql.connector.connect( + user='user', + password='pass', + host='delphi_database_epidata', + database='epidata') + epidata_cur = epidata_cnx.cursor() + + epidata_cur.execute("DELETE FROM `api_user`") + epidata_cur.execute('INSERT INTO `api_user`(`api_key`, `email`) VALUES("key", "email")') + epidata_cnx.commit() + epidata_cur.close() + epidata_cnx.close() + # use the local instance of the Epidata API Epidata.BASE_URL = BASE_URL Epidata.auth = ('epidata', 'key') diff --git a/integrations/acquisition/covidcast/test_csv_uploading.py b/integrations/acquisition/covidcast/test_csv_uploading.py index e4c9d881e..e9c066f6a 100644 --- a/integrations/acquisition/covidcast/test_csv_uploading.py +++ b/integrations/acquisition/covidcast/test_csv_uploading.py @@ -55,6 +55,19 @@ def setUp(self): secrets.db.host = 'delphi_database_epidata' secrets.db.epi = ('user', 'pass') + epidata_cnx = mysql.connector.connect( + user='user', + password='pass', + host='delphi_database_epidata', + database='epidata') + epidata_cur = epidata_cnx.cursor() + + epidata_cur.execute("DELETE FROM `api_user`") + epidata_cur.execute('INSERT INTO `api_user`(`api_key`, `email`) VALUES("key", "email")') + epidata_cnx.commit() + epidata_cur.close() + epidata_cnx.close() + # use the local instance of the Epidata API Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' Epidata.auth = ('epidata', 'key') diff --git a/integrations/server/test_api_keys.py b/integrations/server/test_api_keys.py new file mode 100644 index 000000000..418e265ac --- /dev/null +++ b/integrations/server/test_api_keys.py @@ -0,0 +1,249 @@ +"""Integration tests for the API Keys""" +import requests + +# first party +from delphi.epidata.common.integration_test_base_class import DelphiTestBase + + +class APIKeysTets(DelphiTestBase): + """Tests the API Keys behaviour""" + + def localSetUp(self): + self.role_name = "cdc" + + def _make_request(self, url: str = None, params: dict = {}, auth: tuple = None): + if not url: + url = self.epidata_client.BASE_URL + response = requests.get(url, params=params, auth=auth) + return response + + def test_public_route(self): + """Test public route""" + public_route = "http://delphi_web_epidata/epidata/version" + status_codes = set() + for _ in range(10): + status_codes.add(self._make_request(public_route).status_code) + self.assertEqual(status_codes, {200}) + + def test_no_multiples_data_source(self): + """Test requests with no multiples and with provided `data_source` and `signal` as a separate query params.""" + params = { + "source": "covidcast", + "data_source": "fb-survey", + "signal": "smoothed_wcli", + "time_type": "day", + "geo_type": "state", + "geo_value": "pa", + "time_values": "20200406", + } + status_codes = set() + for _ in range(10): + status_codes.add(self._make_request(params=params).status_code) + self.assertEqual(status_codes, {200}) + + def test_no_multiples_source_signal(self): + """Test requests with colon-delimited source-signal param presentation.""" + params = { + "source": "covidcast", + "signal": "fb-survey:smoothed_wcli", + "time_type": "day", + "geo_type": "state", + "geo_value": "pa", + "time_values": "20200406", + } + status_codes = set() + for _ in range(10): + status_codes.add(self._make_request(params=params).status_code) + self.assertEqual(status_codes, {200}) + + def test_multiples_allowed_signal_two_multiples(self): + """Test requests with 2 multiples and allowed dashboard signal""" + params = { + "source": "covidcast", + "signal": "fb-survey:smoothed_wcli", + "time_type": "day", + "geo_type": "state", + "geo_value": "pa,ny", + "time_values": "20200406,20200407", + } + status_codes = set() + for _ in range(10): + status_codes.add(self._make_request(params=params).status_code) + self.assertEqual(status_codes, {200}) + + def test_multiples_non_allowed_signal(self): + """Test requests with 2 multiples and non-allowed dashboard signal""" + params = { + "source": "covidcast", + "signal": "hospital-admissions:smoothed_adj_covid19_from_claims", + "time_type": "day", + "geo_type": "state", + "geo_value": "pa,ny", + "time_values": "20200406,20200407", + } + status_codes = set() + for _ in range(10): + status_codes.add(self._make_request(params=params).status_code) + self.assertEqual(status_codes, {200, 429}) + + def test_multiples_mixed_allowed_signal_two_multiples(self): + """Test requests with 2 multiples and mixed-allowed dashboard signal""" + params = { + "source": "covidcast", + "signal": "fb-survey:smoothed_wcli,hospital-admissions:smoothed_adj_covid19_from_claims", + "time_type": "day", + "geo_type": "state", + "geo_value": "pa", + "time_values": "20200406,20200407", + } + status_codes = set() + for _ in range(10): + status_codes.add(self._make_request(params=params).status_code) + self.assertEqual(status_codes, {200, 429}) + + def test_multiples_allowed_signal_three_multiples(self): + """Test requests with 3 multiples and allowed dashboard signal""" + params = { + "source": "covidcast", + "signal": "fb-survey:smoothed_wcli,fb-survey:smoothed_wcli", + "time_type": "day", + "geo_type": "state", + "geo_value": "pa,ny", + "time_values": "20200406,20200407", + } + status_codes = set() + for _ in range(10): + status_codes.add(self._make_request(params=params).status_code) + self.assertEqual(status_codes, {401}) + + def test_multiples_mixed_allowed_signal_three_multiples(self): + """Test requests with 3 multiples and mixed-allowed dashboard signal""" + params = { + "source": "covidcast", + "signal": "fb-survey:smoothed_wcli,fb-survey:smoothed_wcli1", + "time_type": "day", + "geo_type": "state", + "geo_value": "pa,ny", + "time_values": "20200406,20200407", + } + status_codes = set() + for _ in range(10): + status_codes.add(self._make_request(params=params).status_code) + self.assertEqual(status_codes, {401}) + + def test_multiples_mixed_allowed_signal_api_key(self): + """Test requests with 3 multiples and mixed-allowed dashboard signal + valid API Key""" + params = { + "source": "covidcast", + "signal": "fb-survey:smoothed_wcli,fb-survey:smoothed_wcli1", + "time_type": "day", + "geo_type": "state", + "geo_value": "pa,ny", + "time_values": "20200406,20200407", + } + status_codes = set() + for _ in range(10): + status_codes.add( + self._make_request(params=params, auth=self.epidata_client.auth).status_code + ) + self.assertEqual(status_codes, {200}) + + def test_multiples_allowed_signal_api_key(self): + """Test requests with 3 multiples and allowed dashboard signal + valid API Key""" + params = { + "source": "covidcast", + "signal": "fb-survey:smoothed_wcli,fb-survey:smoothed_wcli", + "time_type": "day", + "geo_type": "state", + "geo_value": "pa,ny", + "time_values": "20200406,20200407", + } + status_codes = set() + for _ in range(10): + status_codes.add( + self._make_request(params=params, auth=self.epidata_client.auth).status_code + ) + self.assertEqual(status_codes, {200}) + + def test_no_multiples_allowed_signal_api_key(self): + """Test requests with no multiples and allowed dashboard signal + valid API Key""" + params = { + "source": "covidcast", + "signal": "fb-survey:smoothed_wcli", + "time_type": "day", + "geo_type": "state", + "geo_value": "pa", + "time_values": "20200406", + } + status_codes = set() + for _ in range(10): + status_codes.add( + self._make_request(params=params, auth=self.epidata_client.auth).status_code + ) + self.assertEqual(status_codes, {200}) + + def test_no_multiples_allowed_signal_bad_api_key(self): + """Test requests with no multiples and allowed dashboard signal + bad API Key""" + params = { + "source": "covidcast", + "signal": "fb-survey:smoothed_wcli", + "time_type": "day", + "geo_type": "state", + "geo_value": "pa", + "time_values": "20200406", + } + status_codes = set() + for _ in range(10): + status_codes.add( + self._make_request( + params=params, auth=("bad_key", "bad_email") + ).status_code + ) + self.assertEqual(status_codes, {200}) + + def test_restricted_endpoint_no_key(self): + """Test restricted endpoint with no auth key""" + params = {"source": "cdc", "regions": "1as", "epiweeks": "202020"} + status_codes = set() + for _ in range(10): + status_codes.add(self._make_request(params=params).status_code) + self.assertEqual(status_codes, {401}) + + def test_restricted_endpoint_invalid_key(self): + """Test restricted endpoint with invalid auth key""" + params = { + "source": "cdc", + "regions": "1as", + "epiweeks": "202020", + "auth": "invalid_key", + } + status_codes = set() + for _ in range(10): + status_codes.add(self._make_request(params=params).status_code) + self.assertEqual(status_codes, {401}) + + def test_restricted_endpoint_no_roles_key(self): + """Test restricted endpoint with no roles key""" + params = { + "source": "cdc", + "regions": "1as", + "epiweeks": "202020", + "auth": "key", + } + status_codes = set() + for _ in range(10): + status_codes.add(self._make_request(params=params).status_code) + self.assertEqual(status_codes, {401}) + + def test_restricted_endpoint_valid_roles_key(self): + """Test restricted endpoint with valid auth key with required role""" + params = { + "source": "cdc", + "regions": "1as", + "epiweeks": "202020", + "auth": "cdc_key", + } + status_codes = set() + for _ in range(10): + status_codes.add(self._make_request(params=params).status_code) + self.assertEqual(status_codes, {200}) diff --git a/integrations/server/test_cdc.py b/integrations/server/test_cdc.py new file mode 100644 index 000000000..d468bd162 --- /dev/null +++ b/integrations/server/test_cdc.py @@ -0,0 +1,42 @@ +# first party +from delphi.epidata.common.integration_test_base_class import DelphiTestBase + + +class CdcTest(DelphiTestBase): + """Basic integration tests for cdc endpint.""" + + def localSetUp(self) -> None: + self.truncate_tables_list = ["cdc_extract"] + self.role_name = "cdc" + + def test_cdc(self): + """Basic integration test for cdc endpoint""" + self.cur.execute( + "INSERT INTO `cdc_extract`(`epiweek`, `state`, `num1`, `num2`, `num3`, `num4`, `num5`, `num6`, `num7`, `num8`, `total`) VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", + ("201102", "AK", "16", "35", "51", "96", "30", "748", "243", "433", "65"), + ) + self.cnx.commit() + response = self.epidata_client.cdc(auth="cdc_key", epiweeks=201102, locations="cen9") + self.assertEqual( + response, + { + "epidata": [ + { + "location": "cen9", + "epiweek": 201102, + "num1": 16, + "num2": 35, + "num3": 51, + "num4": 96, + "num5": 30, + "num6": 748, + "num7": 243, + "num8": 433, + "total": 65, + "value": None, + } + ], + "result": 1, + "message": "success", + }, + ) diff --git a/integrations/server/test_delphi.py b/integrations/server/test_delphi.py new file mode 100644 index 000000000..fc3e3bc7e --- /dev/null +++ b/integrations/server/test_delphi.py @@ -0,0 +1,58 @@ +import json + +# first party +from delphi.epidata.common.integration_test_base_class import DelphiTestBase + + +class DelphiTest(DelphiTestBase): + """Basic integration tests for delphi endpint.""" + + def localSetUp(self): + self.truncate_tables_list = ["forecasts"] + + def test_delphi(self): + """Basic integration test for delphi endpoint""" + self.cur.execute( + "INSERT INTO `forecasts` (`system`, `epiweek`, `json`) VALUES(%s, %s, %s)", + ( + "eb", + "201441", + json.dumps( + { + "_version": "version", + "name": "name", + "season": "season", + "epiweek": "epiweek", + "year_weeks": 222, + "season_weeks": 111, + "ili_bins": "ili_bins_123", + "ili_bin_size": "ili_bin_size231", + } + ), + ), + ) + self.cnx.commit() + response = self.epidata_client.delphi(system="eb", epiweek=201441) + self.assertEqual( + response, + { + "epidata": [ + { + "epiweek": 201441, + "forecast": { + "_version": "version", + "epiweek": "epiweek", + "ili_bin_size": "ili_bin_size231", + "ili_bins": "ili_bins_123", + "name": "name", + "season": "season", + "season_weeks": 111, + "year_weeks": 222, + }, + "system": "eb", + } + ], + "message": "success", + "result": 1, + }, + ) diff --git a/integrations/server/test_dengue_nowcast.py b/integrations/server/test_dengue_nowcast.py new file mode 100644 index 000000000..79f9765f4 --- /dev/null +++ b/integrations/server/test_dengue_nowcast.py @@ -0,0 +1,42 @@ +# first party +from delphi.epidata.common.integration_test_base_class import DelphiTestBase + + +class DengueNowcastTest(DelphiTestBase): + """Basic integration tests for dengue_nowcast endpint.""" + + def localSetUp(self): + create_dengue_nowcasts = """ + CREATE TABLE IF NOT EXISTS `dengue_nowcasts` ( + `id` int NOT NULL AUTO_INCREMENT, + `target` varchar(32) CHARACTER SET utf8mb3 COLLATE utf8mb3_general_ci NOT NULL, + `epiweek` int NOT NULL, + `location` varchar(12) CHARACTER SET utf8mb3 COLLATE utf8mb3_general_ci DEFAULT NULL, + `value` float NOT NULL, + `std` float NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `entry` (`target`,`epiweek`,`location`), + KEY `target` (`target`), + KEY `epiweek` (`epiweek`), + KEY `location` (`location`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3; + """ + self.create_tables_list = [create_dengue_nowcasts] + self.truncate_tables_list = ["dengue_nowcasts"] + + def test_dengue_nowcasts(self): + """Basic integration test for dengue_nowcasts endpoint""" + self.cur.execute( + "INSERT INTO dengue_nowcasts(target, epiweek, location, value, std) VALUES(%s, %s, %s, %s, %s)", + ("num_dengue", "201409", "ar", "85263", "351456"), + ) + self.cnx.commit() + response = self.epidata_client.dengue_nowcast(locations="ar", epiweeks=201409) + self.assertEqual( + response, + { + "epidata": [{"location": "ar", "epiweek": 201409, "value": 85263.0, "std": 351456.0}], + "result": 1, + "message": "success", + }, + ) diff --git a/integrations/server/test_dengue_sensors.py b/integrations/server/test_dengue_sensors.py new file mode 100644 index 000000000..55d103367 --- /dev/null +++ b/integrations/server/test_dengue_sensors.py @@ -0,0 +1,43 @@ +# first party +from delphi.epidata.common.integration_test_base_class import DelphiTestBase + + +class DengueSensorsTest(DelphiTestBase): + """Basic integration tests for dengue_sensors endpint.""" + + def localSetUp(self): + create_dengue_sensors = """ + CREATE TABLE IF NOT EXISTS `dengue_sensors` ( + `id` int NOT NULL AUTO_INCREMENT, + `target` varchar(32) CHARACTER SET utf8mb3 COLLATE utf8mb3_general_ci NOT NULL, + `name` varchar(8) CHARACTER SET utf8mb3 COLLATE utf8mb3_general_ci NOT NULL, + `epiweek` int NOT NULL, + `location` varchar(12) CHARACTER SET utf8mb3 COLLATE utf8mb3_general_ci DEFAULT NULL, + `value` float NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `entry` (`target`,`name`,`epiweek`,`location`), + KEY `sensor` (`target`,`name`), + KEY `epiweek` (`epiweek`), + KEY `location` (`location`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3; + """ + self.create_tables_list = [create_dengue_sensors] + self.truncate_tables_list = ["dengue_sensors"] + self.role_name = "sensors" + + def test_dengue_sensors(self): + """Basic integration test for dengue_sensors endpoint""" + self.cur.execute( + "INSERT INTO `dengue_sensors`(`target`, `name`, `epiweek`, `location`, `value`) VALUES(%s, %s, %s, %s, %s)", + ("num_dengue", "ght", "201432", "ag", "1234"), + ) + self.cnx.commit() + response = self.epidata_client.dengue_sensors(auth="sensors_key", names="ght", locations="ag", epiweeks="201432") + self.assertEqual( + response, + { + "epidata": [{"name": "ght", "location": "ag", "epiweek": 201432, "value": 1234.0}], + "result": 1, + "message": "success", + }, + ) diff --git a/integrations/server/test_ecdc_ili.py b/integrations/server/test_ecdc_ili.py new file mode 100644 index 000000000..5fe24bcd3 --- /dev/null +++ b/integrations/server/test_ecdc_ili.py @@ -0,0 +1,35 @@ +# first party +from delphi.epidata.common.integration_test_base_class import DelphiTestBase + + +class EcdcIliTest(DelphiTestBase): + """Basic integration tests for edcd_ili endpint.""" + + def localSetUp(self): + self.truncate_tables_list = ["ecdc_ili"] + + def test_ecdc_ili(self): + """Basic integration test for ecdc_ili endpoint""" + self.cur.execute( + "INSERT INTO `ecdc_ili`(`release_date`, `issue`, `epiweek`, `lag`, `region`, `incidence_rate`) VALUES(%s, %s, %s, %s, %s, %s)", + ("2020-03-26", "202012", "201840", "76", "Armenia", "0"), + ) + self.cnx.commit() + response = self.epidata_client.ecdc_ili(regions="Armenia", epiweeks="201840") + self.assertEqual( + response, + { + "epidata": [ + { + "release_date": "2020-03-26", + "region": "Armenia", + "issue": 202012, + "epiweek": 201840, + "lag": 76, + "incidence_rate": 0.0, + } + ], + "result": 1, + "message": "success", + }, + ) diff --git a/integrations/server/test_flusurv.py b/integrations/server/test_flusurv.py new file mode 100644 index 000000000..33f0f00b8 --- /dev/null +++ b/integrations/server/test_flusurv.py @@ -0,0 +1,40 @@ +# first party +from delphi.epidata.common.integration_test_base_class import DelphiTestBase + + +class FlusurvTest(DelphiTestBase): + """Basic integration tests for flusurv endpint.""" + + def localSetUp(self): + self.truncate_tables_list = ["flusurv"] + + def test_flusurv(self): + """Basic integration test for flusurv endpoint""" + self.cur.execute( + "INSERT INTO `flusurv`(`release_date`, `issue`, `epiweek`, `location`, `lag`, `rate_age_0`, `rate_age_1`, `rate_age_2`, `rate_age_3`, `rate_age_4`, `rate_overall`, `rate_age_5`, `rate_age_6`, `rate_age_7`) VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", + ("2012-11-02", "201243", "201143", "CA", "52", "0", "0", "0", "0.151", "0", "0.029", "0", "0", "0"), + ) + self.cnx.commit() + response = self.epidata_client.flusurv(epiweeks=201143, locations="CA") + self.assertEqual( + response, + { + "epidata": [ + { + "release_date": "2012-11-02", + "location": "CA", + "issue": 201243, + "epiweek": 201143, + "lag": 52, + "rate_age_0": 0.0, + "rate_age_1": 0.0, + "rate_age_2": 0.0, + "rate_age_3": 0.151, + "rate_age_4": 0.0, + "rate_overall": 0.029, + } + ], + "result": 1, + "message": "success", + }, + ) diff --git a/integrations/server/test_fluview_clinical.py b/integrations/server/test_fluview_clinical.py new file mode 100644 index 000000000..8ebccfc9e --- /dev/null +++ b/integrations/server/test_fluview_clinical.py @@ -0,0 +1,40 @@ +# first party +from delphi.epidata.common.integration_test_base_class import DelphiTestBase + + +class FluviewClinicalTest(DelphiTestBase): + """Basic integration tests for fluview_clinical endpint.""" + + def localSetUp(self): + self.truncate_tables_list = ["fluview_clinical"] + + def test_fluview_clinical(self): + """Basic integration test for fluview_clinical endpoint""" + self.cur.execute( + "INSERT INTO `fluview_clinical`(`release_date`, `issue`, `epiweek`, `region`, `lag`, `total_specimens`, `total_a`, `total_b`, `percent_positive`, `percent_a`, `percent_b`) VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", + ("2018-10-10", "201839", "201640", "al", "103", "406", "4", "1", "1.32", "0.99", "0.25"), + ) + self.cnx.commit() + response = self.epidata_client.fluview_clinical(epiweeks=201640, regions="al") + self.assertEqual( + response, + { + "epidata": [ + { + "release_date": "2018-10-10", + "region": "al", + "issue": 201839, + "epiweek": 201640, + "lag": 103, + "total_specimens": 406, + "total_a": 4, + "total_b": 1, + "percent_positive": 1.32, + "percent_a": 0.99, + "percent_b": 0.25, + } + ], + "result": 1, + "message": "success", + }, + ) diff --git a/integrations/server/test_gft.py b/integrations/server/test_gft.py new file mode 100644 index 000000000..3f2a68526 --- /dev/null +++ b/integrations/server/test_gft.py @@ -0,0 +1,22 @@ +# first party +from delphi.epidata.common.integration_test_base_class import DelphiTestBase + + +class GftTest(DelphiTestBase): + """Basic integration tests for gft endpint.""" + + def localSetUp(self): + self.truncate_tables_list = ["gft"] + + def test_gft(self): + """Basic integration test for gft endpoint""" + self.cur.execute( + "INSERT INTO `gft`(`epiweek`, `location`, `num`) VALUES(%s, %s, %s)", + ("200340", "nat", "902"), + ) + self.cnx.commit() + response = self.epidata_client.gft(locations="nat", epiweeks="200340") + self.assertEqual( + response, + {"epidata": [{"location": "nat", "epiweek": 200340, "num": 902}], "result": 1, "message": "success"}, + ) diff --git a/integrations/server/test_ght.py b/integrations/server/test_ght.py new file mode 100644 index 000000000..67b135aef --- /dev/null +++ b/integrations/server/test_ght.py @@ -0,0 +1,23 @@ +# first party +from delphi.epidata.common.integration_test_base_class import DelphiTestBase + + +class GhtTest(DelphiTestBase): + """Basic integration tests for ght endpint.""" + + def localSetUp(self): + self.truncate_tables_list = ["ght"] + self.role_name = "ght" + + def test_ght(self): + """Basic integration test for ght endpoint""" + self.cur.execute( + "INSERT INTO `ght`(`query`, `location`, `epiweek`, `value`) VALUES(%s, %s, %s, %s)", + ("/n/query", "US", "200101", "12345"), + ) + self.cnx.commit() + response = self.epidata_client.ght(locations="US", epiweeks="200101", query="/n/query", auth="ght_key") + self.assertEqual( + response, + {"epidata": [{"location": "US", "epiweek": 200101, "value": 12345.0}], "result": 1, "message": "success"}, + ) diff --git a/integrations/server/test_kcdc_ili.py b/integrations/server/test_kcdc_ili.py new file mode 100644 index 000000000..aab40baa6 --- /dev/null +++ b/integrations/server/test_kcdc_ili.py @@ -0,0 +1,35 @@ +# first party +from delphi.epidata.common.integration_test_base_class import DelphiTestBase + + +class KcdcIliTest(DelphiTestBase): + """Basic integration tests for kcdc_ili endpint.""" + + def localSetUp(self): + self.truncate_tables_list = ["kcdc_ili"] + + def test_kcdc_ili(self): + """Basic integration test for kcdc_ili endpoint""" + self.cur.execute( + "INSERT INTO `kcdc_ili`(`release_date`, `issue`, `epiweek`, `lag`, `region`, `ili`) VALUES(%s, %s, %s, %s, %s, %s)", + ("2020-03-27", "202013", "200432", "222", "REG", "0.25"), + ) + self.cnx.commit() + response = self.epidata_client.kcdc_ili(regions="REG", epiweeks="200432") + self.assertEqual( + response, + { + "epidata": [ + { + "release_date": "2020-03-27", + "region": "REG", + "issue": 202013, + "epiweek": 200432, + "lag": 222, + "ili": 0.25, + } + ], + "result": 1, + "message": "success", + }, + ) diff --git a/integrations/server/test_meta.py b/integrations/server/test_meta.py new file mode 100644 index 000000000..295a5cd98 --- /dev/null +++ b/integrations/server/test_meta.py @@ -0,0 +1,28 @@ +# first party +from delphi.epidata.common.integration_test_base_class import DelphiTestBase + + +class MetaTest(DelphiTestBase): + """Basic integration tests for meta endpint.""" + + def localSetUp(self): + self.truncate_tables_list = ["forecasts", "fluview", "wiki", "wiki_meta", "twitter"] + + def test_meta(self): + """Basic integration test for meta endpoint""" + response = self.epidata_client.meta() + self.assertEqual( + response, + { + "epidata": [ + { + "delphi": [], + "fluview": [{"latest_issue": None, "latest_update": None, "table_rows": 0}], + "twitter": [], + "wiki": [{"latest_update": None, "table_rows": 0}], + } + ], + "message": "success", + "result": 1, + }, + ) diff --git a/integrations/server/test_nidss_dengue.py b/integrations/server/test_nidss_dengue.py new file mode 100644 index 000000000..679937ea6 --- /dev/null +++ b/integrations/server/test_nidss_dengue.py @@ -0,0 +1,22 @@ +# first party +from delphi.epidata.common.integration_test_base_class import DelphiTestBase + + +class NiddsDengueTest(DelphiTestBase): + """Basic integration tests for nids_dengue endpint.""" + + def localSetUp(self): + self.truncate_tables_list = ["nidss_dengue"] + + def test_nidss_dengue(self): + """Basic integration test for nidds_dengue endpoint""" + self.cur.execute( + "INSERT INTO `nidss_dengue`(`epiweek`, `location`, `region`, `count`) VALUES(%s, %s, %s, %s)", + ("200340", "SomeCity", "Central", "0"), + ) + self.cnx.commit() + response = self.epidata_client.nidss_dengue(locations="SomeCity", epiweeks="200340") + self.assertEqual( + response, + {"epidata": [{"location": "SomeCity", "epiweek": 200340, "count": 0}], "result": 1, "message": "success"}, + ) diff --git a/integrations/server/test_nidss_flu.py b/integrations/server/test_nidss_flu.py new file mode 100644 index 000000000..0b13ee67f --- /dev/null +++ b/integrations/server/test_nidss_flu.py @@ -0,0 +1,36 @@ +# first party +from delphi.epidata.common.integration_test_base_class import DelphiTestBase + + +class NiddsFluTest(DelphiTestBase): + """Basic integration tests for nids_flu endpint.""" + + def localSetUp(self): + self.truncate_tables_list = ["nidss_flu"] + + def test_nidss_flu(self): + """Basic integration test for nidds_flu endpoint""" + self.cur.execute( + "INSERT INTO `nidss_flu`(`release_date`, `issue`, `epiweek`, `region`, `lag`, `visits`, `ili`) VALUES(%s, %s, %s, %s, %s, %s, %s)", + ("2015-09-05", "201530", "200111", "SomeRegion", "222", "333", "444"), + ) + self.cnx.commit() + response = self.epidata_client.nidss_flu(regions="SomeRegion", epiweeks="200111") + self.assertEqual( + response, + { + "epidata": [ + { + "release_date": "2015-09-05", + "region": "SomeRegion", + "issue": 201530, + "epiweek": 200111, + "lag": 222, + "visits": 333, + "ili": 444.0, + } + ], + "result": 1, + "message": "success", + }, + ) diff --git a/integrations/server/test_norostat.py b/integrations/server/test_norostat.py new file mode 100644 index 000000000..a7866a91d --- /dev/null +++ b/integrations/server/test_norostat.py @@ -0,0 +1,113 @@ +# first party +from delphi.epidata.common.integration_test_base_class import DelphiTestBase + + +class NorostatTest(DelphiTestBase): + """Basic integration tests for norostat endpint.""" + + def localSetUp(self): + create_norostat_point_diffs = """ + CREATE TABLE IF NOT EXISTS `norostat_point_diffs` ( + `release_date` date NOT NULL, + `parse_time` datetime NOT NULL, + `location_id` int NOT NULL, + `epiweek` int NOT NULL, + `new_value` varchar(255) CHARACTER SET utf8mb3 COLLATE utf8mb3_general_ci DEFAULT NULL, + PRIMARY KEY (`release_date`,`parse_time`,`location_id`,`epiweek`), + UNIQUE KEY `location_id` (`location_id`,`epiweek`,`release_date`,`parse_time`,`new_value`), + CONSTRAINT `norostat_point_diffs_ibfk_1` FOREIGN KEY (`release_date`, `parse_time`) REFERENCES `norostat_point_version_list` (`release_date`, `parse_time`), + CONSTRAINT `norostat_point_diffs_ibfk_2` FOREIGN KEY (`location_id`) REFERENCES `norostat_raw_datatable_location_pool` (`location_id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3; + """ + + create_raw_datatable_version_list = """ + CREATE TABLE IF NOT EXISTS `norostat_raw_datatable_version_list` ( + `release_date` date NOT NULL, + `parse_time` datetime NOT NULL, + PRIMARY KEY (`release_date`,`parse_time`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3; + """ + + create_norostat_version_list = """ + CREATE TABLE IF NOT EXISTS `norostat_point_version_list` ( + `release_date` date NOT NULL, + `parse_time` datetime NOT NULL, + PRIMARY KEY (`release_date`,`parse_time`), + CONSTRAINT `norostat_point_version_list_ibfk_1` FOREIGN KEY (`release_date`, `parse_time`) REFERENCES `norostat_raw_datatable_version_list` (`release_date`, `parse_time`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3; + """ + + create_norostat_datatable_location_pool = """ + CREATE TABLE IF NOT EXISTS `norostat_raw_datatable_location_pool` ( + `location_id` int NOT NULL AUTO_INCREMENT, + `location` varchar(255) CHARACTER SET utf8mb3 COLLATE utf8mb3_general_ci NOT NULL, + PRIMARY KEY (`location_id`), + UNIQUE KEY `location` (`location`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3; + """ + + self.create_tables_list = [ + create_raw_datatable_version_list, + create_norostat_version_list, + create_norostat_datatable_location_pool, + create_norostat_point_diffs, + ] + + self.delete_from_tables_list = [ + "norostat_point_diffs", + "norostat_point_version_list", + "norostat_raw_datatable_location_pool", + "norostat_raw_datatable_version_list", + ] + + self.role_name = "norostat" + + def test_norostat(self): + """Basic integration test for norostat endpoint""" + self.cur.execute( + 'INSERT INTO `norostat_raw_datatable_version_list`(`release_date`, `parse_time`) VALUES ("2023-07-19", "2023-07-10 15:24:51")' + ) + self.cur.execute( + 'INSERT INTO `norostat_raw_datatable_location_pool`(`location_id`, `location`) VALUES("1", "SomeTestLocation")' + ) + self.cur.execute( + 'INSERT INTO `norostat_point_version_list`(`release_date`, `parse_time`) VALUES("2023-07-19", "2023-07-10 15:24:51")' + ) + self.cur.execute( + 'INSERT INTO `norostat_point_diffs`(`release_date`, `parse_time`, `location_id`, `epiweek`, `new_value`) VALUES("2023-07-19", "2023-07-10 15:24:51", "1", "202329", 10)' + ) + self.cnx.commit() + response = self.epidata_client.norostat(auth="norostat_key", location="SomeTestLocation", epiweeks="202329") + self.assertEqual( + response, + { + "epidata": [{"release_date": "2023-07-19", "epiweek": 202329, "value": 10}], + "result": 1, + "message": "success", + }, + ) + return True + + def test_meta_norostat(self): + """Basic integration test for meta_norostat endpoint""" + + self.cur.execute( + "INSERT INTO `norostat_raw_datatable_version_list`(`release_date`, `parse_time`) VALUES (%s, %s)", + ("2014-10-22", "2048-12-08 15:22:51"), + ) + self.cur.execute( + 'INSERT INTO `norostat_raw_datatable_location_pool`(`location`) VALUES ("Minnesota, Ohio, Oregon, Tennessee, and Wisconsin")' + ) + self.cnx.commit() + response = self.epidata_client.meta_norostat(auth="norostat_key") + self.assertEqual( + response, + { + "epidata": { + "locations": [{"location": "Minnesota, Ohio, Oregon, Tennessee, and Wisconsin"}], + "releases": [{"release_date": "2014-10-22"}], + }, + "message": "success", + "result": 1, + }, + ) diff --git a/integrations/server/test_nowcast.py b/integrations/server/test_nowcast.py new file mode 100644 index 000000000..2b48dd0da --- /dev/null +++ b/integrations/server/test_nowcast.py @@ -0,0 +1,26 @@ +# first party +from delphi.epidata.common.integration_test_base_class import DelphiTestBase + + +class NowcastTest(DelphiTestBase): + """Basic integration tests for nowcast endpint.""" + + def localSetUp(self): + self.truncate_tables_list = ["nowcasts"] + + def test_nowcast(self): + """Basic integration test for nowcast endpoint""" + self.cur.execute( + "INSERT INTO `nowcasts`(`epiweek`, `location`, `value`, `std`) VALUES(%s, %s, %s, %s)", + ("201145", "nat", "12345", "0.01234"), + ) + self.cnx.commit() + response = self.epidata_client.nowcast(locations="nat", epiweeks="201145") + self.assertEqual( + response, + { + "epidata": [{"location": "nat", "epiweek": 201145, "value": 12345.0, "std": 0.01234}], + "result": 1, + "message": "success", + }, + ) diff --git a/integrations/server/test_paho_dengue.py b/integrations/server/test_paho_dengue.py new file mode 100644 index 000000000..bbe8953f8 --- /dev/null +++ b/integrations/server/test_paho_dengue.py @@ -0,0 +1,40 @@ +# first party +from delphi.epidata.common.integration_test_base_class import DelphiTestBase + + +class PahoDengueTest(DelphiTestBase): + """Basic integration tests for paho_dengue endpint.""" + + def localSetUp(self): + self.truncate_tables_list = ["paho_dengue"] + + def test_paho_dengue(self): + """Basic integration test for paho_dengue endpoint""" + self.cur.execute( + "INSERT INTO `paho_dengue`(`release_date`, `issue`, `epiweek`, `lag`, `region`, `total_pop`, `serotype`, `num_dengue`, `incidence_rate`, `num_severe`, `num_deaths`) VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", + ("2018-12-01", "201848", "201454", "204", "AG", "91", "DEN 1,4", "37", "40.66", "0", "0"), + ) + self.cnx.commit() + response = self.epidata_client.paho_dengue(regions="AG", epiweeks="201454") + self.assertEqual( + response, + { + "epidata": [ + { + "release_date": "2018-12-01", + "region": "AG", + "serotype": "DEN 1,4", + "issue": 201848, + "epiweek": 201454, + "lag": 204, + "total_pop": 91, + "num_dengue": 37, + "num_severe": 0, + "num_deaths": 0, + "incidence_rate": 40.66, + } + ], + "result": 1, + "message": "success", + }, + ) diff --git a/integrations/server/test_quidel.py b/integrations/server/test_quidel.py new file mode 100644 index 000000000..696a7ee41 --- /dev/null +++ b/integrations/server/test_quidel.py @@ -0,0 +1,23 @@ +# first party +from delphi.epidata.common.integration_test_base_class import DelphiTestBase + + +class QuidelTest(DelphiTestBase): + """Basic integration tests for quidel endpint.""" + + def localSetUp(self): + self.truncate_tables_list = ["quidel"] + self.role_name = "quidel" + + def test_quidel(self): + """Basic integration test for quidel endpoint""" + self.cur.execute( + "INSERT INTO `quidel`(`location`, `epiweek`, `value`, `num_rows`, `num_devices`) VALUES(%s, %s, %s, %s, %s)", + ("loc1", "201111", "1", "0", "0"), + ) + self.cnx.commit() + response = self.epidata_client.quidel(locations="loc1", epiweeks="201111", auth="quidel_key") + self.assertEqual( + response, + {"epidata": [{"location": "loc1", "epiweek": 201111, "value": 1.0}], "result": 1, "message": "success"}, + ) diff --git a/integrations/server/test_sensors.py b/integrations/server/test_sensors.py new file mode 100644 index 000000000..835b53893 --- /dev/null +++ b/integrations/server/test_sensors.py @@ -0,0 +1,27 @@ +# first party +from delphi.epidata.common.integration_test_base_class import DelphiTestBase + + +class SensorsTest(DelphiTestBase): + """Basic integration tests for sensors endpint.""" + + def localSetUp(self): + self.truncate_tables_list = ["sensors"] + self.role_name = "sensors" + + def test_sensors(self): + """Basic integration test for sensors endpoint""" + self.cur.execute( + "INSERT INTO `sensors`(`name`, `epiweek`, `location`, `value`) VALUES(%s, %s, %s, %s)", + ("sens1", "201111", "loc1", "222"), + ) + self.cnx.commit() + response = self.epidata_client.sensors(names="sens1", locations="loc1", epiweeks="201111", auth="sensors_key") + self.assertEqual( + response, + { + "epidata": [{"name": "sens1", "location": "loc1", "epiweek": 201111, "value": 222.0}], + "result": 1, + "message": "success", + }, + ) diff --git a/integrations/server/test_signal_dashboard.py b/integrations/server/test_signal_dashboard.py new file mode 100644 index 000000000..e99a464fd --- /dev/null +++ b/integrations/server/test_signal_dashboard.py @@ -0,0 +1,64 @@ +# first party +from delphi.epidata.common.integration_test_base_class import DelphiTestBase + + +class SignalDashboardTest(DelphiTestBase): + """Basic integration tests for signal_dashboard_coverage and signal_dashboard_status endpints.""" + + def setUp(self) -> None: + """Perform per-test setup.""" + + self.delete_from_tables_list = ["dashboard_signal_coverage", "dashboard_signal"] + super().setUp() + + self.cur.execute( + "INSERT INTO `dashboard_signal`(`id`, `name`, `source`, `covidcast_signal`, `enabled`, `latest_coverage_update`, `latest_status_update`) VALUES(%s, %s, %s, %s, %s, %s, %s)", + ("1", "Change", "chng", "smoothed_outpatient_covid", "1", "2021-10-02", "2021-11-27"), + ) + self.cur.execute( + "INSERT INTO `dashboard_signal_coverage`(`signal_id`, `date`, `geo_type`, `count`) VALUES(%s, %s, %s, %s)", + ("1", "2021-10-02", "county", "2222"), + ) + + self.cnx.commit() + + def test_signal_dashboard_coverage(self): + """Basic integration test for signal_dashboard_coverage endpoint""" + + params = { + "endpoint": "signal_dashboard_coverage", + } + response = self.epidata_client._request(params=params) + self.assertEqual( + response, + { + "epidata": {"Change": {"county": [{"count": 2222, "date": "2021-10-02"}]}}, + "message": "success", + "result": 1, + }, + ) + + def test_signal_dashboard_status(self): + """Basic integration test for signal_dashboard_status endpoint""" + + params = { + "endpoint": "signal_dashboard_status", + } + response = self.epidata_client._request(params=params) + self.assertEqual( + response, + { + "epidata": [ + { + "name": "Change", + "source": "chng", + "covidcast_signal": "smoothed_outpatient_covid", + "latest_issue": None, + "latest_time_value": None, + "coverage": {"county": [{"date": "2021-10-02", "count": 2222}]}, + } + ], + "result": 1, + "message": "success", + }, + ) diff --git a/integrations/server/test_twitter.py b/integrations/server/test_twitter.py new file mode 100644 index 000000000..7ba162480 --- /dev/null +++ b/integrations/server/test_twitter.py @@ -0,0 +1,27 @@ +# first party +from delphi.epidata.common.integration_test_base_class import DelphiTestBase + + +class TwitterTest(DelphiTestBase): + """Basic integration tests for twitter endpint.""" + + def localSetUp(self): + self.truncate_tables_list = ["twitter"] + self.role_name = "twitter" + + def test_twitter(self): + """Basic integration test for twitter endpoint""" + + self.cur.execute( + 'INSERT INTO `twitter`(`date`, `state`, `num`, `total`) VALUES ("2015-07-29", "AK", "1", "223"), ("2020-07-29", "CT", "12", "778")', + ) + self.cnx.commit() + response = self.epidata_client.twitter(auth="twitter_key", locations="cen9", dates="20150701-20160101") + self.assertEqual( + response, + { + "epidata": [{"location": "cen9", "date": "2015-07-29", "num": 1, "total": 223, "percent": 0.4484}], + "result": 1, + "message": "success", + }, + ) diff --git a/integrations/server/test_wiki.py b/integrations/server/test_wiki.py new file mode 100644 index 000000000..d53cef0c9 --- /dev/null +++ b/integrations/server/test_wiki.py @@ -0,0 +1,31 @@ +# first party +from delphi.epidata.common.integration_test_base_class import DelphiTestBase + + +class WikiTest(DelphiTestBase): + """Basic integration tests for wiki endpint.""" + + def localSetUp(self): + self.truncate_tables_list = ["wiki", "wiki_meta"] + + def test_wiki(self): + """Basic integration test for wiki endpoint""" + + self.cur.execute( + 'INSERT INTO `wiki`(`datetime`, `article`, `count`, `language`) VALUES ("2007-12-09 18:00:00", "amantadine", "3", "en"), ("2008-12-09 18:00:00", "test", "5", "en")', + ) + self.cur.execute( + 'INSERT INTO `wiki_meta`(`datetime`, `date`, `epiweek`, `total`, `language`) VALUES ("2007-12-09 18:00:00", "2007-12-09", "200750", "969214", "en"), ("2008-12-09 18:00:00", "2008-12-09", "200750", "123321", "en")' + ) + self.cnx.commit() + response = self.epidata_client.wiki(articles="test", epiweeks="200701-200801") + self.assertEqual( + response, + { + "epidata": [ + {"article": "test", "count": 5, "total": 123321, "hour": -1, "epiweek": 200750, "value": 40.544595} + ], + "result": 1, + "message": "success", + }, + ) diff --git a/src/common/integration_test_base_class.py b/src/common/integration_test_base_class.py new file mode 100644 index 000000000..387c0f617 --- /dev/null +++ b/src/common/integration_test_base_class.py @@ -0,0 +1,77 @@ +# standard library +import unittest + +# third party +import mysql.connector + +# first party +from delphi.epidata.client.delphi_epidata import Epidata +from delphi.epidata.server._limiter import limiter + + +class DelphiTestBase(unittest.TestCase): + """Basic integration test class""" + + def __init__(self, methodName: str = "runTest") -> None: + super().__init__(methodName) + self.delete_from_tables_list = [] + self.truncate_tables_list = [] + self.create_tables_list = [] + self.role_name = None + self.epidata_client = Epidata + self.epidata_client.BASE_URL = "http://delphi_web_epidata/epidata/api.php" + self.epidata_client.auth = ("epidata", "key") + + def create_key_with_role(self, cur, role_name: str): + cur.execute(f'INSERT INTO `api_user`(`api_key`, `email`) VALUES("{role_name}_key", "{role_name}_email")') + cur.execute(f'INSERT INTO `user_role`(`name`) VALUES("{role_name}")') + cur.execute( + f'INSERT INTO `user_role_link`(`user_id`, `role_id`) SELECT `api_user`.`id`, `user_role`.`id` FROM `api_user` JOIN `user_role` WHERE `api_user`.`api_key`="{role_name}_key" AND `user_role`.`name`="{role_name}"' + ) + + def setUp(self) -> None: + """Perform per-test setup.""" + + # connect to the `epidata` database + cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") + cur = cnx.cursor() + + cur.execute("DELETE FROM `api_user`") + cur.execute("TRUNCATE TABLE `user_role`") + cur.execute("TRUNCATE TABLE `user_role_link`") + cur.execute('INSERT INTO `api_user`(`api_key`, `email`) VALUES ("key", "email")') + + self.localSetUp() + + for stmt in self.create_tables_list: + cur.execute(stmt) + + for table_name in self.delete_from_tables_list: + cur.execute(f"DELETE FROM `{table_name}`") + + for table_name in self.truncate_tables_list: + cur.execute(f"TRUNCATE TABLE `{table_name}`") + + if self.role_name: + self.create_key_with_role(cur, self.role_name) + + cnx.commit() + cur.close() + + self.cnx = cnx + self.cur = cnx.cursor() + + def localSetUp(self): + # stub; override in subclasses to perform custom setup. + # runs after user/api_key tables have been truncated, but before test-specific tables are created/deleted/truncated and before database changes have been committed + pass + + @staticmethod + def _clear_limits() -> None: + limiter.storage.reset() + + def tearDown(self) -> None: + """Perform per-test teardown.""" + self.cur.close() + self.cnx.close() + self._clear_limits() diff --git a/src/server/_config.py b/src/server/_config.py index 4a96250c6..cf743300c 100644 --- a/src/server/_config.py +++ b/src/server/_config.py @@ -93,8 +93,16 @@ REDIS_HOST = os.environ.get("REDIS_HOST", "delphi_redis") REDIS_PASSWORD = os.environ.get("REDIS_PASSWORD", "1234") +# mode to reduce number of required requests to hit rate limit while running tests, +# by default is set to False +TESTING_MODE = os.environ.get("TESTING_MODE", False) + # https://flask-limiter.readthedocs.io/en/stable/#rate-limit-string-notation RATE_LIMIT = os.environ.get("RATE_LIMIT", "60/hour") + +if TESTING_MODE is not False: + RATE_LIMIT = "5/hour" + # fixed-window, fixed-window-elastic-expiry, or moving-window # see also https://flask-limiter.readthedocs.io/en/stable/#rate-limiting-strategies RATELIMIT_STRATEGY = os.environ.get("RATELIMIT_STRATEGY", "fixed-window")