Skip to content

Commit

Permalink
Enhanced download error handling
Browse files Browse the repository at this point in the history
  • Loading branch information
@DahyannAraya
DahyannAraya committed Oct 28, 2024
1 parent aa5bda3 commit 21c6c0a
Show file tree
Hide file tree
Showing 2 changed files with 67 additions and 22 deletions.
13 changes: 9 additions & 4 deletions climada_petals/hazard/copernicus_forecast/handler.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
from pathlib import Path
import datetime as dt
import matplotlib.pyplot as plt
import requests

import xarray as xr
import pandas as pd
Expand Down Expand Up @@ -444,17 +445,21 @@ def _download_data(
missing_params = [param for param, value in download_params.items() if not value]

Check warning on line 445 in climada_petals/hazard/copernicus_forecast/handler.py

View check run for this annotation

Jenkins - WCR / Pylint

line-too-long 

LOW:
Line too long (109/100)
Raw output 
Used when a line is longer than a given number of characters.
missing_info = (
f"No data returned for parameters: {', '.join(missing_params) or 'specified request'}. "

Check warning on line 447 in climada_petals/hazard/copernicus_forecast/handler.py

View check run for this annotation

Jenkins - WCR / Pylint

line-too-long 

LOW:
Line too long (120/100)
Raw output 
Used when a line is longer than a given number of characters.
"This may indicate unavailable or incorrect parameter selection. Please verify the existence and accuracy of the selected parameters on the CDS website."
"This may indicate unavailable or incorrect parameter selection. Please verify the existence "

Check warning on line 448 in climada_petals/hazard/copernicus_forecast/handler.py

View check run for this annotation

Jenkins - WCR / Pylint

line-too-long 

LOW:
Line too long (126/100)
Raw output 
Used when a line is longer than a given number of characters.
"and accuracy of the selected parameters on the Climate Data Store website. "

Check warning on line 449 in climada_petals/hazard/copernicus_forecast/handler.py

View check run for this annotation

Jenkins - WCR / Pylint

line-too-long 

LOW:
Line too long (109/100)
Raw output 
Used when a line is longer than a given number of characters.
"If you require more information about this error, please visit: "
"https://confluence.ecmwf.int/display/CKB/Common+Error+Messages+for+CDS+Requests"

Check warning on line 451 in climada_petals/hazard/copernicus_forecast/handler.py

View check run for this annotation

Jenkins - WCR / Pylint

line-too-long 

LOW:
Line too long (113/100)
Raw output 
Used when a line is longer than a given number of characters.
)

self.logger.error(missing_info)
raise ValueError(missing_info) from e
return # Exit function gracefully without traceback

else:
self.logger.error(f"Failed to download due to HTTPError: {e}")

Check warning on line 458 in climada_petals/hazard/copernicus_forecast/handler.py

View check run for this annotation

Jenkins - WCR / Pylint

logging-fstring-interpolation 

NORMAL:
Use lazy % formatting in logging functions
Raw output 
no description found
raise
return # Exit function gracefully without traceback
except Exception as e:

Check warning on line 460 in climada_petals/hazard/copernicus_forecast/handler.py

View check run for this annotation

Jenkins - WCR / Pylint

invalid-name 

LOW:
Variable name "e" doesn't conform to '(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$' pattern
Raw output 
Used when the name doesn't match the regular expression associated to its type(constant, variable, class...).
self.logger.error(f"Unexpected error during download: {e}")

Check warning on line 461 in climada_petals/hazard/copernicus_forecast/handler.py

View check run for this annotation

Jenkins - WCR / Pylint

logging-fstring-interpolation 

NORMAL:
Use lazy % formatting in logging functions
Raw output 
no description found
raise e
return # Exit function gracefully without traceback

def _process_data(

Check warning on line 464 in climada_petals/hazard/copernicus_forecast/handler.py

View check run for this annotation

Jenkins - WCR / Pylint

too-many-arguments 

LOW:
Too many arguments (9/7)
Raw output 
Used when a function or method takes too many arguments.

Check warning on line 464 in climada_petals/hazard/copernicus_forecast/handler.py

View check run for this annotation

Jenkins - WCR / Pylint

too-many-positional-arguments 

LOW:
Too many positional arguments (9/5)
Raw output 
no description found

Check warning on line 464 in climada_petals/hazard/copernicus_forecast/handler.py

View check run for this annotation

Jenkins - WCR / Pylint

too-many-locals 

LOW:
Too many local variables (25/15)
Raw output 
Used when a function or method has too many local variables.
self, data_out, year_list, month_list, bounds, overwrite, index_metric, format, originating_centre

Check warning on line 465 in climada_petals/hazard/copernicus_forecast/handler.py

View check run for this annotation

Jenkins - WCR / Pylint

line-too-long 

LOW:
Line too long (106/100)
Raw output 
Used when a line is longer than a given number of characters.

Check warning on line 465 in climada_petals/hazard/copernicus_forecast/handler.py

View check run for this annotation

Jenkins - WCR / Pylint

redefined-builtin 

NORMAL:
Redefining built-in 'format'
Raw output 
Used when a variable or function override a built-in.
Expand Down
76 changes: 58 additions & 18 deletions doc/tutorial/climada_hazard_copernicus_forecast.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
},
{
"cell_type": "code",
"execution_count": 11,
"execution_count": 1,
"id": "170187ce",
"metadata": {},
"outputs": [],
Expand Down Expand Up @@ -113,29 +113,29 @@
},
{
"cell_type": "code",
"execution_count": 12,
"execution_count": 2,
"id": "a2acc712-baa8-45d0-ad98-85814ad13f32",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Explanation for 'Tmax': Maximum Temperature: This indicator tracks the highest temperature recorded over a specified period.\n",
"Explanation for 'TR': Tropical Nights: This indicator counts the number of nights where the minimum temperature remains above a certain threshold, typically 20°C.\n",
"Required Input Data: ['2m temperature (t2m)']\n"
]
}
],
"source": [
"# We define above parameters for an example \n",
"index_metric = \"Tmax\" # maximum temperature\n",
"index_metric = \"TR\" # maximum temperature\n",
"format = \"grib\" # 'grib' or 'netcdf'\n",
"originating_centre = \"dwd\"\n",
"system = \"21\" \n",
"originating_centre = \"CMCC\"\n",
"system = \"3\" \n",
"max_lead_month = 4\n",
"year_list = [2018] \n",
"month_list = [9]\n",
"area_selection = ['DEU', 'CHE'] \n",
"year_list = [2024] \n",
"month_list = [6,7]\n",
"area_selection = ['ITA'] \n",
"overwrite = False\n",
"\n",
"# Details on the characteristics of the selected indicator and the input data used\n",
Expand Down Expand Up @@ -165,17 +165,45 @@
},
{
"cell_type": "code",
"execution_count": 13,
"execution_count": 3,
"id": "69229b74-d49b-4696-ac56-2530950548fb",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"2024-10-28 15:04:17,608 | WARNING : Before downloading, please ensure you have reviewed and accepted the terms and conditions for the use of this dataset. Access the terms here: https://cds.climate.copernicus.eu/datasets/seasonal-original-single-levels?tab=download\n",
"2024-10-28 15:04:17,610 | INFO : File /Users/daraya/climada/data/copernicus_forecast/input_data/dwd/grib/2018/09/dwd_t2m_area4_56_45_16_201809.grib already exists.\n",
"2024-10-28 15:04:17,610 | INFO : Daily file /Users/daraya/climada/data/copernicus_forecast/input_data/netcdf/daily/dwd/2018/09/dwd_t2m_area4_56_45_16_201809.nc already exists.\n"
"2024-10-28 15:47:18,244 | WARNING : Before downloading, please ensure you have reviewed and accepted the terms and conditions for the use of this dataset. Access the terms here: https://cds.climate.copernicus.eu/datasets/seasonal-original-single-levels?tab=download\n",
"2024-10-28 15:47:18,245 | INFO : 489 leadtimes to download.\n",
"2024-10-28 15:47:18,458 INFO [2024-09-28T00:00:00] **Welcome to the New Climate Data Store (CDS)!** This new system is in its early days of full operations and still undergoing enhancements and fine tuning. Some disruptions are to be expected. Your \n",
"[feedback](https://jira.ecmwf.int/plugins/servlet/desk/portal/1/create/202) is key to improve the user experience on the new CDS for the benefit of everyone. Thank you.\n",
"2024-10-28 15:47:18,458 | INFO : [2024-09-28T00:00:00] **Welcome to the New Climate Data Store (CDS)!** This new system is in its early days of full operations and still undergoing enhancements and fine tuning. Some disruptions are to be expected. Your \n",
"[feedback](https://jira.ecmwf.int/plugins/servlet/desk/portal/1/create/202) is key to improve the user experience on the new CDS for the benefit of everyone. Thank you.\n",
"2024-10-28 15:47:18,459 WARNING [2024-09-26T00:00:00] Should you have not yet migrated from the old CDS system to the new CDS, please check our [informative page](https://confluence.ecmwf.int/x/uINmFw) for guidance.\n",
"2024-10-28 15:47:18,459 | WARNING : [2024-09-26T00:00:00] Should you have not yet migrated from the old CDS system to the new CDS, please check our [informative page](https://confluence.ecmwf.int/x/uINmFw) for guidance.\n",
"2024-10-28 15:47:18,460 INFO [2024-09-26T00:00:00] Watch our [Forum](https://forum.ecmwf.int/) for Announcements, news and other discussed topics.\n",
"2024-10-28 15:47:18,460 | INFO : [2024-09-26T00:00:00] Watch our [Forum](https://forum.ecmwf.int/) for Announcements, news and other discussed topics.\n",
"2024-10-28 15:47:18,461 INFO [2024-09-16T00:00:00] Remember that you need to have an ECMWF account to use the new CDS. **Your old CDS credentials will not work in new CDS!**\n",
"2024-10-28 15:47:18,461 | INFO : [2024-09-16T00:00:00] Remember that you need to have an ECMWF account to use the new CDS. **Your old CDS credentials will not work in new CDS!**\n",
"2024-10-28 15:47:18,462 WARNING [2024-06-16T00:00:00] CDS API syntax is changed and some keys or parameter names may have also changed. To avoid requests failing, please use the \"Show API request code\" tool on the dataset Download Form to check you are using the correct syntax for your API request.\n",
"2024-10-28 15:47:18,462 | WARNING : [2024-06-16T00:00:00] CDS API syntax is changed and some keys or parameter names may have also changed. To avoid requests failing, please use the \"Show API request code\" tool on the dataset Download Form to check you are using the correct syntax for your API request.\n",
"2024-10-28 15:47:19,318 INFO Request ID is 53013f55-d4ef-48ff-bd97-e1705ba4acf4\n",
"2024-10-28 15:47:19,318 | INFO : Request ID is 53013f55-d4ef-48ff-bd97-e1705ba4acf4\n",
"2024-10-28 15:47:19,447 INFO status has been updated to accepted\n",
"2024-10-28 15:47:19,447 | INFO : status has been updated to accepted\n",
"2024-10-28 15:47:21,164 INFO status has been updated to running\n",
"2024-10-28 15:47:21,164 | INFO : status has been updated to running\n",
"2024-10-28 15:47:23,457 INFO status has been updated to failed\n",
"2024-10-28 15:47:23,457 | INFO : status has been updated to failed\n",
"2024-10-28 15:47:25,087 | ERROR : Error downloading file /Users/daraya/climada/data/copernicus_forecast/input_data/CMCC/grib/2024/06/CMCC_t2m_area4_49_33_20_202406.grib: 400 Client Error: Bad Request for url: https://cds.climate.copernicus.eu/api/retrieve/v1/jobs/53013f55-d4ef-48ff-bd97-e1705ba4acf4/results\n",
"The job has failed.\n",
"MARS returned no data, please check your selection.Request submitted to the MARS server:\n",
"[{'area': [49.404409157600064, 4.219788779000066, 33.170049669400036, 20.900365510000075], 'step': [0, 6, 12, 18, 24, 30, 36, 42, 48, 54, 60, 66, 72, 78, 84, 90, 96, 102, 108, 114, 120, 126, 132, 138, 144, 150, 156, 162, 168, 174, 180, 186, 192, 198, 204, 210, 216, 222, 228, 234, 240, 246, 252, 258, 264, 270, 276, 282, 288, 294, 300, 306, 312, 318, 324, 330, 336, 342, 348, 354, 360, 366, 372, 378, 384, 390, 396, 402, 408, 414, 420, 426, 432, 438, 444, 450, 456, 462, 468, 474, 480, 486, 492, 498, 504, 510, 516, 522, 528, 534, 540, 546, 552, 558, 564, 570, 576, 582, 588, 594, 600, 606, 612, 618, 624, 630, 636, 642, 648, 654, 660, 666, 672, 678, 684, 690, 696, 702, 708, 714, 720, 726, 732, 738, 744, 750, 756, 762, 768, 774, 780, 786, 792, 798, 804, 810, 816, 822, 828, 834, 840, 846, 852, 858, 864, 870, 876, 882, 888, 894, 900, 906, 912, 918, 924, 930, 936, 942, 948, 954, 960, 966, 972, 978, 984, 990, 996, 1002, 1008, 1014, 1020, 1026, 1032, 1038, 1044, 1050, 1056, 1062, 1068, 1074, 1080, 1086, 1092, 1098, 1104, 1110, 1116, 1122, 1128, 1134, 1140, 1146, 1152, 1158, 1164, 1170, 1176, 1182, 1188, 1194, 1200, 1206, 1212, 1218, 1224, 1230, 1236, 1242, 1248, 1254, 1260, 1266, 1272, 1278, 1284, 1290, 1296, 1302, 1308, 1314, 1320, 1326, 1332, 1338, 1344, 1350, 1356, 1362, 1368, 1374, 1380, 1386, 1392, 1398, 1404, 1410, 1416, 1422, 1428, 1434, 1440, 1446, 1452, 1458, 1464, 1470, 1476, 1482, 1488, 1494, 1500, 1506, 1512, 1518, 1524, 1530, 1536, 1542, 1548, 1554, 1560, 1566, 1572, 1578, 1584, 1590, 1596, 1602, 1608, 1614, 1620, 1626, 1632, 1638, 1644, 1650, 1656, 1662, 1668, 1674, 1680, 1686, 1692, 1698, 1704, 1710, 1716, 1722, 1728, 1734, 1740, 1746, 1752, 1758, 1764, 1770, 1776, 1782, 1788, 1794, 1800, 1806, 1812, 1818, 1824, 1830, 1836, 1842, 1848, 1854, 1860, 1866, 1872, 1878, 1884, 1890, 1896, 1902, 1908, 1914, 1920, 1926, 1932, 1938, 1944, 1950, 1956, 1962, 1968, 1974, 1980, 1986, 1992, 1998, 2004, 2010, 2016, 2022, 2028, 2034, 2040, 2046, 2052, 2058, 2064, 2070, 2076, 2082, 2088, 2094, 2100, 2106, 2112, 2118, 2124, 2130, 2136, 2142, 2148, 2154, 2160, 2166, 2172, 2178, 2184, 2190, 2196, 2202, 2208, 2214, 2220, 2226, 2232, 2238, 2244, 2250, 2256, 2262, 2268, 2274, 2280, 2286, 2292, 2298, 2304, 2310, 2316, 2322, 2328, 2334, 2340, 2346, 2352, 2358, 2364, 2370, 2376, 2382, 2388, 2394, 2400, 2406, 2412, 2418, 2424, 2430, 2436, 2442, 2448, 2454, 2460, 2466, 2472, 2478, 2484, 2490, 2496, 2502, 2508, 2514, 2520, 2526, 2532, 2538, 2544, 2550, 2556, 2562, 2568, 2574, 2580, 2586, 2592, 2598, 2604, 2610, 2616, 2622, 2628, 2634, 2640, 2646, 2652, 2658, 2664, 2670, 2676, 2682, 2688, 2694, 2700, 2706, 2712, 2718, 2724, 2730, 2736, 2742, 2748, 2754, 2760, 2766, 2772, 2778, 2784, 2790, 2796, 2802, 2808, 2814, 2820, 2826, 2832, 2838, 2844, 2850, 2856, 2862, 2868, 2874, 2880, 2886, 2892, 2898, 2904, 2910, 2916, 2922, 2928], 'origin': 'CMCC', 'system': '3', 'param': ['167'], 'time': ['all'], 'type': ['fc'], 'class': ['c3'], 'expect': ['any'], 'expver': ['1'], 'method': ['1'], 'number': ['all'], 'stream': ['mmsf'], 'levtype': ['all'], 'database': 'ecmwf', 'levelist': ['all'], 'date': ['2024-06-01']}]\n",
"\n",
"The job failed with: MarsNoDataError\n",
"2024-10-28 15:47:25,089 | ERROR : No data returned for parameters: specified request. This may indicate unavailable or incorrect parameter selection. Please verify the existence and accuracy of the selected parameters on the Climate Data Store website. If you require more information about this error, please visit: https://confluence.ecmwf.int/display/CKB/Common+Error+Messages+for+CDS+Requests\n",
"2024-10-28 15:47:25,098 | ERROR : Input file None not found. Skipping processing for 2024-06.\n",
"2024-10-28 15:47:25,099 | ERROR : Input file None not found. Skipping processing for 2024-07.\n"
]
}
],
Expand Down Expand Up @@ -210,15 +238,27 @@
},
{
"cell_type": "code",
"execution_count": 14,
"execution_count": 4,
"id": "8d703f84-0749-4003-9fef-6f0ffaccb8d8",
"metadata": {},
"outputs": [
{
"name": "stderr",
"name": "stdout",
"output_type": "stream",
"text": [
"2024-10-28 15:04:20,510 | INFO : Index file /Users/daraya/climada/data/copernicus_forecast/indices/dwd/Tmax/2018/09/Tmax_dwd_area4_56_45_16_201809.nc already exists.\n"
"An error occurred: expected str, bytes or os.PathLike object, not NoneType\n"
]
},
{
"ename": "TypeError",
"evalue": "cannot unpack non-iterable NoneType object",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn[4], line 2\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[39m# Calculate index\u001b[39;00m\n\u001b[0;32m----> 2\u001b[0m handler\u001b[39m.\u001b[39;49mcalculate_index(index_metric, year_list, month_list, area_selection, overwrite, originating_centre)\n",
"File \u001b[0;32m~/Documents/CLIMADA/climada_petals/climada_petals/hazard/copernicus_forecast/handler.py:747\u001b[0m, in \u001b[0;36mForecastHandler.calculate_index\u001b[0;34m(self, index_metric, year_list, month_list, area_selection, overwrite, originating_centre, data_out)\u001b[0m\n\u001b[1;32m 741\u001b[0m ds_daily, ds_monthly, ds_stats \u001b[39m=\u001b[39m (\n\u001b[1;32m 742\u001b[0m indicator\u001b[39m.\u001b[39mcalculate_heat_indices_metrics(\n\u001b[1;32m 743\u001b[0m input_file_name, index_metric\n\u001b[1;32m 744\u001b[0m )\n\u001b[1;32m 745\u001b[0m )\n\u001b[1;32m 746\u001b[0m \u001b[39melif\u001b[39;00m index_metric \u001b[39m==\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mTR\u001b[39m\u001b[39m\"\u001b[39m:\n\u001b[0;32m--> 747\u001b[0m ds_daily, ds_monthly, ds_stats \u001b[39m=\u001b[39m indicator\u001b[39m.\u001b[39mcalculate_TR(\n\u001b[1;32m 748\u001b[0m grib_file_name, index_metric\n\u001b[1;32m 749\u001b[0m )\n\u001b[1;32m 750\u001b[0m \u001b[39melif\u001b[39;00m index_metric \u001b[39m==\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mTX30\u001b[39m\u001b[39m\"\u001b[39m:\n\u001b[1;32m 751\u001b[0m ds_daily, ds_monthly, ds_stats \u001b[39m=\u001b[39m indicator\u001b[39m.\u001b[39mcalculate_tx30(\n\u001b[1;32m 752\u001b[0m grib_file_name, index_metric\n\u001b[1;32m 753\u001b[0m )\n",
"\u001b[0;31mTypeError\u001b[0m: cannot unpack non-iterable NoneType object"
]
}
],
Expand Down Expand Up @@ -263,7 +303,7 @@
},
{
"cell_type": "code",
"execution_count": 15,
"execution_count": null,
"id": "8d0d6332-99f8-49f1-b925-26758005c152",
"metadata": {},
"outputs": [
Expand Down Expand Up @@ -309,7 +349,7 @@
},
{
"cell_type": "code",
"execution_count": 16,
"execution_count": null,
"id": "8275a52a",
"metadata": {},
"outputs": [
Expand Down

0 comments on commit 21c6c0a

Please sign in to comment.