From c1f35449f5ec1023f33aa353f0fa3a756d6c9c3b Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Thu, 21 Mar 2024 14:58:59 -0400 Subject: [PATCH] add icap_mme This feature adds the ICAP_MME results to MONETIO. ICAP_MME is a 9 model ensemble from multiple operational data sources for aerosols. It is available through https://usgodae.org/metools/ensemble/ --- monetio/__init__.py | 1 + monetio/models/__init__.py | 2 + monetio/models/icap_mme.py | 216 +++++++++++++++++++++++++++++++++++++ 3 files changed, 219 insertions(+) create mode 100644 monetio/models/icap_mme.py diff --git a/monetio/__init__.py b/monetio/__init__.py index 9c60c152..e8fa7fd2 100644 --- a/monetio/__init__.py +++ b/monetio/__init__.py @@ -49,6 +49,7 @@ "pardump", "prepchem", "raqms", + "icap_mme", ] diff --git a/monetio/models/__init__.py b/monetio/models/__init__.py index 0d0f64b6..7649a14a 100644 --- a/monetio/models/__init__.py +++ b/monetio/models/__init__.py @@ -9,6 +9,7 @@ fv3chem, hysplit, hytraj, + icap_mme, ncep_grib, pardump, prepchem, @@ -30,6 +31,7 @@ "prepchem", "pardump", "raqms", + "icap_mme", ] __name__ = "models" diff --git a/monetio/models/icap_mme.py b/monetio/models/icap_mme.py new file mode 100644 index 00000000..15a7d81f --- /dev/null +++ b/monetio/models/icap_mme.py @@ -0,0 +1,216 @@ +import pandas as pd + +server = "ftp.star.nesdis.noaa.gov" +base_dir = "/pub/smcd/VIIRS_Aerosol/npp.viirs.aerosol.data/epsaot550/" + + +def build_urls(dates, *, filetype="mmc", data_var="dustaod550"): + """Construct URLs for downloading NEPS data. + + Parameters + ---------- + dates : pd.DatetimeIndex or iterable of datetime + Dates to download data for. + filetype : str, optional + mmc or c4 + res : float, optional + Resolution of data in km, only used for sub-daily data. + sat : str, optional + variable + + Returns + ------- + pd.Series + Series with URLs and corresponding file names. + + Notes + ----- + The `res` and `sat` parameters are only used for sub-daily data. + """ + + from collections.abc import Iterable + + if isinstance(dates, Iterable): + dates = pd.DatetimeIndex(dates) + else: + dates = pd.DatetimeIndex([dates]) + if filetype == "mmc": + ft = "MMC" + elif filetype == "c4": + ft = "C4" + else: + ft = "MME" + + urls = [] + fnames = [] + print("Building VIIRS URLs...") # + base_url = "https://usgodae.org/ftp/outgoing/nrl/ICAP-MME/" + + for dt in dates: + fname = "icap_{}_{}_{}.nc".format(dt.strftime("%Y%m%d%H"), ft, data_var.lower()) + url = base_url + dt.strftime(r"%Y/%Y%m/") + fname + urls.append(url) + fnames.append(fname) + + # Note: files needed for comparison + urls = pd.Series(urls, index=None) + fnames = pd.Series(fnames, index=None) + return urls, fnames + + +def check_remote_file_exists(file_url): + import requests + + r = requests.head(file_url, stream=True, verify=False) + + if r.status_code == 200: + return True + else: + print(f"HTTP Error {r.status_code} - {r.reason}") + return False + + +def retrieve(url, fname): + """Download files from the airnowtech S3 server. + + Parameters + ---------- + url : string + Description of parameter `url`. + fname : string + Description of parameter `fname`. + + Returns + ------- + None + + """ + import os + + import requests + + if not os.path.isfile(fname): + print("\n Retrieving: " + fname) + print(url) + print("\n") + r = requests.get(url) + r.raise_for_status() + with open(fname, "wb") as f: + f.write(r.content) + else: + print("\n File Exists: " + fname) + + +def open_dataset(date, product="mmc", data_var="modeaod550"): + """ + Parameters + ---------- + datestr : str or datetime-like + The date for which to open the dataset. + 2022-10-29 to current is available. + """ + import pandas as pd + import xarray as xr + + if not isinstance(date, pd.Timestamp): + d = pd.to_datetime(date) + else: + d = date + + try: + if product.lower() not in ("mmc", "c4", "mme"): + raise ValueError + except ValueError: + print("Invalid input for 'product': Valid values are 'MMC' 'C4' 'MME'") + + try: + if data_var.lower() not in ( + "modeaod550", + "dustaod550", + "pm", + "seasaltaod550", + "smokeaod550", + "totaldustaod550", + ): + raise ValueError + except ValueError: + print( + "Invalid input for 'data_var': Valid values are 'modeaod550' 'dustaod550' 'pm' 'seasaltaod550' 'smokeaod550' 'totaldustaod550'" + ) + + urls, fnames = build_urls(d, filetype=product, data_var=data_var) + url = urls.values[0] + fname = fnames.values[0] + print(url) + print(fname) + try: + if check_remote_file_exists(url) is False: + raise ValueError + except ValueError: + print("File does not exist on ICAP HTTPS server.", url) + return ValueError + retrieve(url, fname) + + dset = xr.open_dataset(fname) + + return dset + + +def open_mfdataset(dates, product="mmc", data_var="modeaod550"): + """ + Parameters + ---------- + datestr : str or datetime-like + The date for which to open the dataset. + 2022-10-29 to current is available. + """ + import pandas as pd + import xarray as xr + + try: + if isinstance(dates, pd.DatetimeIndex): + d = dates + else: + raise TypeError + except TypeError: + print("Please provide a pandas.DatetimeIndex") + return + + try: + if product.lower() not in ("mmc", "c4", "mme"): + raise ValueError + except ValueError: + print("Invalid input for 'product': Valid values are 'MMC' 'C4' 'MME'") + + try: + if data_var.lower() not in ( + "modeaod550", + "dustaod550", + "pm", + "seasaltaod550", + "smokeaod550", + "totaldustaod550", + ): + raise ValueError + except ValueError: + print( + "Invalid input for 'data_var': Valid values are 'modeaod550' 'dustaod550' 'pm' 'seasaltaod550' 'smokeaod550' 'totaldustaod550'" + ) + + urls, fnames = build_urls(d, filetype=product, data_var=data_var) + url = urls.values[0] + fname = fnames.values[0] + + for url, fname in zip(urls, fnames): + try: + if check_remote_file_exists(url) is False: + raise ValueError + except ValueError: + print("File does not exist on ICAP HTTPS server.", url) + return + retrieve(url, fname) + + dset = xr.open_mfdataset(fnames, combine="nested", concat_dim="time") + # dset["time"] = d + + return dset