Merge pull request #746 from metno/ipc-forests

add Ipc-forests

pyaerocom/aeroval/coldatatojson_helpers.py

@@ -5,6 +5,7 @@
 import os
 from copy import deepcopy
 from datetime import datetime
+from typing import Literal
 
 import numpy as np
 import pandas as pd
@@ -381,12 +382,16 @@ def _create_diurnal_weekly_data_object(coldata, resolution):
     return output_array
 
 
-def _get_period_keys(resolution):
-    if resolution == "seasonal":
-        period_keys = ["DJF", "MAM", "JJA", "SON"]
-    elif resolution == "yearly":
-        period_keys = ["Annual"]
-    return period_keys
+def _get_period_keys(resolution: Literal["seasonal", "yearly"]):
+    period_keys = dict(
+        seasonal=["DJF", "MAM", "JJA", "SON"],
+        yearly=["All"],
+    )
+
+    if resolution not in period_keys:
+        raise ValueError(f"Unknown {resolution=}")
+
+    return period_keys[resolution]
 
 
 def _process_one_station_weekly(stat_name, i, repw_res, meta_glob, time):
@@ -426,8 +431,8 @@ def _process_one_station_weekly(stat_name, i, repw_res, meta_glob, time):
 
     ts_data = {
         "time": time,
-        "seasonal": {"obs": yeardict, "mod": yeardict},
-        "yearly": {"obs": yeardict, "mod": yeardict},
+        "seasonal": {"obs": deepcopy(yeardict), "mod": deepcopy(yeardict)},
+        "yearly": {"obs": deepcopy(yeardict), "mod": deepcopy(yeardict)},
     }
     ts_data["station_name"] = stat_name
     ts_data.update(meta_glob)
@@ -526,8 +531,8 @@ def _process_weekly_object_to_country_time_series(repw_res, meta_glob, regions_h
         for regid, regname in region_ids.items():
             ts_data = {
                 "time": time,
-                "seasonal": {"obs": yeardict, "mod": yeardict},
-                "yearly": {"obs": yeardict, "mod": yeardict},
+                "seasonal": {"obs": deepcopy(yeardict), "mod": deepcopy(yeardict)},
+                "yearly": {"obs": deepcopy(yeardict), "mod": deepcopy(yeardict)},
             }
             ts_data["station_name"] = regname
             ts_data.update(meta_glob)

pyaerocom/aeroval/experiment_output.py

@@ -16,6 +16,7 @@
 from pyaerocom.aeroval.glob_defaults import (
     extended_statistics,
     statistics_defaults,
+    statistics_model_only,
     statistics_obs_only,
     statistics_trend,
     var_ranges_defaults,
@@ -574,6 +575,8 @@ def _create_var_ranges_json(self):
     def _create_statistics_json(self):
         if self.cfg.statistics_opts.obs_only_stats:
             stats_info = statistics_obs_only
+        elif self.cfg.statistics_opts.model_only_stats:
+            stats_info = statistics_model_only
         else:
             stats_info = statistics_defaults
             stats_info.update(extended_statistics)

pyaerocom/aeroval/glob_defaults.py

@@ -200,10 +200,17 @@
         "scale": [0, 0.5, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5.0],
         "colmap": "coolwarm",
     },
-    "wetoxs": {"scale": [0, 1.25, 2.5, 3.75, 5, 6.25, 7.5, 8.75, 10], "colmap": "coolwarm"},
-    "wetoxn": {"scale": [0, 0.25, 0.5, 0.75, 1, 1.25, 1.5, 1.75, 2], "colmap": "coolwarm"},
+    "wetoxs": {"scale": [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.7, 1, 1.5], "colmap": "coolwarm"},
+    "wetna": {"scale": [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.7, 1, 1.5], "colmap": "coolwarm"},
+    "wetoxn": {"scale": [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.7, 1, 1.5], "colmap": "coolwarm"},
     "wetrdn": {
-        "scale": [0, 0.75, 1.5, 2.25, 3.0, 3.75, 4.5, 5.25, 6.0, 6.75, 7.5, 8.25],
+        "scale": [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.7, 1, 1.5],
+        "colmap": "coolwarm",
+    },
+    "wetoxsf": {"scale": [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.7, 1, 1.5], "colmap": "coolwarm"},
+    "wetoxnf": {"scale": [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.7, 1, 1.5], "colmap": "coolwarm"},
+    "wetrdnf": {
+        "scale": [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.7, 1, 1.5],
         "colmap": "coolwarm",
     },
     "prmm": {"scale": [0, 1.25, 2.5, 3.75, 5, 6.25, 7.5, 8.75, 10], "colmap": "coolwarm"},
@@ -215,6 +222,7 @@
         "colmap": "coolwarm",
     },
     "drydust": {
+        # "scale": [0.0, 10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0, 90.0, 100.0],
         "scale": [0.0, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0, 20.0, 50.0, 100.0, 200.0, 500.0, 1000.0],
         "colmap": "coolwarm",
     },
@@ -255,6 +263,58 @@
         "colmap": "coolwarm",
     },
     "ts": {"scale": [265, 270, 275, 280, 285, 290, 300, 305, 310, 315, 320], "colmap": "coolwarm"},
+    "proxydryo3": {"scale": [0, 0.5, 1, 15, 20, 25, 0.30, 40, 50], "colmap": "coolwarm"},
+    "proxydrypm10": {"scale": [0, 0.25, 0.5, 0.75, 1, 1.25, 1.5, 1.75, 2], "colmap": "coolwarm"},
+    "proxydrypm25": {"scale": [0, 0.25, 0.5, 0.75, 1, 1.25, 1.5, 1.75, 2], "colmap": "coolwarm"},
+    "proxydryno2": {
+        "scale": [0, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4],
+        "colmap": "coolwarm",
+    },
+    "proxydryhono": {"scale": [0.001, 0.002, 0.003, 0.004, 0.005, 0.006], "colmap": "coolwarm"},
+    "proxydryn2o5": {"scale": [0.01, 0.02, 0.03, 0.04, 0.05], "colmap": "coolwarm"},
+    "proxydryhno3": {
+        "scale": [0, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4],
+        "colmap": "coolwarm",
+    },
+    "proxydryno3c": {
+        "scale": [0.01, 0.02, 0.03, 0.04, 0.05],
+        "colmap": "coolwarm",
+    },
+    "proxydryno3f": {"scale": [0.01, 0.02, 0.03, 0.04, 0.05], "colmap": "coolwarm"},
+    "proxydrynh3": {
+        "scale": [0, 0.10, 0.20, 0.30, 0.40, 0.50, 0.60, 0.70, 0.80],
+        "colmap": "coolwarm",
+    },
+    "proxydrynh4": {
+        "scale": [0, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4],
+        "colmap": "coolwarm",
+    },
+    "proxydryso2": {
+        "scale": [0, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4],
+        "colmap": "coolwarm",
+    },
+    "proxydryso4": {
+        "scale": [0.01, 0.02, 0.03, 0.04, 0.05],
+        "colmap": "coolwarm",
+    },
+    "proxydryoxs": {
+        "scale": [0, 0.05, 0.1, 0.15, 0.20, 0.25, 0.30, 0.35, 0.40],
+        "colmap": "coolwarm",
+    },
+    "proxydryoxn": {
+        "scale": [0, 0.10, 0.20, 0.30, 0.40, 0.50, 0.60, 0.70, 0.80],
+        "colmap": "coolwarm",
+    },
+    "proxydryrdn": {
+        "scale": [0, 0.10, 0.20, 0.30, 0.40, 0.50, 0.60, 0.70, 0.80],
+        "colmap": "coolwarm",
+    },
+    "depoxs": {"scale": [0, 0.25, 0.5, 0.75, 1, 1.25, 1.5, 1.75, 2], "colmap": "coolwarm"},
+    "depoxn": {"scale": [0, 0.25, 0.5, 0.75, 1, 1.25, 1.5, 1.75, 2], "colmap": "coolwarm"},
+    "deprdn": {"scale": [0, 0.25, 0.5, 0.75, 1, 1.25, 1.5, 1.75, 2], "colmap": "coolwarm"},
+    "depoxsf": {"scale": [0, 0.25, 0.5, 0.75, 1, 1.25, 1.5, 1.75, 2], "colmap": "coolwarm"},
+    "depoxnf": {"scale": [0, 0.25, 0.5, 0.75, 1, 1.25, 1.5, 1.75, 2], "colmap": "coolwarm"},
+    "deprdnf": {"scale": [0, 0.25, 0.5, 0.75, 1, 1.25, 1.5, 1.75, 2], "colmap": "coolwarm"},
 }
 
 #: Default information for statistical parameters
@@ -451,6 +511,18 @@
     },
 }
 
+# For experiments where only model data is interesting, as with proxy drydep
+statistics_model_only = {
+    "data_mean": {
+        "name": "Mean-Mod",
+        "longname": "Model Mean",
+        "scale": None,
+        "colmap": "coolwarm",
+        "unit": "1",
+        "decimals": 2,
+    },
+}
+
 #: Mapping of pyaerocom variable names to web naming conventions
 ## Note: A 2D variable is defined under Column on the website, 3D is defined under Surface
 var_web_info = dict(
@@ -487,6 +559,7 @@
     concNno=["NO", "3D", "Concentration"],
     concno2=["NO2", "3D", "Gas concentrations"],
     concNno2=["NO2", "3D", "Gas concentrations"],
+    vmrno=["NO", "3D", "Volume mixing ratios"],
     vmrno2=["NO2", "3D", "Volume mixing ratios"],
     concno3=["NO3", "3D", "Gas concentrations"],
     conctno3=["tNO3", "3D", "Concentration"],
@@ -509,6 +582,11 @@
     concco=["CO", "3D", "Particle concentration"],
     vmrco=["CO", "3D", "Volume mixing ratios"],
     vmrco2=["CO2", "3D", "Volume mixing ratios"],
+    vmrc2h2=["Ethyne", "3D", "Volume mixing ratios"],
+    vmrc2h4=["Ethylene", "3D", "Volume mixing ratios"],
+    vmrc2h6=["Ethane", "3D", "Volume mixing ratios"],
+    vmrhcho=["Formaldehyde", "3D", "Volume mixing ratios"],
+    vmrisop=["Isoprene", "3D", "Volume mixing ratios"],
     vmrch4=["CH4", "3D", "Volume mixing ratios"],
     # PMs
     concpm10=["PM10", "3D", "Particle concentrations"],
@@ -543,9 +621,47 @@
     drydust=["DryDustDep", "3D", "Deposition"],
     wetdust=["WetDustDep", "3D", "Deposition"],
     wetoxs=["WetOXS", "3D", "Deposition"],
+    wetoxsc=["WetOXScorr", "3D", "Deposition"],
+    wetoxst=["WetOXStot", "3D", "Deposition"],
     wetoxn=["WetOXN", "3D", "Deposition"],
     wetrdn=["WetRDN", "3D", "Deposition"],
     prmm=["Precipitation", "3D", "Deposition"],
     # Temperature
     ts=["Surface Temperature", "3D", "Temperature"],
+    # proxy drydep
+    proxydryoxs=["proxyDryOXS", "3D", "Deposition"],
+    proxydryso2=["proxyDrySO2", "3D", "Deposition"],
+    proxydryso4=["proxyDrySO4", "3D", "Deposition"],
+    proxydryoxn=["proxyDryOXN", "3D", "Deposition"],
+    proxydryno2=["proxyDryNO2", "3D", "Deposition"],
+    proxydryno2no2=["proxyDryNO2NO2", "3D", "Deposition"],
+    proxydryhono=["proxyDryHONO", "3D", "Deposition"],
+    proxydryn2o5=["proxyDryN2O5", "3D", "Deposition"],
+    proxydryhno3=["proxyDryHNO3", "3D", "Deposition"],
+    proxydryno3c=["proxyDryNO3Coarse", "3D", "Deposition"],
+    proxydryno3f=["proxyDryNO3Fine", "3D", "Deposition"],
+    proxydryrdn=["proxyDryRDN", "3D", "Deposition"],
+    proxydrynh3=["proxyDryNH3", "3D", "Deposition"],
+    proxydrynh4=["proxyDryNH4", "3D", "Deposition"],
+    proxydryo3=["proxyDryO3", "3D", "Deposition"],
+    proxydrypm10=["proxyDryPM10", "3D", "Deposition"],
+    proxydrypm25=["proxyDryPM2.5", "3D", "Deposition"],
+    # proxy wetdep
+    proxywetoxs=["proxyWetOXS", "3D", "Deposition"],
+    proxywetso2=["proxyWetSO2", "3D", "Deposition"],
+    proxywetso4=["proxyWetSO4", "3D", "Deposition"],
+    proxywetoxn=["proxyWetOXN", "3D", "Deposition"],
+    proxywetno2=["proxyWetNO2", "3D", "Deposition"],
+    proxywetno2no2=["proxyWetNO2NO2", "3D", "Deposition"],
+    proxywethono=["proxyWetHONO", "3D", "Deposition"],
+    proxywetn2o5=["proxyWetN2O5", "3D", "Deposition"],
+    proxywethno3=["proxyWetHNO3", "3D", "Deposition"],
+    proxywetno3c=["proxyWetNO3Coarse", "3D", "Deposition"],
+    proxywetno3f=["proxyWetNO3Fine", "3D", "Deposition"],
+    proxywetrdn=["proxyWetRDN", "3D", "Deposition"],
+    proxywetnh3=["proxyWetNH3", "3D", "Deposition"],
+    proxywetnh4=["proxyWetNH4", "3D", "Deposition"],
+    proxyweto3=["proxyWetO3", "3D", "Deposition"],
+    proxywetpm10=["proxyWetPM10", "3D", "Deposition"],
+    proxywetpm25=["proxyWetPM2.5", "3D", "Deposition"],
 )

pyaerocom/aeroval/obsentry.py

@@ -51,7 +51,7 @@ class ObsEntry(BrowseDict):
         (c.g. :class:`pyaerocom.io.ReadUngridded`).
     """
 
-    SUPPORTED_VERT_CODES = ["Column", "Profile", "Surface"]
+    SUPPORTED_VERT_CODES = ["Column", "Profile", "Surface"]  # , "2D"]
     ALT_NAMES_VERT_CODES = dict(ModelLevel="Profile")
 
     SUPPORTED_VERT_LOCS = DataSource.SUPPORTED_VERT_LOCS

pyaerocom/aeroval/setupclasses.py

@@ -155,6 +155,7 @@ def __init__(self, **kwargs):
         self.use_fairmode = False
         self.use_diurnal = False
         self.obs_only_stats = False
+        self.model_only_stats = False
         self.update(**kwargs)
 
 

pyaerocom/aux_var_helpers.py

@@ -563,6 +563,52 @@ def compute_wetoxs_from_concprcpoxs(data):
     return _compute_wdep_from_concprcp_helper(data, "wetoxs", "concprcpoxs", "pr")
 
 
+def compute_wetoxs_from_concprcpoxst(data):
+    """Compute wdep from conc in precip and precip data
+
+    Note
+    ----
+    In addition to the returned numpy array, the input instance of
+    :class:`StationData` is modified by additional metadata and flags for
+    the new variable. See also :func:`_compute_wdep_from_concprcp_helper`.
+
+    Parameters
+    ----------
+    StationData
+        data object containing concprcp and precip data
+
+    Returns
+    -------
+    numpy.ndarray
+        array with wet deposition values
+
+    """
+    return _compute_wdep_from_concprcp_helper(data, "wetoxs", "concprcpoxst", "pr")
+
+
+def compute_wetoxs_from_concprcpoxsc(data):
+    """Compute wdep from conc in precip and precip data
+
+    Note
+    ----
+    In addition to the returned numpy array, the input instance of
+    :class:`StationData` is modified by additional metadata and flags for
+    the new variable. See also :func:`_compute_wdep_from_concprcp_helper`.
+
+    Parameters
+    ----------
+    StationData
+        data object containing concprcp and precip data
+
+    Returns
+    -------
+    numpy.ndarray
+        array with wet deposition values
+
+    """
+    return _compute_wdep_from_concprcp_helper(data, "wetoxs", "concprcpoxsc", "pr")
+
+
 def compute_wetoxn_from_concprcpoxn(data):
     """Compute wdep from conc in precip and precip data
 
@@ -609,6 +655,22 @@ def compute_wetrdn_from_concprcprdn(data):
     return _compute_wdep_from_concprcp_helper(data, "wetrdn", "concprcprdn", "pr")
 
 
+def compute_wetnh4_from_concprcpnh4(data):
+    return _compute_wdep_from_concprcp_helper(data, "wetnh4", "concprcpnh4", "pr")
+
+
+def compute_wetno3_from_concprcpno3(data):
+    return _compute_wdep_from_concprcp_helper(data, "wetno3", "concprcpno3", "pr")
+
+
+def compute_wetso4_from_concprcpso4(data):
+    return _compute_wdep_from_concprcp_helper(data, "wetso4", "concprcpso4", "pr")
+
+
+def compute_wetna_from_concprcpna(data):
+    return _compute_wdep_from_concprcp_helper(data, "wetna", "concprcpna", "pr")
+
+
 def vmrx_to_concx(data, p_pascal, T_kelvin, vmr_unit, mmol_var, mmol_air=None, to_unit=None):
     """
     Convert volume mixing ratio (vmr) to mass concentration
@@ -725,3 +787,51 @@ def calc_vmro3max(data):
     # print(data.var_info)
     # exit()
     return o3max
+
+
+def identity(data):
+    return data
+
+
+def make_proxy_drydep_from_O3(data):
+    # sort of prototype to add a compted variable
+    # one has to extend the data structures of the station data object
+    # 'right', but has to return just the data array
+    # That concept is a bit confusing (why not do everything in data here?)
+    var_name = "vmro3"
+    new_var_name = "proxydryo3"
+
+    flags = data.data_flagged[var_name]
+    new_var_data = data[var_name]
+    units = data.var_info[var_name]["units"]
+    # data.var_info[new_var_name]["units"] = units
+
+    if not new_var_name in data.var_info:
+        data.var_info[new_var_name] = {}
+    data.var_info[new_var_name] = data.var_info[var_name]
+    data.var_info[new_var_name]["units"] = "mg m-2 d-1"
+
+    data.data_flagged[new_var_name] = flags
+    return new_var_data
+
+
+def make_proxy_wetdep_from_O3(data):
+    # sort of prototype to add a compted variable
+    # one has to extend the data structures of the station data object
+    # 'right', but has to return just the data array
+    # That concept is a bit confusing (why not do everything in data here?)
+    var_name = "vmro3"
+    new_var_name = "proxyweto3"
+
+    flags = data.data_flagged[var_name]
+    new_var_data = data[var_name]
+    units = data.var_info[var_name]["units"]
+    # data.var_info[new_var_name]["units"] = units
+
+    if not new_var_name in data.var_info:
+        data.var_info[new_var_name] = {}
+    data.var_info[new_var_name] = data.var_info[var_name]
+    data.var_info[new_var_name]["units"] = "mg m-2 d-1"
+
+    data.data_flagged[new_var_name] = flags
+    return new_var_data

pyaerocom/colocation_auto.py

@@ -295,7 +295,7 @@ class ColocationSetup(BrowseDict):
     #: file for ec550aer at the surface ('*ec550aer*Surface*.nc'), then, the
     #: colocation routine will look for '*ec550aer*ModelLevel*.nc' and if this
     #: exists, it will load it and extract the surface level.
-    OBS_VERT_TYPES_ALT = {"Surface": "ModelLevel"}
+    OBS_VERT_TYPES_ALT = {"Surface": "ModelLevel", "2D": "2D"}
 
     #: do not raise Exception if invalid item is attempted to be assigned
     #: (Overwritten from base class)