diff --git a/e3sm_to_cmip/__main__.py b/e3sm_to_cmip/__main__.py index 5ecf3504..9da65f83 100755 --- a/e3sm_to_cmip/__main__.py +++ b/e3sm_to_cmip/__main__.py @@ -21,8 +21,8 @@ import yaml from tqdm import tqdm -from e3sm_to_cmip import ROOT_HANDLERS_DIR, __version__, resources -from e3sm_to_cmip._logger import _setup_logger, _setup_root_logger +from e3sm_to_cmip import ROOT_HANDLERS_DIR, __version__, _logger, resources +from e3sm_to_cmip.cmor_handlers.handler import instantiate_handler_logger from e3sm_to_cmip.cmor_handlers.utils import ( MPAS_REALMS, REALMS, @@ -31,6 +31,7 @@ Realm, _get_mpas_handlers, derive_handlers, + instantiate_h_utils_logger, load_all_handlers, ) from e3sm_to_cmip.util import ( @@ -41,6 +42,7 @@ find_atm_files, find_mpas_files, get_handler_info_msg, + instantiate_util_logger, precheck, print_debug, print_message, @@ -51,11 +53,6 @@ warnings.filterwarnings("ignore") -# Setup the root logger and this module's logger. -log_filename = _setup_root_logger() -logger = _setup_logger(__name__, propagate=True) - - @dataclass class CLIArguments: """A data class storing the command line arguments for e3sm_to_cmip. @@ -98,9 +95,19 @@ class CLIArguments: class E3SMtoCMIP: def __init__(self, args: Optional[List[str]] = None): + # logger assignment is moved into __init__ AFTER the call to _parse_args + # to prevent the default logfile directory being created whenever a call + # to "--help" or "--version" is invoked. Doing so, however, makes the + # logger unavailable to the functions in this class unless made global. + global logger + # A dictionary of command line arguments. parsed_args = self._parse_args(args) + # Setup this module's logger AFTER args are parsed in __init__, so that + # default log file is NOT created for "--help" or "--version" calls. + logger = _logger._logger(name=__name__, to_logfile=True) + # NOTE: The order of these attributes align with class CLIArguments. # ====================================================================== # Run Mode settings. @@ -141,6 +148,9 @@ def __init__(self, args: Optional[List[str]] = None): if self.precheck_path is not None: self._run_precheck() + self.handlers = self._get_handlers() + + def print_config(self): logger.info("--------------------------------------") logger.info("| E3SM to CMIP Configuration") logger.info("--------------------------------------") @@ -150,28 +160,24 @@ def __init__(self, args: Optional[List[str]] = None): logger.info(f" * precheck_path='{self.precheck_path}'") logger.info(f" * freq='{self.freq}'") logger.info(f" * realm='{self.realm}'") - logger.info(f" * Writing log output file to: {log_filename}") - - self.handlers = self._get_handlers() def run(self): - # Setup logger information and print out e3sm_to_cmip CLI arguments. + # If info_mode, call and then exit. # ====================================================================== - if self.output_path is not None: - self.new_metadata_path = os.path.join( - self.output_path, "user_metadata.json" - ) + if self.info_mode: + self._run_info_mode() + sys.exit(0) # Setup directories using the CLI argument paths (e.g., output dir). # ====================================================================== - if not self.info_mode: - self._setup_dirs_with_paths() + self._setup_dirs_with_paths() - # Run e3sm_to_cmip with info mode. + # Set new metadata path if output path was provided. # ====================================================================== - if self.info_mode: - self._run_info_mode() - sys.exit(0) + if self.output_path is not None: + self.new_metadata_path = os.path.join( + self.output_path, "user_metadata.json" + ) # Run e3sm_to_cmip to CMORize serially or in parallel. # ====================================================================== @@ -229,6 +235,10 @@ def _get_handlers(self): elif self.realm in MPAS_REALMS: handlers = _get_mpas_handlers(self.var_list) + else: + logger.error(f"No such realm: {self.realm}") + sys.exit(0) + if len(handlers) == 0: logger.error( "No CMIP6 variable handlers were derived from the variables found " @@ -960,7 +970,15 @@ def _timeout_exit(self): def main(args: Optional[List[str]] = None): app = E3SMtoCMIP(args) - app.run() + + # These calls allow module loggers that create default logfiles to avoid being + # instantiated by arguments "--help" or "--version" upon import. + instantiate_util_logger() + instantiate_h_utils_logger() + instantiate_handler_logger() + + app.print_config() + return app.run() if __name__ == "__main__": diff --git a/e3sm_to_cmip/_logger.py b/e3sm_to_cmip/_logger.py index 2e14df7c..90b12c3b 100644 --- a/e3sm_to_cmip/_logger.py +++ b/e3sm_to_cmip/_logger.py @@ -1,68 +1,70 @@ +from __future__ import annotations + import logging import os -import time -from datetime import datetime - -from pytz import UTC +from datetime import datetime, timezone +DEFAULT_LOG_LEVEL = logging.INFO +DEFAULT_LOG_DIR = "e2c_logs" +DEFAULT_LOGPATH = f"{DEFAULT_LOG_DIR}/e2c_root_log-{datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S_%f')}.log" -def _setup_root_logger() -> str: # pragma: no cover - """Sets up the root logger. +# Logger message and date formats. +MSGFMT = "%(asctime)s_%(msecs)03d:%(levelname)s:%(name)s:%(funcName)s:%(message)s" +DATEFMT = "%Y%m%d_%H%M%S" - The logger module will write to a log file and stream the console - simultaneously. - The log files are saved in a `/logs` directory relative to where - `e3sm_to_cmip` is executed. +def _logger( + name: str | None = None, + log_filename: str = DEFAULT_LOGPATH, + log_level: int = DEFAULT_LOG_LEVEL, + to_console: bool = False, + to_logfile: bool = False, + propagate: bool = False, +): + """Return a root or named logger with variable configuration. - Returns - ------- - str - The name of the logfile. + Parameters + ---------- + name : str | None + The name displayed for the logger in messages. + If name == None or name == "__main__", the root logger is returned + log_filename : str + If logfile handling is requested, any logfile may be specified. + log_level : LogLevel + Either logging.DEBUG (10), logging.INFO (20), logging.WARNING (30), + logging.ERROR (40), logging.CRITICAL (50), by default logging.INFO. + to_console : boolean + If True, a logging.StreamHandler is supplied, by default False. + to_logfile : boolean + If True, a logging.FileHandler is supplied, by default False. + propagate : boolean + If True, messages logged are propagated to the root logger, by default + False. """ - os.makedirs("logs", exist_ok=True) - filename = f'logs/{UTC.localize(datetime.utcnow()).strftime("%Y%m%d_%H%M%S_%f")}' - log_format = "%(asctime)s_%(msecs)03d:%(levelname)s:%(funcName)s:%(message)s" - - # Setup the logging module. - logging.basicConfig( - filename=filename, - format=log_format, - datefmt="%Y%m%d_%H%M%S", - level=logging.DEBUG, - ) - logging.captureWarnings(True) - logging.Formatter.converter = time.gmtime - - # Configure and add a console stream handler. - console_handler = logging.StreamHandler() - console_handler.setLevel(logging.INFO) - log_formatter = logging.Formatter(log_format) - console_handler.setFormatter(log_formatter) - logging.getLogger().addHandler(console_handler) + if to_logfile: + dn = os.path.dirname(log_filename) + if len(dn) and not os.path.exists(dn): + os.makedirs(dn) - return filename + if name is None or name == "__main__": + # FIXME: F821 Undefined name `logger` + logger = logger.root # noqa: F821 + else: + logger = logging.getLogger(name) + logger.propagate = propagate + logger.setLevel(log_level) -def _setup_logger(name, propagate=True) -> logging.Logger: - """Sets up a logger object. - - This function is intended to be used at the top-level of a module. + logger.handlers = [] - Parameters - ---------- - name : str - Name of the file where this function is called. - propagate : bool, optional - Propogate this logger module's messages to the root logger or not, by - default True. + if to_console: + logStreamHandler = logging.StreamHandler() + logStreamHandler.setFormatter(logging.Formatter(MSGFMT, datefmt=DATEFMT)) + logger.addHandler(logStreamHandler) - Returns - ------- - logging.Logger - The logger. - """ - logger = logging.getLogger(name) - logger.propagate = propagate + if to_logfile: + logFileHandler = logging.FileHandler(log_filename) + logFileHandler.setFormatter(logging.Formatter(MSGFMT, datefmt=DATEFMT)) + logger.addHandler(logFileHandler) return logger diff --git a/e3sm_to_cmip/cmor_handlers/handler.py b/e3sm_to_cmip/cmor_handlers/handler.py index c7e4510f..99a19285 100644 --- a/e3sm_to_cmip/cmor_handlers/handler.py +++ b/e3sm_to_cmip/cmor_handlers/handler.py @@ -1,7 +1,6 @@ from __future__ import annotations import json -import logging import os from typing import Any, Dict, KeysView, List, Literal, Optional, Tuple, TypedDict @@ -11,11 +10,16 @@ import xcdat as xc import yaml -from e3sm_to_cmip._logger import _setup_logger +from e3sm_to_cmip import _logger from e3sm_to_cmip.cmor_handlers import FILL_VALUE, _formulas from e3sm_to_cmip.util import _get_table_for_non_monthly_freq -logger = _setup_logger(__name__) + +def instantiate_handler_logger(): + global logger + + logger = _logger._logger(name=__name__, to_logfile=True, propagate=True) + # The names for valid hybrid sigma levels. HYBRID_SIGMA_LEVEL_NAMES = [ @@ -289,7 +293,7 @@ def _all_vars_have_filepaths( """ for var, filepaths in vars_to_filespaths.items(): if len(filepaths) == 0: - logging.error(f"{var}: Unable to find input files for {var}") + logger.error(f"{var}: Unable to find input files for {var}") return False return True @@ -331,7 +335,7 @@ def _setup_cmor_module( cmor.dataset_json(metadata_path) cmor.load_table(self.table) - logging.info(f"{var_name}: CMOR setup complete") + logger.info(f"{var_name}: CMOR setup complete") def _get_var_time_dim(self, table_path: str) -> str | None: """Get the CMIP variable's time dimension, if it exists. diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/areacello.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/areacello.py index dcd6f802..d3833aec 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/areacello.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/areacello.py @@ -2,11 +2,9 @@ compute Grid-Cell Area for Ocean Variables areacello """ -import logging - import xarray -from e3sm_to_cmip import mpas, util +from e3sm_to_cmip import _logger, mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed @@ -17,6 +15,8 @@ VAR_UNITS = "m2" TABLE = "CMIP6_Ofx.json" +logger = _logger._logger(name=__name__, to_logfile=True, propagate=False) + def handle(infiles, tables, user_input_path, **kwargs): """ @@ -43,7 +43,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = "Starting {name}".format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles["MPAS_mesh"] mappingFileName = infiles["MPAS_map"] diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/fsitherm.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/fsitherm.py index 235365ed..8d950f26 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/fsitherm.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/fsitherm.py @@ -4,11 +4,9 @@ from __future__ import absolute_import, division, print_function -import logging - import xarray -from e3sm_to_cmip import mpas, util +from e3sm_to_cmip import _logger, mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed @@ -19,6 +17,8 @@ VAR_UNITS = "kg m-2 s-1" TABLE = "CMIP6_Omon.json" +logger = _logger._logger(name=__name__, to_logfile=True, propagate=False) + def handle(infiles, tables, user_input_path, **kwargs): """ @@ -44,7 +44,7 @@ def handle(infiles, tables, user_input_path, **kwargs): print_message(f"Simple CMOR output not supported for {VAR_NAME}", "error") return None - logging.info(f"Starting {VAR_NAME}") + logger.info(f"Starting {VAR_NAME}") mappingFileName = infiles["MPAS_map"] timeSeriesFiles = infiles["MPASO"] diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/hfds.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/hfds.py index 47595ada..5275ac2f 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/hfds.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/hfds.py @@ -4,11 +4,9 @@ from __future__ import absolute_import, division, print_function -import logging - import xarray -from e3sm_to_cmip import mpas, util +from e3sm_to_cmip import _logger, mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed @@ -19,6 +17,8 @@ VAR_UNITS = "W m-2" TABLE = "CMIP6_Omon.json" +logger = _logger._logger(name=__name__, to_logfile=True, propagate=False) + def handle(infiles, tables, user_input_path, **kwargs): """ @@ -47,7 +47,7 @@ def handle(infiles, tables, user_input_path, **kwargs): print_message(f"Simple CMOR output not supported for {VAR_NAME}", "error") return None - logging.info(f"Starting {VAR_NAME}") + logger.info(f"Starting {VAR_NAME}") mappingFileName = infiles["MPAS_map"] timeSeriesFiles = infiles["MPASO"] diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/hfsifrazil.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/hfsifrazil.py index 455c3a4b..7e13b0db 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/hfsifrazil.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/hfsifrazil.py @@ -4,11 +4,9 @@ from __future__ import absolute_import, division, print_function -import logging - import xarray -from e3sm_to_cmip import mpas, util +from e3sm_to_cmip import _logger, mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed @@ -19,6 +17,8 @@ VAR_UNITS = "W m-2" TABLE = "CMIP6_Omon.json" +logger = _logger._logger(name=__name__, to_logfile=True, propagate=False) + def handle(infiles, tables, user_input_path, **kwargs): """ @@ -45,7 +45,7 @@ def handle(infiles, tables, user_input_path, **kwargs): print_message(f"Simple CMOR output not supported for {VAR_NAME}", "error") return None - logging.info(f"Starting {VAR_NAME}") + logger.info(f"Starting {VAR_NAME}") timeSeriesFiles = infiles["MPASO"] mappingFileName = infiles["MPAS_map"] diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/masscello.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/masscello.py index b691a5bc..a92ca946 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/masscello.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/masscello.py @@ -4,12 +4,10 @@ from __future__ import absolute_import, division, print_function -import logging - import netCDF4 import xarray -from e3sm_to_cmip import mpas, util +from e3sm_to_cmip import _logger, mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed @@ -20,6 +18,8 @@ VAR_UNITS = "kg m-2" TABLE = "CMIP6_Omon.json" +logger = _logger._logger(name=__name__, to_logfile=True, propagate=False) + def handle(infiles, tables, user_input_path, **kwargs): """ @@ -47,7 +47,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = "Starting {name}".format(name=__name__) - logging.info(msg) + logger.info(msg) namelistFileName = infiles["MPASO_namelist"] meshFileName = infiles["MPAS_mesh"] diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/masso.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/masso.py index 98d4e5b3..57d710ff 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/masso.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/masso.py @@ -4,11 +4,9 @@ from __future__ import absolute_import, division, print_function -import logging - import xarray -from e3sm_to_cmip import mpas, util +from e3sm_to_cmip import _logger, mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed @@ -19,6 +17,8 @@ VAR_UNITS = "kg" TABLE = "CMIP6_Omon.json" +logger = _logger._logger(name=__name__, to_logfile=True, propagate=False) + def handle(infiles, tables, user_input_path, **kwargs): """ @@ -46,7 +46,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = "Starting {name}".format(name=__name__) - logging.info(msg) + logger.info(msg) namelistFileName = infiles["MPASO_namelist"] meshFileName = infiles["MPAS_mesh"] diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/mlotst.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/mlotst.py index e128c29d..48f37ccc 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/mlotst.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/mlotst.py @@ -4,11 +4,9 @@ from __future__ import absolute_import, division, print_function -import logging - import xarray -from e3sm_to_cmip import mpas, util +from e3sm_to_cmip import _logger, mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed @@ -19,6 +17,8 @@ VAR_UNITS = "m" TABLE = "CMIP6_Omon.json" +logger = _logger._logger(name=__name__, to_logfile=True, propagate=False) + def handle(infiles, tables, user_input_path, **kwargs): """ @@ -46,7 +46,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = "Starting {name}".format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles["MPAS_mesh"] mappingFileName = infiles["MPAS_map"] diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/msftmz.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/msftmz.py index bf16ab73..2ffc18ae 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/msftmz.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/msftmz.py @@ -4,11 +4,9 @@ from __future__ import absolute_import, division, print_function -import logging - import xarray -from e3sm_to_cmip import mpas, util +from e3sm_to_cmip import _logger, mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed @@ -19,6 +17,8 @@ VAR_UNITS = "kg s-1" TABLE = "CMIP6_Omon.json" +logger = _logger._logger(name=__name__, to_logfile=True, propagate=False) + def handle(infiles, tables, user_input_path, **kwargs): """ @@ -49,7 +49,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = "Starting {name}".format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles["MPAS_mesh"] timeSeriesFiles = infiles["MPASO"] diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/pbo.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/pbo.py index 1e458004..6b64890f 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/pbo.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/pbo.py @@ -4,11 +4,9 @@ from __future__ import absolute_import, division, print_function -import logging - import xarray -from e3sm_to_cmip import mpas, util +from e3sm_to_cmip import _logger, mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed @@ -19,6 +17,8 @@ VAR_UNITS = "Pa" TABLE = "CMIP6_Omon.json" +logger = _logger._logger(name=__name__, to_logfile=True, propagate=False) + def handle(infiles, tables, user_input_path, **kwargs): """ @@ -48,7 +48,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = "Starting {name}".format(name=__name__) - logging.info(msg) + logger.info(msg) namelistFileName = infiles["MPASO_namelist"] meshFileName = infiles["MPAS_mesh"] diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/pso.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/pso.py index 52b005ac..eaac827c 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/pso.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/pso.py @@ -4,11 +4,9 @@ from __future__ import absolute_import, division, print_function -import logging - import xarray -from e3sm_to_cmip import mpas, util +from e3sm_to_cmip import _logger, mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed @@ -19,6 +17,8 @@ VAR_UNITS = "Pa" TABLE = "CMIP6_Omon.json" +logger = _logger._logger(name=__name__, to_logfile=True, propagate=False) + def handle(infiles, tables, user_input_path, **kwargs): """ @@ -47,7 +47,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = "Starting {name}".format(name=__name__) - logging.info(msg) + logger.info(msg) namelistFileName = infiles["MPASO_namelist"] meshFileName = infiles["MPAS_mesh"] diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/sfdsi.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/sfdsi.py index ca69fdb3..8b162e75 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/sfdsi.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/sfdsi.py @@ -4,11 +4,9 @@ from __future__ import absolute_import, division, print_function -import logging - import xarray -from e3sm_to_cmip import mpas, util +from e3sm_to_cmip import _logger, mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed @@ -19,6 +17,8 @@ VAR_UNITS = "kg m-2 s-1" TABLE = "CMIP6_Omon.json" +logger = _logger._logger(name=__name__, to_logfile=True, propagate=False) + def handle(infiles, tables, user_input_path, **kwargs): """ @@ -46,7 +46,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = "Starting {name}".format(name=__name__) - logging.info(msg) + logger.info(msg) mappingFileName = infiles["MPAS_map"] timeSeriesFiles = infiles["MPASO"] diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/siconc.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/siconc.py index f7b8784c..db0b9933 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/siconc.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/siconc.py @@ -4,11 +4,9 @@ from __future__ import absolute_import, division, print_function -import logging - import xarray -from e3sm_to_cmip import mpas, util +from e3sm_to_cmip import _logger, mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed @@ -19,6 +17,8 @@ VAR_UNITS = "%" TABLE = "CMIP6_SImon.json" +logger = _logger._logger(name=__name__, to_logfile=True, propagate=False) + def handle(infiles, tables, user_input_path, **kwargs): """ @@ -46,7 +46,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = "Starting {name}".format(name=__name__) - logging.info(msg) + logger.info(msg) mappingFileName = infiles["MPAS_map"] timeSeriesFiles = infiles["MPASSI"] diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/simass.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/simass.py index 224a86bf..ecc4bed9 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/simass.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/simass.py @@ -4,11 +4,9 @@ from __future__ import absolute_import, division, print_function -import logging - import xarray -from e3sm_to_cmip import mpas, util +from e3sm_to_cmip import _logger, mpas, util # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ["MPASSI", "MPAS_mesh", "MPAS_map"] @@ -18,6 +16,8 @@ VAR_UNITS = "kg m-2" TABLE = "CMIP6_SImon.json" +logger = _logger._logger(name=__name__, to_logfile=True, propagate=False) + def handle(infiles, tables, user_input_path, **kwargs): """ @@ -41,7 +41,7 @@ def handle(infiles, tables, user_input_path, **kwargs): the name of the processed variable after processing is complete """ msg = "Starting {name}".format(name=__name__) - logging.info(msg) + logger.info(msg) mappingFileName = infiles["MPAS_map"] timeSeriesFiles = infiles["MPASSI"] diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/sisnmass.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/sisnmass.py index 91c28fd4..ede2a923 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/sisnmass.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/sisnmass.py @@ -4,11 +4,9 @@ from __future__ import absolute_import, division, print_function -import logging - import xarray -from e3sm_to_cmip import mpas, util +from e3sm_to_cmip import _logger, mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed @@ -19,6 +17,8 @@ VAR_UNITS = "kg m-2" TABLE = "CMIP6_SImon.json" +logger = _logger._logger(name=__name__, to_logfile=True, propagate=False) + def handle(infiles, tables, user_input_path, **kwargs): """ @@ -47,7 +47,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = "Starting {name}".format(name=__name__) - logging.info(msg) + logger.info(msg) mappingFileName = infiles["MPAS_map"] timeSeriesFiles = infiles["MPASSI"] diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/sisnthick.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/sisnthick.py index 183200f7..5a9891da 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/sisnthick.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/sisnthick.py @@ -4,11 +4,9 @@ from __future__ import absolute_import, division, print_function -import logging - import xarray -from e3sm_to_cmip import mpas, util +from e3sm_to_cmip import _logger, mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed @@ -19,6 +17,8 @@ VAR_UNITS = "m" TABLE = "CMIP6_SImon.json" +logger = _logger._logger(name=__name__, to_logfile=True, propagate=False) + def handle(infiles, tables, user_input_path, **kwargs): """ @@ -47,7 +47,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = "Starting {name}".format(name=__name__) - logging.info(msg) + logger.info(msg) mappingFileName = infiles["MPAS_map"] timeSeriesFiles = infiles["MPASSI"] diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/sitemptop.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/sitemptop.py index 12684368..0edda0d5 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/sitemptop.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/sitemptop.py @@ -4,11 +4,9 @@ from __future__ import absolute_import, division, print_function -import logging - import xarray -from e3sm_to_cmip import mpas, util +from e3sm_to_cmip import _logger, mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed @@ -19,6 +17,8 @@ VAR_UNITS = "K" TABLE = "CMIP6_SImon.json" +logger = _logger._logger(name=__name__, to_logfile=True, propagate=False) + def handle(infiles, tables, user_input_path, **kwargs): """ @@ -47,7 +47,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = "Starting {name}".format(name=__name__) - logging.info(msg) + logger.info(msg) mappingFileName = infiles["MPAS_map"] timeSeriesFiles = infiles["MPASSI"] diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/sithick.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/sithick.py index 39efbe9b..26d414de 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/sithick.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/sithick.py @@ -4,11 +4,9 @@ from __future__ import absolute_import, division, print_function -import logging - import xarray -from e3sm_to_cmip import mpas, util +from e3sm_to_cmip import _logger, mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed @@ -19,6 +17,8 @@ VAR_UNITS = "m" TABLE = "CMIP6_SImon.json" +logger = _logger._logger(name=__name__, to_logfile=True, propagate=False) + def handle(infiles, tables, user_input_path, **kwargs): """ @@ -47,7 +47,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = "Starting {name}".format(name=__name__) - logging.info(msg) + logger.info(msg) mappingFileName = infiles["MPAS_map"] timeSeriesFiles = infiles["MPASSI"] diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/sitimefrac.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/sitimefrac.py index 7d9e9a9a..1ba9ec06 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/sitimefrac.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/sitimefrac.py @@ -4,11 +4,9 @@ from __future__ import absolute_import, division, print_function -import logging - import xarray -from e3sm_to_cmip import mpas, util +from e3sm_to_cmip import _logger, mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed @@ -19,6 +17,8 @@ VAR_UNITS = "1" TABLE = "CMIP6_SImon.json" +logger = _logger._logger(name=__name__, to_logfile=True, propagate=False) + def handle(infiles, tables, user_input_path, **kwargs): """ @@ -46,7 +46,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = "Starting {name}".format(name=__name__) - logging.info(msg) + logger.info(msg) mappingFileName = infiles["MPAS_map"] timeSeriesFiles = infiles["MPASSI"] diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/siu.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/siu.py index 12f31da4..9b1eb8a1 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/siu.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/siu.py @@ -4,11 +4,9 @@ from __future__ import absolute_import, division, print_function -import logging - import xarray -from e3sm_to_cmip import mpas, util +from e3sm_to_cmip import _logger, mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed @@ -19,6 +17,8 @@ VAR_UNITS = "m s-1" TABLE = "CMIP6_SImon.json" +logger = _logger._logger(name=__name__, to_logfile=True, propagate=False) + def handle(infiles, tables, user_input_path, **kwargs): """ @@ -46,7 +46,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = "Starting {name}".format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles["MPAS_mesh"] mappingFileName = infiles["MPAS_map"] diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/siv.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/siv.py index edd5d90b..0ef61b01 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/siv.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/siv.py @@ -4,11 +4,9 @@ from __future__ import absolute_import, division, print_function -import logging - import xarray -from e3sm_to_cmip import mpas, util +from e3sm_to_cmip import _logger, mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed @@ -19,6 +17,8 @@ VAR_UNITS = "m s-1" TABLE = "CMIP6_SImon.json" +logger = _logger._logger(name=__name__, to_logfile=True, propagate=False) + def handle(infiles, tables, user_input_path, **kwargs): """ @@ -46,7 +46,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = "Starting {name}".format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles["MPAS_mesh"] mappingFileName = infiles["MPAS_map"] diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/so.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/so.py index 7d16405d..11a53aa9 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/so.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/so.py @@ -4,11 +4,9 @@ from __future__ import absolute_import, division, print_function -import logging - import xarray -from e3sm_to_cmip import mpas, util +from e3sm_to_cmip import _logger, mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed @@ -19,6 +17,8 @@ VAR_UNITS = "0.001" TABLE = "CMIP6_Omon.json" +logger = _logger._logger(name=__name__, to_logfile=True, propagate=False) + def handle(infiles, tables, user_input_path, **kwargs): """ @@ -46,7 +46,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = "Starting {name}".format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles["MPAS_mesh"] mappingFileName = infiles["MPAS_map"] diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/sob.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/sob.py index 84af9492..a7e26fea 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/sob.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/sob.py @@ -4,11 +4,9 @@ from __future__ import absolute_import, division, print_function -import logging - import xarray -from e3sm_to_cmip import mpas, util +from e3sm_to_cmip import _logger, mpas, util # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ["MPASO", "MPAS_mesh", "MPAS_map"] @@ -18,6 +16,8 @@ VAR_UNITS = "0.001" TABLE = "CMIP6_Omon.json" +logger = _logger._logger(name=__name__, to_logfile=True, propagate=False) + def handle(infiles, tables, user_input_path, **kwargs): """ @@ -45,7 +45,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = "Starting {name}".format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles["MPAS_mesh"] mappingFileName = infiles["MPAS_map"] diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/soga.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/soga.py index c9685359..b3b8b717 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/soga.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/soga.py @@ -4,11 +4,9 @@ from __future__ import absolute_import, division, print_function -import logging - import xarray -from e3sm_to_cmip import mpas, util +from e3sm_to_cmip import _logger, mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed @@ -19,6 +17,8 @@ VAR_UNITS = "0.001" TABLE = "CMIP6_Omon.json" +logger = _logger._logger(name=__name__, to_logfile=True, propagate=False) + def handle(infiles, tables, user_input_path, **kwargs): """ @@ -47,7 +47,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = "Starting {name}".format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles["MPAS_mesh"] timeSeriesFiles = infiles["MPASO"] diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/sos.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/sos.py index ab4be65e..d6a0b946 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/sos.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/sos.py @@ -4,11 +4,9 @@ from __future__ import absolute_import, division, print_function -import logging - import xarray -from e3sm_to_cmip import mpas, util +from e3sm_to_cmip import _logger, mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed @@ -19,6 +17,8 @@ VAR_UNITS = "0.001" TABLE = "CMIP6_Omon.json" +logger = _logger._logger(name=__name__, to_logfile=True, propagate=False) + def handle(infiles, tables, user_input_path, **kwargs): """ @@ -46,7 +46,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = "Starting {name}".format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles["MPAS_mesh"] mappingFileName = infiles["MPAS_map"] diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/sosga.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/sosga.py index bfa23cec..619bc14b 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/sosga.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/sosga.py @@ -4,11 +4,9 @@ from __future__ import absolute_import, division, print_function -import logging - import xarray -from e3sm_to_cmip import mpas, util +from e3sm_to_cmip import _logger, mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed @@ -19,6 +17,8 @@ VAR_UNITS = "0.001" TABLE = "CMIP6_Omon.json" +logger = _logger._logger(name=__name__, to_logfile=True, propagate=False) + def handle(infiles, tables, user_input_path, **kwargs): """ @@ -46,7 +46,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = "Starting {name}".format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles["MPAS_mesh"] timeSeriesFiles = infiles["MPASO"] diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/tauuo.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/tauuo.py index c1b6ae90..3315dd3d 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/tauuo.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/tauuo.py @@ -4,11 +4,9 @@ from __future__ import absolute_import, division, print_function -import logging - import xarray -from e3sm_to_cmip import mpas, util +from e3sm_to_cmip import _logger, mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed @@ -19,6 +17,8 @@ VAR_UNITS = "N m-2" TABLE = "CMIP6_Omon.json" +logger = _logger._logger(name=__name__, to_logfile=True, propagate=False) + def handle(infiles, tables, user_input_path, **kwargs): """ @@ -46,7 +46,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = "Starting {name}".format(name=__name__) - logging.info(msg) + logger.info(msg) mappingFileName = infiles["MPAS_map"] timeSeriesFiles = infiles["MPASO"] diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/tauvo.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/tauvo.py index 570a431a..84de5870 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/tauvo.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/tauvo.py @@ -4,11 +4,9 @@ from __future__ import absolute_import, division, print_function -import logging - import xarray -from e3sm_to_cmip import mpas, util +from e3sm_to_cmip import _logger, mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed @@ -19,6 +17,8 @@ VAR_UNITS = "N m-2" TABLE = "CMIP6_Omon.json" +logger = _logger._logger(name=__name__, to_logfile=True, propagate=False) + def handle(infiles, tables, user_input_path, **kwargs): """ @@ -46,7 +46,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = "Starting {name}".format(name=__name__) - logging.info(msg) + logger.info(msg) mappingFileName = infiles["MPAS_map"] timeSeriesFiles = infiles["MPASO"] diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/thetao.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/thetao.py index 59781f99..2a469a9f 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/thetao.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/thetao.py @@ -4,11 +4,9 @@ from __future__ import absolute_import, division, print_function -import logging - import xarray -from e3sm_to_cmip import mpas, util +from e3sm_to_cmip import _logger, mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed @@ -19,6 +17,8 @@ VAR_UNITS = "degC" TABLE = "CMIP6_Omon.json" +logger = _logger._logger(name=__name__, to_logfile=True, propagate=False) + def handle(infiles, tables, user_input_path, **kwargs): """ @@ -46,7 +46,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = "Starting {name}".format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles["MPAS_mesh"] mappingFileName = infiles["MPAS_map"] diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/thetaoga.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/thetaoga.py index 0ef0e1ce..efdeaa5f 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/thetaoga.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/thetaoga.py @@ -4,11 +4,9 @@ from __future__ import absolute_import, division, print_function -import logging - import xarray -from e3sm_to_cmip import mpas, util +from e3sm_to_cmip import _logger, mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed @@ -19,6 +17,8 @@ VAR_UNITS = "degC" TABLE = "CMIP6_Omon.json" +logger = _logger._logger(name=__name__, to_logfile=True, propagate=False) + def handle(infiles, tables, user_input_path, **kwargs): """ @@ -47,7 +47,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = "Starting {name}".format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles["MPAS_mesh"] timeSeriesFiles = infiles["MPASO"] diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/thkcello.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/thkcello.py index 389bc428..4d0fe196 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/thkcello.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/thkcello.py @@ -2,12 +2,10 @@ compute Ocean Model Cell Thickness, thkcello """ -import logging - import netCDF4 import xarray -from e3sm_to_cmip import mpas, util +from e3sm_to_cmip import _logger, mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed @@ -18,6 +16,8 @@ VAR_UNITS = "m" TABLE = "CMIP6_Omon.json" +logger = _logger._logger(name=__name__, to_logfile=True, propagate=False) + def handle(infiles, tables, user_input_path, **kwargs): """ @@ -45,7 +45,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = "Starting {name}".format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles["MPAS_mesh"] mappingFileName = infiles["MPAS_map"] diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/tob.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/tob.py index e1503232..2bfcdeda 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/tob.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/tob.py @@ -4,11 +4,9 @@ from __future__ import absolute_import, division, print_function -import logging - import xarray -from e3sm_to_cmip import mpas, util +from e3sm_to_cmip import _logger, mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed @@ -19,6 +17,8 @@ VAR_UNITS = "degC" TABLE = "CMIP6_Omon.json" +logger = _logger._logger(name=__name__, to_logfile=True, propagate=False) + def handle(infiles, tables, user_input_path, **kwargs): """ @@ -46,7 +46,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = "Starting {name}".format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles["MPAS_mesh"] mappingFileName = infiles["MPAS_map"] diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/tos.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/tos.py index 3766a1f8..dab1c782 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/tos.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/tos.py @@ -4,11 +4,9 @@ from __future__ import absolute_import, division, print_function -import logging - import xarray -from e3sm_to_cmip import mpas, util +from e3sm_to_cmip import _logger, mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed @@ -19,6 +17,8 @@ VAR_UNITS = "degC" TABLE = "CMIP6_Omon.json" +logger = _logger._logger(name=__name__, to_logfile=True, propagate=False) + def handle(infiles, tables, user_input_path, **kwargs): """ @@ -46,7 +46,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = "Starting {name}".format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles["MPAS_mesh"] mappingFileName = infiles["MPAS_map"] diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/tosga.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/tosga.py index 55853352..4c48c279 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/tosga.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/tosga.py @@ -4,11 +4,9 @@ from __future__ import absolute_import, division, print_function -import logging - import xarray -from e3sm_to_cmip import mpas, util +from e3sm_to_cmip import _logger, mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed @@ -19,6 +17,8 @@ VAR_UNITS = "degC" TABLE = "CMIP6_Omon.json" +logger = _logger._logger(name=__name__, to_logfile=True, propagate=False) + def handle(infiles, tables, user_input_path, **kwargs): """ @@ -46,7 +46,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = "Starting {name}".format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles["MPAS_mesh"] timeSeriesFiles = infiles["MPASO"] diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/uo.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/uo.py index 3f90cef4..cf7bafa5 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/uo.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/uo.py @@ -4,11 +4,9 @@ from __future__ import absolute_import, division, print_function -import logging - import xarray -from e3sm_to_cmip import mpas, util +from e3sm_to_cmip import _logger, mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed @@ -19,6 +17,8 @@ VAR_UNITS = "m s-1" TABLE = "CMIP6_Omon.json" +logger = _logger._logger(name=__name__, to_logfile=True, propagate=False) + def handle(infiles, tables, user_input_path, **kwargs): """ @@ -46,7 +46,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = "Starting {name}".format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles["MPAS_mesh"] mappingFileName = infiles["MPAS_map"] diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/vo.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/vo.py index ed230a72..698d1261 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/vo.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/vo.py @@ -4,11 +4,9 @@ from __future__ import absolute_import, division, print_function -import logging - import xarray -from e3sm_to_cmip import mpas, util +from e3sm_to_cmip import _logger, mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed @@ -19,6 +17,8 @@ VAR_UNITS = "m s-1" TABLE = "CMIP6_Omon.json" +logger = _logger._logger(name=__name__, to_logfile=True, propagate=False) + def handle(infiles, tables, user_input_path, **kwargs): """ @@ -46,7 +46,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = "Starting {name}".format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles["MPAS_mesh"] mappingFileName = infiles["MPAS_map"] diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/volcello.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/volcello.py index 73b63287..d29c81cf 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/volcello.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/volcello.py @@ -2,12 +2,10 @@ compute Ocean Grid-Cell Volume, volcello """ -import logging - import netCDF4 import xarray -from e3sm_to_cmip import mpas, util +from e3sm_to_cmip import _logger, mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed @@ -18,6 +16,8 @@ VAR_UNITS = "m3" TABLE = "CMIP6_Omon.json" +logger = _logger._logger(name=__name__, to_logfile=True, propagate=False) + def handle(infiles, tables, user_input_path, **kwargs): """ @@ -45,7 +45,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = "Starting {name}".format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles["MPAS_mesh"] mappingFileName = infiles["MPAS_map"] diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/volo.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/volo.py index 53058a48..64c3cc82 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/volo.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/volo.py @@ -4,11 +4,9 @@ from __future__ import absolute_import, division, print_function -import logging - import xarray -from e3sm_to_cmip import mpas, util +from e3sm_to_cmip import _logger, mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed @@ -19,6 +17,8 @@ VAR_UNITS = "m3" TABLE = "CMIP6_Omon.json" +logger = _logger._logger(name=__name__, to_logfile=True, propagate=False) + def handle(infiles, tables, user_input_path, **kwargs): """ @@ -46,7 +46,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = "Starting {name}".format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles["MPAS_mesh"] timeSeriesFiles = infiles["MPASO"] diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/wfo.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/wfo.py index ec982518..3f47708a 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/wfo.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/wfo.py @@ -4,11 +4,9 @@ from __future__ import absolute_import, division, print_function -import logging - import xarray -from e3sm_to_cmip import mpas, util +from e3sm_to_cmip import _logger, mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed @@ -19,6 +17,8 @@ VAR_UNITS = "kg m-2 s-1" TABLE = "CMIP6_Omon.json" +logger = _logger._logger(name=__name__, to_logfile=True, propagate=False) + def handle(infiles, tables, user_input_path, **kwargs): """ @@ -48,7 +48,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = "Starting {name}".format(name=__name__) - logging.info(msg) + logger.info(msg) mappingFileName = infiles["MPAS_map"] timeSeriesFiles = infiles["MPASO"] diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/wo.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/wo.py index b0136c43..c1bd2238 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/wo.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/wo.py @@ -4,11 +4,9 @@ from __future__ import absolute_import, division, print_function -import logging - import xarray -from e3sm_to_cmip import mpas, util +from e3sm_to_cmip import _logger, mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed @@ -19,6 +17,8 @@ VAR_UNITS = "m s-1" TABLE = "CMIP6_Omon.json" +logger = _logger._logger(name=__name__, to_logfile=True, propagate=False) + def handle(infiles, tables, user_input_path, **kwargs): """ @@ -46,7 +46,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = "Starting {name}".format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles["MPAS_mesh"] mappingFileName = infiles["MPAS_map"] diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/zhalfo.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/zhalfo.py index dc26dac3..5c092403 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/zhalfo.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/zhalfo.py @@ -4,12 +4,10 @@ from __future__ import absolute_import, division, print_function -import logging - import numpy import xarray -from e3sm_to_cmip import mpas, util +from e3sm_to_cmip import _logger, mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed @@ -20,6 +18,8 @@ VAR_UNITS = "m" TABLE = "CMIP6_Omon.json" +logger = _logger._logger(name=__name__, to_logfile=True, propagate=False) + def handle(infiles, tables, user_input_path, **kwargs): """ @@ -47,7 +47,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = "Starting {name}".format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles["MPAS_mesh"] mappingFileName = infiles["MPAS_map"] diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/zos.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/zos.py index eb439a54..1fbc3a9a 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/zos.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/zos.py @@ -4,11 +4,9 @@ from __future__ import absolute_import, division, print_function -import logging - import xarray -from e3sm_to_cmip import mpas, util +from e3sm_to_cmip import _logger, mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed @@ -19,6 +17,8 @@ VAR_UNITS = "m" TABLE = "CMIP6_Omon.json" +logger = _logger._logger(name=__name__, to_logfile=True, propagate=False) + def handle(infiles, tables, user_input_path, **kwargs): """ @@ -46,7 +46,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = "Starting {name}".format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles["MPAS_mesh"] mappingFileName = infiles["MPAS_map"] diff --git a/e3sm_to_cmip/cmor_handlers/utils.py b/e3sm_to_cmip/cmor_handlers/utils.py index 8f946e8e..15c55af8 100644 --- a/e3sm_to_cmip/cmor_handlers/utils.py +++ b/e3sm_to_cmip/cmor_handlers/utils.py @@ -10,12 +10,17 @@ HANDLER_DEFINITIONS_PATH, LEGACY_HANDLER_DIR_PATH, MPAS_HANDLER_DIR_PATH, + _logger, ) -from e3sm_to_cmip._logger import _setup_logger from e3sm_to_cmip.cmor_handlers.handler import VarHandler from e3sm_to_cmip.util import _get_table_for_non_monthly_freq -logger = _setup_logger(__name__) + +def instantiate_h_utils_logger(): + global logger + + logger = _logger._logger(name=__name__, to_logfile=True, propagate=False) + # Type aliases Frequency = Literal["mon", "day", "6hrLev", "6hrPlev", "6hrPlevPt", "3hr", "1hr"] diff --git a/e3sm_to_cmip/cmor_handlers/vars/areacella.py b/e3sm_to_cmip/cmor_handlers/vars/areacella.py index 9f0d50b8..d5207f04 100644 --- a/e3sm_to_cmip/cmor_handlers/vars/areacella.py +++ b/e3sm_to_cmip/cmor_handlers/vars/areacella.py @@ -5,19 +5,17 @@ from __future__ import absolute_import, division, print_function, unicode_literals import json -import logging import os import cmor import numpy as np import xarray as xr -from e3sm_to_cmip import resources -from e3sm_to_cmip._logger import _setup_logger +from e3sm_to_cmip import _logger, resources from e3sm_to_cmip.mpas import write_netcdf from e3sm_to_cmip.util import print_message, setup_cmor -logger = _setup_logger(__name__) +logger = _logger._logger(name=__name__, to_logfile=True, propagate=False) # list of raw variable names needed RAW_VARIABLES = [str("area")] @@ -66,7 +64,7 @@ def handle(infiles, tables, user_input_path, table, logdir): if len(infiles[variable]) == 0: msg = "{}: Unable to find input files for {}".format(VAR_NAME, variable) print_message(msg) - logging.error(msg) + logger.error(msg) zerofiles = True if zerofiles: return None @@ -128,7 +126,7 @@ def handle(infiles, tables, user_input_path, table, logdir): ] msg = "areacella: running CMOR" - logging.info(msg) + logger.info(msg) axis_ids = list() for axis in axes: diff --git a/e3sm_to_cmip/cmor_handlers/vars/clisccp.py b/e3sm_to_cmip/cmor_handlers/vars/clisccp.py index 5402ef72..d077de2a 100644 --- a/e3sm_to_cmip/cmor_handlers/vars/clisccp.py +++ b/e3sm_to_cmip/cmor_handlers/vars/clisccp.py @@ -6,17 +6,16 @@ from __future__ import absolute_import, annotations, division, unicode_literals -import logging from typing import Dict, List, Union import cmor import numpy as np import xarray as xr -from e3sm_to_cmip._logger import _setup_logger +from e3sm_to_cmip import _logger from e3sm_to_cmip.util import print_message, setup_cmor -logger = _setup_logger(__name__) +logger = _logger._logger(name=__name__, to_logfile=True, propagate=False) # list of raw variable names needed RAW_VARIABLES = [str("FISCCP1_COSP")] @@ -59,14 +58,14 @@ def handle( # noqa: C901 If CMORizing was successful, return the output CMIP variable name to indicate success. If failed, return None .""" - logging.info(f"Starting {VAR_NAME}") + logger.info(f"Starting {VAR_NAME}") nonzero = False for variable in RAW_VARIABLES: if len(vars_to_filepaths[variable]) == 0: msg = f"{variable}: Unable to find input files for {RAW_VARIABLES}" print_message(msg) - logging.error(msg) + logger.error(msg) nonzero = True if nonzero: return None diff --git a/e3sm_to_cmip/cmor_handlers/vars/orog.py b/e3sm_to_cmip/cmor_handlers/vars/orog.py index 7de757df..4741a0ef 100644 --- a/e3sm_to_cmip/cmor_handlers/vars/orog.py +++ b/e3sm_to_cmip/cmor_handlers/vars/orog.py @@ -5,19 +5,17 @@ from __future__ import absolute_import, division, print_function, unicode_literals import json -import logging import os import cmor import numpy as np import xarray as xr -from e3sm_to_cmip import resources -from e3sm_to_cmip._logger import _setup_logger +from e3sm_to_cmip import _logger, resources from e3sm_to_cmip.mpas import write_netcdf from e3sm_to_cmip.util import print_message, setup_cmor -logger = _setup_logger(__name__) +logger = _logger._logger(name=__name__, to_logfile=True, propagate=False) # list of raw variable names needed RAW_VARIABLES = [str("PHIS")] @@ -66,7 +64,7 @@ def handle(infiles, tables, user_input_path, table, logdir): if len(infiles[variable]) == 0: msg = f"{VAR_NAME}: Unable to find input files for {variable}" print_message(msg) - logging.error(msg) + logger.error(msg) zerofiles = True if zerofiles: return None @@ -79,7 +77,7 @@ def handle(infiles, tables, user_input_path, table, logdir): ) msg = "{}: CMOR setup complete".format(VAR_NAME) - logging.info(msg) + logger.info(msg) # extract data from the input file msg = "orog: loading PHIS" @@ -120,7 +118,7 @@ def handle(infiles, tables, user_input_path, table, logdir): ] msg = "orog: running CMOR" - logging.info(msg) + logger.info(msg) axis_ids = list() for axis in axes: diff --git a/e3sm_to_cmip/cmor_handlers/vars/sftlf.py b/e3sm_to_cmip/cmor_handlers/vars/sftlf.py index 127f7788..56a0f593 100644 --- a/e3sm_to_cmip/cmor_handlers/vars/sftlf.py +++ b/e3sm_to_cmip/cmor_handlers/vars/sftlf.py @@ -5,19 +5,17 @@ from __future__ import absolute_import, division, print_function, unicode_literals import json -import logging import os import cmor import numpy as np import xarray as xr -from e3sm_to_cmip import resources -from e3sm_to_cmip._logger import _setup_logger +from e3sm_to_cmip import _logger, resources from e3sm_to_cmip.mpas import write_netcdf from e3sm_to_cmip.util import print_message, setup_cmor -logger = _setup_logger(__name__) +logger = _logger._logger(name=__name__, to_logfile=True, propagate=False) # list of raw variable names needed RAW_VARIABLES = [str("LANDFRAC")] @@ -65,7 +63,7 @@ def handle(infiles, tables, user_input_path, table, logdir): if len(infiles[variable]) == 0: msg = f"{VAR_NAME}: Unable to find input files for {variable}" print_message(msg) - logging.error(msg) + logger.error(msg) zerofiles = True if zerofiles: return None @@ -78,7 +76,7 @@ def handle(infiles, tables, user_input_path, table, logdir): ) msg = "{}: CMOR setup complete".format(VAR_NAME) - logging.info(msg) + logger.info(msg) # extract data from the input file msg = "sftlf: loading LANDFRAC" @@ -119,7 +117,7 @@ def handle(infiles, tables, user_input_path, table, logdir): ] msg = "sftlf: running CMOR" - logging.info(msg) + logger.info(msg) axis_ids = list() for axis in axes: diff --git a/e3sm_to_cmip/mpas.py b/e3sm_to_cmip/mpas.py index b3f101fa..e7ed0123 100644 --- a/e3sm_to_cmip/mpas.py +++ b/e3sm_to_cmip/mpas.py @@ -5,7 +5,6 @@ from __future__ import absolute_import, division, print_function import argparse -import logging import multiprocessing import os import re @@ -23,21 +22,23 @@ import xarray from dask.diagnostics import ProgressBar +from e3sm_to_cmip import _logger + +logger = _logger._logger(name=__name__, to_logfile=True, propagate=False) -def run_ncremap_cmd(args, env): - logtext = f"mpas.py: remap: ncremap args = {args}" - logging.info(logtext) +def run_ncremap_cmd(args, env): proc = subprocess.Popen( args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env ) (out, err) = proc.communicate() - logging.info(out) + logger.info(out) if proc.returncode: - print("Error running ncremap command: {}".format(" ".join(args))) + arglist = " ".join(args) + logger.error(f"Error running ncremap command: {arglist}") print(err.decode("utf-8")) raise subprocess.CalledProcessError( # type: ignore - "ncremap returned {}".format(proc.returncode) # type: ignore + f"ncremap returned {proc.returncode}" # type: ignore ) @@ -55,6 +56,11 @@ def remap_seaice_sgs(inFileName, outFileName, mappingFileName, renorm_threshold= ds_in = xarray.open_dataset(inFileName, decode_times=False) outFilePath = f"{outFileName}sub" os.makedirs(outFilePath) + + logger.info( + f"Calling run_ncremap_cmd for each ds_slice in {range(ds_in.sizes['time'])}" + ) + for t_index in range(ds_in.sizes["time"]): ds_slice = ds_in.isel(time=slice(t_index, t_index + 1)) ds_slice.to_netcdf(f"{outFilePath}/temp_in{t_index}.nc") @@ -117,6 +123,7 @@ def remap(ds, pcode, mappingFileName): outFileName = _get_temp_path() if "depth" in ds.dims: + logger.info("Calling ds.transpose") ds = ds.transpose("time", "depth", "nCells", "nbnd") # missing_value_mask attribute has undesired impacts in ncremap @@ -143,8 +150,8 @@ def remap(ds, pcode, mappingFileName): # remove the temporary files keep_temp_files = False if keep_temp_files: - logging.info(f"Retaining inFileName {inFileName}") - logging.info(f"Retaining outFileName {outFileName}") + logger.info(f"Retaining inFileName {inFileName}") + logger.info(f"Retaining outFileName {outFileName}") else: os.remove(inFileName) os.remove(outFileName) @@ -189,6 +196,8 @@ def add_time(ds, dsIn, referenceDate="0001-01-01", offsetYears=0): dateStrings=xtimeEnd, referenceDate=referenceDate ) + logger.info(f"add_time: daysStart={daysStart} daysEnd={daysEnd}") + time_bnds = np.zeros((len(daysStart), 2)) time_bnds[:, 0] = daysStart time_bnds[:, 1] = daysEnd @@ -431,7 +440,7 @@ def write_cmor(axes, ds, varname, varunits, d2f=True, **kwargs): time_bnds=ds.time_bnds.values, ) except Exception as error: - logging.exception(f"Error in cmor.write for {varname}") + logger.exception(f"Error in cmor.write for {varname}") raise Exception(error) from error finally: cmor.close(varid) diff --git a/e3sm_to_cmip/util.py b/e3sm_to_cmip/util.py index 3627d9e5..330cb0a0 100644 --- a/e3sm_to_cmip/util.py +++ b/e3sm_to_cmip/util.py @@ -14,9 +14,13 @@ import yaml from tqdm import tqdm -from e3sm_to_cmip._logger import _setup_logger +from e3sm_to_cmip import _logger -logger = _setup_logger(__name__) + +def instantiate_util_logger(): + global logger + + logger = _logger._logger(name=__name__, to_logfile=True, propagate=False) ATMOS_TABLES = [ @@ -112,7 +116,7 @@ def setup_cmor(var_name, table_path, table_name, user_input_path): table_name = str(table_name) user_input_path = str(user_input_path) - logfile = os.path.join(os.getcwd(), "logs") + logfile = os.path.join(os.getcwd(), "cmor_logs") if not os.path.exists(logfile): os.makedirs(logfile) @@ -562,7 +566,7 @@ def find_mpas_files(component, path, map_path=None): # noqa: C901 pattern_v2 = "mocBasinsAndTransects" for infile in contents: if pattern_v1 in infile or pattern_v2 in infile: - logger.info(f"component mpas0_moc_regions found: {infile}") + logger.info(f"component mpaso_moc_regions found: {infile}") return os.path.abspath(os.path.join(path, infile)) raise IOError("Unable to find mpaso_moc_regions in the input directory") diff --git a/tests/run_script.py b/tests/run_script.py new file mode 100644 index 00000000..dc53a411 --- /dev/null +++ b/tests/run_script.py @@ -0,0 +1,24 @@ +""" +A test script to run the `e3sm_to_cmip` package with a set of arguments. + +This script is meant to be ran into LCRC. +""" + +from e3sm_to_cmip.__main__ import main + +args = [ + "--var-list", + "pfull, phalf, tas, ts, psl, ps, sfcWind, huss, pr, prc, prsn, evspsbl, tauu, tauv, hfls, clt, rlds, rlus, rsds, rsus, hfss, cl, clw, cli, clivi, clwvi, prw, rldscs, rlut, rlutcs, rsdt, rsuscs, rsut, rsutcs, rtmt, abs550aer, od550aer, rsdscs, hur", + "--input", + "/lcrc/group/e3sm/e3sm_to_cmip/input/atm-unified-eam-ncclimo", + "--output", + "../qa/tmp", + "--tables-path", + "/lcrc/group/e3sm/e3sm_to_cmip/cmip6-cmor-tables/Tables/", + "--user-metadata", + "/lcrc/group/e3sm/e3sm_to_cmip/template.json", + "--serial", +] + +# `main()` creates an `E3SMtoCMIP` object and passes `args` to it, which sets the object parameters to execute a run. +main(args)