From ba4f863bee949a7b2e5b9297fca2137ec2e2436b Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Thu, 23 Jun 2022 17:38:45 +0200 Subject: [PATCH 1/5] Refactor the logging and add support for the use of a log-config file Signed-off-by: Adam.Dybbroe --- bin/dispatcher.py | 45 +++---------------- bin/move_it_client.py | 6 +-- bin/move_it_server.py | 7 ++- examples/log_config.yaml | 11 +++++ trollmoves/logger.py | 92 ++++++++++++++++++++++++++++++++++++++ trollmoves/move_it_base.py | 41 ++++------------- 6 files changed, 126 insertions(+), 76 deletions(-) create mode 100644 trollmoves/logger.py diff --git a/bin/dispatcher.py b/bin/dispatcher.py index 8924386b..ccfc5dcb 100644 --- a/bin/dispatcher.py +++ b/bin/dispatcher.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # -# Copyright (c) 2012-2019 +# Copyright (c) 2012-2019, 2022 # # Author(s): # @@ -28,48 +28,13 @@ import argparse import logging -import logging.config -import logging.handlers import os import sys -import yaml - +from trollmoves.logger import LoggerSetup from trollmoves.dispatcher import Dispatcher -LOG_FORMAT = "[%(asctime)s %(levelname)-8s] %(message)s" -logger = logging.getLogger('dispatcher') - -log_levels = { - 0: logging.WARN, - 1: logging.INFO, - 2: logging.DEBUG, -} - - -def setup_logging(cmd_args): - """Set up logging.""" - if cmd_args.log_config is not None: - with open(cmd_args.log_config) as fd: - log_dict = yaml.safe_load(fd.read()) - logging.config.dictConfig(log_dict) - return - - root = logging.getLogger('') - root.setLevel(log_levels[cmd_args.verbosity]) - - if cmd_args.log: - fh_ = logging.handlers.TimedRotatingFileHandler( - os.path.join(cmd_args.log), - "midnight", - backupCount=7) - else: - fh_ = logging.StreamHandler() - - formatter = logging.Formatter(LOG_FORMAT) - fh_.setFormatter(formatter) - - root.addHandler(fh_) +LOG = logging.getLogger('dispatcher') def parse_args(): @@ -99,7 +64,9 @@ def parse_args(): def main(): """Start and run the dispatcher.""" cmd_args = parse_args() - setup_logging(cmd_args) + logger = LoggerSetup(cmd_args) + logger.setup_logging() + LOG = logger.get_logger() logger.info("Starting up.") try: diff --git a/bin/move_it_client.py b/bin/move_it_client.py index 5425ff23..c887354f 100644 --- a/bin/move_it_client.py +++ b/bin/move_it_client.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # -# Copyright (c) 2012, 2013, 2014, 2015, 2016 +# Copyright (c) 2012-2022 Pytroll Developers # # Author(s): # @@ -77,7 +77,6 @@ # TODO: implement ping and server selection import logging -import logging.handlers import argparse import signal import time @@ -85,7 +84,6 @@ from trollmoves.move_it_base import MoveItBase LOGGER = logging.getLogger("move_it_client") -LOG_FORMAT = "[%(asctime)s %(levelname)-8s %(name)s] %(message)s" class MoveItClient(MoveItBase): @@ -116,6 +114,8 @@ def parse_args(): help="The configuration file to run on.") parser.add_argument("-l", "--log", help="The file to log to. stdout otherwise.") + parser.add_argument("-c", "--log-config", + help="Log config file to use instead of the standard logging.") parser.add_argument("-v", "--verbose", default=False, action="store_true", help="Toggle verbose logging") return parser.parse_args() diff --git a/bin/move_it_server.py b/bin/move_it_server.py index 4fa3f269..e02d4816 100644 --- a/bin/move_it_server.py +++ b/bin/move_it_server.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # -# Copyright (c) 2012, 2013, 2014, 2015, 2016 +# Copyright (c) 2012, 2013, 2014, 2015, 2016, 2022 # # Author(s): # @@ -93,9 +93,9 @@ """ import logging -import logging.handlers import argparse from trollmoves.server import MoveItServer +from trollmoves.logger import setup_logger LOGGER = logging.getLogger("move_it_server") LOG_FORMAT = "[%(asctime)s %(levelname)-8s %(name)s] %(message)s" @@ -108,6 +108,8 @@ def parse_args(): help="The configuration file to run on.") parser.add_argument("-l", "--log", help="The file to log to. stdout otherwise.") + parser.add_argument("-c", "--log-config", + help="Log config file to use instead of the standard logging.") parser.add_argument("-p", "--port", help="The port to publish on. 9010 is the default", default=9010) @@ -125,6 +127,7 @@ def parse_args(): def main(): """Start the server.""" cmd_args = parse_args() + setup_logging(cmd_args) server = MoveItServer(cmd_args) try: diff --git a/examples/log_config.yaml b/examples/log_config.yaml index 72ff5be6..4027c369 100644 --- a/examples/log_config.yaml +++ b/examples/log_config.yaml @@ -9,6 +9,17 @@ handlers: level: DEBUG formatter: pytroll stream: ext://sys.stdout + monitor: + (): pytroll_monitor.op5_logger.AsyncOP5Handler + auth: [{{ monitor_user }}, {{ monitor_password}}] + service: check_{{ chain_name }}_regional + server: {{op5_server}}/api/command/PROCESS_SERVICE_CHECK_RESULT + host: {{ inventory_hostname }} +loggers: + posttroll: + level: ERROR + propagate: false + handlers: [console, monitor] root: level: DEBUG handlers: [console, monitor] diff --git a/trollmoves/logger.py b/trollmoves/logger.py new file mode 100644 index 00000000..ab6e636a --- /dev/null +++ b/trollmoves/logger.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (c) 2022 Pytroll Developers + +# Author(s): + +# Adam Dybbroe + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""The log handling. +""" + +import logging +import logging.config +import logging.handlers +import yaml +import pyinotify + +LOG_FORMAT = "[%(asctime)s %(levelname)-8s] %(message)s" + +log_levels = { + 0: logging.WARN, + 1: logging.INFO, + 2: logging.DEBUG, +} + + +class LoggerSetup(): + """Setup logging.""" + + def __init__(self, cmd_args, logger=None): + """Init the logging setup class.""" + self._cmd_args = cmd_args + self._file_handler = None + self._logger = logger + + def setup_logging(self, chain_type=None): + """Set up logging.""" + if self._cmd_args.log_config is not None: + with open(self._cmd_args.log_config) as fd_: + log_dict = yaml.safe_load(fd_.read()) + logging.config.dictConfig(log_dict) + return + + self._logger = logging.getLogger('') + self._logger.setLevel(log_levels[self._cmd_args.verbosity]) + + if self._cmd_args.log: + self._file_handler = logging.handlers.TimedRotatingFileHandler( + os.path.join(self._cmd_args.log), + "midnight", + backupCount=7) + else: + self._file_handler = logging.StreamHandler() + + formatter = logging.Formatter(LOG_FORMAT) + self._file_handler.setFormatter(formatter) + + self._logger.addHandler(self._file_handler) + self._set_loggername(chain_type) + + def _set_loggername(self, chain_type): + if not chain_type: + return + + logger_name = "move_it_server" + if chain_type == "client": + logger_name = "move_it_client" + elif chain_type == "mirror": + logger_name = "move_it_mirror" + self._logger = logging.getLogger(logger_name) + + def get_logger(self): + """Get the logger to use for logging.""" + return self._logger + + def init_pyinotify_logging(self): + """Initialize the pyinotify handler.""" + pyinotify.log.handlers = [self._file_handler] diff --git a/trollmoves/move_it_base.py b/trollmoves/move_it_base.py index 591cf6fe..fc81580e 100644 --- a/trollmoves/move_it_base.py +++ b/trollmoves/move_it_base.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # -# Copyright (c) 2012, 2013, 2014, 2015, 2016 +# Copyright (c) 2012, 2013, 2014, 2015, 2016, 2022 # # Author(s): # @@ -29,9 +29,8 @@ import pyinotify from posttroll.publisher import Publisher - +from trollmoves.logger import LoggerSetup LOGGER = logging.getLogger("move_it_base") -LOG_FORMAT = "[%(asctime)s %(levelname)-8s %(name)s] %(message)s" class MoveItBase(object): @@ -39,6 +38,7 @@ class MoveItBase(object): def __init__(self, cmd_args, chain_type, publisher=None): """Initialize the class.""" + global LOGGER self.cmd_args = cmd_args self.chain_type = chain_type self.running = False @@ -47,7 +47,12 @@ def __init__(self, cmd_args, chain_type, publisher=None): self.publisher = publisher self._np = None self.chains = {} - setup_logging(cmd_args, chain_type) + + logger = LoggerSetup(cmd_args, LOGGER) + logger.setup_logging(chain_type) + LOGGER = logger.get_logger() + logger.init_pyinotify_logging() + LOGGER.info("Starting up.") self.setup_watchers(cmd_args) @@ -107,34 +112,6 @@ def setup_watchers(self, cmd_args): self.watchman.add_watch(os.path.dirname(cmd_args.config_file), mask) -def setup_logging(cmd_args, chain_type): - """Set up logging.""" - global LOGGER - LOGGER = logging.getLogger('') - if cmd_args.verbose: - LOGGER.setLevel(logging.DEBUG) - - if cmd_args.log: - fh_ = logging.handlers.TimedRotatingFileHandler( - os.path.join(cmd_args.log), - "midnight", - backupCount=7) - else: - fh_ = logging.StreamHandler() - - formatter = logging.Formatter(LOG_FORMAT) - fh_.setFormatter(formatter) - - LOGGER.addHandler(fh_) - logger_name = "move_it_server" - if chain_type == "client": - logger_name = "move_it_client" - elif chain_type == "mirror": - logger_name = "move_it_mirror" - LOGGER = logging.getLogger(logger_name) - pyinotify.log.handlers = [fh_] - - def create_publisher(port, publisher_name): """Create a publisher using port *port*.""" LOGGER.info("Starting publisher on port %s.", str(port)) From 6c217a7d3c310ae7feb56742800c15519c68427f Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Fri, 8 Jul 2022 15:47:05 +0200 Subject: [PATCH 2/5] Add testing of the logging configuration setup Signed-off-by: Adam.Dybbroe --- trollmoves/logger.py | 7 +- trollmoves/tests/test_logger.py | 141 ++++++++++++++++++++++++++++++++ 2 files changed, 147 insertions(+), 1 deletion(-) create mode 100644 trollmoves/tests/test_logger.py diff --git a/trollmoves/logger.py b/trollmoves/logger.py index ab6e636a..791309d3 100644 --- a/trollmoves/logger.py +++ b/trollmoves/logger.py @@ -23,6 +23,7 @@ """The log handling. """ +import os import logging import logging.config import logging.handlers @@ -53,8 +54,12 @@ def setup_logging(self, chain_type=None): with open(self._cmd_args.log_config) as fd_: log_dict = yaml.safe_load(fd_.read()) logging.config.dictConfig(log_dict) - return + self._logger = logging.getLogger('') + else: + self._setup_default_logging(chain_type) + def _setup_default_logging(self, chain_type): + """Setup default logging without using a log-config file.""" self._logger = logging.getLogger('') self._logger.setLevel(log_levels[self._cmd_args.verbosity]) diff --git a/trollmoves/tests/test_logger.py b/trollmoves/tests/test_logger.py new file mode 100644 index 00000000..18278a3d --- /dev/null +++ b/trollmoves/tests/test_logger.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (c) 2022 Pytroll Developers + +# Author(s): + +# Adam Dybbroe + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Test the logging setup +""" + +import pytest +from trollmoves.logger import LoggerSetup +from dataclasses import dataclass +import logging + +TEST_LOGGER = logging.getLogger("mytest") + + +TEST_LOG_YAML_CONTENT = """ +version: 1 +disable_existing_loggers: false +formatters: + pytroll: + format: '[%(asctime)s %(levelname)-8s %(name)s] %(message)s' +handlers: + console: + class: logging.StreamHandler + level: DEBUG + formatter: pytroll + stream: ext://sys.stdout + timed_log_rotation: + class: logging.handlers.TimedRotatingFileHandler + level: DEBUG + formatter: pytroll + filename: name_of_the_process + when: h + monitor: + (): pytroll_monitor.op5_logger.AsyncOP5Handler + auth: [username, passwd] + service: check_name_of_the_process + server: https://monitor-xxx.somewhere.yy/api/command/PROCESS_SERVICE_CHECK_RESULT + host: myhost +loggers: + posttroll: + level: ERROR + propagate: false + handlers: [console, monitor, timed_log_rotation] +root: + level: DEBUG + handlers: [console, monitor, timed_log_rotation] +""" + + +@dataclass +class FakeArgparseOutput: + config_file: str + log: str + log_config: str + verbosity: bool + + +@pytest.fixture +def fake_cmd_args_no_logfile(): + """Return a fake argparse Namespace.""" + args_namespace = FakeArgparseOutput('./example_move_it_client.cfg', None, + None, + False) + return args_namespace + + +@pytest.fixture +def fake_cmd_args(fake_yamlconfig_file): + """Return a fake argparse Namespace.""" + args_namespace = FakeArgparseOutput('./example_move_it_client.cfg', None, + fake_yamlconfig_file, + False) + return args_namespace + + +@pytest.fixture +def fake_yamlconfig_file(tmp_path): + """Write fake yaml config file.""" + file_path = tmp_path / 'test_file_log_config.yaml' + with open(file_path, 'w') as fpt: + fpt.write(TEST_LOG_YAML_CONTENT) + + yield file_path + + +def test_setup_logging_init(fake_cmd_args): + """Test initializing the LoggerSetup class.""" + mylogger = LoggerSetup(fake_cmd_args) + + assert mylogger._cmd_args == fake_cmd_args + assert mylogger._file_handler is None + assert mylogger._logger is None + + mylogger = LoggerSetup(fake_cmd_args, TEST_LOGGER) + assert mylogger._logger == TEST_LOGGER + + +def test_setup_logging_from_log_config(fake_cmd_args): + """Test getting and setting logging config from file.""" + mylogger = LoggerSetup(fake_cmd_args) + mylogger.setup_logging('client') + + this_logger = mylogger.get_logger() + assert this_logger.name == 'root' + ahandler = this_logger.handlers[2] + assert ahandler.when == 'H' + assert ahandler.interval == 3600 + assert ahandler.name == 'timed_log_rotation' + assert ahandler.level == 10 + + # To be continued!? FIXME! + + +def test_setup_logging_default(fake_cmd_args_no_logfile): + """Test setting the logging config from without a log-config file.""" + mylogger = LoggerSetup(fake_cmd_args_no_logfile) + mylogger.setup_logging('client') + + this_logger = mylogger.get_logger() + assert this_logger.name == 'move_it_client' + + # To be continued!? FIXME! From aa2d0f6c61ced5ee8ba1d296c9c0cd46f8246d64 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Fri, 8 Jul 2022 16:00:16 +0200 Subject: [PATCH 3/5] Add pytroll_monitor as a requirement for testing Signed-off-by: Adam.Dybbroe --- setup.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 61a84cb1..3f3b5e2a 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (c) 2015, 2020. +# Copyright (c) 2015 - 2022. # # Author(s): @@ -36,6 +36,8 @@ all_extras.extend(extra_deps) extras_require['all'] = list(set(all_extras)) +test_requires = ['pytroll_monitor'], + setup(name="trollmoves", version=versioneer.get_version(), description='Pytroll file utilities', @@ -64,5 +66,6 @@ 'trollsift', 'netifaces', 'pyzmq', 'inotify', 'scp', 'paramiko', 'pyyaml', 'watchdog'], + tests_require=test_requires, extras_require=extras_require, ) From cd9c19fe0736eed59b33969b9114cf9abdb88fda Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Fri, 8 Jul 2022 19:51:44 +0200 Subject: [PATCH 4/5] Make sure pytroll-monitor is installed for github actions Signed-off-by: Adam.Dybbroe --- .github/workflows/ci.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ec91721c..56e71fba 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -27,7 +27,7 @@ jobs: - name: Install dependencies run: | - pip install -U codecov pytest pytest-cov pyyaml trollsift posttroll inotify pyinotify paramiko scp watchdog + pip install -U codecov pytest pytest-cov pyyaml trollsift posttroll pytroll-monitor inotify pyinotify paramiko scp watchdog - name: Install trollmoves run: | @@ -43,4 +43,3 @@ jobs: flags: unittests file: ./coverage.xml env_vars: OS,PYTHON_VERSION,UNSTABLE - From 8565e4ae41fad93cfb577c2263a0788d650fa5ba Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Fri, 8 Jul 2022 20:11:55 +0200 Subject: [PATCH 5/5] Use underscore package name instead of dash Signed-off-by: Adam.Dybbroe --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 56e71fba..1102551f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -27,7 +27,7 @@ jobs: - name: Install dependencies run: | - pip install -U codecov pytest pytest-cov pyyaml trollsift posttroll pytroll-monitor inotify pyinotify paramiko scp watchdog + pip install -U codecov pytest pytest-cov pyyaml trollsift posttroll pytroll_monitor inotify pyinotify paramiko scp watchdog - name: Install trollmoves run: |