Refactor the logging and add support for the use of a log-config file

.github/workflows/ci.yaml

@@ -27,7 +27,7 @@ jobs:
 
       - name: Install dependencies
         run: |
-          pip install -U codecov pytest pytest-cov pyyaml trollsift posttroll inotify pyinotify paramiko scp watchdog
+          pip install -U codecov pytest pytest-cov pyyaml trollsift posttroll pytroll_monitor inotify pyinotify paramiko scp watchdog
 
       - name: Install trollmoves
         run: |
@@ -43,4 +43,3 @@ jobs:
           flags: unittests
           file: ./coverage.xml
           env_vars: OS,PYTHON_VERSION,UNSTABLE
-

bin/dispatcher.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
-# Copyright (c) 2012-2019
+# Copyright (c) 2012-2019, 2022
 #
 # Author(s):
 #
@@ -28,48 +28,13 @@
 
 import argparse
 import logging
-import logging.config
-import logging.handlers
 import os
 import sys
 
-import yaml
-
+from trollmoves.logger import LoggerSetup
 from trollmoves.dispatcher import Dispatcher
 
-LOG_FORMAT = "[%(asctime)s %(levelname)-8s] %(message)s"
-logger = logging.getLogger('dispatcher')
-
-log_levels = {
-    0: logging.WARN,
-    1: logging.INFO,
-    2: logging.DEBUG,
-}
-
-
-def setup_logging(cmd_args):
-    """Set up logging."""
-    if cmd_args.log_config is not None:
-        with open(cmd_args.log_config) as fd:
-            log_dict = yaml.safe_load(fd.read())
-            logging.config.dictConfig(log_dict)
-            return
-
-    root = logging.getLogger('')
-    root.setLevel(log_levels[cmd_args.verbosity])
-
-    if cmd_args.log:
-        fh_ = logging.handlers.TimedRotatingFileHandler(
-            os.path.join(cmd_args.log),
-            "midnight",
-            backupCount=7)
-    else:
-        fh_ = logging.StreamHandler()
-
-    formatter = logging.Formatter(LOG_FORMAT)
-    fh_.setFormatter(formatter)
-
-    root.addHandler(fh_)
+LOG = logging.getLogger('dispatcher')
 
 
 def parse_args():
@@ -99,7 +64,9 @@ def parse_args():
 def main():
     """Start and run the dispatcher."""
     cmd_args = parse_args()
-    setup_logging(cmd_args)
+    logger = LoggerSetup(cmd_args)
+    logger.setup_logging()
+    LOG = logger.get_logger()
     logger.info("Starting up.")
 
     try:

bin/move_it_client.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
-# Copyright (c) 2012, 2013, 2014, 2015, 2016
+# Copyright (c) 2012-2022 Pytroll Developers
 #
 # Author(s):
 #
@@ -77,15 +77,13 @@
 
 # TODO: implement ping and server selection
 import logging
-import logging.handlers
 import argparse
 import signal
 import time
 
 from trollmoves.move_it_base import MoveItBase
 
 LOGGER = logging.getLogger("move_it_client")
-LOG_FORMAT = "[%(asctime)s %(levelname)-8s %(name)s] %(message)s"
 
 
 class MoveItClient(MoveItBase):
@@ -116,6 +114,8 @@ def parse_args():
                         help="The configuration file to run on.")
     parser.add_argument("-l", "--log",
                         help="The file to log to. stdout otherwise.")
+    parser.add_argument("-c", "--log-config",
+                        help="Log config file to use instead of the standard logging.")
     parser.add_argument("-v", "--verbose", default=False, action="store_true",
                         help="Toggle verbose logging")
     return parser.parse_args()

bin/move_it_server.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
-# Copyright (c) 2012, 2013, 2014, 2015, 2016
+# Copyright (c) 2012, 2013, 2014, 2015, 2016, 2022
 #
 # Author(s):
 #
@@ -93,9 +93,9 @@
 """
 
 import logging
-import logging.handlers
 import argparse
 from trollmoves.server import MoveItServer
+from trollmoves.logger import setup_logger
 
 LOGGER = logging.getLogger("move_it_server")
 LOG_FORMAT = "[%(asctime)s %(levelname)-8s %(name)s] %(message)s"
@@ -108,6 +108,8 @@ def parse_args():
                         help="The configuration file to run on.")
     parser.add_argument("-l", "--log",
                         help="The file to log to. stdout otherwise.")
+    parser.add_argument("-c", "--log-config",
+                        help="Log config file to use instead of the standard logging.")
     parser.add_argument("-p", "--port",
                         help="The port to publish on. 9010 is the default",
                         default=9010)
@@ -125,6 +127,7 @@ def parse_args():
 def main():
     """Start the server."""
     cmd_args = parse_args()
+    setup_logging(cmd_args)
     server = MoveItServer(cmd_args)
 
     try:

examples/log_config.yaml

@@ -9,6 +9,17 @@ handlers:
     level: DEBUG
     formatter: pytroll
     stream: ext://sys.stdout
+  monitor:
+    (): pytroll_monitor.op5_logger.AsyncOP5Handler
+    auth: [{{ monitor_user }}, {{ monitor_password}}]
+    service: check_{{ chain_name }}_regional
+    server: {{op5_server}}/api/command/PROCESS_SERVICE_CHECK_RESULT
+    host: {{ inventory_hostname }}
+loggers:
+  posttroll:
+    level: ERROR
+    propagate: false
+    handlers: [console, monitor]
 root:
   level: DEBUG
   handlers: [console, monitor]

setup.py

@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# Copyright (c) 2015, 2020.
+# Copyright (c) 2015 - 2022.
 #
 
 # Author(s):
@@ -42,6 +42,8 @@
     all_extras.extend(extra_deps)
 extras_require['all'] = list(set(all_extras))
 
+test_requires = ['pytroll_monitor'],
+
 setup(name="trollmoves",
       version=versioneer.get_version(),
       description='Pytroll file utilities',
@@ -66,13 +68,9 @@
       data_files=[],
       packages=['trollmoves'],
       zip_safe=False,
-      install_requires=[
-          'posttroll>=1.5.1',
-          'trollsift',
-          'netifaces',
-          'pyinotify',
-          'pyyaml',
-          'pyzmq',
-      ],
+      install_requires=['pyinotify', 'posttroll>=1.5.1',
+                        'trollsift', 'netifaces',
+                        'pyzmq', 'scp', 'pyyaml', 'watchdog'],
+      tests_require=test_requires,
       extras_require=extras_require,
       )

trollmoves/logger.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright (c) 2022 Pytroll Developers
+
+# Author(s):
+
+#   Adam Dybbroe <[email protected]>
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""The log handling.
+"""
+
+import os
+import logging
+import logging.config
+import logging.handlers
+import yaml
+import pyinotify
+
+LOG_FORMAT = "[%(asctime)s %(levelname)-8s] %(message)s"
+
+log_levels = {
+    0: logging.WARN,
+    1: logging.INFO,
+    2: logging.DEBUG,
+}
+
+
+class LoggerSetup():
+    """Setup logging."""
+
+    def __init__(self, cmd_args, logger=None):
+        """Init the logging setup class."""
+        self._cmd_args = cmd_args
+        self._file_handler = None
+        self._logger = logger
+
+    def setup_logging(self, chain_type=None):
+        """Set up logging."""
+        if self._cmd_args.log_config is not None:
+            with open(self._cmd_args.log_config) as fd_:
+                log_dict = yaml.safe_load(fd_.read())
+                logging.config.dictConfig(log_dict)
+            self._logger = logging.getLogger('')
+        else:
+            self._setup_default_logging(chain_type)
+
+    def _setup_default_logging(self, chain_type):
+        """Setup default logging without using a log-config file."""
+        self._logger = logging.getLogger('')
+        self._logger.setLevel(log_levels[self._cmd_args.verbosity])
+
+        if self._cmd_args.log:
+            self._file_handler = logging.handlers.TimedRotatingFileHandler(
+                os.path.join(self._cmd_args.log),
+                "midnight",
+                backupCount=7)
+        else:
+            self._file_handler = logging.StreamHandler()
+
+        formatter = logging.Formatter(LOG_FORMAT)
+        self._file_handler.setFormatter(formatter)
+
+        self._logger.addHandler(self._file_handler)
+        self._set_loggername(chain_type)
+
+    def _set_loggername(self, chain_type):
+        if not chain_type:
+            return
+
+        logger_name = "move_it_server"
+        if chain_type == "client":
+            logger_name = "move_it_client"
+        elif chain_type == "mirror":
+            logger_name = "move_it_mirror"
+        self._logger = logging.getLogger(logger_name)
+
+    def get_logger(self):
+        """Get the logger to use for logging."""
+        return self._logger
+
+    def init_pyinotify_logging(self):
+        """Initialize the pyinotify handler."""
+        pyinotify.log.handlers = [self._file_handler]

trollmoves/move_it_base.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
-# Copyright (c) 2012, 2013, 2014, 2015, 2016
+# Copyright (c) 2012, 2013, 2014, 2015, 2016, 2022
 #
 # Author(s):
 #
@@ -29,16 +29,16 @@
 
 import pyinotify
 from posttroll.publisher import Publisher
-
+from trollmoves.logger import LoggerSetup
 LOGGER = logging.getLogger("move_it_base")
-LOG_FORMAT = "[%(asctime)s %(levelname)-8s %(name)s] %(message)s"
 
 
 class MoveItBase(object):
     """Base class for Trollmoves."""
 
     def __init__(self, cmd_args, chain_type, publisher=None):
         """Initialize the class."""
+        global LOGGER
         self.cmd_args = cmd_args
         self.chain_type = chain_type
         self.running = False
@@ -47,7 +47,12 @@ def __init__(self, cmd_args, chain_type, publisher=None):
         self.publisher = publisher
         self._np = None
         self.chains = {}
-        setup_logging(cmd_args, chain_type)
+
+        logger = LoggerSetup(cmd_args, LOGGER)
+        logger.setup_logging(chain_type)
+        LOGGER = logger.get_logger()
+        logger.init_pyinotify_logging()
+
         LOGGER.info("Starting up.")
         self.setup_watchers(cmd_args)
 
@@ -107,34 +112,6 @@ def setup_watchers(self, cmd_args):
         self.watchman.add_watch(os.path.dirname(cmd_args.config_file), mask)
 
 
-def setup_logging(cmd_args, chain_type):
-    """Set up logging."""
-    global LOGGER
-    LOGGER = logging.getLogger('')
-    if cmd_args.verbose:
-        LOGGER.setLevel(logging.DEBUG)
-
-    if cmd_args.log:
-        fh_ = logging.handlers.TimedRotatingFileHandler(
-            os.path.join(cmd_args.log),
-            "midnight",
-            backupCount=7)
-    else:
-        fh_ = logging.StreamHandler()
-
-    formatter = logging.Formatter(LOG_FORMAT)
-    fh_.setFormatter(formatter)
-
-    LOGGER.addHandler(fh_)
-    logger_name = "move_it_server"
-    if chain_type == "client":
-        logger_name = "move_it_client"
-    elif chain_type == "mirror":
-        logger_name = "move_it_mirror"
-    LOGGER = logging.getLogger(logger_name)
-    pyinotify.log.handlers = [fh_]
-
-
 def create_publisher(port, publisher_name):
     """Create a publisher using port *port*."""
     LOGGER.info("Starting publisher on port %s.", str(port))