File based Python logging config

cartography/cli.py

@@ -1,6 +1,6 @@
 import argparse
 import getpass
-import logging
+import logging.config
 import os
 import sys
 from typing import Optional
@@ -111,6 +111,14 @@ def _build_parser(self):
                 'See https://neo4j.com/docs/api/python-driver/4.4/api.html#database.'
             ),
         )
+        parser.add_argument(
+            '--logging-config',
+            type=str,
+            default=None,
+            help=(
+                'Path to file containing Python logging configuration'
+            ),
+        )
         parser.add_argument(
             '--selected-modules',
             type=str,
@@ -575,6 +583,9 @@ def main(self, argv: str) -> int:
         # TODO support parameter lookup in environment variables if not present on command line
         config: argparse.Namespace = self.parser.parse_args(argv)
         # Logging config
+        if config.logging_config:
+            logging.config.fileConfig(config.logging_config)
+            logger.info("Using logging config from %s", config.logging_config)
         if config.verbose:
             logging.getLogger('cartography').setLevel(logging.DEBUG)
         elif config.quiet:
@@ -802,9 +813,5 @@ def main(argv=None):
     :rtype: int
     :return: The return code.
     """
-    logging.basicConfig(level=logging.INFO)
-    logging.getLogger('botocore').setLevel(logging.WARNING)
-    logging.getLogger('googleapiclient').setLevel(logging.WARNING)
-    logging.getLogger('neo4j').setLevel(logging.WARNING)
     argv = argv if argv is not None else sys.argv[1:]
     sys.exit(CLI(prog='cartography').main(argv))

cartography/config.py

@@ -19,6 +19,8 @@ class Config:
     :param neo4j_database: The name of the database in Neo4j to connect to. If not specified, uses your Neo4j database
     settings to infer which database is set to default.
     See https://neo4j.com/docs/api/python-driver/4.4/api.html#database. Optional.
+    :type logging_config: str
+    :param logging_config: Path to the file containing logging configuration.
     :type selected_modules: str
     :param selected_modules: Comma-separated list of cartography top-level modules to sync. Optional.
     :type update_tag: int
@@ -126,6 +128,7 @@ def __init__(
         neo4j_password=None,
         neo4j_max_connection_lifetime=None,
         neo4j_database=None,
+        logging_config=None,
         selected_modules=None,
         update_tag=None,
         aws_sync_all_profiles=False,
@@ -185,6 +188,7 @@ def __init__(
         self.neo4j_password = neo4j_password
         self.neo4j_max_connection_lifetime = neo4j_max_connection_lifetime
         self.neo4j_database = neo4j_database
+        self.logging_config = logging_config
         self.selected_modules = selected_modules
         self.update_tag = update_tag
         self.aws_sync_all_profiles = aws_sync_all_profiles

logging.cfg

@@ -0,0 +1,65 @@
+[logger_root]
+handlers=stdout,file
+level=NOTSET
+
+[logger_cartography.cli]
+qualname=cartography.cli
+handlers=stdout,file
+# stop duplicate log massage
+propagate=0
+level=DEBUG
+
+[logger_cartography.sync]
+qualname=cartography.sync
+handlers=stdout,file
+# stop duplicate log massage
+propagate=0
+
+[logger_botocore]
+qualname=botocore
+level=WARNING
+handlers=stdout,file
+
+[logger_googleapiclient]
+qualname=googleapiclient
+level=WARNING
+handlers=stdout,file
+
+[logger_neo4j]
+qualname=neo4j
+handlers=stdout,file
+level=WARNING
+
+### Loggers ###
+[loggers]
+keys=root,cartography.cli,cartography.sync,neo4j,botocore,googleapiclient
+
+
+### Formatters ###
+[formatters]
+keys=simple,detailed
+
+[formatter_simple]
+format=%(asctime)s - %(name)s - %(levelname)s - %(message)s
+
+[formatter_detailed]
+format=%(asctime)s - %(name)s - %(levelname)s - %(module)s : %(lineno)d - %(message)s
+
+
+### Handlers ###
+[handlers]
+keys=file,stdout
+
+[handler_file]
+class=handlers.TimedRotatingFileHandler
+interval=midnight
+backupCount=5
+formatter=detailed
+args=('logs/cartography.log',)
+level=DEBUG
+
+[handler_stdout]
+class=StreamHandler
+args=(sys.stdout,)
+formatter=simple
+level=INFO