From 27eb9787cd3279725b4813cdd92072a79f2ca71f Mon Sep 17 00:00:00 2001 From: Tom Close Date: Sat, 28 Sep 2024 14:22:31 +1000 Subject: [PATCH] cleaned logging and added discord option --- pyproject.toml | 1 + xnat_ingest/cli/stage.py | 72 +++++---------------- xnat_ingest/cli/upload.py | 69 +++++---------------- xnat_ingest/utils.py | 127 +++++++++++++------------------------- 4 files changed, 74 insertions(+), 195 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9c4566a..899cab0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,7 @@ readme = "README.md" requires-python = ">=3.8" dependencies = [ "click >=8.1", + "discord", "fileformats-medimage-extras", "pydicom >=2.3.1", "tqdm >=4.64.1", diff --git a/xnat_ingest/cli/stage.py b/xnat_ingest/cli/stage.py index 671651e..79eba75 100644 --- a/xnat_ingest/cli/stage.py +++ b/xnat_ingest/cli/stage.py @@ -13,9 +13,7 @@ from xnat_ingest.utils import ( AssociatedFiles, logger, - LogFile, - LogEmail, - MailServer, + LoggerConfig, XnatLogin, set_logger_handling, ) @@ -145,63 +143,29 @@ help="Whether to delete the session directories after they have been uploaded or not", ) @click.option( - "--log-level", - default="info", - type=str, - envvar="XINGEST_LOGLEVEL", - help=("The level of the logging printed to stdout"), -) -@click.option( - "--log-file", - "log_files", - default=None, - type=LogFile.cli_type, + "--logger", + "loggers", multiple=True, - nargs=2, - metavar=" ", - envvar="XINGEST_LOGFILE", - help=( - 'Location to write the output logs to, defaults to "upload-logs" in the ' - "export directory" - ), -) -@click.option( - "--log-email", - "log_emails", - type=LogEmail.cli_type, + type=LoggerConfig.cli_type, + envvar="XINGEST_LOGGERS", nargs=3, - metavar="
", - multiple=True, - envvar="XINGEST_LOGEMAIL", - help=( - "Email(s) to send logs to. When provided in an environment variable, " - "mail and log level are delimited by ',' and separate destinations by ';'" - ), + default=(), + metavar=" ", + help=("Setup handles to capture logs that are generated"), ) @click.option( - "--add-logger", + "--additional-logger", + "additional_loggers", type=str, multiple=True, default=(), - envvar="XINGEST_LOGGERS", + envvar="XINGEST_ADDITIONAL_LOGGERS", help=( "The loggers to use for logging. By default just the 'xnat-ingest' logger is used. " "But additional loggers can be included (e.g. 'xnat') can be " "specified here" ), ) -@click.option( - "--mail-server", - type=MailServer.cli_type, - nargs=4, - metavar=" ", - default=None, - envvar="XINGEST_MAILSERVER", - help=( - "the mail server to send logger emails to. When provided in an environment variable, " - "args are delimited by ';'" - ), -) @click.option( "--raise-errors/--dont-raise-errors", default=False, @@ -297,11 +261,8 @@ def stage( resource_field: str, project_id: str | None, delete: bool, - log_level: str, - log_files: ty.List[LogFile], - log_emails: ty.List[LogEmail], - add_logger: ty.List[str], - mail_server: MailServer, + loggers: ty.List[LoggerConfig], + additional_loggers: ty.List[str], raise_errors: bool, deidentify: bool, xnat_login: XnatLogin, @@ -315,11 +276,8 @@ def stage( work_dir: Path | None = None, ) -> None: set_logger_handling( - log_level=log_level, - log_emails=log_emails, - log_files=log_files, - mail_server=mail_server, - add_logger=add_logger, + logger_configs=loggers, + additional_loggers=additional_loggers, ) if xnat_login: diff --git a/xnat_ingest/cli/upload.py b/xnat_ingest/cli/upload.py index a03261c..e9b7a32 100644 --- a/xnat_ingest/cli/upload.py +++ b/xnat_ingest/cli/upload.py @@ -17,9 +17,7 @@ from xnat_ingest.resource import ImagingResource from xnat_ingest.utils import ( logger, - LogFile, - LogEmail, - MailServer, + LoggerConfig, set_logger_handling, StoreCredentials, ) @@ -55,41 +53,19 @@ @click.argument("user", type=str, envvar="XINGEST_USER") @click.option("--password", default=None, type=str, envvar="XINGEST_PASS") @click.option( - "--log-level", - default="info", - type=str, - envvar="XINGEST_LOGLEVEL", - help=("The level of the logging printed to stdout"), -) -@click.option( - "--log-file", - "log_files", - default=None, - type=LogFile.cli_type, - nargs=2, - metavar=" ", + "--logger", + "loggers", multiple=True, - envvar="XINGEST_LOGFILE", - help=( - 'Location to write the output logs to, defaults to "upload-logs" in the ' - "export directory" - ), -) -@click.option( - "--log-email", - "log_emails", - type=LogEmail.cli_type, + type=LoggerConfig.cli_type, + envvar="XINGEST_LOGGERS", nargs=3, - metavar="
", - multiple=True, - envvar="XINGEST_LOGEMAIL", - help=( - "Email(s) to send logs to. When provided in an environment variable, " - "mail and log level are delimited by ',' and separate destinations by ';'" - ), + default=(), + metavar=" ", + help=("Setup handles to capture logs that are generated"), ) @click.option( - "--add-logger", + "--additional-logger", + "additional_loggers", type=str, multiple=True, default=(), @@ -100,17 +76,6 @@ "specified here" ), ) -@click.option( - "--mail-server", - type=MailServer.cli_type, - metavar=" ", - default=None, - envvar="XINGEST_MAILSERVER", - help=( - "the mail server to send logger emails to. When provided in an environment variable, " - "args are delimited by ';'" - ), -) @click.option( "--always-include", "-i", @@ -215,12 +180,9 @@ def upload( server: str, user: str, password: str, - log_level: str, - log_files: ty.List[LogFile], - log_emails: ty.List[LogEmail], - mail_server: MailServer, + loggers: ty.List[LoggerConfig], + additional_loggers: ty.List[str], always_include: ty.Sequence[str], - add_logger: ty.List[str], raise_errors: bool, store_credentials: StoreCredentials, temp_dir: ty.Optional[Path], @@ -234,11 +196,8 @@ def upload( ) -> None: set_logger_handling( - log_level=log_level, - log_emails=log_emails, - log_files=log_files, - mail_server=mail_server, - add_logger=add_logger, + logger_configs=loggers, + additional_loggers=additional_loggers, ) # Set the directory to create temporary files/directories in away from system default diff --git a/xnat_ingest/utils.py b/xnat_ingest/utils.py index 061adf5..045c25a 100644 --- a/xnat_ingest/utils.py +++ b/xnat_ingest/utils.py @@ -8,6 +8,7 @@ import attrs import click.types import click.testing +import discord from fileformats.core import DataType, FileSet, from_mime @@ -80,40 +81,20 @@ def cli_type(cls) -> CliType: return CliType(cls, multiple=True) # type: ignore[arg-type] -@attrs.define -class LogEmail(CliTyped): - - address: str - loglevel: str - subject: str - - def __str__(self) -> str: - return self.address +def to_upper(value: str) -> str: + return value.upper() @attrs.define -class LogFile(MultiCliTyped): +class LoggerConfig(MultiCliTyped): - path: Path = attrs.field(converter=Path) + type: str loglevel: str + location: str - def __bool__(self) -> bool: - return bool(self.path) - - def __str__(self) -> str: - return str(self.path) - - def __fspath__(self) -> str: - return str(self.path) - - -@attrs.define -class MailServer(CliTyped): - - host: str - sender_email: str - user: str - password: str + @property + def loglevel_int(self) -> int: + return getattr(logging, self.loglevel.upper()) # type: ignore[no-any-return] @attrs.define @@ -140,71 +121,39 @@ class StoreCredentials(CliTyped): def set_logger_handling( - log_level: str, - log_emails: ty.List[LogEmail] | None, - log_files: ty.List[LogFile] | None, - mail_server: MailServer, - add_logger: ty.Sequence[str] = (), + logger_configs: ty.Sequence[LoggerConfig], + additional_loggers: ty.Sequence[str] = (), ) -> None: + """Set up logging for the application""" loggers = [logger] - for log in add_logger: + for log in additional_loggers: loggers.append(logging.getLogger(log)) - levels = [log_level] - if log_emails: - levels.extend(le.loglevel for le in log_emails) - if log_files: - levels.extend(lf.loglevel for lf in log_files) - - min_log_level = min(getattr(logging, ll.upper()) for ll in levels) + min_log_level = min(ll.loglevel_int for ll in logger_configs) for logr in loggers: logr.setLevel(min_log_level) - # Configure the email logger - if log_emails: - if not mail_server: - raise ValueError( - "Mail server needs to be provided, either by `--mail-server` option or " - "XNAT_INGEST_MAILSERVER environment variable if logger emails " - "are provided: " + ", ".join(str(le) for le in log_emails) - ) - for log_email in log_emails: - smtp_hdle = logging.handlers.SMTPHandler( - mailhost=mail_server.host, - fromaddr=mail_server.sender_email, - toaddrs=[log_email.address], - subject=log_email.subject, - credentials=(mail_server.user, mail_server.password), - secure=None, - ) - smtp_hdle.setLevel(getattr(logging, log_email.loglevel.upper())) - for logr in loggers: - logr.addHandler(smtp_hdle) - # Configure the file logger - if log_files: - for log_file in log_files: - log_file.path.parent.mkdir(exist_ok=True) - log_file_hdle = logging.FileHandler(log_file) - if log_file.loglevel: - log_file_hdle.setLevel(getattr(logging, log_file.loglevel.upper())) - log_file_hdle.setFormatter( - logging.Formatter( - "%(asctime)s - %(name)s - %(levelname)s - %(message)s" - ) - ) - for logr in loggers: - logr.addHandler(log_file_hdle) - - console_hdle = logging.StreamHandler(sys.stdout) - console_hdle.setLevel(getattr(logging, log_level.upper())) - console_hdle.setFormatter( - logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") - ) - for logr in loggers: - logr.addHandler(console_hdle) + for config in logger_configs: + log_handle: logging.Handler + if config.type == "file": + Path(config.location).parent.mkdir(exist_ok=True) + log_handle = logging.FileHandler(config.location) + elif config.type == "stream": + stream = sys.stdout if config.location == "stdout" else sys.stderr + log_handle = logging.StreamHandler(stream) + elif config.type == "discord": + log_handle = DiscordHandler(config.location) + else: + raise ValueError(f"Unknown logger type: {config.type}") + log_handle.setLevel(config.loglevel_int) + log_handle.setFormatter( + logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") + ) + for logr in loggers: + logr.addHandler(log_handle) def show_cli_trace(result: click.testing.Result) -> str: @@ -214,6 +163,18 @@ def show_cli_trace(result: click.testing.Result) -> str: return "".join(traceback.format_exception(exc_type, value=exc, tb=tb)) +class DiscordHandler(logging.Handler): + """A logging handler that sends log messages to a Discord webhook""" + + def __init__(self, webhook_url: str): + super().__init__() + self.webhook_url = webhook_url + self.client = discord.Webhook.from_url(webhook_url) + + def emit(self, record: logging.LogRecord) -> None: + self.client.send(record.msg) + + class RegexExtractor: """Helper callable for extracting a substring from a string with a predefined pattern"""