From a6297dc351449a3c128f593278d034804509165e Mon Sep 17 00:00:00 2001 From: emphasize Date: Wed, 27 Dec 2023 23:14:41 +0100 Subject: [PATCH 1/2] rename var for clearity remove extra env var option readme docs add requirements revert env var situation make ovos-config optional add ovos-logs reduce command optics adjustments add `ovos-logs` console script --- ovos_utils/log.py | 98 ++++- ovos_utils/log_parser.py | 663 ++++++++++++++++++++++++++++++++++ readme.md | 79 +++- requirements/requirements.txt | 3 + setup.py | 7 +- 5 files changed, 840 insertions(+), 10 deletions(-) create mode 100644 ovos_utils/log_parser.py diff --git a/ovos_utils/log.py b/ovos_utils/log.py index 5f8fe83d..06f9dcb7 100644 --- a/ovos_utils/log.py +++ b/ovos_utils/log.py @@ -17,7 +17,21 @@ import sys from logging.handlers import RotatingFileHandler from os.path import join -from typing import List +from pathlib import Path +from typing import Optional, List, Set + + +ALL_SERVICES = {"bus", + "audio", + "skills", + "voice", + "gui", + "ovos", + "phal", + "phal-admin", + "hivemind", + "hivemind-voice-sat"} + class LOG: @@ -78,18 +92,17 @@ def __init__(cls, name='OVOS'): @classmethod def init(cls, config=None): - + from ovos_utils.xdg_utils import xdg_state_home try: from ovos_config.meta import get_xdg_base - default_base = get_xdg_base() + xdg_base = get_xdg_base() except ImportError: - default_base = os.environ.get("OVOS_CONFIG_BASE_FOLDER") or \ - "mycroft" - from ovos_utils.xdg_utils import xdg_state_home + xdg_base = os.environ.get("OVOS_CONFIG_BASE_FOLDER") or "mycroft" + xdg_path = os.path.join(xdg_state_home(), xdg_base) + config = config or {} - cls.base_path = config.get("path") or \ - f"{xdg_state_home()}/{default_base}" + cls.base_path = config.get("path") or xdg_path cls.max_bytes = config.get("max_bytes", 50000000) cls.backup_count = config.get("backup_count", 3) cls.level = config.get("level") or LOG.level @@ -278,3 +291,72 @@ def log_wrapper(*args, **kwargs): return log_wrapper return wrapped + + +def get_log_path(service: str, directories: Optional[List[str]] = None) \ + -> Optional[str]: + """ + Get the path to the log directory for a given service. + Default behaviour is to check the configured paths for the service. + If a list of directories is provided, check that list for the service log + + Args: + service: service name + directories: (optional) list of directories to check for service + + Returns: + path to log directory for service + """ + from ovos_utils.xdg_utils import xdg_state_home + try: + from ovos_config import Configuration + from ovos_config.meta import get_xdg_base + except ImportError: + xdg_base = os.environ.get("OVOS_CONFIG_BASE_FOLDER", "mycroft") + return os.path.join(xdg_state_home(), xdg_base) + + if directories: + for directory in directories: + file = os.path.join(directory, f"{service}.log") + if os.path.exists(file): + return directory + return None + + config = Configuration().get("logging", dict()).get("logs", dict()) + # service specific config or default config location + path = config.get(service, {}).get("path") or config.get("path") + # default xdg location + if not path: + path = os.path.join(xdg_state_home(), get_xdg_base()) + + return path + + +def get_log_paths() -> Set[str]: + """ + Get all log paths for all service logs + Different services may have different log paths + + Returns: + set of paths to log directories + """ + paths = set() + ALL_SERVICES.union({s.replace("-", "_") for s in ALL_SERVICES}) + for service in ALL_SERVICES: + paths.add(get_log_path(service)) + + return paths + +def get_available_logs(directories: Optional[List[str]] = None) -> List[str]: + """ + Get a list of all available log files + Args: + directories: (optional) list of directories to check for service + + Returns: + list of log files + """ + directories = directories or get_log_paths() + return [Path(f).stem for path in directories + for f in os.listdir(path) if Path(f).suffix == ".log"] + \ No newline at end of file diff --git a/ovos_utils/log_parser.py b/ovos_utils/log_parser.py new file mode 100644 index 00000000..c29979db --- /dev/null +++ b/ovos_utils/log_parser.py @@ -0,0 +1,663 @@ +import re +import os +from datetime import datetime +from traceback import FrameSummary +from dataclasses import dataclass +from typing import Any, Tuple, List, Generator, Dict, Union, Optional + +from dateutil.parser import parse +import rich_click as click +from rich.console import Console +from rich.style import Style +from rich.table import Table +import pydoc +from combo_lock import ComboLock + +try: + from ovos_config import Configuration + use24h = Configuration().get("time_format", "full") == "full" + date_format = Configuration().get("date_format", "DMY") +except ImportError: + use24h = True + date_format = "DMY" + +from ovos_utils.log import get_log_path, get_log_paths, get_available_logs + + +TIME_FORMAT = '%Y-%m-%d %H:%M:%S.%f' +LOGLOCK = ComboLock("ovos_logs_console_script") + + +@dataclass +class LogLine: + timestamp: datetime = None + source: str = "" + location: str = "" + level: str = "" + message: str = "" + + def __str__(self): + # sytsem messages etc. + if not all([self.source, self.location, self.level]): + return self.message + return f"{self.format_timestamp()} - {self.source} - {self.location} - {self.level} - {self.message}" + + def format_timestamp(self): + if self.timestamp: + return self.timestamp.strftime(TIME_FORMAT)[:-3] + return "" + + +# Traceback frame +class Frame(FrameSummary): + def __init__(self, filename, lineno, name, line): + super().__init__(filename, lineno, name, line=line) + + def as_dict(self): + return { + "location": self.format_location(), + "level": "TRACEBACK", + "message": self.line + } + + def as_logline(self): + return LogLine(**self.as_dict()) + + def format_location(self): + if "/bin/" in self.filename: + package = self.filename.split("/bin/")[-1].replace(".py", "")\ + .replace("-", "_").replace("/", ".") + elif "site-packages" not in self.filename and \ + (pyver := re.search(r"python\d\.\d+[\\/]", self.filename)): + package = self.filename.split(pyver.group())[-1].replace(".py", "")\ + .replace("-", "_").replace("/", ".") + else: + package = self.filename.split("site-packages/")[-1].replace(".py", "")\ + .replace("-", "_").replace("/", ".") + method = self.name.replace(".py", "").replace("-", "_") + return f"{package}:{method}:{self.lineno}" + + def __str__(self): + return f' File "{self.filename}", line {self.lineno}, in {self.name}\n {self.line}\n' + + +class Traceback: + PATTERN = r'File "(?P[^"]+)", line (?P\d+), in (?P\S+)\n\s*(?P.+)' + + def __init__(self, frames: List[Frame], exception: str, timestamp: datetime = None): + self.frames = frames + self.exception = exception + self._timestamp = timestamp + + @property + def exception_location(self): + return self.frames[-1].format_location() + + @property + def timestamp(self): + return self._timestamp + + @timestamp.setter + def timestamp(self, value): + self._timestamp = value + + def to_loglines(self) -> List[LogLine]: + + lines = [LogLine(timestamp=self.timestamp, + location=self.exception_location, + level="EXCEPTION", + message=self.exception)] + + for frame in self.frames: + lines.append(frame.as_logline()) + + return lines + + @classmethod + def from_list(cls, lines): + lines = [line if line.endswith("\n") else line + "\n" for line in lines] + multiline = "".join(lines) + return cls.from_string(multiline) + + @classmethod + def from_string(cls, s): + matches = re.findall(cls.PATTERN, s, re.MULTILINE) + frames = [] + for match in matches: + data = dict(zip(["filename", "lineno", "name", "line"], match)) + frames.append(Frame(**data)) + exception = next(line for line in s.split("\n")[::-1] if line) + return cls(frames, exception) + + def __str__(self): + multiline = "Traceback (most recent call last):\n" + for frame in self.frames: + multiline += str(frame) + multiline += f"{self.exception}\n" + return multiline + + +class OVOSLogParser: + LOG_PATTERN = r'(?P\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{1,6}) - (?P.+?) - (?P.+?) - (?P\w+) - (?P.*)' + + @classmethod + def parse(self, log_line, last_timestamp=None) -> LogLine: + log_line.rstrip("\n") + match = re.match(self.LOG_PATTERN, log_line) + data = {} + if match: + data = match.groupdict() + data['timestamp'] = datetime.strptime(data['timestamp'], TIME_FORMAT) + return LogLine(**data) + + data["timestamp"] = last_timestamp or "" + data["message"] = log_line + return LogLine(**data) + + @classmethod + def parse_file(self, source) -> Generator[Union[LogLine, Traceback], None, None]: + if not os.path.exists(source): + raise FileNotFoundError(f"File {source} does not exist") + + with open(source, 'r') as file: + trace = None + last_timestamp = None + for line in file: + # gather all lines of the traceback + if line == "Traceback (most recent call last):\n": + trace = [line] + continue + # TODO do tracebacks always end on a empty line? + elif trace and line == "\n": + trace.append(line) + traceback = Traceback.from_list(trace) + traceback.timestamp = last_timestamp + yield traceback + trace = None + elif trace: + trace.append(line) + else: + log = self.parse(line, last_timestamp) + if log.message == "\n": + continue + timestamp = log.timestamp + if timestamp: + last_timestamp = timestamp + yield log + + +console = Console() + +EXPECTED_DATE_FORMAT = "YYYY-MM-DD" if date_format == "YMD" else "DD-MM-YYYY" +EXPECTED_DATE = "2023-12-01" if date_format == "YMD" else "01-12-2023" +EXPECTED_DATETIME_FORMAT = f"[{EXPECTED_DATE_FORMAT}] HH:MM[:SS] {'AM/PM' if not use24h else ''}" +EXPECTED_TIME = f"09:00:05 {'PM' if not use24h else ''}" + +LOGSOPTHELP = """logs to be sliced +\nmultiple: -l bus -l audio""" +STARTTIMEHELP = f"""start time of the log slice (default: since service restart, input format: {EXPECTED_DATETIME_FORMAT.strip()}) +\n Example: -s \"{EXPECTED_DATE} 12:00{' AM/PM' if not use24h else ''}\" / -s 12:00:05{' AM/PM' if not use24h else ''}""" + +click.rich_click.STYLE_ARGUMENT = "dark_red" +click.rich_click.STYLE_OPTION = "dark_red" +click.rich_click.STYLE_SWITCH = "indian_red" +click.rich_click.USE_MARKDOWN = True +click.rich_click.COMMAND_GROUPS = { + "ovos-logs": [ + { + "name": "Slice logs by time", + "commands": ["slice"], + "table_styles": { + "row_styles": ["white"], + "padding": (0, 2), + "title_justify": "left" + }, + }, + { + "name": "List logs by severity", + "commands": ["list"], + "table_styles": { + "row_styles": ["white"], + "padding": (0, 2), + }, + }, + { + "name": "Downsize logs", + "commands": ["reduce"], + "table_styles": { + "row_styles": ["white"], + "padding": (0, 1), + }, + }, + { + "name": "Show logs (using less)", + "commands": ["show"], + "table_styles": { + "row_styles": ["white"], + "padding": (0, 2), + }, + } + ] +} + + +def get_last_load_time(directories: Optional[List[str]] = None) -> Optional[datetime]: + # if nothing's found return the beginning of unix time + last_timestamp = datetime.fromtimestamp(0) + if directories is None: + directory = get_log_path("skills") + else: + directory = get_log_path("skills", directories) + + if directory: + with open(os.path.join(directory,"skills.log"), "r") as f: + for line in f.readlines()[::-1]: + logline = OVOSLogParser.parse(line) + if logline.timestamp: + last_timestamp = logline.timestamp + if logline.message == "Loading message bus configs": + break + return last_timestamp + + +def valid_log(logs, paths): + for log in logs: + if log.lower() not in get_available_logs(paths): + return False + return True + + +def parse_time(time_str): + try: + time = parse(time_str) + except ValueError: + return None + return time + + +def get_timestamped_filename(basename: str, ext: str, basedir = "~", timeformat = '%Y%m%d_%H%M%S'): + if basedir == "~": + basedir = os.path.expanduser("~") + + t = datetime.now().strftime(timeformat) + return os.path.join(basedir, f"{basename}_{t}.{ext}") + + +def parse_timeframe(start, end, directories: Optional[List[str]] = None) -> Tuple[Any, Any]: + """ + Parses the start and end time given a string input. + If the start is None, parse the skill log to determine the last service load time and + if that fails return the beginning starting datetime of the log. + If the end is None, return the current datetime. + + :param start: start time of the log slice (default: since service restart) + :param end: end time of the log slice (default: now) + :param directories: the directory logs reside in + :return: start and end time + """ + if start is None: + start = get_last_load_time(directories) + else: + start = parse_time(start) + + if end is None: + end = datetime.now() + else: + end = parse_time(end) + return start, end + + +@click.group() +def ovos_logs(): + """\b + Small helper tool to quickly navigate the logs, create slices and quickview errors + + `ovos-logs [COMMAND] --help` for further information about the specific command ARGUMENTS + \b + """ + pass + + +@ovos_logs.command() +@click.option("--start", "-s", help=STARTTIMEHELP) +@click.option("--until", "-u", help=f"end time of the log slice [default: now]") +@click.option("--logs", "-l", multiple=True, default=get_available_logs(), help=LOGSOPTHELP, show_default=True) +@click.option("--paths", "-p", multiple=True, default=get_log_paths(), help=f"the directory logs reside in", show_default=True) +@click.option("--file", "-f", is_flag=False, flag_value=get_timestamped_filename("slice", "log"), + default=None, help=f"output as file (if flagged, but not specified: {get_timestamped_filename('slice', 'log')})") +def slice(start, until, logs, paths, file): + """\b + Optionally define start (`-s`) and the time until (`-u`) the slice should be limited to. + \b + Different logs can be included using the `-l` option. If not specified, all logs will be included. + Optionally the directory where the logs are stored (`-p`) and the file where the slices should be dumped (`-f`) + can be specified. + \b + > Examples: + > ovos-logs slice # Slice all logs from service start up until now + > ovos-logs slice -s 01-12-2023 -u 01-12-2023 17:00:20 # Slice all logs from the start of december the first until 17:00:20 + > ovos-logs slice -l bus -l skills -f ~/myslice.log # Slice skills.log and bus.log from service start up until now and dump it to the file ~/myslice.log + """ + logs_present = [] + + if not all(os.path.exists(path) for path in paths): + return console.print(f"Directory [{[p for p in paths if not os.path.exists(p)]}] does not exist") + else: + logs_present = get_available_logs(paths) + + start, end = parse_timeframe(start, until, paths) + if end is None or start is None: + return console.print(f"Need a valid end time in the format ") + elif start > end: + return console.print(f"Start time [{start}] is after end time [{end}]") + + if not logs: + logs = logs_present + elif not valid_log(logs, paths): + return console.print(f"Invalid log name, valid logs are {logs_present}") + + _templog: Dict[str, List[LogLine]] = dict() + + for service in logs: + path = get_log_path(service, paths) + logfile = os.path.join(path, f"{service}.log") + if not os.path.exists(logfile): + continue + _templog[service] = [] + for log in OVOSLogParser.parse_file(logfile): + if start <= log.timestamp < end: + if isinstance(log, Traceback): + _templog[service].extend(log.to_loglines()) + else: + _templog[service].append(log) + if not _templog[service]: + del _templog[service] + + if not _templog: + return console.print("No logs found in the specified time frame") + + if file is not None: + # test if file is writable + try: + with open(file, 'w') as f: + pass + except: + return console.print(f"File [{file}] is not writable. Aborted") + else: + console.print(f"Log slice saved to [bold]{file}[/bold]") + + for service in _templog: + table = Table(title=service) + table.add_column("Time", style="cyan", no_wrap=True) + table.add_column() + table.add_column("Message", style="magenta") + table.add_column("Origin", style="green") + lineno = 0 + for logline in _templog[service]: + lineno += 1 + style = None + timestamp = logline.timestamp or "" + if isinstance(timestamp, datetime): + timestamp = timestamp.strftime("%H:%M:%S.%f" if use24h else "%I:%M:%S.%f")[:-3] + if not use24h: + timestamp += logline.timestamp.strftime(" %p") + + level = logline.level or "" + message = logline.message or "" + if level == "ERROR": + level = "[bold red]" + level[:1] + elif level == "EXCEPTION": + level = "[bold red]" + level[:3] + elif level == "WARNING": + level = "[bold yellow]" + level[:1] + elif level == "DEBUG": + level = "[bold blue]" + level[:1] + elif level == "TRACEBACK": + level = "[white]" + level[:5] + message = "[grey42]" + message + elif level == "INFO": + level = "" + message = "[navajo_white1]" + message + if lineno % 2 == 0: + style = Style(bgcolor="grey7") + table.add_row( + timestamp, + level, + message, + logline.location or "", + style=style + ) + if len(logline.message) > 200: + table.add_row() + + console.print(table) + if file: + Console(file=open(file, 'a')).print(table) + + +@ovos_logs.command() +@click.option("--error", "-e", is_flag=True, help="display error messages") +@click.option("--warning", "-w", is_flag=True, help="display warning messages") +@click.option("--exception", "-x", is_flag=True, help="display exceptions") +@click.option("--debug", "-d", is_flag=True, help="display debug messages") +@click.option("--start", "-s", help=STARTTIMEHELP) +@click.option("--until", "-u", help=f"end time of the log slice [default: now]") +@click.option("--logs", "-l", multiple=True, default=get_available_logs(), help=LOGSOPTHELP, show_default=True) +@click.option("--paths", "-p", multiple=True, type=click.Path(), default=get_log_paths(), help=f"the directory logs reside in", show_default=True) +@click.option("--file", "-f", is_flag=False, type=click.Path(), flag_value=get_timestamped_filename("list", "log"), default=None, + help=f"output as file (if flagged, but not specified: {get_timestamped_filename('list', 'log')})") +def list(error, warning, exception, debug, start, until, logs, paths, file): + """\b + Log level has to be specified. + \b + Optionally define start (`-s`) and the time until (`-u`) the slice should be limited to. + \b + Different logs can be included using the `-l` option. If not specified, all logs will be included. + \b + Optionally the directory where the logs are stored (`-p`) and the file where the slices should be dumped (`-f`) + can be specified. + \b + > Examples: + > ovos-logs list -x # List all exceptions from service start up until now + > ovos-logs list -e -w -s 01-12-2023 -u 01-12-2023 17:00:20 # List all errors and warnings from the start of december the first until 17:00:20 + > ovos-logs list -x -l bus -l skills -f # List all exceptions from skills.log and bus.log and dump it to the file ~/list_xxx_xxx.log + """ + if not any([error, warning, debug, exception]): + return console.print("Need at least one of --error, --warning, --exception or --debug") + else: + log_levels = [lv_str for lv, lv_str in [(error, "ERROR"), (warning, "WARNING"), + (debug, "DEBUG"), (exception, "EXCEPTION")] if lv] + + if not all(os.path.exists(path) for path in paths): + return console.print(f"Directory [{[p for p in paths if not os.path.exists(p)]}] does not exist") + else: + logs_present = get_available_logs(paths) + + start, end = parse_timeframe(start, until, paths) + if end is None or start is None: + return console.print(f"Need a valid end time in the format {EXPECTED_DATETIME_FORMAT}") + elif start > end: + return console.print(f"Start time [{start}] is after end time [{end}]") + + if not logs: + logs = logs_present + elif not valid_log(logs, paths): + return console.print(f"Invalid log name, valid logs are {logs_present}") + + _templog: Dict[str, List[LogLine]] = dict() + + for service in logs: + path = get_log_path(service, paths) + if path is None: + continue + logfile = os.path.join(path, f"{service}.log") + _templog[service] = [] + for log in OVOSLogParser.parse_file(logfile): + if isinstance(log, Traceback): + if exception: + _templog[service].extend(log.to_loglines()) + continue + # LOG.exception + if exception and log.level == "EXCEPTION": + _templog[service].append(log) + if error and log.level == "ERROR": + _templog[service].append(log) + if warning and log.level == "WARNING": + _templog[service].append(log) + if debug and log.level == "DEBUG": + _templog[service].append(log) + if not _templog[service]: + del _templog[service] + + if not _templog: + return console.print("No logs found for the specified log level") + + if file is not None: + # test if file is writable + try: + with open(file, 'w') as f: + pass + except: + return console.print(f"File [{file}] is not writable. Aborted") + else: + console.print(f"Log list saved to [bold]{file}[/bold]") + + for service in _templog: + table = Table(title=f"{service} ({','.join(log_levels)})") + # for traceback indication + table.add_column("Time", style="cyan", no_wrap=True) + if exception or len(log_levels) > 1: + table.add_column() + table.add_column("Message", style="magenta") + table.add_column("Origin", style="green") + lineno = 0 + for log in _templog[service]: + style = None + lineno += 1 + timestamp = log.timestamp or "" + if timestamp: + timestamp = timestamp.strftime("%H:%M:%S.%f" if use24h else "%I:%M:%S.%f")[:-3] + if not use24h and timestamp: + timestamp += log.timestamp.strftime(" %p") + level = log.level.upper() + message = log.message.rstrip("\n") + if level == "ERROR": + level = "[bold red]" + level[:1] + elif level == "EXCEPTION": + level = "[bold red]" + level[:3] + elif level == "WARNING": + level = "[bold yellow]" + level[:1] + elif level == "DEBUG": + level = "[bold blue]" + level[:1] + elif level == "TRACEBACK": + level = "[white]" + level[:5] + message = "[grey42]" + message + elif level == "INFO": + level = "" + message = "[navajo_white1]" + message + + if lineno % 2 == 0: + style = Style(bgcolor="grey7") + row = [timestamp, level, message, log.location] + if not exception and len(log_levels) < 2: + row.pop(1) + table.add_row(*row, style=style) + if len(log.message) > 200: + table.add_row() + + console.print(table) + if file: + Console(file=open(file, 'a')).print(table) + + +@ovos_logs.command() +@click.option("--log", "-l", required=True, type=click.Choice(get_available_logs(), case_sensitive=False), help=f"log to show; available: {get_available_logs()}") +@click.option("--paths", "-p", multiple=True, type=click.Path(), default=get_log_paths(), help=f"the directory logs reside in", show_default=True) +def show(log, paths): + """\b + A service log has to be specified (`-l`). + \b + Optionally the directory where the logs are stored (`-p`) can be specified. + \b + > Examples: + > ovos-logs show -l skills # Display skills.log + > ovos-logs show -l debug -p ~/custom_path/ # Display debug.log from a custom path + """ + if not any(os.path.exists(os.path.join(path, f"{log}.log")) for path in paths): + return console.print(f"File does not exist") + else: + log = os.path.join(get_log_path(log, paths), f"{log}.log") + + pydoc.pager(open(log).read()) + + +@ovos_logs.command() +@click.option("--size", "-s", is_flag=False, flag_value=None, default=0, help="truncate logs to a given size (in bytes)") +@click.option("--date", "-d", help="truncate logs to a given date") +@click.option("--logs", "-l", multiple=True, default=get_available_logs(), help=LOGSOPTHELP, show_default=True) +@click.option("--paths", "-p", multiple=True, type=click.Path(), default=get_log_paths(), help=f"the directory logs reside in", show_default=True) +def reduce(size, date, logs, paths): + """\b + Reduce logs to a given size (in bytes) or remove entries before a given date. + \b + Different logs can be included using the `-l` option. If not specified, all logs will be included. + Optionally the directory where the logs are stored (`-p`) can be specified. + \b + > Examples: + > ovos-logs reduce # Reduce all logs to 0 bytes + > ovos-logs reduce -s 1000000 # Reduce all logs to ~1MB (latest logs) + > ovos-logs reduce -d "1-12-2023 17:00" # Reduce all logs to entries after the specified date/time + > ovos-logs reduce -s 1000000 -l skills -l bus # Reduce skills.log and bus.log to ~1MB (latest logs) + """ + + if date: + size = None + date = parse_time(date) + if date is None: + return console.print(f"The date/time provided couldn't be parsed. Expected format: {EXPECTED_DATETIME_FORMAT}") + + if not all(os.path.exists(path) for path in paths): + return console.print(f"Directory [{[p for p in paths if not os.path.exists(p)]}] does not exist") + else: + logs_present = get_available_logs(paths) + + if not logs: + logs = logs_present + elif not valid_log(logs, paths): + return console.print(f"Invalid log name, valid logs are {logs_present}") + + for service in logs: + path = get_log_path(service, paths) + logfile = os.path.join(path, f"{service}.log") + reduced = False + with LOGLOCK: + if size: + with open(logfile, 'r') as f: + f.seek(0, os.SEEK_END) + fullsize = f.tell() + f.seek(max(fullsize - size, 0)) + # skip cutoff line + f.readline() + remaining_lines = f.readlines() + if fullsize > size and remaining_lines: + reduced = True + with open(logfile, 'w') as f: + f.writelines(remaining_lines) + elif date: + loglines = [] + for log in OVOSLogParser.parse_file(logfile): + if log.timestamp and log.timestamp < date: + reduced = True + continue + loglines.append(log) + if reduced: + with open(logfile, 'w') as f: + for log in loglines: + f.write(str(log) + "\n") + else: + reduced = True + with open(logfile, 'w') as f: + f.write("") + + if reduced: + console.print(f"{service} log reduced") diff --git a/readme.md b/readme.md index fe8103d7..7dd896e6 100644 --- a/readme.md +++ b/readme.md @@ -1,4 +1,4 @@ -# OVOS - utils +# OVOS-utils collection of simple utilities for use across the mycroft ecosystem @@ -8,3 +8,80 @@ collection of simple utilities for use across the mycroft ecosystem pip install ovos_utils ``` +## Commandline scripts +### ovos-logs + Small helper tool to quickly navigate the logs, create slices and quickview errors + +--------------- +- **ovos-logs slice [options]** + + **Slice logs of a given time period. Defaults on the last service start (`-s`) until now (`-u`)** + + _Different logs can be picked using the `-l` option. All logs will be included if not specified._ + _Optionally the directory where the logs are stored (`-p`) and the file where the slices should be dumped (`-f`) can be specified._ + + + _[ex: `ovos-logs slice`]_ + _Slice all logs from service start up until now._ + + _[ex: `ovos-logs slice -s 17:05:20 -u 17:05:25`]_ + _Slice all logs from 17:05:20 until 17:05:25._ + _**no logs in that timeframe in other present logs_ + Screenshot 2023-12-25 185004 + + _[ex: `ovos-logs slice -s 17:05:20 -u 17:05:25 -l skills`]_ + _Slice skills.log from 17:05:20 until 17:05:25._ + + _[ex: `ovos-logs slice -s 17:05:20 -u 17:05:25 -f ~/testslice.log`]_ + _Slice the logs from 17:05:20 until 17:05:25 on all log files and dump the slices in the file ~/testslice.log (default: `~/slice_.log`)._ + Screenshot 2023-12-25 190732 +-------------- + +- **ovos-logs list [-e|-w|-d|-x] [options]** + + **List logs by severity (error/warning/debug/exception). A log level has to be specified - more than one can be listed** + + _A start and end date can be specified using the `-s` and `-u` options. Defaults to the last service start until now._ + _Different logs can be picked using the `-l` option. All logs will be included if not specified._ + _Optionally, the directory where the logs are stored (`-p`) and the file where the slices should be dumped (`-f`) can be passed as arguments._ + + _[ex: `ovos-logs list -x`]_ + _List the logs with level EXCEPTION (plus tracebacks) from the last service start until now._ + Screenshot 2023-12-25 184321 + + _[ex: `ovos-logs list -w -e -s 20-12-2023 -l bus -l skills`]_ + _List the logs with level WARNING and ERROR from the 20th of December 2023 until now from the logs bus.log and skills.log._ + Screenshot 2023-12-25 173739 +--------------------- + +- **ovos-logs reduce [options]** + + **Downsize logs to a given size (in bytes) or remove entries before a given date.** + + _Different logs can be included using the `-l` option. If not specified, all logs will be included._ + _Optionally the directory where the logs are stored (`-p`) can be specified._ + + _[ex: `ovos-logs reduce`]_ + _Downsize all logs to 0 bytes_ + + _[ex: `ovos-logs reduce -s 1000000`]_ + _Downsize all logs to ~1MB (latest logs)_ + + _[ex: `ovos-logs reduce -d "1-12-2023 17:00"`]_ + _Downsize all logs to entries after the specified date/time_ + + _[ex: `ovos-logs reduce -s 1000000 -l skills -l bus`]_ + _Downsize skills.log and bus.log to ~1MB (latest logs)_ + +--------------------- + +- **ovos-logs show -l [servicelog]** + + **Show logs** + + _[ex: `ovos-logs show -l bus`]_ + _Show the logs from bus.log._ + + _[ex: wrong servicelog]_ + _**logs shown depending on the logs present in the folder_ + diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 0c05ebaf..95f6e3e5 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -4,3 +4,6 @@ json_database~=0.7 kthread~=0.2 watchdog pyee +combo-lock~=0.2 +rich-click~=1.7 +rich~=13.7 diff --git a/setup.py b/setup.py index 4b2cbaec..97e29527 100644 --- a/setup.py +++ b/setup.py @@ -67,5 +67,10 @@ def required(requirements_file): license='Apache', author='jarbasAI', author_email='jarbasai@mailfence.com', - description='collection of simple utilities for use across the mycroft ecosystem' + description='collection of simple utilities for use across the mycroft ecosystem', + entry_points={ + 'console_scripts': [ + 'ovos-logs=ovos_utils.log_parser:ovos_logs' + ] + } ) From 80f6dc6523794216e139965ea86b84afe04aac22 Mon Sep 17 00:00:00 2001 From: JarbasAi Date: Sat, 30 Dec 2023 18:11:42 +0000 Subject: [PATCH 2/2] reorder to account for directories kwarg --- ovos_utils/log.py | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/ovos_utils/log.py b/ovos_utils/log.py index 06f9dcb7..86c30048 100644 --- a/ovos_utils/log.py +++ b/ovos_utils/log.py @@ -20,7 +20,6 @@ from pathlib import Path from typing import Optional, List, Set - ALL_SERVICES = {"bus", "audio", "skills", @@ -33,7 +32,6 @@ "hivemind-voice-sat"} - class LOG: """ Custom logger class that acts like logging.Logger @@ -100,7 +98,7 @@ def init(cls, config=None): xdg_base = os.environ.get("OVOS_CONFIG_BASE_FOLDER") or "mycroft" xdg_path = os.path.join(xdg_state_home(), xdg_base) - + config = config or {} cls.base_path = config.get("path") or xdg_path cls.max_bytes = config.get("max_bytes", 50000000) @@ -242,7 +240,6 @@ def log_deprecation(log_message: str = "DEPRECATED", determination. i.e. an internal exception handling method should log the first call external to that package """ - import inspect stack = inspect.stack()[1:] # [0] is this method call_info = "Unknown Origin" origin_module = func_module @@ -294,7 +291,7 @@ def log_wrapper(*args, **kwargs): def get_log_path(service: str, directories: Optional[List[str]] = None) \ - -> Optional[str]: + -> Optional[str]: """ Get the path to the log directory for a given service. Default behaviour is to check the configured paths for the service. @@ -307,6 +304,13 @@ def get_log_path(service: str, directories: Optional[List[str]] = None) \ Returns: path to log directory for service """ + if directories: + for directory in directories: + file = os.path.join(directory, f"{service}.log") + if os.path.exists(file): + return directory + return None + from ovos_utils.xdg_utils import xdg_state_home try: from ovos_config import Configuration @@ -315,20 +319,13 @@ def get_log_path(service: str, directories: Optional[List[str]] = None) \ xdg_base = os.environ.get("OVOS_CONFIG_BASE_FOLDER", "mycroft") return os.path.join(xdg_state_home(), xdg_base) - if directories: - for directory in directories: - file = os.path.join(directory, f"{service}.log") - if os.path.exists(file): - return directory - return None - config = Configuration().get("logging", dict()).get("logs", dict()) # service specific config or default config location path = config.get(service, {}).get("path") or config.get("path") # default xdg location if not path: path = os.path.join(xdg_state_home(), get_xdg_base()) - + return path @@ -344,9 +341,10 @@ def get_log_paths() -> Set[str]: ALL_SERVICES.union({s.replace("-", "_") for s in ALL_SERVICES}) for service in ALL_SERVICES: paths.add(get_log_path(service)) - + return paths + def get_available_logs(directories: Optional[List[str]] = None) -> List[str]: """ Get a list of all available log files @@ -359,4 +357,3 @@ def get_available_logs(directories: Optional[List[str]] = None) -> List[str]: directories = directories or get_log_paths() return [Path(f).stem for path in directories for f in os.listdir(path) if Path(f).suffix == ".log"] - \ No newline at end of file