openstack_log_colorizer

#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Author: Gaël Lambert (gaelL) <gael.lambert@netwiki.fr>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import sys
import re
import argparse

PARSER = argparse.ArgumentParser()
PARSER.add_argument("-l", "--level",
            help="Set log level you want display",
            metavar='level',
            type=str)
PARSER.add_argument("-e", "--exclude",
            help="Set log level you want exclude",
            metavar='level',
            type=str,
            nargs='+')
PARSER.add_argument("-i", "--include",
            help="Set log level you only want display",
            metavar='level',
            type=str,
            nargs='+')
ARGS = PARSER.parse_args()

LOG_LEVEL = {
    'critical': 50,
    'emer': 50,
    'error': 40,
    'err': 40,
    'warning': 30,
    'warn': 30,
    'wrn': 30,
    'info': 20,
    'inf': 20,
    'debug': 10,
    'dbg': 10,
    'notset': 100,
}

class colors:
    grey = '\033[1;30m'
    red =  '\033[1;31m'
    green = '\033[1;32m'
    yellow = '\033[1;33m'
    blue = '\033[1;34m'
    magenta = '\033[1;35m'
    cyan = '\033[1;36m'
    white = '\033[0;37m'
    end = '\033[1;m'

def grey(text):
    return '%s%s%s' % (colors.grey, text, colors.end)

def red(text):
    return '%s%s%s' % (colors.red, text, colors.end)

def green(text):
    return '%s%s%s' % (colors.green, text, colors.end)

def yellow(text):
    return '%s%s%s' % (colors.yellow, text, colors.end)

def blue(text):
    return '%s%s%s' % (colors.blue, text, colors.end)

def magenta(text):
    return '%s%s%s' % (colors.magenta, text, colors.end)

def cyan(text):
    return '%s%s%s' % (colors.cyan, text, colors.end)

def white(text):
    return '%s%s%s' % (colors.white, text, colors.end)

def colored_level(level):
    "Return level text with selected color tag"
    level = level.upper()
    if level in ['TRACE', 'SEC']:
        return magenta(level)
    elif level in ['DEBUG', 'DBG', 'INFO', 'INF']:
        return green(level)
    elif level in ['WARNING', 'WRN', 'WARN', '???']:
        return yellow(level)
    elif level in ['CRITICAL', 'ERROR', 'ERR', 'EMER']:
        return red(level)
    else:
        return white(level)


def parse_line(line):
    "Parse line and return dict of each elements"
    # Line example : Openstack
    # 2014-08-14 18:43:58.950 4092 INFO neutron.plugins.openvswitch.agent.ovs_neutron_agent
    regex = (r'^([0-9]+-[0-9]+-[0-9]+) ' # date
             '([0-9]+:[0-9]+:[0-9]+\.[0-9]+) ' # time
             '([0-9]+):? ' # process
             '([A-Z]+) ' # level
             '([^ ]+) ' # name
             '(.+)') # text

    result = re.match(regex, line)
    if result is not None:
        return { 'date': result.group(1),
          'time': result.group(2),
          'process': result.group(3),
          'level': result.group(4),
          'name': result.group(5),
          'text': result.group(6),
          '_type': 'default',
        }

    # Line example : Openstack without date and time
    # 4092 INFO neutron.plugins.openvswitch.agent.ovs_neutron_agent
    regex = (r'([0-9]+):? ' # process
             '([A-Z]+) ' # level
             '([^ ]+) ' # name
             '(.+)') # text

    result = re.match(regex, line)
    if result is not None:
        return { 'process': result.group(1),
          'level': result.group(2),
          'name': result.group(3),
          'text': result.group(4),
          '_type': 'default_without_datatime',
        }

    # Line example : Ceph.log
    # 2014-11-17 18:27:11.731721 mon.0 10.142.63.18:6789/0 2005 : [INF]
    regex = (r'^([0-9]+-[0-9]+-[0-9]+) ' # date
             '([0-9]+:[0-9]+:[0-9]+\.[0-9]+) ' # time
             '([^\[]+) ' # who
             '\[([A-Z\?]+)\] ' # level
             '(.+)') # text

    result = re.match(regex, line)
    if result is not None:
        return { 'date': result.group(1),
          'time': result.group(2),
          'who': result.group(3),
          'level': result.group(4),
          'text': result.group(5),
          '_type': 'ceph.log',
        }

    # Line example : ovs.log
    #2014-11-21T06:25:09.549Z|00012|vlog|INFO|opened log file /var/log/op...
    regex = (r'^([0-9]+-[0-9]+-[0-9]+)T' # date
             '([0-9]+:[0-9]+:[0-9]+\.[0-9]+)Z\|' # time
             '([0-9]+)\|' # serial
             '([^\|]+)\|' # name
             '([^\|]+)\|' # level
             '(.+)') # text

    result = re.match(regex, line)
    if result is not None:
        return { 'date': result.group(1),
          'time': result.group(2),
          'serial': result.group(3),
          'name': result.group(4),
          'level': result.group(5),
          'text': result.group(6),
          '_type': 'ovs.log',
        }

    return {'text': line, '_type': None}


def colorize(line):
    "Apply color tag on line"

    # Default parsed line
    if line.get('_type') == 'default':
        if line.get('level') == 'TRACE':
            colored_text=grey(line['text'])
        else:
            colored_text=white(line['text'])
        return '%s %s %s: %s %s %s' % ( grey(line['date']),
                                   grey(line['time']),
                                   grey(line['process']),
                                   colored_level(line['level']),
                                   blue(line['name']),
                                   colored_text,)
    # Default parsed line without date and time
    elif line.get('_type') == 'default_without_datatime':
        if line.get('level') == 'TRACE':
            colored_text=grey(line['text'])
        else:
            colored_text=white(line['text'])
        return '%s: %s %s %s' % ( grey(line['process']),
                                 colored_level(line['level']),
                                 blue(line['name']),
                                 colored_text,)
    # Ceph.log
    elif line.get('_type') == 'ceph.log':
        return '%s %s %s [%s] %s' % ( grey(line['date']),
                                 grey(line['time']),
                                 blue(line['who']),
                                 colored_level(line['level']),
                                 white(line['text']),)
    # ovs.log
    elif line.get('_type') == 'ovs.log':
        return '%s %s %s %s %s %s' % ( grey(line['date']),
                                 grey(line['time']),
                                 grey(line['serial']),
                                 blue(line['name']),
                                 colored_level(line['level']),
                                 white(line['text']),)

    # Unparsed line (without level)
    else:
        return '%s' % (line.get('text'))

def check_args():
    # Just allow one arg
    num_args = sum(1 for i in [ARGS.level, ARGS.exclude, ARGS.include] if i)
    if num_args > 1:
        print 'Args conflicts select just one arg'
        PARSER.print_help()
        return False
    return True

def level_filter(line):
    "Return true if line must be filtered. Never filter line without level"
    level = ARGS.level.lower()
    line_level = line.get('level', 'notset').lower()
    if LOG_LEVEL.get(line_level, 100) < LOG_LEVEL.get(level, 0):
        return True
    else:
        return False

def include_filter(line):
    "Return true if line must be filtered. Never filter line without level"
    includes = [i.lower() for i in ARGS.include]
    line_level = line.get('level', 'notset').lower()
    if line_level == 'notset':
        return False
    elif line_level in includes:
        return False
    return True

def exclude_filter(line):
    "Return true if line must be filtered. Never filter line without level"
    excludes = [e.lower() for e in ARGS.exclude]
    line_level = line.get('level', 'notset').lower()
    if line_level in excludes:
        return True
    return False

def line_is_filtered(line):
    "Skip the line ?"

    if ARGS.level:
        return level_filter(line)
    elif ARGS.include:
        return include_filter(line)
    elif ARGS.exclude:
        return exclude_filter(line)
    return False


if __name__ == '__main__':
    # Read each line in stdin
    if not check_args():
        sys.exit(1)

    while 1:
        try:
            line = sys.stdin.readline()
        except KeyboardInterrupt:
            break

        if not line:
            break

        # get parsed line
        parsed_line = parse_line(line.rstrip('\n'))

        # Skip line if filtred (never skip line without log level)
        if line_is_filtered(parsed_line):
            continue

        # Print parsed and colored line
        print colorize(parsed_line)