camomilla

#!/usr/bin/env python3

# Copyright (c) 2016-2017 Vittorio Romeo
# License: MIT License
# MIT License page: https://opensource.org/licenses/MIT
# http://vittorioromeo.info | vittorio.romeo@outlook.com

import os
import sys
import re
import argparse
import json
import tempfile
import time



#
#
#
# -----------------------------------------------------------------------------------------------------------
# Configuration keys
# -----------------------------------------------------------------------------------------------------------

# Command-line argument name constants
akEnableTemplateCollapsing = 'template-collapsing'
akEnableNamespaceReplacements = 'namespace-replacements'
akEnableGenericReplacements = 'generic-replacements'
akEnableTempCache = 'temp-cache'
akReprocess = 'reprocess'
akReprocessPrevConfig = 'reprocess-prev-config'
akEnableProcessByLine = 'process-by-line'

# Configuration dictionary key constants
ckEnableTemplateCollapsing = 'enableTemplateCollapsing'
ckEnableNamespaceReplacements = 'enableNamespaceReplacements'
ckEnableGenericReplacements = 'enableGenericReplacements'
ckEnableTempCache = 'enableTempCache'
ckTemplateCollapsingDepth = 'templateCollapsingDepth'
ckConfigPaths = 'configPaths'
ckNamespaceReplacements = 'namespaceReplacements'
ckGenericReplacements = 'genericReplacements'
ckCompilationCommand = 'compilationCommand'
ckReprocess = 'reprocess'
ckReprocessPrevConfig = 'reprocessPrevConfig'
ckEnableProcessByLine = 'enableProcessByLine'



#
#
#
# -----------------------------------------------------------------------------------------------------------
# Cache
# -----------------------------------------------------------------------------------------------------------

fkCacheConfigFilePath = tempfile.gettempdir() + "/lastCamomillaConfig.json"
fkCacheFilePath = tempfile.gettempdir() + "/lastCamomilla.out"

def writeTempConfigCache(contents):
    with open(fkCacheConfigFilePath, "w+") as f:
        f.write(contents)

def writeTempCache():
    return open(fkCacheFilePath, "w+")

def readTempCache():
    return open(fkCacheFilePath)

def closeTempCache(f):
    f.close()


#
#
#
# -----------------------------------------------------------------------------------------------------------
# Arg parsing
# -----------------------------------------------------------------------------------------------------------

def addFeatureOpt(ap, longFlagName, key, defaultValue, helpStr):
    ap_feature = ap.add_mutually_exclusive_group(required=False)
    ap_feature.add_argument('--' + longFlagName, dest=key, action='store_true', help="| Control " + helpStr)
    ap_feature.add_argument('--no-' + longFlagName, dest=key, action='store_false', help="'")
    ap.set_defaults(**{key:defaultValue})

def addFeatureOptS(ap, shortFlagName, longFlagName, key, defaultValue, helpStr):
    ap_feature = ap.add_mutually_exclusive_group(required=False)
    ap_feature.add_argument(shortFlagName, '--' + longFlagName, dest=key, action='store_true', help="| Control " + helpStr)
    ap_feature.add_argument('--no-' + longFlagName, dest=key, action='store_false', help="'")
    ap.set_defaults(**{key:defaultValue})

def argDefault(noDefaults, x):
    return None if noDefaults else x

def makeArgParser(noDefaults, parseReprocess):
    # `None` if `noDefaults`, otherwise `x`
    def ad(x):
        return argDefault(noDefaults, x)

    ap = argparse.ArgumentParser(\
        formatter_class=lambda prog: argparse.HelpFormatter(prog,max_help_position=256))

    addFeatureOpt(ap, akEnableTemplateCollapsing, ckEnableTemplateCollapsing, ad(True), \
        "template collapsing")

    addFeatureOpt(ap, akEnableNamespaceReplacements, ckEnableNamespaceReplacements, ad(True), \
        "namespace replacements")

    addFeatureOpt(ap, akEnableGenericReplacements, ckEnableGenericReplacements, ad(True), \
        "generic replacements")

    addFeatureOpt(ap, akEnableProcessByLine, ckEnableProcessByLine, ad(True), \
        "process by line")

    if parseReprocess:
        addFeatureOpt(ap, akEnableTempCache, ckEnableTempCache, ad(True), \
            "temp cache")

        addFeatureOptS(ap, '-r', akReprocess, ckReprocess, ad(False), \
            "reprocess previous source")

        addFeatureOpt(ap, akReprocessPrevConfig, ckReprocessPrevConfig, ad(True), \
            "reprocess with previous configuration")

    ap.add_argument('-d', '--depth', \
        help="Template collapsing depth", \
        type=int, default=ad(1), \
        dest=ckTemplateCollapsingDepth, metavar='X')

    ap.add_argument('-c', '--config', \
        help="Configuration file path(s)", \
        dest=ckConfigPaths, default=[], metavar='P', \
        action="append")

    # TODO:
    #
    # ap.add_argument('-x', '--exec', \
    #     help="Execute compilation command", \
    #     type=str, default="", nargs=argparse.REMAINDER, dest=ckCompilationCommand, metavar='X')

    return ap



#
#
#
# -----------------------------------------------------------------------------------------------------------
# Argument propagation
# -----------------------------------------------------------------------------------------------------------

def canPropagate(key, origin):
    return key in origin.keys() and origin[key] != None

def propagateArgOverride(key, origin, target):
    if canPropagate(key, origin):
        target[key] = origin[key]

def propagateReplacements(key, origin, target):
    if canPropagate(key, origin):
        for k, v in origin[key].items():
            target[key][k] = v

def propagateConfigPaths(key, origin, target):
    if canPropagate(key, origin):
        for x in origin[key]:
            target[key].add(x)

def propagateEverything(origin, target):
    propagateArgOverride(ckEnableNamespaceReplacements, origin, target)
    propagateArgOverride(ckEnableGenericReplacements, origin, target)
    propagateArgOverride(ckEnableTemplateCollapsing, origin, target)
    propagateArgOverride(ckEnableProcessByLine, origin, target)

    # TODO: ?
    # propagateArgOverride(ckEnableTempCache, origin, target)

    propagateArgOverride(ckTemplateCollapsingDepth, origin, target)

    propagateReplacements(ckNamespaceReplacements, origin, target)
    propagateReplacements(ckGenericReplacements, origin, target)



#
#
#
# -----------------------------------------------------------------------------------------------------------
# Driver
# -----------------------------------------------------------------------------------------------------------

class Camomilla:
    def __init__(self, conf):
        self._conf = conf
        self._compiledNamespaceRegex = None
        self._compiledGenericRegex = None

        self._initRegexes()

    # Configuration getters
    def namespaceReplacements(self):        return self._conf[ckNamespaceReplacements]
    def genericReplacements(self):          return self._conf[ckGenericReplacements]
    def templateCollapsingDepth(self):      return self._conf[ckTemplateCollapsingDepth]
    def enableNamespaceReplacements(self):  return self._conf[ckEnableNamespaceReplacements]
    def enableGenericReplacements(self):    return self._conf[ckEnableGenericReplacements]
    def enableTemplateCollapsing(self):     return self._conf[ckEnableTemplateCollapsing]
    def enableTempCache(self):              return self._conf[ckEnableTempCache]
    def enableProcessByLine(self):          return self._conf[ckEnableProcessByLine]
    def reprocess(self):                    return self._conf[ckReprocess]

    # From http://stackoverflow.com/questions/15175142/
    def _initRegex(self, k_mapper, v_mapper, xdict):
        # Map keys and values in dictionary
        mapped_dict = {k_mapper(k): v_mapper(v) for k, v in xdict.items()}

        # Create a regular expression from the dictionary keys
        escaped_keys =  mapped_dict.keys()
        joined_regex_matcher = "({})".format("|".join(escaped_keys))

        # Compile and return regex
        return re.compile(joined_regex_matcher)

    # Compiles and caches namespace and generic replacement regexes
    def _initRegexes(self):
        if self.enableNamespaceReplacements():
            self._compiledNamespaceRegex = \
                self._initRegex( \
                    lambda k: k + r'::', \
                    lambda v: v + r'::' if len(v) > 0 else '', \
                    self.namespaceReplacements())

        if self.enableGenericReplacements():
            self._compiledGenericRegex = \
                self._initRegex( \
                    lambda k: k, \
                    lambda v: v, \
                    self.genericReplacements())

    # Find all `<...>` pair ranges
    def _find_angles(self, s, xi_start, xi_end):
        result_angle_pairs = []
        open_angles = []

        def matchOpen(i):
            if s[i] != '<': return False

            if s[i-1] == ' ': return False # No space before '<'
            if s[i-1] == '<': return False # Stream operator

            if s[i+1] == ' ': return False # No space after '<'
            if s[i+1] == '<': return False # Stream operator

            return True

        def matchClose(i):
            if s[i] != '>': return False

            if s[i-1] == '-': return False # Dereference arrow

            # Relational operator or stream operator
            if s[i-1] == ' ' and (s[i+1] == ' ' or s[i+1] == '>') and s[i+2] != '>': 
                return False 
            
            if s[i-1] == '>' and s[i-2] == ' ': return False # Stream operator

            return True

        for i in range(xi_start, xi_end):
            if matchOpen(i):
                open_angles.append(i)

            elif matchClose(i):
                if len(open_angles) == 0:
                    continue

                depth = len(open_angles)
                x = open_angles.pop()
                result_angle_pairs.append((x, i+1, depth))

        return result_angle_pairs

    # Mark a `<...>` pair for removal
    def _mark(self, i_start, i_end):
        return (i_start + 1, i_end - 1)

    # Yield merged overlapping intervals
    # (From 'http://codereview.stackexchange.com/questions/69242')
    def _merged(self, intervals):
        sorted_intervals = sorted(intervals, key=lambda x: x[0])

        if not sorted_intervals:  # no intervals to merge
            return

        # low and high represent the bounds of the current run of merges
        low, high = sorted_intervals[0]

        for iv in sorted_intervals[1:]:
            if iv[0] <= high:  # new interval overlaps current run
                high = max(high, iv[1])  # merge with the current run
            else:  # current run is over
                yield low, high  # yield accumulated interval
                low, high = iv  # start new run

        yield low, high  # end the final run

    def _subDisambiguator(self, dict, x):
        matched_str = x.string[x.start():x.end()]

        for k, v in dict.items():
            if re.match(k, matched_str) != None:
                return v

        raise Exception("No valid replacement for " + matched_str)

    def _multiple_replace(self, k_mapper, v_mapper, xdict, compiled_regex, src):
        # Bail out if there are no replacements
        if len(xdict) == 0:
            return src

        # Map keys and values in dictionary
        mapped_dict = {k_mapper(k): v_mapper(v) for k, v in xdict.items()}

        # For each match, look-up corresponding value in dictionary
        return compiled_regex.sub(lambda x: self._subDisambiguator(mapped_dict, x), src)

    def _processImpl(self, src, ostream, temp_cache):
        out = ""
        marked = []

        # Write temp cache ______________________________________
        #
        if self.reprocess() == False and self.enableTempCache():
            # Cache source error
            temp_cache.write(src)

            # Cache config
            writeTempConfigCache(json.dumps(self._conf))
        # _______________________________________________________
        #

        # Template collapsing ___________________________________
        #
        if self.enableTemplateCollapsing():
            # Find all angle bracket pairs
            angle_pairs = self._find_angles(src, 0, len(src))

            # Mark pairs matching desired depth for removal
            for p in angle_pairs:
                if p[2] > self.templateCollapsingDepth():
                    marked.append(self._mark(p[0], p[1]))

            # Build output string by avoiding marked intervals
            last = 0
            for m in self._merged(marked):
                out += src[last:m[0]]
                last = m[1]
            out += src[last:len(src)]
        else:
            out = src
        # _______________________________________________________
        #

        # Namespace replacements_________________________________
        #
        if self.enableNamespaceReplacements():
            # Replace namespace matches
            out = self._multiple_replace( \
                lambda k: k + r'::', \
                lambda v: v + r'::' if len(v) > 0 else '', \
                self.namespaceReplacements(), self._compiledNamespaceRegex, out)
        # _______________________________________________________
        #

        # Generic replacements __________________________________
        #
        if self.enableGenericReplacements():
            # Replace generic matches
            out = self._multiple_replace( \
                lambda k: k, \
                lambda v: v, \
                self.genericReplacements(), self._compiledGenericRegex, out)
        # _______________________________________________________
        #

        ostream.write(out)

    def process(self, istream, ostream, temp_cache):
        if self.enableProcessByLine():
            # Process error line by line
            lastTime = time.perf_counter()
            while True:
                src = istream.readline()

                if src == '':
                    break

                self._processImpl(src, ostream, temp_cache)

                # Flush once per second
                t = time.perf_counter()
                if t - lastTime >= 1.0:
                    ostream.flush()
                lastTime = t
        else:
            # Process error at all once
            src = istream.read()
            self._processImpl(src, ostream, temp_cache)



#
#
#
# -----------------------------------------------------------------------------------------------------------
# Configuration
# -----------------------------------------------------------------------------------------------------------

def loadConfig(path):
    with open(path) as data:
        j = json.load(data)
        res = {
            ckNamespaceReplacements: dict(),
            ckGenericReplacements: dict(),
            ckConfigPaths: set()
        }

        propagateEverything(j, res)
        propagateConfigPaths(ckConfigPaths, j, res)

        return res

def mergeConfig(parent, x):
    # Recursively merge configs
    if ckConfigPaths in x.keys():
        for child in x[ckConfigPaths]:
            x = mergeConfig(x, loadConfig(child))

    propagateEverything(x, parent)
    return parent

def makeInitialBaseConfig():
    conf = dict()
    conf[ckNamespaceReplacements] = dict()
    conf[ckGenericReplacements] = dict()
    conf[ckConfigPaths] = []

    return conf

def makeInitialConfig(args):
    conf = vars(args[0])
    conf[ckNamespaceReplacements] = dict()
    conf[ckGenericReplacements] = dict()
    conf[ckConfigPaths] = conf[ckConfigPaths]

    conf[ckNamespaceReplacements]["std"] = ""
    conf[ckGenericReplacements]["<>"] = "<?>"

    return conf

def makeFinalConfig():
    ap = makeArgParser(False, True)
    args = ap.parse_known_args()
    conf = makeInitialConfig(args)

    # TODO:
    #
    # compilationCmd = ' '.join(conf[ckCompilationCommand])
    #
    # if compilationCmd == "":
    #     src = sys.stdin.read()
    # else:
    #     print("Warning: stdin data will be ignored because '-x' is being used.")
    #     os.system('(' + compilationCmd + ')')

    for child in conf[ckConfigPaths]:
        conf = mergeConfig(conf, loadConfig(child))

    return conf

def confDispatch():
    conf = makeFinalConfig()

    # If reprocessing with previous config, load it and override with command line
    if conf[ckReprocess] and conf[ckReprocessPrevConfig]:

        # Make initial base config and merge it with cached config
        conf = mergeConfig(makeInitialBaseConfig(), loadConfig(fkCacheConfigFilePath))

        # Override cached config with eventual command-line arguments
        ap = makeArgParser(True, False)
        args = vars(ap.parse_known_args()[0])
        propagateEverything(args, conf)

        # Override config with eventual command-line config paths
        if canPropagate(ckConfigPaths, args):
            for child in args[ckConfigPaths]:
                conf = mergeConfig(conf, loadConfig(child))

        # Prevent altering the cache
        conf[ckReprocess] = True
        conf[ckEnableTempCache] = False

    # Convert all paths to absolute
    conf[ckConfigPaths] = [os.path.abspath(x) for x in conf[ckConfigPaths]]

    return conf



#
#
#
# -----------------------------------------------------------------------------------------------------------
# Main
# -----------------------------------------------------------------------------------------------------------

def main():
    conf = confDispatch()
    camomilla = Camomilla(conf)

    # Read from temp cache if reprocessing, otherwise from stdin
    src = readTempCache() if conf[ckReprocess] else sys.stdin

    if conf[ckEnableTempCache]:
        tempCache = writeTempCache()
    else:
        tempCache = None

    # Process and stream error line by line
    camomilla.process(src, sys.stdout, tempCache)

    # Close temp cache if required
    if conf[ckEnableTempCache]:
        closeTempCache(tempCache)

    if conf[ckReprocess]:
        closeTempCache(src)

    return 0

if __name__ == "__main__":
    sys.exit(main())



### TODO
#
# * gcc, g++, clang, clang++ aliases that pass error to camomilla.
#
# * Makefile/install script.
#
# * --depth=-1 or --alldepth or whatever.
#
# * Refactor code.
#
# * Blog article.
#
# * tuple/pair sugar