makehelper.py

#! /usr/bin/env python

########################################################################
#
# Perform a variety of unrelated build-time tasks
#
# By Scott Pakin <pakin@lanl.gov>
#
# ----------------------------------------------------------------------
#
# 
# Copyright (C) 2003, Triad National Security, LLC
# All rights reserved.
# 
# Copyright (2003).  Triad National Security, LLC.  This software
# was produced under U.S. Government contract 89233218CNA000001 for
# Los Alamos National Laboratory (LANL), which is operated by Los
# Alamos National Security, LLC (Triad) for the U.S. Department
# of Energy. The U.S. Government has rights to use, reproduce,
# and distribute this software.  NEITHER THE GOVERNMENT NOR TRIAD
# MAKES ANY WARRANTY, EXPRESS OR IMPLIED, OR ASSUMES ANY LIABILITY
# FOR THE USE OF THIS SOFTWARE. If software is modified to produce
# derivative works, such modified software should be clearly marked,
# so as not to confuse it with the version available from LANL.
# 
# Additionally, redistribution and use in source and binary forms,
# with or without modification, are permitted provided that the
# following conditions are met:
# 
#   * Redistributions of source code must retain the above copyright
#     notice, this list of conditions and the following disclaimer.
# 
#   * Redistributions in binary form must reproduce the above copyright
#     notice, this list of conditions and the following disclaimer
#     in the documentation and/or other materials provided with the
#     distribution.
# 
#   * Neither the name of Triad National Security, LLC, Los Alamos
#     National Laboratory, the U.S. Government, nor the names of its
#     contributors may be used to endorse or promote products derived
#     from this software without specific prior written permission.
# 
# THIS SOFTWARE IS PROVIDED BY TRIAD AND CONTRIBUTORS "AS IS" AND ANY
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL TRIAD OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
# OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
# OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
# BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# 
#
########################################################################

import sys
import string
import re
import os
import tempfile
import random

def abend(errmsg):
    "Abort the program with an error message."
    sys.stderr.write("%s: %s\n" % (os.path.basename(sys.argv[0]), errmsg))
    sys.exit(1)

def say_generated_by(commentchar="#", outfile=sys.stdout):
    'Output a "generated by" warning.'
    outfile.write("%s This file was generated from the coNCePTuaL Makefile\n" % commentchar)
    outfile.write('%s by "%s %s".\n' % (commentchar, os.path.abspath(sys.argv[0]), sys.argv[1]))
    outfile.write("%s Don't edit this file; edit %s instead.\n" % (commentchar, os.path.basename(sys.argv[0])))

def say_license_here(commentchar="#", outfile=sys.stdout):
    'Output a "put license text here" message.'
    outfile.write(commentchar*72 + "\n")
    outfile.write(commentchar + " PUT_LICENSE_" + "TEXT_HERE\n")
    outfile.write(commentchar*72 + "\n")

def cpp_quote(somestr):
    "Quote a string for the C preprocessor."
    oldstr = repr(somestr)
    oldstr = oldstr[1:-1]
    newstr = ""
    escaped = 0
    for onechar in oldstr:
        if escaped:
            if onechar != "'":
                newstr = newstr + "\\"
            newstr = newstr + onechar
            escaped = 0
        elif onechar == "\\":
            escaped = 1
        elif onechar == '"':
            newstr = newstr + "\\" + onechar
        else:
            newstr = newstr + onechar
    newstr = '"' + newstr + '"'
    return newstr

#----------------------------------------------------------------------

def make_jspark():
    """
         Make a jythonc-friendly version of spark.py with coNCePTuaL's
         doc strings hardwired into it.
    """
    from ncptl_lexer import NCPTL_Lexer
    from ncptl_parser import NCPTL_Parser

    # Open the input files.
    if len(sys.argv) < 3:
        infile = open("spark.py", "r")
    else:
        infile = open(sys.argv[2], "r")
    say_generated_by()
    print

    # Rewrite references to __doc__ to use docstring.
    for oneline in infile.readlines():
        oneline = string.replace(oneline,
                                 'doc = getattr(self, name).__doc__',
                                 'doc = docstring[name]')
        oneline = string.replace(oneline,
                                 'doc = func.__doc__',
                                 'doc = docstring[name]')
        sys.stdout.write(oneline)

    # Create a doc-string dictionary.
    print
    print "# Hardwire all relevant doc strings for jythonc's sake."
    print "docstring = {}"
    for theclass in [NCPTL_Lexer, NCPTL_Parser]:
        for themethod in dir(theclass):
            try:
                print "docstring[%s] = %s" % (repr(themethod),
                                              repr(getattr(theclass, themethod).__doc__))
            except:
                pass

    # Finish up.
    infile.close()

#----------------------------------------------------------------------

def list_backends():
    """
        Given the names of Python scripts which implement coNCePTuaL
        backends, output a Python list assignment which defines
        backend_list to the list of sanitized backend names.
    """
    say_generated_by()
    print
    print ("backend_list = %s" %
           string.replace(repr(map(lambda s: re.sub(r"codegen_([^.]+).py", r"\1", s),
                                   sys.argv[2:])),
                          " ", "\n                "))

#----------------------------------------------------------------------

def make_ncptl_config():
    "Convert config.h to a ncptl_config.py script."

    if len(sys.argv) < 4:
        abend('"config" requires the name of the C preprocessor and at least one configuration filename')
    try:
        # Copy the named config.h file(s) to a temporary file in which
        # #define is replaced by a dummy string.
        dummydefine = "DEFINE" + str(random.randint(100000, 999999))
        try:
            tempfile.tempdir = os.environ["TEMP"]
        except KeyError:
            pass
        tempfile.template = "config-temp."
        config_tmp_name = tempfile.mktemp() + ".c"
        config_tmp = open(config_tmp_name, "w", 0644)
        for config_h_name in sys.argv[3:]:
            config_h = open(config_h_name, "r")
            oneline = config_h.readline()
            while oneline:
                oneline = string.strip(oneline)
                oneline = re.sub(r'# *define', dummydefine, oneline)
                if string.find(oneline, dummydefine) != -1:
                    # Store only parameterless macros and put double quotes
                    # around the macro definition.
                    defparts = string.split(oneline, None, 2)
                    if string.find(defparts[1], "(") == -1:
                        if len(defparts) == 2:
                            defparts.append("")
                        defparts[2] = cpp_quote(defparts[2])
                    oneline = string.join(defparts, " ")
                config_tmp.write(oneline)
                config_tmp.write("\n")
                oneline = config_h.readline()
            config_h.close()
        config_tmp.close()

        # Run the C preprocessor on the temporary file and read all of
        # the defined values.
        definitions = []
        cpp_output = os.popen("%s %s" % (sys.argv[2], config_tmp_name), "r")
        oneline = cpp_output.readline()
        while oneline:
            oneline = string.strip(oneline)
            if string.find(oneline, dummydefine) != -1:
                defparts = string.split(oneline, None, 2)
                if string.find(defparts[1], "(") == -1:
                    if len(defparts) == 2:
                        defparts.append("")
                    definitions.append(defparts[1:])
            oneline = cpp_output.readline()

        # Delete the temporary file.
        os.remove(config_tmp_name)

        # Ensure the popen worked as expected.
        if cpp_output.close() != None:
            raise IOError, (0, '"%s" failed' % sys.argv[2])

        # Clean up the list of definitions.
        newdefs = {}
        for key, value in definitions:
            newdefs[key] = re.sub(r'^""(.*)""$', r'\1', value)
        definitions = newdefs.items()
        definitions.sort()

        # Output the list of definitions as a Python assignment to the
        # ncptl_config dictionary.
        say_generated_by()
        print
        print "ncptl_config = {"
        print "   ", string.join(map(lambda d: "%s : %s" % (repr(d[0]), repr(eval(d[1]))),
                                     definitions),
                                 ",\n    ")
        print "}"

        # Output some boilerplate to expand ncptl_config into
        # expanded_ncptl_config.
        print """
def _expand_ncptl_config():
    'Expand as many variables as possible from ncptl_config to produce expanded_ncptl_config.'
    global expanded_ncptl_config
    expanded_ncptl_config = ncptl_config.copy()
    import re, string
    variable_re = re.compile(r'\$[\{\(](\w+)[\)\}]')
    for key, value in ncptl_config.items():
        changes_made = 1
        while changes_made:
            changes_made = 0
            variable_match = variable_re.search(value)
            while variable_match:
                try:
                    value = string.replace(value, variable_match.group(0),
                                           expanded_ncptl_config[variable_match.group(1)])
                    changes_made = 1
                except KeyError:
                    pass
                variable_match = variable_re.search(value, variable_match.end(0))
        expanded_ncptl_config[key] = value

_expand_ncptl_config()
"""
    except IOError, (errno, strerror):
        abend(strerror)

#----------------------------------------------------------------------

def grammar_summary():
    "Given Texinfo source, output a summary of the coNCePTuaL grammar in Texinfo format."

    say_generated_by("@c")
    print
    grammar_pattern = re.compile(r"@multitable.*?@end multitable", re.DOTALL)
    for ebnf in re.findall(grammar_pattern, string.join(sys.stdin.readlines(), "")):
        if string.find(ebnf, "::=") != -1:
            print ebnf, "\n"

#----------------------------------------------------------------------

def compiler_version():
    '''
         Given a command to invoke the C compiler, create a C header
         file which defines RT_COMPILER_VERSION to a string
         representing the compiler version or "unknown" if the version
         can\'t be determined.
    '''
    if len(sys.argv) < 3:
        abend('"compiler" requires the name of the C compiler')
    c_compiler = sys.argv[2]

    # Create a zero-byte C test file.
    c_file_name = "conftest.c"
    c_file = open(c_file_name, "w")
    c_file.write('/* Temporary file produced by "%s" */\n' % string.join(sys.argv, " "))
    c_file.write("int someval = 123;\n")
    c_file.close()

    # Try various compiler flags until one of them gives us a version string.
    unknown_str = '"unknown"'
    compiler_version = unknown_str
    for compiler_flag in ("--version", "-version", "-V", "-v", "--help",
                          "-help", "-h"):
        try:
            cc_output = os.popen('echo "" | %s %s -c %s 2>&1' % (c_compiler, compiler_flag, c_file_name), "r")
            for oneline in cc_output.readlines():
                if re.search(r'(unknown|recognized|defined|valid)', oneline, re.IGNORECASE):
                    break
                oneline_no_paths = re.sub(r'(^|\s)(\S*/\S*)(\s|$)', r'\1[PATH]\3', oneline)
                if re.search(r'\d\.\d[^/\s]*\b', oneline_no_paths):
                    compiler_version = cpp_quote(string.strip(oneline))
                    break
            cc_output.close()
        except IOError, (errno, strerror):
            abend(strerror)
        if compiler_version != unknown_str:
            break
    if compiler_version == unknown_str:
        augmented_version = ""
    else:
        augmented_version = ' " [" %s "]"' % compiler_version
    os.remove(c_file_name)

    # Output a C header file.
    print "/*"
    say_generated_by(" *")
    print " */"
    print '''
#ifndef _COMPILER_VERSION_H_
#define _COMPILER_VERSION_H_

#ifdef __VERSION__
# define RT_COMPILER_VERSION __VERSION__%s
#else
# define RT_COMPILER_VERSION %s
#endif

#endif
    ''' % (augmented_version, compiler_version)

#----------------------------------------------------------------------

def placeholders():
    """
         Given the name of a coNCePTuaL backend (presumably
         codegen_latex_vis.py), output a list of all placeholder
         comments in Texinfo format.
    """
    # Acquire a list of placeholder strings.
    if len(sys.argv) < 3:
        abend('"placeholders" requires the name of a file to search')
    placeholders = {}
    pyfile = open(sys.argv[2])
    for oneline in pyfile.readlines():
        has_placeholder = re.search(r'% PLACEHOLDER: (\w+)', oneline)
        if has_placeholder:
            placeholders[has_placeholder.group(1)] = 1
    pyfile.close()
    placeholders = placeholders.keys()

    # Output the list in Texinfo format.
    if len(placeholders) < 3:
        abend("Expected to find at least three placeholder strings")
    placeholders.sort()
    say_generated_by("@c")
    phlist = map(lambda ph: "@ocode{%s}" % ph, placeholders)
    print string.join(phlist[:-1], ",\n") + ", and\n%s@." % phlist[-1]

#----------------------------------------------------------------------

def signallist_c():
    "Output a signallist.c file which maps signal names to C preprocessor symbols."
    print "/*"
    say_generated_by(" *")
    print " */"
    print ""
    print "#include <signal.h>"
    import commands
    shell_kill = commands.getoutput("kill -l")
    bin_kill = commands.getoutput("/bin/kill -l")
    unique_signames = {}
    for signame in string.split(shell_kill) + string.split(bin_kill):
        if re.match(r'[A-Za-z]', signame):
            unique_signames[string.replace(string.upper(signame), "SIG", "")] = 1
    unique_signames = unique_signames.keys()
    unique_signames.sort()
    for signame in unique_signames:
        print '"SIG%s", SIG%s' % (signame, signame)

#----------------------------------------------------------------------

def signalmap_c():
    """
         Given a mapping from signal names to signal numbers, output C
         code to define an ncptl_sig2num() function which maps a
         signal name to its number.
    """
    if len(sys.argv) >= 3:
        # We were passed the name of a working gperf executable.
        gperf = os.popen('%s --key-positions="*" --struct-type --includes --enum --initializer-suffix=", 0" --lookup-fn-name=ncptl_sig2num --language=ANSI-C' %
                         sys.argv[2], "w")
        signallist = string.join(sys.stdin.readlines(), "")
        signallist_noquotes = string.replace(signallist, '"', '')
        gperf.write('struct NAMENUMBER {char *name; int number;};\n')
        gperf.write('%%\n')
        gperf.write(signallist_noquotes)
        gperf.write(re.compile(r'^SIG', re.MULTILINE).sub("", signallist_noquotes))
        gperf.close()
    else:
        # Generate code to perform a linear search.
        print "/*"
        say_generated_by(" *")
        print " */"
        print '''
#include <string.h>

typedef struct {char *name; int number;} NAMENUMBER;

const NAMENUMBER *ncptl_sig2num (const char *str, unsigned int len)
{
  static const NAMENUMBER signalmap[] = {'''
        signallist = sys.stdin.readlines()
        for signame in signallist:
            print "    {%s}," % string.replace(signame[:-1], "SIG", "", 1)
        for signame in signallist:
            print "    {%s}," % signame[:-1]
        print '''  };
  int maplen = sizeof(signalmap)/sizeof(NAMENUMBER);
  int i;

  for (i=0; i<maplen; i++)
    if (!strncmp (str, signalmap[i].name, len))
      return &signalmap[i];
  return 0;
}'''

#----------------------------------------------------------------------

def modulefile():
    "Output a modulefile suitable for use with Environment Modules (http://modules.sf.net)."
    if len(sys.argv) < 5:
        abend('"modulefile" requires the coNCePTuaL version number and strings for bindir, mandir, and libdir')
    version, bindir, mandir, libdir = sys.argv[2:6]
    print "#%Module-1.0\n"
    say_generated_by()
    print '''
proc ModulesHelp { } {
   puts stderr "The [module-info name] modulefile defines the default system paths and"
   puts stderr "environment variables needed to use version %s of the coNCePTuaL"
   puts stderr "compiler."
}

module-whatis "coNCePTuaL compiler version %s"
conflict coNCePTuaL

prepend-path PATH %s
prepend-path MANPATH %s
prepend-path LD_LIBRARY_PATH %s''' % (version, version, bindir, mandir, libdir)

#----------------------------------------------------------------------

def license_text():
    "Given a license.html HTML file, output a LICENSE text file."
    htmlfile = string.join(sys.stdin.readlines(), "")
    htmlfile = re.compile(r'^.*<body>\n', re.DOTALL).sub("", htmlfile)
    htmlfile = re.compile(r'</body>.*$', re.DOTALL).sub("", htmlfile)
    htmlfile = re.sub(r'</*q>', '"', htmlfile)
    htmlfile = re.sub(r'&copy;', "(C)", htmlfile)
    htmlfile = re.sub(r'<li>', "  * ", htmlfile)
    htmlfile = re.compile(r'<strong>(.*?)</strong>',
                          re.DOTALL).sub(lambda mo: string.upper(mo.group(1)),
                                         htmlfile)
    htmlfile = re.sub(r'<[^>]*>', "", htmlfile)
    copyright, body = string.split(htmlfile, "\n\n", 1)
    fmt = os.popen("fmt -70", "w")    # Older Pythons lack the textwrap module.
    sys.stdout.write("%s\n\n" % copyright)
    sys.stdout.flush()
    fmt.write(body)
    fmt.close()

#----------------------------------------------------------------------

def license_texi():
    "Given a license.html HTML file, output a license.texi Texinfo file."
    htmlfile = string.join(sys.stdin.readlines(), "")
    htmlfile = re.compile(r'^.*<body>\n', re.DOTALL).sub("", htmlfile)
    htmlfile = re.compile(r'</body>.*$', re.DOTALL).sub("", htmlfile)
    htmlfile = string.replace(htmlfile, "&copy;", "@copyright{}")
    htmlfile = string.replace(htmlfile, "W-7405-ENG-36", "@w{W-7405-ENG-36}")
    htmlfile = string.replace(htmlfile, "<q>", "``")
    htmlfile = string.replace(htmlfile, "</q>", "''")
    htmlfile = string.replace(htmlfile, "<br />", "@*")
    htmlfile = string.replace(htmlfile, "<ul>", "@itemize @bullet{}\n")
    htmlfile = string.replace(htmlfile, "</ul>", "\n@end itemize")
    htmlfile = string.replace(htmlfile, "<li>", "@item\n")
    htmlfile = string.replace(htmlfile, "<strong>", "@sc{")
    htmlfile = string.replace(htmlfile, "</strong>", "}")
    htmlfile = re.sub(r'<[^>]*>', "", htmlfile)
    htmlfile = string.replace(htmlfile, "reserved.\n", "reserved.\n\n@sp 1\n")
    htmlfile = string.strip(htmlfile)
    print htmlfile

#----------------------------------------------------------------------

def add_license():
    "Given a license file and a target file, insert the license file into the target file."
    if len(sys.argv) < 4:
        abend('"add-license" requires a license file and at least one target file')
    import stat
    lfile = open(sys.argv[2])
    license = lfile.readlines()
    lfile.close()
    tag = "PUT_LICENSE" + "_TEXT_HERE"
    for tfilename in sys.argv[3:]:
        output = []
        tfile = open(tfilename)
        oneline = tfile.readline()
        added_license = 0
        while oneline:
            tagoffset = string.find(oneline, tag)
            if tagoffset == -1:
                output.append(oneline)
            else:
                added_license = 1
                prefix = oneline[0:tagoffset]
                for lline in license:
                    output.append("%s%s" % (prefix, lline))
            oneline = tfile.readline()
        tfile.close()
        tfile_mode = os.stat(tfilename)[stat.ST_MODE]
        os.chmod(tfilename, 0666)
        tfile = open(tfilename, "w", tfile_mode)
        for oneline in output:
            tfile.write(oneline)
        tfile.close()
        os.chmod(tfilename, tfile_mode)
        if added_license:
            sys.stderr.write("Added license text to %s\n" % tfilename)

#----------------------------------------------------------------------

def clean_man_page():
    "Clean up a man page produced by pod2man."
    manpage = sys.stdin.readlines()
    for oneline in manpage:
        if string.find(oneline, ".if n .Ip") != -1:
            continue
        oneline = string.replace(oneline, ".el .Ip", ".Ip")
        oneline = string.replace(oneline, "coNCePTuaL", "\\*(co")
        oneline = string.replace(oneline, "\\*(--", "--")
        if oneline[0:9] == ".IX Title":
            print '''.\\" Define a properly typeset version of the name "coNCePTuaL\".
.ie t .ds co \\s-2CO\\s+2NC\\s-2E\\s+2PT\\s-2UA\\s+2L
.el   .ds co coNCePTuaL
.'''
        print oneline,

#----------------------------------------------------------------------

def c_hooks():
    """
        Extract names of hook functions from codegen_c_generic.py into
        a c_hooks.texi Texinfo file.
    """
    # Acquire a list of hook functions.
    hook_re = re.compile(r'self.invoke_hook')
    hooklist = []
    oneline = sys.stdin.readline()
    while oneline:
        if hook_re.search(oneline):
            hooklist.append(string.split(oneline, '"')[1])
        oneline = sys.stdin.readline()
    hooklist.sort()

    # Output the list categorized by parent function.
    say_generated_by("@c")
    print ""
    print "@itemize @bullet"
    prevfname = None
    name_re = re.compile(r'^([_a-z]+)_([_A-Z0-9]+)$')
    for hook in hooklist:
        name_matches = name_re.search(hook)
        if not name_matches:
            abend("I don't know how to process hook \"%s\"" % hook)
        fname, hname = name_matches.groups()
        if fname == prevfname:
            print "@item"
            print "@ocode{%s}" % hook
        else:
            if prevfname != None:
                print "@end itemize\n"
            print "@item"
            print "@ocode{%s}" % fname
            print "@itemize @minus"
            print "@item"
            print "@ocode{%s}" % hook
            prevfname = fname
    print "@end itemize"
    print "@end itemize"

#----------------------------------------------------------------------

def methods():
    "Given a lowest-level backend, output in Texinfo all methods it defines."
    # Acquire a list of unique method names.
    methodlist = {}
    oneline = sys.stdin.readline()
    func_re = re.compile(r'def\s+(n_\w+)')
    while oneline:
        func_match = func_re.search(oneline)
        if func_match:
            methodlist[func_match.group(1)] = 1
        oneline = sys.stdin.readline()
    methodlist = methodlist.keys()
    methodlist.sort()

    # Output a list in Texinfo format.
    say_generated_by("@c")
    print ""
    print "@itemize @bullet"
    print string.join(map(lambda m: "@item\n@ocode{%s}" % m, methodlist), "\n\n")
    print "@end itemize"

#----------------------------------------------------------------------

def eventlist():
    "Output a list of events defined by codegen_c_generic.py."
    # Acquire a list of event names and associated comments.
    events = {}
    event_re = re.compile(r'(EV_[A-Z]+),?\s+/\*\s+(.*?)\s+\*/')
    oneline = sys.stdin.readline()
    while oneline:
        event_match = event_re.search(oneline)
        if event_match:
            events[event_match.group(1)] = event_match.group(2)
        oneline = sys.stdin.readline()
    sorted_events = events.keys()
    sorted_events.sort()

    # Output a Texinfo table.  EV_CODE goes last because its
    # description is "None of the above" and "above" needs to have
    # some meaning.
    say_generated_by("@c")
    print ""
    print "@table @asis"
    print string.join(map(lambda e, events=events: "@item @ocode{%s}\n%s" % (e, events[e]),
                          filter(lambda e: e != "EV_CODE", sorted_events)),
                      "\n\n")
    print ""
    print "@item EV_CODE"
    print events["EV_CODE"]
    print "@end table"

#----------------------------------------------------------------------

def inventory_ac_subst():
    """
         Output an inventory of all variables that were both
         AC_SUBST'ed and defined in the shell.
    """
    if len(sys.argv) < 4:
        abend('"ac_subst" requires a file of shell variables and a file of substitutions')

    # Read the list of shell variables into a dictionary.
    shellvars = {}
    try:
        shellvarfile = open(sys.argv[2])
        for oneline in shellvarfile.readlines():
            try:
                shellvars[oneline[:string.index(oneline, "=")]] = 1
            except ValueError:
                pass
        shellvarfile.close()
    except IOError, (errno, strerror):
        abend(strerror)

    # Read the list of substitutions.
    substitutions = {}
    try:
        substfile = open(sys.argv[3])
        for oneline in substfile.readlines():
            sedmatch = re.search(r'@([^@]+)@', oneline)
            if sedmatch != None:
                # Handle old versions of Autoconf (e.g., 2.53).
                substitutions[sedmatch.group(1)] = 1
            else:
                # Handle new versions of Autoconf (e.g., 2.57).
                for varname in string.split(oneline, None):
                    substitutions[varname] = 1
        substfile.close()
    except IOError, (errno, strerror):
        abend(strerror)
    substitutions = substitutions.keys()
    substitutions.sort()

    # Output only those substitutions which exist as shell variables.
    for varname in substitutions:
        if shellvars.has_key(varname):
            sys.stdout.write("#define %s @%s@\n" % (varname, varname))

#----------------------------------------------------------------------

def make_pyncptl_module():
    "Create pyncptl.py and __init__.py."
    from ncptl_config import expanded_ncptl_config
    from distutils.file_util import move_file

    # If we were given a directory name, create our module files
    # there.  Otherwise, create them in the current directory.
    top_builddir = "."
    if len(sys.argv) >= 3:
        top_builddir = sys.argv[2]
    pyncptl_py_name = os.path.join(top_builddir, "pyncptl.py")
    init_py_name = os.path.join(top_builddir, "__init__.py")

    # Our behavior depends on the version of SWIG we're using.
    try:
        swigversion = string.split(expanded_ncptl_config["SWIGVERSION"], ".") + ["0", "0"]
    except KeyError:
        abend("ncptl_config.py does not define SWIGVERSION")
    swigversion = float(swigversion[0] + "." + swigversion[1])
    try:
        if swigversion <= 1.2:
            # SWIG v1.1 does not produce a pyncptl.py file so we have to
            # make one ourselves.
            pyncptl_py = open(pyncptl_py_name, "w", 0644)
            say_license_here(outfile=pyncptl_py)
            pyncptl_py.write('''
import os
import imp
from distutils.sysconfig import get_config_var
imp.load_dynamic("pyncptl",
                 os.path.join(imp.find_module("conceptual")[1],
                              "_pyncptl" + get_config_var("SO")))
''')
            pyncptl_py.close()

            # Because the preceding code loads the shared object by
            # name we merely need a dummy __init__.py to placate the
            # Python module system.
            open(init_py_name, "w", 0644).close()
        else:
            # SWIG v1.3 does produce a pyncptl.py file but we want to
            # rename it to __init__.py and write a trivial pyncptl.py
            # in its place.
            os.rename(pyncptl_py_name, init_py_name)
            pyncptl_py = open(pyncptl_py_name, "w", 0644)
            say_license_here(outfile=pyncptl_py)
            pyncptl_py.write("\n")
            pyncptl_py.write("from conceptual import *\n")
            pyncptl_py.close()
    except IOError, (errno, strerror):
        abend(strerror)

#----------------------------------------------------------------------

def setup_install():
    'Perform the equivalent of "python setup.py install".'
    from distutils.core import setup, Extension
    from distutils.sysconfig import get_config_vars
    from ncptl_config import expanded_ncptl_config

    # Ensure we were passed the location of our input files.
    if len(sys.argv) < 3:
        abend('"install" requires the name of the top build directory')
    top_builddir = sys.argv[2]
    srcfile = os.path.abspath(os.path.join(top_builddir, "libncptl_wrap.c"))
    libdir = os.path.abspath(os.path.join(top_builddir, ".libs"))

    # Be sure to link with whatever libraries libncptl needs.
    linkargs = string.split(expanded_ncptl_config["LDFLAGS"] + " " + expanded_ncptl_config["LIBS"])

    # Use Python's distutils to install everything in a "conceptual" directory.
    distobj = setup(name='conceptual-python',
                    version=expanded_ncptl_config["PACKAGE_VERSION"],
                    script_args=["install"] + sys.argv[3:],
                    package_dir={"conceptual": top_builddir,
                                 "": top_builddir},
                    py_modules=["pyncptl", "conceptual.__init__"],
                    ext_modules=[Extension("conceptual._pyncptl",
                                           sources=[srcfile],
                                           extra_compile_args=get_config_vars("CCSHARED") + string.split(expanded_ncptl_config["CPPFLAGS"]),
                                           extra_link_args=linkargs + string.split(expanded_ncptl_config["LDFLAGS"]),
                                           library_dirs=[libdir],
                                           libraries=["ncptl"])])

#----------------------------------------------------------------------

def configure_command():
    """
         Output the configure command used to build the currently
         installed coNCePTuaL.
    """
    if len(sys.argv) > 2:
        sys.path.extend(sys.argv[2:])
    if os.environ.has_key("NCPTL_PATH"):
        sys.path.extend(string.split(os.environ["NCPTL_PATH"], ":"))
    try:
        from ncptl_config import expanded_ncptl_config
    except ImportError, reason:
        abend("unable to import ncptl_config (reason: %s)" % reason)
    try:
        # Convert backslashes and single/double quotes from Python
        # style to shell style.  Also, escape exclamation marks for
        # the benefit of certain shells.
        print re.sub(r'\\(.)', r'\1',
                     re.sub(r"\\'", '"\'"',
                            expanded_ncptl_config["CONFIGURE_COMMAND"]))
    except KeyError:
        abend("unable to find the configure command (old coNCePTuaL is too old)")

#----------------------------------------------------------------------


def quote_arg_list():
    """
         Return all remaining arguments quoted so as to appear within
         a C double-quoted string.
    """
    def quote_for_c(somestr):
        "Quote a string for use within C double quotes."
        cleanstr = somestr
        cleanstr = string.replace(cleanstr, "\\", "\\\\")
        cleanstr = string.replace(cleanstr, '"', '\\"')
        cleanstr = string.replace(cleanstr, "\n", "\\n")
        cleanstr = string.replace(cleanstr, "\r", "\\r")
        cleanstr = string.replace(cleanstr, "\t", "\\t")
        return cleanstr

    # If arguments were provided, use them.  Otherwise, read from
    # standard input.
    if len(sys.argv) > 2:
        arguments = sys.argv[2:]
    else:
        arguments = map(lambda s: s[:-1], sys.stdin.readlines())
    print string.join(map(quote_for_c, arguments))

#----------------------------------------------------------------------

def eat_xeatspaces():
    "Remove all instances of \\xeatspaces from a Texinfo index."
    if len(sys.argv) < 3:
        abend('"xeatspaces" requires the name of a Texinfo index file')
    indexfile = open(sys.argv[2])
    try:
        oneline = indexfile.readline()
        while oneline:
            # Walk the line character-by-character, removing {\xeatspaces ...}.
            bracedepth = 0          # Current brace nesting level
            ignoredepths = []       # Depths of braces to ignore
            filtered_line = ""      # Line without {\xeatspaces ...}
            c = 0
            while c < len(oneline):
                onechar = oneline[c]
                if onechar == "{":
                    filtered_line = filtered_line + "{"
                    bracedepth = bracedepth + 1
                elif onechar == "}":
                    bracedepth = bracedepth - 1
                    if ignoredepths and ignoredepths[-1] == bracedepth:
                        ignoredepths.pop()
                    else:
                        filtered_line = filtered_line + "}"
                elif string.find(oneline[c:], r'\xeatspaces {') == 0:
                    ignoredepths.append(bracedepth)
                    c = c + len(r'\xeatspaces {') - 1
                    bracedepth = bracedepth + 1
                else:
                        filtered_line = filtered_line + onechar
                c = c + 1

            # Output the filtered line and move onto the next line.
            sys.stdout.write(filtered_line)
            oneline = indexfile.readline()
        indexfile.close()
    except IOError, (errno, strerror):
        abend(strerror)

#----------------------------------------------------------------------

def relative_symlink():
    'Invoke an "ln -s" command but use a relative path for the target.'
    if len(sys.argv) < 5:
        abend('"rsymlink" requires a complete ln command')
    fulltargetname = os.path.abspath(sys.argv[-2])
    fulllinkname = os.path.abspath(sys.argv[-1])

    # Find the longest common prefix of the target and link names.
    # (N.B. The os.path.commonprefix is useless because it doesn't
    # treat directory names as atomic.)
    targetname = fulltargetname
    linkname = fulllinkname
    def rsplit(pathname):
        "Behave like os.path.split() but split from the left."
        firstdir, restofpath = string.split(pathname, "/", 1)
        return (firstdir+"/", restofpath)
    commonprefix = ""
    while 1:
        try:
            targetfirst, targetrest = rsplit(targetname)
            linkfirst, linkrest = rsplit(linkname)
        except ValueError:
            break
        if targetfirst == linkfirst:
            commonprefix = commonprefix + targetfirst
            targetname = targetrest
            linkname = linkrest
        else:
            break
    commonprefix = os.path.normpath(commonprefix)

    # Strip down the link name until it matches the common prefix.
    linkdir, linkfile = os.path.split(fulllinkname)
    parentpath = ""
    while linkdir != commonprefix:
        parentpath = "../" + parentpath
        linkdir = os.path.dirname(linkdir)

    # Replace the target's common prefix with a relative path to it.
    if os.path.dirname(fulltargetname) == commonprefix:
        # If the target and link are in the same directory then omit
        # the directory name.
        reltargetname = linkfile
    else:
        # Specify a path up to the common prefix.
        reltargetname = string.replace(fulltargetname, commonprefix, parentpath, 1)
        reltargetname = os.path.normpath(reltargetname)

    # Execute the revised ln command.
    ln_command = string.join(sys.argv[2:-2] + [reltargetname, sys.argv[-1]], " ")
    sys.exit(os.system(ln_command))

#----------------------------------------------------------------------

def split_parse_table():
    """Split ncptl_parse_table's data structures into smaller units
    for Jython's sake."""
    if len(sys.argv) >= 3:
        infilename = sys.argv[2]
    else:
        infilename = "ncptl_parse_table.py"
    entries_per_update = 1
    updates_per_function = 5
    infile = open(infilename)
    action_re = re.compile(r'^_lr_action_items = (.*)$')
    oneline = infile.readline()
    while oneline:
        # Split the _lr_action_items dictionary.
        action_match = action_re.match(oneline)
        if action_match:
            _lr_action_items = eval(action_match.group(1))
            action_keys = _lr_action_items.keys()
            num_action_funcs = 0
            while action_keys != []:
                # Output a function that executes updates_per_function updates.
                num_action_funcs = num_action_funcs + 1
                print "def make_lr_action_items_%d():" % num_action_funcs
                func_update_keys = action_keys[:updates_per_function*entries_per_update]
                while func_update_keys != []:
                    # Output a string that updates entries_per_update entries.
                    update_dict = {}
                    for key in func_update_keys[:entries_per_update]:
                        update_dict[key] = _lr_action_items[key]
                    print "    _lr_action_items.update(%s)" % repr(update_dict)
                    func_update_keys = func_update_keys[entries_per_update:]
                action_keys = action_keys[updates_per_function*entries_per_update:]
                print ""
            print "_lr_action_items = {}"
            for f in range(num_action_funcs):
                print "make_lr_action_items_%d()" % (f+1)
            print ""
        else:
            sys.stdout.write(oneline)
        oneline = infile.readline()
    infile.close()

#----------------------------------------------------------------------

def patch_ply():
    "Patch PLY's lex.py or yacc.py for use with Jython."
    from ncptl_lexer import NCPTL_Lexer
    from ncptl_parser import NCPTL_Parser
    import types

    # Parse the command line.
    if len(sys.argv) < 3 or sys.argv[2] not in ["lex", "yacc"]:
        abend('"patch-ply" requires either "lex" or "yacc" as an argument')
    infiletype = sys.argv[2]
    if len(sys.argv) >= 4:
        infilename = sys.argv[3]
    else:
        infilename = infiletype + ".py"

    # Get a list of lexer and parser symbols.
    docstrings = []
    linenums = []
    if infiletype == "lex":
        theclass = NCPTL_Lexer
    else:
        theclass = NCPTL_Parser
    for themethodname, themethodbody in theclass.__dict__.items():
        if type(themethodbody) == types.FunctionType:
            docstrings.append((themethodname, themethodbody.__doc__))
            linenums.append((themethodname, themethodbody.func_code.co_firstlineno))

    # Replace all references to __doc__ with a lookup in docstring.
    # Replace various introspection routines with Jython-compatible
    # versions.  Comment out yacc warning messages because we expect
    # to invoke the parser with different start symbols but doing so
    # produces numerous irritating "defined, but not used" warnings.
    # Replace reference to func_code.co_firstlineno with lookups into
    # the firstlineno global variable.
    if infiletype == "lex":
        modname = "ncptl_lexer.NCPTL_Lexer"
    else:
        modname = "ncptl_parser.NCPTL_Parser"
    infile = open(infilename)
    oneline = infile.readline()
    while oneline:
        oneline = re.sub(r'(\w+)\.__doc__', r'docstring[\1.__name__]', oneline)
        oneline = re.sub(r'(isinstance\(module,\s*_INSTANCETYPE\))',
                         r'(\1 or str(module.__class__) == "%s")' % modname,
                         oneline)
        oneline = string.replace(oneline, "dir(module)", "dir(module)+dir(module.__class__)")
        oneline = re.sub(r'sys\.stderr\.write.*', r'pass', oneline)
        oneline = re.sub(r'(\w+)\.func_code\.co_firstlineno', r'firstlineno[\1.__name__]', oneline)
        if infiletype == "yacc":
            # exec'ing an import statement doesn't work in the current Jython.
            if re.match(r'import.*re', oneline):
                sys.stdout.write("import ncptl_parse_table as parsetab\n")
            oneline = re.sub(r'(exec "import %s as parsetab")', r'# \1', oneline)

        # Handle the case in Jython in which file operations fail
        # because of Java access permissions, not because of I/O
        # errors.
        oneline = string.replace(oneline, "except IOError:", "except:")

        sys.stdout.write(oneline)
        oneline = infile.readline()

    # Output an appropriate docstring table.
    print "# Create a table of doc strings."
    print "docstring = {}"
    if infiletype == "lex":
        prefix = "t_"
    else:
        prefix = "p_"
    for mname, mdoc in docstrings:
        if mname[:2] == prefix:
            print "docstring[%s] = %s" % (repr(mname), repr(mdoc))
    print ""

    # Output an appropriate line-number table.
    print "# Create a table of line numbers."
    print "firstlineno = {}"
    if infiletype == "lex":
        prefix = "t_"
    else:
        prefix = "p_"
    for mname, mline in linenums:
        if mname[:2] == prefix:
            print "firstlineno[%s] = %s" % (repr(mname), repr(mline))

#----------------------------------------------------------------------

def list_difference():
    "Given two lists, remove each element in the second list from the first list."
    if len(sys.argv) < 4:
        abend('"list-diff" requires two lists of tokens and an optional default')
    list1 = re.split(r'[^-\w.]+', sys.argv[2])
    list2 = re.split(r'[^-\w.]+', sys.argv[3])
    printed_something = 0
    for elt in list1:
        if elt not in list2:
            print elt
            printed_something = 1
    if not printed_something and len(sys.argv) >= 5:
        print sys.argv[4]

#----------------------------------------------------------------------

def canonicalize_keywords():
    "Canonicalize the keywords in a Texinfo index."
    if len(sys.argv) < 3:
        abend('"canon-keywords" requires the name of a Texinfo index file')

    # Map from a keyword to all its forms.
    from ncptl_lexer import NCPTL_Lexer
    lexer = NCPTL_Lexer()
    keyword_forms = {}
    for keyw, canon_keyw in lexer.canonicalize_kw.items():
        if keyw != canon_keyw:
            keyword_forms[keyw] = (keyw, canon_keyw)
            keyword_forms[canon_keyw] = (keyw, canon_keyw)

    # Map each keyword encountered to its list of alternate forms.
    codeline_re = re.compile(r'\\entry\{([A-Z]+)\}\{(\d+)\}\{\\code \{([A-Z]+)\}\}')
    indexfile = open(sys.argv[2])
    try:
        oneline = indexfile.readline()
        while oneline:
            is_codeline = codeline_re.match(oneline)
            if is_codeline:
                try:
                    kw_alt, kw_main = keyword_forms[is_codeline.group(1)]
                    lineno = is_codeline.group(2)
                    print "\\entry{%s/%s}{%s}{\\code {%s/%s}}" % (kw_alt, kw_main, lineno, kw_alt, kw_main)
                    if kw_main[0:len(kw_alt)] != kw_alt:
                        # ITS/THEIR, IS/ARE, etc. -- index the other way, too.
                        print "\\entry{%s/%s}{%s}{\\code {%s/%s}}" % (kw_main, kw_alt, lineno, kw_main, kw_alt)
                except KeyError:
                    print oneline,
            else:
                print oneline,
            oneline = indexfile.readline()
    except IOError, (errno, strerror):
        abend(strerror)
    indexfile.close()

#----------------------------------------------------------------------

def cache_vars_to_texinfo():
    "Output a Texinfo table that describes various Autoconf cache variables."
    if len(sys.argv) < 3:
        abend('"cache-vars" requires the full filenames for configure.ac and config.log')

    # Process all input files.
    name2desc = {}       # Map from a variable name to a description
    last_desc = ""       # The last AC_CACHE_CHECK text seen
    for infilename in sys.argv[2:]:
        infile = open(infilename)
        oneline = infile.readline()
        while oneline:
            # Process descriptions in configure.ac.
            try:
                last_desc = re.search(r'\bAC_CACHE_CHECK\S+\s*(.*?)\],\s*$', oneline).group(1)
                last_desc = re.sub(r'"([^\"]+)"', r'@samp{\1}', last_desc)
                last_desc = re.sub(r'(\$\w+|round|\.interp|-e)', r'@samp{\1}', last_desc)
                last_desc = re.sub(r"`\?='", r'@samp{?=}', last_desc)
                last_desc = re.sub(r'^version$', r'swig version number', last_desc)
                last_desc = re.sub(r'(gperf|swig|ncptl-logextract)', r'@filespec{\1}', last_desc)
                last_desc = re.sub(r'(?<![\{\$\w])([\$\w]*_\w*)', r'@samp{\1}', last_desc)
            except AttributeError:
                pass
            try:
                name2desc[re.search(r'\[(ax_cv_\w+)\]', oneline).group(1)] = last_desc
            except AttributeError:
                pass

            # Process assignments in config.log.
            try:
                varname = re.match(r'(ac_cv_\w+)=', oneline).group(1)

                # Process ac_cv_func_* variables.
                try:
                    name2desc[varname] = "the @ocodecf{%s} function is available (@samp{yes} or @samp{no})" % \
                        re.match(r'ac_cv_func_(\w+)', varname).group(1)
                except AttributeError:
                    pass

                # Process ac_cv_header_* variables.
                try:
                    headername = re.match(r'ac_cv_header_(\w+_h)', varname).group(1)
                    headername = re.sub(r'_h$', r'.h', headername)
                    headername = re.sub(r'_', r'/', headername, 1)
                    name2desc[varname] = "the @filespec{%s} header file is available (@samp{yes} or @samp{no})" % headername
                except AttributeError:
                    pass

                # Process ac_cv_lib_* variables.
                try:
                    libname = re.match(r'ac_cv_lib_(\w+)', varname).group(1)
                    name2desc[varname] = "@filespec{lib%s.a} contains a function called @ocodecf{%s} (@samp{yes} or @samp{no})" % \
                        re.match(r'([^_]+)_(.+)$', libname).groups()
                except AttributeError:
                    pass

                # Process ac_cv_member_* variables.
                try:
                    name2desc[varname] = "the @samp{%s} @code{struct} contains a field called @samp{%s} (@samp{yes} or @samp{no})" % \
                        re.match(r'ac_cv_member_struct_([^_]+)_(.+)$', varname).groups()
                except AttributeError:
                    pass

                # Process ac_cv_path_* variables.
                try:
                    name2desc[varname] = "the fully qualified filename for @filespec{%s}" % \
                        str.lower(re.match(r'ac_cv_path_([A-Z0-9]+)$', varname).group(1))
                except AttributeError:
                    pass

                # Process ac_cv_prog_* variables.
                try:
                    name2desc[varname] = "command line for running @filespec{%s}" % \
                        str.lower(re.match(r'ac_cv_prog_([A-Z]+)$', varname).group(1))
                except AttributeError:
                    pass

                # Process ac_cv_search_* variables.
                try:
                    name2desc[varname] = "linker flags/libraries needed to use the @ocodecf{%s} function or @samp{none required}" % \
                        re.match(r'ac_cv_search_(\w+)$', varname).group(1)
                except AttributeError:
                    pass

                # Process ac_cv_sizeof_* variables.
                try:
                    datatype = re.match(r'ac_cv_sizeof_(\w+)$', varname).group(1)
                    datatype = re.sub(r'(unsigned|long)_', r'\1 ', datatype)
                    name2desc[varname] = "size in bytes of the @samp{%s} datatype" % datatype
                except AttributeError:
                    pass

                # Process ac_cv_type_* variables.
                try:
                    name2desc[varname] = "the @samp{%s} datatype exists (@samp{yes} or @samp{no})" % \
                        re.match(r'ac_cv_type_(\w+)$', varname).group(1)
                except AttributeError:
                    pass
            except AttributeError:
                # No ac_cv_* variable appeared in the current line.
                pass

            # Proceed with the following input line.
            oneline = infile.readline()

    # Output a Texinfo table.
    print "@table @asis"
    cachevars = name2desc.keys()
    cachevars.sort()
    for cvar in cachevars:
        if cvar != cachevars[0]:
            print
        print "@item @ocode{%s}" % cvar
        print "%s" % name2desc[cvar]
    print "@end table"

#----------------------------------------------------------------------

# The program starts here.  Decide what operation to perform based on
# the first argument provided.
if __name__ == "__main__":
    if len(sys.argv) < 2:
        sys.stderr.write("Usage: %s <command> [<options>]\n" % sys.argv[0])
        sys.exit(1)
    functable = {
        "jspark"         : make_jspark,
        "backends"       : list_backends,
        "config"         : make_ncptl_config,
        "grammar"        : grammar_summary,
        "compiler"       : compiler_version,
        "placeholders"   : placeholders,
        "signallist_c"   : signallist_c,
        "signalmap_c"    : signalmap_c,
        "modulefile"     : modulefile,
        "license-text"   : license_text,
        "license-texi"   : license_texi,
        "add-license"    : add_license,
        "manpage"        : clean_man_page,
        "c_hooks"        : c_hooks,
        "methods"        : methods,
        "eventlist"      : eventlist,
        "ac_subst"       : inventory_ac_subst,
        "pyncptl"        : make_pyncptl_module,
        "install"        : setup_install,
        "config-cmd"     : configure_command,
        "quote"          : quote_arg_list,
        "xeatspaces"     : eat_xeatspaces,
        "rsymlink"       : relative_symlink,
        "split-parsetab" : split_parse_table,
        "patch-ply"      : patch_ply,
        "list-diff"      : list_difference,
        "canon-keywords" : canonicalize_keywords,
        "cache-vars"     : cache_vars_to_texinfo}
    try:
        makefunc = functable[sys.argv[1]]
    except KeyError:
        abend('unrecognized command "%s"' % sys.argv[1])
    makefunc()