path: root/sim/src/runtest.py

#!/usr/bin/python
# Copyright 2005-2006 Nanorex, Inc.  See LICENSE file for details.

"""runtest.py description.test [--generate]

runs the described test and concatenates the results
to stdout (after removing lines which vary inconsequentially (e.g. dates)

if --generate is given, the results are written to $base.out, and
the end structure is written to $base.xyzcmp (if the test type
is "struct").

a description.test file looks like:

comment lines, which start with a #, and are ignored
blank lines, which are ignored
lines containing one of the following directives:

TYPE [ min | struct | dyn | fail ]
  Choose one of the given types of tests.  The test type controls
  the defaults for the other directives.  Test type "min" performs
  a minimization, and checks the program outputs, including the
  trace file and xyz file are identical.  Test type "struct" also
  performs a minimization, but relies on a structure comparison
  to test for success, rather than the exact program output.
  Test type "dyn" performs a dynamics run, and requires that output
  be identical, as for "min".  Default value for TYPE is "min".

INPUT files...
  The listed files are copied to the temporary directory before
  the test is run.  Defaults to the name of the description file
  with the .test replaced by .mmp ($base.mmp).

OUTPUT files...
  The listed files contain significant data to be compared.
  They are concatenated together, run through a sed script to
  remove inconsequential lines, and the result goes to stdout.
  In addition to files explicitly generated by the program
  stdout and stderr are available.  The file exitvalue contains
  the shell exit code $? resulting from executing the command.
  Defaults to "exitvalue stderr stdout $base.trc $base.xyz" for
  test types "min" and "dyn".  Defaults to "exitvalue structurematch stderr"
  for test type "struct".

PROGRAM command...
  This is the complete command line to run (excluding io redirection).
  Defaults to "/tmp/testsimulator -m -x $base.mmp" for test types "min" and
  "struct".  Defaults to "/tmp/testsimulator -f100 -t300 -i10 -x $base.mmp"
  for test type "dyn".

STRUCT file
  The listed file is compared against the $base.xyz file generated
  by PROGRAM.  Defaults to "" for test types "min" and "dyn", and
  no structure comparison is performed.  Defaults to "$base.xyzcmp"
  for test type "struct".

  # runtest.sh description.test [--generate]

runs the described test and concatenates the results
to stdout (after removing lines which vary inconsequentially (e.g. dates)

if --generate is given, the results are written to $base.out, and
  the end structure is written to $base.xyzcmp (if the test type
  is "struct").

a description.test file looks like:

comment lines, which start with a #, and are ignored
blank lines, which are ignored
lines containing one of the following directives:

TYPE [ min | struct | dyn ]
  Choose one of the given types of tests.  The test type controls
  the defaults for the other directives.  Test type "min" performs
  a minimization, and checks the program outputs, including the
  trace file and xyz file are identical.  Test type "struct" also
  performs a minimization, but relies on a structure comparison
  to test for success, rather than the exact program output.
  Test type "dyn" performs a dynamics run, and requires that output
  be identical, as for "min".  Default value for TYPE is "min".

INPUT files...
  The listed files are copied to the temporary directory before
  the test is run.  Defaults to the name of the description file
  with the .test replaced by .mmp ($base.mmp).

OUTPUT files...
  The listed files contain significant data to be compared.
  They are concatenated together, run through a sed script to
  remove inconsequential lines, and the result goes to stdout.
  In addition to files explicitly generated by the program
  stdout and stderr are available.  The file exitvalue contains
  the shell exit code $? resulting from executing the command.
  Defaults to "exitvalue stderr stdout $base.trc $base.xyz" for
  test types "min" and "dyn".  Defaults to "exitvalue structurematch stderr"
  for test type "struct".

PROGRAM command...
  This is the complete command line to run (excluding io redirection).
  Defaults to "/tmp/testsimulator -m -x $base.mmp" for test types "min" and
  "struct".  Defaults to "/tmp/testsimulator -f100 -t300 -i10 -x $base.mmp"
  for test type "dyn".

STRUCT file
  The listed file is compared against the $base.xyz file generated
  by PROGRAM.  Defaults to "" for test types "min" and "dyn", and
  no structure comparison is performed.  Defaults to "$base.xyzcmp"
  for test type "struct".


##################################################

NOTE: make sure you specify -x to generate a text
  file.  Otherwise you'll end up with binary data
  in your output file for comparison, which is
  probably not what you wanted.

##################################################

A minimizer test using the default command line and results
can be performed with an empty description file.  The other
test types can often be performed with just a TYPE directive.
"""

import sys
import os
import re
from os.path import join, dirname, basename, exists
from shutil import copy, rmtree

# I need this because I don't see a better way to get to findterms.py.
sys.path.append("tests/scripts")
import findterms

# I've used os.linesep several places, assuming it would be the right
# way to get cross-platform compatibility. When the time comes, we'll
# need to check that I got that right.

def appendFiles(flist, outfile,
                dtregexp = re.compile("Date and Time: ")):
    """for f in $flist; do
        echo ======= $f =======
        cat $f
    done | sed '/Date and Time: /d' >> outfile"""
    outf = open(outfile, "a")
    for f in flist:
        outf.write("======= " + f + " =======" + os.linesep)
        for line in open(f).readlines():
            if dtregexp.match(line) == None:
                line = line.rstrip()
                outf.write(line + os.linesep)
    outf.close()

def run(prog, resultFile=None):
    # Redirect standard ouput to "stdout", appending
    # Redirect standard error to "stderr", appending
    # Will this work on Windows?
    prog += " >> stdout 2>> stderr"
    rc = os.system(prog)
    if resultFile != None:
        outf = open(resultFile, "w")
        # Exit status is shifted 8 bits left
        # http://www.python.org/search/hypermail/python-1994q2/0407.html
        # Does this apply to Windows?
        outf.write(repr(rc >> 8))
        outf.close()

def main(argv, myStdout=sys.stdout, generate=False):
    home = os.getcwd()
    testSpecFile = argv[0]
    assert testSpecFile[-5:] == ".test"

    if not exists("/tmp/testsimulator"):
        raise Exception, "Can't find /tmp/testsimulator"

    tmpDir = "/tmp/runtest%d" % os.getpid()
    rmtree(tmpDir, ignore_errors=True)
    os.mkdir(tmpDir)

    base = basename(testSpecFile[:-5])
    here = os.getcwd()
    dir = dirname(testSpecFile)
    hereDir = join(here, dir)

    altoutFile = join(hereDir, base + ".altout")

    DEFAULT_INPUT = ["%s.mmp" % base]
    DEFAULT_OUTPUT_MIN = ["exitvalue", "stderr", "stdout",
                          base+".trc", base + ".xyz"]
    DEFAULT_OUTPUT_STRUCT = ["exitvalue", "structurematch",
                             "lengthsangles", "stderr"]
    DEFAULT_PROGRAM_MIN = "/tmp/testsimulator --minimize " + \
                          "--dump-as-text " + base + ".mmp"
    DEFAULT_PROGRAM_DYN = "/tmp/testsimulator --num-frames=100" + \
                          "--temperature=300 --iters-per-frame=10 " + \
                          "--dump-as-text " + base + ".mmp"
    DEFAULT_STRUCT_MIN = ""
    DEFAULT_STRUCT_STRUCT = base + ".xyzcmp"

    ALT_OUTPUT_FOR_STRUCT = ["exitvalue", "structurematch", "stderr", "stdout",
                             base + ".trc", base + ".xyz"]

    userType = "min"
    userInput = [ ]
    userOutput = [ ]
    userProgram = ""
    userStruct = ""

    inf = open(testSpecFile)
    for line in inf.read().split(os.linesep):
        if line[:4] == "TYPE":
            userType = line[4:].strip()
        elif line[:5] == "INPUT":
            # This is a list, not a string
            userInput = line[5:].split()
        elif line[:6] == "OUTPUT":
            # This is a list, not a string
            userOutput = line[6:].split()
        elif line[:7] == "PROGRAM":
            userProgram = line[7:].strip()
        elif line[:6] == "STRUCT":
            userStruct = line[6:].strip()
        elif line[:1] == '#':
            pass
        elif len(line) > 0:
            raise Exception, \
                  "%s has unrecognied line:\n%s" % (testSpecFile, line)
    inf.close()

    if userType == "min":
        input = userInput or DEFAULT_INPUT
        output = userOutput or DEFAULT_OUTPUT_MIN
        program = userProgram or DEFAULT_PROGRAM_MIN
        struct = userStruct or DEFAULT_STRUCT_MIN
    elif userType == "struct":
        input = userInput or DEFAULT_INPUT
        output = userOutput or DEFAULT_OUTPUT_STRUCT
        program = userProgram or DEFAULT_PROGRAM_MIN
        struct = userStruct or DEFAULT_STRUCT_STRUCT
    elif userType == "dyn":
        input = userInput or DEFAULT_INPUT
        output = userOutput or DEFAULT_OUTPUT_MIN
        program = userProgram or DEFAULT_PROGRAM_DYN
        struct = userStruct or DEFAULT_STRUCT_MIN
    elif userType == "fail":
        input = userInput  # might be None
        output = userOutput
        # will this work in Windows?
        program = userProgram or "echo fail"
        struct = userStruct
    else:
        raise Exception, \
              "%s has unrecognied type:\n%s" % (testSpecFile, userType)


    if generate:
        outxyz = join(hereDir, struct)
        outstd = join(hereDir, base + ".out")
        outba = join(hereDir, base + ".ba")

    structCompare = (struct != "")

    for inputFile in input:
        copy(join(dir, inputFile), tmpDir)

    if structCompare and not generate:
        copy(join(dir, struct), tmpDir)
        copy(join(dir, base + ".ba"), tmpDir)

    results = open(join(tmpDir, "results"), "w")
    results.write("======= " + base + ".test =======" + os.linesep)
    results.write(open(testSpecFile).read())
    results.close()

    os.chdir(tmpDir)

    run(program, "exitvalue")

    if structCompare:

        stdout = open("stdout", "a")
        stderr = open("stderr", "a")
        str = "== structure comparison ==" + os.linesep
        stdout.write(str)
        stderr.write(str)
        stdout.close()
        stderr.close()

        mmpFile = base + ".mmp"
        bondAngleFile = base + ".ba"
        if generate:
            # when generating a test case, assume the structure
            # comparison will succeed, therefore zero exit status
            sm = open("structurematch", "w")
            sm.write("0")
            sm.close()
            la = open("lengthsangles", "w")
            la.write("OK")
            la.close()
            findterms.main(mmpFile,
                           outf=bondAngleFile,
                           generateFlag=True)
        else:
            run("/tmp/testsimulator " + \
                "--base-file=" + base + ".xyzcmp " + \
                base + ".xyz", "structurematch")
            findterms.main(mmpFile,
                           outf=open("lengthsangles", "w"),
                           referenceInputFile=bondAngleFile)

        copy("results", altoutFile)
        appendFiles(ALT_OUTPUT_FOR_STRUCT, altoutFile)

    appendFiles(output, "results")

    if generate:
        copy("results", outstd)
        if structCompare:
            copy(base + ".xyz", outxyz)
            copy(base + ".ba", outba)
    else:
        myStdout.write(open("results").read())

    os.chdir(home)
    return 0

if __name__ == "__main__":
    generate = False
    if len(sys.argv) > 2 and sys.argv[2] == "--generate":
        generate = True
    main(sys.argv[1:], generate=generate)