Home | History | Annotate | Download | only in tools 
      1 #!/usr/bin/env python2.7
      2 
      3 # Copyright 2013, ARM Limited
      4 # All rights reserved.
      5 #
      6 # Redistribution and use in source and binary forms, with or without
      7 # modification, are permitted provided that the following conditions are met:
      8 #
      9 #   * Redistributions of source code must retain the above copyright notice,
     10 #     this list of conditions and the following disclaimer.
     11 #   * Redistributions in binary form must reproduce the above copyright notice,
     12 #     this list of conditions and the following disclaimer in the documentation
     13 #     and/or other materials provided with the distribution.
     14 #   * Neither the name of ARM Limited nor the names of its contributors may be
     15 #     used to endorse or promote products derived from this software without
     16 #     specific prior written permission.
     17 #
     18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
     19 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
     20 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
     21 # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
     22 # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     23 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
     24 # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
     25 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
     26 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
     27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     28 
     29 import os
     30 import sys
     31 import argparse
     32 import re
     33 import subprocess
     34 import time
     35 import util
     36 
     37 
     38 from printer import EnsureNewLine, Print, UpdateProgress
     39 
     40 
     41 def BuildOptions():
     42   result = argparse.ArgumentParser(description =
     43       '''This tool runs each test reported by $CCTEST --list (and filtered as
     44          specified). A summary will be printed, and detailed test output will be
     45          stored in log/$CCTEST.''')
     46   result.add_argument('filters', metavar='filter', nargs='*',
     47                       help='Run tests matching all of the (regexp) filters.')
     48   result.add_argument('--cctest', action='store', required=True,
     49                       help='The cctest executable to run.')
     50   result.add_argument('--coloured_trace', action='store_true',
     51                       help='''Pass --coloured_trace to cctest. This will put
     52                               colour codes in the log files. The coloured output
     53                               can be viewed by "less -R", for example.''')
     54   result.add_argument('--coverage', action='store_true',
     55                       help='Run coverage tests.')
     56   result.add_argument('--debugger', action='store_true',
     57                       help='''Pass --debugger to cctest, so that the debugger is
     58                               used instead of the simulator. This has no effect
     59                               when running natively.''')
     60   result.add_argument('--verbose', action='store_true',
     61                       help='Print verbose output.')
     62   return result.parse_args()
     63 
     64 
     65 def VerbosePrint(string):
     66   if args.verbose:
     67     Print(string)
     68 
     69 
     70 # A class representing an individual test.
     71 class Test:
     72   def __init__(self, name):
     73     self.name = name
     74     self.logpath = os.path.join('log', os.path.basename(args.cctest))
     75     if args.debugger:
     76       basename = name + '_debugger'
     77     else:
     78       basename = name
     79     self.logout = os.path.join(self.logpath, basename + '.stdout')
     80     self.logerr = os.path.join(self.logpath, basename + '.stderr')
     81 
     82   # Run the test.
     83   # Use a thread to be able to control the test.
     84   def Run(self):
     85     command = [args.cctest, '--trace_sim', '--trace_reg', self.name]
     86     if args.coloured_trace:
     87       command.append('--coloured_trace')
     88 
     89     VerbosePrint('==== Running ' + self.name + '... ====')
     90     sys.stdout.flush()
     91 
     92     process = subprocess.Popen(command,
     93                                stdout=subprocess.PIPE,
     94                                stderr=subprocess.PIPE)
     95     # Get the output and return status of the test.
     96     stdout, stderr = process.communicate()
     97     retcode = process.poll()
     98 
     99     # Write stdout and stderr to the log.
    100     if not os.path.exists(self.logpath): os.makedirs(self.logpath)
    101     with open(self.logout, 'w') as f: f.write(stdout)
    102     with open(self.logerr, 'w') as f: f.write(stderr)
    103 
    104     if retcode == 0:
    105       # Success.
    106       # We normally only print the command on failure, but with --verbose we
    107       # should also print it on success.
    108       VerbosePrint('COMMAND: ' + ' '.join(command))
    109       VerbosePrint('LOG (stdout): ' + self.logout)
    110       VerbosePrint('LOG (stderr): ' + self.logerr + '\n')
    111     else:
    112       # Failure.
    113       Print('--- FAILED ' + self.name + ' ---')
    114       Print('COMMAND: ' + ' '.join(command))
    115       Print('LOG (stdout): ' + self.logout)
    116       Print('LOG (stderr): ' + self.logerr + '\n')
    117 
    118     return retcode
    119 
    120 
    121 # Scan matching tests and return a test manifest.
    122 def ReadManifest(filters):
    123   status, output = util.getstatusoutput(args.cctest +  ' --list')
    124   if status != 0: util.abort('Failed to list all tests')
    125 
    126   names = output.split()
    127   for f in filters:
    128     names = filter(re.compile(f).search, names)
    129 
    130   return map(Test, names)
    131 
    132 
    133 # Run all tests in the manifest.
    134 def RunTests(manifest):
    135   count = len(manifest)
    136   passed = 0
    137   failed = 0
    138 
    139   if count == 0:
    140     Print('No tests to run.')
    141     return 0
    142 
    143   Print('Running %d tests...' % (count))
    144   start_time = time.time()
    145 
    146   for test in manifest:
    147     # Update the progress counter with the name of the test we're about to run.
    148     UpdateProgress(start_time, passed, failed, count, args.verbose, test.name)
    149     retcode = test.Run()
    150     # Update the counters and progress indicator.
    151     if retcode == 0:
    152       passed += 1
    153     else:
    154       failed += 1
    155   UpdateProgress(start_time, passed, failed, count, args.verbose, '== Done ==')
    156 
    157   return failed     # 0 indicates success.
    158 
    159 
    160 if __name__ == '__main__':
    161   # $ROOT/tools/test.py
    162   root_dir = os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0])))
    163 
    164   # Parse the arguments.
    165   args = BuildOptions()
    166 
    167   # Find a valid path to args.cctest (in case it doesn't begin with './').
    168   args.cctest = os.path.join('.', args.cctest)
    169 
    170   if not os.access(args.cctest, os.X_OK):
    171     print "'" + args.cctest + "' is not executable or does not exist."
    172     sys.exit(1)
    173 
    174   # List all matching tests.
    175   manifest = ReadManifest(args.filters)
    176 
    177   # Delete coverage data files.
    178   if args.coverage:
    179     status, output = util.getstatusoutput('find obj/coverage -name "*.gcda" -exec rm {} \;')
    180 
    181   # Run the tests.
    182   status = RunTests(manifest)
    183   EnsureNewLine()
    184 
    185   # Print coverage information.
    186   if args.coverage:
    187     cmd = 'tggcov -R summary_all,untested_functions_per_file obj/coverage/src/a64'
    188     p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,
    189                          stderr=subprocess.PIPE)
    190     stdout, stderr = p.communicate()
    191     print(stdout)
    192 
    193   sys.exit(status)
    194 
    195