Home | History | Annotate | Download | only in npb 
      1 import os, shutil, logging, re
      2 from autotest_lib.client.bin import test, utils
      3 from autotest_lib.client.common_lib import error
      4 
      5 class npb(test.test):
      6     """
      7     This module runs the NAS Parallel Benchmarks on the client machine
      8 
      9     @note: Since we use gfortran to complie these benchmarks, this test might
     10             not be able to run on older Operating Systems.
     11     @see: http://www.nas.nasa.gov/Resources/Software/npb.html
     12     """
     13     version = 1
     14     def initialize(self, tests=''):
     15         # Initialize failure counter
     16         self.n_fail = 0
     17         # Get the parameters for run_once()
     18         self.tests = tests
     19         # Ratio is the reason between 1 and the number of CPUs of the system.
     20         self.ratio = 1.0 / utils.count_cpus()
     21         logging.debug('Ratio (1/n_cpus) found for this system: %s' % self.ratio)
     22 
     23 
     24     def setup(self, tarball='NPB3.3.tar.gz'):
     25         tarball = utils.unmap_url(self.bindir, tarball, self.tmpdir)
     26         utils.extract_tarball_to_dir(tarball, self.srcdir)
     27         os.chdir(self.srcdir)
     28         # Prepare the makefile and benchmarks to generate.
     29         utils.system('patch -p1 < ../enable-all-tests.patch')
     30         utils.system('cd NPB3.3-OMP && make suite')
     31 
     32 
     33     def run_once(self):
     34         """
     35         Run each benchmark twice, with different number of threads.
     36 
     37         A sanity check is made on each benchmark executed:
     38         The ratio between the times
     39         time_ratio = time_one_thrd / time_full_thrds
     40 
     41         Has to be contained inside an envelope:
     42         upper_bound = full_thrds * (1 + (1/n_cpus))
     43         lower_bound = full_thrds * (1 - (1/n_cpus))
     44 
     45         Otherwise, we throw an exception (this test might be running under a
     46         virtual machine and sanity check failure might mean bugs on smp
     47         implementation).
     48         """
     49         os.chdir(self.srcdir)
     50 
     51         # get the tests to run
     52         test_list = self.tests.split()
     53 
     54         if len(test_list) == 0:
     55             raise error.TestError('No tests (benchmarks) provided. Exit.')
     56 
     57         for itest in test_list:
     58             itest_cmd = os.path.join('NPB3.3-OMP/bin/', itest)
     59             try:
     60                 itest = utils.run(itest_cmd)
     61             except:
     62                 logging.error('NPB benchmark %s has failed. Output: %s',
     63                               itest_cmd, itest.stdout)
     64                 self.n_fail += 1
     65             logging.debug(itest.stdout)
     66 
     67             # Get the number of threads that the test ran
     68             # (which is supposed to be equal to the number of system cores)
     69             m = re.search('Total threads\s*=\s*(.*)\n', itest.stdout)
     70 
     71             # Gather benchmark results
     72             ts = re.search('Time in seconds\s*=\s*(.*)\n', itest.stdout)
     73             mt = re.search('Mop/s total\s*=\s*(.*)\n', itest.stdout)
     74             mp = re.search('Mop/s/thread\s*=\s*(.*)\n', itest.stdout)
     75 
     76             time_seconds = float(ts.groups()[0])
     77             mops_total = float(mt.groups()[0])
     78             mops_per_thread = float(mp.groups()[0])
     79 
     80             logging.info('Test: %s', itest_cmd)
     81             logging.info('Time (s): %s', time_seconds)
     82             logging.info('Total operations executed (mops/s): %s', mops_total)
     83             logging.info('Total operations per thread (mops/s/thread): %s',
     84                           mops_per_thread)
     85 
     86             self.write_test_keyval({'test': itest_cmd})
     87             self.write_test_keyval({'time_seconds': time_seconds})
     88             self.write_test_keyval({'mops_total': mops_total})
     89             self.write_test_keyval({'mops_per_thread': mops_per_thread})
     90 
     91             # A little extra sanity check comes handy
     92             if int(m.groups()[0]) != utils.count_cpus():
     93                 raise error.TestError("NPB test suite evaluated the number "
     94                                       "of threads incorrectly: System appears "
     95                                       "to have %s cores, but %s threads were "
     96                                       "executed.")
     97 
     98             # We will use this integer with float point vars later.
     99             full_thrds = float(m.groups()[0])
    100 
    101             # get duration for full_threads running.
    102             m = re.search('Time in seconds\s*=\s*(.*)\n', itest.stdout)
    103             time_full_thrds = float(m.groups()[0])
    104 
    105             # repeat the execution with single thread.
    106             itest_single_cmd = ''.join(['OMP_NUM_THREADS=1 ', itest_cmd])
    107             try:
    108                 itest_single = utils.run(itest_single_cmd)
    109             except:
    110                 logging.error('NPB benchmark single thread %s has failed. '
    111                               'Output: %s',
    112                               itest_single_cmd,
    113                               itest_single.stdout)
    114                 self.n_fail += 1
    115 
    116             m = re.search('Time in seconds\s*=\s*(.*)\n', itest_single.stdout)
    117             time_one_thrd = float(m.groups()[0])
    118 
    119             # check durations
    120             ratio = self.ratio
    121             time_ratio = float(time_one_thrd / time_full_thrds)
    122             upper_bound = full_thrds * (1 + ratio)
    123             lower_bound = full_thrds * (1 - ratio)
    124             logging.debug('Time ratio for %s: %s', itest_cmd, time_ratio)
    125             logging.debug('Upper bound: %s', upper_bound)
    126             logging.debug('Lower bound: %s', lower_bound)
    127 
    128             violates_upper_bound = time_ratio > upper_bound
    129             violates_lower_bound = time_ratio < lower_bound
    130             if violates_upper_bound or violates_lower_bound:
    131                 logging.error('NPB benchmark %s failed sanity check '
    132                               '- time ratio outside bounds' % itest_cmd)
    133                 self.n_fail += 1
    134             else:
    135                 logging.debug('NPB benchmark %s sanity check PASS' % itest_cmd)
    136 
    137 
    138     def cleanup(self):
    139         """
    140         Raise TestError if failures were detected during test execution.
    141         """
    142         if self.n_fail != 0:
    143             raise error.TestError('NPB test failed.')
    144         else:
    145             logging.info('NPB test passed.')
    146