Home | History | Annotate | Download | only in tests 
      1 #! /usr/bin/python
      2 
      3 import os
      4 import sys
      5 import glob
      6 import optparse
      7 import tempfile
      8 import logging
      9 import shutil
     10 import ConfigParser
     11 
     12 class Fail(Exception):
     13     def __init__(self, test, msg):
     14         self.msg = msg
     15         self.test = test
     16     def getMsg(self):
     17         return '\'%s\' - %s' % (self.test.path, self.msg)
     18 
     19 class Unsup(Exception):
     20     def __init__(self, test):
     21         self.test = test
     22     def getMsg(self):
     23         return '\'%s\'' % self.test.path
     24 
     25 class Event(dict):
     26     terms = [
     27         'cpu',
     28         'flags',
     29         'type',
     30         'size',
     31         'config',
     32         'sample_period',
     33         'sample_type',
     34         'read_format',
     35         'disabled',
     36         'inherit',
     37         'pinned',
     38         'exclusive',
     39         'exclude_user',
     40         'exclude_kernel',
     41         'exclude_hv',
     42         'exclude_idle',
     43         'mmap',
     44         'comm',
     45         'freq',
     46         'inherit_stat',
     47         'enable_on_exec',
     48         'task',
     49         'watermark',
     50         'precise_ip',
     51         'mmap_data',
     52         'sample_id_all',
     53         'exclude_host',
     54         'exclude_guest',
     55         'exclude_callchain_kernel',
     56         'exclude_callchain_user',
     57         'wakeup_events',
     58         'bp_type',
     59         'config1',
     60         'config2',
     61         'branch_sample_type',
     62         'sample_regs_user',
     63         'sample_stack_user',
     64     ]
     65 
     66     def add(self, data):
     67         for key, val in data:
     68             log.debug("      %s = %s" % (key, val))
     69             self[key] = val
     70 
     71     def __init__(self, name, data, base):
     72         log.debug("    Event %s" % name);
     73         self.name  = name;
     74         self.group = ''
     75         self.add(base)
     76         self.add(data)
     77 
     78     def compare_data(self, a, b):
     79         # Allow multiple values in assignment separated by '|'
     80         a_list = a.split('|')
     81         b_list = b.split('|')
     82 
     83         for a_item in a_list:
     84             for b_item in b_list:
     85                 if (a_item == b_item):
     86                     return True
     87                 elif (a_item == '*') or (b_item == '*'):
     88                     return True
     89 
     90         return False
     91 
     92     def equal(self, other):
     93         for t in Event.terms:
     94             log.debug("      [%s] %s %s" % (t, self[t], other[t]));
     95             if not self.has_key(t) or not other.has_key(t):
     96                 return False
     97             if not self.compare_data(self[t], other[t]):
     98                 return False
     99         return True
    100 
    101     def diff(self, other):
    102         for t in Event.terms:
    103             if not self.has_key(t) or not other.has_key(t):
    104                 continue
    105             if not self.compare_data(self[t], other[t]):
    106 		log.warning("expected %s=%s, got %s" % (t, self[t], other[t]))
    107                 
    108 
    109 # Test file description needs to have following sections:
    110 # [config]
    111 #   - just single instance in file
    112 #   - needs to specify:
    113 #     'command' - perf command name
    114 #     'args'    - special command arguments
    115 #     'ret'     - expected command return value (0 by default)
    116 #
    117 # [eventX:base]
    118 #   - one or multiple instances in file
    119 #   - expected values assignments
    120 class Test(object):
    121     def __init__(self, path, options):
    122         parser = ConfigParser.SafeConfigParser()
    123         parser.read(path)
    124 
    125         log.warning("running '%s'" % path)
    126 
    127         self.path     = path
    128         self.test_dir = options.test_dir
    129         self.perf     = options.perf
    130         self.command  = parser.get('config', 'command')
    131         self.args     = parser.get('config', 'args')
    132 
    133         try:
    134             self.ret  = parser.get('config', 'ret')
    135         except:
    136             self.ret  = 0
    137 
    138         self.expect   = {}
    139         self.result   = {}
    140         log.debug("  loading expected events");
    141         self.load_events(path, self.expect)
    142 
    143     def is_event(self, name):
    144         if name.find("event") == -1:
    145             return False
    146         else:
    147             return True
    148 
    149     def load_events(self, path, events):
    150         parser_event = ConfigParser.SafeConfigParser()
    151         parser_event.read(path)
    152 
    153         # The event record section header contains 'event' word,
    154         # optionaly followed by ':' allowing to load 'parent
    155         # event' first as a base
    156         for section in filter(self.is_event, parser_event.sections()):
    157 
    158             parser_items = parser_event.items(section);
    159             base_items   = {}
    160 
    161             # Read parent event if there's any
    162             if (':' in section):
    163                 base = section[section.index(':') + 1:]
    164                 parser_base = ConfigParser.SafeConfigParser()
    165                 parser_base.read(self.test_dir + '/' + base)
    166                 base_items = parser_base.items('event')
    167 
    168             e = Event(section, parser_items, base_items)
    169             events[section] = e
    170 
    171     def run_cmd(self, tempdir):
    172         cmd = "PERF_TEST_ATTR=%s %s %s -o %s/perf.data %s" % (tempdir,
    173               self.perf, self.command, tempdir, self.args)
    174         ret = os.WEXITSTATUS(os.system(cmd))
    175 
    176         log.info("  '%s' ret %d " % (cmd, ret))
    177 
    178         if ret != int(self.ret):
    179             raise Unsup(self)
    180 
    181     def compare(self, expect, result):
    182         match = {}
    183 
    184         log.debug("  compare");
    185 
    186         # For each expected event find all matching
    187         # events in result. Fail if there's not any.
    188         for exp_name, exp_event in expect.items():
    189             exp_list = []
    190             log.debug("    matching [%s]" % exp_name)
    191             for res_name, res_event in result.items():
    192                 log.debug("      to [%s]" % res_name)
    193                 if (exp_event.equal(res_event)):
    194                     exp_list.append(res_name)
    195                     log.debug("    ->OK")
    196                 else:
    197                     log.debug("    ->FAIL");
    198 
    199             log.debug("    match: [%s] matches %s" % (exp_name, str(exp_list)))
    200 
    201             # we did not any matching event - fail
    202             if (not exp_list):
    203 		exp_event.diff(res_event)
    204                 raise Fail(self, 'match failure');
    205 
    206             match[exp_name] = exp_list
    207 
    208         # For each defined group in the expected events
    209         # check we match the same group in the result.
    210         for exp_name, exp_event in expect.items():
    211             group = exp_event.group
    212 
    213             if (group == ''):
    214                 continue
    215 
    216             for res_name in match[exp_name]:
    217                 res_group = result[res_name].group
    218                 if res_group not in match[group]:
    219                     raise Fail(self, 'group failure')
    220 
    221                 log.debug("    group: [%s] matches group leader %s" %
    222                          (exp_name, str(match[group])))
    223 
    224         log.debug("  matched")
    225 
    226     def resolve_groups(self, events):
    227         for name, event in events.items():
    228             group_fd = event['group_fd'];
    229             if group_fd == '-1':
    230                 continue;
    231 
    232             for iname, ievent in events.items():
    233                 if (ievent['fd'] == group_fd):
    234                     event.group = iname
    235                     log.debug('[%s] has group leader [%s]' % (name, iname))
    236                     break;
    237 
    238     def run(self):
    239         tempdir = tempfile.mkdtemp();
    240 
    241         try:
    242             # run the test script
    243             self.run_cmd(tempdir);
    244 
    245             # load events expectation for the test
    246             log.debug("  loading result events");
    247             for f in glob.glob(tempdir + '/event*'):
    248                 self.load_events(f, self.result);
    249 
    250             # resolve group_fd to event names
    251             self.resolve_groups(self.expect);
    252             self.resolve_groups(self.result);
    253 
    254             # do the expectation - results matching - both ways
    255             self.compare(self.expect, self.result)
    256             self.compare(self.result, self.expect)
    257 
    258         finally:
    259             # cleanup
    260             shutil.rmtree(tempdir)
    261 
    262 
    263 def run_tests(options):
    264     for f in glob.glob(options.test_dir + '/' + options.test):
    265         try:
    266             Test(f, options).run()
    267         except Unsup, obj:
    268             log.warning("unsupp  %s" % obj.getMsg())
    269 
    270 def setup_log(verbose):
    271     global log
    272     level = logging.CRITICAL
    273 
    274     if verbose == 1:
    275         level = logging.WARNING
    276     if verbose == 2:
    277         level = logging.INFO
    278     if verbose >= 3:
    279         level = logging.DEBUG
    280 
    281     log = logging.getLogger('test')
    282     log.setLevel(level)
    283     ch  = logging.StreamHandler()
    284     ch.setLevel(level)
    285     formatter = logging.Formatter('%(message)s')
    286     ch.setFormatter(formatter)
    287     log.addHandler(ch)
    288 
    289 USAGE = '''%s [OPTIONS]
    290   -d dir  # tests dir
    291   -p path # perf binary
    292   -t test # single test
    293   -v      # verbose level
    294 ''' % sys.argv[0]
    295 
    296 def main():
    297     parser = optparse.OptionParser(usage=USAGE)
    298 
    299     parser.add_option("-t", "--test",
    300                       action="store", type="string", dest="test")
    301     parser.add_option("-d", "--test-dir",
    302                       action="store", type="string", dest="test_dir")
    303     parser.add_option("-p", "--perf",
    304                       action="store", type="string", dest="perf")
    305     parser.add_option("-v", "--verbose",
    306                       action="count", dest="verbose")
    307 
    308     options, args = parser.parse_args()
    309     if args:
    310         parser.error('FAILED wrong arguments %s' %  ' '.join(args))
    311         return -1
    312 
    313     setup_log(options.verbose)
    314 
    315     if not options.test_dir:
    316         print 'FAILED no -d option specified'
    317         sys.exit(-1)
    318 
    319     if not options.test:
    320         options.test = 'test*'
    321 
    322     try:
    323         run_tests(options)
    324 
    325     except Fail, obj:
    326         print "FAILED %s" % obj.getMsg();
    327         sys.exit(-1)
    328 
    329     sys.exit(0)
    330 
    331 if __name__ == '__main__':
    332     main()
    333