Home | History | Annotate | Download | only in runtest 
      1 #!/usr/bin/env python
      2 #
      3 # Copyright (C) 2015 The Android Open Source Project
      4 #
      5 # Licensed under the Apache License, Version 2.0 (the "License");
      6 # you may not use this file except in compliance with the License.
      7 # You may obtain a copy of the License at
      8 #
      9 #      http://www.apache.org/licenses/LICENSE-2.0
     10 #
     11 # Unless required by applicable law or agreed to in writing, software
     12 # distributed under the License is distributed on an "AS IS" BASIS,
     13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     14 # See the License for the specific language governing permissions and
     15 # limitations under the License.
     16 #
     17 """Simpleperf runtest runner: run simpleperf runtests on host or on device.
     18 
     19 For a simpleperf runtest like one_function test, it contains following steps:
     20 1. Run simpleperf record command to record simpleperf_runtest_one_function's
     21    running samples, which is generated in perf.data.
     22 2. Run simpleperf report command to parse perf.data, generate perf.report.
     23 4. Parse perf.report and see if it matches expectation.
     24 
     25 The information of all runtests is stored in runtest.conf.
     26 """
     27 
     28 import os
     29 import os.path
     30 import re
     31 import subprocess
     32 import sys
     33 import xml.etree.ElementTree as ET
     34 
     35 
     36 class CallTreeNode(object):
     37 
     38   def __init__(self, name):
     39     self.name = name
     40     self.children = []
     41 
     42   def add_child(self, child):
     43     self.children.append(child)
     44 
     45   def __str__(self):
     46     return 'CallTreeNode:\n' + '\n'.join(self._dump(1))
     47 
     48   def _dump(self, indent):
     49     indent_str = '  ' * indent
     50     strs = [indent_str + self.name]
     51     for child in self.children:
     52       strs.extend(child._dump(indent + 1))
     53     return strs
     54 
     55 
     56 class Symbol(object):
     57 
     58   def __init__(self, name, comm, overhead, children_overhead):
     59     self.name = name
     60     self.comm = comm
     61     self.overhead = overhead
     62     # children_overhead is the overhead sum of this symbol and functions
     63     # called by this symbol.
     64     self.children_overhead = children_overhead
     65     self.call_tree = None
     66 
     67   def set_call_tree(self, call_tree):
     68     self.call_tree = call_tree
     69 
     70   def __str__(self):
     71     strs = []
     72     strs.append('Symbol name=%s comm=%s overhead=%f children_overhead=%f' % (
     73         self.name, self.comm, self.overhead, self.children_overhead))
     74     if self.call_tree:
     75       strs.append('\t%s' % self.call_tree)
     76     return '\n'.join(strs)
     77 
     78 
     79 class SymbolOverheadRequirement(object):
     80 
     81   def __init__(self, symbol_name=None, comm=None, min_overhead=None,
     82                max_overhead=None):
     83     self.symbol_name = symbol_name
     84     self.comm = comm
     85     self.min_overhead = min_overhead
     86     self.max_overhead = max_overhead
     87 
     88   def __str__(self):
     89     strs = []
     90     strs.append('SymbolOverheadRequirement')
     91     if self.symbol_name is not None:
     92       strs.append('symbol_name=%s' % self.symbol_name)
     93     if self.comm is not None:
     94       strs.append('comm=%s' % self.comm)
     95     if self.min_overhead is not None:
     96       strs.append('min_overhead=%f' % self.min_overhead)
     97     if self.max_overhead is not None:
     98       strs.append('max_overhead=%f' % self.max_overhead)
     99     return ' '.join(strs)
    100 
    101   def is_match(self, symbol):
    102     if self.symbol_name is not None:
    103       if self.symbol_name != symbol.name:
    104         return False
    105     if self.comm is not None:
    106       if self.comm != symbol.comm:
    107         return False
    108     return True
    109 
    110   def check_overhead(self, overhead):
    111     if self.min_overhead is not None:
    112       if self.min_overhead > overhead:
    113         return False
    114     if self.max_overhead is not None:
    115       if self.max_overhead < overhead:
    116         return False
    117     return True
    118 
    119 
    120 class SymbolRelationRequirement(object):
    121 
    122   def __init__(self, symbol_name, comm=None):
    123     self.symbol_name = symbol_name
    124     self.comm = comm
    125     self.children = []
    126 
    127   def add_child(self, child):
    128     self.children.append(child)
    129 
    130   def __str__(self):
    131     return 'SymbolRelationRequirement:\n' + '\n'.join(self._dump(1))
    132 
    133   def _dump(self, indent):
    134     indent_str = '  ' * indent
    135     strs = [indent_str + self.symbol_name +
    136             (' ' + self.comm if self.comm else '')]
    137     for child in self.children:
    138       strs.extend(child._dump(indent + 1))
    139     return strs
    140 
    141   def is_match(self, symbol):
    142     if symbol.name != self.symbol_name:
    143       return False
    144     if self.comm is not None:
    145       if symbol.comm != self.comm:
    146         return False
    147     return True
    148 
    149   def check_relation(self, call_tree):
    150     if not call_tree:
    151       return False
    152     if self.symbol_name != call_tree.name:
    153       return False
    154     for child in self.children:
    155       child_matched = False
    156       for node in call_tree.children:
    157         if child.check_relation(node):
    158           child_matched = True
    159           break
    160       if not child_matched:
    161         return False
    162     return True
    163 
    164 
    165 class Test(object):
    166 
    167   def __init__(
    168           self,
    169           test_name,
    170           executable_name,
    171           report_options,
    172           symbol_overhead_requirements,
    173           symbol_children_overhead_requirements,
    174           symbol_relation_requirements):
    175     self.test_name = test_name
    176     self.executable_name = executable_name
    177     self.report_options = report_options
    178     self.symbol_overhead_requirements = symbol_overhead_requirements
    179     self.symbol_children_overhead_requirements = (
    180         symbol_children_overhead_requirements)
    181     self.symbol_relation_requirements = symbol_relation_requirements
    182 
    183   def __str__(self):
    184     strs = []
    185     strs.append('Test test_name=%s' % self.test_name)
    186     strs.append('\texecutable_name=%s' % self.executable_name)
    187     strs.append('\treport_options=%s' % (' '.join(self.report_options)))
    188     strs.append('\tsymbol_overhead_requirements:')
    189     for req in self.symbol_overhead_requirements:
    190       strs.append('\t\t%s' % req)
    191     strs.append('\tsymbol_children_overhead_requirements:')
    192     for req in self.symbol_children_overhead_requirements:
    193       strs.append('\t\t%s' % req)
    194     strs.append('\tsymbol_relation_requirements:')
    195     for req in self.symbol_relation_requirements:
    196       strs.append('\t\t%s' % req)
    197     return '\n'.join(strs)
    198 
    199 
    200 def load_config_file(config_file):
    201   tests = []
    202   tree = ET.parse(config_file)
    203   root = tree.getroot()
    204   assert root.tag == 'runtests'
    205   for test in root:
    206     assert test.tag == 'test'
    207     test_name = test.attrib['name']
    208     executable_name = None
    209     report_options = []
    210     symbol_overhead_requirements = []
    211     symbol_children_overhead_requirements = []
    212     symbol_relation_requirements = []
    213     for test_item in test:
    214       if test_item.tag == 'executable':
    215         executable_name = test_item.attrib['name']
    216       elif test_item.tag == 'report':
    217         report_options = test_item.attrib['option'].split()
    218       elif (test_item.tag == 'symbol_overhead' or
    219               test_item.tag == 'symbol_children_overhead'):
    220         for symbol_item in test_item:
    221           assert symbol_item.tag == 'symbol'
    222           symbol_name = None
    223           if 'name' in symbol_item.attrib:
    224             symbol_name = symbol_item.attrib['name']
    225           comm = None
    226           if 'comm' in symbol_item.attrib:
    227             comm = symbol_item.attrib['comm']
    228           overhead_min = None
    229           if 'min' in symbol_item.attrib:
    230             overhead_min = float(symbol_item.attrib['min'])
    231           overhead_max = None
    232           if 'max' in symbol_item.attrib:
    233             overhead_max = float(symbol_item.attrib['max'])
    234 
    235           if test_item.tag == 'symbol_overhead':
    236             symbol_overhead_requirements.append(
    237                 SymbolOverheadRequirement(
    238                     symbol_name,
    239                     comm,
    240                     overhead_min,
    241                     overhead_max)
    242             )
    243           else:
    244             symbol_children_overhead_requirements.append(
    245                 SymbolOverheadRequirement(
    246                     symbol_name,
    247                     comm,
    248                     overhead_min,
    249                     overhead_max))
    250       elif test_item.tag == 'symbol_callgraph_relation':
    251         for symbol_item in test_item:
    252           req = load_symbol_relation_requirement(symbol_item)
    253           symbol_relation_requirements.append(req)
    254 
    255     tests.append(
    256         Test(
    257             test_name,
    258             executable_name,
    259             report_options,
    260             symbol_overhead_requirements,
    261             symbol_children_overhead_requirements,
    262             symbol_relation_requirements))
    263   return tests
    264 
    265 
    266 def load_symbol_relation_requirement(symbol_item):
    267   symbol_name = symbol_item.attrib['name']
    268   comm = None
    269   if 'comm' in symbol_item.attrib:
    270     comm = symbol_item.attrib['comm']
    271   req = SymbolRelationRequirement(symbol_name, comm)
    272   for item in symbol_item:
    273     child_req = load_symbol_relation_requirement(item)
    274     req.add_child(child_req)
    275   return req
    276 
    277 
    278 class Runner(object):
    279 
    280   def __init__(self, target, perf_path):
    281     self.target = target
    282     self.is32 = target.endswith('32')
    283     self.perf_path = perf_path
    284     self.use_callgraph = False
    285     self.sampler = 'cpu-cycles'
    286 
    287   def record(self, test_executable_name, record_file, additional_options=[]):
    288     call_args = [self.perf_path, 'record']
    289     call_args += ['--duration', '2']
    290     call_args += ['-e', '%s:u' % self.sampler]
    291     if self.use_callgraph:
    292       call_args += ['-f', '1000', '-g']
    293     call_args += ['-o', record_file]
    294     call_args += additional_options
    295     test_executable_name += '32' if self.is32 else '64'
    296     call_args += [test_executable_name]
    297     self._call(call_args)
    298 
    299   def report(self, record_file, report_file, additional_options=[]):
    300     call_args = [self.perf_path, 'report']
    301     call_args += ['-i', record_file]
    302     if self.use_callgraph:
    303       call_args += ['-g', 'callee']
    304     call_args += additional_options
    305     self._call(call_args, report_file)
    306 
    307   def _call(self, args, output_file=None):
    308     pass
    309 
    310 
    311 class HostRunner(Runner):
    312 
    313   """Run perf test on host."""
    314 
    315   def __init__(self, target):
    316     perf_path = 'simpleperf32' if target.endswith('32') else 'simpleperf'
    317     super(HostRunner, self).__init__(target, perf_path)
    318 
    319   def _call(self, args, output_file=None):
    320     output_fh = None
    321     if output_file is not None:
    322       output_fh = open(output_file, 'w')
    323     subprocess.check_call(args, stdout=output_fh)
    324     if output_fh is not None:
    325       output_fh.close()
    326 
    327 
    328 class DeviceRunner(Runner):
    329 
    330   """Run perf test on device."""
    331 
    332   def __init__(self, target):
    333     self.tmpdir = '/data/local/tmp/'
    334     perf_path = 'simpleperf32' if target.endswith('32') else 'simpleperf'
    335     super(DeviceRunner, self).__init__(target, self.tmpdir + perf_path)
    336     self._download(os.environ['OUT'] + '/system/xbin/' + perf_path, self.tmpdir)
    337     lib = 'lib' if self.is32 else 'lib64'
    338     self._download(os.environ['OUT'] + '/system/' + lib + '/libsimpleperf_inplace_sampler.so',
    339                    self.tmpdir)
    340 
    341   def _call(self, args, output_file=None):
    342     output_fh = None
    343     if output_file is not None:
    344       output_fh = open(output_file, 'w')
    345     args_with_adb = ['adb', 'shell']
    346     args_with_adb.append('export LD_LIBRARY_PATH=' + self.tmpdir + ' && ' + ' '.join(args))
    347     subprocess.check_call(args_with_adb, stdout=output_fh)
    348     if output_fh is not None:
    349       output_fh.close()
    350 
    351   def _download(self, file, to_dir):
    352     args = ['adb', 'push', file, to_dir]
    353     subprocess.check_call(args)
    354 
    355   def record(self, test_executable_name, record_file, additional_options=[]):
    356     self._download(os.environ['OUT'] + '/system/bin/' + test_executable_name +
    357                    ('32' if self.is32 else '64'), self.tmpdir)
    358     super(DeviceRunner, self).record(self.tmpdir + test_executable_name,
    359                                      self.tmpdir + record_file,
    360                                      additional_options)
    361 
    362   def report(self, record_file, report_file, additional_options=[]):
    363     super(DeviceRunner, self).report(self.tmpdir + record_file,
    364                                      report_file,
    365                                      additional_options)
    366 
    367 class ReportAnalyzer(object):
    368 
    369   """Check if perf.report matches expectation in Configuration."""
    370 
    371   def _read_report_file(self, report_file, has_callgraph):
    372     fh = open(report_file, 'r')
    373     lines = fh.readlines()
    374     fh.close()
    375 
    376     lines = [x.rstrip() for x in lines]
    377     blank_line_index = -1
    378     for i in range(len(lines)):
    379       if not lines[i]:
    380         blank_line_index = i
    381     assert blank_line_index != -1
    382     assert blank_line_index + 1 < len(lines)
    383     title_line = lines[blank_line_index + 1]
    384     report_item_lines = lines[blank_line_index + 2:]
    385 
    386     if has_callgraph:
    387       assert re.search(r'^Children\s+Self\s+Command.+Symbol$', title_line)
    388     else:
    389       assert re.search(r'^Overhead\s+Command.+Symbol$', title_line)
    390 
    391     return self._parse_report_items(report_item_lines, has_callgraph)
    392 
    393   def _parse_report_items(self, lines, has_callgraph):
    394     symbols = []
    395     cur_symbol = None
    396     call_tree_stack = {}
    397     vertical_columns = []
    398     last_node = None
    399     last_depth = -1
    400 
    401     for line in lines:
    402       if not line:
    403         continue
    404       if not line[0].isspace():
    405         if has_callgraph:
    406           m = re.search(r'^([\d\.]+)%\s+([\d\.]+)%\s+(\S+).*\s+(\S+)$', line)
    407           children_overhead = float(m.group(1))
    408           overhead = float(m.group(2))
    409           comm = m.group(3)
    410           symbol_name = m.group(4)
    411           cur_symbol = Symbol(symbol_name, comm, overhead, children_overhead)
    412           symbols.append(cur_symbol)
    413         else:
    414           m = re.search(r'^([\d\.]+)%\s+(\S+).*\s+(\S+)$', line)
    415           overhead = float(m.group(1))
    416           comm = m.group(2)
    417           symbol_name = m.group(3)
    418           cur_symbol = Symbol(symbol_name, comm, overhead, 0)
    419           symbols.append(cur_symbol)
    420         # Each report item can have different column depths.
    421         vertical_columns = []
    422       else:
    423         for i in range(len(line)):
    424           if line[i] == '|':
    425             if not vertical_columns or vertical_columns[-1] < i:
    426               vertical_columns.append(i)
    427 
    428         if not line.strip('| \t'):
    429           continue
    430         if line.find('-') == -1:
    431           function_name = line.strip('| \t')
    432           node = CallTreeNode(function_name)
    433           last_node.add_child(node)
    434           last_node = node
    435           call_tree_stack[last_depth] = node
    436         else:
    437           pos = line.find('-')
    438           depth = -1
    439           for i in range(len(vertical_columns)):
    440             if pos >= vertical_columns[i]:
    441               depth = i
    442           assert depth != -1
    443 
    444           line = line.strip('|- \t')
    445           m = re.search(r'^[\d\.]+%[-\s]+(.+)$', line)
    446           if m:
    447             function_name = m.group(1)
    448           else:
    449             function_name = line
    450 
    451           node = CallTreeNode(function_name)
    452           if depth == 0:
    453             cur_symbol.set_call_tree(node)
    454 
    455           else:
    456             call_tree_stack[depth - 1].add_child(node)
    457           call_tree_stack[depth] = node
    458           last_node = node
    459           last_depth = depth
    460 
    461     return symbols
    462 
    463   def check_report_file(self, test, report_file, has_callgraph):
    464     symbols = self._read_report_file(report_file, has_callgraph)
    465     if not self._check_symbol_overhead_requirements(test, symbols):
    466       return False
    467     if has_callgraph:
    468       if not self._check_symbol_children_overhead_requirements(test, symbols):
    469         return False
    470       if not self._check_symbol_relation_requirements(test, symbols):
    471         return False
    472     return True
    473 
    474   def _check_symbol_overhead_requirements(self, test, symbols):
    475     result = True
    476     matched = [False] * len(test.symbol_overhead_requirements)
    477     matched_overhead = [0] * len(test.symbol_overhead_requirements)
    478     for symbol in symbols:
    479       for i in range(len(test.symbol_overhead_requirements)):
    480         req = test.symbol_overhead_requirements[i]
    481         if req.is_match(symbol):
    482           matched[i] = True
    483           matched_overhead[i] += symbol.overhead
    484     for i in range(len(matched)):
    485       if not matched[i]:
    486         print 'requirement (%s) has no matched symbol in test %s' % (
    487             test.symbol_overhead_requirements[i], test)
    488         result = False
    489       else:
    490         fulfilled = req.check_overhead(matched_overhead[i])
    491         if not fulfilled:
    492           print "Symbol (%s) doesn't match requirement (%s) in test %s" % (
    493               symbol, req, test)
    494           result = False
    495     return result
    496 
    497   def _check_symbol_children_overhead_requirements(self, test, symbols):
    498     result = True
    499     matched = [False] * len(test.symbol_children_overhead_requirements)
    500     for symbol in symbols:
    501       for i in range(len(test.symbol_children_overhead_requirements)):
    502         req = test.symbol_children_overhead_requirements[i]
    503         if req.is_match(symbol):
    504           matched[i] = True
    505           fulfilled = req.check_overhead(symbol.children_overhead)
    506           if not fulfilled:
    507             print "Symbol (%s) doesn't match requirement (%s) in test %s" % (
    508                 symbol, req, test)
    509             result = False
    510     for i in range(len(matched)):
    511       if not matched[i]:
    512         print 'requirement (%s) has no matched symbol in test %s' % (
    513             test.symbol_children_overhead_requirements[i], test)
    514         result = False
    515     return result
    516 
    517   def _check_symbol_relation_requirements(self, test, symbols):
    518     result = True
    519     matched = [False] * len(test.symbol_relation_requirements)
    520     for symbol in symbols:
    521       for i in range(len(test.symbol_relation_requirements)):
    522         req = test.symbol_relation_requirements[i]
    523         if req.is_match(symbol):
    524           matched[i] = True
    525           fulfilled = req.check_relation(symbol.call_tree)
    526           if not fulfilled:
    527             print "Symbol (%s) doesn't match requirement (%s) in test %s" % (
    528                 symbol, req, test)
    529             result = False
    530     for i in range(len(matched)):
    531       if not matched[i]:
    532         print 'requirement (%s) has no matched symbol in test %s' % (
    533             test.symbol_relation_requirements[i], test)
    534         result = False
    535     return result
    536 
    537 
    538 def build_runner(target, use_callgraph, sampler):
    539   if target == 'host32' and use_callgraph:
    540     print "Current 64bit linux host doesn't support `simpleperf32 record -g`"
    541     return None
    542   if target.startswith('host'):
    543     runner = HostRunner(target)
    544   else:
    545     runner = DeviceRunner(target)
    546   runner.use_callgraph = use_callgraph
    547   runner.sampler = sampler
    548   return runner
    549 
    550 
    551 def test_with_runner(runner, tests):
    552   report_analyzer = ReportAnalyzer()
    553   for test in tests:
    554     runner.record(test.executable_name, 'perf.data')
    555     if runner.sampler == 'inplace-sampler':
    556       # TODO: fix this when inplace-sampler actually works.
    557       runner.report('perf.data', 'perf.report')
    558       symbols = report_analyzer._read_report_file('perf.report', runner.use_callgraph)
    559       result = False
    560       if len(symbols) == 1 and symbols[0].name.find('FakeFunction()') != -1:
    561         result = True
    562     else:
    563       runner.report('perf.data', 'perf.report', additional_options = test.report_options)
    564       result = report_analyzer.check_report_file(test, 'perf.report', runner.use_callgraph)
    565     str = 'test %s on %s ' % (test.test_name, runner.target)
    566     if runner.use_callgraph:
    567       str += 'with call graph '
    568     str += 'using %s ' % runner.sampler
    569     str += ' Succeeded' if result else 'Failed'
    570     print str
    571     if not result:
    572       exit(1)
    573 
    574 
    575 def runtest(target_options, use_callgraph_options, sampler_options, selected_tests):
    576   tests = load_config_file(os.path.dirname(os.path.realpath(__file__)) + \
    577                            '/runtest.conf')
    578   if selected_tests is not None:
    579     new_tests = []
    580     for test in tests:
    581       if test.test_name in selected_tests:
    582         new_tests.append(test)
    583     tests = new_tests
    584   for target in target_options:
    585     for use_callgraph in use_callgraph_options:
    586       for sampler in sampler_options:
    587         runner = build_runner(target, use_callgraph, sampler)
    588         if runner is not None:
    589           test_with_runner(runner, tests)
    590 
    591 
    592 def main():
    593   target_options = ['host64', 'host32', 'device64', 'device32']
    594   use_callgraph_options = [False, True]
    595   sampler_options = ['cpu-cycles', 'inplace-sampler']
    596   selected_tests = None
    597   i = 1
    598   while i < len(sys.argv):
    599     if sys.argv[i] == '--host':
    600       target_options = ['host64', 'host32']
    601     elif sys.argv[i] == '--device':
    602       target_options = ['device64', 'device32']
    603     elif sys.argv[i] == '--normal':
    604       use_callgraph_options = [False]
    605     elif sys.argv[i] == '--callgraph':
    606       use_callgraph_options = [True]
    607     elif sys.argv[i] == '--no-inplace-sampler':
    608       sampler_options = ['cpu-cycles']
    609     elif sys.argv[i] == '--inplace-sampler':
    610       sampler_options = ['inplace-sampler']
    611     elif sys.argv[i] == '--test':
    612       if i < len(sys.argv):
    613         i += 1
    614         for test in sys.argv[i].split(','):
    615           if selected_tests is None:
    616             selected_tests = {}
    617           selected_tests[test] = True
    618     i += 1
    619   runtest(target_options, use_callgraph_options, sampler_options, selected_tests)
    620 
    621 if __name__ == '__main__':
    622   main()
    623