Home | History | Annotate | Download | only in cts-media 
      1 #!/usr/bin/python
      2 # Copyright (C) 2015 The Android Open Source Project
      3 #
      4 # Licensed under the Apache License, Version 2.0 (the "License");
      5 # you may not use this file except in compliance with the License.
      6 # You may obtain a copy of the License at
      7 #
      8 #      http://www.apache.org/licenses/LICENSE-2.0
      9 #
     10 # Unless required by applicable law or agreed to in writing, software
     11 # distributed under the License is distributed on an "AS IS" BASIS,
     12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13 # See the License for the specific language governing permissions and
     14 # limitations under the License.
     15 #
     16 
     17 import argparse, math, re, sys
     18 import xml.etree.ElementTree as ET
     19 from collections import defaultdict, namedtuple
     20 import itertools
     21 
     22 
     23 def createLookup(values, key):
     24   """Creates a lookup table for a collection of values based on keys.
     25 
     26   Arguments:
     27     values: a collection of arbitrary values. Must be iterable.
     28     key: a function of one argument that returns the key for a value.
     29 
     30   Returns:
     31     A dict mapping keys (as generated by the key argument) to lists of
     32     values. All values in the lists have the same key, and are in the order
     33     they appeared in the collection.
     34   """
     35   lookup = defaultdict(list)
     36   for v in values:
     37     lookup[key(v)].append(v)
     38   return lookup
     39 
     40 
     41 def _intify(value):
     42   """Returns a value converted to int if possible, else the original value."""
     43   try:
     44     return int(value)
     45   except ValueError:
     46     return value
     47 
     48 
     49 class Size(namedtuple('Size', ['width', 'height'])):
     50   """A namedtuple with width and height fields."""
     51   def __str__(self):
     52     return '%dx%d' % (self.width, self.height)
     53 
     54 
     55 class _VideoResultBase(object):
     56   """Helper methods for results. Not for use by applications.
     57 
     58   Attributes:
     59     codec: The name of the codec (string) or None
     60     size: Size representing the video size or None
     61     mime: The mime-type of the codec (string) or None
     62     rates: The measured achievable frame rates
     63     is_decoder: True iff codec is a decoder.
     64   """
     65 
     66   def __init__(self, is_decoder):
     67     self.codec = None
     68     self.mime = None
     69     self.size = None
     70     self._rates_from_failure = []
     71     self._rates_from_message = []
     72     self.is_decoder = is_decoder
     73 
     74   def _inited(self):
     75     """Returns true iff codec, mime and size was set."""
     76     return None not in (self.codec, self.mime, self.size)
     77 
     78   def __len__(self):
     79     # don't report any result if codec name, mime type and size is unclear
     80     if not self._inited():
     81       return 0
     82     return len(self.rates)
     83 
     84   @property
     85   def rates(self):
     86     return self._rates_from_failure or self._rates_from_message
     87 
     88   def _parseDict(self, value):
     89     """Parses a MediaFormat from its string representation sans brackets."""
     90     return dict((k, _intify(v))
     91                 for k, v in re.findall(r'([^ =]+)=([^ [=]+(?:|\[[^\]]+\]))(?:, |$)', value))
     92 
     93   def _cleanFormat(self, format):
     94     """Removes internal fields from a parsed MediaFormat."""
     95     format.pop('what', None)
     96     format.pop('image-data', None)
     97 
     98   MESSAGE_PATTERN = r'(?P<key>\w+)=(?P<value>\{[^}]*\}|[^ ,{}]+)'
     99 
    100   def _parsePartialResult(self, message_match):
    101     """Parses a partial test result conforming to the message pattern.
    102 
    103     Returns:
    104       A tuple of string key and int, string or dict value, where dict has
    105       string keys mapping to int or string values.
    106     """
    107     key, value = message_match.group('key', 'value')
    108     if value.startswith('{'):
    109       value = self._parseDict(value[1:-1])
    110       if key.endswith('Format'):
    111         self._cleanFormat(value)
    112     else:
    113       value = _intify(value)
    114     return key, value
    115 
    116   def _parseValuesFromBracket(self, line):
    117     """Returns the values enclosed in brackets without the brackets.
    118 
    119     Parses a line matching the pattern "<tag>: [<values>]" and returns <values>.
    120 
    121     Raises:
    122       ValueError: if the line does not match the pattern.
    123     """
    124     try:
    125       return re.match(r'^[^:]+: *\[(?P<values>.*)\]\.$', line).group('values')
    126     except AttributeError:
    127       raise ValueError('line does not match "tag: [value]": %s' % line)
    128 
    129   def _parseRawData(self, line):
    130     """Parses the raw data line for video performance tests.
    131 
    132     Yields:
    133       Dict objects corresponding to parsed results, mapping string keys to
    134       int, string or dict values.
    135     """
    136     try:
    137       values = self._parseValuesFromBracket(line)
    138       result = {}
    139       for m in re.finditer(self.MESSAGE_PATTERN + r'(?P<sep>,? +|$)', values):
    140         key, value = self._parsePartialResult(m)
    141         result[key] = value
    142         if m.group('sep') != ' ':
    143           yield result
    144           result = {}
    145     except ValueError:
    146       print >> sys.stderr, 'could not parse line %s' % repr(line)
    147 
    148   def _tryParseMeasuredFrameRate(self, line):
    149     """Parses a line starting with 'Measured frame rate:'."""
    150     if line.startswith('Measured frame rate: '):
    151       try:
    152         values = self._parseValuesFromBracket(line)
    153         values = re.split(r' *, *', values)
    154         self._rates_from_failure = list(map(float, values))
    155       except ValueError:
    156         print >> sys.stderr, 'could not parse line %s' % repr(line)
    157 
    158   def parse(self, test):
    159     """Parses the ValueArray and FailedScene lines of a test result.
    160 
    161     Arguments:
    162       test: An ElementTree <Test> element.
    163     """
    164     failure = test.find('FailedScene')
    165     if failure is not None:
    166       trace = failure.find('StackTrace')
    167       if trace is not None:
    168         for line in re.split(r'[\r\n]+', trace.text):
    169           self._parseFailureLine(line)
    170     details = test.find('Details')
    171     if details is not None:
    172       for array in details.iter('ValueArray'):
    173         message = array.get('message')
    174         self._parseMessage(message, array)
    175 
    176   def _parseFailureLine(self, line):
    177     raise NotImplementedError
    178 
    179   def _parseMessage(self, message, array):
    180     raise NotImplementedError
    181 
    182   def getData(self):
    183     """Gets the parsed test result data.
    184 
    185     Yields:
    186        Result objects containing at least codec, size, mime and rates attributes."""
    187     yield self
    188 
    189 
    190 class VideoEncoderDecoderTestResult(_VideoResultBase):
    191   """Represents a result from a VideoEncoderDecoderTest performance case."""
    192 
    193   def __init__(self, unused_m):
    194     super(VideoEncoderDecoderTestResult, self).__init__(is_decoder=False)
    195 
    196   # If a VideoEncoderDecoderTest succeeds, it provides the results in the
    197   # message of a ValueArray. If fails, it provides the results in the failure
    198   # using raw data. (For now it also includes some data in the ValueArrays even
    199   # if it fails, which we ignore.)
    200 
    201   def _parseFailureLine(self, line):
    202     """Handles parsing a line from the failure log."""
    203     self._tryParseMeasuredFrameRate(line)
    204     if line.startswith('Raw data: '):
    205       for result in self._parseRawData(line):
    206         fmt = result['EncOutputFormat']
    207         self.size = Size(fmt['width'], fmt['height'])
    208         self.codec = result['codec']
    209         self.mime = fmt['mime']
    210 
    211   def _parseMessage(self, message, array):
    212     """Handles parsing a message from ValueArrays."""
    213     if message.startswith('codec='):
    214       result = dict(self._parsePartialResult(m)
    215                   for m in re.finditer(self.MESSAGE_PATTERN + '(?: |$)', message))
    216       if 'EncInputFormat' in result:
    217         self.codec = result['codec']
    218         fmt = result['EncInputFormat']
    219         self.size = Size(fmt['width'], fmt['height'])
    220         self.mime = result['EncOutputFormat']['mime']
    221         self._rates_from_message.append(1000000./result['min'])
    222 
    223 
    224 class VideoDecoderPerfTestResult(_VideoResultBase):
    225   """Represents a result from a VideoDecoderPerfTest performance case."""
    226 
    227   # If a VideoDecoderPerfTest succeeds, it provides the results in the message
    228   # of a ValueArray. If fails, it provides the results in the failure only
    229   # using raw data.
    230 
    231   def __init__(self, unused_m):
    232     super(VideoDecoderPerfTestResult, self).__init__(is_decoder=True)
    233 
    234   def _parseFailureLine(self, line):
    235     """Handles parsing a line from the failure log."""
    236     self._tryParseMeasuredFrameRate(line)
    237     # if the test failed, we can only get the codec/size/mime from the raw data.
    238     if line.startswith('Raw data: '):
    239       for result in self._parseRawData(line):
    240         fmt = result['DecOutputFormat']
    241         self.size = Size(fmt['width'], fmt['height'])
    242         self.codec = result['codec']
    243         self.mime = result['mime']
    244 
    245   def _parseMessage(self, message, array):
    246     """Handles parsing a message from ValueArrays."""
    247     if message.startswith('codec='):
    248       result = dict(self._parsePartialResult(m)
    249                   for m in re.finditer(self.MESSAGE_PATTERN + '(?: |$)', message))
    250       if result.get('decodeto') == 'surface':
    251         self.codec = result['codec']
    252         fmt = result['DecOutputFormat']
    253         self.size = Size(fmt['width'], fmt['height'])
    254         self.mime = result['mime']
    255         self._rates_from_message.append(1000000. / result['min'])
    256 
    257 
    258 class Results(object):
    259   """Container that keeps all test results."""
    260   def __init__(self):
    261       self._results = [] # namedtuples
    262       self._device = None
    263 
    264   VIDEO_ENCODER_DECODER_TEST_REGEX = re.compile(
    265       'test(.*)(\d{4})x(\d{4})(Goog|Other)$')
    266 
    267   VIDEO_DECODER_PERF_TEST_REGEX = re.compile(
    268       'test(VP[89]|H26[34]|MPEG4|HEVC)(\d+)x(\d+)(.*)$')
    269 
    270   TestCaseSpec = namedtuple('TestCaseSpec', 'package path class_ regex result_class')
    271 
    272   def _getTestCases(self):
    273     return [
    274       self.TestCaseSpec(package='CtsDeviceVideoPerf',
    275                    path='TestSuite/TestSuite/TestSuite/TestSuite/TestCase',
    276                    class_='VideoEncoderDecoderTest',
    277                    regex=self.VIDEO_ENCODER_DECODER_TEST_REGEX,
    278                    result_class=VideoEncoderDecoderTestResult),
    279       self.TestCaseSpec(package='CtsMediaTestCases',
    280                    path='TestSuite/TestSuite/TestSuite/TestCase',
    281                    class_='VideoDecoderPerfTest',
    282                    regex=self.VIDEO_DECODER_PERF_TEST_REGEX,
    283                    result_class=VideoDecoderPerfTestResult)
    284     ]
    285 
    286   def _verifyDeviceInfo(self, device):
    287     assert self._device in (None, device), "expected %s device" % self._device
    288     self._device = device
    289 
    290   def importXml(self, xml):
    291     self._verifyDeviceInfo(xml.find('DeviceInfo/BuildInfo').get('buildName'))
    292 
    293     packages = createLookup(self._getTestCases(), lambda tc: tc.package)
    294 
    295     for pkg in xml.iter('TestPackage'):
    296       tests_in_package = packages.get(pkg.get('name'))
    297       if not tests_in_package:
    298         continue
    299       paths = createLookup(tests_in_package, lambda tc: tc.path)
    300       for path, tests_in_path in paths.items():
    301         classes = createLookup(tests_in_path, lambda tc: tc.class_)
    302         for tc in pkg.iterfind(path):
    303           tests_in_class = classes.get(tc.get('name'))
    304           if not tests_in_class:
    305             continue
    306           for test in tc.iter('Test'):
    307             for tc in tests_in_class:
    308               m = tc.regex.match(test.get('name'))
    309               if m:
    310                 result = tc.result_class(m)
    311                 result.parse(test)
    312                 self._results.append(result)
    313 
    314   def importFile(self, path):
    315     print >> sys.stderr, 'Importing "%s"...' % path
    316     try:
    317       return self.importXml(ET.parse(path))
    318     except ET.ParseError:
    319       raise ValueError('not a valid XML file')
    320 
    321   def getData(self):
    322     for result in self._results:
    323       for data in result.getData():
    324         yield data
    325 
    326   def dumpXml(self, results):
    327     yield '<?xml version="1.0" encoding="utf-8" ?>'
    328     yield '<!-- Copyright 2015 The Android Open Source Project'
    329     yield ''
    330     yield '     Licensed under the Apache License, Version 2.0 (the "License");'
    331     yield '     you may not use this file except in compliance with the License.'
    332     yield '     You may obtain a copy of the License at'
    333     yield ''
    334     yield '          http://www.apache.org/licenses/LICENSE-2.0'
    335     yield ''
    336     yield '     Unless required by applicable law or agreed to in writing, software'
    337     yield '     distributed under the License is distributed on an "AS IS" BASIS,'
    338     yield '     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.'
    339     yield '     See the License for the specific language governing permissions and'
    340     yield '     limitations under the License.'
    341     yield '-->'
    342     yield ''
    343     yield '<MediaCodecs>'
    344     last_section = None
    345     Comp = namedtuple('Comp', 'is_decoder google mime name')
    346     by_comp = createLookup(results,
    347                            lambda e: Comp(is_decoder=e.is_decoder, google='.google.' in e.codec, mime=e.mime, name=e.codec))
    348     for comp in sorted(by_comp):
    349       section = 'Decoders' if comp.is_decoder else 'Encoders'
    350       if section != last_section:
    351         if last_section:
    352           yield '    </%s>' % last_section
    353         yield '    <%s>' % section
    354         last_section = section
    355       yield '        <MediaCodec name="%s" type="%s" update="true">' % (comp.name, comp.mime)
    356       by_size = createLookup(by_comp[comp], lambda e: e.size)
    357       for size in sorted(by_size):
    358         values = list(itertools.chain(*(e.rates for e in by_size[size])))
    359         min_, max_ = min(values), max(values)
    360         med_ = int(math.sqrt(min_ * max_))
    361         yield '            <Limit name="measured-frame-rate-%s" range="%d-%d" />' % (size, med_, med_)
    362       yield '        </MediaCodec>'
    363     if last_section:
    364       yield '    </%s>' % last_section
    365     yield '</MediaCodecs>'
    366 
    367 
    368 class Main(object):
    369   """Executor of this utility."""
    370 
    371   def __init__(self):
    372     self._result = Results()
    373 
    374     self._parser = argparse.ArgumentParser('get_achievable_framerates')
    375     self._parser.add_argument('result_xml', nargs='+')
    376 
    377   def _parseArgs(self):
    378     self._args = self._parser.parse_args()
    379 
    380   def _importXml(self, xml):
    381     self._result.importFile(xml)
    382 
    383   def _report(self):
    384     for line in self._result.dumpXml(r for r in self._result.getData() if r):
    385       print line
    386 
    387   def run(self):
    388     self._parseArgs()
    389     try:
    390       for xml in self._args.result_xml:
    391         try:
    392           self._importXml(xml)
    393         except (ValueError, IOError, AssertionError) as e:
    394           print >> sys.stderr, e
    395           raise KeyboardInterrupt
    396       self._report()
    397     except KeyboardInterrupt:
    398       print >> sys.stderr, 'Interrupted.'
    399 
    400 if __name__ == '__main__':
    401   Main().run()
    402 
    403