Home | History | Annotate | Download | only in outproc
      1 # Copyright 2017 the V8 project authors. All rights reserved.
      2 # Use of this source code is governed by a BSD-style license that can be
      3 # found in the LICENSE file.
      4 
      5 import itertools
      6 
      7 from ..testproc.base import (
      8     DROP_RESULT, DROP_OUTPUT, DROP_PASS_OUTPUT, DROP_PASS_STDOUT)
      9 from ..local import statusfile
     10 from ..testproc.result import Result
     11 
     12 
     13 OUTCOMES_PASS = [statusfile.PASS]
     14 OUTCOMES_FAIL = [statusfile.FAIL]
     15 OUTCOMES_PASS_OR_TIMEOUT = [statusfile.PASS, statusfile.TIMEOUT]
     16 OUTCOMES_FAIL_OR_TIMEOUT = [statusfile.FAIL, statusfile.TIMEOUT]
     17 
     18 
     19 class BaseOutProc(object):
     20   def process(self, output, reduction=None):
     21     has_unexpected_output = self.has_unexpected_output(output)
     22     return self._create_result(has_unexpected_output, output, reduction)
     23 
     24   def has_unexpected_output(self, output):
     25     return self.get_outcome(output) not in self.expected_outcomes
     26 
     27   def _create_result(self, has_unexpected_output, output, reduction):
     28     """Creates Result instance. When reduction is passed it tries to drop some
     29     parts of the result to save memory and time needed to send the result
     30     across process boundary. None disables reduction and full result is created.
     31     """
     32     if reduction == DROP_RESULT:
     33       return None
     34     if reduction == DROP_OUTPUT:
     35       return Result(has_unexpected_output, None)
     36     if not has_unexpected_output:
     37       if reduction == DROP_PASS_OUTPUT:
     38         return Result(has_unexpected_output, None)
     39       if reduction == DROP_PASS_STDOUT:
     40         return Result(has_unexpected_output, output.without_text())
     41 
     42     return Result(has_unexpected_output, output)
     43 
     44   def get_outcome(self, output):
     45     if output.HasCrashed():
     46       return statusfile.CRASH
     47     elif output.HasTimedOut():
     48       return statusfile.TIMEOUT
     49     elif self._has_failed(output):
     50       return statusfile.FAIL
     51     else:
     52       return statusfile.PASS
     53 
     54   def _has_failed(self, output):
     55     execution_failed = self._is_failure_output(output)
     56     if self.negative:
     57       return not execution_failed
     58     return execution_failed
     59 
     60   def _is_failure_output(self, output):
     61     return output.exit_code != 0
     62 
     63   @property
     64   def negative(self):
     65     return False
     66 
     67   @property
     68   def expected_outcomes(self):
     69     raise NotImplementedError()
     70 
     71 
     72 class Negative(object):
     73   @property
     74   def negative(self):
     75     return True
     76 
     77 
     78 class PassOutProc(BaseOutProc):
     79   """Output processor optimized for positive tests expected to PASS."""
     80   def has_unexpected_output(self, output):
     81     return self.get_outcome(output) != statusfile.PASS
     82 
     83   @property
     84   def expected_outcomes(self):
     85     return OUTCOMES_PASS
     86 
     87 
     88 class NegPassOutProc(Negative, PassOutProc):
     89   """Output processor optimized for negative tests expected to PASS"""
     90   pass
     91 
     92 
     93 class OutProc(BaseOutProc):
     94   """Output processor optimized for positive tests with expected outcomes
     95   different than a single PASS.
     96   """
     97   def __init__(self, expected_outcomes):
     98     self._expected_outcomes = expected_outcomes
     99 
    100   @property
    101   def expected_outcomes(self):
    102     return self._expected_outcomes
    103 
    104   # TODO(majeski): Inherit from PassOutProc in case of OUTCOMES_PASS and remove
    105   # custom get/set state.
    106   def __getstate__(self):
    107     d = self.__dict__
    108     if self._expected_outcomes is OUTCOMES_PASS:
    109       d = d.copy()
    110       del d['_expected_outcomes']
    111     return d
    112 
    113   def __setstate__(self, d):
    114     if '_expected_outcomes' not in d:
    115       d['_expected_outcomes'] = OUTCOMES_PASS
    116     self.__dict__.update(d)
    117 
    118 
    119 # TODO(majeski): Override __reduce__ to make it deserialize as one instance.
    120 DEFAULT = PassOutProc()
    121 DEFAULT_NEGATIVE = NegPassOutProc()
    122 
    123 
    124 class ExpectedOutProc(OutProc):
    125   """Output processor that has is_failure_output depending on comparing the
    126   output with the expected output.
    127   """
    128   def __init__(self, expected_outcomes, expected_filename):
    129     super(ExpectedOutProc, self).__init__(expected_outcomes)
    130     self._expected_filename = expected_filename
    131 
    132   def _is_failure_output(self, output):
    133     with open(self._expected_filename, 'r') as f:
    134       expected_lines = f.readlines()
    135 
    136     for act_iterator in self._act_block_iterator(output):
    137       for expected, actual in itertools.izip_longest(
    138           self._expected_iterator(expected_lines),
    139           act_iterator,
    140           fillvalue=''
    141       ):
    142         if expected != actual:
    143           return True
    144       return False
    145 
    146   def _act_block_iterator(self, output):
    147     """Iterates over blocks of actual output lines."""
    148     lines = output.stdout.splitlines()
    149     start_index = 0
    150     found_eqeq = False
    151     for index, line in enumerate(lines):
    152       # If a stress test separator is found:
    153       if line.startswith('=='):
    154         # Iterate over all lines before a separator except the first.
    155         if not found_eqeq:
    156           found_eqeq = True
    157         else:
    158           yield self._actual_iterator(lines[start_index:index])
    159         # The next block of output lines starts after the separator.
    160         start_index = index + 1
    161     # Iterate over complete output if no separator was found.
    162     if not found_eqeq:
    163       yield self._actual_iterator(lines)
    164 
    165   def _actual_iterator(self, lines):
    166     return self._iterator(lines, self._ignore_actual_line)
    167 
    168   def _expected_iterator(self, lines):
    169     return self._iterator(lines, self._ignore_expected_line)
    170 
    171   def _ignore_actual_line(self, line):
    172     """Ignore empty lines, valgrind output, Android output and trace
    173     incremental marking output.
    174     """
    175     if not line:
    176       return True
    177     return (line.startswith('==') or
    178             line.startswith('**') or
    179             line.startswith('ANDROID') or
    180             line.startswith('###') or
    181             # FIXME(machenbach): The test driver shouldn't try to use slow
    182             # asserts if they weren't compiled. This fails in optdebug=2.
    183             line == 'Warning: unknown flag --enable-slow-asserts.' or
    184             line == 'Try --help for options')
    185 
    186   def _ignore_expected_line(self, line):
    187     return not line
    188 
    189   def _iterator(self, lines, ignore_predicate):
    190     for line in lines:
    191       line = line.strip()
    192       if not ignore_predicate(line):
    193         yield line
    194