Home | History | Annotate | Download | only in bindings 
      1 # Copyright (C) 2011 Google Inc.  All rights reserved.
      2 #
      3 # Redistribution and use in source and binary forms, with or without
      4 # modification, are permitted provided that the following conditions
      5 # are met:
      6 # 1. Redistributions of source code must retain the above copyright
      7 #    notice, this list of conditions and the following disclaimer.
      8 # 2. Redistributions in binary form must reproduce the above copyright
      9 #    notice, this list of conditions and the following disclaimer in the
     10 #    documentation and/or other materials provided with the distribution.
     11 #
     12 # THIS SOFTWARE IS PROVIDED BY APPLE COMPUTER, INC. ``AS IS'' AND ANY
     13 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     14 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     15 # PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE COMPUTER, INC. OR
     16 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
     17 # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
     18 # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
     19 # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
     20 # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     21 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
     22 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     23 #
     24 
     25 from contextlib import contextmanager
     26 import filecmp
     27 import fnmatch
     28 import os
     29 import shutil
     30 import sys
     31 import tempfile
     32 
     33 from webkitpy.common.system.executive import Executive
     34 
     35 # Source/ path is needed both to find input IDL files, and to import other
     36 # Python modules.
     37 module_path = os.path.dirname(__file__)
     38 source_path = os.path.normpath(os.path.join(module_path, os.pardir, os.pardir,
     39                                             os.pardir, os.pardir, 'Source'))
     40 sys.path.append(source_path)  # for Source/bindings imports
     41 
     42 import bindings.scripts.compute_interfaces_info_individual
     43 from bindings.scripts.compute_interfaces_info_individual import compute_info_individual, info_individual
     44 import bindings.scripts.compute_interfaces_info_overall
     45 from bindings.scripts.compute_interfaces_info_overall import compute_interfaces_info_overall, interfaces_info
     46 from bindings.scripts.idl_compiler import IdlCompilerV8
     47 
     48 
     49 PASS_MESSAGE = 'All tests PASS!'
     50 FAIL_MESSAGE = """Some tests FAIL!
     51 To update the reference files, execute:
     52     run-bindings-tests --reset-results
     53 
     54 If the failures are not due to your changes, test results may be out of sync;
     55 please rebaseline them in a separate CL, after checking that tests fail in ToT.
     56 In CL, please set:
     57 NOTRY=true
     58 TBR=(someone in Source/bindings/OWNERS or WATCHLISTS:bindings)
     59 """
     60 
     61 DEPENDENCY_IDL_FILES = frozenset([
     62     'TestImplements.idl',
     63     'TestImplements2.idl',
     64     'TestImplements3.idl',
     65     'TestPartialInterface.idl',
     66     'TestPartialInterface2.idl',
     67 ])
     68 
     69 
     70 test_input_directory = os.path.join(source_path, 'bindings', 'tests', 'idls')
     71 reference_directory = os.path.join(source_path, 'bindings', 'tests', 'results')
     72 
     73 
     74 @contextmanager
     75 def TemporaryDirectory():
     76     """Wrapper for tempfile.mkdtemp() so it's usable with 'with' statement.
     77 
     78     Simple backport of tempfile.TemporaryDirectory from Python 3.2.
     79     """
     80     name = tempfile.mkdtemp()
     81     try:
     82         yield name
     83     finally:
     84         shutil.rmtree(name)
     85 
     86 
     87 def generate_interface_dependencies():
     88     def idl_paths_recursive(directory):
     89         # This is slow, especially on Windows, due to os.walk making
     90         # excess stat() calls. Faster versions may appear in Python 3.5 or
     91         # later:
     92         # https://github.com/benhoyt/scandir
     93         # http://bugs.python.org/issue11406
     94         idl_paths = []
     95         for dirpath, _, files in os.walk(directory):
     96             idl_paths.extend(os.path.join(dirpath, filename)
     97                              for filename in fnmatch.filter(files, '*.idl'))
     98         return idl_paths
     99 
    100     # We compute interfaces info for *all* IDL files, not just test IDL
    101     # files, as code generator output depends on inheritance (both ancestor
    102     # chain and inherited extended attributes), and some real interfaces
    103     # are special-cased, such as Node.
    104     #
    105     # For example, when testing the behavior of interfaces that inherit
    106     # from Node, we also need to know that these inherit from EventTarget,
    107     # since this is also special-cased and Node inherits from EventTarget,
    108     # but this inheritance information requires computing dependencies for
    109     # the real Node.idl file.
    110 
    111     # 2-stage computation: individual, then overall
    112     #
    113     # Properly should compute separately by component (currently test
    114     # includes are invalid), but that's brittle (would need to update this file
    115     # for each new component) and doesn't test the code generator any better
    116     # than using a single component.
    117     for idl_filename in idl_paths_recursive(source_path):
    118         compute_info_individual(idl_filename, 'tests')
    119     info_individuals = [info_individual()]
    120     compute_interfaces_info_overall(info_individuals)
    121 
    122 
    123 def bindings_tests(output_directory, verbose):
    124     executive = Executive()
    125 
    126     def diff(filename1, filename2):
    127         # Python's difflib module is too slow, especially on long output, so
    128         # run external diff(1) command
    129         cmd = ['diff',
    130                '-u',  # unified format
    131                '-N',  # treat absent files as empty
    132                filename1,
    133                filename2]
    134         # Return output and don't raise exception, even though diff(1) has
    135         # non-zero exit if files differ.
    136         return executive.run_command(cmd, error_handler=lambda x: None)
    137 
    138     def delete_cache_files():
    139         # FIXME: Instead of deleting cache files, don't generate them.
    140         cache_files = [os.path.join(output_directory, output_file)
    141                        for output_file in os.listdir(output_directory)
    142                        if (output_file in ('lextab.py',  # PLY lex
    143                                            'lextab.pyc',
    144                                            'parsetab.pickle') or  # PLY yacc
    145                            output_file.endswith('.cache'))]  # Jinja
    146         for cache_file in cache_files:
    147             os.remove(cache_file)
    148 
    149     def identical_file(reference_filename, output_filename):
    150         reference_basename = os.path.basename(reference_filename)
    151 
    152         if not os.path.isfile(reference_filename):
    153             print 'Missing reference file!'
    154             print '(if adding new test, update reference files)'
    155             print reference_basename
    156             print
    157             return False
    158 
    159         if not filecmp.cmp(reference_filename, output_filename):
    160             # cmp is much faster than diff, and usual case is "no differance",
    161             # so only run diff if cmp detects a difference
    162             print 'FAIL: %s' % reference_basename
    163             print diff(reference_filename, output_filename)
    164             return False
    165 
    166         if verbose:
    167             print 'PASS: %s' % reference_basename
    168         return True
    169 
    170     def identical_output_files():
    171         file_pairs = [(os.path.join(reference_directory, output_file),
    172                        os.path.join(output_directory, output_file))
    173                       for output_file in os.listdir(output_directory)]
    174         return all([identical_file(reference_filename, output_filename)
    175                     for (reference_filename, output_filename) in file_pairs])
    176 
    177     def no_excess_files():
    178         generated_files = set(os.listdir(output_directory))
    179         generated_files.add('.svn')  # Subversion working copy directory
    180         excess_files = [output_file
    181                         for output_file in os.listdir(reference_directory)
    182                         if output_file not in generated_files]
    183         if excess_files:
    184             print ('Excess reference files! '
    185                   '(probably cruft from renaming or deleting):\n' +
    186                   '\n'.join(excess_files))
    187             return False
    188         return True
    189 
    190     try:
    191         generate_interface_dependencies()
    192         idl_compiler = IdlCompilerV8(output_directory,
    193                                      interfaces_info=interfaces_info,
    194                                      only_if_changed=True)
    195 
    196         idl_basenames = [filename
    197                          for filename in os.listdir(test_input_directory)
    198                          if (filename.endswith('.idl') and
    199                              # Dependencies aren't built
    200                              # (they are used by the dependent)
    201                              filename not in DEPENDENCY_IDL_FILES)]
    202         for idl_basename in idl_basenames:
    203             idl_path = os.path.realpath(
    204                 os.path.join(test_input_directory, idl_basename))
    205             idl_compiler.compile_file(idl_path)
    206             if verbose:
    207                 print 'Compiled: %s' % filename
    208     finally:
    209         delete_cache_files()
    210 
    211     # Detect all changes
    212     passed = identical_output_files()
    213     passed &= no_excess_files()
    214 
    215     if passed:
    216         if verbose:
    217             print
    218             print PASS_MESSAGE
    219         return 0
    220     print
    221     print FAIL_MESSAGE
    222     return 1
    223 
    224 
    225 def run_bindings_tests(reset_results, verbose):
    226     # Generate output into the reference directory if resetting results, or
    227     # a temp directory if not.
    228     if reset_results:
    229         print 'Resetting results'
    230         return bindings_tests(reference_directory, verbose)
    231     with TemporaryDirectory() as temp_dir:
    232         return bindings_tests(temp_dir, verbose)
    233