Home | History | Annotate | Download | only in Scripts 
      1 #!/usr/bin/python
      2 import json
      3 import optparse
      4 import os
      5 import sys
      6 
      7 from webkitpy.common.host import Host
      8 from webkitpy.layout_tests.port import platform_options, configuration_options
      9 
     10 
     11 def main(argv):
     12     parser = optparse.OptionParser(usage='%prog [path-to-results.json]')
     13     parser.add_option('--failures', action='store_true',
     14                       help='show failing tests')
     15     parser.add_option('--flakes', action='store_true',
     16                       help='show flaky tests')
     17     parser.add_option('--expected', action='store_true',
     18                       help='include expected results along with unexpected')
     19     parser.add_option('--passes', action='store_true',
     20                       help='show passing tests')
     21     parser.add_option('--ignored-failures-path', action='store',
     22                       help='ignore failures seen in a previous run')
     23     parser.add_options(platform_options())
     24     parser.add_options(configuration_options())
     25     options, args = parser.parse_args(argv)
     26 
     27     host = Host()
     28     if args:
     29         if args[0] == '-':
     30             txt = sys.stdin.read()
     31         elif os.path.exists(args[0]):
     32             with open(args[0], 'r') as fp:
     33                 txt = fp.read()
     34         else:
     35             print >> sys.stderr, "file not found: %s" % args[0]
     36             sys.exit(1)
     37     else:
     38         txt = host.filesystem.read_text_file(host.filesystem.join(host.port_factory.get(options=options).results_directory(), 'full_results.json'))
     39 
     40     if txt.startswith('ADD_RESULTS(') and txt.endswith(');'):
     41         txt = txt[12:-2]  # ignore optional JSONP wrapper
     42     results = json.loads(txt)
     43 
     44     passes, failures, flakes = decode_results(results, options.expected)
     45 
     46     tests_to_print = []
     47     if options.passes:
     48         tests_to_print += passes.keys()
     49     if options.failures:
     50         tests_to_print += failures.keys()
     51     if options.flakes:
     52         tests_to_print += flakes.keys()
     53     print "\n".join(sorted(tests_to_print))
     54 
     55     if options.ignored_failures_path:
     56         with open(options.ignored_failures_path, 'r') as fp:
     57             txt = fp.read()
     58         if txt.startswith('ADD_RESULTS(') and txt.endswith(');'):
     59             txt = txt[12:-2]  # ignore optional JSONP wrapper
     60         results = json.loads(txt)
     61         _, ignored_failures, _ = decode_results(results, options.expected)
     62         new_failures = set(failures.keys()) - set(ignored_failures.keys())
     63         if new_failures:
     64             print "New failures:"
     65             print "\n".join(sorted(new_failures))
     66             print
     67         if ignored_failures:
     68             print "Ignored failures:"
     69             print "\n".join(sorted(ignored_failures.keys()))
     70         if new_failures:
     71             return 1
     72         return 0
     73 
     74 
     75 def decode_results(results, include_expected=False):
     76     tests = convert_trie_to_flat_paths(results['tests'])
     77     failures = {}
     78     flakes = {}
     79     passes = {}
     80     for (test, result) in tests.iteritems():
     81         if include_expected or result.get('is_unexpected'):
     82             actual_results = result['actual'].split()
     83             expected_results = result['expected'].split()
     84             if len(actual_results) > 1:
     85                 if actual_results[1] in expected_results:
     86                     flakes[test] = actual_results[0]
     87                 else:
     88                     # We report the first failure type back, even if the second
     89                     # was more severe.
     90                     failures[test] = actual_results[0]
     91             elif actual_results[0] == 'PASS':
     92                 passes[test] = result
     93             else:
     94                 failures[test] = actual_results[0]
     95 
     96     return (passes, failures, flakes)
     97 
     98 
     99 def convert_trie_to_flat_paths(trie, prefix=None):
    100     # Cloned from webkitpy.layout_tests.layout_package.json_results_generator
    101     # so that this code can stand alone.
    102     result = {}
    103     for name, data in trie.iteritems():
    104         if prefix:
    105             name = prefix + "/" + name
    106 
    107         if len(data) and not "actual" in data and not "expected" in data:
    108             result.update(convert_trie_to_flat_paths(data, name))
    109         else:
    110             result[name] = data
    111 
    112     return result
    113 
    114 
    115 if __name__ ==  '__main__':
    116     sys.exit(main(sys.argv[1:]))
    117