Home | History | Annotate | Download | only in layout_tests 
      1 #!/usr/bin/env python
      2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
      3 # Use of this source code is governed by a BSD-style license that can be
      4 # found in the LICENSE file.
      5 
      6 """Main function to run the layout test analyzer.
      7 
      8 The purpose of this script is to run the layout test analyzer for various
      9 teams based on the run configuration file in CSV format. The CSV file is based
     10 on https://sites.google.com/a/chromium.org/dev/developers/testing/
     11 webkit-layout-tests/layout-test-stats-1.
     12 """
     13 
     14 import optparse
     15 import os
     16 import shutil
     17 from subprocess import Popen
     18 
     19 # TODO(shadi): Re-examine the need of external files. Inline data instead?
     20 DEFAULT_RUN_CONFIG = {
     21     # test_group_name: ('test_files.csv', 'report_email_address')
     22     'media': ('testname/media.csv', 'layout-test-analyzer-result (at] google.com')
     23 }
     24 
     25 # Predefined result/graph directory.
     26 DEFAULT_RESULT_DIR = 'result'
     27 DEFAULT_GRAPH_DIR = 'graph'
     28 
     29 
     30 def ParseOption():
     31   """Parse command-line options using OptionParser.
     32 
     33   Returns:
     34       an object containing all command-line option information.
     35   """
     36   option_parser = optparse.OptionParser()
     37   option_parser.add_option('-d', '--result-directory-location',
     38                            dest='result_directory_location',
     39                            help=('Name of result directory location '
     40                                  '(default to %default)'),
     41                            default=DEFAULT_RESULT_DIR)
     42   option_parser.add_option('-p', '--graph-directory-location',
     43                            dest='graph_directory_location',
     44                            help=('Name of graph directory location '
     45                                  '(default to %default)'),
     46                            default=DEFAULT_GRAPH_DIR)
     47   option_parser.add_option('-e', '--email-only-change-mode',
     48                            dest='email_only_change_mode',
     49                            help=('With this mode, email is sent out '
     50                                  'only when there is a change in the '
     51                                  'analyzer result compared to the previous '
     52                                  'result (off by default)'),
     53                            action='store_true', default=False)
     54   option_parser.add_option('-z', '--issue-detail-mode',
     55                            dest='issue_detail_mode',
     56                            help=('With this mode, email includes issue details'
     57                                  ' including links to the flakiness dashboard'
     58                                  ' (off by default)'),
     59                            action='store_true', default=False)
     60   return option_parser.parse_args()[0]
     61 
     62 
     63 def GenerateDashboardHTMLFile(file_name, test_group_list):
     64   """Generate dashboard HTML file.
     65 
     66   Currently, it is simple table that shows all the analyzer results.
     67 
     68   Args:
     69     file_name: the file name of the dashboard.
     70     test_group_list: a list of test group names such as 'media' or 'composite'.
     71   """
     72   file_object = open(file_name, 'wb')
     73   legend_txt = """
     74 <style type="text/css">
     75 th {
     76   width: 30px; overflow: hidden;
     77 }
     78 tr.d0 td {
     79   background-color: #CC9999; color: black;
     80   text-align: right;
     81   width: 30px; overflow: hidden;
     82 }
     83 tr.d1 td {
     84   background-color: #9999CC; color: black;
     85   text-align: right;
     86   width: 30px; overflow: hidden;
     87 }
     88 </style>
     89 <h2>Chromium Layout Test Analyzer Result</h2>
     90 Legend:
     91 <ul>
     92 <li>#Tests: the number of tests for the given test group
     93 <li>#Skipped Tests: the number of tests that are skipped in the
     94 <a href='http://svn.webkit.org/repository/webkit/trunk/LayoutTests/platform/\
     95 chromium/test_expectations.txt'>test expectaion file</a> (e.g., BUGWK60877
     96 SKIP : loader/navigation-while-deferring-loads.html = FAIL)
     97 <li>#Non-Skipped Failing Tests: the number of tests that appeared in the
     98 test expectation file and were not skipped.
     99 <li>Failing rate: #NonSkippedFailing / (#Tests - #Skipped)
    100 <li>Passing rate: 100 - (Failing rate)
    101 </ul>
    102   """
    103   file_object.write(legend_txt)
    104   file_object.write('<table border="1">')
    105   file_object.write('<tr><th>Base Directory</th>')
    106   file_object.write('<th>Trend Graph</th>')
    107   file_object.write('<th>#Tests</th>')
    108   file_object.write('<th>#Skipped Tests</th>')
    109   file_object.write('<th>#Non-Skipped Failing Tests</th>')
    110   file_object.write('<th>Failing Rate</th>')
    111   file_object.write('<th>Passing Rate</th>')
    112   file_object.write('<th>Last Revision Number</th>')
    113   file_object.write('<th>Last Revision Date</th>')
    114   file_object.write('<th>Owner Email</th>')
    115   file_object.write('<th>Bug Information</th></tr>\n')
    116   test_group_list.sort()
    117   for i, test_group in enumerate(test_group_list):
    118     file_object.write('<tr class="d' + str(i % 2) + '">\n')
    119     file_object.write('<td>' + test_group + '</td>\n')
    120     file_object.write('</tr>\n')
    121   file_object.write('</table>')
    122   file_object.close()
    123 
    124 
    125 # TODO(shadi): Use only one file with main()! Remove this file in favor of
    126 # layouttest_analyzer.py main().
    127 def main():
    128   """A main function for the analyzer runner."""
    129   options = ParseOption()
    130   run_config_map = DEFAULT_RUN_CONFIG
    131   test_group_list = run_config_map.keys()
    132   dashboard_file_location = os.path.join(options.graph_directory_location,
    133                                          'index.html')
    134   if not os.path.exists(dashboard_file_location):
    135     GenerateDashboardHTMLFile(dashboard_file_location, test_group_list)
    136   for test_group in test_group_list:
    137     # Prepare the result if it does not exist.
    138     # The directory name should be changed to avoid collision
    139     # with the file separator.
    140     test_group_name_for_data = test_group.replace('/', '_')
    141     result_dir = os.path.join(options.result_directory_location,
    142                               test_group_name_for_data)
    143     if not os.path.exists(result_dir):
    144       os.mkdir(result_dir)
    145     graph_file = os.path.join(options.graph_directory_location,
    146                               test_group_name_for_data + '.html')
    147     if not os.path.exists(graph_file):
    148       # Copy the template file.
    149       shutil.copy(os.path.join('graph', 'graph.html'),
    150                   graph_file)
    151       os.chmod(graph_file, 0744)
    152     cmd = ('python layouttest_analyzer.py -x %s -d %s -t %s'
    153            ' -q %s ') % (
    154                test_group, result_dir, graph_file, dashboard_file_location)
    155     if run_config_map[test_group][0]:
    156       cmd += '-n ' + run_config_map[test_group][0] + ' '
    157     if run_config_map[test_group][1]:
    158       cmd += '-r ' + run_config_map[test_group][1] + ' '
    159     if options.email_only_change_mode:
    160       cmd += ' -c '
    161     if options.issue_detail_mode:
    162       cmd += ' -z '
    163     print 'Running ' + cmd
    164     proc = Popen(cmd, shell=True)
    165     proc.communicate()
    166 
    167 
    168 if '__main__' == __name__:
    169   main()
    170