1 #!/usr/bin/env python 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 """Main function to run the layout test analyzer. 7 8 The purpose of this script is to run the layout test analyzer for various 9 teams based on the run configuration file in CSV format. The CSV file is based 10 on https://sites.google.com/a/chromium.org/dev/developers/testing/ 11 webkit-layout-tests/layout-test-stats-1. 12 """ 13 14 import optparse 15 import os 16 import shutil 17 from subprocess import Popen 18 19 # TODO(shadi): Re-examine the need of external files. Inline data instead? 20 DEFAULT_RUN_CONFIG = { 21 # test_group_name: ('test_files.csv', 'report_email_address') 22 'media': ('testname/media.csv', 'layout-test-analyzer-result (at] google.com') 23 } 24 25 # Predefined result/graph directory. 26 DEFAULT_RESULT_DIR = 'result' 27 DEFAULT_GRAPH_DIR = 'graph' 28 29 30 def ParseOption(): 31 """Parse command-line options using OptionParser. 32 33 Returns: 34 an object containing all command-line option information. 35 """ 36 option_parser = optparse.OptionParser() 37 option_parser.add_option('-d', '--result-directory-location', 38 dest='result_directory_location', 39 help=('Name of result directory location ' 40 '(default to %default)'), 41 default=DEFAULT_RESULT_DIR) 42 option_parser.add_option('-p', '--graph-directory-location', 43 dest='graph_directory_location', 44 help=('Name of graph directory location ' 45 '(default to %default)'), 46 default=DEFAULT_GRAPH_DIR) 47 option_parser.add_option('-e', '--email-only-change-mode', 48 dest='email_only_change_mode', 49 help=('With this mode, email is sent out ' 50 'only when there is a change in the ' 51 'analyzer result compared to the previous ' 52 'result (off by default)'), 53 action='store_true', default=False) 54 option_parser.add_option('-z', '--issue-detail-mode', 55 dest='issue_detail_mode', 56 help=('With this mode, email includes issue details' 57 ' including links to the flakiness dashboard' 58 ' (off by default)'), 59 action='store_true', default=False) 60 return option_parser.parse_args()[0] 61 62 63 def GenerateDashboardHTMLFile(file_name, test_group_list): 64 """Generate dashboard HTML file. 65 66 Currently, it is simple table that shows all the analyzer results. 67 68 Args: 69 file_name: the file name of the dashboard. 70 test_group_list: a list of test group names such as 'media' or 'composite'. 71 """ 72 file_object = open(file_name, 'wb') 73 legend_txt = """ 74 <style type="text/css"> 75 th { 76 width: 30px; overflow: hidden; 77 } 78 tr.d0 td { 79 background-color: #CC9999; color: black; 80 text-align: right; 81 width: 30px; overflow: hidden; 82 } 83 tr.d1 td { 84 background-color: #9999CC; color: black; 85 text-align: right; 86 width: 30px; overflow: hidden; 87 } 88 </style> 89 <h2>Chromium Layout Test Analyzer Result</h2> 90 Legend: 91 <ul> 92 <li>#Tests: the number of tests for the given test group 93 <li>#Skipped Tests: the number of tests that are skipped in the 94 <a href='http://svn.webkit.org/repository/webkit/trunk/LayoutTests/platform/\ 95 chromium/test_expectations.txt'>test expectaion file</a> (e.g., BUGWK60877 96 SKIP : loader/navigation-while-deferring-loads.html = FAIL) 97 <li>#Non-Skipped Failing Tests: the number of tests that appeared in the 98 test expectation file and were not skipped. 99 <li>Failing rate: #NonSkippedFailing / (#Tests - #Skipped) 100 <li>Passing rate: 100 - (Failing rate) 101 </ul> 102 """ 103 file_object.write(legend_txt) 104 file_object.write('<table border="1">') 105 file_object.write('<tr><th>Base Directory</th>') 106 file_object.write('<th>Trend Graph</th>') 107 file_object.write('<th>#Tests</th>') 108 file_object.write('<th>#Skipped Tests</th>') 109 file_object.write('<th>#Non-Skipped Failing Tests</th>') 110 file_object.write('<th>Failing Rate</th>') 111 file_object.write('<th>Passing Rate</th>') 112 file_object.write('<th>Last Revision Number</th>') 113 file_object.write('<th>Last Revision Date</th>') 114 file_object.write('<th>Owner Email</th>') 115 file_object.write('<th>Bug Information</th></tr>\n') 116 test_group_list.sort() 117 for i, test_group in enumerate(test_group_list): 118 file_object.write('<tr class="d' + str(i % 2) + '">\n') 119 file_object.write('<td>' + test_group + '</td>\n') 120 file_object.write('</tr>\n') 121 file_object.write('</table>') 122 file_object.close() 123 124 125 # TODO(shadi): Use only one file with main()! Remove this file in favor of 126 # layouttest_analyzer.py main(). 127 def main(): 128 """A main function for the analyzer runner.""" 129 options = ParseOption() 130 run_config_map = DEFAULT_RUN_CONFIG 131 test_group_list = run_config_map.keys() 132 dashboard_file_location = os.path.join(options.graph_directory_location, 133 'index.html') 134 if not os.path.exists(dashboard_file_location): 135 GenerateDashboardHTMLFile(dashboard_file_location, test_group_list) 136 for test_group in test_group_list: 137 # Prepare the result if it does not exist. 138 # The directory name should be changed to avoid collision 139 # with the file separator. 140 test_group_name_for_data = test_group.replace('/', '_') 141 result_dir = os.path.join(options.result_directory_location, 142 test_group_name_for_data) 143 if not os.path.exists(result_dir): 144 os.mkdir(result_dir) 145 graph_file = os.path.join(options.graph_directory_location, 146 test_group_name_for_data + '.html') 147 if not os.path.exists(graph_file): 148 # Copy the template file. 149 shutil.copy(os.path.join('graph', 'graph.html'), 150 graph_file) 151 os.chmod(graph_file, 0744) 152 cmd = ('python layouttest_analyzer.py -x %s -d %s -t %s' 153 ' -q %s ') % ( 154 test_group, result_dir, graph_file, dashboard_file_location) 155 if run_config_map[test_group][0]: 156 cmd += '-n ' + run_config_map[test_group][0] + ' ' 157 if run_config_map[test_group][1]: 158 cmd += '-r ' + run_config_map[test_group][1] + ' ' 159 if options.email_only_change_mode: 160 cmd += ' -c ' 161 if options.issue_detail_mode: 162 cmd += ' -z ' 163 print 'Running ' + cmd 164 proc = Popen(cmd, shell=True) 165 proc.communicate() 166 167 168 if '__main__' == __name__: 169 main() 170