1 #!/usr/bin/env python 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 """Main functions for the Layout Test Analyzer module.""" 7 8 from datetime import datetime 9 import optparse 10 import os 11 import sys 12 import time 13 14 import layouttest_analyzer_helpers 15 from layouttest_analyzer_helpers import DEFAULT_REVISION_VIEW_URL 16 import layouttests 17 from layouttests import DEFAULT_LAYOUTTEST_SVN_VIEW_LOCATION 18 19 from test_expectations import TestExpectations 20 from trend_graph import TrendGraph 21 22 # Predefined result directory. 23 DEFAULT_RESULT_DIR = 'result' 24 # TODO(shadi): Remove graph functions as they are not used any more. 25 DEFAULT_GRAPH_FILE = os.path.join('graph', 'graph.html') 26 # TODO(shadi): Check if these files are needed any more. 27 DEFAULT_STATS_CSV_FILENAME = 'stats.csv' 28 DEFAULT_ISSUES_CSV_FILENAME = 'issues.csv' 29 # TODO(shadi): These are used only for |debug| mode. What is debug mode for? 30 # AFAIK, we don't run debug mode, should be safe to remove. 31 # Predefined result files for debug. 32 CUR_TIME_FOR_DEBUG = '2011-09-11-19' 33 CURRENT_RESULT_FILE_FOR_DEBUG = os.path.join(DEFAULT_RESULT_DIR, 34 CUR_TIME_FOR_DEBUG) 35 PREV_TIME_FOR_DEBUG = '2011-09-11-18' 36 37 # Text to append at the end of every analyzer result email. 38 DEFAULT_EMAIL_APPEND_TEXT = ( 39 '<b><a href="https://groups.google.com/a/google.com/group/' 40 'layout-test-analyzer-result/topics">Email History</a></b><br>' 41 ) 42 43 44 def ParseOption(): 45 """Parse command-line options using OptionParser. 46 47 Returns: 48 an object containing all command-line option information. 49 """ 50 option_parser = optparse.OptionParser() 51 52 option_parser.add_option('-r', '--receiver-email-address', 53 dest='receiver_email_address', 54 help=('receiver\'s email address. ' 55 'Result email is not sent if this is not ' 56 'specified.')) 57 option_parser.add_option('-g', '--debug-mode', dest='debug', 58 help=('Debug mode is used when you want to debug ' 59 'the analyzer by using local file rather ' 60 'than getting data from SVN. This shortens ' 61 'the debugging time (off by default).'), 62 action='store_true', default=False) 63 option_parser.add_option('-t', '--trend-graph-location', 64 dest='trend_graph_location', 65 help=('Location of the bug trend file; ' 66 'file is expected to be in Google ' 67 'Visualization API trend-line format ' 68 '(defaults to %default).'), 69 default=DEFAULT_GRAPH_FILE) 70 option_parser.add_option('-n', '--test-group-file-location', 71 dest='test_group_file_location', 72 help=('Location of the test group file; ' 73 'file is expected to be in CSV format ' 74 'and lists all test name patterns. ' 75 'When this option is not specified, ' 76 'the value of --test-group-name is used ' 77 'for a test name pattern.'), 78 default=None) 79 option_parser.add_option('-x', '--test-group-name', 80 dest='test_group_name', 81 help=('A name of test group. Either ' 82 '--test_group_file_location or this option ' 83 'needs to be specified.')) 84 option_parser.add_option('-d', '--result-directory-location', 85 dest='result_directory_location', 86 help=('Name of result directory location ' 87 '(default to %default).'), 88 default=DEFAULT_RESULT_DIR) 89 option_parser.add_option('-b', '--email-appended-text-file-location', 90 dest='email_appended_text_file_location', 91 help=('File location of the email appended text. ' 92 'The text is appended in the status email. ' 93 '(default to %default and no text is ' 94 'appended in that case).'), 95 default=None) 96 option_parser.add_option('-c', '--email-only-change-mode', 97 dest='email_only_change_mode', 98 help=('With this mode, email is sent out ' 99 'only when there is a change in the ' 100 'analyzer result compared to the previous ' 101 'result (off by default)'), 102 action='store_true', default=False) 103 option_parser.add_option('-q', '--dashboard-file-location', 104 dest='dashboard_file_location', 105 help=('Location of dashboard file. The results are ' 106 'not reported to the dashboard if this ' 107 'option is not specified.')) 108 option_parser.add_option('-z', '--issue-detail-mode', 109 dest='issue_detail_mode', 110 help=('With this mode, email includes issue details ' 111 '(links to the flakiness dashboard)' 112 ' (off by default)'), 113 action='store_true', default=False) 114 return option_parser.parse_args()[0] 115 116 117 def GetCurrentAndPreviousResults(debug, test_group_file_location, 118 test_group_name, result_directory_location): 119 """Get current and the latest previous analyzer results. 120 121 In debug mode, they are read from predefined files. In non-debug mode, 122 current analyzer results are dynamically obtained from Blink SVN and 123 the latest previous result is read from the corresponding file. 124 125 Args: 126 debug: please refer to |options|. 127 test_group_file_location: please refer to |options|. 128 test_group_name: please refer to |options|. 129 result_directory_location: please refer to |options|. 130 131 Returns: 132 a tuple of the following: 133 prev_time: the previous time string that is compared against. 134 prev_analyzer_result_map: previous analyzer result map. Please refer to 135 layouttest_analyzer_helpers.AnalyzerResultMap. 136 analyzer_result_map: current analyzer result map. Please refer to 137 layouttest_analyzer_helpers.AnalyzerResultMap. 138 """ 139 if not debug: 140 if not test_group_file_location and not test_group_name: 141 print ('Either --test-group-name or --test_group_file_location must be ' 142 'specified. Exiting this program.') 143 sys.exit() 144 filter_names = [] 145 if test_group_file_location and os.path.exists(test_group_file_location): 146 filter_names = layouttests.LayoutTests.GetLayoutTestNamesFromCSV( 147 test_group_file_location) 148 parent_location_list = ( 149 layouttests.LayoutTests.GetParentDirectoryList(filter_names)) 150 recursion = True 151 else: 152 # When test group CSV file is not specified, test group name 153 # (e.g., 'media') is used for getting layout tests. 154 # The tests are in 155 # http://src.chromium.org/blink/trunk/LayoutTests/media 156 # Filtering is not set so all HTML files are considered as valid tests. 157 # Also, we look for the tests recursively. 158 if not test_group_file_location or ( 159 not os.path.exists(test_group_file_location)): 160 print ('Warning: CSV file (%s) does not exist. So it is ignored and ' 161 '%s is used for obtaining test names') % ( 162 test_group_file_location, test_group_name) 163 if not test_group_name.endswith('/'): 164 test_group_name += '/' 165 parent_location_list = [test_group_name] 166 filter_names = None 167 recursion = True 168 layouttests_object = layouttests.LayoutTests( 169 parent_location_list=parent_location_list, recursion=recursion, 170 filter_names=filter_names) 171 analyzer_result_map = layouttest_analyzer_helpers.AnalyzerResultMap( 172 layouttests_object.JoinWithTestExpectation(TestExpectations())) 173 result = layouttest_analyzer_helpers.FindLatestResult( 174 result_directory_location) 175 if result: 176 (prev_time, prev_analyzer_result_map) = result 177 else: 178 prev_time = None 179 prev_analyzer_result_map = None 180 else: 181 analyzer_result_map = layouttest_analyzer_helpers.AnalyzerResultMap.Load( 182 CURRENT_RESULT_FILE_FOR_DEBUG) 183 prev_time = PREV_TIME_FOR_DEBUG 184 prev_analyzer_result_map = ( 185 layouttest_analyzer_helpers.AnalyzerResultMap.Load( 186 os.path.join(DEFAULT_RESULT_DIR, prev_time))) 187 return (prev_time, prev_analyzer_result_map, analyzer_result_map) 188 189 190 def SendEmail(prev_time, prev_analyzer_result_map, analyzer_result_map, 191 appended_text_to_email, email_only_change_mode, debug, 192 receiver_email_address, test_group_name, issue_detail_mode): 193 """Send result status email. 194 195 Args: 196 prev_time: the previous time string that is compared against. 197 prev_analyzer_result_map: previous analyzer result map. Please refer to 198 layouttest_analyzer_helpers.AnalyzerResultMap. 199 analyzer_result_map: current analyzer result map. Please refer to 200 layouttest_analyzer_helpers.AnalyzerResultMap. 201 appended_text_to_email: the text string to append to the status email. 202 email_only_change_mode: please refer to |options|. 203 debug: please refer to |options|. 204 receiver_email_address: please refer to |options|. 205 test_group_name: please refer to |options|. 206 issue_detail_mode: please refer to |options|. 207 208 Returns: 209 a tuple of the following: 210 result_change: a boolean indicating whether there is a change in the 211 result compared with the latest past result. 212 diff_map: please refer to 213 layouttest_analyzer_helpers.SendStatusEmail(). 214 simple_rev_str: a simple version of revision string that is sent in 215 the email. 216 rev: the latest revision number for the given test group. 217 rev_date: the latest revision date for the given test group. 218 email_content: email content string (without 219 |appended_text_to_email|) that will be shown on the dashboard. 220 """ 221 rev = '' 222 rev_date = '' 223 email_content = '' 224 if prev_analyzer_result_map: 225 diff_map = analyzer_result_map.CompareToOtherResultMap( 226 prev_analyzer_result_map) 227 result_change = (any(diff_map['whole']) or any(diff_map['skip']) or 228 any(diff_map['nonskip'])) 229 # Email only when |email_only_change_mode| is False or there 230 # is a change in the result compared to the last result. 231 simple_rev_str = '' 232 if not email_only_change_mode or result_change: 233 prev_time_in_float = datetime.strptime(prev_time, '%Y-%m-%d-%H') 234 prev_time_in_float = time.mktime(prev_time_in_float.timetuple()) 235 if debug: 236 cur_time_in_float = datetime.strptime(CUR_TIME_FOR_DEBUG, 237 '%Y-%m-%d-%H') 238 cur_time_in_float = time.mktime(cur_time_in_float.timetuple()) 239 else: 240 cur_time_in_float = time.time() 241 (rev_str, simple_rev_str, rev, rev_date) = ( 242 layouttest_analyzer_helpers.GetRevisionString(prev_time_in_float, 243 cur_time_in_float, 244 diff_map)) 245 email_content = analyzer_result_map.ConvertToString(prev_time, 246 diff_map, 247 issue_detail_mode) 248 if receiver_email_address: 249 layouttest_analyzer_helpers.SendStatusEmail( 250 prev_time, analyzer_result_map, diff_map, 251 receiver_email_address, test_group_name, 252 appended_text_to_email, email_content, rev_str, 253 email_only_change_mode) 254 if simple_rev_str: 255 simple_rev_str = '\'' + simple_rev_str + '\'' 256 else: 257 simple_rev_str = 'undefined' # GViz uses undefined for NONE. 258 else: 259 # Initial result should be written to tread-graph if there are no previous 260 # results. 261 result_change = True 262 diff_map = None 263 simple_rev_str = 'undefined' 264 email_content = analyzer_result_map.ConvertToString(None, diff_map, 265 issue_detail_mode) 266 return (result_change, diff_map, simple_rev_str, rev, rev_date, 267 email_content) 268 269 270 def UpdateTrendGraph(start_time, analyzer_result_map, diff_map, simple_rev_str, 271 trend_graph_location): 272 """Update trend graph in GViz. 273 274 Annotate the graph with revision information. 275 276 Args: 277 start_time: the script starting time as a float value. 278 analyzer_result_map: current analyzer result map. Please refer to 279 layouttest_analyzer_helpers.AnalyzerResultMap. 280 diff_map: a map that has 'whole', 'skip' and 'nonskip' as keys. 281 Please refer to |diff_map| in 282 |layouttest_analyzer_helpers.SendStatusEmail()|. 283 simple_rev_str: a simple version of revision string that is sent in 284 the email. 285 trend_graph_location: the location of the trend graph that needs to be 286 updated. 287 288 Returns: 289 a dictionary that maps result data category ('whole', 'skip', 'nonskip', 290 'passingrate') to information tuple (a dictionary that maps test name 291 to its description, annotation, simple_rev_string) of the given result 292 data category. These tuples are used for trend graph update. 293 """ 294 # Trend graph update (if specified in the command-line argument) when 295 # there is change from the last result. 296 # Currently, there are two graphs (graph1 is for 'whole', 'skip', 297 # 'nonskip' and the graph2 is for 'passingrate'). Please refer to 298 # graph/graph.html. 299 # Sample JS annotation for graph1: 300 # [new Date(2011,8,12,10,41,32),224,undefined,'',52,undefined, 301 # undefined, 12, 'test1,','<a href="http://t</a>,',], 302 # This example lists 'whole' triple and 'skip' triple and 303 # 'nonskip' triple. Each triple is (the number of tests that belong to 304 # the test group, linked text, a link). The following code generates this 305 # automatically based on rev_string etc. 306 trend_graph = TrendGraph(trend_graph_location) 307 datetime_string = start_time.strftime('%Y,%m,%d,%H,%M,%S') 308 data_map = {} 309 passingrate_anno = '' 310 for test_group in ['whole', 'skip', 'nonskip']: 311 anno = 'undefined' 312 # Extract test description. 313 test_map = {} 314 for (test_name, value) in ( 315 analyzer_result_map.result_map[test_group].iteritems()): 316 test_map[test_name] = value['desc'] 317 test_str = '' 318 links = '' 319 if diff_map and diff_map[test_group]: 320 for i in [0, 1]: 321 for (name, _) in diff_map[test_group][i]: 322 test_str += name + ',' 323 # This is link to test HTML in SVN. 324 links += ('<a href="%s%s">%s</a>' % 325 (DEFAULT_LAYOUTTEST_SVN_VIEW_LOCATION, name, name)) 326 if test_str: 327 anno = '\'' + test_str + '\'' 328 # The annotation of passing rate is a union of all annotations. 329 passingrate_anno += anno 330 if links: 331 links = '\'' + links + '\'' 332 else: 333 links = 'undefined' 334 if test_group is 'whole': 335 data_map[test_group] = (test_map, anno, links) 336 else: 337 data_map[test_group] = (test_map, anno, simple_rev_str) 338 if not passingrate_anno: 339 passingrate_anno = 'undefined' 340 data_map['passingrate'] = ( 341 str(analyzer_result_map.GetPassingRate()), passingrate_anno, 342 simple_rev_str) 343 trend_graph.Update(datetime_string, data_map) 344 return data_map 345 346 347 def UpdateDashboard(dashboard_file_location, test_group_name, data_map, 348 layouttest_root_path, rev, rev_date, email, 349 email_content): 350 """Update dashboard HTML file. 351 352 Args: 353 dashboard_file_location: the file location for the dashboard file. 354 test_group_name: please refer to |options|. 355 data_map: a dictionary that maps result data category ('whole', 'skip', 356 'nonskip', 'passingrate') to information tuple (a dictionary that maps 357 test name to its description, annotation, simple_rev_string) of the 358 given result data category. These tuples are used for trend graph 359 update. 360 layouttest_root_path: A location string where layout tests are stored. 361 rev: the latest revision number for the given test group. 362 rev_date: the latest revision date for the given test group. 363 email: email address of the owner for the given test group. 364 email_content: email content string (without |appended_text_to_email|) 365 that will be shown on the dashboard. 366 """ 367 # Generate a HTML file that contains all test names for each test group. 368 escaped_tg_name = test_group_name.replace('/', '_') 369 for tg in ['whole', 'skip', 'nonskip']: 370 file_name = os.path.join( 371 os.path.dirname(dashboard_file_location), 372 escaped_tg_name + '_' + tg + '.html') 373 file_object = open(file_name, 'wb') 374 file_object.write('<table border="1">') 375 sorted_testnames = data_map[tg][0].keys() 376 sorted_testnames.sort() 377 for testname in sorted_testnames: 378 file_object.write(( 379 '<tr><td><a href="%s">%s</a></td><td><a href="%s">dashboard</a>' 380 '</td><td>%s</td></tr>') % ( 381 layouttest_root_path + testname, testname, 382 ('http://test-results.appspot.com/dashboards/' 383 'flakiness_dashboard.html#tests=%s') % testname, 384 data_map[tg][0][testname])) 385 file_object.write('</table>') 386 file_object.close() 387 email_content_with_link = '' 388 if email_content: 389 file_name = os.path.join(os.path.dirname(dashboard_file_location), 390 escaped_tg_name + '_email.html') 391 file_object = open(file_name, 'wb') 392 file_object.write(email_content) 393 file_object.close() 394 email_content_with_link = '<a href="%s_email.html">info</a>' % ( 395 escaped_tg_name) 396 test_group_str = ( 397 '<td><a href="%(test_group_path)s">%(test_group_name)s</a></td>' 398 '<td><a href="%(graph_path)s">graph</a></td>' 399 '<td><a href="%(all_tests_path)s">%(all_tests_count)d</a></td>' 400 '<td><a href="%(skip_tests_path)s">%(skip_tests_count)d</a></td>' 401 '<td><a href="%(nonskip_tests_path)s">%(nonskip_tests_count)d</a></td>' 402 '<td>%(fail_rate)d%%</td>' 403 '<td>%(passing_rate)d%%</td>' 404 '<td><a href="%(rev_url)s">%(rev)s</a></td>' 405 '<td>%(rev_date)s</td>' 406 '<td><a href="mailto:%(email)s">%(email)s</a></td>' 407 '<td>%(email_content)s</td>\n') % { 408 # Dashboard file and graph must be in the same directory 409 # to make the following link work. 410 'test_group_path': layouttest_root_path + '/' + test_group_name, 411 'test_group_name': test_group_name, 412 'graph_path': escaped_tg_name + '.html', 413 'all_tests_path': escaped_tg_name + '_whole.html', 414 'all_tests_count': len(data_map['whole'][0]), 415 'skip_tests_path': escaped_tg_name + '_skip.html', 416 'skip_tests_count': len(data_map['skip'][0]), 417 'nonskip_tests_path': escaped_tg_name + '_nonskip.html', 418 'nonskip_tests_count': len(data_map['nonskip'][0]), 419 'fail_rate': 100 - float(data_map['passingrate'][0]), 420 'passing_rate': float(data_map['passingrate'][0]), 421 'rev_url': DEFAULT_REVISION_VIEW_URL % rev, 422 'rev': rev, 423 'rev_date': rev_date, 424 'email': email, 425 'email_content': email_content_with_link 426 } 427 layouttest_analyzer_helpers.ReplaceLineInFile( 428 dashboard_file_location, '<td>' + test_group_name + '</td>', 429 test_group_str) 430 431 432 def main(): 433 """A main function for the analyzer.""" 434 options = ParseOption() 435 start_time = datetime.now() 436 437 (prev_time, prev_analyzer_result_map, analyzer_result_map) = ( 438 GetCurrentAndPreviousResults(options.debug, 439 options.test_group_file_location, 440 options.test_group_name, 441 options.result_directory_location)) 442 (result_change, diff_map, simple_rev_str, rev, rev_date, email_content) = ( 443 SendEmail(prev_time, prev_analyzer_result_map, analyzer_result_map, 444 DEFAULT_EMAIL_APPEND_TEXT, 445 options.email_only_change_mode, options.debug, 446 options.receiver_email_address, options.test_group_name, 447 options.issue_detail_mode)) 448 449 # Create CSV texts and save them for bug spreadsheet. 450 (stats, issues_txt) = analyzer_result_map.ConvertToCSVText( 451 start_time.strftime('%Y-%m-%d-%H')) 452 file_object = open(os.path.join(options.result_directory_location, 453 DEFAULT_STATS_CSV_FILENAME), 'wb') 454 file_object.write(stats) 455 file_object.close() 456 file_object = open(os.path.join(options.result_directory_location, 457 DEFAULT_ISSUES_CSV_FILENAME), 'wb') 458 file_object.write(issues_txt) 459 file_object.close() 460 461 if not options.debug and (result_change or not prev_analyzer_result_map): 462 # Save the current result when result is changed or the script is 463 # executed for the first time. 464 date = start_time.strftime('%Y-%m-%d-%H') 465 file_path = os.path.join(options.result_directory_location, date) 466 analyzer_result_map.Save(file_path) 467 if result_change or not prev_analyzer_result_map: 468 data_map = UpdateTrendGraph(start_time, analyzer_result_map, diff_map, 469 simple_rev_str, options.trend_graph_location) 470 # Report the result to dashboard. 471 if options.dashboard_file_location: 472 UpdateDashboard(options.dashboard_file_location, options.test_group_name, 473 data_map, layouttests.DEFAULT_LAYOUTTEST_LOCATION, rev, 474 rev_date, options.receiver_email_address, 475 email_content) 476 477 478 if '__main__' == __name__: 479 main() 480
