1 # Copyright 2016 The Chromium Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 import copy 6 import datetime 7 import json 8 import unittest 9 10 import mock 11 import webapp2 12 import webtest 13 14 from dashboard import bisect_fyi 15 from dashboard import bisect_fyi_test 16 from dashboard import layered_cache 17 from dashboard import rietveld_service 18 from dashboard import stored_object 19 from dashboard import testing_common 20 from dashboard import update_bug_with_results 21 from dashboard import utils 22 from dashboard.models import anomaly 23 from dashboard.models import bug_data 24 from dashboard.models import try_job 25 26 _SAMPLE_BISECT_RESULTS_JSON = { 27 'try_job_id': 6789, 28 'bug_id': 4567, 29 'status': 'completed', 30 'bisect_bot': 'linux', 31 'buildbot_log_url': '', 32 'command': ('tools/perf/run_benchmark -v ' 33 '--browser=release page_cycler.intl_ar_fa_he'), 34 'metric': 'warm_times/page_load_time', 35 'change': '', 36 'score': 99.9, 37 'good_revision': '306475', 38 'bad_revision': '306478', 39 'warnings': None, 40 'abort_reason': None, 41 'issue_url': 'https://issue_url/123456', 42 'culprit_data': { 43 'subject': 'subject', 44 'author': 'author', 45 'email': 'author (at] email.com', 46 'cl_date': '1/2/2015', 47 'commit_info': 'commit_info', 48 'revisions_links': ['http://src.chromium.org/viewvc/chrome?view=' 49 'revision&revision=20798'], 50 'cl': '2a1781d64d' # Should match config in bisect_fyi_test.py. 51 }, 52 'revision_data': [ 53 { 54 'depot_name': 'chromium', 55 'deps_revision': 1234, 56 'commit_hash': '1234abcdf', 57 'mean_value': 70, 58 'std_dev': 0, 59 'values': [70, 70, 70], 60 'result': 'good' 61 }, { 62 'depot_name': 'chromium', 63 'deps_revision': 1235, 64 'commit_hash': '1235abdcf', 65 'mean_value': 80, 66 'std_dev': 0, 67 'values': [80, 80, 80], 68 'result': 'bad' 69 } 70 ] 71 } 72 73 _REVISION_RESPONSE = """ 74 <html xmlns=....> 75 <head><title>[chrome] Revision 207985</title></head><body><table>.... 76 <tr align="left"> 77 <th>Log Message:</th> 78 <td> Message....</td> 79 > > Review URL: <a href="https://codereview.chromium.org/81533002">\ 80 https://codereview.chromium.org/81533002</a> 81 > 82 > Review URL: <a href="https://codereview.chromium.org/96073002">\ 83 https://codereview.chromium.org/96073002</a> 84 85 Review URL: <a href="https://codereview.chromium.org/17504006">\ 86 https://codereview.chromium.org/96363002</a></pre></td></tr></table>....</body> 87 </html> 88 """ 89 90 _PERF_TEST_CONFIG = """config = { 91 'command': 'tools/perf/run_benchmark -v --browser=release\ 92 dromaeo.jslibstylejquery --profiler=trace', 93 'good_revision': '215806', 94 'bad_revision': '215828', 95 'repeat_count': '1', 96 'max_time_minutes': '120' 97 }""" 98 99 _ISSUE_RESPONSE = """ 100 { 101 "description": "Issue Description.", 102 "cc": [ 103 "chromium-reviews@chromium.org", 104 "cc-bugs@chromium.org", 105 "sullivan@google.com" 106 ], 107 "reviewers": [ 108 "prasadv@google.com" 109 ], 110 "owner_email": "sullivan@google.com", 111 "private": false, 112 "base_url": "svn://chrome-svn/chrome/trunk/src/", 113 "owner":"sullivan", 114 "subject":"Issue Subject", 115 "created":"2013-06-20 22:23:27.227150", 116 "patchsets":[1,21001,29001], 117 "modified":"2013-06-22 00:59:38.530190", 118 "closed":true, 119 "commit":false, 120 "issue":17504006 121 } 122 """ 123 124 125 def _MockFetch(url=None): 126 url_to_response_map = { 127 'http://src.chromium.org/viewvc/chrome?view=revision&revision=20798': [ 128 200, _REVISION_RESPONSE 129 ], 130 'http://src.chromium.org/viewvc/chrome?view=revision&revision=20799': [ 131 200, 'REVISION REQUEST FAILED!' 132 ], 133 'https://codereview.chromium.org/api/17504006': [ 134 200, json.dumps(json.loads(_ISSUE_RESPONSE)) 135 ], 136 } 137 138 if url not in url_to_response_map: 139 assert False, 'Bad url %s' % url 140 141 response_code = url_to_response_map[url][0] 142 response = url_to_response_map[url][1] 143 return testing_common.FakeResponseObject(response_code, response) 144 145 146 # In this class, we patch apiclient.discovery.build so as to not make network 147 # requests, which are normally made when the IssueTrackerService is initialized. 148 @mock.patch('apiclient.discovery.build', mock.MagicMock()) 149 @mock.patch.object(utils, 'TickMonitoringCustomMetric', mock.MagicMock()) 150 class UpdateBugWithResultsTest(testing_common.TestCase): 151 152 def setUp(self): 153 super(UpdateBugWithResultsTest, self).setUp() 154 app = webapp2.WSGIApplication([( 155 '/update_bug_with_results', 156 update_bug_with_results.UpdateBugWithResultsHandler)]) 157 self.testapp = webtest.TestApp(app) 158 self._AddRietveldConfig() 159 160 def _AddRietveldConfig(self): 161 """Adds a RietveldConfig entity to the datastore. 162 163 This is used in order to get the Rietveld URL when requests are made to the 164 handler in te tests below. In the real datastore, the RietveldConfig entity 165 would contain credentials. 166 """ 167 rietveld_service.RietveldConfig( 168 id='default_rietveld_config', 169 client_email='sullivan (at] google.com', 170 service_account_key='Fake Account Key', 171 server_url='https://test-rietveld.appspot.com', 172 internal_server_url='https://test-rietveld.appspot.com').put() 173 174 def _AddTryJob(self, bug_id, status, bot, **kwargs): 175 job = try_job.TryJob(bug_id=bug_id, status=status, bot=bot, **kwargs) 176 job.put() 177 bug_data.Bug(id=bug_id).put() 178 return job 179 180 @mock.patch( 181 'google.appengine.api.urlfetch.fetch', 182 mock.MagicMock(side_effect=_MockFetch)) 183 @mock.patch.object( 184 update_bug_with_results.issue_tracker_service, 'IssueTrackerService', 185 mock.MagicMock()) 186 def testGet(self): 187 # Put succeeded, failed, staled, and not yet finished jobs in the 188 # datastore. 189 self._AddTryJob(11111, 'started', 'win_perf', 190 results_data=_SAMPLE_BISECT_RESULTS_JSON) 191 staled_timestamp = (datetime.datetime.now() - 192 update_bug_with_results._STALE_TRYJOB_DELTA) 193 self._AddTryJob(22222, 'started', 'win_perf', 194 last_ran_timestamp=staled_timestamp) 195 self._AddTryJob(33333, 'failed', 'win_perf') 196 self._AddTryJob(44444, 'started', 'win_perf') 197 198 self.testapp.get('/update_bug_with_results') 199 pending_jobs = try_job.TryJob.query().fetch() 200 # Expects no jobs to be deleted. 201 self.assertEqual(4, len(pending_jobs)) 202 self.assertEqual(11111, pending_jobs[0].bug_id) 203 self.assertEqual('completed', pending_jobs[0].status) 204 self.assertEqual(22222, pending_jobs[1].bug_id) 205 self.assertEqual('staled', pending_jobs[1].status) 206 self.assertEqual(33333, pending_jobs[2].bug_id) 207 self.assertEqual('failed', pending_jobs[2].status) 208 self.assertEqual(44444, pending_jobs[3].bug_id) 209 self.assertEqual('started', pending_jobs[3].status) 210 211 @mock.patch( 212 'google.appengine.api.urlfetch.fetch', 213 mock.MagicMock(side_effect=_MockFetch)) 214 @mock.patch.object( 215 update_bug_with_results.issue_tracker_service, 'IssueTrackerService', 216 mock.MagicMock()) 217 def testCreateTryJob_WithoutExistingBug(self): 218 # Put succeeded job in the datastore. 219 try_job.TryJob( 220 bug_id=12345, status='started', bot='win_perf', 221 results_data=_SAMPLE_BISECT_RESULTS_JSON).put() 222 223 self.testapp.get('/update_bug_with_results') 224 pending_jobs = try_job.TryJob.query().fetch() 225 226 # Expects job to finish. 227 self.assertEqual(1, len(pending_jobs)) 228 self.assertEqual(12345, pending_jobs[0].bug_id) 229 self.assertEqual('completed', pending_jobs[0].status) 230 231 @mock.patch.object(utils, 'ServiceAccountCredentials', mock.MagicMock()) 232 @mock.patch( 233 'google.appengine.api.urlfetch.fetch', 234 mock.MagicMock(side_effect=_MockFetch)) 235 @mock.patch.object( 236 update_bug_with_results.issue_tracker_service.IssueTrackerService, 237 'AddBugComment', mock.MagicMock(return_value=False)) 238 @mock.patch('logging.error') 239 def testGet_FailsToUpdateBug_LogsErrorAndMovesOn(self, mock_logging_error): 240 # Put a successful job and a failed job with partial results. 241 # Note that AddBugComment is mocked to always returns false, which 242 # simulates failing to post results to the issue tracker for all bugs. 243 self._AddTryJob(12345, 'started', 'win_perf', 244 results_data=_SAMPLE_BISECT_RESULTS_JSON) 245 self._AddTryJob(54321, 'started', 'win_perf', 246 results_data=_SAMPLE_BISECT_RESULTS_JSON) 247 self.testapp.get('/update_bug_with_results') 248 249 # Two errors should be logged. 250 self.assertEqual(2, mock_logging_error.call_count) 251 252 # The pending jobs should still be there. 253 pending_jobs = try_job.TryJob.query().fetch() 254 self.assertEqual(2, len(pending_jobs)) 255 self.assertEqual('started', pending_jobs[0].status) 256 self.assertEqual('started', pending_jobs[1].status) 257 258 @mock.patch( 259 'google.appengine.api.urlfetch.fetch', 260 mock.MagicMock(side_effect=_MockFetch)) 261 @mock.patch.object( 262 update_bug_with_results.issue_tracker_service.IssueTrackerService, 263 'AddBugComment') 264 def testGet_BisectCulpritHasAuthor_AssignsAuthor(self, mock_update_bug): 265 # When a bisect has a culprit for a perf regression, 266 # author and reviewer of the CL should be cc'ed on issue update. 267 self._AddTryJob(12345, 'started', 'win_perf', 268 results_data=_SAMPLE_BISECT_RESULTS_JSON) 269 270 self.testapp.get('/update_bug_with_results') 271 mock_update_bug.assert_called_once_with( 272 mock.ANY, mock.ANY, 273 cc_list=['author (at] email.com', 'prasadv (at] google.com'], 274 merge_issue=None, labels=None, owner='author (at] email.com') 275 276 @mock.patch( 277 'google.appengine.api.urlfetch.fetch', 278 mock.MagicMock(side_effect=_MockFetch)) 279 @mock.patch.object( 280 update_bug_with_results.issue_tracker_service.IssueTrackerService, 281 'AddBugComment') 282 def testGet_FailedRevisionResponse(self, mock_add_bug): 283 # When a Rietveld CL link fails to respond, only update CL owner in CC 284 # list. 285 sample_bisect_results = copy.deepcopy(_SAMPLE_BISECT_RESULTS_JSON) 286 sample_bisect_results['revisions_links'] = [ 287 'http://src.chromium.org/viewvc/chrome?view=revision&revision=20799'] 288 self._AddTryJob(12345, 'started', 'win_perf', 289 results_data=sample_bisect_results) 290 291 self.testapp.get('/update_bug_with_results') 292 mock_add_bug.assert_called_once_with(mock.ANY, 293 mock.ANY, 294 cc_list=['author (at] email.com', 295 'prasadv (at] google.com'], 296 merge_issue=None, 297 labels=None, 298 owner='author (at] email.com') 299 300 @mock.patch( 301 'google.appengine.api.urlfetch.fetch', 302 mock.MagicMock(side_effect=_MockFetch)) 303 @mock.patch.object( 304 update_bug_with_results.issue_tracker_service.IssueTrackerService, 305 'AddBugComment', mock.MagicMock()) 306 def testGet_PositiveResult_StoresCommitHash(self): 307 self._AddTryJob(12345, 'started', 'win_perf', 308 results_data=_SAMPLE_BISECT_RESULTS_JSON) 309 310 self.testapp.get('/update_bug_with_results') 311 self.assertEqual('12345', 312 layered_cache.GetExternal('commit_hash_2a1781d64d')) 313 314 @mock.patch( 315 'google.appengine.api.urlfetch.fetch', 316 mock.MagicMock(side_effect=_MockFetch)) 317 @mock.patch.object( 318 update_bug_with_results.issue_tracker_service.IssueTrackerService, 319 'AddBugComment', mock.MagicMock()) 320 def testGet_NegativeResult_DoesNotStoreCommitHash(self): 321 sample_bisect_results = copy.deepcopy(_SAMPLE_BISECT_RESULTS_JSON) 322 sample_bisect_results['culprit_data'] = None 323 self._AddTryJob(12345, 'started', 'win_perf', 324 results_data=sample_bisect_results) 325 self.testapp.get('/update_bug_with_results') 326 327 caches = layered_cache.CachedPickledString.query().fetch() 328 # Only 1 cache for bisect stats. 329 self.assertEqual(1, len(caches)) 330 331 def testMapAnomaliesToMergeIntoBug(self): 332 # Add anomalies. 333 test_keys = map(utils.TestKey, [ 334 'ChromiumGPU/linux-release/scrolling-benchmark/first_paint', 335 'ChromiumGPU/linux-release/scrolling-benchmark/mean_frame_time']) 336 anomaly.Anomaly( 337 start_revision=9990, end_revision=9997, test=test_keys[0], 338 median_before_anomaly=100, median_after_anomaly=200, 339 sheriff=None, bug_id=12345).put() 340 anomaly.Anomaly( 341 start_revision=9990, end_revision=9996, test=test_keys[0], 342 median_before_anomaly=100, median_after_anomaly=200, 343 sheriff=None, bug_id=54321).put() 344 # Map anomalies to base(dest_bug_id) bug. 345 update_bug_with_results._MapAnomaliesToMergeIntoBug( 346 dest_bug_id=12345, source_bug_id=54321) 347 anomalies = anomaly.Anomaly.query( 348 anomaly.Anomaly.bug_id == int(54321)).fetch() 349 self.assertEqual(0, len(anomalies)) 350 351 @mock.patch( 352 'google.appengine.api.urlfetch.fetch', 353 mock.MagicMock(side_effect=_MockFetch)) 354 @mock.patch.object( 355 update_bug_with_results.email_template, 356 'GetPerfTryJobEmailReport', mock.MagicMock(return_value=None)) 357 def testSendPerfTryJobEmail_EmptyEmailReport_DontSendEmail(self): 358 self._AddTryJob(12345, 'started', 'win_perf', job_type='perf-try', 359 results_data=_SAMPLE_BISECT_RESULTS_JSON) 360 self.testapp.get('/update_bug_with_results') 361 messages = self.mail_stub.get_sent_messages() 362 self.assertEqual(0, len(messages)) 363 364 @mock.patch( 365 'google.appengine.api.urlfetch.fetch', 366 mock.MagicMock(side_effect=_MockFetch)) 367 @mock.patch.object( 368 update_bug_with_results.issue_tracker_service.IssueTrackerService, 369 'AddBugComment') 370 def testGet_InternalOnlyTryJob_AddsInternalOnlyBugLabel( 371 self, mock_update_bug): 372 self._AddTryJob(12345, 'started', 'win_perf', 373 results_data=_SAMPLE_BISECT_RESULTS_JSON, 374 internal_only=True) 375 376 self.testapp.get('/update_bug_with_results') 377 mock_update_bug.assert_called_once_with( 378 mock.ANY, mock.ANY, 379 cc_list=mock.ANY, 380 merge_issue=None, labels=['Restrict-View-Google'], owner=mock.ANY) 381 382 @mock.patch( 383 'google.appengine.api.urlfetch.fetch', 384 mock.MagicMock(side_effect=_MockFetch)) 385 @mock.patch.object( 386 update_bug_with_results.issue_tracker_service, 'IssueTrackerService', 387 mock.MagicMock()) 388 def testFYI_Send_No_Email_On_Success(self): 389 stored_object.Set( 390 bisect_fyi._BISECT_FYI_CONFIGS_KEY, 391 bisect_fyi_test.TEST_FYI_CONFIGS) 392 test_config = bisect_fyi_test.TEST_FYI_CONFIGS['positive_culprit'] 393 bisect_config = test_config.get('bisect_config') 394 self._AddTryJob(12345, 'started', 'win_perf', 395 results_data=_SAMPLE_BISECT_RESULTS_JSON, 396 internal_only=True, 397 config=utils.BisectConfigPythonString(bisect_config), 398 job_type='bisect-fyi', 399 job_name='positive_culprit', 400 email='chris (at] email.com') 401 402 self.testapp.get('/update_bug_with_results') 403 messages = self.mail_stub.get_sent_messages() 404 self.assertEqual(0, len(messages)) 405 406 @mock.patch( 407 'google.appengine.api.urlfetch.fetch', 408 mock.MagicMock(side_effect=_MockFetch)) 409 @mock.patch.object( 410 update_bug_with_results.bisect_fyi, 'IsBugUpdated', 411 mock.MagicMock(return_value=True)) 412 @mock.patch.object( 413 update_bug_with_results.issue_tracker_service, 'IssueTrackerService', 414 mock.MagicMock()) 415 def testFYI_Failed_Job_SendEmail(self): 416 stored_object.Set( 417 bisect_fyi._BISECT_FYI_CONFIGS_KEY, 418 bisect_fyi_test.TEST_FYI_CONFIGS) 419 test_config = bisect_fyi_test.TEST_FYI_CONFIGS['positive_culprit'] 420 bisect_config = test_config.get('bisect_config') 421 sample_bisect_results = copy.deepcopy(_SAMPLE_BISECT_RESULTS_JSON) 422 sample_bisect_results['status'] = 'failed' 423 self._AddTryJob(12345, 'started', 'win_perf', 424 results_data=sample_bisect_results, 425 internal_only=True, 426 config=utils.BisectConfigPythonString(bisect_config), 427 job_type='bisect-fyi', 428 job_name='positive_culprit', 429 email='chris (at] email.com') 430 431 self.testapp.get('/update_bug_with_results') 432 messages = self.mail_stub.get_sent_messages() 433 self.assertEqual(1, len(messages)) 434 435 @mock.patch.object( 436 update_bug_with_results.quick_logger.QuickLogger, 437 'Log', mock.MagicMock(return_value='record_key_123')) 438 @mock.patch('logging.error') 439 def testUpdateQuickLog_WithJobResults_NoError(self, mock_logging_error): 440 job = self._AddTryJob(111, 'started', 'win_perf', 441 results_data=_SAMPLE_BISECT_RESULTS_JSON) 442 update_bug_with_results.UpdateQuickLog(job) 443 self.assertEqual(0, mock_logging_error.call_count) 444 445 @mock.patch('logging.error') 446 @mock.patch('update_bug_with_results.quick_logger.QuickLogger.Log') 447 def testUpdateQuickLog_NoResultsData_ReportsError( 448 self, mock_log, mock_logging_error): 449 job = self._AddTryJob(111, 'started', 'win_perf') 450 update_bug_with_results.UpdateQuickLog(job) 451 self.assertEqual(0, mock_log.call_count) 452 mock_logging_error.assert_called_once_with( 453 'Bisect report returns empty for job id %s, bug_id %s.', 1, 111) 454 455 @mock.patch( 456 'google.appengine.api.urlfetch.fetch', 457 mock.MagicMock(side_effect=_MockFetch)) 458 @mock.patch.object( 459 update_bug_with_results.issue_tracker_service.IssueTrackerService, 460 'AddBugComment') 461 def testGet_PostResult_WithoutBugEntity( 462 self, mock_update_bug): 463 job = try_job.TryJob(bug_id=12345, status='started', bot='win_perf', 464 results_data=_SAMPLE_BISECT_RESULTS_JSON) 465 job.put() 466 self.testapp.get('/update_bug_with_results') 467 mock_update_bug.assert_called_once_with( 468 12345, mock.ANY, cc_list=mock.ANY, merge_issue=mock.ANY, 469 labels=mock.ANY, owner=mock.ANY) 470 471 472 if __name__ == '__main__': 473 unittest.main() 474
