1 #!/usr/bin/python 2 # 3 # Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 4 # Use of this source code is governed by a BSD-style license that can be 5 # found in the LICENSE file. 6 7 import unittest 8 9 import mox 10 11 import common 12 from autotest_lib.server.cros.dynamic_suite import constants 13 from autotest_lib.server.cros.dynamic_suite import job_status 14 from autotest_lib.server.cros.dynamic_suite import reporting 15 from autotest_lib.server.cros.dynamic_suite import reporting_utils 16 from autotest_lib.server.cros.dynamic_suite import tools 17 from autotest_lib.site_utils import phapi_lib 18 from chromite.lib import gdata_lib 19 20 21 class ReportingTest(mox.MoxTestBase): 22 """Unittests to verify basic control flow for automatic bug filing.""" 23 24 # fake issue id to use in testing duplicate issues 25 _FAKE_ISSUE_ID = 123 26 27 # test report used to generate failure 28 test_report = { 29 'build':'build-build/R1-1', 30 'chrome_version':'28.0', 31 'suite':'suite', 32 'test':'bad_test', 33 'reason':'dreadful_reason', 34 'owner':'user', 35 'hostname':'myhost', 36 'job_id':'myjob', 37 'status': 'FAIL', 38 } 39 40 bug_template = { 41 'labels': ['Cr-Internals-WebRTC'], 42 'owner': 'myself', 43 'status': 'Fixed', 44 'summary': 'This is a short summary', 45 'title': None, 46 } 47 48 def _get_failure(self, is_server_job=False): 49 """Get a TestBug so we can report it. 50 51 @param is_server_job: Set to True of failed job is a server job. Server 52 job's test name is formated as build/suite/test_name. 53 @return: a failure object initialized with values from test_report. 54 """ 55 if is_server_job: 56 test_name = tools.create_job_name( 57 self.test_report.get('build'), 58 self.test_report.get('suite'), 59 self.test_report.get('test')) 60 else: 61 test_name = self.test_report.get('test') 62 expected_result = job_status.Status(self.test_report.get('status'), 63 test_name, 64 reason=self.test_report.get('reason'), 65 job_id=self.test_report.get('job_id'), 66 owner=self.test_report.get('owner'), 67 hostname=self.test_report.get('hostname')) 68 69 return reporting.TestBug(self.test_report.get('build'), 70 self.test_report.get('chrome_version'), 71 self.test_report.get('suite'), expected_result) 72 73 74 def setUp(self): 75 super(ReportingTest, self).setUp() 76 self.mox.StubOutClassWithMocks(phapi_lib, 'ProjectHostingApiClient') 77 self._orig_project_name = reporting.Reporter._project_name 78 79 # We want to have some data so that the Reporter doesn't fail at 80 # initialization. 81 reporting.Reporter._project_name = 'project' 82 83 84 def tearDown(self): 85 reporting.Reporter._project_name = self._orig_project_name 86 super(ReportingTest, self).tearDown() 87 88 89 def testNewIssue(self): 90 """Add a new issue to the tracker when a matching issue isn't found. 91 92 Confirms that we call CreateTrackerIssue when an Issue search 93 returns None. 94 """ 95 self.mox.StubOutWithMock(reporting.Reporter, 'find_issue_by_marker') 96 self.mox.StubOutWithMock(reporting.TestBug, 'summary') 97 98 client = phapi_lib.ProjectHostingApiClient(mox.IgnoreArg(), 99 mox.IgnoreArg()) 100 client.create_issue(mox.IgnoreArg()).AndReturn( 101 {'id': self._FAKE_ISSUE_ID}) 102 reporting.Reporter.find_issue_by_marker(mox.IgnoreArg()).AndReturn( 103 None) 104 reporting.TestBug.summary().AndReturn('') 105 106 self.mox.ReplayAll() 107 bug_id, bug_count = reporting.Reporter().report(self._get_failure()) 108 109 self.assertEqual(bug_id, self._FAKE_ISSUE_ID) 110 self.assertEqual(bug_count, 1) 111 112 113 def testDuplicateIssue(self): 114 """Dedupe to an existing issue when one is found. 115 116 Confirms that we call AppendTrackerIssueById with the same issue 117 returned by the issue search. 118 """ 119 self.mox.StubOutWithMock(reporting.Reporter, 'find_issue_by_marker') 120 self.mox.StubOutWithMock(reporting.TestBug, 'summary') 121 122 issue = self.mox.CreateMock(phapi_lib.Issue) 123 issue.id = self._FAKE_ISSUE_ID 124 issue.labels = [] 125 issue.state = constants.ISSUE_OPEN 126 127 client = phapi_lib.ProjectHostingApiClient(mox.IgnoreArg(), 128 mox.IgnoreArg()) 129 client.update_issue(self._FAKE_ISSUE_ID, mox.IgnoreArg()) 130 reporting.Reporter.find_issue_by_marker(mox.IgnoreArg()).AndReturn( 131 issue) 132 133 reporting.TestBug.summary().AndReturn('') 134 135 self.mox.ReplayAll() 136 bug_id, bug_count = reporting.Reporter().report(self._get_failure()) 137 138 self.assertEqual(bug_id, self._FAKE_ISSUE_ID) 139 self.assertEqual(bug_count, 2) 140 141 142 def testSuiteIssueConfig(self): 143 """Test that the suite bug template values are not overridden.""" 144 145 def check_suite_options(issue): 146 """ 147 Checks to see if the options specified in bug_template reflect in 148 the issue we're about to file, and that the autofiled label was not 149 lost in the process. 150 151 @param issue: issue to check labels on. 152 """ 153 assert('autofiled' in issue.labels) 154 for k, v in self.bug_template.iteritems(): 155 if (isinstance(v, list) 156 and all(item in getattr(issue, k) for item in v)): 157 continue 158 if v and getattr(issue, k) is not v: 159 return False 160 return True 161 162 self.mox.StubOutWithMock(reporting.Reporter, 'find_issue_by_marker') 163 self.mox.StubOutWithMock(reporting.TestBug, 'summary') 164 165 reporting.Reporter.find_issue_by_marker(mox.IgnoreArg()).AndReturn( 166 None) 167 reporting.TestBug.summary().AndReturn('Summary') 168 169 mock_host = phapi_lib.ProjectHostingApiClient(mox.IgnoreArg(), 170 mox.IgnoreArg()) 171 mock_host.create_issue(mox.IgnoreArg()).AndReturn( 172 {'id': self._FAKE_ISSUE_ID}) 173 174 self.mox.ReplayAll() 175 bug_id, bug_count = reporting.Reporter().report(self._get_failure(), 176 self.bug_template) 177 178 self.assertEqual(bug_id, self._FAKE_ISSUE_ID) 179 self.assertEqual(bug_count, 1) 180 181 182 def testGenericBugCanBeFiled(self): 183 """Test that we can use a Bug object to file a bug report.""" 184 self.mox.StubOutWithMock(reporting.Reporter, 'find_issue_by_marker') 185 186 bug = reporting.Bug('title', 'summary', 'marker') 187 188 reporting.Reporter.find_issue_by_marker(mox.IgnoreArg()).AndReturn( 189 None) 190 191 mock_host = phapi_lib.ProjectHostingApiClient(mox.IgnoreArg(), 192 mox.IgnoreArg()) 193 mock_host.create_issue(mox.IgnoreArg()).AndReturn( 194 {'id': self._FAKE_ISSUE_ID}) 195 196 self.mox.ReplayAll() 197 bug_id, bug_count = reporting.Reporter().report(bug) 198 199 self.assertEqual(bug_id, self._FAKE_ISSUE_ID) 200 self.assertEqual(bug_count, 1) 201 202 203 def testWithSearchMarkerSetToNoneIsNotDeduped(self): 204 """Test that we do not dedupe bugs that have no search marker.""" 205 206 bug = reporting.Bug('title', 'summary', search_marker=None) 207 208 mock_host = phapi_lib.ProjectHostingApiClient(mox.IgnoreArg(), 209 mox.IgnoreArg()) 210 mock_host.create_issue(mox.IgnoreArg()).AndReturn( 211 {'id': self._FAKE_ISSUE_ID}) 212 213 self.mox.ReplayAll() 214 bug_id, bug_count = reporting.Reporter().report(bug) 215 216 self.assertEqual(bug_id, self._FAKE_ISSUE_ID) 217 self.assertEqual(bug_count, 1) 218 219 220 def testSearchMarkerNoBuildSuiteInfo(self): 221 """Test that the search marker does not include build and suite info.""" 222 test_failure = self._get_failure(is_server_job=True) 223 search_marker = test_failure.search_marker() 224 self.assertFalse(test_failure.build in search_marker, 225 ('Build information should not be presented in search ' 226 'marker.')) 227 228 229 class FindIssueByMarkerTests(mox.MoxTestBase): 230 """Tests the find_issue_by_marker function.""" 231 232 def setUp(self): 233 super(FindIssueByMarkerTests, self).setUp() 234 self.mox.StubOutClassWithMocks(phapi_lib, 'ProjectHostingApiClient') 235 self._orig_project_name = reporting.Reporter._project_name 236 237 # We want to have some data so that the Reporter doesn't fail at 238 # initialization. 239 reporting.Reporter._project_name = 'project' 240 241 242 def tearDown(self): 243 reporting.Reporter._project_name = self._orig_project_name 244 super(FindIssueByMarkerTests, self).tearDown() 245 246 247 def testReturnNoneIfMarkerIsNone(self): 248 """Test that we do not look up an issue if the search marker is None.""" 249 mock_host = phapi_lib.ProjectHostingApiClient(mox.IgnoreArg(), 250 mox.IgnoreArg()) 251 252 self.mox.ReplayAll() 253 result = reporting.Reporter().find_issue_by_marker(None) 254 self.assertTrue(result is None) 255 256 257 class AnchorSummaryTests(mox.MoxTestBase): 258 """Tests the _anchor_summary function.""" 259 260 def setUp(self): 261 super(AnchorSummaryTests, self).setUp() 262 self.mox.StubOutClassWithMocks(phapi_lib, 'ProjectHostingApiClient') 263 self._orig_project_name = reporting.Reporter._project_name 264 265 # We want to have some data so that the Reporter doesn't fail at 266 # initialization. 267 reporting.Reporter._project_name = 'project' 268 269 270 def tearDown(self): 271 reporting.Reporter._project_name = self._orig_project_name 272 super(AnchorSummaryTests, self).tearDown() 273 274 275 def test_summary_returned_untouched_if_no_search_maker(self): 276 """Test that we just return the summary if we have no search marker.""" 277 mock_host = phapi_lib.ProjectHostingApiClient(mox.IgnoreArg(), 278 mox.IgnoreArg()) 279 280 bug = reporting.Bug('title', 'summary', None) 281 282 self.mox.ReplayAll() 283 result = reporting.Reporter()._anchor_summary(bug) 284 285 self.assertEqual(result, 'summary') 286 287 288 def test_append_anchor_to_summary_if_search_marker(self): 289 """Test that we add an anchor to the search marker.""" 290 mock_host = phapi_lib.ProjectHostingApiClient(mox.IgnoreArg(), 291 mox.IgnoreArg()) 292 293 bug = reporting.Bug('title', 'summary', 'marker') 294 295 self.mox.ReplayAll() 296 result = reporting.Reporter()._anchor_summary(bug) 297 298 self.assertEqual(result, 'summary\n\n%smarker\n' % 299 reporting.Reporter._SEARCH_MARKER) 300 301 302 class LabelUpdateTests(mox.MoxTestBase): 303 """Test the _create_autofiled_count_update() function.""" 304 305 def setUp(self): 306 super(LabelUpdateTests, self).setUp() 307 self.mox.StubOutClassWithMocks(phapi_lib, 'ProjectHostingApiClient') 308 self._orig_project_name = reporting.Reporter._project_name 309 310 # We want to have some data so that the Reporter doesn't fail at 311 # initialization. 312 reporting.Reporter._project_name = 'project' 313 314 315 def tearDown(self): 316 reporting.Reporter._project_name = self._orig_project_name 317 super(LabelUpdateTests, self).tearDown() 318 319 320 def _create_count_label(self, n): 321 return '%s%d' % (reporting.Reporter.AUTOFILED_COUNT, n) 322 323 324 def _test_count_label_update(self, labels, remove, expected_count): 325 """Utility to test _create_autofiled_count_update(). 326 327 @param labels Input list of labels. 328 @param remove List of labels expected to be removed 329 in the result. 330 @param expected_count Count value expected to be returned 331 from the call. 332 """ 333 client = phapi_lib.ProjectHostingApiClient(mox.IgnoreArg(), 334 mox.IgnoreArg()) 335 self.mox.ReplayAll() 336 issue = self.mox.CreateMock(gdata_lib.Issue) 337 issue.labels = labels 338 339 reporter = reporting.Reporter() 340 new_labels, count = reporter._create_autofiled_count_update(issue) 341 expected = map(lambda l: '-' + l, remove) 342 expected.append(self._create_count_label(expected_count)) 343 self.assertEqual(new_labels, expected) 344 self.assertEqual(count, expected_count) 345 346 347 def testProjectLabelExtraction(self): 348 """Test that the project label is correctly extracted from the title.""" 349 TITLE_EMPTY = '' 350 TITLE_NO_PROJ = '[stress] platformDevice Failure on release/47-75.0.0' 351 TITLE_PROJ = '[stress] p_Device Failure on rikku-release/R44-7075.0.0' 352 TITLE_PROJ2 = '[stress] p_Device Failure on ' \ 353 'rikku-freon-release/R44-7075.0.0' 354 TITLE_PROJ_SUBBOARD = '[stress] p_Device Failure on ' \ 355 'veyron_rikku-release/R44-7075.0.0' 356 357 client = phapi_lib.ProjectHostingApiClient(mox.IgnoreArg(), 358 mox.IgnoreArg()) 359 self.mox.ReplayAll() 360 361 reporter = reporting.Reporter() 362 self.assertEqual(reporter._get_project_label_from_title(TITLE_EMPTY), 363 '') 364 self.assertEqual(reporter._get_project_label_from_title( 365 TITLE_NO_PROJ), '') 366 self.assertEqual(reporter._get_project_label_from_title(TITLE_PROJ), 367 'Proj-rikku') 368 self.assertEqual(reporter._get_project_label_from_title(TITLE_PROJ2), 369 'Proj-rikku') 370 self.assertEqual(reporter._get_project_label_from_title( 371 TITLE_PROJ_SUBBOARD), 'Proj-rikku') 372 373 374 def testCountLabelIncrement(self): 375 """Test that incrementing an autofiled-count label should work.""" 376 n = 3 377 old_label = self._create_count_label(n) 378 self._test_count_label_update([old_label], [old_label], n + 1) 379 380 381 def testCountLabelIncrementPredefined(self): 382 """Test that Reporter._PREDEFINED_LABELS has a sane autofiled-count.""" 383 self._test_count_label_update( 384 reporting.Reporter._PREDEFINED_LABELS, 385 [self._create_count_label(1)], 2) 386 387 388 def testCountLabelCreate(self): 389 """Test that old bugs should get a correct autofiled-count.""" 390 self._test_count_label_update([], [], 2) 391 392 393 def testCountLabelIncrementMultiple(self): 394 """Test that duplicate autofiled-count labels are handled.""" 395 old_count1 = self._create_count_label(2) 396 old_count2 = self._create_count_label(3) 397 self._test_count_label_update([old_count1, old_count2], 398 [old_count1, old_count2], 4) 399 400 401 def testCountLabelSkipUnknown(self): 402 """Test that autofiled-count increment ignores unknown labels.""" 403 old_count = self._create_count_label(3) 404 self._test_count_label_update(['unknown-label', old_count], 405 [old_count], 4) 406 407 408 def testCountLabelSkipMalformed(self): 409 """Test that autofiled-count increment ignores unusual labels.""" 410 old_count = self._create_count_label(3) 411 self._test_count_label_update( 412 [reporting.Reporter.AUTOFILED_COUNT + 'bogus', 413 self._create_count_label(8) + '-bogus', 414 old_count], 415 [old_count], 4) 416 417 418 class TestSubmitGenericBugReport(mox.MoxTestBase, unittest.TestCase): 419 """Test the submit_generic_bug_report function.""" 420 421 def setUp(self): 422 super(TestSubmitGenericBugReport, self).setUp() 423 self.mox.StubOutClassWithMocks(reporting, 'Reporter') 424 425 426 def test_accepts_required_arguments(self): 427 """ 428 Test that the function accepts the required arguments. 429 430 This basically tests that no exceptions are thrown. 431 432 """ 433 reporter = reporting.Reporter() 434 reporter.report(mox.IgnoreArg()).AndReturn((11,1)) 435 436 self.mox.ReplayAll() 437 reporting.submit_generic_bug_report('title', 'summary') 438 439 440 def test_rejects_too_few_required_arguments(self): 441 """Test that the function rejects too few required arguments.""" 442 self.mox.ReplayAll() 443 self.assertRaises(TypeError, 444 reporting.submit_generic_bug_report, 'too_few') 445 446 447 def test_accepts_key_word_arguments(self): 448 """ 449 Test that the functions accepts the key_word arguments. 450 451 This basically tests that no exceptions are thrown. 452 453 """ 454 reporter = reporting.Reporter() 455 reporter.report(mox.IgnoreArg()).AndReturn((11,1)) 456 457 self.mox.ReplayAll() 458 reporting.submit_generic_bug_report('test', 'summary', labels=[]) 459 460 461 def test_rejects_invalid_keyword_arguments(self): 462 """Test that the function rejects invalid keyword arguments.""" 463 self.mox.ReplayAll() 464 self.assertRaises(TypeError, reporting.submit_generic_bug_report, 465 'title', 'summary', wrong='wrong') 466 467 468 class TestMergeBugTemplate(mox.MoxTestBase): 469 """Test bug can be properly merged and validated.""" 470 def test_validate_success(self): 471 """Test a valid bug can be verified successfully.""" 472 bug_template= {} 473 bug_template['owner'] = 'someone (at] company.com' 474 reporting_utils.BugTemplate.validate_bug_template(bug_template) 475 476 477 def test_validate_success(self): 478 """Test a valid bug can be verified successfully.""" 479 # Bug template must be a dictionary. 480 bug_template = ['test'] 481 self.assertRaises(reporting_utils.InvalidBugTemplateException, 482 reporting_utils.BugTemplate.validate_bug_template, 483 bug_template) 484 485 # Bug template must contain value for essential attribute, e.g., owner. 486 bug_template= {'no-owner': 'user1'} 487 self.assertRaises(reporting_utils.InvalidBugTemplateException, 488 reporting_utils.BugTemplate.validate_bug_template, 489 bug_template) 490 491 # Bug template must contain value for essential attribute, e.g., owner. 492 bug_template= {'owner': 'invalid_email_address'} 493 self.assertRaises(reporting_utils.InvalidBugTemplateException, 494 reporting_utils.BugTemplate.validate_bug_template, 495 bug_template) 496 497 # Check unexpected attributes. 498 bug_template= {} 499 bug_template['random tag'] = 'test' 500 self.assertRaises(reporting_utils.InvalidBugTemplateException, 501 reporting_utils.BugTemplate.validate_bug_template, 502 bug_template) 503 504 # Value for cc must be a list 505 bug_template= {} 506 bug_template['cc'] = 'test' 507 self.assertRaises(reporting_utils.InvalidBugTemplateException, 508 reporting_utils.BugTemplate.validate_bug_template, 509 bug_template) 510 511 # Value for labels must be a list 512 bug_template= {} 513 bug_template['labels'] = 'test' 514 self.assertRaises(reporting_utils.InvalidBugTemplateException, 515 reporting_utils.BugTemplate.validate_bug_template, 516 bug_template) 517 518 519 def test_merge_success(self): 520 """Test test and suite bug templates can be merged successfully.""" 521 test_bug_template = { 522 'labels': ['l1'], 523 'owner': 'user1 (at] chromium.org', 524 'status': 'Assigned', 525 'title': None, 526 'cc': ['cc1 (at] chromium.org', 'cc2 (at] chromium.org'] 527 } 528 suite_bug_template = { 529 'labels': ['l2'], 530 'owner': 'user2 (at] chromium.org', 531 'status': 'Fixed', 532 'summary': 'This is a short summary for suite bug', 533 'title': 'Title for suite bug', 534 'cc': ['cc2 (at] chromium.org', 'cc3 (at] chromium.org'] 535 } 536 bug_template = reporting_utils.BugTemplate(suite_bug_template) 537 merged_bug_template = bug_template.finalize_bug_template( 538 test_bug_template) 539 self.assertEqual(merged_bug_template['owner'], 540 test_bug_template['owner'], 541 'Value in test bug template should prevail.') 542 543 self.assertEqual(merged_bug_template['title'], 544 suite_bug_template['title'], 545 'If an attribute has value None in test bug template, ' 546 'use the value given in suite bug template.') 547 548 self.assertEqual(merged_bug_template['summary'], 549 suite_bug_template['summary'], 550 'If an attribute does not exist in test bug template, ' 551 'but exists in suite bug template, it should be ' 552 'included in the merged template.') 553 554 self.assertEqual(merged_bug_template['cc'], 555 test_bug_template['cc'] + suite_bug_template['cc'], 556 'List values for an attribute should be merged.') 557 558 self.assertEqual(merged_bug_template['labels'], 559 test_bug_template['labels'] + 560 suite_bug_template['labels'], 561 'List values for an attribute should be merged.') 562 563 test_bug_template['owner'] = '' 564 test_bug_template['cc'] = [''] 565 suite_bug_template['owner'] = '' 566 suite_bug_template['cc'] = [''] 567 bug_template = reporting_utils.BugTemplate(suite_bug_template) 568 merged_bug_template = bug_template.finalize_bug_template( 569 test_bug_template) 570 self.assertFalse('owner' in merged_bug_template, 571 'owner should be removed from the merged template.') 572 self.assertFalse('cc' in merged_bug_template, 573 'cc should be removed from the merged template.') 574 575 576 if __name__ == '__main__': 577 unittest.main() 578
