1 #!/usr/bin/python 2 # 3 # Copyright (c) 2012 The Chromium OS Authors. All rights reserved. 4 # Use of this source code is governed by a BSD-style license that can be 5 # found in the LICENSE file. 6 7 # pylint: disable-msg=C0111 8 9 """Unit tests for server/cros/dynamic_suite/job_status.py.""" 10 11 import mox 12 import shutil 13 import tempfile 14 import time 15 import unittest 16 import os 17 import common 18 19 from autotest_lib.server import frontend 20 from autotest_lib.server.cros import host_lock_manager 21 from autotest_lib.server.cros.dynamic_suite import host_spec 22 from autotest_lib.server.cros.dynamic_suite import job_status 23 from autotest_lib.server.cros.dynamic_suite.fakes import FakeHost, FakeJob 24 from autotest_lib.server.cros.dynamic_suite.fakes import FakeStatus 25 26 27 DEFAULT_WAITTIMEOUT_MINS = 60 * 4 28 29 30 class StatusTest(mox.MoxTestBase): 31 """Unit tests for job_status.Status. 32 """ 33 34 35 def setUp(self): 36 super(StatusTest, self).setUp() 37 self.afe = self.mox.CreateMock(frontend.AFE) 38 self.tko = self.mox.CreateMock(frontend.TKO) 39 40 self.tmpdir = tempfile.mkdtemp(suffix=type(self).__name__) 41 42 43 def tearDown(self): 44 super(StatusTest, self).tearDown() 45 shutil.rmtree(self.tmpdir, ignore_errors=True) 46 47 48 def testGatherJobHostnamesAllRan(self): 49 """All entries for the job were assigned hosts.""" 50 job = FakeJob(0, []) 51 expected_hosts = ['host2', 'host1'] 52 entries = [{'status': 'Running', 53 'host': {'hostname': h}} for h in expected_hosts] 54 self.afe.run('get_host_queue_entries', job=job.id).AndReturn(entries) 55 self.mox.ReplayAll() 56 57 self.assertEquals(sorted(expected_hosts), 58 sorted(job_status.gather_job_hostnames(self.afe, 59 job))) 60 61 62 def testGatherJobHostnamesSomeRan(self): 63 """Not all entries for the job were assigned hosts.""" 64 job = FakeJob(0, []) 65 expected_hosts = ['host2', 'host1'] 66 entries = [{'status': 'Running', 67 'host': {'hostname': h}} for h in expected_hosts] 68 entries.append({'status': 'Running', 'host': None}) 69 self.afe.run('get_host_queue_entries', job=job.id).AndReturn(entries) 70 self.mox.ReplayAll() 71 72 self.assertEquals(sorted(expected_hosts + [None]), 73 sorted(job_status.gather_job_hostnames(self.afe, 74 job))) 75 76 77 def testGatherJobHostnamesSomeStillQueued(self): 78 """Not all entries for the job were Running, though all had hosts.""" 79 job = FakeJob(0, []) 80 expected_hosts = ['host2', 'host1'] 81 entries = [{'status': 'Running', 82 'host': {'hostname': h}} for h in expected_hosts] 83 entries[-1]['status'] = 'Queued' 84 self.afe.run('get_host_queue_entries', job=job.id).AndReturn(entries) 85 self.mox.ReplayAll() 86 87 self.assertTrue(expected_hosts[-1] not in 88 job_status.gather_job_hostnames(self.afe, job)) 89 90 91 def testWaitForJobToStart(self): 92 """Ensure we detect when a job has started running.""" 93 self.mox.StubOutWithMock(time, 'sleep') 94 95 job = FakeJob(0, []) 96 self.afe.get_jobs(id=job.id, not_yet_run=True).AndReturn([job]) 97 self.afe.get_jobs(id=job.id, not_yet_run=True).AndReturn([]) 98 time.sleep(mox.IgnoreArg()).MultipleTimes() 99 self.mox.ReplayAll() 100 101 job_status.wait_for_jobs_to_start(self.afe, [job]) 102 103 104 def testWaitForMultipleJobsToStart(self): 105 """Ensure we detect when all jobs have started running.""" 106 self.mox.StubOutWithMock(time, 'sleep') 107 108 job0 = FakeJob(0, []) 109 job1 = FakeJob(1, []) 110 self.afe.get_jobs(id=job0.id, not_yet_run=True).AndReturn([job0]) 111 self.afe.get_jobs(id=job1.id, not_yet_run=True).AndReturn([job1]) 112 self.afe.get_jobs(id=job0.id, not_yet_run=True).AndReturn([]) 113 self.afe.get_jobs(id=job1.id, not_yet_run=True).AndReturn([job1]) 114 self.afe.get_jobs(id=job1.id, not_yet_run=True).AndReturn([]) 115 time.sleep(mox.IgnoreArg()).MultipleTimes() 116 self.mox.ReplayAll() 117 118 job_status.wait_for_jobs_to_start(self.afe, [job0, job1]) 119 120 121 def testWaitForJobToStartAlreadyStarted(self): 122 """Ensure we don't wait forever if a job already started.""" 123 job = FakeJob(0, []) 124 self.afe.get_jobs(id=job.id, not_yet_run=True).AndReturn([]) 125 self.mox.ReplayAll() 126 job_status.wait_for_jobs_to_start(self.afe, [job]) 127 128 129 def testWaitForJobToFinish(self): 130 """Ensure we detect when a job has finished.""" 131 self.mox.StubOutWithMock(time, 'sleep') 132 133 job = FakeJob(0, []) 134 self.afe.get_jobs(id=job.id, finished=True).AndReturn([]) 135 self.afe.get_jobs(id=job.id, finished=True).AndReturn([job]) 136 time.sleep(mox.IgnoreArg()).MultipleTimes() 137 self.mox.ReplayAll() 138 139 job_status.wait_for_jobs_to_finish(self.afe, [job]) 140 141 142 def testWaitForMultipleJobsToFinish(self): 143 """Ensure we detect when all jobs have stopped running.""" 144 self.mox.StubOutWithMock(time, 'sleep') 145 146 job0 = FakeJob(0, []) 147 job1 = FakeJob(1, []) 148 self.afe.get_jobs(id=job0.id, finished=True).AndReturn([]) 149 self.afe.get_jobs(id=job1.id, finished=True).AndReturn([]) 150 self.afe.get_jobs(id=job0.id, finished=True).AndReturn([]) 151 self.afe.get_jobs(id=job1.id, finished=True).AndReturn([job1]) 152 self.afe.get_jobs(id=job0.id, finished=True).AndReturn([job0]) 153 time.sleep(mox.IgnoreArg()).MultipleTimes() 154 self.mox.ReplayAll() 155 156 job_status.wait_for_jobs_to_finish(self.afe, [job0, job1]) 157 158 159 def testWaitForJobToFinishAlreadyFinished(self): 160 """Ensure we don't wait forever if a job already finished.""" 161 job = FakeJob(0, []) 162 self.afe.get_jobs(id=job.id, finished=True).AndReturn([job]) 163 self.mox.ReplayAll() 164 job_status.wait_for_jobs_to_finish(self.afe, [job]) 165 166 167 def expect_hosts_query_and_lock(self, jobs, manager, running_hosts, 168 do_lock=True): 169 """Expect asking for a job's hosts and, potentially, lock them. 170 171 job_status.gather_job_hostnames() should be mocked out prior to call. 172 173 @param jobs: a lists of FakeJobs with a valid ID. 174 @param manager: mocked out HostLockManager 175 @param running_hosts: list of FakeHosts that should be listed as 176 'Running'. 177 @param do_lock: If |manager| should expect |running_hosts| to get 178 added and locked. 179 @return nothing, but self.afe, job_status.gather_job_hostnames, and 180 manager will have expectations set. 181 """ 182 used_hostnames = [] 183 for job in jobs: 184 job_status.gather_job_hostnames( 185 mox.IgnoreArg(), job).InAnyOrder().AndReturn(job.hostnames) 186 used_hostnames.extend([h for h in job.hostnames if h]) 187 188 if used_hostnames: 189 self.afe.get_hosts(mox.SameElementsAs(used_hostnames), 190 status='Running').AndReturn(running_hosts) 191 if do_lock: 192 manager.lock([h.hostname for h in running_hosts]) 193 194 195 def testWaitForSingleJobHostsToRunAndGetLocked(self): 196 """Ensure we lock all running hosts as they're discovered.""" 197 self.mox.StubOutWithMock(time, 'sleep') 198 self.mox.StubOutWithMock(job_status, 'gather_job_hostnames') 199 200 manager = self.mox.CreateMock(host_lock_manager.HostLockManager) 201 expected_hostnames=['host1', 'host0'] 202 expected_hosts = [FakeHost(h) for h in expected_hostnames] 203 job = FakeJob(7, hostnames=[None, None]) 204 205 time.sleep(mox.IgnoreArg()).MultipleTimes() 206 self.expect_hosts_query_and_lock([job], manager, [], False) 207 # First, only one test in the job has had a host assigned at all. 208 # Since no hosts are running, expect no locking. 209 job.hostnames = [None] + expected_hostnames[1:] 210 self.expect_hosts_query_and_lock([job], manager, [], False) 211 212 # Then, that host starts running, but no other tests have hosts. 213 self.expect_hosts_query_and_lock([job], manager, expected_hosts[1:]) 214 215 # The second test gets a host assigned, but it's not yet running. 216 # Since no new running hosts are found, no locking should happen. 217 job.hostnames = expected_hostnames 218 self.expect_hosts_query_and_lock([job], manager, expected_hosts[1:], 219 False) 220 # The second test's host starts running as well. 221 self.expect_hosts_query_and_lock([job], manager, expected_hosts) 222 223 # The last loop update; doesn't impact behavior. 224 job_status.gather_job_hostnames(mox.IgnoreArg(), 225 job).AndReturn(expected_hostnames) 226 self.mox.ReplayAll() 227 self.assertEquals( 228 sorted(expected_hostnames), 229 sorted(job_status.wait_for_and_lock_job_hosts(self.afe, 230 [job], 231 manager))) 232 233 234 def testWaitForAndLockWithTimeOutInStartJobs(self): 235 """If we experience a timeout, no locked hosts are returned""" 236 self.mox.StubOutWithMock(job_status, 'gather_job_hostnames') 237 self.mox.StubOutWithMock(job_status, '_abort_jobs_if_timedout') 238 239 job_status._abort_jobs_if_timedout(mox.IgnoreArg(), mox.IgnoreArg(), 240 mox.IgnoreArg(), mox.IgnoreArg()).AndReturn(True) 241 manager = self.mox.CreateMock(host_lock_manager.HostLockManager) 242 expected_hostnames=['host1', 'host0'] 243 expected_hosts = [FakeHost(h) for h in expected_hostnames] 244 job = FakeJob(7, hostnames=[None, None]) 245 job_status.gather_job_hostnames(mox.IgnoreArg(), 246 job).AndReturn(expected_hostnames) 247 self.mox.ReplayAll() 248 self.assertFalse(job_status.wait_for_and_lock_job_hosts(self.afe, 249 [job], manager, wait_timeout_mins=DEFAULT_WAITTIMEOUT_MINS)) 250 251 252 def testWaitForAndLockWithTimedOutSubJobs(self): 253 """If we experience a timeout, no locked hosts are returned""" 254 self.mox.StubOutWithMock(job_status, 'gather_job_hostnames') 255 self.mox.StubOutWithMock(job_status, '_abort_jobs_if_timedout') 256 257 job_status._abort_jobs_if_timedout(mox.IgnoreArg(), mox.IgnoreArg(), 258 mox.IgnoreArg(), mox.IgnoreArg()).AndReturn(True) 259 manager = self.mox.CreateMock(host_lock_manager.HostLockManager) 260 expected_hostnames=['host1', 'host0'] 261 expected_hosts = [FakeHost(h) for h in expected_hostnames] 262 job = FakeJob(7, hostnames=[None, None]) 263 job_status.gather_job_hostnames(mox.IgnoreArg(), 264 job).AndReturn(expected_hostnames) 265 self.mox.ReplayAll() 266 self.assertEquals(set(), 267 job_status.wait_for_and_lock_job_hosts(self.afe, [job], 268 manager, wait_timeout_mins=DEFAULT_WAITTIMEOUT_MINS)) 269 270 271 def testWaitForSingleJobHostsWithTimeout(self): 272 """Discover a single host for this job then timeout.""" 273 self.mox.StubOutWithMock(time, 'sleep') 274 self.mox.StubOutWithMock(job_status, 'gather_job_hostnames') 275 self.mox.StubOutWithMock(job_status, '_abort_jobs_if_timedout') 276 277 manager = self.mox.CreateMock(host_lock_manager.HostLockManager) 278 expected_hostnames=['host1', 'host0'] 279 expected_hosts = [FakeHost(h) for h in expected_hostnames] 280 job = FakeJob(7, hostnames=[None, None]) 281 282 time.sleep(mox.IgnoreArg()).MultipleTimes() 283 job_status._abort_jobs_if_timedout(mox.IgnoreArg(), mox.IgnoreArg(), 284 mox.IgnoreArg(), mox.IgnoreArg()).AndReturn(False) 285 self.expect_hosts_query_and_lock([job], manager, [], False) 286 287 # First, only one test in the job has had a host assigned at all. 288 # Since no hosts are running, expect no locking. 289 job_status._abort_jobs_if_timedout(mox.IgnoreArg(), mox.IgnoreArg(), 290 mox.IgnoreArg(), mox.IgnoreArg()).AndReturn(False) 291 job.hostnames = [None] + expected_hostnames[1:] 292 self.expect_hosts_query_and_lock([job], manager, [], False) 293 294 # Then, that host starts running, but no other tests have hosts. 295 job_status._abort_jobs_if_timedout(mox.IgnoreArg(), mox.IgnoreArg(), 296 mox.IgnoreArg(), mox.IgnoreArg()).AndReturn(False) 297 self.expect_hosts_query_and_lock([job], manager, expected_hosts[1:]) 298 299 # The second test gets a host assigned, but it's not yet running. 300 # Since no new running hosts are found, no locking should happen. 301 job_status._abort_jobs_if_timedout(mox.IgnoreArg(), mox.IgnoreArg(), 302 mox.IgnoreArg(), mox.IgnoreArg()).AndReturn(False) 303 job.hostnames = expected_hostnames 304 self.expect_hosts_query_and_lock([job], manager, expected_hosts[1:], 305 False) 306 307 # A timeout occurs, and only the locked hosts should be returned. 308 job_status._abort_jobs_if_timedout(mox.IgnoreArg(), mox.IgnoreArg(), 309 mox.IgnoreArg(), mox.IgnoreArg()).AndReturn(True) 310 311 # The last loop update; doesn't impact behavior. 312 job_status.gather_job_hostnames(mox.IgnoreArg(), 313 job).AndReturn(expected_hostnames) 314 self.mox.ReplayAll() 315 316 # Because of the timeout only one host is returned. 317 expect_timeout_hostnames = ['host0'] 318 self.assertEquals(sorted(expect_timeout_hostnames),sorted( 319 job_status.wait_for_and_lock_job_hosts(self.afe, 320 [job],manager, wait_timeout_mins=DEFAULT_WAITTIMEOUT_MINS))) 321 322 323 def testWaitForSingleJobHostsToRunAndGetLockedSerially(self): 324 """Lock running hosts as discovered, serially.""" 325 self.mox.StubOutWithMock(time, 'sleep') 326 self.mox.StubOutWithMock(job_status, 'gather_job_hostnames') 327 328 manager = self.mox.CreateMock(host_lock_manager.HostLockManager) 329 expected_hostnames=['host1', 'host0'] 330 expected_hosts = [FakeHost(h) for h in expected_hostnames] 331 job = FakeJob(7, hostnames=[None, None]) 332 333 time.sleep(mox.IgnoreArg()).MultipleTimes() 334 self.expect_hosts_query_and_lock([job], manager, [], False) 335 # First, only one test in the job has had a host assigned at all. 336 # Since no hosts are running, expect no locking. 337 job.hostnames = [None] + expected_hostnames[1:] 338 self.expect_hosts_query_and_lock([job], manager, [], False) 339 340 # Then, that host starts running, but no other tests have hosts. 341 self.expect_hosts_query_and_lock([job], manager, expected_hosts[1:]) 342 343 # The second test gets a host assigned, but it's not yet running. 344 # Since no new running hosts are found, no locking should happen. 345 job.hostnames = expected_hostnames 346 self.expect_hosts_query_and_lock([job], manager, expected_hosts[1:], 347 False) 348 # The second test's host starts running as well, and the first stops. 349 self.expect_hosts_query_and_lock([job], manager, expected_hosts[:1]) 350 351 # The last loop update; doesn't impact behavior. 352 job_status.gather_job_hostnames(mox.IgnoreArg(), 353 job).AndReturn(expected_hostnames) 354 self.mox.ReplayAll() 355 self.assertEquals( 356 sorted(expected_hostnames), 357 sorted(job_status.wait_for_and_lock_job_hosts(self.afe, 358 [job], 359 manager))) 360 361 362 def testWaitForMultiJobHostsToRunAndGetLocked(self): 363 """Ensure we lock all running hosts for all jobs as discovered.""" 364 self.mox.StubOutWithMock(time, 'sleep') 365 self.mox.StubOutWithMock(job_status, 'gather_job_hostnames') 366 367 manager = self.mox.CreateMock(host_lock_manager.HostLockManager) 368 expected_hostnames = ['host1', 'host0', 'host2'] 369 expected_hosts = [FakeHost(h) for h in expected_hostnames] 370 job0 = FakeJob(0, hostnames=[]) 371 job1 = FakeJob(1, hostnames=[]) 372 373 time.sleep(mox.IgnoreArg()).MultipleTimes() 374 # First, only one test in either job has had a host assigned at all. 375 # Since no hosts are running, expect no locking. 376 job0.hostnames = [None, expected_hostnames[2]] 377 job1.hostnames = [None] 378 self.expect_hosts_query_and_lock([job0, job1], manager, [], False) 379 380 # Then, that host starts running, but no other tests have hosts. 381 self.expect_hosts_query_and_lock([job0, job1], manager, 382 expected_hosts[2:]) 383 384 # The test in the second job gets a host assigned, but it's not yet 385 # running. 386 # Since no new running hosts are found, no locking should happen. 387 job1.hostnames = expected_hostnames[1:2] 388 self.expect_hosts_query_and_lock([job0, job1], manager, 389 expected_hosts[2:], False) 390 391 # The second job's test's host starts running as well. 392 self.expect_hosts_query_and_lock([job0, job1], manager, 393 expected_hosts[1:]) 394 395 # All three hosts across both jobs are now running. 396 job0.hostnames = [expected_hostnames[0], expected_hostnames[2]] 397 self.expect_hosts_query_and_lock([job0, job1], manager, expected_hosts) 398 399 # The last loop update; doesn't impact behavior. 400 job_status.gather_job_hostnames(mox.IgnoreArg(), 401 job0).AndReturn(job0.hostnames) 402 job_status.gather_job_hostnames(mox.IgnoreArg(), 403 job1).AndReturn(job1.hostnames) 404 405 self.mox.ReplayAll() 406 self.assertEquals( 407 sorted(expected_hostnames), 408 sorted(job_status.wait_for_and_lock_job_hosts(self.afe, 409 [job0, job1], 410 manager))) 411 412 413 def expect_result_gathering(self, job): 414 self.afe.get_jobs(id=job.id, finished=True).AndReturn(job) 415 self.expect_yield_job_entries(job) 416 417 418 def expect_yield_job_entries(self, job): 419 entries = [s.entry for s in job.statuses] 420 self.afe.run('get_host_queue_entries', 421 job=job.id).AndReturn(entries) 422 if True not in map(lambda e: 'aborted' in e and e['aborted'], entries): 423 self.tko.get_job_test_statuses_from_db(job.id).AndReturn( 424 job.statuses) 425 426 427 def testWaitForResults(self): 428 """Should gather status and return records for job summaries.""" 429 jobs = [FakeJob(0, [FakeStatus('GOOD', 'T0', ''), 430 FakeStatus('GOOD', 'T1', '')]), 431 FakeJob(1, [FakeStatus('ERROR', 'T0', 'err', False), 432 FakeStatus('GOOD', 'T1', '')]), 433 FakeJob(2, [FakeStatus('TEST_NA', 'T0', 'no')]), 434 FakeJob(3, [FakeStatus('FAIL', 'T0', 'broken')]), 435 FakeJob(4, [FakeStatus('ERROR', 'SERVER_JOB', 'server error'), 436 FakeStatus('GOOD', 'T0', '')]),] 437 438 # TODO: Write a better test for the case where we yield 439 # results for aborts vs cannot yield results because of 440 # a premature abort. Currently almost all client aborts 441 # have been converted to failures, and when aborts do happen 442 # they result in server job failures for which we always 443 # want results. 444 # FakeJob(5, [FakeStatus('ERROR', 'T0', 'gah', True)]), 445 # The next job shouldn't be recorded in the results. 446 # FakeJob(6, [FakeStatus('GOOD', 'SERVER_JOB', '')])] 447 448 for status in jobs[4].statuses: 449 status.entry['job'] = {'name': 'broken_infra_job'} 450 451 # To simulate a job that isn't ready the first time we check. 452 self.afe.get_jobs(id=jobs[0].id, finished=True).AndReturn([]) 453 # Expect all the rest of the jobs to be good to go the first time. 454 for job in jobs[1:]: 455 self.expect_result_gathering(job) 456 # Then, expect job[0] to be ready. 457 self.expect_result_gathering(jobs[0]) 458 # Expect us to poll twice. 459 self.mox.StubOutWithMock(time, 'sleep') 460 time.sleep(5) 461 time.sleep(5) 462 self.mox.ReplayAll() 463 464 results = [result for result in job_status.wait_for_results(self.afe, 465 self.tko, 466 jobs)] 467 for job in jobs[:6]: # the 'GOOD' SERVER_JOB shouldn't be there. 468 for status in job.statuses: 469 self.assertTrue(True in map(status.equals_record, results)) 470 471 472 def testWaitForChildResults(self): 473 """Should gather status and return records for job summaries.""" 474 parent_job_id = 54321 475 jobs = [FakeJob(0, [FakeStatus('GOOD', 'T0', ''), 476 FakeStatus('GOOD', 'T1', '')], 477 parent_job_id=parent_job_id), 478 FakeJob(1, [FakeStatus('ERROR', 'T0', 'err', False), 479 FakeStatus('GOOD', 'T1', '')], 480 parent_job_id=parent_job_id), 481 FakeJob(2, [FakeStatus('TEST_NA', 'T0', 'no')], 482 parent_job_id=parent_job_id), 483 FakeJob(3, [FakeStatus('FAIL', 'T0', 'broken')], 484 parent_job_id=parent_job_id), 485 FakeJob(4, [FakeStatus('ERROR', 'SERVER_JOB', 'server error'), 486 FakeStatus('GOOD', 'T0', '')], 487 parent_job_id=parent_job_id),] 488 489 # TODO: Write a better test for the case where we yield 490 # results for aborts vs cannot yield results because of 491 # a premature abort. Currently almost all client aborts 492 # have been converted to failures and when aborts do happen 493 # they result in server job failures for which we always 494 # want results. 495 #FakeJob(5, [FakeStatus('ERROR', 'T0', 'gah', True)], 496 # parent_job_id=parent_job_id), 497 # The next job shouldn't be recorded in the results. 498 #FakeJob(6, [FakeStatus('GOOD', 'SERVER_JOB', '')], 499 # parent_job_id=12345)] 500 for status in jobs[4].statuses: 501 status.entry['job'] = {'name': 'broken_infra_job'} 502 503 # Expect one call to get a list of all child jobs. 504 self.afe.get_jobs(parent_job_id=parent_job_id).AndReturn(jobs[:6]) 505 506 # Have the first two jobs be finished by the first polling, 507 # and the remaining ones (not including #6) for the second polling. 508 self.afe.get_jobs(parent_job_id=parent_job_id, 509 finished=True).AndReturn([jobs[1]]) 510 self.expect_yield_job_entries(jobs[1]) 511 512 self.afe.get_jobs(parent_job_id=parent_job_id, 513 finished=True).AndReturn(jobs[:2]) 514 self.expect_yield_job_entries(jobs[0]) 515 516 self.afe.get_jobs(parent_job_id=parent_job_id, 517 finished=True).AndReturn(jobs[:6]) 518 for job in jobs[2:6]: 519 self.expect_yield_job_entries(job) 520 # Then, expect job[0] to be ready. 521 522 # Expect us to poll thrice 523 self.mox.StubOutWithMock(time, 'sleep') 524 time.sleep(5) 525 time.sleep(5) 526 time.sleep(5) 527 self.mox.ReplayAll() 528 529 results = [result for result in job_status.wait_for_child_results( 530 self.afe, 531 self.tko, 532 parent_job_id)] 533 for job in jobs[:6]: # the 'GOOD' SERVER_JOB shouldn't be there. 534 for status in job.statuses: 535 self.assertTrue(True in map(status.equals_record, results)) 536 537 538 def testYieldSubdir(self): 539 """Make sure subdir are properly set for test and non-test status.""" 540 job_tag = '0-owner/172.33.44.55' 541 job_name = 'broken_infra_job' 542 job = FakeJob(0, [FakeStatus('ERROR', 'SERVER_JOB', 'server error', 543 subdir='---', job_tag=job_tag), 544 FakeStatus('GOOD', 'T0', '', 545 subdir='T0.subdir', job_tag=job_tag)], 546 parent_job_id=54321) 547 for status in job.statuses: 548 status.entry['job'] = {'name': job_name} 549 self.expect_yield_job_entries(job) 550 self.mox.ReplayAll() 551 results = list(job_status._yield_job_results(self.afe, self.tko, job)) 552 for i in range(len(results)): 553 result = results[i] 554 if result.test_name.endswith('SERVER_JOB'): 555 expected_name = '%s_%s' % (job_name, job.statuses[i].test_name) 556 expected_subdir = job_tag 557 else: 558 expected_name = job.statuses[i].test_name 559 expected_subdir = os.path.join(job_tag, job.statuses[i].subdir) 560 self.assertEqual(results[i].test_name, expected_name) 561 self.assertEqual(results[i].subdir, expected_subdir) 562 563 564 def testGatherPerHostResults(self): 565 """Should gather per host results.""" 566 # For the 0th job, the 1st entry is more bad/specific. 567 # For all the others, it's the 0th that we expect. 568 jobs = [FakeJob(0, [FakeStatus('FAIL', 'T0', '', hostname='h0'), 569 FakeStatus('FAIL', 'T1', 'bad', hostname='h0')]), 570 FakeJob(1, [FakeStatus('ERROR', 'T0', 'err', False, 'h1'), 571 FakeStatus('GOOD', 'T1', '', hostname='h1')]), 572 FakeJob(2, [FakeStatus('TEST_NA', 'T0', 'no', hostname='h2')]), 573 FakeJob(3, [FakeStatus('FAIL', 'T0', 'broken', hostname='h3')]), 574 FakeJob(4, [FakeStatus('ERROR', 'T0', 'gah', True, 'h4')]), 575 FakeJob(5, [FakeStatus('GOOD', 'T0', 'Yay', hostname='h5')])] 576 # Method under test returns status available right now. 577 for job in jobs: 578 entries = map(lambda s: s.entry, job.statuses) 579 self.afe.run('get_host_queue_entries', 580 job=job.id).AndReturn(entries) 581 self.tko.get_job_test_statuses_from_db(job.id).AndReturn( 582 job.statuses) 583 self.mox.ReplayAll() 584 585 results = job_status.gather_per_host_results(self.afe, 586 self.tko, 587 jobs).values() 588 for status in [jobs[0].statuses[1]] + [j.statuses[0] for j in jobs[1:]]: 589 self.assertTrue(True in map(status.equals_hostname_record, results)) 590 591 592 def _prepareForReporting(self, results): 593 def callable(x): 594 pass 595 596 record_entity = self.mox.CreateMock(callable) 597 group = self.mox.CreateMock(host_spec.HostGroup) 598 599 statuses = {} 600 all_bad = True not in results.itervalues() 601 for hostname, result in results.iteritems(): 602 status = self.mox.CreateMock(job_status.Status) 603 status.record_all(record_entity).InAnyOrder('recording') 604 status.is_good().InAnyOrder('recording').AndReturn(result) 605 if not result: 606 status.test_name = 'test' 607 if not all_bad: 608 status.override_status('WARN').InAnyOrder('recording') 609 else: 610 group.mark_host_success(hostname).InAnyOrder('recording') 611 statuses[hostname] = status 612 613 return (statuses, group, record_entity) 614 615 616 def testRecordAndReportGoodResults(self): 617 """Record and report success across the board.""" 618 results = {'h1': True, 'h2': True} 619 (statuses, group, record_entity) = self._prepareForReporting(results) 620 group.enough_hosts_succeeded().AndReturn(True) 621 self.mox.ReplayAll() 622 623 success = job_status.check_and_record_reimage_results(statuses, 624 group, 625 record_entity) 626 self.assertTrue(success) 627 628 629 def testRecordAndReportOkayResults(self): 630 """Record and report success of at least one host.""" 631 results = {'h1': False, 'h2': True} 632 (statuses, group, record_entity) = self._prepareForReporting(results) 633 group.enough_hosts_succeeded().AndReturn(True) 634 self.mox.ReplayAll() 635 636 success = job_status.check_and_record_reimage_results(statuses, 637 group, 638 record_entity) 639 self.assertTrue(success) 640 641 642 def testRecordAndReportBadResults(self): 643 """Record and report failure across the board.""" 644 results = {'h1': False, 'h2': False} 645 (statuses, group, record_entity) = self._prepareForReporting(results) 646 group.enough_hosts_succeeded().AndReturn(False) 647 self.mox.ReplayAll() 648 649 success = job_status.check_and_record_reimage_results(statuses, 650 group, 651 record_entity) 652 self.assertFalse(success) 653 654 655 if __name__ == '__main__': 656 unittest.main() 657
