1 #!/usr/bin/python 2 # Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 import mox 7 import unittest 8 9 import common 10 import time 11 12 import autoupdater 13 from autotest_lib.client.common_lib import error 14 15 class TestAutoUpdater(mox.MoxTestBase): 16 """Test autoupdater module.""" 17 18 19 def testParseBuildFromUpdateUrlwithUpdate(self): 20 """Test that we properly parse the build from an update_url.""" 21 update_url = ('http://172.22.50.205:8082/update/lumpy-release/' 22 'R27-3837.0.0') 23 expected_value = 'lumpy-release/R27-3837.0.0' 24 self.assertEqual(autoupdater.url_to_image_name(update_url), 25 expected_value) 26 27 28 def testCheckVersion_1(self): 29 """Test version check methods work for any build. 30 31 Test two methods used to check version, check_version and 32 check_version_to_confirm_install, for: 33 1. trybot paladin build. 34 update version: trybot-lumpy-paladin/R27-3837.0.0-b123 35 booted version: 3837.0.2013_03_21_1340 36 37 """ 38 update_url = ('http://172.22.50.205:8082/update/trybot-lumpy-paladin/' 39 'R27-1111.0.0-b123') 40 updater = autoupdater.ChromiumOSUpdater( 41 update_url, host=self.mox.CreateMockAnything()) 42 43 self.mox.UnsetStubs() 44 self.mox.StubOutWithMock(updater.host, 'get_release_version') 45 updater.host.get_release_version().MultipleTimes().AndReturn( 46 '1111.0.2013_03_21_1340') 47 self.mox.ReplayAll() 48 49 self.assertFalse(updater.check_version()) 50 self.assertTrue(updater.check_version_to_confirm_install()) 51 52 self.mox.UnsetStubs() 53 self.mox.StubOutWithMock(updater.host, 'get_release_version') 54 updater.host.get_release_version().MultipleTimes().AndReturn( 55 '1111.0.0-rc1') 56 self.mox.ReplayAll() 57 58 self.assertFalse(updater.check_version()) 59 self.assertFalse(updater.check_version_to_confirm_install()) 60 61 self.mox.UnsetStubs() 62 self.mox.StubOutWithMock(updater.host, 'get_release_version') 63 updater.host.get_release_version().MultipleTimes().AndReturn('1111.0.0') 64 self.mox.ReplayAll() 65 66 self.assertFalse(updater.check_version()) 67 self.assertFalse(updater.check_version_to_confirm_install()) 68 69 self.mox.UnsetStubs() 70 self.mox.StubOutWithMock(updater.host, 'get_release_version') 71 updater.host.get_release_version().MultipleTimes().AndReturn( 72 '4444.0.0-pgo-generate') 73 self.mox.ReplayAll() 74 75 self.assertFalse(updater.check_version()) 76 self.assertFalse(updater.check_version_to_confirm_install()) 77 78 79 def testCheckVersion_2(self): 80 """Test version check methods work for any build. 81 82 Test two methods used to check version, check_version and 83 check_version_to_confirm_install, for: 84 2. trybot release build. 85 update version: trybot-lumpy-release/R27-3837.0.0-b456 86 booted version: 3837.0.0 87 88 """ 89 update_url = ('http://172.22.50.205:8082/update/trybot-lumpy-release/' 90 'R27-2222.0.0-b456') 91 updater = autoupdater.ChromiumOSUpdater( 92 update_url, host=self.mox.CreateMockAnything()) 93 94 self.mox.UnsetStubs() 95 self.mox.StubOutWithMock(updater.host, 'get_release_version') 96 updater.host.get_release_version().MultipleTimes().AndReturn( 97 '2222.0.2013_03_21_1340') 98 self.mox.ReplayAll() 99 100 self.assertFalse(updater.check_version()) 101 self.assertFalse(updater.check_version_to_confirm_install()) 102 103 self.mox.UnsetStubs() 104 self.mox.StubOutWithMock(updater.host, 'get_release_version') 105 updater.host.get_release_version().MultipleTimes().AndReturn( 106 '2222.0.0-rc1') 107 self.mox.ReplayAll() 108 109 self.assertFalse(updater.check_version()) 110 self.assertFalse(updater.check_version_to_confirm_install()) 111 112 self.mox.UnsetStubs() 113 self.mox.StubOutWithMock(updater.host, 'get_release_version') 114 updater.host.get_release_version().MultipleTimes().AndReturn('2222.0.0') 115 self.mox.ReplayAll() 116 117 self.assertFalse(updater.check_version()) 118 self.assertTrue(updater.check_version_to_confirm_install()) 119 120 self.mox.UnsetStubs() 121 self.mox.StubOutWithMock(updater.host, 'get_release_version') 122 updater.host.get_release_version().MultipleTimes().AndReturn( 123 '4444.0.0-pgo-generate') 124 self.mox.ReplayAll() 125 126 self.assertFalse(updater.check_version()) 127 self.assertFalse(updater.check_version_to_confirm_install()) 128 129 130 def testCheckVersion_3(self): 131 """Test version check methods work for any build. 132 133 Test two methods used to check version, check_version and 134 check_version_to_confirm_install, for: 135 3. buildbot official release build. 136 update version: lumpy-release/R27-3837.0.0 137 booted version: 3837.0.0 138 139 """ 140 update_url = ('http://172.22.50.205:8082/update/lumpy-release/' 141 'R27-3333.0.0') 142 updater = autoupdater.ChromiumOSUpdater( 143 update_url, host=self.mox.CreateMockAnything()) 144 145 self.mox.UnsetStubs() 146 self.mox.StubOutWithMock(updater.host, 'get_release_version') 147 updater.host.get_release_version().MultipleTimes().AndReturn( 148 '3333.0.2013_03_21_1340') 149 self.mox.ReplayAll() 150 151 self.assertFalse(updater.check_version()) 152 self.assertFalse(updater.check_version_to_confirm_install()) 153 154 self.mox.UnsetStubs() 155 self.mox.StubOutWithMock(updater.host, 'get_release_version') 156 updater.host.get_release_version().MultipleTimes().AndReturn( 157 '3333.0.0-rc1') 158 self.mox.ReplayAll() 159 160 self.assertFalse(updater.check_version()) 161 self.assertFalse(updater.check_version_to_confirm_install()) 162 163 self.mox.UnsetStubs() 164 self.mox.StubOutWithMock(updater.host, 'get_release_version') 165 updater.host.get_release_version().MultipleTimes().AndReturn('3333.0.0') 166 self.mox.ReplayAll() 167 168 self.assertTrue(updater.check_version()) 169 self.assertTrue(updater.check_version_to_confirm_install()) 170 171 self.mox.UnsetStubs() 172 self.mox.StubOutWithMock(updater.host, 'get_release_version') 173 updater.host.get_release_version().MultipleTimes().AndReturn( 174 '4444.0.0-pgo-generate') 175 self.mox.ReplayAll() 176 177 self.assertFalse(updater.check_version()) 178 self.assertFalse(updater.check_version_to_confirm_install()) 179 180 181 def testCheckVersion_4(self): 182 """Test version check methods work for any build. 183 184 Test two methods used to check version, check_version and 185 check_version_to_confirm_install, for: 186 4. non-official paladin rc build. 187 update version: lumpy-paladin/R27-3837.0.0-rc7 188 booted version: 3837.0.0-rc7 189 190 """ 191 update_url = ('http://172.22.50.205:8082/update/lumpy-paladin/' 192 'R27-4444.0.0-rc7') 193 updater = autoupdater.ChromiumOSUpdater( 194 update_url, host=self.mox.CreateMockAnything()) 195 196 self.mox.UnsetStubs() 197 self.mox.StubOutWithMock(updater.host, 'get_release_version') 198 updater.host.get_release_version().MultipleTimes().AndReturn( 199 '4444.0.2013_03_21_1340') 200 self.mox.ReplayAll() 201 202 self.assertFalse(updater.check_version()) 203 self.assertFalse(updater.check_version_to_confirm_install()) 204 205 self.mox.UnsetStubs() 206 self.mox.StubOutWithMock(updater.host, 'get_release_version') 207 updater.host.get_release_version().MultipleTimes().AndReturn( 208 '4444.0.0-rc7') 209 self.mox.ReplayAll() 210 211 self.assertTrue(updater.check_version()) 212 self.assertTrue(updater.check_version_to_confirm_install()) 213 214 self.mox.UnsetStubs() 215 self.mox.StubOutWithMock(updater.host, 'get_release_version') 216 updater.host.get_release_version().MultipleTimes().AndReturn('4444.0.0') 217 self.mox.ReplayAll() 218 219 self.assertFalse(updater.check_version()) 220 self.assertFalse(updater.check_version_to_confirm_install()) 221 222 self.mox.UnsetStubs() 223 self.mox.StubOutWithMock(updater.host, 'get_release_version') 224 updater.host.get_release_version().MultipleTimes().AndReturn( 225 '4444.0.0-pgo-generate') 226 self.mox.ReplayAll() 227 228 self.assertFalse(updater.check_version()) 229 self.assertFalse(updater.check_version_to_confirm_install()) 230 231 232 def testCheckVersion_5(self): 233 """Test version check methods work for any build. 234 235 Test two methods used to check version, check_version and 236 check_version_to_confirm_install, for: 237 5. chrome-perf build. 238 update version: lumpy-chrome-perf/R28-3837.0.0-b2996 239 booted version: 3837.0.0 240 241 """ 242 update_url = ('http://172.22.50.205:8082/update/lumpy-chrome-perf/' 243 'R28-4444.0.0-b2996') 244 updater = autoupdater.ChromiumOSUpdater( 245 update_url, host=self.mox.CreateMockAnything()) 246 247 self.mox.UnsetStubs() 248 self.mox.StubOutWithMock(updater.host, 'get_release_version') 249 updater.host.get_release_version().MultipleTimes().AndReturn( 250 '4444.0.2013_03_21_1340') 251 self.mox.ReplayAll() 252 253 self.assertFalse(updater.check_version()) 254 self.assertFalse(updater.check_version_to_confirm_install()) 255 256 self.mox.UnsetStubs() 257 self.mox.StubOutWithMock(updater.host, 'get_release_version') 258 updater.host.get_release_version().MultipleTimes().AndReturn( 259 '4444.0.0-rc7') 260 self.mox.ReplayAll() 261 262 self.assertFalse(updater.check_version()) 263 self.assertFalse(updater.check_version_to_confirm_install()) 264 265 self.mox.UnsetStubs() 266 self.mox.StubOutWithMock(updater.host, 'get_release_version') 267 updater.host.get_release_version().MultipleTimes().AndReturn('4444.0.0') 268 self.mox.ReplayAll() 269 270 self.assertFalse(updater.check_version()) 271 self.assertTrue(updater.check_version_to_confirm_install()) 272 273 self.mox.UnsetStubs() 274 self.mox.StubOutWithMock(updater.host, 'get_release_version') 275 updater.host.get_release_version().MultipleTimes().AndReturn( 276 '4444.0.0-pgo-generate') 277 self.mox.ReplayAll() 278 279 self.assertFalse(updater.check_version()) 280 self.assertFalse(updater.check_version_to_confirm_install()) 281 282 283 def testCheckVersion_6(self): 284 """Test version check methods work for any build. 285 286 Test two methods used to check version, check_version and 287 check_version_to_confirm_install, for: 288 6. pgo-generate build. 289 update version: lumpy-release-pgo-generate/R28-3837.0.0-b2996 290 booted version: 3837.0.0-pgo-generate 291 292 """ 293 update_url = ('http://172.22.50.205:8082/update/lumpy-release-pgo-' 294 'generate/R28-4444.0.0-b2996') 295 updater = autoupdater.ChromiumOSUpdater( 296 update_url, host=self.mox.CreateMockAnything()) 297 298 self.mox.UnsetStubs() 299 self.mox.StubOutWithMock(updater.host, 'get_release_version') 300 updater.host.get_release_version().MultipleTimes().AndReturn( 301 '4444.0.0-2013_03_21_1340') 302 self.mox.ReplayAll() 303 304 self.assertFalse(updater.check_version()) 305 self.assertFalse(updater.check_version_to_confirm_install()) 306 307 self.mox.UnsetStubs() 308 self.mox.StubOutWithMock(updater.host, 'get_release_version') 309 updater.host.get_release_version().MultipleTimes().AndReturn( 310 '4444.0.0-rc7') 311 self.mox.ReplayAll() 312 313 self.assertFalse(updater.check_version()) 314 self.assertFalse(updater.check_version_to_confirm_install()) 315 316 self.mox.UnsetStubs() 317 self.mox.StubOutWithMock(updater.host, 'get_release_version') 318 updater.host.get_release_version().MultipleTimes().AndReturn('4444.0.0') 319 self.mox.ReplayAll() 320 321 self.assertFalse(updater.check_version()) 322 self.assertFalse(updater.check_version_to_confirm_install()) 323 324 self.mox.UnsetStubs() 325 self.mox.StubOutWithMock(updater.host, 'get_release_version') 326 updater.host.get_release_version().MultipleTimes().AndReturn( 327 '4444.0.0-pgo-generate') 328 self.mox.ReplayAll() 329 330 self.assertFalse(updater.check_version()) 331 self.assertTrue(updater.check_version_to_confirm_install()) 332 333 334 def testCheckVersion_7(self): 335 """Test version check methods work for a test-ap build. 336 337 Test two methods used to check version, check_version and 338 check_version_to_confirm_install, for: 339 6. test-ap build. 340 update version: trybot-stumpy-test-ap/R46-7298.0.0-b23 341 booted version: 7298.0.0 342 343 """ 344 update_url = ('http://100.107.160.2:8082/update/trybot-stumpy-test-api' 345 '/R46-7298.0.0-b23') 346 updater = autoupdater.ChromiumOSUpdater( 347 update_url, host=self.mox.CreateMockAnything()) 348 349 self.mox.UnsetStubs() 350 self.mox.StubOutWithMock(updater.host, 'get_release_version') 351 updater.host.get_release_version().MultipleTimes().AndReturn( 352 '7298.0.2015_07_24_1640') 353 self.mox.ReplayAll() 354 355 self.assertFalse(updater.check_version()) 356 self.assertTrue(updater.check_version_to_confirm_install()) 357 358 self.mox.UnsetStubs() 359 self.mox.StubOutWithMock(updater.host, 'get_release_version') 360 updater.host.get_release_version().MultipleTimes().AndReturn( 361 '7298.0.2015_07_24_1640') 362 self.mox.ReplayAll() 363 364 self.assertFalse(updater.check_version()) 365 self.assertTrue(updater.check_version_to_confirm_install()) 366 367 self.mox.UnsetStubs() 368 self.mox.StubOutWithMock(updater.host, 'get_release_version') 369 updater.host.get_release_version().MultipleTimes().AndReturn('7298.0.0') 370 self.mox.ReplayAll() 371 372 self.assertFalse(updater.check_version()) 373 self.assertFalse(updater.check_version_to_confirm_install()) 374 375 self.mox.UnsetStubs() 376 self.mox.StubOutWithMock(updater.host, 'get_release_version') 377 updater.host.get_release_version().MultipleTimes().AndReturn( 378 '7298.0.0') 379 self.mox.ReplayAll() 380 381 self.assertFalse(updater.check_version()) 382 self.assertFalse(updater.check_version_to_confirm_install()) 383 384 385 def _host_run_for_update(self, cmd, exception=None, 386 bad_update_status=False): 387 """Helper function for AU tests. 388 389 @param host: the test host 390 @param cmd: the command to be recorded 391 @param exception: the exception to be recorded, or None 392 """ 393 if exception: 394 self.host.run(command=cmd).AndRaise(exception) 395 else: 396 result = self.mox.CreateMockAnything() 397 if bad_update_status: 398 # Pick randomly one unexpected status 399 result.stdout = 'UPDATE_STATUS_UPDATED_NEED_REBOOT' 400 else: 401 result.stdout = 'UPDATE_STATUS_IDLE' 402 result.status = 0 403 self.host.run(command=cmd).AndReturn(result) 404 405 406 def testTriggerUpdate(self): 407 """Tests that we correctly handle updater errors.""" 408 update_url = 'http://server/test/url' 409 self.host = self.mox.CreateMockAnything() 410 self.mox.StubOutWithMock(self.host, 'run') 411 self.mox.StubOutWithMock(autoupdater.ChromiumOSUpdater, 412 'get_last_update_error') 413 self.host.hostname = 'test_host' 414 updater_control_bin = '/usr/bin/update_engine_client' 415 test_url = 'http://server/test/url' 416 expected_wait_cmd = ('%s -status | grep CURRENT_OP' % 417 updater_control_bin) 418 expected_cmd = ('%s --check_for_update --omaha_url=%s' % 419 (updater_control_bin, test_url)) 420 self.mox.StubOutWithMock(time, "sleep") 421 UPDATE_ENGINE_RETRY_WAIT_TIME=5 422 423 # Generic SSH Error. 424 cmd_result_255 = self.mox.CreateMockAnything() 425 cmd_result_255.exit_status = 255 426 427 # Command Failed Error 428 cmd_result_1 = self.mox.CreateMockAnything() 429 cmd_result_1.exit_status = 1 430 431 # Error 37 432 cmd_result_37 = self.mox.CreateMockAnything() 433 cmd_result_37.exit_status = 37 434 435 updater = autoupdater.ChromiumOSUpdater(update_url, host=self.host) 436 437 # (SUCCESS) Expect one wait command and one status command. 438 self._host_run_for_update(expected_wait_cmd) 439 self._host_run_for_update(expected_cmd) 440 441 # (SUCCESS) Test with one retry to wait for update-engine. 442 self._host_run_for_update(expected_wait_cmd, exception= 443 error.AutoservRunError('non-zero status', cmd_result_1)) 444 time.sleep(UPDATE_ENGINE_RETRY_WAIT_TIME) 445 self._host_run_for_update(expected_wait_cmd) 446 self._host_run_for_update(expected_cmd) 447 448 # (SUCCESS) One-time SSH timeout, then success on retry. 449 self._host_run_for_update(expected_wait_cmd) 450 self._host_run_for_update(expected_cmd, exception= 451 error.AutoservSSHTimeout('ssh timed out', cmd_result_255)) 452 self._host_run_for_update(expected_cmd) 453 454 # (SUCCESS) One-time ERROR 37, then success. 455 self._host_run_for_update(expected_wait_cmd) 456 self._host_run_for_update(expected_cmd, exception= 457 error.AutoservRunError('ERROR_CODE=37', cmd_result_37)) 458 self._host_run_for_update(expected_cmd) 459 460 # (FAILURE) Bad status of update engine. 461 self._host_run_for_update(expected_wait_cmd) 462 self._host_run_for_update(expected_cmd, bad_update_status=True, 463 exception=error.InstallError( 464 'host is not in installable state')) 465 466 # (FAILURE) Two-time SSH timeout. 467 self._host_run_for_update(expected_wait_cmd) 468 self._host_run_for_update(expected_cmd, exception= 469 error.AutoservSSHTimeout('ssh timed out', cmd_result_255)) 470 self._host_run_for_update(expected_cmd, exception= 471 error.AutoservSSHTimeout('ssh timed out', cmd_result_255)) 472 473 # (FAILURE) SSH Permission Error 474 self._host_run_for_update(expected_wait_cmd) 475 self._host_run_for_update(expected_cmd, exception= 476 error.AutoservSshPermissionDeniedError('no permission', 477 cmd_result_255)) 478 479 # (FAILURE) Other ssh failure 480 self._host_run_for_update(expected_wait_cmd) 481 self._host_run_for_update(expected_cmd, exception= 482 error.AutoservSshPermissionDeniedError('no permission', 483 cmd_result_255)) 484 # (FAILURE) Other error 485 self._host_run_for_update(expected_wait_cmd) 486 self._host_run_for_update(expected_cmd, exception= 487 error.AutoservRunError("unknown error", cmd_result_1)) 488 489 self.mox.ReplayAll() 490 491 # Expect success 492 updater.trigger_update() 493 updater.trigger_update() 494 updater.trigger_update() 495 updater.trigger_update() 496 497 # Expect errors as listed above 498 self.assertRaises(autoupdater.RootFSUpdateError, updater.trigger_update) 499 self.assertRaises(autoupdater.RootFSUpdateError, updater.trigger_update) 500 self.assertRaises(autoupdater.RootFSUpdateError, updater.trigger_update) 501 self.assertRaises(autoupdater.RootFSUpdateError, updater.trigger_update) 502 self.assertRaises(autoupdater.RootFSUpdateError, updater.trigger_update) 503 504 self.mox.VerifyAll() 505 506 507 def testUpdateStateful(self): 508 """Tests that we call the stateful update script with the correct args. 509 """ 510 self.mox.StubOutWithMock(autoupdater.ChromiumOSUpdater, '_run') 511 self.mox.StubOutWithMock(autoupdater.ChromiumOSUpdater, 512 'get_stateful_update_script') 513 update_url = ('http://172.22.50.205:8082/update/lumpy-chrome-perf/' 514 'R28-4444.0.0-b2996') 515 static_update_url = ('http://172.22.50.205:8082/static/' 516 'lumpy-chrome-perf/R28-4444.0.0-b2996') 517 518 # Test with clobber=False. 519 autoupdater.ChromiumOSUpdater.get_stateful_update_script().AndReturn( 520 autoupdater.ChromiumOSUpdater.REMOTE_STATEUL_UPDATE_PATH) 521 autoupdater.ChromiumOSUpdater._run( 522 mox.And( 523 mox.StrContains( 524 autoupdater.ChromiumOSUpdater. 525 REMOTE_STATEUL_UPDATE_PATH), 526 mox.StrContains(static_update_url), 527 mox.Not(mox.StrContains('--stateful_change=clean'))), 528 timeout=mox.IgnoreArg()) 529 530 self.mox.ReplayAll() 531 updater = autoupdater.ChromiumOSUpdater(update_url) 532 updater.update_stateful(clobber=False) 533 self.mox.VerifyAll() 534 535 # Test with clobber=True. 536 self.mox.ResetAll() 537 autoupdater.ChromiumOSUpdater.get_stateful_update_script().AndReturn( 538 autoupdater.ChromiumOSUpdater.REMOTE_STATEUL_UPDATE_PATH) 539 autoupdater.ChromiumOSUpdater._run( 540 mox.And( 541 mox.StrContains( 542 autoupdater.ChromiumOSUpdater. 543 REMOTE_STATEUL_UPDATE_PATH), 544 mox.StrContains(static_update_url), 545 mox.StrContains('--stateful_change=clean')), 546 timeout=mox.IgnoreArg()) 547 self.mox.ReplayAll() 548 updater = autoupdater.ChromiumOSUpdater(update_url) 549 updater.update_stateful(clobber=True) 550 self.mox.VerifyAll() 551 552 553 def testRollbackRootfs(self): 554 """Tests that we correctly rollback the rootfs when requested.""" 555 self.mox.StubOutWithMock(autoupdater.ChromiumOSUpdater, '_run') 556 self.mox.StubOutWithMock(autoupdater.ChromiumOSUpdater, 557 '_verify_update_completed') 558 host = self.mox.CreateMockAnything() 559 update_url = 'http://server/test/url' 560 host.hostname = 'test_host' 561 562 can_rollback_cmd = ('/usr/bin/update_engine_client --can_rollback') 563 rollback_cmd = ('/usr/bin/update_engine_client --rollback ' 564 '--follow') 565 566 updater = autoupdater.ChromiumOSUpdater(update_url, host=host) 567 568 # Return an old build which shouldn't call can_rollback. 569 updater.host.get_release_version().AndReturn('1234.0.0') 570 autoupdater.ChromiumOSUpdater._run(rollback_cmd) 571 autoupdater.ChromiumOSUpdater._verify_update_completed() 572 573 self.mox.ReplayAll() 574 updater.rollback_rootfs(powerwash=True) 575 self.mox.VerifyAll() 576 577 self.mox.ResetAll() 578 cmd_result_1 = self.mox.CreateMockAnything() 579 cmd_result_1.exit_status = 1 580 581 # Rollback but can_rollback says we can't -- return an error. 582 updater.host.get_release_version().AndReturn('5775.0.0') 583 autoupdater.ChromiumOSUpdater._run(can_rollback_cmd).AndRaise( 584 error.AutoservRunError('can_rollback failed', cmd_result_1)) 585 self.mox.ReplayAll() 586 self.assertRaises(autoupdater.RootFSUpdateError, 587 updater.rollback_rootfs, True) 588 self.mox.VerifyAll() 589 590 self.mox.ResetAll() 591 # Rollback >= version blacklisted. 592 updater.host.get_release_version().AndReturn('5775.0.0') 593 autoupdater.ChromiumOSUpdater._run(can_rollback_cmd) 594 autoupdater.ChromiumOSUpdater._run(rollback_cmd) 595 autoupdater.ChromiumOSUpdater._verify_update_completed() 596 self.mox.ReplayAll() 597 updater.rollback_rootfs(powerwash=True) 598 self.mox.VerifyAll() 599 600 601 if __name__ == '__main__': 602 unittest.main() 603