1 # Copyright 2017 The Chromium OS Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 import random 6 import logging 7 import time 8 from autotest_lib.client.common_lib import error 9 from autotest_lib.server.cros.cfm import cfm_base_test 10 from autotest_lib.client.common_lib.cros.manual import audio_helper 11 from autotest_lib.client.common_lib.cros.manual import cfm_helper 12 from autotest_lib.client.common_lib.cros.manual import meet_helper 13 from autotest_lib.client.common_lib.cros.manual import video_helper 14 from autotest_lib.client.common_lib.cros.manual import get_usb_devices 15 16 ATRUS = "Hangouts Meet speakermic" 17 CORE_DIR_LINES = 5 18 FAILURE_REASON = { 19 'telemetry': 20 ['No hangouts or meet telemetry API available', 21 'telemetry api', 22 'RPC: cfm_main_screen.', 23 'Failed RPC', 24 'cfm_main_screen', 25 'Webview with screen param', 26 'autotest_lib.client.common_lib.error.TestFail'], 27 'chrome':[], 28 'kernel':[], 29 'atrus':[] 30 } 31 NUM_AUDIO_STREAM_IN_MEETING = 3 32 LOG_CHECK_LIST = ['check_kernel_errorlog', 33 'check_video_errorlog', 34 'check_audio_errorlog', 35 'check_chrome_errorlog', 36 'check_atrus_errorlog', 37 'check_usb_stability'] 38 TEST_DELAY = 0 39 40 class enterprise_CFM_Test(cfm_base_test.CfmBaseTest): 41 """Executes multiple tests on CFM device based on control file, 42 after each test, perform mulitple verifications based on control 43 file. Test flow can be controlled by control file, such as abort 44 on failure, or continue on failure. 45 """ 46 version = 1 47 48 def gpio_test(self): 49 """ 50 Powercycle USB port on Guado. 51 """ 52 if self.run_meeting_test: 53 if self.random: 54 min_meetings = random.randrange(-1, self.gpio_min_meets) 55 else: 56 min_meetings = self.gpio_min_meets 57 if self.meets_last_gpio <= min_meetings: 58 logging.debug('Skip gpio test.') 59 return True, None 60 if cfm_helper.check_is_platform(self.client, 'guado'): 61 status, errmsg = cfm_helper.gpio_usb_test(self.client, 62 self.gpio_list, 63 self.puts, self.gpio_pause, 64 'guado') 65 self.gpio_no += 1 66 self.meets_last_gpio = 0 67 else: 68 logging.info('Skip gpio_test for non-guado CfM.') 69 return True, None 70 71 ## workaround for bug b/69261543 72 if self.is_in_meet: 73 self.is_camera_muted = self.cfm_facade.is_camera_muted() 74 return status, errmsg 75 76 77 def meeting_test(self): 78 """ 79 Join/leave meeting. 80 """ 81 if self.is_in_meet: 82 status, errmsg = meet_helper.leave_meeting(self.cfm_facade, 83 self.is_meeting) 84 if status: 85 self.is_camera_muted = True 86 self.is_in_meet = False 87 return True, None 88 else: 89 if self.recover_on_failure or self.meets_last_reboot == 0: 90 self.is_in_meet = False 91 else: 92 self.is_in_meet = True 93 return False, errmsg 94 else: 95 status, errmsg = meet_helper.join_meeting(self.cfm_facade, 96 self.is_meeting, self.meeting_code) 97 if status: 98 self.is_camera_muted = False 99 self.is_in_meet = True 100 self.meet_no += 1 101 self.meets_last_reboot += 1 102 self.meets_last_gpio += 1 103 return True, None 104 else: 105 if self.recover_on_failure or self.meets_last_reboot == 0: 106 self.is_in_meet = True 107 else: 108 self.is_in_meet = False 109 return False, errmsg 110 111 def reboot_test(self): 112 """ 113 Reboot CfM. 114 """ 115 if self.run_meeting_test: 116 if self.random: 117 min_meetings = random.randrange(-1, self.reboot_min_meets) 118 else: 119 min_meetings = self.reboot_min_meets 120 if self.meets_last_reboot < min_meetings: 121 logging.info('Skip reboot CfM test') 122 return True, None 123 try: 124 self.client.reboot() 125 time.sleep(self.reboot_timeout) 126 except Exception as e: 127 errmsg = 'Reboot test fails for %s' % self.ip_addr 128 logging.exception(errmsg) 129 self.reboot_log = cfm_helper.check_last_reboot(self.client) 130 self.chrome_file = cfm_helper.check_chrome_logfile(self.client) 131 return False, errmsg 132 self.reboot_no += 1 133 self.meets_last_reboot = 0 134 if 'meeting_test' in self.action_config: 135 self.reboot_log = cfm_helper.check_last_reboot(self.client) 136 if list(set(self.verification_config) & set(LOG_CHECK_LIST)): 137 self.log_checking_point = cfm_helper.find_last_log(self.client, 138 self.speaker) 139 return self.restart_chrome_and_meeting(True) 140 self.reboot_log = cfm_helper.check_last_reboot(self.client) 141 self.chrome_file = cfm_helper.check_chrome_logfile(self.client) 142 if list(set(self.verification_config) & set(LOG_CHECK_LIST)): 143 self.log_checking_point = cfm_helper.find_last_log(self.client, 144 self.speaker) 145 return True, None 146 147 def restart_chrome_and_meeting(self, recovery): 148 """ 149 Restart Chrome and Join/Start meeting if previous state is in meeting. 150 """ 151 result, errmsg = meet_helper.restart_chrome(self.cfm_facade, 152 self.is_meeting, 153 recovery) 154 logging.info('restart chrome result:%s, msg = %s', result,errmsg) 155 if not result: 156 logging.info('restart chrome result: False, msg = %s', errmsg) 157 self.chrome_file = cfm_helper.check_chrome_logfile(self.client) 158 return False, errmsg 159 if self.is_in_meet: 160 logging.info('start meeting if needed') 161 self.is_in_meet = False 162 self.chrome_file = cfm_helper.check_chrome_logfile(self.client) 163 test_result, ret_msg = self.meeting_test() 164 if test_result: 165 try: 166 self.is_camera_muted = self.cfm_facade.is_camera_muted() 167 self.is_mic_muted = self.cfm_facade.is_mic_muted() 168 except Exception as e: 169 errmsg = 'Fail to run telemetry api to check camera..' 170 logging.exception(errmsg) 171 return False, errmsg 172 self.chrome_file = cfm_helper.check_chrome_logfile(self.client) 173 return True, None 174 175 # TODO(mzhuo): Adding resetusb test. 176 def reset_usb_test(self): 177 """ 178 Reset USB port 179 """ 180 return True, None 181 182 def mute_unmute_camera_test(self): 183 """ 184 Mute or unmute camera. 185 """ 186 if not self.camera: 187 logging.info('Skip mute/unmute camera testing.') 188 return True, None 189 if self.is_in_meet: 190 if self.is_camera_muted: 191 status, errmsg = meet_helper.mute_unmute_camera( 192 self.cfm_facade, True) 193 if status: 194 self.is_camera_muted = False 195 else: 196 return False, errmsg 197 else: 198 status, errmsg = meet_helper.mute_unmute_camera( 199 self.cfm_facade, False) 200 if status: 201 self.is_camera_muted = True 202 else: 203 return False, errmsg 204 return True, None 205 206 def mute_unmute_mic_test(self): 207 """ 208 Mute or unmute microphone. 209 """ 210 if not self.speaker and not cfm_helper.check_is_platform(self.client, 211 'buddy'): 212 logging.info('Skip mute/unmute microphone testing.') 213 return True, None 214 if self.is_in_meet: 215 if self.is_mic_muted: 216 status, errmsg = meet_helper.mute_unmute_mic(self.cfm_facade, 217 True) 218 if status: 219 self.is_mic_muted = False 220 else: 221 return False, errmsg 222 else: 223 status, errmsg = meet_helper.mute_unmute_mic(self.cfm_facade, 224 False) 225 if status: 226 self.is_mic_muted = True 227 else: 228 return False, errmsg 229 return True, None 230 231 232 def speaker_volume_test(self): 233 """ 234 Update speaker volume. 235 """ 236 if not self.speaker and not cfm_helper.check_is_platform(self.client, 237 'buddy'): 238 logging.info('Skip update volume of speaker testing.') 239 return True, None 240 if self.is_in_meet: 241 test_result, ret_msg = meet_helper.speaker_volume_test( 242 self.cfm_facade, 243 self.vol_change_step, 244 self.vol_change_mode, self.random) 245 if test_result: 246 self.speaker_volume = int(ret_msg) 247 return True, None 248 else: 249 return False, ret_msg 250 else: 251 return True, None 252 253 # TODO(mzhuo): Adding test to turn on/off monite. 254 def flap_monitor_test(self): 255 """ 256 Connect or disconnect monitor. 257 """ 258 return True, None 259 260 def check_usb_enumeration(self): 261 """ 262 Verify all usb devices which were enumerated originally are enumerated. 263 """ 264 return cfm_helper.check_usb_enumeration(self.client, 265 self.puts) 266 267 def check_usb_inf_init(self): 268 """ 269 Verify all usb devices which were enumerated originally have 270 valid interfaces: video interface, audio interface or touch 271 interface. 272 """ 273 return cfm_helper.check_usb_interface_initializion(self.client, 274 self.puts) 275 276 def check_v4l2_interface(self): 277 """ 278 Verify camera has v4l2 file handler created. 279 """ 280 if not self.camera: 281 return True, None 282 return video_helper.check_v4l2_interface(self.client, 283 self.camera, self.name_camera) 284 285 def check_audio_card(self): 286 """ 287 Verify speaker/microphone has audip file handler created. 288 """ 289 if not self.speaker: 290 return True, None 291 return audio_helper.check_soundcard_by_name(self.client, 292 self.name_speaker) 293 294 def check_cras_speaker(self): 295 """ 296 Verify cras server detects speaker. 297 """ 298 if not self.speaker: 299 return True, None 300 return audio_helper.check_speaker_exist_cras(self.client, 301 self.name_speaker) 302 303 def check_cras_mic(self): 304 """ 305 Verify cras server detects microphone. 306 """ 307 if not self.speaker: 308 return True, None 309 return audio_helper.check_microphone_exist_cras(self.client, 310 self.name_speaker) 311 312 def check_cras_mic_mute(self): 313 """ 314 Verify cras shows mic muted or unmuted as expected. 315 """ 316 if not self.speaker or not self.is_in_meet: 317 return True, None 318 return audio_helper.check_cras_mic_mute(self.client, self.cfm_facade) 319 320 def check_cras_pspeaker(self): 321 """ 322 Verify cras shows correct preferred speaker. 323 """ 324 if not self.speaker: 325 return True, None 326 return audio_helper.check_is_preferred_speaker(self.client, 327 self.name_speaker) 328 329 def check_cras_speaker_vol(self): 330 """ 331 Verify cras shows correct volume for speaker. 332 """ 333 if not self.speaker or not self.is_in_meet: 334 return True, None 335 return audio_helper.check_default_speaker_volume(self.client, 336 self.cfm_facade) 337 338 def check_cras_pmic(self): 339 """ 340 Verify cras shows correct preferred microphone. 341 """ 342 if not self.speaker: 343 return True, None 344 return audio_helper.check_is_preferred_mic(self.client, 345 self.name_speaker) 346 347 # TODO(mzhuo): add verification for preferred camera 348 def check_prefer_camera(self): 349 """ 350 Verify preferred camera is correct. 351 """ 352 return True, None 353 354 #TODO(mzhuo): add verification to verify camera is muted or unmuted 355 #in video stack in kernel space. 356 def check_camera_mute(self): 357 """ 358 Verify camera is muted or unmuted as expected. 359 """ 360 return True, None 361 362 def check_video_stream(self): 363 """ 364 Verify camera is streaming or not streaming as expected. 365 """ 366 if not self.camera: 367 return True, None 368 return video_helper.check_video_stream(self.client, 369 self.is_camera_muted, self.camera, self.name_camera) 370 371 def check_audio_stream(self): 372 """ 373 Verify speaker is streaming or not streaming as expected. 374 """ 375 if not self.speaker: 376 return True, None 377 return audio_helper.check_audio_stream(self.client, 378 self.is_in_meet) 379 380 # TODO(mzhuo): Adding verification for speaker in Hotrod App 381 def check_hotrod_speaker(self): 382 """ 383 Verify hotrod shows all speakers. 384 """ 385 return True, None 386 387 # TODO(mzhuo): Adding verification for mic in Hotrod App 388 def check_hotrod_mic(self): 389 """ 390 Verify hotrod shows all microphone. 391 """ 392 return True, None 393 394 # TODO(mzhuo): Adding verification for camera in Hotrod App 395 def check_hotrod_camera(self): 396 """ 397 Verify hotrod shows all cameras. 398 """ 399 return True, None 400 401 # TODO(mzhuo): Adding verification for speaker in Hotrod App 402 def check_hotrod_pspeaker(self): 403 """ 404 Verify hotrod selects correct preferred speaker. 405 """ 406 return True, None 407 408 # TODO(mzhuo): Adding verification for mic in Hotrod App 409 def check_hotrod_pmic(self): 410 """ 411 Verify hotrod selects correct preferred microphone. 412 """ 413 return True, None 414 415 416 # TODO(mzhuo): Adding verification for camera in Hotrod App 417 def check_hotrod_pcamera(self): 418 """ 419 Verify hotrod selects correct preferred camera. 420 """ 421 return True, None 422 423 #TODO(mzhuo): Adding verififaction in hotrod layer for speaker volume 424 def check_hotrod_speaker_vol(self): 425 """ 426 Verify hotrod can set volume for speaker. 427 """ 428 return True, None 429 430 #TODO(mzhuo): Adding verififaction in hotrod layer for mic mute status 431 def check_hotrod_mic_state(self): 432 """ 433 Verify hotrod can mute or unmute microphone. 434 """ 435 return True, None 436 437 #TODO(mzhuo): Adding verififaction in hotrod layer for camera status 438 def check_hotrod_camera_state(self): 439 """ 440 Verify hotrod can mute or unmute camera. 441 """ 442 return True, None 443 444 def check_kernel_errorlog(self): 445 """ 446 Check /var/log/message does not contain any element in 447 error_key_words['kernel']. 448 """ 449 return cfm_helper.check_log(self.client, self.log_checking_point, 450 self.error_key_words, 'kernel', 451 'messages') 452 453 def check_chrome_errorlog(self): 454 """ 455 Check /var/log/chrome/chrome does not contain any element in 456 error_key_words['chrome']. 457 """ 458 return cfm_helper.check_log(self.client, self.log_checking_point, 459 self.error_key_words, 'chrome', 460 'chrome') 461 462 def check_atrus_errorlog(self): 463 """ 464 Check /var/log/atrus.log does not contain any element in 465 error_key_words['atrus']. 466 """ 467 if self.current_test in ['gpio_test', 'resetusb_test']: 468 return True, None 469 if not self.name_speaker: 470 return True, None 471 if self.name_speaker not in ATRUS: 472 logging.info('Speaker %s', self.name_speaker) 473 return True, None 474 if cfm_helper.check_is_platform(self.client, 'guado'): 475 return cfm_helper.check_log(self.client, self.log_checking_point, 476 self.error_key_words, 'atrus', 'atrus') 477 else: 478 return True, None 479 480 def check_video_errorlog(self): 481 """ 482 Check /var/log/message does not contain any element in 483 error_key_words['video']. 484 """ 485 return cfm_helper.check_log(self.client, self.log_checking_point, 486 self.error_key_words, 'video', 487 'messages') 488 489 def check_audio_errorlog(self): 490 """ 491 Check /var/log/message does not contain any element in 492 error_key_words['audio']. 493 """ 494 return cfm_helper.check_log(self.client, self.log_checking_point, 495 self.error_key_words, 'audio', 496 'messages') 497 498 def check_usb_errorlog(self): 499 """ 500 Check /var/log/message does not contain any element in 501 error_key_words['usb']. 502 """ 503 return cfm_helper.check_log(self.client, self.log_checking_point, 504 self.error_key_words, 'usb', 'messages') 505 506 def check_usb_stability(self): 507 """ 508 Check if no disruptive test done, USB device should not go offline. 509 """ 510 if self.current_test in ['gpio_test', 'reboot_test', 'resetusb_test']: 511 return True, None 512 return cfm_helper.check_log(self.client, self.log_checking_point, 513 self.error_key_words, 514 'usb_stability', 'messages') 515 516 def check_process_crash(self): 517 """ 518 check no process crashing. 519 """ 520 test_result, self.cdlines = cfm_helper.check_process_crash(self.client, 521 self.cdlines) 522 return test_result, str(self.cdlines) 523 524 #TODO(mzhuo): Adding verififaction to check whether there is kernel panic 525 def check_kernel_panic(self): 526 """ 527 Check no kernel panic reported. 528 """ 529 return True, None 530 531 532 def check_chrome_restarted(self): 533 """ 534 Check whether chrome is killed and restarted. 535 """ 536 if self.chrome_file == cfm_helper.check_chrome_logfile(self.client): 537 return True, None 538 else: 539 self.chrome_file = cfm_helper.check_chrome_logfile(self.client) 540 return False, 'Chrome is restarted unexpectly.' 541 542 def check_cfm_rebooted(self): 543 """ 544 Check whether CfM is rebooted. 545 """ 546 logging.info('Last reboot: %s', self.reboot_log) 547 if self.reboot_log == cfm_helper.check_last_reboot(self.client): 548 return True, None 549 else: 550 self.reboot_log = cfm_helper.check_last_reboot(self.client) 551 return False, 'CfM is rebooted unexpectly.' 552 553 def initialize_action_check_config(self, action_config, verification_config, 554 fixedmode): 555 """ 556 Initialize action list based on control file. 557 @param action_config: dict defines the number of test should be done 558 for each test 559 @param fixedmode: if True all tests are executed in fixed order; 560 if False all tests are executed in random order. 561 """ 562 self.action_config = [] 563 if action_config['meeting_test'] == 1: 564 self.action_config = ['meeting_test'] 565 if not self.camera: 566 action_config['mute_unmute_camera_test'] = 0 567 verification_config['check_v4l2_interface'] = False 568 verification_config['check_video_stream'] = False 569 verification_config['check_video_errorlog'] = False 570 if not self.speaker: 571 if not cfm_helper.check_is_platform(self.client, 'buddy'): 572 action_config['mute_unmute_mic_test'] = 0 573 action_config['speaker_volume_test'] = 0 574 verification_config['check_audio_card'] = False 575 verification_config['check_cras_speaker'] = False 576 verification_config['check_cras_mic'] = False 577 verification_config['check_cras_pspeaker'] = False 578 verification_config['check_cras_pmic'] = False 579 verification_config['check_audio_stream'] = False 580 verification_config['check_audio_errorlog'] = False 581 verification_config['check_cras_speaker_vol'] = False 582 verification_config['check_cras_mic_mute'] = False 583 not_in_meeting_action = ['meeting_test', 'gpio_test', 'reboot_test'] 584 585 if fixedmode: 586 for action, nof_times in action_config.iteritems(): 587 if not action in not_in_meeting_action: 588 self.action_config.extend(nof_times * [action]) 589 else: 590 for action, nof_times in action_config.iteritems(): 591 if not action in not_in_meeting_action and nof_times > 0: 592 dup_test = max(1, random.randrange(0, nof_times)) 593 for _ in range(dup_test): 594 self.action_config.insert(max(1, random.randrange(-1, 595 len(self.action_config))), action) 596 597 if action_config['meeting_test'] == 1: 598 self.action_config.append('meeting_test') 599 600 for action, nof_times in action_config.iteritems(): 601 if action == 'meeting_test': 602 continue 603 if action in not_in_meeting_action and nof_times > 0: 604 dup_test = max(1, random.randrange(0, nof_times)) 605 for _ in range(dup_test): 606 self.action_config.insert(max(0, random.randrange(-1, 607 len(self.action_config))), action) 608 609 logging.info('Test list = %s', self.action_config) 610 self.verification_config = [v for v in verification_config.keys() 611 if verification_config[v]] 612 logging.info('Verification list = %s', self.verification_config) 613 614 615 def initialize_test(self, test_config, action_config, verification_config, 616 error_key_words, test_flow_control): 617 """ 618 Initialize the list of tests and verifications; 619 and polulate data needed for test: 620 USB devices: which can be retrieved from control file or 621 automatically detected by script; 622 Test loop, meeting mode, meeting code, test flow contro 623 variables. 624 """ 625 self.gpio_pause = test_config['gpiopause'] 626 self.reboot_timeout = test_config['reboot_timeout'] 627 self.vol_change_step = test_config['vol_change_step'] 628 self.vol_change_mode = test_config['vol_change_mode'] 629 self.gpio_list = test_config['gpio_list'] 630 self.is_meeting = test_config['is_meeting'] 631 self.meeting_code = test_config ['meeting_code'] 632 self.reboot_min_meets = test_config['reboot_after_min_meets'] 633 self.gpio_min_meets = test_config['gpio_after_min_meets'] 634 self.run_meeting_test = action_config['meeting_test'] 635 self.random = test_flow_control['random_mode'] 636 self.recover_on_failure = test_flow_control['recovery_on_fatal_failure'] 637 self.error_key_words = error_key_words 638 if test_config['puts']: 639 self.puts = test_config['puts'].split(',') 640 else: 641 self.puts = None 642 643 if verification_config['check_process_crash']: 644 cfm_helper.clear_core_file(self.client) 645 646 self.action_fun = { 647 'gpio_test': self.gpio_test, 648 'meeting_test': self.meeting_test, 649 'reboot_test': self.reboot_test, 650 'reset_usb_test': self.reset_usb_test, 651 'mute_unmute_camera_test': self.mute_unmute_camera_test, 652 'mute_unmute_mic_test': self.mute_unmute_mic_test, 653 'speaker_volume_test': self.speaker_volume_test, 654 'flap_monitor_test': self.flap_monitor_test 655 } 656 self.veri_fun = { 657 'check_usb_enumeration': self.check_usb_enumeration, 658 'check_usb_inf_init': self.check_usb_inf_init, 659 'check_v4l2_interface': self.check_v4l2_interface, 660 'check_audio_card': self.check_audio_card, 661 'check_cras_speaker': self.check_cras_speaker, 662 'check_cras_mic': self.check_cras_mic, 663 'check_cras_pspeaker': self.check_cras_pspeaker, 664 'check_cras_pmic': self.check_cras_pmic, 665 'check_cras_speaker_vol': self.check_cras_speaker_vol, 666 'check_cras_mic_mute': self.check_cras_mic_mute, 667 'check_prefer_camera': self.check_prefer_camera, 668 'check_camera_mute': self.check_camera_mute, 669 'check_audio_stream': self.check_audio_stream, 670 'check_video_stream': self.check_video_stream, 671 'check_hotrod_speaker': self.check_hotrod_speaker, 672 'check_hotrod_mic': self.check_hotrod_mic, 673 'check_hotrod_camera': self.check_hotrod_camera, 674 'check_hotrod_pspeaker': self.check_hotrod_pspeaker, 675 'check_hotrod_pmic': self.check_hotrod_pmic, 676 'check_hotrod_pcamera': self.check_hotrod_pcamera, 677 'check_hotrod_speaker_vol': self.check_hotrod_speaker_vol, 678 'check_hotrod_mic_state': self.check_hotrod_mic_state, 679 'check_hotrod_camera_state': self.check_hotrod_camera_state, 680 'check_usb_errorlog': self.check_usb_errorlog, 681 'check_kernel_errorlog': self.check_kernel_errorlog, 682 'check_video_errorlog': self.check_video_errorlog, 683 'check_audio_errorlog': self.check_audio_errorlog, 684 'check_chrome_errorlog': self.check_chrome_errorlog, 685 'check_atrus_errorlog': self.check_atrus_errorlog, 686 'check_usb_stability': self.check_usb_stability, 687 'check_process_crash': self.check_process_crash, 688 'check_kernel_panic': self.check_kernel_panic, 689 'check_cfm_rebooted':self.check_cfm_rebooted, 690 'check_chrome_restarted':self.check_chrome_restarted 691 } 692 693 self.usb_data = [] 694 self.speaker = None 695 self.speaker_volume = None 696 self.camera = None 697 self.name_speaker = None 698 self.mimo_sis = None 699 self.mimo_display = None 700 self.is_in_meet = False 701 self.is_camera_muted = True 702 self.is_mic_muted = False 703 self.meets_last_reboot = 0 704 self.meets_last_gpio = 0 705 self.meet_no = 0 706 self.reboot_no = 0 707 self.gpio_no = 0 708 self.cdlines = CORE_DIR_LINES 709 710 usb_data = cfm_helper.retrieve_usb_devices(self.client) 711 if not usb_data: 712 logging.info('\n\nEnterprise_CFM_Test_Failed.') 713 raise error.TestFail('Fails to find any usb devices on CfM.') 714 peripherals = cfm_helper.extract_peripherals_for_cfm(usb_data) 715 if not peripherals: 716 logging.info('\n\nEnterprise_CFM_Test_Failed.') 717 raise error.TestFail('Fails to find any periphereal on CfM.') 718 if not self.puts: 719 self.puts = peripherals.keys() 720 else: 721 if [put for put in self.puts if not put in peripherals.keys()]: 722 logging.info('Fails to find target device %s', put) 723 logging.info('\nEnterprise_CFM_Test_Failed.') 724 raise error.TestFail('Fails to find device') 725 for _put in self.puts: 726 if _put in get_usb_devices.CAMERA_MAP.keys(): 727 self.camera = _put 728 if _put in get_usb_devices.SPEAKER_MAP.keys(): 729 self.speaker = _put 730 if self.camera: 731 self.name_camera = get_usb_devices.get_device_prod(self.camera) 732 logging.info('Camera under test: %s %s', 733 self.camera, self.name_camera) 734 if self.speaker: 735 self.name_speaker = get_usb_devices.get_device_prod(self.speaker) 736 logging.info('Speaker under test: %s %s', 737 self.speaker, self.name_speaker) 738 if not test_flow_control['skip_cfm_check']: 739 if cfm_helper.check_is_platform(self.client, 'guado'): 740 if not cfm_helper.check_peripherals_for_cfm(peripherals): 741 logging.info('Sanity Check on CfM fails.') 742 logging.info('\n\nEnterprise_CFM_Test_Failed.') 743 raise error.TestFail('Sanity Check on CfM fails.') 744 self.ip_addr = cfm_helper.get_mgmt_ipv4(self.client) 745 logging.info('CfM %s passes sanity check, will start test.', 746 self.ip_addr) 747 748 self.initialize_action_check_config(action_config, 749 verification_config, True) 750 751 if list(set(self.verification_config) & set(LOG_CHECK_LIST)): 752 self.log_checking_point = cfm_helper.find_last_log(self.client, 753 self.speaker) 754 self.chrome_file = cfm_helper.check_chrome_logfile(self.client) 755 self.reboot_log = cfm_helper.check_last_reboot(self.client) 756 757 758 def recovery_routine(self, failure_msg): 759 """ 760 Telemetry api often returns API timeout, or javascript timeout due to 761 various reasons. To workout the problem before the fix is ready script 762 checks if errmsg from test failure or verification failure has keywords 763 defined in FAILURE_REASON['telemetry']. If yes, script checks whether 764 CfM is rebooted or Chrome browser is rebooted unexpectly. If no, script 765 restarts chrome. If yes no chrome restart is done to preserve valid 766 failure state. 767 @param failure_msg: failure message returned from test failure or 768 verification failure 769 @return True if recovery is successfully done, 770 False, if recovery is not done, or fails. 771 """ 772 loop_result = False 773 to_be_recovered = False 774 if not self.check_chrome_restarted(): 775 self.chrome_file = cfm_helper.check_chrome_logfile(self.client) 776 return False 777 if not self.check_cfm_rebooted(): 778 self.reboot_log = cfm_helper.check_last_reboot(self.client) 779 return False 780 for _err in FAILURE_REASON['telemetry']: 781 if _err in failure_msg: 782 to_be_recovered = True 783 break 784 if to_be_recovered: 785 logging.info('Restart Chrome to recover......') 786 result, emsg = self.restart_chrome_and_meeting(True) 787 if result: 788 loop_result = True 789 if list(set(self.verification_config) & set(LOG_CHECK_LIST)): 790 self.log_checking_point = cfm_helper.find_last_log( 791 self.client, 792 self.speaker) 793 return loop_result 794 795 796 def process_test_result(self, loop_result, loop_no, test_no, 797 failed_tests, failed_verifications, 798 failed_tests_loop, 799 failed_verifications_loop, test_flow_control, 800 test_config, finished_tests_verifications, 801 test_done): 802 """ 803 Proceess test result data, and print out test report. 804 @params loop_result: True when all tests and verifications pass, 805 False if any test or verification fails. 806 @param loop_no: sequence number of the loop. 807 @param test_no: sequence number of the test. 808 @param failed_tests: failed tests. 809 @param failed_verifications: failed verifications. 810 @param failed_tests_loop: failed tests in the loop. 811 @param failed_verifications_loop: failed verifications in the loop. 812 @param test_flow_control: variable of flow control defined in 813 control file 814 @param test_config: variable of test config defined in control file 815 @param finished_tests_verifications: data to keep track number of 816 tests and verifications performed. 817 @param test_done: True if all loops are done; False otherwose. 818 """ 819 if 'reboot_test' in finished_tests_verifications.keys(): 820 finished_tests_verifications['reboot_test'] = self.reboot_no 821 if not loop_result and not test_done: 822 logging.info('\n\nVerification_or_Test_Fail on %s for loop NO:' 823 ' %d, Test: %d', self.ip_addr, loop_no, test_no) 824 if failed_tests_loop: 825 logging.info('----- Failed_Tests: %s', failed_tests_loop) 826 if failed_verifications_loop: 827 logging.info('----- Failed_Verifications: %s', 828 failed_verifications_loop) 829 if test_flow_control['report']: 830 logging.info('\n\n\n----------------Summary---------------') 831 logging.info('---Loop %d, Test: %d, Meet: %d, Reboot: %d, Gpio: %s', 832 loop_no, test_no, self.meet_no, self.reboot_no, 833 self.gpio_no) 834 for testname, counter in failed_tests.iteritems(): 835 logging.info('----Test: %s, Failed times: %d, Total Run: %d', 836 testname, counter, 837 finished_tests_verifications[testname]) 838 for veriname, counter in failed_verifications.iteritems(): 839 logging.info('----Verification: %s, Failed times: %d,' 840 'Total Run: %d', 841 veriname, counter, 842 finished_tests_verifications[veriname]) 843 if self.random: 844 time.sleep(random.randrange(0, test_config['loop_timeout'])) 845 else: 846 time.sleep(test_config['loop_timeout']) 847 if not test_done: 848 if list(set(self.verification_config) & set(LOG_CHECK_LIST)): 849 self.log_checking_point = cfm_helper.find_last_log(self.client, 850 self.speaker) 851 852 def run_once(self, host, run_test_only, test_config, action_config, 853 verification_config, 854 error_key_words, test_flow_control): 855 """Runs the test.""" 856 logging.info('Start_Test_Script:Enterprise_CFM_Test') 857 self.client = host 858 if test_flow_control['reboot_before_start']: 859 try: 860 self.client.reboot() 861 except Exception as e: 862 errmsg = ('Reboot test fails for %s.' 863 '% self.ip_addr') 864 logging.exception(errmsg) 865 raise error.TestFail(errmsg) 866 if action_config['meeting_test'] > 0: 867 result, errmsg = meet_helper.restart_chrome(self.cfm_facade, 868 test_config['is_meeting'], True) 869 if not result: 870 logging.info('Restart chrome fails, msg = %s', errmsg) 871 raise error.TestFail(errmsg) 872 873 self.initialize_test(test_config, action_config, verification_config, 874 error_key_words, test_flow_control) 875 test_no = 0 876 failed_tests = {} 877 failed_verifications = {} 878 finished_tests_verifications = {} 879 test_failure_reason = [] 880 verification_failure_reason = [] 881 882 883 for loop_no in xrange(1, test_config['repeat'] + 1): 884 logging.info('=============%s:Test_Loop_No:%d=============', 885 self.ip_addr, loop_no) 886 logging.info('Action list: %s', self.action_config) 887 failed_tests_loop = [] 888 failed_verifications_loop = [] 889 for test in self.action_config: 890 loop_result = True 891 if not test in finished_tests_verifications.keys(): 892 finished_tests_verifications[test] = 1 893 else: 894 finished_tests_verifications[test] += 1 895 self.current_test = test 896 logging.info('\nStart_test:%s', test) 897 test_result, test_msg = self.action_fun[test]() 898 test_no += 1 899 if test_result: 900 logging.info('Test_Result:%s:SUCCESS', test) 901 else: 902 logging.info('Test_Result:%s:FAILURE:%s', test, test_msg) 903 test_failure_reason.append(test_msg) 904 failed_tests_loop.append(test) 905 loop_result = False 906 if not test in failed_tests.keys(): 907 failed_tests[test] = 1 908 else: 909 failed_tests[test] += 1 910 logging.info('\n%s:Test_failure:%s:%s', self.ip_addr, 911 test, test_msg) 912 if self.recover_on_failure or self.meets_last_reboot < 1: 913 loop_result = self.recovery_routine(test_msg) 914 if self.random: 915 time.sleep(random.randrange(test_config['min_timeout'], 916 test_config['action_timeout'])) 917 else: 918 time.sleep(test_config['min_timeout']) 919 920 for verification in self.verification_config: 921 if not verification in finished_tests_verifications.keys(): 922 finished_tests_verifications[verification] = 1 923 else: 924 finished_tests_verifications[verification] += 1 925 926 logging.info('\nStart_verification:%s', verification) 927 veri_result, veri_msg = self.veri_fun[verification]() 928 if veri_result: 929 logging.info('Verification_Result:%s:SUCCESS', 930 verification) 931 else: 932 logging.info('Verification_Result:%s:FAILURE:%s', 933 verification, veri_msg) 934 verification_failure_reason.append(veri_msg) 935 failed_verifications_loop.append(verification) 936 if not verification in failed_verifications.keys(): 937 failed_verifications[verification] = 1 938 else: 939 failed_verifications[verification] += 1 940 logging.info('%s:Verification_fail:%s:%s', 941 self.ip_addr, verification, veri_msg) 942 loop_result = False 943 if self.recover_on_failure: 944 loop_result = self.recovery_routine(veri_msg) 945 946 self.process_test_result(loop_result, loop_no, test_no, 947 failed_tests, 948 failed_verifications, 949 failed_tests_loop, 950 failed_verifications_loop, 951 test_flow_control, 952 test_config, 953 finished_tests_verifications, False) 954 if not loop_result: 955 if test_flow_control['abort_on_failure']: 956 logging.info('Enterprise_CFM_Test_Failed.') 957 time.sleep(test_config['debug_timeout']) 958 logging.info('Enterprise_CFM_Test_Finished.') 959 raise error.TestFail( 960 'Test_or_Verification_fails after {}.'.format(test)) 961 else: 962 logging.info('Enterprise_CFM_Test_Failure_Detected.') 963 964 if self.random: 965 self.initialize_action_check_config(action_config, 966 verification_config, False) 967 968 logging.info('Enterprise_CFM_Test_Finished.') 969 self.process_test_result(loop_result, loop_no, test_no, 970 failed_tests, 971 failed_verifications, 972 failed_tests_loop, 973 failed_verifications_loop, 974 test_flow_control, 975 test_config, 976 finished_tests_verifications, True) 977 if test_failure_reason: 978 logging.debug('Test failure reason %s', test_failure_reason) 979 if verification_failure_reason: 980 logging.debug('Verification failure reason %s', 981 verification_failure_reason) 982
