1 # Copyright 2016 The Chromium OS Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 # repohooks/pre-upload.py currently does not run pylint. But for developers who 6 # want to check their code manually we disable several harmless pylint warnings 7 # which just distract from more serious remaining issues. 8 # 9 # The instance variable _android_cts is not defined in __init__(). 10 # pylint: disable=attribute-defined-outside-init 11 # 12 # Many short variable names don't follow the naming convention. 13 # pylint: disable=invalid-name 14 15 import logging 16 import os 17 18 from autotest_lib.server import utils 19 from autotest_lib.server.cros import tradefed_test 20 21 # Maximum default time allowed for each individual CTS module. 22 _CTS_TIMEOUT_SECONDS = 3600 23 24 # Public download locations for android cts bundles. 25 _DL_CTS = 'https://dl.google.com/dl/android/cts/' 26 _CTS_URI = { 27 'arm': _DL_CTS + 'android-cts-7.1_r14-linux_x86-arm.zip', 28 'x86': _DL_CTS + 'android-cts-7.1_r14-linux_x86-x86.zip', 29 'media': _DL_CTS + 'android-cts-media-1.4.zip', 30 } 31 32 33 class cheets_CTS_N(tradefed_test.TradefedTest): 34 """Sets up tradefed to run CTS tests.""" 35 version = 1 36 37 # TODO(bmgordon): Remove kahlee once the bulk of failing tests are fixed. 38 _BOARD_RETRY = {'betty': 0, 'kahlee': 0} 39 _CHANNEL_RETRY = {'dev': 5, 'beta': 5, 'stable': 5} 40 41 def _get_default_bundle_url(self, bundle): 42 return _CTS_URI[bundle] 43 44 def _get_tradefed_base_dir(self): 45 return 'android-cts' 46 47 def _tradefed_run_command(self, 48 module=None, 49 plan=None, 50 session_id=None, 51 test_class=None, 52 test_method=None): 53 """Builds the CTS tradefed 'run' command line. 54 55 There are five different usages: 56 57 1. Test a module: assign the module name via |module|. 58 2. Test a plan: assign the plan name via |plan|. 59 3. Continue a session: assign the session ID via |session_id|. 60 4. Run all test cases of a class: assign the class name via 61 |test_class|. 62 5. Run a specific test case: assign the class name and method name in 63 |test_class| and |test_method|. 64 65 @param module: the name of test module to be run. 66 @param plan: name of the plan to be run. 67 @param session_id: tradefed session id to continue. 68 @param test_class: the name of the class of which all test cases will 69 be run. 70 @param test_name: the name of the method of |test_class| to be run. 71 Must be used with |test_class|. 72 @return: list of command tokens for the 'run' command. 73 """ 74 if module is not None: 75 # Run a particular module (used to be called package in M). 76 cmd = ['run', 'commandAndExit', 'cts', '--module', module] 77 if test_class is not None: 78 if test_method is not None: 79 cmd += ['-t', test_class + '#' + test_method] 80 else: 81 cmd += ['-t', test_class] 82 elif plan is not None and session_id is not None: 83 # In 7.1 R2 we can only retry session_id with the original plan. 84 cmd = ['run', 'commandAndExit', 'cts', '--plan', plan, 85 '--retry', '%d' % session_id] 86 elif plan is not None: 87 # Subplan for any customized CTS test plan in form of xml. 88 cmd = ['run', 'commandAndExit', 'cts', '--subplan', plan] 89 else: 90 logging.warning('Running all tests. This can take several days.') 91 cmd = ['run', 'commandAndExit', 'cts'] 92 # We handle media download ourselves in the lab, as lazy as possible. 93 cmd.append('--precondition-arg') 94 cmd.append('skip-media-download') 95 # If we are running outside of the lab we can collect more data. 96 if not utils.is_in_container(): 97 logging.info('Running outside of lab, adding extra debug options.') 98 cmd.append('--log-level-display=DEBUG') 99 cmd.append('--screenshot-on-failure') 100 # TODO(ihf): Add log collection once b/28333587 fixed. 101 #cmd.append('--collect-deqp-logs') 102 # TODO(ihf): Add tradefed_test.adb_keepalive() and remove 103 # --disable-reboot. This might be more efficient. 104 # At early stage, cts-tradefed tries to reboot the device by 105 # "adb reboot" command. In a real Android device case, when the 106 # rebooting is completed, adb connection is re-established 107 # automatically, and cts-tradefed expects that behavior. 108 # However, in ARC, it doesn't work, so the whole test process 109 # is just stuck. Here, disable the feature. 110 cmd.append('--disable-reboot') 111 # Create a logcat file for each individual failure. 112 cmd.append('--logcat-on-failure') 113 return cmd 114 115 def _get_timeout_factor(self): 116 """Returns the factor to be multiplied to the timeout parameter. 117 The factor is determined by the number of ABIs to run.""" 118 if self._timeoutfactor is None: 119 abilist = self._run('adb', args=('shell', 'getprop', 120 'ro.product.cpu.abilist')).stdout.split(',') 121 prefix = {'x86': 'x86', 'arm': 'armeabi-'}.get(self._abi) 122 self._timeoutfactor = (1 if prefix is None else 123 sum(1 for abi in abilist if abi.startswith(prefix))) 124 return self._timeoutfactor 125 126 def _run_tradefed(self, commands): 127 """Kick off CTS. 128 129 @param commands: the command(s) to pass to CTS. 130 @param datetime_id: For 'continue' datetime of previous run is known. 131 @return: The result object from utils.run. 132 """ 133 cts_tradefed = os.path.join(self._repository, 'tools', 'cts-tradefed') 134 with tradefed_test.adb_keepalive(self._get_adb_target(), 135 self._install_paths): 136 for command in commands: 137 logging.info('RUN: ./cts-tradefed %s', ' '.join(command)) 138 output = self._run( 139 cts_tradefed, 140 args=tuple(command), 141 timeout=self._timeout * self._get_timeout_factor(), 142 verbose=True, 143 ignore_status=False, 144 # Make sure to tee tradefed stdout/stderr to autotest logs 145 # continuously during the test run. 146 stdout_tee=utils.TEE_TO_LOGS, 147 stderr_tee=utils.TEE_TO_LOGS) 148 logging.info('END: ./cts-tradefed %s\n', ' '.join(command)) 149 return output 150 151 def _should_skip_test(self): 152 """Some tests are expected to fail and are skipped.""" 153 # newbie and novato are x86 VMs without binary translation. Skip the ARM 154 # tests. 155 no_ARM_ABI_test_boards = ('newbie', 'novato', 'novato-arc64') 156 if self._get_board_name(self._host) in no_ARM_ABI_test_boards: 157 if self._abi == 'arm': 158 return True 159 return False 160 161 def generate_test_command(self, target_module, target_plan, target_class, 162 target_method, tradefed_args, session_id=0): 163 """Generates the CTS command and name to use based on test arguments. 164 165 @param target_module: the name of test module to run. 166 @param target_plan: the name of the test plan to run. 167 @param target_class: the name of the class to be tested. 168 @param target_method: the name of the method to be tested. 169 @param tradefed_args: a list of args to pass to tradefed. 170 @param session_id: tradefed session_id. 171 """ 172 if target_module is not None: 173 if target_class is not None: 174 test_name = 'testcase.%s' % target_class 175 if target_method is not None: 176 test_name += '.' + target_method 177 test_command = self._tradefed_run_command( 178 module=target_module, 179 test_class=target_class, 180 test_method=target_method, 181 session_id=session_id) 182 else: 183 test_name = 'module.%s' % target_module 184 test_command = self._tradefed_run_command( 185 module=target_module, session_id=session_id) 186 elif target_plan is not None: 187 test_name = 'plan.%s' % target_plan 188 test_command = self._tradefed_run_command(plan=target_plan) 189 elif tradefed_args is not None: 190 test_name = 'run tradefed %s' % ' '.join(tradefed_args) 191 test_command = tradefed_args 192 else: 193 test_command = self._tradefed_run_command() 194 test_name = 'all_CTS' 195 196 logging.info('CTS command: %s', test_command) 197 return test_command, test_name 198 199 def run_once(self, 200 target_module=None, 201 target_plan=None, 202 target_class=None, 203 target_method=None, 204 needs_push_media=False, 205 tradefed_args=None, 206 precondition_commands=[], 207 login_precondition_commands=[], 208 timeout=_CTS_TIMEOUT_SECONDS): 209 """Runs the specified CTS once, but with several retries. 210 211 There are four usages: 212 1. Test the whole module named |target_module|. 213 2. Test with a plan named |target_plan|. 214 3. Run all the test cases of class named |target_class|. 215 4. Run a specific test method named |target_method| of class 216 |target_class|. 217 5. Run an arbitrary tradefed command. 218 219 @param target_module: the name of test module to run. 220 @param target_plan: the name of the test plan to run. 221 @param target_class: the name of the class to be tested. 222 @param target_method: the name of the method to be tested. 223 @param needs_push_media: need to push test media streams. 224 @param timeout: time after which tradefed can be interrupted. 225 @param precondition_commands: a list of scripts to be run on the 226 dut before the test is run, the scripts must already be installed. 227 @param login_precondition_commands: a list of scripts to be run on the 228 dut before the log-in for the test is performed. 229 @param tradefed_args: a list of args to pass to tradefed. 230 """ 231 232 # On dev and beta channels timeouts are sharp, lenient on stable. 233 self._timeout = timeout 234 if self._get_release_channel(self._host) == 'stable': 235 self._timeout += 3600 236 # Retries depend on channel. 237 self._timeoutfactor = None 238 239 test_command, test_name = self.generate_test_command(target_module, 240 target_plan, 241 target_class, 242 target_method, 243 tradefed_args) 244 245 self._run_tradefed_with_retries(target_module, test_command, test_name, 246 target_plan, needs_push_media, _CTS_URI, 247 login_precondition_commands, 248 precondition_commands) 249
