1 # Copyright (C) 2016 The Android Open Source Project 2 # 3 # Licensed under the Apache License, Version 2.0 (the "License"); 4 # you may not use this file except in compliance with the License. 5 # You may obtain a copy of the License at 6 # 7 # http://www.apache.org/licenses/LICENSE-2.0 8 # 9 # Unless required by applicable law or agreed to in writing, software 10 # distributed under the License is distributed on an "AS IS" BASIS, 11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 # See the License for the specific language governing permissions and 13 # limitations under the License. 14 15 '''Module that contains TestBase, the base class of all tests.''' 16 17 from __future__ import absolute_import 18 19 import logging 20 import os 21 import re 22 import tempfile 23 import inspect 24 import traceback 25 26 from .exception import DisconnectedException, TestSuiteException 27 28 from . import util_log 29 30 31 class TestBase(object): 32 '''Base class for all tests. Provides some common functionality.''' 33 34 bundle_target = {} 35 36 class TestFail(Exception): 37 '''Exception that is thrown when a line in a test fails. 38 39 This exception is thrown if a lldb command does not return the expected 40 string. 41 ''' 42 pass 43 44 def __init__(self, device_port, device, timer, app_type, wimpy=False, **kwargs): 45 # Keep argument names for documentation purposes. This method is 46 # overwritten by test_base_remote. 47 # pylint: disable=unused-argument 48 self._lldb = None # handle to the lldb module 49 self._ci = None # instance of the lldb command interpreter for this test 50 self._timer = timer # timer instance, to check whether the test froze 51 self.app_type = app_type # The type of bundle that is being executed 52 self.wimpy = wimpy 53 54 def setup(self, android): 55 '''Set up environment for the test. 56 57 Override to specify commands to be run before the test APK launch. 58 Useful for setting Android properties or environment variables. See also 59 the teardown method. 60 61 Args: 62 android: Handler to the android device, see the UtilAndroid class. 63 ''' 64 pass 65 66 def teardown(self, android): 67 '''Clean up environment after test. 68 69 Override this procedure to specify commands to be run after the test has 70 finished. This method is run regardless the outcome of the test. 71 72 Args: 73 android: Handler to the android device, see the UtilAndroid class. 74 ''' 75 pass 76 77 def run(self, dbg, remote_pid, lldb): 78 '''Execute the actual test suite. 79 80 Args: 81 dbg: The instance of the SBDebugger that is used to test commands. 82 remote_pid: The integer that is the process id of the binary that 83 the debugger is attached to. 84 lldb: A handle to the lldb module. 85 86 Returns: 87 A list of (test, failure) tuples. 88 ''' 89 log = util_log.get_logger() 90 91 def predicate(obj): 92 '''check whether we're interested in the function''' 93 if not callable(obj): 94 return False 95 if self.wimpy and not getattr(obj, 'wimpy', False): 96 log.debug("skipping non-wimpy test in wimpy mode:%r", obj) 97 return False 98 return True 99 100 test_methods = [ 101 method for name, method in inspect.getmembers(self, predicate) 102 if name.startswith('test_') 103 ] 104 log.debug("Found the following tests %r", test_methods) 105 test_errors = [] 106 107 for test in sorted( 108 test_methods, 109 key=lambda item: getattr(item, 'test_order', float('Inf')) 110 ): 111 try: 112 log.info("running test %r", test.__name__) 113 result = test() 114 except (self.TestFail, TestSuiteException) as e: 115 test_errors.append((method, e)) 116 117 return test_errors 118 119 def post_run(self): 120 '''Clean up after test execution.''' 121 pass 122 123 def assert_true(self, cond): 124 '''Check a given condition and raise TestFail if it is False. 125 126 Args: 127 cond: The boolean condition to check. 128 129 Raises: 130 TestFail: The condition was false. 131 ''' 132 if not cond: 133 raise self.TestFail() 134 135 def assert_lang_renderscript(self): 136 '''Check that LLDB is stopped in a RenderScript frame 137 138 Use the LLDB API to check that the language of the current frame 139 is RenderScript, fail otherwise. 140 141 Raises: 142 TestFail: Detected language not RenderScript. 143 ''' 144 assert self._lldb 145 assert self._ci 146 147 proc = self._ci.GetProcess() 148 frame = proc.GetSelectedThread().GetSelectedFrame() 149 lang = frame.GetCompileUnit().GetLanguage() 150 151 if lang != self._lldb.eLanguageTypeExtRenderScript: 152 raise self.TestFail('Frame language not RenderScript, instead {0}' 153 .format(lang)) 154 155 def do_command(self, cmd): 156 '''Run an lldb command and return the output. 157 158 Args: 159 cmd: The string representing the lldb command to run. 160 161 Raises: 162 TestFail: The lldb command failed. 163 ''' 164 assert self._lldb 165 assert self._ci 166 167 log = util_log.get_logger() 168 res = self._lldb.SBCommandReturnObject() 169 170 log.info('[Command] {0}'.format(cmd)) 171 172 # before issuing the command, restart the current timer to check 173 # whether the command is going to freeze the test 174 if self._timer: 175 self._timer.reset() 176 177 self._ci.HandleCommand(cmd, res) 178 179 if not res.Succeeded(): 180 error = res.GetError() 181 error = error if error else res.GetOutput() 182 raise self.TestFail('The command "{0}" failed with the error: {1}' 183 .format(cmd, error if error else '<N/a>')) 184 185 output = res.GetOutput() or '' 186 log.debug('[Output] {0}'.format(output.rstrip())) 187 188 return output 189 190 def try_command(self, cmd, expected=None, expected_regex=None): 191 '''Run an lldb command and match the expected response. 192 193 Args: 194 cmd: The string representing the lldb command to run. 195 expected: A list of strings that should be present in lldb's 196 output. 197 expected_regex: A list of regular expressions that should 198 match lldb's output. 199 200 Raises: 201 TestFail: One of the expected strings were not found in the lldb 202 output. 203 204 Returns: 205 str: raw lldb command output. 206 ''' 207 assert self._lldb 208 assert self._ci 209 log = util_log.get_logger() 210 output = '' 211 try: 212 output = self.do_command(cmd) 213 214 if 'lost connection' in output: 215 raise DisconnectedException('Lost connection to lldb-server.') 216 217 # check the expected strings 218 if expected: 219 self._match_literals(output, expected) 220 221 # check the regexp patterns 222 if expected_regex: 223 self._match_regexp_patterns(output, expected_regex) 224 225 except self.TestFail as exception: 226 # if the command failed, ensure the output retrieved from the 227 # command is printed even in verbose mode 228 if log.getEffectiveLevel() > logging.DEBUG: 229 log.error('[Output] {0}'.format(output.rstrip() if output 230 else '<empty>')) 231 232 # print the back trace, it should help to identify the error in 233 # the test 234 backtrace = ['[Back trace]'] 235 for (filename, line, function, text) in \ 236 traceback.extract_stack()[:-1]: 237 backtrace.append(' [{0} line: {2} fn: {1}] {3}'.format( 238 filename, function, line, text 239 ) 240 ) 241 log.error('\n'.join(backtrace)) 242 log.error('[TEST ERROR] {0}'.format(exception.message)) 243 raise # pass through 244 245 return output 246 247 def _match_literals(self, text, literals): 248 '''Checks the text against the array of literals. 249 250 Raises a TestFail exception in case one of the literals is not contained 251 in the text. 252 253 Args: 254 text: String, it represents the text to match. 255 literals: an array of string literals to match in the output. 256 257 Throws: self.TestFail: if it cannot match one of the literals in 258 the output. 259 ''' 260 for string in literals: 261 if string not in text: 262 raise self.TestFail('Cannot find "{0}" in the output' 263 .format(string)) 264 265 def _match_regexp_patterns(self, text, patterns): 266 '''Checks the text against the array of regular expression patterns. 267 268 Raises a TestFail exception in case one of the patterns is not matched 269 in the given text. 270 271 Args: 272 text: String, it represents the text to match. 273 patterns: an array of strings, each of them representing a regular 274 expression to match in text. 275 276 Throws: self.TestFail: if it cannot match one of the literals in 277 the output. 278 ''' 279 log = util_log.get_logger() 280 281 for regex in patterns: 282 match = re.search(regex, text) 283 if not match: 284 raise self.TestFail('Cannot match the regexp "{0}" in ' 285 'the output'.format(regex)) 286 else: 287 msg = 'Found match to regex {0}: {1}'.format(regex, 288 match.group()) 289 log.debug(msg) 290 291 @staticmethod 292 def get_tmp_file_path(): 293 '''Get the path of a temporary file that is then deleted. 294 295 Returns: 296 A string that is the path to a temporary file. 297 ''' 298 file_desc, name = tempfile.mkstemp() 299 os.close(file_desc) 300 os.remove(name) 301 return name 302 303 304 class TestBaseNoTargetProcess(TestBase): 305 '''lldb target that doesn't require a binary to be running.''' 306 307 def get_bundle_target(self): 308 '''Get bundle executable to run. 309 310 Returns: None 311 ''' 312 return None 313 314 @property 315 def bundle_target(self): 316 return self.get_bundle_target() 317 318 def run(self, dbg, remote_pid, lldb): 319 '''Execute the test case. 320 321 Args: 322 dbg: The instance of the SBDebugger that is used to test commands. 323 lldb: A handle to the lldb module. 324 325 Returns: 326 True: test passed, False: test failed. 327 ''' 328 self._lldb = lldb 329 self._dbg = dbg 330 self._ci = dbg.GetCommandInterpreter() 331 assert self._ci.IsValid() 332 return super(TestBaseNoTargetProcess, self).run(self, dbg, remote_pid) 333
