1 # Copyright 2016 The Chromium OS Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 import grp 6 import logging 7 import os 8 import pwd 9 import re 10 import shutil 11 import signal 12 import stat 13 import subprocess 14 15 import crash_test 16 from autotest_lib.client.bin import utils 17 from autotest_lib.client.common_lib import error 18 19 20 CRASHER = 'crasher_nobreakpad' 21 22 23 class UserCrashTest(crash_test.CrashTest): 24 """ 25 Base class for tests that verify crash reporting for user processes. Shared 26 functionality includes installing a crasher executable, generating Breakpad 27 symbols, running the crasher process, and verifying collection and sending. 28 """ 29 30 31 # Every crash report needs one of these to be valid. 32 REPORT_REQUIRED_FILETYPES = {'meta'} 33 # Reports might have these and that's OK! 34 REPORT_OPTIONAL_FILETYPES = {'dmp', 'log', 'proclog'} 35 36 37 def setup(self): 38 """Copy the crasher source code under |srcdir| and build it.""" 39 src = os.path.join(os.path.dirname(__file__), 'crasher') 40 dest = os.path.join(self.srcdir, 'crasher') 41 shutil.copytree(src, dest) 42 43 os.chdir(dest) 44 utils.make() 45 46 47 def initialize(self, expected_tag='user', expected_version=None, 48 force_user_crash_dir=False): 49 """Initialize and configure the test. 50 51 @param expected_tag: Expected tag in crash_reporter log message. 52 @param expected_version: Expected version included in the crash report, 53 or None to use the Chrome OS version. 54 @param force_user_crash_dir: Always look for crash reports in the crash 55 directory of the current user session, or 56 the fallback directory if no sessions. 57 """ 58 crash_test.CrashTest.initialize(self) 59 self._expected_tag = expected_tag 60 self._expected_version = expected_version 61 self._force_user_crash_dir = force_user_crash_dir 62 63 64 def _prepare_crasher(self, root_path='/'): 65 """Extract the crasher and set its permissions. 66 67 crasher is only gzipped to subvert Portage stripping. 68 69 @param root_path: Root directory of the chroot environment in which the 70 crasher is installed and run. 71 """ 72 self._root_path = root_path 73 self._crasher_path = os.path.join(self.srcdir, 'crasher', CRASHER) 74 utils.system('cd %s; tar xzf crasher.tgz-unmasked' % 75 os.path.dirname(self._crasher_path)) 76 # Make sure all users (specifically chronos) have access to 77 # this directory and its decendents in order to run crasher 78 # executable as different users. 79 utils.system('chmod -R a+rx ' + self.bindir) 80 81 82 def _populate_symbols(self): 83 """Set up Breakpad's symbol structure. 84 85 Breakpad's minidump processor expects symbols to be in a directory 86 hierarchy: 87 <symbol-root>/<module_name>/<file_id>/<module_name>.sym 88 """ 89 self._symbol_dir = os.path.join(os.path.dirname(self._crasher_path), 90 'symbols') 91 utils.system('rm -rf %s' % self._symbol_dir) 92 os.mkdir(self._symbol_dir) 93 94 basename = os.path.basename(self._crasher_path) 95 utils.system('/usr/bin/dump_syms %s > %s.sym' % 96 (self._crasher_path, 97 basename)) 98 sym_name = '%s.sym' % basename 99 symbols = utils.read_file(sym_name) 100 # First line should be like: 101 # MODULE Linux x86 7BC3323FBDBA2002601FA5BA3186D6540 crasher_XXX 102 # or 103 # MODULE Linux arm C2FE4895B203D87DD4D9227D5209F7890 crasher_XXX 104 first_line = symbols.split('\n')[0] 105 tokens = first_line.split() 106 if tokens[0] != 'MODULE' or tokens[1] != 'Linux': 107 raise error.TestError('Unexpected symbols format: %s', 108 first_line) 109 file_id = tokens[3] 110 target_dir = os.path.join(self._symbol_dir, basename, file_id) 111 os.makedirs(target_dir) 112 os.rename(sym_name, os.path.join(target_dir, sym_name)) 113 114 115 def _is_frame_in_stack(self, frame_index, module_name, 116 function_name, file_name, 117 line_number, stack): 118 """Search for frame entries in the given stack dump text. 119 120 A frame entry looks like (alone on a line): 121 16 crasher_nobreakpad!main [crasher.cc : 21 + 0xb] 122 123 Args: 124 frame_index: number of the stack frame (0 is innermost frame) 125 module_name: name of the module (executable or dso) 126 function_name: name of the function in the stack 127 file_name: name of the file containing the function 128 line_number: line number 129 stack: text string of stack frame entries on separate lines. 130 131 Returns: 132 Boolean indicating if an exact match is present. 133 134 Note: 135 We do not care about the full function signature - ie, is it 136 foo or foo(ClassA *). These are present in function names 137 pulled by dump_syms for Stabs but not for DWARF. 138 """ 139 regexp = (r'\n\s*%d\s+%s!%s.*\[\s*%s\s*:\s*%d\s.*\]' % 140 (frame_index, module_name, 141 function_name, file_name, 142 line_number)) 143 logging.info('Searching for regexp %s', regexp) 144 return re.search(regexp, stack) is not None 145 146 147 def _verify_stack(self, stack, basename, from_crash_reporter): 148 # Should identify cause as SIGSEGV at address 0x16. 149 logging.debug('minidump_stackwalk output:\n%s', stack) 150 151 # Look for a line like: 152 # Crash reason: SIGSEGV 153 # Crash reason: SIGSEGV /0x00000000 154 match = re.search(r'Crash reason:\s+([^\s]*)', stack) 155 expected_address = '0x16' 156 if from_crash_reporter: 157 # We cannot yet determine the crash address when coming 158 # through core files via crash_reporter. 159 expected_address = '0x0' 160 if not match or match.group(1) != 'SIGSEGV': 161 raise error.TestFail('Did not identify SIGSEGV cause') 162 match = re.search(r'Crash address:\s+(.*)', stack) 163 if not match or match.group(1) != expected_address: 164 raise error.TestFail('Did not identify crash address %s' % 165 expected_address) 166 167 # Should identify crash at *(char*)0x16 assignment line 168 if not self._is_frame_in_stack(0, basename, 169 'recbomb', 'bomb.cc', 9, stack): 170 raise error.TestFail('Did not show crash line on stack') 171 172 # Should identify recursion line which is on the stack 173 # for 15 levels 174 if not self._is_frame_in_stack(15, basename, 'recbomb', 175 'bomb.cc', 12, stack): 176 raise error.TestFail('Did not show recursion line on stack') 177 178 # Should identify main line 179 if not self._is_frame_in_stack(16, basename, 'main', 180 'crasher.cc', 24, stack): 181 raise error.TestFail('Did not show main on stack') 182 183 184 def _run_crasher_process(self, username, cause_crash=True, consent=True, 185 crasher_path=None, run_crasher=None, 186 expected_uid=None, expected_gid=None, 187 expected_exit_code=None, expected_reason=None): 188 """Runs the crasher process. 189 190 Will wait up to 10 seconds for crash_reporter to report the crash. 191 crash_reporter_caught will be marked as true when the "Received crash 192 notification message..." appears. While associated logs are likely to be 193 available at this point, the function does not guarantee this. 194 195 @param username: Unix user of the crasher process. 196 @param cause_crash: Whether the crasher should crash. 197 @param consent: Whether the user consents to crash reporting. 198 @param crasher_path: Path to which the crasher should be copied before 199 execution. Relative to |_root_path|. 200 @param run_crasher: A closure to override the default |crasher_command| 201 invocation. It should return a tuple describing the 202 process, where |pid| can be None if it should be 203 parsed from the |output|: 204 205 def run_crasher(username, crasher_command): 206 ... 207 return (exit_code, output, pid) 208 209 @param expected_uid: The uid the crash happens under. 210 @param expected_gid: The gid the crash happens under. 211 @param expected_exit_code: 212 @param expected_reason: 213 Expected information in crash_reporter log message. 214 215 @returns: 216 A dictionary with keys: 217 returncode: return code of the crasher 218 crashed: did the crasher return segv error code 219 crash_reporter_caught: did crash_reporter catch a segv 220 output: stderr output of the crasher process 221 """ 222 if crasher_path is None: 223 crasher_path = self._crasher_path 224 else: 225 dest = os.path.join(self._root_path, 226 crasher_path[os.path.isabs(crasher_path):]) 227 228 utils.system('cp -a "%s" "%s"' % (self._crasher_path, dest)) 229 230 self.enable_crash_filtering(os.path.basename(crasher_path)) 231 232 crasher_command = [] 233 234 if username == 'root': 235 if expected_exit_code is None: 236 expected_exit_code = -signal.SIGSEGV 237 else: 238 if expected_exit_code is None: 239 expected_exit_code = 128 + signal.SIGSEGV 240 241 if not run_crasher: 242 crasher_command.extend(['su', username, '-c']) 243 244 crasher_command.append(crasher_path) 245 basename = os.path.basename(crasher_path) 246 if not cause_crash: 247 crasher_command.append('--nocrash') 248 self._set_consent(consent) 249 250 logging.debug('Running crasher: %s', crasher_command) 251 252 if run_crasher: 253 (exit_code, output, pid) = run_crasher(username, crasher_command) 254 255 else: 256 crasher = subprocess.Popen(crasher_command, 257 stdout=subprocess.PIPE, 258 stderr=subprocess.PIPE) 259 260 output = crasher.communicate()[1] 261 exit_code = crasher.returncode 262 pid = None 263 264 logging.debug('Crasher output:\n%s', output) 265 266 if pid is None: 267 # Get the PID from the output, since |crasher.pid| may be su's PID. 268 match = re.search(r'pid=(\d+)', output) 269 if not match: 270 raise error.TestFail('Missing PID in crasher output') 271 pid = int(match.group(1)) 272 273 if expected_uid is None: 274 expected_uid = pwd.getpwnam(username).pw_uid 275 276 if expected_gid is None: 277 expected_gid = pwd.getpwnam(username).pw_gid 278 279 if expected_reason is None: 280 expected_reason = 'handling' if consent else 'ignoring - no consent' 281 282 expected_message = ( 283 ('[%s] Received crash notification for %s[%d] sig 11, user %d ' 284 'group %d (%s)') % 285 (self._expected_tag, basename, pid, expected_uid, expected_gid, 286 expected_reason)) 287 288 # Wait until no crash_reporter is running. 289 utils.poll_for_condition( 290 lambda: utils.system('pgrep -f crash_reporter.*:%s' % basename, 291 ignore_status=True) != 0, 292 timeout=10, 293 exception=error.TestError( 294 'Timeout waiting for crash_reporter to finish: ' + 295 self._log_reader.get_logs())) 296 297 is_caught = False 298 try: 299 utils.poll_for_condition( 300 lambda: self._log_reader.can_find(expected_message), 301 timeout=5, 302 desc='Logs contain crash_reporter message: ' + expected_message) 303 is_caught = True 304 except utils.TimeoutError: 305 pass 306 307 result = {'crashed': exit_code == expected_exit_code, 308 'crash_reporter_caught': is_caught, 309 'output': output, 310 'returncode': exit_code} 311 logging.debug('Crasher process result: %s', result) 312 return result 313 314 315 def _check_crash_directory_permissions(self, crash_dir): 316 stat_info = os.stat(crash_dir) 317 user = pwd.getpwuid(stat_info.st_uid).pw_name 318 group = grp.getgrgid(stat_info.st_gid).gr_name 319 mode = stat.S_IMODE(stat_info.st_mode) 320 321 expected_mode = 0o700 322 if crash_dir == '/var/spool/crash': 323 expected_user = 'root' 324 expected_group = 'root' 325 else: 326 expected_user = 'chronos' 327 expected_group = 'chronos' 328 329 if user != expected_user or group != expected_group: 330 raise error.TestFail( 331 'Expected %s.%s ownership of %s (actual %s.%s)' % 332 (expected_user, expected_group, crash_dir, user, group)) 333 if mode != expected_mode: 334 raise error.TestFail( 335 'Expected %s to have mode %o (actual %o)' % 336 (crash_dir, expected_mode, mode)) 337 338 339 def _check_minidump_stackwalk(self, minidump_path, basename, 340 from_crash_reporter): 341 stack = utils.system_output('/usr/bin/minidump_stackwalk %s %s' % 342 (minidump_path, self._symbol_dir)) 343 self._verify_stack(stack, basename, from_crash_reporter) 344 345 346 def _check_generated_report_sending(self, meta_path, payload_path, 347 exec_name, report_kind, 348 expected_sig=None): 349 # Now check that the sending works 350 result = self._call_sender_one_crash( 351 report=os.path.basename(payload_path)) 352 if (not result['send_attempt'] or not result['send_success'] or 353 result['report_exists']): 354 raise error.TestFail('Report not sent properly') 355 if result['exec_name'] != exec_name: 356 raise error.TestFail('Executable name incorrect') 357 if result['report_kind'] != report_kind: 358 raise error.TestFail('Expected a %s report' % report_kind) 359 if result['report_payload'] != payload_path: 360 raise error.TestFail('Sent the wrong minidump payload %s vs %s' % ( 361 result['report_payload'], payload_path)) 362 if result['meta_path'] != meta_path: 363 raise error.TestFail('Used the wrong meta file %s vs %s' % ( 364 result['meta_path'], meta_path)) 365 if expected_sig is None: 366 if result['sig'] is not None: 367 raise error.TestFail('Report should not have signature') 368 else: 369 if not 'sig' in result or result['sig'] != expected_sig: 370 raise error.TestFail('Report signature mismatch: %s vs %s' % 371 (result['sig'], expected_sig)) 372 373 version = self._expected_version 374 if version is None: 375 lsb_release = utils.read_file('/etc/lsb-release') 376 version = re.search( 377 r'CHROMEOS_RELEASE_VERSION=(.*)', lsb_release).group(1) 378 379 if not ('Version: %s' % version) in result['output']: 380 raise error.TestFail('Missing version %s in log output' % version) 381 382 383 def _run_crasher_process_and_analyze(self, username, 384 cause_crash=True, consent=True, 385 crasher_path=None, run_crasher=None, 386 expected_uid=None, expected_gid=None, 387 expected_exit_code=None): 388 self._log_reader.set_start_by_current() 389 390 result = self._run_crasher_process( 391 username, cause_crash=cause_crash, consent=consent, 392 crasher_path=crasher_path, run_crasher=run_crasher, 393 expected_uid=expected_uid, expected_gid=expected_gid, 394 expected_exit_code=expected_exit_code) 395 396 if not result['crashed'] or not result['crash_reporter_caught']: 397 return result 398 399 crash_dir = self._get_crash_dir(username, self._force_user_crash_dir) 400 401 if not consent: 402 if os.path.exists(crash_dir): 403 raise error.TestFail('Crash directory should not exist') 404 return result 405 406 if not os.path.exists(crash_dir): 407 raise error.TestFail('Crash directory does not exist') 408 409 crash_contents = os.listdir(crash_dir) 410 basename = os.path.basename(crasher_path or self._crasher_path) 411 412 # A dict tracking files for each crash report. 413 crash_report_files = {} 414 415 self._check_crash_directory_permissions(crash_dir) 416 417 logging.debug('Contents in %s: %s', crash_dir, crash_contents) 418 419 # Variables and their typical contents: 420 # basename: crasher_nobreakpad 421 # filename: crasher_nobreakpad.20181023.135339.16890.dmp 422 # ext: dmp 423 for filename in crash_contents: 424 if filename.endswith('.core'): 425 # Ignore core files. We'll test them later. 426 pass 427 elif filename.startswith(basename + '.'): 428 ext = filename.rsplit('.', 1)[1] 429 logging.debug('Found crash report file (%s): %s', ext, filename) 430 if ext in crash_report_files: 431 raise error.TestFail( 432 'Found multiple files with .%s: %s and %s' % 433 (ext, filename, crash_report_files[ext])) 434 crash_report_files[ext] = filename 435 else: 436 # Flag all unknown files. 437 raise error.TestFail('Crash reporter created an unknown file: ' 438 '%s' % (filename,)) 439 440 # Make sure we generated the exact set of files we expected. 441 found_filetypes = set(crash_report_files.keys()) 442 missing_filetypes = self.REPORT_REQUIRED_FILETYPES - found_filetypes 443 unknown_filetypes = (found_filetypes - self.REPORT_REQUIRED_FILETYPES - 444 self.REPORT_OPTIONAL_FILETYPES) 445 if missing_filetypes: 446 raise error.TestFail('crash report is missing files: %s' % ( 447 ['.' + x for x in missing_filetypes],)) 448 if unknown_filetypes: 449 raise error.TestFail('crash report includes unknown files: %s' % ( 450 [crash_report_files[x] for x in unknown_filetypes],)) 451 452 # Create full paths for the logging code below. 453 for key in (self.REPORT_REQUIRED_FILETYPES | 454 self.REPORT_OPTIONAL_FILETYPES): 455 if key in crash_report_files: 456 crash_report_files[key] = os.path.join( 457 crash_dir, crash_report_files[key]) 458 else: 459 crash_report_files[key] = None 460 461 result['minidump'] = crash_report_files['dmp'] 462 result['basename'] = basename 463 result['meta'] = crash_report_files['meta'] 464 result['log'] = crash_report_files['log'] 465 return result 466 467 468 def _check_crashed_and_caught(self, result): 469 if not result['crashed']: 470 raise error.TestFail('Crasher returned %d instead of crashing' % 471 result['returncode']) 472 473 if not result['crash_reporter_caught']: 474 logging.debug('Logs do not contain crash_reporter message:\n%s', 475 self._log_reader.get_logs()) 476 raise error.TestFail('crash_reporter did not catch crash') 477 478 479 def _check_crashing_process(self, username, consent=True, 480 crasher_path=None, run_crasher=None, 481 expected_uid=None, expected_gid=None, 482 expected_exit_code=None): 483 result = self._run_crasher_process_and_analyze( 484 username, consent=consent, 485 crasher_path=crasher_path, 486 run_crasher=run_crasher, 487 expected_uid=expected_uid, 488 expected_gid=expected_gid, 489 expected_exit_code=expected_exit_code) 490 491 self._check_crashed_and_caught(result) 492 493 if not consent: 494 return 495 496 if not result['minidump']: 497 raise error.TestFail('crash reporter did not generate minidump') 498 499 if not self._log_reader.can_find('Stored minidump to ' + 500 result['minidump']): 501 raise error.TestFail('crash reporter did not announce minidump') 502 503 self._check_minidump_stackwalk(result['minidump'], 504 result['basename'], 505 from_crash_reporter=True) 506 self._check_generated_report_sending(result['meta'], 507 result['minidump'], 508 result['basename'], 509 'minidump') 510
