Home | History | Annotate | Download | only in logging_UserCrash
      1 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
      2 # Use of this source code is governed by a BSD-style license that can be
      3 # found in the LICENSE file.
      4 
      5 import grp, logging, os, pwd, re, stat, subprocess
      6 from signal import SIGSEGV
      7 from autotest_lib.client.bin import utils
      8 from autotest_lib.client.common_lib import error
      9 from autotest_lib.client.cros import crash_test, cros_ui, upstart
     10 
     11 
     12 _COLLECTION_ERROR_SIGNATURE = 'crash_reporter-user-collection'
     13 _CORE2MD_PATH = '/usr/bin/core2md'
     14 _LEAVE_CORE_PATH = '/root/.leave_core'
     15 _MAX_CRASH_DIRECTORY_SIZE = 32
     16 
     17 
     18 class logging_UserCrash(crash_test.CrashTest):
     19     version = 1
     20 
     21 
     22     def setup(self):
     23         os.chdir(self.srcdir)
     24         utils.make('clean')
     25         utils.make('all')
     26 
     27 
     28     def _test_reporter_startup(self):
     29         """Test that the core_pattern is set up by crash reporter."""
     30         # Turn off crash filtering so we see the original setting.
     31         self.disable_crash_filtering()
     32         output = utils.read_file(self._CORE_PATTERN).rstrip()
     33         expected_core_pattern = ('|%s --user=%%P:%%s:%%u:%%e' %
     34                                  self._CRASH_REPORTER_PATH)
     35         if output != expected_core_pattern:
     36             raise error.TestFail('core pattern should have been %s, not %s' %
     37                                  (expected_core_pattern, output))
     38 
     39         self._log_reader.set_start_by_reboot(-1)
     40 
     41         if not self._log_reader.can_find('Enabling user crash handling'):
     42             raise error.TestFail(
     43                 'user space crash handling was not started during last boot')
     44 
     45 
     46     def _test_reporter_shutdown(self):
     47         """Test the crash_reporter shutdown code works."""
     48         self._log_reader.set_start_by_current()
     49         utils.system('%s --clean_shutdown' % self._CRASH_REPORTER_PATH)
     50         output = utils.read_file(self._CORE_PATTERN).rstrip()
     51         if output != 'core':
     52             raise error.TestFail('core pattern should have been core, not %s' %
     53                                  output)
     54 
     55 
     56     def _prepare_crasher(self):
     57         """Extract the crasher and set its permissions.
     58 
     59         crasher is only gzipped to subvert Portage stripping.
     60         """
     61         self._crasher_path = os.path.join(self.srcdir, 'crasher_nobreakpad')
     62         utils.system('cd %s; tar xzf crasher.tgz-unmasked' %
     63                      self.srcdir)
     64         # Make sure all users (specifically chronos) have access to
     65         # this directory and its decendents in order to run crasher
     66         # executable as different users.
     67         utils.system('chmod -R a+rx ' + self.bindir)
     68 
     69 
     70     def _populate_symbols(self):
     71         """Set up Breakpad's symbol structure.
     72 
     73         Breakpad's minidump processor expects symbols to be in a directory
     74         hierarchy:
     75           <symbol-root>/<module_name>/<file_id>/<module_name>.sym
     76         """
     77         # Dump the symbols from the crasher
     78         self._symbol_dir = os.path.join(self.srcdir, 'symbols')
     79         utils.system('rm -rf %s' % self._symbol_dir)
     80         os.mkdir(self._symbol_dir)
     81 
     82         basename = os.path.basename(self._crasher_path)
     83         utils.system('/usr/bin/dump_syms %s > %s.sym' %
     84                      (self._crasher_path,
     85                       basename))
     86         sym_name = '%s.sym' % basename
     87         symbols = utils.read_file(sym_name)
     88         # First line should be like:
     89         # MODULE Linux x86 7BC3323FBDBA2002601FA5BA3186D6540 crasher_XXX
     90         #  or
     91         # MODULE Linux arm C2FE4895B203D87DD4D9227D5209F7890 crasher_XXX
     92         first_line = symbols.split('\n')[0]
     93         tokens = first_line.split()
     94         if tokens[0] != 'MODULE' or tokens[1] != 'Linux':
     95           raise error.TestError('Unexpected symbols format: %s',
     96                                 first_line)
     97         file_id = tokens[3]
     98         target_dir = os.path.join(self._symbol_dir, basename, file_id)
     99         os.makedirs(target_dir)
    100         os.rename(sym_name, os.path.join(target_dir, sym_name))
    101 
    102 
    103     def _is_frame_in_stack(self, frame_index, module_name,
    104                            function_name, file_name,
    105                            line_number, stack):
    106         """Search for frame entries in the given stack dump text.
    107 
    108         A frame entry looks like (alone on a line):
    109           16  crasher_nobreakpad!main [crasher.cc : 21 + 0xb]
    110 
    111         Args:
    112           frame_index: number of the stack frame (0 is innermost frame)
    113           module_name: name of the module (executable or dso)
    114           function_name: name of the function in the stack
    115           file_name: name of the file containing the function
    116           line_number: line number
    117           stack: text string of stack frame entries on separate lines.
    118 
    119         Returns:
    120           Boolean indicating if an exact match is present.
    121 
    122         Note:
    123           We do not care about the full function signature - ie, is it
    124           foo or foo(ClassA *).  These are present in function names
    125           pulled by dump_syms for Stabs but not for DWARF.
    126         """
    127         regexp = (r'\n\s*%d\s+%s!%s.*\[\s*%s\s*:\s*%d\s.*\]' %
    128                   (frame_index, module_name,
    129                    function_name, file_name,
    130                    line_number))
    131         logging.info('Searching for regexp ' + regexp)
    132         return re.search(regexp, stack) is not None
    133 
    134 
    135     def _verify_stack(self, stack, basename, from_crash_reporter):
    136         logging.debug('Crash stackwalk was: %s' % stack)
    137 
    138         # Should identify cause as SIGSEGV at address 0x16
    139         match = re.search(r'Crash reason:\s+(.*)', stack)
    140         expected_address = '0x16'
    141         if from_crash_reporter:
    142             # We cannot yet determine the crash address when coming
    143             # through core files via crash_reporter.
    144             expected_address = '0x0'
    145         if not match or match.group(1) != 'SIGSEGV':
    146             raise error.TestFail('Did not identify SIGSEGV cause')
    147         match = re.search(r'Crash address:\s+(.*)', stack)
    148         if not match or match.group(1) != expected_address:
    149             raise error.TestFail('Did not identify crash address %s' %
    150                                  expected_address)
    151 
    152         # Should identify crash at *(char*)0x16 assignment line
    153         if not self._is_frame_in_stack(0, basename,
    154                                        'recbomb', 'bomb.cc', 9, stack):
    155             raise error.TestFail('Did not show crash line on stack')
    156 
    157         # Should identify recursion line which is on the stack
    158         # for 15 levels
    159         if not self._is_frame_in_stack(15, basename, 'recbomb',
    160                                        'bomb.cc', 12, stack):
    161             raise error.TestFail('Did not show recursion line on stack')
    162 
    163         # Should identify main line
    164         if not self._is_frame_in_stack(16, basename, 'main',
    165                                        'crasher.cc', 20, stack):
    166             raise error.TestFail('Did not show main on stack')
    167 
    168 
    169     def _run_crasher_process(self, username, cause_crash=True, consent=True,
    170                              crasher_path=None):
    171         """Runs the crasher process.
    172 
    173         Will wait up to 5 seconds for crash_reporter to report the crash.
    174         crash_reporter_caught will be marked as true when the "Received crash
    175         notification message..." appears. While associated logs are likely to be
    176         available at this point, the function does not guarantee this.
    177 
    178         Args:
    179           username: runs as given user
    180           extra_args: additional parameters to pass to crasher process
    181 
    182         Returns:
    183           A dictionary with keys:
    184             returncode: return code of the crasher
    185             crashed: did the crasher return segv error code
    186             crash_reporter_caught: did crash_reporter catch a segv
    187             output: stderr/stdout output of the crasher process
    188         """
    189         if crasher_path is None: crasher_path = self._crasher_path
    190         self.enable_crash_filtering(os.path.basename(crasher_path))
    191 
    192         if username != 'root':
    193             crasher_command = ['su', username, '-c']
    194             expected_result = 128 + SIGSEGV
    195         else:
    196             crasher_command = []
    197             expected_result = -SIGSEGV
    198 
    199         crasher_command.append(crasher_path)
    200         basename = os.path.basename(crasher_path)
    201         if not cause_crash:
    202             crasher_command.append('--nocrash')
    203         self._set_consent(consent)
    204         crasher = subprocess.Popen(crasher_command,
    205                                    stdout=subprocess.PIPE,
    206                                    stderr=subprocess.PIPE)
    207         output = crasher.communicate()[1]
    208         logging.debug('Output from %s: %s' %
    209                       (crasher_command, output))
    210 
    211         # Grab the pid from the process output.  We can't just use
    212         # crasher.pid unfortunately because that may be the PID of su.
    213         match = re.search(r'pid=(\d+)', output)
    214         if not match:
    215             raise error.TestFail('Could not find pid output from crasher: %s' %
    216                                  output)
    217         pid = int(match.group(1))
    218 
    219         expected_uid = pwd.getpwnam(username)[2]
    220         if consent:
    221             handled_string = 'handling'
    222         else:
    223             handled_string = 'ignoring - no consent'
    224         expected_message = (
    225             'Received crash notification for %s[%d] sig 11, user %d (%s)' %
    226             (basename, pid, expected_uid, handled_string))
    227 
    228         # Wait until no crash_reporter is running.
    229         utils.poll_for_condition(
    230             lambda: utils.system('pgrep -f crash_reporter.*:%s' % basename,
    231                                  ignore_status=True) != 0,
    232             timeout=10,
    233             exception=error.TestError(
    234                 'Timeout waiting for crash_reporter to finish: ' +
    235                 self._log_reader.get_logs()))
    236 
    237         logging.debug('crash_reporter_caught message: ' + expected_message)
    238         is_caught = False
    239         try:
    240             utils.poll_for_condition(
    241                 lambda: self._log_reader.can_find(expected_message),
    242                 timeout=5)
    243             is_caught = True
    244         except utils.TimeoutError:
    245             pass
    246 
    247         result = {'crashed': crasher.returncode == expected_result,
    248                   'crash_reporter_caught': is_caught,
    249                   'output': output,
    250                   'returncode': crasher.returncode}
    251         logging.debug('Crasher process result: %s' % result)
    252         return result
    253 
    254 
    255     def _check_crash_directory_permissions(self, crash_dir):
    256         stat_info = os.stat(crash_dir)
    257         user = pwd.getpwuid(stat_info.st_uid)[0]
    258         group = grp.getgrgid(stat_info.st_gid)[0]
    259         mode = stat.S_IMODE(stat_info.st_mode)
    260 
    261         if crash_dir == '/var/spool/crash':
    262             expected_user = 'root'
    263             expected_group = 'root'
    264             expected_mode = 01755
    265         else:
    266             expected_user = 'chronos'
    267             expected_group = 'chronos'
    268             expected_mode = 0755
    269 
    270         if user != expected_user or group != expected_group:
    271             raise error.TestFail(
    272                 'Expected %s.%s ownership of %s (actual %s.%s)' %
    273                 (expected_user, expected_group, crash_dir, user, group))
    274         if mode != expected_mode:
    275             raise error.TestFail(
    276                 'Expected %s to have mode %o (actual %o)' %
    277                 (crash_dir, expected_mode, mode))
    278 
    279 
    280     def _check_minidump_stackwalk(self, minidump_path, basename,
    281                                   from_crash_reporter):
    282         # Now stackwalk the minidump
    283         stack = utils.system_output('/usr/bin/minidump_stackwalk %s %s' %
    284                                     (minidump_path, self._symbol_dir))
    285         self._verify_stack(stack, basename, from_crash_reporter)
    286 
    287 
    288     def _check_generated_report_sending(self, meta_path, payload_path,
    289                                         username, exec_name, report_kind,
    290                                         expected_sig=None):
    291         # Now check that the sending works
    292         result = self._call_sender_one_crash(
    293             username=username,
    294             report=os.path.basename(payload_path))
    295         if (not result['send_attempt'] or not result['send_success'] or
    296             result['report_exists']):
    297             raise error.TestFail('Report not sent properly')
    298         if result['exec_name'] != exec_name:
    299             raise error.TestFail('Executable name incorrect')
    300         if result['report_kind'] != report_kind:
    301             raise error.TestFail('Expected a minidump report')
    302         if result['report_payload'] != payload_path:
    303             raise error.TestFail('Sent the wrong minidump payload')
    304         if result['meta_path'] != meta_path:
    305             raise error.TestFail('Used the wrong meta file')
    306         if expected_sig is None:
    307             if result['sig'] is not None:
    308                 raise error.TestFail('Report should not have signature')
    309         else:
    310             if not 'sig' in result or result['sig'] != expected_sig:
    311                 raise error.TestFail('Report signature mismatch: %s vs %s' %
    312                                      (result['sig'], expected_sig))
    313 
    314         # Check version matches.
    315         lsb_release = utils.read_file('/etc/lsb-release')
    316         version_match = re.search(r'CHROMEOS_RELEASE_VERSION=(.*)', lsb_release)
    317         if not ('Version: %s' % version_match.group(1)) in result['output']:
    318             raise error.TestFail('Did not find version %s in log output' %
    319                                  version_match.group(1))
    320 
    321 
    322     def _run_crasher_process_and_analyze(self, username,
    323                                          cause_crash=True, consent=True,
    324                                          crasher_path=None):
    325         self._log_reader.set_start_by_current()
    326 
    327         if crasher_path is None: crasher_path = self._crasher_path
    328         result = self._run_crasher_process(username, cause_crash=cause_crash,
    329                                            consent=consent,
    330                                            crasher_path=crasher_path)
    331 
    332         if not result['crashed'] or not result['crash_reporter_caught']:
    333             return result;
    334 
    335         crash_dir = self._get_crash_dir(username)
    336 
    337         if not consent:
    338             if os.path.exists(crash_dir):
    339                 raise error.TestFail('Crash directory should not exist')
    340             return result
    341 
    342         crash_contents = os.listdir(crash_dir)
    343         basename = os.path.basename(crasher_path)
    344 
    345         breakpad_minidump = None
    346         crash_reporter_minidump = None
    347         crash_reporter_meta = None
    348         crash_reporter_log = None
    349 
    350         self._check_crash_directory_permissions(crash_dir)
    351 
    352         logging.debug('Contents in %s: %s' % (crash_dir, crash_contents))
    353 
    354         for filename in crash_contents:
    355             if filename.endswith('.core'):
    356                 # Ignore core files.  We'll test them later.
    357                 pass
    358             elif (filename.startswith(basename) and
    359                   filename.endswith('.dmp')):
    360                 # This appears to be a minidump created by the crash reporter.
    361                 if not crash_reporter_minidump is None:
    362                     raise error.TestFail('Crash reporter wrote multiple '
    363                                          'minidumps')
    364                 crash_reporter_minidump = os.path.join(crash_dir, filename)
    365             elif (filename.startswith(basename) and
    366                   filename.endswith('.meta')):
    367                 if not crash_reporter_meta is None:
    368                     raise error.TestFail('Crash reporter wrote multiple '
    369                                          'meta files')
    370                 crash_reporter_meta = os.path.join(crash_dir, filename)
    371             elif (filename.startswith(basename) and
    372                   filename.endswith('.log')):
    373                 if not crash_reporter_log is None:
    374                     raise error.TestFail('Crash reporter wrote multiple '
    375                                          'log files')
    376                 crash_reporter_log = os.path.join(crash_dir, filename)
    377             else:
    378                 # This appears to be a breakpad created minidump.
    379                 if not breakpad_minidump is None:
    380                     raise error.TestFail('Breakpad wrote multimpe minidumps')
    381                 breakpad_minidump = os.path.join(crash_dir, filename)
    382 
    383         if breakpad_minidump:
    384             raise error.TestFail('%s did generate breakpad minidump' % basename)
    385 
    386         if not crash_reporter_meta:
    387             raise error.TestFail('crash reporter did not generate meta')
    388 
    389         result['minidump'] = crash_reporter_minidump
    390         result['basename'] = basename
    391         result['meta'] = crash_reporter_meta
    392         result['log'] = crash_reporter_log
    393         return result
    394 
    395 
    396     def _check_crashed_and_caught(self, result):
    397         if not result['crashed']:
    398             raise error.TestFail('crasher did not do its job of crashing: %d' %
    399                                  result['returncode'])
    400 
    401         if not result['crash_reporter_caught']:
    402             logging.debug('Messages that should have included segv: %s' %
    403                           self._log_reader.get_logs())
    404             raise error.TestFail('Did not find segv message')
    405 
    406 
    407     def _check_crashing_process(self, username, consent=True):
    408         result = self._run_crasher_process_and_analyze(username,
    409                                                        consent=consent)
    410 
    411         self._check_crashed_and_caught(result)
    412 
    413         if not consent:
    414             return
    415 
    416         if not result['minidump']:
    417             raise error.TestFail('crash reporter did not generate minidump')
    418 
    419         if not self._log_reader.can_find('Stored minidump to ' +
    420                                          result['minidump']):
    421             raise error.TestFail('crash reporter did not announce minidump')
    422 
    423         self._check_minidump_stackwalk(result['minidump'],
    424                                        result['basename'],
    425                                        from_crash_reporter=True)
    426         self._check_generated_report_sending(result['meta'],
    427                                              result['minidump'],
    428                                              username,
    429                                              result['basename'],
    430                                              'minidump')
    431 
    432     def _test_no_crash(self):
    433         """Test a program linked against libcrash_dumper can exit normally."""
    434         self._log_reader.set_start_by_current()
    435         result = self._run_crasher_process_and_analyze(username='root',
    436                                                        cause_crash=False)
    437         if (result['crashed'] or
    438             result['crash_reporter_caught'] or
    439             result['returncode'] != 0):
    440             raise error.TestFail('Normal exit of program with dumper failed')
    441 
    442 
    443     def _test_chronos_crasher(self):
    444         """Test a user space crash when running as chronos is handled."""
    445         self._check_crashing_process('chronos')
    446 
    447 
    448     def _test_chronos_crasher_no_consent(self):
    449         """Test that without consent no files are stored."""
    450         results = self._check_crashing_process('chronos', consent=False)
    451 
    452 
    453     def _test_root_crasher(self):
    454         """Test a user space crash when running as root is handled."""
    455         self._check_crashing_process('root')
    456 
    457 
    458     def _test_root_crasher_no_consent(self):
    459         """Test that without consent no files are stored."""
    460         results = self._check_crashing_process('root', consent=False)
    461 
    462 
    463     def _check_filter_crasher(self, should_receive):
    464         self._log_reader.set_start_by_current()
    465         crasher_basename = os.path.basename(self._crasher_path)
    466         utils.system(self._crasher_path, ignore_status=True);
    467         if should_receive:
    468             to_find = 'Received crash notification for ' + crasher_basename
    469         else:
    470             to_find = 'Ignoring crash from ' + crasher_basename
    471         utils.poll_for_condition(
    472             lambda: self._log_reader.can_find(to_find),
    473             timeout=10,
    474             exception=error.TestError(
    475               'Timeout waiting for: ' + to_find + ' in ' +
    476               self._log_reader.get_logs()))
    477 
    478 
    479     def _test_crash_filtering(self):
    480         """Test that crash filtering (a feature needed for testing) works."""
    481         crasher_basename = os.path.basename(self._crasher_path)
    482         self._log_reader.set_start_by_current()
    483 
    484         self.enable_crash_filtering('none')
    485         self._check_filter_crasher(False)
    486 
    487         self.enable_crash_filtering('sleep')
    488         self._check_filter_crasher(False)
    489 
    490         self.disable_crash_filtering()
    491         self._check_filter_crasher(True)
    492 
    493 
    494     def _test_max_enqueued_crashes(self):
    495         """Test that _MAX_CRASH_DIRECTORY_SIZE is enforced."""
    496         self._log_reader.set_start_by_current()
    497         username = 'root'
    498 
    499         crash_dir = self._get_crash_dir(username)
    500         full_message = ('Crash directory %s already full with %d pending '
    501                         'reports' % (crash_dir, _MAX_CRASH_DIRECTORY_SIZE))
    502 
    503         # Fill up the queue.
    504         for i in range(0, _MAX_CRASH_DIRECTORY_SIZE):
    505           result = self._run_crasher_process(username)
    506           if not result['crashed']:
    507             raise error.TestFail('failure while setting up queue: %d' %
    508                                  result['returncode'])
    509           if self._log_reader.can_find(full_message):
    510             raise error.TestFail('unexpected full message: ' + full_message)
    511 
    512         crash_dir_size = len(os.listdir(crash_dir))
    513         # For debugging
    514         utils.system('ls -l %s' % crash_dir)
    515         logging.info('Crash directory had %d entries' % crash_dir_size)
    516 
    517         # Crash a bunch more times, but make sure no new reports
    518         # are enqueued.
    519         for i in range(0, 10):
    520           self._log_reader.set_start_by_current()
    521           result = self._run_crasher_process(username)
    522           logging.info('New log messages: %s' % self._log_reader.get_logs())
    523           if not result['crashed']:
    524             raise error.TestFail('failure after setting up queue: %d' %
    525                                  result['returncode'])
    526           utils.poll_for_condition(
    527               lambda: self._log_reader.can_find(full_message),
    528               timeout=20,
    529               exception=error.TestFail('expected full message: ' +
    530                                        full_message))
    531           if crash_dir_size != len(os.listdir(crash_dir)):
    532             utils.system('ls -l %s' % crash_dir)
    533             raise error.TestFail('expected no new files (now %d were %d)',
    534                                  len(os.listdir(crash_dir)),
    535                                  crash_dir_size)
    536 
    537 
    538     def _check_collection_failure(self, test_option, failure_string):
    539         # Add parameter to core_pattern.
    540         old_core_pattern = utils.read_file(self._CORE_PATTERN)[:-1]
    541         try:
    542             utils.system('echo "%s %s" > %s' % (old_core_pattern, test_option,
    543                                                 self._CORE_PATTERN))
    544             result = self._run_crasher_process_and_analyze('root',
    545                                                            consent=True)
    546             self._check_crashed_and_caught(result)
    547             if not self._log_reader.can_find(failure_string):
    548                 raise error.TestFail('Did not find fail string in log %s' %
    549                                      failure_string)
    550             if result['minidump']:
    551                 raise error.TestFail('failed collection resulted in minidump')
    552             if not result['log']:
    553                 raise error.TestFail('failed collection had no log')
    554             log_contents = utils.read_file(result['log'])
    555             logging.debug('Log contents were: ' + log_contents)
    556             if not failure_string in log_contents:
    557                 raise error.TestFail('Expected logged error '
    558                                      '\"%s\" was \"%s\"' %
    559                                      (failure_string, log_contents))
    560             # Verify we are generating appropriate diagnostic output.
    561             if ((not '===ps output===' in log_contents) or
    562                 (not '===meminfo===' in log_contents)):
    563                 raise error.TestFail('Expected full logs, got: ' + log_contents)
    564             self._check_generated_report_sending(result['meta'],
    565                                                  result['log'],
    566                                                  'root',
    567                                                  result['basename'],
    568                                                  'log',
    569                                                  _COLLECTION_ERROR_SIGNATURE)
    570         finally:
    571             utils.system('echo "%s" > %s' % (old_core_pattern,
    572                                              self._CORE_PATTERN))
    573 
    574 
    575     def _test_core2md_failure(self):
    576         self._check_collection_failure('--core2md_failure',
    577                                        'Problem during %s [result=1]: Usage:' %
    578                                        _CORE2MD_PATH)
    579 
    580 
    581     def _test_internal_directory_failure(self):
    582         self._check_collection_failure('--directory_failure',
    583                                        'Purposefully failing to create')
    584 
    585 
    586     def _test_crash_logs_creation(self):
    587         logs_triggering_crasher = os.path.join(os.path.dirname(self.bindir),
    588                                                'crash_log_test')
    589         # Copy crasher_path to a test location with correct mode and a
    590         # special name to trigger crash log creation.
    591         utils.system('cp -a "%s" "%s"' % (self._crasher_path,
    592                                           logs_triggering_crasher))
    593         result = self._run_crasher_process_and_analyze(
    594             'root', crasher_path=logs_triggering_crasher)
    595         self._check_crashed_and_caught(result)
    596         contents = utils.read_file(result['log'])
    597         if contents != 'hello world\n':
    598             raise error.TestFail('Crash log contents unexpected: %s' % contents)
    599         if not ('log=' + result['log']) in utils.read_file(result['meta']):
    600             raise error.TestFail('Meta file does not reference log')
    601 
    602 
    603     def _test_crash_log_infinite_recursion(self):
    604         recursion_triggering_crasher = os.path.join(
    605             os.path.dirname(self.bindir), 'crash_log_recursion_test')
    606         # The configuration file hardcodes this path, so make sure it's still
    607         # the same.
    608         if (recursion_triggering_crasher !=
    609             '/usr/local/autotest/tests/crash_log_recursion_test'):
    610           raise error.TestError('Path to recursion test changed')
    611         # Copy crasher_path to a test location with correct mode and a
    612         # special name to trigger crash log creation.
    613         utils.system('cp -a "%s" "%s"' % (self._crasher_path,
    614                                           recursion_triggering_crasher))
    615         # Simply completing this command means that we avoided
    616         # infinite recursion.
    617         result = self._run_crasher_process(
    618             'root', crasher_path=recursion_triggering_crasher)
    619 
    620 
    621     def _check_core_file_persisting(self, expect_persist):
    622         self._log_reader.set_start_by_current()
    623 
    624         result = self._run_crasher_process('root')
    625 
    626         if not result['crashed']:
    627             raise error.TestFail('crasher did not crash')
    628 
    629         crash_contents = os.listdir(self._get_crash_dir('root'))
    630 
    631         logging.debug('Contents of crash directory: %s', crash_contents)
    632         logging.debug('Log messages: %s' % self._log_reader.get_logs())
    633 
    634         if expect_persist:
    635             if not self._log_reader.can_find('Leaving core file at'):
    636                 raise error.TestFail('Missing log message')
    637             expected_core_files = 1
    638         else:
    639             if self._log_reader.can_find('Leaving core file at'):
    640                 raise error.TestFail('Unexpected log message')
    641             expected_core_files = 0
    642 
    643         dmp_files = 0
    644         core_files = 0
    645         for filename in crash_contents:
    646             if filename.endswith('.dmp'):
    647                 dmp_files += 1
    648             if filename.endswith('.core'):
    649                 core_files += 1
    650 
    651         if dmp_files != 1:
    652             raise error.TestFail('Should have been exactly 1 dmp file')
    653         if core_files != expected_core_files:
    654             raise error.TestFail('Should have been exactly %d core files' %
    655                                  expected_core_files)
    656 
    657 
    658     def _test_core_file_removed_in_production(self):
    659         """Test that core files do not stick around for production builds."""
    660         # Avoid remounting / rw by instead creating a tmpfs in /root and
    661         # populating it with everything but the
    662         utils.system('tar -cvz -C /root -f /tmp/root.tgz .')
    663         utils.system('mount -t tmpfs tmpfs /root')
    664         try:
    665             utils.system('tar -xvz -C /root -f /tmp/root.tgz .')
    666             os.remove(_LEAVE_CORE_PATH)
    667             if os.path.exists(_LEAVE_CORE_PATH):
    668                 raise error.TestFail('.leave_core file did not disappear')
    669             self._check_core_file_persisting(False)
    670         finally:
    671             os.system('umount /root')
    672 
    673 
    674     def initialize(self):
    675         super(logging_UserCrash, self).initialize()
    676 
    677         # If the device has a GUI, return the device to the sign-in screen, as
    678         # some tests will fail inside a user session.
    679         if upstart.has_service('ui'):
    680             cros_ui.restart()
    681 
    682 
    683     # TODO(kmixter): Test crashing a process as ntp or some other
    684     # non-root, non-chronos user.
    685 
    686     def run_once(self):
    687         self._prepare_crasher()
    688         self._populate_symbols()
    689 
    690         # Run the test once without re-initializing
    691         # to catch problems with the default crash reporting setup
    692         self.run_crash_tests(['reporter_startup'],
    693                               initialize_crash_reporter=False,
    694                               must_run_all=False)
    695 
    696         self.run_crash_tests(['reporter_startup',
    697                               'reporter_shutdown',
    698                               'no_crash',
    699                               'chronos_crasher',
    700                               'chronos_crasher_no_consent',
    701                               'root_crasher',
    702                               'root_crasher_no_consent',
    703                               'crash_filtering',
    704                               'max_enqueued_crashes',
    705                               'core2md_failure',
    706                               'internal_directory_failure',
    707                               'crash_logs_creation',
    708                               'crash_log_infinite_recursion',
    709                               'core_file_removed_in_production'],
    710                               initialize_crash_reporter=True)
    711