Home | History | Annotate | Download | only in valgrind 
      1 #!/usr/bin/env python
      2 # Copyright (c) 2011 The Chromium Authors. All rights reserved.
      3 # Use of this source code is governed by a BSD-style license that can be
      4 # found in the LICENSE file.
      5 
      6 # tsan_analyze.py
      7 
      8 ''' Given a ThreadSanitizer output file, parses errors and uniques them.'''
      9 
     10 import gdb_helper
     11 
     12 from collections import defaultdict
     13 import hashlib
     14 import logging
     15 import optparse
     16 import os
     17 import re
     18 import subprocess
     19 import sys
     20 import time
     21 
     22 import common
     23 
     24 # Global symbol table (ugh)
     25 TheAddressTable = None
     26 
     27 class _StackTraceLine(object):
     28   def __init__(self, line, address, binary):
     29     self.raw_line_ = line
     30     self.address = address
     31     self.binary = binary
     32   def __str__(self):
     33     global TheAddressTable
     34     file, line = TheAddressTable.GetFileLine(self.binary, self.address)
     35     if (file is None) or (line is None):
     36       return self.raw_line_
     37     else:
     38       return self.raw_line_.replace(self.binary, '%s:%s' % (file, line))
     39 
     40 class TsanAnalyzer(object):
     41   ''' Given a set of ThreadSanitizer output files, parse all the errors out of
     42   them, unique them and output the results.'''
     43 
     44   LOAD_LIB_RE = re.compile('--[0-9]+-- ([^(:]*) \((0x[0-9a-f]+)\)')
     45   TSAN_LINE_RE = re.compile('==[0-9]+==\s*[#0-9]+\s*'
     46                             '([0-9A-Fa-fx]+):'
     47                             '(?:[^ ]* )*'
     48                             '([^ :\n]+)'
     49                             '')
     50   THREAD_CREATION_STR = ("INFO: T.* "
     51       "(has been created by T.* at this point|is program's main thread)")
     52 
     53   SANITY_TEST_SUPPRESSION = ("ThreadSanitizer sanity test "
     54                              "(ToolsSanityTest.DataRace)")
     55   TSAN_RACE_DESCRIPTION = "Possible data race"
     56   TSAN_WARNING_DESCRIPTION =  ("Unlocking a non-locked lock"
     57       "|accessing an invalid lock"
     58       "|which did not acquire this lock")
     59   RACE_VERIFIER_LINE = "Confirmed a race|unexpected race"
     60   TSAN_ASSERTION = "Assertion failed: "
     61 
     62   def __init__(self, source_dir, use_gdb=False):
     63     '''Reads in a set of files.
     64 
     65     Args:
     66       source_dir: Path to top of source tree for this build
     67     '''
     68 
     69     self._use_gdb = use_gdb
     70     self._cur_testcase = None
     71 
     72   def ReadLine(self):
     73     self.line_ = self.cur_fd_.readline()
     74     self.stack_trace_line_ = None
     75     if not self._use_gdb:
     76       return
     77     global TheAddressTable
     78     match = TsanAnalyzer.LOAD_LIB_RE.match(self.line_)
     79     if match:
     80       binary, ip = match.groups()
     81       TheAddressTable.AddBinaryAt(binary, ip)
     82       return
     83     match = TsanAnalyzer.TSAN_LINE_RE.match(self.line_)
     84     if match:
     85       address, binary_name = match.groups()
     86       stack_trace_line = _StackTraceLine(self.line_, address, binary_name)
     87       TheAddressTable.Add(stack_trace_line.binary, stack_trace_line.address)
     88       self.stack_trace_line_ = stack_trace_line
     89 
     90   def ReadSection(self):
     91     """ Example of a section:
     92     ==4528== WARNING: Possible data race: {{{
     93     ==4528==    T20 (L{}):
     94     ==4528==     #0  MyTest::Foo1
     95     ==4528==     #1  MyThread::ThreadBody
     96     ==4528==   Concurrent write happened at this point:
     97     ==4528==    T19 (L{}):
     98     ==4528==     #0  MyTest::Foo2
     99     ==4528==     #1  MyThread::ThreadBody
    100     ==4528== }}}
    101     ------- suppression -------
    102     {
    103       <Put your suppression name here>
    104       ThreadSanitizer:Race
    105       fun:MyTest::Foo1
    106       fun:MyThread::ThreadBody
    107     }
    108     ------- end suppression -------
    109     """
    110     result = [self.line_]
    111     if re.search("{{{", self.line_):
    112       while not re.search('}}}', self.line_):
    113         self.ReadLine()
    114         if self.stack_trace_line_ is None:
    115           result.append(self.line_)
    116         else:
    117           result.append(self.stack_trace_line_)
    118       self.ReadLine()
    119       if re.match('-+ suppression -+', self.line_):
    120         # We need to calculate the suppression hash and prepend a line like
    121         # "Suppression (error hash=#0123456789ABCDEF#):" so the buildbot can
    122         # extract the suppression snippet.
    123         supp = ""
    124         while not re.match('-+ end suppression -+', self.line_):
    125           self.ReadLine()
    126           supp += self.line_
    127         self.ReadLine()
    128         if self._cur_testcase:
    129           result.append("The report came from the `%s` test.\n" % \
    130                         self._cur_testcase)
    131         result.append("Suppression (error hash=#%016X#):\n" % \
    132                       (int(hashlib.md5(supp).hexdigest()[:16], 16)))
    133         result.append("  For more info on using suppressions see "
    134             "http://dev.chromium.org/developers/how-tos/using-valgrind/threadsanitizer#TOC-Suppressing-data-races\n")
    135         result.append(supp)
    136     else:
    137       self.ReadLine()
    138 
    139     return result
    140 
    141   def ReadTillTheEnd(self):
    142     result = [self.line_]
    143     while self.line_:
    144       self.ReadLine()
    145       result.append(self.line_)
    146     return result
    147 
    148   def ParseReportFile(self, filename):
    149     '''Parses a report file and returns a list of ThreadSanitizer reports.
    150 
    151 
    152     Args:
    153       filename: report filename.
    154     Returns:
    155       list of (list of (str iff self._use_gdb, _StackTraceLine otherwise)).
    156     '''
    157     ret = []
    158     self.cur_fd_ = open(filename, 'r')
    159 
    160     while True:
    161       # Read ThreadSanitizer reports.
    162       self.ReadLine()
    163       if not self.line_:
    164         break
    165 
    166       while True:
    167         tmp = []
    168         while re.search(TsanAnalyzer.RACE_VERIFIER_LINE, self.line_):
    169           tmp.append(self.line_)
    170           self.ReadLine()
    171         while re.search(TsanAnalyzer.THREAD_CREATION_STR, self.line_):
    172           tmp.extend(self.ReadSection())
    173         if re.search(TsanAnalyzer.TSAN_RACE_DESCRIPTION, self.line_):
    174           tmp.extend(self.ReadSection())
    175           ret.append(tmp)  # includes RaceVerifier and thread creation stacks
    176         elif (re.search(TsanAnalyzer.TSAN_WARNING_DESCRIPTION, self.line_) and
    177             not common.IsWindows()): # workaround for http://crbug.com/53198
    178           tmp.extend(self.ReadSection())
    179           ret.append(tmp)
    180         else:
    181           break
    182 
    183       tmp = []
    184       if re.search(TsanAnalyzer.TSAN_ASSERTION, self.line_):
    185         tmp.extend(self.ReadTillTheEnd())
    186         ret.append(tmp)
    187         break
    188 
    189       match = re.search("used_suppression:\s+([0-9]+)\s(.*)", self.line_)
    190       if match:
    191         count, supp_name = match.groups()
    192         count = int(count)
    193         self.used_suppressions[supp_name] += count
    194     self.cur_fd_.close()
    195     return ret
    196 
    197   def GetReports(self, files):
    198     '''Extracts reports from a set of files.
    199 
    200     Reads a set of files and returns a list of all discovered
    201     ThreadSanitizer race reports. As a side effect, populates
    202     self.used_suppressions with appropriate info.
    203     '''
    204 
    205     global TheAddressTable
    206     if self._use_gdb:
    207       TheAddressTable = gdb_helper.AddressTable()
    208     else:
    209       TheAddressTable = None
    210     reports = []
    211     self.used_suppressions = defaultdict(int)
    212     for file in files:
    213       reports.extend(self.ParseReportFile(file))
    214     if self._use_gdb:
    215       TheAddressTable.ResolveAll()
    216       # Make each line of each report a string.
    217       reports = map(lambda(x): map(str, x), reports)
    218     return [''.join(report_lines) for report_lines in reports]
    219 
    220   def Report(self, files, testcase, check_sanity=False):
    221     '''Reads in a set of files and prints ThreadSanitizer report.
    222 
    223     Args:
    224       files: A list of filenames.
    225       check_sanity: if true, search for SANITY_TEST_SUPPRESSIONS
    226     '''
    227 
    228     # We set up _cur_testcase class-wide variable to avoid passing it through
    229     # about 5 functions.
    230     self._cur_testcase = testcase
    231     reports = self.GetReports(files)
    232     self._cur_testcase = None  # just in case, shouldn't be used anymore
    233 
    234     common.PrintUsedSuppressionsList(self.used_suppressions)
    235 
    236 
    237     retcode = 0
    238     if reports:
    239       sys.stdout.flush()
    240       sys.stderr.flush()
    241       logging.info("FAIL! Found %i report(s)" % len(reports))
    242       for report in reports:
    243         logging.info('\n' + report)
    244       sys.stdout.flush()
    245       retcode = -1
    246 
    247     # Report tool's insanity even if there were errors.
    248     if (check_sanity and
    249         TsanAnalyzer.SANITY_TEST_SUPPRESSION not in self.used_suppressions):
    250       logging.error("FAIL! Sanity check failed!")
    251       retcode = -3
    252 
    253     if retcode != 0:
    254       return retcode
    255 
    256     logging.info("PASS: No reports found")
    257     return 0
    258 
    259 
    260 def main():
    261   '''For testing only. The TsanAnalyzer class should be imported instead.'''
    262   parser = optparse.OptionParser("usage: %prog [options] <files to analyze>")
    263   parser.add_option("", "--source_dir",
    264                     help="path to top of source tree for this build"
    265                     "(used to normalize source paths in baseline)")
    266 
    267   (options, args) = parser.parse_args()
    268   if not args:
    269     parser.error("no filename specified")
    270   filenames = args
    271 
    272   logging.getLogger().setLevel(logging.INFO)
    273   analyzer = TsanAnalyzer(options.source_dir, use_gdb=True)
    274   return analyzer.Report(filenames, None)
    275 
    276 
    277 if __name__ == '__main__':
    278   sys.exit(main())
    279