Home | History | Annotate | Download | only in analyzer 
      1 #!/usr/bin/env python
      2 
      3 """
      4 Static Analyzer qualification infrastructure.
      5 
      6 The goal is to test the analyzer against different projects, check for failures,
      7 compare results, and measure performance.
      8 
      9 Repository Directory will contain sources of the projects as well as the 
     10 information on how to build them and the expected output. 
     11 Repository Directory structure:
     12    - ProjectMap file
     13    - Historical Performance Data
     14    - Project Dir1
     15      - ReferenceOutput
     16    - Project Dir2
     17      - ReferenceOutput
     18    ..
     19 
     20 To test the build of the analyzer one would:
     21    - Copy over a copy of the Repository Directory. (TODO: Prefer to ensure that 
     22      the build directory does not pollute the repository to min network traffic).
     23    - Build all projects, until error. Produce logs to report errors.
     24    - Compare results.  
     25 
     26 The files which should be kept around for failure investigations: 
     27    RepositoryCopy/Project DirI/ScanBuildResults
     28    RepositoryCopy/Project DirI/run_static_analyzer.log      
     29    
     30 Assumptions (TODO: shouldn't need to assume these.):   
     31    The script is being run from the Repository Directory.
     32    The compiler for scan-build and scan-build are in the PATH.
     33    export PATH=/Users/zaks/workspace/c2llvm/build/Release+Asserts/bin:$PATH
     34 
     35 For more logging, set the  env variables:
     36    zaks:TI zaks$ export CCC_ANALYZER_LOG=1
     37    zaks:TI zaks$ export CCC_ANALYZER_VERBOSE=1
     38 """
     39 import CmpRuns
     40 
     41 import os
     42 import csv
     43 import sys
     44 import glob
     45 import shutil
     46 import time
     47 import plistlib
     48 from subprocess import check_call, CalledProcessError
     49 
     50 # Project map stores info about all the "registered" projects.
     51 ProjectMapFile = "projectMap.csv"
     52 
     53 # Names of the project specific scripts.
     54 # The script that needs to be executed before the build can start.
     55 CleanupScript = "cleanup_run_static_analyzer.sh"
     56 # This is a file containing commands for scan-build.  
     57 BuildScript = "run_static_analyzer.cmd"
     58 
     59 # The log file name.
     60 LogFolderName = "Logs"
     61 BuildLogName = "run_static_analyzer.log"
     62 # Summary file - contains the summary of the failures. Ex: This info can be be  
     63 # displayed when buildbot detects a build failure.
     64 NumOfFailuresInSummary = 10
     65 FailuresSummaryFileName = "failures.txt"
     66 # Summary of the result diffs.
     67 DiffsSummaryFileName = "diffs.txt"
     68 
     69 # The scan-build result directory.
     70 SBOutputDirName = "ScanBuildResults"
     71 SBOutputDirReferencePrefix = "Ref"
     72 
     73 # The list of checkers used during analyzes.
     74 # Currently, consists of all the non experimental checkers.
     75 Checkers="experimental.security.taint,core,deadcode,cplusplus,security,unix,osx,cocoa"
     76 
     77 Verbose = 1
     78 
     79 IsReferenceBuild = False
     80 
     81 # Make sure we flush the output after every print statement.
     82 class flushfile(object):
     83     def __init__(self, f):
     84         self.f = f
     85     def write(self, x):
     86         self.f.write(x)
     87         self.f.flush()
     88 
     89 sys.stdout = flushfile(sys.stdout)
     90 
     91 def getProjectMapPath():
     92     ProjectMapPath = os.path.join(os.path.abspath(os.curdir), 
     93                                   ProjectMapFile)
     94     if not os.path.exists(ProjectMapPath):
     95         print "Error: Cannot find the Project Map file " + ProjectMapPath +\
     96                 "\nRunning script for the wrong directory?"
     97         sys.exit(-1)  
     98     return ProjectMapPath         
     99 
    100 def getProjectDir(ID):
    101     return os.path.join(os.path.abspath(os.curdir), ID)        
    102 
    103 def getSBOutputDirName() :
    104     if IsReferenceBuild == True :
    105         return SBOutputDirReferencePrefix + SBOutputDirName
    106     else :
    107         return SBOutputDirName
    108 
    109 # Run pre-processing script if any.
    110 def runCleanupScript(Dir, PBuildLogFile):
    111     ScriptPath = os.path.join(Dir, CleanupScript)
    112     if os.path.exists(ScriptPath):
    113         try:
    114             if Verbose == 1:        
    115                 print "  Executing: %s" % (ScriptPath,)
    116             check_call("chmod +x %s" % ScriptPath, cwd = Dir, 
    117                                               stderr=PBuildLogFile,
    118                                               stdout=PBuildLogFile, 
    119                                               shell=True)    
    120             check_call(ScriptPath, cwd = Dir, stderr=PBuildLogFile,
    121                                               stdout=PBuildLogFile, 
    122                                               shell=True)
    123         except:
    124             print "Error: The pre-processing step failed. See ", \
    125                   PBuildLogFile.name, " for details."
    126             sys.exit(-1)
    127 
    128 # Build the project with scan-build by reading in the commands and 
    129 # prefixing them with the scan-build options.
    130 def runScanBuild(Dir, SBOutputDir, PBuildLogFile):
    131     BuildScriptPath = os.path.join(Dir, BuildScript)
    132     if not os.path.exists(BuildScriptPath):
    133         print "Error: build script is not defined: %s" % BuildScriptPath
    134         sys.exit(-1)       
    135     SBOptions = "-plist-html -o " + SBOutputDir + " "
    136     SBOptions += "-enable-checker " + Checkers + " "  
    137     try:
    138         SBCommandFile = open(BuildScriptPath, "r")
    139         SBPrefix = "scan-build " + SBOptions + " "
    140         for Command in SBCommandFile:
    141             SBCommand = SBPrefix + Command
    142             if Verbose == 1:        
    143                 print "  Executing: %s" % (SBCommand,)
    144             check_call(SBCommand, cwd = Dir, stderr=PBuildLogFile,
    145                                              stdout=PBuildLogFile, 
    146                                              shell=True)
    147     except:
    148         print "Error: scan-build failed. See ",PBuildLogFile.name,\
    149               " for details."
    150         raise
    151 
    152 def hasNoExtension(FileName):
    153     (Root, Ext) = os.path.splitext(FileName)
    154     if ((Ext == "")) :
    155         return True
    156     return False
    157 
    158 def isValidSingleInputFile(FileName):
    159     (Root, Ext) = os.path.splitext(FileName)
    160     if ((Ext == ".i") | (Ext == ".ii") | 
    161         (Ext == ".c") | (Ext == ".cpp") | 
    162         (Ext == ".m") | (Ext == "")) :
    163         return True
    164     return False
    165 
    166 # Run analysis on a set of preprocessed files.
    167 def runAnalyzePreprocessed(Dir, SBOutputDir):
    168     if os.path.exists(os.path.join(Dir, BuildScript)):
    169         print "Error: The preprocessed files project should not contain %s" % \
    170                BuildScript
    171         raise Exception()       
    172 
    173     CmdPrefix = "clang -cc1 -analyze -analyzer-output=plist -w "
    174     CmdPrefix += "-analyzer-checker=" + Checkers +" -fcxx-exceptions -fblocks "   
    175     
    176     PlistPath = os.path.join(Dir, SBOutputDir, "date")
    177     FailPath = os.path.join(PlistPath, "failures");
    178     os.makedirs(FailPath);
    179  
    180     for FullFileName in glob.glob(Dir + "/*"):
    181         FileName = os.path.basename(FullFileName)
    182         Failed = False
    183         
    184         # Only run the analyzes on supported files.
    185         if (hasNoExtension(FileName)):
    186             continue
    187         if (isValidSingleInputFile(FileName) == False):
    188             print "Error: Invalid single input file %s." % (FullFileName,)
    189             raise Exception()
    190         
    191         # Build and call the analyzer command.
    192         OutputOption = "-o " + os.path.join(PlistPath, FileName) + ".plist "
    193         Command = CmdPrefix + OutputOption + os.path.join(Dir, FileName)
    194         LogFile = open(os.path.join(FailPath, FileName + ".stderr.txt"), "w+b")
    195         try:
    196             if Verbose == 1:        
    197                 print "  Executing: %s" % (Command,)
    198             check_call(Command, cwd = Dir, stderr=LogFile,
    199                                            stdout=LogFile, 
    200                                            shell=True)
    201         except CalledProcessError, e:
    202             print "Error: Analyzes of %s failed. See %s for details." \
    203                   "Error code %d." % \
    204                    (FullFileName, LogFile.name, e.returncode)
    205             Failed = True       
    206         finally:
    207             LogFile.close()            
    208         
    209         # If command did not fail, erase the log file.
    210         if Failed == False:
    211             os.remove(LogFile.name);
    212 
    213 def buildProject(Dir, SBOutputDir, IsScanBuild):
    214     TBegin = time.time() 
    215 
    216     BuildLogPath = os.path.join(SBOutputDir, LogFolderName, BuildLogName)
    217     print "Log file: %s" % (BuildLogPath,) 
    218     print "Output directory: %s" %(SBOutputDir, )
    219     
    220     # Clean up the log file.
    221     if (os.path.exists(BuildLogPath)) :
    222         RmCommand = "rm " + BuildLogPath
    223         if Verbose == 1:
    224             print "  Executing: %s" % (RmCommand,)
    225         check_call(RmCommand, shell=True)
    226     
    227     # Clean up scan build results.
    228     if (os.path.exists(SBOutputDir)) :
    229         RmCommand = "rm -r " + SBOutputDir
    230         if Verbose == 1: 
    231             print "  Executing: %s" % (RmCommand,)
    232             check_call(RmCommand, shell=True)
    233     assert(not os.path.exists(SBOutputDir))
    234     os.makedirs(os.path.join(SBOutputDir, LogFolderName))
    235         
    236     # Open the log file.
    237     PBuildLogFile = open(BuildLogPath, "wb+")
    238     
    239     # Build and analyze the project.
    240     try:
    241         runCleanupScript(Dir, PBuildLogFile)
    242         
    243         if IsScanBuild:
    244             runScanBuild(Dir, SBOutputDir, PBuildLogFile)
    245         else:
    246             runAnalyzePreprocessed(Dir, SBOutputDir)
    247         
    248         if IsReferenceBuild :
    249             runCleanupScript(Dir, PBuildLogFile)
    250            
    251     finally:
    252         PBuildLogFile.close()
    253         
    254     print "Build complete (time: %.2f). See the log for more details: %s" % \
    255            ((time.time()-TBegin), BuildLogPath) 
    256        
    257 # A plist file is created for each call to the analyzer(each source file).
    258 # We are only interested on the once that have bug reports, so delete the rest.        
    259 def CleanUpEmptyPlists(SBOutputDir):
    260     for F in glob.glob(SBOutputDir + "/*/*.plist"):
    261         P = os.path.join(SBOutputDir, F)
    262         
    263         Data = plistlib.readPlist(P)
    264         # Delete empty reports.
    265         if not Data['files']:
    266             os.remove(P)
    267             continue
    268 
    269 # Given the scan-build output directory, checks if the build failed 
    270 # (by searching for the failures directories). If there are failures, it 
    271 # creates a summary file in the output directory.         
    272 def checkBuild(SBOutputDir):
    273     # Check if there are failures.
    274     Failures = glob.glob(SBOutputDir + "/*/failures/*.stderr.txt")
    275     TotalFailed = len(Failures);
    276     if TotalFailed == 0:
    277         CleanUpEmptyPlists(SBOutputDir)
    278         Plists = glob.glob(SBOutputDir + "/*/*.plist")
    279         print "Number of bug reports (non empty plist files) produced: %d" %\
    280            len(Plists)
    281         return;
    282     
    283     # Create summary file to display when the build fails.
    284     SummaryPath = os.path.join(SBOutputDir, LogFolderName, FailuresSummaryFileName)
    285     if (Verbose > 0):
    286         print "  Creating the failures summary file %s" % (SummaryPath,)
    287     
    288     SummaryLog = open(SummaryPath, "w+")
    289     try:
    290         SummaryLog.write("Total of %d failures discovered.\n" % (TotalFailed,))
    291         if TotalFailed > NumOfFailuresInSummary:
    292             SummaryLog.write("See the first %d below.\n" 
    293                                                    % (NumOfFailuresInSummary,))
    294         # TODO: Add a line "See the results folder for more."
    295     
    296         FailuresCopied = NumOfFailuresInSummary
    297         Idx = 0
    298         for FailLogPathI in glob.glob(SBOutputDir + "/*/failures/*.stderr.txt"):
    299             if Idx >= NumOfFailuresInSummary:
    300                 break;
    301             Idx += 1 
    302             SummaryLog.write("\n-- Error #%d -----------\n" % (Idx,));
    303             FailLogI = open(FailLogPathI, "r");
    304             try: 
    305                 shutil.copyfileobj(FailLogI, SummaryLog);
    306             finally:
    307                 FailLogI.close()
    308     finally:
    309         SummaryLog.close()
    310     
    311     print "Error: analysis failed. See ", SummaryPath
    312     sys.exit(-1)       
    313 
    314 # Auxiliary object to discard stdout.
    315 class Discarder(object):
    316     def write(self, text):
    317         pass # do nothing
    318 
    319 # Compare the warnings produced by scan-build.
    320 def runCmpResults(Dir):   
    321     TBegin = time.time() 
    322 
    323     RefDir = os.path.join(Dir, SBOutputDirReferencePrefix + SBOutputDirName)
    324     NewDir = os.path.join(Dir, SBOutputDirName)
    325     
    326     # We have to go one level down the directory tree.
    327     RefList = glob.glob(RefDir + "/*") 
    328     NewList = glob.glob(NewDir + "/*")
    329     
    330     # Log folders are also located in the results dir, so ignore them. 
    331     RefList.remove(os.path.join(RefDir, LogFolderName))
    332     NewList.remove(os.path.join(NewDir, LogFolderName))
    333     
    334     if len(RefList) == 0 or len(NewList) == 0:
    335         return False
    336     assert(len(RefList) == len(NewList))
    337 
    338     # There might be more then one folder underneath - one per each scan-build 
    339     # command (Ex: one for configure and one for make).
    340     if (len(RefList) > 1):
    341         # Assume that the corresponding folders have the same names.
    342         RefList.sort()
    343         NewList.sort()
    344     
    345     # Iterate and find the differences.
    346     NumDiffs = 0
    347     PairList = zip(RefList, NewList)    
    348     for P in PairList:    
    349         RefDir = P[0] 
    350         NewDir = P[1]
    351     
    352         assert(RefDir != NewDir) 
    353         if Verbose == 1:        
    354             print "  Comparing Results: %s %s" % (RefDir, NewDir)
    355     
    356         DiffsPath = os.path.join(NewDir, DiffsSummaryFileName)
    357         Opts = CmpRuns.CmpOptions(DiffsPath)
    358         # Discard everything coming out of stdout (CmpRun produces a lot of them).
    359         OLD_STDOUT = sys.stdout
    360         sys.stdout = Discarder()
    361         # Scan the results, delete empty plist files.
    362         NumDiffs = CmpRuns.cmpScanBuildResults(RefDir, NewDir, Opts, False)
    363         sys.stdout = OLD_STDOUT
    364         if (NumDiffs > 0) :
    365             print "Warning: %r differences in diagnostics. See %s" % \
    366                   (NumDiffs, DiffsPath,)
    367                     
    368     print "Diagnostic comparison complete (time: %.2f)." % (time.time()-TBegin) 
    369     return (NumDiffs > 0)
    370     
    371 def updateSVN(Mode, ProjectsMap):
    372     try:
    373         ProjectsMap.seek(0)    
    374         for I in csv.reader(ProjectsMap):
    375             ProjName = I[0] 
    376             Path = os.path.join(ProjName, getSBOutputDirName())
    377     
    378             if Mode == "delete":
    379                 Command = "svn delete %s" % (Path,)
    380             else:
    381                 Command = "svn add %s" % (Path,)
    382 
    383             if Verbose == 1:        
    384                 print "  Executing: %s" % (Command,)
    385                 check_call(Command, shell=True)    
    386     
    387         if Mode == "delete":
    388             CommitCommand = "svn commit -m \"[analyzer tests] Remove " \
    389                             "reference results.\""     
    390         else:
    391             CommitCommand = "svn commit -m \"[analyzer tests] Add new " \
    392                             "reference results.\""
    393         if Verbose == 1:        
    394             print "  Executing: %s" % (CommitCommand,)
    395             check_call(CommitCommand, shell=True)    
    396     except:
    397         print "Error: SVN update failed."
    398         sys.exit(-1)
    399         
    400 def testProject(ID, IsScanBuild, Dir=None):
    401     print " \n\n--- Building project %s" % (ID,)
    402 
    403     TBegin = time.time() 
    404 
    405     if Dir is None :
    406         Dir = getProjectDir(ID)        
    407     if Verbose == 1:        
    408         print "  Build directory: %s." % (Dir,)
    409     
    410     # Set the build results directory.
    411     RelOutputDir = getSBOutputDirName()
    412     SBOutputDir = os.path.join(Dir, RelOutputDir)
    413                 
    414     buildProject(Dir, SBOutputDir, IsScanBuild)    
    415 
    416     checkBuild(SBOutputDir)
    417     
    418     if IsReferenceBuild == False:
    419         runCmpResults(Dir)
    420         
    421     print "Completed tests for project %s (time: %.2f)." % \
    422           (ID, (time.time()-TBegin))
    423     
    424 def testAll(InIsReferenceBuild = False, UpdateSVN = False):
    425     global IsReferenceBuild
    426     IsReferenceBuild = InIsReferenceBuild
    427 
    428     PMapFile = open(getProjectMapPath(), "rb")
    429     try:        
    430         # Validate the input.
    431         for I in csv.reader(PMapFile):
    432             if (len(I) != 2) :
    433                 print "Error: Rows in the ProjectMapFile should have 3 entries."
    434                 raise Exception()
    435             if (not ((I[1] == "1") | (I[1] == "0"))):
    436                 print "Error: Second entry in the ProjectMapFile should be 0 or 1."
    437                 raise Exception()              
    438 
    439         # When we are regenerating the reference results, we might need to 
    440         # update svn. Remove reference results from SVN.
    441         if UpdateSVN == True:
    442             assert(InIsReferenceBuild == True);
    443             updateSVN("delete",  PMapFile);
    444             
    445         # Test the projects.
    446         PMapFile.seek(0)    
    447         for I in csv.reader(PMapFile):
    448             testProject(I[0], int(I[1]))
    449 
    450         # Add reference results to SVN.
    451         if UpdateSVN == True:
    452             updateSVN("add",  PMapFile);
    453 
    454     except:
    455         print "Error occurred. Premature termination."
    456         raise                            
    457     finally:
    458         PMapFile.close()    
    459             
    460 if __name__ == '__main__':
    461     IsReference = False
    462     UpdateSVN = False
    463     if len(sys.argv) >= 2:
    464         if sys.argv[1] == "-r":
    465             IsReference = True
    466         elif sys.argv[1] == "-rs":
    467             IsReference = True
    468             UpdateSVN = True
    469         else:     
    470           print >> sys.stderr, 'Usage: ', sys.argv[0],\
    471                              '[-r|-rs]' \
    472                              'Use -r to regenerate reference output' \
    473                              'Use -rs to regenerate reference output and update svn'
    474 
    475     testAll(IsReference, UpdateSVN)
    476