1 #!/usr/bin/env python 2 3 """ 4 Static Analyzer qualification infrastructure. 5 6 The goal is to test the analyzer against different projects, check for failures, 7 compare results, and measure performance. 8 9 Repository Directory will contain sources of the projects as well as the 10 information on how to build them and the expected output. 11 Repository Directory structure: 12 - ProjectMap file 13 - Historical Performance Data 14 - Project Dir1 15 - ReferenceOutput 16 - Project Dir2 17 - ReferenceOutput 18 .. 19 Note that the build tree must be inside the project dir. 20 21 To test the build of the analyzer one would: 22 - Copy over a copy of the Repository Directory. (TODO: Prefer to ensure that 23 the build directory does not pollute the repository to min network traffic). 24 - Build all projects, until error. Produce logs to report errors. 25 - Compare results. 26 27 The files which should be kept around for failure investigations: 28 RepositoryCopy/Project DirI/ScanBuildResults 29 RepositoryCopy/Project DirI/run_static_analyzer.log 30 31 Assumptions (TODO: shouldn't need to assume these.): 32 The script is being run from the Repository Directory. 33 The compiler for scan-build and scan-build are in the PATH. 34 export PATH=/Users/zaks/workspace/c2llvm/build/Release+Asserts/bin:$PATH 35 36 For more logging, set the env variables: 37 zaks:TI zaks$ export CCC_ANALYZER_LOG=1 38 zaks:TI zaks$ export CCC_ANALYZER_VERBOSE=1 39 40 The list of checkers tested are hardcoded in the Checkers variable. 41 For testing additional checkers, use the SA_ADDITIONAL_CHECKERS environment 42 variable. It should contain a comma separated list. 43 """ 44 import CmpRuns 45 46 import os 47 import csv 48 import sys 49 import glob 50 import math 51 import shutil 52 import time 53 import plistlib 54 import argparse 55 from subprocess import check_call, check_output, CalledProcessError 56 57 #------------------------------------------------------------------------------ 58 # Helper functions. 59 #------------------------------------------------------------------------------ 60 61 def detectCPUs(): 62 """ 63 Detects the number of CPUs on a system. Cribbed from pp. 64 """ 65 # Linux, Unix and MacOS: 66 if hasattr(os, "sysconf"): 67 if os.sysconf_names.has_key("SC_NPROCESSORS_ONLN"): 68 # Linux & Unix: 69 ncpus = os.sysconf("SC_NPROCESSORS_ONLN") 70 if isinstance(ncpus, int) and ncpus > 0: 71 return ncpus 72 else: # OSX: 73 return int(capture(['sysctl', '-n', 'hw.ncpu'])) 74 # Windows: 75 if os.environ.has_key("NUMBER_OF_PROCESSORS"): 76 ncpus = int(os.environ["NUMBER_OF_PROCESSORS"]) 77 if ncpus > 0: 78 return ncpus 79 return 1 # Default 80 81 def which(command, paths = None): 82 """which(command, [paths]) - Look up the given command in the paths string 83 (or the PATH environment variable, if unspecified).""" 84 85 if paths is None: 86 paths = os.environ.get('PATH','') 87 88 # Check for absolute match first. 89 if os.path.exists(command): 90 return command 91 92 # Would be nice if Python had a lib function for this. 93 if not paths: 94 paths = os.defpath 95 96 # Get suffixes to search. 97 # On Cygwin, 'PATHEXT' may exist but it should not be used. 98 if os.pathsep == ';': 99 pathext = os.environ.get('PATHEXT', '').split(';') 100 else: 101 pathext = [''] 102 103 # Search the paths... 104 for path in paths.split(os.pathsep): 105 for ext in pathext: 106 p = os.path.join(path, command + ext) 107 if os.path.exists(p): 108 return p 109 110 return None 111 112 # Make sure we flush the output after every print statement. 113 class flushfile(object): 114 def __init__(self, f): 115 self.f = f 116 def write(self, x): 117 self.f.write(x) 118 self.f.flush() 119 120 sys.stdout = flushfile(sys.stdout) 121 122 def getProjectMapPath(): 123 ProjectMapPath = os.path.join(os.path.abspath(os.curdir), 124 ProjectMapFile) 125 if not os.path.exists(ProjectMapPath): 126 print "Error: Cannot find the Project Map file " + ProjectMapPath +\ 127 "\nRunning script for the wrong directory?" 128 sys.exit(-1) 129 return ProjectMapPath 130 131 def getProjectDir(ID): 132 return os.path.join(os.path.abspath(os.curdir), ID) 133 134 def getSBOutputDirName(IsReferenceBuild) : 135 if IsReferenceBuild == True : 136 return SBOutputDirReferencePrefix + SBOutputDirName 137 else : 138 return SBOutputDirName 139 140 #------------------------------------------------------------------------------ 141 # Configuration setup. 142 #------------------------------------------------------------------------------ 143 144 # Find Clang for static analysis. 145 Clang = which("clang", os.environ['PATH']) 146 if not Clang: 147 print "Error: cannot find 'clang' in PATH" 148 sys.exit(-1) 149 150 # Number of jobs. 151 Jobs = int(math.ceil(detectCPUs() * 0.75)) 152 153 # Project map stores info about all the "registered" projects. 154 ProjectMapFile = "projectMap.csv" 155 156 # Names of the project specific scripts. 157 # The script that downloads the project. 158 DownloadScript = "download_project.sh" 159 # The script that needs to be executed before the build can start. 160 CleanupScript = "cleanup_run_static_analyzer.sh" 161 # This is a file containing commands for scan-build. 162 BuildScript = "run_static_analyzer.cmd" 163 164 # The log file name. 165 LogFolderName = "Logs" 166 BuildLogName = "run_static_analyzer.log" 167 # Summary file - contains the summary of the failures. Ex: This info can be be 168 # displayed when buildbot detects a build failure. 169 NumOfFailuresInSummary = 10 170 FailuresSummaryFileName = "failures.txt" 171 # Summary of the result diffs. 172 DiffsSummaryFileName = "diffs.txt" 173 174 # The scan-build result directory. 175 SBOutputDirName = "ScanBuildResults" 176 SBOutputDirReferencePrefix = "Ref" 177 178 # The name of the directory storing the cached project source. If this directory 179 # does not exist, the download script will be executed. That script should 180 # create the "CachedSource" directory and download the project source into it. 181 CachedSourceDirName = "CachedSource" 182 183 # The name of the directory containing the source code that will be analyzed. 184 # Each time a project is analyzed, a fresh copy of its CachedSource directory 185 # will be copied to the PatchedSource directory and then the local patches 186 # in PatchfileName will be applied (if PatchfileName exists). 187 PatchedSourceDirName = "PatchedSource" 188 189 # The name of the patchfile specifying any changes that should be applied 190 # to the CachedSource before analyzing. 191 PatchfileName = "changes_for_analyzer.patch" 192 193 # The list of checkers used during analyzes. 194 # Currently, consists of all the non-experimental checkers, plus a few alpha 195 # checkers we don't want to regress on. 196 Checkers="alpha.unix.SimpleStream,alpha.security.taint,cplusplus.NewDeleteLeaks,core,cplusplus,deadcode,security,unix,osx" 197 198 Verbose = 1 199 200 #------------------------------------------------------------------------------ 201 # Test harness logic. 202 #------------------------------------------------------------------------------ 203 204 # Run pre-processing script if any. 205 def runCleanupScript(Dir, PBuildLogFile): 206 Cwd = os.path.join(Dir, PatchedSourceDirName) 207 ScriptPath = os.path.join(Dir, CleanupScript) 208 runScript(ScriptPath, PBuildLogFile, Cwd) 209 210 # Run the script to download the project, if it exists. 211 def runDownloadScript(Dir, PBuildLogFile): 212 ScriptPath = os.path.join(Dir, DownloadScript) 213 runScript(ScriptPath, PBuildLogFile, Dir) 214 215 # Run the provided script if it exists. 216 def runScript(ScriptPath, PBuildLogFile, Cwd): 217 if os.path.exists(ScriptPath): 218 try: 219 if Verbose == 1: 220 print " Executing: %s" % (ScriptPath,) 221 check_call("chmod +x '%s'" % ScriptPath, cwd = Cwd, 222 stderr=PBuildLogFile, 223 stdout=PBuildLogFile, 224 shell=True) 225 check_call("'%s'" % ScriptPath, cwd = Cwd, stderr=PBuildLogFile, 226 stdout=PBuildLogFile, 227 shell=True) 228 except: 229 print "Error: Running %s failed. See %s for details." % (ScriptPath, 230 PBuildLogFile.name) 231 sys.exit(-1) 232 233 # Download the project and apply the local patchfile if it exists. 234 def downloadAndPatch(Dir, PBuildLogFile): 235 CachedSourceDirPath = os.path.join(Dir, CachedSourceDirName) 236 237 # If the we don't already have the cached source, run the project's 238 # download script to download it. 239 if not os.path.exists(CachedSourceDirPath): 240 runDownloadScript(Dir, PBuildLogFile) 241 if not os.path.exists(CachedSourceDirPath): 242 print "Error: '%s' not found after download." % (CachedSourceDirPath) 243 exit(-1) 244 245 PatchedSourceDirPath = os.path.join(Dir, PatchedSourceDirName) 246 247 # Remove potentially stale patched source. 248 if os.path.exists(PatchedSourceDirPath): 249 shutil.rmtree(PatchedSourceDirPath) 250 251 # Copy the cached source and apply any patches to the copy. 252 shutil.copytree(CachedSourceDirPath, PatchedSourceDirPath, symlinks=True) 253 applyPatch(Dir, PBuildLogFile) 254 255 def applyPatch(Dir, PBuildLogFile): 256 PatchfilePath = os.path.join(Dir, PatchfileName) 257 PatchedSourceDirPath = os.path.join(Dir, PatchedSourceDirName) 258 if not os.path.exists(PatchfilePath): 259 print " No local patches." 260 return 261 262 print " Applying patch." 263 try: 264 check_call("patch -p1 < '%s'" % (PatchfilePath), 265 cwd = PatchedSourceDirPath, 266 stderr=PBuildLogFile, 267 stdout=PBuildLogFile, 268 shell=True) 269 except: 270 print "Error: Patch failed. See %s for details." % (PBuildLogFile.name) 271 sys.exit(-1) 272 273 # Build the project with scan-build by reading in the commands and 274 # prefixing them with the scan-build options. 275 def runScanBuild(Dir, SBOutputDir, PBuildLogFile): 276 BuildScriptPath = os.path.join(Dir, BuildScript) 277 if not os.path.exists(BuildScriptPath): 278 print "Error: build script is not defined: %s" % BuildScriptPath 279 sys.exit(-1) 280 281 AllCheckers = Checkers 282 if os.environ.has_key('SA_ADDITIONAL_CHECKERS'): 283 AllCheckers = AllCheckers + ',' + os.environ['SA_ADDITIONAL_CHECKERS'] 284 285 # Run scan-build from within the patched source directory. 286 SBCwd = os.path.join(Dir, PatchedSourceDirName) 287 288 SBOptions = "--use-analyzer '%s' " % Clang 289 SBOptions += "-plist-html -o '%s' " % SBOutputDir 290 SBOptions += "-enable-checker " + AllCheckers + " " 291 SBOptions += "--keep-empty " 292 # Always use ccc-analyze to ensure that we can locate the failures 293 # directory. 294 SBOptions += "--override-compiler " 295 try: 296 SBCommandFile = open(BuildScriptPath, "r") 297 SBPrefix = "scan-build " + SBOptions + " " 298 for Command in SBCommandFile: 299 Command = Command.strip() 300 if len(Command) == 0: 301 continue; 302 # If using 'make', auto imply a -jX argument 303 # to speed up analysis. xcodebuild will 304 # automatically use the maximum number of cores. 305 if (Command.startswith("make ") or Command == "make") and \ 306 "-j" not in Command: 307 Command += " -j%d" % Jobs 308 SBCommand = SBPrefix + Command 309 if Verbose == 1: 310 print " Executing: %s" % (SBCommand,) 311 check_call(SBCommand, cwd = SBCwd, stderr=PBuildLogFile, 312 stdout=PBuildLogFile, 313 shell=True) 314 except: 315 print "Error: scan-build failed. See ",PBuildLogFile.name,\ 316 " for details." 317 raise 318 319 def hasNoExtension(FileName): 320 (Root, Ext) = os.path.splitext(FileName) 321 if ((Ext == "")) : 322 return True 323 return False 324 325 def isValidSingleInputFile(FileName): 326 (Root, Ext) = os.path.splitext(FileName) 327 if ((Ext == ".i") | (Ext == ".ii") | 328 (Ext == ".c") | (Ext == ".cpp") | 329 (Ext == ".m") | (Ext == "")) : 330 return True 331 return False 332 333 # Get the path to the SDK for the given SDK name. Returns None if 334 # the path cannot be determined. 335 def getSDKPath(SDKName): 336 if which("xcrun") is None: 337 return None 338 339 Cmd = "xcrun --sdk " + SDKName + " --show-sdk-path" 340 return check_output(Cmd, shell=True).rstrip() 341 342 # Run analysis on a set of preprocessed files. 343 def runAnalyzePreprocessed(Dir, SBOutputDir, Mode): 344 if os.path.exists(os.path.join(Dir, BuildScript)): 345 print "Error: The preprocessed files project should not contain %s" % \ 346 BuildScript 347 raise Exception() 348 349 CmdPrefix = Clang + " -cc1 " 350 351 # For now, we assume the preprocessed files should be analyzed 352 # with the OS X SDK. 353 SDKPath = getSDKPath("macosx") 354 if SDKPath is not None: 355 CmdPrefix += "-isysroot " + SDKPath + " " 356 357 CmdPrefix += "-analyze -analyzer-output=plist -w " 358 CmdPrefix += "-analyzer-checker=" + Checkers +" -fcxx-exceptions -fblocks " 359 360 if (Mode == 2) : 361 CmdPrefix += "-std=c++11 " 362 363 PlistPath = os.path.join(Dir, SBOutputDir, "date") 364 FailPath = os.path.join(PlistPath, "failures"); 365 os.makedirs(FailPath); 366 367 for FullFileName in glob.glob(Dir + "/*"): 368 FileName = os.path.basename(FullFileName) 369 Failed = False 370 371 # Only run the analyzes on supported files. 372 if (hasNoExtension(FileName)): 373 continue 374 if (isValidSingleInputFile(FileName) == False): 375 print "Error: Invalid single input file %s." % (FullFileName,) 376 raise Exception() 377 378 # Build and call the analyzer command. 379 OutputOption = "-o '%s.plist' " % os.path.join(PlistPath, FileName) 380 Command = CmdPrefix + OutputOption + ("'%s'" % FileName) 381 LogFile = open(os.path.join(FailPath, FileName + ".stderr.txt"), "w+b") 382 try: 383 if Verbose == 1: 384 print " Executing: %s" % (Command,) 385 check_call(Command, cwd = Dir, stderr=LogFile, 386 stdout=LogFile, 387 shell=True) 388 except CalledProcessError, e: 389 print "Error: Analyzes of %s failed. See %s for details." \ 390 "Error code %d." % \ 391 (FullFileName, LogFile.name, e.returncode) 392 Failed = True 393 finally: 394 LogFile.close() 395 396 # If command did not fail, erase the log file. 397 if Failed == False: 398 os.remove(LogFile.name); 399 400 def getBuildLogPath(SBOutputDir): 401 return os.path.join(SBOutputDir, LogFolderName, BuildLogName) 402 403 def removeLogFile(SBOutputDir): 404 BuildLogPath = getBuildLogPath(SBOutputDir) 405 # Clean up the log file. 406 if (os.path.exists(BuildLogPath)) : 407 RmCommand = "rm '%s'" % BuildLogPath 408 if Verbose == 1: 409 print " Executing: %s" % (RmCommand,) 410 check_call(RmCommand, shell=True) 411 412 def buildProject(Dir, SBOutputDir, ProjectBuildMode, IsReferenceBuild): 413 TBegin = time.time() 414 415 BuildLogPath = getBuildLogPath(SBOutputDir) 416 print "Log file: %s" % (BuildLogPath,) 417 print "Output directory: %s" %(SBOutputDir, ) 418 419 removeLogFile(SBOutputDir) 420 421 # Clean up scan build results. 422 if (os.path.exists(SBOutputDir)) : 423 RmCommand = "rm -r '%s'" % SBOutputDir 424 if Verbose == 1: 425 print " Executing: %s" % (RmCommand,) 426 check_call(RmCommand, shell=True) 427 assert(not os.path.exists(SBOutputDir)) 428 os.makedirs(os.path.join(SBOutputDir, LogFolderName)) 429 430 # Open the log file. 431 PBuildLogFile = open(BuildLogPath, "wb+") 432 433 # Build and analyze the project. 434 try: 435 if (ProjectBuildMode == 1): 436 downloadAndPatch(Dir, PBuildLogFile) 437 runCleanupScript(Dir, PBuildLogFile) 438 runScanBuild(Dir, SBOutputDir, PBuildLogFile) 439 else: 440 runAnalyzePreprocessed(Dir, SBOutputDir, ProjectBuildMode) 441 442 if IsReferenceBuild : 443 runCleanupScript(Dir, PBuildLogFile) 444 445 # Make the absolute paths relative in the reference results. 446 for (DirPath, Dirnames, Filenames) in os.walk(SBOutputDir): 447 for F in Filenames: 448 if (not F.endswith('plist')): 449 continue 450 Plist = os.path.join(DirPath, F) 451 Data = plistlib.readPlist(Plist) 452 PathPrefix = Dir 453 if (ProjectBuildMode == 1): 454 PathPrefix = os.path.join(Dir, PatchedSourceDirName) 455 Paths = [SourceFile[len(PathPrefix)+1:]\ 456 if SourceFile.startswith(PathPrefix)\ 457 else SourceFile for SourceFile in Data['files']] 458 Data['files'] = Paths 459 plistlib.writePlist(Data, Plist) 460 461 finally: 462 PBuildLogFile.close() 463 464 print "Build complete (time: %.2f). See the log for more details: %s" % \ 465 ((time.time()-TBegin), BuildLogPath) 466 467 # A plist file is created for each call to the analyzer(each source file). 468 # We are only interested on the once that have bug reports, so delete the rest. 469 def CleanUpEmptyPlists(SBOutputDir): 470 for F in glob.glob(SBOutputDir + "/*/*.plist"): 471 P = os.path.join(SBOutputDir, F) 472 473 Data = plistlib.readPlist(P) 474 # Delete empty reports. 475 if not Data['files']: 476 os.remove(P) 477 continue 478 479 # Given the scan-build output directory, checks if the build failed 480 # (by searching for the failures directories). If there are failures, it 481 # creates a summary file in the output directory. 482 def checkBuild(SBOutputDir): 483 # Check if there are failures. 484 Failures = glob.glob(SBOutputDir + "/*/failures/*.stderr.txt") 485 TotalFailed = len(Failures); 486 if TotalFailed == 0: 487 CleanUpEmptyPlists(SBOutputDir) 488 Plists = glob.glob(SBOutputDir + "/*/*.plist") 489 print "Number of bug reports (non-empty plist files) produced: %d" %\ 490 len(Plists) 491 return; 492 493 # Create summary file to display when the build fails. 494 SummaryPath = os.path.join(SBOutputDir, LogFolderName, FailuresSummaryFileName) 495 if (Verbose > 0): 496 print " Creating the failures summary file %s" % (SummaryPath,) 497 498 SummaryLog = open(SummaryPath, "w+") 499 try: 500 SummaryLog.write("Total of %d failures discovered.\n" % (TotalFailed,)) 501 if TotalFailed > NumOfFailuresInSummary: 502 SummaryLog.write("See the first %d below.\n" 503 % (NumOfFailuresInSummary,)) 504 # TODO: Add a line "See the results folder for more." 505 506 FailuresCopied = NumOfFailuresInSummary 507 Idx = 0 508 for FailLogPathI in Failures: 509 if Idx >= NumOfFailuresInSummary: 510 break; 511 Idx += 1 512 SummaryLog.write("\n-- Error #%d -----------\n" % (Idx,)); 513 FailLogI = open(FailLogPathI, "r"); 514 try: 515 shutil.copyfileobj(FailLogI, SummaryLog); 516 finally: 517 FailLogI.close() 518 finally: 519 SummaryLog.close() 520 521 print "Error: analysis failed. See ", SummaryPath 522 sys.exit(-1) 523 524 # Auxiliary object to discard stdout. 525 class Discarder(object): 526 def write(self, text): 527 pass # do nothing 528 529 # Compare the warnings produced by scan-build. 530 # Strictness defines the success criteria for the test: 531 # 0 - success if there are no crashes or analyzer failure. 532 # 1 - success if there are no difference in the number of reported bugs. 533 # 2 - success if all the bug reports are identical. 534 def runCmpResults(Dir, Strictness = 0): 535 TBegin = time.time() 536 537 RefDir = os.path.join(Dir, SBOutputDirReferencePrefix + SBOutputDirName) 538 NewDir = os.path.join(Dir, SBOutputDirName) 539 540 # We have to go one level down the directory tree. 541 RefList = glob.glob(RefDir + "/*") 542 NewList = glob.glob(NewDir + "/*") 543 544 # Log folders are also located in the results dir, so ignore them. 545 RefLogDir = os.path.join(RefDir, LogFolderName) 546 if RefLogDir in RefList: 547 RefList.remove(RefLogDir) 548 NewList.remove(os.path.join(NewDir, LogFolderName)) 549 550 if len(RefList) == 0 or len(NewList) == 0: 551 return False 552 assert(len(RefList) == len(NewList)) 553 554 # There might be more then one folder underneath - one per each scan-build 555 # command (Ex: one for configure and one for make). 556 if (len(RefList) > 1): 557 # Assume that the corresponding folders have the same names. 558 RefList.sort() 559 NewList.sort() 560 561 # Iterate and find the differences. 562 NumDiffs = 0 563 PairList = zip(RefList, NewList) 564 for P in PairList: 565 RefDir = P[0] 566 NewDir = P[1] 567 568 assert(RefDir != NewDir) 569 if Verbose == 1: 570 print " Comparing Results: %s %s" % (RefDir, NewDir) 571 572 DiffsPath = os.path.join(NewDir, DiffsSummaryFileName) 573 PatchedSourceDirPath = os.path.join(Dir, PatchedSourceDirName) 574 Opts = CmpRuns.CmpOptions(DiffsPath, "", PatchedSourceDirPath) 575 # Discard everything coming out of stdout (CmpRun produces a lot of them). 576 OLD_STDOUT = sys.stdout 577 sys.stdout = Discarder() 578 # Scan the results, delete empty plist files. 579 NumDiffs, ReportsInRef, ReportsInNew = \ 580 CmpRuns.dumpScanBuildResultsDiff(RefDir, NewDir, Opts, False) 581 sys.stdout = OLD_STDOUT 582 if (NumDiffs > 0) : 583 print "Warning: %r differences in diagnostics. See %s" % \ 584 (NumDiffs, DiffsPath,) 585 if Strictness >= 2 and NumDiffs > 0: 586 print "Error: Diffs found in strict mode (2)." 587 sys.exit(-1) 588 elif Strictness >= 1 and ReportsInRef != ReportsInNew: 589 print "Error: The number of results are different in strict mode (1)." 590 sys.exit(-1) 591 592 print "Diagnostic comparison complete (time: %.2f)." % (time.time()-TBegin) 593 return (NumDiffs > 0) 594 595 def cleanupReferenceResults(SBOutputDir): 596 # Delete html, css, and js files from reference results. These can 597 # include multiple copies of the benchmark source and so get very large. 598 Extensions = ["html", "css", "js"] 599 for E in Extensions: 600 for F in glob.glob("%s/*/*.%s" % (SBOutputDir, E)): 601 P = os.path.join(SBOutputDir, F) 602 RmCommand = "rm '%s'" % P 603 check_call(RmCommand, shell=True) 604 605 # Remove the log file. It leaks absolute path names. 606 removeLogFile(SBOutputDir) 607 608 def updateSVN(Mode, ProjectsMap): 609 try: 610 ProjectsMap.seek(0) 611 for I in csv.reader(ProjectsMap): 612 ProjName = I[0] 613 Path = os.path.join(ProjName, getSBOutputDirName(True)) 614 615 if Mode == "delete": 616 Command = "svn delete '%s'" % (Path,) 617 else: 618 Command = "svn add '%s'" % (Path,) 619 620 if Verbose == 1: 621 print " Executing: %s" % (Command,) 622 check_call(Command, shell=True) 623 624 if Mode == "delete": 625 CommitCommand = "svn commit -m \"[analyzer tests] Remove " \ 626 "reference results.\"" 627 else: 628 CommitCommand = "svn commit -m \"[analyzer tests] Add new " \ 629 "reference results.\"" 630 if Verbose == 1: 631 print " Executing: %s" % (CommitCommand,) 632 check_call(CommitCommand, shell=True) 633 except: 634 print "Error: SVN update failed." 635 sys.exit(-1) 636 637 def testProject(ID, ProjectBuildMode, IsReferenceBuild=False, Dir=None, Strictness = 0): 638 print " \n\n--- Building project %s" % (ID,) 639 640 TBegin = time.time() 641 642 if Dir is None : 643 Dir = getProjectDir(ID) 644 if Verbose == 1: 645 print " Build directory: %s." % (Dir,) 646 647 # Set the build results directory. 648 RelOutputDir = getSBOutputDirName(IsReferenceBuild) 649 SBOutputDir = os.path.join(Dir, RelOutputDir) 650 651 buildProject(Dir, SBOutputDir, ProjectBuildMode, IsReferenceBuild) 652 653 checkBuild(SBOutputDir) 654 655 if IsReferenceBuild == False: 656 runCmpResults(Dir, Strictness) 657 else: 658 cleanupReferenceResults(SBOutputDir) 659 660 print "Completed tests for project %s (time: %.2f)." % \ 661 (ID, (time.time()-TBegin)) 662 663 def testAll(IsReferenceBuild = False, UpdateSVN = False, Strictness = 0): 664 PMapFile = open(getProjectMapPath(), "rb") 665 try: 666 # Validate the input. 667 for I in csv.reader(PMapFile): 668 if (len(I) != 2) : 669 print "Error: Rows in the ProjectMapFile should have 3 entries." 670 raise Exception() 671 if (not ((I[1] == "0") | (I[1] == "1") | (I[1] == "2"))): 672 print "Error: Second entry in the ProjectMapFile should be 0" \ 673 " (single file), 1 (project), or 2(single file c++11)." 674 raise Exception() 675 676 # When we are regenerating the reference results, we might need to 677 # update svn. Remove reference results from SVN. 678 if UpdateSVN == True: 679 assert(IsReferenceBuild == True); 680 updateSVN("delete", PMapFile); 681 682 # Test the projects. 683 PMapFile.seek(0) 684 for I in csv.reader(PMapFile): 685 testProject(I[0], int(I[1]), IsReferenceBuild, None, Strictness) 686 687 # Add reference results to SVN. 688 if UpdateSVN == True: 689 updateSVN("add", PMapFile); 690 691 except: 692 print "Error occurred. Premature termination." 693 raise 694 finally: 695 PMapFile.close() 696 697 if __name__ == '__main__': 698 # Parse command line arguments. 699 Parser = argparse.ArgumentParser(description='Test the Clang Static Analyzer.') 700 Parser.add_argument('--strictness', dest='strictness', type=int, default=0, 701 help='0 to fail on runtime errors, 1 to fail when the number\ 702 of found bugs are different from the reference, 2 to \ 703 fail on any difference from the reference. Default is 0.') 704 Parser.add_argument('-r', dest='regenerate', action='store_true', default=False, 705 help='Regenerate reference output.') 706 Parser.add_argument('-rs', dest='update_reference', action='store_true', 707 default=False, help='Regenerate reference output and update svn.') 708 Args = Parser.parse_args() 709 710 IsReference = False 711 UpdateSVN = False 712 Strictness = Args.strictness 713 if Args.regenerate: 714 IsReference = True 715 elif Args.update_reference: 716 IsReference = True 717 UpdateSVN = True 718 719 testAll(IsReference, UpdateSVN, Strictness) 720