1 // 2 // The LLVM Compiler Infrastructure 3 // 4 // This file is distributed under the University of Illinois Open Source 5 // License. See LICENSE.TXT for details. 6 7 // 8 // testfilerunner.m 9 // testObjects 10 // 11 // Created by Blaine Garst on 9/24/08. 12 // 13 14 #import "testfilerunner.h" 15 #import <Foundation/Foundation.h> 16 #include <stdio.h> 17 #include <unistd.h> 18 #include <fcntl.h> 19 #include <string.h> 20 #include <stdlib.h> 21 #include <stdbool.h> 22 23 bool Everything = false; // do it also with 3 levels of optimization 24 bool DoClang = false; 25 26 static bool isDirectory(char *path); 27 static bool isExecutable(char *path); 28 static bool isYounger(char *source, char *binary); 29 static bool readErrorFile(char *buffer, const char *from); 30 31 __strong char *gcstrcpy2(__strong const char *arg, char *endp) { 32 unsigned size = endp - arg + 1; 33 __strong char *result = NSAllocateCollectable(size, 0); 34 strncpy(result, arg, size); 35 result[size-1] = 0; 36 return result; 37 } 38 __strong char *gcstrcpy1(__strong char *arg) { 39 unsigned size = strlen(arg) + 1; 40 __strong char *result = NSAllocateCollectable(size, 0); 41 strncpy(result, arg, size); 42 result[size-1] = 0; 43 return result; 44 } 45 46 @implementation TestFileExe 47 48 @synthesize options, compileLine, shouldFail, binaryName, sourceName; 49 @synthesize generator; 50 @synthesize libraryPath, frameworkPath; 51 52 - (NSString *)description { 53 NSMutableString *result = [NSMutableString new]; 54 if (shouldFail) [result appendString:@"fail"]; 55 for (id x in compileLine) { 56 [result appendString:[NSString stringWithFormat:@" %s", (char *)x]]; 57 } 58 return result; 59 } 60 61 - (__strong char *)radar { 62 return generator.radar; 63 } 64 65 - (bool) compileUnlessExists:(bool)skip { 66 if (shouldFail) { 67 printf("don't use this to compile anymore!\n"); 68 return false; 69 } 70 if (skip && isExecutable(binaryName) && !isYounger(sourceName, binaryName)) return true; 71 int argc = [compileLine count]; 72 char *argv[argc+1]; 73 for (int i = 0; i < argc; ++i) 74 argv[i] = (char *)[compileLine pointerAtIndex:i]; 75 argv[argc] = NULL; 76 pid_t child = fork(); 77 if (child == 0) { 78 execv(argv[0], argv); 79 exit(10); // shouldn't happen 80 } 81 if (child < 0) { 82 printf("fork failed\n"); 83 return false; 84 } 85 int status = 0; 86 pid_t deadchild = wait(&status); 87 if (deadchild != child) { 88 printf("wait got %d instead of %d\n", deadchild, child); 89 exit(1); 90 } 91 if (WEXITSTATUS(status) == 0) { 92 return true; 93 } 94 printf("run failed\n"); 95 return false; 96 } 97 98 bool lookforIn(char *lookfor, const char *format, pid_t child) { 99 char buffer[512]; 100 char got[512]; 101 sprintf(buffer, format, child); 102 bool gotOutput = readErrorFile(got, buffer); 103 if (!gotOutput) { 104 printf("**** didn't get an output file %s to analyze!!??\n", buffer); 105 return false; 106 } 107 char *where = strstr(got, lookfor); 108 if (!where) { 109 printf("didn't find '%s' in output file %s\n", lookfor, buffer); 110 return false; 111 } 112 unlink(buffer); 113 return true; 114 } 115 116 - (bool) compileWithExpectedFailure { 117 if (!shouldFail) { 118 printf("Why am I being called?\n"); 119 return false; 120 } 121 int argc = [compileLine count]; 122 char *argv[argc+1]; 123 for (int i = 0; i < argc; ++i) 124 argv[i] = (char *)[compileLine pointerAtIndex:i]; 125 argv[argc] = NULL; 126 pid_t child = fork(); 127 char buffer[512]; 128 if (child == 0) { 129 // in child 130 sprintf(buffer, "/tmp/errorfile_%d", getpid()); 131 close(1); 132 int fd = creat(buffer, 0777); 133 if (fd != 1) { 134 fprintf(stderr, "didn't open custom error file %s as 1, got %d\n", buffer, fd); 135 exit(1); 136 } 137 close(2); 138 dup(1); 139 int result = execv(argv[0], argv); 140 exit(10); 141 } 142 if (child < 0) { 143 printf("fork failed\n"); 144 return false; 145 } 146 int status = 0; 147 pid_t deadchild = wait(&status); 148 if (deadchild != child) { 149 printf("wait got %d instead of %d\n", deadchild, child); 150 exit(11); 151 } 152 if (WIFEXITED(status)) { 153 if (WEXITSTATUS(status) == 0) { 154 return false; 155 } 156 } 157 else { 158 printf("***** compiler borked/ICEd/died unexpectedly (status %x)\n", status); 159 return false; 160 } 161 char *error = generator.errorString; 162 163 if (!error) return true; 164 #if 0 165 char got[512]; 166 sprintf(buffer, "/tmp/errorfile_%d", child); 167 bool gotOutput = readErrorFile(got, buffer); 168 if (!gotOutput) { 169 printf("**** didn't get an error file %s to analyze!!??\n", buffer); 170 return false; 171 } 172 char *where = strstr(got, error); 173 if (!where) { 174 printf("didn't find '%s' in error file %s\n", error, buffer); 175 return false; 176 } 177 unlink(buffer); 178 #else 179 if (!lookforIn(error, "/tmp/errorfile_%d", child)) return false; 180 #endif 181 return true; 182 } 183 184 - (bool) run { 185 if (shouldFail) return true; 186 if (sizeof(long) == 4 && options & Do64) { 187 return true; // skip 64-bit tests 188 } 189 int argc = 1; 190 char *argv[argc+1]; 191 argv[0] = binaryName; 192 argv[argc] = NULL; 193 pid_t child = fork(); 194 if (child == 0) { 195 // set up environment 196 char lpath[1024]; 197 char fpath[1024]; 198 char *myenv[3]; 199 int counter = 0; 200 if (libraryPath) { 201 sprintf(lpath, "DYLD_LIBRARY_PATH=%s", libraryPath); 202 myenv[counter++] = lpath; 203 } 204 if (frameworkPath) { 205 sprintf(fpath, "DYLD_FRAMEWORK_PATH=%s", frameworkPath); 206 myenv[counter++] = fpath; 207 } 208 myenv[counter] = NULL; 209 if (generator.warningString) { 210 // set up stdout/stderr 211 char outfile[1024]; 212 sprintf(outfile, "/tmp/stdout_%d", getpid()); 213 close(2); 214 close(1); 215 creat(outfile, 0700); 216 dup(1); 217 } 218 execve(argv[0], argv, myenv); 219 exit(10); // shouldn't happen 220 } 221 if (child < 0) { 222 printf("fork failed\n"); 223 return false; 224 } 225 int status = 0; 226 pid_t deadchild = wait(&status); 227 if (deadchild != child) { 228 printf("wait got %d instead of %d\n", deadchild, child); 229 exit(1); 230 } 231 if (WIFEXITED(status) && WEXITSTATUS(status) == 0) { 232 if (generator.warningString) { 233 if (!lookforIn(generator.warningString, "/tmp/stdout_%d", child)) return false; 234 } 235 return true; 236 } 237 printf("**** run failed for %s\n", binaryName); 238 return false; 239 } 240 241 @end 242 243 @implementation TestFileExeGenerator 244 @synthesize filename, compilerPath, errorString; 245 @synthesize hasObjC, hasRR, hasGC, hasCPlusPlus, wantsC99, supposedToNotCompile, open, wants32, wants64; 246 @synthesize radar; 247 @synthesize warningString; 248 249 - (void)setFilename:(__strong char *)name { 250 filename = gcstrcpy1(name); 251 } 252 - (void)setCompilerPath:(__strong char *)name { 253 compilerPath = gcstrcpy1(name); 254 } 255 256 - (void)forMostThings:(NSMutableArray *)lines options:(int)options { 257 TestFileExe *item = nil; 258 item = [self lineForOptions:options]; 259 if (item) [lines addObject:item]; 260 item = [self lineForOptions:options|Do64]; 261 if (item) [lines addObject:item]; 262 item = [self lineForOptions:options|DoCPP]; 263 if (item) [lines addObject:item]; 264 item = [self lineForOptions:options|Do64|DoCPP]; 265 if (item) [lines addObject:item]; 266 } 267 268 /* 269 DoDashG = (1 << 8), 270 DoDashO = (1 << 9), 271 DoDashOs = (1 << 10), 272 DoDashO2 = (1 << 11), 273 */ 274 275 - (void)forAllThings:(NSMutableArray *)lines options:(int)options { 276 [self forMostThings:lines options:options]; 277 if (!Everything) { 278 return; 279 } 280 // now do it with three explicit optimization flags 281 [self forMostThings:lines options:options | DoDashO]; 282 [self forMostThings:lines options:options | DoDashOs]; 283 [self forMostThings:lines options:options | DoDashO2]; 284 } 285 286 - (NSArray *)allLines { 287 NSMutableArray *result = [NSMutableArray new]; 288 TestFileExe *item = nil; 289 290 int options = 0; 291 [self forAllThings:result options:0]; 292 [self forAllThings:result options:DoOBJC | DoRR]; 293 [self forAllThings:result options:DoOBJC | DoGC]; 294 [self forAllThings:result options:DoOBJC | DoGCRR]; 295 //[self forAllThings:result options:DoOBJC | DoRRGC]; 296 297 return result; 298 } 299 300 - (void)addLibrary:(const char *)dashLSomething { 301 if (!extraLibraries) { 302 extraLibraries = [NSPointerArray pointerArrayWithOptions: 303 NSPointerFunctionsStrongMemory | 304 NSPointerFunctionsCStringPersonality]; 305 } 306 [extraLibraries addPointer:(void *)dashLSomething]; 307 } 308 309 - (TestFileExe *)lineForOptions:(int)options { // nil if no can do 310 if (hasObjC && !(options & DoOBJC)) return nil; 311 if (hasCPlusPlus && !(options & DoCPP)) return nil; 312 if (hasObjC) { 313 if (!hasGC && (options & (DoGC|DoGCRR))) return nil; // not smart enough 314 if (!hasRR && (options & (DoRR|DoRRGC))) return nil; 315 } 316 NSPointerArray *pa = [NSPointerArray pointerArrayWithOptions: 317 NSPointerFunctionsStrongMemory | 318 NSPointerFunctionsCStringPersonality]; 319 // construct path 320 char path[512]; 321 path[0] = 0; 322 if (!compilerPath) compilerPath = "/usr/bin"; 323 if (compilerPath) { 324 strcat(path, compilerPath); 325 strcat(path, "/"); 326 } 327 if (options & DoCPP) { 328 strcat(path, DoClang ? "clang++" : "g++-4.2"); 329 } 330 else { 331 strcat(path, DoClang ? "clang" : "gcc-4.2"); 332 } 333 [pa addPointer:gcstrcpy1(path)]; 334 if (options & DoOBJC) { 335 if (options & DoCPP) { 336 [pa addPointer:"-ObjC++"]; 337 } 338 else { 339 [pa addPointer:"-ObjC"]; 340 } 341 } 342 [pa addPointer:"-g"]; 343 if (options & DoDashO) [pa addPointer:"-O"]; 344 else if (options & DoDashO2) [pa addPointer:"-O2"]; 345 else if (options & DoDashOs) [pa addPointer:"-Os"]; 346 if (wantsC99 && (! (options & DoCPP))) { 347 [pa addPointer:"-std=c99"]; 348 [pa addPointer:"-fblocks"]; 349 } 350 [pa addPointer:"-arch"]; 351 [pa addPointer: (options & Do64) ? "x86_64" : "i386"]; 352 353 if (options & DoOBJC) { 354 switch (options & (DoRR|DoGC|DoGCRR|DoRRGC)) { 355 case DoRR: 356 break; 357 case DoGC: 358 [pa addPointer:"-fobjc-gc-only"]; 359 break; 360 case DoGCRR: 361 [pa addPointer:"-fobjc-gc"]; 362 break; 363 case DoRRGC: 364 printf("DoRRGC unsupported right now\n"); 365 [pa addPointer:"-c"]; 366 return nil; 367 } 368 [pa addPointer:"-framework"]; 369 [pa addPointer:"Foundation"]; 370 } 371 [pa addPointer:gcstrcpy1(filename)]; 372 [pa addPointer:"-o"]; 373 374 path[0] = 0; 375 strcat(path, filename); 376 strcat(path, "."); 377 strcat(path, (options & Do64) ? "64" : "32"); 378 if (options & DoOBJC) { 379 switch (options & (DoRR|DoGC|DoGCRR|DoRRGC)) { 380 case DoRR: strcat(path, "-rr"); break; 381 case DoGC: strcat(path, "-gconly"); break; 382 case DoGCRR: strcat(path, "-gcrr"); break; 383 case DoRRGC: strcat(path, "-rrgc"); break; 384 } 385 } 386 if (options & DoCPP) strcat(path, "++"); 387 if (options & DoDashO) strcat(path, "-O"); 388 else if (options & DoDashO2) strcat(path, "-O2"); 389 else if (options & DoDashOs) strcat(path, "-Os"); 390 if (wantsC99) strcat(path, "-C99"); 391 strcat(path, DoClang ? "-clang" : "-gcc"); 392 strcat(path, "-bin"); 393 TestFileExe *result = [TestFileExe new]; 394 result.binaryName = gcstrcpy1(path); // could snarf copy in pa 395 [pa addPointer:result.binaryName]; 396 for (id cString in extraLibraries) { 397 [pa addPointer:cString]; 398 } 399 400 result.sourceName = gcstrcpy1(filename); // could snarf copy in pa 401 result.compileLine = pa; 402 result.options = options; 403 result.shouldFail = supposedToNotCompile; 404 result.generator = self; 405 return result; 406 } 407 408 + (NSArray *)generatorsFromPath:(NSString *)path { 409 FILE *fp = fopen([path fileSystemRepresentation], "r"); 410 if (fp == NULL) return nil; 411 NSArray *result = [self generatorsFromFILE:fp]; 412 fclose(fp); 413 return result; 414 } 415 416 #define LOOKFOR "CON" "FIG" 417 418 char *__strong parseRadar(char *line) { 419 line = strstr(line, "rdar:"); // returns beginning 420 char *endp = line + strlen("rdar:"); 421 while (*endp && *endp != ' ' && *endp != '\n') 422 ++endp; 423 return gcstrcpy2(line, endp); 424 } 425 426 - (void)parseLibraries:(const char *)line { 427 start: 428 line = strstr(line, "-l"); 429 char *endp = (char *)line + 2; 430 while (*endp && *endp != ' ' && *endp != '\n') 431 ++endp; 432 [self addLibrary:gcstrcpy2(line, endp)]; 433 if (strstr(endp, "-l")) { 434 line = endp; 435 goto start; 436 } 437 } 438 439 + (TestFileExeGenerator *)generatorFromLine:(char *)line filename:(char *)filename { 440 TestFileExeGenerator *item = [TestFileExeGenerator new]; 441 item.filename = gcstrcpy1(filename); 442 if (strstr(line, "GC")) item.hasGC = true; 443 if (strstr(line, "RR")) item.hasRR = true; 444 if (strstr(line, "C++")) item.hasCPlusPlus = true; 445 if (strstr(line, "-C99")) { 446 item.wantsC99 = true; 447 } 448 if (strstr(line, "64")) item.wants64 = true; 449 if (strstr(line, "32")) item.wants32 = true; 450 if (strstr(line, "-l")) [item parseLibraries:line]; 451 if (strstr(line, "open")) item.open = true; 452 if (strstr(line, "FAIL")) item.supposedToNotCompile = true; // old 453 // compile time error 454 if (strstr(line, "error:")) { 455 item.supposedToNotCompile = true; 456 // zap newline 457 char *error = strstr(line, "error:") + strlen("error:"); 458 // make sure we have something before the newline 459 char *newline = strstr(error, "\n"); 460 if (newline && ((newline-error) > 1)) { 461 *newline = 0; 462 item.errorString = gcstrcpy1(strstr(line, "error:") + strlen("error: ")); 463 } 464 } 465 // run time warning 466 if (strstr(line, "runtime:")) { 467 // zap newline 468 char *error = strstr(line, "runtime:") + strlen("runtime:"); 469 // make sure we have something before the newline 470 char *newline = strstr(error, "\n"); 471 if (newline && ((newline-error) > 1)) { 472 *newline = 0; 473 item.warningString = gcstrcpy1(strstr(line, "runtime:") + strlen("runtime:")); 474 } 475 } 476 if (strstr(line, "rdar:")) item.radar = parseRadar(line); 477 if (item.hasGC || item.hasRR) item.hasObjC = true; 478 if (!item.wants32 && !item.wants64) { // give them both if they ask for neither 479 item.wants32 = item.wants64 = true; 480 } 481 return item; 482 } 483 484 + (NSArray *)generatorsFromFILE:(FILE *)fp { 485 NSMutableArray *result = [NSMutableArray new]; 486 // pretend this is a grep LOOKFOR *.[cmCM][cmCM] input 487 // look for 488 // filename: ... LOOKFOR [GC] [RR] [C++] [FAIL ...] 489 char buf[512]; 490 while (fgets(buf, 512, fp)) { 491 char *config = strstr(buf, LOOKFOR); 492 if (!config) continue; 493 char *filename = buf; 494 char *end = strchr(buf, ':'); 495 *end = 0; 496 [result addObject:[self generatorFromLine:config filename:filename]]; 497 } 498 return result; 499 } 500 501 + (TestFileExeGenerator *)generatorFromFilename:(char *)filename { 502 FILE *fp = fopen(filename, "r"); 503 if (!fp) { 504 printf("didn't open %s!!\n", filename); 505 return nil; 506 } 507 char buf[512]; 508 while (fgets(buf, 512, fp)) { 509 char *config = strstr(buf, LOOKFOR); 510 if (!config) continue; 511 fclose(fp); 512 return [self generatorFromLine:config filename:filename]; 513 } 514 fclose(fp); 515 // guess from filename 516 char *ext = strrchr(filename, '.'); 517 if (!ext) return nil; 518 TestFileExeGenerator *result = [TestFileExeGenerator new]; 519 result.filename = gcstrcpy1(filename); 520 if (!strncmp(ext, ".m", 2)) { 521 result.hasObjC = true; 522 result.hasRR = true; 523 result.hasGC = true; 524 } 525 else if (!strcmp(ext, ".c")) { 526 ; 527 } 528 else if (!strcmp(ext, ".M") || !strcmp(ext, ".mm")) { 529 result.hasObjC = true; 530 result.hasRR = true; 531 result.hasGC = true; 532 result.hasCPlusPlus = true; 533 } 534 else if (!strcmp(ext, ".cc") 535 || !strcmp(ext, ".cp") 536 || !strcmp(ext, ".cxx") 537 || !strcmp(ext, ".cpp") 538 || !strcmp(ext, ".CPP") 539 || !strcmp(ext, ".c++") 540 || !strcmp(ext, ".C")) { 541 result.hasCPlusPlus = true; 542 } 543 else { 544 printf("unknown extension, file %s ignored\n", filename); 545 result = nil; 546 } 547 return result; 548 549 } 550 551 - (NSString *)description { 552 return [NSString stringWithFormat:@"%s: %s%s%s%s%s%s", 553 filename, 554 LOOKFOR, 555 hasGC ? " GC" : "", 556 hasRR ? " RR" : "", 557 hasCPlusPlus ? " C++" : "", 558 wantsC99 ? "C99" : "", 559 supposedToNotCompile ? " FAIL" : ""]; 560 } 561 562 @end 563 564 void printDetails(NSArray *failures, const char *whatAreThey) { 565 if ([failures count]) { 566 NSMutableString *output = [NSMutableString new]; 567 printf("%s:\n", whatAreThey); 568 for (TestFileExe *line in failures) { 569 printf("%s", line.binaryName); 570 char *radar = line.generator.radar; 571 if (radar) 572 printf(" (due to %s?),", radar); 573 printf(" recompile via:\n%s\n\n", line.description.UTF8String); 574 } 575 printf("\n"); 576 } 577 } 578 579 void help(const char *whoami) { 580 printf("Usage: %s [-fast] [-e] [-dyld librarypath] [gcc4.2dir] [-- | source1 ...]\n", whoami); 581 printf(" -fast don't recompile if binary younger than source\n"); 582 printf(" -open only run tests that are thought to still be unresolved\n"); 583 printf(" -clang use the clang and clang++ compilers\n"); 584 printf(" -e compile all variations also with -Os, -O2, -O3\n"); 585 printf(" -dyld p override DYLD_LIBRARY_PATH and DYLD_FRAMEWORK_PATH to p when running tests\n"); 586 printf(" <compilerpath> directory containing gcc-4.2 (or clang) that you wish to use to compile the tests\n"); 587 printf(" -- assume stdin is a grep CON" "FIG across the test sources\n"); 588 printf(" otherwise treat each remaining argument as a single test file source\n"); 589 printf("%s will compile and run individual test files under a variety of compilers, c, obj-c, c++, and objc++\n", whoami); 590 printf(" .c files are compiled with all four compilers\n"); 591 printf(" .m files are compiled with objc and objc++ compilers\n"); 592 printf(" .C files are compiled with c++ and objc++ compilers\n"); 593 printf(" .M files are compiled only with the objc++ compiler\n"); 594 printf("(actually all forms of extensions recognized by the compilers are honored, .cc, .c++ etc.)\n"); 595 printf("\nTest files should run to completion with no output and exit (return) 0 on success.\n"); 596 printf("Further they should be able to be compiled and run with GC on or off and by the C++ compilers\n"); 597 printf("A line containing the string CON" "FIG within the source enables restrictions to the above assumptions\n"); 598 printf("and other options.\n"); 599 printf("Following CON" "FIG the string\n"); 600 printf(" C++ restricts the test to only be run by c++ and objc++ compilers\n"); 601 printf(" GC restricts the test to only be compiled and run with GC on\n"); 602 printf(" RR (retain/release) restricts the test to only be compiled and run with GC off\n"); 603 printf("Additionally,\n"); 604 printf(" -C99 restricts the C versions of the test to -fstd=c99 -fblocks\n"); 605 printf(" -O adds the -O optimization level\n"); 606 printf(" -O2 adds the -O2 optimization level\n"); 607 printf(" -Os adds the -Os optimization level\n"); 608 printf("Files that are known to exhibit unresolved problems can provide the term \"open\" and this can"); 609 printf("in turn allow highlighting of fixes that have regressed as well as identify that fixes are now available.\n"); 610 printf("Files that exhibit known bugs may provide\n"); 611 printf(" rdar://whatever such that if they fail the rdar will get cited\n"); 612 printf("Files that are expected to fail to compile should provide, as their last token sequence,\n"); 613 printf(" error:\n"); 614 printf(" or error: substring to match.\n"); 615 printf("Files that are expected to produce a runtime error message should provide, as their last token sequence,\n"); 616 printf(" warning: string to match\n"); 617 printf("\n%s will compile and run all configurations of the test files and report a summary at the end. Good luck.\n", whoami); 618 printf(" Blaine Garst blaine (a] apple.com\n"); 619 } 620 621 int main(int argc, char *argv[]) { 622 printf("running on %s-bit architecture\n", sizeof(long) == 4 ? "32" : "64"); 623 char *compilerDir = "/usr/bin"; 624 NSMutableArray *generators = [NSMutableArray new]; 625 bool doFast = false; 626 bool doStdin = false; 627 bool onlyOpen = false; 628 char *libraryPath = getenv("DYLD_LIBRARY_PATH"); 629 char *frameworkPath = getenv("DYLD_FRAMEWORK_PATH"); 630 // process options 631 while (argc > 1) { 632 if (!strcmp(argv[1], "-fast")) { 633 doFast = true; 634 --argc; 635 ++argv; 636 } 637 else if (!strcmp(argv[1], "-dyld")) { 638 doFast = true; 639 --argc; 640 ++argv; 641 frameworkPath = argv[1]; 642 libraryPath = argv[1]; 643 --argc; 644 ++argv; 645 } 646 else if (!strcmp(argv[1], "-open")) { 647 onlyOpen = true; 648 --argc; 649 ++argv; 650 } 651 else if (!strcmp(argv[1], "-clang")) { 652 DoClang = true; 653 --argc; 654 ++argv; 655 } 656 else if (!strcmp(argv[1], "-e")) { 657 Everything = true; 658 --argc; 659 ++argv; 660 } 661 else if (!strcmp(argv[1], "--")) { 662 doStdin = true; 663 --argc; 664 ++argv; 665 } 666 else if (!strcmp(argv[1], "-")) { 667 help(argv[0]); 668 return 1; 669 } 670 else if (argc > 1 && isDirectory(argv[1])) { 671 compilerDir = argv[1]; 672 ++argv; 673 --argc; 674 } 675 else 676 break; 677 } 678 // process remaining arguments, or stdin 679 if (argc == 1) { 680 if (doStdin) 681 generators = (NSMutableArray *)[TestFileExeGenerator generatorsFromFILE:stdin]; 682 else { 683 help(argv[0]); 684 return 1; 685 } 686 } 687 else while (argc > 1) { 688 TestFileExeGenerator *generator = [TestFileExeGenerator generatorFromFilename:argv[1]]; 689 if (generator) [generators addObject:generator]; 690 ++argv; 691 --argc; 692 } 693 // see if we can generate all possibilities 694 NSMutableArray *failureToCompile = [NSMutableArray new]; 695 NSMutableArray *failureToFailToCompile = [NSMutableArray new]; 696 NSMutableArray *failureToRun = [NSMutableArray new]; 697 NSMutableArray *successes = [NSMutableArray new]; 698 for (TestFileExeGenerator *generator in generators) { 699 //NSLog(@"got %@", generator); 700 if (onlyOpen && !generator.open) { 701 //printf("skipping resolved test %s\n", generator.filename); 702 continue; // skip closed if onlyOpen 703 } 704 if (!onlyOpen && generator.open) { 705 //printf("skipping open test %s\n", generator.filename); 706 continue; // skip open if not asked for onlyOpen 707 } 708 generator.compilerPath = compilerDir; 709 NSArray *tests = [generator allLines]; 710 for (TestFileExe *line in tests) { 711 line.frameworkPath = frameworkPath; // tell generators about it instead XXX 712 line.libraryPath = libraryPath; // tell generators about it instead XXX 713 if ([line shouldFail]) { 714 if (doFast) continue; // don't recompile & don't count as success 715 if ([line compileWithExpectedFailure]) { 716 [successes addObject:line]; 717 } 718 else 719 [failureToFailToCompile addObject:line]; 720 } 721 else if ([line compileUnlessExists:doFast]) { 722 if ([line run]) { 723 printf("%s ran successfully\n", line.binaryName); 724 [successes addObject:line]; 725 } 726 else { 727 [failureToRun addObject:line]; 728 } 729 } 730 else { 731 [failureToCompile addObject:line]; 732 } 733 } 734 } 735 printf("\n--- results ---\n\n%lu successes\n%lu unexpected compile failures\n%lu failure to fail to compile errors\n%lu run failures\n", 736 [successes count], [failureToCompile count], [failureToFailToCompile count], [failureToRun count]); 737 printDetails(failureToCompile, "unexpected compile failures"); 738 printDetails(failureToFailToCompile, "should have failed to compile but didn't failures"); 739 printDetails(failureToRun, "run failures"); 740 741 if (onlyOpen && [successes count]) { 742 NSMutableSet *radars = [NSMutableSet new]; 743 printf("The following tests ran successfully suggesting that they are now resolved:\n"); 744 for (TestFileExe *line in successes) { 745 printf("%s\n", line.binaryName); 746 if (line.radar) [radars addObject:line.generator]; 747 } 748 if ([radars count]) { 749 printf("The following radars may be resolved:\n"); 750 for (TestFileExeGenerator *line in radars) { 751 printf("%s\n", line.radar); 752 } 753 } 754 } 755 756 return [failureToCompile count] + [failureToRun count]; 757 } 758 759 #include <sys/stat.h> 760 761 static bool isDirectory(char *path) { 762 struct stat statb; 763 int retval = stat(path, &statb); 764 if (retval != 0) return false; 765 if (statb.st_mode & S_IFDIR) return true; 766 return false; 767 } 768 769 static bool isExecutable(char *path) { 770 struct stat statb; 771 int retval = stat(path, &statb); 772 if (retval != 0) return false; 773 if (!(statb.st_mode & S_IFREG)) return false; 774 if (statb.st_mode & S_IXUSR) return true; 775 return false; 776 } 777 778 static bool isYounger(char *source, char *binary) { 779 struct stat statb; 780 int retval = stat(binary, &statb); 781 if (retval != 0) return true; // if doesn't exit, lie 782 783 struct stat stata; 784 retval = stat(source, &stata); 785 if (retval != 0) return true; // we're hosed 786 // the greater the timeval the younger it is 787 if (stata.st_mtimespec.tv_sec > statb.st_mtimespec.tv_sec) return true; 788 if (stata.st_mtimespec.tv_nsec > statb.st_mtimespec.tv_nsec) return true; 789 return false; 790 } 791 792 static bool readErrorFile(char *buffer, const char *from) { 793 int fd = open(from, 0); 794 if (fd < 0) { 795 printf("didn't open %s, (might not have been created?)\n", buffer); 796 return false; 797 } 798 int count = read(fd, buffer, 512); 799 if (count < 1) { 800 printf("read error on %s\n", buffer); 801 return false; 802 } 803 buffer[count-1] = 0; // zap newline 804 return true; 805 } 806
