Home | History | Annotate | Download | only in Scripts 
      1 #!/usr/bin/perl -w
      2 
      3 # Copyright (C) 2010, 2011 Apple Inc. All rights reserved.
      4 #
      5 # Redistribution and use in source and binary forms, with or without
      6 # modification, are permitted provided that the following conditions
      7 # are met:
      8 # 1. Redistributions of source code must retain the above copyright
      9 #    notice, this list of conditions and the following disclaimer.
     10 # 2. Redistributions in binary form must reproduce the above copyright
     11 #    notice, this list of conditions and the following disclaimer in the
     12 #    documentation and/or other materials provided with the distribution.
     13 #
     14 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
     15 # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
     16 # THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     17 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
     18 # BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     19 # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     20 # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     21 # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     22 # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     23 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
     24 # THE POSSIBILITY OF SUCH DAMAGE.
     25 
     26 # Features to add:
     27 #   - Command line option to run a single test.
     28 #   - Command line option to run all tests in a suite.
     29 
     30 use strict;
     31 use warnings;
     32 
     33 use File::Basename;
     34 use FindBin;
     35 use Getopt::Long qw(:config pass_through);
     36 use IPC::Open3;
     37 use lib $FindBin::Bin;
     38 use webkitdirs;
     39 use Term::ANSIColor qw(:constants);
     40 
     41 sub dumpAllTests();
     42 sub runAllTests();
     43 sub runAllTestsInSuite($);
     44 sub runTest($$);
     45 sub populateTests();
     46 sub buildTestTool();
     47 
     48 my $showHelp = 0;
     49 my $verbose = 0;
     50 my $dump = 0;
     51 
     52 my $programName = basename($0);
     53 my $usage = <<EOF;
     54 Usage: $programName [options]
     55   --help                Show this help message
     56   -v|--verbose          Verbose output
     57   -d|--dump-tests       Dump the names of testcases without running them
     58 EOF
     59 
     60 GetOptions(
     61     'help' => \$showHelp,
     62     'verbose|v' => \$verbose,
     63     'dump|d' => \$dump,
     64 );
     65 
     66 if ($showHelp) {
     67    print STDERR $usage;
     68    exit 1;
     69 }
     70 
     71 setConfiguration();
     72 buildTestTool();
     73 setPathForRunningWebKitApp(\%ENV);
     74 my %testsToRun = populateTests();
     75 
     76 if ($dump) {
     77     dumpAllTests();
     78     exit 0;
     79 }
     80 
     81 runAllTests();
     82 
     83 sub dumpAllTests()
     84 {
     85     print "Dumping test cases\n";
     86     print "------------------\n";
     87     for my $suite (keys %testsToRun) {
     88         print $suite . ":\n";
     89         print map { "    " . $_ . "\n" } @{ $testsToRun{$suite} };
     90     }
     91     print "------------------\n";
     92 }
     93 
     94 sub runAllTests()
     95 {
     96     my $anyFailures = 0;
     97     for my $suite (keys %testsToRun) {
     98         my $failed = runAllTestsInSuite($suite);
     99         if ($failed) {
    100             $anyFailures = 1;
    101         }
    102     }
    103     return $anyFailures;
    104 }
    105 
    106 sub runAllTestsInSuite($)
    107 {
    108     my ($suite) = @_;
    109     print "Suite: $suite\n";
    110 
    111     my $anyFailures = 0;
    112     for my $test (@{$testsToRun{$suite}}) {
    113         my $failed = runTest($suite, $test);
    114         if ($failed) {
    115             $anyFailures = 1;
    116         }
    117     }
    118     
    119     return $anyFailures;
    120 }
    121 
    122 sub runTest($$)
    123 {
    124     my ($suite, $testName) = @_;
    125     my $test = $suite . "/" . $testName;
    126 
    127     print "    Test: $testName -> ";
    128 
    129     my $result = 0;
    130     if (isAppleMacWebKit()) {
    131         my $productDir = productDir();
    132         $ENV{DYLD_FRAMEWORK_PATH} = $productDir;
    133         $ENV{WEBKIT_UNSET_DYLD_FRAMEWORK_PATH} = "YES";
    134         my $apiTesterPath = "$productDir/TestWebKitAPI";
    135 
    136         local *DEVNULL;
    137         my ($childIn, $childOut, $childErr);
    138         unless ($verbose) {
    139             open(DEVNULL, ">", File::Spec->devnull()) or die "Failed to open /dev/null";
    140             $childOut = ">&DEVNULL";
    141             $childErr = ">&DEVNULL";
    142         } else {
    143             $childOut = ">&STDOUT";
    144             $childErr = ">&STDERR";
    145         }
    146 
    147         my $pid;
    148         if (architecture()) {
    149             $pid = open3($childIn, $childOut, $childErr, "arch", "-" . architecture(), $apiTesterPath, $test, @ARGV) or die "Failed to run test: $test.";
    150         } else {
    151             $pid = open3($childIn, $childOut, $childErr, $apiTesterPath, $test, @ARGV) or die "Failed to run test: $test.";
    152         }
    153 
    154         close($childIn);
    155         close($childOut);
    156         close($childErr);
    157         close(DEVNULL) unless ($verbose);
    158 
    159         waitpid($pid, 0);
    160         $result = $?;
    161     } elsif (isAppleWinWebKit()) {
    162         my $apiTesterNameSuffix;
    163         if (configurationForVisualStudio() ne "Debug_All") {
    164             $apiTesterNameSuffix = "";
    165         } else {
    166             $apiTesterNameSuffix = "_debug";
    167         }
    168         my $apiTesterPath = File::Spec->catfile(productDir(), "TestWebKitAPI$apiTesterNameSuffix.exe");
    169         $result = system $apiTesterPath, $test, @ARGV;
    170     } else {
    171         die "run-api-tests is not supported on this platform.\n"
    172     }
    173     
    174     if ($result == 0) {
    175         print BOLD GREEN, "Passed", RESET, "\n";
    176     } else {
    177         print BOLD RED, "Failed", RESET, "\n";
    178     }
    179 }
    180 
    181 sub populateTests()
    182 {
    183     my @tests;
    184 
    185     if (isAppleMacWebKit()) {
    186         my $productDir = productDir();
    187         $ENV{DYLD_FRAMEWORK_PATH} = $productDir;
    188         $ENV{WEBKIT_UNSET_DYLD_FRAMEWORK_PATH} = "YES";
    189         my $apiTesterPath = "$productDir/TestWebKitAPI";
    190 
    191         local *DEVNULL;
    192         my ($childIn, $childOut, $childErr);
    193         unless ($verbose) {
    194             open(DEVNULL, ">", File::Spec->devnull()) or die "Failed to open /dev/null";
    195             $childErr = ">&DEVNULL";
    196         } else {
    197             $childErr = ">&STDERR";
    198         }
    199 
    200         my $pid;
    201         if (architecture()) {
    202             $pid = open3($childIn, $childOut, $childErr, "arch", "-" . architecture(), $apiTesterPath, "--dump-tests") or die "Failed to build list of tests!";
    203         } else {
    204             $pid = open3($childIn, $childOut, $childErr, $apiTesterPath, "--dump-tests") or die "Failed to build list of tests!";
    205         }
    206 
    207         close($childIn);
    208         @tests = <$childOut>;
    209         close($childOut);
    210         close($childErr);
    211         close(DEVNULL) unless ($verbose);
    212 
    213         waitpid($pid, 0);
    214         my $result = $?;
    215 
    216         if ($result) {
    217             print STDERR "Failed to build list of tests!\n";
    218             exit exitStatus($result);
    219         }
    220     } elsif (isAppleWinWebKit()) {
    221         my $apiTesterNameSuffix;
    222         if (configurationForVisualStudio() ne "Debug_All") {
    223             $apiTesterNameSuffix = "";
    224         } else {
    225             $apiTesterNameSuffix = "_debug";
    226         }
    227         my $apiTesterPath = File::Spec->catfile(productDir(), "TestWebKitAPI$apiTesterNameSuffix.exe");
    228         open(TESTS, "-|", $apiTesterPath, "--dump-tests") or die $!;
    229         @tests = <TESTS>;
    230         close(TESTS) or die $!;
    231     } else {
    232         die "run-api-tests is not supported on this platform.\n"
    233     }
    234 
    235     my %keyedTests = ();
    236     for my $test (@tests) {
    237         $test =~ s/[\r\n]*$//;
    238         my ($suite, $testName) = split(/\//, $test);
    239         push @{$keyedTests{$suite}}, $testName;
    240     }
    241     
    242     return %keyedTests;
    243 }
    244 
    245 sub buildTestTool()
    246 {
    247     chdirWebKit();
    248 
    249     my $buildTestTool = "build-api-tests";
    250     print STDERR "Running $buildTestTool\n";
    251 
    252     local *DEVNULL;
    253     my ($childIn, $childOut, $childErr);
    254     unless ($verbose) {
    255         open(DEVNULL, ">", File::Spec->devnull()) or die "Failed to open /dev/null";
    256         $childOut = ">&DEVNULL";
    257         $childErr = ">&DEVNULL";
    258     } else {
    259         # When not quiet, let the child use our stdout/stderr.
    260         $childOut = ">&STDOUT";
    261         $childErr = ">&STDERR";
    262     }
    263 
    264     my @args = argumentsForConfiguration();
    265     my $buildProcess = open3($childIn, $childOut, $childErr, "Tools/Scripts/$buildTestTool", @args) or die "Failed to run " . $buildTestTool;
    266 
    267     close($childIn);
    268     close($childOut);
    269     close($childErr);
    270     close(DEVNULL) unless ($verbose);
    271 
    272     waitpid($buildProcess, 0);
    273     my $buildResult = $?;
    274 
    275     if ($buildResult) {
    276         print STDERR "Compiling TestWebKitAPI failed!\n";
    277         exit exitStatus($buildResult);
    278     }
    279 }
    280