Home | History | Annotate | Download | only in SunSpider 
      1 #!/usr/bin/perl -w
      2 
      3 # Copyright (C) 2007 Apple Inc. All rights reserved.
      4 # Copyright (C) 2007 Eric Seidel <eric (at] webkit.org>
      5 #
      6 # Redistribution and use in source and binary forms, with or without
      7 # modification, are permitted provided that the following conditions
      8 # are met:
      9 # 1. Redistributions of source code must retain the above copyright
     10 #    notice, this list of conditions and the following disclaimer.
     11 # 2. Redistributions in binary form must reproduce the above copyright
     12 #    notice, this list of conditions and the following disclaimer in the
     13 #    documentation and/or other materials provided with the distribution.
     14 #
     15 # THIS SOFTWARE IS PROVIDED BY APPLE COMPUTER, INC. ``AS IS'' AND ANY
     16 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     17 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     18 # PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE COMPUTER, INC. OR
     19 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
     20 # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
     21 # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
     22 # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
     23 # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     24 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
     25 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
     26 
     27 use strict;
     28 use Getopt::Long;
     29 use File::Basename;
     30 use File::Spec;
     31 use Cwd;
     32 use POSIX qw(strftime);
     33 use Time::HiRes qw(gettimeofday tv_interval);
     34 
     35 my $showHelp = 0;
     36 my $runInstruments = 0;
     37 my $ubench = 0;
     38 my $v8suite = 0;
     39 my $suite = "";
     40 my $parseOnly = 0;
     41 my $jsShellPath;
     42 my $jsShellArgs = "";
     43 my $setBaseline = 0;
     44 my $testsPattern;
     45 my $testRuns = 10;
     46 
     47 my $programName = basename($0);
     48 my $usage = <<EOF;
     49 Usage: $programName --shell=[path] [options]
     50   --help            Show this help message
     51   --set-baseline    Set baseline for future comparisons
     52   --shell           Path to JavaScript shell
     53   --args            Arguments to pass to JavaScript shell
     54   --runs            Number of times to run tests (default: $testRuns)
     55   --tests           Only run tests matching provided pattern
     56   --instruments     Sample execution time with the Mac OS X "Instruments" tool (Time Profile) (implies --runs=1)
     57   --suite           Select a specific benchmark suite. The default is sunspider-1.0
     58   --ubench          Use microbenchmark suite instead of regular tests. Same as --suite=ubench
     59   --v8-suite        Use the V8 benchmark suite. Same as --suite=v8-v4
     60   --parse-only      Use the parse-only benchmark suite. Same as --suite=parse-only
     61 EOF
     62 
     63 GetOptions('runs=i' => \$testRuns,
     64            'set-baseline' => \$setBaseline,
     65            'shell=s' => \$jsShellPath,
     66            'args=s' => \$jsShellArgs,
     67            'instruments' => \$runInstruments,
     68            'suite=s' => \$suite,
     69            'ubench' => \$ubench,
     70            'v8-suite' => \$v8suite,
     71            'parse-only' => \$parseOnly,
     72            'tests=s' => \$testsPattern,
     73            'help' => \$showHelp);
     74 
     75 
     76 $suite = "ubench" if ($ubench);
     77 $suite = "v8-v4" if ($v8suite);
     78 $suite = "parse-only" if ($parseOnly);
     79 $suite = "sunspider-1.0" if (!$suite);
     80 
     81 my $resultDirectory = "${suite}-results";
     82 
     83 my $suitePath = $suite;
     84 $suitePath = "tests/" . $suitePath unless ($suite =~ /\//);
     85 
     86 $testRuns = 1 if $runInstruments;
     87 
     88 if (!$jsShellPath || $showHelp) {
     89    print STDERR $usage;
     90    exit 1;
     91 }
     92 
     93 sub dumpToFile($$)
     94 {
     95     my ($contents, $path) = @_;
     96     open FILE, ">", $path or die "Failed to open $path";
     97     print FILE $contents;
     98     close FILE;
     99 }
    100 
    101 my @tests = ();
    102 my @categories = ();
    103 my %uniqueCategories = ();
    104 
    105 sub loadTestsList()
    106 {
    107     open TESTLIST, "<", "${suitePath}/LIST" or die "Can't find ${suitePath}/LIST";
    108     while (<TESTLIST>) {
    109         chomp;
    110         next unless !$testsPattern || /$testsPattern/;
    111         
    112         push @tests, $_;
    113         my $category = $_;
    114         $category =~ s/-.*//;
    115         if (!$uniqueCategories{$category}) {
    116             push @categories, $category;
    117             $uniqueCategories{$category} = $category;
    118         }
    119     }
    120     close TESTLIST;
    121 }
    122 
    123 my $timeString = strftime "%Y-%m-%d-%H.%M.%S", localtime $^T;
    124 my $prefixFile = "$resultDirectory/sunspider-test-prefix.js";
    125 my $resultsFile = "$resultDirectory/sunspider-results-$timeString.js";
    126 
    127 sub writePrefixFile()
    128 {
    129     my $prefix = "var suitePath = " . '"' . $suitePath . '"' . ";\n";
    130     $prefix .= "var tests = [ " . join(", ", map { '"' . $_ . '"' } @tests) . " ];\n";
    131     $prefix .= "var categories = [ " . join(", ", map { '"' . $_ . '"' } @categories) . " ];\n";
    132 
    133     mkdir "$resultDirectory";
    134     dumpToFile($prefix, $prefixFile);
    135 }
    136 
    137 sub runTestsOnce($)
    138 {
    139     my ($useInstruments) = @_;
    140     my $shellArgs = $jsShellArgs . " -f $prefixFile -f resources/sunspider-standalone-driver.js 2> " . File::Spec->devnull();
    141     my $output;
    142     if ($useInstruments) {
    143         $output = `instruments -t "resources/TimeProfile20us.tracetemplate" "$jsShellPath" $shellArgs`;
    144     } else {
    145         $output = `"$jsShellPath" $shellArgs | grep -v break`;
    146     }
    147     return $output;
    148 }
    149 
    150 sub newestFile($$)
    151 {
    152     my ($dir, $pattern) = @_;
    153 
    154     my $newestAge;
    155     my $newestFile = "";
    156     opendir DIR, $dir or die;
    157     for my $file (readdir DIR) {
    158         if ($file =~ $pattern) {
    159             my $age = -M "$dir/$file";
    160             if (!defined $newestAge || $age < $newestAge) {
    161                 $newestFile = $file;
    162                 $newestAge = $age;
    163             }
    164         }
    165     }
    166     closedir DIR;
    167 
    168     return "$dir/$newestFile";
    169 }
    170 
    171 loadTestsList();
    172 if ($testsPattern) {
    173     print STDERR "Found " . scalar(@tests) . " tests matching '" . $testsPattern . "'\n";
    174 } else {
    175     print STDERR "Found " . scalar(@tests) . " tests\n";
    176 }
    177 die "No tests to run"  unless scalar(@tests);
    178 print STDERR "Running SunSpider once for warmup, then " . ($runInstruments ? "under Instruments" : "$testRuns time" . ($testRuns == 1 ? "" : "s")) . "\n";
    179 writePrefixFile();
    180 
    181 runTestsOnce(0);
    182 print "Discarded first run.\n";
    183 
    184 my $result;
    185 my $count = 0;
    186 my @results = ();
    187 my $total = 0;
    188 print "[";
    189 while ($count++ < $testRuns) {
    190     $result = runTestsOnce($runInstruments);
    191     $result =~ s/\r\n/\n/g;
    192     chomp $result;
    193     push @results, $result;
    194     print $result;
    195     print ",\n" unless ($count == $testRuns);
    196 }
    197 print "]\n";
    198 
    199 my $output = "var output = [\n" . join(",\n", @results) . "\n];\n";
    200 dumpToFile($output, $resultsFile);
    201 dumpToFile(File::Spec->rel2abs($resultsFile), "$resultDirectory/baseline-filename.txt") if $setBaseline;
    202 
    203 system("$jsShellPath", "-f", $prefixFile, "-f", $resultsFile, "-f", "resources/sunspider-analyze-results.js");
    204 
    205 print("\nResults are located at $resultsFile\n");
    206 
    207 if ($runInstruments) {
    208     my $newestTrace = newestFile(".", qr/\.trace$/);
    209     if ($newestTrace) {
    210         my $profileFile = "$resultDirectory/sunspider-profile-$timeString.trace";
    211         rename $newestTrace, $profileFile or die;
    212         exec "/usr/bin/open", $profileFile;
    213     }
    214 }
    215