Home | History | Annotate | Download | only in Scripts 
      1 #!/usr/bin/perl -w
      2 
      3 # Copyright (C) 2007, 2008, 2011 Apple Inc.  All rights reserved.
      4 #
      5 # Redistribution and use in source and binary forms, with or without
      6 # modification, are permitted provided that the following conditions
      7 # are met:
      8 #
      9 # 1.  Redistributions of source code must retain the above copyright
     10 #     notice, this list of conditions and the following disclaimer. 
     11 # 2.  Redistributions in binary form must reproduce the above copyright
     12 #     notice, this list of conditions and the following disclaimer in the
     13 #     documentation and/or other materials provided with the distribution. 
     14 # 3.  Neither the name of Apple Computer, Inc. ("Apple") nor the names of
     15 #     its contributors may be used to endorse or promote products derived
     16 #     from this software without specific prior written permission. 
     17 #
     18 # THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
     19 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
     20 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
     21 # DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
     22 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
     23 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
     24 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
     25 # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
     27 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     28 
     29 # This script attempts to find the point at which a regression (or progression)
     30 # of behavior occurred by searching WebKit nightly builds.
     31 
     32 # To override the location where the nightly builds are downloaded or the path
     33 # to the Safari web browser, create a ~/.bisect-buildsrc file with one or more of
     34 # the following lines (use "~/" to specify a path from your home directory):
     35 #
     36 # $branch = "branch-name";
     37 # $nightlyDownloadDirectory = "~/path/to/nightly/downloads";
     38 # $safariPath = "/path/to/Safari.app";
     39 
     40 use strict;
     41 
     42 use File::Basename;
     43 use File::Path;
     44 use File::Spec;
     45 use File::Temp qw(tempfile);
     46 use Getopt::Long;
     47 use Time::HiRes qw(usleep);
     48 
     49 sub createTempFile($);
     50 sub downloadNightly($$$);
     51 sub findMacOSXVersion();
     52 sub findNearestNightlyIndex(\@$$);
     53 sub findSafariVersion($);
     54 sub loadSettings();
     55 sub makeNightlyList($$$$);
     56 sub max($$) { return $_[0] > $_[1] ? $_[0] : $_[1]; }
     57 sub mountAndRunNightly($$$$);
     58 sub parseRevisions($$;$);
     59 sub printStatus($$$);
     60 sub printTracLink($$);
     61 sub promptForTest($);
     62 
     63 loadSettings();
     64 
     65 my %validBranches = map { $_ => 1 } qw(feature-branch trunk);
     66 my $branch = $Settings::branch;
     67 my $nightlyDownloadDirectory = $Settings::nightlyDownloadDirectory;
     68 my $safariPath = $Settings::safariPath;
     69 
     70 my @nightlies;
     71 
     72 my $isProgression;
     73 my $localOnly;
     74 my @revisions;
     75 my $sanityCheck;
     76 my $showHelp;
     77 my $testURL;
     78 
     79 # Fix up -r switches in @ARGV
     80 @ARGV = map { /^(-r)(.+)$/ ? ($1, $2) : $_ } @ARGV;
     81 
     82 my $result = GetOptions(
     83     "b|branch=s"             => \$branch,
     84     "d|download-directory=s" => \$nightlyDownloadDirectory,
     85     "h|help"                 => \$showHelp,
     86     "l|local!"               => \$localOnly,
     87     "p|progression!"         => \$isProgression,
     88     "r|revisions=s"          => \&parseRevisions,
     89     "safari-path=s"          => \$safariPath,
     90     "s|sanity-check!"        => \$sanityCheck,
     91 );
     92 $testURL = shift @ARGV;
     93 
     94 $branch = "feature-branch" if $branch eq "feature";
     95 if (!exists $validBranches{$branch}) {
     96     print STDERR "ERROR: Invalid branch '$branch'\n";
     97     $showHelp = 1;
     98 }
     99 
    100 if (!$result || $showHelp || scalar(@ARGV) > 0) {
    101     print STDERR "Search WebKit nightly builds for changes in behavior.\n";
    102     print STDERR "Usage: " . basename($0) . " [options] [url]\n";
    103     print STDERR <<END;
    104   [-b|--branch name]             name of the nightly build branch (default: trunk)
    105   [-d|--download-directory dir]  nightly build download directory (default: ~/Library/Caches/WebKit-Nightlies)
    106   [-h|--help]                    show this help message
    107   [-l|--local]                   only use local (already downloaded) nightlies
    108   [-p|--progression]             searching for a progression, not a regression
    109   [-r|--revision M[:N]]          specify starting (and optional ending) revisions to search
    110   [--safari-path path]           path to Safari application bundle (default: /Applications/Safari.app)
    111   [-s|--sanity-check]            verify both starting and ending revisions before bisecting
    112 END
    113     exit 1;
    114 }
    115 
    116 my $nightlyWebSite = "http://nightly.webkit.org";
    117 my $nightlyBuildsURLBase = $nightlyWebSite . File::Spec->catdir("/builds", $branch, "mac");
    118 my $nightlyFilesURLBase = $nightlyWebSite . File::Spec->catdir("/files", $branch, "mac");
    119 
    120 $nightlyDownloadDirectory = glob($nightlyDownloadDirectory) if $nightlyDownloadDirectory =~ /^~/;
    121 $safariPath = glob($safariPath) if $safariPath =~ /^~/;
    122 $safariPath = File::Spec->catdir($safariPath, "Contents/MacOS/Safari") if $safariPath =~ m#\.app/*#;
    123 
    124 $nightlyDownloadDirectory = File::Spec->catdir($nightlyDownloadDirectory, $branch);
    125 if (! -d $nightlyDownloadDirectory) {
    126     mkpath($nightlyDownloadDirectory, 0, 0755) || die "Could not create $nightlyDownloadDirectory: $!";
    127 }
    128 
    129 @nightlies = makeNightlyList($localOnly, $nightlyDownloadDirectory, findMacOSXVersion(), findSafariVersion($safariPath));
    130 
    131 my $startIndex = $revisions[0] ? findNearestNightlyIndex(@nightlies, $revisions[0], 'ceil') : 0;
    132 my $endIndex = $revisions[1] ? findNearestNightlyIndex(@nightlies, $revisions[1], 'floor') : $#nightlies;
    133 
    134 my $tempFile = createTempFile($testURL);
    135 
    136 if ($sanityCheck) {
    137     my $didReproduceBug;
    138 
    139     do {
    140         printf "\nChecking starting revision r%s...\n",
    141             $nightlies[$startIndex]->{rev};
    142         downloadNightly($nightlies[$startIndex]->{file}, $nightlyFilesURLBase, $nightlyDownloadDirectory);
    143         mountAndRunNightly($nightlies[$startIndex]->{file}, $nightlyDownloadDirectory, $safariPath, $tempFile);
    144         $didReproduceBug = promptForTest($nightlies[$startIndex]->{rev});
    145         $startIndex-- if $didReproduceBug < 0;
    146     } while ($didReproduceBug < 0);
    147     die "ERROR: Bug reproduced in starting revision!  Do you need to test an earlier revision or for a progression?"
    148         if $didReproduceBug && !$isProgression;
    149     die "ERROR: Bug not reproduced in starting revision!  Do you need to test an earlier revision or for a regression?"
    150         if !$didReproduceBug && $isProgression;
    151 
    152     do {
    153         printf "\nChecking ending revision r%s...\n",
    154             $nightlies[$endIndex]->{rev};
    155         downloadNightly($nightlies[$endIndex]->{file}, $nightlyFilesURLBase, $nightlyDownloadDirectory);
    156         mountAndRunNightly($nightlies[$endIndex]->{file}, $nightlyDownloadDirectory, $safariPath, $tempFile);
    157         $didReproduceBug = promptForTest($nightlies[$endIndex]->{rev});
    158         $endIndex++ if $didReproduceBug < 0;
    159     } while ($didReproduceBug < 0);
    160     die "ERROR: Bug NOT reproduced in ending revision!  Do you need to test a later revision or for a progression?"
    161         if !$didReproduceBug && !$isProgression;
    162     die "ERROR: Bug reproduced in ending revision!  Do you need to test a later revision or for a regression?"
    163         if $didReproduceBug && $isProgression;
    164 }
    165 
    166 printStatus($nightlies[$startIndex]->{rev}, $nightlies[$endIndex]->{rev}, $isProgression);
    167 
    168 my %brokenRevisions = ();
    169 while (abs($endIndex - $startIndex) > 1) {
    170     my $index = $startIndex + int(($endIndex - $startIndex) / 2);
    171 
    172     my $didReproduceBug;
    173     do {
    174         if (exists $nightlies[$index]) {
    175             my $buildsLeft = max(max(0, $endIndex - $index - 1), max(0, $index - $startIndex - 1));
    176             my $plural = $buildsLeft == 1 ? "" : "s";
    177             printf "\nChecking revision r%s (%d build%s left to test after this)...\n", $nightlies[$index]->{rev}, $buildsLeft, $plural;
    178             downloadNightly($nightlies[$index]->{file}, $nightlyFilesURLBase, $nightlyDownloadDirectory);
    179             mountAndRunNightly($nightlies[$index]->{file}, $nightlyDownloadDirectory, $safariPath, $tempFile);
    180             $didReproduceBug = promptForTest($nightlies[$index]->{rev});
    181         }
    182         if ($didReproduceBug < 0) {
    183             $brokenRevisions{$nightlies[$index]->{rev}} = $nightlies[$index]->{file};
    184             delete $nightlies[$index];
    185             $endIndex--;
    186             $index = $startIndex + int(($endIndex - $startIndex) / 2);
    187         }
    188     } while ($didReproduceBug < 0);
    189 
    190     if ($didReproduceBug && !$isProgression || !$didReproduceBug && $isProgression) {
    191         $endIndex = $index;
    192     } else {
    193         $startIndex = $index;
    194     }
    195 
    196     print "\nBroken revisions skipped: r" . join(", r", keys %brokenRevisions) . "\n"
    197         if scalar keys %brokenRevisions > 0;
    198     printStatus($nightlies[$startIndex]->{rev}, $nightlies[$endIndex]->{rev}, $isProgression);
    199 }
    200 
    201 printTracLink($nightlies[$startIndex]->{rev}, $nightlies[$endIndex]->{rev});
    202 
    203 unlink $tempFile if $tempFile;
    204 
    205 exit 0;
    206 
    207 sub createTempFile($)
    208 {
    209     my ($url) = @_;
    210 
    211     return undef if !$url;
    212 
    213     my ($fh, $tempFile) = tempfile(
    214         basename($0) . "-XXXXXXXX",
    215         DIR => File::Spec->tmpdir(),
    216         SUFFIX => ".html",
    217         UNLINK => 0,
    218     );
    219     print $fh "<meta http-equiv=\"refresh\" content=\"0; $url\">\n";
    220     close($fh);
    221 
    222     return $tempFile;
    223 }
    224 
    225 sub downloadNightly($$$)
    226 {
    227     my ($filename, $urlBase, $directory) = @_;
    228     my $path = File::Spec->catfile($directory, $filename);
    229     if (! -f $path) {
    230         print "Downloading $filename to $directory...\n";
    231         `curl -# -o '$path' '$urlBase/$filename'`;
    232     }
    233 }
    234 
    235 sub findMacOSXVersion()
    236 {
    237     my $version;
    238     open(SW_VERS, "-|", "/usr/bin/sw_vers") || die;
    239     while (<SW_VERS>) {
    240         $version = $1 if /^ProductVersion:\s+([^\s]+)/;
    241     }
    242     close(SW_VERS);
    243     return $version;
    244 }
    245 
    246 sub findNearestNightlyIndex(\@$$)
    247 {
    248     my ($nightlies, $revision, $round) = @_;
    249 
    250     my $lowIndex = 0;
    251     my $highIndex = $#{$nightlies};
    252 
    253     return $highIndex if uc($revision) eq 'HEAD' || $revision >= $nightlies->[$highIndex]->{rev};
    254     return $lowIndex if $revision <= $nightlies->[$lowIndex]->{rev};
    255 
    256     while (abs($highIndex - $lowIndex) > 1) {
    257         my $index = $lowIndex + int(($highIndex - $lowIndex) / 2);
    258         if ($revision < $nightlies->[$index]->{rev}) {
    259             $highIndex = $index;
    260         } elsif ($revision > $nightlies->[$index]->{rev}) {
    261             $lowIndex = $index;
    262         } else {
    263             return $index;
    264         }
    265     }
    266 
    267     return ($round eq "floor") ? $lowIndex : $highIndex;
    268 }
    269 
    270 sub findSafariVersion($)
    271 {
    272     my ($path) = @_;
    273     my $versionPlist = File::Spec->catdir(dirname(dirname($path)), "version.plist");
    274     my $version;
    275     open(PLIST, "< $versionPlist") || die;
    276     while (<PLIST>) {
    277         if (m#^\s*<key>CFBundleShortVersionString</key>#) {
    278             $version = <PLIST>;
    279             $version =~ s#^\s*<string>([0-9.]+)[^<]*</string>\s*[\r\n]*#$1#;
    280         }
    281     }
    282     close(PLIST);
    283     return $version;
    284 }
    285 
    286 sub loadSettings()
    287 {
    288     package Settings;
    289 
    290     our $branch = "trunk";
    291     our $nightlyDownloadDirectory = File::Spec->catdir($ENV{HOME}, "Library/Caches/WebKit-Nightlies");
    292     our $safariPath = "/Applications/Safari.app";
    293 
    294     my $rcfile = File::Spec->catdir($ENV{HOME}, ".bisect-buildsrc");
    295     return if !-f $rcfile;
    296 
    297     my $result = do $rcfile;
    298     die "Could not parse $rcfile: $@" if $@;
    299 }
    300 
    301 sub makeNightlyList($$$$)
    302 {
    303     my ($useLocalFiles, $localDirectory, $macOSXVersion, $safariVersion) = @_;
    304     my @files;
    305 
    306     if ($useLocalFiles) {
    307         opendir(DIR, $localDirectory) || die "$!";
    308         foreach my $file (readdir(DIR)) {
    309             if ($file =~ /^WebKit-SVN-r([0-9]+)\.dmg$/) {
    310                 push(@files, +{ rev => $1, file => $file });
    311             }
    312         }
    313         closedir(DIR);
    314     } else {
    315         open(NIGHTLIES, "curl -s $nightlyBuildsURLBase/all |") || die;
    316 
    317         while (my $line = <NIGHTLIES>) {
    318             chomp $line;
    319             my ($revision, $timestamp, $url) = split(/,/, $line);
    320             my $nightly = basename($url);
    321             push(@files, +{ rev => $revision, file => $nightly });
    322         }
    323         close(NIGHTLIES);
    324     }
    325 
    326     if (eval "v$macOSXVersion" ge v10.5) {
    327         if ($safariVersion eq "4 Public Beta") {
    328             @files = grep { $_->{rev} >= 39682 } @files;
    329         } elsif (eval "v$safariVersion" ge v3.2) {
    330             @files = grep { $_->{rev} >= 37348 } @files;
    331         } elsif (eval "v$safariVersion" ge v3.1) {
    332             @files = grep { $_->{rev} >= 29711 } @files;
    333         } elsif (eval "v$safariVersion" ge v3.0) {
    334             @files = grep { $_->{rev} >= 25124 } @files;
    335         } elsif (eval "v$safariVersion" ge v2.0) {
    336             @files = grep { $_->{rev} >= 19594 } @files;
    337         } else {
    338             die "Requires Safari 2.0 or newer";
    339         }
    340     } elsif (eval "v$macOSXVersion" ge v10.4) {
    341         if ($safariVersion eq "4 Public Beta") {
    342             @files = grep { $_->{rev} >= 39682 } @files;
    343         } elsif (eval "v$safariVersion" ge v3.2) {
    344             @files = grep { $_->{rev} >= 37348 } @files;
    345         } elsif (eval "v$safariVersion" ge v3.1) {
    346             @files = grep { $_->{rev} >= 29711 } @files;
    347         } elsif (eval "v$safariVersion" ge v3.0) {
    348             @files = grep { $_->{rev} >= 19992 } @files;
    349         } elsif (eval "v$safariVersion" ge v2.0) {
    350             @files = grep { $_->{rev} >= 11976 } @files;
    351         } else {
    352             die "Requires Safari 2.0 or newer";
    353         }
    354     } else {
    355         die "Requires Mac OS X 10.4 (Tiger) or 10.5 (Leopard)";
    356     }
    357 
    358     my $nightlycmp = sub { return $a->{rev} <=> $b->{rev}; };
    359 
    360     return sort $nightlycmp @files;
    361 }
    362 
    363 sub mountAndRunNightly($$$$)
    364 {
    365     my ($filename, $directory, $safari, $tempFile) = @_;
    366     my $mountPath = "/Volumes/WebKit";
    367     my $webkitApp = File::Spec->catfile($mountPath, "WebKit.app");
    368     my $diskImage = File::Spec->catfile($directory, $filename);
    369     my $devNull = File::Spec->devnull();
    370 
    371     my $i = 0;
    372     while (-e $mountPath) {
    373         $i++;
    374         usleep 100 if $i > 1;
    375         `hdiutil detach '$mountPath' 2> $devNull`;
    376         die "Could not unmount $diskImage at $mountPath" if $i > 100;
    377     }
    378     die "Can't mount $diskImage: $mountPath already exists!" if -e $mountPath;
    379 
    380     print "Mounting disk image and running WebKit...\n";
    381     `hdiutil attach '$diskImage'`;
    382     $i = 0;
    383     while (! -e $webkitApp) {
    384         usleep 100;
    385         $i++;
    386         die "Could not mount $diskImage at $mountPath" if $i > 100;
    387     }
    388 
    389     my $frameworkPath;
    390     if (-d "/Volumes/WebKit/WebKit.app/Contents/Frameworks") {
    391         my $osXVersion = join('.', (split(/\./, findMacOSXVersion()))[0..1]);
    392         $frameworkPath = "/Volumes/WebKit/WebKit.app/Contents/Frameworks/$osXVersion";
    393     } else {
    394         $frameworkPath = "/Volumes/WebKit/WebKit.app/Contents/Resources";
    395     }
    396 
    397     $tempFile ||= "";
    398     `DYLD_FRAMEWORK_PATH=$frameworkPath WEBKIT_UNSET_DYLD_FRAMEWORK_PATH=YES $safari $tempFile`;
    399 
    400     `hdiutil detach '$mountPath' 2> $devNull`;
    401 }
    402 
    403 sub parseRevisions($$;$)
    404 {
    405     my ($optionName, $value, $ignored) = @_;
    406 
    407     if ($value =~ /^r?([0-9]+|HEAD):?$/i) {
    408         push(@revisions, $1);
    409         die "Too many revision arguments specified" if scalar @revisions > 2;
    410     } elsif ($value =~ /^r?([0-9]+):?r?([0-9]+|HEAD)$/i) {
    411         $revisions[0] = $1;
    412         $revisions[1] = $2;
    413     } else {
    414         die "Unknown revision '$value':  expected 'M' or 'M:N'";
    415     }
    416 }
    417 
    418 sub printStatus($$$)
    419 {
    420     my ($startRevision, $endRevision, $isProgression) = @_;
    421     printf "\n%s: r%s  %s: r%s\n",
    422         $isProgression ? "Fails" : "Works", $startRevision,
    423         $isProgression ? "Works" : "Fails", $endRevision;
    424 }
    425 
    426 sub printTracLink($$)
    427 {
    428     my ($startRevision, $endRevision) = @_;
    429     printf("http://trac.webkit.org/log/trunk/?rev=%s&stop_rev=%s\n", $endRevision, $startRevision + 1);
    430 }
    431 
    432 sub promptForTest($)
    433 {
    434     my ($revision) = @_;
    435     print "Did the bug reproduce in r$revision (yes/no/broken)? ";
    436     my $answer = <STDIN>;
    437     return 1 if $answer =~ /^(1|y.*)$/i;
    438     return -1 if $answer =~ /^(-1|b.*)$/i; # Broken
    439     return 0;
    440 }
    441 
    442