Home | History | Annotate | Download | only in Scripts 
      1 #!/usr/bin/perl -w
      2 
      3 # Copyright (C) 2007, 2008 Apple Inc.  All rights reserved.
      4 #
      5 # Redistribution and use in source and binary forms, with or without
      6 # modification, are permitted provided that the following conditions
      7 # are met:
      8 #
      9 # 1.  Redistributions of source code must retain the above copyright
     10 #     notice, this list of conditions and the following disclaimer. 
     11 # 2.  Redistributions in binary form must reproduce the above copyright
     12 #     notice, this list of conditions and the following disclaimer in the
     13 #     documentation and/or other materials provided with the distribution. 
     14 # 3.  Neither the name of Apple Computer, Inc. ("Apple") nor the names of
     15 #     its contributors may be used to endorse or promote products derived
     16 #     from this software without specific prior written permission. 
     17 #
     18 # THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
     19 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
     20 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
     21 # DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
     22 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
     23 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
     24 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
     25 # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
     27 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     28 
     29 # This script attempts to find the point at which a regression (or progression)
     30 # of behavior occurred by searching WebKit nightly builds.
     31 
     32 # To override the location where the nightly builds are downloaded or the path
     33 # to the Safari web browser, create a ~/.bisect-buildsrc file with one or more of
     34 # the following lines (use "~/" to specify a path from your home directory):
     35 #
     36 # $branch = "branch-name";
     37 # $nightlyDownloadDirectory = "~/path/to/nightly/downloads";
     38 # $safariPath = "/path/to/Safari.app";
     39 
     40 use strict;
     41 
     42 use File::Basename;
     43 use File::Path;
     44 use File::Spec;
     45 use File::Temp qw(tempfile);
     46 use Getopt::Long;
     47 use Time::HiRes qw(usleep);
     48 
     49 sub createTempFile($);
     50 sub downloadNightly($$$);
     51 sub findMacOSXVersion();
     52 sub findNearestNightlyIndex(\@$$);
     53 sub findSafariVersion($);
     54 sub loadSettings();
     55 sub makeNightlyList($$$$);
     56 sub max($$) { return $_[0] > $_[1] ? $_[0] : $_[1]; }
     57 sub mountAndRunNightly($$$$);
     58 sub parseRevisions($$;$);
     59 sub printStatus($$$);
     60 sub promptForTest($);
     61 
     62 loadSettings();
     63 
     64 my %validBranches = map { $_ => 1 } qw(feature-branch trunk);
     65 my $branch = $Settings::branch;
     66 my $nightlyDownloadDirectory = $Settings::nightlyDownloadDirectory;
     67 my $safariPath = $Settings::safariPath;
     68 
     69 my @nightlies;
     70 
     71 my $isProgression;
     72 my $localOnly;
     73 my @revisions;
     74 my $sanityCheck;
     75 my $showHelp;
     76 my $testURL;
     77 
     78 # Fix up -r switches in @ARGV
     79 @ARGV = map { /^(-r)(.+)$/ ? ($1, $2) : $_ } @ARGV;
     80 
     81 my $result = GetOptions(
     82     "b|branch=s"             => \$branch,
     83     "d|download-directory=s" => \$nightlyDownloadDirectory,
     84     "h|help"                 => \$showHelp,
     85     "l|local!"               => \$localOnly,
     86     "p|progression!"         => \$isProgression,
     87     "r|revisions=s"          => \&parseRevisions,
     88     "safari-path=s"          => \$safariPath,
     89     "s|sanity-check!"        => \$sanityCheck,
     90 );
     91 $testURL = shift @ARGV;
     92 
     93 $branch = "feature-branch" if $branch eq "feature";
     94 if (!exists $validBranches{$branch}) {
     95     print STDERR "ERROR: Invalid branch '$branch'\n";
     96     $showHelp = 1;
     97 }
     98 
     99 if (!$result || $showHelp || scalar(@ARGV) > 0) {
    100     print STDERR "Search WebKit nightly builds for changes in behavior.\n";
    101     print STDERR "Usage: " . basename($0) . " [options] [url]\n";
    102     print STDERR <<END;
    103   [-b|--branch name]             name of the nightly build branch (default: trunk)
    104   [-d|--download-directory dir]  nightly build download directory (default: ~/Library/Caches/WebKit-Nightlies)
    105   [-h|--help]                    show this help message
    106   [-l|--local]                   only use local (already downloaded) nightlies
    107   [-p|--progression]             searching for a progression, not a regression
    108   [-r|--revision M[:N]]          specify starting (and optional ending) revisions to search
    109   [--safari-path path]           path to Safari application bundle (default: /Applications/Safari.app)
    110   [-s|--sanity-check]            verify both starting and ending revisions before bisecting
    111 END
    112     exit 1;
    113 }
    114 
    115 my $nightlyWebSite = "http://nightly.webkit.org";
    116 my $nightlyBuildsURLBase = $nightlyWebSite . File::Spec->catdir("/builds", $branch, "mac");
    117 my $nightlyFilesURLBase = $nightlyWebSite . File::Spec->catdir("/files", $branch, "mac");
    118 
    119 $nightlyDownloadDirectory = glob($nightlyDownloadDirectory) if $nightlyDownloadDirectory =~ /^~/;
    120 $safariPath = glob($safariPath) if $safariPath =~ /^~/;
    121 $safariPath = File::Spec->catdir($safariPath, "Contents/MacOS/Safari") if $safariPath =~ m#\.app/*#;
    122 
    123 $nightlyDownloadDirectory = File::Spec->catdir($nightlyDownloadDirectory, $branch);
    124 if (! -d $nightlyDownloadDirectory) {
    125     mkpath($nightlyDownloadDirectory, 0, 0755) || die "Could not create $nightlyDownloadDirectory: $!";
    126 }
    127 
    128 @nightlies = makeNightlyList($localOnly, $nightlyDownloadDirectory, findMacOSXVersion(), findSafariVersion($safariPath));
    129 
    130 my $startIndex = $revisions[0] ? findNearestNightlyIndex(@nightlies, $revisions[0], 'ceil') : 0;
    131 my $endIndex = $revisions[1] ? findNearestNightlyIndex(@nightlies, $revisions[1], 'floor') : $#nightlies;
    132 
    133 my $tempFile = createTempFile($testURL);
    134 
    135 if ($sanityCheck) {
    136     my $didReproduceBug;
    137 
    138     do {
    139         printf "\nChecking starting revision r%s...\n",
    140             $nightlies[$startIndex]->{rev};
    141         downloadNightly($nightlies[$startIndex]->{file}, $nightlyFilesURLBase, $nightlyDownloadDirectory);
    142         mountAndRunNightly($nightlies[$startIndex]->{file}, $nightlyDownloadDirectory, $safariPath, $tempFile);
    143         $didReproduceBug = promptForTest($nightlies[$startIndex]->{rev});
    144         $startIndex-- if $didReproduceBug < 0;
    145     } while ($didReproduceBug < 0);
    146     die "ERROR: Bug reproduced in starting revision!  Do you need to test an earlier revision or for a progression?"
    147         if $didReproduceBug && !$isProgression;
    148     die "ERROR: Bug not reproduced in starting revision!  Do you need to test an earlier revision or for a regression?"
    149         if !$didReproduceBug && $isProgression;
    150 
    151     do {
    152         printf "\nChecking ending revision r%s...\n",
    153             $nightlies[$endIndex]->{rev};
    154         downloadNightly($nightlies[$endIndex]->{file}, $nightlyFilesURLBase, $nightlyDownloadDirectory);
    155         mountAndRunNightly($nightlies[$endIndex]->{file}, $nightlyDownloadDirectory, $safariPath, $tempFile);
    156         $didReproduceBug = promptForTest($nightlies[$endIndex]->{rev});
    157         $endIndex++ if $didReproduceBug < 0;
    158     } while ($didReproduceBug < 0);
    159     die "ERROR: Bug NOT reproduced in ending revision!  Do you need to test a later revision or for a progression?"
    160         if !$didReproduceBug && !$isProgression;
    161     die "ERROR: Bug reproduced in ending revision!  Do you need to test a later revision or for a regression?"
    162         if $didReproduceBug && $isProgression;
    163 }
    164 
    165 printStatus($nightlies[$startIndex]->{rev}, $nightlies[$endIndex]->{rev}, $isProgression);
    166 
    167 my %brokenRevisions = ();
    168 while (abs($endIndex - $startIndex) > 1) {
    169     my $index = $startIndex + int(($endIndex - $startIndex) / 2);
    170 
    171     my $didReproduceBug;
    172     do {
    173         if (exists $nightlies[$index]) {
    174             my $buildsLeft = max(max(0, $endIndex - $index - 1), max(0, $index - $startIndex - 1));
    175             my $plural = $buildsLeft == 1 ? "" : "s";
    176             printf "\nChecking revision r%s (%d build%s left to test after this)...\n", $nightlies[$index]->{rev}, $buildsLeft, $plural;
    177             downloadNightly($nightlies[$index]->{file}, $nightlyFilesURLBase, $nightlyDownloadDirectory);
    178             mountAndRunNightly($nightlies[$index]->{file}, $nightlyDownloadDirectory, $safariPath, $tempFile);
    179             $didReproduceBug = promptForTest($nightlies[$index]->{rev});
    180         }
    181         if ($didReproduceBug < 0) {
    182             $brokenRevisions{$nightlies[$index]->{rev}} = $nightlies[$index]->{file};
    183             delete $nightlies[$index];
    184             $endIndex--;
    185             $index = $startIndex + int(($endIndex - $startIndex) / 2);
    186         }
    187     } while ($didReproduceBug < 0);
    188 
    189     if ($didReproduceBug && !$isProgression || !$didReproduceBug && $isProgression) {
    190         $endIndex = $index;
    191     } else {
    192         $startIndex = $index;
    193     }
    194 
    195     print "\nBroken revisions skipped: r" . join(", r", keys %brokenRevisions) . "\n"
    196         if scalar keys %brokenRevisions > 0;
    197     printStatus($nightlies[$startIndex]->{rev}, $nightlies[$endIndex]->{rev}, $isProgression);
    198 }
    199 
    200 unlink $tempFile if $tempFile;
    201 
    202 exit 0;
    203 
    204 sub createTempFile($)
    205 {
    206     my ($url) = @_;
    207 
    208     return undef if !$url;
    209 
    210     my ($fh, $tempFile) = tempfile(
    211         basename($0) . "-XXXXXXXX",
    212         DIR => File::Spec->tmpdir(),
    213         SUFFIX => ".html",
    214         UNLINK => 0,
    215     );
    216     print $fh "<meta http-equiv=\"refresh\" content=\"0; $url\">\n";
    217     close($fh);
    218 
    219     return $tempFile;
    220 }
    221 
    222 sub downloadNightly($$$)
    223 {
    224     my ($filename, $urlBase, $directory) = @_;
    225     my $path = File::Spec->catfile($directory, $filename);
    226     if (! -f $path) {
    227         print "Downloading $filename to $directory...\n";
    228         `curl -# -o '$path' '$urlBase/$filename'`;
    229     }
    230 }
    231 
    232 sub findMacOSXVersion()
    233 {
    234     my $version;
    235     open(SW_VERS, "-|", "/usr/bin/sw_vers") || die;
    236     while (<SW_VERS>) {
    237         $version = $1 if /^ProductVersion:\s+([^\s]+)/;
    238     }
    239     close(SW_VERS);
    240     return $version;
    241 }
    242 
    243 sub findNearestNightlyIndex(\@$$)
    244 {
    245     my ($nightlies, $revision, $round) = @_;
    246 
    247     my $lowIndex = 0;
    248     my $highIndex = $#{$nightlies};
    249 
    250     return $highIndex if uc($revision) eq 'HEAD' || $revision >= $nightlies->[$highIndex]->{rev};
    251     return $lowIndex if $revision <= $nightlies->[$lowIndex]->{rev};
    252 
    253     while (abs($highIndex - $lowIndex) > 1) {
    254         my $index = $lowIndex + int(($highIndex - $lowIndex) / 2);
    255         if ($revision < $nightlies->[$index]->{rev}) {
    256             $highIndex = $index;
    257         } elsif ($revision > $nightlies->[$index]->{rev}) {
    258             $lowIndex = $index;
    259         } else {
    260             return $index;
    261         }
    262     }
    263 
    264     return ($round eq "floor") ? $lowIndex : $highIndex;
    265 }
    266 
    267 sub findSafariVersion($)
    268 {
    269     my ($path) = @_;
    270     my $versionPlist = File::Spec->catdir(dirname(dirname($path)), "version.plist");
    271     my $version;
    272     open(PLIST, "< $versionPlist") || die;
    273     while (<PLIST>) {
    274         if (m#^\s*<key>CFBundleShortVersionString</key>#) {
    275             $version = <PLIST>;
    276             $version =~ s#^\s*<string>([0-9.]+)[^<]*</string>\s*[\r\n]*#$1#;
    277         }
    278     }
    279     close(PLIST);
    280     return $version;
    281 }
    282 
    283 sub loadSettings()
    284 {
    285     package Settings;
    286 
    287     our $branch = "trunk";
    288     our $nightlyDownloadDirectory = File::Spec->catdir($ENV{HOME}, "Library/Caches/WebKit-Nightlies");
    289     our $safariPath = "/Applications/Safari.app";
    290 
    291     my $rcfile = File::Spec->catdir($ENV{HOME}, ".bisect-buildsrc");
    292     return if !-f $rcfile;
    293 
    294     my $result = do $rcfile;
    295     die "Could not parse $rcfile: $@" if $@;
    296 }
    297 
    298 sub makeNightlyList($$$$)
    299 {
    300     my ($useLocalFiles, $localDirectory, $macOSXVersion, $safariVersion) = @_;
    301     my @files;
    302 
    303     if ($useLocalFiles) {
    304         opendir(DIR, $localDirectory) || die "$!";
    305         foreach my $file (readdir(DIR)) {
    306             if ($file =~ /^WebKit-SVN-r([0-9]+)\.dmg$/) {
    307                 push(@files, +{ rev => $1, file => $file });
    308             }
    309         }
    310         closedir(DIR);
    311     } else {
    312         open(NIGHTLIES, "curl -s $nightlyBuildsURLBase/all |") || die;
    313 
    314         while (my $line = <NIGHTLIES>) {
    315             chomp $line;
    316             my ($revision, $timestamp, $url) = split(/,/, $line);
    317             my $nightly = basename($url);
    318             push(@files, +{ rev => $revision, file => $nightly });
    319         }
    320         close(NIGHTLIES);
    321     }
    322 
    323     if (eval "v$macOSXVersion" ge v10.5) {
    324         if ($safariVersion eq "4 Public Beta") {
    325             @files = grep { $_->{rev} >= 39682 } @files;
    326         } elsif (eval "v$safariVersion" ge v3.2) {
    327             @files = grep { $_->{rev} >= 37348 } @files;
    328         } elsif (eval "v$safariVersion" ge v3.1) {
    329             @files = grep { $_->{rev} >= 29711 } @files;
    330         } elsif (eval "v$safariVersion" ge v3.0) {
    331             @files = grep { $_->{rev} >= 25124 } @files;
    332         } elsif (eval "v$safariVersion" ge v2.0) {
    333             @files = grep { $_->{rev} >= 19594 } @files;
    334         } else {
    335             die "Requires Safari 2.0 or newer";
    336         }
    337     } elsif (eval "v$macOSXVersion" ge v10.4) {
    338         if ($safariVersion eq "4 Public Beta") {
    339             @files = grep { $_->{rev} >= 39682 } @files;
    340         } elsif (eval "v$safariVersion" ge v3.2) {
    341             @files = grep { $_->{rev} >= 37348 } @files;
    342         } elsif (eval "v$safariVersion" ge v3.1) {
    343             @files = grep { $_->{rev} >= 29711 } @files;
    344         } elsif (eval "v$safariVersion" ge v3.0) {
    345             @files = grep { $_->{rev} >= 19992 } @files;
    346         } elsif (eval "v$safariVersion" ge v2.0) {
    347             @files = grep { $_->{rev} >= 11976 } @files;
    348         } else {
    349             die "Requires Safari 2.0 or newer";
    350         }
    351     } else {
    352         die "Requires Mac OS X 10.4 (Tiger) or 10.5 (Leopard)";
    353     }
    354 
    355     my $nightlycmp = sub { return $a->{rev} <=> $b->{rev}; };
    356 
    357     return sort $nightlycmp @files;
    358 }
    359 
    360 sub mountAndRunNightly($$$$)
    361 {
    362     my ($filename, $directory, $safari, $tempFile) = @_;
    363     my $mountPath = "/Volumes/WebKit";
    364     my $webkitApp = File::Spec->catfile($mountPath, "WebKit.app");
    365     my $diskImage = File::Spec->catfile($directory, $filename);
    366     my $devNull = File::Spec->devnull();
    367 
    368     my $i = 0;
    369     while (-e $mountPath) {
    370         $i++;
    371         usleep 100 if $i > 1;
    372         `hdiutil detach '$mountPath' 2> $devNull`;
    373         die "Could not unmount $diskImage at $mountPath" if $i > 100;
    374     }
    375     die "Can't mount $diskImage: $mountPath already exists!" if -e $mountPath;
    376 
    377     print "Mounting disk image and running WebKit...\n";
    378     `hdiutil attach '$diskImage'`;
    379     $i = 0;
    380     while (! -e $webkitApp) {
    381         usleep 100;
    382         $i++;
    383         die "Could not mount $diskImage at $mountPath" if $i > 100;
    384     }
    385 
    386     my $frameworkPath;
    387     if (-d "/Volumes/WebKit/WebKit.app/Contents/Frameworks") {
    388         my $osXVersion = join('.', (split(/\./, findMacOSXVersion()))[0..1]);
    389         $frameworkPath = "/Volumes/WebKit/WebKit.app/Contents/Frameworks/$osXVersion";
    390     } else {
    391         $frameworkPath = "/Volumes/WebKit/WebKit.app/Contents/Resources";
    392     }
    393 
    394     $tempFile ||= "";
    395     `DYLD_FRAMEWORK_PATH=$frameworkPath WEBKIT_UNSET_DYLD_FRAMEWORK_PATH=YES $safari $tempFile`;
    396 
    397     `hdiutil detach '$mountPath' 2> $devNull`;
    398 }
    399 
    400 sub parseRevisions($$;$)
    401 {
    402     my ($optionName, $value, $ignored) = @_;
    403 
    404     if ($value =~ /^r?([0-9]+|HEAD):?$/i) {
    405         push(@revisions, $1);
    406         die "Too many revision arguments specified" if scalar @revisions > 2;
    407     } elsif ($value =~ /^r?([0-9]+):?r?([0-9]+|HEAD)$/i) {
    408         $revisions[0] = $1;
    409         $revisions[1] = $2;
    410     } else {
    411         die "Unknown revision '$value':  expected 'M' or 'M:N'";
    412     }
    413 }
    414 
    415 sub printStatus($$$)
    416 {
    417     my ($startRevision, $endRevision, $isProgression) = @_;
    418     printf "\n%s: r%s  %s: r%s\n",
    419         $isProgression ? "Fails" : "Works", $startRevision,
    420         $isProgression ? "Works" : "Fails", $endRevision;
    421 }
    422 
    423 sub promptForTest($)
    424 {
    425     my ($revision) = @_;
    426     print "Did the bug reproduce in r$revision (yes/no/broken)? ";
    427     my $answer = <STDIN>;
    428     return 1 if $answer =~ /^(1|y.*)$/i;
    429     return -1 if $answer =~ /^(-1|b.*)$/i; # Broken
    430     return 0;
    431 }
    432 
    433