Home | History | Annotate | Download | only in Scripts 
      1 #!/usr/bin/perl -w
      2 
      3 # Copyright (C) 2005, 2006 Apple Computer, Inc.  All rights reserved.
      4 #
      5 # Redistribution and use in source and binary forms, with or without
      6 # modification, are permitted provided that the following conditions
      7 # are met:
      8 #
      9 # 1.  Redistributions of source code must retain the above copyright
     10 #     notice, this list of conditions and the following disclaimer.
     11 # 2.  Redistributions in binary form must reproduce the above copyright
     12 #     notice, this list of conditions and the following disclaimer in the
     13 #     documentation and/or other materials provided with the distribution.
     14 # 3.  Neither the name of Apple Computer, Inc. ("Apple") nor the names of
     15 #     its contributors may be used to endorse or promote products derived
     16 #     from this software without specific prior written permission.
     17 #
     18 # THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
     19 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
     20 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
     21 # DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
     22 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
     23 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
     24 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
     25 # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
     27 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     28 
     29 # Extended "svn diff" script for WebKit Open Source Project, used to make patches.
     30 
     31 # Differences from standard "svn diff":
     32 #
     33 #   Uses the real diff, not svn's built-in diff.
     34 #   Always passes "-p" to diff so it will try to include function names.
     35 #   Handles binary files (encoded as a base64 chunk of text).
     36 #   Sorts the diffs alphabetically by text files, then binary files.
     37 #   Handles copied and moved files.
     38 #
     39 # Missing features:
     40 #
     41 #   Handle copied and moved directories.
     42 
     43 use strict;
     44 use warnings;
     45 
     46 use Config;
     47 use File::Basename;
     48 use File::Spec;
     49 use File::stat;
     50 use FindBin;
     51 use Getopt::Long;
     52 use lib $FindBin::Bin;
     53 use MIME::Base64;
     54 use POSIX qw(:errno_h);
     55 use Time::gmtime;
     56 use VCSUtils;
     57 
     58 sub binarycmp($$);
     59 sub diffOptionsForFile($);
     60 sub findBaseUrl($);
     61 sub findMimeType($;$);
     62 sub findModificationType($);
     63 sub findSourceFileAndRevision($);
     64 sub generateDiff($$);
     65 sub generateFileList($\%);
     66 sub hunkHeaderLineRegExForFile($);
     67 sub isBinaryMimeType($);
     68 sub manufacturePatchForAdditionWithHistory($);
     69 sub numericcmp($$);
     70 sub outputBinaryContent($);
     71 sub patchpathcmp($$);
     72 sub pathcmp($$);
     73 sub processPaths(\@);
     74 sub splitpath($);
     75 sub testfilecmp($$);
     76 
     77 $ENV{'LC_ALL'} = 'C';
     78 
     79 my $showHelp;
     80 my $ignoreChangelogs = 0;
     81 my $devNull = File::Spec->devnull();
     82 
     83 my $result = GetOptions(
     84     "help"       => \$showHelp,
     85     "ignore-changelogs"    => \$ignoreChangelogs
     86 );
     87 if (!$result || $showHelp) {
     88     print STDERR basename($0) . " [-h|--help] [--ignore-changelogs] [svndir1 [svndir2 ...]]\n";
     89     exit 1;
     90 }
     91 
     92 # Sort the diffs for easier reviewing.
     93 my %paths = processPaths(@ARGV);
     94 
     95 # Generate a list of files requiring diffs.
     96 my %diffFiles;
     97 for my $path (keys %paths) {
     98     generateFileList($path, %diffFiles);
     99 }
    100 
    101 my $svnRoot = determineSVNRoot();
    102 my $prefix = chdirReturningRelativePath($svnRoot);
    103 
    104 my $patchSize = 0;
    105 
    106 # Generate the diffs, in a order chosen for easy reviewing.
    107 for my $path (sort patchpathcmp values %diffFiles) {
    108     $patchSize += generateDiff($path, $prefix);
    109 }
    110 
    111 if ($patchSize > 20480) {
    112     print STDERR "WARNING: Patch's size is " . int($patchSize/1024) . " kbytes.\n";
    113     print STDERR "Patches 20k or smaller are more likely to be reviewed. Larger patches may sit unreviewed for a long time.\n";
    114 }
    115 
    116 exit 0;
    117 
    118 # Overall sort, considering multiple criteria.
    119 sub patchpathcmp($$)
    120 {
    121     my ($a, $b) = @_;
    122 
    123     # All binary files come after all non-binary files.
    124     my $result = binarycmp($a, $b);
    125     return $result if $result;
    126 
    127     # All test files come after all non-test files.
    128     $result = testfilecmp($a, $b);
    129     return $result if $result;
    130 
    131     # Final sort is a "smart" sort by directory and file name.
    132     return pathcmp($a, $b);
    133 }
    134 
    135 # Sort so text files appear before binary files.
    136 sub binarycmp($$)
    137 {
    138     my ($fileDataA, $fileDataB) = @_;
    139     return $fileDataA->{isBinary} <=> $fileDataB->{isBinary};
    140 }
    141 
    142 sub diffOptionsForFile($)
    143 {
    144     my ($file) = @_;
    145 
    146     my $options = "uaNp";
    147 
    148     if (my $hunkHeaderLineRegEx = hunkHeaderLineRegExForFile($file)) {
    149         $options .= "F'$hunkHeaderLineRegEx'";
    150     }
    151 
    152     return $options;
    153 }
    154 
    155 sub findBaseUrl($)
    156 {
    157     my ($infoPath) = @_;
    158     my $baseUrl;
    159     my $escapedInfoPath = escapeSubversionPath($infoPath);
    160     open INFO, "svn info '$escapedInfoPath' |" or die;
    161     while (<INFO>) {
    162         if (/^URL: (.+?)[\r\n]*$/) {
    163             $baseUrl = $1;
    164         }
    165     }
    166     close INFO;
    167     return $baseUrl;
    168 }
    169 
    170 sub findMimeType($;$)
    171 {
    172     my ($file, $revision) = @_;
    173     my $args = $revision ? "--revision $revision" : "";
    174     my $escapedFile = escapeSubversionPath($file);
    175     open PROPGET, "svn propget svn:mime-type $args '$escapedFile' |" or die;
    176     my $mimeType = <PROPGET>;
    177     close PROPGET;
    178     # svn may output a different EOL sequence than $/, so avoid chomp.
    179     if ($mimeType) {
    180         $mimeType =~ s/[\r\n]+$//g;
    181     }
    182     return $mimeType;
    183 }
    184 
    185 sub findModificationType($)
    186 {
    187     my ($stat) = @_;
    188     my $fileStat = substr($stat, 0, 1);
    189     my $propertyStat = substr($stat, 1, 1);
    190     if ($fileStat eq "A" || $fileStat eq "R") {
    191         my $additionWithHistory = substr($stat, 3, 1);
    192         return $additionWithHistory eq "+" ? "additionWithHistory" : "addition";
    193     }
    194     return "modification" if ($fileStat eq "M" || $propertyStat eq "M");
    195     return "deletion" if ($fileStat eq "D");
    196     return undef;
    197 }
    198 
    199 sub findSourceFileAndRevision($)
    200 {
    201     my ($file) = @_;
    202     my $baseUrl = findBaseUrl(".");
    203     my $sourceFile;
    204     my $sourceRevision;
    205     my $escapedFile = escapeSubversionPath($file);
    206     open INFO, "svn info '$escapedFile' |" or die;
    207     while (<INFO>) {
    208         if (/^Copied From URL: (.+?)[\r\n]*$/) {
    209             $sourceFile = File::Spec->abs2rel($1, $baseUrl);
    210         } elsif (/^Copied From Rev: ([0-9]+)/) {
    211             $sourceRevision = $1;
    212         }
    213     }
    214     close INFO;
    215     return ($sourceFile, $sourceRevision);
    216 }
    217 
    218 sub generateDiff($$)
    219 {
    220     my ($fileData, $prefix) = @_;
    221     my $file = File::Spec->catdir($prefix, $fileData->{path});
    222 
    223     if ($ignoreChangelogs && basename($file) eq "ChangeLog") {
    224         return 0;
    225     }
    226 
    227     my $patch = "";
    228     if ($fileData->{modificationType} eq "additionWithHistory") {
    229         manufacturePatchForAdditionWithHistory($fileData);
    230     }
    231 
    232     my $diffOptions = diffOptionsForFile($file);
    233     my $escapedFile = escapeSubversionPath($file);
    234     open DIFF, "svn diff --diff-cmd diff -x -$diffOptions '$escapedFile' |" or die;
    235     while (<DIFF>) {
    236         $patch .= $_;
    237     }
    238     close DIFF;
    239     if (basename($file) eq "ChangeLog") {
    240         my $changeLogHash = fixChangeLogPatch($patch);
    241         $patch = $changeLogHash->{patch};
    242     }
    243     print $patch;
    244     if ($fileData->{isBinary}) {
    245         print "\n" if ($patch && $patch =~ m/\n\S+$/m);
    246         outputBinaryContent($file);
    247     }
    248     return length($patch);
    249 }
    250 
    251 sub generateFileList($\%)
    252 {
    253     my ($statPath, $diffFiles) = @_;
    254     my %testDirectories = map { $_ => 1 } qw(LayoutTests);
    255     my $escapedStatPath = escapeSubversionPath($statPath);
    256     open STAT, "svn stat '$escapedStatPath' |" or die;
    257     while (my $line = <STAT>) {
    258         # svn may output a different EOL sequence than $/, so avoid chomp.
    259         $line =~ s/[\r\n]+$//g;
    260         my $stat;
    261         my $path;
    262         if (isSVNVersion16OrNewer()) {
    263             $stat = substr($line, 0, 8);
    264             $path = substr($line, 8);
    265         } else {
    266             $stat = substr($line, 0, 7);
    267             $path = substr($line, 7);
    268         }
    269         next if -d $path;
    270         my $modificationType = findModificationType($stat);
    271         if ($modificationType) {
    272             $diffFiles->{$path}->{path} = $path;
    273             $diffFiles->{$path}->{modificationType} = $modificationType;
    274             $diffFiles->{$path}->{isBinary} = isBinaryMimeType($path);
    275             $diffFiles->{$path}->{isTestFile} = exists $testDirectories{(File::Spec->splitdir($path))[0]} ? 1 : 0;
    276             if ($modificationType eq "additionWithHistory") {
    277                 my ($sourceFile, $sourceRevision) = findSourceFileAndRevision($path);
    278                 $diffFiles->{$path}->{sourceFile} = $sourceFile;
    279                 $diffFiles->{$path}->{sourceRevision} = $sourceRevision;
    280             }
    281         } else {
    282             print STDERR $line, "\n";
    283         }
    284     }
    285     close STAT;
    286 }
    287 
    288 sub hunkHeaderLineRegExForFile($)
    289 {
    290     my ($file) = @_;
    291 
    292     my $startOfObjCInterfaceRegEx = "@(implementation\\|interface\\|protocol)";
    293     return "^[-+]\\|$startOfObjCInterfaceRegEx" if $file =~ /\.mm?$/;
    294     return "^$startOfObjCInterfaceRegEx" if $file =~ /^(.*\/)?(mac|objc)\// && $file =~ /\.h$/;
    295 }
    296 
    297 sub isBinaryMimeType($)
    298 {
    299     my ($file) = @_;
    300     my $mimeType = findMimeType($file);
    301     return 0 if (!$mimeType || substr($mimeType, 0, 5) eq "text/");
    302     return 1;
    303 }
    304 
    305 sub manufacturePatchForAdditionWithHistory($)
    306 {
    307     my ($fileData) = @_;
    308     my $file = $fileData->{path};
    309     print "Index: ${file}\n";
    310     print "=" x 67, "\n";
    311     my $sourceFile = $fileData->{sourceFile};
    312     my $sourceRevision = $fileData->{sourceRevision};
    313     print "--- ${file}\t(revision ${sourceRevision})\t(from ${sourceFile}:${sourceRevision})\n";
    314     print "+++ ${file}\t(working copy)\n";
    315     if ($fileData->{isBinary}) {
    316         print "\nCannot display: file marked as a binary type.\n";
    317         my $mimeType = findMimeType($file, $sourceRevision);
    318         print "svn:mime-type = ${mimeType}\n\n";
    319     } else {
    320         my $escapedSourceFile = escapeSubversionPath($sourceFile);
    321         print `svn cat ${escapedSourceFile} | diff -u $devNull - | tail -n +3`;
    322     }
    323 }
    324 
    325 # Sort numeric parts of strings as numbers, other parts as strings.
    326 # Makes 1.33 come after 1.3, which is cool.
    327 sub numericcmp($$)
    328 {
    329     my ($aa, $bb) = @_;
    330 
    331     my @a = split /(\d+)/, $aa;
    332     my @b = split /(\d+)/, $bb;
    333 
    334     # Compare one chunk at a time.
    335     # Each chunk is either all numeric digits, or all not numeric digits.
    336     while (@a && @b) {
    337         my $a = shift @a;
    338         my $b = shift @b;
    339 
    340         # Use numeric comparison if chunks are non-equal numbers.
    341         return $a <=> $b if $a =~ /^\d/ && $b =~ /^\d/ && $a != $b;
    342 
    343         # Use string comparison if chunks are any other kind of non-equal string.
    344         return $a cmp $b if $a ne $b;
    345     }
    346 
    347     # One of the two is now empty; compare lengths for result in this case.
    348     return @a <=> @b;
    349 }
    350 
    351 sub outputBinaryContent($)
    352 {
    353     my ($path) = @_;
    354     # Deletion
    355     return if (! -e $path);
    356     # Addition or Modification
    357     my $buffer;
    358     open BINARY, $path  or die;
    359     while (read(BINARY, $buffer, 60*57)) {
    360         print encode_base64($buffer);
    361     }
    362     close BINARY;
    363     print "\n";
    364 }
    365 
    366 # Sort first by directory, then by file, so all paths in one directory are grouped
    367 # rather than being interspersed with items from subdirectories.
    368 # Use numericcmp to sort directory and filenames to make order logical.
    369 # Also include a special case for ChangeLog, which comes first in any directory.
    370 sub pathcmp($$)
    371 {
    372     my ($fileDataA, $fileDataB) = @_;
    373 
    374     my ($dira, $namea) = splitpath($fileDataA->{path});
    375     my ($dirb, $nameb) = splitpath($fileDataB->{path});
    376 
    377     return numericcmp($dira, $dirb) if $dira ne $dirb;
    378     return -1 if $namea eq "ChangeLog" && $nameb ne "ChangeLog";
    379     return +1 if $namea ne "ChangeLog" && $nameb eq "ChangeLog";
    380     return numericcmp($namea, $nameb);
    381 }
    382 
    383 sub processPaths(\@)
    384 {
    385     my ($paths) = @_;
    386     return ("." => 1) if (!@{$paths});
    387 
    388     my %result = ();
    389 
    390     for my $file (@{$paths}) {
    391         die "can't handle absolute paths like \"$file\"\n" if File::Spec->file_name_is_absolute($file);
    392         die "can't handle empty string path\n" if $file eq "";
    393         die "can't handle path with single quote in the name like \"$file\"\n" if $file =~ /'/; # ' (keep Xcode syntax highlighting happy)
    394 
    395         my $untouchedFile = $file;
    396 
    397         $file = canonicalizePath($file);
    398 
    399         die "can't handle paths with .. like \"$untouchedFile\"\n" if $file =~ m|/\.\./|;
    400 
    401         $result{$file} = 1;
    402     }
    403 
    404     return ("." => 1) if ($result{"."});
    405 
    406     # Remove any paths that also have a parent listed.
    407     for my $path (keys %result) {
    408         for (my $parent = dirname($path); $parent ne '.'; $parent = dirname($parent)) {
    409             if ($result{$parent}) {
    410                 delete $result{$path};
    411                 last;
    412             }
    413         }
    414     }
    415 
    416     return %result;
    417 }
    418 
    419 # Break up a path into the directory (with slash) and base name.
    420 sub splitpath($)
    421 {
    422     my ($path) = @_;
    423 
    424     my $pathSeparator = "/";
    425     my $dirname = dirname($path) . $pathSeparator;
    426     $dirname = "" if $dirname eq "." . $pathSeparator;
    427 
    428     return ($dirname, basename($path));
    429 }
    430 
    431 # Sort so source code files appear before test files.
    432 sub testfilecmp($$)
    433 {
    434     my ($fileDataA, $fileDataB) = @_;
    435     return $fileDataA->{isTestFile} <=> $fileDataB->{isTestFile};
    436 }
    437 
    438