Home | History | Annotate | Download | only in Scripts 
      1 #!/usr/bin/perl -w
      2 
      3 # Copyright (C) 2005, 2006, 2007 Apple Inc.  All rights reserved.
      4 # Copyright (C) 2009 Cameron McCormack <cam (at] mcc.id.au>
      5 # Copyright (C) 2010 Chris Jerdonek (chris.jerdonek (at] gmail.com)
      6 #
      7 # Redistribution and use in source and binary forms, with or without
      8 # modification, are permitted provided that the following conditions
      9 # are met:
     10 #
     11 # 1.  Redistributions of source code must retain the above copyright
     12 #     notice, this list of conditions and the following disclaimer.
     13 # 2.  Redistributions in binary form must reproduce the above copyright
     14 #     notice, this list of conditions and the following disclaimer in the
     15 #     documentation and/or other materials provided with the distribution.
     16 # 3.  Neither the name of Apple Computer, Inc. ("Apple") nor the names of
     17 #     its contributors may be used to endorse or promote products derived
     18 #     from this software without specific prior written permission.
     19 #
     20 # THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
     21 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
     22 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
     23 # DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
     24 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
     25 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
     26 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
     27 # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
     29 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     30 
     31 # "patch" script for WebKit Open Source Project, used to apply patches.
     32 
     33 # Differences from invoking "patch -p0":
     34 #
     35 #   Handles added files (does a svn add with logic to handle local changes).
     36 #   Handles added directories (does a svn add).
     37 #   Handles removed files (does a svn rm with logic to handle local changes).
     38 #   Handles removed directories--those with no more files or directories left in them
     39 #       (does a svn rm).
     40 #   Has mode where it will roll back to svn version numbers in the patch file so svn
     41 #       can do a 3-way merge.
     42 #   Paths from Index: lines are used rather than the paths on the patch lines, which
     43 #       makes patches generated by "cvs diff" work (increasingly unimportant since we
     44 #       use Subversion now).
     45 #   ChangeLog patches use --fuzz=3 to prevent rejects.
     46 #   Handles binary files (requires patches made by svn-create-patch).
     47 #   Handles copied and moved files (requires patches made by svn-create-patch).
     48 #   Handles git-diff patches (without binary changes) created at the top-level directory
     49 #
     50 # Missing features:
     51 #
     52 #   Handle property changes.
     53 #   Handle copied and moved directories (would require patches made by svn-create-patch).
     54 #   When doing a removal, check that old file matches what's being removed.
     55 #   Notice a patch that's being applied at the "wrong level" and make it work anyway.
     56 #   Do a dry run on the whole patch and don't do anything if part of the patch is
     57 #       going to fail (probably too strict unless we exclude ChangeLog).
     58 #   Handle git-diff patches with binary delta
     59 
     60 use strict;
     61 use warnings;
     62 
     63 use Digest::MD5;
     64 use File::Basename;
     65 use File::Spec;
     66 use Getopt::Long;
     67 use MIME::Base64;
     68 use POSIX qw(strftime);
     69 
     70 use FindBin;
     71 use lib $FindBin::Bin;
     72 use VCSUtils;
     73 
     74 sub addDirectoriesIfNeeded($);
     75 sub applyPatch($$;$);
     76 sub checksum($);
     77 sub handleBinaryChange($$);
     78 sub handleGitBinaryChange($$);
     79 sub isDirectoryEmptyForRemoval($);
     80 sub patch($);
     81 sub removeDirectoriesIfNeeded();
     82 
     83 # These should be replaced by an scm class/module:
     84 sub scmKnowsOfFile($);
     85 sub scmCopy($$);
     86 sub scmAdd($);
     87 sub scmRemove($);
     88 
     89 my $merge = 0;
     90 my $showHelp = 0;
     91 my $reviewer;
     92 my $force = 0;
     93 
     94 my $optionParseSuccess = GetOptions(
     95     "merge!" => \$merge,
     96     "help!" => \$showHelp,
     97     "reviewer=s" => \$reviewer,
     98     "force!" => \$force
     99 );
    100 
    101 if (!$optionParseSuccess || $showHelp) {
    102     print STDERR basename($0) . " [-h|--help] [--force] [-m|--merge] [-r|--reviewer name] patch1 [patch2 ...]\n";
    103     exit 1;
    104 }
    105 
    106 my %removeDirectoryIgnoreList = (
    107     '.' => 1,
    108     '..' => 1,
    109     '.git' => 1,
    110     '.svn' => 1,
    111     '_svn' => 1,
    112 );
    113 
    114 my $epochTime = time(); # This is used to set the date in ChangeLog files.
    115 my $globalExitStatus = 0;
    116 
    117 my $repositoryRootPath = determineVCSRoot();
    118 
    119 my %checkedDirectories;
    120 
    121 # Need to use a typeglob to pass the file handle as a parameter,
    122 # otherwise get a bareword error.
    123 my @diffHashRefs = parsePatch(*ARGV);
    124 
    125 print "Parsed " . @diffHashRefs . " diffs from patch file(s).\n";
    126 
    127 my $preparedPatchHash = prepareParsedPatch($force, @diffHashRefs);
    128 
    129 my @copyDiffHashRefs = @{$preparedPatchHash->{copyDiffHashRefs}};
    130 my @nonCopyDiffHashRefs = @{$preparedPatchHash->{nonCopyDiffHashRefs}};
    131 my %sourceRevisions = %{$preparedPatchHash->{sourceRevisionHash}};
    132 
    133 if ($merge) {
    134     die "--merge is currently only supported for SVN" unless isSVN();
    135     # How do we handle Git patches applied to an SVN checkout here?
    136     for my $file (sort keys %sourceRevisions) {
    137         my $version = $sourceRevisions{$file};
    138         print "Getting version $version of $file\n";
    139         my $escapedFile = escapeSubversionPath($file);
    140         system("svn", "update", "-r", $version, $escapedFile) == 0 or die "Failed to run svn update -r $version $escapedFile.";
    141     }
    142 }
    143 
    144 # Handle copied and moved files first since moved files may have their
    145 # source deleted before the move.
    146 for my $copyDiffHashRef (@copyDiffHashRefs) {
    147     my $indexPath = $copyDiffHashRef->{indexPath};
    148     my $copiedFromPath = $copyDiffHashRef->{copiedFromPath};
    149 
    150     addDirectoriesIfNeeded(dirname($indexPath));
    151     scmCopy($copiedFromPath, $indexPath);
    152 }
    153 
    154 for my $diffHashRef (@nonCopyDiffHashRefs) {
    155     patch($diffHashRef);
    156 }
    157 
    158 removeDirectoriesIfNeeded();
    159 
    160 exit $globalExitStatus;
    161 
    162 sub addDirectoriesIfNeeded($)
    163 {
    164     # Git removes a directory once the last file in it is removed. We need
    165     # explicitly check for the existence of each directory along the path
    166     # (and create it if it doesn't) so as to support patches that move all files in
    167     # directory A to A/B. That is, we cannot depend on %checkedDirectories.
    168     my ($path) = @_;
    169     my @dirs = File::Spec->splitdir($path);
    170     my $dir = ".";
    171     while (scalar @dirs) {
    172         $dir = File::Spec->catdir($dir, shift @dirs);
    173         next if !isGit() && exists $checkedDirectories{$dir};
    174         if (! -e $dir) {
    175             mkdir $dir or die "Failed to create required directory '$dir' for path '$path'\n";
    176             scmAdd($dir);
    177             $checkedDirectories{$dir} = 1;
    178         }
    179         elsif (-d $dir) {
    180             # SVN prints "svn: warning: 'directory' is already under version control"
    181             # if you try and add a directory which is already in the repository.
    182             # Git will ignore the add, but re-adding large directories can be sloooow.
    183             # So we check first to see if the directory is under version control first.
    184             if (!scmKnowsOfFile($dir)) {
    185                 scmAdd($dir);
    186             }
    187             $checkedDirectories{$dir} = 1;
    188         }
    189         else {
    190             die "'$dir' exists, but is not a directory";
    191         }
    192     }
    193 }
    194 
    195 # Args:
    196 #   $patch: a patch string.
    197 #   $pathRelativeToRoot: the path of the file to be patched, relative to the
    198 #                        repository root. This should normally be the path
    199 #                        found in the patch's "Index:" line.
    200 #   $options: a reference to an array of options to pass to the patch command.
    201 sub applyPatch($$;$)
    202 {
    203     my ($patch, $pathRelativeToRoot, $options) = @_;
    204 
    205     my $optionalArgs = {options => $options, ensureForce => $force};
    206 
    207     my $exitStatus = runPatchCommand($patch, $repositoryRootPath, $pathRelativeToRoot, $optionalArgs);
    208 
    209     if ($exitStatus) {
    210         $globalExitStatus = $exitStatus;
    211     }
    212 }
    213 
    214 sub checksum($)
    215 {
    216     my $file = shift;
    217     open(FILE, $file) or die "Can't open '$file': $!";
    218     binmode(FILE);
    219     my $checksum = Digest::MD5->new->addfile(*FILE)->hexdigest();
    220     close(FILE);
    221     return $checksum;
    222 }
    223 
    224 sub handleBinaryChange($$)
    225 {
    226     my ($fullPath, $contents) = @_;
    227     # [A-Za-z0-9+/] is the class of allowed base64 characters.
    228     # One or more lines, at most 76 characters in length.
    229     # The last line is allowed to have up to two '=' characters at the end (to signify padding).
    230     if ($contents =~ m#((\n[A-Za-z0-9+/]{76})*\n[A-Za-z0-9+/]{2,74}?[A-Za-z0-9+/=]{2}\n)#) {
    231         # Addition or Modification
    232         open FILE, ">", $fullPath or die "Failed to open $fullPath.";
    233         print FILE decode_base64($1);
    234         close FILE;
    235         if (!scmKnowsOfFile($fullPath)) {
    236             # Addition
    237             scmAdd($fullPath);
    238         }
    239     } else {
    240         # Deletion
    241         scmRemove($fullPath);
    242     }
    243 }
    244 
    245 sub handleGitBinaryChange($$)
    246 {
    247     my ($fullPath, $diffHashRef) = @_;
    248 
    249     my $contents = $diffHashRef->{svnConvertedText};
    250 
    251     my ($binaryChunkType, $binaryChunk, $reverseBinaryChunkType, $reverseBinaryChunk) = decodeGitBinaryPatch($contents, $fullPath);
    252 
    253     my $isFileAddition = $diffHashRef->{isNew};
    254     my $isFileDeletion = $diffHashRef->{isDeletion};
    255 
    256     my $originalContents = "";
    257     if (open FILE, $fullPath) {
    258         die "$fullPath already exists" if $isFileAddition;
    259 
    260         $originalContents = join("", <FILE>);
    261         close FILE;
    262     }
    263 
    264     if ($reverseBinaryChunkType eq "literal") {
    265         die "Original content of $fullPath mismatches" if $originalContents ne $reverseBinaryChunk;
    266     }
    267 
    268     if ($isFileDeletion) {
    269         scmRemove($fullPath);
    270     } else {
    271         # Addition or Modification
    272         my $out = "";
    273         if ($binaryChunkType eq "delta") {
    274             $out = applyGitBinaryPatchDelta($binaryChunk, $originalContents);
    275         } else {
    276             $out = $binaryChunk;
    277         }
    278         if ($reverseBinaryChunkType eq "delta") {
    279             die "Original content of $fullPath mismatches" if $originalContents ne applyGitBinaryPatchDelta($reverseBinaryChunk, $out);
    280         }
    281         open FILE, ">", $fullPath or die "Failed to open $fullPath.";
    282         print FILE $out;
    283         close FILE;
    284         if ($isFileAddition) {
    285             scmAdd($fullPath);
    286         }
    287     }
    288 }
    289 
    290 sub isDirectoryEmptyForRemoval($)
    291 {
    292     my ($dir) = @_;
    293     return 1 unless -d $dir;
    294     my $directoryIsEmpty = 1;
    295     opendir DIR, $dir or die "Could not open '$dir' to list files: $?";
    296     for (my $item = readdir DIR; $item && $directoryIsEmpty; $item = readdir DIR) {
    297         next if exists $removeDirectoryIgnoreList{$item};
    298         if (-d File::Spec->catdir($dir, $item)) {
    299             $directoryIsEmpty = 0;
    300         } else {
    301             next if (scmWillDeleteFile(File::Spec->catdir($dir, $item)));
    302             $directoryIsEmpty = 0;
    303         }
    304     }
    305     closedir DIR;
    306     return $directoryIsEmpty;
    307 }
    308 
    309 # Args:
    310 #   $diffHashRef: a diff hash reference of the type returned by parsePatch().
    311 sub patch($)
    312 {
    313     my ($diffHashRef) = @_;
    314 
    315     # Make sure $patch is initialized to some value.  A deletion can have no
    316     # svnConvertedText property in the case of a deletion resulting from a
    317     # Git rename.
    318     my $patch = $diffHashRef->{svnConvertedText} || "";
    319 
    320     my $fullPath = $diffHashRef->{indexPath};
    321     my $isBinary = $diffHashRef->{isBinary};
    322     my $isGit = $diffHashRef->{isGit};
    323     my $hasTextChunks = $patch && $diffHashRef->{numTextChunks};
    324 
    325     my $deletion = 0;
    326     my $addition = 0;
    327 
    328     $addition = 1 if ($diffHashRef->{isNew} || $patch =~ /\n@@ -0,0 .* @@/);
    329     $deletion = 1 if ($diffHashRef->{isDeletion} || $patch =~ /\n@@ .* \+0,0 @@/);
    330 
    331     if (!$addition && !$deletion && !$isBinary && $hasTextChunks) {
    332         # Standard patch, patch tool can handle this.
    333         if (basename($fullPath) eq "ChangeLog") {
    334             my $changeLogDotOrigExisted = -f "${fullPath}.orig";
    335             my $changeLogHash = fixChangeLogPatch($patch);
    336             my $newPatch = setChangeLogDateAndReviewer($changeLogHash->{patch}, $reviewer, $epochTime);
    337             applyPatch($newPatch, $fullPath, ["--fuzz=3"]);
    338             unlink("${fullPath}.orig") if (! $changeLogDotOrigExisted);
    339         } else {
    340             applyPatch($patch, $fullPath);
    341         }
    342     } else {
    343         # Either a deletion, an addition or a binary change.
    344 
    345         addDirectoriesIfNeeded(dirname($fullPath));
    346 
    347         if ($isBinary) {
    348             if ($isGit) {
    349                 handleGitBinaryChange($fullPath, $diffHashRef);
    350             } else {
    351                 handleBinaryChange($fullPath, $patch) if $patch;
    352             }
    353         } elsif ($deletion) {
    354             applyPatch($patch, $fullPath, ["--force"]) if $patch;
    355             scmRemove($fullPath);
    356         } elsif ($addition) {
    357             # Addition
    358             rename($fullPath, "$fullPath.orig") if -e $fullPath;
    359             applyPatch($patch, $fullPath) if $patch;
    360             unlink("$fullPath.orig") if -e "$fullPath.orig" && checksum($fullPath) eq checksum("$fullPath.orig");
    361             scmAdd($fullPath);
    362             my $escapedFullPath = escapeSubversionPath("$fullPath.orig");
    363             # What is this for?
    364             system("svn", "stat", "$escapedFullPath") if isSVN() && -e "$fullPath.orig";
    365         }
    366     }
    367 
    368     scmToggleExecutableBit($fullPath, $diffHashRef->{executableBitDelta}) if defined($diffHashRef->{executableBitDelta});
    369 }
    370 
    371 sub removeDirectoriesIfNeeded()
    372 {
    373     foreach my $dir (reverse sort keys %checkedDirectories) {
    374         if (isDirectoryEmptyForRemoval($dir)) {
    375             scmRemove($dir);
    376         }
    377     }
    378 }
    379 
    380 # This could be made into a more general "status" call, except svn and git
    381 # have different ideas about "moving" files which might get confusing.
    382 sub scmWillDeleteFile($)
    383 {
    384     my ($path) = @_;
    385     if (isSVN()) {
    386         my $svnOutput = svnStatus($path);
    387         return 1 if $svnOutput && substr($svnOutput, 0, 1) eq "D";
    388     } elsif (isGit()) {
    389         my $command = runCommand("git", "diff-index", "--name-status", "HEAD", "--", $path);
    390         return 1 if $command->{stdout} && substr($command->{stdout}, 0, 1) eq "D";
    391     }
    392     return 0;
    393 }
    394 
    395 # Return whether the file at the given path is known to Git.
    396 #
    397 # This method outputs a message like the following to STDERR when
    398 # returning false:
    399 #
    400 # "error: pathspec 'test.png' did not match any file(s) known to git.
    401 #  Did you forget to 'git add'?"
    402 sub gitKnowsOfFile($)
    403 {
    404     my $path = shift;
    405 
    406     `git ls-files --error-unmatch -- $path`;
    407     my $exitStatus = exitStatus($?);
    408     return $exitStatus == 0;
    409 }
    410 
    411 sub scmKnowsOfFile($)
    412 {
    413     my ($path) = @_;
    414     if (isSVN()) {
    415         my $svnOutput = svnStatus($path);
    416         # This will match more than intended.  ? might not be the first field in the status
    417         if ($svnOutput && $svnOutput =~ m#\?\s+$path\n#) {
    418             return 0;
    419         }
    420         # This does not handle errors well.
    421         return 1;
    422     } elsif (isGit()) {
    423         my @result = callSilently(\&gitKnowsOfFile, $path);
    424         return $result[0];
    425     }
    426 }
    427 
    428 sub scmCopy($$)
    429 {
    430     my ($source, $destination) = @_;
    431     if (isSVN()) {
    432         my $escapedSource = escapeSubversionPath($source);
    433         my $escapedDestination = escapeSubversionPath($destination);
    434         system("svn", "copy", $escapedSource, $escapedDestination) == 0 or die "Failed to svn copy $escapedSource $escapedDestination.";
    435     } elsif (isGit()) {
    436         system("cp", $source, $destination) == 0 or die "Failed to copy $source $destination.";
    437         system("git", "add", $destination) == 0 or die "Failed to git add $destination.";
    438     }
    439 }
    440 
    441 sub scmAdd($)
    442 {
    443     my ($path) = @_;
    444     if (isSVN()) {
    445         my $escapedPath = escapeSubversionPath($path);
    446         system("svn", "add", $escapedPath) == 0 or die "Failed to svn add $escapedPath.";
    447     } elsif (isGit()) {
    448         system("git", "add", $path) == 0 or die "Failed to git add $path.";
    449     }
    450 }
    451 
    452 sub scmRemove($)
    453 {
    454     my ($path) = @_;
    455     if (isSVN()) {
    456         # SVN is very verbose when removing directories.  Squelch all output except the last line.
    457         my $svnOutput;
    458         my $escapedPath = escapeSubversionPath($path);
    459         open SVN, "svn rm --force '$escapedPath' |" or die "svn rm --force '$escapedPath' failed!";
    460         # Only print the last line.  Subversion outputs all changed statuses below $dir
    461         while (<SVN>) {
    462             $svnOutput = $_;
    463         }
    464         close SVN;
    465         print $svnOutput if $svnOutput;
    466     } elsif (isGit()) {
    467         # Git removes a directory if it becomes empty when the last file it contains is
    468         # removed by `git rm`. In svn-apply this can happen when a directory is being
    469         # removed in a patch, and all of the files inside of the directory are removed
    470         # before attemping to remove the directory itself. In this case, Git will have
    471         # already deleted the directory and `git rm` would exit with an error claiming
    472         # there was no file. The --ignore-unmatch switch gracefully handles this case.
    473         system("git", "rm", "--force", "--ignore-unmatch", $path) == 0 or die "Failed to git rm --force --ignore-unmatch $path.";
    474     }
    475 }
    476