Home | History | Annotate | Download | only in push-to-trunk 
      1 #!/usr/bin/env python
      2 # Copyright 2013 the V8 project authors. All rights reserved.
      3 # Redistribution and use in source and binary forms, with or without
      4 # modification, are permitted provided that the following conditions are
      5 # met:
      6 #
      7 #     * Redistributions of source code must retain the above copyright
      8 #       notice, this list of conditions and the following disclaimer.
      9 #     * Redistributions in binary form must reproduce the above
     10 #       copyright notice, this list of conditions and the following
     11 #       disclaimer in the documentation and/or other materials provided
     12 #       with the distribution.
     13 #     * Neither the name of Google Inc. nor the names of its
     14 #       contributors may be used to endorse or promote products derived
     15 #       from this software without specific prior written permission.
     16 #
     17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
     18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
     19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
     20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
     21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
     22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
     23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
     27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     28 
     29 import datetime
     30 import os
     31 import re
     32 import subprocess
     33 import sys
     34 import textwrap
     35 import time
     36 import urllib2
     37 
     38 PERSISTFILE_BASENAME = "PERSISTFILE_BASENAME"
     39 TEMP_BRANCH = "TEMP_BRANCH"
     40 BRANCHNAME = "BRANCHNAME"
     41 DOT_GIT_LOCATION = "DOT_GIT_LOCATION"
     42 VERSION_FILE = "VERSION_FILE"
     43 CHANGELOG_FILE = "CHANGELOG_FILE"
     44 CHANGELOG_ENTRY_FILE = "CHANGELOG_ENTRY_FILE"
     45 COMMITMSG_FILE = "COMMITMSG_FILE"
     46 PATCH_FILE = "PATCH_FILE"
     47 
     48 
     49 def TextToFile(text, file_name):
     50   with open(file_name, "w") as f:
     51     f.write(text)
     52 
     53 
     54 def AppendToFile(text, file_name):
     55   with open(file_name, "a") as f:
     56     f.write(text)
     57 
     58 
     59 def LinesInFile(file_name):
     60   with open(file_name) as f:
     61     for line in f:
     62       yield line
     63 
     64 
     65 def FileToText(file_name):
     66   with open(file_name) as f:
     67     return f.read()
     68 
     69 
     70 def MSub(rexp, replacement, text):
     71   return re.sub(rexp, replacement, text, flags=re.MULTILINE)
     72 
     73 
     74 def Fill80(line):
     75   # Replace tabs and remove surrounding space.
     76   line = re.sub(r"\t", r"        ", line.strip())
     77 
     78   # Format with 8 characters indentation and line width 80.
     79   return textwrap.fill(line, width=80, initial_indent="        ",
     80                        subsequent_indent="        ")
     81 
     82 
     83 def GetLastChangeLogEntries(change_log_file):
     84   result = []
     85   for line in LinesInFile(change_log_file):
     86     if re.search(r"^\d{4}-\d{2}-\d{2}:", line) and result: break
     87     result.append(line)
     88   return "".join(result)
     89 
     90 
     91 def MakeComment(text):
     92   return MSub(r"^( ?)", "#", text)
     93 
     94 
     95 def StripComments(text):
     96   # Use split not splitlines to keep terminal newlines.
     97   return "\n".join(filter(lambda x: not x.startswith("#"), text.split("\n")))
     98 
     99 
    100 def MakeChangeLogBody(commit_messages, auto_format=False):
    101   result = ""
    102   added_titles = set()
    103   for (title, body, author) in commit_messages:
    104     # TODO(machenbach): Better check for reverts. A revert should remove the
    105     # original CL from the actual log entry.
    106     title = title.strip()
    107     if auto_format:
    108       # Only add commits that set the LOG flag correctly.
    109       log_exp = r"^[ \t]*LOG[ \t]*=[ \t]*(?:(?:Y(?:ES)?)|TRUE)"
    110       if not re.search(log_exp, body, flags=re.I | re.M):
    111         continue
    112       # Never include reverts.
    113       if title.startswith("Revert "):
    114         continue
    115       # Don't include duplicates.
    116       if title in added_titles:
    117         continue
    118 
    119     # Add and format the commit's title and bug reference. Move dot to the end.
    120     added_titles.add(title)
    121     raw_title = re.sub(r"(\.|\?|!)$", "", title)
    122     bug_reference = MakeChangeLogBugReference(body)
    123     space = " " if bug_reference else ""
    124     result += "%s\n" % Fill80("%s%s%s." % (raw_title, space, bug_reference))
    125 
    126     # Append the commit's author for reference if not in auto-format mode.
    127     if not auto_format:
    128       result += "%s\n" % Fill80("(%s)" % author.strip())
    129 
    130     result += "\n"
    131   return result
    132 
    133 
    134 def MakeChangeLogBugReference(body):
    135   """Grep for "BUG=xxxx" lines in the commit message and convert them to
    136   "(issue xxxx)".
    137   """
    138   crbugs = []
    139   v8bugs = []
    140 
    141   def AddIssues(text):
    142     ref = re.match(r"^BUG[ \t]*=[ \t]*(.+)$", text.strip())
    143     if not ref:
    144       return
    145     for bug in ref.group(1).split(","):
    146       bug = bug.strip()
    147       match = re.match(r"^v8:(\d+)$", bug)
    148       if match: v8bugs.append(int(match.group(1)))
    149       else:
    150         match = re.match(r"^(?:chromium:)?(\d+)$", bug)
    151         if match: crbugs.append(int(match.group(1)))
    152 
    153   # Add issues to crbugs and v8bugs.
    154   map(AddIssues, body.splitlines())
    155 
    156   # Filter duplicates, sort, stringify.
    157   crbugs = map(str, sorted(set(crbugs)))
    158   v8bugs = map(str, sorted(set(v8bugs)))
    159 
    160   bug_groups = []
    161   def FormatIssues(prefix, bugs):
    162     if len(bugs) > 0:
    163       plural = "s" if len(bugs) > 1 else ""
    164       bug_groups.append("%sissue%s %s" % (prefix, plural, ", ".join(bugs)))
    165 
    166   FormatIssues("", v8bugs)
    167   FormatIssues("Chromium ", crbugs)
    168 
    169   if len(bug_groups) > 0:
    170     return "(%s)" % ", ".join(bug_groups)
    171   else:
    172     return ""
    173 
    174 
    175 # Some commands don't like the pipe, e.g. calling vi from within the script or
    176 # from subscripts like git cl upload.
    177 def Command(cmd, args="", prefix="", pipe=True):
    178   # TODO(machenbach): Use timeout.
    179   cmd_line = "%s %s %s" % (prefix, cmd, args)
    180   print "Command: %s" % cmd_line
    181   try:
    182     if pipe:
    183       return subprocess.check_output(cmd_line, shell=True)
    184     else:
    185       return subprocess.check_call(cmd_line, shell=True)
    186   except subprocess.CalledProcessError:
    187     return None
    188 
    189 
    190 # Wrapper for side effects.
    191 class SideEffectHandler(object):
    192   def Command(self, cmd, args="", prefix="", pipe=True):
    193     return Command(cmd, args, prefix, pipe)
    194 
    195   def ReadLine(self):
    196     return sys.stdin.readline().strip()
    197 
    198   def ReadURL(self, url):
    199     # pylint: disable=E1121
    200     url_fh = urllib2.urlopen(url, None, 60)
    201     try:
    202       return url_fh.read()
    203     finally:
    204       url_fh.close()
    205 
    206   def Sleep(self, seconds):
    207     time.sleep(seconds)
    208 
    209   def GetDate(self):
    210     return datetime.date.today().strftime("%Y-%m-%d")
    211 
    212 DEFAULT_SIDE_EFFECT_HANDLER = SideEffectHandler()
    213 
    214 
    215 class Step(object):
    216   def __init__(self, text, requires, number, config, state, options, handler):
    217     self._text = text
    218     self._requires = requires
    219     self._number = number
    220     self._config = config
    221     self._state = state
    222     self._options = options
    223     self._side_effect_handler = handler
    224     assert self._number >= 0
    225     assert self._config is not None
    226     assert self._state is not None
    227     assert self._side_effect_handler is not None
    228 
    229   def Config(self, key):
    230     return self._config[key]
    231 
    232   def IsForced(self):
    233     return self._options and self._options.f
    234 
    235   def IsManual(self):
    236     return self._options and self._options.m
    237 
    238   def Run(self):
    239     if self._requires:
    240       self.RestoreIfUnset(self._requires)
    241       if not self._state[self._requires]:
    242         return
    243     print ">>> Step %d: %s" % (self._number, self._text)
    244     self.RunStep()
    245 
    246   def RunStep(self):
    247     raise NotImplementedError
    248 
    249   def Retry(self, cb, retry_on=None, wait_plan=None):
    250     """ Retry a function.
    251     Params:
    252       cb: The function to retry.
    253       retry_on: A callback that takes the result of the function and returns
    254                 True if the function should be retried. A function throwing an
    255                 exception is always retried.
    256       wait_plan: A list of waiting delays between retries in seconds. The
    257                  maximum number of retries is len(wait_plan).
    258     """
    259     retry_on = retry_on or (lambda x: False)
    260     wait_plan = list(wait_plan or [])
    261     wait_plan.reverse()
    262     while True:
    263       got_exception = False
    264       try:
    265         result = cb()
    266       except Exception:
    267         got_exception = True
    268       if got_exception or retry_on(result):
    269         if not wait_plan:
    270           raise Exception("Retried too often. Giving up.")
    271         wait_time = wait_plan.pop()
    272         print "Waiting for %f seconds." % wait_time
    273         self._side_effect_handler.Sleep(wait_time)
    274         print "Retrying..."
    275       else:
    276         return result
    277 
    278   def ReadLine(self, default=None):
    279     # Don't prompt in forced mode.
    280     if not self.IsManual() and default is not None:
    281       print "%s (forced)" % default
    282       return default
    283     else:
    284       return self._side_effect_handler.ReadLine()
    285 
    286   def Git(self, args="", prefix="", pipe=True, retry_on=None):
    287     cmd = lambda: self._side_effect_handler.Command("git", args, prefix, pipe)
    288     return self.Retry(cmd, retry_on, [5, 30])
    289 
    290   def Editor(self, args):
    291     if not self.IsForced():
    292       return self._side_effect_handler.Command(os.environ["EDITOR"], args,
    293                                                pipe=False)
    294 
    295   def ReadURL(self, url, retry_on=None, wait_plan=None):
    296     wait_plan = wait_plan or [3, 60, 600]
    297     cmd = lambda: self._side_effect_handler.ReadURL(url)
    298     return self.Retry(cmd, retry_on, wait_plan)
    299 
    300   def GetDate(self):
    301     return self._side_effect_handler.GetDate()
    302 
    303   def Die(self, msg=""):
    304     if msg != "":
    305       print "Error: %s" % msg
    306     print "Exiting"
    307     raise Exception(msg)
    308 
    309   def DieNoManualMode(self, msg=""):
    310     if not self.IsManual():
    311       msg = msg or "Only available in manual mode."
    312       self.Die(msg)
    313 
    314   def Confirm(self, msg):
    315     print "%s [Y/n] " % msg,
    316     answer = self.ReadLine(default="Y")
    317     return answer == "" or answer == "Y" or answer == "y"
    318 
    319   def DeleteBranch(self, name):
    320     git_result = self.Git("branch").strip()
    321     for line in git_result.splitlines():
    322       if re.match(r".*\s+%s$" % name, line):
    323         msg = "Branch %s exists, do you want to delete it?" % name
    324         if self.Confirm(msg):
    325           if self.Git("branch -D %s" % name) is None:
    326             self.Die("Deleting branch '%s' failed." % name)
    327           print "Branch %s deleted." % name
    328         else:
    329           msg = "Can't continue. Please delete branch %s and try again." % name
    330           self.Die(msg)
    331 
    332   def Persist(self, var, value):
    333     value = value or "__EMPTY__"
    334     TextToFile(value, "%s-%s" % (self._config[PERSISTFILE_BASENAME], var))
    335 
    336   def Restore(self, var):
    337     value = FileToText("%s-%s" % (self._config[PERSISTFILE_BASENAME], var))
    338     value = value or self.Die("Variable '%s' could not be restored." % var)
    339     return "" if value == "__EMPTY__" else value
    340 
    341   def RestoreIfUnset(self, var_name):
    342     if self._state.get(var_name) is None:
    343       self._state[var_name] = self.Restore(var_name)
    344 
    345   def InitialEnvironmentChecks(self):
    346     # Cancel if this is not a git checkout.
    347     if not os.path.exists(self._config[DOT_GIT_LOCATION]):
    348       self.Die("This is not a git checkout, this script won't work for you.")
    349 
    350     # Cancel if EDITOR is unset or not executable.
    351     if (not self.IsForced() and (not os.environ.get("EDITOR") or
    352         Command("which", os.environ["EDITOR"]) is None)):
    353       self.Die("Please set your EDITOR environment variable, you'll need it.")
    354 
    355   def CommonPrepare(self):
    356     # Check for a clean workdir.
    357     if self.Git("status -s -uno").strip() != "":
    358       self.Die("Workspace is not clean. Please commit or undo your changes.")
    359 
    360     # Persist current branch.
    361     current_branch = ""
    362     git_result = self.Git("status -s -b -uno").strip()
    363     for line in git_result.splitlines():
    364       match = re.match(r"^## (.+)", line)
    365       if match:
    366         current_branch = match.group(1)
    367         break
    368     self.Persist("current_branch", current_branch)
    369 
    370     # Fetch unfetched revisions.
    371     if self.Git("svn fetch") is None:
    372       self.Die("'git svn fetch' failed.")
    373 
    374   def PrepareBranch(self):
    375     # Get ahold of a safe temporary branch and check it out.
    376     self.RestoreIfUnset("current_branch")
    377     if self._state["current_branch"] != self._config[TEMP_BRANCH]:
    378       self.DeleteBranch(self._config[TEMP_BRANCH])
    379       self.Git("checkout -b %s" % self._config[TEMP_BRANCH])
    380 
    381     # Delete the branch that will be created later if it exists already.
    382     self.DeleteBranch(self._config[BRANCHNAME])
    383 
    384   def CommonCleanup(self):
    385     self.RestoreIfUnset("current_branch")
    386     self.Git("checkout -f %s" % self._state["current_branch"])
    387     if self._config[TEMP_BRANCH] != self._state["current_branch"]:
    388       self.Git("branch -D %s" % self._config[TEMP_BRANCH])
    389     if self._config[BRANCHNAME] != self._state["current_branch"]:
    390       self.Git("branch -D %s" % self._config[BRANCHNAME])
    391 
    392     # Clean up all temporary files.
    393     Command("rm", "-f %s*" % self._config[PERSISTFILE_BASENAME])
    394 
    395   def ReadAndPersistVersion(self, prefix=""):
    396     def ReadAndPersist(var_name, def_name):
    397       match = re.match(r"^#define %s\s+(\d*)" % def_name, line)
    398       if match:
    399         value = match.group(1)
    400         self.Persist("%s%s" % (prefix, var_name), value)
    401         self._state["%s%s" % (prefix, var_name)] = value
    402     for line in LinesInFile(self._config[VERSION_FILE]):
    403       for (var_name, def_name) in [("major", "MAJOR_VERSION"),
    404                                    ("minor", "MINOR_VERSION"),
    405                                    ("build", "BUILD_NUMBER"),
    406                                    ("patch", "PATCH_LEVEL")]:
    407         ReadAndPersist(var_name, def_name)
    408 
    409   def RestoreVersionIfUnset(self, prefix=""):
    410     for v in ["major", "minor", "build", "patch"]:
    411       self.RestoreIfUnset("%s%s" % (prefix, v))
    412 
    413   def WaitForLGTM(self):
    414     print ("Please wait for an LGTM, then type \"LGTM<Return>\" to commit "
    415            "your change. (If you need to iterate on the patch or double check "
    416            "that it's sane, do so in another shell, but remember to not "
    417            "change the headline of the uploaded CL.")
    418     answer = ""
    419     while answer != "LGTM":
    420       print "> ",
    421       answer = self.ReadLine("LGTM" if self.IsForced() else None)
    422       if answer != "LGTM":
    423         print "That was not 'LGTM'."
    424 
    425   def WaitForResolvingConflicts(self, patch_file):
    426     print("Applying the patch \"%s\" failed. Either type \"ABORT<Return>\", "
    427           "or resolve the conflicts, stage *all* touched files with "
    428           "'git add', and type \"RESOLVED<Return>\"")
    429     self.DieNoManualMode()
    430     answer = ""
    431     while answer != "RESOLVED":
    432       if answer == "ABORT":
    433         self.Die("Applying the patch failed.")
    434       if answer != "":
    435         print "That was not 'RESOLVED' or 'ABORT'."
    436       print "> ",
    437       answer = self.ReadLine()
    438 
    439   # Takes a file containing the patch to apply as first argument.
    440   def ApplyPatch(self, patch_file, reverse_patch=""):
    441     args = "apply --index --reject %s \"%s\"" % (reverse_patch, patch_file)
    442     if self.Git(args) is None:
    443       self.WaitForResolvingConflicts(patch_file)
    444 
    445 
    446 class UploadStep(Step):
    447   MESSAGE = "Upload for code review."
    448 
    449   def RunStep(self):
    450     if self._options.r:
    451       print "Using account %s for review." % self._options.r
    452       reviewer = self._options.r
    453     else:
    454       print "Please enter the email address of a V8 reviewer for your patch: ",
    455       self.DieNoManualMode("A reviewer must be specified in forced mode.")
    456       reviewer = self.ReadLine()
    457     force_flag = " -f" if not self.IsManual() else ""
    458     args = "cl upload -r \"%s\" --send-mail%s" % (reviewer, force_flag)
    459     # TODO(machenbach): Check output in forced mode. Verify that all required
    460     # base files were uploaded, if not retry.
    461     if self.Git(args, pipe=False) is None:
    462       self.Die("'git cl upload' failed, please try again.")
    463 
    464 
    465 def MakeStep(step_class=Step, number=0, state=None, config=None,
    466              options=None, side_effect_handler=DEFAULT_SIDE_EFFECT_HANDLER):
    467     # Allow to pass in empty dictionaries.
    468     state = state if state is not None else {}
    469     config = config if config is not None else {}
    470 
    471     try:
    472       message = step_class.MESSAGE
    473     except AttributeError:
    474       message = step_class.__name__
    475     try:
    476       requires = step_class.REQUIRES
    477     except AttributeError:
    478       requires = None
    479 
    480     return step_class(message, requires, number=number, config=config,
    481                       state=state, options=options,
    482                       handler=side_effect_handler)
    483 
    484 
    485 def RunScript(step_classes,
    486               config,
    487               options,
    488               side_effect_handler=DEFAULT_SIDE_EFFECT_HANDLER):
    489   state = {}
    490   steps = []
    491   for (number, step_class) in enumerate(step_classes):
    492     steps.append(MakeStep(step_class, number, state, config,
    493                           options, side_effect_handler))
    494 
    495   for step in steps[options.s:]:
    496     step.Run()
    497