Home | History | Annotate | Download | only in push-to-trunk 
      1 #!/usr/bin/env python
      2 # Copyright 2013 the V8 project authors. All rights reserved.
      3 # Redistribution and use in source and binary forms, with or without
      4 # modification, are permitted provided that the following conditions are
      5 # met:
      6 #
      7 #     * Redistributions of source code must retain the above copyright
      8 #       notice, this list of conditions and the following disclaimer.
      9 #     * Redistributions in binary form must reproduce the above
     10 #       copyright notice, this list of conditions and the following
     11 #       disclaimer in the documentation and/or other materials provided
     12 #       with the distribution.
     13 #     * Neither the name of Google Inc. nor the names of its
     14 #       contributors may be used to endorse or promote products derived
     15 #       from this software without specific prior written permission.
     16 #
     17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
     18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
     19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
     20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
     21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
     22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
     23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
     27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     28 
     29 import argparse
     30 import datetime
     31 import imp
     32 import json
     33 import os
     34 import re
     35 import subprocess
     36 import sys
     37 import textwrap
     38 import time
     39 import urllib2
     40 
     41 from git_recipes import GitRecipesMixin
     42 from git_recipes import GitFailedException
     43 
     44 PERSISTFILE_BASENAME = "PERSISTFILE_BASENAME"
     45 BRANCHNAME = "BRANCHNAME"
     46 DOT_GIT_LOCATION = "DOT_GIT_LOCATION"
     47 VERSION_FILE = "VERSION_FILE"
     48 CHANGELOG_FILE = "CHANGELOG_FILE"
     49 CHANGELOG_ENTRY_FILE = "CHANGELOG_ENTRY_FILE"
     50 COMMITMSG_FILE = "COMMITMSG_FILE"
     51 PATCH_FILE = "PATCH_FILE"
     52 
     53 
     54 def TextToFile(text, file_name):
     55   with open(file_name, "w") as f:
     56     f.write(text)
     57 
     58 
     59 def AppendToFile(text, file_name):
     60   with open(file_name, "a") as f:
     61     f.write(text)
     62 
     63 
     64 def LinesInFile(file_name):
     65   with open(file_name) as f:
     66     for line in f:
     67       yield line
     68 
     69 
     70 def FileToText(file_name):
     71   with open(file_name) as f:
     72     return f.read()
     73 
     74 
     75 def MSub(rexp, replacement, text):
     76   return re.sub(rexp, replacement, text, flags=re.MULTILINE)
     77 
     78 
     79 def Fill80(line):
     80   # Replace tabs and remove surrounding space.
     81   line = re.sub(r"\t", r"        ", line.strip())
     82 
     83   # Format with 8 characters indentation and line width 80.
     84   return textwrap.fill(line, width=80, initial_indent="        ",
     85                        subsequent_indent="        ")
     86 
     87 
     88 def MakeComment(text):
     89   return MSub(r"^( ?)", "#", text)
     90 
     91 
     92 def StripComments(text):
     93   # Use split not splitlines to keep terminal newlines.
     94   return "\n".join(filter(lambda x: not x.startswith("#"), text.split("\n")))
     95 
     96 
     97 def MakeChangeLogBody(commit_messages, auto_format=False):
     98   result = ""
     99   added_titles = set()
    100   for (title, body, author) in commit_messages:
    101     # TODO(machenbach): Better check for reverts. A revert should remove the
    102     # original CL from the actual log entry.
    103     title = title.strip()
    104     if auto_format:
    105       # Only add commits that set the LOG flag correctly.
    106       log_exp = r"^[ \t]*LOG[ \t]*=[ \t]*(?:(?:Y(?:ES)?)|TRUE)"
    107       if not re.search(log_exp, body, flags=re.I | re.M):
    108         continue
    109       # Never include reverts.
    110       if title.startswith("Revert "):
    111         continue
    112       # Don't include duplicates.
    113       if title in added_titles:
    114         continue
    115 
    116     # Add and format the commit's title and bug reference. Move dot to the end.
    117     added_titles.add(title)
    118     raw_title = re.sub(r"(\.|\?|!)$", "", title)
    119     bug_reference = MakeChangeLogBugReference(body)
    120     space = " " if bug_reference else ""
    121     result += "%s\n" % Fill80("%s%s%s." % (raw_title, space, bug_reference))
    122 
    123     # Append the commit's author for reference if not in auto-format mode.
    124     if not auto_format:
    125       result += "%s\n" % Fill80("(%s)" % author.strip())
    126 
    127     result += "\n"
    128   return result
    129 
    130 
    131 def MakeChangeLogBugReference(body):
    132   """Grep for "BUG=xxxx" lines in the commit message and convert them to
    133   "(issue xxxx)".
    134   """
    135   crbugs = []
    136   v8bugs = []
    137 
    138   def AddIssues(text):
    139     ref = re.match(r"^BUG[ \t]*=[ \t]*(.+)$", text.strip())
    140     if not ref:
    141       return
    142     for bug in ref.group(1).split(","):
    143       bug = bug.strip()
    144       match = re.match(r"^v8:(\d+)$", bug)
    145       if match: v8bugs.append(int(match.group(1)))
    146       else:
    147         match = re.match(r"^(?:chromium:)?(\d+)$", bug)
    148         if match: crbugs.append(int(match.group(1)))
    149 
    150   # Add issues to crbugs and v8bugs.
    151   map(AddIssues, body.splitlines())
    152 
    153   # Filter duplicates, sort, stringify.
    154   crbugs = map(str, sorted(set(crbugs)))
    155   v8bugs = map(str, sorted(set(v8bugs)))
    156 
    157   bug_groups = []
    158   def FormatIssues(prefix, bugs):
    159     if len(bugs) > 0:
    160       plural = "s" if len(bugs) > 1 else ""
    161       bug_groups.append("%sissue%s %s" % (prefix, plural, ", ".join(bugs)))
    162 
    163   FormatIssues("", v8bugs)
    164   FormatIssues("Chromium ", crbugs)
    165 
    166   if len(bug_groups) > 0:
    167     return "(%s)" % ", ".join(bug_groups)
    168   else:
    169     return ""
    170 
    171 
    172 # Some commands don't like the pipe, e.g. calling vi from within the script or
    173 # from subscripts like git cl upload.
    174 def Command(cmd, args="", prefix="", pipe=True):
    175   # TODO(machenbach): Use timeout.
    176   cmd_line = "%s %s %s" % (prefix, cmd, args)
    177   print "Command: %s" % cmd_line
    178   sys.stdout.flush()
    179   try:
    180     if pipe:
    181       return subprocess.check_output(cmd_line, shell=True)
    182     else:
    183       return subprocess.check_call(cmd_line, shell=True)
    184   except subprocess.CalledProcessError:
    185     return None
    186   finally:
    187     sys.stdout.flush()
    188     sys.stderr.flush()
    189 
    190 
    191 # Wrapper for side effects.
    192 class SideEffectHandler(object):  # pragma: no cover
    193   def Call(self, fun, *args, **kwargs):
    194     return fun(*args, **kwargs)
    195 
    196   def Command(self, cmd, args="", prefix="", pipe=True):
    197     return Command(cmd, args, prefix, pipe)
    198 
    199   def ReadLine(self):
    200     return sys.stdin.readline().strip()
    201 
    202   def ReadURL(self, url, params=None):
    203     # pylint: disable=E1121
    204     url_fh = urllib2.urlopen(url, params, 60)
    205     try:
    206       return url_fh.read()
    207     finally:
    208       url_fh.close()
    209 
    210   def Sleep(self, seconds):
    211     time.sleep(seconds)
    212 
    213   def GetDate(self):
    214     return datetime.date.today().strftime("%Y-%m-%d")
    215 
    216 DEFAULT_SIDE_EFFECT_HANDLER = SideEffectHandler()
    217 
    218 
    219 class NoRetryException(Exception):
    220   pass
    221 
    222 
    223 class Step(GitRecipesMixin):
    224   def __init__(self, text, requires, number, config, state, options, handler):
    225     self._text = text
    226     self._requires = requires
    227     self._number = number
    228     self._config = config
    229     self._state = state
    230     self._options = options
    231     self._side_effect_handler = handler
    232     assert self._number >= 0
    233     assert self._config is not None
    234     assert self._state is not None
    235     assert self._side_effect_handler is not None
    236 
    237   def __getitem__(self, key):
    238     # Convenience method to allow direct [] access on step classes for
    239     # manipulating the backed state dict.
    240     return self._state[key]
    241 
    242   def __setitem__(self, key, value):
    243     # Convenience method to allow direct [] access on step classes for
    244     # manipulating the backed state dict.
    245     self._state[key] = value
    246 
    247   def Config(self, key):
    248     return self._config[key]
    249 
    250   def Run(self):
    251     # Restore state.
    252     state_file = "%s-state.json" % self._config[PERSISTFILE_BASENAME]
    253     if not self._state and os.path.exists(state_file):
    254       self._state.update(json.loads(FileToText(state_file)))
    255 
    256     # Skip step if requirement is not met.
    257     if self._requires and not self._state.get(self._requires):
    258       return
    259 
    260     print ">>> Step %d: %s" % (self._number, self._text)
    261     try:
    262       return self.RunStep()
    263     finally:
    264       # Persist state.
    265       TextToFile(json.dumps(self._state), state_file)
    266 
    267   def RunStep(self):  # pragma: no cover
    268     raise NotImplementedError
    269 
    270   def Retry(self, cb, retry_on=None, wait_plan=None):
    271     """ Retry a function.
    272     Params:
    273       cb: The function to retry.
    274       retry_on: A callback that takes the result of the function and returns
    275                 True if the function should be retried. A function throwing an
    276                 exception is always retried.
    277       wait_plan: A list of waiting delays between retries in seconds. The
    278                  maximum number of retries is len(wait_plan).
    279     """
    280     retry_on = retry_on or (lambda x: False)
    281     wait_plan = list(wait_plan or [])
    282     wait_plan.reverse()
    283     while True:
    284       got_exception = False
    285       try:
    286         result = cb()
    287       except NoRetryException, e:
    288         raise e
    289       except Exception:
    290         got_exception = True
    291       if got_exception or retry_on(result):
    292         if not wait_plan:  # pragma: no cover
    293           raise Exception("Retried too often. Giving up.")
    294         wait_time = wait_plan.pop()
    295         print "Waiting for %f seconds." % wait_time
    296         self._side_effect_handler.Sleep(wait_time)
    297         print "Retrying..."
    298       else:
    299         return result
    300 
    301   def ReadLine(self, default=None):
    302     # Don't prompt in forced mode.
    303     if self._options.force_readline_defaults and default is not None:
    304       print "%s (forced)" % default
    305       return default
    306     else:
    307       return self._side_effect_handler.ReadLine()
    308 
    309   def Git(self, args="", prefix="", pipe=True, retry_on=None):
    310     cmd = lambda: self._side_effect_handler.Command("git", args, prefix, pipe)
    311     result = self.Retry(cmd, retry_on, [5, 30])
    312     if result is None:
    313       raise GitFailedException("'git %s' failed." % args)
    314     return result
    315 
    316   def SVN(self, args="", prefix="", pipe=True, retry_on=None):
    317     cmd = lambda: self._side_effect_handler.Command("svn", args, prefix, pipe)
    318     return self.Retry(cmd, retry_on, [5, 30])
    319 
    320   def Editor(self, args):
    321     if self._options.requires_editor:
    322       return self._side_effect_handler.Command(os.environ["EDITOR"], args,
    323                                                pipe=False)
    324 
    325   def ReadURL(self, url, params=None, retry_on=None, wait_plan=None):
    326     wait_plan = wait_plan or [3, 60, 600]
    327     cmd = lambda: self._side_effect_handler.ReadURL(url, params)
    328     return self.Retry(cmd, retry_on, wait_plan)
    329 
    330   def GetDate(self):
    331     return self._side_effect_handler.GetDate()
    332 
    333   def Die(self, msg=""):
    334     if msg != "":
    335       print "Error: %s" % msg
    336     print "Exiting"
    337     raise Exception(msg)
    338 
    339   def DieNoManualMode(self, msg=""):
    340     if not self._options.manual:  # pragma: no cover
    341       msg = msg or "Only available in manual mode."
    342       self.Die(msg)
    343 
    344   def Confirm(self, msg):
    345     print "%s [Y/n] " % msg,
    346     answer = self.ReadLine(default="Y")
    347     return answer == "" or answer == "Y" or answer == "y"
    348 
    349   def DeleteBranch(self, name):
    350     for line in self.GitBranch().splitlines():
    351       if re.match(r".*\s+%s$" % name, line):
    352         msg = "Branch %s exists, do you want to delete it?" % name
    353         if self.Confirm(msg):
    354           self.GitDeleteBranch(name)
    355           print "Branch %s deleted." % name
    356         else:
    357           msg = "Can't continue. Please delete branch %s and try again." % name
    358           self.Die(msg)
    359 
    360   def InitialEnvironmentChecks(self):
    361     # Cancel if this is not a git checkout.
    362     if not os.path.exists(self._config[DOT_GIT_LOCATION]):  # pragma: no cover
    363       self.Die("This is not a git checkout, this script won't work for you.")
    364 
    365     # Cancel if EDITOR is unset or not executable.
    366     if (self._options.requires_editor and (not os.environ.get("EDITOR") or
    367         Command("which", os.environ["EDITOR"]) is None)):  # pragma: no cover
    368       self.Die("Please set your EDITOR environment variable, you'll need it.")
    369 
    370   def CommonPrepare(self):
    371     # Check for a clean workdir.
    372     if not self.GitIsWorkdirClean():  # pragma: no cover
    373       self.Die("Workspace is not clean. Please commit or undo your changes.")
    374 
    375     # Persist current branch.
    376     self["current_branch"] = self.GitCurrentBranch()
    377 
    378     # Fetch unfetched revisions.
    379     self.GitSVNFetch()
    380 
    381   def PrepareBranch(self):
    382     # Delete the branch that will be created later if it exists already.
    383     self.DeleteBranch(self._config[BRANCHNAME])
    384 
    385   def CommonCleanup(self):
    386     self.GitCheckout(self["current_branch"])
    387     if self._config[BRANCHNAME] != self["current_branch"]:
    388       self.GitDeleteBranch(self._config[BRANCHNAME])
    389 
    390     # Clean up all temporary files.
    391     Command("rm", "-f %s*" % self._config[PERSISTFILE_BASENAME])
    392 
    393   def ReadAndPersistVersion(self, prefix=""):
    394     def ReadAndPersist(var_name, def_name):
    395       match = re.match(r"^#define %s\s+(\d*)" % def_name, line)
    396       if match:
    397         value = match.group(1)
    398         self["%s%s" % (prefix, var_name)] = value
    399     for line in LinesInFile(self._config[VERSION_FILE]):
    400       for (var_name, def_name) in [("major", "MAJOR_VERSION"),
    401                                    ("minor", "MINOR_VERSION"),
    402                                    ("build", "BUILD_NUMBER"),
    403                                    ("patch", "PATCH_LEVEL")]:
    404         ReadAndPersist(var_name, def_name)
    405 
    406   def WaitForLGTM(self):
    407     print ("Please wait for an LGTM, then type \"LGTM<Return>\" to commit "
    408            "your change. (If you need to iterate on the patch or double check "
    409            "that it's sane, do so in another shell, but remember to not "
    410            "change the headline of the uploaded CL.")
    411     answer = ""
    412     while answer != "LGTM":
    413       print "> ",
    414       answer = self.ReadLine(None if self._options.wait_for_lgtm else "LGTM")
    415       if answer != "LGTM":
    416         print "That was not 'LGTM'."
    417 
    418   def WaitForResolvingConflicts(self, patch_file):
    419     print("Applying the patch \"%s\" failed. Either type \"ABORT<Return>\", "
    420           "or resolve the conflicts, stage *all* touched files with "
    421           "'git add', and type \"RESOLVED<Return>\"")
    422     self.DieNoManualMode()
    423     answer = ""
    424     while answer != "RESOLVED":
    425       if answer == "ABORT":
    426         self.Die("Applying the patch failed.")
    427       if answer != "":
    428         print "That was not 'RESOLVED' or 'ABORT'."
    429       print "> ",
    430       answer = self.ReadLine()
    431 
    432   # Takes a file containing the patch to apply as first argument.
    433   def ApplyPatch(self, patch_file, revert=False):
    434     try:
    435       self.GitApplyPatch(patch_file, revert)
    436     except GitFailedException:
    437       self.WaitForResolvingConflicts(patch_file)
    438 
    439   def FindLastTrunkPush(self, parent_hash="", include_patches=False):
    440     push_pattern = "^Version [[:digit:]]*\.[[:digit:]]*\.[[:digit:]]*"
    441     if not include_patches:
    442       # Non-patched versions only have three numbers followed by the "(based
    443       # on...) comment."
    444       push_pattern += " (based"
    445     branch = "" if parent_hash else "svn/trunk"
    446     return self.GitLog(n=1, format="%H", grep=push_pattern,
    447                        parent_hash=parent_hash, branch=branch)
    448 
    449 
    450 class UploadStep(Step):
    451   MESSAGE = "Upload for code review."
    452 
    453   def RunStep(self):
    454     if self._options.reviewer:
    455       print "Using account %s for review." % self._options.reviewer
    456       reviewer = self._options.reviewer
    457     else:
    458       print "Please enter the email address of a V8 reviewer for your patch: ",
    459       self.DieNoManualMode("A reviewer must be specified in forced mode.")
    460       reviewer = self.ReadLine()
    461     self.GitUpload(reviewer, self._options.author, self._options.force_upload)
    462 
    463 
    464 class DetermineV8Sheriff(Step):
    465   MESSAGE = "Determine the V8 sheriff for code review."
    466 
    467   def RunStep(self):
    468     self["sheriff"] = None
    469     if not self._options.sheriff:  # pragma: no cover
    470       return
    471 
    472     try:
    473       # The googlers mapping maps @google.com accounts to @chromium.org
    474       # accounts.
    475       googlers = imp.load_source('googlers_mapping',
    476                                  self._options.googlers_mapping)
    477       googlers = googlers.list_to_dict(googlers.get_list())
    478     except:  # pragma: no cover
    479       print "Skip determining sheriff without googler mapping."
    480       return
    481 
    482     # The sheriff determined by the rotation on the waterfall has a
    483     # @google.com account.
    484     url = "https://chromium-build.appspot.com/p/chromium/sheriff_v8.js"
    485     match = re.match(r"document\.write\('(\w+)'\)", self.ReadURL(url))
    486 
    487     # If "channel is sheriff", we can't match an account.
    488     if match:
    489       g_name = match.group(1)
    490       self["sheriff"] = googlers.get(g_name + "@google.com",
    491                                      g_name + "@chromium.org")
    492       self._options.reviewer = self["sheriff"]
    493       print "Found active sheriff: %s" % self["sheriff"]
    494     else:
    495       print "No active sheriff found."
    496 
    497 
    498 def MakeStep(step_class=Step, number=0, state=None, config=None,
    499              options=None, side_effect_handler=DEFAULT_SIDE_EFFECT_HANDLER):
    500     # Allow to pass in empty dictionaries.
    501     state = state if state is not None else {}
    502     config = config if config is not None else {}
    503 
    504     try:
    505       message = step_class.MESSAGE
    506     except AttributeError:
    507       message = step_class.__name__
    508     try:
    509       requires = step_class.REQUIRES
    510     except AttributeError:
    511       requires = None
    512 
    513     return step_class(message, requires, number=number, config=config,
    514                       state=state, options=options,
    515                       handler=side_effect_handler)
    516 
    517 
    518 class ScriptsBase(object):
    519   # TODO(machenbach): Move static config here.
    520   def __init__(self, config, side_effect_handler=DEFAULT_SIDE_EFFECT_HANDLER,
    521                state=None):
    522     self._config = config
    523     self._side_effect_handler = side_effect_handler
    524     self._state = state if state is not None else {}
    525 
    526   def _Description(self):
    527     return None
    528 
    529   def _PrepareOptions(self, parser):
    530     pass
    531 
    532   def _ProcessOptions(self, options):
    533     return True
    534 
    535   def _Steps(self):  # pragma: no cover
    536     raise Exception("Not implemented.")
    537 
    538   def MakeOptions(self, args=None):
    539     parser = argparse.ArgumentParser(description=self._Description())
    540     parser.add_argument("-a", "--author", default="",
    541                         help="The author email used for rietveld.")
    542     parser.add_argument("-g", "--googlers-mapping",
    543                         help="Path to the script mapping google accounts.")
    544     parser.add_argument("-r", "--reviewer", default="",
    545                         help="The account name to be used for reviews.")
    546     parser.add_argument("--sheriff", default=False, action="store_true",
    547                         help=("Determine current sheriff to review CLs. On "
    548                               "success, this will overwrite the reviewer "
    549                               "option."))
    550     parser.add_argument("-s", "--step",
    551         help="Specify the step where to start work. Default: 0.",
    552         default=0, type=int)
    553 
    554     self._PrepareOptions(parser)
    555 
    556     if args is None:  # pragma: no cover
    557       options = parser.parse_args()
    558     else:
    559       options = parser.parse_args(args)
    560 
    561     # Process common options.
    562     if options.step < 0:  # pragma: no cover
    563       print "Bad step number %d" % options.step
    564       parser.print_help()
    565       return None
    566     if options.sheriff and not options.googlers_mapping:  # pragma: no cover
    567       print "To determine the current sheriff, requires the googler mapping"
    568       parser.print_help()
    569       return None
    570 
    571     # Defaults for options, common to all scripts.
    572     options.manual = getattr(options, "manual", True)
    573     options.force = getattr(options, "force", False)
    574 
    575     # Derived options.
    576     options.requires_editor = not options.force
    577     options.wait_for_lgtm = not options.force
    578     options.force_readline_defaults = not options.manual
    579     options.force_upload = not options.manual
    580 
    581     # Process script specific options.
    582     if not self._ProcessOptions(options):
    583       parser.print_help()
    584       return None
    585     return options
    586 
    587   def RunSteps(self, step_classes, args=None):
    588     options = self.MakeOptions(args)
    589     if not options:
    590       return 1
    591 
    592     state_file = "%s-state.json" % self._config[PERSISTFILE_BASENAME]
    593     if options.step == 0 and os.path.exists(state_file):
    594       os.remove(state_file)
    595 
    596     steps = []
    597     for (number, step_class) in enumerate(step_classes):
    598       steps.append(MakeStep(step_class, number, self._state, self._config,
    599                             options, self._side_effect_handler))
    600     for step in steps[options.step:]:
    601       if step.Run():
    602         return 1
    603     return 0
    604 
    605   def Run(self, args=None):
    606     return self.RunSteps(self._Steps(), args)
    607