Home | History | Annotate | Download | only in scripts
      1 #!/usr/bin/env python
      2 #
      3 # Copyright 2007 Google Inc.
      4 #
      5 # Licensed under the Apache License, Version 2.0 (the "License");
      6 # you may not use this file except in compliance with the License.
      7 # You may obtain a copy of the License at
      8 #
      9 #     http://www.apache.org/licenses/LICENSE-2.0
     10 #
     11 # Unless required by applicable law or agreed to in writing, software
     12 # distributed under the License is distributed on an "AS IS" BASIS,
     13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     14 # See the License for the specific language governing permissions and
     15 # limitations under the License.
     16 
     17 """Tool for uploading diffs from a version control system to the codereview app.
     18 
     19 Usage summary: upload.py [options] [-- diff_options]
     20 
     21 Diff options are passed to the diff command of the underlying system.
     22 
     23 Supported version control systems:
     24   Git
     25   Mercurial
     26   Subversion
     27 
     28 It is important for Git/Mercurial users to specify a tree/node/branch to diff
     29 against by using the '--rev' option.
     30 """
     31 # This code is derived from appcfg.py in the App Engine SDK (open source),
     32 # and from ASPN recipe #146306.
     33 
     34 import cookielib
     35 import getpass
     36 import logging
     37 import md5
     38 import mimetypes
     39 import optparse
     40 import os
     41 import re
     42 import socket
     43 import subprocess
     44 import sys
     45 import urllib
     46 import urllib2
     47 import urlparse
     48 
     49 try:
     50   import readline
     51 except ImportError:
     52   pass
     53 
     54 # The logging verbosity:
     55 #  0: Errors only.
     56 #  1: Status messages.
     57 #  2: Info logs.
     58 #  3: Debug logs.
     59 verbosity = 1
     60 
     61 # Max size of patch or base file.
     62 MAX_UPLOAD_SIZE = 900 * 1024
     63 
     64 
     65 def GetEmail(prompt):
     66   """Prompts the user for their email address and returns it.
     67 
     68   The last used email address is saved to a file and offered up as a suggestion
     69   to the user. If the user presses enter without typing in anything the last
     70   used email address is used. If the user enters a new address, it is saved
     71   for next time we prompt.
     72 
     73   """
     74   last_email_file_name = os.path.expanduser("~/.last_codereview_email_address")
     75   last_email = ""
     76   if os.path.exists(last_email_file_name):
     77     try:
     78       last_email_file = open(last_email_file_name, "r")
     79       last_email = last_email_file.readline().strip("\n")
     80       last_email_file.close()
     81       prompt += " [%s]" % last_email
     82     except IOError, e:
     83       pass
     84   email = raw_input(prompt + ": ").strip()
     85   if email:
     86     try:
     87       last_email_file = open(last_email_file_name, "w")
     88       last_email_file.write(email)
     89       last_email_file.close()
     90     except IOError, e:
     91       pass
     92   else:
     93     email = last_email
     94   return email
     95 
     96 
     97 def StatusUpdate(msg):
     98   """Print a status message to stdout.
     99 
    100   If 'verbosity' is greater than 0, print the message.
    101 
    102   Args:
    103     msg: The string to print.
    104   """
    105   if verbosity > 0:
    106     print msg
    107 
    108 
    109 def ErrorExit(msg):
    110   """Print an error message to stderr and exit."""
    111   print >>sys.stderr, msg
    112   sys.exit(1)
    113 
    114 
    115 class ClientLoginError(urllib2.HTTPError):
    116   """Raised to indicate there was an error authenticating with ClientLogin."""
    117 
    118   def __init__(self, url, code, msg, headers, args):
    119     urllib2.HTTPError.__init__(self, url, code, msg, headers, None)
    120     self.args = args
    121     self.reason = args["Error"]
    122 
    123 
    124 class AbstractRpcServer(object):
    125   """Provides a common interface for a simple RPC server."""
    126 
    127   def __init__(self, host, auth_function, host_override=None, extra_headers={},
    128                save_cookies=False):
    129     """Creates a new HttpRpcServer.
    130 
    131     Args:
    132       host: The host to send requests to.
    133       auth_function: A function that takes no arguments and returns an
    134         (email, password) tuple when called. Will be called if authentication
    135         is required.
    136       host_override: The host header to send to the server (defaults to host).
    137       extra_headers: A dict of extra headers to append to every request.
    138       save_cookies: If True, save the authentication cookies to local disk.
    139         If False, use an in-memory cookiejar instead.  Subclasses must
    140         implement this functionality.  Defaults to False.
    141     """
    142     self.host = host
    143     self.host_override = host_override
    144     self.auth_function = auth_function
    145     self.authenticated = False
    146     self.extra_headers = extra_headers
    147     self.save_cookies = save_cookies
    148     self.opener = self._GetOpener()
    149     if self.host_override:
    150       logging.info("Server: %s; Host: %s", self.host, self.host_override)
    151     else:
    152       logging.info("Server: %s", self.host)
    153 
    154   def _GetOpener(self):
    155     """Returns an OpenerDirector for making HTTP requests.
    156 
    157     Returns:
    158       A urllib2.OpenerDirector object.
    159     """
    160     raise NotImplementedError()
    161 
    162   def _CreateRequest(self, url, data=None):
    163     """Creates a new urllib request."""
    164     logging.debug("Creating request for: '%s' with payload:\n%s", url, data)
    165     req = urllib2.Request(url, data=data)
    166     if self.host_override:
    167       req.add_header("Host", self.host_override)
    168     for key, value in self.extra_headers.iteritems():
    169       req.add_header(key, value)
    170     return req
    171 
    172   def _GetAuthToken(self, email, password):
    173     """Uses ClientLogin to authenticate the user, returning an auth token.
    174 
    175     Args:
    176       email:    The user's email address
    177       password: The user's password
    178 
    179     Raises:
    180       ClientLoginError: If there was an error authenticating with ClientLogin.
    181       HTTPError: If there was some other form of HTTP error.
    182 
    183     Returns:
    184       The authentication token returned by ClientLogin.
    185     """
    186     account_type = "GOOGLE"
    187     if self.host.endswith(".google.com"):
    188       # Needed for use inside Google.
    189       account_type = "HOSTED"
    190     req = self._CreateRequest(
    191         url="https://www.google.com/accounts/ClientLogin",
    192         data=urllib.urlencode({
    193             "Email": email,
    194             "Passwd": password,
    195             "service": "ah",
    196             "source": "rietveld-codereview-upload",
    197             "accountType": account_type,
    198         }),
    199     )
    200     try:
    201       response = self.opener.open(req)
    202       response_body = response.read()
    203       response_dict = dict(x.split("=")
    204                            for x in response_body.split("\n") if x)
    205       return response_dict["Auth"]
    206     except urllib2.HTTPError, e:
    207       if e.code == 403:
    208         body = e.read()
    209         response_dict = dict(x.split("=", 1) for x in body.split("\n") if x)
    210         raise ClientLoginError(req.get_full_url(), e.code, e.msg,
    211                                e.headers, response_dict)
    212       else:
    213         raise
    214 
    215   def _GetAuthCookie(self, auth_token):
    216     """Fetches authentication cookies for an authentication token.
    217 
    218     Args:
    219       auth_token: The authentication token returned by ClientLogin.
    220 
    221     Raises:
    222       HTTPError: If there was an error fetching the authentication cookies.
    223     """
    224     # This is a dummy value to allow us to identify when we're successful.
    225     continue_location = "http://localhost/"
    226     args = {"continue": continue_location, "auth": auth_token}
    227     req = self._CreateRequest("http://%s/_ah/login?%s" %
    228                               (self.host, urllib.urlencode(args)))
    229     try:
    230       response = self.opener.open(req)
    231     except urllib2.HTTPError, e:
    232       response = e
    233     if (response.code != 302 or
    234         response.info()["location"] != continue_location):
    235       raise urllib2.HTTPError(req.get_full_url(), response.code, response.msg,
    236                               response.headers, response.fp)
    237     self.authenticated = True
    238 
    239   def _Authenticate(self):
    240     """Authenticates the user.
    241 
    242     The authentication process works as follows:
    243      1) We get a username and password from the user
    244      2) We use ClientLogin to obtain an AUTH token for the user
    245         (see http://code.google.com/apis/accounts/AuthForInstalledApps.html).
    246      3) We pass the auth token to /_ah/login on the server to obtain an
    247         authentication cookie. If login was successful, it tries to redirect
    248         us to the URL we provided.
    249 
    250     If we attempt to access the upload API without first obtaining an
    251     authentication cookie, it returns a 401 response and directs us to
    252     authenticate ourselves with ClientLogin.
    253     """
    254     for i in range(3):
    255       credentials = self.auth_function()
    256       try:
    257         auth_token = self._GetAuthToken(credentials[0], credentials[1])
    258       except ClientLoginError, e:
    259         if e.reason == "BadAuthentication":
    260           print >>sys.stderr, "Invalid username or password."
    261           continue
    262         if e.reason == "CaptchaRequired":
    263           print >>sys.stderr, (
    264               "Please go to\n"
    265               "https://www.google.com/accounts/DisplayUnlockCaptcha\n"
    266               "and verify you are a human.  Then try again.")
    267           break
    268         if e.reason == "NotVerified":
    269           print >>sys.stderr, "Account not verified."
    270           break
    271         if e.reason == "TermsNotAgreed":
    272           print >>sys.stderr, "User has not agreed to TOS."
    273           break
    274         if e.reason == "AccountDeleted":
    275           print >>sys.stderr, "The user account has been deleted."
    276           break
    277         if e.reason == "AccountDisabled":
    278           print >>sys.stderr, "The user account has been disabled."
    279           break
    280         if e.reason == "ServiceDisabled":
    281           print >>sys.stderr, ("The user's access to the service has been "
    282                                "disabled.")
    283           break
    284         if e.reason == "ServiceUnavailable":
    285           print >>sys.stderr, "The service is not available; try again later."
    286           break
    287         raise
    288       self._GetAuthCookie(auth_token)
    289       return
    290 
    291   def Send(self, request_path, payload=None,
    292            content_type="application/octet-stream",
    293            timeout=None,
    294            **kwargs):
    295     """Sends an RPC and returns the response.
    296 
    297     Args:
    298       request_path: The path to send the request to, eg /api/appversion/create.
    299       payload: The body of the request, or None to send an empty request.
    300       content_type: The Content-Type header to use.
    301       timeout: timeout in seconds; default None i.e. no timeout.
    302         (Note: for large requests on OS X, the timeout doesn't work right.)
    303       kwargs: Any keyword arguments are converted into query string parameters.
    304 
    305     Returns:
    306       The response body, as a string.
    307     """
    308     # TODO: Don't require authentication.  Let the server say
    309     # whether it is necessary.
    310     if not self.authenticated:
    311       self._Authenticate()
    312 
    313     old_timeout = socket.getdefaulttimeout()
    314     socket.setdefaulttimeout(timeout)
    315     try:
    316       tries = 0
    317       while True:
    318         tries += 1
    319         args = dict(kwargs)
    320         url = "http://%s%s" % (self.host, request_path)
    321         if args:
    322           url += "?" + urllib.urlencode(args)
    323         req = self._CreateRequest(url=url, data=payload)
    324         req.add_header("Content-Type", content_type)
    325         try:
    326           f = self.opener.open(req)
    327           response = f.read()
    328           f.close()
    329           return response
    330         except urllib2.HTTPError, e:
    331           if tries > 3:
    332             raise
    333           elif e.code == 401:
    334             self._Authenticate()
    335 ##           elif e.code >= 500 and e.code < 600:
    336 ##             # Server Error - try again.
    337 ##             continue
    338           else:
    339             raise
    340     finally:
    341       socket.setdefaulttimeout(old_timeout)
    342 
    343 
    344 class HttpRpcServer(AbstractRpcServer):
    345   """Provides a simplified RPC-style interface for HTTP requests."""
    346 
    347   def _Authenticate(self):
    348     """Save the cookie jar after authentication."""
    349     super(HttpRpcServer, self)._Authenticate()
    350     if self.save_cookies:
    351       StatusUpdate("Saving authentication cookies to %s" % self.cookie_file)
    352       self.cookie_jar.save()
    353 
    354   def _GetOpener(self):
    355     """Returns an OpenerDirector that supports cookies and ignores redirects.
    356 
    357     Returns:
    358       A urllib2.OpenerDirector object.
    359     """
    360     opener = urllib2.OpenerDirector()
    361     opener.add_handler(urllib2.ProxyHandler())
    362     opener.add_handler(urllib2.UnknownHandler())
    363     opener.add_handler(urllib2.HTTPHandler())
    364     opener.add_handler(urllib2.HTTPDefaultErrorHandler())
    365     opener.add_handler(urllib2.HTTPSHandler())
    366     opener.add_handler(urllib2.HTTPErrorProcessor())
    367     if self.save_cookies:
    368       self.cookie_file = os.path.expanduser("~/.codereview_upload_cookies")
    369       self.cookie_jar = cookielib.MozillaCookieJar(self.cookie_file)
    370       if os.path.exists(self.cookie_file):
    371         try:
    372           self.cookie_jar.load()
    373           self.authenticated = True
    374           StatusUpdate("Loaded authentication cookies from %s" %
    375                        self.cookie_file)
    376         except (cookielib.LoadError, IOError):
    377           # Failed to load cookies - just ignore them.
    378           pass
    379       else:
    380         # Create an empty cookie file with mode 600
    381         fd = os.open(self.cookie_file, os.O_CREAT, 0600)
    382         os.close(fd)
    383       # Always chmod the cookie file
    384       os.chmod(self.cookie_file, 0600)
    385     else:
    386       # Don't save cookies across runs of update.py.
    387       self.cookie_jar = cookielib.CookieJar()
    388     opener.add_handler(urllib2.HTTPCookieProcessor(self.cookie_jar))
    389     return opener
    390 
    391 
    392 parser = optparse.OptionParser(usage="%prog [options] [-- diff_options]")
    393 parser.add_option("-y", "--assume_yes", action="store_true",
    394                   dest="assume_yes", default=False,
    395                   help="Assume that the answer to yes/no questions is 'yes'.")
    396 # Logging
    397 group = parser.add_option_group("Logging options")
    398 group.add_option("-q", "--quiet", action="store_const", const=0,
    399                  dest="verbose", help="Print errors only.")
    400 group.add_option("-v", "--verbose", action="store_const", const=2,
    401                  dest="verbose", default=1,
    402                  help="Print info level logs (default).")
    403 group.add_option("--noisy", action="store_const", const=3,
    404                  dest="verbose", help="Print all logs.")
    405 # Review server
    406 group = parser.add_option_group("Review server options")
    407 group.add_option("-s", "--server", action="store", dest="server",
    408                  default="codereview.appspot.com",
    409                  metavar="SERVER",
    410                  help=("The server to upload to. The format is host[:port]. "
    411                        "Defaults to 'codereview.appspot.com'."))
    412 group.add_option("-e", "--email", action="store", dest="email",
    413                  metavar="EMAIL", default=None,
    414                  help="The username to use. Will prompt if omitted.")
    415 group.add_option("-H", "--host", action="store", dest="host",
    416                  metavar="HOST", default=None,
    417                  help="Overrides the Host header sent with all RPCs.")
    418 group.add_option("--no_cookies", action="store_false",
    419                  dest="save_cookies", default=True,
    420                  help="Do not save authentication cookies to local disk.")
    421 # Issue
    422 group = parser.add_option_group("Issue options")
    423 group.add_option("-d", "--description", action="store", dest="description",
    424                  metavar="DESCRIPTION", default=None,
    425                  help="Optional description when creating an issue.")
    426 group.add_option("-f", "--description_file", action="store",
    427                  dest="description_file", metavar="DESCRIPTION_FILE",
    428                  default=None,
    429                  help="Optional path of a file that contains "
    430                       "the description when creating an issue.")
    431 group.add_option("-r", "--reviewers", action="store", dest="reviewers",
    432                  metavar="REVIEWERS", default=None,
    433                  help="Add reviewers (comma separated email addresses).")
    434 group.add_option("--cc", action="store", dest="cc",
    435                  metavar="CC", default=None,
    436                  help="Add CC (comma separated email addresses).")
    437 # Upload options
    438 group = parser.add_option_group("Patch options")
    439 group.add_option("-m", "--message", action="store", dest="message",
    440                  metavar="MESSAGE", default=None,
    441                  help="A message to identify the patch. "
    442                       "Will prompt if omitted.")
    443 group.add_option("-i", "--issue", type="int", action="store",
    444                  metavar="ISSUE", default=None,
    445                  help="Issue number to which to add. Defaults to new issue.")
    446 group.add_option("--download_base", action="store_true",
    447                  dest="download_base", default=False,
    448                  help="Base files will be downloaded by the server "
    449                  "(side-by-side diffs may not work on files with CRs).")
    450 group.add_option("--rev", action="store", dest="revision",
    451                  metavar="REV", default=None,
    452                  help="Branch/tree/revision to diff against (used by DVCS).")
    453 group.add_option("--send_mail", action="store_true",
    454                  dest="send_mail", default=False,
    455                  help="Send notification email to reviewers.")
    456 
    457 
    458 def GetRpcServer(options):
    459   """Returns an instance of an AbstractRpcServer.
    460 
    461   Returns:
    462     A new AbstractRpcServer, on which RPC calls can be made.
    463   """
    464 
    465   rpc_server_class = HttpRpcServer
    466 
    467   def GetUserCredentials():
    468     """Prompts the user for a username and password."""
    469     email = options.email
    470     if email is None:
    471       email = GetEmail("Email (login for uploading to %s)" % options.server)
    472     password = getpass.getpass("Password for %s: " % email)
    473     return (email, password)
    474 
    475   # If this is the dev_appserver, use fake authentication.
    476   host = (options.host or options.server).lower()
    477   if host == "localhost" or host.startswith("localhost:"):
    478     email = options.email
    479     if email is None:
    480       email = "test (at] example.com"
    481       logging.info("Using debug user %s.  Override with --email" % email)
    482     server = rpc_server_class(
    483         options.server,
    484         lambda: (email, "password"),
    485         host_override=options.host,
    486         extra_headers={"Cookie":
    487                        'dev_appserver_login="%s:False"' % email},
    488         save_cookies=options.save_cookies)
    489     # Don't try to talk to ClientLogin.
    490     server.authenticated = True
    491     return server
    492 
    493   return rpc_server_class(options.server, GetUserCredentials,
    494                           host_override=options.host,
    495                           save_cookies=options.save_cookies)
    496 
    497 
    498 def EncodeMultipartFormData(fields, files):
    499   """Encode form fields for multipart/form-data.
    500 
    501   Args:
    502     fields: A sequence of (name, value) elements for regular form fields.
    503     files: A sequence of (name, filename, value) elements for data to be
    504            uploaded as files.
    505   Returns:
    506     (content_type, body) ready for httplib.HTTP instance.
    507 
    508   Source:
    509     http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/146306
    510   """
    511   BOUNDARY = '-M-A-G-I-C---B-O-U-N-D-A-R-Y-'
    512   CRLF = '\r\n'
    513   lines = []
    514   for (key, value) in fields:
    515     lines.append('--' + BOUNDARY)
    516     lines.append('Content-Disposition: form-data; name="%s"' % key)
    517     lines.append('')
    518     lines.append(value)
    519   for (key, filename, value) in files:
    520     lines.append('--' + BOUNDARY)
    521     lines.append('Content-Disposition: form-data; name="%s"; filename="%s"' %
    522              (key, filename))
    523     lines.append('Content-Type: %s' % GetContentType(filename))
    524     lines.append('')
    525     lines.append(value)
    526   lines.append('--' + BOUNDARY + '--')
    527   lines.append('')
    528   body = CRLF.join(lines)
    529   content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
    530   return content_type, body
    531 
    532 
    533 def GetContentType(filename):
    534   """Helper to guess the content-type from the filename."""
    535   return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
    536 
    537 
    538 # Use a shell for subcommands on Windows to get a PATH search.
    539 use_shell = sys.platform.startswith("win")
    540 
    541 def RunShellWithReturnCode(command, print_output=False,
    542                            universal_newlines=True):
    543   """Executes a command and returns the output from stdout and the return code.
    544 
    545   Args:
    546     command: Command to execute.
    547     print_output: If True, the output is printed to stdout.
    548                   If False, both stdout and stderr are ignored.
    549     universal_newlines: Use universal_newlines flag (default: True).
    550 
    551   Returns:
    552     Tuple (output, return code)
    553   """
    554   logging.info("Running %s", command)
    555   p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
    556                        shell=use_shell, universal_newlines=universal_newlines)
    557   if print_output:
    558     output_array = []
    559     while True:
    560       line = p.stdout.readline()
    561       if not line:
    562         break
    563       print line.strip("\n")
    564       output_array.append(line)
    565     output = "".join(output_array)
    566   else:
    567     output = p.stdout.read()
    568   p.wait()
    569   errout = p.stderr.read()
    570   if print_output and errout:
    571     print >>sys.stderr, errout
    572   p.stdout.close()
    573   p.stderr.close()
    574   return output, p.returncode
    575 
    576 
    577 def RunShell(command, silent_ok=False, universal_newlines=True,
    578              print_output=False):
    579   data, retcode = RunShellWithReturnCode(command, print_output,
    580                                          universal_newlines)
    581   if retcode:
    582     ErrorExit("Got error status from %s:\n%s" % (command, data))
    583   if not silent_ok and not data:
    584     ErrorExit("No output from %s" % command)
    585   return data
    586 
    587 
    588 class VersionControlSystem(object):
    589   """Abstract base class providing an interface to the VCS."""
    590 
    591   def __init__(self, options):
    592     """Constructor.
    593 
    594     Args:
    595       options: Command line options.
    596     """
    597     self.options = options
    598 
    599   def GenerateDiff(self, args):
    600     """Return the current diff as a string.
    601 
    602     Args:
    603       args: Extra arguments to pass to the diff command.
    604     """
    605     raise NotImplementedError(
    606         "abstract method -- subclass %s must override" % self.__class__)
    607 
    608   def GetUnknownFiles(self):
    609     """Return a list of files unknown to the VCS."""
    610     raise NotImplementedError(
    611         "abstract method -- subclass %s must override" % self.__class__)
    612 
    613   def CheckForUnknownFiles(self):
    614     """Show an "are you sure?" prompt if there are unknown files."""
    615     unknown_files = self.GetUnknownFiles()
    616     if unknown_files:
    617       print "The following files are not added to version control:"
    618       for line in unknown_files:
    619         print line
    620       prompt = "Are you sure to continue?(y/N) "
    621       answer = raw_input(prompt).strip()
    622       if answer != "y":
    623         ErrorExit("User aborted")
    624 
    625   def GetBaseFile(self, filename):
    626     """Get the content of the upstream version of a file.
    627 
    628     Returns:
    629       A tuple (base_content, new_content, is_binary, status)
    630         base_content: The contents of the base file.
    631         new_content: For text files, this is empty.  For binary files, this is
    632           the contents of the new file, since the diff output won't contain
    633           information to reconstruct the current file.
    634         is_binary: True iff the file is binary.
    635         status: The status of the file.
    636     """
    637 
    638     raise NotImplementedError(
    639         "abstract method -- subclass %s must override" % self.__class__)
    640 
    641 
    642   def GetBaseFiles(self, diff):
    643     """Helper that calls GetBase file for each file in the patch.
    644 
    645     Returns:
    646       A dictionary that maps from filename to GetBaseFile's tuple.  Filenames
    647       are retrieved based on lines that start with "Index:" or
    648       "Property changes on:".
    649     """
    650     files = {}
    651     for line in diff.splitlines(True):
    652       if line.startswith('Index:') or line.startswith('Property changes on:'):
    653         unused, filename = line.split(':', 1)
    654         # On Windows if a file has property changes its filename uses '\'
    655         # instead of '/'.
    656         filename = filename.strip().replace('\\', '/')
    657         files[filename] = self.GetBaseFile(filename)
    658     return files
    659 
    660 
    661   def UploadBaseFiles(self, issue, rpc_server, patch_list, patchset, options,
    662                       files):
    663     """Uploads the base files (and if necessary, the current ones as well)."""
    664 
    665     def UploadFile(filename, file_id, content, is_binary, status, is_base):
    666       """Uploads a file to the server."""
    667       file_too_large = False
    668       if is_base:
    669         type = "base"
    670       else:
    671         type = "current"
    672       if len(content) > MAX_UPLOAD_SIZE:
    673         print ("Not uploading the %s file for %s because it's too large." %
    674                (type, filename))
    675         file_too_large = True
    676         content = ""
    677       checksum = md5.new(content).hexdigest()
    678       if options.verbose > 0 and not file_too_large:
    679         print "Uploading %s file for %s" % (type, filename)
    680       url = "/%d/upload_content/%d/%d" % (int(issue), int(patchset), file_id)
    681       form_fields = [("filename", filename),
    682                      ("status", status),
    683                      ("checksum", checksum),
    684                      ("is_binary", str(is_binary)),
    685                      ("is_current", str(not is_base)),
    686                     ]
    687       if file_too_large:
    688         form_fields.append(("file_too_large", "1"))
    689       if options.email:
    690         form_fields.append(("user", options.email))
    691       ctype, body = EncodeMultipartFormData(form_fields,
    692                                             [("data", filename, content)])
    693       response_body = rpc_server.Send(url, body,
    694                                       content_type=ctype)
    695       if not response_body.startswith("OK"):
    696         StatusUpdate("  --> %s" % response_body)
    697         sys.exit(1)
    698 
    699     patches = dict()
    700     [patches.setdefault(v, k) for k, v in patch_list]
    701     for filename in patches.keys():
    702       base_content, new_content, is_binary, status = files[filename]
    703       file_id_str = patches.get(filename)
    704       if file_id_str.find("nobase") != -1:
    705         base_content = None
    706         file_id_str = file_id_str[file_id_str.rfind("_") + 1:]
    707       file_id = int(file_id_str)
    708       if base_content != None:
    709         UploadFile(filename, file_id, base_content, is_binary, status, True)
    710       if new_content != None:
    711         UploadFile(filename, file_id, new_content, is_binary, status, False)
    712 
    713   def IsImage(self, filename):
    714     """Returns true if the filename has an image extension."""
    715     mimetype =  mimetypes.guess_type(filename)[0]
    716     if not mimetype:
    717       return False
    718     return mimetype.startswith("image/")
    719 
    720 
    721 class SubversionVCS(VersionControlSystem):
    722   """Implementation of the VersionControlSystem interface for Subversion."""
    723 
    724   def __init__(self, options):
    725     super(SubversionVCS, self).__init__(options)
    726     if self.options.revision:
    727       match = re.match(r"(\d+)(:(\d+))?", self.options.revision)
    728       if not match:
    729         ErrorExit("Invalid Subversion revision %s." % self.options.revision)
    730       self.rev_start = match.group(1)
    731       self.rev_end = match.group(3)
    732     else:
    733       self.rev_start = self.rev_end = None
    734     # Cache output from "svn list -r REVNO dirname".
    735     # Keys: dirname, Values: 2-tuple (ouput for start rev and end rev).
    736     self.svnls_cache = {}
    737     # SVN base URL is required to fetch files deleted in an older revision.
    738     # Result is cached to not guess it over and over again in GetBaseFile().
    739     required = self.options.download_base or self.options.revision is not None
    740     self.svn_base = self._GuessBase(required)
    741 
    742   def GuessBase(self, required):
    743     """Wrapper for _GuessBase."""
    744     return self.svn_base
    745 
    746   def _GuessBase(self, required):
    747     """Returns the SVN base URL.
    748 
    749     Args:
    750       required: If true, exits if the url can't be guessed, otherwise None is
    751         returned.
    752     """
    753     info = RunShell(["svn", "info"])
    754     for line in info.splitlines():
    755       words = line.split()
    756       if len(words) == 2 and words[0] == "URL:":
    757         url = words[1]
    758         scheme, netloc, path, params, query, fragment = urlparse.urlparse(url)
    759         username, netloc = urllib.splituser(netloc)
    760         if username:
    761           logging.info("Removed username from base URL")
    762         if netloc.endswith("svn.python.org"):
    763           if netloc == "svn.python.org":
    764             if path.startswith("/projects/"):
    765               path = path[9:]
    766           elif netloc != "pythondev (at] svn.python.org":
    767             ErrorExit("Unrecognized Python URL: %s" % url)
    768           base = "http://svn.python.org/view/*checkout*%s/" % path
    769           logging.info("Guessed Python base = %s", base)
    770         elif netloc.endswith("svn.collab.net"):
    771           if path.startswith("/repos/"):
    772             path = path[6:]
    773           base = "http://svn.collab.net/viewvc/*checkout*%s/" % path
    774           logging.info("Guessed CollabNet base = %s", base)
    775         elif netloc.endswith(".googlecode.com"):
    776           path = path + "/"
    777           base = urlparse.urlunparse(("http", netloc, path, params,
    778                                       query, fragment))
    779           logging.info("Guessed Google Code base = %s", base)
    780         else:
    781           path = path + "/"
    782           base = urlparse.urlunparse((scheme, netloc, path, params,
    783                                       query, fragment))
    784           logging.info("Guessed base = %s", base)
    785         return base
    786     if required:
    787       ErrorExit("Can't find URL in output from svn info")
    788     return None
    789 
    790   def GenerateDiff(self, args):
    791     cmd = ["svn", "diff"]
    792     if self.options.revision:
    793       cmd += ["-r", self.options.revision]
    794     cmd.extend(args)
    795     data = RunShell(cmd)
    796     count = 0
    797     for line in data.splitlines():
    798       if line.startswith("Index:") or line.startswith("Property changes on:"):
    799         count += 1
    800         logging.info(line)
    801     if not count:
    802       ErrorExit("No valid patches found in output from svn diff")
    803     return data
    804 
    805   def _CollapseKeywords(self, content, keyword_str):
    806     """Collapses SVN keywords."""
    807     # svn cat translates keywords but svn diff doesn't. As a result of this
    808     # behavior patching.PatchChunks() fails with a chunk mismatch error.
    809     # This part was originally written by the Review Board development team
    810     # who had the same problem (http://reviews.review-board.org/r/276/).
    811     # Mapping of keywords to known aliases
    812     svn_keywords = {
    813       # Standard keywords
    814       'Date':                ['Date', 'LastChangedDate'],
    815       'Revision':            ['Revision', 'LastChangedRevision', 'Rev'],
    816       'Author':              ['Author', 'LastChangedBy'],
    817       'HeadURL':             ['HeadURL', 'URL'],
    818       'Id':                  ['Id'],
    819 
    820       # Aliases
    821       'LastChangedDate':     ['LastChangedDate', 'Date'],
    822       'LastChangedRevision': ['LastChangedRevision', 'Rev', 'Revision'],
    823       'LastChangedBy':       ['LastChangedBy', 'Author'],
    824       'URL':                 ['URL', 'HeadURL'],
    825     }
    826 
    827     def repl(m):
    828        if m.group(2):
    829          return "$%s::%s$" % (m.group(1), " " * len(m.group(3)))
    830        return "$%s$" % m.group(1)
    831     keywords = [keyword
    832                 for name in keyword_str.split(" ")
    833                 for keyword in svn_keywords.get(name, [])]
    834     return re.sub(r"\$(%s):(:?)([^\$]+)\$" % '|'.join(keywords), repl, content)
    835 
    836   def GetUnknownFiles(self):
    837     status = RunShell(["svn", "status", "--ignore-externals"], silent_ok=True)
    838     unknown_files = []
    839     for line in status.split("\n"):
    840       if line and line[0] == "?":
    841         unknown_files.append(line)
    842     return unknown_files
    843 
    844   def ReadFile(self, filename):
    845     """Returns the contents of a file."""
    846     file = open(filename, 'rb')
    847     result = ""
    848     try:
    849       result = file.read()
    850     finally:
    851       file.close()
    852     return result
    853 
    854   def GetStatus(self, filename):
    855     """Returns the status of a file."""
    856     if not self.options.revision:
    857       status = RunShell(["svn", "status", "--ignore-externals", filename])
    858       if not status:
    859         ErrorExit("svn status returned no output for %s" % filename)
    860       status_lines = status.splitlines()
    861       # If file is in a cl, the output will begin with
    862       # "\n--- Changelist 'cl_name':\n".  See
    863       # http://svn.collab.net/repos/svn/trunk/notes/changelist-design.txt
    864       if (len(status_lines) == 3 and
    865           not status_lines[0] and
    866           status_lines[1].startswith("--- Changelist")):
    867         status = status_lines[2]
    868       else:
    869         status = status_lines[0]
    870     # If we have a revision to diff against we need to run "svn list"
    871     # for the old and the new revision and compare the results to get
    872     # the correct status for a file.
    873     else:
    874       dirname, relfilename = os.path.split(filename)
    875       if dirname not in self.svnls_cache:
    876         cmd = ["svn", "list", "-r", self.rev_start, dirname or "."]
    877         out, returncode = RunShellWithReturnCode(cmd)
    878         if returncode:
    879           ErrorExit("Failed to get status for %s." % filename)
    880         old_files = out.splitlines()
    881         args = ["svn", "list"]
    882         if self.rev_end:
    883           args += ["-r", self.rev_end]
    884         cmd = args + [dirname or "."]
    885         out, returncode = RunShellWithReturnCode(cmd)
    886         if returncode:
    887           ErrorExit("Failed to run command %s" % cmd)
    888         self.svnls_cache[dirname] = (old_files, out.splitlines())
    889       old_files, new_files = self.svnls_cache[dirname]
    890       if relfilename in old_files and relfilename not in new_files:
    891         status = "D   "
    892       elif relfilename in old_files and relfilename in new_files:
    893         status = "M   "
    894       else:
    895         status = "A   "
    896     return status
    897 
    898   def GetBaseFile(self, filename):
    899     status = self.GetStatus(filename)
    900     base_content = None
    901     new_content = None
    902 
    903     # If a file is copied its status will be "A  +", which signifies
    904     # "addition-with-history".  See "svn st" for more information.  We need to
    905     # upload the original file or else diff parsing will fail if the file was
    906     # edited.
    907     if status[0] == "A" and status[3] != "+":
    908       # We'll need to upload the new content if we're adding a binary file
    909       # since diff's output won't contain it.
    910       mimetype = RunShell(["svn", "propget", "svn:mime-type", filename],
    911                           silent_ok=True)
    912       base_content = ""
    913       is_binary = mimetype and not mimetype.startswith("text/")
    914       if is_binary and self.IsImage(filename):
    915         new_content = self.ReadFile(filename)
    916     elif (status[0] in ("M", "D", "R") or
    917           (status[0] == "A" and status[3] == "+") or  # Copied file.
    918           (status[0] == " " and status[1] == "M")):  # Property change.
    919       args = []
    920       if self.options.revision:
    921         url = "%s/%s@%s" % (self.svn_base, filename, self.rev_start)
    922       else:
    923         # Don't change filename, it's needed later.
    924         url = filename
    925         args += ["-r", "BASE"]
    926       cmd = ["svn"] + args + ["propget", "svn:mime-type", url]
    927       mimetype, returncode = RunShellWithReturnCode(cmd)
    928       if returncode:
    929         # File does not exist in the requested revision.
    930         # Reset mimetype, it contains an error message.
    931         mimetype = ""
    932       get_base = False
    933       is_binary = mimetype and not mimetype.startswith("text/")
    934       if status[0] == " ":
    935         # Empty base content just to force an upload.
    936         base_content = ""
    937       elif is_binary:
    938         if self.IsImage(filename):
    939           get_base = True
    940           if status[0] == "M":
    941             if not self.rev_end:
    942               new_content = self.ReadFile(filename)
    943             else:
    944               url = "%s/%s@%s" % (self.svn_base, filename, self.rev_end)
    945               new_content = RunShell(["svn", "cat", url],
    946                                      universal_newlines=True, silent_ok=True)
    947         else:
    948           base_content = ""
    949       else:
    950         get_base = True
    951 
    952       if get_base:
    953         if is_binary:
    954           universal_newlines = False
    955         else:
    956           universal_newlines = True
    957         if self.rev_start:
    958           # "svn cat -r REV delete_file.txt" doesn't work. cat requires
    959           # the full URL with "@REV" appended instead of using "-r" option.
    960           url = "%s/%s@%s" % (self.svn_base, filename, self.rev_start)
    961           base_content = RunShell(["svn", "cat", url],
    962                                   universal_newlines=universal_newlines,
    963                                   silent_ok=True)
    964         else:
    965           base_content = RunShell(["svn", "cat", filename],
    966                                   universal_newlines=universal_newlines,
    967                                   silent_ok=True)
    968         if not is_binary:
    969           args = []
    970           if self.rev_start:
    971             url = "%s/%s@%s" % (self.svn_base, filename, self.rev_start)
    972           else:
    973             url = filename
    974             args += ["-r", "BASE"]
    975           cmd = ["svn"] + args + ["propget", "svn:keywords", url]
    976           keywords, returncode = RunShellWithReturnCode(cmd)
    977           if keywords and not returncode:
    978             base_content = self._CollapseKeywords(base_content, keywords)
    979     else:
    980       StatusUpdate("svn status returned unexpected output: %s" % status)
    981       sys.exit(1)
    982     return base_content, new_content, is_binary, status[0:5]
    983 
    984 
    985 class GitVCS(VersionControlSystem):
    986   """Implementation of the VersionControlSystem interface for Git."""
    987 
    988   def __init__(self, options):
    989     super(GitVCS, self).__init__(options)
    990     # Map of filename -> hash of base file.
    991     self.base_hashes = {}
    992 
    993   def GenerateDiff(self, extra_args):
    994     # This is more complicated than svn's GenerateDiff because we must convert
    995     # the diff output to include an svn-style "Index:" line as well as record
    996     # the hashes of the base files, so we can upload them along with our diff.
    997     if self.options.revision:
    998       extra_args = [self.options.revision] + extra_args
    999     gitdiff = RunShell(["git", "diff", "--full-index"] + extra_args)
   1000     svndiff = []
   1001     filecount = 0
   1002     filename = None
   1003     for line in gitdiff.splitlines():
   1004       match = re.match(r"diff --git a/(.*) b/.*$", line)
   1005       if match:
   1006         filecount += 1
   1007         filename = match.group(1)
   1008         svndiff.append("Index: %s\n" % filename)
   1009       else:
   1010         # The "index" line in a git diff looks like this (long hashes elided):
   1011         #   index 82c0d44..b2cee3f 100755
   1012         # We want to save the left hash, as that identifies the base file.
   1013         match = re.match(r"index (\w+)\.\.", line)
   1014         if match:
   1015           self.base_hashes[filename] = match.group(1)
   1016       svndiff.append(line + "\n")
   1017     if not filecount:
   1018       ErrorExit("No valid patches found in output from git diff")
   1019     return "".join(svndiff)
   1020 
   1021   def GetUnknownFiles(self):
   1022     status = RunShell(["git", "ls-files", "--exclude-standard", "--others"],
   1023                       silent_ok=True)
   1024     return status.splitlines()
   1025 
   1026   def GetBaseFile(self, filename):
   1027     hash = self.base_hashes[filename]
   1028     base_content = None
   1029     new_content = None
   1030     is_binary = False
   1031     if hash == "0" * 40:  # All-zero hash indicates no base file.
   1032       status = "A"
   1033       base_content = ""
   1034     else:
   1035       status = "M"
   1036       base_content, returncode = RunShellWithReturnCode(["git", "show", hash])
   1037       if returncode:
   1038         ErrorExit("Got error status from 'git show %s'" % hash)
   1039     return (base_content, new_content, is_binary, status)
   1040 
   1041 
   1042 class MercurialVCS(VersionControlSystem):
   1043   """Implementation of the VersionControlSystem interface for Mercurial."""
   1044 
   1045   def __init__(self, options, repo_dir):
   1046     super(MercurialVCS, self).__init__(options)
   1047     # Absolute path to repository (we can be in a subdir)
   1048     self.repo_dir = os.path.normpath(repo_dir)
   1049     # Compute the subdir
   1050     cwd = os.path.normpath(os.getcwd())
   1051     assert cwd.startswith(self.repo_dir)
   1052     self.subdir = cwd[len(self.repo_dir):].lstrip(r"\/")
   1053     if self.options.revision:
   1054       self.base_rev = self.options.revision
   1055     else:
   1056       self.base_rev = RunShell(["hg", "parent", "-q"]).split(':')[1].strip()
   1057 
   1058   def _GetRelPath(self, filename):
   1059     """Get relative path of a file according to the current directory,
   1060     given its logical path in the repo."""
   1061     assert filename.startswith(self.subdir), filename
   1062     return filename[len(self.subdir):].lstrip(r"\/")
   1063 
   1064   def GenerateDiff(self, extra_args):
   1065     # If no file specified, restrict to the current subdir
   1066     extra_args = extra_args or ["."]
   1067     cmd = ["hg", "diff", "--git", "-r", self.base_rev] + extra_args
   1068     data = RunShell(cmd, silent_ok=True)
   1069     svndiff = []
   1070     filecount = 0
   1071     for line in data.splitlines():
   1072       m = re.match("diff --git a/(\S+) b/(\S+)", line)
   1073       if m:
   1074         # Modify line to make it look like as it comes from svn diff.
   1075         # With this modification no changes on the server side are required
   1076         # to make upload.py work with Mercurial repos.
   1077         # NOTE: for proper handling of moved/copied files, we have to use
   1078         # the second filename.
   1079         filename = m.group(2)
   1080         svndiff.append("Index: %s" % filename)
   1081         svndiff.append("=" * 67)
   1082         filecount += 1
   1083         logging.info(line)
   1084       else:
   1085         svndiff.append(line)
   1086     if not filecount:
   1087       ErrorExit("No valid patches found in output from hg diff")
   1088     return "\n".join(svndiff) + "\n"
   1089 
   1090   def GetUnknownFiles(self):
   1091     """Return a list of files unknown to the VCS."""
   1092     args = []
   1093     status = RunShell(["hg", "status", "--rev", self.base_rev, "-u", "."],
   1094         silent_ok=True)
   1095     unknown_files = []
   1096     for line in status.splitlines():
   1097       st, fn = line.split(" ", 1)
   1098       if st == "?":
   1099         unknown_files.append(fn)
   1100     return unknown_files
   1101 
   1102   def GetBaseFile(self, filename):
   1103     # "hg status" and "hg cat" both take a path relative to the current subdir
   1104     # rather than to the repo root, but "hg diff" has given us the full path
   1105     # to the repo root.
   1106     base_content = ""
   1107     new_content = None
   1108     is_binary = False
   1109     oldrelpath = relpath = self._GetRelPath(filename)
   1110     # "hg status -C" returns two lines for moved/copied files, one otherwise
   1111     out = RunShell(["hg", "status", "-C", "--rev", self.base_rev, relpath])
   1112     out = out.splitlines()
   1113     # HACK: strip error message about missing file/directory if it isn't in
   1114     # the working copy
   1115     if out[0].startswith('%s: ' % relpath):
   1116       out = out[1:]
   1117     if len(out) > 1:
   1118       # Moved/copied => considered as modified, use old filename to
   1119       # retrieve base contents
   1120       oldrelpath = out[1].strip()
   1121       status = "M"
   1122     else:
   1123       status, _ = out[0].split(' ', 1)
   1124     if status != "A":
   1125       base_content = RunShell(["hg", "cat", "-r", self.base_rev, oldrelpath],
   1126         silent_ok=True)
   1127       is_binary = "\0" in base_content  # Mercurial's heuristic
   1128     if status != "R":
   1129       new_content = open(relpath, "rb").read()
   1130       is_binary = is_binary or "\0" in new_content
   1131     if is_binary and base_content:
   1132       # Fetch again without converting newlines
   1133       base_content = RunShell(["hg", "cat", "-r", self.base_rev, oldrelpath],
   1134         silent_ok=True, universal_newlines=False)
   1135     if not is_binary or not self.IsImage(relpath):
   1136       new_content = None
   1137     return base_content, new_content, is_binary, status
   1138 
   1139 
   1140 # NOTE: The SplitPatch function is duplicated in engine.py, keep them in sync.
   1141 def SplitPatch(data):
   1142   """Splits a patch into separate pieces for each file.
   1143 
   1144   Args:
   1145     data: A string containing the output of svn diff.
   1146 
   1147   Returns:
   1148     A list of 2-tuple (filename, text) where text is the svn diff output
   1149       pertaining to filename.
   1150   """
   1151   patches = []
   1152   filename = None
   1153   diff = []
   1154   for line in data.splitlines(True):
   1155     new_filename = None
   1156     if line.startswith('Index:'):
   1157       unused, new_filename = line.split(':', 1)
   1158       new_filename = new_filename.strip()
   1159     elif line.startswith('Property changes on:'):
   1160       unused, temp_filename = line.split(':', 1)
   1161       # When a file is modified, paths use '/' between directories, however
   1162       # when a property is modified '\' is used on Windows.  Make them the same
   1163       # otherwise the file shows up twice.
   1164       temp_filename = temp_filename.strip().replace('\\', '/')
   1165       if temp_filename != filename:
   1166         # File has property changes but no modifications, create a new diff.
   1167         new_filename = temp_filename
   1168     if new_filename:
   1169       if filename and diff:
   1170         patches.append((filename, ''.join(diff)))
   1171       filename = new_filename
   1172       diff = [line]
   1173       continue
   1174     if diff is not None:
   1175       diff.append(line)
   1176   if filename and diff:
   1177     patches.append((filename, ''.join(diff)))
   1178   return patches
   1179 
   1180 
   1181 def UploadSeparatePatches(issue, rpc_server, patchset, data, options):
   1182   """Uploads a separate patch for each file in the diff output.
   1183 
   1184   Returns a list of [patch_key, filename] for each file.
   1185   """
   1186   patches = SplitPatch(data)
   1187   rv = []
   1188   for patch in patches:
   1189     if len(patch[1]) > MAX_UPLOAD_SIZE:
   1190       print ("Not uploading the patch for " + patch[0] +
   1191              " because the file is too large.")
   1192       continue
   1193     form_fields = [("filename", patch[0])]
   1194     if not options.download_base:
   1195       form_fields.append(("content_upload", "1"))
   1196     files = [("data", "data.diff", patch[1])]
   1197     ctype, body = EncodeMultipartFormData(form_fields, files)
   1198     url = "/%d/upload_patch/%d" % (int(issue), int(patchset))
   1199     print "Uploading patch for " + patch[0]
   1200     response_body = rpc_server.Send(url, body, content_type=ctype)
   1201     lines = response_body.splitlines()
   1202     if not lines or lines[0] != "OK":
   1203       StatusUpdate("  --> %s" % response_body)
   1204       sys.exit(1)
   1205     rv.append([lines[1], patch[0]])
   1206   return rv
   1207 
   1208 
   1209 def GuessVCS(options):
   1210   """Helper to guess the version control system.
   1211 
   1212   This examines the current directory, guesses which VersionControlSystem
   1213   we're using, and returns an instance of the appropriate class.  Exit with an
   1214   error if we can't figure it out.
   1215 
   1216   Returns:
   1217     A VersionControlSystem instance. Exits if the VCS can't be guessed.
   1218   """
   1219   # Mercurial has a command to get the base directory of a repository
   1220   # Try running it, but don't die if we don't have hg installed.
   1221   # NOTE: we try Mercurial first as it can sit on top of an SVN working copy.
   1222   try:
   1223     out, returncode = RunShellWithReturnCode(["hg", "root"])
   1224     if returncode == 0:
   1225       return MercurialVCS(options, out.strip())
   1226   except OSError, (errno, message):
   1227     if errno != 2:  # ENOENT -- they don't have hg installed.
   1228       raise
   1229 
   1230   # Subversion has a .svn in all working directories.
   1231   if os.path.isdir('.svn'):
   1232     logging.info("Guessed VCS = Subversion")
   1233     return SubversionVCS(options)
   1234 
   1235   # Git has a command to test if you're in a git tree.
   1236   # Try running it, but don't die if we don't have git installed.
   1237   try:
   1238     out, returncode = RunShellWithReturnCode(["git", "rev-parse",
   1239                                               "--is-inside-work-tree"])
   1240     if returncode == 0:
   1241       return GitVCS(options)
   1242   except OSError, (errno, message):
   1243     if errno != 2:  # ENOENT -- they don't have git installed.
   1244       raise
   1245 
   1246   ErrorExit(("Could not guess version control system. "
   1247              "Are you in a working copy directory?"))
   1248 
   1249 
   1250 def RealMain(argv, data=None):
   1251   """The real main function.
   1252 
   1253   Args:
   1254     argv: Command line arguments.
   1255     data: Diff contents. If None (default) the diff is generated by
   1256       the VersionControlSystem implementation returned by GuessVCS().
   1257 
   1258   Returns:
   1259     A 2-tuple (issue id, patchset id).
   1260     The patchset id is None if the base files are not uploaded by this
   1261     script (applies only to SVN checkouts).
   1262   """
   1263   logging.basicConfig(format=("%(asctime).19s %(levelname)s %(filename)s:"
   1264                               "%(lineno)s %(message)s "))
   1265   os.environ['LC_ALL'] = 'C'
   1266   options, args = parser.parse_args(argv[1:])
   1267   global verbosity
   1268   verbosity = options.verbose
   1269   if verbosity >= 3:
   1270     logging.getLogger().setLevel(logging.DEBUG)
   1271   elif verbosity >= 2:
   1272     logging.getLogger().setLevel(logging.INFO)
   1273   vcs = GuessVCS(options)
   1274   if isinstance(vcs, SubversionVCS):
   1275     # base field is only allowed for Subversion.
   1276     # Note: Fetching base files may become deprecated in future releases.
   1277     base = vcs.GuessBase(options.download_base)
   1278   else:
   1279     base = None
   1280   if not base and options.download_base:
   1281     options.download_base = True
   1282     logging.info("Enabled upload of base file")
   1283   if not options.assume_yes:
   1284     vcs.CheckForUnknownFiles()
   1285   if data is None:
   1286     data = vcs.GenerateDiff(args)
   1287   files = vcs.GetBaseFiles(data)
   1288   if verbosity >= 1:
   1289     print "Upload server:", options.server, "(change with -s/--server)"
   1290   if options.issue:
   1291     prompt = "Message describing this patch set: "
   1292   else:
   1293     prompt = "New issue subject: "
   1294   message = options.message or raw_input(prompt).strip()
   1295   if not message:
   1296     ErrorExit("A non-empty message is required")
   1297   rpc_server = GetRpcServer(options)
   1298   form_fields = [("subject", message)]
   1299   if base:
   1300     form_fields.append(("base", base))
   1301   if options.issue:
   1302     form_fields.append(("issue", str(options.issue)))
   1303   if options.email:
   1304     form_fields.append(("user", options.email))
   1305   if options.reviewers:
   1306     for reviewer in options.reviewers.split(','):
   1307       if "@" in reviewer and not reviewer.split("@")[1].count(".") == 1:
   1308         ErrorExit("Invalid email address: %s" % reviewer)
   1309     form_fields.append(("reviewers", options.reviewers))
   1310   if options.cc:
   1311     for cc in options.cc.split(','):
   1312       if "@" in cc and not cc.split("@")[1].count(".") == 1:
   1313         ErrorExit("Invalid email address: %s" % cc)
   1314     form_fields.append(("cc", options.cc))
   1315   description = options.description
   1316   if options.description_file:
   1317     if options.description:
   1318       ErrorExit("Can't specify description and description_file")
   1319     file = open(options.description_file, 'r')
   1320     description = file.read()
   1321     file.close()
   1322   if description:
   1323     form_fields.append(("description", description))
   1324   # Send a hash of all the base file so the server can determine if a copy
   1325   # already exists in an earlier patchset.
   1326   base_hashes = ""
   1327   for file, info in files.iteritems():
   1328     if not info[0] is None:
   1329       checksum = md5.new(info[0]).hexdigest()
   1330       if base_hashes:
   1331         base_hashes += "|"
   1332       base_hashes += checksum + ":" + file
   1333   form_fields.append(("base_hashes", base_hashes))
   1334   # If we're uploading base files, don't send the email before the uploads, so
   1335   # that it contains the file status.
   1336   if options.send_mail and options.download_base:
   1337     form_fields.append(("send_mail", "1"))
   1338   if not options.download_base:
   1339     form_fields.append(("content_upload", "1"))
   1340   if len(data) > MAX_UPLOAD_SIZE:
   1341     print "Patch is large, so uploading file patches separately."
   1342     uploaded_diff_file = []
   1343     form_fields.append(("separate_patches", "1"))
   1344   else:
   1345     uploaded_diff_file = [("data", "data.diff", data)]
   1346   ctype, body = EncodeMultipartFormData(form_fields, uploaded_diff_file)
   1347   response_body = rpc_server.Send("/upload", body, content_type=ctype)
   1348   patchset = None
   1349   if not options.download_base or not uploaded_diff_file:
   1350     lines = response_body.splitlines()
   1351     if len(lines) >= 2:
   1352       msg = lines[0]
   1353       patchset = lines[1].strip()
   1354       patches = [x.split(" ", 1) for x in lines[2:]]
   1355     else:
   1356       msg = response_body
   1357   else:
   1358     msg = response_body
   1359   StatusUpdate(msg)
   1360   if not response_body.startswith("Issue created.") and \
   1361   not response_body.startswith("Issue updated."):
   1362     sys.exit(0)
   1363   issue = msg[msg.rfind("/")+1:]
   1364 
   1365   if not uploaded_diff_file:
   1366     result = UploadSeparatePatches(issue, rpc_server, patchset, data, options)
   1367     if not options.download_base:
   1368       patches = result
   1369 
   1370   if not options.download_base:
   1371     vcs.UploadBaseFiles(issue, rpc_server, patches, patchset, options, files)
   1372     if options.send_mail:
   1373       rpc_server.Send("/" + issue + "/mail", payload="")
   1374   return issue, patchset
   1375 
   1376 
   1377 def main():
   1378   try:
   1379     RealMain(sys.argv)
   1380   except KeyboardInterrupt:
   1381     print
   1382     StatusUpdate("Interrupted.")
   1383     sys.exit(1)
   1384 
   1385 
   1386 if __name__ == "__main__":
   1387   main()
   1388