Home | History | Annotate | Download | only in webrtc 
      1 #!/usr/bin/env python
      2 # Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
      3 #
      4 # Use of this source code is governed by a BSD-style license
      5 # that can be found in the LICENSE file in the root of the source
      6 # tree. An additional intellectual property rights grant can be found
      7 # in the file PATENTS.  All contributing project authors may
      8 # be found in the AUTHORS file in the root of the source tree.
      9 
     10 """Setup links to a Chromium checkout for WebRTC.
     11 
     12 WebRTC standalone shares a lot of dependencies and build tools with Chromium.
     13 To do this, many of the paths of a Chromium checkout is emulated by creating
     14 symlinks to files and directories. This script handles the setup of symlinks to
     15 achieve this.
     16 
     17 It also handles cleanup of the legacy Subversion-based approach that was used
     18 before Chrome switched over their master repo from Subversion to Git.
     19 """
     20 
     21 
     22 import ctypes
     23 import errno
     24 import logging
     25 import optparse
     26 import os
     27 import shelve
     28 import shutil
     29 import subprocess
     30 import sys
     31 import textwrap
     32 
     33 
     34 DIRECTORIES = [
     35   'build',
     36   'buildtools',
     37   'testing',
     38   'third_party/binutils',
     39   'third_party/boringssl',
     40   'third_party/colorama',
     41   'third_party/drmemory',
     42   'third_party/expat',
     43   'third_party/ffmpeg',
     44   'third_party/instrumented_libraries',
     45   'third_party/jsoncpp',
     46   'third_party/libc++-static',
     47   'third_party/libjpeg',
     48   'third_party/libjpeg_turbo',
     49   'third_party/libsrtp',
     50   'third_party/libudev',
     51   'third_party/libvpx_new',
     52   'third_party/libyuv',
     53   'third_party/llvm-build',
     54   'third_party/lss',
     55   'third_party/nss',
     56   'third_party/ocmock',
     57   'third_party/openh264',
     58   'third_party/openmax_dl',
     59   'third_party/opus',
     60   'third_party/proguard',
     61   'third_party/protobuf',
     62   'third_party/sqlite',
     63   'third_party/syzygy',
     64   'third_party/usrsctp',
     65   'third_party/yasm',
     66   'third_party/zlib',
     67   'tools/clang',
     68   'tools/generate_library_loader',
     69   'tools/gn',
     70   'tools/gyp',
     71   'tools/memory',
     72   'tools/protoc_wrapper',
     73   'tools/python',
     74   'tools/swarming_client',
     75   'tools/valgrind',
     76   'tools/vim',
     77   'tools/win',
     78   'tools/xdisplaycheck',
     79 ]
     80 
     81 from sync_chromium import get_target_os_list
     82 target_os = get_target_os_list()
     83 if 'android' in target_os:
     84   DIRECTORIES += [
     85     'base',
     86     'third_party/android_platform',
     87     'third_party/android_tools',
     88     'third_party/appurify-python',
     89     'third_party/ashmem',
     90     'third_party/catapult',
     91     'third_party/icu',
     92     'third_party/ijar',
     93     'third_party/jsr-305',
     94     'third_party/junit',
     95     'third_party/libevent',
     96     'third_party/libxml',
     97     'third_party/mockito',
     98     'third_party/modp_b64',
     99     'third_party/requests',
    100     'third_party/robolectric',
    101     'tools/android',
    102     'tools/grit',
    103     'tools/telemetry',
    104   ]
    105 if 'ios' in target_os:
    106   DIRECTORIES.append('third_party/class-dump')
    107 
    108 FILES = {
    109   'tools/isolate_driver.py': None,
    110   'third_party/BUILD.gn': None,
    111 }
    112 
    113 ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
    114 CHROMIUM_CHECKOUT = os.path.join('chromium', 'src')
    115 LINKS_DB = 'links'
    116 
    117 # Version management to make future upgrades/downgrades easier to support.
    118 SCHEMA_VERSION = 1
    119 
    120 
    121 def query_yes_no(question, default=False):
    122   """Ask a yes/no question via raw_input() and return their answer.
    123 
    124   Modified from http://stackoverflow.com/a/3041990.
    125   """
    126   prompt = " [%s/%%s]: "
    127   prompt = prompt % ('Y' if default is True  else 'y')
    128   prompt = prompt % ('N' if default is False else 'n')
    129 
    130   if default is None:
    131     default = 'INVALID'
    132 
    133   while True:
    134     sys.stdout.write(question + prompt)
    135     choice = raw_input().lower()
    136     if choice == '' and default != 'INVALID':
    137       return default
    138 
    139     if 'yes'.startswith(choice):
    140       return True
    141     elif 'no'.startswith(choice):
    142       return False
    143 
    144     print "Please respond with 'yes' or 'no' (or 'y' or 'n')."
    145 
    146 
    147 # Actions
    148 class Action(object):
    149   def __init__(self, dangerous):
    150     self.dangerous = dangerous
    151 
    152   def announce(self, planning):
    153     """Log a description of this action.
    154 
    155     Args:
    156       planning - True iff we're in the planning stage, False if we're in the
    157                  doit stage.
    158     """
    159     pass
    160 
    161   def doit(self, links_db):
    162     """Execute the action, recording what we did to links_db, if necessary."""
    163     pass
    164 
    165 
    166 class Remove(Action):
    167   def __init__(self, path, dangerous):
    168     super(Remove, self).__init__(dangerous)
    169     self._priority = 0
    170     self._path = path
    171 
    172   def announce(self, planning):
    173     log = logging.warn
    174     filesystem_type = 'file'
    175     if not self.dangerous:
    176       log = logging.info
    177       filesystem_type = 'link'
    178     if planning:
    179       log('Planning to remove %s: %s', filesystem_type, self._path)
    180     else:
    181       log('Removing %s: %s', filesystem_type, self._path)
    182 
    183   def doit(self, _):
    184     os.remove(self._path)
    185 
    186 
    187 class Rmtree(Action):
    188   def __init__(self, path):
    189     super(Rmtree, self).__init__(dangerous=True)
    190     self._priority = 0
    191     self._path = path
    192 
    193   def announce(self, planning):
    194     if planning:
    195       logging.warn('Planning to remove directory: %s', self._path)
    196     else:
    197       logging.warn('Removing directory: %s', self._path)
    198 
    199   def doit(self, _):
    200     if sys.platform.startswith('win'):
    201       # shutil.rmtree() doesn't work on Windows if any of the directories are
    202       # read-only, which svn repositories are.
    203       subprocess.check_call(['rd', '/q', '/s', self._path], shell=True)
    204     else:
    205       shutil.rmtree(self._path)
    206 
    207 
    208 class Makedirs(Action):
    209   def __init__(self, path):
    210     super(Makedirs, self).__init__(dangerous=False)
    211     self._priority = 1
    212     self._path = path
    213 
    214   def doit(self, _):
    215     try:
    216       os.makedirs(self._path)
    217     except OSError as e:
    218       if e.errno != errno.EEXIST:
    219         raise
    220 
    221 
    222 class Symlink(Action):
    223   def __init__(self, source_path, link_path):
    224     super(Symlink, self).__init__(dangerous=False)
    225     self._priority = 2
    226     self._source_path = source_path
    227     self._link_path = link_path
    228 
    229   def announce(self, planning):
    230     if planning:
    231       logging.info(
    232           'Planning to create link from %s to %s', self._link_path,
    233           self._source_path)
    234     else:
    235       logging.debug(
    236           'Linking from %s to %s', self._link_path, self._source_path)
    237 
    238   def doit(self, links_db):
    239     # Files not in the root directory need relative path calculation.
    240     # On Windows, use absolute paths instead since NTFS doesn't seem to support
    241     # relative paths for symlinks.
    242     if sys.platform.startswith('win'):
    243       source_path = os.path.abspath(self._source_path)
    244     else:
    245       if os.path.dirname(self._link_path) != self._link_path:
    246         source_path = os.path.relpath(self._source_path,
    247                                       os.path.dirname(self._link_path))
    248 
    249     os.symlink(source_path, os.path.abspath(self._link_path))
    250     links_db[self._source_path] = self._link_path
    251 
    252 
    253 class LinkError(IOError):
    254   """Failed to create a link."""
    255   pass
    256 
    257 
    258 # Handles symlink creation on the different platforms.
    259 if sys.platform.startswith('win'):
    260   def symlink(source_path, link_path):
    261     flag = 1 if os.path.isdir(source_path) else 0
    262     if not ctypes.windll.kernel32.CreateSymbolicLinkW(
    263         unicode(link_path), unicode(source_path), flag):
    264       raise OSError('Failed to create symlink to %s. Notice that only NTFS '
    265                     'version 5.0 and up has all the needed APIs for '
    266                     'creating symlinks.' % source_path)
    267   os.symlink = symlink
    268 
    269 
    270 class WebRTCLinkSetup(object):
    271   def __init__(self, links_db, force=False, dry_run=False, prompt=False):
    272     self._force = force
    273     self._dry_run = dry_run
    274     self._prompt = prompt
    275     self._links_db = links_db
    276 
    277   def CreateLinks(self, on_bot):
    278     logging.debug('CreateLinks')
    279     # First, make a plan of action
    280     actions = []
    281 
    282     for source_path, link_path in FILES.iteritems():
    283       actions += self._ActionForPath(
    284           source_path, link_path, check_fn=os.path.isfile, check_msg='files')
    285     for source_dir in DIRECTORIES:
    286       actions += self._ActionForPath(
    287           source_dir, None, check_fn=os.path.isdir,
    288           check_msg='directories')
    289 
    290     if not on_bot and self._force:
    291       # When making the manual switch from legacy SVN checkouts to the new
    292       # Git-based Chromium DEPS, the .gclient_entries file that contains cached
    293       # URLs for all DEPS entries must be removed to avoid future sync problems.
    294       entries_file = os.path.join(os.path.dirname(ROOT_DIR), '.gclient_entries')
    295       if os.path.exists(entries_file):
    296         actions.append(Remove(entries_file, dangerous=True))
    297 
    298     actions.sort()
    299 
    300     if self._dry_run:
    301       for action in actions:
    302         action.announce(planning=True)
    303       logging.info('Not doing anything because dry-run was specified.')
    304       sys.exit(0)
    305 
    306     if any(a.dangerous for a in actions):
    307       logging.warn('Dangerous actions:')
    308       for action in (a for a in actions if a.dangerous):
    309         action.announce(planning=True)
    310       print
    311 
    312       if not self._force:
    313         logging.error(textwrap.dedent("""\
    314         @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
    315                               A C T I O N     R E Q I R E D
    316         @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
    317 
    318         Because chromium/src is transitioning to Git (from SVN), we needed to
    319         change the way that the WebRTC standalone checkout works. Instead of
    320         individually syncing subdirectories of Chromium in SVN, we're now
    321         syncing Chromium (and all of its DEPS, as defined by its own DEPS file),
    322         into the `chromium/src` directory.
    323 
    324         As such, all Chromium directories which are currently pulled by DEPS are
    325         now replaced with a symlink into the full Chromium checkout.
    326 
    327         To avoid disrupting developers, we've chosen to not delete your
    328         directories forcibly, in case you have some work in progress in one of
    329         them :).
    330 
    331         ACTION REQUIRED:
    332         Before running `gclient sync|runhooks` again, you must run:
    333         %s%s --force
    334 
    335         Which will replace all directories which now must be symlinks, after
    336         prompting with a summary of the work-to-be-done.
    337         """), 'python ' if sys.platform.startswith('win') else '', sys.argv[0])
    338         sys.exit(1)
    339       elif self._prompt:
    340         if not query_yes_no('Would you like to perform the above plan?'):
    341           sys.exit(1)
    342 
    343     for action in actions:
    344       action.announce(planning=False)
    345       action.doit(self._links_db)
    346 
    347     if not on_bot and self._force:
    348       logging.info('Completed!\n\nNow run `gclient sync|runhooks` again to '
    349                    'let the remaining hooks (that probably were interrupted) '
    350                    'execute.')
    351 
    352   def CleanupLinks(self):
    353     logging.debug('CleanupLinks')
    354     for source, link_path  in self._links_db.iteritems():
    355       if source == 'SCHEMA_VERSION':
    356         continue
    357       if os.path.islink(link_path) or sys.platform.startswith('win'):
    358         # os.path.islink() always returns false on Windows
    359         # See http://bugs.python.org/issue13143.
    360         logging.debug('Removing link to %s at %s', source, link_path)
    361         if not self._dry_run:
    362           if os.path.exists(link_path):
    363             if sys.platform.startswith('win') and os.path.isdir(link_path):
    364               subprocess.check_call(['rmdir', '/q', '/s', link_path],
    365                                     shell=True)
    366             else:
    367               os.remove(link_path)
    368           del self._links_db[source]
    369 
    370   @staticmethod
    371   def _ActionForPath(source_path, link_path=None, check_fn=None,
    372                      check_msg=None):
    373     """Create zero or more Actions to link to a file or directory.
    374 
    375     This will be a symlink on POSIX platforms. On Windows this requires
    376     that NTFS is version 5.0 or higher (Vista or newer).
    377 
    378     Args:
    379       source_path: Path relative to the Chromium checkout root.
    380         For readability, the path may contain slashes, which will
    381         automatically be converted to the right path delimiter on Windows.
    382       link_path: The location for the link to create. If omitted it will be the
    383         same path as source_path.
    384       check_fn: A function returning true if the type of filesystem object is
    385         correct for the attempted call. Otherwise an error message with
    386         check_msg will be printed.
    387       check_msg: String used to inform the user of an invalid attempt to create
    388         a file.
    389     Returns:
    390       A list of Action objects.
    391     """
    392     def fix_separators(path):
    393       if sys.platform.startswith('win'):
    394         return path.replace(os.altsep, os.sep)
    395       else:
    396         return path
    397 
    398     assert check_fn
    399     assert check_msg
    400     link_path = link_path or source_path
    401     link_path = fix_separators(link_path)
    402 
    403     source_path = fix_separators(source_path)
    404     source_path = os.path.join(CHROMIUM_CHECKOUT, source_path)
    405     if os.path.exists(source_path) and not check_fn:
    406       raise LinkError('_LinkChromiumPath can only be used to link to %s: '
    407                       'Tried to link to: %s' % (check_msg, source_path))
    408 
    409     if not os.path.exists(source_path):
    410       logging.debug('Silently ignoring missing source: %s. This is to avoid '
    411                     'errors on platform-specific dependencies.', source_path)
    412       return []
    413 
    414     actions = []
    415 
    416     if os.path.exists(link_path) or os.path.islink(link_path):
    417       if os.path.islink(link_path):
    418         actions.append(Remove(link_path, dangerous=False))
    419       elif os.path.isfile(link_path):
    420         actions.append(Remove(link_path, dangerous=True))
    421       elif os.path.isdir(link_path):
    422         actions.append(Rmtree(link_path))
    423       else:
    424         raise LinkError('Don\'t know how to plan: %s' % link_path)
    425 
    426     # Create parent directories to the target link if needed.
    427     target_parent_dirs = os.path.dirname(link_path)
    428     if (target_parent_dirs and
    429         target_parent_dirs != link_path and
    430         not os.path.exists(target_parent_dirs)):
    431       actions.append(Makedirs(target_parent_dirs))
    432 
    433     actions.append(Symlink(source_path, link_path))
    434 
    435     return actions
    436 
    437 def _initialize_database(filename):
    438   links_database = shelve.open(filename)
    439 
    440   # Wipe the database if this version of the script ends up looking at a
    441   # newer (future) version of the links db, just to be sure.
    442   version = links_database.get('SCHEMA_VERSION')
    443   if version and version != SCHEMA_VERSION:
    444     logging.info('Found database with schema version %s while this script only '
    445                  'supports %s. Wiping previous database contents.', version,
    446                  SCHEMA_VERSION)
    447     links_database.clear()
    448   links_database['SCHEMA_VERSION'] = SCHEMA_VERSION
    449   return links_database
    450 
    451 
    452 def main():
    453   on_bot = os.environ.get('CHROME_HEADLESS') == '1'
    454 
    455   parser = optparse.OptionParser()
    456   parser.add_option('-d', '--dry-run', action='store_true', default=False,
    457                     help='Print what would be done, but don\'t perform any '
    458                          'operations. This will automatically set logging to '
    459                          'verbose.')
    460   parser.add_option('-c', '--clean-only', action='store_true', default=False,
    461                     help='Only clean previously created links, don\'t create '
    462                          'new ones. This will automatically set logging to '
    463                          'verbose.')
    464   parser.add_option('-f', '--force', action='store_true', default=on_bot,
    465                     help='Force link creation. CAUTION: This deletes existing '
    466                          'folders and files in the locations where links are '
    467                          'about to be created.')
    468   parser.add_option('-n', '--no-prompt', action='store_false', dest='prompt',
    469                     default=(not on_bot),
    470                     help='Prompt if we\'re planning to do a dangerous action')
    471   parser.add_option('-v', '--verbose', action='store_const',
    472                     const=logging.DEBUG, default=logging.INFO,
    473                     help='Print verbose output for debugging.')
    474   options, _ = parser.parse_args()
    475 
    476   if options.dry_run or options.force or options.clean_only:
    477     options.verbose = logging.DEBUG
    478   logging.basicConfig(format='%(message)s', level=options.verbose)
    479 
    480   # Work from the root directory of the checkout.
    481   script_dir = os.path.dirname(os.path.abspath(__file__))
    482   os.chdir(script_dir)
    483 
    484   if sys.platform.startswith('win'):
    485     def is_admin():
    486       try:
    487         return os.getuid() == 0
    488       except AttributeError:
    489         return ctypes.windll.shell32.IsUserAnAdmin() != 0
    490     if not is_admin():
    491       logging.error('On Windows, you now need to have administrator '
    492                     'privileges for the shell running %s (or '
    493                     '`gclient sync|runhooks`).\nPlease start another command '
    494                     'prompt as Administrator and try again.', sys.argv[0])
    495       return 1
    496 
    497   if not os.path.exists(CHROMIUM_CHECKOUT):
    498     logging.error('Cannot find a Chromium checkout at %s. Did you run "gclient '
    499                   'sync" before running this script?', CHROMIUM_CHECKOUT)
    500     return 2
    501 
    502   links_database = _initialize_database(LINKS_DB)
    503   try:
    504     symlink_creator = WebRTCLinkSetup(links_database, options.force,
    505                                       options.dry_run, options.prompt)
    506     symlink_creator.CleanupLinks()
    507     if not options.clean_only:
    508       symlink_creator.CreateLinks(on_bot)
    509   except LinkError as e:
    510     print >> sys.stderr, e.message
    511     return 3
    512   finally:
    513     links_database.close()
    514   return 0
    515 
    516 
    517 if __name__ == '__main__':
    518   sys.exit(main())
    519