Home | History | Annotate | Download | only in utils 
      1 #!/usr/bin/python
      2 #
      3 # Please keep this code python 2.4 compatible and stand alone.
      4 
      5 """
      6 Fetch, build and install external Python library dependancies.
      7 
      8 This fetches external python libraries, builds them using your host's
      9 python and installs them under our own autotest/site-packages/ directory.
     10 
     11 Usage?  Just run it.
     12     utils/build_externals.py
     13 """
     14 
     15 import argparse
     16 import compileall
     17 import logging
     18 import os
     19 import sys
     20 
     21 import common
     22 from autotest_lib.client.common_lib import logging_config, logging_manager
     23 from autotest_lib.client.common_lib import utils
     24 from autotest_lib.utils import external_packages
     25 
     26 # bring in site packages as well
     27 utils.import_site_module(__file__, 'autotest_lib.utils.site_external_packages')
     28 
     29 # Where package source be fetched to relative to the top of the autotest tree.
     30 PACKAGE_DIR = 'ExternalSource'
     31 
     32 # Where packages will be installed to relative to the top of the autotest tree.
     33 INSTALL_DIR = 'site-packages'
     34 
     35 # Installs all packages, even if the system already has the version required
     36 INSTALL_ALL = False
     37 
     38 
     39 # Want to add more packages to fetch, build and install?  See the class
     40 # definitions at the end of external_packages.py for examples of how to do it.
     41 
     42 
     43 class BuildExternalsLoggingConfig(logging_config.LoggingConfig):
     44     def configure_logging(self, results_dir=None, verbose=False):
     45         super(BuildExternalsLoggingConfig, self).configure_logging(
     46                                                                use_console=True,
     47                                                                verbose=verbose)
     48 
     49 
     50 def main():
     51     """
     52     Find all ExternalPackage classes defined in this file and ask them to
     53     fetch, build and install themselves.
     54     """
     55     options = parse_arguments(sys.argv[1:])
     56     logging_manager.configure_logging(BuildExternalsLoggingConfig(),
     57                                       verbose=True)
     58     os.umask(022)
     59 
     60     top_of_tree = external_packages.find_top_of_autotest_tree()
     61     package_dir = os.path.join(top_of_tree, PACKAGE_DIR)
     62     install_dir = os.path.join(top_of_tree, INSTALL_DIR)
     63 
     64     # Make sure the install_dir is in our python module search path
     65     # as well as the PYTHONPATH being used by all our setup.py
     66     # install subprocesses.
     67     if install_dir not in sys.path:
     68         sys.path.insert(0, install_dir)
     69     env_python_path_varname = 'PYTHONPATH'
     70     env_python_path = os.environ.get(env_python_path_varname, '')
     71     if install_dir+':' not in env_python_path:
     72         os.environ[env_python_path_varname] = ':'.join([
     73             install_dir, env_python_path])
     74 
     75     fetched_packages, fetch_errors = fetch_necessary_packages(
     76         package_dir, install_dir, set(options.names_to_check))
     77     install_errors = build_and_install_packages(
     78         fetched_packages, install_dir, options.use_chromite_master)
     79 
     80     # Byte compile the code after it has been installed in its final
     81     # location as .pyc files contain the path passed to compile_dir().
     82     # When printing exception tracebacks, python uses that path first to look
     83     # for the source code before checking the directory of the .pyc file.
     84     # Don't leave references to our temporary build dir in the files.
     85     logging.info('compiling .py files in %s to .pyc', install_dir)
     86     compileall.compile_dir(install_dir, quiet=True)
     87 
     88     # Some things install with whacky permissions, fix that.
     89     external_packages.system("chmod -R a+rX '%s'" % install_dir)
     90 
     91     errors = fetch_errors + install_errors
     92     for error_msg in errors:
     93         logging.error(error_msg)
     94 
     95     return len(errors)
     96 
     97 
     98 def parse_arguments(args):
     99     """Parse command line arguments.
    100 
    101     @param args: The command line arguments to parse. (ususally sys.argsv[1:])
    102 
    103     @returns An argparse.Namespace populated with argument values.
    104     """
    105     parser = argparse.ArgumentParser(
    106             description='Command to build third party dependencies required '
    107                         'for autotest.')
    108     parser.add_argument('--use_chromite_master', action='store_true',
    109                         help='Update chromite to master branch, rather than '
    110                              'prod.')
    111     parser.add_argument('--names_to_check', nargs='*', type=str, default=set(),
    112                         help='Package names to check whether they are needed '
    113                              'in current system.')
    114     return parser.parse_args(args)
    115 
    116 
    117 def fetch_necessary_packages(dest_dir, install_dir, names_to_check=set()):
    118     """
    119     Fetches all ExternalPackages into dest_dir.
    120 
    121     @param dest_dir: Directory the packages should be fetched into.
    122     @param install_dir: Directory where packages will later installed.
    123     @param names_to_check: A set of package names to check whether they are
    124                            needed on current system. Default is empty.
    125 
    126     @returns A tuple containing two lists:
    127              * A list of ExternalPackage instances that were fetched and
    128                need to be installed.
    129              * A list of error messages for any failed fetches.
    130     """
    131     errors = []
    132     fetched_packages = []
    133     for package_class in external_packages.ExternalPackage.subclasses:
    134         package = package_class()
    135         if names_to_check and package.name.lower() not in names_to_check:
    136             continue
    137         if not package.is_needed(install_dir):
    138             logging.info('A new %s is not needed on this system.',
    139                          package.name)
    140             if INSTALL_ALL:
    141                 logging.info('Installing anyways...')
    142             else:
    143                 continue
    144         if not package.fetch(dest_dir):
    145             msg = 'Unable to download %s' % package.name
    146             logging.error(msg)
    147             errors.append(msg)
    148         else:
    149             fetched_packages.append(package)
    150 
    151     return fetched_packages, errors
    152 
    153 
    154 def build_and_install_packages(packages, install_dir,
    155                                use_chromite_master=False):
    156     """
    157     Builds and installs all packages into install_dir.
    158 
    159     @param packages - A list of already fetched ExternalPackage instances.
    160     @param install_dir - Directory the packages should be installed into.
    161     @param use_chromite_master: True if updating chromite to master branch.
    162 
    163     @returns A list of error messages for any installs that failed.
    164     """
    165     errors = []
    166     for package in packages:
    167         if use_chromite_master and package.name.lower() == 'chromiterepo':
    168             result = package.build_and_install(install_dir, master_branch=True)
    169         else:
    170             result = package.build_and_install(install_dir)
    171         if isinstance(result, bool):
    172             success = result
    173             message = None
    174         else:
    175             success = result[0]
    176             message = result[1]
    177         if not success:
    178             msg = ('Unable to build and install %s.\nError: %s' %
    179                    (package.name, message))
    180             logging.error(msg)
    181             errors.append(msg)
    182     return errors
    183 
    184 
    185 if __name__ == '__main__':
    186     sys.exit(main())
    187