Home | History | Annotate | Download | only in utils 
      1 #!/usr/bin/python
      2 #
      3 # Please keep this code python 2.4 compatible and standalone.
      4 
      5 """
      6 Fetch, build and install external Python library dependancies.
      7 
      8 This fetches external python libraries, builds them using your host's
      9 python and installs them under our own autotest/site-packages/ directory.
     10 
     11 Usage?  Just run it.
     12     utils/build_externals.py
     13 """
     14 
     15 import argparse
     16 import compileall
     17 import logging
     18 import os
     19 import sys
     20 
     21 import common
     22 from autotest_lib.client.common_lib import logging_config, logging_manager
     23 from autotest_lib.client.common_lib import utils
     24 from autotest_lib.utils import external_packages
     25 
     26 # bring in site packages as well
     27 utils.import_site_module(__file__, 'autotest_lib.utils.site_external_packages')
     28 
     29 # Where package source be fetched to relative to the top of the autotest tree.
     30 PACKAGE_DIR = 'ExternalSource'
     31 
     32 # Where packages will be installed to relative to the top of the autotest tree.
     33 INSTALL_DIR = 'site-packages'
     34 
     35 # Installs all packages, even if the system already has the version required
     36 INSTALL_ALL = False
     37 
     38 
     39 # Want to add more packages to fetch, build and install?  See the class
     40 # definitions at the end of external_packages.py for examples of how to do it.
     41 
     42 
     43 class BuildExternalsLoggingConfig(logging_config.LoggingConfig):
     44     """Logging manager config."""
     45 
     46     def configure_logging(self, results_dir=None, verbose=False):
     47         """Configure logging."""
     48         super(BuildExternalsLoggingConfig, self).configure_logging(
     49                                                                use_console=True,
     50                                                                verbose=verbose)
     51 
     52 
     53 def main():
     54     """
     55     Find all ExternalPackage classes defined in this file and ask them to
     56     fetch, build and install themselves.
     57     """
     58     options = parse_arguments(sys.argv[1:])
     59     logging_manager.configure_logging(BuildExternalsLoggingConfig(),
     60                                       verbose=True)
     61     os.umask(022)
     62 
     63     top_of_tree = external_packages.find_top_of_autotest_tree()
     64     package_dir = os.path.join(top_of_tree, PACKAGE_DIR)
     65     install_dir = os.path.join(top_of_tree, INSTALL_DIR)
     66 
     67     # Make sure the install_dir is in our python module search path
     68     # as well as the PYTHONPATH being used by all our setup.py
     69     # install subprocesses.
     70     if install_dir not in sys.path:
     71         sys.path.insert(0, install_dir)
     72     env_python_path_varname = 'PYTHONPATH'
     73     env_python_path = os.environ.get(env_python_path_varname, '')
     74     if install_dir+':' not in env_python_path:
     75         os.environ[env_python_path_varname] = ':'.join([
     76             install_dir, env_python_path])
     77 
     78     fetched_packages, fetch_errors = fetch_necessary_packages(
     79         package_dir, install_dir, set(options.names_to_check))
     80     install_errors = build_and_install_packages(
     81         fetched_packages, install_dir, options.use_chromite_master)
     82 
     83     # Byte compile the code after it has been installed in its final
     84     # location as .pyc files contain the path passed to compile_dir().
     85     # When printing exception tracebacks, python uses that path first to look
     86     # for the source code before checking the directory of the .pyc file.
     87     # Don't leave references to our temporary build dir in the files.
     88     logging.info('compiling .py files in %s to .pyc', install_dir)
     89     compileall.compile_dir(install_dir, quiet=True)
     90 
     91     # Some things install with whacky permissions, fix that.
     92     external_packages.system("chmod -R a+rX '%s'" % install_dir)
     93 
     94     errors = fetch_errors + install_errors
     95     for error_msg in errors:
     96         logging.error(error_msg)
     97 
     98     if not errors:
     99       logging.info("Syntax errors from pylint above are expected, not "
    100                    "problematic. SUCCESS.")
    101     else:
    102       logging.info("Problematic errors encountered. FAILURE.")
    103     return len(errors)
    104 
    105 
    106 def parse_arguments(args):
    107     """Parse command line arguments.
    108 
    109     @param args: The command line arguments to parse. (ususally sys.argsv[1:])
    110 
    111     @returns An argparse.Namespace populated with argument values.
    112     """
    113     parser = argparse.ArgumentParser(
    114             description='Command to build third party dependencies required '
    115                         'for autotest.')
    116     parser.add_argument('--use_chromite_master', action='store_true',
    117                         help='Update chromite to master branch, rather than '
    118                              'prod.')
    119     parser.add_argument('--names_to_check', nargs='*', type=str, default=set(),
    120                         help='Package names to check whether they are needed '
    121                              'in current system.')
    122     return parser.parse_args(args)
    123 
    124 
    125 def fetch_necessary_packages(dest_dir, install_dir, names_to_check=set()):
    126     """
    127     Fetches all ExternalPackages into dest_dir.
    128 
    129     @param dest_dir: Directory the packages should be fetched into.
    130     @param install_dir: Directory where packages will later installed.
    131     @param names_to_check: A set of package names to check whether they are
    132                            needed on current system. Default is empty.
    133 
    134     @returns A tuple containing two lists:
    135              * A list of ExternalPackage instances that were fetched and
    136                need to be installed.
    137              * A list of error messages for any failed fetches.
    138     """
    139     errors = []
    140     fetched_packages = []
    141     for package_class in external_packages.ExternalPackage.subclasses:
    142         package = package_class()
    143         if names_to_check and package.name.lower() not in names_to_check:
    144             continue
    145         if not package.is_needed(install_dir):
    146             logging.info('A new %s is not needed on this system.',
    147                          package.name)
    148             if INSTALL_ALL:
    149                 logging.info('Installing anyways...')
    150             else:
    151                 continue
    152         if not package.fetch(dest_dir):
    153             msg = 'Unable to download %s' % package.name
    154             logging.error(msg)
    155             errors.append(msg)
    156         else:
    157             fetched_packages.append(package)
    158 
    159     return fetched_packages, errors
    160 
    161 
    162 def build_and_install_packages(packages, install_dir,
    163                                use_chromite_master=False):
    164     """
    165     Builds and installs all packages into install_dir.
    166 
    167     @param packages - A list of already fetched ExternalPackage instances.
    168     @param install_dir - Directory the packages should be installed into.
    169     @param use_chromite_master: True if updating chromite to master branch.
    170 
    171     @returns A list of error messages for any installs that failed.
    172     """
    173     errors = []
    174     for package in packages:
    175         if use_chromite_master and package.name.lower() == 'chromiterepo':
    176             result = package.build_and_install(install_dir, master_branch=True)
    177         else:
    178             result = package.build_and_install(install_dir)
    179         if isinstance(result, bool):
    180             success = result
    181             message = None
    182         else:
    183             success = result[0]
    184             message = result[1]
    185         if not success:
    186             msg = ('Unable to build and install %s.\nError: %s' %
    187                    (package.name, message))
    188             logging.error(msg)
    189             errors.append(msg)
    190     return errors
    191 
    192 
    193 if __name__ == '__main__':
    194     sys.exit(main())
    195