Home | History | Annotate | Download | only in utils 
      1 #!/usr/bin/python
      2 #
      3 # Please keep this code python 2.4 compatible and stand alone.
      4 
      5 """
      6 Fetch, build and install external Python library dependancies.
      7 
      8 This fetches external python libraries, builds them using your host's
      9 python and installs them under our own autotest/site-packages/ directory.
     10 
     11 Usage?  Just run it.
     12     utils/build_externals.py
     13 """
     14 
     15 import compileall, logging, os, sys
     16 import common
     17 from autotest_lib.client.common_lib import logging_config, logging_manager
     18 from autotest_lib.client.common_lib import utils
     19 from autotest_lib.utils import external_packages
     20 
     21 # bring in site packages as well
     22 utils.import_site_module(__file__, 'autotest_lib.utils.site_external_packages')
     23 
     24 # Where package source be fetched to relative to the top of the autotest tree.
     25 PACKAGE_DIR = 'ExternalSource'
     26 
     27 # Where packages will be installed to relative to the top of the autotest tree.
     28 INSTALL_DIR = 'site-packages'
     29 
     30 # Installs all packages, even if the system already has the version required
     31 INSTALL_ALL = False
     32 
     33 
     34 # Want to add more packages to fetch, build and install?  See the class
     35 # definitions at the end of external_packages.py for examples of how to do it.
     36 
     37 
     38 class BuildExternalsLoggingConfig(logging_config.LoggingConfig):
     39     def configure_logging(self, results_dir=None, verbose=False):
     40         super(BuildExternalsLoggingConfig, self).configure_logging(
     41                                                                use_console=True,
     42                                                                verbose=verbose)
     43 
     44 
     45 def main():
     46     """
     47     Find all ExternalPackage classes defined in this file and ask them to
     48     fetch, build and install themselves.
     49     """
     50     logging_manager.configure_logging(BuildExternalsLoggingConfig(),
     51                                       verbose=True)
     52     os.umask(022)
     53 
     54     top_of_tree = external_packages.find_top_of_autotest_tree()
     55     package_dir = os.path.join(top_of_tree, PACKAGE_DIR)
     56     install_dir = os.path.join(top_of_tree, INSTALL_DIR)
     57 
     58     # Make sure the install_dir is in our python module search path
     59     # as well as the PYTHONPATH being used by all our setup.py
     60     # install subprocesses.
     61     if install_dir not in sys.path:
     62         sys.path.insert(0, install_dir)
     63     env_python_path_varname = 'PYTHONPATH'
     64     env_python_path = os.environ.get(env_python_path_varname, '')
     65     if install_dir+':' not in env_python_path:
     66         os.environ[env_python_path_varname] = ':'.join([
     67             install_dir, env_python_path])
     68 
     69     fetched_packages, fetch_errors = fetch_necessary_packages(package_dir,
     70                                                               install_dir)
     71     install_errors = build_and_install_packages(fetched_packages, install_dir)
     72 
     73     # Byte compile the code after it has been installed in its final
     74     # location as .pyc files contain the path passed to compile_dir().
     75     # When printing exception tracebacks, python uses that path first to look
     76     # for the source code before checking the directory of the .pyc file.
     77     # Don't leave references to our temporary build dir in the files.
     78     logging.info('compiling .py files in %s to .pyc', install_dir)
     79     compileall.compile_dir(install_dir, quiet=True)
     80 
     81     # Some things install with whacky permissions, fix that.
     82     external_packages.system("chmod -R a+rX '%s'" % install_dir)
     83 
     84     errors = fetch_errors + install_errors
     85     for error_msg in errors:
     86         logging.error(error_msg)
     87 
     88     return len(errors)
     89 
     90 
     91 def fetch_necessary_packages(dest_dir, install_dir):
     92     """
     93     Fetches all ExternalPackages into dest_dir.
     94 
     95     @param dest_dir: Directory the packages should be fetched into.
     96     @param install_dir: Directory where packages will later installed.
     97 
     98     @returns A tuple containing two lists:
     99              * A list of ExternalPackage instances that were fetched and
    100                need to be installed.
    101              * A list of error messages for any failed fetches.
    102     """
    103     names_to_check = sys.argv[1:]
    104     errors = []
    105     fetched_packages = []
    106     for package_class in external_packages.ExternalPackage.subclasses:
    107         package = package_class()
    108         if names_to_check and package.name.lower() not in names_to_check:
    109             continue
    110         if not package.is_needed(install_dir):
    111             logging.info('A new %s is not needed on this system.',
    112                          package.name)
    113             if INSTALL_ALL:
    114                 logging.info('Installing anyways...')
    115             else:
    116                 continue
    117         if not package.fetch(dest_dir):
    118             msg = 'Unable to download %s' % package.name
    119             logging.error(msg)
    120             errors.append(msg)
    121         else:
    122             fetched_packages.append(package)
    123 
    124     return fetched_packages, errors
    125 
    126 
    127 def build_and_install_packages(packages, install_dir):
    128     """
    129     Builds and installs all packages into install_dir.
    130 
    131     @param packages - A list of already fetched ExternalPackage instances.
    132     @param install_dir - Directory the packages should be installed into.
    133 
    134     @returns A list of error messages for any installs that failed.
    135     """
    136     errors = []
    137     for package in packages:
    138         result = package.build_and_install(install_dir)
    139         if isinstance(result, bool):
    140             success = result
    141             message = None
    142         else:
    143             success = result[0]
    144             message = result[1]
    145         if not success:
    146             msg = ('Unable to build and install %s.\nError: %s' %
    147                    (package.name, message))
    148             logging.error(msg)
    149             errors.append(msg)
    150     return errors
    151 
    152 
    153 if __name__ == '__main__':
    154     sys.exit(main())
    155