1 # Please keep this code python 2.4 compatible and stand alone. 2 3 import logging, os, shutil, sys, tempfile, time, urllib2 4 import subprocess, re 5 from distutils.version import LooseVersion 6 7 from autotest_lib.client.common_lib import autotemp, revision_control, utils 8 9 _READ_SIZE = 64*1024 10 _MAX_PACKAGE_SIZE = 100*1024*1024 11 _CHROMEOS_MIRROR = ('http://commondatastorage.googleapis.com/' 12 'chromeos-mirror/gentoo/distfiles/') 13 14 15 class Error(Exception): 16 """Local exception to be raised by code in this file.""" 17 18 class FetchError(Error): 19 """Failed to fetch a package from any of its listed URLs.""" 20 21 22 def _checksum_file(full_path): 23 """@returns The hex checksum of a file given its pathname.""" 24 inputfile = open(full_path, 'rb') 25 try: 26 hex_sum = utils.hash('sha1', inputfile.read()).hexdigest() 27 finally: 28 inputfile.close() 29 return hex_sum 30 31 32 def system(commandline): 33 """Same as os.system(commandline) but logs the command first. 34 35 @param commandline: commandline to be called. 36 """ 37 logging.info(commandline) 38 return os.system(commandline) 39 40 41 def find_top_of_autotest_tree(): 42 """@returns The full path to the top of the autotest directory tree.""" 43 dirname = os.path.dirname(__file__) 44 autotest_dir = os.path.abspath(os.path.join(dirname, '..')) 45 return autotest_dir 46 47 48 class ExternalPackage(object): 49 """ 50 Defines an external package with URLs to fetch its sources from and 51 a build_and_install() method to unpack it, build it and install it 52 beneath our own autotest/site-packages directory. 53 54 Base Class. Subclass this to define packages. 55 Note: Unless your subclass has a specific reason to, it should not 56 re-install the package every time build_externals is invoked, as this 57 happens periodically through the scheduler. To avoid doing so the is_needed 58 method needs to return an appropriate value. 59 60 Attributes: 61 @attribute urls - A tuple of URLs to try fetching the package from. 62 @attribute local_filename - A local filename to use when saving the 63 fetched package. 64 @attribute dist_name - The name of the Python distribution. For example, 65 the package MySQLdb is included in the distribution named 66 MySQL-python. This is generally the PyPI name. Defaults to the 67 name part of the local_filename. 68 @attribute hex_sum - The hex digest (currently SHA1) of this package 69 to be used to verify its contents. 70 @attribute module_name - The installed python module name to be used for 71 for a version check. Defaults to the lower case class name with 72 the word Package stripped off. 73 @attribute extracted_package_path - The path to package directory after 74 extracting. 75 @attribute version - The desired minimum package version. 76 @attribute os_requirements - A dictionary mapping pathname tuples on the 77 the OS distribution to a likely name of a package the user 78 needs to install on their system in order to get this file. 79 One of the files in the tuple must exist. 80 @attribute name - Read only, the printable name of the package. 81 @attribute subclasses - This class attribute holds a list of all defined 82 subclasses. It is constructed dynamically using the metaclass. 83 """ 84 # Modules that are meant to be installed in system directory, rather than 85 # autotest/site-packages. These modules should be skipped if the module 86 # is already installed in system directory. This prevents an older version 87 # of the module from being installed in system directory. 88 SYSTEM_MODULES = ['setuptools'] 89 90 subclasses = [] 91 urls = () 92 local_filename = None 93 dist_name = None 94 hex_sum = None 95 module_name = None 96 version = None 97 os_requirements = None 98 99 100 class __metaclass__(type): 101 """Any time a subclass is defined, add it to our list.""" 102 def __init__(mcs, name, bases, dict): 103 if name != 'ExternalPackage' and not name.startswith('_'): 104 mcs.subclasses.append(mcs) 105 106 107 def __init__(self): 108 self.verified_package = '' 109 if not self.module_name: 110 self.module_name = self.name.lower() 111 if not self.dist_name and self.local_filename: 112 self.dist_name = self.local_filename[:self.local_filename.rindex('-')] 113 self.installed_version = '' 114 115 116 @property 117 def extracted_package_path(self): 118 """Return the package path after extracting. 119 120 If the package has assigned its own extracted_package_path, use it. 121 Or use part of its local_filename as the extracting path. 122 """ 123 return self.local_filename[:-len(self._get_extension( 124 self.local_filename))] 125 126 127 @property 128 def name(self): 129 """Return the class name with any trailing 'Package' stripped off.""" 130 class_name = self.__class__.__name__ 131 if class_name.endswith('Package'): 132 return class_name[:-len('Package')] 133 return class_name 134 135 136 def is_needed(self, install_dir): 137 """ 138 Check to see if we need to reinstall a package. This is contingent on: 139 1. Module name: If the name of the module is different from the package, 140 the class that installs it needs to specify a module_name string, 141 so we can try importing the module. 142 143 2. Installed version: If the module doesn't contain a __version__ the 144 class that installs it needs to override the 145 _get_installed_version_from_module method to return an appropriate 146 version string. 147 148 3. Version/Minimum version: The class that installs the package should 149 contain a version string, and an optional minimum version string. 150 151 4. install_dir: If the module exists in a different directory, e.g., 152 /usr/lib/python2.7/dist-packages/, the module will be forced to be 153 installed in install_dir. 154 155 @param install_dir: install directory. 156 @returns True if self.module_name needs to be built and installed. 157 """ 158 if not self.module_name or not self.version: 159 logging.warning('version and module_name required for ' 160 'is_needed() check to work.') 161 return True 162 try: 163 module = __import__(self.module_name) 164 except ImportError, e: 165 logging.info("%s isn't present. Will install.", self.module_name) 166 return True 167 if (not module.__file__.startswith(install_dir) and 168 not self.module_name in self.SYSTEM_MODULES): 169 logging.info('Module %s is installed in %s, rather than %s. The ' 170 'module will be forced to be installed in %s.', 171 self.module_name, module.__file__, install_dir, 172 install_dir) 173 return True 174 self.installed_version = self._get_installed_version_from_module(module) 175 if not self.installed_version: 176 return True 177 178 logging.info('imported %s version %s.', self.module_name, 179 self.installed_version) 180 if hasattr(self, 'minimum_version'): 181 return LooseVersion(self.minimum_version) > LooseVersion( 182 self.installed_version) 183 else: 184 return LooseVersion(self.version) > LooseVersion( 185 self.installed_version) 186 187 188 def _get_installed_version_from_module(self, module): 189 """Ask our module its version string and return it or '' if unknown.""" 190 try: 191 return module.__version__ 192 except AttributeError: 193 logging.error('could not get version from %s', module) 194 return '' 195 196 197 def _build_and_install(self, install_dir): 198 """Subclasses MUST provide their own implementation.""" 199 raise NotImplementedError 200 201 202 def _build_and_install_current_dir(self, install_dir): 203 """ 204 Subclasses that use _build_and_install_from_package() MUST provide 205 their own implementation of this method. 206 """ 207 raise NotImplementedError 208 209 210 def build_and_install(self, install_dir): 211 """ 212 Builds and installs the package. It must have been fetched already. 213 214 @param install_dir - The package installation directory. If it does 215 not exist it will be created. 216 """ 217 if not self.verified_package: 218 raise Error('Must call fetch() first. - %s' % self.name) 219 self._check_os_requirements() 220 return self._build_and_install(install_dir) 221 222 223 def _check_os_requirements(self): 224 if not self.os_requirements: 225 return 226 failed = False 227 for file_names, package_name in self.os_requirements.iteritems(): 228 if not any(os.path.exists(file_name) for file_name in file_names): 229 failed = True 230 logging.error('Can\'t find %s, %s probably needs it.', 231 ' or '.join(file_names), self.name) 232 logging.error('Perhaps you need to install something similar ' 233 'to the %s package for OS first.', package_name) 234 if failed: 235 raise Error('Missing OS requirements for %s. (see above)' % 236 self.name) 237 238 239 def _build_and_install_current_dir_setup_py(self, install_dir): 240 """For use as a _build_and_install_current_dir implementation.""" 241 egg_path = self._build_egg_using_setup_py(setup_py='setup.py') 242 if not egg_path: 243 return False 244 return self._install_from_egg(install_dir, egg_path) 245 246 247 def _build_and_install_current_dir_setupegg_py(self, install_dir): 248 """For use as a _build_and_install_current_dir implementation.""" 249 egg_path = self._build_egg_using_setup_py(setup_py='setupegg.py') 250 if not egg_path: 251 return False 252 return self._install_from_egg(install_dir, egg_path) 253 254 255 def _build_and_install_current_dir_noegg(self, install_dir): 256 if not self._build_using_setup_py(): 257 return False 258 return self._install_using_setup_py_and_rsync(install_dir) 259 260 261 def _get_extension(self, package): 262 """Get extension of package.""" 263 valid_package_extensions = ['.tar.gz', '.tar.bz2', '.zip'] 264 extension = None 265 266 for ext in valid_package_extensions: 267 if package.endswith(ext): 268 extension = ext 269 break 270 271 if not extension: 272 raise Error('Unexpected package file extension on %s' % package) 273 274 return extension 275 276 277 def _build_and_install_from_package(self, install_dir): 278 """ 279 This method may be used as a _build_and_install() implementation 280 for subclasses if they implement _build_and_install_current_dir(). 281 282 Extracts the .tar.gz file, chdirs into the extracted directory 283 (which is assumed to match the tar filename) and calls 284 _build_and_isntall_current_dir from there. 285 286 Afterwards the build (regardless of failure) extracted .tar.gz 287 directory is cleaned up. 288 289 @returns True on success, False otherwise. 290 291 @raises OSError If the expected extraction directory does not exist. 292 """ 293 self._extract_compressed_package() 294 extension = self._get_extension(self.verified_package) 295 os.chdir(os.path.dirname(self.verified_package)) 296 os.chdir(self.extracted_package_path) 297 extracted_dir = os.getcwd() 298 try: 299 return self._build_and_install_current_dir(install_dir) 300 finally: 301 os.chdir(os.path.join(extracted_dir, '..')) 302 shutil.rmtree(extracted_dir) 303 304 305 def _extract_compressed_package(self): 306 """Extract the fetched compressed .tar or .zip within its directory.""" 307 if not self.verified_package: 308 raise Error('Package must have been fetched first.') 309 os.chdir(os.path.dirname(self.verified_package)) 310 if self.verified_package.endswith('gz'): 311 status = system("tar -xzf '%s'" % self.verified_package) 312 elif self.verified_package.endswith('bz2'): 313 status = system("tar -xjf '%s'" % self.verified_package) 314 elif self.verified_package.endswith('zip'): 315 status = system("unzip '%s'" % self.verified_package) 316 else: 317 raise Error('Unknown compression suffix on %s.' % 318 self.verified_package) 319 if status: 320 raise Error('tar failed with %s' % (status,)) 321 322 323 def _build_using_setup_py(self, setup_py='setup.py'): 324 """ 325 Assuming the cwd is the extracted python package, execute a simple 326 python setup.py build. 327 328 @param setup_py - The name of the setup.py file to execute. 329 330 @returns True on success, False otherwise. 331 """ 332 if not os.path.exists(setup_py): 333 raise Error('%s does not exist in %s' % (setup_py, os.getcwd())) 334 status = system("'%s' %s build" % (sys.executable, setup_py)) 335 if status: 336 logging.error('%s build failed.', self.name) 337 return False 338 return True 339 340 341 def _build_egg_using_setup_py(self, setup_py='setup.py'): 342 """ 343 Assuming the cwd is the extracted python package, execute a simple 344 python setup.py bdist_egg. 345 346 @param setup_py - The name of the setup.py file to execute. 347 348 @returns The relative path to the resulting egg file or '' on failure. 349 """ 350 if not os.path.exists(setup_py): 351 raise Error('%s does not exist in %s' % (setup_py, os.getcwd())) 352 egg_subdir = 'dist' 353 if os.path.isdir(egg_subdir): 354 shutil.rmtree(egg_subdir) 355 status = system("'%s' %s bdist_egg" % (sys.executable, setup_py)) 356 if status: 357 logging.error('bdist_egg of setuptools failed.') 358 return '' 359 # I've never seen a bdist_egg lay multiple .egg files. 360 for filename in os.listdir(egg_subdir): 361 if filename.endswith('.egg'): 362 return os.path.join(egg_subdir, filename) 363 364 365 def _install_from_egg(self, install_dir, egg_path): 366 """ 367 Install a module from an egg file by unzipping the necessary parts 368 into install_dir. 369 370 @param install_dir - The installation directory. 371 @param egg_path - The pathname of the egg file. 372 """ 373 status = system("unzip -q -o -d '%s' '%s'" % (install_dir, egg_path)) 374 if status: 375 logging.error('unzip of %s failed', egg_path) 376 return False 377 egg_info_dir = os.path.join(install_dir, 'EGG-INFO') 378 if os.path.isdir(egg_info_dir): 379 egg_info_new_path = self._get_egg_info_path(install_dir) 380 if egg_info_new_path: 381 if os.path.exists(egg_info_new_path): 382 shutil.rmtree(egg_info_new_path) 383 os.rename(egg_info_dir, egg_info_new_path) 384 else: 385 shutil.rmtree(egg_info_dir) 386 return True 387 388 389 def _get_egg_info_path(self, install_dir): 390 """Get egg-info path for this package. 391 392 Example path: install_dir/MySQL_python-1.2.3.egg-info 393 394 """ 395 if self.dist_name: 396 egg_info_name_part = self.dist_name.replace('-', '_') 397 if self.version: 398 egg_info_filename = '%s-%s.egg-info' % (egg_info_name_part, 399 self.version) 400 else: 401 egg_info_filename = '%s.egg-info' % (egg_info_name_part,) 402 return os.path.join(install_dir, egg_info_filename) 403 else: 404 return None 405 406 407 def _get_temp_dir(self): 408 return tempfile.mkdtemp(dir='/var/tmp') 409 410 411 def _site_packages_path(self, temp_dir): 412 # This makes assumptions about what python setup.py install 413 # does when given a prefix. Is this always correct? 414 python_xy = 'python%s' % sys.version[:3] 415 return os.path.join(temp_dir, 'lib', python_xy, 'site-packages') 416 417 418 def _rsync (self, temp_site_dir, install_dir): 419 """Rsync contents. """ 420 status = system("rsync -r '%s/' '%s/'" % 421 (os.path.normpath(temp_site_dir), 422 os.path.normpath(install_dir))) 423 if status: 424 logging.error('%s rsync to install_dir failed.', self.name) 425 return False 426 return True 427 428 429 def _install_using_setup_py_and_rsync(self, install_dir, 430 setup_py='setup.py', 431 temp_dir=None): 432 """ 433 Assuming the cwd is the extracted python package, execute a simple: 434 435 python setup.py install --prefix=BLA 436 437 BLA will be a temporary directory that everything installed will 438 be picked out of and rsynced to the appropriate place under 439 install_dir afterwards. 440 441 Afterwards, it deconstructs the extra lib/pythonX.Y/site-packages/ 442 directory tree that setuptools created and moves all installed 443 site-packages directly up into install_dir itself. 444 445 @param install_dir the directory for the install to happen under. 446 @param setup_py - The name of the setup.py file to execute. 447 448 @returns True on success, False otherwise. 449 """ 450 if not os.path.exists(setup_py): 451 raise Error('%s does not exist in %s' % (setup_py, os.getcwd())) 452 453 if temp_dir is None: 454 temp_dir = self._get_temp_dir() 455 456 try: 457 status = system("'%s' %s install --no-compile --prefix='%s'" 458 % (sys.executable, setup_py, temp_dir)) 459 if status: 460 logging.error('%s install failed.', self.name) 461 return False 462 463 if os.path.isdir(os.path.join(temp_dir, 'lib')): 464 # NOTE: This ignores anything outside of the lib/ dir that 465 # was installed. 466 temp_site_dir = self._site_packages_path(temp_dir) 467 else: 468 temp_site_dir = temp_dir 469 470 return self._rsync(temp_site_dir, install_dir) 471 finally: 472 shutil.rmtree(temp_dir) 473 474 475 476 def _build_using_make(self, install_dir): 477 """Build the current package using configure/make. 478 479 @returns True on success, False otherwise. 480 """ 481 install_prefix = os.path.join(install_dir, 'usr', 'local') 482 status = system('./configure --prefix=%s' % install_prefix) 483 if status: 484 logging.error('./configure failed for %s', self.name) 485 return False 486 status = system('make') 487 if status: 488 logging.error('make failed for %s', self.name) 489 return False 490 status = system('make check') 491 if status: 492 logging.error('make check failed for %s', self.name) 493 return False 494 return True 495 496 497 def _install_using_make(self): 498 """Install the current package using make install. 499 500 Assumes the install path was set up while running ./configure (in 501 _build_using_make()). 502 503 @returns True on success, False otherwise. 504 """ 505 status = system('make install') 506 return status == 0 507 508 509 def fetch(self, dest_dir): 510 """ 511 Fetch the package from one its URLs and save it in dest_dir. 512 513 If the the package already exists in dest_dir and the checksum 514 matches this code will not fetch it again. 515 516 Sets the 'verified_package' attribute with the destination pathname. 517 518 @param dest_dir - The destination directory to save the local file. 519 If it does not exist it will be created. 520 521 @returns A boolean indicating if we the package is now in dest_dir. 522 @raises FetchError - When something unexpected happens. 523 """ 524 if not os.path.exists(dest_dir): 525 os.makedirs(dest_dir) 526 local_path = os.path.join(dest_dir, self.local_filename) 527 528 # If the package exists, verify its checksum and be happy if it is good. 529 if os.path.exists(local_path): 530 actual_hex_sum = _checksum_file(local_path) 531 if self.hex_sum == actual_hex_sum: 532 logging.info('Good checksum for existing %s package.', 533 self.name) 534 self.verified_package = local_path 535 return True 536 logging.warning('Bad checksum for existing %s package. ' 537 'Re-downloading', self.name) 538 os.rename(local_path, local_path + '.wrong-checksum') 539 540 # Download the package from one of its urls, rejecting any if the 541 # checksum does not match. 542 for url in self.urls: 543 logging.info('Fetching %s', url) 544 try: 545 url_file = urllib2.urlopen(url) 546 except (urllib2.URLError, EnvironmentError): 547 logging.warning('Could not fetch %s package from %s.', 548 self.name, url) 549 continue 550 551 data_length = int(url_file.info().get('Content-Length', 552 _MAX_PACKAGE_SIZE)) 553 if data_length <= 0 or data_length > _MAX_PACKAGE_SIZE: 554 raise FetchError('%s from %s fails Content-Length %d ' 555 'sanity check.' % (self.name, url, 556 data_length)) 557 checksum = utils.hash('sha1') 558 total_read = 0 559 output = open(local_path, 'wb') 560 try: 561 while total_read < data_length: 562 data = url_file.read(_READ_SIZE) 563 if not data: 564 break 565 output.write(data) 566 checksum.update(data) 567 total_read += len(data) 568 finally: 569 output.close() 570 if self.hex_sum != checksum.hexdigest(): 571 logging.warning('Bad checksum for %s fetched from %s.', 572 self.name, url) 573 logging.warning('Got %s', checksum.hexdigest()) 574 logging.warning('Expected %s', self.hex_sum) 575 os.unlink(local_path) 576 continue 577 logging.info('Good checksum.') 578 self.verified_package = local_path 579 return True 580 else: 581 return False 582 583 584 # NOTE: This class definition must come -before- all other ExternalPackage 585 # classes that need to use this version of setuptools so that is is inserted 586 # into the ExternalPackage.subclasses list before them. 587 class SetuptoolsPackage(ExternalPackage): 588 """setuptools package""" 589 # For all known setuptools releases a string compare works for the 590 # version string. Hopefully they never release a 0.10. (Their own 591 # version comparison code would break if they did.) 592 # Any system with setuptools > 18.0.1 is fine. If none installed, then 593 # try to install the latest found on the upstream. 594 minimum_version = '18.0.1' 595 version = '18.0.1' 596 urls = (_CHROMEOS_MIRROR + 'setuptools-%s.tar.gz' % (version,),) 597 local_filename = 'setuptools-%s.tar.gz' % version 598 hex_sum = 'ebc4fe81b7f6d61d923d9519f589903824044f52' 599 600 SUDO_SLEEP_DELAY = 15 601 602 603 def _build_and_install(self, install_dir): 604 """Install setuptools on the system.""" 605 logging.info('NOTE: setuptools install does not use install_dir.') 606 return self._build_and_install_from_package(install_dir) 607 608 609 def _build_and_install_current_dir(self, install_dir): 610 egg_path = self._build_egg_using_setup_py() 611 if not egg_path: 612 return False 613 614 print '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n' 615 print 'About to run sudo to install setuptools', self.version 616 print 'on your system for use by', sys.executable, '\n' 617 print '!! ^C within', self.SUDO_SLEEP_DELAY, 'seconds to abort.\n' 618 time.sleep(self.SUDO_SLEEP_DELAY) 619 620 # Copy the egg to the local filesystem /var/tmp so that root can 621 # access it properly (avoid NFS squashroot issues). 622 temp_dir = self._get_temp_dir() 623 try: 624 shutil.copy(egg_path, temp_dir) 625 egg_name = os.path.split(egg_path)[1] 626 temp_egg = os.path.join(temp_dir, egg_name) 627 p = subprocess.Popen(['sudo', '/bin/sh', temp_egg], 628 stdout=subprocess.PIPE) 629 regex = re.compile('Copying (.*?) to (.*?)\n') 630 match = regex.search(p.communicate()[0]) 631 status = p.wait() 632 633 if match: 634 compiled = os.path.join(match.group(2), match.group(1)) 635 os.system("sudo chmod a+r '%s'" % compiled) 636 finally: 637 shutil.rmtree(temp_dir) 638 639 if status: 640 logging.error('install of setuptools from egg failed.') 641 return False 642 return True 643 644 645 class MySQLdbPackage(ExternalPackage): 646 """mysql package, used in scheduler.""" 647 module_name = 'MySQLdb' 648 version = '1.2.3' 649 local_filename = 'MySQL-python-%s.tar.gz' % version 650 urls = ('http://commondatastorage.googleapis.com/chromeos-mirror/gentoo/' 651 'distfiles/%s' % local_filename,) 652 hex_sum = '3511bb8c57c6016eeafa531d5c3ea4b548915e3c' 653 654 _build_and_install_current_dir = ( 655 ExternalPackage._build_and_install_current_dir_setup_py) 656 657 658 def _build_and_install(self, install_dir): 659 if not os.path.exists('/usr/bin/mysql_config'): 660 error_msg = '''\ 661 You need to install /usr/bin/mysql_config. 662 On recent Debian based distros, run: \ 663 sudo apt-get install libmariadbclient-dev-compat 664 On older Debian based distros, run: sudo apt-get install libmysqlclient15-dev 665 ''' 666 logging.error(error_msg) 667 return False, error_msg 668 return self._build_and_install_from_package(install_dir) 669 670 671 class DjangoPackage(ExternalPackage): 672 """django package.""" 673 version = '1.5.1' 674 local_filename = 'Django-%s.tar.gz' % version 675 urls = (_CHROMEOS_MIRROR + local_filename,) 676 hex_sum = '0ab97b90c4c79636e56337f426f1e875faccbba1' 677 678 _build_and_install = ExternalPackage._build_and_install_from_package 679 _build_and_install_current_dir = ( 680 ExternalPackage._build_and_install_current_dir_noegg) 681 682 683 def _get_installed_version_from_module(self, module): 684 try: 685 return module.get_version().split()[0] 686 except AttributeError: 687 return '0.9.6' 688 689 690 691 class NumpyPackage(ExternalPackage): 692 """numpy package, required by matploglib.""" 693 version = '1.7.0' 694 local_filename = 'numpy-%s.tar.gz' % version 695 urls = (_CHROMEOS_MIRROR + local_filename,) 696 hex_sum = 'ba328985f20390b0f969a5be2a6e1141d5752cf9' 697 698 _build_and_install = ExternalPackage._build_and_install_from_package 699 _build_and_install_current_dir = ( 700 ExternalPackage._build_and_install_current_dir_setupegg_py) 701 702 703 704 class JsonRPCLib(ExternalPackage): 705 """jsonrpclib package""" 706 version = '0.1.3' 707 module_name = 'jsonrpclib' 708 local_filename = '%s-%s.tar.gz' % (module_name, version) 709 urls = (_CHROMEOS_MIRROR + local_filename,) 710 hex_sum = '431714ed19ab677f641ce5d678a6a95016f5c452' 711 712 def _get_installed_version_from_module(self, module): 713 # jsonrpclib doesn't contain a proper version 714 return self.version 715 716 _build_and_install = ExternalPackage._build_and_install_from_package 717 _build_and_install_current_dir = ( 718 ExternalPackage._build_and_install_current_dir_noegg) 719 720 721 class GwtPackage(ExternalPackage): 722 """Fetch and extract a local copy of GWT used to build the frontend.""" 723 724 version = '2.3.0' 725 local_filename = 'gwt-%s.zip' % version 726 urls = (_CHROMEOS_MIRROR + local_filename,) 727 hex_sum = 'd51fce9166e6b31349659ffca89baf93e39bc84b' 728 name = 'gwt' 729 about_filename = 'about.txt' 730 module_name = None # Not a Python module. 731 732 733 def is_needed(self, install_dir): 734 gwt_dir = os.path.join(install_dir, self.name) 735 about_file = os.path.join(install_dir, self.name, self.about_filename) 736 737 if not os.path.exists(gwt_dir) or not os.path.exists(about_file): 738 logging.info('gwt not installed for autotest') 739 return True 740 741 f = open(about_file, 'r') 742 version_line = f.readline() 743 f.close() 744 745 match = re.match(r'Google Web Toolkit (.*)', version_line) 746 if not match: 747 logging.info('did not find gwt version') 748 return True 749 750 logging.info('found gwt version %s', match.group(1)) 751 return match.group(1) != self.version 752 753 754 def _build_and_install(self, install_dir): 755 os.chdir(install_dir) 756 self._extract_compressed_package() 757 extracted_dir = self.local_filename[:-len('.zip')] 758 target_dir = os.path.join(install_dir, self.name) 759 if os.path.exists(target_dir): 760 shutil.rmtree(target_dir) 761 os.rename(extracted_dir, target_dir) 762 return True 763 764 765 class PyudevPackage(ExternalPackage): 766 """ 767 pyudev module 768 769 Used in unittests. 770 """ 771 version = '0.16.1' 772 url_filename = 'pyudev-%s.tar.gz' % version 773 local_filename = url_filename 774 urls = (_CHROMEOS_MIRROR + local_filename,) 775 hex_sum = 'b36bc5c553ce9b56d32a5e45063a2c88156771c0' 776 777 _build_and_install = ExternalPackage._build_and_install_from_package 778 _build_and_install_current_dir = ( 779 ExternalPackage._build_and_install_current_dir_setup_py) 780 781 782 class PyMoxPackage(ExternalPackage): 783 """ 784 mox module 785 786 Used in unittests. 787 """ 788 module_name = 'mox' 789 version = '0.5.3' 790 # Note: url_filename does not match local_filename, because of 791 # an uncontrolled fork at some point in time of mox versions. 792 url_filename = 'mox-%s-autotest.tar.gz' % version 793 local_filename = 'mox-%s.tar.gz' % version 794 urls = (_CHROMEOS_MIRROR + url_filename,) 795 hex_sum = '1c502d2c0a8aefbba2c7f385a83d33e7d822452a' 796 797 _build_and_install = ExternalPackage._build_and_install_from_package 798 _build_and_install_current_dir = ( 799 ExternalPackage._build_and_install_current_dir_noegg) 800 801 def _get_installed_version_from_module(self, module): 802 # mox doesn't contain a proper version 803 return self.version 804 805 806 class PySeleniumPackage(ExternalPackage): 807 """ 808 selenium module 809 810 Used in wifi_interop suite. 811 """ 812 module_name = 'selenium' 813 version = '2.37.2' 814 url_filename = 'selenium-%s.tar.gz' % version 815 local_filename = url_filename 816 urls = (_CHROMEOS_MIRROR + local_filename,) 817 hex_sum = '66946d5349e36d946daaad625c83c30c11609e36' 818 819 _build_and_install = ExternalPackage._build_and_install_from_package 820 _build_and_install_current_dir = ( 821 ExternalPackage._build_and_install_current_dir_setup_py) 822 823 824 class FaultHandlerPackage(ExternalPackage): 825 """ 826 faulthandler module 827 """ 828 module_name = 'faulthandler' 829 version = '2.3' 830 url_filename = '%s-%s.tar.gz' % (module_name, version) 831 local_filename = url_filename 832 urls = (_CHROMEOS_MIRROR + local_filename,) 833 hex_sum = 'efb30c068414fba9df892e48fcf86170cbf53589' 834 835 _build_and_install = ExternalPackage._build_and_install_from_package 836 _build_and_install_current_dir = ( 837 ExternalPackage._build_and_install_current_dir_noegg) 838 839 840 class PsutilPackage(ExternalPackage): 841 """ 842 psutil module 843 """ 844 module_name = 'psutil' 845 version = '2.1.1' 846 url_filename = '%s-%s.tar.gz' % (module_name, version) 847 local_filename = url_filename 848 urls = (_CHROMEOS_MIRROR + local_filename,) 849 hex_sum = '0c20a20ed316e69f2b0881530439213988229916' 850 851 _build_and_install = ExternalPackage._build_and_install_from_package 852 _build_and_install_current_dir = ( 853 ExternalPackage._build_and_install_current_dir_setup_py) 854 855 856 class ElasticSearchPackage(ExternalPackage): 857 """elasticsearch-py package.""" 858 version = '1.6.0' 859 url_filename = 'elasticsearch-%s.tar.gz' % version 860 local_filename = url_filename 861 urls = ('https://pypi.python.org/packages/source/e/elasticsearch/%s' % 862 (url_filename),) 863 hex_sum = '3e676c96f47935b1f52df82df3969564bd356b1c' 864 _build_and_install = ExternalPackage._build_and_install_from_package 865 _build_and_install_current_dir = ( 866 ExternalPackage._build_and_install_current_dir_setup_py) 867 868 def _get_installed_version_from_module(self, module): 869 # Elastic's version format is like tuple (1, 6, 0), which needs to be 870 # transferred to 1.6.0. 871 try: 872 return '.'.join(str(i) for i in module.__version__) 873 except: 874 return self.version 875 876 877 class Urllib3Package(ExternalPackage): 878 """elasticsearch-py package.""" 879 version = '1.9' 880 url_filename = 'urllib3-%s.tar.gz' % version 881 local_filename = url_filename 882 urls = (_CHROMEOS_MIRROR + local_filename,) 883 hex_sum = '9522197efb2a2b49ce804de3a515f06d97b6602f' 884 _build_and_install = ExternalPackage._build_and_install_from_package 885 _build_and_install_current_dir = ( 886 ExternalPackage._build_and_install_current_dir_setup_py) 887 888 class ImagingLibraryPackage(ExternalPackage): 889 """Python Imaging Library (PIL).""" 890 version = '1.1.7' 891 url_filename = 'Imaging-%s.tar.gz' % version 892 local_filename = url_filename 893 urls = ('http://commondatastorage.googleapis.com/chromeos-mirror/gentoo/' 894 'distfiles/%s' % url_filename,) 895 hex_sum = '76c37504251171fda8da8e63ecb8bc42a69a5c81' 896 897 def _build_and_install(self, install_dir): 898 #The path of zlib library might be different from what PIL setup.py is 899 #expected. Following change does the best attempt to link the library 900 #to a path PIL setup.py will try. 901 libz_possible_path = '/usr/lib/x86_64-linux-gnu/libz.so' 902 libz_expected_path = '/usr/lib/libz.so' 903 if (os.path.exists(libz_possible_path) and 904 not os.path.exists(libz_expected_path)): 905 utils.run('sudo ln -s %s %s' % 906 (libz_possible_path, libz_expected_path)) 907 return self._build_and_install_from_package(install_dir) 908 909 _build_and_install_current_dir = ( 910 ExternalPackage._build_and_install_current_dir_noegg) 911 912 913 class AstroidPackage(ExternalPackage): 914 """astroid package.""" 915 version = '1.5.3' 916 url_filename = 'astroid-%s.tar.gz' % version 917 local_filename = url_filename 918 urls = (_CHROMEOS_MIRROR + local_filename,) 919 hex_sum = 'e654225ab5bd2788e5e246b156910990bf33cde6' 920 _build_and_install = ExternalPackage._build_and_install_from_package 921 _build_and_install_current_dir = ( 922 ExternalPackage._build_and_install_current_dir_setup_py) 923 924 925 class LazyObjectProxyPackage(ExternalPackage): 926 """lazy-object-proxy package (dependency for astroid).""" 927 version = '1.3.1' 928 url_filename = 'lazy-object-proxy-%s.tar.gz' % version 929 local_filename = url_filename 930 urls = (_CHROMEOS_MIRROR + local_filename,) 931 hex_sum = '984828d8f672986ca926373986214d7057b772fb' 932 _build_and_install = ExternalPackage._build_and_install_from_package 933 _build_and_install_current_dir = ( 934 ExternalPackage._build_and_install_current_dir_setup_py) 935 936 937 class SingleDispatchPackage(ExternalPackage): 938 """singledispatch package (dependency for astroid).""" 939 version = '3.4.0.3' 940 url_filename = 'singledispatch-%s.tar.gz' % version 941 local_filename = url_filename 942 urls = (_CHROMEOS_MIRROR + local_filename,) 943 hex_sum = 'f93241b06754a612af8bb7aa208c4d1805637022' 944 _build_and_install = ExternalPackage._build_and_install_from_package 945 _build_and_install_current_dir = ( 946 ExternalPackage._build_and_install_current_dir_setup_py) 947 948 949 class Enum34Package(ExternalPackage): 950 """enum34 package (dependency for astroid).""" 951 version = '1.1.6' 952 url_filename = 'enum34-%s.tar.gz' % version 953 local_filename = url_filename 954 urls = (_CHROMEOS_MIRROR + local_filename,) 955 hex_sum = '014ef5878333ff91099893d615192c8cd0b1525a' 956 _build_and_install = ExternalPackage._build_and_install_from_package 957 _build_and_install_current_dir = ( 958 ExternalPackage._build_and_install_current_dir_setup_py) 959 960 961 class WraptPackage(ExternalPackage): 962 """wrapt package (dependency for astroid).""" 963 version = '1.10.10' 964 url_filename = 'wrapt-%s.tar.gz' % version 965 local_filename = url_filename 966 #md5=97365e906afa8b431f266866ec4e2e18 967 urls = ('https://pypi.python.org/packages/a3/bb/' 968 '525e9de0a220060394f4aa34fdf6200853581803d92714ae41fc3556e7d7/%s' % 969 (url_filename),) 970 hex_sum = '6be4f1bb50db879863f4247692360eb830a3eb33' 971 _build_and_install = ExternalPackage._build_and_install_from_package 972 _build_and_install_current_dir = ( 973 ExternalPackage._build_and_install_current_dir_noegg) 974 975 976 class SixPackage(ExternalPackage): 977 """six package (dependency for astroid).""" 978 version = '1.10.0' 979 url_filename = 'six-%s.tar.gz' % version 980 local_filename = url_filename 981 urls = (_CHROMEOS_MIRROR + local_filename,) 982 hex_sum = '30d480d2e352e8e4c2aae042cf1bf33368ff0920' 983 _build_and_install = ExternalPackage._build_and_install_from_package 984 _build_and_install_current_dir = ( 985 ExternalPackage._build_and_install_current_dir_setup_py) 986 987 988 class LruCachePackage(ExternalPackage): 989 """backports.functools_lru_cache package (dependency for astroid).""" 990 version = '1.4' 991 url_filename = 'backports.functools_lru_cache-%s.tar.gz' % version 992 local_filename = url_filename 993 urls = (_CHROMEOS_MIRROR + local_filename,) 994 hex_sum = '8a546e7887e961c2873c9b053f4e2cd2a96bd71d' 995 _build_and_install = ExternalPackage._build_and_install_from_package 996 _build_and_install_current_dir = ( 997 ExternalPackage._build_and_install_current_dir_setup_py) 998 999 1000 class LogilabCommonPackage(ExternalPackage): 1001 """logilab-common package.""" 1002 version = '1.2.2' 1003 module_name = 'logilab' 1004 url_filename = 'logilab-common-%s.tar.gz' % version 1005 local_filename = url_filename 1006 urls = (_CHROMEOS_MIRROR + local_filename,) 1007 hex_sum = 'ecad2d10c31dcf183c8bed87b6ec35e7ed397d27' 1008 _build_and_install = ExternalPackage._build_and_install_from_package 1009 _build_and_install_current_dir = ( 1010 ExternalPackage._build_and_install_current_dir_setup_py) 1011 1012 1013 class PyLintPackage(ExternalPackage): 1014 """pylint package.""" 1015 version = '1.7.2' 1016 url_filename = 'pylint-%s.tar.gz' % version 1017 local_filename = url_filename 1018 urls = (_CHROMEOS_MIRROR + local_filename,) 1019 hex_sum = '42d8b9394e5a485377ae128b01350f25d8b131e0' 1020 _build_and_install = ExternalPackage._build_and_install_from_package 1021 _build_and_install_current_dir = ( 1022 ExternalPackage._build_and_install_current_dir_setup_py) 1023 1024 1025 class ConfigParserPackage(ExternalPackage): 1026 """configparser package (dependency for pylint).""" 1027 version = '3.5.0' 1028 url_filename = 'configparser-%s.tar.gz' % version 1029 local_filename = url_filename 1030 urls = (_CHROMEOS_MIRROR + local_filename,) 1031 hex_sum = '8ee6b29c6a11977c0e094da1d4f5f71e7e7ac78b' 1032 _build_and_install = ExternalPackage._build_and_install_from_package 1033 _build_and_install_current_dir = ( 1034 ExternalPackage._build_and_install_current_dir_setup_py) 1035 1036 1037 class IsortPackage(ExternalPackage): 1038 """isort package (dependency for pylint).""" 1039 version = '4.2.15' 1040 url_filename = 'isort-%s.tar.gz' % version 1041 local_filename = url_filename 1042 urls = (_CHROMEOS_MIRROR + local_filename,) 1043 hex_sum = 'acacc36e476b70e13e6fda812c193f4c3c187781' 1044 _build_and_install = ExternalPackage._build_and_install_from_package 1045 _build_and_install_current_dir = ( 1046 ExternalPackage._build_and_install_current_dir_setup_py) 1047 1048 1049 class Pytz(ExternalPackage): 1050 """Pytz package.""" 1051 version = '2016.10' 1052 url_filename = 'pytz-%s.tar.gz' % version 1053 local_filename = url_filename 1054 #md5=cc9f16ba436efabdcef3c4d32ae4919c 1055 urls = ('https://pypi.python.org/packages/42/00/' 1056 '5c89fc6c9b305df84def61863528e899e9dccb196f8438f6cbe960758fc5/%s' % 1057 (url_filename),) 1058 hex_sum = '8d63f1e9b1ee862841b990a7d8ad1d4508d9f0be' 1059 _build_and_install = ExternalPackage._build_and_install_from_package 1060 _build_and_install_current_dir = ( 1061 ExternalPackage._build_and_install_current_dir_setup_py) 1062 1063 1064 class Tzlocal(ExternalPackage): 1065 """Tzlocal package.""" 1066 version = '1.3' 1067 url_filename = 'tzlocal-%s.tar.gz' % version 1068 local_filename = url_filename 1069 urls = (_CHROMEOS_MIRROR + local_filename,) 1070 hex_sum = '730e9d7112335865a1dcfabec69c8c3086be424f' 1071 _build_and_install = ExternalPackage._build_and_install_from_package 1072 _build_and_install_current_dir = ( 1073 ExternalPackage._build_and_install_current_dir_setup_py) 1074 1075 1076 class PyYAMLPackage(ExternalPackage): 1077 """pyyaml package.""" 1078 version = '3.12' 1079 local_filename = 'PyYAML-%s.tar.gz' % version 1080 urls = (_CHROMEOS_MIRROR + local_filename,) 1081 hex_sum = 'cb7fd3e58c129494ee86e41baedfec69eb7dafbe' 1082 _build_and_install = ExternalPackage._build_and_install_from_package 1083 _build_and_install_current_dir = ( 1084 ExternalPackage._build_and_install_current_dir_noegg) 1085 1086 1087 class _ExternalGitRepo(ExternalPackage): 1088 """ 1089 Parent class for any package which needs to pull a git repo. 1090 1091 This class inherits from ExternalPackage only so we can sync git 1092 repos through the build_externals script. We do not reuse any of 1093 ExternalPackage's other methods. Any package that needs a git repo 1094 should subclass this and override build_and_install or fetch as 1095 they see appropriate. 1096 """ 1097 1098 os_requirements = {('/usr/bin/git') : 'git-core'} 1099 1100 # All the chromiumos projects used on the lab servers should have a 'prod' 1101 # branch used to track the software version deployed in prod. 1102 PROD_BRANCH = 'prod' 1103 MASTER_BRANCH = 'master' 1104 1105 def is_needed(self, unused_install_dir): 1106 """Tell build_externals that we need to fetch.""" 1107 # TODO(beeps): check if we're already upto date. 1108 return True 1109 1110 1111 def build_and_install(self, unused_install_dir): 1112 """ 1113 Fall through method to install a package. 1114 1115 Overwritten in base classes to pull a git repo. 1116 """ 1117 raise NotImplementedError 1118 1119 1120 def fetch(self, unused_dest_dir): 1121 """Fallthrough method to fetch a package.""" 1122 return True 1123 1124 1125 class HdctoolsRepo(_ExternalGitRepo): 1126 """Clones or updates the hdctools repo.""" 1127 1128 module_name = 'servo' 1129 temp_hdctools_dir = tempfile.mktemp(suffix='hdctools') 1130 _GIT_URL = ('https://chromium.googlesource.com/' 1131 'chromiumos/third_party/hdctools') 1132 1133 def fetch(self, unused_dest_dir): 1134 """ 1135 Fetch repo to a temporary location. 1136 1137 We use an intermediate temp directory to stage our 1138 installation because we only care about the servo package. 1139 If we can't get at the top commit hash after fetching 1140 something is wrong. This can happen when we've cloned/pulled 1141 an empty repo. Not something we expect to do. 1142 1143 @parma unused_dest_dir: passed in because we inherit from 1144 ExternalPackage. 1145 1146 @return: True if repo sync was successful. 1147 """ 1148 git_repo = revision_control.GitRepo( 1149 self.temp_hdctools_dir, 1150 self._GIT_URL, 1151 None, 1152 abs_work_tree=self.temp_hdctools_dir) 1153 git_repo.reinit_repo_at(self.PROD_BRANCH) 1154 1155 if git_repo.get_latest_commit_hash(): 1156 return True 1157 return False 1158 1159 1160 def build_and_install(self, install_dir): 1161 """Reach into the hdctools repo and rsync only the servo directory.""" 1162 1163 servo_dir = os.path.join(self.temp_hdctools_dir, 'servo') 1164 if not os.path.exists(servo_dir): 1165 return False 1166 1167 rv = self._rsync(servo_dir, os.path.join(install_dir, 'servo')) 1168 shutil.rmtree(self.temp_hdctools_dir) 1169 return rv 1170 1171 1172 class ChromiteRepo(_ExternalGitRepo): 1173 """Clones or updates the chromite repo.""" 1174 1175 _GIT_URL = ('https://chromium.googlesource.com/chromiumos/chromite') 1176 1177 def build_and_install(self, install_dir, master_branch=False): 1178 """ 1179 Clone if the repo isn't initialized, pull clean bits if it is. 1180 1181 Unlike it's hdctools counterpart the chromite repo clones master 1182 directly into site-packages. It doesn't use an intermediate temp 1183 directory because it doesn't need installation. 1184 1185 @param install_dir: destination directory for chromite installation. 1186 @param master_branch: if True, install master branch. Otherwise, 1187 install prod branch. 1188 """ 1189 init_branch = (self.MASTER_BRANCH if master_branch 1190 else self.PROD_BRANCH) 1191 local_chromite_dir = os.path.join(install_dir, 'chromite') 1192 git_repo = revision_control.GitRepo( 1193 local_chromite_dir, 1194 self._GIT_URL, 1195 abs_work_tree=local_chromite_dir) 1196 git_repo.reinit_repo_at(init_branch) 1197 1198 1199 if git_repo.get_latest_commit_hash(): 1200 return True 1201 return False 1202 1203 1204 class BtsocketRepo(_ExternalGitRepo): 1205 """Clones or updates the btsocket repo.""" 1206 1207 _GIT_URL = ('https://chromium.googlesource.com/' 1208 'chromiumos/platform/btsocket') 1209 1210 def fetch(self, unused_dest_dir): 1211 """ 1212 Fetch repo to a temporary location. 1213 1214 We use an intermediate temp directory because we have to build an 1215 egg for installation. If we can't get at the top commit hash after 1216 fetching something is wrong. This can happen when we've cloned/pulled 1217 an empty repo. Not something we expect to do. 1218 1219 @parma unused_dest_dir: passed in because we inherit from 1220 ExternalPackage. 1221 1222 @return: True if repo sync was successful. 1223 """ 1224 self.temp_btsocket_dir = autotemp.tempdir(unique_id='btsocket') 1225 try: 1226 git_repo = revision_control.GitRepo( 1227 self.temp_btsocket_dir.name, 1228 self._GIT_URL, 1229 None, 1230 abs_work_tree=self.temp_btsocket_dir.name) 1231 git_repo.reinit_repo_at(self.PROD_BRANCH) 1232 1233 if git_repo.get_latest_commit_hash(): 1234 return True 1235 except: 1236 self.temp_btsocket_dir.clean() 1237 raise 1238 1239 self.temp_btsocket_dir.clean() 1240 return False 1241 1242 1243 def build_and_install(self, install_dir): 1244 """ 1245 Install the btsocket module using setup.py 1246 1247 @param install_dir: Target installation directory. 1248 1249 @return: A boolean indicating success of failure. 1250 """ 1251 work_dir = os.getcwd() 1252 try: 1253 os.chdir(self.temp_btsocket_dir.name) 1254 rv = self._build_and_install_current_dir_setup_py(install_dir) 1255 finally: 1256 os.chdir(work_dir) 1257 self.temp_btsocket_dir.clean() 1258 return rv 1259
