Home | History | Annotate | Download | only in toolchain-utils 
      1 #!/usr/bin/env python2
      2 """Script for running llvm validation tests on ChromeOS.
      3 
      4 This script launches a buildbot to build ChromeOS with the llvm on
      5 a particular board; then it finds and downloads the trybot image and the
      6 corresponding official image, and runs test for correctness.
      7 It then generates a report, emails it to the c-compiler-chrome, as
      8 well as copying the result into a directory.
      9 """
     10 
     11 # Script to test different toolchains against ChromeOS benchmarks.
     12 
     13 from __future__ import print_function
     14 
     15 import argparse
     16 import datetime
     17 import os
     18 import sys
     19 import time
     20 
     21 from cros_utils import command_executer
     22 from cros_utils import logger
     23 
     24 from cros_utils import buildbot_utils
     25 
     26 # CL that uses LLVM to build the peppy image.
     27 USE_LLVM_PATCH = '295217'
     28 
     29 CROSTC_ROOT = '/usr/local/google/crostc'
     30 ROLE_ACCOUNT = 'mobiletc-prebuild'
     31 TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__))
     32 MAIL_PROGRAM = '~/var/bin/mail-sheriff'
     33 VALIDATION_RESULT_DIR = os.path.join(CROSTC_ROOT, 'validation_result')
     34 START_DATE = datetime.date(2016, 1, 1)
     35 TEST_PER_DAY = 2
     36 TEST_BOARD = [
     37     'squawks',      # x86_64, rambi  (baytrail)
     38     'terra',        # x86_64, strago (braswell)
     39     'lulu',         # x86_64, auron  (broadwell)
     40     'peach_pit',    # arm,    peach  (exynos-5420)
     41     'peppy',        # x86_64, slippy (haswell celeron)
     42     'link',         # x86_64, ivybridge (ivybridge)
     43     'nyan_big',     # arm,    nyan   (tegra)
     44     'sentry',       # x86_64, kunimitsu (skylake-u)
     45     'chell',        # x86_64, glados (skylake-y)
     46     'daisy',        # arm,    daisy  (exynos)
     47     'caroline',     # amd64
     48     'kevin',        # arm,    gru  (Rockchip)
     49 ]
     50 
     51 
     52 class ToolchainVerifier(object):
     53   """Class for the toolchain verifier."""
     54 
     55   def __init__(self, board, chromeos_root, weekday, patches, compiler):
     56     self._board = board
     57     self._chromeos_root = chromeos_root
     58     self._base_dir = os.getcwd()
     59     self._ce = command_executer.GetCommandExecuter()
     60     self._l = logger.GetLogger()
     61     self._compiler = compiler
     62     self._build = '%s-%s-toolchain' % (board, compiler)
     63     self._patches = patches.split(',') if patches else []
     64     self._patches_string = '_'.join(str(p) for p in self._patches)
     65 
     66     if not weekday:
     67       self._weekday = time.strftime('%a')
     68     else:
     69       self._weekday = weekday
     70     self._reports = os.path.join(VALIDATION_RESULT_DIR, compiler, board)
     71 
     72   def _FinishSetup(self):
     73     """Make sure testing_rsa file is properly set up."""
     74     # Fix protections on ssh key
     75     command = ('chmod 600 /var/cache/chromeos-cache/distfiles/target'
     76                '/chrome-src-internal/src/third_party/chromite/ssh_keys'
     77                '/testing_rsa')
     78     ret_val = self._ce.ChrootRunCommand(self._chromeos_root, command)
     79     if ret_val != 0:
     80       raise RuntimeError('chmod for testing_rsa failed')
     81 
     82   def DoAll(self):
     83     """Main function inside ToolchainComparator class.
     84 
     85     Launch trybot, get image names, create crosperf experiment file, run
     86     crosperf, and copy images into seven-day report directories.
     87     """
     88     flags = ['--hwtest']
     89     date_str = datetime.date.today()
     90     description = 'master_%s_%s_%s' % (self._patches_string, self._build,
     91                                        date_str)
     92     _ = buildbot_utils.GetTrybotImage(
     93         self._chromeos_root,
     94         self._build,
     95         self._patches,
     96         description,
     97         other_flags=flags,
     98         async=True)
     99 
    100     return 0
    101 
    102 def Main(argv):
    103   """The main function."""
    104 
    105   # Common initializations
    106   command_executer.InitCommandExecuter()
    107   parser = argparse.ArgumentParser()
    108   parser.add_argument(
    109       '--chromeos_root',
    110       dest='chromeos_root',
    111       help='The chromeos root from which to run tests.')
    112   parser.add_argument(
    113       '--weekday',
    114       default='',
    115       dest='weekday',
    116       help='The day of the week for which to run tests.')
    117   parser.add_argument(
    118       '--board', default='', dest='board', help='The board to test.')
    119   parser.add_argument(
    120       '--patch',
    121       dest='patches',
    122       default='',
    123       help='The patches to use for the testing, '
    124       "seprate the patch numbers with ',' "
    125       'for more than one patches.')
    126   parser.add_argument(
    127       '--compiler',
    128       dest='compiler',
    129       help='Which compiler (llvm, llvm-next or gcc) to use for '
    130       'testing.')
    131 
    132   options = parser.parse_args(argv[1:])
    133   if not options.chromeos_root:
    134     print('Please specify the ChromeOS root directory.')
    135     return 1
    136   if not options.compiler:
    137     print('Please specify which compiler to test (gcc, llvm, or llvm-next).')
    138     return 1
    139   patches = options.patches
    140   if not patches and options.compiler == 'llvm':
    141     patches = USE_LLVM_PATCH
    142 
    143   if options.board:
    144     fv = ToolchainVerifier(options.board, options.chromeos_root,
    145                            options.weekday, patches, options.compiler)
    146     return fv.Doall()
    147 
    148   today = datetime.date.today()
    149   delta = today - START_DATE
    150   days = delta.days
    151 
    152   start_board = (days * TEST_PER_DAY) % len(TEST_BOARD)
    153   for i in range(TEST_PER_DAY):
    154     try:
    155       board = TEST_BOARD[(start_board + i) % len(TEST_BOARD)]
    156       fv = ToolchainVerifier(board, options.chromeos_root, options.weekday,
    157                              patches, options.compiler)
    158       fv.DoAll()
    159     except SystemExit:
    160       logfile = os.path.join(VALIDATION_RESULT_DIR, options.compiler, board)
    161       with open(logfile, 'w') as f:
    162         f.write('Verifier got an exception, please check the log.\n')
    163 
    164 
    165 if __name__ == '__main__':
    166   retval = Main(sys.argv)
    167   sys.exit(retval)
    168