Home | History | Annotate | Download | only in toolchain-utils 
      1 #!/usr/bin/env python2
      2 """Script for running llvm validation tests on ChromeOS.
      3 
      4 This script launches a buildbot to build ChromeOS with the llvm on
      5 a particular board; then it finds and downloads the trybot image and the
      6 corresponding official image, and runs test for correctness.
      7 It then generates a report, emails it to the c-compiler-chrome, as
      8 well as copying the result into a directory.
      9 """
     10 
     11 # Script to test different toolchains against ChromeOS benchmarks.
     12 
     13 from __future__ import print_function
     14 
     15 import argparse
     16 import datetime
     17 import os
     18 import sys
     19 import time
     20 
     21 from cros_utils import command_executer
     22 from cros_utils import logger
     23 
     24 from cros_utils import buildbot_utils
     25 
     26 CROSTC_ROOT = '/usr/local/google/crostc'
     27 ROLE_ACCOUNT = 'mobiletc-prebuild'
     28 TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__))
     29 MAIL_PROGRAM = '~/var/bin/mail-sheriff'
     30 VALIDATION_RESULT_DIR = os.path.join(CROSTC_ROOT, 'validation_result')
     31 START_DATE = datetime.date(2016, 1, 1)
     32 TEST_PER_DAY = 3
     33 TEST_BOARD = [
     34     'squawks',  # x86_64, rambi  (baytrail)
     35     'terra',  # x86_64, strago (braswell)
     36     'lulu',  # x86_64, auron  (broadwell)
     37     'peach_pit',  # arm,    peach  (exynos-5420)
     38     'peppy',  # x86_64, slippy (haswell celeron)
     39     'link',  # x86_64, ivybridge (ivybridge)
     40     'nyan_big',  # arm,    nyan   (tegra)
     41     'sentry',  # x86_64, kunimitsu (skylake-u)
     42     'chell',  # x86_64, glados (skylake-y)
     43     'daisy',  # arm,    daisy  (exynos)
     44     'caroline',  # x86_64, glados (skylake-y)
     45     'kevin',  # arm,    gru  (Rockchip)
     46     'reef',  # x86_64, reef  (Apollo Lake)
     47     'lakitu',
     48     'whirlwind',
     49 ]
     50 
     51 
     52 class ToolchainVerifier(object):
     53   """Class for the toolchain verifier."""
     54 
     55   def __init__(self, board, chromeos_root, weekday, patches, compiler):
     56     self._board = board
     57     self._chromeos_root = chromeos_root
     58     self._base_dir = os.getcwd()
     59     self._ce = command_executer.GetCommandExecuter()
     60     self._l = logger.GetLogger()
     61     self._compiler = compiler
     62     self._build = '%s-%s-toolchain' % (board, compiler)
     63     self._patches = patches.split(',') if patches else []
     64     self._patches_string = '_'.join(str(p) for p in self._patches)
     65 
     66     if not weekday:
     67       self._weekday = time.strftime('%a')
     68     else:
     69       self._weekday = weekday
     70     self._reports = os.path.join(VALIDATION_RESULT_DIR, compiler, board)
     71 
     72   def _FinishSetup(self):
     73     """Make sure testing_rsa file is properly set up."""
     74     # Fix protections on ssh key
     75     command = ('chmod 600 /var/cache/chromeos-cache/distfiles/target'
     76                '/chrome-src-internal/src/third_party/chromite/ssh_keys'
     77                '/testing_rsa')
     78     ret_val = self._ce.ChrootRunCommand(self._chromeos_root, command)
     79     if ret_val != 0:
     80       raise RuntimeError('chmod for testing_rsa failed')
     81 
     82   def DoAll(self):
     83     """Main function inside ToolchainComparator class.
     84 
     85     Launch trybot, get image names, create crosperf experiment file, run
     86     crosperf, and copy images into seven-day report directories.
     87     """
     88     flags = ['--hwtest']
     89     date_str = datetime.date.today()
     90     description = 'master_%s_%s_%s' % (self._patches_string, self._build,
     91                                        date_str)
     92     _ = buildbot_utils.GetTrybotImage(
     93         self._chromeos_root,
     94         self._build,
     95         self._patches,
     96         description,
     97         other_flags=flags,
     98         async=True)
     99 
    100     return 0
    101 
    102 
    103 def Main(argv):
    104   """The main function."""
    105 
    106   # Common initializations
    107   command_executer.InitCommandExecuter()
    108   parser = argparse.ArgumentParser()
    109   parser.add_argument(
    110       '--chromeos_root',
    111       dest='chromeos_root',
    112       help='The chromeos root from which to run tests.')
    113   parser.add_argument(
    114       '--weekday',
    115       default='',
    116       dest='weekday',
    117       help='The day of the week for which to run tests.')
    118   parser.add_argument(
    119       '--board', default='', dest='board', help='The board to test.')
    120   parser.add_argument(
    121       '--patch',
    122       dest='patches',
    123       default='',
    124       help='The patches to use for the testing, '
    125       "seprate the patch numbers with ',' "
    126       'for more than one patches.')
    127   parser.add_argument(
    128       '--compiler',
    129       dest='compiler',
    130       help='Which compiler (llvm, llvm-next or gcc) to use for '
    131       'testing.')
    132 
    133   options = parser.parse_args(argv[1:])
    134   if not options.chromeos_root:
    135     print('Please specify the ChromeOS root directory.')
    136     return 1
    137   if not options.compiler:
    138     print('Please specify which compiler to test (gcc, llvm, or llvm-next).')
    139     return 1
    140 
    141   if options.board:
    142     fv = ToolchainVerifier(options.board, options.chromeos_root,
    143                            options.weekday, options.patches, options.compiler)
    144     return fv.Doall()
    145 
    146   today = datetime.date.today()
    147   delta = today - START_DATE
    148   days = delta.days
    149 
    150   start_board = (days * TEST_PER_DAY) % len(TEST_BOARD)
    151   for i in range(TEST_PER_DAY):
    152     try:
    153       board = TEST_BOARD[(start_board + i) % len(TEST_BOARD)]
    154       fv = ToolchainVerifier(board, options.chromeos_root, options.weekday,
    155                              options.patches, options.compiler)
    156       fv.DoAll()
    157     except SystemExit:
    158       logfile = os.path.join(VALIDATION_RESULT_DIR, options.compiler, board)
    159       with open(logfile, 'w') as f:
    160         f.write('Verifier got an exception, please check the log.\n')
    161 
    162 
    163 if __name__ == '__main__':
    164   retval = Main(sys.argv)
    165   sys.exit(retval)
    166