1 # Copyright 2011 Google Inc. All Rights Reserved. 2 3 __author__ = 'kbaclawski (at] google.com (Krystian Baclawski)' 4 5 import os.path 6 import time 7 8 from automation.clients.helper import jobs 9 from automation.clients.helper import perforce 10 from automation.common import command as cmd 11 from automation.common import job 12 13 14 class JobsFactory(object): 15 16 def __init__(self): 17 self.commands = CommandsFactory() 18 19 def CheckoutCrosstool(self, target): 20 command = self.commands.CheckoutCrosstool() 21 new_job = jobs.CreateLinuxJob('CheckoutCrosstool(%s)' % target, command) 22 checkout_dir_dep = job.FolderDependency(new_job, 23 CommandsFactory.CHECKOUT_DIR) 24 manifests_dir_dep = job.FolderDependency( 25 new_job, os.path.join(self.commands.buildit_path, target), 'manifests') 26 return new_job, checkout_dir_dep, manifests_dir_dep 27 28 def BuildRelease(self, checkout_dir, target): 29 command = self.commands.BuildRelease(target) 30 new_job = jobs.CreateLinuxJob('BuildRelease(%s)' % target, command) 31 new_job.DependsOnFolder(checkout_dir) 32 build_tree_dep = job.FolderDependency(new_job, 33 self.commands.buildit_work_dir_path) 34 return new_job, build_tree_dep 35 36 def RunTests(self, checkout_dir, build_tree_dir, target, board, component): 37 command = self.commands.RunTests(target, board, component) 38 new_job = jobs.CreateLinuxJob('RunTests(%s, %s, %s)' % 39 (target, component, board), command) 40 new_job.DependsOnFolder(checkout_dir) 41 new_job.DependsOnFolder(build_tree_dir) 42 testrun_dir_dep = job.FolderDependency( 43 new_job, self.commands.dejagnu_output_path, board) 44 return new_job, testrun_dir_dep 45 46 def GenerateReport(self, testrun_dirs, manifests_dir, target, boards): 47 command = self.commands.GenerateReport(boards) 48 new_job = jobs.CreateLinuxJob('GenerateReport(%s)' % target, command) 49 new_job.DependsOnFolder(manifests_dir) 50 for testrun_dir in testrun_dirs: 51 new_job.DependsOnFolder(testrun_dir) 52 return new_job 53 54 55 class CommandsFactory(object): 56 CHECKOUT_DIR = 'crosstool-checkout-dir' 57 58 def __init__(self): 59 self.buildit_path = os.path.join(self.CHECKOUT_DIR, 'gcctools', 'crosstool', 60 'v15') 61 62 self.buildit_work_dir = 'buildit-tmp' 63 self.buildit_work_dir_path = os.path.join('$JOB_TMP', self.buildit_work_dir) 64 self.dejagnu_output_path = os.path.join(self.buildit_work_dir_path, 65 'dejagnu-output') 66 67 paths = { 68 'gcctools': [ 69 'crosstool/v15/...', 'scripts/...' 70 ], 71 'gcctools/google_vendor_src_branch': [ 72 'binutils/binutils-2.21/...', 'gdb/gdb-7.2.x/...', 73 'zlib/zlib-1.2.3/...' 74 ], 75 'gcctools/vendor_src': [ 76 'gcc/google/gcc-4_6/...' 77 ] 78 } 79 80 p4view = perforce.View('depot2', 81 perforce.PathMapping.ListFromPathDict(paths)) 82 83 self.p4client = perforce.CommandsFactory(self.CHECKOUT_DIR, p4view) 84 85 def CheckoutCrosstool(self): 86 p4client = self.p4client 87 88 return p4client.SetupAndDo(p4client.Sync(), 89 p4client.SaveCurrentCLNumber('CLNUM'), 90 p4client.Remove()) 91 92 def BuildRelease(self, target): 93 clnum_path = os.path.join('$JOB_TMP', self.CHECKOUT_DIR, 'CLNUM') 94 95 toolchain_root = os.path.join('/google/data/rw/projects/toolchains', target, 96 'unstable') 97 toolchain_path = os.path.join(toolchain_root, '${CLNUM}') 98 99 build_toolchain = cmd.Wrapper( 100 cmd.Chain( 101 cmd.MakeDir(toolchain_path), 102 cmd.Shell('buildit', 103 '--keep-work-dir', 104 '--build-type=release', 105 '--work-dir=%s' % self.buildit_work_dir_path, 106 '--results-dir=%s' % toolchain_path, 107 '--force-release=%s' % '${CLNUM}', 108 target, 109 path='.')), 110 cwd=self.buildit_path, 111 umask='0022', 112 env={'CLNUM': '$(< %s)' % clnum_path}) 113 114 # remove all but 10 most recent directories 115 remove_old_toolchains_from_x20 = cmd.Wrapper( 116 cmd.Pipe( 117 cmd.Shell('ls', '-1', '-r'), cmd.Shell('sed', '-e', '1,10d'), 118 cmd.Shell('xargs', 'rm', '-r', '-f')), 119 cwd=toolchain_root) 120 121 return cmd.Chain(build_toolchain, remove_old_toolchains_from_x20) 122 123 def RunTests(self, target, board, component='gcc'): 124 dejagnu_flags = ['--outdir=%s' % self.dejagnu_output_path, 125 '--target_board=%s' % board] 126 127 # Look for {pandaboard,qemu}.exp files in 128 # //depot/google3/experimental/users/kbaclawski/dejagnu/boards 129 130 site_exp_file = os.path.join('/google/src/head/depot/google3', 131 'experimental/users/kbaclawski', 132 'dejagnu/site.exp') 133 134 build_dir_path = os.path.join(target, 'rpmbuild/BUILD/crosstool*-0.0', 135 'build-%s' % component) 136 137 run_dejagnu = cmd.Wrapper( 138 cmd.Chain( 139 cmd.MakeDir(self.dejagnu_output_path), 140 cmd.Shell('make', 141 'check', 142 '-k', 143 '-j $(grep -c processor /proc/cpuinfo)', 144 'RUNTESTFLAGS="%s"' % ' '.join(dejagnu_flags), 145 'DEJAGNU="%s"' % site_exp_file, 146 ignore_error=True)), 147 cwd=os.path.join(self.buildit_work_dir_path, build_dir_path), 148 env={'REMOTE_TMPDIR': 'job-$JOB_ID'}) 149 150 save_results = cmd.Copy(self.dejagnu_output_path, 151 to_dir='$JOB_TMP/results', 152 recursive=True) 153 154 return cmd.Chain(run_dejagnu, save_results) 155 156 def GenerateReport(self, boards): 157 sumfiles = [os.path.join('$JOB_TMP', board, '*.sum') for board in boards] 158 159 return cmd.Wrapper( 160 cmd.Shell('dejagnu.sh', 161 'report', 162 '-m', 163 '$JOB_TMP/manifests/*.xfail', 164 '-o', 165 '$JOB_TMP/results/report.html', 166 *sumfiles, 167 path='.'), 168 cwd='$HOME/automation/clients/report') 169
