1 # Copyright 2011 Google Inc. All Rights Reserved. 2 """Helper modules for Android toolchain test infrastructure. 3 4 Provides following Android toolchain test jobs and commands. 5 . Checkout Android toolchain source code 6 . Build Android toolchain 7 . Checkout and build Android tree 8 . Checkout/build/run Android benchmarks, generate size dashboard 9 . Transform size dashboard to report, send perflab jobid to 10 perflab dashboard server.(TODO) 11 """ 12 13 __author__ = 'jingyu (at] google.com (Jing Yu)' 14 15 import os.path 16 17 from automation.clients.helper import jobs 18 from automation.clients.helper import perforce 19 from automation.common import command as cmd 20 from automation.common import job 21 22 23 class JobsFactory(object): 24 25 def __init__(self, gcc_version='4.4.3', build_type='DEVELOPMENT'): 26 assert gcc_version in ['4.4.3', '4.6', 'google_main', 'fsf_trunk'] 27 assert build_type in ['DEVELOPMENT', 'RELEASE'] 28 29 self.gcc_version = gcc_version 30 self.commands = CommandsFactory(gcc_version, build_type) 31 self.tc_tag = 'gcc-%s-%s' % (gcc_version, build_type) 32 33 def CheckoutAndroidToolchain(self): 34 """Check out Android toolchain sources by release and gcc version.""" 35 command = self.commands.CheckoutAndroidToolchain() 36 new_job = jobs.CreateLinuxJob('AndroidCheckoutToolchain(%s)' % self.tc_tag, 37 command) 38 checkout_dir_dep = job.FolderDependency(new_job, self.commands.CHECKOUT_DIR) 39 return new_job, checkout_dir_dep 40 41 def BuildAndroidToolchain(self, checkout_dir_dep): 42 """Build Android Toolchain.""" 43 command = self.commands.BuildAndroidToolchain() 44 new_job = jobs.CreateLinuxJob('AndroidBuildToolchain(%s)' % self.tc_tag, 45 command) 46 new_job.DependsOnFolder(checkout_dir_dep) 47 tc_prefix_dep = job.FolderDependency(new_job, 48 self.commands.toolchain_prefix_dir) 49 return new_job, tc_prefix_dep 50 51 def BuildAndroidImage(self, 52 tc_prefix_dep, 53 product='stingray', 54 branch='ics-release'): 55 assert product in ['stingray', 'passion', 'trygon', 'soju'] 56 assert branch in ['honeycomb-release', 'ics-release'] 57 command = self.commands.BuildAndroidImage(product, branch) 58 new_job = jobs.CreateLinuxJob('AndroidGetBuildTree(%s)' % self.tc_tag, 59 command) 60 new_job.DependsOnFolder(tc_prefix_dep) 61 return new_job 62 63 def Benchmark(self, tc_prefix_dep, arch='soju'): 64 assert arch in ['soju', 'stingray'] 65 script_cmd = self.commands.CheckoutScripts() 66 experiment_tag = 'android/nightly/%s/%s/$JOB_ID' % (self.tc_tag, arch) 67 build_run_benchmark_cmd = self.commands.BuildRunBenchmark(arch, 68 experiment_tag) 69 command = cmd.Chain(script_cmd, build_run_benchmark_cmd) 70 new_job = jobs.CreateLinuxJob('AndroidBenchmarking(%s)' % self.tc_tag, 71 command) 72 new_job.DependsOnFolder(tc_prefix_dep) 73 return new_job 74 75 76 class CommandsFactory(object): 77 CHECKOUT_DIR = 'androidtc-checkout-dir' 78 TOOLCHAIN_SRC_DIR = os.path.join(CHECKOUT_DIR, 'src') 79 TOOLCHAIN_BUILD_DIR = 'obj' 80 ANDROID_TREES_DIR = 'android_trees' 81 TOOLS_DIR = 'android-tools' 82 BENCHMARK_OUT_DIR = 'results' 83 84 def __init__(self, gcc_version, build_type): 85 assert gcc_version in ['4.4.3', '4.6', 'google_main', 'fsf_trunk'] 86 assert build_type in ['DEVELOPMENT', 'RELEASE'] 87 88 self.build_type = build_type 89 self.gcc_version = gcc_version 90 self.binutils_version = '2.21' 91 self.gold_version = '2.21' 92 self.toolchain_prefix_dir = 'install-gcc-%s-%s' % (gcc_version, build_type) 93 self.p4client = self._CreatePerforceClient() 94 self.scripts = ScriptsFactory(self.gcc_version, self.binutils_version, 95 self.gold_version) 96 97 def _CreatePerforceClient(self): 98 p4_dev_path = 'gcctools/google_vendor_src_branch' 99 mobile_rel_branch = ('branches/' 100 'mobile_toolchain_v15_release_branch/gcctools/' 101 'google_vendor_src_branch') 102 gcc_443_rel_branch = ('branches/' 103 'android_compiler_v14_release_branch/gcctools/' 104 'google_vendor_src_branch') 105 106 # Common views for tools 107 p4view = perforce.View('depot2', perforce.PathMapping.ListFromPathTuples([( 108 'gcctools/android/build/...', 'src/build/...'), ( 109 'gcctools/android/Tarballs/...', 'src/tarballs/...')])) 110 for mapping in perforce.PathMapping.ListFromPathDict( 111 {'gcctools/android': ['tools/scripts/...', 'master/...']}): 112 p4view.add(mapping) 113 114 # Add views for gdb 115 p4view.add(perforce.PathMapping(p4_dev_path, 'src', 116 'gdb/gdb-7.1.x-android/...')) 117 118 # Add view for binutils for ld and gold 119 if self.build_type is 'RELEASE': 120 binutils_branch = mobile_rel_branch 121 else: 122 binutils_branch = p4_dev_path 123 p4view.add(perforce.PathMapping(binutils_branch, 'src', ( 124 'binutils/binutils-%s/...' % self.binutils_version))) 125 if self.binutils_version != self.gold_version: 126 p4view.add(perforce.PathMapping(binutils_branch, 'src', ( 127 'binutils/binutils-%s/...' % self.gold_version))) 128 129 # Add view for gcc if gcc_version is '4.4.3'. 130 if self.gcc_version == '4.4.3': 131 gcc443_path = 'gcc/gcc-4.4.3/...' 132 if self.build_type is 'RELEASE': 133 p4view.add(perforce.PathMapping(gcc_443_rel_branch, 'src', gcc443_path)) 134 else: 135 p4view.add(perforce.PathMapping(p4_dev_path, 'src', gcc443_path)) 136 137 return perforce.CommandsFactory(self.CHECKOUT_DIR, p4view) 138 139 def _CheckoutGCCFromSVN(self): 140 """Check out gcc from fsf svn. 141 142 Return the command that check out gcc from svn 143 to gcc_required_dir (=TOOLCHAIN_SRC_DIR/src/gcc/gcc-xxx). 144 145 TODO: 146 Create a svn class that does these jobs. 147 Parallelize p4 checkout and svn checkout. 148 """ 149 if self.gcc_version == '4.4.3': 150 return '' 151 assert self.gcc_version in ['4.6', 'google_main', 'fsf_trunk'] 152 153 gcc_branches_dir = {'4.6': 'branches/google/gcc-4_6', 154 'google_main': 'branches/google/main', 155 'fsf_trunk': 'trunk'} 156 157 # Find GCC revision number, output it to TOOLCHAIN_SRC_DIR/CLNUM_GCC 158 svn_get_revision = cmd.Pipe( 159 cmd.Shell('svn', 'info'), 160 cmd.Shell('grep', '"Revision:"'), 161 cmd.Shell('sed', '-E', '"s,Revision: ([0-9]+).*,\\1,"'), 162 output='../../../CLNUM_GCC') 163 164 svn_co_command = 'svn co svn://gcc.gnu.org/svn/gcc/%s .' % ( 165 gcc_branches_dir[self.gcc_version]) 166 167 gcc_required_dir = os.path.join(self.TOOLCHAIN_SRC_DIR, 'gcc', 168 'gcc-%s' % self.gcc_version) 169 170 return cmd.Chain( 171 cmd.MakeDir(gcc_required_dir), 172 cmd.Wrapper( 173 cmd.Chain(svn_co_command, svn_get_revision), 174 cwd=gcc_required_dir)) 175 176 def CheckoutAndroidToolchain(self): 177 p4client = self.p4client 178 command = p4client.SetupAndDo(p4client.Sync(), 179 p4client.SaveCurrentCLNumber('CLNUM'), 180 p4client.Remove()) 181 if self.gcc_version != '4.4.3': 182 command.append(self._CheckoutGCCFromSVN()) 183 184 return command 185 186 def BuildAndroidToolchain(self): 187 script_cmd = self.scripts.BuildAndroidToolchain( 188 self.toolchain_prefix_dir, self.CHECKOUT_DIR, self.TOOLCHAIN_BUILD_DIR, 189 self.TOOLCHAIN_SRC_DIR) 190 191 # Record toolchain and gcc CL number 192 record_cl_cmd = cmd.Copy( 193 os.path.join(self.CHECKOUT_DIR, 'CLNUM*'), 194 to_dir=self.toolchain_prefix_dir) 195 save_cmd = cmd.Tar( 196 os.path.join('$JOB_TMP', 'results', '%s.tar.bz2' % 197 self.toolchain_prefix_dir), self.toolchain_prefix_dir) 198 return cmd.Chain(script_cmd, record_cl_cmd, save_cmd) 199 200 def _BuildAndroidTree(self, local_android_branch_dir, product): 201 target_tools_prefix = os.path.join('$JOB_TMP', self.toolchain_prefix_dir, 202 'bin', 'arm-linux-androideabi-') 203 java_path = '/usr/lib/jvm/java-6-sun/bin' 204 build_cmd = cmd.Shell('make', '-j8', 'PRODUCT-%s-userdebug' % product, 205 'TARGET_TOOLS_PREFIX=%s' % target_tools_prefix, 206 'PATH=%s:$PATH' % java_path) 207 return cmd.Wrapper(build_cmd, cwd=local_android_branch_dir) 208 209 def BuildAndroidImage(self, product, branch): 210 assert product in ['stingray', 'passion', 'trygon', 'soju'] 211 212 # Copy the tree from atree.mtv.corp to ANDROID_TREES_DIR/branch 213 androidtrees_host = 'atree.mtv.corp.google.com' 214 androidtrees_path = ('/usr/local/google2/home/mobiletc-prebuild/' 215 'android_trees') 216 remote_android_branch_path = os.path.join(androidtrees_path, branch) 217 local_android_branch_dir = os.path.join(self.ANDROID_TREES_DIR, branch) 218 gettree_cmd = cmd.RemoteCopyFrom( 219 androidtrees_host, remote_android_branch_path, local_android_branch_dir) 220 221 # Configure and build the tree 222 buildtree_cmd = self._BuildAndroidTree(local_android_branch_dir, product) 223 224 # Compress and copy system.img to result 225 result_system_img = os.path.join(local_android_branch_dir, 'out', 'target', 226 'product', product, 'system.img') 227 copy_img = cmd.Copy(result_system_img, to_dir='results') 228 compress_img = cmd.Shell('bzip2', os.path.join('results', 'system.img')) 229 230 return cmd.Chain(gettree_cmd, buildtree_cmd, copy_img, compress_img) 231 232 def CheckoutScripts(self): 233 p4view = perforce.View('depot2', 234 [perforce.PathMapping('gcctools/android/tools/...', 235 'tools/...')]) 236 p4client = perforce.CommandsFactory(self.TOOLS_DIR, p4view) 237 return p4client.SetupAndDo(p4client.Sync(), p4client.Remove()) 238 239 def BuildRunBenchmark(self, arch, run_experiment): 240 # Copy base benchmark binaries from atree.mtv.corp 241 base_benchbin_host = 'atree.mtv.corp.google.com' 242 base_benchbin_path = ('/usr/local/google2/home/mobiletc-prebuild/' 243 'archive/v3binaries/2011-10-18') 244 local_basebenchbin_dir = 'base_benchmark_bin' 245 getbase_cmd = cmd.RemoteCopyFrom(base_benchbin_host, base_benchbin_path, 246 local_basebenchbin_dir) 247 248 # Build and run benchmark. 249 android_arch = 'android_%s' % arch 250 run_label = 'normal' 251 benchmark_cmd = self.scripts.RunBenchmark( 252 self.toolchain_prefix_dir, self.TOOLS_DIR, self.BENCHMARK_OUT_DIR, 253 run_label, run_experiment, android_arch, local_basebenchbin_dir) 254 255 # Extract jobid from BENCHMARK_OUT_DIR/log/jobid_normal.log file. 256 # Copy jobid to www server to generate performance dashboard. 257 # TODO(jingyu) 258 259 return cmd.Chain(getbase_cmd, benchmark_cmd) 260 261 262 class ScriptsFactory(object): 263 264 def __init__(self, gcc_version, binutils_version, gold_version): 265 self._gcc_version = gcc_version 266 self._binutils_version = binutils_version 267 self._gold_version = gold_version 268 269 def BuildAndroidToolchain(self, toolchain_prefix_dir, checkout_dir, 270 toolchain_build_dir, androidtc_src_dir): 271 if self._gcc_version == '4.4.3': 272 gold_option = 'both/gold' 273 else: 274 gold_option = 'default' 275 276 return cmd.Shell( 277 'build_androidtoolchain.sh', 278 '--toolchain-src=%s' % os.path.join('$JOB_TMP', androidtc_src_dir), 279 '--build-path=%s' % os.path.join('$JOB_TMP', toolchain_build_dir), 280 '--install-prefix=%s' % os.path.join('$JOB_TMP', toolchain_prefix_dir), 281 '--target=arm-linux-androideabi', 282 '--enable-gold=%s' % gold_option, 283 '--with-gcc-version=%s' % self._gcc_version, 284 '--with-binutils-version=%s' % self._binutils_version, 285 '--with-gold-version=%s' % self._gold_version, 286 '--with-gdb-version=7.1.x-android', 287 '--log-path=%s/logs' % '$JOB_HOME', 288 '--android-sysroot=%s' % os.path.join('$JOB_TMP', checkout_dir, 289 'gcctools', 'android', 'master', 290 'honeycomb_generic_sysroot'), 291 path=os.path.join(checkout_dir, 'gcctools', 'android', 'tools', 292 'scripts')) 293 294 def RunBenchmark(self, 295 toolchain_prefix_dir, 296 checkout_dir, 297 output_dir, 298 run_label, 299 run_experiment, 300 arch, 301 base_bench_bin=None): 302 if base_bench_bin: 303 base_bench_opt = '--base_benchmark_bin=%s' % base_bench_bin 304 else: 305 base_bench_opt = '' 306 307 return cmd.Shell( 308 'benchmark.sh', 309 '--android_toolchain=%s' % os.path.join('$JOB_TMP', 310 toolchain_prefix_dir), 311 '--bench_space=%s' % os.path.join('$JOB_TMP', 'bench'), 312 '--benchmark_bin=%s' % os.path.join('$JOB_TMP', output_dir, 313 'bench_bin'), 314 base_bench_opt, 315 '--log_path=%s' % os.path.join('$JOB_TMP', output_dir, 'log'), 316 '--arch=%s' % arch, 317 '--run_label=%s' % run_label, 318 '--run_experiment=%s' % run_experiment, 319 path=os.path.join(checkout_dir, 'tools', 'scripts')) 320
