1 # Copyright 2016 The Chromium Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 import default_flavor 6 7 """GN flavor utils, used for building Skia with GN.""" 8 class GNFlavorUtils(default_flavor.DefaultFlavorUtils): 9 def _run(self, title, cmd, infra_step=False, **kwargs): 10 return self.m.run(self.m.step, title, cmd=cmd, 11 infra_step=infra_step, **kwargs) 12 13 def _py(self, title, script, infra_step=True, args=()): 14 return self.m.run(self.m.python, title, script=script, args=args, 15 infra_step=infra_step) 16 17 def build_command_buffer(self): 18 self.m.run(self.m.python, 'build command_buffer', 19 script=self.m.vars.skia_dir.join('tools', 'build_command_buffer.py'), 20 args=[ 21 '--chrome-dir', self.m.vars.checkout_root, 22 '--output-dir', self.m.vars.skia_out.join(self.m.vars.configuration), 23 '--no-sync', '--make-output-dir']) 24 25 def _get_goma_json(self): 26 json_key = 'jwt_service_account_goma-client' 27 json_filename = json_key + '.json' 28 29 # Ensure that the tmp_dir exists. 30 self.m.run.run_once(self.m.file.ensure_directory, 31 'makedirs tmp_dir', 32 self.m.vars.tmp_dir) 33 34 json_file = self.m.vars.tmp_dir.join(json_filename) 35 self.m.python.inline( 36 'download ' + json_filename, 37 """ 38 import os 39 import sys 40 import urllib2 41 42 TOKEN_URL = ( 43 'http://metadata/computeMetadata/v1/project/attributes/%s') 44 45 req = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'}) 46 contents = urllib2.urlopen(req).read() 47 48 with open(sys.argv[1], 'w') as f: 49 f.write(contents) 50 """ % json_key, 51 args=[json_file], 52 infra_step=True) 53 return json_file 54 55 def compile(self, unused_target): 56 """Build Skia with GN.""" 57 compiler = self.m.vars.builder_cfg.get('compiler', '') 58 configuration = self.m.vars.builder_cfg.get('configuration', '') 59 extra_tokens = self.m.vars.extra_tokens 60 os = self.m.vars.builder_cfg.get('os', '') 61 target_arch = self.m.vars.builder_cfg.get('target_arch', '') 62 63 goma_dir = None 64 clang_linux = str(self.m.vars.slave_dir.join('clang_linux')) 65 emscripten_sdk = str(self.m.vars.slave_dir.join('emscripten_sdk')) 66 linux_vulkan_sdk = str(self.m.vars.slave_dir.join('linux_vulkan_sdk')) 67 win_toolchain = str(self.m.vars.slave_dir.join( 68 't', 'depot_tools', 'win_toolchain', 'vs_files', 69 'a9e1098bba66d2acccc377d5ee81265910f29272')) 70 win_vulkan_sdk = str(self.m.vars.slave_dir.join('win_vulkan_sdk')) 71 72 cc, cxx = None, None 73 extra_cflags = [] 74 extra_ldflags = [] 75 76 if compiler == 'Clang' and self.m.vars.is_linux: 77 cc = clang_linux + '/bin/clang' 78 cxx = clang_linux + '/bin/clang++' 79 extra_cflags .append('-B%s/bin' % clang_linux) 80 extra_ldflags.append('-B%s/bin' % clang_linux) 81 extra_ldflags.append('-fuse-ld=lld') 82 elif compiler == 'Clang': 83 cc, cxx = 'clang', 'clang++' 84 elif compiler == 'GCC' and os == "Ubuntu14": 85 cc, cxx = 'gcc-4.8', 'g++-4.8' 86 elif compiler == 'GCC': 87 cc, cxx = 'gcc', 'g++' 88 elif compiler == 'EMCC': 89 cc = emscripten_sdk + '/emscripten/incoming/emcc' 90 cxx = emscripten_sdk + '/emscripten/incoming/em++' 91 extra_cflags.append('-Wno-unknown-warning-option') 92 93 if 'Coverage' in extra_tokens: 94 # See https://clang.llvm.org/docs/SourceBasedCodeCoverage.html for 95 # more info on using llvm to gather coverage information. 96 extra_cflags.append('-fprofile-instr-generate') 97 extra_cflags.append('-fcoverage-mapping') 98 extra_ldflags.append('-fprofile-instr-generate') 99 extra_ldflags.append('-fcoverage-mapping') 100 101 if compiler != 'MSVC' and configuration == 'Debug': 102 extra_cflags.append('-O1') 103 104 if 'Exceptions' in extra_tokens: 105 extra_cflags.append('/EHsc') 106 if 'Fast' in extra_tokens: 107 extra_cflags.extend(['-march=native', '-fomit-frame-pointer', '-O3', 108 '-ffp-contract=off']) 109 110 # TODO(benjaminwagner): Same appears in compile.py to set CPPFLAGS. Are 111 # both needed? 112 if len(extra_tokens) == 1 and extra_tokens[0].startswith('SK'): 113 extra_cflags.append('-D' + extra_tokens[0]) 114 115 if 'MSAN' in extra_tokens: 116 extra_ldflags.append('-L' + clang_linux + '/msan') 117 118 args = {} 119 ninja_args = ['-k', '0', '-C', self.out_dir] 120 env = {} 121 122 if configuration != 'Debug': 123 args['is_debug'] = 'false' 124 if 'ANGLE' in extra_tokens: 125 args['skia_use_angle'] = 'true' 126 if 'CommandBuffer' in extra_tokens: 127 self.m.run.run_once(self.build_command_buffer) 128 if 'MSAN' in extra_tokens: 129 args['skia_enable_gpu'] = 'false' 130 args['skia_use_fontconfig'] = 'false' 131 if 'ASAN' in extra_tokens or 'UBSAN' in extra_tokens: 132 args['skia_enable_spirv_validation'] = 'false' 133 if 'Mini' in extra_tokens: 134 args.update({ 135 'is_component_build': 'true', # Proves we can link a coherent .so. 136 'is_official_build': 'true', # No debug symbols, no tools. 137 'skia_enable_effects': 'false', 138 'skia_enable_gpu': 'true', 139 'skia_enable_pdf': 'false', 140 'skia_use_expat': 'false', 141 'skia_use_libjpeg_turbo': 'false', 142 'skia_use_libpng': 'false', 143 'skia_use_libwebp': 'false', 144 'skia_use_zlib': 'false', 145 }) 146 if 'NoGPU' in extra_tokens: 147 args['skia_enable_gpu'] = 'false' 148 if 'EmbededResouces' in extra_tokens: 149 args['skia_embed_resoucres'] = 'true' 150 if 'Shared' in extra_tokens: 151 args['is_component_build'] = 'true' 152 if 'Vulkan' in extra_tokens and not 'Android' in extra_tokens: 153 args['skia_enable_vulkan_debug_layers'] = 'false' 154 if self.m.vars.is_linux: 155 args['skia_vulkan_sdk'] = '"%s"' % linux_vulkan_sdk 156 if 'Win' in os: 157 args['skia_vulkan_sdk'] = '"%s"' % win_vulkan_sdk 158 if 'Metal' in extra_tokens: 159 args['skia_use_metal'] = 'true' 160 if 'iOS' in extra_tokens: 161 # Bots use Chromium signing cert. 162 args['skia_ios_identity'] = '".*GS9WA.*"' 163 args['skia_ios_profile'] = '"Upstream Testing Provisioning Profile"' 164 if 'CheckGeneratedFiles' in extra_tokens: 165 args['skia_compile_processors'] = 'true' 166 if compiler == 'Clang' and 'Win' in os: 167 args['clang_win'] = '"%s"' % self.m.vars.slave_dir.join('clang_win') 168 if target_arch == 'wasm': 169 args.update({ 170 'skia_use_freetype': 'false', 171 'skia_use_fontconfig': 'false', 172 'skia_use_dng_sdk': 'false', 173 'skia_use_icu': 'false', 174 'skia_enable_gpu': 'false', 175 }) 176 if 'Goma' in extra_tokens or 'GomaNoFallback' in extra_tokens: 177 json_file = self._get_goma_json() 178 self.m.cipd.set_service_account_credentials(json_file) 179 goma_package = ('infra_internal/goma/client/%s' % 180 self.m.cipd.platform_suffix()) 181 goma_dir = self.m.path['cache'].join('goma') 182 self.m.cipd.ensure(goma_dir, {goma_package: 'release'}) 183 env['GOMA_SERVICE_ACCOUNT_JSON_FILE'] = json_file 184 if 'GomaNoFallback' in extra_tokens: 185 env['GOMA_HERMETIC'] = 'error' 186 env['GOMA_USE_LOCAL'] = '0' 187 env['GOMA_FALLBACK'] = '0' 188 with self.m.context(cwd=goma_dir, env=env): 189 self._py('start goma', 'goma_ctl.py', args=['ensure_start']) 190 args['cc_wrapper'] = '"%s"' % goma_dir.join('gomacc') 191 if 'ANGLE' in extra_tokens and 'Win' in os: 192 # ANGLE uses case-insensitive include paths in D3D code. Not sure why 193 # only Goma warns about this. 194 extra_cflags.append('-Wno-nonportable-include-path') 195 ninja_args.extend(['-j', '2000']) 196 197 sanitize = '' 198 for t in extra_tokens: 199 if t.endswith('SAN'): 200 sanitize = t 201 if 'SafeStack' in extra_tokens: 202 assert sanitize == '' 203 sanitize = 'safe-stack' 204 205 for (k,v) in { 206 'cc': cc, 207 'cxx': cxx, 208 'sanitize': sanitize, 209 'target_cpu': target_arch, 210 'target_os': 'ios' if 'iOS' in extra_tokens else '', 211 'win_sdk': win_toolchain + '/win_sdk' if 'Win' in os else '', 212 'win_vc': win_toolchain + '/VC' if 'Win' in os else '', 213 }.iteritems(): 214 if v: 215 args[k] = '"%s"' % v 216 if extra_cflags: 217 args['extra_cflags'] = repr(extra_cflags).replace("'", '"') 218 if extra_ldflags: 219 args['extra_ldflags'] = repr(extra_ldflags).replace("'", '"') 220 221 gn_args = ' '.join('%s=%s' % (k,v) for (k,v) in sorted(args.iteritems())) 222 223 gn = 'gn.exe' if 'Win' in os else 'gn' 224 ninja = 'ninja.exe' if 'Win' in os else 'ninja' 225 gn = self.m.vars.skia_dir.join('bin', gn) 226 227 try: 228 with self.m.context(cwd=self.m.vars.skia_dir): 229 self._py('fetch-gn', self.m.vars.skia_dir.join('bin', 'fetch-gn')) 230 if 'CheckGeneratedFiles' in extra_tokens: 231 env['PATH'] = '%s:%%(PATH)s' % self.m.vars.skia_dir.join('bin') 232 self._py( 233 'fetch-clang-format', 234 self.m.vars.skia_dir.join('bin', 'fetch-clang-format')) 235 if target_arch == 'wasm': 236 fastcomp = emscripten_sdk + '/clang/fastcomp/build_incoming_64/bin' 237 env['PATH'] = '%s:%%(PATH)s' % fastcomp 238 239 with self.m.env(env): 240 self._run('gn gen', [gn, 'gen', self.out_dir, '--args=' + gn_args]) 241 self._run('ninja', [ninja] + ninja_args) 242 finally: 243 if goma_dir: 244 with self.m.context(cwd=goma_dir, env=env): 245 self.m.run(self.m.python, 'print goma stats', 246 script='goma_ctl.py', args=['stat'], infra_step=True, 247 abort_on_failure=False, fail_build_on_failure=False) 248 self.m.run(self.m.python, 'stop goma', 249 script='goma_ctl.py', args=['stop'], infra_step=True, 250 abort_on_failure=False, fail_build_on_failure=False) 251 # Hack: goma_ctl stop is asynchronous, so the process often does not 252 # stop before the recipe exits, which causes Swarming to freak out. 253 # Wait a couple seconds for it to exit normally. 254 # TODO(dogben): Remove after internal b/72128121 is resolved. 255 self.m.run(self.m.python.inline, 'wait for goma_ctl stop', 256 program="""import time; time.sleep(2)""", 257 infra_step=True, 258 abort_on_failure=False, fail_build_on_failure=False) 259 260 def copy_extra_build_products(self, swarming_out_dir): 261 configuration = self.m.vars.builder_cfg.get('configuration', '') 262 extra_tokens = self.m.vars.extra_tokens 263 os = self.m.vars.builder_cfg.get('os', '') 264 265 win_vulkan_sdk = str(self.m.vars.slave_dir.join('win_vulkan_sdk')) 266 if 'Win' in os and 'Vulkan' in extra_tokens: 267 self.m.run.copy_build_products( 268 win_vulkan_sdk, 269 swarming_out_dir.join('out', configuration + '_x64')) 270 271 def step(self, name, cmd): 272 app = self.m.vars.skia_out.join(self.m.vars.configuration, cmd[0]) 273 cmd = [app] + cmd[1:] 274 env = self.m.context.env 275 path = [] 276 ld_library_path = [] 277 278 slave_dir = self.m.vars.slave_dir 279 clang_linux = str(slave_dir.join('clang_linux')) 280 extra_tokens = self.m.vars.extra_tokens 281 282 if self.m.vars.is_linux: 283 if (self.m.vars.builder_cfg.get('cpu_or_gpu', '') == 'GPU' 284 and 'Intel' in self.m.vars.builder_cfg.get('cpu_or_gpu_value', '')): 285 # The vulkan in this asset name simply means that the graphics driver 286 # supports Vulkan. It is also the driver used for GL code. 287 dri_path = slave_dir.join('linux_vulkan_intel_driver_release') 288 if self.m.vars.builder_cfg.get('configuration', '') == 'Debug': 289 dri_path = slave_dir.join('linux_vulkan_intel_driver_debug') 290 ld_library_path.append(dri_path) 291 env['LIBGL_DRIVERS_PATH'] = str(dri_path) 292 env['VK_ICD_FILENAMES'] = str(dri_path.join('intel_icd.x86_64.json')) 293 294 if 'Vulkan' in extra_tokens: 295 path.append(slave_dir.join('linux_vulkan_sdk', 'bin')) 296 ld_library_path.append(slave_dir.join('linux_vulkan_sdk', 'lib')) 297 298 if any('SAN' in t for t in extra_tokens): 299 # Sanitized binaries may want to run clang_linux/bin/llvm-symbolizer. 300 path.append(clang_linux + '/bin') 301 elif self.m.vars.is_linux: 302 cmd = ['catchsegv'] + cmd 303 304 if 'ASAN' in extra_tokens or 'UBSAN' in extra_tokens: 305 env[ 'ASAN_OPTIONS'] = 'symbolize=1 detect_leaks=1' 306 env[ 'LSAN_OPTIONS'] = 'symbolize=1 print_suppressions=1' 307 env['UBSAN_OPTIONS'] = 'symbolize=1 print_stacktrace=1' 308 309 if 'MSAN' in extra_tokens: 310 # Find the MSAN-built libc++. 311 ld_library_path.append(clang_linux + '/msan') 312 313 if 'TSAN' in extra_tokens: 314 # We don't care about malloc(), fprintf, etc. used in signal handlers. 315 # If we're in a signal handler, we're already crashing... 316 env['TSAN_OPTIONS'] = 'report_signal_unsafe=0' 317 318 if 'Coverage' in extra_tokens: 319 # This is the output file for the coverage data. Just running the binary 320 # will produce the output. The output_file is in the swarming_out_dir and 321 # thus will be an isolated output of the Test step. 322 profname = '%s.profraw' % self.m.vars.builder_cfg.get('test_filter','o') 323 env['LLVM_PROFILE_FILE'] = self.m.path.join(self.m.vars.swarming_out_dir, 324 profname) 325 326 if path: 327 env['PATH'] = '%%(PATH)s:%s' % ':'.join('%s' % p for p in path) 328 if ld_library_path: 329 env['LD_LIBRARY_PATH'] = ':'.join('%s' % p for p in ld_library_path) 330 331 to_symbolize = ['dm', 'nanobench'] 332 if name in to_symbolize and self.m.vars.is_linux: 333 # Convert path objects or placeholders into strings such that they can 334 # be passed to symbolize_stack_trace.py 335 args = [slave_dir] + [str(x) for x in cmd] 336 with self.m.context(cwd=self.m.vars.skia_dir, env=env): 337 self._py('symbolized %s' % name, 338 self.module.resource('symbolize_stack_trace.py'), 339 args=args, 340 infra_step=False) 341 342 else: 343 with self.m.context(env=env): 344 self._run(name, cmd) 345
