1 #!/usr/bin/env python 2 # Copyright (c) 2013 The Chromium Authors. All rights reserved. 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 import multiprocessing 7 import optparse 8 import os 9 import posixpath 10 import sys 11 import urllib2 12 13 import buildbot_common 14 import build_version 15 import generate_make 16 import parse_dsc 17 18 from build_paths import SDK_SRC_DIR, OUT_DIR, SDK_RESOURCE_DIR 19 from build_paths import GSTORE 20 from generate_index import LandingPage 21 22 sys.path.append(os.path.join(SDK_SRC_DIR, 'tools')) 23 import getos 24 25 26 MAKE = 'nacl_sdk/make_3.99.90-26-gf80222c/make.exe' 27 LIB_DICT = { 28 'linux': [], 29 'mac': [], 30 'win': ['x86_32'] 31 } 32 VALID_TOOLCHAINS = [ 33 'bionic', 34 'newlib', 35 'glibc', 36 'pnacl', 37 'win', 38 'linux', 39 'mac', 40 ] 41 42 # Global verbosity setting. 43 # If set to True (normally via a command line arg) then build_projects will 44 # add V=1 to all calls to 'make' 45 verbose = False 46 47 48 def Trace(msg): 49 if verbose: 50 sys.stderr.write(str(msg) + '\n') 51 52 53 def CopyFilesFromTo(filelist, srcdir, dstdir): 54 for filename in filelist: 55 srcpath = os.path.join(srcdir, filename) 56 dstpath = os.path.join(dstdir, filename) 57 buildbot_common.CopyFile(srcpath, dstpath) 58 59 60 def UpdateHelpers(pepperdir, clobber=False): 61 tools_dir = os.path.join(pepperdir, 'tools') 62 if not os.path.exists(tools_dir): 63 buildbot_common.ErrorExit('SDK tools dir is missing: %s' % tools_dir) 64 65 exampledir = os.path.join(pepperdir, 'examples') 66 if clobber: 67 buildbot_common.RemoveDir(exampledir) 68 buildbot_common.MakeDir(exampledir) 69 70 # Copy files for individual build and landing page 71 files = ['favicon.ico', 'httpd.cmd', 'index.css', 'index.js', 72 'button_close.png', 'button_close_hover.png'] 73 CopyFilesFromTo(files, SDK_RESOURCE_DIR, exampledir) 74 75 # Copy tools scripts and make includes 76 buildbot_common.CopyDir(os.path.join(SDK_SRC_DIR, 'tools', '*.py'), 77 tools_dir) 78 buildbot_common.CopyDir(os.path.join(SDK_SRC_DIR, 'tools', '*.mk'), 79 tools_dir) 80 81 # Copy tools/lib scripts 82 tools_lib_dir = os.path.join(pepperdir, 'tools', 'lib') 83 buildbot_common.MakeDir(tools_lib_dir) 84 buildbot_common.CopyDir(os.path.join(SDK_SRC_DIR, 'tools', 'lib', '*.py'), 85 tools_lib_dir) 86 87 # On Windows add a prebuilt make 88 if getos.GetPlatform() == 'win': 89 buildbot_common.BuildStep('Add MAKE') 90 make_url = posixpath.join(GSTORE, MAKE) 91 make_exe = os.path.join(tools_dir, 'make.exe') 92 with open(make_exe, 'wb') as f: 93 f.write(urllib2.urlopen(make_url).read()) 94 95 96 def ValidateToolchains(toolchains): 97 invalid_toolchains = set(toolchains) - set(VALID_TOOLCHAINS) 98 if invalid_toolchains: 99 buildbot_common.ErrorExit('Invalid toolchain(s): %s' % ( 100 ', '.join(invalid_toolchains))) 101 102 def GetDeps(projects): 103 out = {} 104 105 # Build list of all project names 106 localtargets = [proj['NAME'] for proj in projects] 107 108 # For each project 109 for proj in projects: 110 deplist = [] 111 # generate a list of dependencies 112 for targ in proj.get('TARGETS', []): 113 deplist.extend(targ.get('DEPS', []) + targ.get('LIBS', [])) 114 115 # and add dependencies to targets built in this subtree 116 localdeps = [dep for dep in deplist if dep in localtargets] 117 if localdeps: 118 out[proj['NAME']] = localdeps 119 120 return out 121 122 123 def UpdateProjects(pepperdir, project_tree, toolchains, 124 clobber=False, configs=None, first_toolchain=False): 125 if configs is None: 126 configs = ['Debug', 'Release'] 127 if not os.path.exists(os.path.join(pepperdir, 'tools')): 128 buildbot_common.ErrorExit('Examples depend on missing tools.') 129 if not os.path.exists(os.path.join(pepperdir, 'toolchain')): 130 buildbot_common.ErrorExit('Examples depend on missing toolchains.') 131 132 ValidateToolchains(toolchains) 133 134 # Create the library output directories 135 libdir = os.path.join(pepperdir, 'lib') 136 platform = getos.GetPlatform() 137 for config in configs: 138 for arch in LIB_DICT[platform]: 139 dirpath = os.path.join(libdir, '%s_%s_host' % (platform, arch), config) 140 if clobber: 141 buildbot_common.RemoveDir(dirpath) 142 buildbot_common.MakeDir(dirpath) 143 144 landing_page = None 145 for branch, projects in project_tree.iteritems(): 146 dirpath = os.path.join(pepperdir, branch) 147 if clobber: 148 buildbot_common.RemoveDir(dirpath) 149 buildbot_common.MakeDir(dirpath) 150 targets = [desc['NAME'] for desc in projects] 151 deps = GetDeps(projects) 152 153 # Generate master make for this branch of projects 154 generate_make.GenerateMasterMakefile(pepperdir, 155 os.path.join(pepperdir, branch), 156 targets, deps) 157 158 if branch.startswith('examples') and not landing_page: 159 landing_page = LandingPage() 160 161 # Generate individual projects 162 for desc in projects: 163 srcroot = os.path.dirname(desc['FILEPATH']) 164 generate_make.ProcessProject(pepperdir, srcroot, pepperdir, desc, 165 toolchains, configs=configs, 166 first_toolchain=first_toolchain) 167 168 if branch.startswith('examples'): 169 landing_page.AddDesc(desc) 170 171 if landing_page: 172 # Generate the landing page text file. 173 index_html = os.path.join(pepperdir, 'examples', 'index.html') 174 index_template = os.path.join(SDK_RESOURCE_DIR, 'index.html.template') 175 with open(index_html, 'w') as fh: 176 out = landing_page.GeneratePage(index_template) 177 fh.write(out) 178 179 # Generate top Make for examples 180 targets = ['api', 'demo', 'getting_started', 'tutorial'] 181 targets = [x for x in targets if 'examples/'+x in project_tree] 182 branch_name = 'examples' 183 generate_make.GenerateMasterMakefile(pepperdir, 184 os.path.join(pepperdir, branch_name), 185 targets, {}) 186 187 188 def BuildProjectsBranch(pepperdir, branch, deps, clean, config, args=None): 189 make_dir = os.path.join(pepperdir, branch) 190 print "\nMake: " + make_dir 191 192 if getos.GetPlatform() == 'win': 193 # We need to modify the environment to build host on Windows. 194 make = os.path.join(make_dir, 'make.bat') 195 else: 196 make = 'make' 197 198 env = None 199 if os.environ.get('USE_GOMA') == '1': 200 env = dict(os.environ) 201 env['NACL_COMPILER_PREFIX'] = 'gomacc' 202 # Add -m32 to the CFLAGS when building using i686-nacl-gcc 203 # otherwise goma won't recognise it as different to the x86_64 204 # build. 205 env['X86_32_CFLAGS'] = '-m32' 206 env['X86_32_CXXFLAGS'] = '-m32' 207 jobs = '50' 208 else: 209 jobs = str(multiprocessing.cpu_count()) 210 211 make_cmd = [make, '-j', jobs] 212 213 make_cmd.append('CONFIG='+config) 214 # We always ENABLE_BIONIC in case we need it. If neither --bionic nor 215 # -t bionic have been provided on the command line, then VALID_TOOLCHAINS 216 # will not contain a bionic target. 217 make_cmd.append('ENABLE_BIONIC=1') 218 if not deps: 219 make_cmd.append('IGNORE_DEPS=1') 220 221 if verbose: 222 make_cmd.append('V=1') 223 224 if args: 225 make_cmd += args 226 else: 227 make_cmd.append('TOOLCHAIN=all') 228 229 buildbot_common.Run(make_cmd, cwd=make_dir, env=env) 230 if clean: 231 # Clean to remove temporary files but keep the built 232 buildbot_common.Run(make_cmd + ['clean'], cwd=make_dir, env=env) 233 234 235 def BuildProjects(pepperdir, project_tree, deps=True, 236 clean=False, config='Debug'): 237 # Make sure we build libraries (which live in 'src') before 238 # any of the examples. 239 build_first = [p for p in project_tree if p != 'src'] 240 build_second = [p for p in project_tree if p == 'src'] 241 242 for branch in build_first + build_second: 243 BuildProjectsBranch(pepperdir, branch, deps, clean, config) 244 245 246 def main(argv): 247 parser = optparse.OptionParser() 248 parser.add_option('-c', '--clobber', 249 help='Clobber project directories before copying new files', 250 action='store_true', default=False) 251 parser.add_option('-b', '--build', 252 help='Build the projects. Otherwise the projects are only copied.', 253 action='store_true') 254 parser.add_option('--config', 255 help='Choose configuration to build (Debug or Release). Builds both ' 256 'by default') 257 parser.add_option('--bionic', 258 help='Enable bionic projects', action='store_true') 259 parser.add_option('-x', '--experimental', 260 help='Build experimental projects', action='store_true') 261 parser.add_option('-t', '--toolchain', 262 help='Build using toolchain. Can be passed more than once.', 263 action='append', default=[]) 264 parser.add_option('-d', '--dest', 265 help='Select which build destinations (project types) are valid.', 266 action='append') 267 parser.add_option('-v', '--verbose', action='store_true') 268 269 # To setup bash completion for this command first install optcomplete 270 # and then add this line to your .bashrc: 271 # complete -F _optcomplete build_projects.py 272 try: 273 import optcomplete 274 optcomplete.autocomplete(parser) 275 except ImportError: 276 pass 277 278 options, args = parser.parse_args(argv[1:]) 279 280 global verbose 281 if options.verbose: 282 verbose = True 283 284 buildbot_common.verbose = verbose 285 286 if 'NACL_SDK_ROOT' in os.environ: 287 # We don't want the currently configured NACL_SDK_ROOT to have any effect 288 # on the build. 289 del os.environ['NACL_SDK_ROOT'] 290 291 pepper_ver = str(int(build_version.ChromeMajorVersion())) 292 pepperdir = os.path.join(OUT_DIR, 'pepper_' + pepper_ver) 293 294 if not options.toolchain: 295 # Order matters here: the default toolchain for an example's Makefile will 296 # be the first toolchain in this list that is available in the example. 297 # e.g. If an example supports newlib and glibc, then the default will be 298 # newlib. 299 options.toolchain = ['pnacl', 'newlib', 'glibc', 'host'] 300 if options.experimental or options.bionic: 301 options.toolchain.append('bionic') 302 303 if 'host' in options.toolchain: 304 options.toolchain.remove('host') 305 options.toolchain.append(getos.GetPlatform()) 306 Trace('Adding platform: ' + getos.GetPlatform()) 307 308 ValidateToolchains(options.toolchain) 309 310 filters = {} 311 if options.toolchain: 312 filters['TOOLS'] = options.toolchain 313 Trace('Filter by toolchain: ' + str(options.toolchain)) 314 if not options.experimental: 315 filters['EXPERIMENTAL'] = False 316 if options.dest: 317 filters['DEST'] = options.dest 318 Trace('Filter by type: ' + str(options.dest)) 319 if args: 320 filters['NAME'] = args 321 Trace('Filter by name: ' + str(args)) 322 323 try: 324 project_tree = parse_dsc.LoadProjectTree(SDK_SRC_DIR, include=filters) 325 except parse_dsc.ValidationError as e: 326 buildbot_common.ErrorExit(str(e)) 327 328 if verbose: 329 parse_dsc.PrintProjectTree(project_tree) 330 331 UpdateHelpers(pepperdir, clobber=options.clobber) 332 UpdateProjects(pepperdir, project_tree, options.toolchain, 333 clobber=options.clobber) 334 335 if options.build: 336 if options.config: 337 configs = [options.config] 338 else: 339 configs = ['Debug', 'Release'] 340 for config in configs: 341 BuildProjects(pepperdir, project_tree, config=config, deps=False) 342 343 return 0 344 345 346 if __name__ == '__main__': 347 script_name = os.path.basename(sys.argv[0]) 348 try: 349 sys.exit(main(sys.argv)) 350 except parse_dsc.ValidationError as e: 351 buildbot_common.ErrorExit('%s: %s' % (script_name, e)) 352 except KeyboardInterrupt: 353 buildbot_common.ErrorExit('%s: interrupted' % script_name) 354
