1 #!/usr/bin/python3 2 ##===- utils/llvmbuild - Build the LLVM project ----------------*-python-*-===## 3 # 4 # The LLVM Compiler Infrastructure 5 # 6 # This file is distributed under the University of Illinois Open Source 7 # License. See LICENSE.TXT for details. 8 # 9 ##===----------------------------------------------------------------------===## 10 # 11 # This script builds many different flavors of the LLVM ecosystem. It 12 # will build LLVM, Clang and dragonegg as well as run tests on them. 13 # This script is convenient to use to check builds and tests before 14 # committing changes to the upstream repository 15 # 16 # A typical source setup uses three trees and looks like this: 17 # 18 # official 19 # dragonegg 20 # llvm 21 # tools 22 # clang 23 # staging 24 # dragonegg 25 # llvm 26 # tools 27 # clang 28 # commit 29 # dragonegg 30 # llvm 31 # tools 32 # clang 33 # 34 # In a typical workflow, the "official" tree always contains unchanged 35 # sources from the main LLVM project repositories. The "staging" tree 36 # is where local work is done. A set of changes resides there waiting 37 # to be moved upstream. The "commit" tree is where changes from 38 # "staging" make their way upstream. Individual incremental changes 39 # from "staging" are applied to "commit" and committed upstream after 40 # a successful build and test run. A successful build is one in which 41 # testing results in no more failures than seen in the testing of the 42 # "official" tree. 43 # 44 # A build may be invoked as such: 45 # 46 # llvmbuild --src=~/llvm/commit --src=~/llvm/staging --src=~/llvm/official 47 # --build=debug --build=release --build=paranoid 48 # --prefix=/home/greened/install --builddir=/home/greened/build 49 # 50 # This will build the LLVM ecosystem, including LLVM, Clangand 51 # dragonegg, putting build results in ~/build and installing tools in 52 # ~/install. llvm-compilers-check creates separate build and install 53 # directories for each source/build flavor. In the above example, 54 # llvmbuild will build debug, release and paranoid (debug+checks) 55 # flavors from each source tree (official, staging and commit) for a 56 # total of nine builds. All builds will be run in parallel. 57 # 58 # The user may control parallelism via the --jobs and --threads 59 # switches. --jobs tells llvm-compilers-checl the maximum total 60 # number of builds to activate in parallel. The user may think of it 61 # as equivalent to the GNU make -j switch. --threads tells 62 # llvm-compilers-check how many worker threads to use to accomplish 63 # those builds. If --threads is less than --jobs, --threads workers 64 # will be launched and each one will pick a source/flavor combination 65 # to build. Then llvm-compilers-check will invoke GNU make with -j 66 # (--jobs / --threads) to use up the remaining job capacity. Once a 67 # worker is finished with a build, it will pick another combination 68 # off the list and start building it. 69 # 70 ##===----------------------------------------------------------------------===## 71 72 import optparse 73 import os 74 import sys 75 import threading 76 import queue 77 import logging 78 import traceback 79 import subprocess 80 import re 81 82 # TODO: Use shutil.which when it is available (3.2 or later) 83 def find_executable(executable, path=None): 84 """Try to find 'executable' in the directories listed in 'path' (a 85 string listing directories separated by 'os.pathsep'; defaults to 86 os.environ['PATH']). Returns the complete filename or None if not 87 found 88 """ 89 if path is None: 90 path = os.environ['PATH'] 91 paths = path.split(os.pathsep) 92 extlist = [''] 93 if os.name == 'os2': 94 (base, ext) = os.path.splitext(executable) 95 # executable files on OS/2 can have an arbitrary extension, but 96 # .exe is automatically appended if no dot is present in the name 97 if not ext: 98 executable = executable + ".exe" 99 elif sys.platform == 'win32': 100 pathext = os.environ['PATHEXT'].lower().split(os.pathsep) 101 (base, ext) = os.path.splitext(executable) 102 if ext.lower() not in pathext: 103 extlist = pathext 104 for ext in extlist: 105 execname = executable + ext 106 if os.path.isfile(execname): 107 return execname 108 else: 109 for p in paths: 110 f = os.path.join(p, execname) 111 if os.path.isfile(f): 112 return f 113 else: 114 return None 115 116 def is_executable(fpath): 117 return os.path.exists(fpath) and os.access(fpath, os.X_OK) 118 119 def add_options(parser): 120 parser.add_option("-v", "--verbose", action="store_true", 121 default=False, 122 help=("Output informational messages" 123 " [default: %default]")) 124 parser.add_option("--src", action="append", 125 help=("Top-level source directory [default: %default]")) 126 parser.add_option("--build", action="append", 127 help=("Build types to run [default: %default]")) 128 parser.add_option("--cc", default=find_executable("cc"), 129 help=("The C compiler to use [default: %default]")) 130 parser.add_option("--cxx", default=find_executable("c++"), 131 help=("The C++ compiler to use [default: %default]")) 132 parser.add_option("--threads", default=4, type="int", 133 help=("The number of worker threads to use " 134 "[default: %default]")) 135 parser.add_option("--jobs", "-j", default=8, type="int", 136 help=("The number of simultaneous build jobs " 137 "[default: %default]")) 138 parser.add_option("--prefix", 139 help=("Root install directory [default: %default]")) 140 parser.add_option("--builddir", 141 help=("Root build directory [default: %default]")) 142 parser.add_option("--extra-llvm-config-flags", default="", 143 help=("Extra flags to pass to llvm configure [default: %default]")) 144 parser.add_option("--force-configure", default=False, action="store_true", 145 help=("Force reconfigure of all components")) 146 parser.add_option("--no-dragonegg", default=False, action="store_true", 147 help=("Do not build dragonegg")) 148 parser.add_option("--no-install", default=False, action="store_true", 149 help=("Do not do installs")) 150 parser.add_option("--keep-going", default=False, action="store_true", 151 help=("Keep going after failures")) 152 parser.add_option("--no-flavor-prefix", default=False, action="store_true", 153 help=("Do not append the build flavor to the install path")) 154 parser.add_option("--enable-werror", default=False, action="store_true", 155 help=("Build with -Werror")) 156 return 157 158 def check_options(parser, options, valid_builds): 159 # See if we're building valid flavors. 160 for build in options.build: 161 if (build not in valid_builds): 162 parser.error("'" + build + "' is not a valid build flavor " 163 + str(valid_builds)) 164 165 # See if we can find source directories. 166 for src in options.src: 167 for component in components: 168 component = component.rstrip("2") 169 compsrc = src + "/" + component 170 if (not os.path.isdir(compsrc)): 171 parser.error("'" + compsrc + "' does not exist") 172 173 # See if we can find the compilers 174 options.cc = find_executable(options.cc) 175 options.cxx = find_executable(options.cxx) 176 177 return 178 179 # Find a unique short name for the given set of paths. This searches 180 # back through path components until it finds unique component names 181 # among all given paths. 182 def get_path_abbrevs(paths): 183 # Find the number of common starting characters in the last component 184 # of the paths. 185 unique_paths = list(paths) 186 187 class NotFoundException(Exception): pass 188 189 # Find a unique component of each path. 190 unique_bases = unique_paths[:] 191 found = 0 192 while len(unique_paths) > 0: 193 bases = [os.path.basename(src) for src in unique_paths] 194 components = { c for c in bases } 195 # Account for single entry in paths. 196 if len(components) > 1 or len(components) == len(bases): 197 # We found something unique. 198 for c in components: 199 if bases.count(c) == 1: 200 index = bases.index(c) 201 unique_bases[index] = c 202 # Remove the corresponding path from the set under 203 # consideration. 204 unique_paths[index] = None 205 unique_paths = [ p for p in unique_paths if p is not None ] 206 unique_paths = [os.path.dirname(src) for src in unique_paths] 207 208 if len(unique_paths) > 0: 209 raise NotFoundException() 210 211 abbrevs = dict(zip(paths, [base for base in unique_bases])) 212 213 return abbrevs 214 215 # Given a set of unique names, find a short character sequence that 216 # uniquely identifies them. 217 def get_short_abbrevs(unique_bases): 218 # Find a unique start character for each path base. 219 my_unique_bases = unique_bases[:] 220 unique_char_starts = unique_bases[:] 221 while len(my_unique_bases) > 0: 222 for start, char_tuple in enumerate(zip(*[base 223 for base in my_unique_bases])): 224 chars = { c for c in char_tuple } 225 # Account for single path. 226 if len(chars) > 1 or len(chars) == len(char_tuple): 227 # We found something unique. 228 for c in chars: 229 if char_tuple.count(c) == 1: 230 index = char_tuple.index(c) 231 unique_char_starts[index] = start 232 # Remove the corresponding path from the set under 233 # consideration. 234 my_unique_bases[index] = None 235 my_unique_bases = [ b for b in my_unique_bases 236 if b is not None ] 237 break 238 239 if len(my_unique_bases) > 0: 240 raise NotFoundException() 241 242 abbrevs = [abbrev[start_index:start_index+3] 243 for abbrev, start_index 244 in zip([base for base in unique_bases], 245 [index for index in unique_char_starts])] 246 247 abbrevs = dict(zip(unique_bases, abbrevs)) 248 249 return abbrevs 250 251 class Builder(threading.Thread): 252 class ExecutableNotFound(Exception): pass 253 class FileNotExecutable(Exception): pass 254 255 def __init__(self, work_queue, jobs, 256 build_abbrev, source_abbrev, 257 options): 258 super().__init__() 259 self.work_queue = work_queue 260 self.jobs = jobs 261 self.cc = options.cc 262 self.cxx = options.cxx 263 self.build_abbrev = build_abbrev 264 self.source_abbrev = source_abbrev 265 self.build_prefix = options.builddir 266 self.install_prefix = options.prefix 267 self.options = options 268 self.component_abbrev = dict( 269 llvm="llvm", 270 dragonegg="degg") 271 def run(self): 272 while True: 273 try: 274 source, build = self.work_queue.get() 275 self.dobuild(source, build) 276 except: 277 traceback.print_exc() 278 finally: 279 self.work_queue.task_done() 280 281 def execute(self, command, execdir, env, component): 282 prefix = self.component_abbrev[component.replace("-", "_")] 283 pwd = os.getcwd() 284 if not os.path.exists(execdir): 285 os.makedirs(execdir) 286 287 execenv = os.environ.copy() 288 289 for key, value in env.items(): 290 execenv[key] = value 291 292 self.logger.debug("[" + prefix + "] " + "env " + str(env) + " " 293 + " ".join(command)); 294 295 try: 296 proc = subprocess.Popen(command, 297 cwd=execdir, 298 env=execenv, 299 stdout=subprocess.PIPE, 300 stderr=subprocess.STDOUT) 301 302 line = proc.stdout.readline() 303 while line: 304 self.logger.info("[" + prefix + "] " 305 + str(line, "utf-8").rstrip()) 306 line = proc.stdout.readline() 307 308 (stdoutdata, stderrdata) = proc.communicate() 309 retcode = proc.wait() 310 311 return retcode 312 313 except: 314 traceback.print_exc() 315 316 # Get a list of C++ include directories to pass to clang. 317 def get_includes(self): 318 # Assume we're building with g++ for now. 319 command = [self.cxx] 320 command += ["-v", "-x", "c++", "/dev/null", "-fsyntax-only"] 321 includes = [] 322 self.logger.debug(command) 323 try: 324 proc = subprocess.Popen(command, 325 stdout=subprocess.PIPE, 326 stderr=subprocess.STDOUT) 327 328 gather = False 329 line = proc.stdout.readline() 330 while line: 331 self.logger.debug(line) 332 if re.search("End of search list", str(line)) is not None: 333 self.logger.debug("Stop Gather") 334 gather = False 335 if gather: 336 includes.append(str(line, "utf-8").strip()) 337 if re.search("#include <...> search starts", str(line)) is not None: 338 self.logger.debug("Start Gather") 339 gather = True 340 line = proc.stdout.readline() 341 342 except: 343 traceback.print_exc() 344 self.logger.debug(includes) 345 return includes 346 347 def dobuild(self, source, build): 348 build_suffix = "" 349 350 ssabbrev = get_short_abbrevs([ab for ab in self.source_abbrev.values()]) 351 352 prefix = "[" + ssabbrev[self.source_abbrev[source]] + "-" + self.build_abbrev[build] + "]" 353 if (not self.options.no_flavor_prefix): 354 self.install_prefix += "/" + self.source_abbrev[source] + "/" + build 355 356 build_suffix += "/" + self.source_abbrev[source] + "/" + build 357 358 self.logger = logging.getLogger(prefix) 359 360 self.logger.debug(self.install_prefix) 361 362 # Assume we're building with gcc for now. 363 cxxincludes = self.get_includes() 364 cxxroot = os.path.dirname(cxxincludes[0]) # Remove the version 365 cxxroot = os.path.dirname(cxxroot) # Remove the c++ 366 cxxroot = os.path.dirname(cxxroot) # Remove the include 367 368 configure_flags = dict( 369 llvm=dict(debug=["--prefix=" + self.install_prefix, 370 "--enable-assertions", 371 "--disable-optimized", 372 "--with-gcc-toolchain=" + cxxroot], 373 release=["--prefix=" + self.install_prefix, 374 "--enable-optimized", 375 "--with-gcc-toolchain=" + cxxroot], 376 paranoid=["--prefix=" + self.install_prefix, 377 "--enable-assertions", 378 "--enable-expensive-checks", 379 "--disable-optimized", 380 "--with-gcc-toolchain=" + cxxroot]), 381 dragonegg=dict(debug=[], 382 release=[], 383 paranoid=[])) 384 385 if (self.options.enable_werror): 386 configure_flags["llvm"]["debug"].append("--enable-werror") 387 configure_flags["llvm"]["release"].append("--enable-werror") 388 configure_flags["llvm"]["paranoid"].append("--enable-werror") 389 390 configure_env = dict( 391 llvm=dict(debug=dict(CC=self.cc, 392 CXX=self.cxx), 393 release=dict(CC=self.cc, 394 CXX=self.cxx), 395 paranoid=dict(CC=self.cc, 396 CXX=self.cxx)), 397 dragonegg=dict(debug=dict(CC=self.cc, 398 CXX=self.cxx), 399 release=dict(CC=self.cc, 400 CXX=self.cxx), 401 paranoid=dict(CC=self.cc, 402 CXX=self.cxx))) 403 404 make_flags = dict( 405 llvm=dict(debug=["-j" + str(self.jobs)], 406 release=["-j" + str(self.jobs)], 407 paranoid=["-j" + str(self.jobs)]), 408 dragonegg=dict(debug=["-j" + str(self.jobs)], 409 release=["-j" + str(self.jobs)], 410 paranoid=["-j" + str(self.jobs)])) 411 412 make_env = dict( 413 llvm=dict(debug=dict(), 414 release=dict(), 415 paranoid=dict()), 416 dragonegg=dict(debug=dict(GCC=self.cc, 417 LLVM_CONFIG=self.install_prefix + "/bin/llvm-config"), 418 release=dict(GCC=self.cc, 419 LLVM_CONFIG=self.install_prefix + "/bin/llvm-config"), 420 paranoid=dict(GCC=self.cc, 421 LLVM_CONFIG=self.install_prefix + "/bin/llvm-config"))) 422 423 make_install_flags = dict( 424 llvm=dict(debug=["install"], 425 release=["install"], 426 paranoid=["install"]), 427 dragonegg=dict(debug=["install"], 428 release=["install"], 429 paranoid=["install"])) 430 431 make_install_env = dict( 432 llvm=dict(debug=dict(), 433 release=dict(), 434 paranoid=dict()), 435 dragonegg=dict(debug=dict(), 436 release=dict(), 437 paranoid=dict())) 438 439 make_check_flags = dict( 440 llvm=dict(debug=["check"], 441 release=["check"], 442 paranoid=["check"]), 443 dragonegg=dict(debug=["check"], 444 release=["check"], 445 paranoid=["check"])) 446 447 make_check_env = dict( 448 llvm=dict(debug=dict(), 449 release=dict(), 450 paranoid=dict()), 451 dragonegg=dict(debug=dict(), 452 release=dict(), 453 paranoid=dict())) 454 455 for component in components: 456 comp = component[:] 457 458 if (self.options.no_dragonegg): 459 if (comp == 'dragonegg'): 460 self.logger.info("Skipping " + component + " in " 461 + builddir) 462 continue 463 464 srcdir = source + "/" + comp.rstrip("2") 465 builddir = self.build_prefix + "/" + comp + "/" + build_suffix 466 installdir = self.install_prefix 467 468 comp_key = comp.replace("-", "_") 469 470 config_args = configure_flags[comp_key][build][:] 471 config_args.extend(getattr(self.options, 472 "extra_" + comp_key.rstrip("2") 473 + "_config_flags", 474 "").split()) 475 476 self.logger.info("Configuring " + component + " in " + builddir) 477 configrc = self.configure(component, srcdir, builddir, 478 config_args, 479 configure_env[comp_key][build]) 480 481 if (configrc == None) : 482 self.logger.info("[None] Failed to configure " + component + " in " + installdir) 483 484 if (configrc == 0 or self.options.keep_going) : 485 self.logger.info("Building " + component + " in " + builddir) 486 self.logger.info("Build: make " + str(make_flags[comp_key][build])) 487 buildrc = self.make(component, srcdir, builddir, 488 make_flags[comp_key][build], 489 make_env[comp_key][build]) 490 491 if (buildrc == None) : 492 self.logger.info("[None] Failed to build " + component + " in " + installdir) 493 494 if (buildrc == 0 or self.options.keep_going) : 495 self.logger.info("Testing " + component + " in " + builddir) 496 self.logger.info("Test: make " 497 + str(make_check_flags[comp_key][build])) 498 testrc = self.make(component, srcdir, builddir, 499 make_check_flags[comp_key][build], 500 make_check_env[comp_key][build]) 501 502 if (testrc == None) : 503 self.logger.info("[None] Failed to test " + component + " in " + installdir) 504 505 if ((testrc == 0 or self.options.keep_going) 506 and not self.options.no_install): 507 self.logger.info("Installing " + component + " in " + installdir) 508 self.make(component, srcdir, builddir, 509 make_install_flags[comp_key][build], 510 make_install_env[comp_key][build]) 511 else : 512 self.logger.info("Failed testing " + component + " in " + installdir) 513 514 else : 515 self.logger.info("Failed to build " + component + " in " + installdir) 516 517 else : 518 self.logger.info("Failed to configure " + component + " in " + installdir) 519 520 def configure(self, component, srcdir, builddir, flags, env): 521 prefix = self.component_abbrev[component.replace("-", "_")] 522 523 self.logger.debug("Configure " + str(flags) + " " + str(srcdir) + " -> " 524 + str(builddir)) 525 526 configure_files = dict( 527 llvm=[(srcdir + "/configure", builddir + "/Makefile")], 528 dragonegg=[(None,None)]) 529 530 531 doconfig = False 532 for conf, mf in configure_files[component.replace("-", "_")]: 533 if conf is None: 534 # No configure necessary 535 return 0 536 537 if not os.path.exists(conf): 538 self.logger.info("[" + prefix + "] Configure failed, no configure script " + conf) 539 return -1 540 541 if os.path.exists(conf) and os.path.exists(mf): 542 confstat = os.stat(conf) 543 makestat = os.stat(mf) 544 if confstat.st_mtime > makestat.st_mtime: 545 doconfig = True 546 break 547 else: 548 doconfig = True 549 break 550 551 if not doconfig and not self.options.force_configure: 552 return 0 553 554 program = srcdir + "/configure" 555 if not is_executable(program): 556 self.logger.info("[" + prefix + "] Configure failed, cannot execute " + program) 557 return -1 558 559 args = [program] 560 args += ["--verbose"] 561 args += flags 562 return self.execute(args, builddir, env, component) 563 564 def make(self, component, srcdir, builddir, flags, env): 565 program = find_executable("make") 566 if program is None: 567 raise ExecutableNotFound 568 569 if not is_executable(program): 570 raise FileNotExecutable 571 572 args = [program] 573 args += flags 574 return self.execute(args, builddir, env, component) 575 576 # Global constants 577 build_abbrev = dict(debug="dbg", release="opt", paranoid="par") 578 components = ["llvm", "dragonegg"] 579 580 # Parse options 581 parser = optparse.OptionParser(version="%prog 1.0") 582 add_options(parser) 583 (options, args) = parser.parse_args() 584 check_options(parser, options, build_abbrev.keys()); 585 586 if options.verbose: 587 logging.basicConfig(level=logging.DEBUG, 588 format='%(name)-13s: %(message)s') 589 else: 590 logging.basicConfig(level=logging.INFO, 591 format='%(name)-13s: %(message)s') 592 593 source_abbrev = get_path_abbrevs(set(options.src)) 594 595 work_queue = queue.Queue() 596 597 jobs = options.jobs // options.threads 598 if jobs == 0: 599 jobs = 1 600 601 numthreads = options.threads 602 603 logging.getLogger().info("Building with " + str(options.jobs) + " jobs and " 604 + str(numthreads) + " threads using " + str(jobs) 605 + " make jobs") 606 607 logging.getLogger().info("CC = " + str(options.cc)) 608 logging.getLogger().info("CXX = " + str(options.cxx)) 609 610 for t in range(numthreads): 611 builder = Builder(work_queue, jobs, 612 build_abbrev, source_abbrev, 613 options) 614 builder.daemon = True 615 builder.start() 616 617 for build in set(options.build): 618 for source in set(options.src): 619 work_queue.put((source, build)) 620 621 work_queue.join() 622
