1 # SPDX-License-Identifier: GPL-2.0+ 2 # Copyright (c) 2012 The Chromium OS Authors. 3 # 4 5 import re 6 import glob 7 from HTMLParser import HTMLParser 8 import os 9 import sys 10 import tempfile 11 import urllib2 12 13 import bsettings 14 import command 15 import terminal 16 17 (PRIORITY_FULL_PREFIX, PRIORITY_PREFIX_GCC, PRIORITY_PREFIX_GCC_PATH, 18 PRIORITY_CALC) = range(4) 19 20 # Simple class to collect links from a page 21 class MyHTMLParser(HTMLParser): 22 def __init__(self, arch): 23 """Create a new parser 24 25 After the parser runs, self.links will be set to a list of the links 26 to .xz archives found in the page, and self.arch_link will be set to 27 the one for the given architecture (or None if not found). 28 29 Args: 30 arch: Architecture to search for 31 """ 32 HTMLParser.__init__(self) 33 self.arch_link = None 34 self.links = [] 35 self.re_arch = re.compile('[-_]%s-' % arch) 36 37 def handle_starttag(self, tag, attrs): 38 if tag == 'a': 39 for tag, value in attrs: 40 if tag == 'href': 41 if value and value.endswith('.xz'): 42 self.links.append(value) 43 if self.re_arch.search(value): 44 self.arch_link = value 45 46 47 class Toolchain: 48 """A single toolchain 49 50 Public members: 51 gcc: Full path to C compiler 52 path: Directory path containing C compiler 53 cross: Cross compile string, e.g. 'arm-linux-' 54 arch: Architecture of toolchain as determined from the first 55 component of the filename. E.g. arm-linux-gcc becomes arm 56 priority: Toolchain priority (0=highest, 20=lowest) 57 """ 58 def __init__(self, fname, test, verbose=False, priority=PRIORITY_CALC, 59 arch=None): 60 """Create a new toolchain object. 61 62 Args: 63 fname: Filename of the gcc component 64 test: True to run the toolchain to test it 65 verbose: True to print out the information 66 priority: Priority to use for this toolchain, or PRIORITY_CALC to 67 calculate it 68 """ 69 self.gcc = fname 70 self.path = os.path.dirname(fname) 71 72 # Find the CROSS_COMPILE prefix to use for U-Boot. For example, 73 # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'. 74 basename = os.path.basename(fname) 75 pos = basename.rfind('-') 76 self.cross = basename[:pos + 1] if pos != -1 else '' 77 78 # The architecture is the first part of the name 79 pos = self.cross.find('-') 80 if arch: 81 self.arch = arch 82 else: 83 self.arch = self.cross[:pos] if pos != -1 else 'sandbox' 84 85 env = self.MakeEnvironment(False) 86 87 # As a basic sanity check, run the C compiler with --version 88 cmd = [fname, '--version'] 89 if priority == PRIORITY_CALC: 90 self.priority = self.GetPriority(fname) 91 else: 92 self.priority = priority 93 if test: 94 result = command.RunPipe([cmd], capture=True, env=env, 95 raise_on_error=False) 96 self.ok = result.return_code == 0 97 if verbose: 98 print 'Tool chain test: ', 99 if self.ok: 100 print "OK, arch='%s', priority %d" % (self.arch, 101 self.priority) 102 else: 103 print 'BAD' 104 print 'Command: ', cmd 105 print result.stdout 106 print result.stderr 107 else: 108 self.ok = True 109 110 def GetPriority(self, fname): 111 """Return the priority of the toolchain. 112 113 Toolchains are ranked according to their suitability by their 114 filename prefix. 115 116 Args: 117 fname: Filename of toolchain 118 Returns: 119 Priority of toolchain, PRIORITY_CALC=highest, 20=lowest. 120 """ 121 priority_list = ['-elf', '-unknown-linux-gnu', '-linux', 122 '-none-linux-gnueabi', '-none-linux-gnueabihf', '-uclinux', 123 '-none-eabi', '-gentoo-linux-gnu', '-linux-gnueabi', 124 '-linux-gnueabihf', '-le-linux', '-uclinux'] 125 for prio in range(len(priority_list)): 126 if priority_list[prio] in fname: 127 return PRIORITY_CALC + prio 128 return PRIORITY_CALC + prio 129 130 def GetWrapper(self, show_warning=True): 131 """Get toolchain wrapper from the setting file. 132 """ 133 value = '' 134 for name, value in bsettings.GetItems('toolchain-wrapper'): 135 if not value: 136 print "Warning: Wrapper not found" 137 if value: 138 value = value + ' ' 139 140 return value 141 142 def MakeEnvironment(self, full_path): 143 """Returns an environment for using the toolchain. 144 145 Thie takes the current environment and adds CROSS_COMPILE so that 146 the tool chain will operate correctly. This also disables localized 147 output and possibly unicode encoded output of all build tools by 148 adding LC_ALL=C. 149 150 Args: 151 full_path: Return the full path in CROSS_COMPILE and don't set 152 PATH 153 """ 154 env = dict(os.environ) 155 wrapper = self.GetWrapper() 156 157 if full_path: 158 env['CROSS_COMPILE'] = wrapper + os.path.join(self.path, self.cross) 159 else: 160 env['CROSS_COMPILE'] = wrapper + self.cross 161 env['PATH'] = self.path + ':' + env['PATH'] 162 163 env['LC_ALL'] = 'C' 164 165 return env 166 167 168 class Toolchains: 169 """Manage a list of toolchains for building U-Boot 170 171 We select one toolchain for each architecture type 172 173 Public members: 174 toolchains: Dict of Toolchain objects, keyed by architecture name 175 prefixes: Dict of prefixes to check, keyed by architecture. This can 176 be a full path and toolchain prefix, for example 177 {'x86', 'opt/i386-linux/bin/i386-linux-'}, or the name of 178 something on the search path, for example 179 {'arm', 'arm-linux-gnueabihf-'}. Wildcards are not supported. 180 paths: List of paths to check for toolchains (may contain wildcards) 181 """ 182 183 def __init__(self): 184 self.toolchains = {} 185 self.prefixes = {} 186 self.paths = [] 187 self._make_flags = dict(bsettings.GetItems('make-flags')) 188 189 def GetPathList(self, show_warning=True): 190 """Get a list of available toolchain paths 191 192 Args: 193 show_warning: True to show a warning if there are no tool chains. 194 195 Returns: 196 List of strings, each a path to a toolchain mentioned in the 197 [toolchain] section of the settings file. 198 """ 199 toolchains = bsettings.GetItems('toolchain') 200 if show_warning and not toolchains: 201 print ("Warning: No tool chains. Please run 'buildman " 202 "--fetch-arch all' to download all available toolchains, or " 203 "add a [toolchain] section to your buildman config file " 204 "%s. See README for details" % 205 bsettings.config_fname) 206 207 paths = [] 208 for name, value in toolchains: 209 if '*' in value: 210 paths += glob.glob(value) 211 else: 212 paths.append(value) 213 return paths 214 215 def GetSettings(self, show_warning=True): 216 """Get toolchain settings from the settings file. 217 218 Args: 219 show_warning: True to show a warning if there are no tool chains. 220 """ 221 self.prefixes = bsettings.GetItems('toolchain-prefix') 222 self.paths += self.GetPathList(show_warning) 223 224 def Add(self, fname, test=True, verbose=False, priority=PRIORITY_CALC, 225 arch=None): 226 """Add a toolchain to our list 227 228 We select the given toolchain as our preferred one for its 229 architecture if it is a higher priority than the others. 230 231 Args: 232 fname: Filename of toolchain's gcc driver 233 test: True to run the toolchain to test it 234 priority: Priority to use for this toolchain 235 arch: Toolchain architecture, or None if not known 236 """ 237 toolchain = Toolchain(fname, test, verbose, priority, arch) 238 add_it = toolchain.ok 239 if toolchain.arch in self.toolchains: 240 add_it = (toolchain.priority < 241 self.toolchains[toolchain.arch].priority) 242 if add_it: 243 self.toolchains[toolchain.arch] = toolchain 244 elif verbose: 245 print ("Toolchain '%s' at priority %d will be ignored because " 246 "another toolchain for arch '%s' has priority %d" % 247 (toolchain.gcc, toolchain.priority, toolchain.arch, 248 self.toolchains[toolchain.arch].priority)) 249 250 def ScanPath(self, path, verbose): 251 """Scan a path for a valid toolchain 252 253 Args: 254 path: Path to scan 255 verbose: True to print out progress information 256 Returns: 257 Filename of C compiler if found, else None 258 """ 259 fnames = [] 260 for subdir in ['.', 'bin', 'usr/bin']: 261 dirname = os.path.join(path, subdir) 262 if verbose: print " - looking in '%s'" % dirname 263 for fname in glob.glob(dirname + '/*gcc'): 264 if verbose: print " - found '%s'" % fname 265 fnames.append(fname) 266 return fnames 267 268 def ScanPathEnv(self, fname): 269 """Scan the PATH environment variable for a given filename. 270 271 Args: 272 fname: Filename to scan for 273 Returns: 274 List of matching pathanames, or [] if none 275 """ 276 pathname_list = [] 277 for path in os.environ["PATH"].split(os.pathsep): 278 path = path.strip('"') 279 pathname = os.path.join(path, fname) 280 if os.path.exists(pathname): 281 pathname_list.append(pathname) 282 return pathname_list 283 284 def Scan(self, verbose): 285 """Scan for available toolchains and select the best for each arch. 286 287 We look for all the toolchains we can file, figure out the 288 architecture for each, and whether it works. Then we select the 289 highest priority toolchain for each arch. 290 291 Args: 292 verbose: True to print out progress information 293 """ 294 if verbose: print 'Scanning for tool chains' 295 for name, value in self.prefixes: 296 if verbose: print " - scanning prefix '%s'" % value 297 if os.path.exists(value): 298 self.Add(value, True, verbose, PRIORITY_FULL_PREFIX, name) 299 continue 300 fname = value + 'gcc' 301 if os.path.exists(fname): 302 self.Add(fname, True, verbose, PRIORITY_PREFIX_GCC, name) 303 continue 304 fname_list = self.ScanPathEnv(fname) 305 for f in fname_list: 306 self.Add(f, True, verbose, PRIORITY_PREFIX_GCC_PATH, name) 307 if not fname_list: 308 raise ValueError, ("No tool chain found for prefix '%s'" % 309 value) 310 for path in self.paths: 311 if verbose: print " - scanning path '%s'" % path 312 fnames = self.ScanPath(path, verbose) 313 for fname in fnames: 314 self.Add(fname, True, verbose) 315 316 def List(self): 317 """List out the selected toolchains for each architecture""" 318 col = terminal.Color() 319 print col.Color(col.BLUE, 'List of available toolchains (%d):' % 320 len(self.toolchains)) 321 if len(self.toolchains): 322 for key, value in sorted(self.toolchains.iteritems()): 323 print '%-10s: %s' % (key, value.gcc) 324 else: 325 print 'None' 326 327 def Select(self, arch): 328 """Returns the toolchain for a given architecture 329 330 Args: 331 args: Name of architecture (e.g. 'arm', 'ppc_8xx') 332 333 returns: 334 toolchain object, or None if none found 335 """ 336 for tag, value in bsettings.GetItems('toolchain-alias'): 337 if arch == tag: 338 for alias in value.split(): 339 if alias in self.toolchains: 340 return self.toolchains[alias] 341 342 if not arch in self.toolchains: 343 raise ValueError, ("No tool chain found for arch '%s'" % arch) 344 return self.toolchains[arch] 345 346 def ResolveReferences(self, var_dict, args): 347 """Resolve variable references in a string 348 349 This converts ${blah} within the string to the value of blah. 350 This function works recursively. 351 352 Args: 353 var_dict: Dictionary containing variables and their values 354 args: String containing make arguments 355 Returns: 356 Resolved string 357 358 >>> bsettings.Setup() 359 >>> tcs = Toolchains() 360 >>> tcs.Add('fred', False) 361 >>> var_dict = {'oblique' : 'OBLIQUE', 'first' : 'fi${second}rst', \ 362 'second' : '2nd'} 363 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set') 364 'this=OBLIQUE_set' 365 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set${first}nd') 366 'this=OBLIQUE_setfi2ndrstnd' 367 """ 368 re_var = re.compile('(\$\{[-_a-z0-9A-Z]{1,}\})') 369 370 while True: 371 m = re_var.search(args) 372 if not m: 373 break 374 lookup = m.group(0)[2:-1] 375 value = var_dict.get(lookup, '') 376 args = args[:m.start(0)] + value + args[m.end(0):] 377 return args 378 379 def GetMakeArguments(self, board): 380 """Returns 'make' arguments for a given board 381 382 The flags are in a section called 'make-flags'. Flags are named 383 after the target they represent, for example snapper9260=TESTING=1 384 will pass TESTING=1 to make when building the snapper9260 board. 385 386 References to other boards can be added in the string also. For 387 example: 388 389 [make-flags] 390 at91-boards=ENABLE_AT91_TEST=1 391 snapper9260=${at91-boards} BUILD_TAG=442 392 snapper9g45=${at91-boards} BUILD_TAG=443 393 394 This will return 'ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260 395 and 'ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45. 396 397 A special 'target' variable is set to the board target. 398 399 Args: 400 board: Board object for the board to check. 401 Returns: 402 'make' flags for that board, or '' if none 403 """ 404 self._make_flags['target'] = board.target 405 arg_str = self.ResolveReferences(self._make_flags, 406 self._make_flags.get(board.target, '')) 407 args = arg_str.split(' ') 408 i = 0 409 while i < len(args): 410 if not args[i]: 411 del args[i] 412 else: 413 i += 1 414 return args 415 416 def LocateArchUrl(self, fetch_arch): 417 """Find a toolchain available online 418 419 Look in standard places for available toolchains. At present the 420 only standard place is at kernel.org. 421 422 Args: 423 arch: Architecture to look for, or 'list' for all 424 Returns: 425 If fetch_arch is 'list', a tuple: 426 Machine architecture (e.g. x86_64) 427 List of toolchains 428 else 429 URL containing this toolchain, if avaialble, else None 430 """ 431 arch = command.OutputOneLine('uname', '-m') 432 base = 'https://www.kernel.org/pub/tools/crosstool/files/bin' 433 versions = ['7.3.0', '6.4.0', '4.9.4'] 434 links = [] 435 for version in versions: 436 url = '%s/%s/%s/' % (base, arch, version) 437 print 'Checking: %s' % url 438 response = urllib2.urlopen(url) 439 html = response.read() 440 parser = MyHTMLParser(fetch_arch) 441 parser.feed(html) 442 if fetch_arch == 'list': 443 links += parser.links 444 elif parser.arch_link: 445 return url + parser.arch_link 446 if fetch_arch == 'list': 447 return arch, links 448 return None 449 450 def Download(self, url): 451 """Download a file to a temporary directory 452 453 Args: 454 url: URL to download 455 Returns: 456 Tuple: 457 Temporary directory name 458 Full path to the downloaded archive file in that directory, 459 or None if there was an error while downloading 460 """ 461 print 'Downloading: %s' % url 462 leaf = url.split('/')[-1] 463 tmpdir = tempfile.mkdtemp('.buildman') 464 response = urllib2.urlopen(url) 465 fname = os.path.join(tmpdir, leaf) 466 fd = open(fname, 'wb') 467 meta = response.info() 468 size = int(meta.getheaders('Content-Length')[0]) 469 done = 0 470 block_size = 1 << 16 471 status = '' 472 473 # Read the file in chunks and show progress as we go 474 while True: 475 buffer = response.read(block_size) 476 if not buffer: 477 print chr(8) * (len(status) + 1), '\r', 478 break 479 480 done += len(buffer) 481 fd.write(buffer) 482 status = r'%10d MiB [%3d%%]' % (done / 1024 / 1024, 483 done * 100 / size) 484 status = status + chr(8) * (len(status) + 1) 485 print status, 486 sys.stdout.flush() 487 fd.close() 488 if done != size: 489 print 'Error, failed to download' 490 os.remove(fname) 491 fname = None 492 return tmpdir, fname 493 494 def Unpack(self, fname, dest): 495 """Unpack a tar file 496 497 Args: 498 fname: Filename to unpack 499 dest: Destination directory 500 Returns: 501 Directory name of the first entry in the archive, without the 502 trailing / 503 """ 504 stdout = command.Output('tar', 'xvfJ', fname, '-C', dest) 505 return stdout.splitlines()[0][:-1] 506 507 def TestSettingsHasPath(self, path): 508 """Check if buildman will find this toolchain 509 510 Returns: 511 True if the path is in settings, False if not 512 """ 513 paths = self.GetPathList(False) 514 return path in paths 515 516 def ListArchs(self): 517 """List architectures with available toolchains to download""" 518 host_arch, archives = self.LocateArchUrl('list') 519 re_arch = re.compile('[-a-z0-9.]*_([^-]*)-.*') 520 arch_set = set() 521 for archive in archives: 522 # Remove the host architecture from the start 523 arch = re_arch.match(archive[len(host_arch):]) 524 if arch: 525 arch_set.add(arch.group(1)) 526 return sorted(arch_set) 527 528 def FetchAndInstall(self, arch): 529 """Fetch and install a new toolchain 530 531 arch: 532 Architecture to fetch, or 'list' to list 533 """ 534 # Fist get the URL for this architecture 535 col = terminal.Color() 536 print col.Color(col.BLUE, "Downloading toolchain for arch '%s'" % arch) 537 url = self.LocateArchUrl(arch) 538 if not url: 539 print ("Cannot find toolchain for arch '%s' - use 'list' to list" % 540 arch) 541 return 2 542 home = os.environ['HOME'] 543 dest = os.path.join(home, '.buildman-toolchains') 544 if not os.path.exists(dest): 545 os.mkdir(dest) 546 547 # Download the tar file for this toolchain and unpack it 548 tmpdir, tarfile = self.Download(url) 549 if not tarfile: 550 return 1 551 print col.Color(col.GREEN, 'Unpacking to: %s' % dest), 552 sys.stdout.flush() 553 path = self.Unpack(tarfile, dest) 554 os.remove(tarfile) 555 os.rmdir(tmpdir) 556 print 557 558 # Check that the toolchain works 559 print col.Color(col.GREEN, 'Testing') 560 dirpath = os.path.join(dest, path) 561 compiler_fname_list = self.ScanPath(dirpath, True) 562 if not compiler_fname_list: 563 print 'Could not locate C compiler - fetch failed.' 564 return 1 565 if len(compiler_fname_list) != 1: 566 print col.Color(col.RED, 'Warning, ambiguous toolchains: %s' % 567 ', '.join(compiler_fname_list)) 568 toolchain = Toolchain(compiler_fname_list[0], True, True) 569 570 # Make sure that it will be found by buildman 571 if not self.TestSettingsHasPath(dirpath): 572 print ("Adding 'download' to config file '%s'" % 573 bsettings.config_fname) 574 bsettings.SetItem('toolchain', 'download', '%s/*/*' % dest) 575 return 0 576
