1# Copyright (c) 2012 The Chromium OS Authors. 2# 3# SPDX-License-Identifier: GPL-2.0+ 4# 5 6import re 7import glob 8from HTMLParser import HTMLParser 9import os 10import sys 11import tempfile 12import urllib2 13 14import bsettings 15import command 16 17(PRIORITY_FULL_PREFIX, PRIORITY_PREFIX_GCC, PRIORITY_PREFIX_GCC_PATH, 18 PRIORITY_CALC) = range(4) 19 20# Simple class to collect links from a page 21class MyHTMLParser(HTMLParser): 22 def __init__(self, arch): 23 """Create a new parser 24 25 After the parser runs, self.links will be set to a list of the links 26 to .xz archives found in the page, and self.arch_link will be set to 27 the one for the given architecture (or None if not found). 28 29 Args: 30 arch: Architecture to search for 31 """ 32 HTMLParser.__init__(self) 33 self.arch_link = None 34 self.links = [] 35 self._match = '_%s-' % arch 36 37 def handle_starttag(self, tag, attrs): 38 if tag == 'a': 39 for tag, value in attrs: 40 if tag == 'href': 41 if value and value.endswith('.xz'): 42 self.links.append(value) 43 if self._match in value: 44 self.arch_link = value 45 46 47class Toolchain: 48 """A single toolchain 49 50 Public members: 51 gcc: Full path to C compiler 52 path: Directory path containing C compiler 53 cross: Cross compile string, e.g. 'arm-linux-' 54 arch: Architecture of toolchain as determined from the first 55 component of the filename. E.g. arm-linux-gcc becomes arm 56 priority: Toolchain priority (0=highest, 20=lowest) 57 """ 58 def __init__(self, fname, test, verbose=False, priority=PRIORITY_CALC, 59 arch=None): 60 """Create a new toolchain object. 61 62 Args: 63 fname: Filename of the gcc component 64 test: True to run the toolchain to test it 65 verbose: True to print out the information 66 priority: Priority to use for this toolchain, or PRIORITY_CALC to 67 calculate it 68 """ 69 self.gcc = fname 70 self.path = os.path.dirname(fname) 71 72 # Find the CROSS_COMPILE prefix to use for U-Boot. For example, 73 # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'. 74 basename = os.path.basename(fname) 75 pos = basename.rfind('-') 76 self.cross = basename[:pos + 1] if pos != -1 else '' 77 78 # The architecture is the first part of the name 79 pos = self.cross.find('-') 80 if arch: 81 self.arch = arch 82 else: 83 self.arch = self.cross[:pos] if pos != -1 else 'sandbox' 84 85 env = self.MakeEnvironment(False) 86 87 # As a basic sanity check, run the C compiler with --version 88 cmd = [fname, '--version'] 89 if priority == PRIORITY_CALC: 90 self.priority = self.GetPriority(fname) 91 else: 92 self.priority = priority 93 if test: 94 result = command.RunPipe([cmd], capture=True, env=env, 95 raise_on_error=False) 96 self.ok = result.return_code == 0 97 if verbose: 98 print 'Tool chain test: ', 99 if self.ok: 100 print "OK, arch='%s', priority %d" % (self.arch, 101 self.priority) 102 else: 103 print 'BAD' 104 print 'Command: ', cmd 105 print result.stdout 106 print result.stderr 107 else: 108 self.ok = True 109 110 def GetPriority(self, fname): 111 """Return the priority of the toolchain. 112 113 Toolchains are ranked according to their suitability by their 114 filename prefix. 115 116 Args: 117 fname: Filename of toolchain 118 Returns: 119 Priority of toolchain, PRIORITY_CALC=highest, 20=lowest. 120 """ 121 priority_list = ['-elf', '-unknown-linux-gnu', '-linux', 122 '-none-linux-gnueabi', '-uclinux', '-none-eabi', 123 '-gentoo-linux-gnu', '-linux-gnueabi', '-le-linux', '-uclinux'] 124 for prio in range(len(priority_list)): 125 if priority_list[prio] in fname: 126 return PRIORITY_CALC + prio 127 return PRIORITY_CALC + prio 128 129 def MakeEnvironment(self, full_path): 130 """Returns an environment for using the toolchain. 131 132 Thie takes the current environment and adds CROSS_COMPILE so that 133 the tool chain will operate correctly. 134 135 Args: 136 full_path: Return the full path in CROSS_COMPILE and don't set 137 PATH 138 """ 139 env = dict(os.environ) 140 if full_path: 141 env['CROSS_COMPILE'] = os.path.join(self.path, self.cross) 142 else: 143 env['CROSS_COMPILE'] = self.cross 144 env['PATH'] = self.path + ':' + env['PATH'] 145 146 return env 147 148 149class Toolchains: 150 """Manage a list of toolchains for building U-Boot 151 152 We select one toolchain for each architecture type 153 154 Public members: 155 toolchains: Dict of Toolchain objects, keyed by architecture name 156 prefixes: Dict of prefixes to check, keyed by architecture. This can 157 be a full path and toolchain prefix, for example 158 {'x86', 'opt/i386-linux/bin/i386-linux-'}, or the name of 159 something on the search path, for example 160 {'arm', 'arm-linux-gnueabihf-'}. Wildcards are not supported. 161 paths: List of paths to check for toolchains (may contain wildcards) 162 """ 163 164 def __init__(self): 165 self.toolchains = {} 166 self.prefixes = {} 167 self.paths = [] 168 self._make_flags = dict(bsettings.GetItems('make-flags')) 169 170 def GetPathList(self, show_warning=True): 171 """Get a list of available toolchain paths 172 173 Args: 174 show_warning: True to show a warning if there are no tool chains. 175 176 Returns: 177 List of strings, each a path to a toolchain mentioned in the 178 [toolchain] section of the settings file. 179 """ 180 toolchains = bsettings.GetItems('toolchain') 181 if show_warning and not toolchains: 182 print ('Warning: No tool chains - please add a [toolchain] section' 183 ' to your buildman config file %s. See README for details' % 184 bsettings.config_fname) 185 186 paths = [] 187 for name, value in toolchains: 188 if '*' in value: 189 paths += glob.glob(value) 190 else: 191 paths.append(value) 192 return paths 193 194 def GetSettings(self, show_warning=True): 195 """Get toolchain settings from the settings file. 196 197 Args: 198 show_warning: True to show a warning if there are no tool chains. 199 """ 200 self.prefixes = bsettings.GetItems('toolchain-prefix') 201 self.paths += self.GetPathList(show_warning) 202 203 def Add(self, fname, test=True, verbose=False, priority=PRIORITY_CALC, 204 arch=None): 205 """Add a toolchain to our list 206 207 We select the given toolchain as our preferred one for its 208 architecture if it is a higher priority than the others. 209 210 Args: 211 fname: Filename of toolchain's gcc driver 212 test: True to run the toolchain to test it 213 priority: Priority to use for this toolchain 214 arch: Toolchain architecture, or None if not known 215 """ 216 toolchain = Toolchain(fname, test, verbose, priority, arch) 217 add_it = toolchain.ok 218 if toolchain.arch in self.toolchains: 219 add_it = (toolchain.priority < 220 self.toolchains[toolchain.arch].priority) 221 if add_it: 222 self.toolchains[toolchain.arch] = toolchain 223 elif verbose: 224 print ("Toolchain '%s' at priority %d will be ignored because " 225 "another toolchain for arch '%s' has priority %d" % 226 (toolchain.gcc, toolchain.priority, toolchain.arch, 227 self.toolchains[toolchain.arch].priority)) 228 229 def ScanPath(self, path, verbose): 230 """Scan a path for a valid toolchain 231 232 Args: 233 path: Path to scan 234 verbose: True to print out progress information 235 Returns: 236 Filename of C compiler if found, else None 237 """ 238 fnames = [] 239 for subdir in ['.', 'bin', 'usr/bin']: 240 dirname = os.path.join(path, subdir) 241 if verbose: print " - looking in '%s'" % dirname 242 for fname in glob.glob(dirname + '/*gcc'): 243 if verbose: print " - found '%s'" % fname 244 fnames.append(fname) 245 return fnames 246 247 def ScanPathEnv(self, fname): 248 """Scan the PATH environment variable for a given filename. 249 250 Args: 251 fname: Filename to scan for 252 Returns: 253 List of matching pathanames, or [] if none 254 """ 255 pathname_list = [] 256 for path in os.environ["PATH"].split(os.pathsep): 257 path = path.strip('"') 258 pathname = os.path.join(path, fname) 259 if os.path.exists(pathname): 260 pathname_list.append(pathname) 261 return pathname_list 262 263 def Scan(self, verbose): 264 """Scan for available toolchains and select the best for each arch. 265 266 We look for all the toolchains we can file, figure out the 267 architecture for each, and whether it works. Then we select the 268 highest priority toolchain for each arch. 269 270 Args: 271 verbose: True to print out progress information 272 """ 273 if verbose: print 'Scanning for tool chains' 274 for name, value in self.prefixes: 275 if verbose: print " - scanning prefix '%s'" % value 276 if os.path.exists(value): 277 self.Add(value, True, verbose, PRIORITY_FULL_PREFIX, name) 278 continue 279 fname = value + 'gcc' 280 if os.path.exists(fname): 281 self.Add(fname, True, verbose, PRIORITY_PREFIX_GCC, name) 282 continue 283 fname_list = self.ScanPathEnv(fname) 284 for f in fname_list: 285 self.Add(f, True, verbose, PRIORITY_PREFIX_GCC_PATH, name) 286 if not fname_list: 287 raise ValueError, ("No tool chain found for prefix '%s'" % 288 value) 289 for path in self.paths: 290 if verbose: print " - scanning path '%s'" % path 291 fnames = self.ScanPath(path, verbose) 292 for fname in fnames: 293 self.Add(fname, True, verbose) 294 295 def List(self): 296 """List out the selected toolchains for each architecture""" 297 print 'List of available toolchains (%d):' % len(self.toolchains) 298 if len(self.toolchains): 299 for key, value in sorted(self.toolchains.iteritems()): 300 print '%-10s: %s' % (key, value.gcc) 301 else: 302 print 'None' 303 304 def Select(self, arch): 305 """Returns the toolchain for a given architecture 306 307 Args: 308 args: Name of architecture (e.g. 'arm', 'ppc_8xx') 309 310 returns: 311 toolchain object, or None if none found 312 """ 313 for tag, value in bsettings.GetItems('toolchain-alias'): 314 if arch == tag: 315 for alias in value.split(): 316 if alias in self.toolchains: 317 return self.toolchains[alias] 318 319 if not arch in self.toolchains: 320 raise ValueError, ("No tool chain found for arch '%s'" % arch) 321 return self.toolchains[arch] 322 323 def ResolveReferences(self, var_dict, args): 324 """Resolve variable references in a string 325 326 This converts ${blah} within the string to the value of blah. 327 This function works recursively. 328 329 Args: 330 var_dict: Dictionary containing variables and their values 331 args: String containing make arguments 332 Returns: 333 Resolved string 334 335 >>> bsettings.Setup() 336 >>> tcs = Toolchains() 337 >>> tcs.Add('fred', False) 338 >>> var_dict = {'oblique' : 'OBLIQUE', 'first' : 'fi${second}rst', \ 339 'second' : '2nd'} 340 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set') 341 'this=OBLIQUE_set' 342 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set${first}nd') 343 'this=OBLIQUE_setfi2ndrstnd' 344 """ 345 re_var = re.compile('(\$\{[-_a-z0-9A-Z]{1,}\})') 346 347 while True: 348 m = re_var.search(args) 349 if not m: 350 break 351 lookup = m.group(0)[2:-1] 352 value = var_dict.get(lookup, '') 353 args = args[:m.start(0)] + value + args[m.end(0):] 354 return args 355 356 def GetMakeArguments(self, board): 357 """Returns 'make' arguments for a given board 358 359 The flags are in a section called 'make-flags'. Flags are named 360 after the target they represent, for example snapper9260=TESTING=1 361 will pass TESTING=1 to make when building the snapper9260 board. 362 363 References to other boards can be added in the string also. For 364 example: 365 366 [make-flags] 367 at91-boards=ENABLE_AT91_TEST=1 368 snapper9260=${at91-boards} BUILD_TAG=442 369 snapper9g45=${at91-boards} BUILD_TAG=443 370 371 This will return 'ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260 372 and 'ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45. 373 374 A special 'target' variable is set to the board target. 375 376 Args: 377 board: Board object for the board to check. 378 Returns: 379 'make' flags for that board, or '' if none 380 """ 381 self._make_flags['target'] = board.target 382 arg_str = self.ResolveReferences(self._make_flags, 383 self._make_flags.get(board.target, '')) 384 args = arg_str.split(' ') 385 i = 0 386 while i < len(args): 387 if not args[i]: 388 del args[i] 389 else: 390 i += 1 391 return args 392 393 def LocateArchUrl(self, fetch_arch): 394 """Find a toolchain available online 395 396 Look in standard places for available toolchains. At present the 397 only standard place is at kernel.org. 398 399 Args: 400 arch: Architecture to look for, or 'list' for all 401 Returns: 402 If fetch_arch is 'list', a tuple: 403 Machine architecture (e.g. x86_64) 404 List of toolchains 405 else 406 URL containing this toolchain, if avaialble, else None 407 """ 408 arch = command.OutputOneLine('uname', '-m') 409 base = 'https://www.kernel.org/pub/tools/crosstool/files/bin' 410 versions = ['4.9.0', '4.6.3', '4.6.2', '4.5.1', '4.2.4'] 411 links = [] 412 for version in versions: 413 url = '%s/%s/%s/' % (base, arch, version) 414 print 'Checking: %s' % url 415 response = urllib2.urlopen(url) 416 html = response.read() 417 parser = MyHTMLParser(fetch_arch) 418 parser.feed(html) 419 if fetch_arch == 'list': 420 links += parser.links 421 elif parser.arch_link: 422 return url + parser.arch_link 423 if fetch_arch == 'list': 424 return arch, links 425 return None 426 427 def Download(self, url): 428 """Download a file to a temporary directory 429 430 Args: 431 url: URL to download 432 Returns: 433 Tuple: 434 Temporary directory name 435 Full path to the downloaded archive file in that directory, 436 or None if there was an error while downloading 437 """ 438 print 'Downloading: %s' % url 439 leaf = url.split('/')[-1] 440 tmpdir = tempfile.mkdtemp('.buildman') 441 response = urllib2.urlopen(url) 442 fname = os.path.join(tmpdir, leaf) 443 fd = open(fname, 'wb') 444 meta = response.info() 445 size = int(meta.getheaders('Content-Length')[0]) 446 done = 0 447 block_size = 1 << 16 448 status = '' 449 450 # Read the file in chunks and show progress as we go 451 while True: 452 buffer = response.read(block_size) 453 if not buffer: 454 print chr(8) * (len(status) + 1), '\r', 455 break 456 457 done += len(buffer) 458 fd.write(buffer) 459 status = r'%10d MiB [%3d%%]' % (done / 1024 / 1024, 460 done * 100 / size) 461 status = status + chr(8) * (len(status) + 1) 462 print status, 463 sys.stdout.flush() 464 fd.close() 465 if done != size: 466 print 'Error, failed to download' 467 os.remove(fname) 468 fname = None 469 return tmpdir, fname 470 471 def Unpack(self, fname, dest): 472 """Unpack a tar file 473 474 Args: 475 fname: Filename to unpack 476 dest: Destination directory 477 Returns: 478 Directory name of the first entry in the archive, without the 479 trailing / 480 """ 481 stdout = command.Output('tar', 'xvfJ', fname, '-C', dest) 482 return stdout.splitlines()[0][:-1] 483 484 def TestSettingsHasPath(self, path): 485 """Check if builmand will find this toolchain 486 487 Returns: 488 True if the path is in settings, False if not 489 """ 490 paths = self.GetPathList(False) 491 return path in paths 492 493 def ListArchs(self): 494 """List architectures with available toolchains to download""" 495 host_arch, archives = self.LocateArchUrl('list') 496 re_arch = re.compile('[-a-z0-9.]*_([^-]*)-.*') 497 arch_set = set() 498 for archive in archives: 499 # Remove the host architecture from the start 500 arch = re_arch.match(archive[len(host_arch):]) 501 if arch: 502 arch_set.add(arch.group(1)) 503 return sorted(arch_set) 504 505 def FetchAndInstall(self, arch): 506 """Fetch and install a new toolchain 507 508 arch: 509 Architecture to fetch, or 'list' to list 510 """ 511 # Fist get the URL for this architecture 512 url = self.LocateArchUrl(arch) 513 if not url: 514 print ("Cannot find toolchain for arch '%s' - use 'list' to list" % 515 arch) 516 return 2 517 home = os.environ['HOME'] 518 dest = os.path.join(home, '.buildman-toolchains') 519 if not os.path.exists(dest): 520 os.mkdir(dest) 521 522 # Download the tar file for this toolchain and unpack it 523 tmpdir, tarfile = self.Download(url) 524 if not tarfile: 525 return 1 526 print 'Unpacking to: %s' % dest, 527 sys.stdout.flush() 528 path = self.Unpack(tarfile, dest) 529 os.remove(tarfile) 530 os.rmdir(tmpdir) 531 print 532 533 # Check that the toolchain works 534 print 'Testing' 535 dirpath = os.path.join(dest, path) 536 compiler_fname_list = self.ScanPath(dirpath, True) 537 if not compiler_fname_list: 538 print 'Could not locate C compiler - fetch failed.' 539 return 1 540 if len(compiler_fname_list) != 1: 541 print ('Internal error, ambiguous toolchains: %s' % 542 (', '.join(compiler_fname))) 543 return 1 544 toolchain = Toolchain(compiler_fname_list[0], True, True) 545 546 # Make sure that it will be found by buildman 547 if not self.TestSettingsHasPath(dirpath): 548 print ("Adding 'download' to config file '%s'" % 549 bsettings.config_fname) 550 tools_dir = os.path.dirname(dirpath) 551 bsettings.SetItem('toolchain', 'download', '%s/*' % tools_dir) 552 return 0 553