1# Copyright (c) 2012 The Chromium OS Authors. 2# 3# SPDX-License-Identifier: GPL-2.0+ 4# 5 6import re 7import glob 8from HTMLParser import HTMLParser 9import os 10import sys 11import tempfile 12import urllib2 13 14import bsettings 15import command 16 17PRIORITY_CALC = 0 18 19# Simple class to collect links from a page 20class MyHTMLParser(HTMLParser): 21 def __init__(self, arch): 22 """Create a new parser 23 24 After the parser runs, self.links will be set to a list of the links 25 to .xz archives found in the page, and self.arch_link will be set to 26 the one for the given architecture (or None if not found). 27 28 Args: 29 arch: Architecture to search for 30 """ 31 HTMLParser.__init__(self) 32 self.arch_link = None 33 self.links = [] 34 self._match = '_%s-' % arch 35 36 def handle_starttag(self, tag, attrs): 37 if tag == 'a': 38 for tag, value in attrs: 39 if tag == 'href': 40 if value and value.endswith('.xz'): 41 self.links.append(value) 42 if self._match in value: 43 self.arch_link = value 44 45 46class Toolchain: 47 """A single toolchain 48 49 Public members: 50 gcc: Full path to C compiler 51 path: Directory path containing C compiler 52 cross: Cross compile string, e.g. 'arm-linux-' 53 arch: Architecture of toolchain as determined from the first 54 component of the filename. E.g. arm-linux-gcc becomes arm 55 priority: Toolchain priority (0=highest, 20=lowest) 56 """ 57 def __init__(self, fname, test, verbose=False, priority=PRIORITY_CALC): 58 """Create a new toolchain object. 59 60 Args: 61 fname: Filename of the gcc component 62 test: True to run the toolchain to test it 63 verbose: True to print out the information 64 priority: Priority to use for this toolchain, or PRIORITY_CALC to 65 calculate it 66 """ 67 self.gcc = fname 68 self.path = os.path.dirname(fname) 69 70 # Find the CROSS_COMPILE prefix to use for U-Boot. For example, 71 # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'. 72 basename = os.path.basename(fname) 73 pos = basename.rfind('-') 74 self.cross = basename[:pos + 1] if pos != -1 else '' 75 76 # The architecture is the first part of the name 77 pos = self.cross.find('-') 78 self.arch = self.cross[:pos] if pos != -1 else 'sandbox' 79 80 env = self.MakeEnvironment(False) 81 82 # As a basic sanity check, run the C compiler with --version 83 cmd = [fname, '--version'] 84 if priority == PRIORITY_CALC: 85 self.priority = self.GetPriority(fname) 86 else: 87 self.priority = priority 88 if test: 89 result = command.RunPipe([cmd], capture=True, env=env, 90 raise_on_error=False) 91 self.ok = result.return_code == 0 92 if verbose: 93 print 'Tool chain test: ', 94 if self.ok: 95 print 'OK, priority %d' % self.priority 96 else: 97 print 'BAD' 98 print 'Command: ', cmd 99 print result.stdout 100 print result.stderr 101 else: 102 self.ok = True 103 104 def GetPriority(self, fname): 105 """Return the priority of the toolchain. 106 107 Toolchains are ranked according to their suitability by their 108 filename prefix. 109 110 Args: 111 fname: Filename of toolchain 112 Returns: 113 Priority of toolchain, PRIORITY_CALC=highest, 20=lowest. 114 """ 115 priority_list = ['-elf', '-unknown-linux-gnu', '-linux', 116 '-none-linux-gnueabi', '-uclinux', '-none-eabi', 117 '-gentoo-linux-gnu', '-linux-gnueabi', '-le-linux', '-uclinux'] 118 for prio in range(len(priority_list)): 119 if priority_list[prio] in fname: 120 return PRIORITY_CALC + prio 121 return PRIORITY_CALC + prio 122 123 def MakeEnvironment(self, full_path): 124 """Returns an environment for using the toolchain. 125 126 Thie takes the current environment and adds CROSS_COMPILE so that 127 the tool chain will operate correctly. 128 129 Args: 130 full_path: Return the full path in CROSS_COMPILE and don't set 131 PATH 132 """ 133 env = dict(os.environ) 134 if full_path: 135 env['CROSS_COMPILE'] = os.path.join(self.path, self.cross) 136 else: 137 env['CROSS_COMPILE'] = self.cross 138 env['PATH'] = self.path + ':' + env['PATH'] 139 140 return env 141 142 143class Toolchains: 144 """Manage a list of toolchains for building U-Boot 145 146 We select one toolchain for each architecture type 147 148 Public members: 149 toolchains: Dict of Toolchain objects, keyed by architecture name 150 paths: List of paths to check for toolchains (may contain wildcards) 151 """ 152 153 def __init__(self): 154 self.toolchains = {} 155 self.paths = [] 156 self._make_flags = dict(bsettings.GetItems('make-flags')) 157 158 def GetPathList(self): 159 """Get a list of available toolchain paths 160 161 Returns: 162 List of strings, each a path to a toolchain mentioned in the 163 [toolchain] section of the settings file. 164 """ 165 toolchains = bsettings.GetItems('toolchain') 166 if not toolchains: 167 print ('Warning: No tool chains - please add a [toolchain] section' 168 ' to your buildman config file %s. See README for details' % 169 bsettings.config_fname) 170 171 paths = [] 172 for name, value in toolchains: 173 if '*' in value: 174 paths += glob.glob(value) 175 else: 176 paths.append(value) 177 return paths 178 179 def GetSettings(self): 180 self.paths += self.GetPathList() 181 182 def Add(self, fname, test=True, verbose=False, priority=PRIORITY_CALC): 183 """Add a toolchain to our list 184 185 We select the given toolchain as our preferred one for its 186 architecture if it is a higher priority than the others. 187 188 Args: 189 fname: Filename of toolchain's gcc driver 190 test: True to run the toolchain to test it 191 priority: Priority to use for this toolchain 192 """ 193 toolchain = Toolchain(fname, test, verbose, priority) 194 add_it = toolchain.ok 195 if toolchain.arch in self.toolchains: 196 add_it = (toolchain.priority < 197 self.toolchains[toolchain.arch].priority) 198 if add_it: 199 self.toolchains[toolchain.arch] = toolchain 200 elif verbose: 201 print ("Toolchain '%s' at priority %d will be ignored because " 202 "another toolchain for arch '%s' has priority %d" % 203 (toolchain.gcc, toolchain.priority, toolchain.arch, 204 self.toolchains[toolchain.arch].priority)) 205 206 def ScanPath(self, path, verbose): 207 """Scan a path for a valid toolchain 208 209 Args: 210 path: Path to scan 211 verbose: True to print out progress information 212 Returns: 213 Filename of C compiler if found, else None 214 """ 215 fnames = [] 216 for subdir in ['.', 'bin', 'usr/bin']: 217 dirname = os.path.join(path, subdir) 218 if verbose: print " - looking in '%s'" % dirname 219 for fname in glob.glob(dirname + '/*gcc'): 220 if verbose: print " - found '%s'" % fname 221 fnames.append(fname) 222 return fnames 223 224 225 def Scan(self, verbose): 226 """Scan for available toolchains and select the best for each arch. 227 228 We look for all the toolchains we can file, figure out the 229 architecture for each, and whether it works. Then we select the 230 highest priority toolchain for each arch. 231 232 Args: 233 verbose: True to print out progress information 234 """ 235 if verbose: print 'Scanning for tool chains' 236 for path in self.paths: 237 if verbose: print " - scanning path '%s'" % path 238 fnames = self.ScanPath(path, verbose) 239 for fname in fnames: 240 self.Add(fname, True, verbose) 241 242 def List(self): 243 """List out the selected toolchains for each architecture""" 244 print 'List of available toolchains (%d):' % len(self.toolchains) 245 if len(self.toolchains): 246 for key, value in sorted(self.toolchains.iteritems()): 247 print '%-10s: %s' % (key, value.gcc) 248 else: 249 print 'None' 250 251 def Select(self, arch): 252 """Returns the toolchain for a given architecture 253 254 Args: 255 args: Name of architecture (e.g. 'arm', 'ppc_8xx') 256 257 returns: 258 toolchain object, or None if none found 259 """ 260 for tag, value in bsettings.GetItems('toolchain-alias'): 261 if arch == tag: 262 for alias in value.split(): 263 if alias in self.toolchains: 264 return self.toolchains[alias] 265 266 if not arch in self.toolchains: 267 raise ValueError, ("No tool chain found for arch '%s'" % arch) 268 return self.toolchains[arch] 269 270 def ResolveReferences(self, var_dict, args): 271 """Resolve variable references in a string 272 273 This converts ${blah} within the string to the value of blah. 274 This function works recursively. 275 276 Args: 277 var_dict: Dictionary containing variables and their values 278 args: String containing make arguments 279 Returns: 280 Resolved string 281 282 >>> bsettings.Setup() 283 >>> tcs = Toolchains() 284 >>> tcs.Add('fred', False) 285 >>> var_dict = {'oblique' : 'OBLIQUE', 'first' : 'fi${second}rst', \ 286 'second' : '2nd'} 287 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set') 288 'this=OBLIQUE_set' 289 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set${first}nd') 290 'this=OBLIQUE_setfi2ndrstnd' 291 """ 292 re_var = re.compile('(\$\{[-_a-z0-9A-Z]{1,}\})') 293 294 while True: 295 m = re_var.search(args) 296 if not m: 297 break 298 lookup = m.group(0)[2:-1] 299 value = var_dict.get(lookup, '') 300 args = args[:m.start(0)] + value + args[m.end(0):] 301 return args 302 303 def GetMakeArguments(self, board): 304 """Returns 'make' arguments for a given board 305 306 The flags are in a section called 'make-flags'. Flags are named 307 after the target they represent, for example snapper9260=TESTING=1 308 will pass TESTING=1 to make when building the snapper9260 board. 309 310 References to other boards can be added in the string also. For 311 example: 312 313 [make-flags] 314 at91-boards=ENABLE_AT91_TEST=1 315 snapper9260=${at91-boards} BUILD_TAG=442 316 snapper9g45=${at91-boards} BUILD_TAG=443 317 318 This will return 'ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260 319 and 'ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45. 320 321 A special 'target' variable is set to the board target. 322 323 Args: 324 board: Board object for the board to check. 325 Returns: 326 'make' flags for that board, or '' if none 327 """ 328 self._make_flags['target'] = board.target 329 arg_str = self.ResolveReferences(self._make_flags, 330 self._make_flags.get(board.target, '')) 331 args = arg_str.split(' ') 332 i = 0 333 while i < len(args): 334 if not args[i]: 335 del args[i] 336 else: 337 i += 1 338 return args 339 340 def LocateArchUrl(self, fetch_arch): 341 """Find a toolchain available online 342 343 Look in standard places for available toolchains. At present the 344 only standard place is at kernel.org. 345 346 Args: 347 arch: Architecture to look for, or 'list' for all 348 Returns: 349 If fetch_arch is 'list', a tuple: 350 Machine architecture (e.g. x86_64) 351 List of toolchains 352 else 353 URL containing this toolchain, if avaialble, else None 354 """ 355 arch = command.OutputOneLine('uname', '-m') 356 base = 'https://www.kernel.org/pub/tools/crosstool/files/bin' 357 versions = ['4.9.0', '4.6.3', '4.6.2', '4.5.1', '4.2.4'] 358 links = [] 359 for version in versions: 360 url = '%s/%s/%s/' % (base, arch, version) 361 print 'Checking: %s' % url 362 response = urllib2.urlopen(url) 363 html = response.read() 364 parser = MyHTMLParser(fetch_arch) 365 parser.feed(html) 366 if fetch_arch == 'list': 367 links += parser.links 368 elif parser.arch_link: 369 return url + parser.arch_link 370 if fetch_arch == 'list': 371 return arch, links 372 return None 373 374 def Download(self, url): 375 """Download a file to a temporary directory 376 377 Args: 378 url: URL to download 379 Returns: 380 Tuple: 381 Temporary directory name 382 Full path to the downloaded archive file in that directory, 383 or None if there was an error while downloading 384 """ 385 print 'Downloading: %s' % url 386 leaf = url.split('/')[-1] 387 tmpdir = tempfile.mkdtemp('.buildman') 388 response = urllib2.urlopen(url) 389 fname = os.path.join(tmpdir, leaf) 390 fd = open(fname, 'wb') 391 meta = response.info() 392 size = int(meta.getheaders('Content-Length')[0]) 393 done = 0 394 block_size = 1 << 16 395 status = '' 396 397 # Read the file in chunks and show progress as we go 398 while True: 399 buffer = response.read(block_size) 400 if not buffer: 401 print chr(8) * (len(status) + 1), '\r', 402 break 403 404 done += len(buffer) 405 fd.write(buffer) 406 status = r'%10d MiB [%3d%%]' % (done / 1024 / 1024, 407 done * 100 / size) 408 status = status + chr(8) * (len(status) + 1) 409 print status, 410 sys.stdout.flush() 411 fd.close() 412 if done != size: 413 print 'Error, failed to download' 414 os.remove(fname) 415 fname = None 416 return tmpdir, fname 417 418 def Unpack(self, fname, dest): 419 """Unpack a tar file 420 421 Args: 422 fname: Filename to unpack 423 dest: Destination directory 424 Returns: 425 Directory name of the first entry in the archive, without the 426 trailing / 427 """ 428 stdout = command.Output('tar', 'xvfJ', fname, '-C', dest) 429 return stdout.splitlines()[0][:-1] 430 431 def TestSettingsHasPath(self, path): 432 """Check if builmand will find this toolchain 433 434 Returns: 435 True if the path is in settings, False if not 436 """ 437 paths = self.GetPathList() 438 return path in paths 439 440 def ListArchs(self): 441 """List architectures with available toolchains to download""" 442 host_arch, archives = self.LocateArchUrl('list') 443 re_arch = re.compile('[-a-z0-9.]*_([^-]*)-.*') 444 arch_set = set() 445 for archive in archives: 446 # Remove the host architecture from the start 447 arch = re_arch.match(archive[len(host_arch):]) 448 if arch: 449 arch_set.add(arch.group(1)) 450 return sorted(arch_set) 451 452 def FetchAndInstall(self, arch): 453 """Fetch and install a new toolchain 454 455 arch: 456 Architecture to fetch, or 'list' to list 457 """ 458 # Fist get the URL for this architecture 459 url = self.LocateArchUrl(arch) 460 if not url: 461 print ("Cannot find toolchain for arch '%s' - use 'list' to list" % 462 arch) 463 return 2 464 home = os.environ['HOME'] 465 dest = os.path.join(home, '.buildman-toolchains') 466 if not os.path.exists(dest): 467 os.mkdir(dest) 468 469 # Download the tar file for this toolchain and unpack it 470 tmpdir, tarfile = self.Download(url) 471 if not tarfile: 472 return 1 473 print 'Unpacking to: %s' % dest, 474 sys.stdout.flush() 475 path = self.Unpack(tarfile, dest) 476 os.remove(tarfile) 477 os.rmdir(tmpdir) 478 print 479 480 # Check that the toolchain works 481 print 'Testing' 482 dirpath = os.path.join(dest, path) 483 compiler_fname_list = self.ScanPath(dirpath, True) 484 if not compiler_fname_list: 485 print 'Could not locate C compiler - fetch failed.' 486 return 1 487 if len(compiler_fname_list) != 1: 488 print ('Internal error, ambiguous toolchains: %s' % 489 (', '.join(compiler_fname))) 490 return 1 491 toolchain = Toolchain(compiler_fname_list[0], True, True) 492 493 # Make sure that it will be found by buildman 494 if not self.TestSettingsHasPath(dirpath): 495 print ("Adding 'download' to config file '%s'" % 496 bsettings.config_fname) 497 tools_dir = os.path.dirname(dirpath) 498 bsettings.SetItem('toolchain', 'download', '%s/*' % tools_dir) 499 return 0 500