1# Copyright (c) 2012 The Chromium OS Authors. 2# 3# SPDX-License-Identifier: GPL-2.0+ 4# 5 6import re 7import glob 8from HTMLParser import HTMLParser 9import os 10import sys 11import tempfile 12import urllib2 13 14import bsettings 15import command 16 17# Simple class to collect links from a page 18class MyHTMLParser(HTMLParser): 19 def __init__(self, arch): 20 """Create a new parser 21 22 After the parser runs, self.links will be set to a list of the links 23 to .xz archives found in the page, and self.arch_link will be set to 24 the one for the given architecture (or None if not found). 25 26 Args: 27 arch: Architecture to search for 28 """ 29 HTMLParser.__init__(self) 30 self.arch_link = None 31 self.links = [] 32 self._match = '_%s-' % arch 33 34 def handle_starttag(self, tag, attrs): 35 if tag == 'a': 36 for tag, value in attrs: 37 if tag == 'href': 38 if value and value.endswith('.xz'): 39 self.links.append(value) 40 if self._match in value: 41 self.arch_link = value 42 43 44class Toolchain: 45 """A single toolchain 46 47 Public members: 48 gcc: Full path to C compiler 49 path: Directory path containing C compiler 50 cross: Cross compile string, e.g. 'arm-linux-' 51 arch: Architecture of toolchain as determined from the first 52 component of the filename. E.g. arm-linux-gcc becomes arm 53 """ 54 def __init__(self, fname, test, verbose=False): 55 """Create a new toolchain object. 56 57 Args: 58 fname: Filename of the gcc component 59 test: True to run the toolchain to test it 60 verbose: True to print out the information 61 """ 62 self.gcc = fname 63 self.path = os.path.dirname(fname) 64 65 # Find the CROSS_COMPILE prefix to use for U-Boot. For example, 66 # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'. 67 basename = os.path.basename(fname) 68 pos = basename.rfind('-') 69 self.cross = basename[:pos + 1] if pos != -1 else '' 70 71 # The architecture is the first part of the name 72 pos = self.cross.find('-') 73 self.arch = self.cross[:pos] if pos != -1 else 'sandbox' 74 75 env = self.MakeEnvironment(False) 76 77 # As a basic sanity check, run the C compiler with --version 78 cmd = [fname, '--version'] 79 if test: 80 result = command.RunPipe([cmd], capture=True, env=env, 81 raise_on_error=False) 82 self.ok = result.return_code == 0 83 if verbose: 84 print 'Tool chain test: ', 85 if self.ok: 86 print 'OK' 87 else: 88 print 'BAD' 89 print 'Command: ', cmd 90 print result.stdout 91 print result.stderr 92 else: 93 self.ok = True 94 self.priority = self.GetPriority(fname) 95 96 def GetPriority(self, fname): 97 """Return the priority of the toolchain. 98 99 Toolchains are ranked according to their suitability by their 100 filename prefix. 101 102 Args: 103 fname: Filename of toolchain 104 Returns: 105 Priority of toolchain, 0=highest, 20=lowest. 106 """ 107 priority_list = ['-elf', '-unknown-linux-gnu', '-linux', 108 '-none-linux-gnueabi', '-uclinux', '-none-eabi', 109 '-gentoo-linux-gnu', '-linux-gnueabi', '-le-linux', '-uclinux'] 110 for prio in range(len(priority_list)): 111 if priority_list[prio] in fname: 112 return prio 113 return prio 114 115 def MakeEnvironment(self, full_path): 116 """Returns an environment for using the toolchain. 117 118 Thie takes the current environment and adds CROSS_COMPILE so that 119 the tool chain will operate correctly. 120 121 Args: 122 full_path: Return the full path in CROSS_COMPILE and don't set 123 PATH 124 """ 125 env = dict(os.environ) 126 if full_path: 127 env['CROSS_COMPILE'] = os.path.join(self.path, self.cross) 128 else: 129 env['CROSS_COMPILE'] = self.cross 130 env['PATH'] = self.path + ':' + env['PATH'] 131 132 return env 133 134 135class Toolchains: 136 """Manage a list of toolchains for building U-Boot 137 138 We select one toolchain for each architecture type 139 140 Public members: 141 toolchains: Dict of Toolchain objects, keyed by architecture name 142 paths: List of paths to check for toolchains (may contain wildcards) 143 """ 144 145 def __init__(self): 146 self.toolchains = {} 147 self.paths = [] 148 self._make_flags = dict(bsettings.GetItems('make-flags')) 149 150 def GetPathList(self): 151 """Get a list of available toolchain paths 152 153 Returns: 154 List of strings, each a path to a toolchain mentioned in the 155 [toolchain] section of the settings file. 156 """ 157 toolchains = bsettings.GetItems('toolchain') 158 if not toolchains: 159 print ('Warning: No tool chains - please add a [toolchain] section' 160 ' to your buildman config file %s. See README for details' % 161 bsettings.config_fname) 162 163 paths = [] 164 for name, value in toolchains: 165 if '*' in value: 166 paths += glob.glob(value) 167 else: 168 paths.append(value) 169 return paths 170 171 def GetSettings(self): 172 self.paths += self.GetPathList() 173 174 def Add(self, fname, test=True, verbose=False): 175 """Add a toolchain to our list 176 177 We select the given toolchain as our preferred one for its 178 architecture if it is a higher priority than the others. 179 180 Args: 181 fname: Filename of toolchain's gcc driver 182 test: True to run the toolchain to test it 183 """ 184 toolchain = Toolchain(fname, test, verbose) 185 add_it = toolchain.ok 186 if toolchain.arch in self.toolchains: 187 add_it = (toolchain.priority < 188 self.toolchains[toolchain.arch].priority) 189 if add_it: 190 self.toolchains[toolchain.arch] = toolchain 191 192 def ScanPath(self, path, verbose): 193 """Scan a path for a valid toolchain 194 195 Args: 196 path: Path to scan 197 verbose: True to print out progress information 198 Returns: 199 Filename of C compiler if found, else None 200 """ 201 fnames = [] 202 for subdir in ['.', 'bin', 'usr/bin']: 203 dirname = os.path.join(path, subdir) 204 if verbose: print " - looking in '%s'" % dirname 205 for fname in glob.glob(dirname + '/*gcc'): 206 if verbose: print " - found '%s'" % fname 207 fnames.append(fname) 208 return fnames 209 210 211 def Scan(self, verbose): 212 """Scan for available toolchains and select the best for each arch. 213 214 We look for all the toolchains we can file, figure out the 215 architecture for each, and whether it works. Then we select the 216 highest priority toolchain for each arch. 217 218 Args: 219 verbose: True to print out progress information 220 """ 221 if verbose: print 'Scanning for tool chains' 222 for path in self.paths: 223 if verbose: print " - scanning path '%s'" % path 224 fnames = self.ScanPath(path, verbose) 225 for fname in fnames: 226 self.Add(fname, True, verbose) 227 228 def List(self): 229 """List out the selected toolchains for each architecture""" 230 print 'List of available toolchains (%d):' % len(self.toolchains) 231 if len(self.toolchains): 232 for key, value in sorted(self.toolchains.iteritems()): 233 print '%-10s: %s' % (key, value.gcc) 234 else: 235 print 'None' 236 237 def Select(self, arch): 238 """Returns the toolchain for a given architecture 239 240 Args: 241 args: Name of architecture (e.g. 'arm', 'ppc_8xx') 242 243 returns: 244 toolchain object, or None if none found 245 """ 246 for tag, value in bsettings.GetItems('toolchain-alias'): 247 if arch == tag: 248 for alias in value.split(): 249 if alias in self.toolchains: 250 return self.toolchains[alias] 251 252 if not arch in self.toolchains: 253 raise ValueError, ("No tool chain found for arch '%s'" % arch) 254 return self.toolchains[arch] 255 256 def ResolveReferences(self, var_dict, args): 257 """Resolve variable references in a string 258 259 This converts ${blah} within the string to the value of blah. 260 This function works recursively. 261 262 Args: 263 var_dict: Dictionary containing variables and their values 264 args: String containing make arguments 265 Returns: 266 Resolved string 267 268 >>> bsettings.Setup() 269 >>> tcs = Toolchains() 270 >>> tcs.Add('fred', False) 271 >>> var_dict = {'oblique' : 'OBLIQUE', 'first' : 'fi${second}rst', \ 272 'second' : '2nd'} 273 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set') 274 'this=OBLIQUE_set' 275 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set${first}nd') 276 'this=OBLIQUE_setfi2ndrstnd' 277 """ 278 re_var = re.compile('(\$\{[-_a-z0-9A-Z]{1,}\})') 279 280 while True: 281 m = re_var.search(args) 282 if not m: 283 break 284 lookup = m.group(0)[2:-1] 285 value = var_dict.get(lookup, '') 286 args = args[:m.start(0)] + value + args[m.end(0):] 287 return args 288 289 def GetMakeArguments(self, board): 290 """Returns 'make' arguments for a given board 291 292 The flags are in a section called 'make-flags'. Flags are named 293 after the target they represent, for example snapper9260=TESTING=1 294 will pass TESTING=1 to make when building the snapper9260 board. 295 296 References to other boards can be added in the string also. For 297 example: 298 299 [make-flags] 300 at91-boards=ENABLE_AT91_TEST=1 301 snapper9260=${at91-boards} BUILD_TAG=442 302 snapper9g45=${at91-boards} BUILD_TAG=443 303 304 This will return 'ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260 305 and 'ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45. 306 307 A special 'target' variable is set to the board target. 308 309 Args: 310 board: Board object for the board to check. 311 Returns: 312 'make' flags for that board, or '' if none 313 """ 314 self._make_flags['target'] = board.target 315 arg_str = self.ResolveReferences(self._make_flags, 316 self._make_flags.get(board.target, '')) 317 args = arg_str.split(' ') 318 i = 0 319 while i < len(args): 320 if not args[i]: 321 del args[i] 322 else: 323 i += 1 324 return args 325 326 def LocateArchUrl(self, fetch_arch): 327 """Find a toolchain available online 328 329 Look in standard places for available toolchains. At present the 330 only standard place is at kernel.org. 331 332 Args: 333 arch: Architecture to look for, or 'list' for all 334 Returns: 335 If fetch_arch is 'list', a tuple: 336 Machine architecture (e.g. x86_64) 337 List of toolchains 338 else 339 URL containing this toolchain, if avaialble, else None 340 """ 341 arch = command.OutputOneLine('uname', '-m') 342 base = 'https://www.kernel.org/pub/tools/crosstool/files/bin' 343 versions = ['4.9.0', '4.6.3', '4.6.2', '4.5.1', '4.2.4'] 344 links = [] 345 for version in versions: 346 url = '%s/%s/%s/' % (base, arch, version) 347 print 'Checking: %s' % url 348 response = urllib2.urlopen(url) 349 html = response.read() 350 parser = MyHTMLParser(fetch_arch) 351 parser.feed(html) 352 if fetch_arch == 'list': 353 links += parser.links 354 elif parser.arch_link: 355 return url + parser.arch_link 356 if fetch_arch == 'list': 357 return arch, links 358 return None 359 360 def Download(self, url): 361 """Download a file to a temporary directory 362 363 Args: 364 url: URL to download 365 Returns: 366 Tuple: 367 Temporary directory name 368 Full path to the downloaded archive file in that directory, 369 or None if there was an error while downloading 370 """ 371 print 'Downloading: %s' % url 372 leaf = url.split('/')[-1] 373 tmpdir = tempfile.mkdtemp('.buildman') 374 response = urllib2.urlopen(url) 375 fname = os.path.join(tmpdir, leaf) 376 fd = open(fname, 'wb') 377 meta = response.info() 378 size = int(meta.getheaders('Content-Length')[0]) 379 done = 0 380 block_size = 1 << 16 381 status = '' 382 383 # Read the file in chunks and show progress as we go 384 while True: 385 buffer = response.read(block_size) 386 if not buffer: 387 print chr(8) * (len(status) + 1), '\r', 388 break 389 390 done += len(buffer) 391 fd.write(buffer) 392 status = r'%10d MiB [%3d%%]' % (done / 1024 / 1024, 393 done * 100 / size) 394 status = status + chr(8) * (len(status) + 1) 395 print status, 396 sys.stdout.flush() 397 fd.close() 398 if done != size: 399 print 'Error, failed to download' 400 os.remove(fname) 401 fname = None 402 return tmpdir, fname 403 404 def Unpack(self, fname, dest): 405 """Unpack a tar file 406 407 Args: 408 fname: Filename to unpack 409 dest: Destination directory 410 Returns: 411 Directory name of the first entry in the archive, without the 412 trailing / 413 """ 414 stdout = command.Output('tar', 'xvfJ', fname, '-C', dest) 415 return stdout.splitlines()[0][:-1] 416 417 def TestSettingsHasPath(self, path): 418 """Check if builmand will find this toolchain 419 420 Returns: 421 True if the path is in settings, False if not 422 """ 423 paths = self.GetPathList() 424 return path in paths 425 426 def ListArchs(self): 427 """List architectures with available toolchains to download""" 428 host_arch, archives = self.LocateArchUrl('list') 429 re_arch = re.compile('[-a-z0-9.]*_([^-]*)-.*') 430 arch_set = set() 431 for archive in archives: 432 # Remove the host architecture from the start 433 arch = re_arch.match(archive[len(host_arch):]) 434 if arch: 435 arch_set.add(arch.group(1)) 436 return sorted(arch_set) 437 438 def FetchAndInstall(self, arch): 439 """Fetch and install a new toolchain 440 441 arch: 442 Architecture to fetch, or 'list' to list 443 """ 444 # Fist get the URL for this architecture 445 url = self.LocateArchUrl(arch) 446 if not url: 447 print ("Cannot find toolchain for arch '%s' - use 'list' to list" % 448 arch) 449 return 2 450 home = os.environ['HOME'] 451 dest = os.path.join(home, '.buildman-toolchains') 452 if not os.path.exists(dest): 453 os.mkdir(dest) 454 455 # Download the tar file for this toolchain and unpack it 456 tmpdir, tarfile = self.Download(url) 457 if not tarfile: 458 return 1 459 print 'Unpacking to: %s' % dest, 460 sys.stdout.flush() 461 path = self.Unpack(tarfile, dest) 462 os.remove(tarfile) 463 os.rmdir(tmpdir) 464 print 465 466 # Check that the toolchain works 467 print 'Testing' 468 dirpath = os.path.join(dest, path) 469 compiler_fname_list = self.ScanPath(dirpath, True) 470 if not compiler_fname_list: 471 print 'Could not locate C compiler - fetch failed.' 472 return 1 473 if len(compiler_fname_list) != 1: 474 print ('Internal error, ambiguous toolchains: %s' % 475 (', '.join(compiler_fname))) 476 return 1 477 toolchain = Toolchain(compiler_fname_list[0], True, True) 478 479 # Make sure that it will be found by buildman 480 if not self.TestSettingsHasPath(dirpath): 481 print ("Adding 'download' to config file '%s'" % 482 bsettings.config_fname) 483 tools_dir = os.path.dirname(dirpath) 484 bsettings.SetItem('toolchain', 'download', '%s/*' % tools_dir) 485 return 0 486