1# Copyright (c) 2013 The Chromium OS Authors. 2# 3# Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com> 4# 5# SPDX-License-Identifier: GPL-2.0+ 6# 7 8import collections 9import errno 10from datetime import datetime, timedelta 11import glob 12import os 13import re 14import Queue 15import shutil 16import string 17import sys 18import threading 19import time 20 21import command 22import gitutil 23import terminal 24import toolchain 25 26 27""" 28Theory of Operation 29 30Please see README for user documentation, and you should be familiar with 31that before trying to make sense of this. 32 33Buildman works by keeping the machine as busy as possible, building different 34commits for different boards on multiple CPUs at once. 35 36The source repo (self.git_dir) contains all the commits to be built. Each 37thread works on a single board at a time. It checks out the first commit, 38configures it for that board, then builds it. Then it checks out the next 39commit and builds it (typically without re-configuring). When it runs out 40of commits, it gets another job from the builder and starts again with that 41board. 42 43Clearly the builder threads could work either way - they could check out a 44commit and then built it for all boards. Using separate directories for each 45commit/board pair they could leave their build product around afterwards 46also. 47 48The intent behind building a single board for multiple commits, is to make 49use of incremental builds. Since each commit is built incrementally from 50the previous one, builds are faster. Reconfiguring for a different board 51removes all intermediate object files. 52 53Many threads can be working at once, but each has its own working directory. 54When a thread finishes a build, it puts the output files into a result 55directory. 56 57The base directory used by buildman is normally '../<branch>', i.e. 58a directory higher than the source repository and named after the branch 59being built. 60 61Within the base directory, we have one subdirectory for each commit. Within 62that is one subdirectory for each board. Within that is the build output for 63that commit/board combination. 64 65Buildman also create working directories for each thread, in a .bm-work/ 66subdirectory in the base dir. 67 68As an example, say we are building branch 'us-net' for boards 'sandbox' and 69'seaboard', and say that us-net has two commits. We will have directories 70like this: 71 72us-net/ base directory 73 01_of_02_g4ed4ebc_net--Add-tftp-speed-/ 74 sandbox/ 75 u-boot.bin 76 seaboard/ 77 u-boot.bin 78 02_of_02_g4ed4ebc_net--Check-tftp-comp/ 79 sandbox/ 80 u-boot.bin 81 seaboard/ 82 u-boot.bin 83 .bm-work/ 84 00/ working directory for thread 0 (contains source checkout) 85 build/ build output 86 01/ working directory for thread 1 87 build/ build output 88 ... 89u-boot/ source directory 90 .git/ repository 91""" 92 93# Possible build outcomes 94OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4) 95 96# Translate a commit subject into a valid filename 97trans_valid_chars = string.maketrans("/: ", "---") 98 99 100def Mkdir(dirname): 101 """Make a directory if it doesn't already exist. 102 103 Args: 104 dirname: Directory to create 105 """ 106 try: 107 os.mkdir(dirname) 108 except OSError as err: 109 if err.errno == errno.EEXIST: 110 pass 111 else: 112 raise 113 114class BuilderJob: 115 """Holds information about a job to be performed by a thread 116 117 Members: 118 board: Board object to build 119 commits: List of commit options to build. 120 """ 121 def __init__(self): 122 self.board = None 123 self.commits = [] 124 125 126class ResultThread(threading.Thread): 127 """This thread processes results from builder threads. 128 129 It simply passes the results on to the builder. There is only one 130 result thread, and this helps to serialise the build output. 131 """ 132 def __init__(self, builder): 133 """Set up a new result thread 134 135 Args: 136 builder: Builder which will be sent each result 137 """ 138 threading.Thread.__init__(self) 139 self.builder = builder 140 141 def run(self): 142 """Called to start up the result thread. 143 144 We collect the next result job and pass it on to the build. 145 """ 146 while True: 147 result = self.builder.out_queue.get() 148 self.builder.ProcessResult(result) 149 self.builder.out_queue.task_done() 150 151 152class BuilderThread(threading.Thread): 153 """This thread builds U-Boot for a particular board. 154 155 An input queue provides each new job. We run 'make' to build U-Boot 156 and then pass the results on to the output queue. 157 158 Members: 159 builder: The builder which contains information we might need 160 thread_num: Our thread number (0-n-1), used to decide on a 161 temporary directory 162 """ 163 def __init__(self, builder, thread_num): 164 """Set up a new builder thread""" 165 threading.Thread.__init__(self) 166 self.builder = builder 167 self.thread_num = thread_num 168 169 def Make(self, commit, brd, stage, cwd, *args, **kwargs): 170 """Run 'make' on a particular commit and board. 171 172 The source code will already be checked out, so the 'commit' 173 argument is only for information. 174 175 Args: 176 commit: Commit object that is being built 177 brd: Board object that is being built 178 stage: Stage of the build. Valid stages are: 179 distclean - can be called to clean source 180 config - called to configure for a board 181 build - the main make invocation - it does the build 182 args: A list of arguments to pass to 'make' 183 kwargs: A list of keyword arguments to pass to command.RunPipe() 184 185 Returns: 186 CommandResult object 187 """ 188 return self.builder.do_make(commit, brd, stage, cwd, *args, 189 **kwargs) 190 191 def RunCommit(self, commit_upto, brd, work_dir, do_config, force_build, 192 force_build_failures): 193 """Build a particular commit. 194 195 If the build is already done, and we are not forcing a build, we skip 196 the build and just return the previously-saved results. 197 198 Args: 199 commit_upto: Commit number to build (0...n-1) 200 brd: Board object to build 201 work_dir: Directory to which the source will be checked out 202 do_config: True to run a make <board>_defconfig on the source 203 force_build: Force a build even if one was previously done 204 force_build_failures: Force a bulid if the previous result showed 205 failure 206 207 Returns: 208 tuple containing: 209 - CommandResult object containing the results of the build 210 - boolean indicating whether 'make config' is still needed 211 """ 212 # Create a default result - it will be overwritte by the call to 213 # self.Make() below, in the event that we do a build. 214 result = command.CommandResult() 215 result.return_code = 0 216 if self.builder.in_tree: 217 out_dir = work_dir 218 else: 219 out_dir = os.path.join(work_dir, 'build') 220 221 # Check if the job was already completed last time 222 done_file = self.builder.GetDoneFile(commit_upto, brd.target) 223 result.already_done = os.path.exists(done_file) 224 will_build = (force_build or force_build_failures or 225 not result.already_done) 226 if result.already_done and will_build: 227 # Get the return code from that build and use it 228 with open(done_file, 'r') as fd: 229 result.return_code = int(fd.readline()) 230 err_file = self.builder.GetErrFile(commit_upto, brd.target) 231 if os.path.exists(err_file) and os.stat(err_file).st_size: 232 result.stderr = 'bad' 233 elif not force_build: 234 # The build passed, so no need to build it again 235 will_build = False 236 237 if will_build: 238 # We are going to have to build it. First, get a toolchain 239 if not self.toolchain: 240 try: 241 self.toolchain = self.builder.toolchains.Select(brd.arch) 242 except ValueError as err: 243 result.return_code = 10 244 result.stdout = '' 245 result.stderr = str(err) 246 # TODO(sjg@chromium.org): This gets swallowed, but needs 247 # to be reported. 248 249 if self.toolchain: 250 # Checkout the right commit 251 if self.builder.commits: 252 commit = self.builder.commits[commit_upto] 253 if self.builder.checkout: 254 git_dir = os.path.join(work_dir, '.git') 255 gitutil.Checkout(commit.hash, git_dir, work_dir, 256 force=True) 257 else: 258 commit = 'current' 259 260 # Set up the environment and command line 261 env = self.toolchain.MakeEnvironment() 262 Mkdir(out_dir) 263 args = [] 264 cwd = work_dir 265 if not self.builder.in_tree: 266 if commit_upto is None: 267 # In this case we are building in the original source 268 # directory (i.e. the current directory where buildman 269 # is invoked. The output directory is set to this 270 # thread's selected work directory. 271 # 272 # Symlinks can confuse U-Boot's Makefile since 273 # we may use '..' in our path, so remove them. 274 work_dir = os.path.realpath(work_dir) 275 args.append('O=%s/build' % work_dir) 276 cwd = None 277 else: 278 args.append('O=build') 279 args.append('-s') 280 if self.builder.num_jobs is not None: 281 args.extend(['-j', str(self.builder.num_jobs)]) 282 config_args = ['%s_defconfig' % brd.target] 283 config_out = '' 284 args.extend(self.builder.toolchains.GetMakeArguments(brd)) 285 286 # If we need to reconfigure, do that now 287 if do_config: 288 result = self.Make(commit, brd, 'distclean', cwd, 289 'distclean', *args, env=env) 290 result = self.Make(commit, brd, 'config', cwd, 291 *(args + config_args), env=env) 292 config_out = result.combined 293 do_config = False # No need to configure next time 294 if result.return_code == 0: 295 result = self.Make(commit, brd, 'build', cwd, *args, 296 env=env) 297 result.stdout = config_out + result.stdout 298 else: 299 result.return_code = 1 300 result.stderr = 'No tool chain for %s\n' % brd.arch 301 result.already_done = False 302 303 result.toolchain = self.toolchain 304 result.brd = brd 305 result.commit_upto = commit_upto 306 result.out_dir = out_dir 307 return result, do_config 308 309 def _WriteResult(self, result, keep_outputs): 310 """Write a built result to the output directory. 311 312 Args: 313 result: CommandResult object containing result to write 314 keep_outputs: True to store the output binaries, False 315 to delete them 316 """ 317 # Fatal error 318 if result.return_code < 0: 319 return 320 321 # Aborted? 322 if result.stderr and 'No child processes' in result.stderr: 323 return 324 325 if result.already_done: 326 return 327 328 # Write the output and stderr 329 output_dir = self.builder._GetOutputDir(result.commit_upto) 330 Mkdir(output_dir) 331 build_dir = self.builder.GetBuildDir(result.commit_upto, 332 result.brd.target) 333 Mkdir(build_dir) 334 335 outfile = os.path.join(build_dir, 'log') 336 with open(outfile, 'w') as fd: 337 if result.stdout: 338 fd.write(result.stdout) 339 340 errfile = self.builder.GetErrFile(result.commit_upto, 341 result.brd.target) 342 if result.stderr: 343 with open(errfile, 'w') as fd: 344 fd.write(result.stderr) 345 elif os.path.exists(errfile): 346 os.remove(errfile) 347 348 if result.toolchain: 349 # Write the build result and toolchain information. 350 done_file = self.builder.GetDoneFile(result.commit_upto, 351 result.brd.target) 352 with open(done_file, 'w') as fd: 353 fd.write('%s' % result.return_code) 354 with open(os.path.join(build_dir, 'toolchain'), 'w') as fd: 355 print >>fd, 'gcc', result.toolchain.gcc 356 print >>fd, 'path', result.toolchain.path 357 print >>fd, 'cross', result.toolchain.cross 358 print >>fd, 'arch', result.toolchain.arch 359 fd.write('%s' % result.return_code) 360 361 with open(os.path.join(build_dir, 'toolchain'), 'w') as fd: 362 print >>fd, 'gcc', result.toolchain.gcc 363 print >>fd, 'path', result.toolchain.path 364 365 # Write out the image and function size information and an objdump 366 env = result.toolchain.MakeEnvironment() 367 lines = [] 368 for fname in ['u-boot', 'spl/u-boot-spl']: 369 cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname] 370 nm_result = command.RunPipe([cmd], capture=True, 371 capture_stderr=True, cwd=result.out_dir, 372 raise_on_error=False, env=env) 373 if nm_result.stdout: 374 nm = self.builder.GetFuncSizesFile(result.commit_upto, 375 result.brd.target, fname) 376 with open(nm, 'w') as fd: 377 print >>fd, nm_result.stdout, 378 379 cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname] 380 dump_result = command.RunPipe([cmd], capture=True, 381 capture_stderr=True, cwd=result.out_dir, 382 raise_on_error=False, env=env) 383 rodata_size = '' 384 if dump_result.stdout: 385 objdump = self.builder.GetObjdumpFile(result.commit_upto, 386 result.brd.target, fname) 387 with open(objdump, 'w') as fd: 388 print >>fd, dump_result.stdout, 389 for line in dump_result.stdout.splitlines(): 390 fields = line.split() 391 if len(fields) > 5 and fields[1] == '.rodata': 392 rodata_size = fields[2] 393 394 cmd = ['%ssize' % self.toolchain.cross, fname] 395 size_result = command.RunPipe([cmd], capture=True, 396 capture_stderr=True, cwd=result.out_dir, 397 raise_on_error=False, env=env) 398 if size_result.stdout: 399 lines.append(size_result.stdout.splitlines()[1] + ' ' + 400 rodata_size) 401 402 # Write out the image sizes file. This is similar to the output 403 # of binutil's 'size' utility, but it omits the header line and 404 # adds an additional hex value at the end of each line for the 405 # rodata size 406 if len(lines): 407 sizes = self.builder.GetSizesFile(result.commit_upto, 408 result.brd.target) 409 with open(sizes, 'w') as fd: 410 print >>fd, '\n'.join(lines) 411 412 # Now write the actual build output 413 if keep_outputs: 414 patterns = ['u-boot', '*.bin', 'u-boot.dtb', '*.map', 415 'include/autoconf.mk', 'spl/u-boot-spl', 416 'spl/u-boot-spl.bin'] 417 for pattern in patterns: 418 file_list = glob.glob(os.path.join(result.out_dir, pattern)) 419 for fname in file_list: 420 shutil.copy(fname, build_dir) 421 422 423 def RunJob(self, job): 424 """Run a single job 425 426 A job consists of a building a list of commits for a particular board. 427 428 Args: 429 job: Job to build 430 """ 431 brd = job.board 432 work_dir = self.builder.GetThreadDir(self.thread_num) 433 self.toolchain = None 434 if job.commits: 435 # Run 'make board_defconfig' on the first commit 436 do_config = True 437 commit_upto = 0 438 force_build = False 439 for commit_upto in range(0, len(job.commits), job.step): 440 result, request_config = self.RunCommit(commit_upto, brd, 441 work_dir, do_config, 442 force_build or self.builder.force_build, 443 self.builder.force_build_failures) 444 failed = result.return_code or result.stderr 445 did_config = do_config 446 if failed and not do_config: 447 # If our incremental build failed, try building again 448 # with a reconfig. 449 if self.builder.force_config_on_failure: 450 result, request_config = self.RunCommit(commit_upto, 451 brd, work_dir, True, True, False) 452 did_config = True 453 if not self.builder.force_reconfig: 454 do_config = request_config 455 456 # If we built that commit, then config is done. But if we got 457 # an warning, reconfig next time to force it to build the same 458 # files that created warnings this time. Otherwise an 459 # incremental build may not build the same file, and we will 460 # think that the warning has gone away. 461 # We could avoid this by using -Werror everywhere... 462 # For errors, the problem doesn't happen, since presumably 463 # the build stopped and didn't generate output, so will retry 464 # that file next time. So we could detect warnings and deal 465 # with them specially here. For now, we just reconfigure if 466 # anything goes work. 467 # Of course this is substantially slower if there are build 468 # errors/warnings (e.g. 2-3x slower even if only 10% of builds 469 # have problems). 470 if (failed and not result.already_done and not did_config and 471 self.builder.force_config_on_failure): 472 # If this build failed, try the next one with a 473 # reconfigure. 474 # Sometimes if the board_config.h file changes it can mess 475 # with dependencies, and we get: 476 # make: *** No rule to make target `include/autoconf.mk', 477 # needed by `depend'. 478 do_config = True 479 force_build = True 480 else: 481 force_build = False 482 if self.builder.force_config_on_failure: 483 if failed: 484 do_config = True 485 result.commit_upto = commit_upto 486 if result.return_code < 0: 487 raise ValueError('Interrupt') 488 489 # We have the build results, so output the result 490 self._WriteResult(result, job.keep_outputs) 491 self.builder.out_queue.put(result) 492 else: 493 # Just build the currently checked-out build 494 result, request_config = self.RunCommit(None, brd, work_dir, True, 495 True, self.builder.force_build_failures) 496 result.commit_upto = 0 497 self._WriteResult(result, job.keep_outputs) 498 self.builder.out_queue.put(result) 499 500 def run(self): 501 """Our thread's run function 502 503 This thread picks a job from the queue, runs it, and then goes to the 504 next job. 505 """ 506 alive = True 507 while True: 508 job = self.builder.queue.get() 509 if self.builder.active and alive: 510 self.RunJob(job) 511 ''' 512 try: 513 if self.builder.active and alive: 514 self.RunJob(job) 515 except Exception as err: 516 alive = False 517 print err 518 ''' 519 self.builder.queue.task_done() 520 521 522class Builder: 523 """Class for building U-Boot for a particular commit. 524 525 Public members: (many should ->private) 526 active: True if the builder is active and has not been stopped 527 already_done: Number of builds already completed 528 base_dir: Base directory to use for builder 529 checkout: True to check out source, False to skip that step. 530 This is used for testing. 531 col: terminal.Color() object 532 count: Number of commits to build 533 do_make: Method to call to invoke Make 534 fail: Number of builds that failed due to error 535 force_build: Force building even if a build already exists 536 force_config_on_failure: If a commit fails for a board, disable 537 incremental building for the next commit we build for that 538 board, so that we will see all warnings/errors again. 539 force_build_failures: If a previously-built build (i.e. built on 540 a previous run of buildman) is marked as failed, rebuild it. 541 git_dir: Git directory containing source repository 542 last_line_len: Length of the last line we printed (used for erasing 543 it with new progress information) 544 num_jobs: Number of jobs to run at once (passed to make as -j) 545 num_threads: Number of builder threads to run 546 out_queue: Queue of results to process 547 re_make_err: Compiled regular expression for ignore_lines 548 queue: Queue of jobs to run 549 threads: List of active threads 550 toolchains: Toolchains object to use for building 551 upto: Current commit number we are building (0.count-1) 552 warned: Number of builds that produced at least one warning 553 force_reconfig: Reconfigure U-Boot on each comiit. This disables 554 incremental building, where buildman reconfigures on the first 555 commit for a baord, and then just does an incremental build for 556 the following commits. In fact buildman will reconfigure and 557 retry for any failing commits, so generally the only effect of 558 this option is to slow things down. 559 in_tree: Build U-Boot in-tree instead of specifying an output 560 directory separate from the source code. This option is really 561 only useful for testing in-tree builds. 562 563 Private members: 564 _base_board_dict: Last-summarised Dict of boards 565 _base_err_lines: Last-summarised list of errors 566 _build_period_us: Time taken for a single build (float object). 567 _complete_delay: Expected delay until completion (timedelta) 568 _next_delay_update: Next time we plan to display a progress update 569 (datatime) 570 _show_unknown: Show unknown boards (those not built) in summary 571 _timestamps: List of timestamps for the completion of the last 572 last _timestamp_count builds. Each is a datetime object. 573 _timestamp_count: Number of timestamps to keep in our list. 574 _working_dir: Base working directory containing all threads 575 """ 576 class Outcome: 577 """Records a build outcome for a single make invocation 578 579 Public Members: 580 rc: Outcome value (OUTCOME_...) 581 err_lines: List of error lines or [] if none 582 sizes: Dictionary of image size information, keyed by filename 583 - Each value is itself a dictionary containing 584 values for 'text', 'data' and 'bss', being the integer 585 size in bytes of each section. 586 func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each 587 value is itself a dictionary: 588 key: function name 589 value: Size of function in bytes 590 """ 591 def __init__(self, rc, err_lines, sizes, func_sizes): 592 self.rc = rc 593 self.err_lines = err_lines 594 self.sizes = sizes 595 self.func_sizes = func_sizes 596 597 def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, 598 gnu_make='make', checkout=True, show_unknown=True, step=1): 599 """Create a new Builder object 600 601 Args: 602 toolchains: Toolchains object to use for building 603 base_dir: Base directory to use for builder 604 git_dir: Git directory containing source repository 605 num_threads: Number of builder threads to run 606 num_jobs: Number of jobs to run at once (passed to make as -j) 607 gnu_make: the command name of GNU Make. 608 checkout: True to check out source, False to skip that step. 609 This is used for testing. 610 show_unknown: Show unknown boards (those not built) in summary 611 step: 1 to process every commit, n to process every nth commit 612 """ 613 self.toolchains = toolchains 614 self.base_dir = base_dir 615 self._working_dir = os.path.join(base_dir, '.bm-work') 616 self.threads = [] 617 self.active = True 618 self.do_make = self.Make 619 self.gnu_make = gnu_make 620 self.checkout = checkout 621 self.num_threads = num_threads 622 self.num_jobs = num_jobs 623 self.already_done = 0 624 self.force_build = False 625 self.git_dir = git_dir 626 self._show_unknown = show_unknown 627 self._timestamp_count = 10 628 self._build_period_us = None 629 self._complete_delay = None 630 self._next_delay_update = datetime.now() 631 self.force_config_on_failure = True 632 self.force_build_failures = False 633 self.force_reconfig = False 634 self._step = step 635 self.in_tree = False 636 637 self.col = terminal.Color() 638 639 self.queue = Queue.Queue() 640 self.out_queue = Queue.Queue() 641 for i in range(self.num_threads): 642 t = BuilderThread(self, i) 643 t.setDaemon(True) 644 t.start() 645 self.threads.append(t) 646 647 self.last_line_len = 0 648 t = ResultThread(self) 649 t.setDaemon(True) 650 t.start() 651 self.threads.append(t) 652 653 ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)'] 654 self.re_make_err = re.compile('|'.join(ignore_lines)) 655 656 def __del__(self): 657 """Get rid of all threads created by the builder""" 658 for t in self.threads: 659 del t 660 661 def _AddTimestamp(self): 662 """Add a new timestamp to the list and record the build period. 663 664 The build period is the length of time taken to perform a single 665 build (one board, one commit). 666 """ 667 now = datetime.now() 668 self._timestamps.append(now) 669 count = len(self._timestamps) 670 delta = self._timestamps[-1] - self._timestamps[0] 671 seconds = delta.total_seconds() 672 673 # If we have enough data, estimate build period (time taken for a 674 # single build) and therefore completion time. 675 if count > 1 and self._next_delay_update < now: 676 self._next_delay_update = now + timedelta(seconds=2) 677 if seconds > 0: 678 self._build_period = float(seconds) / count 679 todo = self.count - self.upto 680 self._complete_delay = timedelta(microseconds= 681 self._build_period * todo * 1000000) 682 # Round it 683 self._complete_delay -= timedelta( 684 microseconds=self._complete_delay.microseconds) 685 686 if seconds > 60: 687 self._timestamps.popleft() 688 count -= 1 689 690 def ClearLine(self, length): 691 """Clear any characters on the current line 692 693 Make way for a new line of length 'length', by outputting enough 694 spaces to clear out the old line. Then remember the new length for 695 next time. 696 697 Args: 698 length: Length of new line, in characters 699 """ 700 if length < self.last_line_len: 701 print ' ' * (self.last_line_len - length), 702 print '\r', 703 self.last_line_len = length 704 sys.stdout.flush() 705 706 def SelectCommit(self, commit, checkout=True): 707 """Checkout the selected commit for this build 708 """ 709 self.commit = commit 710 if checkout and self.checkout: 711 gitutil.Checkout(commit.hash) 712 713 def Make(self, commit, brd, stage, cwd, *args, **kwargs): 714 """Run make 715 716 Args: 717 commit: Commit object that is being built 718 brd: Board object that is being built 719 stage: Stage that we are at (distclean, config, build) 720 cwd: Directory where make should be run 721 args: Arguments to pass to make 722 kwargs: Arguments to pass to command.RunPipe() 723 """ 724 cmd = [self.gnu_make] + list(args) 725 result = command.RunPipe([cmd], capture=True, capture_stderr=True, 726 cwd=cwd, raise_on_error=False, **kwargs) 727 return result 728 729 def ProcessResult(self, result): 730 """Process the result of a build, showing progress information 731 732 Args: 733 result: A CommandResult object 734 """ 735 col = terminal.Color() 736 if result: 737 target = result.brd.target 738 739 if result.return_code < 0: 740 self.active = False 741 command.StopAll() 742 return 743 744 self.upto += 1 745 if result.return_code != 0: 746 self.fail += 1 747 elif result.stderr: 748 self.warned += 1 749 if result.already_done: 750 self.already_done += 1 751 else: 752 target = '(starting)' 753 754 # Display separate counts for ok, warned and fail 755 ok = self.upto - self.warned - self.fail 756 line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok) 757 line += self.col.Color(self.col.YELLOW, '%5d' % self.warned) 758 line += self.col.Color(self.col.RED, '%5d' % self.fail) 759 760 name = ' /%-5d ' % self.count 761 762 # Add our current completion time estimate 763 self._AddTimestamp() 764 if self._complete_delay: 765 name += '%s : ' % self._complete_delay 766 # When building all boards for a commit, we can print a commit 767 # progress message. 768 if result and result.commit_upto is None: 769 name += 'commit %2d/%-3d' % (self.commit_upto + 1, 770 self.commit_count) 771 772 name += target 773 print line + name, 774 length = 13 + len(name) 775 self.ClearLine(length) 776 777 def _GetOutputDir(self, commit_upto): 778 """Get the name of the output directory for a commit number 779 780 The output directory is typically .../<branch>/<commit>. 781 782 Args: 783 commit_upto: Commit number to use (0..self.count-1) 784 """ 785 if self.commits: 786 commit = self.commits[commit_upto] 787 subject = commit.subject.translate(trans_valid_chars) 788 commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1, 789 self.commit_count, commit.hash, subject[:20])) 790 else: 791 commit_dir = 'current' 792 output_dir = os.path.join(self.base_dir, commit_dir) 793 return output_dir 794 795 def GetBuildDir(self, commit_upto, target): 796 """Get the name of the build directory for a commit number 797 798 The build directory is typically .../<branch>/<commit>/<target>. 799 800 Args: 801 commit_upto: Commit number to use (0..self.count-1) 802 target: Target name 803 """ 804 output_dir = self._GetOutputDir(commit_upto) 805 return os.path.join(output_dir, target) 806 807 def GetDoneFile(self, commit_upto, target): 808 """Get the name of the done file for a commit number 809 810 Args: 811 commit_upto: Commit number to use (0..self.count-1) 812 target: Target name 813 """ 814 return os.path.join(self.GetBuildDir(commit_upto, target), 'done') 815 816 def GetSizesFile(self, commit_upto, target): 817 """Get the name of the sizes file for a commit number 818 819 Args: 820 commit_upto: Commit number to use (0..self.count-1) 821 target: Target name 822 """ 823 return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes') 824 825 def GetFuncSizesFile(self, commit_upto, target, elf_fname): 826 """Get the name of the funcsizes file for a commit number and ELF file 827 828 Args: 829 commit_upto: Commit number to use (0..self.count-1) 830 target: Target name 831 elf_fname: Filename of elf image 832 """ 833 return os.path.join(self.GetBuildDir(commit_upto, target), 834 '%s.sizes' % elf_fname.replace('/', '-')) 835 836 def GetObjdumpFile(self, commit_upto, target, elf_fname): 837 """Get the name of the objdump file for a commit number and ELF file 838 839 Args: 840 commit_upto: Commit number to use (0..self.count-1) 841 target: Target name 842 elf_fname: Filename of elf image 843 """ 844 return os.path.join(self.GetBuildDir(commit_upto, target), 845 '%s.objdump' % elf_fname.replace('/', '-')) 846 847 def GetErrFile(self, commit_upto, target): 848 """Get the name of the err file for a commit number 849 850 Args: 851 commit_upto: Commit number to use (0..self.count-1) 852 target: Target name 853 """ 854 output_dir = self.GetBuildDir(commit_upto, target) 855 return os.path.join(output_dir, 'err') 856 857 def FilterErrors(self, lines): 858 """Filter out errors in which we have no interest 859 860 We should probably use map(). 861 862 Args: 863 lines: List of error lines, each a string 864 Returns: 865 New list with only interesting lines included 866 """ 867 out_lines = [] 868 for line in lines: 869 if not self.re_make_err.search(line): 870 out_lines.append(line) 871 return out_lines 872 873 def ReadFuncSizes(self, fname, fd): 874 """Read function sizes from the output of 'nm' 875 876 Args: 877 fd: File containing data to read 878 fname: Filename we are reading from (just for errors) 879 880 Returns: 881 Dictionary containing size of each function in bytes, indexed by 882 function name. 883 """ 884 sym = {} 885 for line in fd.readlines(): 886 try: 887 size, type, name = line[:-1].split() 888 except: 889 print "Invalid line in file '%s': '%s'" % (fname, line[:-1]) 890 continue 891 if type in 'tTdDbB': 892 # function names begin with '.' on 64-bit powerpc 893 if '.' in name[1:]: 894 name = 'static.' + name.split('.')[0] 895 sym[name] = sym.get(name, 0) + int(size, 16) 896 return sym 897 898 def GetBuildOutcome(self, commit_upto, target, read_func_sizes): 899 """Work out the outcome of a build. 900 901 Args: 902 commit_upto: Commit number to check (0..n-1) 903 target: Target board to check 904 read_func_sizes: True to read function size information 905 906 Returns: 907 Outcome object 908 """ 909 done_file = self.GetDoneFile(commit_upto, target) 910 sizes_file = self.GetSizesFile(commit_upto, target) 911 sizes = {} 912 func_sizes = {} 913 if os.path.exists(done_file): 914 with open(done_file, 'r') as fd: 915 return_code = int(fd.readline()) 916 err_lines = [] 917 err_file = self.GetErrFile(commit_upto, target) 918 if os.path.exists(err_file): 919 with open(err_file, 'r') as fd: 920 err_lines = self.FilterErrors(fd.readlines()) 921 922 # Decide whether the build was ok, failed or created warnings 923 if return_code: 924 rc = OUTCOME_ERROR 925 elif len(err_lines): 926 rc = OUTCOME_WARNING 927 else: 928 rc = OUTCOME_OK 929 930 # Convert size information to our simple format 931 if os.path.exists(sizes_file): 932 with open(sizes_file, 'r') as fd: 933 for line in fd.readlines(): 934 values = line.split() 935 rodata = 0 936 if len(values) > 6: 937 rodata = int(values[6], 16) 938 size_dict = { 939 'all' : int(values[0]) + int(values[1]) + 940 int(values[2]), 941 'text' : int(values[0]) - rodata, 942 'data' : int(values[1]), 943 'bss' : int(values[2]), 944 'rodata' : rodata, 945 } 946 sizes[values[5]] = size_dict 947 948 if read_func_sizes: 949 pattern = self.GetFuncSizesFile(commit_upto, target, '*') 950 for fname in glob.glob(pattern): 951 with open(fname, 'r') as fd: 952 dict_name = os.path.basename(fname).replace('.sizes', 953 '') 954 func_sizes[dict_name] = self.ReadFuncSizes(fname, fd) 955 956 return Builder.Outcome(rc, err_lines, sizes, func_sizes) 957 958 return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}) 959 960 def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes): 961 """Calculate a summary of the results of building a commit. 962 963 Args: 964 board_selected: Dict containing boards to summarise 965 commit_upto: Commit number to summarize (0..self.count-1) 966 read_func_sizes: True to read function size information 967 968 Returns: 969 Tuple: 970 Dict containing boards which passed building this commit. 971 keyed by board.target 972 List containing a summary of error/warning lines 973 """ 974 board_dict = {} 975 err_lines_summary = [] 976 977 for board in boards_selected.itervalues(): 978 outcome = self.GetBuildOutcome(commit_upto, board.target, 979 read_func_sizes) 980 board_dict[board.target] = outcome 981 for err in outcome.err_lines: 982 if err and not err.rstrip() in err_lines_summary: 983 err_lines_summary.append(err.rstrip()) 984 return board_dict, err_lines_summary 985 986 def AddOutcome(self, board_dict, arch_list, changes, char, color): 987 """Add an output to our list of outcomes for each architecture 988 989 This simple function adds failing boards (changes) to the 990 relevant architecture string, so we can print the results out 991 sorted by architecture. 992 993 Args: 994 board_dict: Dict containing all boards 995 arch_list: Dict keyed by arch name. Value is a string containing 996 a list of board names which failed for that arch. 997 changes: List of boards to add to arch_list 998 color: terminal.Colour object 999 """ 1000 done_arch = {} 1001 for target in changes: 1002 if target in board_dict: 1003 arch = board_dict[target].arch 1004 else: 1005 arch = 'unknown' 1006 str = self.col.Color(color, ' ' + target) 1007 if not arch in done_arch: 1008 str = self.col.Color(color, char) + ' ' + str 1009 done_arch[arch] = True 1010 if not arch in arch_list: 1011 arch_list[arch] = str 1012 else: 1013 arch_list[arch] += str 1014 1015 1016 def ColourNum(self, num): 1017 color = self.col.RED if num > 0 else self.col.GREEN 1018 if num == 0: 1019 return '0' 1020 return self.col.Color(color, str(num)) 1021 1022 def ResetResultSummary(self, board_selected): 1023 """Reset the results summary ready for use. 1024 1025 Set up the base board list to be all those selected, and set the 1026 error lines to empty. 1027 1028 Following this, calls to PrintResultSummary() will use this 1029 information to work out what has changed. 1030 1031 Args: 1032 board_selected: Dict containing boards to summarise, keyed by 1033 board.target 1034 """ 1035 self._base_board_dict = {} 1036 for board in board_selected: 1037 self._base_board_dict[board] = Builder.Outcome(0, [], [], {}) 1038 self._base_err_lines = [] 1039 1040 def PrintFuncSizeDetail(self, fname, old, new): 1041 grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0 1042 delta, common = [], {} 1043 1044 for a in old: 1045 if a in new: 1046 common[a] = 1 1047 1048 for name in old: 1049 if name not in common: 1050 remove += 1 1051 down += old[name] 1052 delta.append([-old[name], name]) 1053 1054 for name in new: 1055 if name not in common: 1056 add += 1 1057 up += new[name] 1058 delta.append([new[name], name]) 1059 1060 for name in common: 1061 diff = new.get(name, 0) - old.get(name, 0) 1062 if diff > 0: 1063 grow, up = grow + 1, up + diff 1064 elif diff < 0: 1065 shrink, down = shrink + 1, down - diff 1066 delta.append([diff, name]) 1067 1068 delta.sort() 1069 delta.reverse() 1070 1071 args = [add, -remove, grow, -shrink, up, -down, up - down] 1072 if max(args) == 0: 1073 return 1074 args = [self.ColourNum(x) for x in args] 1075 indent = ' ' * 15 1076 print ('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' % 1077 tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args)) 1078 print '%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new', 1079 'delta') 1080 for diff, name in delta: 1081 if diff: 1082 color = self.col.RED if diff > 0 else self.col.GREEN 1083 msg = '%s %-38s %7s %7s %+7d' % (indent, name, 1084 old.get(name, '-'), new.get(name,'-'), diff) 1085 print self.col.Color(color, msg) 1086 1087 1088 def PrintSizeDetail(self, target_list, show_bloat): 1089 """Show details size information for each board 1090 1091 Args: 1092 target_list: List of targets, each a dict containing: 1093 'target': Target name 1094 'total_diff': Total difference in bytes across all areas 1095 <part_name>: Difference for that part 1096 show_bloat: Show detail for each function 1097 """ 1098 targets_by_diff = sorted(target_list, reverse=True, 1099 key=lambda x: x['_total_diff']) 1100 for result in targets_by_diff: 1101 printed_target = False 1102 for name in sorted(result): 1103 diff = result[name] 1104 if name.startswith('_'): 1105 continue 1106 if diff != 0: 1107 color = self.col.RED if diff > 0 else self.col.GREEN 1108 msg = ' %s %+d' % (name, diff) 1109 if not printed_target: 1110 print '%10s %-15s:' % ('', result['_target']), 1111 printed_target = True 1112 print self.col.Color(color, msg), 1113 if printed_target: 1114 print 1115 if show_bloat: 1116 target = result['_target'] 1117 outcome = result['_outcome'] 1118 base_outcome = self._base_board_dict[target] 1119 for fname in outcome.func_sizes: 1120 self.PrintFuncSizeDetail(fname, 1121 base_outcome.func_sizes[fname], 1122 outcome.func_sizes[fname]) 1123 1124 1125 def PrintSizeSummary(self, board_selected, board_dict, show_detail, 1126 show_bloat): 1127 """Print a summary of image sizes broken down by section. 1128 1129 The summary takes the form of one line per architecture. The 1130 line contains deltas for each of the sections (+ means the section 1131 got bigger, - means smaller). The nunmbers are the average number 1132 of bytes that a board in this section increased by. 1133 1134 For example: 1135 powerpc: (622 boards) text -0.0 1136 arm: (285 boards) text -0.0 1137 nds32: (3 boards) text -8.0 1138 1139 Args: 1140 board_selected: Dict containing boards to summarise, keyed by 1141 board.target 1142 board_dict: Dict containing boards for which we built this 1143 commit, keyed by board.target. The value is an Outcome object. 1144 show_detail: Show detail for each board 1145 show_bloat: Show detail for each function 1146 """ 1147 arch_list = {} 1148 arch_count = {} 1149 1150 # Calculate changes in size for different image parts 1151 # The previous sizes are in Board.sizes, for each board 1152 for target in board_dict: 1153 if target not in board_selected: 1154 continue 1155 base_sizes = self._base_board_dict[target].sizes 1156 outcome = board_dict[target] 1157 sizes = outcome.sizes 1158 1159 # Loop through the list of images, creating a dict of size 1160 # changes for each image/part. We end up with something like 1161 # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4} 1162 # which means that U-Boot data increased by 5 bytes and SPL 1163 # text decreased by 4. 1164 err = {'_target' : target} 1165 for image in sizes: 1166 if image in base_sizes: 1167 base_image = base_sizes[image] 1168 # Loop through the text, data, bss parts 1169 for part in sorted(sizes[image]): 1170 diff = sizes[image][part] - base_image[part] 1171 col = None 1172 if diff: 1173 if image == 'u-boot': 1174 name = part 1175 else: 1176 name = image + ':' + part 1177 err[name] = diff 1178 arch = board_selected[target].arch 1179 if not arch in arch_count: 1180 arch_count[arch] = 1 1181 else: 1182 arch_count[arch] += 1 1183 if not sizes: 1184 pass # Only add to our list when we have some stats 1185 elif not arch in arch_list: 1186 arch_list[arch] = [err] 1187 else: 1188 arch_list[arch].append(err) 1189 1190 # We now have a list of image size changes sorted by arch 1191 # Print out a summary of these 1192 for arch, target_list in arch_list.iteritems(): 1193 # Get total difference for each type 1194 totals = {} 1195 for result in target_list: 1196 total = 0 1197 for name, diff in result.iteritems(): 1198 if name.startswith('_'): 1199 continue 1200 total += diff 1201 if name in totals: 1202 totals[name] += diff 1203 else: 1204 totals[name] = diff 1205 result['_total_diff'] = total 1206 result['_outcome'] = board_dict[result['_target']] 1207 1208 count = len(target_list) 1209 printed_arch = False 1210 for name in sorted(totals): 1211 diff = totals[name] 1212 if diff: 1213 # Display the average difference in this name for this 1214 # architecture 1215 avg_diff = float(diff) / count 1216 color = self.col.RED if avg_diff > 0 else self.col.GREEN 1217 msg = ' %s %+1.1f' % (name, avg_diff) 1218 if not printed_arch: 1219 print '%10s: (for %d/%d boards)' % (arch, count, 1220 arch_count[arch]), 1221 printed_arch = True 1222 print self.col.Color(color, msg), 1223 1224 if printed_arch: 1225 print 1226 if show_detail: 1227 self.PrintSizeDetail(target_list, show_bloat) 1228 1229 1230 def PrintResultSummary(self, board_selected, board_dict, err_lines, 1231 show_sizes, show_detail, show_bloat): 1232 """Compare results with the base results and display delta. 1233 1234 Only boards mentioned in board_selected will be considered. This 1235 function is intended to be called repeatedly with the results of 1236 each commit. It therefore shows a 'diff' between what it saw in 1237 the last call and what it sees now. 1238 1239 Args: 1240 board_selected: Dict containing boards to summarise, keyed by 1241 board.target 1242 board_dict: Dict containing boards for which we built this 1243 commit, keyed by board.target. The value is an Outcome object. 1244 err_lines: A list of errors for this commit, or [] if there is 1245 none, or we don't want to print errors 1246 show_sizes: Show image size deltas 1247 show_detail: Show detail for each board 1248 show_bloat: Show detail for each function 1249 """ 1250 better = [] # List of boards fixed since last commit 1251 worse = [] # List of new broken boards since last commit 1252 new = [] # List of boards that didn't exist last time 1253 unknown = [] # List of boards that were not built 1254 1255 for target in board_dict: 1256 if target not in board_selected: 1257 continue 1258 1259 # If the board was built last time, add its outcome to a list 1260 if target in self._base_board_dict: 1261 base_outcome = self._base_board_dict[target].rc 1262 outcome = board_dict[target] 1263 if outcome.rc == OUTCOME_UNKNOWN: 1264 unknown.append(target) 1265 elif outcome.rc < base_outcome: 1266 better.append(target) 1267 elif outcome.rc > base_outcome: 1268 worse.append(target) 1269 else: 1270 new.append(target) 1271 1272 # Get a list of errors that have appeared, and disappeared 1273 better_err = [] 1274 worse_err = [] 1275 for line in err_lines: 1276 if line not in self._base_err_lines: 1277 worse_err.append('+' + line) 1278 for line in self._base_err_lines: 1279 if line not in err_lines: 1280 better_err.append('-' + line) 1281 1282 # Display results by arch 1283 if better or worse or unknown or new or worse_err or better_err: 1284 arch_list = {} 1285 self.AddOutcome(board_selected, arch_list, better, '', 1286 self.col.GREEN) 1287 self.AddOutcome(board_selected, arch_list, worse, '+', 1288 self.col.RED) 1289 self.AddOutcome(board_selected, arch_list, new, '*', self.col.BLUE) 1290 if self._show_unknown: 1291 self.AddOutcome(board_selected, arch_list, unknown, '?', 1292 self.col.MAGENTA) 1293 for arch, target_list in arch_list.iteritems(): 1294 print '%10s: %s' % (arch, target_list) 1295 if better_err: 1296 print self.col.Color(self.col.GREEN, '\n'.join(better_err)) 1297 if worse_err: 1298 print self.col.Color(self.col.RED, '\n'.join(worse_err)) 1299 1300 if show_sizes: 1301 self.PrintSizeSummary(board_selected, board_dict, show_detail, 1302 show_bloat) 1303 1304 # Save our updated information for the next call to this function 1305 self._base_board_dict = board_dict 1306 self._base_err_lines = err_lines 1307 1308 # Get a list of boards that did not get built, if needed 1309 not_built = [] 1310 for board in board_selected: 1311 if not board in board_dict: 1312 not_built.append(board) 1313 if not_built: 1314 print "Boards not built (%d): %s" % (len(not_built), 1315 ', '.join(not_built)) 1316 1317 1318 def ShowSummary(self, commits, board_selected, show_errors, show_sizes, 1319 show_detail, show_bloat): 1320 """Show a build summary for U-Boot for a given board list. 1321 1322 Reset the result summary, then repeatedly call GetResultSummary on 1323 each commit's results, then display the differences we see. 1324 1325 Args: 1326 commit: Commit objects to summarise 1327 board_selected: Dict containing boards to summarise 1328 show_errors: Show errors that occured 1329 show_sizes: Show size deltas 1330 show_detail: Show detail for each board 1331 show_bloat: Show detail for each function 1332 """ 1333 self.commit_count = len(commits) if commits else 1 1334 self.commits = commits 1335 self.ResetResultSummary(board_selected) 1336 1337 for commit_upto in range(0, self.commit_count, self._step): 1338 board_dict, err_lines = self.GetResultSummary(board_selected, 1339 commit_upto, read_func_sizes=show_bloat) 1340 if commits: 1341 msg = '%02d: %s' % (commit_upto + 1, 1342 commits[commit_upto].subject) 1343 else: 1344 msg = 'current' 1345 print self.col.Color(self.col.BLUE, msg) 1346 self.PrintResultSummary(board_selected, board_dict, 1347 err_lines if show_errors else [], show_sizes, show_detail, 1348 show_bloat) 1349 1350 1351 def SetupBuild(self, board_selected, commits): 1352 """Set up ready to start a build. 1353 1354 Args: 1355 board_selected: Selected boards to build 1356 commits: Selected commits to build 1357 """ 1358 # First work out how many commits we will build 1359 count = (self.commit_count + self._step - 1) / self._step 1360 self.count = len(board_selected) * count 1361 self.upto = self.warned = self.fail = 0 1362 self._timestamps = collections.deque() 1363 1364 def BuildBoardsForCommit(self, board_selected, keep_outputs): 1365 """Build all boards for a single commit""" 1366 self.SetupBuild(board_selected) 1367 self.count = len(board_selected) 1368 for brd in board_selected.itervalues(): 1369 job = BuilderJob() 1370 job.board = brd 1371 job.commits = None 1372 job.keep_outputs = keep_outputs 1373 self.queue.put(brd) 1374 1375 self.queue.join() 1376 self.out_queue.join() 1377 print 1378 self.ClearLine(0) 1379 1380 def BuildCommits(self, commits, board_selected, show_errors, keep_outputs): 1381 """Build all boards for all commits (non-incremental)""" 1382 self.commit_count = len(commits) 1383 1384 self.ResetResultSummary(board_selected) 1385 for self.commit_upto in range(self.commit_count): 1386 self.SelectCommit(commits[self.commit_upto]) 1387 self.SelectOutputDir() 1388 Mkdir(self.output_dir) 1389 1390 self.BuildBoardsForCommit(board_selected, keep_outputs) 1391 board_dict, err_lines = self.GetResultSummary() 1392 self.PrintResultSummary(board_selected, board_dict, 1393 err_lines if show_errors else []) 1394 1395 if self.already_done: 1396 print '%d builds already done' % self.already_done 1397 1398 def GetThreadDir(self, thread_num): 1399 """Get the directory path to the working dir for a thread. 1400 1401 Args: 1402 thread_num: Number of thread to check. 1403 """ 1404 return os.path.join(self._working_dir, '%02d' % thread_num) 1405 1406 def _PrepareThread(self, thread_num, setup_git): 1407 """Prepare the working directory for a thread. 1408 1409 This clones or fetches the repo into the thread's work directory. 1410 1411 Args: 1412 thread_num: Thread number (0, 1, ...) 1413 setup_git: True to set up a git repo clone 1414 """ 1415 thread_dir = self.GetThreadDir(thread_num) 1416 Mkdir(thread_dir) 1417 git_dir = os.path.join(thread_dir, '.git') 1418 1419 # Clone the repo if it doesn't already exist 1420 # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so 1421 # we have a private index but uses the origin repo's contents? 1422 if setup_git and self.git_dir: 1423 src_dir = os.path.abspath(self.git_dir) 1424 if os.path.exists(git_dir): 1425 gitutil.Fetch(git_dir, thread_dir) 1426 else: 1427 print 'Cloning repo for thread %d' % thread_num 1428 gitutil.Clone(src_dir, thread_dir) 1429 1430 def _PrepareWorkingSpace(self, max_threads, setup_git): 1431 """Prepare the working directory for use. 1432 1433 Set up the git repo for each thread. 1434 1435 Args: 1436 max_threads: Maximum number of threads we expect to need. 1437 setup_git: True to set up a git repo clone 1438 """ 1439 Mkdir(self._working_dir) 1440 for thread in range(max_threads): 1441 self._PrepareThread(thread, setup_git) 1442 1443 def _PrepareOutputSpace(self): 1444 """Get the output directories ready to receive files. 1445 1446 We delete any output directories which look like ones we need to 1447 create. Having left over directories is confusing when the user wants 1448 to check the output manually. 1449 """ 1450 dir_list = [] 1451 for commit_upto in range(self.commit_count): 1452 dir_list.append(self._GetOutputDir(commit_upto)) 1453 1454 for dirname in glob.glob(os.path.join(self.base_dir, '*')): 1455 if dirname not in dir_list: 1456 shutil.rmtree(dirname) 1457 1458 def BuildBoards(self, commits, board_selected, show_errors, keep_outputs): 1459 """Build all commits for a list of boards 1460 1461 Args: 1462 commits: List of commits to be build, each a Commit object 1463 boards_selected: Dict of selected boards, key is target name, 1464 value is Board object 1465 show_errors: True to show summarised error/warning info 1466 keep_outputs: True to save build output files 1467 """ 1468 self.commit_count = len(commits) if commits else 1 1469 self.commits = commits 1470 1471 self.ResetResultSummary(board_selected) 1472 Mkdir(self.base_dir) 1473 self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)), 1474 commits is not None) 1475 self._PrepareOutputSpace() 1476 self.SetupBuild(board_selected, commits) 1477 self.ProcessResult(None) 1478 1479 # Create jobs to build all commits for each board 1480 for brd in board_selected.itervalues(): 1481 job = BuilderJob() 1482 job.board = brd 1483 job.commits = commits 1484 job.keep_outputs = keep_outputs 1485 job.step = self._step 1486 self.queue.put(job) 1487 1488 # Wait until all jobs are started 1489 self.queue.join() 1490 1491 # Wait until we have processed all output 1492 self.out_queue.join() 1493 print 1494 self.ClearLine(0) 1495