1# Copyright (c) 2013 The Chromium OS Authors. 2# 3# Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com> 4# 5# SPDX-License-Identifier: GPL-2.0+ 6# 7 8import collections 9from datetime import datetime, timedelta 10import glob 11import os 12import re 13import Queue 14import shutil 15import signal 16import string 17import sys 18import threading 19import time 20 21import builderthread 22import command 23import gitutil 24import terminal 25from terminal import Print 26import toolchain 27 28 29""" 30Theory of Operation 31 32Please see README for user documentation, and you should be familiar with 33that before trying to make sense of this. 34 35Buildman works by keeping the machine as busy as possible, building different 36commits for different boards on multiple CPUs at once. 37 38The source repo (self.git_dir) contains all the commits to be built. Each 39thread works on a single board at a time. It checks out the first commit, 40configures it for that board, then builds it. Then it checks out the next 41commit and builds it (typically without re-configuring). When it runs out 42of commits, it gets another job from the builder and starts again with that 43board. 44 45Clearly the builder threads could work either way - they could check out a 46commit and then built it for all boards. Using separate directories for each 47commit/board pair they could leave their build product around afterwards 48also. 49 50The intent behind building a single board for multiple commits, is to make 51use of incremental builds. Since each commit is built incrementally from 52the previous one, builds are faster. Reconfiguring for a different board 53removes all intermediate object files. 54 55Many threads can be working at once, but each has its own working directory. 56When a thread finishes a build, it puts the output files into a result 57directory. 58 59The base directory used by buildman is normally '../<branch>', i.e. 60a directory higher than the source repository and named after the branch 61being built. 62 63Within the base directory, we have one subdirectory for each commit. Within 64that is one subdirectory for each board. Within that is the build output for 65that commit/board combination. 66 67Buildman also create working directories for each thread, in a .bm-work/ 68subdirectory in the base dir. 69 70As an example, say we are building branch 'us-net' for boards 'sandbox' and 71'seaboard', and say that us-net has two commits. We will have directories 72like this: 73 74us-net/ base directory 75 01_of_02_g4ed4ebc_net--Add-tftp-speed-/ 76 sandbox/ 77 u-boot.bin 78 seaboard/ 79 u-boot.bin 80 02_of_02_g4ed4ebc_net--Check-tftp-comp/ 81 sandbox/ 82 u-boot.bin 83 seaboard/ 84 u-boot.bin 85 .bm-work/ 86 00/ working directory for thread 0 (contains source checkout) 87 build/ build output 88 01/ working directory for thread 1 89 build/ build output 90 ... 91u-boot/ source directory 92 .git/ repository 93""" 94 95# Possible build outcomes 96OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4) 97 98# Translate a commit subject into a valid filename 99trans_valid_chars = string.maketrans("/: ", "---") 100 101CONFIG_FILENAMES = [ 102 '.config', '.config-spl', '.config-tpl', 103 'autoconf.mk', 'autoconf-spl.mk', 'autoconf-tpl.mk', 104 'autoconf.h', 'autoconf-spl.h','autoconf-tpl.h', 105 'u-boot.cfg', 'u-boot-spl.cfg', 'u-boot-tpl.cfg' 106] 107 108class Config: 109 """Holds information about configuration settings for a board.""" 110 def __init__(self, target): 111 self.target = target 112 self.config = {} 113 for fname in CONFIG_FILENAMES: 114 self.config[fname] = {} 115 116 def Add(self, fname, key, value): 117 self.config[fname][key] = value 118 119 def __hash__(self): 120 val = 0 121 for fname in self.config: 122 for key, value in self.config[fname].iteritems(): 123 print key, value 124 val = val ^ hash(key) & hash(value) 125 return val 126 127class Builder: 128 """Class for building U-Boot for a particular commit. 129 130 Public members: (many should ->private) 131 already_done: Number of builds already completed 132 base_dir: Base directory to use for builder 133 checkout: True to check out source, False to skip that step. 134 This is used for testing. 135 col: terminal.Color() object 136 count: Number of commits to build 137 do_make: Method to call to invoke Make 138 fail: Number of builds that failed due to error 139 force_build: Force building even if a build already exists 140 force_config_on_failure: If a commit fails for a board, disable 141 incremental building for the next commit we build for that 142 board, so that we will see all warnings/errors again. 143 force_build_failures: If a previously-built build (i.e. built on 144 a previous run of buildman) is marked as failed, rebuild it. 145 git_dir: Git directory containing source repository 146 last_line_len: Length of the last line we printed (used for erasing 147 it with new progress information) 148 num_jobs: Number of jobs to run at once (passed to make as -j) 149 num_threads: Number of builder threads to run 150 out_queue: Queue of results to process 151 re_make_err: Compiled regular expression for ignore_lines 152 queue: Queue of jobs to run 153 threads: List of active threads 154 toolchains: Toolchains object to use for building 155 upto: Current commit number we are building (0.count-1) 156 warned: Number of builds that produced at least one warning 157 force_reconfig: Reconfigure U-Boot on each comiit. This disables 158 incremental building, where buildman reconfigures on the first 159 commit for a baord, and then just does an incremental build for 160 the following commits. In fact buildman will reconfigure and 161 retry for any failing commits, so generally the only effect of 162 this option is to slow things down. 163 in_tree: Build U-Boot in-tree instead of specifying an output 164 directory separate from the source code. This option is really 165 only useful for testing in-tree builds. 166 167 Private members: 168 _base_board_dict: Last-summarised Dict of boards 169 _base_err_lines: Last-summarised list of errors 170 _base_warn_lines: Last-summarised list of warnings 171 _build_period_us: Time taken for a single build (float object). 172 _complete_delay: Expected delay until completion (timedelta) 173 _next_delay_update: Next time we plan to display a progress update 174 (datatime) 175 _show_unknown: Show unknown boards (those not built) in summary 176 _timestamps: List of timestamps for the completion of the last 177 last _timestamp_count builds. Each is a datetime object. 178 _timestamp_count: Number of timestamps to keep in our list. 179 _working_dir: Base working directory containing all threads 180 """ 181 class Outcome: 182 """Records a build outcome for a single make invocation 183 184 Public Members: 185 rc: Outcome value (OUTCOME_...) 186 err_lines: List of error lines or [] if none 187 sizes: Dictionary of image size information, keyed by filename 188 - Each value is itself a dictionary containing 189 values for 'text', 'data' and 'bss', being the integer 190 size in bytes of each section. 191 func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each 192 value is itself a dictionary: 193 key: function name 194 value: Size of function in bytes 195 config: Dictionary keyed by filename - e.g. '.config'. Each 196 value is itself a dictionary: 197 key: config name 198 value: config value 199 """ 200 def __init__(self, rc, err_lines, sizes, func_sizes, config): 201 self.rc = rc 202 self.err_lines = err_lines 203 self.sizes = sizes 204 self.func_sizes = func_sizes 205 self.config = config 206 207 def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, 208 gnu_make='make', checkout=True, show_unknown=True, step=1, 209 no_subdirs=False, full_path=False, verbose_build=False, 210 incremental=False, per_board_out_dir=False, 211 config_only=False): 212 """Create a new Builder object 213 214 Args: 215 toolchains: Toolchains object to use for building 216 base_dir: Base directory to use for builder 217 git_dir: Git directory containing source repository 218 num_threads: Number of builder threads to run 219 num_jobs: Number of jobs to run at once (passed to make as -j) 220 gnu_make: the command name of GNU Make. 221 checkout: True to check out source, False to skip that step. 222 This is used for testing. 223 show_unknown: Show unknown boards (those not built) in summary 224 step: 1 to process every commit, n to process every nth commit 225 no_subdirs: Don't create subdirectories when building current 226 source for a single board 227 full_path: Return the full path in CROSS_COMPILE and don't set 228 PATH 229 verbose_build: Run build with V=1 and don't use 'make -s' 230 incremental: Always perform incremental builds; don't run make 231 mrproper when configuring 232 per_board_out_dir: Build in a separate persistent directory per 233 board rather than a thread-specific directory 234 config_only: Only configure each build, don't build it 235 """ 236 self.toolchains = toolchains 237 self.base_dir = base_dir 238 self._working_dir = os.path.join(base_dir, '.bm-work') 239 self.threads = [] 240 self.do_make = self.Make 241 self.gnu_make = gnu_make 242 self.checkout = checkout 243 self.num_threads = num_threads 244 self.num_jobs = num_jobs 245 self.already_done = 0 246 self.force_build = False 247 self.git_dir = git_dir 248 self._show_unknown = show_unknown 249 self._timestamp_count = 10 250 self._build_period_us = None 251 self._complete_delay = None 252 self._next_delay_update = datetime.now() 253 self.force_config_on_failure = True 254 self.force_build_failures = False 255 self.force_reconfig = False 256 self._step = step 257 self.in_tree = False 258 self._error_lines = 0 259 self.no_subdirs = no_subdirs 260 self.full_path = full_path 261 self.verbose_build = verbose_build 262 self.config_only = config_only 263 264 self.col = terminal.Color() 265 266 self._re_function = re.compile('(.*): In function.*') 267 self._re_files = re.compile('In file included from.*') 268 self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*') 269 self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*') 270 271 self.queue = Queue.Queue() 272 self.out_queue = Queue.Queue() 273 for i in range(self.num_threads): 274 t = builderthread.BuilderThread(self, i, incremental, 275 per_board_out_dir) 276 t.setDaemon(True) 277 t.start() 278 self.threads.append(t) 279 280 self.last_line_len = 0 281 t = builderthread.ResultThread(self) 282 t.setDaemon(True) 283 t.start() 284 self.threads.append(t) 285 286 ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)'] 287 self.re_make_err = re.compile('|'.join(ignore_lines)) 288 289 # Handle existing graceful with SIGINT / Ctrl-C 290 signal.signal(signal.SIGINT, self.signal_handler) 291 292 def __del__(self): 293 """Get rid of all threads created by the builder""" 294 for t in self.threads: 295 del t 296 297 def signal_handler(self, signal, frame): 298 sys.exit(1) 299 300 def SetDisplayOptions(self, show_errors=False, show_sizes=False, 301 show_detail=False, show_bloat=False, 302 list_error_boards=False, show_config=False): 303 """Setup display options for the builder. 304 305 show_errors: True to show summarised error/warning info 306 show_sizes: Show size deltas 307 show_detail: Show detail for each board 308 show_bloat: Show detail for each function 309 list_error_boards: Show the boards which caused each error/warning 310 show_config: Show config deltas 311 """ 312 self._show_errors = show_errors 313 self._show_sizes = show_sizes 314 self._show_detail = show_detail 315 self._show_bloat = show_bloat 316 self._list_error_boards = list_error_boards 317 self._show_config = show_config 318 319 def _AddTimestamp(self): 320 """Add a new timestamp to the list and record the build period. 321 322 The build period is the length of time taken to perform a single 323 build (one board, one commit). 324 """ 325 now = datetime.now() 326 self._timestamps.append(now) 327 count = len(self._timestamps) 328 delta = self._timestamps[-1] - self._timestamps[0] 329 seconds = delta.total_seconds() 330 331 # If we have enough data, estimate build period (time taken for a 332 # single build) and therefore completion time. 333 if count > 1 and self._next_delay_update < now: 334 self._next_delay_update = now + timedelta(seconds=2) 335 if seconds > 0: 336 self._build_period = float(seconds) / count 337 todo = self.count - self.upto 338 self._complete_delay = timedelta(microseconds= 339 self._build_period * todo * 1000000) 340 # Round it 341 self._complete_delay -= timedelta( 342 microseconds=self._complete_delay.microseconds) 343 344 if seconds > 60: 345 self._timestamps.popleft() 346 count -= 1 347 348 def ClearLine(self, length): 349 """Clear any characters on the current line 350 351 Make way for a new line of length 'length', by outputting enough 352 spaces to clear out the old line. Then remember the new length for 353 next time. 354 355 Args: 356 length: Length of new line, in characters 357 """ 358 if length < self.last_line_len: 359 Print(' ' * (self.last_line_len - length), newline=False) 360 Print('\r', newline=False) 361 self.last_line_len = length 362 sys.stdout.flush() 363 364 def SelectCommit(self, commit, checkout=True): 365 """Checkout the selected commit for this build 366 """ 367 self.commit = commit 368 if checkout and self.checkout: 369 gitutil.Checkout(commit.hash) 370 371 def Make(self, commit, brd, stage, cwd, *args, **kwargs): 372 """Run make 373 374 Args: 375 commit: Commit object that is being built 376 brd: Board object that is being built 377 stage: Stage that we are at (mrproper, config, build) 378 cwd: Directory where make should be run 379 args: Arguments to pass to make 380 kwargs: Arguments to pass to command.RunPipe() 381 """ 382 cmd = [self.gnu_make] + list(args) 383 result = command.RunPipe([cmd], capture=True, capture_stderr=True, 384 cwd=cwd, raise_on_error=False, **kwargs) 385 if self.verbose_build: 386 result.stdout = '%s\n' % (' '.join(cmd)) + result.stdout 387 result.combined = '%s\n' % (' '.join(cmd)) + result.combined 388 return result 389 390 def ProcessResult(self, result): 391 """Process the result of a build, showing progress information 392 393 Args: 394 result: A CommandResult object, which indicates the result for 395 a single build 396 """ 397 col = terminal.Color() 398 if result: 399 target = result.brd.target 400 401 self.upto += 1 402 if result.return_code != 0: 403 self.fail += 1 404 elif result.stderr: 405 self.warned += 1 406 if result.already_done: 407 self.already_done += 1 408 if self._verbose: 409 Print('\r', newline=False) 410 self.ClearLine(0) 411 boards_selected = {target : result.brd} 412 self.ResetResultSummary(boards_selected) 413 self.ProduceResultSummary(result.commit_upto, self.commits, 414 boards_selected) 415 else: 416 target = '(starting)' 417 418 # Display separate counts for ok, warned and fail 419 ok = self.upto - self.warned - self.fail 420 line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok) 421 line += self.col.Color(self.col.YELLOW, '%5d' % self.warned) 422 line += self.col.Color(self.col.RED, '%5d' % self.fail) 423 424 name = ' /%-5d ' % self.count 425 426 # Add our current completion time estimate 427 self._AddTimestamp() 428 if self._complete_delay: 429 name += '%s : ' % self._complete_delay 430 # When building all boards for a commit, we can print a commit 431 # progress message. 432 if result and result.commit_upto is None: 433 name += 'commit %2d/%-3d' % (self.commit_upto + 1, 434 self.commit_count) 435 436 name += target 437 Print(line + name, newline=False) 438 length = 14 + len(name) 439 self.ClearLine(length) 440 441 def _GetOutputDir(self, commit_upto): 442 """Get the name of the output directory for a commit number 443 444 The output directory is typically .../<branch>/<commit>. 445 446 Args: 447 commit_upto: Commit number to use (0..self.count-1) 448 """ 449 commit_dir = None 450 if self.commits: 451 commit = self.commits[commit_upto] 452 subject = commit.subject.translate(trans_valid_chars) 453 commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1, 454 self.commit_count, commit.hash, subject[:20])) 455 elif not self.no_subdirs: 456 commit_dir = 'current' 457 if not commit_dir: 458 return self.base_dir 459 return os.path.join(self.base_dir, commit_dir) 460 461 def GetBuildDir(self, commit_upto, target): 462 """Get the name of the build directory for a commit number 463 464 The build directory is typically .../<branch>/<commit>/<target>. 465 466 Args: 467 commit_upto: Commit number to use (0..self.count-1) 468 target: Target name 469 """ 470 output_dir = self._GetOutputDir(commit_upto) 471 return os.path.join(output_dir, target) 472 473 def GetDoneFile(self, commit_upto, target): 474 """Get the name of the done file for a commit number 475 476 Args: 477 commit_upto: Commit number to use (0..self.count-1) 478 target: Target name 479 """ 480 return os.path.join(self.GetBuildDir(commit_upto, target), 'done') 481 482 def GetSizesFile(self, commit_upto, target): 483 """Get the name of the sizes file for a commit number 484 485 Args: 486 commit_upto: Commit number to use (0..self.count-1) 487 target: Target name 488 """ 489 return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes') 490 491 def GetFuncSizesFile(self, commit_upto, target, elf_fname): 492 """Get the name of the funcsizes file for a commit number and ELF file 493 494 Args: 495 commit_upto: Commit number to use (0..self.count-1) 496 target: Target name 497 elf_fname: Filename of elf image 498 """ 499 return os.path.join(self.GetBuildDir(commit_upto, target), 500 '%s.sizes' % elf_fname.replace('/', '-')) 501 502 def GetObjdumpFile(self, commit_upto, target, elf_fname): 503 """Get the name of the objdump file for a commit number and ELF file 504 505 Args: 506 commit_upto: Commit number to use (0..self.count-1) 507 target: Target name 508 elf_fname: Filename of elf image 509 """ 510 return os.path.join(self.GetBuildDir(commit_upto, target), 511 '%s.objdump' % elf_fname.replace('/', '-')) 512 513 def GetErrFile(self, commit_upto, target): 514 """Get the name of the err file for a commit number 515 516 Args: 517 commit_upto: Commit number to use (0..self.count-1) 518 target: Target name 519 """ 520 output_dir = self.GetBuildDir(commit_upto, target) 521 return os.path.join(output_dir, 'err') 522 523 def FilterErrors(self, lines): 524 """Filter out errors in which we have no interest 525 526 We should probably use map(). 527 528 Args: 529 lines: List of error lines, each a string 530 Returns: 531 New list with only interesting lines included 532 """ 533 out_lines = [] 534 for line in lines: 535 if not self.re_make_err.search(line): 536 out_lines.append(line) 537 return out_lines 538 539 def ReadFuncSizes(self, fname, fd): 540 """Read function sizes from the output of 'nm' 541 542 Args: 543 fd: File containing data to read 544 fname: Filename we are reading from (just for errors) 545 546 Returns: 547 Dictionary containing size of each function in bytes, indexed by 548 function name. 549 """ 550 sym = {} 551 for line in fd.readlines(): 552 try: 553 size, type, name = line[:-1].split() 554 except: 555 Print("Invalid line in file '%s': '%s'" % (fname, line[:-1])) 556 continue 557 if type in 'tTdDbB': 558 # function names begin with '.' on 64-bit powerpc 559 if '.' in name[1:]: 560 name = 'static.' + name.split('.')[0] 561 sym[name] = sym.get(name, 0) + int(size, 16) 562 return sym 563 564 def _ProcessConfig(self, fname): 565 """Read in a .config, autoconf.mk or autoconf.h file 566 567 This function handles all config file types. It ignores comments and 568 any #defines which don't start with CONFIG_. 569 570 Args: 571 fname: Filename to read 572 573 Returns: 574 Dictionary: 575 key: Config name (e.g. CONFIG_DM) 576 value: Config value (e.g. 1) 577 """ 578 config = {} 579 if os.path.exists(fname): 580 with open(fname) as fd: 581 for line in fd: 582 line = line.strip() 583 if line.startswith('#define'): 584 values = line[8:].split(' ', 1) 585 if len(values) > 1: 586 key, value = values 587 else: 588 key = values[0] 589 value = '' 590 if not key.startswith('CONFIG_'): 591 continue 592 elif not line or line[0] in ['#', '*', '/']: 593 continue 594 else: 595 key, value = line.split('=', 1) 596 config[key] = value 597 return config 598 599 def GetBuildOutcome(self, commit_upto, target, read_func_sizes, 600 read_config): 601 """Work out the outcome of a build. 602 603 Args: 604 commit_upto: Commit number to check (0..n-1) 605 target: Target board to check 606 read_func_sizes: True to read function size information 607 read_config: True to read .config and autoconf.h files 608 609 Returns: 610 Outcome object 611 """ 612 done_file = self.GetDoneFile(commit_upto, target) 613 sizes_file = self.GetSizesFile(commit_upto, target) 614 sizes = {} 615 func_sizes = {} 616 config = {} 617 if os.path.exists(done_file): 618 with open(done_file, 'r') as fd: 619 return_code = int(fd.readline()) 620 err_lines = [] 621 err_file = self.GetErrFile(commit_upto, target) 622 if os.path.exists(err_file): 623 with open(err_file, 'r') as fd: 624 err_lines = self.FilterErrors(fd.readlines()) 625 626 # Decide whether the build was ok, failed or created warnings 627 if return_code: 628 rc = OUTCOME_ERROR 629 elif len(err_lines): 630 rc = OUTCOME_WARNING 631 else: 632 rc = OUTCOME_OK 633 634 # Convert size information to our simple format 635 if os.path.exists(sizes_file): 636 with open(sizes_file, 'r') as fd: 637 for line in fd.readlines(): 638 values = line.split() 639 rodata = 0 640 if len(values) > 6: 641 rodata = int(values[6], 16) 642 size_dict = { 643 'all' : int(values[0]) + int(values[1]) + 644 int(values[2]), 645 'text' : int(values[0]) - rodata, 646 'data' : int(values[1]), 647 'bss' : int(values[2]), 648 'rodata' : rodata, 649 } 650 sizes[values[5]] = size_dict 651 652 if read_func_sizes: 653 pattern = self.GetFuncSizesFile(commit_upto, target, '*') 654 for fname in glob.glob(pattern): 655 with open(fname, 'r') as fd: 656 dict_name = os.path.basename(fname).replace('.sizes', 657 '') 658 func_sizes[dict_name] = self.ReadFuncSizes(fname, fd) 659 660 if read_config: 661 output_dir = self.GetBuildDir(commit_upto, target) 662 for name in CONFIG_FILENAMES: 663 fname = os.path.join(output_dir, name) 664 config[name] = self._ProcessConfig(fname) 665 666 return Builder.Outcome(rc, err_lines, sizes, func_sizes, config) 667 668 return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}, {}) 669 670 def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes, 671 read_config): 672 """Calculate a summary of the results of building a commit. 673 674 Args: 675 board_selected: Dict containing boards to summarise 676 commit_upto: Commit number to summarize (0..self.count-1) 677 read_func_sizes: True to read function size information 678 read_config: True to read .config and autoconf.h files 679 680 Returns: 681 Tuple: 682 Dict containing boards which passed building this commit. 683 keyed by board.target 684 List containing a summary of error lines 685 Dict keyed by error line, containing a list of the Board 686 objects with that error 687 List containing a summary of warning lines 688 Dict keyed by error line, containing a list of the Board 689 objects with that warning 690 Dictionary keyed by board.target. Each value is a dictionary: 691 key: filename - e.g. '.config' 692 value is itself a dictionary: 693 key: config name 694 value: config value 695 """ 696 def AddLine(lines_summary, lines_boards, line, board): 697 line = line.rstrip() 698 if line in lines_boards: 699 lines_boards[line].append(board) 700 else: 701 lines_boards[line] = [board] 702 lines_summary.append(line) 703 704 board_dict = {} 705 err_lines_summary = [] 706 err_lines_boards = {} 707 warn_lines_summary = [] 708 warn_lines_boards = {} 709 config = {} 710 711 for board in boards_selected.itervalues(): 712 outcome = self.GetBuildOutcome(commit_upto, board.target, 713 read_func_sizes, read_config) 714 board_dict[board.target] = outcome 715 last_func = None 716 last_was_warning = False 717 for line in outcome.err_lines: 718 if line: 719 if (self._re_function.match(line) or 720 self._re_files.match(line)): 721 last_func = line 722 else: 723 is_warning = self._re_warning.match(line) 724 is_note = self._re_note.match(line) 725 if is_warning or (last_was_warning and is_note): 726 if last_func: 727 AddLine(warn_lines_summary, warn_lines_boards, 728 last_func, board) 729 AddLine(warn_lines_summary, warn_lines_boards, 730 line, board) 731 else: 732 if last_func: 733 AddLine(err_lines_summary, err_lines_boards, 734 last_func, board) 735 AddLine(err_lines_summary, err_lines_boards, 736 line, board) 737 last_was_warning = is_warning 738 last_func = None 739 tconfig = Config(board.target) 740 for fname in CONFIG_FILENAMES: 741 if outcome.config: 742 for key, value in outcome.config[fname].iteritems(): 743 tconfig.Add(fname, key, value) 744 config[board.target] = tconfig 745 746 return (board_dict, err_lines_summary, err_lines_boards, 747 warn_lines_summary, warn_lines_boards, config) 748 749 def AddOutcome(self, board_dict, arch_list, changes, char, color): 750 """Add an output to our list of outcomes for each architecture 751 752 This simple function adds failing boards (changes) to the 753 relevant architecture string, so we can print the results out 754 sorted by architecture. 755 756 Args: 757 board_dict: Dict containing all boards 758 arch_list: Dict keyed by arch name. Value is a string containing 759 a list of board names which failed for that arch. 760 changes: List of boards to add to arch_list 761 color: terminal.Colour object 762 """ 763 done_arch = {} 764 for target in changes: 765 if target in board_dict: 766 arch = board_dict[target].arch 767 else: 768 arch = 'unknown' 769 str = self.col.Color(color, ' ' + target) 770 if not arch in done_arch: 771 str = ' %s %s' % (self.col.Color(color, char), str) 772 done_arch[arch] = True 773 if not arch in arch_list: 774 arch_list[arch] = str 775 else: 776 arch_list[arch] += str 777 778 779 def ColourNum(self, num): 780 color = self.col.RED if num > 0 else self.col.GREEN 781 if num == 0: 782 return '0' 783 return self.col.Color(color, str(num)) 784 785 def ResetResultSummary(self, board_selected): 786 """Reset the results summary ready for use. 787 788 Set up the base board list to be all those selected, and set the 789 error lines to empty. 790 791 Following this, calls to PrintResultSummary() will use this 792 information to work out what has changed. 793 794 Args: 795 board_selected: Dict containing boards to summarise, keyed by 796 board.target 797 """ 798 self._base_board_dict = {} 799 for board in board_selected: 800 self._base_board_dict[board] = Builder.Outcome(0, [], [], {}, {}) 801 self._base_err_lines = [] 802 self._base_warn_lines = [] 803 self._base_err_line_boards = {} 804 self._base_warn_line_boards = {} 805 self._base_config = None 806 807 def PrintFuncSizeDetail(self, fname, old, new): 808 grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0 809 delta, common = [], {} 810 811 for a in old: 812 if a in new: 813 common[a] = 1 814 815 for name in old: 816 if name not in common: 817 remove += 1 818 down += old[name] 819 delta.append([-old[name], name]) 820 821 for name in new: 822 if name not in common: 823 add += 1 824 up += new[name] 825 delta.append([new[name], name]) 826 827 for name in common: 828 diff = new.get(name, 0) - old.get(name, 0) 829 if diff > 0: 830 grow, up = grow + 1, up + diff 831 elif diff < 0: 832 shrink, down = shrink + 1, down - diff 833 delta.append([diff, name]) 834 835 delta.sort() 836 delta.reverse() 837 838 args = [add, -remove, grow, -shrink, up, -down, up - down] 839 if max(args) == 0: 840 return 841 args = [self.ColourNum(x) for x in args] 842 indent = ' ' * 15 843 Print('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' % 844 tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args)) 845 Print('%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new', 846 'delta')) 847 for diff, name in delta: 848 if diff: 849 color = self.col.RED if diff > 0 else self.col.GREEN 850 msg = '%s %-38s %7s %7s %+7d' % (indent, name, 851 old.get(name, '-'), new.get(name,'-'), diff) 852 Print(msg, colour=color) 853 854 855 def PrintSizeDetail(self, target_list, show_bloat): 856 """Show details size information for each board 857 858 Args: 859 target_list: List of targets, each a dict containing: 860 'target': Target name 861 'total_diff': Total difference in bytes across all areas 862 <part_name>: Difference for that part 863 show_bloat: Show detail for each function 864 """ 865 targets_by_diff = sorted(target_list, reverse=True, 866 key=lambda x: x['_total_diff']) 867 for result in targets_by_diff: 868 printed_target = False 869 for name in sorted(result): 870 diff = result[name] 871 if name.startswith('_'): 872 continue 873 if diff != 0: 874 color = self.col.RED if diff > 0 else self.col.GREEN 875 msg = ' %s %+d' % (name, diff) 876 if not printed_target: 877 Print('%10s %-15s:' % ('', result['_target']), 878 newline=False) 879 printed_target = True 880 Print(msg, colour=color, newline=False) 881 if printed_target: 882 Print() 883 if show_bloat: 884 target = result['_target'] 885 outcome = result['_outcome'] 886 base_outcome = self._base_board_dict[target] 887 for fname in outcome.func_sizes: 888 self.PrintFuncSizeDetail(fname, 889 base_outcome.func_sizes[fname], 890 outcome.func_sizes[fname]) 891 892 893 def PrintSizeSummary(self, board_selected, board_dict, show_detail, 894 show_bloat): 895 """Print a summary of image sizes broken down by section. 896 897 The summary takes the form of one line per architecture. The 898 line contains deltas for each of the sections (+ means the section 899 got bigger, - means smaller). The nunmbers are the average number 900 of bytes that a board in this section increased by. 901 902 For example: 903 powerpc: (622 boards) text -0.0 904 arm: (285 boards) text -0.0 905 nds32: (3 boards) text -8.0 906 907 Args: 908 board_selected: Dict containing boards to summarise, keyed by 909 board.target 910 board_dict: Dict containing boards for which we built this 911 commit, keyed by board.target. The value is an Outcome object. 912 show_detail: Show detail for each board 913 show_bloat: Show detail for each function 914 """ 915 arch_list = {} 916 arch_count = {} 917 918 # Calculate changes in size for different image parts 919 # The previous sizes are in Board.sizes, for each board 920 for target in board_dict: 921 if target not in board_selected: 922 continue 923 base_sizes = self._base_board_dict[target].sizes 924 outcome = board_dict[target] 925 sizes = outcome.sizes 926 927 # Loop through the list of images, creating a dict of size 928 # changes for each image/part. We end up with something like 929 # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4} 930 # which means that U-Boot data increased by 5 bytes and SPL 931 # text decreased by 4. 932 err = {'_target' : target} 933 for image in sizes: 934 if image in base_sizes: 935 base_image = base_sizes[image] 936 # Loop through the text, data, bss parts 937 for part in sorted(sizes[image]): 938 diff = sizes[image][part] - base_image[part] 939 col = None 940 if diff: 941 if image == 'u-boot': 942 name = part 943 else: 944 name = image + ':' + part 945 err[name] = diff 946 arch = board_selected[target].arch 947 if not arch in arch_count: 948 arch_count[arch] = 1 949 else: 950 arch_count[arch] += 1 951 if not sizes: 952 pass # Only add to our list when we have some stats 953 elif not arch in arch_list: 954 arch_list[arch] = [err] 955 else: 956 arch_list[arch].append(err) 957 958 # We now have a list of image size changes sorted by arch 959 # Print out a summary of these 960 for arch, target_list in arch_list.iteritems(): 961 # Get total difference for each type 962 totals = {} 963 for result in target_list: 964 total = 0 965 for name, diff in result.iteritems(): 966 if name.startswith('_'): 967 continue 968 total += diff 969 if name in totals: 970 totals[name] += diff 971 else: 972 totals[name] = diff 973 result['_total_diff'] = total 974 result['_outcome'] = board_dict[result['_target']] 975 976 count = len(target_list) 977 printed_arch = False 978 for name in sorted(totals): 979 diff = totals[name] 980 if diff: 981 # Display the average difference in this name for this 982 # architecture 983 avg_diff = float(diff) / count 984 color = self.col.RED if avg_diff > 0 else self.col.GREEN 985 msg = ' %s %+1.1f' % (name, avg_diff) 986 if not printed_arch: 987 Print('%10s: (for %d/%d boards)' % (arch, count, 988 arch_count[arch]), newline=False) 989 printed_arch = True 990 Print(msg, colour=color, newline=False) 991 992 if printed_arch: 993 Print() 994 if show_detail: 995 self.PrintSizeDetail(target_list, show_bloat) 996 997 998 def PrintResultSummary(self, board_selected, board_dict, err_lines, 999 err_line_boards, warn_lines, warn_line_boards, 1000 config, show_sizes, show_detail, show_bloat, 1001 show_config): 1002 """Compare results with the base results and display delta. 1003 1004 Only boards mentioned in board_selected will be considered. This 1005 function is intended to be called repeatedly with the results of 1006 each commit. It therefore shows a 'diff' between what it saw in 1007 the last call and what it sees now. 1008 1009 Args: 1010 board_selected: Dict containing boards to summarise, keyed by 1011 board.target 1012 board_dict: Dict containing boards for which we built this 1013 commit, keyed by board.target. The value is an Outcome object. 1014 err_lines: A list of errors for this commit, or [] if there is 1015 none, or we don't want to print errors 1016 err_line_boards: Dict keyed by error line, containing a list of 1017 the Board objects with that error 1018 warn_lines: A list of warnings for this commit, or [] if there is 1019 none, or we don't want to print errors 1020 warn_line_boards: Dict keyed by warning line, containing a list of 1021 the Board objects with that warning 1022 config: Dictionary keyed by filename - e.g. '.config'. Each 1023 value is itself a dictionary: 1024 key: config name 1025 value: config value 1026 show_sizes: Show image size deltas 1027 show_detail: Show detail for each board 1028 show_bloat: Show detail for each function 1029 show_config: Show config changes 1030 """ 1031 def _BoardList(line, line_boards): 1032 """Helper function to get a line of boards containing a line 1033 1034 Args: 1035 line: Error line to search for 1036 Return: 1037 String containing a list of boards with that error line, or 1038 '' if the user has not requested such a list 1039 """ 1040 if self._list_error_boards: 1041 names = [] 1042 for board in line_boards[line]: 1043 if not board.target in names: 1044 names.append(board.target) 1045 names_str = '(%s) ' % ','.join(names) 1046 else: 1047 names_str = '' 1048 return names_str 1049 1050 def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards, 1051 char): 1052 better_lines = [] 1053 worse_lines = [] 1054 for line in lines: 1055 if line not in base_lines: 1056 worse_lines.append(char + '+' + 1057 _BoardList(line, line_boards) + line) 1058 for line in base_lines: 1059 if line not in lines: 1060 better_lines.append(char + '-' + 1061 _BoardList(line, base_line_boards) + line) 1062 return better_lines, worse_lines 1063 1064 def _CalcConfig(delta, name, config): 1065 """Calculate configuration changes 1066 1067 Args: 1068 delta: Type of the delta, e.g. '+' 1069 name: name of the file which changed (e.g. .config) 1070 config: configuration change dictionary 1071 key: config name 1072 value: config value 1073 Returns: 1074 String containing the configuration changes which can be 1075 printed 1076 """ 1077 out = '' 1078 for key in sorted(config.keys()): 1079 out += '%s=%s ' % (key, config[key]) 1080 return '%s %s: %s' % (delta, name, out) 1081 1082 def _AddConfig(lines, name, config_plus, config_minus, config_change): 1083 """Add changes in configuration to a list 1084 1085 Args: 1086 lines: list to add to 1087 name: config file name 1088 config_plus: configurations added, dictionary 1089 key: config name 1090 value: config value 1091 config_minus: configurations removed, dictionary 1092 key: config name 1093 value: config value 1094 config_change: configurations changed, dictionary 1095 key: config name 1096 value: config value 1097 """ 1098 if config_plus: 1099 lines.append(_CalcConfig('+', name, config_plus)) 1100 if config_minus: 1101 lines.append(_CalcConfig('-', name, config_minus)) 1102 if config_change: 1103 lines.append(_CalcConfig('c', name, config_change)) 1104 1105 def _OutputConfigInfo(lines): 1106 for line in lines: 1107 if not line: 1108 continue 1109 if line[0] == '+': 1110 col = self.col.GREEN 1111 elif line[0] == '-': 1112 col = self.col.RED 1113 elif line[0] == 'c': 1114 col = self.col.YELLOW 1115 Print(' ' + line, newline=True, colour=col) 1116 1117 1118 better = [] # List of boards fixed since last commit 1119 worse = [] # List of new broken boards since last commit 1120 new = [] # List of boards that didn't exist last time 1121 unknown = [] # List of boards that were not built 1122 1123 for target in board_dict: 1124 if target not in board_selected: 1125 continue 1126 1127 # If the board was built last time, add its outcome to a list 1128 if target in self._base_board_dict: 1129 base_outcome = self._base_board_dict[target].rc 1130 outcome = board_dict[target] 1131 if outcome.rc == OUTCOME_UNKNOWN: 1132 unknown.append(target) 1133 elif outcome.rc < base_outcome: 1134 better.append(target) 1135 elif outcome.rc > base_outcome: 1136 worse.append(target) 1137 else: 1138 new.append(target) 1139 1140 # Get a list of errors that have appeared, and disappeared 1141 better_err, worse_err = _CalcErrorDelta(self._base_err_lines, 1142 self._base_err_line_boards, err_lines, err_line_boards, '') 1143 better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines, 1144 self._base_warn_line_boards, warn_lines, warn_line_boards, 'w') 1145 1146 # Display results by arch 1147 if (better or worse or unknown or new or worse_err or better_err 1148 or worse_warn or better_warn): 1149 arch_list = {} 1150 self.AddOutcome(board_selected, arch_list, better, '', 1151 self.col.GREEN) 1152 self.AddOutcome(board_selected, arch_list, worse, '+', 1153 self.col.RED) 1154 self.AddOutcome(board_selected, arch_list, new, '*', self.col.BLUE) 1155 if self._show_unknown: 1156 self.AddOutcome(board_selected, arch_list, unknown, '?', 1157 self.col.MAGENTA) 1158 for arch, target_list in arch_list.iteritems(): 1159 Print('%10s: %s' % (arch, target_list)) 1160 self._error_lines += 1 1161 if better_err: 1162 Print('\n'.join(better_err), colour=self.col.GREEN) 1163 self._error_lines += 1 1164 if worse_err: 1165 Print('\n'.join(worse_err), colour=self.col.RED) 1166 self._error_lines += 1 1167 if better_warn: 1168 Print('\n'.join(better_warn), colour=self.col.CYAN) 1169 self._error_lines += 1 1170 if worse_warn: 1171 Print('\n'.join(worse_warn), colour=self.col.MAGENTA) 1172 self._error_lines += 1 1173 1174 if show_sizes: 1175 self.PrintSizeSummary(board_selected, board_dict, show_detail, 1176 show_bloat) 1177 1178 if show_config and self._base_config: 1179 summary = {} 1180 arch_config_plus = {} 1181 arch_config_minus = {} 1182 arch_config_change = {} 1183 arch_list = [] 1184 1185 for target in board_dict: 1186 if target not in board_selected: 1187 continue 1188 arch = board_selected[target].arch 1189 if arch not in arch_list: 1190 arch_list.append(arch) 1191 1192 for arch in arch_list: 1193 arch_config_plus[arch] = {} 1194 arch_config_minus[arch] = {} 1195 arch_config_change[arch] = {} 1196 for name in CONFIG_FILENAMES: 1197 arch_config_plus[arch][name] = {} 1198 arch_config_minus[arch][name] = {} 1199 arch_config_change[arch][name] = {} 1200 1201 for target in board_dict: 1202 if target not in board_selected: 1203 continue 1204 1205 arch = board_selected[target].arch 1206 1207 all_config_plus = {} 1208 all_config_minus = {} 1209 all_config_change = {} 1210 tbase = self._base_config[target] 1211 tconfig = config[target] 1212 lines = [] 1213 for name in CONFIG_FILENAMES: 1214 if not tconfig.config[name]: 1215 continue 1216 config_plus = {} 1217 config_minus = {} 1218 config_change = {} 1219 base = tbase.config[name] 1220 for key, value in tconfig.config[name].iteritems(): 1221 if key not in base: 1222 config_plus[key] = value 1223 all_config_plus[key] = value 1224 for key, value in base.iteritems(): 1225 if key not in tconfig.config[name]: 1226 config_minus[key] = value 1227 all_config_minus[key] = value 1228 for key, value in base.iteritems(): 1229 new_value = tconfig.config.get(key) 1230 if new_value and value != new_value: 1231 desc = '%s -> %s' % (value, new_value) 1232 config_change[key] = desc 1233 all_config_change[key] = desc 1234 1235 arch_config_plus[arch][name].update(config_plus) 1236 arch_config_minus[arch][name].update(config_minus) 1237 arch_config_change[arch][name].update(config_change) 1238 1239 _AddConfig(lines, name, config_plus, config_minus, 1240 config_change) 1241 _AddConfig(lines, 'all', all_config_plus, all_config_minus, 1242 all_config_change) 1243 summary[target] = '\n'.join(lines) 1244 1245 lines_by_target = {} 1246 for target, lines in summary.iteritems(): 1247 if lines in lines_by_target: 1248 lines_by_target[lines].append(target) 1249 else: 1250 lines_by_target[lines] = [target] 1251 1252 for arch in arch_list: 1253 lines = [] 1254 all_plus = {} 1255 all_minus = {} 1256 all_change = {} 1257 for name in CONFIG_FILENAMES: 1258 all_plus.update(arch_config_plus[arch][name]) 1259 all_minus.update(arch_config_minus[arch][name]) 1260 all_change.update(arch_config_change[arch][name]) 1261 _AddConfig(lines, name, arch_config_plus[arch][name], 1262 arch_config_minus[arch][name], 1263 arch_config_change[arch][name]) 1264 _AddConfig(lines, 'all', all_plus, all_minus, all_change) 1265 #arch_summary[target] = '\n'.join(lines) 1266 if lines: 1267 Print('%s:' % arch) 1268 _OutputConfigInfo(lines) 1269 1270 for lines, targets in lines_by_target.iteritems(): 1271 if not lines: 1272 continue 1273 Print('%s :' % ' '.join(sorted(targets))) 1274 _OutputConfigInfo(lines.split('\n')) 1275 1276 1277 # Save our updated information for the next call to this function 1278 self._base_board_dict = board_dict 1279 self._base_err_lines = err_lines 1280 self._base_warn_lines = warn_lines 1281 self._base_err_line_boards = err_line_boards 1282 self._base_warn_line_boards = warn_line_boards 1283 self._base_config = config 1284 1285 # Get a list of boards that did not get built, if needed 1286 not_built = [] 1287 for board in board_selected: 1288 if not board in board_dict: 1289 not_built.append(board) 1290 if not_built: 1291 Print("Boards not built (%d): %s" % (len(not_built), 1292 ', '.join(not_built))) 1293 1294 def ProduceResultSummary(self, commit_upto, commits, board_selected): 1295 (board_dict, err_lines, err_line_boards, warn_lines, 1296 warn_line_boards, config) = self.GetResultSummary( 1297 board_selected, commit_upto, 1298 read_func_sizes=self._show_bloat, 1299 read_config=self._show_config) 1300 if commits: 1301 msg = '%02d: %s' % (commit_upto + 1, 1302 commits[commit_upto].subject) 1303 Print(msg, colour=self.col.BLUE) 1304 self.PrintResultSummary(board_selected, board_dict, 1305 err_lines if self._show_errors else [], err_line_boards, 1306 warn_lines if self._show_errors else [], warn_line_boards, 1307 config, self._show_sizes, self._show_detail, 1308 self._show_bloat, self._show_config) 1309 1310 def ShowSummary(self, commits, board_selected): 1311 """Show a build summary for U-Boot for a given board list. 1312 1313 Reset the result summary, then repeatedly call GetResultSummary on 1314 each commit's results, then display the differences we see. 1315 1316 Args: 1317 commit: Commit objects to summarise 1318 board_selected: Dict containing boards to summarise 1319 """ 1320 self.commit_count = len(commits) if commits else 1 1321 self.commits = commits 1322 self.ResetResultSummary(board_selected) 1323 self._error_lines = 0 1324 1325 for commit_upto in range(0, self.commit_count, self._step): 1326 self.ProduceResultSummary(commit_upto, commits, board_selected) 1327 if not self._error_lines: 1328 Print('(no errors to report)', colour=self.col.GREEN) 1329 1330 1331 def SetupBuild(self, board_selected, commits): 1332 """Set up ready to start a build. 1333 1334 Args: 1335 board_selected: Selected boards to build 1336 commits: Selected commits to build 1337 """ 1338 # First work out how many commits we will build 1339 count = (self.commit_count + self._step - 1) / self._step 1340 self.count = len(board_selected) * count 1341 self.upto = self.warned = self.fail = 0 1342 self._timestamps = collections.deque() 1343 1344 def GetThreadDir(self, thread_num): 1345 """Get the directory path to the working dir for a thread. 1346 1347 Args: 1348 thread_num: Number of thread to check. 1349 """ 1350 return os.path.join(self._working_dir, '%02d' % thread_num) 1351 1352 def _PrepareThread(self, thread_num, setup_git): 1353 """Prepare the working directory for a thread. 1354 1355 This clones or fetches the repo into the thread's work directory. 1356 1357 Args: 1358 thread_num: Thread number (0, 1, ...) 1359 setup_git: True to set up a git repo clone 1360 """ 1361 thread_dir = self.GetThreadDir(thread_num) 1362 builderthread.Mkdir(thread_dir) 1363 git_dir = os.path.join(thread_dir, '.git') 1364 1365 # Clone the repo if it doesn't already exist 1366 # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so 1367 # we have a private index but uses the origin repo's contents? 1368 if setup_git and self.git_dir: 1369 src_dir = os.path.abspath(self.git_dir) 1370 if os.path.exists(git_dir): 1371 gitutil.Fetch(git_dir, thread_dir) 1372 else: 1373 Print('\rCloning repo for thread %d' % thread_num, 1374 newline=False) 1375 gitutil.Clone(src_dir, thread_dir) 1376 Print('\r%s\r' % (' ' * 30), newline=False) 1377 1378 def _PrepareWorkingSpace(self, max_threads, setup_git): 1379 """Prepare the working directory for use. 1380 1381 Set up the git repo for each thread. 1382 1383 Args: 1384 max_threads: Maximum number of threads we expect to need. 1385 setup_git: True to set up a git repo clone 1386 """ 1387 builderthread.Mkdir(self._working_dir) 1388 for thread in range(max_threads): 1389 self._PrepareThread(thread, setup_git) 1390 1391 def _PrepareOutputSpace(self): 1392 """Get the output directories ready to receive files. 1393 1394 We delete any output directories which look like ones we need to 1395 create. Having left over directories is confusing when the user wants 1396 to check the output manually. 1397 """ 1398 if not self.commits: 1399 return 1400 dir_list = [] 1401 for commit_upto in range(self.commit_count): 1402 dir_list.append(self._GetOutputDir(commit_upto)) 1403 1404 to_remove = [] 1405 for dirname in glob.glob(os.path.join(self.base_dir, '*')): 1406 if dirname not in dir_list: 1407 to_remove.append(dirname) 1408 if to_remove: 1409 Print('Removing %d old build directories' % len(to_remove), 1410 newline=False) 1411 for dirname in to_remove: 1412 shutil.rmtree(dirname) 1413 1414 def BuildBoards(self, commits, board_selected, keep_outputs, verbose): 1415 """Build all commits for a list of boards 1416 1417 Args: 1418 commits: List of commits to be build, each a Commit object 1419 boards_selected: Dict of selected boards, key is target name, 1420 value is Board object 1421 keep_outputs: True to save build output files 1422 verbose: Display build results as they are completed 1423 Returns: 1424 Tuple containing: 1425 - number of boards that failed to build 1426 - number of boards that issued warnings 1427 """ 1428 self.commit_count = len(commits) if commits else 1 1429 self.commits = commits 1430 self._verbose = verbose 1431 1432 self.ResetResultSummary(board_selected) 1433 builderthread.Mkdir(self.base_dir, parents = True) 1434 self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)), 1435 commits is not None) 1436 self._PrepareOutputSpace() 1437 Print('\rStarting build...', newline=False) 1438 self.SetupBuild(board_selected, commits) 1439 self.ProcessResult(None) 1440 1441 # Create jobs to build all commits for each board 1442 for brd in board_selected.itervalues(): 1443 job = builderthread.BuilderJob() 1444 job.board = brd 1445 job.commits = commits 1446 job.keep_outputs = keep_outputs 1447 job.step = self._step 1448 self.queue.put(job) 1449 1450 term = threading.Thread(target=self.queue.join) 1451 term.setDaemon(True) 1452 term.start() 1453 while term.isAlive(): 1454 term.join(100) 1455 1456 # Wait until we have processed all output 1457 self.out_queue.join() 1458 Print() 1459 self.ClearLine(0) 1460 return (self.fail, self.warned) 1461