1# Copyright (c) 2013 The Chromium OS Authors. 2# 3# Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com> 4# 5# SPDX-License-Identifier: GPL-2.0+ 6# 7 8import collections 9from datetime import datetime, timedelta 10import glob 11import os 12import re 13import Queue 14import shutil 15import string 16import sys 17import time 18 19import builderthread 20import command 21import gitutil 22import terminal 23import toolchain 24 25 26""" 27Theory of Operation 28 29Please see README for user documentation, and you should be familiar with 30that before trying to make sense of this. 31 32Buildman works by keeping the machine as busy as possible, building different 33commits for different boards on multiple CPUs at once. 34 35The source repo (self.git_dir) contains all the commits to be built. Each 36thread works on a single board at a time. It checks out the first commit, 37configures it for that board, then builds it. Then it checks out the next 38commit and builds it (typically without re-configuring). When it runs out 39of commits, it gets another job from the builder and starts again with that 40board. 41 42Clearly the builder threads could work either way - they could check out a 43commit and then built it for all boards. Using separate directories for each 44commit/board pair they could leave their build product around afterwards 45also. 46 47The intent behind building a single board for multiple commits, is to make 48use of incremental builds. Since each commit is built incrementally from 49the previous one, builds are faster. Reconfiguring for a different board 50removes all intermediate object files. 51 52Many threads can be working at once, but each has its own working directory. 53When a thread finishes a build, it puts the output files into a result 54directory. 55 56The base directory used by buildman is normally '../<branch>', i.e. 57a directory higher than the source repository and named after the branch 58being built. 59 60Within the base directory, we have one subdirectory for each commit. Within 61that is one subdirectory for each board. Within that is the build output for 62that commit/board combination. 63 64Buildman also create working directories for each thread, in a .bm-work/ 65subdirectory in the base dir. 66 67As an example, say we are building branch 'us-net' for boards 'sandbox' and 68'seaboard', and say that us-net has two commits. We will have directories 69like this: 70 71us-net/ base directory 72 01_of_02_g4ed4ebc_net--Add-tftp-speed-/ 73 sandbox/ 74 u-boot.bin 75 seaboard/ 76 u-boot.bin 77 02_of_02_g4ed4ebc_net--Check-tftp-comp/ 78 sandbox/ 79 u-boot.bin 80 seaboard/ 81 u-boot.bin 82 .bm-work/ 83 00/ working directory for thread 0 (contains source checkout) 84 build/ build output 85 01/ working directory for thread 1 86 build/ build output 87 ... 88u-boot/ source directory 89 .git/ repository 90""" 91 92# Possible build outcomes 93OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4) 94 95# Translate a commit subject into a valid filename 96trans_valid_chars = string.maketrans("/: ", "---") 97 98 99class Builder: 100 """Class for building U-Boot for a particular commit. 101 102 Public members: (many should ->private) 103 active: True if the builder is active and has not been stopped 104 already_done: Number of builds already completed 105 base_dir: Base directory to use for builder 106 checkout: True to check out source, False to skip that step. 107 This is used for testing. 108 col: terminal.Color() object 109 count: Number of commits to build 110 do_make: Method to call to invoke Make 111 fail: Number of builds that failed due to error 112 force_build: Force building even if a build already exists 113 force_config_on_failure: If a commit fails for a board, disable 114 incremental building for the next commit we build for that 115 board, so that we will see all warnings/errors again. 116 force_build_failures: If a previously-built build (i.e. built on 117 a previous run of buildman) is marked as failed, rebuild it. 118 git_dir: Git directory containing source repository 119 last_line_len: Length of the last line we printed (used for erasing 120 it with new progress information) 121 num_jobs: Number of jobs to run at once (passed to make as -j) 122 num_threads: Number of builder threads to run 123 out_queue: Queue of results to process 124 re_make_err: Compiled regular expression for ignore_lines 125 queue: Queue of jobs to run 126 threads: List of active threads 127 toolchains: Toolchains object to use for building 128 upto: Current commit number we are building (0.count-1) 129 warned: Number of builds that produced at least one warning 130 force_reconfig: Reconfigure U-Boot on each comiit. This disables 131 incremental building, where buildman reconfigures on the first 132 commit for a baord, and then just does an incremental build for 133 the following commits. In fact buildman will reconfigure and 134 retry for any failing commits, so generally the only effect of 135 this option is to slow things down. 136 in_tree: Build U-Boot in-tree instead of specifying an output 137 directory separate from the source code. This option is really 138 only useful for testing in-tree builds. 139 140 Private members: 141 _base_board_dict: Last-summarised Dict of boards 142 _base_err_lines: Last-summarised list of errors 143 _build_period_us: Time taken for a single build (float object). 144 _complete_delay: Expected delay until completion (timedelta) 145 _next_delay_update: Next time we plan to display a progress update 146 (datatime) 147 _show_unknown: Show unknown boards (those not built) in summary 148 _timestamps: List of timestamps for the completion of the last 149 last _timestamp_count builds. Each is a datetime object. 150 _timestamp_count: Number of timestamps to keep in our list. 151 _working_dir: Base working directory containing all threads 152 """ 153 class Outcome: 154 """Records a build outcome for a single make invocation 155 156 Public Members: 157 rc: Outcome value (OUTCOME_...) 158 err_lines: List of error lines or [] if none 159 sizes: Dictionary of image size information, keyed by filename 160 - Each value is itself a dictionary containing 161 values for 'text', 'data' and 'bss', being the integer 162 size in bytes of each section. 163 func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each 164 value is itself a dictionary: 165 key: function name 166 value: Size of function in bytes 167 """ 168 def __init__(self, rc, err_lines, sizes, func_sizes): 169 self.rc = rc 170 self.err_lines = err_lines 171 self.sizes = sizes 172 self.func_sizes = func_sizes 173 174 def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, 175 gnu_make='make', checkout=True, show_unknown=True, step=1): 176 """Create a new Builder object 177 178 Args: 179 toolchains: Toolchains object to use for building 180 base_dir: Base directory to use for builder 181 git_dir: Git directory containing source repository 182 num_threads: Number of builder threads to run 183 num_jobs: Number of jobs to run at once (passed to make as -j) 184 gnu_make: the command name of GNU Make. 185 checkout: True to check out source, False to skip that step. 186 This is used for testing. 187 show_unknown: Show unknown boards (those not built) in summary 188 step: 1 to process every commit, n to process every nth commit 189 """ 190 self.toolchains = toolchains 191 self.base_dir = base_dir 192 self._working_dir = os.path.join(base_dir, '.bm-work') 193 self.threads = [] 194 self.active = True 195 self.do_make = self.Make 196 self.gnu_make = gnu_make 197 self.checkout = checkout 198 self.num_threads = num_threads 199 self.num_jobs = num_jobs 200 self.already_done = 0 201 self.force_build = False 202 self.git_dir = git_dir 203 self._show_unknown = show_unknown 204 self._timestamp_count = 10 205 self._build_period_us = None 206 self._complete_delay = None 207 self._next_delay_update = datetime.now() 208 self.force_config_on_failure = True 209 self.force_build_failures = False 210 self.force_reconfig = False 211 self._step = step 212 self.in_tree = False 213 214 self.col = terminal.Color() 215 216 self.queue = Queue.Queue() 217 self.out_queue = Queue.Queue() 218 for i in range(self.num_threads): 219 t = builderthread.BuilderThread(self, i) 220 t.setDaemon(True) 221 t.start() 222 self.threads.append(t) 223 224 self.last_line_len = 0 225 t = builderthread.ResultThread(self) 226 t.setDaemon(True) 227 t.start() 228 self.threads.append(t) 229 230 ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)'] 231 self.re_make_err = re.compile('|'.join(ignore_lines)) 232 233 def __del__(self): 234 """Get rid of all threads created by the builder""" 235 for t in self.threads: 236 del t 237 238 def _AddTimestamp(self): 239 """Add a new timestamp to the list and record the build period. 240 241 The build period is the length of time taken to perform a single 242 build (one board, one commit). 243 """ 244 now = datetime.now() 245 self._timestamps.append(now) 246 count = len(self._timestamps) 247 delta = self._timestamps[-1] - self._timestamps[0] 248 seconds = delta.total_seconds() 249 250 # If we have enough data, estimate build period (time taken for a 251 # single build) and therefore completion time. 252 if count > 1 and self._next_delay_update < now: 253 self._next_delay_update = now + timedelta(seconds=2) 254 if seconds > 0: 255 self._build_period = float(seconds) / count 256 todo = self.count - self.upto 257 self._complete_delay = timedelta(microseconds= 258 self._build_period * todo * 1000000) 259 # Round it 260 self._complete_delay -= timedelta( 261 microseconds=self._complete_delay.microseconds) 262 263 if seconds > 60: 264 self._timestamps.popleft() 265 count -= 1 266 267 def ClearLine(self, length): 268 """Clear any characters on the current line 269 270 Make way for a new line of length 'length', by outputting enough 271 spaces to clear out the old line. Then remember the new length for 272 next time. 273 274 Args: 275 length: Length of new line, in characters 276 """ 277 if length < self.last_line_len: 278 print ' ' * (self.last_line_len - length), 279 print '\r', 280 self.last_line_len = length 281 sys.stdout.flush() 282 283 def SelectCommit(self, commit, checkout=True): 284 """Checkout the selected commit for this build 285 """ 286 self.commit = commit 287 if checkout and self.checkout: 288 gitutil.Checkout(commit.hash) 289 290 def Make(self, commit, brd, stage, cwd, *args, **kwargs): 291 """Run make 292 293 Args: 294 commit: Commit object that is being built 295 brd: Board object that is being built 296 stage: Stage that we are at (distclean, config, build) 297 cwd: Directory where make should be run 298 args: Arguments to pass to make 299 kwargs: Arguments to pass to command.RunPipe() 300 """ 301 cmd = [self.gnu_make] + list(args) 302 result = command.RunPipe([cmd], capture=True, capture_stderr=True, 303 cwd=cwd, raise_on_error=False, **kwargs) 304 return result 305 306 def ProcessResult(self, result): 307 """Process the result of a build, showing progress information 308 309 Args: 310 result: A CommandResult object 311 """ 312 col = terminal.Color() 313 if result: 314 target = result.brd.target 315 316 if result.return_code < 0: 317 self.active = False 318 command.StopAll() 319 return 320 321 self.upto += 1 322 if result.return_code != 0: 323 self.fail += 1 324 elif result.stderr: 325 self.warned += 1 326 if result.already_done: 327 self.already_done += 1 328 else: 329 target = '(starting)' 330 331 # Display separate counts for ok, warned and fail 332 ok = self.upto - self.warned - self.fail 333 line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok) 334 line += self.col.Color(self.col.YELLOW, '%5d' % self.warned) 335 line += self.col.Color(self.col.RED, '%5d' % self.fail) 336 337 name = ' /%-5d ' % self.count 338 339 # Add our current completion time estimate 340 self._AddTimestamp() 341 if self._complete_delay: 342 name += '%s : ' % self._complete_delay 343 # When building all boards for a commit, we can print a commit 344 # progress message. 345 if result and result.commit_upto is None: 346 name += 'commit %2d/%-3d' % (self.commit_upto + 1, 347 self.commit_count) 348 349 name += target 350 print line + name, 351 length = 13 + len(name) 352 self.ClearLine(length) 353 354 def _GetOutputDir(self, commit_upto): 355 """Get the name of the output directory for a commit number 356 357 The output directory is typically .../<branch>/<commit>. 358 359 Args: 360 commit_upto: Commit number to use (0..self.count-1) 361 """ 362 if self.commits: 363 commit = self.commits[commit_upto] 364 subject = commit.subject.translate(trans_valid_chars) 365 commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1, 366 self.commit_count, commit.hash, subject[:20])) 367 else: 368 commit_dir = 'current' 369 output_dir = os.path.join(self.base_dir, commit_dir) 370 return output_dir 371 372 def GetBuildDir(self, commit_upto, target): 373 """Get the name of the build directory for a commit number 374 375 The build directory is typically .../<branch>/<commit>/<target>. 376 377 Args: 378 commit_upto: Commit number to use (0..self.count-1) 379 target: Target name 380 """ 381 output_dir = self._GetOutputDir(commit_upto) 382 return os.path.join(output_dir, target) 383 384 def GetDoneFile(self, commit_upto, target): 385 """Get the name of the done file for a commit number 386 387 Args: 388 commit_upto: Commit number to use (0..self.count-1) 389 target: Target name 390 """ 391 return os.path.join(self.GetBuildDir(commit_upto, target), 'done') 392 393 def GetSizesFile(self, commit_upto, target): 394 """Get the name of the sizes file for a commit number 395 396 Args: 397 commit_upto: Commit number to use (0..self.count-1) 398 target: Target name 399 """ 400 return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes') 401 402 def GetFuncSizesFile(self, commit_upto, target, elf_fname): 403 """Get the name of the funcsizes file for a commit number and ELF file 404 405 Args: 406 commit_upto: Commit number to use (0..self.count-1) 407 target: Target name 408 elf_fname: Filename of elf image 409 """ 410 return os.path.join(self.GetBuildDir(commit_upto, target), 411 '%s.sizes' % elf_fname.replace('/', '-')) 412 413 def GetObjdumpFile(self, commit_upto, target, elf_fname): 414 """Get the name of the objdump file for a commit number and ELF file 415 416 Args: 417 commit_upto: Commit number to use (0..self.count-1) 418 target: Target name 419 elf_fname: Filename of elf image 420 """ 421 return os.path.join(self.GetBuildDir(commit_upto, target), 422 '%s.objdump' % elf_fname.replace('/', '-')) 423 424 def GetErrFile(self, commit_upto, target): 425 """Get the name of the err file for a commit number 426 427 Args: 428 commit_upto: Commit number to use (0..self.count-1) 429 target: Target name 430 """ 431 output_dir = self.GetBuildDir(commit_upto, target) 432 return os.path.join(output_dir, 'err') 433 434 def FilterErrors(self, lines): 435 """Filter out errors in which we have no interest 436 437 We should probably use map(). 438 439 Args: 440 lines: List of error lines, each a string 441 Returns: 442 New list with only interesting lines included 443 """ 444 out_lines = [] 445 for line in lines: 446 if not self.re_make_err.search(line): 447 out_lines.append(line) 448 return out_lines 449 450 def ReadFuncSizes(self, fname, fd): 451 """Read function sizes from the output of 'nm' 452 453 Args: 454 fd: File containing data to read 455 fname: Filename we are reading from (just for errors) 456 457 Returns: 458 Dictionary containing size of each function in bytes, indexed by 459 function name. 460 """ 461 sym = {} 462 for line in fd.readlines(): 463 try: 464 size, type, name = line[:-1].split() 465 except: 466 print "Invalid line in file '%s': '%s'" % (fname, line[:-1]) 467 continue 468 if type in 'tTdDbB': 469 # function names begin with '.' on 64-bit powerpc 470 if '.' in name[1:]: 471 name = 'static.' + name.split('.')[0] 472 sym[name] = sym.get(name, 0) + int(size, 16) 473 return sym 474 475 def GetBuildOutcome(self, commit_upto, target, read_func_sizes): 476 """Work out the outcome of a build. 477 478 Args: 479 commit_upto: Commit number to check (0..n-1) 480 target: Target board to check 481 read_func_sizes: True to read function size information 482 483 Returns: 484 Outcome object 485 """ 486 done_file = self.GetDoneFile(commit_upto, target) 487 sizes_file = self.GetSizesFile(commit_upto, target) 488 sizes = {} 489 func_sizes = {} 490 if os.path.exists(done_file): 491 with open(done_file, 'r') as fd: 492 return_code = int(fd.readline()) 493 err_lines = [] 494 err_file = self.GetErrFile(commit_upto, target) 495 if os.path.exists(err_file): 496 with open(err_file, 'r') as fd: 497 err_lines = self.FilterErrors(fd.readlines()) 498 499 # Decide whether the build was ok, failed or created warnings 500 if return_code: 501 rc = OUTCOME_ERROR 502 elif len(err_lines): 503 rc = OUTCOME_WARNING 504 else: 505 rc = OUTCOME_OK 506 507 # Convert size information to our simple format 508 if os.path.exists(sizes_file): 509 with open(sizes_file, 'r') as fd: 510 for line in fd.readlines(): 511 values = line.split() 512 rodata = 0 513 if len(values) > 6: 514 rodata = int(values[6], 16) 515 size_dict = { 516 'all' : int(values[0]) + int(values[1]) + 517 int(values[2]), 518 'text' : int(values[0]) - rodata, 519 'data' : int(values[1]), 520 'bss' : int(values[2]), 521 'rodata' : rodata, 522 } 523 sizes[values[5]] = size_dict 524 525 if read_func_sizes: 526 pattern = self.GetFuncSizesFile(commit_upto, target, '*') 527 for fname in glob.glob(pattern): 528 with open(fname, 'r') as fd: 529 dict_name = os.path.basename(fname).replace('.sizes', 530 '') 531 func_sizes[dict_name] = self.ReadFuncSizes(fname, fd) 532 533 return Builder.Outcome(rc, err_lines, sizes, func_sizes) 534 535 return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}) 536 537 def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes): 538 """Calculate a summary of the results of building a commit. 539 540 Args: 541 board_selected: Dict containing boards to summarise 542 commit_upto: Commit number to summarize (0..self.count-1) 543 read_func_sizes: True to read function size information 544 545 Returns: 546 Tuple: 547 Dict containing boards which passed building this commit. 548 keyed by board.target 549 List containing a summary of error/warning lines 550 """ 551 board_dict = {} 552 err_lines_summary = [] 553 554 for board in boards_selected.itervalues(): 555 outcome = self.GetBuildOutcome(commit_upto, board.target, 556 read_func_sizes) 557 board_dict[board.target] = outcome 558 for err in outcome.err_lines: 559 if err and not err.rstrip() in err_lines_summary: 560 err_lines_summary.append(err.rstrip()) 561 return board_dict, err_lines_summary 562 563 def AddOutcome(self, board_dict, arch_list, changes, char, color): 564 """Add an output to our list of outcomes for each architecture 565 566 This simple function adds failing boards (changes) to the 567 relevant architecture string, so we can print the results out 568 sorted by architecture. 569 570 Args: 571 board_dict: Dict containing all boards 572 arch_list: Dict keyed by arch name. Value is a string containing 573 a list of board names which failed for that arch. 574 changes: List of boards to add to arch_list 575 color: terminal.Colour object 576 """ 577 done_arch = {} 578 for target in changes: 579 if target in board_dict: 580 arch = board_dict[target].arch 581 else: 582 arch = 'unknown' 583 str = self.col.Color(color, ' ' + target) 584 if not arch in done_arch: 585 str = self.col.Color(color, char) + ' ' + str 586 done_arch[arch] = True 587 if not arch in arch_list: 588 arch_list[arch] = str 589 else: 590 arch_list[arch] += str 591 592 593 def ColourNum(self, num): 594 color = self.col.RED if num > 0 else self.col.GREEN 595 if num == 0: 596 return '0' 597 return self.col.Color(color, str(num)) 598 599 def ResetResultSummary(self, board_selected): 600 """Reset the results summary ready for use. 601 602 Set up the base board list to be all those selected, and set the 603 error lines to empty. 604 605 Following this, calls to PrintResultSummary() will use this 606 information to work out what has changed. 607 608 Args: 609 board_selected: Dict containing boards to summarise, keyed by 610 board.target 611 """ 612 self._base_board_dict = {} 613 for board in board_selected: 614 self._base_board_dict[board] = Builder.Outcome(0, [], [], {}) 615 self._base_err_lines = [] 616 617 def PrintFuncSizeDetail(self, fname, old, new): 618 grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0 619 delta, common = [], {} 620 621 for a in old: 622 if a in new: 623 common[a] = 1 624 625 for name in old: 626 if name not in common: 627 remove += 1 628 down += old[name] 629 delta.append([-old[name], name]) 630 631 for name in new: 632 if name not in common: 633 add += 1 634 up += new[name] 635 delta.append([new[name], name]) 636 637 for name in common: 638 diff = new.get(name, 0) - old.get(name, 0) 639 if diff > 0: 640 grow, up = grow + 1, up + diff 641 elif diff < 0: 642 shrink, down = shrink + 1, down - diff 643 delta.append([diff, name]) 644 645 delta.sort() 646 delta.reverse() 647 648 args = [add, -remove, grow, -shrink, up, -down, up - down] 649 if max(args) == 0: 650 return 651 args = [self.ColourNum(x) for x in args] 652 indent = ' ' * 15 653 print ('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' % 654 tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args)) 655 print '%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new', 656 'delta') 657 for diff, name in delta: 658 if diff: 659 color = self.col.RED if diff > 0 else self.col.GREEN 660 msg = '%s %-38s %7s %7s %+7d' % (indent, name, 661 old.get(name, '-'), new.get(name,'-'), diff) 662 print self.col.Color(color, msg) 663 664 665 def PrintSizeDetail(self, target_list, show_bloat): 666 """Show details size information for each board 667 668 Args: 669 target_list: List of targets, each a dict containing: 670 'target': Target name 671 'total_diff': Total difference in bytes across all areas 672 <part_name>: Difference for that part 673 show_bloat: Show detail for each function 674 """ 675 targets_by_diff = sorted(target_list, reverse=True, 676 key=lambda x: x['_total_diff']) 677 for result in targets_by_diff: 678 printed_target = False 679 for name in sorted(result): 680 diff = result[name] 681 if name.startswith('_'): 682 continue 683 if diff != 0: 684 color = self.col.RED if diff > 0 else self.col.GREEN 685 msg = ' %s %+d' % (name, diff) 686 if not printed_target: 687 print '%10s %-15s:' % ('', result['_target']), 688 printed_target = True 689 print self.col.Color(color, msg), 690 if printed_target: 691 print 692 if show_bloat: 693 target = result['_target'] 694 outcome = result['_outcome'] 695 base_outcome = self._base_board_dict[target] 696 for fname in outcome.func_sizes: 697 self.PrintFuncSizeDetail(fname, 698 base_outcome.func_sizes[fname], 699 outcome.func_sizes[fname]) 700 701 702 def PrintSizeSummary(self, board_selected, board_dict, show_detail, 703 show_bloat): 704 """Print a summary of image sizes broken down by section. 705 706 The summary takes the form of one line per architecture. The 707 line contains deltas for each of the sections (+ means the section 708 got bigger, - means smaller). The nunmbers are the average number 709 of bytes that a board in this section increased by. 710 711 For example: 712 powerpc: (622 boards) text -0.0 713 arm: (285 boards) text -0.0 714 nds32: (3 boards) text -8.0 715 716 Args: 717 board_selected: Dict containing boards to summarise, keyed by 718 board.target 719 board_dict: Dict containing boards for which we built this 720 commit, keyed by board.target. The value is an Outcome object. 721 show_detail: Show detail for each board 722 show_bloat: Show detail for each function 723 """ 724 arch_list = {} 725 arch_count = {} 726 727 # Calculate changes in size for different image parts 728 # The previous sizes are in Board.sizes, for each board 729 for target in board_dict: 730 if target not in board_selected: 731 continue 732 base_sizes = self._base_board_dict[target].sizes 733 outcome = board_dict[target] 734 sizes = outcome.sizes 735 736 # Loop through the list of images, creating a dict of size 737 # changes for each image/part. We end up with something like 738 # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4} 739 # which means that U-Boot data increased by 5 bytes and SPL 740 # text decreased by 4. 741 err = {'_target' : target} 742 for image in sizes: 743 if image in base_sizes: 744 base_image = base_sizes[image] 745 # Loop through the text, data, bss parts 746 for part in sorted(sizes[image]): 747 diff = sizes[image][part] - base_image[part] 748 col = None 749 if diff: 750 if image == 'u-boot': 751 name = part 752 else: 753 name = image + ':' + part 754 err[name] = diff 755 arch = board_selected[target].arch 756 if not arch in arch_count: 757 arch_count[arch] = 1 758 else: 759 arch_count[arch] += 1 760 if not sizes: 761 pass # Only add to our list when we have some stats 762 elif not arch in arch_list: 763 arch_list[arch] = [err] 764 else: 765 arch_list[arch].append(err) 766 767 # We now have a list of image size changes sorted by arch 768 # Print out a summary of these 769 for arch, target_list in arch_list.iteritems(): 770 # Get total difference for each type 771 totals = {} 772 for result in target_list: 773 total = 0 774 for name, diff in result.iteritems(): 775 if name.startswith('_'): 776 continue 777 total += diff 778 if name in totals: 779 totals[name] += diff 780 else: 781 totals[name] = diff 782 result['_total_diff'] = total 783 result['_outcome'] = board_dict[result['_target']] 784 785 count = len(target_list) 786 printed_arch = False 787 for name in sorted(totals): 788 diff = totals[name] 789 if diff: 790 # Display the average difference in this name for this 791 # architecture 792 avg_diff = float(diff) / count 793 color = self.col.RED if avg_diff > 0 else self.col.GREEN 794 msg = ' %s %+1.1f' % (name, avg_diff) 795 if not printed_arch: 796 print '%10s: (for %d/%d boards)' % (arch, count, 797 arch_count[arch]), 798 printed_arch = True 799 print self.col.Color(color, msg), 800 801 if printed_arch: 802 print 803 if show_detail: 804 self.PrintSizeDetail(target_list, show_bloat) 805 806 807 def PrintResultSummary(self, board_selected, board_dict, err_lines, 808 show_sizes, show_detail, show_bloat): 809 """Compare results with the base results and display delta. 810 811 Only boards mentioned in board_selected will be considered. This 812 function is intended to be called repeatedly with the results of 813 each commit. It therefore shows a 'diff' between what it saw in 814 the last call and what it sees now. 815 816 Args: 817 board_selected: Dict containing boards to summarise, keyed by 818 board.target 819 board_dict: Dict containing boards for which we built this 820 commit, keyed by board.target. The value is an Outcome object. 821 err_lines: A list of errors for this commit, or [] if there is 822 none, or we don't want to print errors 823 show_sizes: Show image size deltas 824 show_detail: Show detail for each board 825 show_bloat: Show detail for each function 826 """ 827 better = [] # List of boards fixed since last commit 828 worse = [] # List of new broken boards since last commit 829 new = [] # List of boards that didn't exist last time 830 unknown = [] # List of boards that were not built 831 832 for target in board_dict: 833 if target not in board_selected: 834 continue 835 836 # If the board was built last time, add its outcome to a list 837 if target in self._base_board_dict: 838 base_outcome = self._base_board_dict[target].rc 839 outcome = board_dict[target] 840 if outcome.rc == OUTCOME_UNKNOWN: 841 unknown.append(target) 842 elif outcome.rc < base_outcome: 843 better.append(target) 844 elif outcome.rc > base_outcome: 845 worse.append(target) 846 else: 847 new.append(target) 848 849 # Get a list of errors that have appeared, and disappeared 850 better_err = [] 851 worse_err = [] 852 for line in err_lines: 853 if line not in self._base_err_lines: 854 worse_err.append('+' + line) 855 for line in self._base_err_lines: 856 if line not in err_lines: 857 better_err.append('-' + line) 858 859 # Display results by arch 860 if better or worse or unknown or new or worse_err or better_err: 861 arch_list = {} 862 self.AddOutcome(board_selected, arch_list, better, '', 863 self.col.GREEN) 864 self.AddOutcome(board_selected, arch_list, worse, '+', 865 self.col.RED) 866 self.AddOutcome(board_selected, arch_list, new, '*', self.col.BLUE) 867 if self._show_unknown: 868 self.AddOutcome(board_selected, arch_list, unknown, '?', 869 self.col.MAGENTA) 870 for arch, target_list in arch_list.iteritems(): 871 print '%10s: %s' % (arch, target_list) 872 if better_err: 873 print self.col.Color(self.col.GREEN, '\n'.join(better_err)) 874 if worse_err: 875 print self.col.Color(self.col.RED, '\n'.join(worse_err)) 876 877 if show_sizes: 878 self.PrintSizeSummary(board_selected, board_dict, show_detail, 879 show_bloat) 880 881 # Save our updated information for the next call to this function 882 self._base_board_dict = board_dict 883 self._base_err_lines = err_lines 884 885 # Get a list of boards that did not get built, if needed 886 not_built = [] 887 for board in board_selected: 888 if not board in board_dict: 889 not_built.append(board) 890 if not_built: 891 print "Boards not built (%d): %s" % (len(not_built), 892 ', '.join(not_built)) 893 894 895 def ShowSummary(self, commits, board_selected, show_errors, show_sizes, 896 show_detail, show_bloat): 897 """Show a build summary for U-Boot for a given board list. 898 899 Reset the result summary, then repeatedly call GetResultSummary on 900 each commit's results, then display the differences we see. 901 902 Args: 903 commit: Commit objects to summarise 904 board_selected: Dict containing boards to summarise 905 show_errors: Show errors that occured 906 show_sizes: Show size deltas 907 show_detail: Show detail for each board 908 show_bloat: Show detail for each function 909 """ 910 self.commit_count = len(commits) if commits else 1 911 self.commits = commits 912 self.ResetResultSummary(board_selected) 913 914 for commit_upto in range(0, self.commit_count, self._step): 915 board_dict, err_lines = self.GetResultSummary(board_selected, 916 commit_upto, read_func_sizes=show_bloat) 917 if commits: 918 msg = '%02d: %s' % (commit_upto + 1, 919 commits[commit_upto].subject) 920 else: 921 msg = 'current' 922 print self.col.Color(self.col.BLUE, msg) 923 self.PrintResultSummary(board_selected, board_dict, 924 err_lines if show_errors else [], show_sizes, show_detail, 925 show_bloat) 926 927 928 def SetupBuild(self, board_selected, commits): 929 """Set up ready to start a build. 930 931 Args: 932 board_selected: Selected boards to build 933 commits: Selected commits to build 934 """ 935 # First work out how many commits we will build 936 count = (self.commit_count + self._step - 1) / self._step 937 self.count = len(board_selected) * count 938 self.upto = self.warned = self.fail = 0 939 self._timestamps = collections.deque() 940 941 def BuildBoardsForCommit(self, board_selected, keep_outputs): 942 """Build all boards for a single commit""" 943 self.SetupBuild(board_selected) 944 self.count = len(board_selected) 945 for brd in board_selected.itervalues(): 946 job = BuilderJob() 947 job.board = brd 948 job.commits = None 949 job.keep_outputs = keep_outputs 950 self.queue.put(brd) 951 952 self.queue.join() 953 self.out_queue.join() 954 print 955 self.ClearLine(0) 956 957 def BuildCommits(self, commits, board_selected, show_errors, keep_outputs): 958 """Build all boards for all commits (non-incremental)""" 959 self.commit_count = len(commits) 960 961 self.ResetResultSummary(board_selected) 962 for self.commit_upto in range(self.commit_count): 963 self.SelectCommit(commits[self.commit_upto]) 964 self.SelectOutputDir() 965 builderthread.Mkdir(self.output_dir) 966 967 self.BuildBoardsForCommit(board_selected, keep_outputs) 968 board_dict, err_lines = self.GetResultSummary() 969 self.PrintResultSummary(board_selected, board_dict, 970 err_lines if show_errors else []) 971 972 if self.already_done: 973 print '%d builds already done' % self.already_done 974 975 def GetThreadDir(self, thread_num): 976 """Get the directory path to the working dir for a thread. 977 978 Args: 979 thread_num: Number of thread to check. 980 """ 981 return os.path.join(self._working_dir, '%02d' % thread_num) 982 983 def _PrepareThread(self, thread_num, setup_git): 984 """Prepare the working directory for a thread. 985 986 This clones or fetches the repo into the thread's work directory. 987 988 Args: 989 thread_num: Thread number (0, 1, ...) 990 setup_git: True to set up a git repo clone 991 """ 992 thread_dir = self.GetThreadDir(thread_num) 993 builderthread.Mkdir(thread_dir) 994 git_dir = os.path.join(thread_dir, '.git') 995 996 # Clone the repo if it doesn't already exist 997 # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so 998 # we have a private index but uses the origin repo's contents? 999 if setup_git and self.git_dir: 1000 src_dir = os.path.abspath(self.git_dir) 1001 if os.path.exists(git_dir): 1002 gitutil.Fetch(git_dir, thread_dir) 1003 else: 1004 print 'Cloning repo for thread %d' % thread_num 1005 gitutil.Clone(src_dir, thread_dir) 1006 1007 def _PrepareWorkingSpace(self, max_threads, setup_git): 1008 """Prepare the working directory for use. 1009 1010 Set up the git repo for each thread. 1011 1012 Args: 1013 max_threads: Maximum number of threads we expect to need. 1014 setup_git: True to set up a git repo clone 1015 """ 1016 builderthread.Mkdir(self._working_dir) 1017 for thread in range(max_threads): 1018 self._PrepareThread(thread, setup_git) 1019 1020 def _PrepareOutputSpace(self): 1021 """Get the output directories ready to receive files. 1022 1023 We delete any output directories which look like ones we need to 1024 create. Having left over directories is confusing when the user wants 1025 to check the output manually. 1026 """ 1027 dir_list = [] 1028 for commit_upto in range(self.commit_count): 1029 dir_list.append(self._GetOutputDir(commit_upto)) 1030 1031 for dirname in glob.glob(os.path.join(self.base_dir, '*')): 1032 if dirname not in dir_list: 1033 shutil.rmtree(dirname) 1034 1035 def BuildBoards(self, commits, board_selected, show_errors, keep_outputs): 1036 """Build all commits for a list of boards 1037 1038 Args: 1039 commits: List of commits to be build, each a Commit object 1040 boards_selected: Dict of selected boards, key is target name, 1041 value is Board object 1042 show_errors: True to show summarised error/warning info 1043 keep_outputs: True to save build output files 1044 """ 1045 self.commit_count = len(commits) if commits else 1 1046 self.commits = commits 1047 1048 self.ResetResultSummary(board_selected) 1049 builderthread.Mkdir(self.base_dir) 1050 self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)), 1051 commits is not None) 1052 self._PrepareOutputSpace() 1053 self.SetupBuild(board_selected, commits) 1054 self.ProcessResult(None) 1055 1056 # Create jobs to build all commits for each board 1057 for brd in board_selected.itervalues(): 1058 job = builderthread.BuilderJob() 1059 job.board = brd 1060 job.commits = commits 1061 job.keep_outputs = keep_outputs 1062 job.step = self._step 1063 self.queue.put(job) 1064 1065 # Wait until all jobs are started 1066 self.queue.join() 1067 1068 # Wait until we have processed all output 1069 self.out_queue.join() 1070 print 1071 self.ClearLine(0) 1072