1# Copyright (c) 2013 The Chromium OS Authors. 2# 3# Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com> 4# 5# SPDX-License-Identifier: GPL-2.0+ 6# 7 8import collections 9from datetime import datetime, timedelta 10import glob 11import os 12import re 13import Queue 14import shutil 15import string 16import sys 17import time 18 19import builderthread 20import command 21import gitutil 22import terminal 23import toolchain 24 25 26""" 27Theory of Operation 28 29Please see README for user documentation, and you should be familiar with 30that before trying to make sense of this. 31 32Buildman works by keeping the machine as busy as possible, building different 33commits for different boards on multiple CPUs at once. 34 35The source repo (self.git_dir) contains all the commits to be built. Each 36thread works on a single board at a time. It checks out the first commit, 37configures it for that board, then builds it. Then it checks out the next 38commit and builds it (typically without re-configuring). When it runs out 39of commits, it gets another job from the builder and starts again with that 40board. 41 42Clearly the builder threads could work either way - they could check out a 43commit and then built it for all boards. Using separate directories for each 44commit/board pair they could leave their build product around afterwards 45also. 46 47The intent behind building a single board for multiple commits, is to make 48use of incremental builds. Since each commit is built incrementally from 49the previous one, builds are faster. Reconfiguring for a different board 50removes all intermediate object files. 51 52Many threads can be working at once, but each has its own working directory. 53When a thread finishes a build, it puts the output files into a result 54directory. 55 56The base directory used by buildman is normally '../<branch>', i.e. 57a directory higher than the source repository and named after the branch 58being built. 59 60Within the base directory, we have one subdirectory for each commit. Within 61that is one subdirectory for each board. Within that is the build output for 62that commit/board combination. 63 64Buildman also create working directories for each thread, in a .bm-work/ 65subdirectory in the base dir. 66 67As an example, say we are building branch 'us-net' for boards 'sandbox' and 68'seaboard', and say that us-net has two commits. We will have directories 69like this: 70 71us-net/ base directory 72 01_of_02_g4ed4ebc_net--Add-tftp-speed-/ 73 sandbox/ 74 u-boot.bin 75 seaboard/ 76 u-boot.bin 77 02_of_02_g4ed4ebc_net--Check-tftp-comp/ 78 sandbox/ 79 u-boot.bin 80 seaboard/ 81 u-boot.bin 82 .bm-work/ 83 00/ working directory for thread 0 (contains source checkout) 84 build/ build output 85 01/ working directory for thread 1 86 build/ build output 87 ... 88u-boot/ source directory 89 .git/ repository 90""" 91 92# Possible build outcomes 93OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4) 94 95# Translate a commit subject into a valid filename 96trans_valid_chars = string.maketrans("/: ", "---") 97 98 99class Builder: 100 """Class for building U-Boot for a particular commit. 101 102 Public members: (many should ->private) 103 active: True if the builder is active and has not been stopped 104 already_done: Number of builds already completed 105 base_dir: Base directory to use for builder 106 checkout: True to check out source, False to skip that step. 107 This is used for testing. 108 col: terminal.Color() object 109 count: Number of commits to build 110 do_make: Method to call to invoke Make 111 fail: Number of builds that failed due to error 112 force_build: Force building even if a build already exists 113 force_config_on_failure: If a commit fails for a board, disable 114 incremental building for the next commit we build for that 115 board, so that we will see all warnings/errors again. 116 force_build_failures: If a previously-built build (i.e. built on 117 a previous run of buildman) is marked as failed, rebuild it. 118 git_dir: Git directory containing source repository 119 last_line_len: Length of the last line we printed (used for erasing 120 it with new progress information) 121 num_jobs: Number of jobs to run at once (passed to make as -j) 122 num_threads: Number of builder threads to run 123 out_queue: Queue of results to process 124 re_make_err: Compiled regular expression for ignore_lines 125 queue: Queue of jobs to run 126 threads: List of active threads 127 toolchains: Toolchains object to use for building 128 upto: Current commit number we are building (0.count-1) 129 warned: Number of builds that produced at least one warning 130 force_reconfig: Reconfigure U-Boot on each comiit. This disables 131 incremental building, where buildman reconfigures on the first 132 commit for a baord, and then just does an incremental build for 133 the following commits. In fact buildman will reconfigure and 134 retry for any failing commits, so generally the only effect of 135 this option is to slow things down. 136 in_tree: Build U-Boot in-tree instead of specifying an output 137 directory separate from the source code. This option is really 138 only useful for testing in-tree builds. 139 140 Private members: 141 _base_board_dict: Last-summarised Dict of boards 142 _base_err_lines: Last-summarised list of errors 143 _build_period_us: Time taken for a single build (float object). 144 _complete_delay: Expected delay until completion (timedelta) 145 _next_delay_update: Next time we plan to display a progress update 146 (datatime) 147 _show_unknown: Show unknown boards (those not built) in summary 148 _timestamps: List of timestamps for the completion of the last 149 last _timestamp_count builds. Each is a datetime object. 150 _timestamp_count: Number of timestamps to keep in our list. 151 _working_dir: Base working directory containing all threads 152 """ 153 class Outcome: 154 """Records a build outcome for a single make invocation 155 156 Public Members: 157 rc: Outcome value (OUTCOME_...) 158 err_lines: List of error lines or [] if none 159 sizes: Dictionary of image size information, keyed by filename 160 - Each value is itself a dictionary containing 161 values for 'text', 'data' and 'bss', being the integer 162 size in bytes of each section. 163 func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each 164 value is itself a dictionary: 165 key: function name 166 value: Size of function in bytes 167 """ 168 def __init__(self, rc, err_lines, sizes, func_sizes): 169 self.rc = rc 170 self.err_lines = err_lines 171 self.sizes = sizes 172 self.func_sizes = func_sizes 173 174 def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, 175 gnu_make='make', checkout=True, show_unknown=True, step=1): 176 """Create a new Builder object 177 178 Args: 179 toolchains: Toolchains object to use for building 180 base_dir: Base directory to use for builder 181 git_dir: Git directory containing source repository 182 num_threads: Number of builder threads to run 183 num_jobs: Number of jobs to run at once (passed to make as -j) 184 gnu_make: the command name of GNU Make. 185 checkout: True to check out source, False to skip that step. 186 This is used for testing. 187 show_unknown: Show unknown boards (those not built) in summary 188 step: 1 to process every commit, n to process every nth commit 189 """ 190 self.toolchains = toolchains 191 self.base_dir = base_dir 192 self._working_dir = os.path.join(base_dir, '.bm-work') 193 self.threads = [] 194 self.active = True 195 self.do_make = self.Make 196 self.gnu_make = gnu_make 197 self.checkout = checkout 198 self.num_threads = num_threads 199 self.num_jobs = num_jobs 200 self.already_done = 0 201 self.force_build = False 202 self.git_dir = git_dir 203 self._show_unknown = show_unknown 204 self._timestamp_count = 10 205 self._build_period_us = None 206 self._complete_delay = None 207 self._next_delay_update = datetime.now() 208 self.force_config_on_failure = True 209 self.force_build_failures = False 210 self.force_reconfig = False 211 self._step = step 212 self.in_tree = False 213 214 self.col = terminal.Color() 215 216 self.queue = Queue.Queue() 217 self.out_queue = Queue.Queue() 218 for i in range(self.num_threads): 219 t = builderthread.BuilderThread(self, i) 220 t.setDaemon(True) 221 t.start() 222 self.threads.append(t) 223 224 self.last_line_len = 0 225 t = builderthread.ResultThread(self) 226 t.setDaemon(True) 227 t.start() 228 self.threads.append(t) 229 230 ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)'] 231 self.re_make_err = re.compile('|'.join(ignore_lines)) 232 233 def __del__(self): 234 """Get rid of all threads created by the builder""" 235 for t in self.threads: 236 del t 237 238 def SetDisplayOptions(self, show_errors=False, show_sizes=False, 239 show_detail=False, show_bloat=False): 240 """Setup display options for the builder. 241 242 show_errors: True to show summarised error/warning info 243 show_sizes: Show size deltas 244 show_detail: Show detail for each board 245 show_bloat: Show detail for each function 246 """ 247 self._show_errors = show_errors 248 self._show_sizes = show_sizes 249 self._show_detail = show_detail 250 self._show_bloat = show_bloat 251 252 def _AddTimestamp(self): 253 """Add a new timestamp to the list and record the build period. 254 255 The build period is the length of time taken to perform a single 256 build (one board, one commit). 257 """ 258 now = datetime.now() 259 self._timestamps.append(now) 260 count = len(self._timestamps) 261 delta = self._timestamps[-1] - self._timestamps[0] 262 seconds = delta.total_seconds() 263 264 # If we have enough data, estimate build period (time taken for a 265 # single build) and therefore completion time. 266 if count > 1 and self._next_delay_update < now: 267 self._next_delay_update = now + timedelta(seconds=2) 268 if seconds > 0: 269 self._build_period = float(seconds) / count 270 todo = self.count - self.upto 271 self._complete_delay = timedelta(microseconds= 272 self._build_period * todo * 1000000) 273 # Round it 274 self._complete_delay -= timedelta( 275 microseconds=self._complete_delay.microseconds) 276 277 if seconds > 60: 278 self._timestamps.popleft() 279 count -= 1 280 281 def ClearLine(self, length): 282 """Clear any characters on the current line 283 284 Make way for a new line of length 'length', by outputting enough 285 spaces to clear out the old line. Then remember the new length for 286 next time. 287 288 Args: 289 length: Length of new line, in characters 290 """ 291 if length < self.last_line_len: 292 print ' ' * (self.last_line_len - length), 293 print '\r', 294 self.last_line_len = length 295 sys.stdout.flush() 296 297 def SelectCommit(self, commit, checkout=True): 298 """Checkout the selected commit for this build 299 """ 300 self.commit = commit 301 if checkout and self.checkout: 302 gitutil.Checkout(commit.hash) 303 304 def Make(self, commit, brd, stage, cwd, *args, **kwargs): 305 """Run make 306 307 Args: 308 commit: Commit object that is being built 309 brd: Board object that is being built 310 stage: Stage that we are at (distclean, config, build) 311 cwd: Directory where make should be run 312 args: Arguments to pass to make 313 kwargs: Arguments to pass to command.RunPipe() 314 """ 315 cmd = [self.gnu_make] + list(args) 316 result = command.RunPipe([cmd], capture=True, capture_stderr=True, 317 cwd=cwd, raise_on_error=False, **kwargs) 318 return result 319 320 def ProcessResult(self, result): 321 """Process the result of a build, showing progress information 322 323 Args: 324 result: A CommandResult object, which indicates the result for 325 a single build 326 """ 327 col = terminal.Color() 328 if result: 329 target = result.brd.target 330 331 if result.return_code < 0: 332 self.active = False 333 command.StopAll() 334 return 335 336 self.upto += 1 337 if result.return_code != 0: 338 self.fail += 1 339 elif result.stderr: 340 self.warned += 1 341 if result.already_done: 342 self.already_done += 1 343 if self._verbose: 344 print '\r', 345 self.ClearLine(0) 346 boards_selected = {target : result.brd} 347 self.ResetResultSummary(boards_selected) 348 self.ProduceResultSummary(result.commit_upto, self.commits, 349 boards_selected) 350 else: 351 target = '(starting)' 352 353 # Display separate counts for ok, warned and fail 354 ok = self.upto - self.warned - self.fail 355 line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok) 356 line += self.col.Color(self.col.YELLOW, '%5d' % self.warned) 357 line += self.col.Color(self.col.RED, '%5d' % self.fail) 358 359 name = ' /%-5d ' % self.count 360 361 # Add our current completion time estimate 362 self._AddTimestamp() 363 if self._complete_delay: 364 name += '%s : ' % self._complete_delay 365 # When building all boards for a commit, we can print a commit 366 # progress message. 367 if result and result.commit_upto is None: 368 name += 'commit %2d/%-3d' % (self.commit_upto + 1, 369 self.commit_count) 370 371 name += target 372 print line + name, 373 length = 14 + len(name) 374 self.ClearLine(length) 375 376 def _GetOutputDir(self, commit_upto): 377 """Get the name of the output directory for a commit number 378 379 The output directory is typically .../<branch>/<commit>. 380 381 Args: 382 commit_upto: Commit number to use (0..self.count-1) 383 """ 384 if self.commits: 385 commit = self.commits[commit_upto] 386 subject = commit.subject.translate(trans_valid_chars) 387 commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1, 388 self.commit_count, commit.hash, subject[:20])) 389 else: 390 commit_dir = 'current' 391 output_dir = os.path.join(self.base_dir, commit_dir) 392 return output_dir 393 394 def GetBuildDir(self, commit_upto, target): 395 """Get the name of the build directory for a commit number 396 397 The build directory is typically .../<branch>/<commit>/<target>. 398 399 Args: 400 commit_upto: Commit number to use (0..self.count-1) 401 target: Target name 402 """ 403 output_dir = self._GetOutputDir(commit_upto) 404 return os.path.join(output_dir, target) 405 406 def GetDoneFile(self, commit_upto, target): 407 """Get the name of the done file for a commit number 408 409 Args: 410 commit_upto: Commit number to use (0..self.count-1) 411 target: Target name 412 """ 413 return os.path.join(self.GetBuildDir(commit_upto, target), 'done') 414 415 def GetSizesFile(self, commit_upto, target): 416 """Get the name of the sizes file for a commit number 417 418 Args: 419 commit_upto: Commit number to use (0..self.count-1) 420 target: Target name 421 """ 422 return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes') 423 424 def GetFuncSizesFile(self, commit_upto, target, elf_fname): 425 """Get the name of the funcsizes file for a commit number and ELF file 426 427 Args: 428 commit_upto: Commit number to use (0..self.count-1) 429 target: Target name 430 elf_fname: Filename of elf image 431 """ 432 return os.path.join(self.GetBuildDir(commit_upto, target), 433 '%s.sizes' % elf_fname.replace('/', '-')) 434 435 def GetObjdumpFile(self, commit_upto, target, elf_fname): 436 """Get the name of the objdump file for a commit number and ELF file 437 438 Args: 439 commit_upto: Commit number to use (0..self.count-1) 440 target: Target name 441 elf_fname: Filename of elf image 442 """ 443 return os.path.join(self.GetBuildDir(commit_upto, target), 444 '%s.objdump' % elf_fname.replace('/', '-')) 445 446 def GetErrFile(self, commit_upto, target): 447 """Get the name of the err file for a commit number 448 449 Args: 450 commit_upto: Commit number to use (0..self.count-1) 451 target: Target name 452 """ 453 output_dir = self.GetBuildDir(commit_upto, target) 454 return os.path.join(output_dir, 'err') 455 456 def FilterErrors(self, lines): 457 """Filter out errors in which we have no interest 458 459 We should probably use map(). 460 461 Args: 462 lines: List of error lines, each a string 463 Returns: 464 New list with only interesting lines included 465 """ 466 out_lines = [] 467 for line in lines: 468 if not self.re_make_err.search(line): 469 out_lines.append(line) 470 return out_lines 471 472 def ReadFuncSizes(self, fname, fd): 473 """Read function sizes from the output of 'nm' 474 475 Args: 476 fd: File containing data to read 477 fname: Filename we are reading from (just for errors) 478 479 Returns: 480 Dictionary containing size of each function in bytes, indexed by 481 function name. 482 """ 483 sym = {} 484 for line in fd.readlines(): 485 try: 486 size, type, name = line[:-1].split() 487 except: 488 print "Invalid line in file '%s': '%s'" % (fname, line[:-1]) 489 continue 490 if type in 'tTdDbB': 491 # function names begin with '.' on 64-bit powerpc 492 if '.' in name[1:]: 493 name = 'static.' + name.split('.')[0] 494 sym[name] = sym.get(name, 0) + int(size, 16) 495 return sym 496 497 def GetBuildOutcome(self, commit_upto, target, read_func_sizes): 498 """Work out the outcome of a build. 499 500 Args: 501 commit_upto: Commit number to check (0..n-1) 502 target: Target board to check 503 read_func_sizes: True to read function size information 504 505 Returns: 506 Outcome object 507 """ 508 done_file = self.GetDoneFile(commit_upto, target) 509 sizes_file = self.GetSizesFile(commit_upto, target) 510 sizes = {} 511 func_sizes = {} 512 if os.path.exists(done_file): 513 with open(done_file, 'r') as fd: 514 return_code = int(fd.readline()) 515 err_lines = [] 516 err_file = self.GetErrFile(commit_upto, target) 517 if os.path.exists(err_file): 518 with open(err_file, 'r') as fd: 519 err_lines = self.FilterErrors(fd.readlines()) 520 521 # Decide whether the build was ok, failed or created warnings 522 if return_code: 523 rc = OUTCOME_ERROR 524 elif len(err_lines): 525 rc = OUTCOME_WARNING 526 else: 527 rc = OUTCOME_OK 528 529 # Convert size information to our simple format 530 if os.path.exists(sizes_file): 531 with open(sizes_file, 'r') as fd: 532 for line in fd.readlines(): 533 values = line.split() 534 rodata = 0 535 if len(values) > 6: 536 rodata = int(values[6], 16) 537 size_dict = { 538 'all' : int(values[0]) + int(values[1]) + 539 int(values[2]), 540 'text' : int(values[0]) - rodata, 541 'data' : int(values[1]), 542 'bss' : int(values[2]), 543 'rodata' : rodata, 544 } 545 sizes[values[5]] = size_dict 546 547 if read_func_sizes: 548 pattern = self.GetFuncSizesFile(commit_upto, target, '*') 549 for fname in glob.glob(pattern): 550 with open(fname, 'r') as fd: 551 dict_name = os.path.basename(fname).replace('.sizes', 552 '') 553 func_sizes[dict_name] = self.ReadFuncSizes(fname, fd) 554 555 return Builder.Outcome(rc, err_lines, sizes, func_sizes) 556 557 return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}) 558 559 def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes): 560 """Calculate a summary of the results of building a commit. 561 562 Args: 563 board_selected: Dict containing boards to summarise 564 commit_upto: Commit number to summarize (0..self.count-1) 565 read_func_sizes: True to read function size information 566 567 Returns: 568 Tuple: 569 Dict containing boards which passed building this commit. 570 keyed by board.target 571 List containing a summary of error/warning lines 572 """ 573 board_dict = {} 574 err_lines_summary = [] 575 576 for board in boards_selected.itervalues(): 577 outcome = self.GetBuildOutcome(commit_upto, board.target, 578 read_func_sizes) 579 board_dict[board.target] = outcome 580 for err in outcome.err_lines: 581 if err and not err.rstrip() in err_lines_summary: 582 err_lines_summary.append(err.rstrip()) 583 return board_dict, err_lines_summary 584 585 def AddOutcome(self, board_dict, arch_list, changes, char, color): 586 """Add an output to our list of outcomes for each architecture 587 588 This simple function adds failing boards (changes) to the 589 relevant architecture string, so we can print the results out 590 sorted by architecture. 591 592 Args: 593 board_dict: Dict containing all boards 594 arch_list: Dict keyed by arch name. Value is a string containing 595 a list of board names which failed for that arch. 596 changes: List of boards to add to arch_list 597 color: terminal.Colour object 598 """ 599 done_arch = {} 600 for target in changes: 601 if target in board_dict: 602 arch = board_dict[target].arch 603 else: 604 arch = 'unknown' 605 str = self.col.Color(color, ' ' + target) 606 if not arch in done_arch: 607 str = self.col.Color(color, char) + ' ' + str 608 done_arch[arch] = True 609 if not arch in arch_list: 610 arch_list[arch] = str 611 else: 612 arch_list[arch] += str 613 614 615 def ColourNum(self, num): 616 color = self.col.RED if num > 0 else self.col.GREEN 617 if num == 0: 618 return '0' 619 return self.col.Color(color, str(num)) 620 621 def ResetResultSummary(self, board_selected): 622 """Reset the results summary ready for use. 623 624 Set up the base board list to be all those selected, and set the 625 error lines to empty. 626 627 Following this, calls to PrintResultSummary() will use this 628 information to work out what has changed. 629 630 Args: 631 board_selected: Dict containing boards to summarise, keyed by 632 board.target 633 """ 634 self._base_board_dict = {} 635 for board in board_selected: 636 self._base_board_dict[board] = Builder.Outcome(0, [], [], {}) 637 self._base_err_lines = [] 638 639 def PrintFuncSizeDetail(self, fname, old, new): 640 grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0 641 delta, common = [], {} 642 643 for a in old: 644 if a in new: 645 common[a] = 1 646 647 for name in old: 648 if name not in common: 649 remove += 1 650 down += old[name] 651 delta.append([-old[name], name]) 652 653 for name in new: 654 if name not in common: 655 add += 1 656 up += new[name] 657 delta.append([new[name], name]) 658 659 for name in common: 660 diff = new.get(name, 0) - old.get(name, 0) 661 if diff > 0: 662 grow, up = grow + 1, up + diff 663 elif diff < 0: 664 shrink, down = shrink + 1, down - diff 665 delta.append([diff, name]) 666 667 delta.sort() 668 delta.reverse() 669 670 args = [add, -remove, grow, -shrink, up, -down, up - down] 671 if max(args) == 0: 672 return 673 args = [self.ColourNum(x) for x in args] 674 indent = ' ' * 15 675 print ('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' % 676 tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args)) 677 print '%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new', 678 'delta') 679 for diff, name in delta: 680 if diff: 681 color = self.col.RED if diff > 0 else self.col.GREEN 682 msg = '%s %-38s %7s %7s %+7d' % (indent, name, 683 old.get(name, '-'), new.get(name,'-'), diff) 684 print self.col.Color(color, msg) 685 686 687 def PrintSizeDetail(self, target_list, show_bloat): 688 """Show details size information for each board 689 690 Args: 691 target_list: List of targets, each a dict containing: 692 'target': Target name 693 'total_diff': Total difference in bytes across all areas 694 <part_name>: Difference for that part 695 show_bloat: Show detail for each function 696 """ 697 targets_by_diff = sorted(target_list, reverse=True, 698 key=lambda x: x['_total_diff']) 699 for result in targets_by_diff: 700 printed_target = False 701 for name in sorted(result): 702 diff = result[name] 703 if name.startswith('_'): 704 continue 705 if diff != 0: 706 color = self.col.RED if diff > 0 else self.col.GREEN 707 msg = ' %s %+d' % (name, diff) 708 if not printed_target: 709 print '%10s %-15s:' % ('', result['_target']), 710 printed_target = True 711 print self.col.Color(color, msg), 712 if printed_target: 713 print 714 if show_bloat: 715 target = result['_target'] 716 outcome = result['_outcome'] 717 base_outcome = self._base_board_dict[target] 718 for fname in outcome.func_sizes: 719 self.PrintFuncSizeDetail(fname, 720 base_outcome.func_sizes[fname], 721 outcome.func_sizes[fname]) 722 723 724 def PrintSizeSummary(self, board_selected, board_dict, show_detail, 725 show_bloat): 726 """Print a summary of image sizes broken down by section. 727 728 The summary takes the form of one line per architecture. The 729 line contains deltas for each of the sections (+ means the section 730 got bigger, - means smaller). The nunmbers are the average number 731 of bytes that a board in this section increased by. 732 733 For example: 734 powerpc: (622 boards) text -0.0 735 arm: (285 boards) text -0.0 736 nds32: (3 boards) text -8.0 737 738 Args: 739 board_selected: Dict containing boards to summarise, keyed by 740 board.target 741 board_dict: Dict containing boards for which we built this 742 commit, keyed by board.target. The value is an Outcome object. 743 show_detail: Show detail for each board 744 show_bloat: Show detail for each function 745 """ 746 arch_list = {} 747 arch_count = {} 748 749 # Calculate changes in size for different image parts 750 # The previous sizes are in Board.sizes, for each board 751 for target in board_dict: 752 if target not in board_selected: 753 continue 754 base_sizes = self._base_board_dict[target].sizes 755 outcome = board_dict[target] 756 sizes = outcome.sizes 757 758 # Loop through the list of images, creating a dict of size 759 # changes for each image/part. We end up with something like 760 # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4} 761 # which means that U-Boot data increased by 5 bytes and SPL 762 # text decreased by 4. 763 err = {'_target' : target} 764 for image in sizes: 765 if image in base_sizes: 766 base_image = base_sizes[image] 767 # Loop through the text, data, bss parts 768 for part in sorted(sizes[image]): 769 diff = sizes[image][part] - base_image[part] 770 col = None 771 if diff: 772 if image == 'u-boot': 773 name = part 774 else: 775 name = image + ':' + part 776 err[name] = diff 777 arch = board_selected[target].arch 778 if not arch in arch_count: 779 arch_count[arch] = 1 780 else: 781 arch_count[arch] += 1 782 if not sizes: 783 pass # Only add to our list when we have some stats 784 elif not arch in arch_list: 785 arch_list[arch] = [err] 786 else: 787 arch_list[arch].append(err) 788 789 # We now have a list of image size changes sorted by arch 790 # Print out a summary of these 791 for arch, target_list in arch_list.iteritems(): 792 # Get total difference for each type 793 totals = {} 794 for result in target_list: 795 total = 0 796 for name, diff in result.iteritems(): 797 if name.startswith('_'): 798 continue 799 total += diff 800 if name in totals: 801 totals[name] += diff 802 else: 803 totals[name] = diff 804 result['_total_diff'] = total 805 result['_outcome'] = board_dict[result['_target']] 806 807 count = len(target_list) 808 printed_arch = False 809 for name in sorted(totals): 810 diff = totals[name] 811 if diff: 812 # Display the average difference in this name for this 813 # architecture 814 avg_diff = float(diff) / count 815 color = self.col.RED if avg_diff > 0 else self.col.GREEN 816 msg = ' %s %+1.1f' % (name, avg_diff) 817 if not printed_arch: 818 print '%10s: (for %d/%d boards)' % (arch, count, 819 arch_count[arch]), 820 printed_arch = True 821 print self.col.Color(color, msg), 822 823 if printed_arch: 824 print 825 if show_detail: 826 self.PrintSizeDetail(target_list, show_bloat) 827 828 829 def PrintResultSummary(self, board_selected, board_dict, err_lines, 830 show_sizes, show_detail, show_bloat): 831 """Compare results with the base results and display delta. 832 833 Only boards mentioned in board_selected will be considered. This 834 function is intended to be called repeatedly with the results of 835 each commit. It therefore shows a 'diff' between what it saw in 836 the last call and what it sees now. 837 838 Args: 839 board_selected: Dict containing boards to summarise, keyed by 840 board.target 841 board_dict: Dict containing boards for which we built this 842 commit, keyed by board.target. The value is an Outcome object. 843 err_lines: A list of errors for this commit, or [] if there is 844 none, or we don't want to print errors 845 show_sizes: Show image size deltas 846 show_detail: Show detail for each board 847 show_bloat: Show detail for each function 848 """ 849 better = [] # List of boards fixed since last commit 850 worse = [] # List of new broken boards since last commit 851 new = [] # List of boards that didn't exist last time 852 unknown = [] # List of boards that were not built 853 854 for target in board_dict: 855 if target not in board_selected: 856 continue 857 858 # If the board was built last time, add its outcome to a list 859 if target in self._base_board_dict: 860 base_outcome = self._base_board_dict[target].rc 861 outcome = board_dict[target] 862 if outcome.rc == OUTCOME_UNKNOWN: 863 unknown.append(target) 864 elif outcome.rc < base_outcome: 865 better.append(target) 866 elif outcome.rc > base_outcome: 867 worse.append(target) 868 else: 869 new.append(target) 870 871 # Get a list of errors that have appeared, and disappeared 872 better_err = [] 873 worse_err = [] 874 for line in err_lines: 875 if line not in self._base_err_lines: 876 worse_err.append('+' + line) 877 for line in self._base_err_lines: 878 if line not in err_lines: 879 better_err.append('-' + line) 880 881 # Display results by arch 882 if better or worse or unknown or new or worse_err or better_err: 883 arch_list = {} 884 self.AddOutcome(board_selected, arch_list, better, '', 885 self.col.GREEN) 886 self.AddOutcome(board_selected, arch_list, worse, '+', 887 self.col.RED) 888 self.AddOutcome(board_selected, arch_list, new, '*', self.col.BLUE) 889 if self._show_unknown: 890 self.AddOutcome(board_selected, arch_list, unknown, '?', 891 self.col.MAGENTA) 892 for arch, target_list in arch_list.iteritems(): 893 print '%10s: %s' % (arch, target_list) 894 if better_err: 895 print self.col.Color(self.col.GREEN, '\n'.join(better_err)) 896 if worse_err: 897 print self.col.Color(self.col.RED, '\n'.join(worse_err)) 898 899 if show_sizes: 900 self.PrintSizeSummary(board_selected, board_dict, show_detail, 901 show_bloat) 902 903 # Save our updated information for the next call to this function 904 self._base_board_dict = board_dict 905 self._base_err_lines = err_lines 906 907 # Get a list of boards that did not get built, if needed 908 not_built = [] 909 for board in board_selected: 910 if not board in board_dict: 911 not_built.append(board) 912 if not_built: 913 print "Boards not built (%d): %s" % (len(not_built), 914 ', '.join(not_built)) 915 916 def ProduceResultSummary(self, commit_upto, commits, board_selected): 917 board_dict, err_lines = self.GetResultSummary(board_selected, 918 commit_upto, read_func_sizes=self._show_bloat) 919 if commits: 920 msg = '%02d: %s' % (commit_upto + 1, 921 commits[commit_upto].subject) 922 print self.col.Color(self.col.BLUE, msg) 923 self.PrintResultSummary(board_selected, board_dict, 924 err_lines if self._show_errors else [], 925 self._show_sizes, self._show_detail, self._show_bloat) 926 927 def ShowSummary(self, commits, board_selected): 928 """Show a build summary for U-Boot for a given board list. 929 930 Reset the result summary, then repeatedly call GetResultSummary on 931 each commit's results, then display the differences we see. 932 933 Args: 934 commit: Commit objects to summarise 935 board_selected: Dict containing boards to summarise 936 """ 937 self.commit_count = len(commits) if commits else 1 938 self.commits = commits 939 self.ResetResultSummary(board_selected) 940 941 for commit_upto in range(0, self.commit_count, self._step): 942 self.ProduceResultSummary(commit_upto, commits, board_selected) 943 944 945 def SetupBuild(self, board_selected, commits): 946 """Set up ready to start a build. 947 948 Args: 949 board_selected: Selected boards to build 950 commits: Selected commits to build 951 """ 952 # First work out how many commits we will build 953 count = (self.commit_count + self._step - 1) / self._step 954 self.count = len(board_selected) * count 955 self.upto = self.warned = self.fail = 0 956 self._timestamps = collections.deque() 957 958 def GetThreadDir(self, thread_num): 959 """Get the directory path to the working dir for a thread. 960 961 Args: 962 thread_num: Number of thread to check. 963 """ 964 return os.path.join(self._working_dir, '%02d' % thread_num) 965 966 def _PrepareThread(self, thread_num, setup_git): 967 """Prepare the working directory for a thread. 968 969 This clones or fetches the repo into the thread's work directory. 970 971 Args: 972 thread_num: Thread number (0, 1, ...) 973 setup_git: True to set up a git repo clone 974 """ 975 thread_dir = self.GetThreadDir(thread_num) 976 builderthread.Mkdir(thread_dir) 977 git_dir = os.path.join(thread_dir, '.git') 978 979 # Clone the repo if it doesn't already exist 980 # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so 981 # we have a private index but uses the origin repo's contents? 982 if setup_git and self.git_dir: 983 src_dir = os.path.abspath(self.git_dir) 984 if os.path.exists(git_dir): 985 gitutil.Fetch(git_dir, thread_dir) 986 else: 987 print 'Cloning repo for thread %d' % thread_num 988 gitutil.Clone(src_dir, thread_dir) 989 990 def _PrepareWorkingSpace(self, max_threads, setup_git): 991 """Prepare the working directory for use. 992 993 Set up the git repo for each thread. 994 995 Args: 996 max_threads: Maximum number of threads we expect to need. 997 setup_git: True to set up a git repo clone 998 """ 999 builderthread.Mkdir(self._working_dir) 1000 for thread in range(max_threads): 1001 self._PrepareThread(thread, setup_git) 1002 1003 def _PrepareOutputSpace(self): 1004 """Get the output directories ready to receive files. 1005 1006 We delete any output directories which look like ones we need to 1007 create. Having left over directories is confusing when the user wants 1008 to check the output manually. 1009 """ 1010 dir_list = [] 1011 for commit_upto in range(self.commit_count): 1012 dir_list.append(self._GetOutputDir(commit_upto)) 1013 1014 for dirname in glob.glob(os.path.join(self.base_dir, '*')): 1015 if dirname not in dir_list: 1016 shutil.rmtree(dirname) 1017 1018 def BuildBoards(self, commits, board_selected, keep_outputs, verbose): 1019 """Build all commits for a list of boards 1020 1021 Args: 1022 commits: List of commits to be build, each a Commit object 1023 boards_selected: Dict of selected boards, key is target name, 1024 value is Board object 1025 keep_outputs: True to save build output files 1026 verbose: Display build results as they are completed 1027 """ 1028 self.commit_count = len(commits) if commits else 1 1029 self.commits = commits 1030 self._verbose = verbose 1031 1032 self.ResetResultSummary(board_selected) 1033 builderthread.Mkdir(self.base_dir) 1034 self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)), 1035 commits is not None) 1036 self._PrepareOutputSpace() 1037 self.SetupBuild(board_selected, commits) 1038 self.ProcessResult(None) 1039 1040 # Create jobs to build all commits for each board 1041 for brd in board_selected.itervalues(): 1042 job = builderthread.BuilderJob() 1043 job.board = brd 1044 job.commits = commits 1045 job.keep_outputs = keep_outputs 1046 job.step = self._step 1047 self.queue.put(job) 1048 1049 # Wait until all jobs are started 1050 self.queue.join() 1051 1052 # Wait until we have processed all output 1053 self.out_queue.join() 1054 print 1055 self.ClearLine(0) 1056