1# Copyright (c) 2013 The Chromium OS Authors. 2# 3# Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com> 4# 5# SPDX-License-Identifier: GPL-2.0+ 6# 7 8import collections 9from datetime import datetime, timedelta 10import glob 11import os 12import re 13import Queue 14import shutil 15import string 16import sys 17import time 18 19import builderthread 20import command 21import gitutil 22import terminal 23import toolchain 24 25 26""" 27Theory of Operation 28 29Please see README for user documentation, and you should be familiar with 30that before trying to make sense of this. 31 32Buildman works by keeping the machine as busy as possible, building different 33commits for different boards on multiple CPUs at once. 34 35The source repo (self.git_dir) contains all the commits to be built. Each 36thread works on a single board at a time. It checks out the first commit, 37configures it for that board, then builds it. Then it checks out the next 38commit and builds it (typically without re-configuring). When it runs out 39of commits, it gets another job from the builder and starts again with that 40board. 41 42Clearly the builder threads could work either way - they could check out a 43commit and then built it for all boards. Using separate directories for each 44commit/board pair they could leave their build product around afterwards 45also. 46 47The intent behind building a single board for multiple commits, is to make 48use of incremental builds. Since each commit is built incrementally from 49the previous one, builds are faster. Reconfiguring for a different board 50removes all intermediate object files. 51 52Many threads can be working at once, but each has its own working directory. 53When a thread finishes a build, it puts the output files into a result 54directory. 55 56The base directory used by buildman is normally '../<branch>', i.e. 57a directory higher than the source repository and named after the branch 58being built. 59 60Within the base directory, we have one subdirectory for each commit. Within 61that is one subdirectory for each board. Within that is the build output for 62that commit/board combination. 63 64Buildman also create working directories for each thread, in a .bm-work/ 65subdirectory in the base dir. 66 67As an example, say we are building branch 'us-net' for boards 'sandbox' and 68'seaboard', and say that us-net has two commits. We will have directories 69like this: 70 71us-net/ base directory 72 01_of_02_g4ed4ebc_net--Add-tftp-speed-/ 73 sandbox/ 74 u-boot.bin 75 seaboard/ 76 u-boot.bin 77 02_of_02_g4ed4ebc_net--Check-tftp-comp/ 78 sandbox/ 79 u-boot.bin 80 seaboard/ 81 u-boot.bin 82 .bm-work/ 83 00/ working directory for thread 0 (contains source checkout) 84 build/ build output 85 01/ working directory for thread 1 86 build/ build output 87 ... 88u-boot/ source directory 89 .git/ repository 90""" 91 92# Possible build outcomes 93OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4) 94 95# Translate a commit subject into a valid filename 96trans_valid_chars = string.maketrans("/: ", "---") 97 98 99class Builder: 100 """Class for building U-Boot for a particular commit. 101 102 Public members: (many should ->private) 103 active: True if the builder is active and has not been stopped 104 already_done: Number of builds already completed 105 base_dir: Base directory to use for builder 106 checkout: True to check out source, False to skip that step. 107 This is used for testing. 108 col: terminal.Color() object 109 count: Number of commits to build 110 do_make: Method to call to invoke Make 111 fail: Number of builds that failed due to error 112 force_build: Force building even if a build already exists 113 force_config_on_failure: If a commit fails for a board, disable 114 incremental building for the next commit we build for that 115 board, so that we will see all warnings/errors again. 116 force_build_failures: If a previously-built build (i.e. built on 117 a previous run of buildman) is marked as failed, rebuild it. 118 git_dir: Git directory containing source repository 119 last_line_len: Length of the last line we printed (used for erasing 120 it with new progress information) 121 num_jobs: Number of jobs to run at once (passed to make as -j) 122 num_threads: Number of builder threads to run 123 out_queue: Queue of results to process 124 re_make_err: Compiled regular expression for ignore_lines 125 queue: Queue of jobs to run 126 threads: List of active threads 127 toolchains: Toolchains object to use for building 128 upto: Current commit number we are building (0.count-1) 129 warned: Number of builds that produced at least one warning 130 force_reconfig: Reconfigure U-Boot on each comiit. This disables 131 incremental building, where buildman reconfigures on the first 132 commit for a baord, and then just does an incremental build for 133 the following commits. In fact buildman will reconfigure and 134 retry for any failing commits, so generally the only effect of 135 this option is to slow things down. 136 in_tree: Build U-Boot in-tree instead of specifying an output 137 directory separate from the source code. This option is really 138 only useful for testing in-tree builds. 139 140 Private members: 141 _base_board_dict: Last-summarised Dict of boards 142 _base_err_lines: Last-summarised list of errors 143 _build_period_us: Time taken for a single build (float object). 144 _complete_delay: Expected delay until completion (timedelta) 145 _next_delay_update: Next time we plan to display a progress update 146 (datatime) 147 _show_unknown: Show unknown boards (those not built) in summary 148 _timestamps: List of timestamps for the completion of the last 149 last _timestamp_count builds. Each is a datetime object. 150 _timestamp_count: Number of timestamps to keep in our list. 151 _working_dir: Base working directory containing all threads 152 """ 153 class Outcome: 154 """Records a build outcome for a single make invocation 155 156 Public Members: 157 rc: Outcome value (OUTCOME_...) 158 err_lines: List of error lines or [] if none 159 sizes: Dictionary of image size information, keyed by filename 160 - Each value is itself a dictionary containing 161 values for 'text', 'data' and 'bss', being the integer 162 size in bytes of each section. 163 func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each 164 value is itself a dictionary: 165 key: function name 166 value: Size of function in bytes 167 """ 168 def __init__(self, rc, err_lines, sizes, func_sizes): 169 self.rc = rc 170 self.err_lines = err_lines 171 self.sizes = sizes 172 self.func_sizes = func_sizes 173 174 def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, 175 gnu_make='make', checkout=True, show_unknown=True, step=1): 176 """Create a new Builder object 177 178 Args: 179 toolchains: Toolchains object to use for building 180 base_dir: Base directory to use for builder 181 git_dir: Git directory containing source repository 182 num_threads: Number of builder threads to run 183 num_jobs: Number of jobs to run at once (passed to make as -j) 184 gnu_make: the command name of GNU Make. 185 checkout: True to check out source, False to skip that step. 186 This is used for testing. 187 show_unknown: Show unknown boards (those not built) in summary 188 step: 1 to process every commit, n to process every nth commit 189 """ 190 self.toolchains = toolchains 191 self.base_dir = base_dir 192 self._working_dir = os.path.join(base_dir, '.bm-work') 193 self.threads = [] 194 self.active = True 195 self.do_make = self.Make 196 self.gnu_make = gnu_make 197 self.checkout = checkout 198 self.num_threads = num_threads 199 self.num_jobs = num_jobs 200 self.already_done = 0 201 self.force_build = False 202 self.git_dir = git_dir 203 self._show_unknown = show_unknown 204 self._timestamp_count = 10 205 self._build_period_us = None 206 self._complete_delay = None 207 self._next_delay_update = datetime.now() 208 self.force_config_on_failure = True 209 self.force_build_failures = False 210 self.force_reconfig = False 211 self._step = step 212 self.in_tree = False 213 214 self.col = terminal.Color() 215 216 self.queue = Queue.Queue() 217 self.out_queue = Queue.Queue() 218 for i in range(self.num_threads): 219 t = builderthread.BuilderThread(self, i) 220 t.setDaemon(True) 221 t.start() 222 self.threads.append(t) 223 224 self.last_line_len = 0 225 t = builderthread.ResultThread(self) 226 t.setDaemon(True) 227 t.start() 228 self.threads.append(t) 229 230 ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)'] 231 self.re_make_err = re.compile('|'.join(ignore_lines)) 232 233 def __del__(self): 234 """Get rid of all threads created by the builder""" 235 for t in self.threads: 236 del t 237 238 def SetDisplayOptions(self, show_errors=False, show_sizes=False, 239 show_detail=False, show_bloat=False): 240 """Setup display options for the builder. 241 242 show_errors: True to show summarised error/warning info 243 show_sizes: Show size deltas 244 show_detail: Show detail for each board 245 show_bloat: Show detail for each function 246 """ 247 self._show_errors = show_errors 248 self._show_sizes = show_sizes 249 self._show_detail = show_detail 250 self._show_bloat = show_bloat 251 252 def _AddTimestamp(self): 253 """Add a new timestamp to the list and record the build period. 254 255 The build period is the length of time taken to perform a single 256 build (one board, one commit). 257 """ 258 now = datetime.now() 259 self._timestamps.append(now) 260 count = len(self._timestamps) 261 delta = self._timestamps[-1] - self._timestamps[0] 262 seconds = delta.total_seconds() 263 264 # If we have enough data, estimate build period (time taken for a 265 # single build) and therefore completion time. 266 if count > 1 and self._next_delay_update < now: 267 self._next_delay_update = now + timedelta(seconds=2) 268 if seconds > 0: 269 self._build_period = float(seconds) / count 270 todo = self.count - self.upto 271 self._complete_delay = timedelta(microseconds= 272 self._build_period * todo * 1000000) 273 # Round it 274 self._complete_delay -= timedelta( 275 microseconds=self._complete_delay.microseconds) 276 277 if seconds > 60: 278 self._timestamps.popleft() 279 count -= 1 280 281 def ClearLine(self, length): 282 """Clear any characters on the current line 283 284 Make way for a new line of length 'length', by outputting enough 285 spaces to clear out the old line. Then remember the new length for 286 next time. 287 288 Args: 289 length: Length of new line, in characters 290 """ 291 if length < self.last_line_len: 292 print ' ' * (self.last_line_len - length), 293 print '\r', 294 self.last_line_len = length 295 sys.stdout.flush() 296 297 def SelectCommit(self, commit, checkout=True): 298 """Checkout the selected commit for this build 299 """ 300 self.commit = commit 301 if checkout and self.checkout: 302 gitutil.Checkout(commit.hash) 303 304 def Make(self, commit, brd, stage, cwd, *args, **kwargs): 305 """Run make 306 307 Args: 308 commit: Commit object that is being built 309 brd: Board object that is being built 310 stage: Stage that we are at (distclean, config, build) 311 cwd: Directory where make should be run 312 args: Arguments to pass to make 313 kwargs: Arguments to pass to command.RunPipe() 314 """ 315 cmd = [self.gnu_make] + list(args) 316 result = command.RunPipe([cmd], capture=True, capture_stderr=True, 317 cwd=cwd, raise_on_error=False, **kwargs) 318 return result 319 320 def ProcessResult(self, result): 321 """Process the result of a build, showing progress information 322 323 Args: 324 result: A CommandResult object 325 """ 326 col = terminal.Color() 327 if result: 328 target = result.brd.target 329 330 if result.return_code < 0: 331 self.active = False 332 command.StopAll() 333 return 334 335 self.upto += 1 336 if result.return_code != 0: 337 self.fail += 1 338 elif result.stderr: 339 self.warned += 1 340 if result.already_done: 341 self.already_done += 1 342 else: 343 target = '(starting)' 344 345 # Display separate counts for ok, warned and fail 346 ok = self.upto - self.warned - self.fail 347 line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok) 348 line += self.col.Color(self.col.YELLOW, '%5d' % self.warned) 349 line += self.col.Color(self.col.RED, '%5d' % self.fail) 350 351 name = ' /%-5d ' % self.count 352 353 # Add our current completion time estimate 354 self._AddTimestamp() 355 if self._complete_delay: 356 name += '%s : ' % self._complete_delay 357 # When building all boards for a commit, we can print a commit 358 # progress message. 359 if result and result.commit_upto is None: 360 name += 'commit %2d/%-3d' % (self.commit_upto + 1, 361 self.commit_count) 362 363 name += target 364 print line + name, 365 length = 13 + len(name) 366 self.ClearLine(length) 367 368 def _GetOutputDir(self, commit_upto): 369 """Get the name of the output directory for a commit number 370 371 The output directory is typically .../<branch>/<commit>. 372 373 Args: 374 commit_upto: Commit number to use (0..self.count-1) 375 """ 376 if self.commits: 377 commit = self.commits[commit_upto] 378 subject = commit.subject.translate(trans_valid_chars) 379 commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1, 380 self.commit_count, commit.hash, subject[:20])) 381 else: 382 commit_dir = 'current' 383 output_dir = os.path.join(self.base_dir, commit_dir) 384 return output_dir 385 386 def GetBuildDir(self, commit_upto, target): 387 """Get the name of the build directory for a commit number 388 389 The build directory is typically .../<branch>/<commit>/<target>. 390 391 Args: 392 commit_upto: Commit number to use (0..self.count-1) 393 target: Target name 394 """ 395 output_dir = self._GetOutputDir(commit_upto) 396 return os.path.join(output_dir, target) 397 398 def GetDoneFile(self, commit_upto, target): 399 """Get the name of the done file for a commit number 400 401 Args: 402 commit_upto: Commit number to use (0..self.count-1) 403 target: Target name 404 """ 405 return os.path.join(self.GetBuildDir(commit_upto, target), 'done') 406 407 def GetSizesFile(self, commit_upto, target): 408 """Get the name of the sizes file for a commit number 409 410 Args: 411 commit_upto: Commit number to use (0..self.count-1) 412 target: Target name 413 """ 414 return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes') 415 416 def GetFuncSizesFile(self, commit_upto, target, elf_fname): 417 """Get the name of the funcsizes file for a commit number and ELF file 418 419 Args: 420 commit_upto: Commit number to use (0..self.count-1) 421 target: Target name 422 elf_fname: Filename of elf image 423 """ 424 return os.path.join(self.GetBuildDir(commit_upto, target), 425 '%s.sizes' % elf_fname.replace('/', '-')) 426 427 def GetObjdumpFile(self, commit_upto, target, elf_fname): 428 """Get the name of the objdump file for a commit number and ELF file 429 430 Args: 431 commit_upto: Commit number to use (0..self.count-1) 432 target: Target name 433 elf_fname: Filename of elf image 434 """ 435 return os.path.join(self.GetBuildDir(commit_upto, target), 436 '%s.objdump' % elf_fname.replace('/', '-')) 437 438 def GetErrFile(self, commit_upto, target): 439 """Get the name of the err file for a commit number 440 441 Args: 442 commit_upto: Commit number to use (0..self.count-1) 443 target: Target name 444 """ 445 output_dir = self.GetBuildDir(commit_upto, target) 446 return os.path.join(output_dir, 'err') 447 448 def FilterErrors(self, lines): 449 """Filter out errors in which we have no interest 450 451 We should probably use map(). 452 453 Args: 454 lines: List of error lines, each a string 455 Returns: 456 New list with only interesting lines included 457 """ 458 out_lines = [] 459 for line in lines: 460 if not self.re_make_err.search(line): 461 out_lines.append(line) 462 return out_lines 463 464 def ReadFuncSizes(self, fname, fd): 465 """Read function sizes from the output of 'nm' 466 467 Args: 468 fd: File containing data to read 469 fname: Filename we are reading from (just for errors) 470 471 Returns: 472 Dictionary containing size of each function in bytes, indexed by 473 function name. 474 """ 475 sym = {} 476 for line in fd.readlines(): 477 try: 478 size, type, name = line[:-1].split() 479 except: 480 print "Invalid line in file '%s': '%s'" % (fname, line[:-1]) 481 continue 482 if type in 'tTdDbB': 483 # function names begin with '.' on 64-bit powerpc 484 if '.' in name[1:]: 485 name = 'static.' + name.split('.')[0] 486 sym[name] = sym.get(name, 0) + int(size, 16) 487 return sym 488 489 def GetBuildOutcome(self, commit_upto, target, read_func_sizes): 490 """Work out the outcome of a build. 491 492 Args: 493 commit_upto: Commit number to check (0..n-1) 494 target: Target board to check 495 read_func_sizes: True to read function size information 496 497 Returns: 498 Outcome object 499 """ 500 done_file = self.GetDoneFile(commit_upto, target) 501 sizes_file = self.GetSizesFile(commit_upto, target) 502 sizes = {} 503 func_sizes = {} 504 if os.path.exists(done_file): 505 with open(done_file, 'r') as fd: 506 return_code = int(fd.readline()) 507 err_lines = [] 508 err_file = self.GetErrFile(commit_upto, target) 509 if os.path.exists(err_file): 510 with open(err_file, 'r') as fd: 511 err_lines = self.FilterErrors(fd.readlines()) 512 513 # Decide whether the build was ok, failed or created warnings 514 if return_code: 515 rc = OUTCOME_ERROR 516 elif len(err_lines): 517 rc = OUTCOME_WARNING 518 else: 519 rc = OUTCOME_OK 520 521 # Convert size information to our simple format 522 if os.path.exists(sizes_file): 523 with open(sizes_file, 'r') as fd: 524 for line in fd.readlines(): 525 values = line.split() 526 rodata = 0 527 if len(values) > 6: 528 rodata = int(values[6], 16) 529 size_dict = { 530 'all' : int(values[0]) + int(values[1]) + 531 int(values[2]), 532 'text' : int(values[0]) - rodata, 533 'data' : int(values[1]), 534 'bss' : int(values[2]), 535 'rodata' : rodata, 536 } 537 sizes[values[5]] = size_dict 538 539 if read_func_sizes: 540 pattern = self.GetFuncSizesFile(commit_upto, target, '*') 541 for fname in glob.glob(pattern): 542 with open(fname, 'r') as fd: 543 dict_name = os.path.basename(fname).replace('.sizes', 544 '') 545 func_sizes[dict_name] = self.ReadFuncSizes(fname, fd) 546 547 return Builder.Outcome(rc, err_lines, sizes, func_sizes) 548 549 return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}) 550 551 def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes): 552 """Calculate a summary of the results of building a commit. 553 554 Args: 555 board_selected: Dict containing boards to summarise 556 commit_upto: Commit number to summarize (0..self.count-1) 557 read_func_sizes: True to read function size information 558 559 Returns: 560 Tuple: 561 Dict containing boards which passed building this commit. 562 keyed by board.target 563 List containing a summary of error/warning lines 564 """ 565 board_dict = {} 566 err_lines_summary = [] 567 568 for board in boards_selected.itervalues(): 569 outcome = self.GetBuildOutcome(commit_upto, board.target, 570 read_func_sizes) 571 board_dict[board.target] = outcome 572 for err in outcome.err_lines: 573 if err and not err.rstrip() in err_lines_summary: 574 err_lines_summary.append(err.rstrip()) 575 return board_dict, err_lines_summary 576 577 def AddOutcome(self, board_dict, arch_list, changes, char, color): 578 """Add an output to our list of outcomes for each architecture 579 580 This simple function adds failing boards (changes) to the 581 relevant architecture string, so we can print the results out 582 sorted by architecture. 583 584 Args: 585 board_dict: Dict containing all boards 586 arch_list: Dict keyed by arch name. Value is a string containing 587 a list of board names which failed for that arch. 588 changes: List of boards to add to arch_list 589 color: terminal.Colour object 590 """ 591 done_arch = {} 592 for target in changes: 593 if target in board_dict: 594 arch = board_dict[target].arch 595 else: 596 arch = 'unknown' 597 str = self.col.Color(color, ' ' + target) 598 if not arch in done_arch: 599 str = self.col.Color(color, char) + ' ' + str 600 done_arch[arch] = True 601 if not arch in arch_list: 602 arch_list[arch] = str 603 else: 604 arch_list[arch] += str 605 606 607 def ColourNum(self, num): 608 color = self.col.RED if num > 0 else self.col.GREEN 609 if num == 0: 610 return '0' 611 return self.col.Color(color, str(num)) 612 613 def ResetResultSummary(self, board_selected): 614 """Reset the results summary ready for use. 615 616 Set up the base board list to be all those selected, and set the 617 error lines to empty. 618 619 Following this, calls to PrintResultSummary() will use this 620 information to work out what has changed. 621 622 Args: 623 board_selected: Dict containing boards to summarise, keyed by 624 board.target 625 """ 626 self._base_board_dict = {} 627 for board in board_selected: 628 self._base_board_dict[board] = Builder.Outcome(0, [], [], {}) 629 self._base_err_lines = [] 630 631 def PrintFuncSizeDetail(self, fname, old, new): 632 grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0 633 delta, common = [], {} 634 635 for a in old: 636 if a in new: 637 common[a] = 1 638 639 for name in old: 640 if name not in common: 641 remove += 1 642 down += old[name] 643 delta.append([-old[name], name]) 644 645 for name in new: 646 if name not in common: 647 add += 1 648 up += new[name] 649 delta.append([new[name], name]) 650 651 for name in common: 652 diff = new.get(name, 0) - old.get(name, 0) 653 if diff > 0: 654 grow, up = grow + 1, up + diff 655 elif diff < 0: 656 shrink, down = shrink + 1, down - diff 657 delta.append([diff, name]) 658 659 delta.sort() 660 delta.reverse() 661 662 args = [add, -remove, grow, -shrink, up, -down, up - down] 663 if max(args) == 0: 664 return 665 args = [self.ColourNum(x) for x in args] 666 indent = ' ' * 15 667 print ('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' % 668 tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args)) 669 print '%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new', 670 'delta') 671 for diff, name in delta: 672 if diff: 673 color = self.col.RED if diff > 0 else self.col.GREEN 674 msg = '%s %-38s %7s %7s %+7d' % (indent, name, 675 old.get(name, '-'), new.get(name,'-'), diff) 676 print self.col.Color(color, msg) 677 678 679 def PrintSizeDetail(self, target_list, show_bloat): 680 """Show details size information for each board 681 682 Args: 683 target_list: List of targets, each a dict containing: 684 'target': Target name 685 'total_diff': Total difference in bytes across all areas 686 <part_name>: Difference for that part 687 show_bloat: Show detail for each function 688 """ 689 targets_by_diff = sorted(target_list, reverse=True, 690 key=lambda x: x['_total_diff']) 691 for result in targets_by_diff: 692 printed_target = False 693 for name in sorted(result): 694 diff = result[name] 695 if name.startswith('_'): 696 continue 697 if diff != 0: 698 color = self.col.RED if diff > 0 else self.col.GREEN 699 msg = ' %s %+d' % (name, diff) 700 if not printed_target: 701 print '%10s %-15s:' % ('', result['_target']), 702 printed_target = True 703 print self.col.Color(color, msg), 704 if printed_target: 705 print 706 if show_bloat: 707 target = result['_target'] 708 outcome = result['_outcome'] 709 base_outcome = self._base_board_dict[target] 710 for fname in outcome.func_sizes: 711 self.PrintFuncSizeDetail(fname, 712 base_outcome.func_sizes[fname], 713 outcome.func_sizes[fname]) 714 715 716 def PrintSizeSummary(self, board_selected, board_dict, show_detail, 717 show_bloat): 718 """Print a summary of image sizes broken down by section. 719 720 The summary takes the form of one line per architecture. The 721 line contains deltas for each of the sections (+ means the section 722 got bigger, - means smaller). The nunmbers are the average number 723 of bytes that a board in this section increased by. 724 725 For example: 726 powerpc: (622 boards) text -0.0 727 arm: (285 boards) text -0.0 728 nds32: (3 boards) text -8.0 729 730 Args: 731 board_selected: Dict containing boards to summarise, keyed by 732 board.target 733 board_dict: Dict containing boards for which we built this 734 commit, keyed by board.target. The value is an Outcome object. 735 show_detail: Show detail for each board 736 show_bloat: Show detail for each function 737 """ 738 arch_list = {} 739 arch_count = {} 740 741 # Calculate changes in size for different image parts 742 # The previous sizes are in Board.sizes, for each board 743 for target in board_dict: 744 if target not in board_selected: 745 continue 746 base_sizes = self._base_board_dict[target].sizes 747 outcome = board_dict[target] 748 sizes = outcome.sizes 749 750 # Loop through the list of images, creating a dict of size 751 # changes for each image/part. We end up with something like 752 # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4} 753 # which means that U-Boot data increased by 5 bytes and SPL 754 # text decreased by 4. 755 err = {'_target' : target} 756 for image in sizes: 757 if image in base_sizes: 758 base_image = base_sizes[image] 759 # Loop through the text, data, bss parts 760 for part in sorted(sizes[image]): 761 diff = sizes[image][part] - base_image[part] 762 col = None 763 if diff: 764 if image == 'u-boot': 765 name = part 766 else: 767 name = image + ':' + part 768 err[name] = diff 769 arch = board_selected[target].arch 770 if not arch in arch_count: 771 arch_count[arch] = 1 772 else: 773 arch_count[arch] += 1 774 if not sizes: 775 pass # Only add to our list when we have some stats 776 elif not arch in arch_list: 777 arch_list[arch] = [err] 778 else: 779 arch_list[arch].append(err) 780 781 # We now have a list of image size changes sorted by arch 782 # Print out a summary of these 783 for arch, target_list in arch_list.iteritems(): 784 # Get total difference for each type 785 totals = {} 786 for result in target_list: 787 total = 0 788 for name, diff in result.iteritems(): 789 if name.startswith('_'): 790 continue 791 total += diff 792 if name in totals: 793 totals[name] += diff 794 else: 795 totals[name] = diff 796 result['_total_diff'] = total 797 result['_outcome'] = board_dict[result['_target']] 798 799 count = len(target_list) 800 printed_arch = False 801 for name in sorted(totals): 802 diff = totals[name] 803 if diff: 804 # Display the average difference in this name for this 805 # architecture 806 avg_diff = float(diff) / count 807 color = self.col.RED if avg_diff > 0 else self.col.GREEN 808 msg = ' %s %+1.1f' % (name, avg_diff) 809 if not printed_arch: 810 print '%10s: (for %d/%d boards)' % (arch, count, 811 arch_count[arch]), 812 printed_arch = True 813 print self.col.Color(color, msg), 814 815 if printed_arch: 816 print 817 if show_detail: 818 self.PrintSizeDetail(target_list, show_bloat) 819 820 821 def PrintResultSummary(self, board_selected, board_dict, err_lines, 822 show_sizes, show_detail, show_bloat): 823 """Compare results with the base results and display delta. 824 825 Only boards mentioned in board_selected will be considered. This 826 function is intended to be called repeatedly with the results of 827 each commit. It therefore shows a 'diff' between what it saw in 828 the last call and what it sees now. 829 830 Args: 831 board_selected: Dict containing boards to summarise, keyed by 832 board.target 833 board_dict: Dict containing boards for which we built this 834 commit, keyed by board.target. The value is an Outcome object. 835 err_lines: A list of errors for this commit, or [] if there is 836 none, or we don't want to print errors 837 show_sizes: Show image size deltas 838 show_detail: Show detail for each board 839 show_bloat: Show detail for each function 840 """ 841 better = [] # List of boards fixed since last commit 842 worse = [] # List of new broken boards since last commit 843 new = [] # List of boards that didn't exist last time 844 unknown = [] # List of boards that were not built 845 846 for target in board_dict: 847 if target not in board_selected: 848 continue 849 850 # If the board was built last time, add its outcome to a list 851 if target in self._base_board_dict: 852 base_outcome = self._base_board_dict[target].rc 853 outcome = board_dict[target] 854 if outcome.rc == OUTCOME_UNKNOWN: 855 unknown.append(target) 856 elif outcome.rc < base_outcome: 857 better.append(target) 858 elif outcome.rc > base_outcome: 859 worse.append(target) 860 else: 861 new.append(target) 862 863 # Get a list of errors that have appeared, and disappeared 864 better_err = [] 865 worse_err = [] 866 for line in err_lines: 867 if line not in self._base_err_lines: 868 worse_err.append('+' + line) 869 for line in self._base_err_lines: 870 if line not in err_lines: 871 better_err.append('-' + line) 872 873 # Display results by arch 874 if better or worse or unknown or new or worse_err or better_err: 875 arch_list = {} 876 self.AddOutcome(board_selected, arch_list, better, '', 877 self.col.GREEN) 878 self.AddOutcome(board_selected, arch_list, worse, '+', 879 self.col.RED) 880 self.AddOutcome(board_selected, arch_list, new, '*', self.col.BLUE) 881 if self._show_unknown: 882 self.AddOutcome(board_selected, arch_list, unknown, '?', 883 self.col.MAGENTA) 884 for arch, target_list in arch_list.iteritems(): 885 print '%10s: %s' % (arch, target_list) 886 if better_err: 887 print self.col.Color(self.col.GREEN, '\n'.join(better_err)) 888 if worse_err: 889 print self.col.Color(self.col.RED, '\n'.join(worse_err)) 890 891 if show_sizes: 892 self.PrintSizeSummary(board_selected, board_dict, show_detail, 893 show_bloat) 894 895 # Save our updated information for the next call to this function 896 self._base_board_dict = board_dict 897 self._base_err_lines = err_lines 898 899 # Get a list of boards that did not get built, if needed 900 not_built = [] 901 for board in board_selected: 902 if not board in board_dict: 903 not_built.append(board) 904 if not_built: 905 print "Boards not built (%d): %s" % (len(not_built), 906 ', '.join(not_built)) 907 908 def ProduceResultSummary(self, commit_upto, commits, board_selected): 909 board_dict, err_lines = self.GetResultSummary(board_selected, 910 commit_upto, read_func_sizes=self._show_bloat) 911 if commits: 912 msg = '%02d: %s' % (commit_upto + 1, 913 commits[commit_upto].subject) 914 print self.col.Color(self.col.BLUE, msg) 915 self.PrintResultSummary(board_selected, board_dict, 916 err_lines if self._show_errors else [], 917 self._show_sizes, self._show_detail, self._show_bloat) 918 919 def ShowSummary(self, commits, board_selected): 920 """Show a build summary for U-Boot for a given board list. 921 922 Reset the result summary, then repeatedly call GetResultSummary on 923 each commit's results, then display the differences we see. 924 925 Args: 926 commit: Commit objects to summarise 927 board_selected: Dict containing boards to summarise 928 """ 929 self.commit_count = len(commits) if commits else 1 930 self.commits = commits 931 self.ResetResultSummary(board_selected) 932 933 for commit_upto in range(0, self.commit_count, self._step): 934 self.ProduceResultSummary(commit_upto, commits, board_selected) 935 936 937 def SetupBuild(self, board_selected, commits): 938 """Set up ready to start a build. 939 940 Args: 941 board_selected: Selected boards to build 942 commits: Selected commits to build 943 """ 944 # First work out how many commits we will build 945 count = (self.commit_count + self._step - 1) / self._step 946 self.count = len(board_selected) * count 947 self.upto = self.warned = self.fail = 0 948 self._timestamps = collections.deque() 949 950 def BuildBoardsForCommit(self, board_selected, keep_outputs): 951 """Build all boards for a single commit""" 952 self.SetupBuild(board_selected) 953 self.count = len(board_selected) 954 for brd in board_selected.itervalues(): 955 job = BuilderJob() 956 job.board = brd 957 job.commits = None 958 job.keep_outputs = keep_outputs 959 self.queue.put(brd) 960 961 self.queue.join() 962 self.out_queue.join() 963 print 964 self.ClearLine(0) 965 966 def BuildCommits(self, commits, board_selected, show_errors, keep_outputs): 967 """Build all boards for all commits (non-incremental)""" 968 self.commit_count = len(commits) 969 970 self.ResetResultSummary(board_selected) 971 for self.commit_upto in range(self.commit_count): 972 self.SelectCommit(commits[self.commit_upto]) 973 self.SelectOutputDir() 974 builderthread.Mkdir(self.output_dir) 975 976 self.BuildBoardsForCommit(board_selected, keep_outputs) 977 board_dict, err_lines = self.GetResultSummary() 978 self.PrintResultSummary(board_selected, board_dict, 979 err_lines if show_errors else []) 980 981 if self.already_done: 982 print '%d builds already done' % self.already_done 983 984 def GetThreadDir(self, thread_num): 985 """Get the directory path to the working dir for a thread. 986 987 Args: 988 thread_num: Number of thread to check. 989 """ 990 return os.path.join(self._working_dir, '%02d' % thread_num) 991 992 def _PrepareThread(self, thread_num, setup_git): 993 """Prepare the working directory for a thread. 994 995 This clones or fetches the repo into the thread's work directory. 996 997 Args: 998 thread_num: Thread number (0, 1, ...) 999 setup_git: True to set up a git repo clone 1000 """ 1001 thread_dir = self.GetThreadDir(thread_num) 1002 builderthread.Mkdir(thread_dir) 1003 git_dir = os.path.join(thread_dir, '.git') 1004 1005 # Clone the repo if it doesn't already exist 1006 # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so 1007 # we have a private index but uses the origin repo's contents? 1008 if setup_git and self.git_dir: 1009 src_dir = os.path.abspath(self.git_dir) 1010 if os.path.exists(git_dir): 1011 gitutil.Fetch(git_dir, thread_dir) 1012 else: 1013 print 'Cloning repo for thread %d' % thread_num 1014 gitutil.Clone(src_dir, thread_dir) 1015 1016 def _PrepareWorkingSpace(self, max_threads, setup_git): 1017 """Prepare the working directory for use. 1018 1019 Set up the git repo for each thread. 1020 1021 Args: 1022 max_threads: Maximum number of threads we expect to need. 1023 setup_git: True to set up a git repo clone 1024 """ 1025 builderthread.Mkdir(self._working_dir) 1026 for thread in range(max_threads): 1027 self._PrepareThread(thread, setup_git) 1028 1029 def _PrepareOutputSpace(self): 1030 """Get the output directories ready to receive files. 1031 1032 We delete any output directories which look like ones we need to 1033 create. Having left over directories is confusing when the user wants 1034 to check the output manually. 1035 """ 1036 dir_list = [] 1037 for commit_upto in range(self.commit_count): 1038 dir_list.append(self._GetOutputDir(commit_upto)) 1039 1040 for dirname in glob.glob(os.path.join(self.base_dir, '*')): 1041 if dirname not in dir_list: 1042 shutil.rmtree(dirname) 1043 1044 def BuildBoards(self, commits, board_selected, keep_outputs): 1045 """Build all commits for a list of boards 1046 1047 Args: 1048 commits: List of commits to be build, each a Commit object 1049 boards_selected: Dict of selected boards, key is target name, 1050 value is Board object 1051 keep_outputs: True to save build output files 1052 """ 1053 self.commit_count = len(commits) if commits else 1 1054 self.commits = commits 1055 1056 self.ResetResultSummary(board_selected) 1057 builderthread.Mkdir(self.base_dir) 1058 self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)), 1059 commits is not None) 1060 self._PrepareOutputSpace() 1061 self.SetupBuild(board_selected, commits) 1062 self.ProcessResult(None) 1063 1064 # Create jobs to build all commits for each board 1065 for brd in board_selected.itervalues(): 1066 job = builderthread.BuilderJob() 1067 job.board = brd 1068 job.commits = commits 1069 job.keep_outputs = keep_outputs 1070 job.step = self._step 1071 self.queue.put(job) 1072 1073 # Wait until all jobs are started 1074 self.queue.join() 1075 1076 # Wait until we have processed all output 1077 self.out_queue.join() 1078 print 1079 self.ClearLine(0) 1080