1# Copyright (c) 2013 The Chromium OS Authors. 2# 3# Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com> 4# 5# SPDX-License-Identifier: GPL-2.0+ 6# 7 8import collections 9from datetime import datetime, timedelta 10import glob 11import os 12import re 13import Queue 14import shutil 15import string 16import sys 17import time 18 19import builderthread 20import command 21import gitutil 22import terminal 23import toolchain 24 25 26""" 27Theory of Operation 28 29Please see README for user documentation, and you should be familiar with 30that before trying to make sense of this. 31 32Buildman works by keeping the machine as busy as possible, building different 33commits for different boards on multiple CPUs at once. 34 35The source repo (self.git_dir) contains all the commits to be built. Each 36thread works on a single board at a time. It checks out the first commit, 37configures it for that board, then builds it. Then it checks out the next 38commit and builds it (typically without re-configuring). When it runs out 39of commits, it gets another job from the builder and starts again with that 40board. 41 42Clearly the builder threads could work either way - they could check out a 43commit and then built it for all boards. Using separate directories for each 44commit/board pair they could leave their build product around afterwards 45also. 46 47The intent behind building a single board for multiple commits, is to make 48use of incremental builds. Since each commit is built incrementally from 49the previous one, builds are faster. Reconfiguring for a different board 50removes all intermediate object files. 51 52Many threads can be working at once, but each has its own working directory. 53When a thread finishes a build, it puts the output files into a result 54directory. 55 56The base directory used by buildman is normally '../<branch>', i.e. 57a directory higher than the source repository and named after the branch 58being built. 59 60Within the base directory, we have one subdirectory for each commit. Within 61that is one subdirectory for each board. Within that is the build output for 62that commit/board combination. 63 64Buildman also create working directories for each thread, in a .bm-work/ 65subdirectory in the base dir. 66 67As an example, say we are building branch 'us-net' for boards 'sandbox' and 68'seaboard', and say that us-net has two commits. We will have directories 69like this: 70 71us-net/ base directory 72 01_of_02_g4ed4ebc_net--Add-tftp-speed-/ 73 sandbox/ 74 u-boot.bin 75 seaboard/ 76 u-boot.bin 77 02_of_02_g4ed4ebc_net--Check-tftp-comp/ 78 sandbox/ 79 u-boot.bin 80 seaboard/ 81 u-boot.bin 82 .bm-work/ 83 00/ working directory for thread 0 (contains source checkout) 84 build/ build output 85 01/ working directory for thread 1 86 build/ build output 87 ... 88u-boot/ source directory 89 .git/ repository 90""" 91 92# Possible build outcomes 93OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4) 94 95# Translate a commit subject into a valid filename 96trans_valid_chars = string.maketrans("/: ", "---") 97 98 99class Builder: 100 """Class for building U-Boot for a particular commit. 101 102 Public members: (many should ->private) 103 active: True if the builder is active and has not been stopped 104 already_done: Number of builds already completed 105 base_dir: Base directory to use for builder 106 checkout: True to check out source, False to skip that step. 107 This is used for testing. 108 col: terminal.Color() object 109 count: Number of commits to build 110 do_make: Method to call to invoke Make 111 fail: Number of builds that failed due to error 112 force_build: Force building even if a build already exists 113 force_config_on_failure: If a commit fails for a board, disable 114 incremental building for the next commit we build for that 115 board, so that we will see all warnings/errors again. 116 force_build_failures: If a previously-built build (i.e. built on 117 a previous run of buildman) is marked as failed, rebuild it. 118 git_dir: Git directory containing source repository 119 last_line_len: Length of the last line we printed (used for erasing 120 it with new progress information) 121 num_jobs: Number of jobs to run at once (passed to make as -j) 122 num_threads: Number of builder threads to run 123 out_queue: Queue of results to process 124 re_make_err: Compiled regular expression for ignore_lines 125 queue: Queue of jobs to run 126 threads: List of active threads 127 toolchains: Toolchains object to use for building 128 upto: Current commit number we are building (0.count-1) 129 warned: Number of builds that produced at least one warning 130 force_reconfig: Reconfigure U-Boot on each comiit. This disables 131 incremental building, where buildman reconfigures on the first 132 commit for a baord, and then just does an incremental build for 133 the following commits. In fact buildman will reconfigure and 134 retry for any failing commits, so generally the only effect of 135 this option is to slow things down. 136 in_tree: Build U-Boot in-tree instead of specifying an output 137 directory separate from the source code. This option is really 138 only useful for testing in-tree builds. 139 140 Private members: 141 _base_board_dict: Last-summarised Dict of boards 142 _base_err_lines: Last-summarised list of errors 143 _base_warn_lines: Last-summarised list of warnings 144 _build_period_us: Time taken for a single build (float object). 145 _complete_delay: Expected delay until completion (timedelta) 146 _next_delay_update: Next time we plan to display a progress update 147 (datatime) 148 _show_unknown: Show unknown boards (those not built) in summary 149 _timestamps: List of timestamps for the completion of the last 150 last _timestamp_count builds. Each is a datetime object. 151 _timestamp_count: Number of timestamps to keep in our list. 152 _working_dir: Base working directory containing all threads 153 """ 154 class Outcome: 155 """Records a build outcome for a single make invocation 156 157 Public Members: 158 rc: Outcome value (OUTCOME_...) 159 err_lines: List of error lines or [] if none 160 sizes: Dictionary of image size information, keyed by filename 161 - Each value is itself a dictionary containing 162 values for 'text', 'data' and 'bss', being the integer 163 size in bytes of each section. 164 func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each 165 value is itself a dictionary: 166 key: function name 167 value: Size of function in bytes 168 """ 169 def __init__(self, rc, err_lines, sizes, func_sizes): 170 self.rc = rc 171 self.err_lines = err_lines 172 self.sizes = sizes 173 self.func_sizes = func_sizes 174 175 def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, 176 gnu_make='make', checkout=True, show_unknown=True, step=1): 177 """Create a new Builder object 178 179 Args: 180 toolchains: Toolchains object to use for building 181 base_dir: Base directory to use for builder 182 git_dir: Git directory containing source repository 183 num_threads: Number of builder threads to run 184 num_jobs: Number of jobs to run at once (passed to make as -j) 185 gnu_make: the command name of GNU Make. 186 checkout: True to check out source, False to skip that step. 187 This is used for testing. 188 show_unknown: Show unknown boards (those not built) in summary 189 step: 1 to process every commit, n to process every nth commit 190 """ 191 self.toolchains = toolchains 192 self.base_dir = base_dir 193 self._working_dir = os.path.join(base_dir, '.bm-work') 194 self.threads = [] 195 self.active = True 196 self.do_make = self.Make 197 self.gnu_make = gnu_make 198 self.checkout = checkout 199 self.num_threads = num_threads 200 self.num_jobs = num_jobs 201 self.already_done = 0 202 self.force_build = False 203 self.git_dir = git_dir 204 self._show_unknown = show_unknown 205 self._timestamp_count = 10 206 self._build_period_us = None 207 self._complete_delay = None 208 self._next_delay_update = datetime.now() 209 self.force_config_on_failure = True 210 self.force_build_failures = False 211 self.force_reconfig = False 212 self._step = step 213 self.in_tree = False 214 self._error_lines = 0 215 216 self.col = terminal.Color() 217 218 self._re_function = re.compile('(.*): In function.*') 219 self._re_files = re.compile('In file included from.*') 220 self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*') 221 self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*') 222 223 self.queue = Queue.Queue() 224 self.out_queue = Queue.Queue() 225 for i in range(self.num_threads): 226 t = builderthread.BuilderThread(self, i) 227 t.setDaemon(True) 228 t.start() 229 self.threads.append(t) 230 231 self.last_line_len = 0 232 t = builderthread.ResultThread(self) 233 t.setDaemon(True) 234 t.start() 235 self.threads.append(t) 236 237 ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)'] 238 self.re_make_err = re.compile('|'.join(ignore_lines)) 239 240 def __del__(self): 241 """Get rid of all threads created by the builder""" 242 for t in self.threads: 243 del t 244 245 def SetDisplayOptions(self, show_errors=False, show_sizes=False, 246 show_detail=False, show_bloat=False, 247 list_error_boards=False): 248 """Setup display options for the builder. 249 250 show_errors: True to show summarised error/warning info 251 show_sizes: Show size deltas 252 show_detail: Show detail for each board 253 show_bloat: Show detail for each function 254 list_error_boards: Show the boards which caused each error/warning 255 """ 256 self._show_errors = show_errors 257 self._show_sizes = show_sizes 258 self._show_detail = show_detail 259 self._show_bloat = show_bloat 260 self._list_error_boards = list_error_boards 261 262 def _AddTimestamp(self): 263 """Add a new timestamp to the list and record the build period. 264 265 The build period is the length of time taken to perform a single 266 build (one board, one commit). 267 """ 268 now = datetime.now() 269 self._timestamps.append(now) 270 count = len(self._timestamps) 271 delta = self._timestamps[-1] - self._timestamps[0] 272 seconds = delta.total_seconds() 273 274 # If we have enough data, estimate build period (time taken for a 275 # single build) and therefore completion time. 276 if count > 1 and self._next_delay_update < now: 277 self._next_delay_update = now + timedelta(seconds=2) 278 if seconds > 0: 279 self._build_period = float(seconds) / count 280 todo = self.count - self.upto 281 self._complete_delay = timedelta(microseconds= 282 self._build_period * todo * 1000000) 283 # Round it 284 self._complete_delay -= timedelta( 285 microseconds=self._complete_delay.microseconds) 286 287 if seconds > 60: 288 self._timestamps.popleft() 289 count -= 1 290 291 def ClearLine(self, length): 292 """Clear any characters on the current line 293 294 Make way for a new line of length 'length', by outputting enough 295 spaces to clear out the old line. Then remember the new length for 296 next time. 297 298 Args: 299 length: Length of new line, in characters 300 """ 301 if length < self.last_line_len: 302 print ' ' * (self.last_line_len - length), 303 print '\r', 304 self.last_line_len = length 305 sys.stdout.flush() 306 307 def SelectCommit(self, commit, checkout=True): 308 """Checkout the selected commit for this build 309 """ 310 self.commit = commit 311 if checkout and self.checkout: 312 gitutil.Checkout(commit.hash) 313 314 def Make(self, commit, brd, stage, cwd, *args, **kwargs): 315 """Run make 316 317 Args: 318 commit: Commit object that is being built 319 brd: Board object that is being built 320 stage: Stage that we are at (mrproper, config, build) 321 cwd: Directory where make should be run 322 args: Arguments to pass to make 323 kwargs: Arguments to pass to command.RunPipe() 324 """ 325 cmd = [self.gnu_make] + list(args) 326 result = command.RunPipe([cmd], capture=True, capture_stderr=True, 327 cwd=cwd, raise_on_error=False, **kwargs) 328 return result 329 330 def ProcessResult(self, result): 331 """Process the result of a build, showing progress information 332 333 Args: 334 result: A CommandResult object, which indicates the result for 335 a single build 336 """ 337 col = terminal.Color() 338 if result: 339 target = result.brd.target 340 341 if result.return_code < 0: 342 self.active = False 343 command.StopAll() 344 return 345 346 self.upto += 1 347 if result.return_code != 0: 348 self.fail += 1 349 elif result.stderr: 350 self.warned += 1 351 if result.already_done: 352 self.already_done += 1 353 if self._verbose: 354 print '\r', 355 self.ClearLine(0) 356 boards_selected = {target : result.brd} 357 self.ResetResultSummary(boards_selected) 358 self.ProduceResultSummary(result.commit_upto, self.commits, 359 boards_selected) 360 else: 361 target = '(starting)' 362 363 # Display separate counts for ok, warned and fail 364 ok = self.upto - self.warned - self.fail 365 line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok) 366 line += self.col.Color(self.col.YELLOW, '%5d' % self.warned) 367 line += self.col.Color(self.col.RED, '%5d' % self.fail) 368 369 name = ' /%-5d ' % self.count 370 371 # Add our current completion time estimate 372 self._AddTimestamp() 373 if self._complete_delay: 374 name += '%s : ' % self._complete_delay 375 # When building all boards for a commit, we can print a commit 376 # progress message. 377 if result and result.commit_upto is None: 378 name += 'commit %2d/%-3d' % (self.commit_upto + 1, 379 self.commit_count) 380 381 name += target 382 print line + name, 383 length = 14 + len(name) 384 self.ClearLine(length) 385 386 def _GetOutputDir(self, commit_upto): 387 """Get the name of the output directory for a commit number 388 389 The output directory is typically .../<branch>/<commit>. 390 391 Args: 392 commit_upto: Commit number to use (0..self.count-1) 393 """ 394 if self.commits: 395 commit = self.commits[commit_upto] 396 subject = commit.subject.translate(trans_valid_chars) 397 commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1, 398 self.commit_count, commit.hash, subject[:20])) 399 else: 400 commit_dir = 'current' 401 output_dir = os.path.join(self.base_dir, commit_dir) 402 return output_dir 403 404 def GetBuildDir(self, commit_upto, target): 405 """Get the name of the build directory for a commit number 406 407 The build directory is typically .../<branch>/<commit>/<target>. 408 409 Args: 410 commit_upto: Commit number to use (0..self.count-1) 411 target: Target name 412 """ 413 output_dir = self._GetOutputDir(commit_upto) 414 return os.path.join(output_dir, target) 415 416 def GetDoneFile(self, commit_upto, target): 417 """Get the name of the done file for a commit number 418 419 Args: 420 commit_upto: Commit number to use (0..self.count-1) 421 target: Target name 422 """ 423 return os.path.join(self.GetBuildDir(commit_upto, target), 'done') 424 425 def GetSizesFile(self, commit_upto, target): 426 """Get the name of the sizes file for a commit number 427 428 Args: 429 commit_upto: Commit number to use (0..self.count-1) 430 target: Target name 431 """ 432 return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes') 433 434 def GetFuncSizesFile(self, commit_upto, target, elf_fname): 435 """Get the name of the funcsizes file for a commit number and ELF file 436 437 Args: 438 commit_upto: Commit number to use (0..self.count-1) 439 target: Target name 440 elf_fname: Filename of elf image 441 """ 442 return os.path.join(self.GetBuildDir(commit_upto, target), 443 '%s.sizes' % elf_fname.replace('/', '-')) 444 445 def GetObjdumpFile(self, commit_upto, target, elf_fname): 446 """Get the name of the objdump file for a commit number and ELF file 447 448 Args: 449 commit_upto: Commit number to use (0..self.count-1) 450 target: Target name 451 elf_fname: Filename of elf image 452 """ 453 return os.path.join(self.GetBuildDir(commit_upto, target), 454 '%s.objdump' % elf_fname.replace('/', '-')) 455 456 def GetErrFile(self, commit_upto, target): 457 """Get the name of the err file for a commit number 458 459 Args: 460 commit_upto: Commit number to use (0..self.count-1) 461 target: Target name 462 """ 463 output_dir = self.GetBuildDir(commit_upto, target) 464 return os.path.join(output_dir, 'err') 465 466 def FilterErrors(self, lines): 467 """Filter out errors in which we have no interest 468 469 We should probably use map(). 470 471 Args: 472 lines: List of error lines, each a string 473 Returns: 474 New list with only interesting lines included 475 """ 476 out_lines = [] 477 for line in lines: 478 if not self.re_make_err.search(line): 479 out_lines.append(line) 480 return out_lines 481 482 def ReadFuncSizes(self, fname, fd): 483 """Read function sizes from the output of 'nm' 484 485 Args: 486 fd: File containing data to read 487 fname: Filename we are reading from (just for errors) 488 489 Returns: 490 Dictionary containing size of each function in bytes, indexed by 491 function name. 492 """ 493 sym = {} 494 for line in fd.readlines(): 495 try: 496 size, type, name = line[:-1].split() 497 except: 498 print "Invalid line in file '%s': '%s'" % (fname, line[:-1]) 499 continue 500 if type in 'tTdDbB': 501 # function names begin with '.' on 64-bit powerpc 502 if '.' in name[1:]: 503 name = 'static.' + name.split('.')[0] 504 sym[name] = sym.get(name, 0) + int(size, 16) 505 return sym 506 507 def GetBuildOutcome(self, commit_upto, target, read_func_sizes): 508 """Work out the outcome of a build. 509 510 Args: 511 commit_upto: Commit number to check (0..n-1) 512 target: Target board to check 513 read_func_sizes: True to read function size information 514 515 Returns: 516 Outcome object 517 """ 518 done_file = self.GetDoneFile(commit_upto, target) 519 sizes_file = self.GetSizesFile(commit_upto, target) 520 sizes = {} 521 func_sizes = {} 522 if os.path.exists(done_file): 523 with open(done_file, 'r') as fd: 524 return_code = int(fd.readline()) 525 err_lines = [] 526 err_file = self.GetErrFile(commit_upto, target) 527 if os.path.exists(err_file): 528 with open(err_file, 'r') as fd: 529 err_lines = self.FilterErrors(fd.readlines()) 530 531 # Decide whether the build was ok, failed or created warnings 532 if return_code: 533 rc = OUTCOME_ERROR 534 elif len(err_lines): 535 rc = OUTCOME_WARNING 536 else: 537 rc = OUTCOME_OK 538 539 # Convert size information to our simple format 540 if os.path.exists(sizes_file): 541 with open(sizes_file, 'r') as fd: 542 for line in fd.readlines(): 543 values = line.split() 544 rodata = 0 545 if len(values) > 6: 546 rodata = int(values[6], 16) 547 size_dict = { 548 'all' : int(values[0]) + int(values[1]) + 549 int(values[2]), 550 'text' : int(values[0]) - rodata, 551 'data' : int(values[1]), 552 'bss' : int(values[2]), 553 'rodata' : rodata, 554 } 555 sizes[values[5]] = size_dict 556 557 if read_func_sizes: 558 pattern = self.GetFuncSizesFile(commit_upto, target, '*') 559 for fname in glob.glob(pattern): 560 with open(fname, 'r') as fd: 561 dict_name = os.path.basename(fname).replace('.sizes', 562 '') 563 func_sizes[dict_name] = self.ReadFuncSizes(fname, fd) 564 565 return Builder.Outcome(rc, err_lines, sizes, func_sizes) 566 567 return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}) 568 569 def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes): 570 """Calculate a summary of the results of building a commit. 571 572 Args: 573 board_selected: Dict containing boards to summarise 574 commit_upto: Commit number to summarize (0..self.count-1) 575 read_func_sizes: True to read function size information 576 577 Returns: 578 Tuple: 579 Dict containing boards which passed building this commit. 580 keyed by board.target 581 List containing a summary of error lines 582 Dict keyed by error line, containing a list of the Board 583 objects with that error 584 List containing a summary of warning lines 585 Dict keyed by error line, containing a list of the Board 586 objects with that warning 587 """ 588 def AddLine(lines_summary, lines_boards, line, board): 589 line = line.rstrip() 590 if line in lines_boards: 591 lines_boards[line].append(board) 592 else: 593 lines_boards[line] = [board] 594 lines_summary.append(line) 595 596 board_dict = {} 597 err_lines_summary = [] 598 err_lines_boards = {} 599 warn_lines_summary = [] 600 warn_lines_boards = {} 601 602 for board in boards_selected.itervalues(): 603 outcome = self.GetBuildOutcome(commit_upto, board.target, 604 read_func_sizes) 605 board_dict[board.target] = outcome 606 last_func = None 607 last_was_warning = False 608 for line in outcome.err_lines: 609 if line: 610 if (self._re_function.match(line) or 611 self._re_files.match(line)): 612 last_func = line 613 else: 614 is_warning = self._re_warning.match(line) 615 is_note = self._re_note.match(line) 616 if is_warning or (last_was_warning and is_note): 617 if last_func: 618 AddLine(warn_lines_summary, warn_lines_boards, 619 last_func, board) 620 AddLine(warn_lines_summary, warn_lines_boards, 621 line, board) 622 else: 623 if last_func: 624 AddLine(err_lines_summary, err_lines_boards, 625 last_func, board) 626 AddLine(err_lines_summary, err_lines_boards, 627 line, board) 628 last_was_warning = is_warning 629 last_func = None 630 return (board_dict, err_lines_summary, err_lines_boards, 631 warn_lines_summary, warn_lines_boards) 632 633 def AddOutcome(self, board_dict, arch_list, changes, char, color): 634 """Add an output to our list of outcomes for each architecture 635 636 This simple function adds failing boards (changes) to the 637 relevant architecture string, so we can print the results out 638 sorted by architecture. 639 640 Args: 641 board_dict: Dict containing all boards 642 arch_list: Dict keyed by arch name. Value is a string containing 643 a list of board names which failed for that arch. 644 changes: List of boards to add to arch_list 645 color: terminal.Colour object 646 """ 647 done_arch = {} 648 for target in changes: 649 if target in board_dict: 650 arch = board_dict[target].arch 651 else: 652 arch = 'unknown' 653 str = self.col.Color(color, ' ' + target) 654 if not arch in done_arch: 655 str = self.col.Color(color, char) + ' ' + str 656 done_arch[arch] = True 657 if not arch in arch_list: 658 arch_list[arch] = str 659 else: 660 arch_list[arch] += str 661 662 663 def ColourNum(self, num): 664 color = self.col.RED if num > 0 else self.col.GREEN 665 if num == 0: 666 return '0' 667 return self.col.Color(color, str(num)) 668 669 def ResetResultSummary(self, board_selected): 670 """Reset the results summary ready for use. 671 672 Set up the base board list to be all those selected, and set the 673 error lines to empty. 674 675 Following this, calls to PrintResultSummary() will use this 676 information to work out what has changed. 677 678 Args: 679 board_selected: Dict containing boards to summarise, keyed by 680 board.target 681 """ 682 self._base_board_dict = {} 683 for board in board_selected: 684 self._base_board_dict[board] = Builder.Outcome(0, [], [], {}) 685 self._base_err_lines = [] 686 self._base_warn_lines = [] 687 self._base_err_line_boards = {} 688 self._base_warn_line_boards = {} 689 690 def PrintFuncSizeDetail(self, fname, old, new): 691 grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0 692 delta, common = [], {} 693 694 for a in old: 695 if a in new: 696 common[a] = 1 697 698 for name in old: 699 if name not in common: 700 remove += 1 701 down += old[name] 702 delta.append([-old[name], name]) 703 704 for name in new: 705 if name not in common: 706 add += 1 707 up += new[name] 708 delta.append([new[name], name]) 709 710 for name in common: 711 diff = new.get(name, 0) - old.get(name, 0) 712 if diff > 0: 713 grow, up = grow + 1, up + diff 714 elif diff < 0: 715 shrink, down = shrink + 1, down - diff 716 delta.append([diff, name]) 717 718 delta.sort() 719 delta.reverse() 720 721 args = [add, -remove, grow, -shrink, up, -down, up - down] 722 if max(args) == 0: 723 return 724 args = [self.ColourNum(x) for x in args] 725 indent = ' ' * 15 726 print ('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' % 727 tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args)) 728 print '%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new', 729 'delta') 730 for diff, name in delta: 731 if diff: 732 color = self.col.RED if diff > 0 else self.col.GREEN 733 msg = '%s %-38s %7s %7s %+7d' % (indent, name, 734 old.get(name, '-'), new.get(name,'-'), diff) 735 print self.col.Color(color, msg) 736 737 738 def PrintSizeDetail(self, target_list, show_bloat): 739 """Show details size information for each board 740 741 Args: 742 target_list: List of targets, each a dict containing: 743 'target': Target name 744 'total_diff': Total difference in bytes across all areas 745 <part_name>: Difference for that part 746 show_bloat: Show detail for each function 747 """ 748 targets_by_diff = sorted(target_list, reverse=True, 749 key=lambda x: x['_total_diff']) 750 for result in targets_by_diff: 751 printed_target = False 752 for name in sorted(result): 753 diff = result[name] 754 if name.startswith('_'): 755 continue 756 if diff != 0: 757 color = self.col.RED if diff > 0 else self.col.GREEN 758 msg = ' %s %+d' % (name, diff) 759 if not printed_target: 760 print '%10s %-15s:' % ('', result['_target']), 761 printed_target = True 762 print self.col.Color(color, msg), 763 if printed_target: 764 print 765 if show_bloat: 766 target = result['_target'] 767 outcome = result['_outcome'] 768 base_outcome = self._base_board_dict[target] 769 for fname in outcome.func_sizes: 770 self.PrintFuncSizeDetail(fname, 771 base_outcome.func_sizes[fname], 772 outcome.func_sizes[fname]) 773 774 775 def PrintSizeSummary(self, board_selected, board_dict, show_detail, 776 show_bloat): 777 """Print a summary of image sizes broken down by section. 778 779 The summary takes the form of one line per architecture. The 780 line contains deltas for each of the sections (+ means the section 781 got bigger, - means smaller). The nunmbers are the average number 782 of bytes that a board in this section increased by. 783 784 For example: 785 powerpc: (622 boards) text -0.0 786 arm: (285 boards) text -0.0 787 nds32: (3 boards) text -8.0 788 789 Args: 790 board_selected: Dict containing boards to summarise, keyed by 791 board.target 792 board_dict: Dict containing boards for which we built this 793 commit, keyed by board.target. The value is an Outcome object. 794 show_detail: Show detail for each board 795 show_bloat: Show detail for each function 796 """ 797 arch_list = {} 798 arch_count = {} 799 800 # Calculate changes in size for different image parts 801 # The previous sizes are in Board.sizes, for each board 802 for target in board_dict: 803 if target not in board_selected: 804 continue 805 base_sizes = self._base_board_dict[target].sizes 806 outcome = board_dict[target] 807 sizes = outcome.sizes 808 809 # Loop through the list of images, creating a dict of size 810 # changes for each image/part. We end up with something like 811 # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4} 812 # which means that U-Boot data increased by 5 bytes and SPL 813 # text decreased by 4. 814 err = {'_target' : target} 815 for image in sizes: 816 if image in base_sizes: 817 base_image = base_sizes[image] 818 # Loop through the text, data, bss parts 819 for part in sorted(sizes[image]): 820 diff = sizes[image][part] - base_image[part] 821 col = None 822 if diff: 823 if image == 'u-boot': 824 name = part 825 else: 826 name = image + ':' + part 827 err[name] = diff 828 arch = board_selected[target].arch 829 if not arch in arch_count: 830 arch_count[arch] = 1 831 else: 832 arch_count[arch] += 1 833 if not sizes: 834 pass # Only add to our list when we have some stats 835 elif not arch in arch_list: 836 arch_list[arch] = [err] 837 else: 838 arch_list[arch].append(err) 839 840 # We now have a list of image size changes sorted by arch 841 # Print out a summary of these 842 for arch, target_list in arch_list.iteritems(): 843 # Get total difference for each type 844 totals = {} 845 for result in target_list: 846 total = 0 847 for name, diff in result.iteritems(): 848 if name.startswith('_'): 849 continue 850 total += diff 851 if name in totals: 852 totals[name] += diff 853 else: 854 totals[name] = diff 855 result['_total_diff'] = total 856 result['_outcome'] = board_dict[result['_target']] 857 858 count = len(target_list) 859 printed_arch = False 860 for name in sorted(totals): 861 diff = totals[name] 862 if diff: 863 # Display the average difference in this name for this 864 # architecture 865 avg_diff = float(diff) / count 866 color = self.col.RED if avg_diff > 0 else self.col.GREEN 867 msg = ' %s %+1.1f' % (name, avg_diff) 868 if not printed_arch: 869 print '%10s: (for %d/%d boards)' % (arch, count, 870 arch_count[arch]), 871 printed_arch = True 872 print self.col.Color(color, msg), 873 874 if printed_arch: 875 print 876 if show_detail: 877 self.PrintSizeDetail(target_list, show_bloat) 878 879 880 def PrintResultSummary(self, board_selected, board_dict, err_lines, 881 err_line_boards, warn_lines, warn_line_boards, 882 show_sizes, show_detail, show_bloat): 883 """Compare results with the base results and display delta. 884 885 Only boards mentioned in board_selected will be considered. This 886 function is intended to be called repeatedly with the results of 887 each commit. It therefore shows a 'diff' between what it saw in 888 the last call and what it sees now. 889 890 Args: 891 board_selected: Dict containing boards to summarise, keyed by 892 board.target 893 board_dict: Dict containing boards for which we built this 894 commit, keyed by board.target. The value is an Outcome object. 895 err_lines: A list of errors for this commit, or [] if there is 896 none, or we don't want to print errors 897 err_line_boards: Dict keyed by error line, containing a list of 898 the Board objects with that error 899 warn_lines: A list of warnings for this commit, or [] if there is 900 none, or we don't want to print errors 901 warn_line_boards: Dict keyed by warning line, containing a list of 902 the Board objects with that warning 903 show_sizes: Show image size deltas 904 show_detail: Show detail for each board 905 show_bloat: Show detail for each function 906 """ 907 def _BoardList(line, line_boards): 908 """Helper function to get a line of boards containing a line 909 910 Args: 911 line: Error line to search for 912 Return: 913 String containing a list of boards with that error line, or 914 '' if the user has not requested such a list 915 """ 916 if self._list_error_boards: 917 names = [] 918 for board in line_boards[line]: 919 names.append(board.target) 920 names_str = '(%s) ' % ','.join(names) 921 else: 922 names_str = '' 923 return names_str 924 925 def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards, 926 char): 927 better_lines = [] 928 worse_lines = [] 929 for line in lines: 930 if line not in base_lines: 931 worse_lines.append(char + '+' + 932 _BoardList(line, line_boards) + line) 933 for line in base_lines: 934 if line not in lines: 935 better_lines.append(char + '-' + 936 _BoardList(line, base_line_boards) + line) 937 return better_lines, worse_lines 938 939 better = [] # List of boards fixed since last commit 940 worse = [] # List of new broken boards since last commit 941 new = [] # List of boards that didn't exist last time 942 unknown = [] # List of boards that were not built 943 944 for target in board_dict: 945 if target not in board_selected: 946 continue 947 948 # If the board was built last time, add its outcome to a list 949 if target in self._base_board_dict: 950 base_outcome = self._base_board_dict[target].rc 951 outcome = board_dict[target] 952 if outcome.rc == OUTCOME_UNKNOWN: 953 unknown.append(target) 954 elif outcome.rc < base_outcome: 955 better.append(target) 956 elif outcome.rc > base_outcome: 957 worse.append(target) 958 else: 959 new.append(target) 960 961 # Get a list of errors that have appeared, and disappeared 962 better_err, worse_err = _CalcErrorDelta(self._base_err_lines, 963 self._base_err_line_boards, err_lines, err_line_boards, '') 964 better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines, 965 self._base_warn_line_boards, warn_lines, warn_line_boards, 'w') 966 967 # Display results by arch 968 if (better or worse or unknown or new or worse_err or better_err 969 or worse_warn or better_warn): 970 arch_list = {} 971 self.AddOutcome(board_selected, arch_list, better, '', 972 self.col.GREEN) 973 self.AddOutcome(board_selected, arch_list, worse, '+', 974 self.col.RED) 975 self.AddOutcome(board_selected, arch_list, new, '*', self.col.BLUE) 976 if self._show_unknown: 977 self.AddOutcome(board_selected, arch_list, unknown, '?', 978 self.col.MAGENTA) 979 for arch, target_list in arch_list.iteritems(): 980 print '%10s: %s' % (arch, target_list) 981 self._error_lines += 1 982 if better_err: 983 print self.col.Color(self.col.GREEN, '\n'.join(better_err)) 984 self._error_lines += 1 985 if worse_err: 986 print self.col.Color(self.col.RED, '\n'.join(worse_err)) 987 self._error_lines += 1 988 if better_warn: 989 print self.col.Color(self.col.YELLOW, '\n'.join(better_warn)) 990 self._error_lines += 1 991 if worse_warn: 992 print self.col.Color(self.col.MAGENTA, '\n'.join(worse_warn)) 993 self._error_lines += 1 994 995 if show_sizes: 996 self.PrintSizeSummary(board_selected, board_dict, show_detail, 997 show_bloat) 998 999 # Save our updated information for the next call to this function 1000 self._base_board_dict = board_dict 1001 self._base_err_lines = err_lines 1002 self._base_warn_lines = warn_lines 1003 self._base_err_line_boards = err_line_boards 1004 self._base_warn_line_boards = warn_line_boards 1005 1006 # Get a list of boards that did not get built, if needed 1007 not_built = [] 1008 for board in board_selected: 1009 if not board in board_dict: 1010 not_built.append(board) 1011 if not_built: 1012 print "Boards not built (%d): %s" % (len(not_built), 1013 ', '.join(not_built)) 1014 1015 def ProduceResultSummary(self, commit_upto, commits, board_selected): 1016 (board_dict, err_lines, err_line_boards, warn_lines, 1017 warn_line_boards) = self.GetResultSummary( 1018 board_selected, commit_upto, 1019 read_func_sizes=self._show_bloat) 1020 if commits: 1021 msg = '%02d: %s' % (commit_upto + 1, 1022 commits[commit_upto].subject) 1023 print self.col.Color(self.col.BLUE, msg) 1024 self.PrintResultSummary(board_selected, board_dict, 1025 err_lines if self._show_errors else [], err_line_boards, 1026 warn_lines if self._show_errors else [], warn_line_boards, 1027 self._show_sizes, self._show_detail, self._show_bloat) 1028 1029 def ShowSummary(self, commits, board_selected): 1030 """Show a build summary for U-Boot for a given board list. 1031 1032 Reset the result summary, then repeatedly call GetResultSummary on 1033 each commit's results, then display the differences we see. 1034 1035 Args: 1036 commit: Commit objects to summarise 1037 board_selected: Dict containing boards to summarise 1038 """ 1039 self.commit_count = len(commits) if commits else 1 1040 self.commits = commits 1041 self.ResetResultSummary(board_selected) 1042 self._error_lines = 0 1043 1044 for commit_upto in range(0, self.commit_count, self._step): 1045 self.ProduceResultSummary(commit_upto, commits, board_selected) 1046 if not self._error_lines: 1047 print self.col.Color(self.col.GREEN, '(no errors to report)') 1048 1049 1050 def SetupBuild(self, board_selected, commits): 1051 """Set up ready to start a build. 1052 1053 Args: 1054 board_selected: Selected boards to build 1055 commits: Selected commits to build 1056 """ 1057 # First work out how many commits we will build 1058 count = (self.commit_count + self._step - 1) / self._step 1059 self.count = len(board_selected) * count 1060 self.upto = self.warned = self.fail = 0 1061 self._timestamps = collections.deque() 1062 1063 def GetThreadDir(self, thread_num): 1064 """Get the directory path to the working dir for a thread. 1065 1066 Args: 1067 thread_num: Number of thread to check. 1068 """ 1069 return os.path.join(self._working_dir, '%02d' % thread_num) 1070 1071 def _PrepareThread(self, thread_num, setup_git): 1072 """Prepare the working directory for a thread. 1073 1074 This clones or fetches the repo into the thread's work directory. 1075 1076 Args: 1077 thread_num: Thread number (0, 1, ...) 1078 setup_git: True to set up a git repo clone 1079 """ 1080 thread_dir = self.GetThreadDir(thread_num) 1081 builderthread.Mkdir(thread_dir) 1082 git_dir = os.path.join(thread_dir, '.git') 1083 1084 # Clone the repo if it doesn't already exist 1085 # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so 1086 # we have a private index but uses the origin repo's contents? 1087 if setup_git and self.git_dir: 1088 src_dir = os.path.abspath(self.git_dir) 1089 if os.path.exists(git_dir): 1090 gitutil.Fetch(git_dir, thread_dir) 1091 else: 1092 print 'Cloning repo for thread %d' % thread_num 1093 gitutil.Clone(src_dir, thread_dir) 1094 1095 def _PrepareWorkingSpace(self, max_threads, setup_git): 1096 """Prepare the working directory for use. 1097 1098 Set up the git repo for each thread. 1099 1100 Args: 1101 max_threads: Maximum number of threads we expect to need. 1102 setup_git: True to set up a git repo clone 1103 """ 1104 builderthread.Mkdir(self._working_dir) 1105 for thread in range(max_threads): 1106 self._PrepareThread(thread, setup_git) 1107 1108 def _PrepareOutputSpace(self): 1109 """Get the output directories ready to receive files. 1110 1111 We delete any output directories which look like ones we need to 1112 create. Having left over directories is confusing when the user wants 1113 to check the output manually. 1114 """ 1115 dir_list = [] 1116 for commit_upto in range(self.commit_count): 1117 dir_list.append(self._GetOutputDir(commit_upto)) 1118 1119 for dirname in glob.glob(os.path.join(self.base_dir, '*')): 1120 if dirname not in dir_list: 1121 shutil.rmtree(dirname) 1122 1123 def BuildBoards(self, commits, board_selected, keep_outputs, verbose): 1124 """Build all commits for a list of boards 1125 1126 Args: 1127 commits: List of commits to be build, each a Commit object 1128 boards_selected: Dict of selected boards, key is target name, 1129 value is Board object 1130 keep_outputs: True to save build output files 1131 verbose: Display build results as they are completed 1132 Returns: 1133 Tuple containing: 1134 - number of boards that failed to build 1135 - number of boards that issued warnings 1136 """ 1137 self.commit_count = len(commits) if commits else 1 1138 self.commits = commits 1139 self._verbose = verbose 1140 1141 self.ResetResultSummary(board_selected) 1142 builderthread.Mkdir(self.base_dir) 1143 self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)), 1144 commits is not None) 1145 self._PrepareOutputSpace() 1146 self.SetupBuild(board_selected, commits) 1147 self.ProcessResult(None) 1148 1149 # Create jobs to build all commits for each board 1150 for brd in board_selected.itervalues(): 1151 job = builderthread.BuilderJob() 1152 job.board = brd 1153 job.commits = commits 1154 job.keep_outputs = keep_outputs 1155 job.step = self._step 1156 self.queue.put(job) 1157 1158 # Wait until all jobs are started 1159 self.queue.join() 1160 1161 # Wait until we have processed all output 1162 self.out_queue.join() 1163 print 1164 self.ClearLine(0) 1165 return (self.fail, self.warned) 1166