1# Copyright (c) 2013 The Chromium OS Authors. 2# 3# Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com> 4# 5# SPDX-License-Identifier: GPL-2.0+ 6# 7 8import collections 9from datetime import datetime, timedelta 10import glob 11import os 12import re 13import Queue 14import shutil 15import string 16import sys 17import time 18 19import builderthread 20import command 21import gitutil 22import terminal 23from terminal import Print 24import toolchain 25 26 27""" 28Theory of Operation 29 30Please see README for user documentation, and you should be familiar with 31that before trying to make sense of this. 32 33Buildman works by keeping the machine as busy as possible, building different 34commits for different boards on multiple CPUs at once. 35 36The source repo (self.git_dir) contains all the commits to be built. Each 37thread works on a single board at a time. It checks out the first commit, 38configures it for that board, then builds it. Then it checks out the next 39commit and builds it (typically without re-configuring). When it runs out 40of commits, it gets another job from the builder and starts again with that 41board. 42 43Clearly the builder threads could work either way - they could check out a 44commit and then built it for all boards. Using separate directories for each 45commit/board pair they could leave their build product around afterwards 46also. 47 48The intent behind building a single board for multiple commits, is to make 49use of incremental builds. Since each commit is built incrementally from 50the previous one, builds are faster. Reconfiguring for a different board 51removes all intermediate object files. 52 53Many threads can be working at once, but each has its own working directory. 54When a thread finishes a build, it puts the output files into a result 55directory. 56 57The base directory used by buildman is normally '../<branch>', i.e. 58a directory higher than the source repository and named after the branch 59being built. 60 61Within the base directory, we have one subdirectory for each commit. Within 62that is one subdirectory for each board. Within that is the build output for 63that commit/board combination. 64 65Buildman also create working directories for each thread, in a .bm-work/ 66subdirectory in the base dir. 67 68As an example, say we are building branch 'us-net' for boards 'sandbox' and 69'seaboard', and say that us-net has two commits. We will have directories 70like this: 71 72us-net/ base directory 73 01_of_02_g4ed4ebc_net--Add-tftp-speed-/ 74 sandbox/ 75 u-boot.bin 76 seaboard/ 77 u-boot.bin 78 02_of_02_g4ed4ebc_net--Check-tftp-comp/ 79 sandbox/ 80 u-boot.bin 81 seaboard/ 82 u-boot.bin 83 .bm-work/ 84 00/ working directory for thread 0 (contains source checkout) 85 build/ build output 86 01/ working directory for thread 1 87 build/ build output 88 ... 89u-boot/ source directory 90 .git/ repository 91""" 92 93# Possible build outcomes 94OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4) 95 96# Translate a commit subject into a valid filename 97trans_valid_chars = string.maketrans("/: ", "---") 98 99 100class Builder: 101 """Class for building U-Boot for a particular commit. 102 103 Public members: (many should ->private) 104 active: True if the builder is active and has not been stopped 105 already_done: Number of builds already completed 106 base_dir: Base directory to use for builder 107 checkout: True to check out source, False to skip that step. 108 This is used for testing. 109 col: terminal.Color() object 110 count: Number of commits to build 111 do_make: Method to call to invoke Make 112 fail: Number of builds that failed due to error 113 force_build: Force building even if a build already exists 114 force_config_on_failure: If a commit fails for a board, disable 115 incremental building for the next commit we build for that 116 board, so that we will see all warnings/errors again. 117 force_build_failures: If a previously-built build (i.e. built on 118 a previous run of buildman) is marked as failed, rebuild it. 119 git_dir: Git directory containing source repository 120 last_line_len: Length of the last line we printed (used for erasing 121 it with new progress information) 122 num_jobs: Number of jobs to run at once (passed to make as -j) 123 num_threads: Number of builder threads to run 124 out_queue: Queue of results to process 125 re_make_err: Compiled regular expression for ignore_lines 126 queue: Queue of jobs to run 127 threads: List of active threads 128 toolchains: Toolchains object to use for building 129 upto: Current commit number we are building (0.count-1) 130 warned: Number of builds that produced at least one warning 131 force_reconfig: Reconfigure U-Boot on each comiit. This disables 132 incremental building, where buildman reconfigures on the first 133 commit for a baord, and then just does an incremental build for 134 the following commits. In fact buildman will reconfigure and 135 retry for any failing commits, so generally the only effect of 136 this option is to slow things down. 137 in_tree: Build U-Boot in-tree instead of specifying an output 138 directory separate from the source code. This option is really 139 only useful for testing in-tree builds. 140 141 Private members: 142 _base_board_dict: Last-summarised Dict of boards 143 _base_err_lines: Last-summarised list of errors 144 _base_warn_lines: Last-summarised list of warnings 145 _build_period_us: Time taken for a single build (float object). 146 _complete_delay: Expected delay until completion (timedelta) 147 _next_delay_update: Next time we plan to display a progress update 148 (datatime) 149 _show_unknown: Show unknown boards (those not built) in summary 150 _timestamps: List of timestamps for the completion of the last 151 last _timestamp_count builds. Each is a datetime object. 152 _timestamp_count: Number of timestamps to keep in our list. 153 _working_dir: Base working directory containing all threads 154 """ 155 class Outcome: 156 """Records a build outcome for a single make invocation 157 158 Public Members: 159 rc: Outcome value (OUTCOME_...) 160 err_lines: List of error lines or [] if none 161 sizes: Dictionary of image size information, keyed by filename 162 - Each value is itself a dictionary containing 163 values for 'text', 'data' and 'bss', being the integer 164 size in bytes of each section. 165 func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each 166 value is itself a dictionary: 167 key: function name 168 value: Size of function in bytes 169 """ 170 def __init__(self, rc, err_lines, sizes, func_sizes): 171 self.rc = rc 172 self.err_lines = err_lines 173 self.sizes = sizes 174 self.func_sizes = func_sizes 175 176 def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, 177 gnu_make='make', checkout=True, show_unknown=True, step=1, 178 no_subdirs=False, full_path=False, verbose_build=False): 179 """Create a new Builder object 180 181 Args: 182 toolchains: Toolchains object to use for building 183 base_dir: Base directory to use for builder 184 git_dir: Git directory containing source repository 185 num_threads: Number of builder threads to run 186 num_jobs: Number of jobs to run at once (passed to make as -j) 187 gnu_make: the command name of GNU Make. 188 checkout: True to check out source, False to skip that step. 189 This is used for testing. 190 show_unknown: Show unknown boards (those not built) in summary 191 step: 1 to process every commit, n to process every nth commit 192 no_subdirs: Don't create subdirectories when building current 193 source for a single board 194 full_path: Return the full path in CROSS_COMPILE and don't set 195 PATH 196 verbose_build: Run build with V=1 and don't use 'make -s' 197 """ 198 self.toolchains = toolchains 199 self.base_dir = base_dir 200 self._working_dir = os.path.join(base_dir, '.bm-work') 201 self.threads = [] 202 self.active = True 203 self.do_make = self.Make 204 self.gnu_make = gnu_make 205 self.checkout = checkout 206 self.num_threads = num_threads 207 self.num_jobs = num_jobs 208 self.already_done = 0 209 self.force_build = False 210 self.git_dir = git_dir 211 self._show_unknown = show_unknown 212 self._timestamp_count = 10 213 self._build_period_us = None 214 self._complete_delay = None 215 self._next_delay_update = datetime.now() 216 self.force_config_on_failure = True 217 self.force_build_failures = False 218 self.force_reconfig = False 219 self._step = step 220 self.in_tree = False 221 self._error_lines = 0 222 self.no_subdirs = no_subdirs 223 self.full_path = full_path 224 self.verbose_build = verbose_build 225 226 self.col = terminal.Color() 227 228 self._re_function = re.compile('(.*): In function.*') 229 self._re_files = re.compile('In file included from.*') 230 self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*') 231 self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*') 232 233 self.queue = Queue.Queue() 234 self.out_queue = Queue.Queue() 235 for i in range(self.num_threads): 236 t = builderthread.BuilderThread(self, i) 237 t.setDaemon(True) 238 t.start() 239 self.threads.append(t) 240 241 self.last_line_len = 0 242 t = builderthread.ResultThread(self) 243 t.setDaemon(True) 244 t.start() 245 self.threads.append(t) 246 247 ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)'] 248 self.re_make_err = re.compile('|'.join(ignore_lines)) 249 250 def __del__(self): 251 """Get rid of all threads created by the builder""" 252 for t in self.threads: 253 del t 254 255 def SetDisplayOptions(self, show_errors=False, show_sizes=False, 256 show_detail=False, show_bloat=False, 257 list_error_boards=False): 258 """Setup display options for the builder. 259 260 show_errors: True to show summarised error/warning info 261 show_sizes: Show size deltas 262 show_detail: Show detail for each board 263 show_bloat: Show detail for each function 264 list_error_boards: Show the boards which caused each error/warning 265 """ 266 self._show_errors = show_errors 267 self._show_sizes = show_sizes 268 self._show_detail = show_detail 269 self._show_bloat = show_bloat 270 self._list_error_boards = list_error_boards 271 272 def _AddTimestamp(self): 273 """Add a new timestamp to the list and record the build period. 274 275 The build period is the length of time taken to perform a single 276 build (one board, one commit). 277 """ 278 now = datetime.now() 279 self._timestamps.append(now) 280 count = len(self._timestamps) 281 delta = self._timestamps[-1] - self._timestamps[0] 282 seconds = delta.total_seconds() 283 284 # If we have enough data, estimate build period (time taken for a 285 # single build) and therefore completion time. 286 if count > 1 and self._next_delay_update < now: 287 self._next_delay_update = now + timedelta(seconds=2) 288 if seconds > 0: 289 self._build_period = float(seconds) / count 290 todo = self.count - self.upto 291 self._complete_delay = timedelta(microseconds= 292 self._build_period * todo * 1000000) 293 # Round it 294 self._complete_delay -= timedelta( 295 microseconds=self._complete_delay.microseconds) 296 297 if seconds > 60: 298 self._timestamps.popleft() 299 count -= 1 300 301 def ClearLine(self, length): 302 """Clear any characters on the current line 303 304 Make way for a new line of length 'length', by outputting enough 305 spaces to clear out the old line. Then remember the new length for 306 next time. 307 308 Args: 309 length: Length of new line, in characters 310 """ 311 if length < self.last_line_len: 312 Print(' ' * (self.last_line_len - length), newline=False) 313 Print('\r', newline=False) 314 self.last_line_len = length 315 sys.stdout.flush() 316 317 def SelectCommit(self, commit, checkout=True): 318 """Checkout the selected commit for this build 319 """ 320 self.commit = commit 321 if checkout and self.checkout: 322 gitutil.Checkout(commit.hash) 323 324 def Make(self, commit, brd, stage, cwd, *args, **kwargs): 325 """Run make 326 327 Args: 328 commit: Commit object that is being built 329 brd: Board object that is being built 330 stage: Stage that we are at (mrproper, config, build) 331 cwd: Directory where make should be run 332 args: Arguments to pass to make 333 kwargs: Arguments to pass to command.RunPipe() 334 """ 335 cmd = [self.gnu_make] + list(args) 336 result = command.RunPipe([cmd], capture=True, capture_stderr=True, 337 cwd=cwd, raise_on_error=False, **kwargs) 338 if self.verbose_build: 339 result.stdout = '%s\n' % (' '.join(cmd)) + result.stdout 340 result.combined = '%s\n' % (' '.join(cmd)) + result.combined 341 return result 342 343 def ProcessResult(self, result): 344 """Process the result of a build, showing progress information 345 346 Args: 347 result: A CommandResult object, which indicates the result for 348 a single build 349 """ 350 col = terminal.Color() 351 if result: 352 target = result.brd.target 353 354 if result.return_code < 0: 355 self.active = False 356 command.StopAll() 357 return 358 359 self.upto += 1 360 if result.return_code != 0: 361 self.fail += 1 362 elif result.stderr: 363 self.warned += 1 364 if result.already_done: 365 self.already_done += 1 366 if self._verbose: 367 Print('\r', newline=False) 368 self.ClearLine(0) 369 boards_selected = {target : result.brd} 370 self.ResetResultSummary(boards_selected) 371 self.ProduceResultSummary(result.commit_upto, self.commits, 372 boards_selected) 373 else: 374 target = '(starting)' 375 376 # Display separate counts for ok, warned and fail 377 ok = self.upto - self.warned - self.fail 378 line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok) 379 line += self.col.Color(self.col.YELLOW, '%5d' % self.warned) 380 line += self.col.Color(self.col.RED, '%5d' % self.fail) 381 382 name = ' /%-5d ' % self.count 383 384 # Add our current completion time estimate 385 self._AddTimestamp() 386 if self._complete_delay: 387 name += '%s : ' % self._complete_delay 388 # When building all boards for a commit, we can print a commit 389 # progress message. 390 if result and result.commit_upto is None: 391 name += 'commit %2d/%-3d' % (self.commit_upto + 1, 392 self.commit_count) 393 394 name += target 395 Print(line + name, newline=False) 396 length = 14 + len(name) 397 self.ClearLine(length) 398 399 def _GetOutputDir(self, commit_upto): 400 """Get the name of the output directory for a commit number 401 402 The output directory is typically .../<branch>/<commit>. 403 404 Args: 405 commit_upto: Commit number to use (0..self.count-1) 406 """ 407 commit_dir = None 408 if self.commits: 409 commit = self.commits[commit_upto] 410 subject = commit.subject.translate(trans_valid_chars) 411 commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1, 412 self.commit_count, commit.hash, subject[:20])) 413 elif not self.no_subdirs: 414 commit_dir = 'current' 415 if not commit_dir: 416 return self.base_dir 417 return os.path.join(self.base_dir, commit_dir) 418 419 def GetBuildDir(self, commit_upto, target): 420 """Get the name of the build directory for a commit number 421 422 The build directory is typically .../<branch>/<commit>/<target>. 423 424 Args: 425 commit_upto: Commit number to use (0..self.count-1) 426 target: Target name 427 """ 428 output_dir = self._GetOutputDir(commit_upto) 429 return os.path.join(output_dir, target) 430 431 def GetDoneFile(self, commit_upto, target): 432 """Get the name of the done file for a commit number 433 434 Args: 435 commit_upto: Commit number to use (0..self.count-1) 436 target: Target name 437 """ 438 return os.path.join(self.GetBuildDir(commit_upto, target), 'done') 439 440 def GetSizesFile(self, commit_upto, target): 441 """Get the name of the sizes file for a commit number 442 443 Args: 444 commit_upto: Commit number to use (0..self.count-1) 445 target: Target name 446 """ 447 return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes') 448 449 def GetFuncSizesFile(self, commit_upto, target, elf_fname): 450 """Get the name of the funcsizes file for a commit number and ELF file 451 452 Args: 453 commit_upto: Commit number to use (0..self.count-1) 454 target: Target name 455 elf_fname: Filename of elf image 456 """ 457 return os.path.join(self.GetBuildDir(commit_upto, target), 458 '%s.sizes' % elf_fname.replace('/', '-')) 459 460 def GetObjdumpFile(self, commit_upto, target, elf_fname): 461 """Get the name of the objdump file for a commit number and ELF file 462 463 Args: 464 commit_upto: Commit number to use (0..self.count-1) 465 target: Target name 466 elf_fname: Filename of elf image 467 """ 468 return os.path.join(self.GetBuildDir(commit_upto, target), 469 '%s.objdump' % elf_fname.replace('/', '-')) 470 471 def GetErrFile(self, commit_upto, target): 472 """Get the name of the err file for a commit number 473 474 Args: 475 commit_upto: Commit number to use (0..self.count-1) 476 target: Target name 477 """ 478 output_dir = self.GetBuildDir(commit_upto, target) 479 return os.path.join(output_dir, 'err') 480 481 def FilterErrors(self, lines): 482 """Filter out errors in which we have no interest 483 484 We should probably use map(). 485 486 Args: 487 lines: List of error lines, each a string 488 Returns: 489 New list with only interesting lines included 490 """ 491 out_lines = [] 492 for line in lines: 493 if not self.re_make_err.search(line): 494 out_lines.append(line) 495 return out_lines 496 497 def ReadFuncSizes(self, fname, fd): 498 """Read function sizes from the output of 'nm' 499 500 Args: 501 fd: File containing data to read 502 fname: Filename we are reading from (just for errors) 503 504 Returns: 505 Dictionary containing size of each function in bytes, indexed by 506 function name. 507 """ 508 sym = {} 509 for line in fd.readlines(): 510 try: 511 size, type, name = line[:-1].split() 512 except: 513 Print("Invalid line in file '%s': '%s'" % (fname, line[:-1])) 514 continue 515 if type in 'tTdDbB': 516 # function names begin with '.' on 64-bit powerpc 517 if '.' in name[1:]: 518 name = 'static.' + name.split('.')[0] 519 sym[name] = sym.get(name, 0) + int(size, 16) 520 return sym 521 522 def GetBuildOutcome(self, commit_upto, target, read_func_sizes): 523 """Work out the outcome of a build. 524 525 Args: 526 commit_upto: Commit number to check (0..n-1) 527 target: Target board to check 528 read_func_sizes: True to read function size information 529 530 Returns: 531 Outcome object 532 """ 533 done_file = self.GetDoneFile(commit_upto, target) 534 sizes_file = self.GetSizesFile(commit_upto, target) 535 sizes = {} 536 func_sizes = {} 537 if os.path.exists(done_file): 538 with open(done_file, 'r') as fd: 539 return_code = int(fd.readline()) 540 err_lines = [] 541 err_file = self.GetErrFile(commit_upto, target) 542 if os.path.exists(err_file): 543 with open(err_file, 'r') as fd: 544 err_lines = self.FilterErrors(fd.readlines()) 545 546 # Decide whether the build was ok, failed or created warnings 547 if return_code: 548 rc = OUTCOME_ERROR 549 elif len(err_lines): 550 rc = OUTCOME_WARNING 551 else: 552 rc = OUTCOME_OK 553 554 # Convert size information to our simple format 555 if os.path.exists(sizes_file): 556 with open(sizes_file, 'r') as fd: 557 for line in fd.readlines(): 558 values = line.split() 559 rodata = 0 560 if len(values) > 6: 561 rodata = int(values[6], 16) 562 size_dict = { 563 'all' : int(values[0]) + int(values[1]) + 564 int(values[2]), 565 'text' : int(values[0]) - rodata, 566 'data' : int(values[1]), 567 'bss' : int(values[2]), 568 'rodata' : rodata, 569 } 570 sizes[values[5]] = size_dict 571 572 if read_func_sizes: 573 pattern = self.GetFuncSizesFile(commit_upto, target, '*') 574 for fname in glob.glob(pattern): 575 with open(fname, 'r') as fd: 576 dict_name = os.path.basename(fname).replace('.sizes', 577 '') 578 func_sizes[dict_name] = self.ReadFuncSizes(fname, fd) 579 580 return Builder.Outcome(rc, err_lines, sizes, func_sizes) 581 582 return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}) 583 584 def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes): 585 """Calculate a summary of the results of building a commit. 586 587 Args: 588 board_selected: Dict containing boards to summarise 589 commit_upto: Commit number to summarize (0..self.count-1) 590 read_func_sizes: True to read function size information 591 592 Returns: 593 Tuple: 594 Dict containing boards which passed building this commit. 595 keyed by board.target 596 List containing a summary of error lines 597 Dict keyed by error line, containing a list of the Board 598 objects with that error 599 List containing a summary of warning lines 600 Dict keyed by error line, containing a list of the Board 601 objects with that warning 602 """ 603 def AddLine(lines_summary, lines_boards, line, board): 604 line = line.rstrip() 605 if line in lines_boards: 606 lines_boards[line].append(board) 607 else: 608 lines_boards[line] = [board] 609 lines_summary.append(line) 610 611 board_dict = {} 612 err_lines_summary = [] 613 err_lines_boards = {} 614 warn_lines_summary = [] 615 warn_lines_boards = {} 616 617 for board in boards_selected.itervalues(): 618 outcome = self.GetBuildOutcome(commit_upto, board.target, 619 read_func_sizes) 620 board_dict[board.target] = outcome 621 last_func = None 622 last_was_warning = False 623 for line in outcome.err_lines: 624 if line: 625 if (self._re_function.match(line) or 626 self._re_files.match(line)): 627 last_func = line 628 else: 629 is_warning = self._re_warning.match(line) 630 is_note = self._re_note.match(line) 631 if is_warning or (last_was_warning and is_note): 632 if last_func: 633 AddLine(warn_lines_summary, warn_lines_boards, 634 last_func, board) 635 AddLine(warn_lines_summary, warn_lines_boards, 636 line, board) 637 else: 638 if last_func: 639 AddLine(err_lines_summary, err_lines_boards, 640 last_func, board) 641 AddLine(err_lines_summary, err_lines_boards, 642 line, board) 643 last_was_warning = is_warning 644 last_func = None 645 return (board_dict, err_lines_summary, err_lines_boards, 646 warn_lines_summary, warn_lines_boards) 647 648 def AddOutcome(self, board_dict, arch_list, changes, char, color): 649 """Add an output to our list of outcomes for each architecture 650 651 This simple function adds failing boards (changes) to the 652 relevant architecture string, so we can print the results out 653 sorted by architecture. 654 655 Args: 656 board_dict: Dict containing all boards 657 arch_list: Dict keyed by arch name. Value is a string containing 658 a list of board names which failed for that arch. 659 changes: List of boards to add to arch_list 660 color: terminal.Colour object 661 """ 662 done_arch = {} 663 for target in changes: 664 if target in board_dict: 665 arch = board_dict[target].arch 666 else: 667 arch = 'unknown' 668 str = self.col.Color(color, ' ' + target) 669 if not arch in done_arch: 670 str = ' %s %s' % (self.col.Color(color, char), str) 671 done_arch[arch] = True 672 if not arch in arch_list: 673 arch_list[arch] = str 674 else: 675 arch_list[arch] += str 676 677 678 def ColourNum(self, num): 679 color = self.col.RED if num > 0 else self.col.GREEN 680 if num == 0: 681 return '0' 682 return self.col.Color(color, str(num)) 683 684 def ResetResultSummary(self, board_selected): 685 """Reset the results summary ready for use. 686 687 Set up the base board list to be all those selected, and set the 688 error lines to empty. 689 690 Following this, calls to PrintResultSummary() will use this 691 information to work out what has changed. 692 693 Args: 694 board_selected: Dict containing boards to summarise, keyed by 695 board.target 696 """ 697 self._base_board_dict = {} 698 for board in board_selected: 699 self._base_board_dict[board] = Builder.Outcome(0, [], [], {}) 700 self._base_err_lines = [] 701 self._base_warn_lines = [] 702 self._base_err_line_boards = {} 703 self._base_warn_line_boards = {} 704 705 def PrintFuncSizeDetail(self, fname, old, new): 706 grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0 707 delta, common = [], {} 708 709 for a in old: 710 if a in new: 711 common[a] = 1 712 713 for name in old: 714 if name not in common: 715 remove += 1 716 down += old[name] 717 delta.append([-old[name], name]) 718 719 for name in new: 720 if name not in common: 721 add += 1 722 up += new[name] 723 delta.append([new[name], name]) 724 725 for name in common: 726 diff = new.get(name, 0) - old.get(name, 0) 727 if diff > 0: 728 grow, up = grow + 1, up + diff 729 elif diff < 0: 730 shrink, down = shrink + 1, down - diff 731 delta.append([diff, name]) 732 733 delta.sort() 734 delta.reverse() 735 736 args = [add, -remove, grow, -shrink, up, -down, up - down] 737 if max(args) == 0: 738 return 739 args = [self.ColourNum(x) for x in args] 740 indent = ' ' * 15 741 Print('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' % 742 tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args)) 743 Print('%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new', 744 'delta')) 745 for diff, name in delta: 746 if diff: 747 color = self.col.RED if diff > 0 else self.col.GREEN 748 msg = '%s %-38s %7s %7s %+7d' % (indent, name, 749 old.get(name, '-'), new.get(name,'-'), diff) 750 Print(msg, colour=color) 751 752 753 def PrintSizeDetail(self, target_list, show_bloat): 754 """Show details size information for each board 755 756 Args: 757 target_list: List of targets, each a dict containing: 758 'target': Target name 759 'total_diff': Total difference in bytes across all areas 760 <part_name>: Difference for that part 761 show_bloat: Show detail for each function 762 """ 763 targets_by_diff = sorted(target_list, reverse=True, 764 key=lambda x: x['_total_diff']) 765 for result in targets_by_diff: 766 printed_target = False 767 for name in sorted(result): 768 diff = result[name] 769 if name.startswith('_'): 770 continue 771 if diff != 0: 772 color = self.col.RED if diff > 0 else self.col.GREEN 773 msg = ' %s %+d' % (name, diff) 774 if not printed_target: 775 Print('%10s %-15s:' % ('', result['_target']), 776 newline=False) 777 printed_target = True 778 Print(msg, colour=color, newline=False) 779 if printed_target: 780 Print() 781 if show_bloat: 782 target = result['_target'] 783 outcome = result['_outcome'] 784 base_outcome = self._base_board_dict[target] 785 for fname in outcome.func_sizes: 786 self.PrintFuncSizeDetail(fname, 787 base_outcome.func_sizes[fname], 788 outcome.func_sizes[fname]) 789 790 791 def PrintSizeSummary(self, board_selected, board_dict, show_detail, 792 show_bloat): 793 """Print a summary of image sizes broken down by section. 794 795 The summary takes the form of one line per architecture. The 796 line contains deltas for each of the sections (+ means the section 797 got bigger, - means smaller). The nunmbers are the average number 798 of bytes that a board in this section increased by. 799 800 For example: 801 powerpc: (622 boards) text -0.0 802 arm: (285 boards) text -0.0 803 nds32: (3 boards) text -8.0 804 805 Args: 806 board_selected: Dict containing boards to summarise, keyed by 807 board.target 808 board_dict: Dict containing boards for which we built this 809 commit, keyed by board.target. The value is an Outcome object. 810 show_detail: Show detail for each board 811 show_bloat: Show detail for each function 812 """ 813 arch_list = {} 814 arch_count = {} 815 816 # Calculate changes in size for different image parts 817 # The previous sizes are in Board.sizes, for each board 818 for target in board_dict: 819 if target not in board_selected: 820 continue 821 base_sizes = self._base_board_dict[target].sizes 822 outcome = board_dict[target] 823 sizes = outcome.sizes 824 825 # Loop through the list of images, creating a dict of size 826 # changes for each image/part. We end up with something like 827 # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4} 828 # which means that U-Boot data increased by 5 bytes and SPL 829 # text decreased by 4. 830 err = {'_target' : target} 831 for image in sizes: 832 if image in base_sizes: 833 base_image = base_sizes[image] 834 # Loop through the text, data, bss parts 835 for part in sorted(sizes[image]): 836 diff = sizes[image][part] - base_image[part] 837 col = None 838 if diff: 839 if image == 'u-boot': 840 name = part 841 else: 842 name = image + ':' + part 843 err[name] = diff 844 arch = board_selected[target].arch 845 if not arch in arch_count: 846 arch_count[arch] = 1 847 else: 848 arch_count[arch] += 1 849 if not sizes: 850 pass # Only add to our list when we have some stats 851 elif not arch in arch_list: 852 arch_list[arch] = [err] 853 else: 854 arch_list[arch].append(err) 855 856 # We now have a list of image size changes sorted by arch 857 # Print out a summary of these 858 for arch, target_list in arch_list.iteritems(): 859 # Get total difference for each type 860 totals = {} 861 for result in target_list: 862 total = 0 863 for name, diff in result.iteritems(): 864 if name.startswith('_'): 865 continue 866 total += diff 867 if name in totals: 868 totals[name] += diff 869 else: 870 totals[name] = diff 871 result['_total_diff'] = total 872 result['_outcome'] = board_dict[result['_target']] 873 874 count = len(target_list) 875 printed_arch = False 876 for name in sorted(totals): 877 diff = totals[name] 878 if diff: 879 # Display the average difference in this name for this 880 # architecture 881 avg_diff = float(diff) / count 882 color = self.col.RED if avg_diff > 0 else self.col.GREEN 883 msg = ' %s %+1.1f' % (name, avg_diff) 884 if not printed_arch: 885 Print('%10s: (for %d/%d boards)' % (arch, count, 886 arch_count[arch]), newline=False) 887 printed_arch = True 888 Print(msg, colour=color, newline=False) 889 890 if printed_arch: 891 Print() 892 if show_detail: 893 self.PrintSizeDetail(target_list, show_bloat) 894 895 896 def PrintResultSummary(self, board_selected, board_dict, err_lines, 897 err_line_boards, warn_lines, warn_line_boards, 898 show_sizes, show_detail, show_bloat): 899 """Compare results with the base results and display delta. 900 901 Only boards mentioned in board_selected will be considered. This 902 function is intended to be called repeatedly with the results of 903 each commit. It therefore shows a 'diff' between what it saw in 904 the last call and what it sees now. 905 906 Args: 907 board_selected: Dict containing boards to summarise, keyed by 908 board.target 909 board_dict: Dict containing boards for which we built this 910 commit, keyed by board.target. The value is an Outcome object. 911 err_lines: A list of errors for this commit, or [] if there is 912 none, or we don't want to print errors 913 err_line_boards: Dict keyed by error line, containing a list of 914 the Board objects with that error 915 warn_lines: A list of warnings for this commit, or [] if there is 916 none, or we don't want to print errors 917 warn_line_boards: Dict keyed by warning line, containing a list of 918 the Board objects with that warning 919 show_sizes: Show image size deltas 920 show_detail: Show detail for each board 921 show_bloat: Show detail for each function 922 """ 923 def _BoardList(line, line_boards): 924 """Helper function to get a line of boards containing a line 925 926 Args: 927 line: Error line to search for 928 Return: 929 String containing a list of boards with that error line, or 930 '' if the user has not requested such a list 931 """ 932 if self._list_error_boards: 933 names = [] 934 for board in line_boards[line]: 935 if not board.target in names: 936 names.append(board.target) 937 names_str = '(%s) ' % ','.join(names) 938 else: 939 names_str = '' 940 return names_str 941 942 def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards, 943 char): 944 better_lines = [] 945 worse_lines = [] 946 for line in lines: 947 if line not in base_lines: 948 worse_lines.append(char + '+' + 949 _BoardList(line, line_boards) + line) 950 for line in base_lines: 951 if line not in lines: 952 better_lines.append(char + '-' + 953 _BoardList(line, base_line_boards) + line) 954 return better_lines, worse_lines 955 956 better = [] # List of boards fixed since last commit 957 worse = [] # List of new broken boards since last commit 958 new = [] # List of boards that didn't exist last time 959 unknown = [] # List of boards that were not built 960 961 for target in board_dict: 962 if target not in board_selected: 963 continue 964 965 # If the board was built last time, add its outcome to a list 966 if target in self._base_board_dict: 967 base_outcome = self._base_board_dict[target].rc 968 outcome = board_dict[target] 969 if outcome.rc == OUTCOME_UNKNOWN: 970 unknown.append(target) 971 elif outcome.rc < base_outcome: 972 better.append(target) 973 elif outcome.rc > base_outcome: 974 worse.append(target) 975 else: 976 new.append(target) 977 978 # Get a list of errors that have appeared, and disappeared 979 better_err, worse_err = _CalcErrorDelta(self._base_err_lines, 980 self._base_err_line_boards, err_lines, err_line_boards, '') 981 better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines, 982 self._base_warn_line_boards, warn_lines, warn_line_boards, 'w') 983 984 # Display results by arch 985 if (better or worse or unknown or new or worse_err or better_err 986 or worse_warn or better_warn): 987 arch_list = {} 988 self.AddOutcome(board_selected, arch_list, better, '', 989 self.col.GREEN) 990 self.AddOutcome(board_selected, arch_list, worse, '+', 991 self.col.RED) 992 self.AddOutcome(board_selected, arch_list, new, '*', self.col.BLUE) 993 if self._show_unknown: 994 self.AddOutcome(board_selected, arch_list, unknown, '?', 995 self.col.MAGENTA) 996 for arch, target_list in arch_list.iteritems(): 997 Print('%10s: %s' % (arch, target_list)) 998 self._error_lines += 1 999 if better_err: 1000 Print('\n'.join(better_err), colour=self.col.GREEN) 1001 self._error_lines += 1 1002 if worse_err: 1003 Print('\n'.join(worse_err), colour=self.col.RED) 1004 self._error_lines += 1 1005 if better_warn: 1006 Print('\n'.join(better_warn), colour=self.col.CYAN) 1007 self._error_lines += 1 1008 if worse_warn: 1009 Print('\n'.join(worse_warn), colour=self.col.MAGENTA) 1010 self._error_lines += 1 1011 1012 if show_sizes: 1013 self.PrintSizeSummary(board_selected, board_dict, show_detail, 1014 show_bloat) 1015 1016 # Save our updated information for the next call to this function 1017 self._base_board_dict = board_dict 1018 self._base_err_lines = err_lines 1019 self._base_warn_lines = warn_lines 1020 self._base_err_line_boards = err_line_boards 1021 self._base_warn_line_boards = warn_line_boards 1022 1023 # Get a list of boards that did not get built, if needed 1024 not_built = [] 1025 for board in board_selected: 1026 if not board in board_dict: 1027 not_built.append(board) 1028 if not_built: 1029 Print("Boards not built (%d): %s" % (len(not_built), 1030 ', '.join(not_built))) 1031 1032 def ProduceResultSummary(self, commit_upto, commits, board_selected): 1033 (board_dict, err_lines, err_line_boards, warn_lines, 1034 warn_line_boards) = self.GetResultSummary( 1035 board_selected, commit_upto, 1036 read_func_sizes=self._show_bloat) 1037 if commits: 1038 msg = '%02d: %s' % (commit_upto + 1, 1039 commits[commit_upto].subject) 1040 Print(msg, colour=self.col.BLUE) 1041 self.PrintResultSummary(board_selected, board_dict, 1042 err_lines if self._show_errors else [], err_line_boards, 1043 warn_lines if self._show_errors else [], warn_line_boards, 1044 self._show_sizes, self._show_detail, self._show_bloat) 1045 1046 def ShowSummary(self, commits, board_selected): 1047 """Show a build summary for U-Boot for a given board list. 1048 1049 Reset the result summary, then repeatedly call GetResultSummary on 1050 each commit's results, then display the differences we see. 1051 1052 Args: 1053 commit: Commit objects to summarise 1054 board_selected: Dict containing boards to summarise 1055 """ 1056 self.commit_count = len(commits) if commits else 1 1057 self.commits = commits 1058 self.ResetResultSummary(board_selected) 1059 self._error_lines = 0 1060 1061 for commit_upto in range(0, self.commit_count, self._step): 1062 self.ProduceResultSummary(commit_upto, commits, board_selected) 1063 if not self._error_lines: 1064 Print('(no errors to report)', colour=self.col.GREEN) 1065 1066 1067 def SetupBuild(self, board_selected, commits): 1068 """Set up ready to start a build. 1069 1070 Args: 1071 board_selected: Selected boards to build 1072 commits: Selected commits to build 1073 """ 1074 # First work out how many commits we will build 1075 count = (self.commit_count + self._step - 1) / self._step 1076 self.count = len(board_selected) * count 1077 self.upto = self.warned = self.fail = 0 1078 self._timestamps = collections.deque() 1079 1080 def GetThreadDir(self, thread_num): 1081 """Get the directory path to the working dir for a thread. 1082 1083 Args: 1084 thread_num: Number of thread to check. 1085 """ 1086 return os.path.join(self._working_dir, '%02d' % thread_num) 1087 1088 def _PrepareThread(self, thread_num, setup_git): 1089 """Prepare the working directory for a thread. 1090 1091 This clones or fetches the repo into the thread's work directory. 1092 1093 Args: 1094 thread_num: Thread number (0, 1, ...) 1095 setup_git: True to set up a git repo clone 1096 """ 1097 thread_dir = self.GetThreadDir(thread_num) 1098 builderthread.Mkdir(thread_dir) 1099 git_dir = os.path.join(thread_dir, '.git') 1100 1101 # Clone the repo if it doesn't already exist 1102 # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so 1103 # we have a private index but uses the origin repo's contents? 1104 if setup_git and self.git_dir: 1105 src_dir = os.path.abspath(self.git_dir) 1106 if os.path.exists(git_dir): 1107 gitutil.Fetch(git_dir, thread_dir) 1108 else: 1109 Print('Cloning repo for thread %d' % thread_num) 1110 gitutil.Clone(src_dir, thread_dir) 1111 1112 def _PrepareWorkingSpace(self, max_threads, setup_git): 1113 """Prepare the working directory for use. 1114 1115 Set up the git repo for each thread. 1116 1117 Args: 1118 max_threads: Maximum number of threads we expect to need. 1119 setup_git: True to set up a git repo clone 1120 """ 1121 builderthread.Mkdir(self._working_dir) 1122 for thread in range(max_threads): 1123 self._PrepareThread(thread, setup_git) 1124 1125 def _PrepareOutputSpace(self): 1126 """Get the output directories ready to receive files. 1127 1128 We delete any output directories which look like ones we need to 1129 create. Having left over directories is confusing when the user wants 1130 to check the output manually. 1131 """ 1132 if not self.commits: 1133 return 1134 dir_list = [] 1135 for commit_upto in range(self.commit_count): 1136 dir_list.append(self._GetOutputDir(commit_upto)) 1137 1138 for dirname in glob.glob(os.path.join(self.base_dir, '*')): 1139 if dirname not in dir_list: 1140 shutil.rmtree(dirname) 1141 1142 def BuildBoards(self, commits, board_selected, keep_outputs, verbose): 1143 """Build all commits for a list of boards 1144 1145 Args: 1146 commits: List of commits to be build, each a Commit object 1147 boards_selected: Dict of selected boards, key is target name, 1148 value is Board object 1149 keep_outputs: True to save build output files 1150 verbose: Display build results as they are completed 1151 Returns: 1152 Tuple containing: 1153 - number of boards that failed to build 1154 - number of boards that issued warnings 1155 """ 1156 self.commit_count = len(commits) if commits else 1 1157 self.commits = commits 1158 self._verbose = verbose 1159 1160 self.ResetResultSummary(board_selected) 1161 builderthread.Mkdir(self.base_dir, parents = True) 1162 self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)), 1163 commits is not None) 1164 self._PrepareOutputSpace() 1165 self.SetupBuild(board_selected, commits) 1166 self.ProcessResult(None) 1167 1168 # Create jobs to build all commits for each board 1169 for brd in board_selected.itervalues(): 1170 job = builderthread.BuilderJob() 1171 job.board = brd 1172 job.commits = commits 1173 job.keep_outputs = keep_outputs 1174 job.step = self._step 1175 self.queue.put(job) 1176 1177 # Wait until all jobs are started 1178 self.queue.join() 1179 1180 # Wait until we have processed all output 1181 self.out_queue.join() 1182 Print() 1183 self.ClearLine(0) 1184 return (self.fail, self.warned) 1185