1# Copyright (c) 2013 The Chromium OS Authors. 2# 3# Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com> 4# 5# SPDX-License-Identifier: GPL-2.0+ 6# 7 8import collections 9import errno 10from datetime import datetime, timedelta 11import glob 12import os 13import re 14import Queue 15import shutil 16import string 17import sys 18import threading 19import time 20 21import command 22import gitutil 23import terminal 24import toolchain 25 26 27""" 28Theory of Operation 29 30Please see README for user documentation, and you should be familiar with 31that before trying to make sense of this. 32 33Buildman works by keeping the machine as busy as possible, building different 34commits for different boards on multiple CPUs at once. 35 36The source repo (self.git_dir) contains all the commits to be built. Each 37thread works on a single board at a time. It checks out the first commit, 38configures it for that board, then builds it. Then it checks out the next 39commit and builds it (typically without re-configuring). When it runs out 40of commits, it gets another job from the builder and starts again with that 41board. 42 43Clearly the builder threads could work either way - they could check out a 44commit and then built it for all boards. Using separate directories for each 45commit/board pair they could leave their build product around afterwards 46also. 47 48The intent behind building a single board for multiple commits, is to make 49use of incremental builds. Since each commit is built incrementally from 50the previous one, builds are faster. Reconfiguring for a different board 51removes all intermediate object files. 52 53Many threads can be working at once, but each has its own working directory. 54When a thread finishes a build, it puts the output files into a result 55directory. 56 57The base directory used by buildman is normally '../<branch>', i.e. 58a directory higher than the source repository and named after the branch 59being built. 60 61Within the base directory, we have one subdirectory for each commit. Within 62that is one subdirectory for each board. Within that is the build output for 63that commit/board combination. 64 65Buildman also create working directories for each thread, in a .bm-work/ 66subdirectory in the base dir. 67 68As an example, say we are building branch 'us-net' for boards 'sandbox' and 69'seaboard', and say that us-net has two commits. We will have directories 70like this: 71 72us-net/ base directory 73 01_of_02_g4ed4ebc_net--Add-tftp-speed-/ 74 sandbox/ 75 u-boot.bin 76 seaboard/ 77 u-boot.bin 78 02_of_02_g4ed4ebc_net--Check-tftp-comp/ 79 sandbox/ 80 u-boot.bin 81 seaboard/ 82 u-boot.bin 83 .bm-work/ 84 00/ working directory for thread 0 (contains source checkout) 85 build/ build output 86 01/ working directory for thread 1 87 build/ build output 88 ... 89u-boot/ source directory 90 .git/ repository 91""" 92 93# Possible build outcomes 94OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4) 95 96# Translate a commit subject into a valid filename 97trans_valid_chars = string.maketrans("/: ", "---") 98 99 100def Mkdir(dirname): 101 """Make a directory if it doesn't already exist. 102 103 Args: 104 dirname: Directory to create 105 """ 106 try: 107 os.mkdir(dirname) 108 except OSError as err: 109 if err.errno == errno.EEXIST: 110 pass 111 else: 112 raise 113 114class BuilderJob: 115 """Holds information about a job to be performed by a thread 116 117 Members: 118 board: Board object to build 119 commits: List of commit options to build. 120 """ 121 def __init__(self): 122 self.board = None 123 self.commits = [] 124 125 126class ResultThread(threading.Thread): 127 """This thread processes results from builder threads. 128 129 It simply passes the results on to the builder. There is only one 130 result thread, and this helps to serialise the build output. 131 """ 132 def __init__(self, builder): 133 """Set up a new result thread 134 135 Args: 136 builder: Builder which will be sent each result 137 """ 138 threading.Thread.__init__(self) 139 self.builder = builder 140 141 def run(self): 142 """Called to start up the result thread. 143 144 We collect the next result job and pass it on to the build. 145 """ 146 while True: 147 result = self.builder.out_queue.get() 148 self.builder.ProcessResult(result) 149 self.builder.out_queue.task_done() 150 151 152class BuilderThread(threading.Thread): 153 """This thread builds U-Boot for a particular board. 154 155 An input queue provides each new job. We run 'make' to build U-Boot 156 and then pass the results on to the output queue. 157 158 Members: 159 builder: The builder which contains information we might need 160 thread_num: Our thread number (0-n-1), used to decide on a 161 temporary directory 162 """ 163 def __init__(self, builder, thread_num): 164 """Set up a new builder thread""" 165 threading.Thread.__init__(self) 166 self.builder = builder 167 self.thread_num = thread_num 168 169 def Make(self, commit, brd, stage, cwd, *args, **kwargs): 170 """Run 'make' on a particular commit and board. 171 172 The source code will already be checked out, so the 'commit' 173 argument is only for information. 174 175 Args: 176 commit: Commit object that is being built 177 brd: Board object that is being built 178 stage: Stage of the build. Valid stages are: 179 distclean - can be called to clean source 180 config - called to configure for a board 181 build - the main make invocation - it does the build 182 args: A list of arguments to pass to 'make' 183 kwargs: A list of keyword arguments to pass to command.RunPipe() 184 185 Returns: 186 CommandResult object 187 """ 188 return self.builder.do_make(commit, brd, stage, cwd, *args, 189 **kwargs) 190 191 def RunCommit(self, commit_upto, brd, work_dir, do_config, force_build, 192 force_build_failures): 193 """Build a particular commit. 194 195 If the build is already done, and we are not forcing a build, we skip 196 the build and just return the previously-saved results. 197 198 Args: 199 commit_upto: Commit number to build (0...n-1) 200 brd: Board object to build 201 work_dir: Directory to which the source will be checked out 202 do_config: True to run a make <board>_config on the source 203 force_build: Force a build even if one was previously done 204 force_build_failures: Force a bulid if the previous result showed 205 failure 206 207 Returns: 208 tuple containing: 209 - CommandResult object containing the results of the build 210 - boolean indicating whether 'make config' is still needed 211 """ 212 # Create a default result - it will be overwritte by the call to 213 # self.Make() below, in the event that we do a build. 214 result = command.CommandResult() 215 result.return_code = 0 216 out_dir = os.path.join(work_dir, 'build') 217 218 # Check if the job was already completed last time 219 done_file = self.builder.GetDoneFile(commit_upto, brd.target) 220 result.already_done = os.path.exists(done_file) 221 will_build = (force_build or force_build_failures or 222 not result.already_done) 223 if result.already_done and will_build: 224 # Get the return code from that build and use it 225 with open(done_file, 'r') as fd: 226 result.return_code = int(fd.readline()) 227 err_file = self.builder.GetErrFile(commit_upto, brd.target) 228 if os.path.exists(err_file) and os.stat(err_file).st_size: 229 result.stderr = 'bad' 230 elif not force_build: 231 # The build passed, so no need to build it again 232 will_build = False 233 234 if will_build: 235 # We are going to have to build it. First, get a toolchain 236 if not self.toolchain: 237 try: 238 self.toolchain = self.builder.toolchains.Select(brd.arch) 239 except ValueError as err: 240 result.return_code = 10 241 result.stdout = '' 242 result.stderr = str(err) 243 # TODO(sjg@chromium.org): This gets swallowed, but needs 244 # to be reported. 245 246 if self.toolchain: 247 # Checkout the right commit 248 if commit_upto is not None: 249 commit = self.builder.commits[commit_upto] 250 if self.builder.checkout: 251 git_dir = os.path.join(work_dir, '.git') 252 gitutil.Checkout(commit.hash, git_dir, work_dir, 253 force=True) 254 else: 255 commit = self.builder.commit # Ick, fix this for BuildCommits() 256 257 # Set up the environment and command line 258 env = self.toolchain.MakeEnvironment() 259 Mkdir(out_dir) 260 args = ['O=build', '-s'] 261 if self.builder.num_jobs is not None: 262 args.extend(['-j', str(self.builder.num_jobs)]) 263 config_args = ['%s_config' % brd.target] 264 config_out = '' 265 args.extend(self.builder.toolchains.GetMakeArguments(brd)) 266 267 # If we need to reconfigure, do that now 268 if do_config: 269 result = self.Make(commit, brd, 'distclean', work_dir, 270 'distclean', *args, env=env) 271 result = self.Make(commit, brd, 'config', work_dir, 272 *(args + config_args), env=env) 273 config_out = result.combined 274 do_config = False # No need to configure next time 275 if result.return_code == 0: 276 result = self.Make(commit, brd, 'build', work_dir, *args, 277 env=env) 278 result.stdout = config_out + result.stdout 279 else: 280 result.return_code = 1 281 result.stderr = 'No tool chain for %s\n' % brd.arch 282 result.already_done = False 283 284 result.toolchain = self.toolchain 285 result.brd = brd 286 result.commit_upto = commit_upto 287 result.out_dir = out_dir 288 return result, do_config 289 290 def _WriteResult(self, result, keep_outputs): 291 """Write a built result to the output directory. 292 293 Args: 294 result: CommandResult object containing result to write 295 keep_outputs: True to store the output binaries, False 296 to delete them 297 """ 298 # Fatal error 299 if result.return_code < 0: 300 return 301 302 # Aborted? 303 if result.stderr and 'No child processes' in result.stderr: 304 return 305 306 if result.already_done: 307 return 308 309 # Write the output and stderr 310 output_dir = self.builder._GetOutputDir(result.commit_upto) 311 Mkdir(output_dir) 312 build_dir = self.builder.GetBuildDir(result.commit_upto, 313 result.brd.target) 314 Mkdir(build_dir) 315 316 outfile = os.path.join(build_dir, 'log') 317 with open(outfile, 'w') as fd: 318 if result.stdout: 319 fd.write(result.stdout) 320 321 errfile = self.builder.GetErrFile(result.commit_upto, 322 result.brd.target) 323 if result.stderr: 324 with open(errfile, 'w') as fd: 325 fd.write(result.stderr) 326 elif os.path.exists(errfile): 327 os.remove(errfile) 328 329 if result.toolchain: 330 # Write the build result and toolchain information. 331 done_file = self.builder.GetDoneFile(result.commit_upto, 332 result.brd.target) 333 with open(done_file, 'w') as fd: 334 fd.write('%s' % result.return_code) 335 with open(os.path.join(build_dir, 'toolchain'), 'w') as fd: 336 print >>fd, 'gcc', result.toolchain.gcc 337 print >>fd, 'path', result.toolchain.path 338 print >>fd, 'cross', result.toolchain.cross 339 print >>fd, 'arch', result.toolchain.arch 340 fd.write('%s' % result.return_code) 341 342 with open(os.path.join(build_dir, 'toolchain'), 'w') as fd: 343 print >>fd, 'gcc', result.toolchain.gcc 344 print >>fd, 'path', result.toolchain.path 345 346 # Write out the image and function size information and an objdump 347 env = result.toolchain.MakeEnvironment() 348 lines = [] 349 for fname in ['u-boot', 'spl/u-boot-spl']: 350 cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname] 351 nm_result = command.RunPipe([cmd], capture=True, 352 capture_stderr=True, cwd=result.out_dir, 353 raise_on_error=False, env=env) 354 if nm_result.stdout: 355 nm = self.builder.GetFuncSizesFile(result.commit_upto, 356 result.brd.target, fname) 357 with open(nm, 'w') as fd: 358 print >>fd, nm_result.stdout, 359 360 cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname] 361 dump_result = command.RunPipe([cmd], capture=True, 362 capture_stderr=True, cwd=result.out_dir, 363 raise_on_error=False, env=env) 364 rodata_size = '' 365 if dump_result.stdout: 366 objdump = self.builder.GetObjdumpFile(result.commit_upto, 367 result.brd.target, fname) 368 with open(objdump, 'w') as fd: 369 print >>fd, dump_result.stdout, 370 for line in dump_result.stdout.splitlines(): 371 fields = line.split() 372 if len(fields) > 5 and fields[1] == '.rodata': 373 rodata_size = fields[2] 374 375 cmd = ['%ssize' % self.toolchain.cross, fname] 376 size_result = command.RunPipe([cmd], capture=True, 377 capture_stderr=True, cwd=result.out_dir, 378 raise_on_error=False, env=env) 379 if size_result.stdout: 380 lines.append(size_result.stdout.splitlines()[1] + ' ' + 381 rodata_size) 382 383 # Write out the image sizes file. This is similar to the output 384 # of binutil's 'size' utility, but it omits the header line and 385 # adds an additional hex value at the end of each line for the 386 # rodata size 387 if len(lines): 388 sizes = self.builder.GetSizesFile(result.commit_upto, 389 result.brd.target) 390 with open(sizes, 'w') as fd: 391 print >>fd, '\n'.join(lines) 392 393 # Now write the actual build output 394 if keep_outputs: 395 patterns = ['u-boot', '*.bin', 'u-boot.dtb', '*.map', 396 'include/autoconf.mk', 'spl/u-boot-spl', 397 'spl/u-boot-spl.bin'] 398 for pattern in patterns: 399 file_list = glob.glob(os.path.join(result.out_dir, pattern)) 400 for fname in file_list: 401 shutil.copy(fname, build_dir) 402 403 404 def RunJob(self, job): 405 """Run a single job 406 407 A job consists of a building a list of commits for a particular board. 408 409 Args: 410 job: Job to build 411 """ 412 brd = job.board 413 work_dir = self.builder.GetThreadDir(self.thread_num) 414 self.toolchain = None 415 if job.commits: 416 # Run 'make board_config' on the first commit 417 do_config = True 418 commit_upto = 0 419 force_build = False 420 for commit_upto in range(0, len(job.commits), job.step): 421 result, request_config = self.RunCommit(commit_upto, brd, 422 work_dir, do_config, 423 force_build or self.builder.force_build, 424 self.builder.force_build_failures) 425 failed = result.return_code or result.stderr 426 did_config = do_config 427 if failed and not do_config: 428 # If our incremental build failed, try building again 429 # with a reconfig. 430 if self.builder.force_config_on_failure: 431 result, request_config = self.RunCommit(commit_upto, 432 brd, work_dir, True, True, False) 433 did_config = True 434 do_config = request_config 435 436 # If we built that commit, then config is done. But if we got 437 # an warning, reconfig next time to force it to build the same 438 # files that created warnings this time. Otherwise an 439 # incremental build may not build the same file, and we will 440 # think that the warning has gone away. 441 # We could avoid this by using -Werror everywhere... 442 # For errors, the problem doesn't happen, since presumably 443 # the build stopped and didn't generate output, so will retry 444 # that file next time. So we could detect warnings and deal 445 # with them specially here. For now, we just reconfigure if 446 # anything goes work. 447 # Of course this is substantially slower if there are build 448 # errors/warnings (e.g. 2-3x slower even if only 10% of builds 449 # have problems). 450 if (failed and not result.already_done and not did_config and 451 self.builder.force_config_on_failure): 452 # If this build failed, try the next one with a 453 # reconfigure. 454 # Sometimes if the board_config.h file changes it can mess 455 # with dependencies, and we get: 456 # make: *** No rule to make target `include/autoconf.mk', 457 # needed by `depend'. 458 do_config = True 459 force_build = True 460 else: 461 force_build = False 462 if self.builder.force_config_on_failure: 463 if failed: 464 do_config = True 465 result.commit_upto = commit_upto 466 if result.return_code < 0: 467 raise ValueError('Interrupt') 468 469 # We have the build results, so output the result 470 self._WriteResult(result, job.keep_outputs) 471 self.builder.out_queue.put(result) 472 else: 473 # Just build the currently checked-out build 474 result = self.RunCommit(None, True) 475 result.commit_upto = self.builder.upto 476 self.builder.out_queue.put(result) 477 478 def run(self): 479 """Our thread's run function 480 481 This thread picks a job from the queue, runs it, and then goes to the 482 next job. 483 """ 484 alive = True 485 while True: 486 job = self.builder.queue.get() 487 try: 488 if self.builder.active and alive: 489 self.RunJob(job) 490 except Exception as err: 491 alive = False 492 print err 493 self.builder.queue.task_done() 494 495 496class Builder: 497 """Class for building U-Boot for a particular commit. 498 499 Public members: (many should ->private) 500 active: True if the builder is active and has not been stopped 501 already_done: Number of builds already completed 502 base_dir: Base directory to use for builder 503 checkout: True to check out source, False to skip that step. 504 This is used for testing. 505 col: terminal.Color() object 506 count: Number of commits to build 507 do_make: Method to call to invoke Make 508 fail: Number of builds that failed due to error 509 force_build: Force building even if a build already exists 510 force_config_on_failure: If a commit fails for a board, disable 511 incremental building for the next commit we build for that 512 board, so that we will see all warnings/errors again. 513 force_build_failures: If a previously-built build (i.e. built on 514 a previous run of buildman) is marked as failed, rebuild it. 515 git_dir: Git directory containing source repository 516 last_line_len: Length of the last line we printed (used for erasing 517 it with new progress information) 518 num_jobs: Number of jobs to run at once (passed to make as -j) 519 num_threads: Number of builder threads to run 520 out_queue: Queue of results to process 521 re_make_err: Compiled regular expression for ignore_lines 522 queue: Queue of jobs to run 523 threads: List of active threads 524 toolchains: Toolchains object to use for building 525 upto: Current commit number we are building (0.count-1) 526 warned: Number of builds that produced at least one warning 527 528 Private members: 529 _base_board_dict: Last-summarised Dict of boards 530 _base_err_lines: Last-summarised list of errors 531 _build_period_us: Time taken for a single build (float object). 532 _complete_delay: Expected delay until completion (timedelta) 533 _next_delay_update: Next time we plan to display a progress update 534 (datatime) 535 _show_unknown: Show unknown boards (those not built) in summary 536 _timestamps: List of timestamps for the completion of the last 537 last _timestamp_count builds. Each is a datetime object. 538 _timestamp_count: Number of timestamps to keep in our list. 539 _working_dir: Base working directory containing all threads 540 """ 541 class Outcome: 542 """Records a build outcome for a single make invocation 543 544 Public Members: 545 rc: Outcome value (OUTCOME_...) 546 err_lines: List of error lines or [] if none 547 sizes: Dictionary of image size information, keyed by filename 548 - Each value is itself a dictionary containing 549 values for 'text', 'data' and 'bss', being the integer 550 size in bytes of each section. 551 func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each 552 value is itself a dictionary: 553 key: function name 554 value: Size of function in bytes 555 """ 556 def __init__(self, rc, err_lines, sizes, func_sizes): 557 self.rc = rc 558 self.err_lines = err_lines 559 self.sizes = sizes 560 self.func_sizes = func_sizes 561 562 def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, 563 checkout=True, show_unknown=True, step=1): 564 """Create a new Builder object 565 566 Args: 567 toolchains: Toolchains object to use for building 568 base_dir: Base directory to use for builder 569 git_dir: Git directory containing source repository 570 num_threads: Number of builder threads to run 571 num_jobs: Number of jobs to run at once (passed to make as -j) 572 checkout: True to check out source, False to skip that step. 573 This is used for testing. 574 show_unknown: Show unknown boards (those not built) in summary 575 step: 1 to process every commit, n to process every nth commit 576 """ 577 self.toolchains = toolchains 578 self.base_dir = base_dir 579 self._working_dir = os.path.join(base_dir, '.bm-work') 580 self.threads = [] 581 self.active = True 582 self.do_make = self.Make 583 self.checkout = checkout 584 self.num_threads = num_threads 585 self.num_jobs = num_jobs 586 self.already_done = 0 587 self.force_build = False 588 self.git_dir = git_dir 589 self._show_unknown = show_unknown 590 self._timestamp_count = 10 591 self._build_period_us = None 592 self._complete_delay = None 593 self._next_delay_update = datetime.now() 594 self.force_config_on_failure = True 595 self.force_build_failures = False 596 self._step = step 597 598 self.col = terminal.Color() 599 600 self.queue = Queue.Queue() 601 self.out_queue = Queue.Queue() 602 for i in range(self.num_threads): 603 t = BuilderThread(self, i) 604 t.setDaemon(True) 605 t.start() 606 self.threads.append(t) 607 608 self.last_line_len = 0 609 t = ResultThread(self) 610 t.setDaemon(True) 611 t.start() 612 self.threads.append(t) 613 614 ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)'] 615 self.re_make_err = re.compile('|'.join(ignore_lines)) 616 617 def __del__(self): 618 """Get rid of all threads created by the builder""" 619 for t in self.threads: 620 del t 621 622 def _AddTimestamp(self): 623 """Add a new timestamp to the list and record the build period. 624 625 The build period is the length of time taken to perform a single 626 build (one board, one commit). 627 """ 628 now = datetime.now() 629 self._timestamps.append(now) 630 count = len(self._timestamps) 631 delta = self._timestamps[-1] - self._timestamps[0] 632 seconds = delta.total_seconds() 633 634 # If we have enough data, estimate build period (time taken for a 635 # single build) and therefore completion time. 636 if count > 1 and self._next_delay_update < now: 637 self._next_delay_update = now + timedelta(seconds=2) 638 if seconds > 0: 639 self._build_period = float(seconds) / count 640 todo = self.count - self.upto 641 self._complete_delay = timedelta(microseconds= 642 self._build_period * todo * 1000000) 643 # Round it 644 self._complete_delay -= timedelta( 645 microseconds=self._complete_delay.microseconds) 646 647 if seconds > 60: 648 self._timestamps.popleft() 649 count -= 1 650 651 def ClearLine(self, length): 652 """Clear any characters on the current line 653 654 Make way for a new line of length 'length', by outputting enough 655 spaces to clear out the old line. Then remember the new length for 656 next time. 657 658 Args: 659 length: Length of new line, in characters 660 """ 661 if length < self.last_line_len: 662 print ' ' * (self.last_line_len - length), 663 print '\r', 664 self.last_line_len = length 665 sys.stdout.flush() 666 667 def SelectCommit(self, commit, checkout=True): 668 """Checkout the selected commit for this build 669 """ 670 self.commit = commit 671 if checkout and self.checkout: 672 gitutil.Checkout(commit.hash) 673 674 def Make(self, commit, brd, stage, cwd, *args, **kwargs): 675 """Run make 676 677 Args: 678 commit: Commit object that is being built 679 brd: Board object that is being built 680 stage: Stage that we are at (distclean, config, build) 681 cwd: Directory where make should be run 682 args: Arguments to pass to make 683 kwargs: Arguments to pass to command.RunPipe() 684 """ 685 cmd = ['make'] + list(args) 686 result = command.RunPipe([cmd], capture=True, capture_stderr=True, 687 cwd=cwd, raise_on_error=False, **kwargs) 688 return result 689 690 def ProcessResult(self, result): 691 """Process the result of a build, showing progress information 692 693 Args: 694 result: A CommandResult object 695 """ 696 col = terminal.Color() 697 if result: 698 target = result.brd.target 699 700 if result.return_code < 0: 701 self.active = False 702 command.StopAll() 703 return 704 705 self.upto += 1 706 if result.return_code != 0: 707 self.fail += 1 708 elif result.stderr: 709 self.warned += 1 710 if result.already_done: 711 self.already_done += 1 712 else: 713 target = '(starting)' 714 715 # Display separate counts for ok, warned and fail 716 ok = self.upto - self.warned - self.fail 717 line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok) 718 line += self.col.Color(self.col.YELLOW, '%5d' % self.warned) 719 line += self.col.Color(self.col.RED, '%5d' % self.fail) 720 721 name = ' /%-5d ' % self.count 722 723 # Add our current completion time estimate 724 self._AddTimestamp() 725 if self._complete_delay: 726 name += '%s : ' % self._complete_delay 727 # When building all boards for a commit, we can print a commit 728 # progress message. 729 if result and result.commit_upto is None: 730 name += 'commit %2d/%-3d' % (self.commit_upto + 1, 731 self.commit_count) 732 733 name += target 734 print line + name, 735 length = 13 + len(name) 736 self.ClearLine(length) 737 738 def _GetOutputDir(self, commit_upto): 739 """Get the name of the output directory for a commit number 740 741 The output directory is typically .../<branch>/<commit>. 742 743 Args: 744 commit_upto: Commit number to use (0..self.count-1) 745 """ 746 commit = self.commits[commit_upto] 747 subject = commit.subject.translate(trans_valid_chars) 748 commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1, 749 self.commit_count, commit.hash, subject[:20])) 750 output_dir = os.path.join(self.base_dir, commit_dir) 751 return output_dir 752 753 def GetBuildDir(self, commit_upto, target): 754 """Get the name of the build directory for a commit number 755 756 The build directory is typically .../<branch>/<commit>/<target>. 757 758 Args: 759 commit_upto: Commit number to use (0..self.count-1) 760 target: Target name 761 """ 762 output_dir = self._GetOutputDir(commit_upto) 763 return os.path.join(output_dir, target) 764 765 def GetDoneFile(self, commit_upto, target): 766 """Get the name of the done file for a commit number 767 768 Args: 769 commit_upto: Commit number to use (0..self.count-1) 770 target: Target name 771 """ 772 return os.path.join(self.GetBuildDir(commit_upto, target), 'done') 773 774 def GetSizesFile(self, commit_upto, target): 775 """Get the name of the sizes file for a commit number 776 777 Args: 778 commit_upto: Commit number to use (0..self.count-1) 779 target: Target name 780 """ 781 return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes') 782 783 def GetFuncSizesFile(self, commit_upto, target, elf_fname): 784 """Get the name of the funcsizes file for a commit number and ELF file 785 786 Args: 787 commit_upto: Commit number to use (0..self.count-1) 788 target: Target name 789 elf_fname: Filename of elf image 790 """ 791 return os.path.join(self.GetBuildDir(commit_upto, target), 792 '%s.sizes' % elf_fname.replace('/', '-')) 793 794 def GetObjdumpFile(self, commit_upto, target, elf_fname): 795 """Get the name of the objdump file for a commit number and ELF file 796 797 Args: 798 commit_upto: Commit number to use (0..self.count-1) 799 target: Target name 800 elf_fname: Filename of elf image 801 """ 802 return os.path.join(self.GetBuildDir(commit_upto, target), 803 '%s.objdump' % elf_fname.replace('/', '-')) 804 805 def GetErrFile(self, commit_upto, target): 806 """Get the name of the err file for a commit number 807 808 Args: 809 commit_upto: Commit number to use (0..self.count-1) 810 target: Target name 811 """ 812 output_dir = self.GetBuildDir(commit_upto, target) 813 return os.path.join(output_dir, 'err') 814 815 def FilterErrors(self, lines): 816 """Filter out errors in which we have no interest 817 818 We should probably use map(). 819 820 Args: 821 lines: List of error lines, each a string 822 Returns: 823 New list with only interesting lines included 824 """ 825 out_lines = [] 826 for line in lines: 827 if not self.re_make_err.search(line): 828 out_lines.append(line) 829 return out_lines 830 831 def ReadFuncSizes(self, fname, fd): 832 """Read function sizes from the output of 'nm' 833 834 Args: 835 fd: File containing data to read 836 fname: Filename we are reading from (just for errors) 837 838 Returns: 839 Dictionary containing size of each function in bytes, indexed by 840 function name. 841 """ 842 sym = {} 843 for line in fd.readlines(): 844 try: 845 size, type, name = line[:-1].split() 846 except: 847 print "Invalid line in file '%s': '%s'" % (fname, line[:-1]) 848 continue 849 if type in 'tTdDbB': 850 # function names begin with '.' on 64-bit powerpc 851 if '.' in name[1:]: 852 name = 'static.' + name.split('.')[0] 853 sym[name] = sym.get(name, 0) + int(size, 16) 854 return sym 855 856 def GetBuildOutcome(self, commit_upto, target, read_func_sizes): 857 """Work out the outcome of a build. 858 859 Args: 860 commit_upto: Commit number to check (0..n-1) 861 target: Target board to check 862 read_func_sizes: True to read function size information 863 864 Returns: 865 Outcome object 866 """ 867 done_file = self.GetDoneFile(commit_upto, target) 868 sizes_file = self.GetSizesFile(commit_upto, target) 869 sizes = {} 870 func_sizes = {} 871 if os.path.exists(done_file): 872 with open(done_file, 'r') as fd: 873 return_code = int(fd.readline()) 874 err_lines = [] 875 err_file = self.GetErrFile(commit_upto, target) 876 if os.path.exists(err_file): 877 with open(err_file, 'r') as fd: 878 err_lines = self.FilterErrors(fd.readlines()) 879 880 # Decide whether the build was ok, failed or created warnings 881 if return_code: 882 rc = OUTCOME_ERROR 883 elif len(err_lines): 884 rc = OUTCOME_WARNING 885 else: 886 rc = OUTCOME_OK 887 888 # Convert size information to our simple format 889 if os.path.exists(sizes_file): 890 with open(sizes_file, 'r') as fd: 891 for line in fd.readlines(): 892 values = line.split() 893 rodata = 0 894 if len(values) > 6: 895 rodata = int(values[6], 16) 896 size_dict = { 897 'all' : int(values[0]) + int(values[1]) + 898 int(values[2]), 899 'text' : int(values[0]) - rodata, 900 'data' : int(values[1]), 901 'bss' : int(values[2]), 902 'rodata' : rodata, 903 } 904 sizes[values[5]] = size_dict 905 906 if read_func_sizes: 907 pattern = self.GetFuncSizesFile(commit_upto, target, '*') 908 for fname in glob.glob(pattern): 909 with open(fname, 'r') as fd: 910 dict_name = os.path.basename(fname).replace('.sizes', 911 '') 912 func_sizes[dict_name] = self.ReadFuncSizes(fname, fd) 913 914 return Builder.Outcome(rc, err_lines, sizes, func_sizes) 915 916 return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}) 917 918 def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes): 919 """Calculate a summary of the results of building a commit. 920 921 Args: 922 board_selected: Dict containing boards to summarise 923 commit_upto: Commit number to summarize (0..self.count-1) 924 read_func_sizes: True to read function size information 925 926 Returns: 927 Tuple: 928 Dict containing boards which passed building this commit. 929 keyed by board.target 930 List containing a summary of error/warning lines 931 """ 932 board_dict = {} 933 err_lines_summary = [] 934 935 for board in boards_selected.itervalues(): 936 outcome = self.GetBuildOutcome(commit_upto, board.target, 937 read_func_sizes) 938 board_dict[board.target] = outcome 939 for err in outcome.err_lines: 940 if err and not err.rstrip() in err_lines_summary: 941 err_lines_summary.append(err.rstrip()) 942 return board_dict, err_lines_summary 943 944 def AddOutcome(self, board_dict, arch_list, changes, char, color): 945 """Add an output to our list of outcomes for each architecture 946 947 This simple function adds failing boards (changes) to the 948 relevant architecture string, so we can print the results out 949 sorted by architecture. 950 951 Args: 952 board_dict: Dict containing all boards 953 arch_list: Dict keyed by arch name. Value is a string containing 954 a list of board names which failed for that arch. 955 changes: List of boards to add to arch_list 956 color: terminal.Colour object 957 """ 958 done_arch = {} 959 for target in changes: 960 if target in board_dict: 961 arch = board_dict[target].arch 962 else: 963 arch = 'unknown' 964 str = self.col.Color(color, ' ' + target) 965 if not arch in done_arch: 966 str = self.col.Color(color, char) + ' ' + str 967 done_arch[arch] = True 968 if not arch in arch_list: 969 arch_list[arch] = str 970 else: 971 arch_list[arch] += str 972 973 974 def ColourNum(self, num): 975 color = self.col.RED if num > 0 else self.col.GREEN 976 if num == 0: 977 return '0' 978 return self.col.Color(color, str(num)) 979 980 def ResetResultSummary(self, board_selected): 981 """Reset the results summary ready for use. 982 983 Set up the base board list to be all those selected, and set the 984 error lines to empty. 985 986 Following this, calls to PrintResultSummary() will use this 987 information to work out what has changed. 988 989 Args: 990 board_selected: Dict containing boards to summarise, keyed by 991 board.target 992 """ 993 self._base_board_dict = {} 994 for board in board_selected: 995 self._base_board_dict[board] = Builder.Outcome(0, [], [], {}) 996 self._base_err_lines = [] 997 998 def PrintFuncSizeDetail(self, fname, old, new): 999 grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0 1000 delta, common = [], {} 1001 1002 for a in old: 1003 if a in new: 1004 common[a] = 1 1005 1006 for name in old: 1007 if name not in common: 1008 remove += 1 1009 down += old[name] 1010 delta.append([-old[name], name]) 1011 1012 for name in new: 1013 if name not in common: 1014 add += 1 1015 up += new[name] 1016 delta.append([new[name], name]) 1017 1018 for name in common: 1019 diff = new.get(name, 0) - old.get(name, 0) 1020 if diff > 0: 1021 grow, up = grow + 1, up + diff 1022 elif diff < 0: 1023 shrink, down = shrink + 1, down - diff 1024 delta.append([diff, name]) 1025 1026 delta.sort() 1027 delta.reverse() 1028 1029 args = [add, -remove, grow, -shrink, up, -down, up - down] 1030 if max(args) == 0: 1031 return 1032 args = [self.ColourNum(x) for x in args] 1033 indent = ' ' * 15 1034 print ('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' % 1035 tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args)) 1036 print '%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new', 1037 'delta') 1038 for diff, name in delta: 1039 if diff: 1040 color = self.col.RED if diff > 0 else self.col.GREEN 1041 msg = '%s %-38s %7s %7s %+7d' % (indent, name, 1042 old.get(name, '-'), new.get(name,'-'), diff) 1043 print self.col.Color(color, msg) 1044 1045 1046 def PrintSizeDetail(self, target_list, show_bloat): 1047 """Show details size information for each board 1048 1049 Args: 1050 target_list: List of targets, each a dict containing: 1051 'target': Target name 1052 'total_diff': Total difference in bytes across all areas 1053 <part_name>: Difference for that part 1054 show_bloat: Show detail for each function 1055 """ 1056 targets_by_diff = sorted(target_list, reverse=True, 1057 key=lambda x: x['_total_diff']) 1058 for result in targets_by_diff: 1059 printed_target = False 1060 for name in sorted(result): 1061 diff = result[name] 1062 if name.startswith('_'): 1063 continue 1064 if diff != 0: 1065 color = self.col.RED if diff > 0 else self.col.GREEN 1066 msg = ' %s %+d' % (name, diff) 1067 if not printed_target: 1068 print '%10s %-15s:' % ('', result['_target']), 1069 printed_target = True 1070 print self.col.Color(color, msg), 1071 if printed_target: 1072 print 1073 if show_bloat: 1074 target = result['_target'] 1075 outcome = result['_outcome'] 1076 base_outcome = self._base_board_dict[target] 1077 for fname in outcome.func_sizes: 1078 self.PrintFuncSizeDetail(fname, 1079 base_outcome.func_sizes[fname], 1080 outcome.func_sizes[fname]) 1081 1082 1083 def PrintSizeSummary(self, board_selected, board_dict, show_detail, 1084 show_bloat): 1085 """Print a summary of image sizes broken down by section. 1086 1087 The summary takes the form of one line per architecture. The 1088 line contains deltas for each of the sections (+ means the section 1089 got bigger, - means smaller). The nunmbers are the average number 1090 of bytes that a board in this section increased by. 1091 1092 For example: 1093 powerpc: (622 boards) text -0.0 1094 arm: (285 boards) text -0.0 1095 nds32: (3 boards) text -8.0 1096 1097 Args: 1098 board_selected: Dict containing boards to summarise, keyed by 1099 board.target 1100 board_dict: Dict containing boards for which we built this 1101 commit, keyed by board.target. The value is an Outcome object. 1102 show_detail: Show detail for each board 1103 show_bloat: Show detail for each function 1104 """ 1105 arch_list = {} 1106 arch_count = {} 1107 1108 # Calculate changes in size for different image parts 1109 # The previous sizes are in Board.sizes, for each board 1110 for target in board_dict: 1111 if target not in board_selected: 1112 continue 1113 base_sizes = self._base_board_dict[target].sizes 1114 outcome = board_dict[target] 1115 sizes = outcome.sizes 1116 1117 # Loop through the list of images, creating a dict of size 1118 # changes for each image/part. We end up with something like 1119 # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4} 1120 # which means that U-Boot data increased by 5 bytes and SPL 1121 # text decreased by 4. 1122 err = {'_target' : target} 1123 for image in sizes: 1124 if image in base_sizes: 1125 base_image = base_sizes[image] 1126 # Loop through the text, data, bss parts 1127 for part in sorted(sizes[image]): 1128 diff = sizes[image][part] - base_image[part] 1129 col = None 1130 if diff: 1131 if image == 'u-boot': 1132 name = part 1133 else: 1134 name = image + ':' + part 1135 err[name] = diff 1136 arch = board_selected[target].arch 1137 if not arch in arch_count: 1138 arch_count[arch] = 1 1139 else: 1140 arch_count[arch] += 1 1141 if not sizes: 1142 pass # Only add to our list when we have some stats 1143 elif not arch in arch_list: 1144 arch_list[arch] = [err] 1145 else: 1146 arch_list[arch].append(err) 1147 1148 # We now have a list of image size changes sorted by arch 1149 # Print out a summary of these 1150 for arch, target_list in arch_list.iteritems(): 1151 # Get total difference for each type 1152 totals = {} 1153 for result in target_list: 1154 total = 0 1155 for name, diff in result.iteritems(): 1156 if name.startswith('_'): 1157 continue 1158 total += diff 1159 if name in totals: 1160 totals[name] += diff 1161 else: 1162 totals[name] = diff 1163 result['_total_diff'] = total 1164 result['_outcome'] = board_dict[result['_target']] 1165 1166 count = len(target_list) 1167 printed_arch = False 1168 for name in sorted(totals): 1169 diff = totals[name] 1170 if diff: 1171 # Display the average difference in this name for this 1172 # architecture 1173 avg_diff = float(diff) / count 1174 color = self.col.RED if avg_diff > 0 else self.col.GREEN 1175 msg = ' %s %+1.1f' % (name, avg_diff) 1176 if not printed_arch: 1177 print '%10s: (for %d/%d boards)' % (arch, count, 1178 arch_count[arch]), 1179 printed_arch = True 1180 print self.col.Color(color, msg), 1181 1182 if printed_arch: 1183 print 1184 if show_detail: 1185 self.PrintSizeDetail(target_list, show_bloat) 1186 1187 1188 def PrintResultSummary(self, board_selected, board_dict, err_lines, 1189 show_sizes, show_detail, show_bloat): 1190 """Compare results with the base results and display delta. 1191 1192 Only boards mentioned in board_selected will be considered. This 1193 function is intended to be called repeatedly with the results of 1194 each commit. It therefore shows a 'diff' between what it saw in 1195 the last call and what it sees now. 1196 1197 Args: 1198 board_selected: Dict containing boards to summarise, keyed by 1199 board.target 1200 board_dict: Dict containing boards for which we built this 1201 commit, keyed by board.target. The value is an Outcome object. 1202 err_lines: A list of errors for this commit, or [] if there is 1203 none, or we don't want to print errors 1204 show_sizes: Show image size deltas 1205 show_detail: Show detail for each board 1206 show_bloat: Show detail for each function 1207 """ 1208 better = [] # List of boards fixed since last commit 1209 worse = [] # List of new broken boards since last commit 1210 new = [] # List of boards that didn't exist last time 1211 unknown = [] # List of boards that were not built 1212 1213 for target in board_dict: 1214 if target not in board_selected: 1215 continue 1216 1217 # If the board was built last time, add its outcome to a list 1218 if target in self._base_board_dict: 1219 base_outcome = self._base_board_dict[target].rc 1220 outcome = board_dict[target] 1221 if outcome.rc == OUTCOME_UNKNOWN: 1222 unknown.append(target) 1223 elif outcome.rc < base_outcome: 1224 better.append(target) 1225 elif outcome.rc > base_outcome: 1226 worse.append(target) 1227 else: 1228 new.append(target) 1229 1230 # Get a list of errors that have appeared, and disappeared 1231 better_err = [] 1232 worse_err = [] 1233 for line in err_lines: 1234 if line not in self._base_err_lines: 1235 worse_err.append('+' + line) 1236 for line in self._base_err_lines: 1237 if line not in err_lines: 1238 better_err.append('-' + line) 1239 1240 # Display results by arch 1241 if better or worse or unknown or new or worse_err or better_err: 1242 arch_list = {} 1243 self.AddOutcome(board_selected, arch_list, better, '', 1244 self.col.GREEN) 1245 self.AddOutcome(board_selected, arch_list, worse, '+', 1246 self.col.RED) 1247 self.AddOutcome(board_selected, arch_list, new, '*', self.col.BLUE) 1248 if self._show_unknown: 1249 self.AddOutcome(board_selected, arch_list, unknown, '?', 1250 self.col.MAGENTA) 1251 for arch, target_list in arch_list.iteritems(): 1252 print '%10s: %s' % (arch, target_list) 1253 if better_err: 1254 print self.col.Color(self.col.GREEN, '\n'.join(better_err)) 1255 if worse_err: 1256 print self.col.Color(self.col.RED, '\n'.join(worse_err)) 1257 1258 if show_sizes: 1259 self.PrintSizeSummary(board_selected, board_dict, show_detail, 1260 show_bloat) 1261 1262 # Save our updated information for the next call to this function 1263 self._base_board_dict = board_dict 1264 self._base_err_lines = err_lines 1265 1266 # Get a list of boards that did not get built, if needed 1267 not_built = [] 1268 for board in board_selected: 1269 if not board in board_dict: 1270 not_built.append(board) 1271 if not_built: 1272 print "Boards not built (%d): %s" % (len(not_built), 1273 ', '.join(not_built)) 1274 1275 1276 def ShowSummary(self, commits, board_selected, show_errors, show_sizes, 1277 show_detail, show_bloat): 1278 """Show a build summary for U-Boot for a given board list. 1279 1280 Reset the result summary, then repeatedly call GetResultSummary on 1281 each commit's results, then display the differences we see. 1282 1283 Args: 1284 commit: Commit objects to summarise 1285 board_selected: Dict containing boards to summarise 1286 show_errors: Show errors that occured 1287 show_sizes: Show size deltas 1288 show_detail: Show detail for each board 1289 show_bloat: Show detail for each function 1290 """ 1291 self.commit_count = len(commits) 1292 self.commits = commits 1293 self.ResetResultSummary(board_selected) 1294 1295 for commit_upto in range(0, self.commit_count, self._step): 1296 board_dict, err_lines = self.GetResultSummary(board_selected, 1297 commit_upto, read_func_sizes=show_bloat) 1298 msg = '%02d: %s' % (commit_upto + 1, commits[commit_upto].subject) 1299 print self.col.Color(self.col.BLUE, msg) 1300 self.PrintResultSummary(board_selected, board_dict, 1301 err_lines if show_errors else [], show_sizes, show_detail, 1302 show_bloat) 1303 1304 1305 def SetupBuild(self, board_selected, commits): 1306 """Set up ready to start a build. 1307 1308 Args: 1309 board_selected: Selected boards to build 1310 commits: Selected commits to build 1311 """ 1312 # First work out how many commits we will build 1313 count = (len(commits) + self._step - 1) / self._step 1314 self.count = len(board_selected) * count 1315 self.upto = self.warned = self.fail = 0 1316 self._timestamps = collections.deque() 1317 1318 def BuildBoardsForCommit(self, board_selected, keep_outputs): 1319 """Build all boards for a single commit""" 1320 self.SetupBuild(board_selected) 1321 self.count = len(board_selected) 1322 for brd in board_selected.itervalues(): 1323 job = BuilderJob() 1324 job.board = brd 1325 job.commits = None 1326 job.keep_outputs = keep_outputs 1327 self.queue.put(brd) 1328 1329 self.queue.join() 1330 self.out_queue.join() 1331 print 1332 self.ClearLine(0) 1333 1334 def BuildCommits(self, commits, board_selected, show_errors, keep_outputs): 1335 """Build all boards for all commits (non-incremental)""" 1336 self.commit_count = len(commits) 1337 1338 self.ResetResultSummary(board_selected) 1339 for self.commit_upto in range(self.commit_count): 1340 self.SelectCommit(commits[self.commit_upto]) 1341 self.SelectOutputDir() 1342 Mkdir(self.output_dir) 1343 1344 self.BuildBoardsForCommit(board_selected, keep_outputs) 1345 board_dict, err_lines = self.GetResultSummary() 1346 self.PrintResultSummary(board_selected, board_dict, 1347 err_lines if show_errors else []) 1348 1349 if self.already_done: 1350 print '%d builds already done' % self.already_done 1351 1352 def GetThreadDir(self, thread_num): 1353 """Get the directory path to the working dir for a thread. 1354 1355 Args: 1356 thread_num: Number of thread to check. 1357 """ 1358 return os.path.join(self._working_dir, '%02d' % thread_num) 1359 1360 def _PrepareThread(self, thread_num): 1361 """Prepare the working directory for a thread. 1362 1363 This clones or fetches the repo into the thread's work directory. 1364 1365 Args: 1366 thread_num: Thread number (0, 1, ...) 1367 """ 1368 thread_dir = self.GetThreadDir(thread_num) 1369 Mkdir(thread_dir) 1370 git_dir = os.path.join(thread_dir, '.git') 1371 1372 # Clone the repo if it doesn't already exist 1373 # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so 1374 # we have a private index but uses the origin repo's contents? 1375 if self.git_dir: 1376 src_dir = os.path.abspath(self.git_dir) 1377 if os.path.exists(git_dir): 1378 gitutil.Fetch(git_dir, thread_dir) 1379 else: 1380 print 'Cloning repo for thread %d' % thread_num 1381 gitutil.Clone(src_dir, thread_dir) 1382 1383 def _PrepareWorkingSpace(self, max_threads): 1384 """Prepare the working directory for use. 1385 1386 Set up the git repo for each thread. 1387 1388 Args: 1389 max_threads: Maximum number of threads we expect to need. 1390 """ 1391 Mkdir(self._working_dir) 1392 for thread in range(max_threads): 1393 self._PrepareThread(thread) 1394 1395 def _PrepareOutputSpace(self): 1396 """Get the output directories ready to receive files. 1397 1398 We delete any output directories which look like ones we need to 1399 create. Having left over directories is confusing when the user wants 1400 to check the output manually. 1401 """ 1402 dir_list = [] 1403 for commit_upto in range(self.commit_count): 1404 dir_list.append(self._GetOutputDir(commit_upto)) 1405 1406 for dirname in glob.glob(os.path.join(self.base_dir, '*')): 1407 if dirname not in dir_list: 1408 shutil.rmtree(dirname) 1409 1410 def BuildBoards(self, commits, board_selected, show_errors, keep_outputs): 1411 """Build all commits for a list of boards 1412 1413 Args: 1414 commits: List of commits to be build, each a Commit object 1415 boards_selected: Dict of selected boards, key is target name, 1416 value is Board object 1417 show_errors: True to show summarised error/warning info 1418 keep_outputs: True to save build output files 1419 """ 1420 self.commit_count = len(commits) 1421 self.commits = commits 1422 1423 self.ResetResultSummary(board_selected) 1424 Mkdir(self.base_dir) 1425 self._PrepareWorkingSpace(min(self.num_threads, len(board_selected))) 1426 self._PrepareOutputSpace() 1427 self.SetupBuild(board_selected, commits) 1428 self.ProcessResult(None) 1429 1430 # Create jobs to build all commits for each board 1431 for brd in board_selected.itervalues(): 1432 job = BuilderJob() 1433 job.board = brd 1434 job.commits = commits 1435 job.keep_outputs = keep_outputs 1436 job.step = self._step 1437 self.queue.put(job) 1438 1439 # Wait until all jobs are started 1440 self.queue.join() 1441 1442 # Wait until we have processed all output 1443 self.out_queue.join() 1444 print 1445 self.ClearLine(0) 1446