1# Copyright (c) 2013 The Chromium OS Authors. 2# 3# Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com> 4# 5# SPDX-License-Identifier: GPL-2.0+ 6# 7 8import collections 9import errno 10from datetime import datetime, timedelta 11import glob 12import os 13import re 14import Queue 15import shutil 16import string 17import sys 18import threading 19import time 20 21import command 22import gitutil 23import terminal 24import toolchain 25 26 27""" 28Theory of Operation 29 30Please see README for user documentation, and you should be familiar with 31that before trying to make sense of this. 32 33Buildman works by keeping the machine as busy as possible, building different 34commits for different boards on multiple CPUs at once. 35 36The source repo (self.git_dir) contains all the commits to be built. Each 37thread works on a single board at a time. It checks out the first commit, 38configures it for that board, then builds it. Then it checks out the next 39commit and builds it (typically without re-configuring). When it runs out 40of commits, it gets another job from the builder and starts again with that 41board. 42 43Clearly the builder threads could work either way - they could check out a 44commit and then built it for all boards. Using separate directories for each 45commit/board pair they could leave their build product around afterwards 46also. 47 48The intent behind building a single board for multiple commits, is to make 49use of incremental builds. Since each commit is built incrementally from 50the previous one, builds are faster. Reconfiguring for a different board 51removes all intermediate object files. 52 53Many threads can be working at once, but each has its own working directory. 54When a thread finishes a build, it puts the output files into a result 55directory. 56 57The base directory used by buildman is normally '../<branch>', i.e. 58a directory higher than the source repository and named after the branch 59being built. 60 61Within the base directory, we have one subdirectory for each commit. Within 62that is one subdirectory for each board. Within that is the build output for 63that commit/board combination. 64 65Buildman also create working directories for each thread, in a .bm-work/ 66subdirectory in the base dir. 67 68As an example, say we are building branch 'us-net' for boards 'sandbox' and 69'seaboard', and say that us-net has two commits. We will have directories 70like this: 71 72us-net/ base directory 73 01_of_02_g4ed4ebc_net--Add-tftp-speed-/ 74 sandbox/ 75 u-boot.bin 76 seaboard/ 77 u-boot.bin 78 02_of_02_g4ed4ebc_net--Check-tftp-comp/ 79 sandbox/ 80 u-boot.bin 81 seaboard/ 82 u-boot.bin 83 .bm-work/ 84 00/ working directory for thread 0 (contains source checkout) 85 build/ build output 86 01/ working directory for thread 1 87 build/ build output 88 ... 89u-boot/ source directory 90 .git/ repository 91""" 92 93# Possible build outcomes 94OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4) 95 96# Translate a commit subject into a valid filename 97trans_valid_chars = string.maketrans("/: ", "---") 98 99 100def Mkdir(dirname): 101 """Make a directory if it doesn't already exist. 102 103 Args: 104 dirname: Directory to create 105 """ 106 try: 107 os.mkdir(dirname) 108 except OSError as err: 109 if err.errno == errno.EEXIST: 110 pass 111 else: 112 raise 113 114class BuilderJob: 115 """Holds information about a job to be performed by a thread 116 117 Members: 118 board: Board object to build 119 commits: List of commit options to build. 120 """ 121 def __init__(self): 122 self.board = None 123 self.commits = [] 124 125 126class ResultThread(threading.Thread): 127 """This thread processes results from builder threads. 128 129 It simply passes the results on to the builder. There is only one 130 result thread, and this helps to serialise the build output. 131 """ 132 def __init__(self, builder): 133 """Set up a new result thread 134 135 Args: 136 builder: Builder which will be sent each result 137 """ 138 threading.Thread.__init__(self) 139 self.builder = builder 140 141 def run(self): 142 """Called to start up the result thread. 143 144 We collect the next result job and pass it on to the build. 145 """ 146 while True: 147 result = self.builder.out_queue.get() 148 self.builder.ProcessResult(result) 149 self.builder.out_queue.task_done() 150 151 152class BuilderThread(threading.Thread): 153 """This thread builds U-Boot for a particular board. 154 155 An input queue provides each new job. We run 'make' to build U-Boot 156 and then pass the results on to the output queue. 157 158 Members: 159 builder: The builder which contains information we might need 160 thread_num: Our thread number (0-n-1), used to decide on a 161 temporary directory 162 """ 163 def __init__(self, builder, thread_num): 164 """Set up a new builder thread""" 165 threading.Thread.__init__(self) 166 self.builder = builder 167 self.thread_num = thread_num 168 169 def Make(self, commit, brd, stage, cwd, *args, **kwargs): 170 """Run 'make' on a particular commit and board. 171 172 The source code will already be checked out, so the 'commit' 173 argument is only for information. 174 175 Args: 176 commit: Commit object that is being built 177 brd: Board object that is being built 178 stage: Stage of the build. Valid stages are: 179 distclean - can be called to clean source 180 config - called to configure for a board 181 build - the main make invocation - it does the build 182 args: A list of arguments to pass to 'make' 183 kwargs: A list of keyword arguments to pass to command.RunPipe() 184 185 Returns: 186 CommandResult object 187 """ 188 return self.builder.do_make(commit, brd, stage, cwd, *args, 189 **kwargs) 190 191 def RunCommit(self, commit_upto, brd, work_dir, do_config, force_build, 192 force_build_failures): 193 """Build a particular commit. 194 195 If the build is already done, and we are not forcing a build, we skip 196 the build and just return the previously-saved results. 197 198 Args: 199 commit_upto: Commit number to build (0...n-1) 200 brd: Board object to build 201 work_dir: Directory to which the source will be checked out 202 do_config: True to run a make <board>_config on the source 203 force_build: Force a build even if one was previously done 204 force_build_failures: Force a bulid if the previous result showed 205 failure 206 207 Returns: 208 tuple containing: 209 - CommandResult object containing the results of the build 210 - boolean indicating whether 'make config' is still needed 211 """ 212 # Create a default result - it will be overwritte by the call to 213 # self.Make() below, in the event that we do a build. 214 result = command.CommandResult() 215 result.return_code = 0 216 out_dir = os.path.join(work_dir, 'build') 217 218 # Check if the job was already completed last time 219 done_file = self.builder.GetDoneFile(commit_upto, brd.target) 220 result.already_done = os.path.exists(done_file) 221 will_build = (force_build or force_build_failures or 222 not result.already_done) 223 if result.already_done and will_build: 224 # Get the return code from that build and use it 225 with open(done_file, 'r') as fd: 226 result.return_code = int(fd.readline()) 227 err_file = self.builder.GetErrFile(commit_upto, brd.target) 228 if os.path.exists(err_file) and os.stat(err_file).st_size: 229 result.stderr = 'bad' 230 elif not force_build: 231 # The build passed, so no need to build it again 232 will_build = False 233 234 if will_build: 235 # We are going to have to build it. First, get a toolchain 236 if not self.toolchain: 237 try: 238 self.toolchain = self.builder.toolchains.Select(brd.arch) 239 except ValueError as err: 240 result.return_code = 10 241 result.stdout = '' 242 result.stderr = str(err) 243 # TODO(sjg@chromium.org): This gets swallowed, but needs 244 # to be reported. 245 246 if self.toolchain: 247 # Checkout the right commit 248 if commit_upto is not None: 249 commit = self.builder.commits[commit_upto] 250 if self.builder.checkout: 251 git_dir = os.path.join(work_dir, '.git') 252 gitutil.Checkout(commit.hash, git_dir, work_dir, 253 force=True) 254 else: 255 commit = self.builder.commit # Ick, fix this for BuildCommits() 256 257 # Set up the environment and command line 258 env = self.toolchain.MakeEnvironment() 259 Mkdir(out_dir) 260 args = ['O=build', '-s'] 261 if self.builder.num_jobs is not None: 262 args.extend(['-j', str(self.builder.num_jobs)]) 263 config_args = ['%s_config' % brd.target] 264 config_out = '' 265 args.extend(self.builder.toolchains.GetMakeArguments(brd)) 266 267 # If we need to reconfigure, do that now 268 if do_config: 269 result = self.Make(commit, brd, 'distclean', work_dir, 270 'distclean', *args, env=env) 271 result = self.Make(commit, brd, 'config', work_dir, 272 *(args + config_args), env=env) 273 config_out = result.combined 274 do_config = False # No need to configure next time 275 if result.return_code == 0: 276 result = self.Make(commit, brd, 'build', work_dir, *args, 277 env=env) 278 result.stdout = config_out + result.stdout 279 else: 280 result.return_code = 1 281 result.stderr = 'No tool chain for %s\n' % brd.arch 282 result.already_done = False 283 284 result.toolchain = self.toolchain 285 result.brd = brd 286 result.commit_upto = commit_upto 287 result.out_dir = out_dir 288 return result, do_config 289 290 def _WriteResult(self, result, keep_outputs): 291 """Write a built result to the output directory. 292 293 Args: 294 result: CommandResult object containing result to write 295 keep_outputs: True to store the output binaries, False 296 to delete them 297 """ 298 # Fatal error 299 if result.return_code < 0: 300 return 301 302 # Aborted? 303 if result.stderr and 'No child processes' in result.stderr: 304 return 305 306 if result.already_done: 307 return 308 309 # Write the output and stderr 310 output_dir = self.builder._GetOutputDir(result.commit_upto) 311 Mkdir(output_dir) 312 build_dir = self.builder.GetBuildDir(result.commit_upto, 313 result.brd.target) 314 Mkdir(build_dir) 315 316 outfile = os.path.join(build_dir, 'log') 317 with open(outfile, 'w') as fd: 318 if result.stdout: 319 fd.write(result.stdout) 320 321 errfile = self.builder.GetErrFile(result.commit_upto, 322 result.brd.target) 323 if result.stderr: 324 with open(errfile, 'w') as fd: 325 fd.write(result.stderr) 326 elif os.path.exists(errfile): 327 os.remove(errfile) 328 329 if result.toolchain: 330 # Write the build result and toolchain information. 331 done_file = self.builder.GetDoneFile(result.commit_upto, 332 result.brd.target) 333 with open(done_file, 'w') as fd: 334 fd.write('%s' % result.return_code) 335 with open(os.path.join(build_dir, 'toolchain'), 'w') as fd: 336 print >>fd, 'gcc', result.toolchain.gcc 337 print >>fd, 'path', result.toolchain.path 338 print >>fd, 'cross', result.toolchain.cross 339 print >>fd, 'arch', result.toolchain.arch 340 fd.write('%s' % result.return_code) 341 342 with open(os.path.join(build_dir, 'toolchain'), 'w') as fd: 343 print >>fd, 'gcc', result.toolchain.gcc 344 print >>fd, 'path', result.toolchain.path 345 346 # Write out the image and function size information and an objdump 347 env = result.toolchain.MakeEnvironment() 348 lines = [] 349 for fname in ['u-boot', 'spl/u-boot-spl']: 350 cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname] 351 nm_result = command.RunPipe([cmd], capture=True, 352 capture_stderr=True, cwd=result.out_dir, 353 raise_on_error=False, env=env) 354 if nm_result.stdout: 355 nm = self.builder.GetFuncSizesFile(result.commit_upto, 356 result.brd.target, fname) 357 with open(nm, 'w') as fd: 358 print >>fd, nm_result.stdout, 359 360 cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname] 361 dump_result = command.RunPipe([cmd], capture=True, 362 capture_stderr=True, cwd=result.out_dir, 363 raise_on_error=False, env=env) 364 rodata_size = '' 365 if dump_result.stdout: 366 objdump = self.builder.GetObjdumpFile(result.commit_upto, 367 result.brd.target, fname) 368 with open(objdump, 'w') as fd: 369 print >>fd, dump_result.stdout, 370 for line in dump_result.stdout.splitlines(): 371 fields = line.split() 372 if len(fields) > 5 and fields[1] == '.rodata': 373 rodata_size = fields[2] 374 375 cmd = ['%ssize' % self.toolchain.cross, fname] 376 size_result = command.RunPipe([cmd], capture=True, 377 capture_stderr=True, cwd=result.out_dir, 378 raise_on_error=False, env=env) 379 if size_result.stdout: 380 lines.append(size_result.stdout.splitlines()[1] + ' ' + 381 rodata_size) 382 383 # Write out the image sizes file. This is similar to the output 384 # of binutil's 'size' utility, but it omits the header line and 385 # adds an additional hex value at the end of each line for the 386 # rodata size 387 if len(lines): 388 sizes = self.builder.GetSizesFile(result.commit_upto, 389 result.brd.target) 390 with open(sizes, 'w') as fd: 391 print >>fd, '\n'.join(lines) 392 393 # Now write the actual build output 394 if keep_outputs: 395 patterns = ['u-boot', '*.bin', 'u-boot.dtb', '*.map', 396 'include/autoconf.mk', 'spl/u-boot-spl', 397 'spl/u-boot-spl.bin'] 398 for pattern in patterns: 399 file_list = glob.glob(os.path.join(result.out_dir, pattern)) 400 for fname in file_list: 401 shutil.copy(fname, build_dir) 402 403 404 def RunJob(self, job): 405 """Run a single job 406 407 A job consists of a building a list of commits for a particular board. 408 409 Args: 410 job: Job to build 411 """ 412 brd = job.board 413 work_dir = self.builder.GetThreadDir(self.thread_num) 414 self.toolchain = None 415 if job.commits: 416 # Run 'make board_config' on the first commit 417 do_config = True 418 commit_upto = 0 419 force_build = False 420 for commit_upto in range(0, len(job.commits), job.step): 421 result, request_config = self.RunCommit(commit_upto, brd, 422 work_dir, do_config, 423 force_build or self.builder.force_build, 424 self.builder.force_build_failures) 425 failed = result.return_code or result.stderr 426 if failed and not do_config: 427 # If our incremental build failed, try building again 428 # with a reconfig. 429 if self.builder.force_config_on_failure: 430 result, request_config = self.RunCommit(commit_upto, 431 brd, work_dir, True, True, False) 432 do_config = request_config 433 434 # If we built that commit, then config is done. But if we got 435 # an warning, reconfig next time to force it to build the same 436 # files that created warnings this time. Otherwise an 437 # incremental build may not build the same file, and we will 438 # think that the warning has gone away. 439 # We could avoid this by using -Werror everywhere... 440 # For errors, the problem doesn't happen, since presumably 441 # the build stopped and didn't generate output, so will retry 442 # that file next time. So we could detect warnings and deal 443 # with them specially here. For now, we just reconfigure if 444 # anything goes work. 445 # Of course this is substantially slower if there are build 446 # errors/warnings (e.g. 2-3x slower even if only 10% of builds 447 # have problems). 448 if (failed and not result.already_done and not do_config and 449 self.builder.force_config_on_failure): 450 # If this build failed, try the next one with a 451 # reconfigure. 452 # Sometimes if the board_config.h file changes it can mess 453 # with dependencies, and we get: 454 # make: *** No rule to make target `include/autoconf.mk', 455 # needed by `depend'. 456 do_config = True 457 force_build = True 458 else: 459 force_build = False 460 if self.builder.force_config_on_failure: 461 if failed: 462 do_config = True 463 result.commit_upto = commit_upto 464 if result.return_code < 0: 465 raise ValueError('Interrupt') 466 467 # We have the build results, so output the result 468 self._WriteResult(result, job.keep_outputs) 469 self.builder.out_queue.put(result) 470 else: 471 # Just build the currently checked-out build 472 result = self.RunCommit(None, True) 473 result.commit_upto = self.builder.upto 474 self.builder.out_queue.put(result) 475 476 def run(self): 477 """Our thread's run function 478 479 This thread picks a job from the queue, runs it, and then goes to the 480 next job. 481 """ 482 alive = True 483 while True: 484 job = self.builder.queue.get() 485 try: 486 if self.builder.active and alive: 487 self.RunJob(job) 488 except Exception as err: 489 alive = False 490 print err 491 self.builder.queue.task_done() 492 493 494class Builder: 495 """Class for building U-Boot for a particular commit. 496 497 Public members: (many should ->private) 498 active: True if the builder is active and has not been stopped 499 already_done: Number of builds already completed 500 base_dir: Base directory to use for builder 501 checkout: True to check out source, False to skip that step. 502 This is used for testing. 503 col: terminal.Color() object 504 count: Number of commits to build 505 do_make: Method to call to invoke Make 506 fail: Number of builds that failed due to error 507 force_build: Force building even if a build already exists 508 force_config_on_failure: If a commit fails for a board, disable 509 incremental building for the next commit we build for that 510 board, so that we will see all warnings/errors again. 511 force_build_failures: If a previously-built build (i.e. built on 512 a previous run of buildman) is marked as failed, rebuild it. 513 git_dir: Git directory containing source repository 514 last_line_len: Length of the last line we printed (used for erasing 515 it with new progress information) 516 num_jobs: Number of jobs to run at once (passed to make as -j) 517 num_threads: Number of builder threads to run 518 out_queue: Queue of results to process 519 re_make_err: Compiled regular expression for ignore_lines 520 queue: Queue of jobs to run 521 threads: List of active threads 522 toolchains: Toolchains object to use for building 523 upto: Current commit number we are building (0.count-1) 524 warned: Number of builds that produced at least one warning 525 526 Private members: 527 _base_board_dict: Last-summarised Dict of boards 528 _base_err_lines: Last-summarised list of errors 529 _build_period_us: Time taken for a single build (float object). 530 _complete_delay: Expected delay until completion (timedelta) 531 _next_delay_update: Next time we plan to display a progress update 532 (datatime) 533 _show_unknown: Show unknown boards (those not built) in summary 534 _timestamps: List of timestamps for the completion of the last 535 last _timestamp_count builds. Each is a datetime object. 536 _timestamp_count: Number of timestamps to keep in our list. 537 _working_dir: Base working directory containing all threads 538 """ 539 class Outcome: 540 """Records a build outcome for a single make invocation 541 542 Public Members: 543 rc: Outcome value (OUTCOME_...) 544 err_lines: List of error lines or [] if none 545 sizes: Dictionary of image size information, keyed by filename 546 - Each value is itself a dictionary containing 547 values for 'text', 'data' and 'bss', being the integer 548 size in bytes of each section. 549 func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each 550 value is itself a dictionary: 551 key: function name 552 value: Size of function in bytes 553 """ 554 def __init__(self, rc, err_lines, sizes, func_sizes): 555 self.rc = rc 556 self.err_lines = err_lines 557 self.sizes = sizes 558 self.func_sizes = func_sizes 559 560 def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, 561 checkout=True, show_unknown=True, step=1): 562 """Create a new Builder object 563 564 Args: 565 toolchains: Toolchains object to use for building 566 base_dir: Base directory to use for builder 567 git_dir: Git directory containing source repository 568 num_threads: Number of builder threads to run 569 num_jobs: Number of jobs to run at once (passed to make as -j) 570 checkout: True to check out source, False to skip that step. 571 This is used for testing. 572 show_unknown: Show unknown boards (those not built) in summary 573 step: 1 to process every commit, n to process every nth commit 574 """ 575 self.toolchains = toolchains 576 self.base_dir = base_dir 577 self._working_dir = os.path.join(base_dir, '.bm-work') 578 self.threads = [] 579 self.active = True 580 self.do_make = self.Make 581 self.checkout = checkout 582 self.num_threads = num_threads 583 self.num_jobs = num_jobs 584 self.already_done = 0 585 self.force_build = False 586 self.git_dir = git_dir 587 self._show_unknown = show_unknown 588 self._timestamp_count = 10 589 self._build_period_us = None 590 self._complete_delay = None 591 self._next_delay_update = datetime.now() 592 self.force_config_on_failure = True 593 self.force_build_failures = False 594 self._step = step 595 596 self.col = terminal.Color() 597 598 self.queue = Queue.Queue() 599 self.out_queue = Queue.Queue() 600 for i in range(self.num_threads): 601 t = BuilderThread(self, i) 602 t.setDaemon(True) 603 t.start() 604 self.threads.append(t) 605 606 self.last_line_len = 0 607 t = ResultThread(self) 608 t.setDaemon(True) 609 t.start() 610 self.threads.append(t) 611 612 ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)'] 613 self.re_make_err = re.compile('|'.join(ignore_lines)) 614 615 def __del__(self): 616 """Get rid of all threads created by the builder""" 617 for t in self.threads: 618 del t 619 620 def _AddTimestamp(self): 621 """Add a new timestamp to the list and record the build period. 622 623 The build period is the length of time taken to perform a single 624 build (one board, one commit). 625 """ 626 now = datetime.now() 627 self._timestamps.append(now) 628 count = len(self._timestamps) 629 delta = self._timestamps[-1] - self._timestamps[0] 630 seconds = delta.total_seconds() 631 632 # If we have enough data, estimate build period (time taken for a 633 # single build) and therefore completion time. 634 if count > 1 and self._next_delay_update < now: 635 self._next_delay_update = now + timedelta(seconds=2) 636 if seconds > 0: 637 self._build_period = float(seconds) / count 638 todo = self.count - self.upto 639 self._complete_delay = timedelta(microseconds= 640 self._build_period * todo * 1000000) 641 # Round it 642 self._complete_delay -= timedelta( 643 microseconds=self._complete_delay.microseconds) 644 645 if seconds > 60: 646 self._timestamps.popleft() 647 count -= 1 648 649 def ClearLine(self, length): 650 """Clear any characters on the current line 651 652 Make way for a new line of length 'length', by outputting enough 653 spaces to clear out the old line. Then remember the new length for 654 next time. 655 656 Args: 657 length: Length of new line, in characters 658 """ 659 if length < self.last_line_len: 660 print ' ' * (self.last_line_len - length), 661 print '\r', 662 self.last_line_len = length 663 sys.stdout.flush() 664 665 def SelectCommit(self, commit, checkout=True): 666 """Checkout the selected commit for this build 667 """ 668 self.commit = commit 669 if checkout and self.checkout: 670 gitutil.Checkout(commit.hash) 671 672 def Make(self, commit, brd, stage, cwd, *args, **kwargs): 673 """Run make 674 675 Args: 676 commit: Commit object that is being built 677 brd: Board object that is being built 678 stage: Stage that we are at (distclean, config, build) 679 cwd: Directory where make should be run 680 args: Arguments to pass to make 681 kwargs: Arguments to pass to command.RunPipe() 682 """ 683 cmd = ['make'] + list(args) 684 result = command.RunPipe([cmd], capture=True, capture_stderr=True, 685 cwd=cwd, raise_on_error=False, **kwargs) 686 return result 687 688 def ProcessResult(self, result): 689 """Process the result of a build, showing progress information 690 691 Args: 692 result: A CommandResult object 693 """ 694 col = terminal.Color() 695 if result: 696 target = result.brd.target 697 698 if result.return_code < 0: 699 self.active = False 700 command.StopAll() 701 return 702 703 self.upto += 1 704 if result.return_code != 0: 705 self.fail += 1 706 elif result.stderr: 707 self.warned += 1 708 if result.already_done: 709 self.already_done += 1 710 else: 711 target = '(starting)' 712 713 # Display separate counts for ok, warned and fail 714 ok = self.upto - self.warned - self.fail 715 line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok) 716 line += self.col.Color(self.col.YELLOW, '%5d' % self.warned) 717 line += self.col.Color(self.col.RED, '%5d' % self.fail) 718 719 name = ' /%-5d ' % self.count 720 721 # Add our current completion time estimate 722 self._AddTimestamp() 723 if self._complete_delay: 724 name += '%s : ' % self._complete_delay 725 # When building all boards for a commit, we can print a commit 726 # progress message. 727 if result and result.commit_upto is None: 728 name += 'commit %2d/%-3d' % (self.commit_upto + 1, 729 self.commit_count) 730 731 name += target 732 print line + name, 733 length = 13 + len(name) 734 self.ClearLine(length) 735 736 def _GetOutputDir(self, commit_upto): 737 """Get the name of the output directory for a commit number 738 739 The output directory is typically .../<branch>/<commit>. 740 741 Args: 742 commit_upto: Commit number to use (0..self.count-1) 743 """ 744 commit = self.commits[commit_upto] 745 subject = commit.subject.translate(trans_valid_chars) 746 commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1, 747 self.commit_count, commit.hash, subject[:20])) 748 output_dir = os.path.join(self.base_dir, commit_dir) 749 return output_dir 750 751 def GetBuildDir(self, commit_upto, target): 752 """Get the name of the build directory for a commit number 753 754 The build directory is typically .../<branch>/<commit>/<target>. 755 756 Args: 757 commit_upto: Commit number to use (0..self.count-1) 758 target: Target name 759 """ 760 output_dir = self._GetOutputDir(commit_upto) 761 return os.path.join(output_dir, target) 762 763 def GetDoneFile(self, commit_upto, target): 764 """Get the name of the done file for a commit number 765 766 Args: 767 commit_upto: Commit number to use (0..self.count-1) 768 target: Target name 769 """ 770 return os.path.join(self.GetBuildDir(commit_upto, target), 'done') 771 772 def GetSizesFile(self, commit_upto, target): 773 """Get the name of the sizes file for a commit number 774 775 Args: 776 commit_upto: Commit number to use (0..self.count-1) 777 target: Target name 778 """ 779 return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes') 780 781 def GetFuncSizesFile(self, commit_upto, target, elf_fname): 782 """Get the name of the funcsizes file for a commit number and ELF file 783 784 Args: 785 commit_upto: Commit number to use (0..self.count-1) 786 target: Target name 787 elf_fname: Filename of elf image 788 """ 789 return os.path.join(self.GetBuildDir(commit_upto, target), 790 '%s.sizes' % elf_fname.replace('/', '-')) 791 792 def GetObjdumpFile(self, commit_upto, target, elf_fname): 793 """Get the name of the objdump file for a commit number and ELF file 794 795 Args: 796 commit_upto: Commit number to use (0..self.count-1) 797 target: Target name 798 elf_fname: Filename of elf image 799 """ 800 return os.path.join(self.GetBuildDir(commit_upto, target), 801 '%s.objdump' % elf_fname.replace('/', '-')) 802 803 def GetErrFile(self, commit_upto, target): 804 """Get the name of the err file for a commit number 805 806 Args: 807 commit_upto: Commit number to use (0..self.count-1) 808 target: Target name 809 """ 810 output_dir = self.GetBuildDir(commit_upto, target) 811 return os.path.join(output_dir, 'err') 812 813 def FilterErrors(self, lines): 814 """Filter out errors in which we have no interest 815 816 We should probably use map(). 817 818 Args: 819 lines: List of error lines, each a string 820 Returns: 821 New list with only interesting lines included 822 """ 823 out_lines = [] 824 for line in lines: 825 if not self.re_make_err.search(line): 826 out_lines.append(line) 827 return out_lines 828 829 def ReadFuncSizes(self, fname, fd): 830 """Read function sizes from the output of 'nm' 831 832 Args: 833 fd: File containing data to read 834 fname: Filename we are reading from (just for errors) 835 836 Returns: 837 Dictionary containing size of each function in bytes, indexed by 838 function name. 839 """ 840 sym = {} 841 for line in fd.readlines(): 842 try: 843 size, type, name = line[:-1].split() 844 except: 845 print "Invalid line in file '%s': '%s'" % (fname, line[:-1]) 846 continue 847 if type in 'tTdDbB': 848 # function names begin with '.' on 64-bit powerpc 849 if '.' in name[1:]: 850 name = 'static.' + name.split('.')[0] 851 sym[name] = sym.get(name, 0) + int(size, 16) 852 return sym 853 854 def GetBuildOutcome(self, commit_upto, target, read_func_sizes): 855 """Work out the outcome of a build. 856 857 Args: 858 commit_upto: Commit number to check (0..n-1) 859 target: Target board to check 860 read_func_sizes: True to read function size information 861 862 Returns: 863 Outcome object 864 """ 865 done_file = self.GetDoneFile(commit_upto, target) 866 sizes_file = self.GetSizesFile(commit_upto, target) 867 sizes = {} 868 func_sizes = {} 869 if os.path.exists(done_file): 870 with open(done_file, 'r') as fd: 871 return_code = int(fd.readline()) 872 err_lines = [] 873 err_file = self.GetErrFile(commit_upto, target) 874 if os.path.exists(err_file): 875 with open(err_file, 'r') as fd: 876 err_lines = self.FilterErrors(fd.readlines()) 877 878 # Decide whether the build was ok, failed or created warnings 879 if return_code: 880 rc = OUTCOME_ERROR 881 elif len(err_lines): 882 rc = OUTCOME_WARNING 883 else: 884 rc = OUTCOME_OK 885 886 # Convert size information to our simple format 887 if os.path.exists(sizes_file): 888 with open(sizes_file, 'r') as fd: 889 for line in fd.readlines(): 890 values = line.split() 891 rodata = 0 892 if len(values) > 6: 893 rodata = int(values[6], 16) 894 size_dict = { 895 'all' : int(values[0]) + int(values[1]) + 896 int(values[2]), 897 'text' : int(values[0]) - rodata, 898 'data' : int(values[1]), 899 'bss' : int(values[2]), 900 'rodata' : rodata, 901 } 902 sizes[values[5]] = size_dict 903 904 if read_func_sizes: 905 pattern = self.GetFuncSizesFile(commit_upto, target, '*') 906 for fname in glob.glob(pattern): 907 with open(fname, 'r') as fd: 908 dict_name = os.path.basename(fname).replace('.sizes', 909 '') 910 func_sizes[dict_name] = self.ReadFuncSizes(fname, fd) 911 912 return Builder.Outcome(rc, err_lines, sizes, func_sizes) 913 914 return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}) 915 916 def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes): 917 """Calculate a summary of the results of building a commit. 918 919 Args: 920 board_selected: Dict containing boards to summarise 921 commit_upto: Commit number to summarize (0..self.count-1) 922 read_func_sizes: True to read function size information 923 924 Returns: 925 Tuple: 926 Dict containing boards which passed building this commit. 927 keyed by board.target 928 List containing a summary of error/warning lines 929 """ 930 board_dict = {} 931 err_lines_summary = [] 932 933 for board in boards_selected.itervalues(): 934 outcome = self.GetBuildOutcome(commit_upto, board.target, 935 read_func_sizes) 936 board_dict[board.target] = outcome 937 for err in outcome.err_lines: 938 if err and not err.rstrip() in err_lines_summary: 939 err_lines_summary.append(err.rstrip()) 940 return board_dict, err_lines_summary 941 942 def AddOutcome(self, board_dict, arch_list, changes, char, color): 943 """Add an output to our list of outcomes for each architecture 944 945 This simple function adds failing boards (changes) to the 946 relevant architecture string, so we can print the results out 947 sorted by architecture. 948 949 Args: 950 board_dict: Dict containing all boards 951 arch_list: Dict keyed by arch name. Value is a string containing 952 a list of board names which failed for that arch. 953 changes: List of boards to add to arch_list 954 color: terminal.Colour object 955 """ 956 done_arch = {} 957 for target in changes: 958 if target in board_dict: 959 arch = board_dict[target].arch 960 else: 961 arch = 'unknown' 962 str = self.col.Color(color, ' ' + target) 963 if not arch in done_arch: 964 str = self.col.Color(color, char) + ' ' + str 965 done_arch[arch] = True 966 if not arch in arch_list: 967 arch_list[arch] = str 968 else: 969 arch_list[arch] += str 970 971 972 def ColourNum(self, num): 973 color = self.col.RED if num > 0 else self.col.GREEN 974 if num == 0: 975 return '0' 976 return self.col.Color(color, str(num)) 977 978 def ResetResultSummary(self, board_selected): 979 """Reset the results summary ready for use. 980 981 Set up the base board list to be all those selected, and set the 982 error lines to empty. 983 984 Following this, calls to PrintResultSummary() will use this 985 information to work out what has changed. 986 987 Args: 988 board_selected: Dict containing boards to summarise, keyed by 989 board.target 990 """ 991 self._base_board_dict = {} 992 for board in board_selected: 993 self._base_board_dict[board] = Builder.Outcome(0, [], [], {}) 994 self._base_err_lines = [] 995 996 def PrintFuncSizeDetail(self, fname, old, new): 997 grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0 998 delta, common = [], {} 999 1000 for a in old: 1001 if a in new: 1002 common[a] = 1 1003 1004 for name in old: 1005 if name not in common: 1006 remove += 1 1007 down += old[name] 1008 delta.append([-old[name], name]) 1009 1010 for name in new: 1011 if name not in common: 1012 add += 1 1013 up += new[name] 1014 delta.append([new[name], name]) 1015 1016 for name in common: 1017 diff = new.get(name, 0) - old.get(name, 0) 1018 if diff > 0: 1019 grow, up = grow + 1, up + diff 1020 elif diff < 0: 1021 shrink, down = shrink + 1, down - diff 1022 delta.append([diff, name]) 1023 1024 delta.sort() 1025 delta.reverse() 1026 1027 args = [add, -remove, grow, -shrink, up, -down, up - down] 1028 if max(args) == 0: 1029 return 1030 args = [self.ColourNum(x) for x in args] 1031 indent = ' ' * 15 1032 print ('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' % 1033 tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args)) 1034 print '%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new', 1035 'delta') 1036 for diff, name in delta: 1037 if diff: 1038 color = self.col.RED if diff > 0 else self.col.GREEN 1039 msg = '%s %-38s %7s %7s %+7d' % (indent, name, 1040 old.get(name, '-'), new.get(name,'-'), diff) 1041 print self.col.Color(color, msg) 1042 1043 1044 def PrintSizeDetail(self, target_list, show_bloat): 1045 """Show details size information for each board 1046 1047 Args: 1048 target_list: List of targets, each a dict containing: 1049 'target': Target name 1050 'total_diff': Total difference in bytes across all areas 1051 <part_name>: Difference for that part 1052 show_bloat: Show detail for each function 1053 """ 1054 targets_by_diff = sorted(target_list, reverse=True, 1055 key=lambda x: x['_total_diff']) 1056 for result in targets_by_diff: 1057 printed_target = False 1058 for name in sorted(result): 1059 diff = result[name] 1060 if name.startswith('_'): 1061 continue 1062 if diff != 0: 1063 color = self.col.RED if diff > 0 else self.col.GREEN 1064 msg = ' %s %+d' % (name, diff) 1065 if not printed_target: 1066 print '%10s %-15s:' % ('', result['_target']), 1067 printed_target = True 1068 print self.col.Color(color, msg), 1069 if printed_target: 1070 print 1071 if show_bloat: 1072 target = result['_target'] 1073 outcome = result['_outcome'] 1074 base_outcome = self._base_board_dict[target] 1075 for fname in outcome.func_sizes: 1076 self.PrintFuncSizeDetail(fname, 1077 base_outcome.func_sizes[fname], 1078 outcome.func_sizes[fname]) 1079 1080 1081 def PrintSizeSummary(self, board_selected, board_dict, show_detail, 1082 show_bloat): 1083 """Print a summary of image sizes broken down by section. 1084 1085 The summary takes the form of one line per architecture. The 1086 line contains deltas for each of the sections (+ means the section 1087 got bigger, - means smaller). The nunmbers are the average number 1088 of bytes that a board in this section increased by. 1089 1090 For example: 1091 powerpc: (622 boards) text -0.0 1092 arm: (285 boards) text -0.0 1093 nds32: (3 boards) text -8.0 1094 1095 Args: 1096 board_selected: Dict containing boards to summarise, keyed by 1097 board.target 1098 board_dict: Dict containing boards for which we built this 1099 commit, keyed by board.target. The value is an Outcome object. 1100 show_detail: Show detail for each board 1101 show_bloat: Show detail for each function 1102 """ 1103 arch_list = {} 1104 arch_count = {} 1105 1106 # Calculate changes in size for different image parts 1107 # The previous sizes are in Board.sizes, for each board 1108 for target in board_dict: 1109 if target not in board_selected: 1110 continue 1111 base_sizes = self._base_board_dict[target].sizes 1112 outcome = board_dict[target] 1113 sizes = outcome.sizes 1114 1115 # Loop through the list of images, creating a dict of size 1116 # changes for each image/part. We end up with something like 1117 # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4} 1118 # which means that U-Boot data increased by 5 bytes and SPL 1119 # text decreased by 4. 1120 err = {'_target' : target} 1121 for image in sizes: 1122 if image in base_sizes: 1123 base_image = base_sizes[image] 1124 # Loop through the text, data, bss parts 1125 for part in sorted(sizes[image]): 1126 diff = sizes[image][part] - base_image[part] 1127 col = None 1128 if diff: 1129 if image == 'u-boot': 1130 name = part 1131 else: 1132 name = image + ':' + part 1133 err[name] = diff 1134 arch = board_selected[target].arch 1135 if not arch in arch_count: 1136 arch_count[arch] = 1 1137 else: 1138 arch_count[arch] += 1 1139 if not sizes: 1140 pass # Only add to our list when we have some stats 1141 elif not arch in arch_list: 1142 arch_list[arch] = [err] 1143 else: 1144 arch_list[arch].append(err) 1145 1146 # We now have a list of image size changes sorted by arch 1147 # Print out a summary of these 1148 for arch, target_list in arch_list.iteritems(): 1149 # Get total difference for each type 1150 totals = {} 1151 for result in target_list: 1152 total = 0 1153 for name, diff in result.iteritems(): 1154 if name.startswith('_'): 1155 continue 1156 total += diff 1157 if name in totals: 1158 totals[name] += diff 1159 else: 1160 totals[name] = diff 1161 result['_total_diff'] = total 1162 result['_outcome'] = board_dict[result['_target']] 1163 1164 count = len(target_list) 1165 printed_arch = False 1166 for name in sorted(totals): 1167 diff = totals[name] 1168 if diff: 1169 # Display the average difference in this name for this 1170 # architecture 1171 avg_diff = float(diff) / count 1172 color = self.col.RED if avg_diff > 0 else self.col.GREEN 1173 msg = ' %s %+1.1f' % (name, avg_diff) 1174 if not printed_arch: 1175 print '%10s: (for %d/%d boards)' % (arch, count, 1176 arch_count[arch]), 1177 printed_arch = True 1178 print self.col.Color(color, msg), 1179 1180 if printed_arch: 1181 print 1182 if show_detail: 1183 self.PrintSizeDetail(target_list, show_bloat) 1184 1185 1186 def PrintResultSummary(self, board_selected, board_dict, err_lines, 1187 show_sizes, show_detail, show_bloat): 1188 """Compare results with the base results and display delta. 1189 1190 Only boards mentioned in board_selected will be considered. This 1191 function is intended to be called repeatedly with the results of 1192 each commit. It therefore shows a 'diff' between what it saw in 1193 the last call and what it sees now. 1194 1195 Args: 1196 board_selected: Dict containing boards to summarise, keyed by 1197 board.target 1198 board_dict: Dict containing boards for which we built this 1199 commit, keyed by board.target. The value is an Outcome object. 1200 err_lines: A list of errors for this commit, or [] if there is 1201 none, or we don't want to print errors 1202 show_sizes: Show image size deltas 1203 show_detail: Show detail for each board 1204 show_bloat: Show detail for each function 1205 """ 1206 better = [] # List of boards fixed since last commit 1207 worse = [] # List of new broken boards since last commit 1208 new = [] # List of boards that didn't exist last time 1209 unknown = [] # List of boards that were not built 1210 1211 for target in board_dict: 1212 if target not in board_selected: 1213 continue 1214 1215 # If the board was built last time, add its outcome to a list 1216 if target in self._base_board_dict: 1217 base_outcome = self._base_board_dict[target].rc 1218 outcome = board_dict[target] 1219 if outcome.rc == OUTCOME_UNKNOWN: 1220 unknown.append(target) 1221 elif outcome.rc < base_outcome: 1222 better.append(target) 1223 elif outcome.rc > base_outcome: 1224 worse.append(target) 1225 else: 1226 new.append(target) 1227 1228 # Get a list of errors that have appeared, and disappeared 1229 better_err = [] 1230 worse_err = [] 1231 for line in err_lines: 1232 if line not in self._base_err_lines: 1233 worse_err.append('+' + line) 1234 for line in self._base_err_lines: 1235 if line not in err_lines: 1236 better_err.append('-' + line) 1237 1238 # Display results by arch 1239 if better or worse or unknown or new or worse_err or better_err: 1240 arch_list = {} 1241 self.AddOutcome(board_selected, arch_list, better, '', 1242 self.col.GREEN) 1243 self.AddOutcome(board_selected, arch_list, worse, '+', 1244 self.col.RED) 1245 self.AddOutcome(board_selected, arch_list, new, '*', self.col.BLUE) 1246 if self._show_unknown: 1247 self.AddOutcome(board_selected, arch_list, unknown, '?', 1248 self.col.MAGENTA) 1249 for arch, target_list in arch_list.iteritems(): 1250 print '%10s: %s' % (arch, target_list) 1251 if better_err: 1252 print self.col.Color(self.col.GREEN, '\n'.join(better_err)) 1253 if worse_err: 1254 print self.col.Color(self.col.RED, '\n'.join(worse_err)) 1255 1256 if show_sizes: 1257 self.PrintSizeSummary(board_selected, board_dict, show_detail, 1258 show_bloat) 1259 1260 # Save our updated information for the next call to this function 1261 self._base_board_dict = board_dict 1262 self._base_err_lines = err_lines 1263 1264 # Get a list of boards that did not get built, if needed 1265 not_built = [] 1266 for board in board_selected: 1267 if not board in board_dict: 1268 not_built.append(board) 1269 if not_built: 1270 print "Boards not built (%d): %s" % (len(not_built), 1271 ', '.join(not_built)) 1272 1273 1274 def ShowSummary(self, commits, board_selected, show_errors, show_sizes, 1275 show_detail, show_bloat): 1276 """Show a build summary for U-Boot for a given board list. 1277 1278 Reset the result summary, then repeatedly call GetResultSummary on 1279 each commit's results, then display the differences we see. 1280 1281 Args: 1282 commit: Commit objects to summarise 1283 board_selected: Dict containing boards to summarise 1284 show_errors: Show errors that occured 1285 show_sizes: Show size deltas 1286 show_detail: Show detail for each board 1287 show_bloat: Show detail for each function 1288 """ 1289 self.commit_count = len(commits) 1290 self.commits = commits 1291 self.ResetResultSummary(board_selected) 1292 1293 for commit_upto in range(0, self.commit_count, self._step): 1294 board_dict, err_lines = self.GetResultSummary(board_selected, 1295 commit_upto, read_func_sizes=show_bloat) 1296 msg = '%02d: %s' % (commit_upto + 1, commits[commit_upto].subject) 1297 print self.col.Color(self.col.BLUE, msg) 1298 self.PrintResultSummary(board_selected, board_dict, 1299 err_lines if show_errors else [], show_sizes, show_detail, 1300 show_bloat) 1301 1302 1303 def SetupBuild(self, board_selected, commits): 1304 """Set up ready to start a build. 1305 1306 Args: 1307 board_selected: Selected boards to build 1308 commits: Selected commits to build 1309 """ 1310 # First work out how many commits we will build 1311 count = (len(commits) + self._step - 1) / self._step 1312 self.count = len(board_selected) * count 1313 self.upto = self.warned = self.fail = 0 1314 self._timestamps = collections.deque() 1315 1316 def BuildBoardsForCommit(self, board_selected, keep_outputs): 1317 """Build all boards for a single commit""" 1318 self.SetupBuild(board_selected) 1319 self.count = len(board_selected) 1320 for brd in board_selected.itervalues(): 1321 job = BuilderJob() 1322 job.board = brd 1323 job.commits = None 1324 job.keep_outputs = keep_outputs 1325 self.queue.put(brd) 1326 1327 self.queue.join() 1328 self.out_queue.join() 1329 print 1330 self.ClearLine(0) 1331 1332 def BuildCommits(self, commits, board_selected, show_errors, keep_outputs): 1333 """Build all boards for all commits (non-incremental)""" 1334 self.commit_count = len(commits) 1335 1336 self.ResetResultSummary(board_selected) 1337 for self.commit_upto in range(self.commit_count): 1338 self.SelectCommit(commits[self.commit_upto]) 1339 self.SelectOutputDir() 1340 Mkdir(self.output_dir) 1341 1342 self.BuildBoardsForCommit(board_selected, keep_outputs) 1343 board_dict, err_lines = self.GetResultSummary() 1344 self.PrintResultSummary(board_selected, board_dict, 1345 err_lines if show_errors else []) 1346 1347 if self.already_done: 1348 print '%d builds already done' % self.already_done 1349 1350 def GetThreadDir(self, thread_num): 1351 """Get the directory path to the working dir for a thread. 1352 1353 Args: 1354 thread_num: Number of thread to check. 1355 """ 1356 return os.path.join(self._working_dir, '%02d' % thread_num) 1357 1358 def _PrepareThread(self, thread_num): 1359 """Prepare the working directory for a thread. 1360 1361 This clones or fetches the repo into the thread's work directory. 1362 1363 Args: 1364 thread_num: Thread number (0, 1, ...) 1365 """ 1366 thread_dir = self.GetThreadDir(thread_num) 1367 Mkdir(thread_dir) 1368 git_dir = os.path.join(thread_dir, '.git') 1369 1370 # Clone the repo if it doesn't already exist 1371 # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so 1372 # we have a private index but uses the origin repo's contents? 1373 if self.git_dir: 1374 src_dir = os.path.abspath(self.git_dir) 1375 if os.path.exists(git_dir): 1376 gitutil.Fetch(git_dir, thread_dir) 1377 else: 1378 print 'Cloning repo for thread %d' % thread_num 1379 gitutil.Clone(src_dir, thread_dir) 1380 1381 def _PrepareWorkingSpace(self, max_threads): 1382 """Prepare the working directory for use. 1383 1384 Set up the git repo for each thread. 1385 1386 Args: 1387 max_threads: Maximum number of threads we expect to need. 1388 """ 1389 Mkdir(self._working_dir) 1390 for thread in range(max_threads): 1391 self._PrepareThread(thread) 1392 1393 def _PrepareOutputSpace(self): 1394 """Get the output directories ready to receive files. 1395 1396 We delete any output directories which look like ones we need to 1397 create. Having left over directories is confusing when the user wants 1398 to check the output manually. 1399 """ 1400 dir_list = [] 1401 for commit_upto in range(self.commit_count): 1402 dir_list.append(self._GetOutputDir(commit_upto)) 1403 1404 for dirname in glob.glob(os.path.join(self.base_dir, '*')): 1405 if dirname not in dir_list: 1406 shutil.rmtree(dirname) 1407 1408 def BuildBoards(self, commits, board_selected, show_errors, keep_outputs): 1409 """Build all commits for a list of boards 1410 1411 Args: 1412 commits: List of commits to be build, each a Commit object 1413 boards_selected: Dict of selected boards, key is target name, 1414 value is Board object 1415 show_errors: True to show summarised error/warning info 1416 keep_outputs: True to save build output files 1417 """ 1418 self.commit_count = len(commits) 1419 self.commits = commits 1420 1421 self.ResetResultSummary(board_selected) 1422 Mkdir(self.base_dir) 1423 self._PrepareWorkingSpace(min(self.num_threads, len(board_selected))) 1424 self._PrepareOutputSpace() 1425 self.SetupBuild(board_selected, commits) 1426 self.ProcessResult(None) 1427 1428 # Create jobs to build all commits for each board 1429 for brd in board_selected.itervalues(): 1430 job = BuilderJob() 1431 job.board = brd 1432 job.commits = commits 1433 job.keep_outputs = keep_outputs 1434 job.step = self._step 1435 self.queue.put(job) 1436 1437 # Wait until all jobs are started 1438 self.queue.join() 1439 1440 # Wait until we have processed all output 1441 self.out_queue.join() 1442 print 1443 self.ClearLine(0) 1444