1# Copyright (c) 2013 The Chromium OS Authors. 2# 3# Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com> 4# 5# SPDX-License-Identifier: GPL-2.0+ 6# 7 8import collections 9import errno 10from datetime import datetime, timedelta 11import glob 12import os 13import re 14import Queue 15import shutil 16import string 17import sys 18import threading 19import time 20 21import command 22import gitutil 23import terminal 24import toolchain 25 26 27""" 28Theory of Operation 29 30Please see README for user documentation, and you should be familiar with 31that before trying to make sense of this. 32 33Buildman works by keeping the machine as busy as possible, building different 34commits for different boards on multiple CPUs at once. 35 36The source repo (self.git_dir) contains all the commits to be built. Each 37thread works on a single board at a time. It checks out the first commit, 38configures it for that board, then builds it. Then it checks out the next 39commit and builds it (typically without re-configuring). When it runs out 40of commits, it gets another job from the builder and starts again with that 41board. 42 43Clearly the builder threads could work either way - they could check out a 44commit and then built it for all boards. Using separate directories for each 45commit/board pair they could leave their build product around afterwards 46also. 47 48The intent behind building a single board for multiple commits, is to make 49use of incremental builds. Since each commit is built incrementally from 50the previous one, builds are faster. Reconfiguring for a different board 51removes all intermediate object files. 52 53Many threads can be working at once, but each has its own working directory. 54When a thread finishes a build, it puts the output files into a result 55directory. 56 57The base directory used by buildman is normally '../<branch>', i.e. 58a directory higher than the source repository and named after the branch 59being built. 60 61Within the base directory, we have one subdirectory for each commit. Within 62that is one subdirectory for each board. Within that is the build output for 63that commit/board combination. 64 65Buildman also create working directories for each thread, in a .bm-work/ 66subdirectory in the base dir. 67 68As an example, say we are building branch 'us-net' for boards 'sandbox' and 69'seaboard', and say that us-net has two commits. We will have directories 70like this: 71 72us-net/ base directory 73 01_of_02_g4ed4ebc_net--Add-tftp-speed-/ 74 sandbox/ 75 u-boot.bin 76 seaboard/ 77 u-boot.bin 78 02_of_02_g4ed4ebc_net--Check-tftp-comp/ 79 sandbox/ 80 u-boot.bin 81 seaboard/ 82 u-boot.bin 83 .bm-work/ 84 00/ working directory for thread 0 (contains source checkout) 85 build/ build output 86 01/ working directory for thread 1 87 build/ build output 88 ... 89u-boot/ source directory 90 .git/ repository 91""" 92 93# Possible build outcomes 94OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4) 95 96# Translate a commit subject into a valid filename 97trans_valid_chars = string.maketrans("/: ", "---") 98 99 100def Mkdir(dirname): 101 """Make a directory if it doesn't already exist. 102 103 Args: 104 dirname: Directory to create 105 """ 106 try: 107 os.mkdir(dirname) 108 except OSError as err: 109 if err.errno == errno.EEXIST: 110 pass 111 else: 112 raise 113 114class BuilderJob: 115 """Holds information about a job to be performed by a thread 116 117 Members: 118 board: Board object to build 119 commits: List of commit options to build. 120 """ 121 def __init__(self): 122 self.board = None 123 self.commits = [] 124 125 126class ResultThread(threading.Thread): 127 """This thread processes results from builder threads. 128 129 It simply passes the results on to the builder. There is only one 130 result thread, and this helps to serialise the build output. 131 """ 132 def __init__(self, builder): 133 """Set up a new result thread 134 135 Args: 136 builder: Builder which will be sent each result 137 """ 138 threading.Thread.__init__(self) 139 self.builder = builder 140 141 def run(self): 142 """Called to start up the result thread. 143 144 We collect the next result job and pass it on to the build. 145 """ 146 while True: 147 result = self.builder.out_queue.get() 148 self.builder.ProcessResult(result) 149 self.builder.out_queue.task_done() 150 151 152class BuilderThread(threading.Thread): 153 """This thread builds U-Boot for a particular board. 154 155 An input queue provides each new job. We run 'make' to build U-Boot 156 and then pass the results on to the output queue. 157 158 Members: 159 builder: The builder which contains information we might need 160 thread_num: Our thread number (0-n-1), used to decide on a 161 temporary directory 162 """ 163 def __init__(self, builder, thread_num): 164 """Set up a new builder thread""" 165 threading.Thread.__init__(self) 166 self.builder = builder 167 self.thread_num = thread_num 168 169 def Make(self, commit, brd, stage, cwd, *args, **kwargs): 170 """Run 'make' on a particular commit and board. 171 172 The source code will already be checked out, so the 'commit' 173 argument is only for information. 174 175 Args: 176 commit: Commit object that is being built 177 brd: Board object that is being built 178 stage: Stage of the build. Valid stages are: 179 distclean - can be called to clean source 180 config - called to configure for a board 181 build - the main make invocation - it does the build 182 args: A list of arguments to pass to 'make' 183 kwargs: A list of keyword arguments to pass to command.RunPipe() 184 185 Returns: 186 CommandResult object 187 """ 188 return self.builder.do_make(commit, brd, stage, cwd, *args, 189 **kwargs) 190 191 def RunCommit(self, commit_upto, brd, work_dir, do_config, force_build, 192 force_build_failures): 193 """Build a particular commit. 194 195 If the build is already done, and we are not forcing a build, we skip 196 the build and just return the previously-saved results. 197 198 Args: 199 commit_upto: Commit number to build (0...n-1) 200 brd: Board object to build 201 work_dir: Directory to which the source will be checked out 202 do_config: True to run a make <board>_config on the source 203 force_build: Force a build even if one was previously done 204 force_build_failures: Force a bulid if the previous result showed 205 failure 206 207 Returns: 208 tuple containing: 209 - CommandResult object containing the results of the build 210 - boolean indicating whether 'make config' is still needed 211 """ 212 # Create a default result - it will be overwritte by the call to 213 # self.Make() below, in the event that we do a build. 214 result = command.CommandResult() 215 result.return_code = 0 216 out_dir = os.path.join(work_dir, 'build') 217 218 # Check if the job was already completed last time 219 done_file = self.builder.GetDoneFile(commit_upto, brd.target) 220 result.already_done = os.path.exists(done_file) 221 will_build = (force_build or force_build_failures or 222 not result.already_done) 223 if result.already_done and will_build: 224 # Get the return code from that build and use it 225 with open(done_file, 'r') as fd: 226 result.return_code = int(fd.readline()) 227 err_file = self.builder.GetErrFile(commit_upto, brd.target) 228 if os.path.exists(err_file) and os.stat(err_file).st_size: 229 result.stderr = 'bad' 230 elif not force_build: 231 # The build passed, so no need to build it again 232 will_build = False 233 234 if will_build: 235 # We are going to have to build it. First, get a toolchain 236 if not self.toolchain: 237 try: 238 self.toolchain = self.builder.toolchains.Select(brd.arch) 239 except ValueError as err: 240 result.return_code = 10 241 result.stdout = '' 242 result.stderr = str(err) 243 # TODO(sjg@chromium.org): This gets swallowed, but needs 244 # to be reported. 245 246 if self.toolchain: 247 # Checkout the right commit 248 if commit_upto is not None: 249 commit = self.builder.commits[commit_upto] 250 if self.builder.checkout: 251 git_dir = os.path.join(work_dir, '.git') 252 gitutil.Checkout(commit.hash, git_dir, work_dir, 253 force=True) 254 else: 255 commit = self.builder.commit # Ick, fix this for BuildCommits() 256 257 # Set up the environment and command line 258 env = self.toolchain.MakeEnvironment() 259 Mkdir(out_dir) 260 args = ['O=build', '-s'] 261 if self.builder.num_jobs is not None: 262 args.extend(['-j', str(self.builder.num_jobs)]) 263 config_args = ['%s_config' % brd.target] 264 config_out = '' 265 args.extend(self.builder.toolchains.GetMakeArguments(brd)) 266 267 # If we need to reconfigure, do that now 268 if do_config: 269 result = self.Make(commit, brd, 'distclean', work_dir, 270 'distclean', *args, env=env) 271 result = self.Make(commit, brd, 'config', work_dir, 272 *(args + config_args), env=env) 273 config_out = result.combined 274 do_config = False # No need to configure next time 275 if result.return_code == 0: 276 result = self.Make(commit, brd, 'build', work_dir, *args, 277 env=env) 278 result.stdout = config_out + result.stdout 279 else: 280 result.return_code = 1 281 result.stderr = 'No tool chain for %s\n' % brd.arch 282 result.already_done = False 283 284 result.toolchain = self.toolchain 285 result.brd = brd 286 result.commit_upto = commit_upto 287 result.out_dir = out_dir 288 return result, do_config 289 290 def _WriteResult(self, result, keep_outputs): 291 """Write a built result to the output directory. 292 293 Args: 294 result: CommandResult object containing result to write 295 keep_outputs: True to store the output binaries, False 296 to delete them 297 """ 298 # Fatal error 299 if result.return_code < 0: 300 return 301 302 # Aborted? 303 if result.stderr and 'No child processes' in result.stderr: 304 return 305 306 if result.already_done: 307 return 308 309 # Write the output and stderr 310 output_dir = self.builder._GetOutputDir(result.commit_upto) 311 Mkdir(output_dir) 312 build_dir = self.builder.GetBuildDir(result.commit_upto, 313 result.brd.target) 314 Mkdir(build_dir) 315 316 outfile = os.path.join(build_dir, 'log') 317 with open(outfile, 'w') as fd: 318 if result.stdout: 319 fd.write(result.stdout) 320 321 errfile = self.builder.GetErrFile(result.commit_upto, 322 result.brd.target) 323 if result.stderr: 324 with open(errfile, 'w') as fd: 325 fd.write(result.stderr) 326 elif os.path.exists(errfile): 327 os.remove(errfile) 328 329 if result.toolchain: 330 # Write the build result and toolchain information. 331 done_file = self.builder.GetDoneFile(result.commit_upto, 332 result.brd.target) 333 with open(done_file, 'w') as fd: 334 fd.write('%s' % result.return_code) 335 with open(os.path.join(build_dir, 'toolchain'), 'w') as fd: 336 print >>fd, 'gcc', result.toolchain.gcc 337 print >>fd, 'path', result.toolchain.path 338 print >>fd, 'cross', result.toolchain.cross 339 print >>fd, 'arch', result.toolchain.arch 340 fd.write('%s' % result.return_code) 341 342 with open(os.path.join(build_dir, 'toolchain'), 'w') as fd: 343 print >>fd, 'gcc', result.toolchain.gcc 344 print >>fd, 'path', result.toolchain.path 345 346 # Write out the image and function size information and an objdump 347 env = result.toolchain.MakeEnvironment() 348 lines = [] 349 for fname in ['u-boot', 'spl/u-boot-spl']: 350 cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname] 351 nm_result = command.RunPipe([cmd], capture=True, 352 capture_stderr=True, cwd=result.out_dir, 353 raise_on_error=False, env=env) 354 if nm_result.stdout: 355 nm = self.builder.GetFuncSizesFile(result.commit_upto, 356 result.brd.target, fname) 357 with open(nm, 'w') as fd: 358 print >>fd, nm_result.stdout, 359 360 cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname] 361 dump_result = command.RunPipe([cmd], capture=True, 362 capture_stderr=True, cwd=result.out_dir, 363 raise_on_error=False, env=env) 364 rodata_size = '' 365 if dump_result.stdout: 366 objdump = self.builder.GetObjdumpFile(result.commit_upto, 367 result.brd.target, fname) 368 with open(objdump, 'w') as fd: 369 print >>fd, dump_result.stdout, 370 for line in dump_result.stdout.splitlines(): 371 fields = line.split() 372 if len(fields) > 5 and fields[1] == '.rodata': 373 rodata_size = fields[2] 374 375 cmd = ['%ssize' % self.toolchain.cross, fname] 376 size_result = command.RunPipe([cmd], capture=True, 377 capture_stderr=True, cwd=result.out_dir, 378 raise_on_error=False, env=env) 379 if size_result.stdout: 380 lines.append(size_result.stdout.splitlines()[1] + ' ' + 381 rodata_size) 382 383 # Write out the image sizes file. This is similar to the output 384 # of binutil's 'size' utility, but it omits the header line and 385 # adds an additional hex value at the end of each line for the 386 # rodata size 387 if len(lines): 388 sizes = self.builder.GetSizesFile(result.commit_upto, 389 result.brd.target) 390 with open(sizes, 'w') as fd: 391 print >>fd, '\n'.join(lines) 392 393 # Now write the actual build output 394 if keep_outputs: 395 patterns = ['u-boot', '*.bin', 'u-boot.dtb', '*.map', 396 'include/autoconf.mk', 'spl/u-boot-spl', 397 'spl/u-boot-spl.bin'] 398 for pattern in patterns: 399 file_list = glob.glob(os.path.join(result.out_dir, pattern)) 400 for fname in file_list: 401 shutil.copy(fname, build_dir) 402 403 404 def RunJob(self, job): 405 """Run a single job 406 407 A job consists of a building a list of commits for a particular board. 408 409 Args: 410 job: Job to build 411 """ 412 brd = job.board 413 work_dir = self.builder.GetThreadDir(self.thread_num) 414 self.toolchain = None 415 if job.commits: 416 # Run 'make board_config' on the first commit 417 do_config = True 418 commit_upto = 0 419 force_build = False 420 for commit_upto in range(0, len(job.commits), job.step): 421 result, request_config = self.RunCommit(commit_upto, brd, 422 work_dir, do_config, 423 force_build or self.builder.force_build, 424 self.builder.force_build_failures) 425 failed = result.return_code or result.stderr 426 did_config = do_config 427 if failed and not do_config: 428 # If our incremental build failed, try building again 429 # with a reconfig. 430 if self.builder.force_config_on_failure: 431 result, request_config = self.RunCommit(commit_upto, 432 brd, work_dir, True, True, False) 433 did_config = True 434 if not self.builder.force_reconfig: 435 do_config = request_config 436 437 # If we built that commit, then config is done. But if we got 438 # an warning, reconfig next time to force it to build the same 439 # files that created warnings this time. Otherwise an 440 # incremental build may not build the same file, and we will 441 # think that the warning has gone away. 442 # We could avoid this by using -Werror everywhere... 443 # For errors, the problem doesn't happen, since presumably 444 # the build stopped and didn't generate output, so will retry 445 # that file next time. So we could detect warnings and deal 446 # with them specially here. For now, we just reconfigure if 447 # anything goes work. 448 # Of course this is substantially slower if there are build 449 # errors/warnings (e.g. 2-3x slower even if only 10% of builds 450 # have problems). 451 if (failed and not result.already_done and not did_config and 452 self.builder.force_config_on_failure): 453 # If this build failed, try the next one with a 454 # reconfigure. 455 # Sometimes if the board_config.h file changes it can mess 456 # with dependencies, and we get: 457 # make: *** No rule to make target `include/autoconf.mk', 458 # needed by `depend'. 459 do_config = True 460 force_build = True 461 else: 462 force_build = False 463 if self.builder.force_config_on_failure: 464 if failed: 465 do_config = True 466 result.commit_upto = commit_upto 467 if result.return_code < 0: 468 raise ValueError('Interrupt') 469 470 # We have the build results, so output the result 471 self._WriteResult(result, job.keep_outputs) 472 self.builder.out_queue.put(result) 473 else: 474 # Just build the currently checked-out build 475 result = self.RunCommit(None, True) 476 result.commit_upto = self.builder.upto 477 self.builder.out_queue.put(result) 478 479 def run(self): 480 """Our thread's run function 481 482 This thread picks a job from the queue, runs it, and then goes to the 483 next job. 484 """ 485 alive = True 486 while True: 487 job = self.builder.queue.get() 488 try: 489 if self.builder.active and alive: 490 self.RunJob(job) 491 except Exception as err: 492 alive = False 493 print err 494 self.builder.queue.task_done() 495 496 497class Builder: 498 """Class for building U-Boot for a particular commit. 499 500 Public members: (many should ->private) 501 active: True if the builder is active and has not been stopped 502 already_done: Number of builds already completed 503 base_dir: Base directory to use for builder 504 checkout: True to check out source, False to skip that step. 505 This is used for testing. 506 col: terminal.Color() object 507 count: Number of commits to build 508 do_make: Method to call to invoke Make 509 fail: Number of builds that failed due to error 510 force_build: Force building even if a build already exists 511 force_config_on_failure: If a commit fails for a board, disable 512 incremental building for the next commit we build for that 513 board, so that we will see all warnings/errors again. 514 force_build_failures: If a previously-built build (i.e. built on 515 a previous run of buildman) is marked as failed, rebuild it. 516 git_dir: Git directory containing source repository 517 last_line_len: Length of the last line we printed (used for erasing 518 it with new progress information) 519 num_jobs: Number of jobs to run at once (passed to make as -j) 520 num_threads: Number of builder threads to run 521 out_queue: Queue of results to process 522 re_make_err: Compiled regular expression for ignore_lines 523 queue: Queue of jobs to run 524 threads: List of active threads 525 toolchains: Toolchains object to use for building 526 upto: Current commit number we are building (0.count-1) 527 warned: Number of builds that produced at least one warning 528 force_reconfig: Reconfigure U-Boot on each comiit. This disables 529 incremental building, where buildman reconfigures on the first 530 commit for a baord, and then just does an incremental build for 531 the following commits. In fact buildman will reconfigure and 532 retry for any failing commits, so generally the only effect of 533 this option is to slow things down. 534 535 Private members: 536 _base_board_dict: Last-summarised Dict of boards 537 _base_err_lines: Last-summarised list of errors 538 _build_period_us: Time taken for a single build (float object). 539 _complete_delay: Expected delay until completion (timedelta) 540 _next_delay_update: Next time we plan to display a progress update 541 (datatime) 542 _show_unknown: Show unknown boards (those not built) in summary 543 _timestamps: List of timestamps for the completion of the last 544 last _timestamp_count builds. Each is a datetime object. 545 _timestamp_count: Number of timestamps to keep in our list. 546 _working_dir: Base working directory containing all threads 547 """ 548 class Outcome: 549 """Records a build outcome for a single make invocation 550 551 Public Members: 552 rc: Outcome value (OUTCOME_...) 553 err_lines: List of error lines or [] if none 554 sizes: Dictionary of image size information, keyed by filename 555 - Each value is itself a dictionary containing 556 values for 'text', 'data' and 'bss', being the integer 557 size in bytes of each section. 558 func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each 559 value is itself a dictionary: 560 key: function name 561 value: Size of function in bytes 562 """ 563 def __init__(self, rc, err_lines, sizes, func_sizes): 564 self.rc = rc 565 self.err_lines = err_lines 566 self.sizes = sizes 567 self.func_sizes = func_sizes 568 569 def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, 570 checkout=True, show_unknown=True, step=1): 571 """Create a new Builder object 572 573 Args: 574 toolchains: Toolchains object to use for building 575 base_dir: Base directory to use for builder 576 git_dir: Git directory containing source repository 577 num_threads: Number of builder threads to run 578 num_jobs: Number of jobs to run at once (passed to make as -j) 579 checkout: True to check out source, False to skip that step. 580 This is used for testing. 581 show_unknown: Show unknown boards (those not built) in summary 582 step: 1 to process every commit, n to process every nth commit 583 """ 584 self.toolchains = toolchains 585 self.base_dir = base_dir 586 self._working_dir = os.path.join(base_dir, '.bm-work') 587 self.threads = [] 588 self.active = True 589 self.do_make = self.Make 590 self.checkout = checkout 591 self.num_threads = num_threads 592 self.num_jobs = num_jobs 593 self.already_done = 0 594 self.force_build = False 595 self.git_dir = git_dir 596 self._show_unknown = show_unknown 597 self._timestamp_count = 10 598 self._build_period_us = None 599 self._complete_delay = None 600 self._next_delay_update = datetime.now() 601 self.force_config_on_failure = True 602 self.force_build_failures = False 603 self.force_reconfig = False 604 self._step = step 605 606 self.col = terminal.Color() 607 608 self.queue = Queue.Queue() 609 self.out_queue = Queue.Queue() 610 for i in range(self.num_threads): 611 t = BuilderThread(self, i) 612 t.setDaemon(True) 613 t.start() 614 self.threads.append(t) 615 616 self.last_line_len = 0 617 t = ResultThread(self) 618 t.setDaemon(True) 619 t.start() 620 self.threads.append(t) 621 622 ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)'] 623 self.re_make_err = re.compile('|'.join(ignore_lines)) 624 625 def __del__(self): 626 """Get rid of all threads created by the builder""" 627 for t in self.threads: 628 del t 629 630 def _AddTimestamp(self): 631 """Add a new timestamp to the list and record the build period. 632 633 The build period is the length of time taken to perform a single 634 build (one board, one commit). 635 """ 636 now = datetime.now() 637 self._timestamps.append(now) 638 count = len(self._timestamps) 639 delta = self._timestamps[-1] - self._timestamps[0] 640 seconds = delta.total_seconds() 641 642 # If we have enough data, estimate build period (time taken for a 643 # single build) and therefore completion time. 644 if count > 1 and self._next_delay_update < now: 645 self._next_delay_update = now + timedelta(seconds=2) 646 if seconds > 0: 647 self._build_period = float(seconds) / count 648 todo = self.count - self.upto 649 self._complete_delay = timedelta(microseconds= 650 self._build_period * todo * 1000000) 651 # Round it 652 self._complete_delay -= timedelta( 653 microseconds=self._complete_delay.microseconds) 654 655 if seconds > 60: 656 self._timestamps.popleft() 657 count -= 1 658 659 def ClearLine(self, length): 660 """Clear any characters on the current line 661 662 Make way for a new line of length 'length', by outputting enough 663 spaces to clear out the old line. Then remember the new length for 664 next time. 665 666 Args: 667 length: Length of new line, in characters 668 """ 669 if length < self.last_line_len: 670 print ' ' * (self.last_line_len - length), 671 print '\r', 672 self.last_line_len = length 673 sys.stdout.flush() 674 675 def SelectCommit(self, commit, checkout=True): 676 """Checkout the selected commit for this build 677 """ 678 self.commit = commit 679 if checkout and self.checkout: 680 gitutil.Checkout(commit.hash) 681 682 def Make(self, commit, brd, stage, cwd, *args, **kwargs): 683 """Run make 684 685 Args: 686 commit: Commit object that is being built 687 brd: Board object that is being built 688 stage: Stage that we are at (distclean, config, build) 689 cwd: Directory where make should be run 690 args: Arguments to pass to make 691 kwargs: Arguments to pass to command.RunPipe() 692 """ 693 cmd = ['make'] + list(args) 694 result = command.RunPipe([cmd], capture=True, capture_stderr=True, 695 cwd=cwd, raise_on_error=False, **kwargs) 696 return result 697 698 def ProcessResult(self, result): 699 """Process the result of a build, showing progress information 700 701 Args: 702 result: A CommandResult object 703 """ 704 col = terminal.Color() 705 if result: 706 target = result.brd.target 707 708 if result.return_code < 0: 709 self.active = False 710 command.StopAll() 711 return 712 713 self.upto += 1 714 if result.return_code != 0: 715 self.fail += 1 716 elif result.stderr: 717 self.warned += 1 718 if result.already_done: 719 self.already_done += 1 720 else: 721 target = '(starting)' 722 723 # Display separate counts for ok, warned and fail 724 ok = self.upto - self.warned - self.fail 725 line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok) 726 line += self.col.Color(self.col.YELLOW, '%5d' % self.warned) 727 line += self.col.Color(self.col.RED, '%5d' % self.fail) 728 729 name = ' /%-5d ' % self.count 730 731 # Add our current completion time estimate 732 self._AddTimestamp() 733 if self._complete_delay: 734 name += '%s : ' % self._complete_delay 735 # When building all boards for a commit, we can print a commit 736 # progress message. 737 if result and result.commit_upto is None: 738 name += 'commit %2d/%-3d' % (self.commit_upto + 1, 739 self.commit_count) 740 741 name += target 742 print line + name, 743 length = 13 + len(name) 744 self.ClearLine(length) 745 746 def _GetOutputDir(self, commit_upto): 747 """Get the name of the output directory for a commit number 748 749 The output directory is typically .../<branch>/<commit>. 750 751 Args: 752 commit_upto: Commit number to use (0..self.count-1) 753 """ 754 commit = self.commits[commit_upto] 755 subject = commit.subject.translate(trans_valid_chars) 756 commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1, 757 self.commit_count, commit.hash, subject[:20])) 758 output_dir = os.path.join(self.base_dir, commit_dir) 759 return output_dir 760 761 def GetBuildDir(self, commit_upto, target): 762 """Get the name of the build directory for a commit number 763 764 The build directory is typically .../<branch>/<commit>/<target>. 765 766 Args: 767 commit_upto: Commit number to use (0..self.count-1) 768 target: Target name 769 """ 770 output_dir = self._GetOutputDir(commit_upto) 771 return os.path.join(output_dir, target) 772 773 def GetDoneFile(self, commit_upto, target): 774 """Get the name of the done file for a commit number 775 776 Args: 777 commit_upto: Commit number to use (0..self.count-1) 778 target: Target name 779 """ 780 return os.path.join(self.GetBuildDir(commit_upto, target), 'done') 781 782 def GetSizesFile(self, commit_upto, target): 783 """Get the name of the sizes file for a commit number 784 785 Args: 786 commit_upto: Commit number to use (0..self.count-1) 787 target: Target name 788 """ 789 return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes') 790 791 def GetFuncSizesFile(self, commit_upto, target, elf_fname): 792 """Get the name of the funcsizes file for a commit number and ELF file 793 794 Args: 795 commit_upto: Commit number to use (0..self.count-1) 796 target: Target name 797 elf_fname: Filename of elf image 798 """ 799 return os.path.join(self.GetBuildDir(commit_upto, target), 800 '%s.sizes' % elf_fname.replace('/', '-')) 801 802 def GetObjdumpFile(self, commit_upto, target, elf_fname): 803 """Get the name of the objdump file for a commit number and ELF file 804 805 Args: 806 commit_upto: Commit number to use (0..self.count-1) 807 target: Target name 808 elf_fname: Filename of elf image 809 """ 810 return os.path.join(self.GetBuildDir(commit_upto, target), 811 '%s.objdump' % elf_fname.replace('/', '-')) 812 813 def GetErrFile(self, commit_upto, target): 814 """Get the name of the err file for a commit number 815 816 Args: 817 commit_upto: Commit number to use (0..self.count-1) 818 target: Target name 819 """ 820 output_dir = self.GetBuildDir(commit_upto, target) 821 return os.path.join(output_dir, 'err') 822 823 def FilterErrors(self, lines): 824 """Filter out errors in which we have no interest 825 826 We should probably use map(). 827 828 Args: 829 lines: List of error lines, each a string 830 Returns: 831 New list with only interesting lines included 832 """ 833 out_lines = [] 834 for line in lines: 835 if not self.re_make_err.search(line): 836 out_lines.append(line) 837 return out_lines 838 839 def ReadFuncSizes(self, fname, fd): 840 """Read function sizes from the output of 'nm' 841 842 Args: 843 fd: File containing data to read 844 fname: Filename we are reading from (just for errors) 845 846 Returns: 847 Dictionary containing size of each function in bytes, indexed by 848 function name. 849 """ 850 sym = {} 851 for line in fd.readlines(): 852 try: 853 size, type, name = line[:-1].split() 854 except: 855 print "Invalid line in file '%s': '%s'" % (fname, line[:-1]) 856 continue 857 if type in 'tTdDbB': 858 # function names begin with '.' on 64-bit powerpc 859 if '.' in name[1:]: 860 name = 'static.' + name.split('.')[0] 861 sym[name] = sym.get(name, 0) + int(size, 16) 862 return sym 863 864 def GetBuildOutcome(self, commit_upto, target, read_func_sizes): 865 """Work out the outcome of a build. 866 867 Args: 868 commit_upto: Commit number to check (0..n-1) 869 target: Target board to check 870 read_func_sizes: True to read function size information 871 872 Returns: 873 Outcome object 874 """ 875 done_file = self.GetDoneFile(commit_upto, target) 876 sizes_file = self.GetSizesFile(commit_upto, target) 877 sizes = {} 878 func_sizes = {} 879 if os.path.exists(done_file): 880 with open(done_file, 'r') as fd: 881 return_code = int(fd.readline()) 882 err_lines = [] 883 err_file = self.GetErrFile(commit_upto, target) 884 if os.path.exists(err_file): 885 with open(err_file, 'r') as fd: 886 err_lines = self.FilterErrors(fd.readlines()) 887 888 # Decide whether the build was ok, failed or created warnings 889 if return_code: 890 rc = OUTCOME_ERROR 891 elif len(err_lines): 892 rc = OUTCOME_WARNING 893 else: 894 rc = OUTCOME_OK 895 896 # Convert size information to our simple format 897 if os.path.exists(sizes_file): 898 with open(sizes_file, 'r') as fd: 899 for line in fd.readlines(): 900 values = line.split() 901 rodata = 0 902 if len(values) > 6: 903 rodata = int(values[6], 16) 904 size_dict = { 905 'all' : int(values[0]) + int(values[1]) + 906 int(values[2]), 907 'text' : int(values[0]) - rodata, 908 'data' : int(values[1]), 909 'bss' : int(values[2]), 910 'rodata' : rodata, 911 } 912 sizes[values[5]] = size_dict 913 914 if read_func_sizes: 915 pattern = self.GetFuncSizesFile(commit_upto, target, '*') 916 for fname in glob.glob(pattern): 917 with open(fname, 'r') as fd: 918 dict_name = os.path.basename(fname).replace('.sizes', 919 '') 920 func_sizes[dict_name] = self.ReadFuncSizes(fname, fd) 921 922 return Builder.Outcome(rc, err_lines, sizes, func_sizes) 923 924 return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}) 925 926 def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes): 927 """Calculate a summary of the results of building a commit. 928 929 Args: 930 board_selected: Dict containing boards to summarise 931 commit_upto: Commit number to summarize (0..self.count-1) 932 read_func_sizes: True to read function size information 933 934 Returns: 935 Tuple: 936 Dict containing boards which passed building this commit. 937 keyed by board.target 938 List containing a summary of error/warning lines 939 """ 940 board_dict = {} 941 err_lines_summary = [] 942 943 for board in boards_selected.itervalues(): 944 outcome = self.GetBuildOutcome(commit_upto, board.target, 945 read_func_sizes) 946 board_dict[board.target] = outcome 947 for err in outcome.err_lines: 948 if err and not err.rstrip() in err_lines_summary: 949 err_lines_summary.append(err.rstrip()) 950 return board_dict, err_lines_summary 951 952 def AddOutcome(self, board_dict, arch_list, changes, char, color): 953 """Add an output to our list of outcomes for each architecture 954 955 This simple function adds failing boards (changes) to the 956 relevant architecture string, so we can print the results out 957 sorted by architecture. 958 959 Args: 960 board_dict: Dict containing all boards 961 arch_list: Dict keyed by arch name. Value is a string containing 962 a list of board names which failed for that arch. 963 changes: List of boards to add to arch_list 964 color: terminal.Colour object 965 """ 966 done_arch = {} 967 for target in changes: 968 if target in board_dict: 969 arch = board_dict[target].arch 970 else: 971 arch = 'unknown' 972 str = self.col.Color(color, ' ' + target) 973 if not arch in done_arch: 974 str = self.col.Color(color, char) + ' ' + str 975 done_arch[arch] = True 976 if not arch in arch_list: 977 arch_list[arch] = str 978 else: 979 arch_list[arch] += str 980 981 982 def ColourNum(self, num): 983 color = self.col.RED if num > 0 else self.col.GREEN 984 if num == 0: 985 return '0' 986 return self.col.Color(color, str(num)) 987 988 def ResetResultSummary(self, board_selected): 989 """Reset the results summary ready for use. 990 991 Set up the base board list to be all those selected, and set the 992 error lines to empty. 993 994 Following this, calls to PrintResultSummary() will use this 995 information to work out what has changed. 996 997 Args: 998 board_selected: Dict containing boards to summarise, keyed by 999 board.target 1000 """ 1001 self._base_board_dict = {} 1002 for board in board_selected: 1003 self._base_board_dict[board] = Builder.Outcome(0, [], [], {}) 1004 self._base_err_lines = [] 1005 1006 def PrintFuncSizeDetail(self, fname, old, new): 1007 grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0 1008 delta, common = [], {} 1009 1010 for a in old: 1011 if a in new: 1012 common[a] = 1 1013 1014 for name in old: 1015 if name not in common: 1016 remove += 1 1017 down += old[name] 1018 delta.append([-old[name], name]) 1019 1020 for name in new: 1021 if name not in common: 1022 add += 1 1023 up += new[name] 1024 delta.append([new[name], name]) 1025 1026 for name in common: 1027 diff = new.get(name, 0) - old.get(name, 0) 1028 if diff > 0: 1029 grow, up = grow + 1, up + diff 1030 elif diff < 0: 1031 shrink, down = shrink + 1, down - diff 1032 delta.append([diff, name]) 1033 1034 delta.sort() 1035 delta.reverse() 1036 1037 args = [add, -remove, grow, -shrink, up, -down, up - down] 1038 if max(args) == 0: 1039 return 1040 args = [self.ColourNum(x) for x in args] 1041 indent = ' ' * 15 1042 print ('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' % 1043 tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args)) 1044 print '%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new', 1045 'delta') 1046 for diff, name in delta: 1047 if diff: 1048 color = self.col.RED if diff > 0 else self.col.GREEN 1049 msg = '%s %-38s %7s %7s %+7d' % (indent, name, 1050 old.get(name, '-'), new.get(name,'-'), diff) 1051 print self.col.Color(color, msg) 1052 1053 1054 def PrintSizeDetail(self, target_list, show_bloat): 1055 """Show details size information for each board 1056 1057 Args: 1058 target_list: List of targets, each a dict containing: 1059 'target': Target name 1060 'total_diff': Total difference in bytes across all areas 1061 <part_name>: Difference for that part 1062 show_bloat: Show detail for each function 1063 """ 1064 targets_by_diff = sorted(target_list, reverse=True, 1065 key=lambda x: x['_total_diff']) 1066 for result in targets_by_diff: 1067 printed_target = False 1068 for name in sorted(result): 1069 diff = result[name] 1070 if name.startswith('_'): 1071 continue 1072 if diff != 0: 1073 color = self.col.RED if diff > 0 else self.col.GREEN 1074 msg = ' %s %+d' % (name, diff) 1075 if not printed_target: 1076 print '%10s %-15s:' % ('', result['_target']), 1077 printed_target = True 1078 print self.col.Color(color, msg), 1079 if printed_target: 1080 print 1081 if show_bloat: 1082 target = result['_target'] 1083 outcome = result['_outcome'] 1084 base_outcome = self._base_board_dict[target] 1085 for fname in outcome.func_sizes: 1086 self.PrintFuncSizeDetail(fname, 1087 base_outcome.func_sizes[fname], 1088 outcome.func_sizes[fname]) 1089 1090 1091 def PrintSizeSummary(self, board_selected, board_dict, show_detail, 1092 show_bloat): 1093 """Print a summary of image sizes broken down by section. 1094 1095 The summary takes the form of one line per architecture. The 1096 line contains deltas for each of the sections (+ means the section 1097 got bigger, - means smaller). The nunmbers are the average number 1098 of bytes that a board in this section increased by. 1099 1100 For example: 1101 powerpc: (622 boards) text -0.0 1102 arm: (285 boards) text -0.0 1103 nds32: (3 boards) text -8.0 1104 1105 Args: 1106 board_selected: Dict containing boards to summarise, keyed by 1107 board.target 1108 board_dict: Dict containing boards for which we built this 1109 commit, keyed by board.target. The value is an Outcome object. 1110 show_detail: Show detail for each board 1111 show_bloat: Show detail for each function 1112 """ 1113 arch_list = {} 1114 arch_count = {} 1115 1116 # Calculate changes in size for different image parts 1117 # The previous sizes are in Board.sizes, for each board 1118 for target in board_dict: 1119 if target not in board_selected: 1120 continue 1121 base_sizes = self._base_board_dict[target].sizes 1122 outcome = board_dict[target] 1123 sizes = outcome.sizes 1124 1125 # Loop through the list of images, creating a dict of size 1126 # changes for each image/part. We end up with something like 1127 # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4} 1128 # which means that U-Boot data increased by 5 bytes and SPL 1129 # text decreased by 4. 1130 err = {'_target' : target} 1131 for image in sizes: 1132 if image in base_sizes: 1133 base_image = base_sizes[image] 1134 # Loop through the text, data, bss parts 1135 for part in sorted(sizes[image]): 1136 diff = sizes[image][part] - base_image[part] 1137 col = None 1138 if diff: 1139 if image == 'u-boot': 1140 name = part 1141 else: 1142 name = image + ':' + part 1143 err[name] = diff 1144 arch = board_selected[target].arch 1145 if not arch in arch_count: 1146 arch_count[arch] = 1 1147 else: 1148 arch_count[arch] += 1 1149 if not sizes: 1150 pass # Only add to our list when we have some stats 1151 elif not arch in arch_list: 1152 arch_list[arch] = [err] 1153 else: 1154 arch_list[arch].append(err) 1155 1156 # We now have a list of image size changes sorted by arch 1157 # Print out a summary of these 1158 for arch, target_list in arch_list.iteritems(): 1159 # Get total difference for each type 1160 totals = {} 1161 for result in target_list: 1162 total = 0 1163 for name, diff in result.iteritems(): 1164 if name.startswith('_'): 1165 continue 1166 total += diff 1167 if name in totals: 1168 totals[name] += diff 1169 else: 1170 totals[name] = diff 1171 result['_total_diff'] = total 1172 result['_outcome'] = board_dict[result['_target']] 1173 1174 count = len(target_list) 1175 printed_arch = False 1176 for name in sorted(totals): 1177 diff = totals[name] 1178 if diff: 1179 # Display the average difference in this name for this 1180 # architecture 1181 avg_diff = float(diff) / count 1182 color = self.col.RED if avg_diff > 0 else self.col.GREEN 1183 msg = ' %s %+1.1f' % (name, avg_diff) 1184 if not printed_arch: 1185 print '%10s: (for %d/%d boards)' % (arch, count, 1186 arch_count[arch]), 1187 printed_arch = True 1188 print self.col.Color(color, msg), 1189 1190 if printed_arch: 1191 print 1192 if show_detail: 1193 self.PrintSizeDetail(target_list, show_bloat) 1194 1195 1196 def PrintResultSummary(self, board_selected, board_dict, err_lines, 1197 show_sizes, show_detail, show_bloat): 1198 """Compare results with the base results and display delta. 1199 1200 Only boards mentioned in board_selected will be considered. This 1201 function is intended to be called repeatedly with the results of 1202 each commit. It therefore shows a 'diff' between what it saw in 1203 the last call and what it sees now. 1204 1205 Args: 1206 board_selected: Dict containing boards to summarise, keyed by 1207 board.target 1208 board_dict: Dict containing boards for which we built this 1209 commit, keyed by board.target. The value is an Outcome object. 1210 err_lines: A list of errors for this commit, or [] if there is 1211 none, or we don't want to print errors 1212 show_sizes: Show image size deltas 1213 show_detail: Show detail for each board 1214 show_bloat: Show detail for each function 1215 """ 1216 better = [] # List of boards fixed since last commit 1217 worse = [] # List of new broken boards since last commit 1218 new = [] # List of boards that didn't exist last time 1219 unknown = [] # List of boards that were not built 1220 1221 for target in board_dict: 1222 if target not in board_selected: 1223 continue 1224 1225 # If the board was built last time, add its outcome to a list 1226 if target in self._base_board_dict: 1227 base_outcome = self._base_board_dict[target].rc 1228 outcome = board_dict[target] 1229 if outcome.rc == OUTCOME_UNKNOWN: 1230 unknown.append(target) 1231 elif outcome.rc < base_outcome: 1232 better.append(target) 1233 elif outcome.rc > base_outcome: 1234 worse.append(target) 1235 else: 1236 new.append(target) 1237 1238 # Get a list of errors that have appeared, and disappeared 1239 better_err = [] 1240 worse_err = [] 1241 for line in err_lines: 1242 if line not in self._base_err_lines: 1243 worse_err.append('+' + line) 1244 for line in self._base_err_lines: 1245 if line not in err_lines: 1246 better_err.append('-' + line) 1247 1248 # Display results by arch 1249 if better or worse or unknown or new or worse_err or better_err: 1250 arch_list = {} 1251 self.AddOutcome(board_selected, arch_list, better, '', 1252 self.col.GREEN) 1253 self.AddOutcome(board_selected, arch_list, worse, '+', 1254 self.col.RED) 1255 self.AddOutcome(board_selected, arch_list, new, '*', self.col.BLUE) 1256 if self._show_unknown: 1257 self.AddOutcome(board_selected, arch_list, unknown, '?', 1258 self.col.MAGENTA) 1259 for arch, target_list in arch_list.iteritems(): 1260 print '%10s: %s' % (arch, target_list) 1261 if better_err: 1262 print self.col.Color(self.col.GREEN, '\n'.join(better_err)) 1263 if worse_err: 1264 print self.col.Color(self.col.RED, '\n'.join(worse_err)) 1265 1266 if show_sizes: 1267 self.PrintSizeSummary(board_selected, board_dict, show_detail, 1268 show_bloat) 1269 1270 # Save our updated information for the next call to this function 1271 self._base_board_dict = board_dict 1272 self._base_err_lines = err_lines 1273 1274 # Get a list of boards that did not get built, if needed 1275 not_built = [] 1276 for board in board_selected: 1277 if not board in board_dict: 1278 not_built.append(board) 1279 if not_built: 1280 print "Boards not built (%d): %s" % (len(not_built), 1281 ', '.join(not_built)) 1282 1283 1284 def ShowSummary(self, commits, board_selected, show_errors, show_sizes, 1285 show_detail, show_bloat): 1286 """Show a build summary for U-Boot for a given board list. 1287 1288 Reset the result summary, then repeatedly call GetResultSummary on 1289 each commit's results, then display the differences we see. 1290 1291 Args: 1292 commit: Commit objects to summarise 1293 board_selected: Dict containing boards to summarise 1294 show_errors: Show errors that occured 1295 show_sizes: Show size deltas 1296 show_detail: Show detail for each board 1297 show_bloat: Show detail for each function 1298 """ 1299 self.commit_count = len(commits) 1300 self.commits = commits 1301 self.ResetResultSummary(board_selected) 1302 1303 for commit_upto in range(0, self.commit_count, self._step): 1304 board_dict, err_lines = self.GetResultSummary(board_selected, 1305 commit_upto, read_func_sizes=show_bloat) 1306 msg = '%02d: %s' % (commit_upto + 1, commits[commit_upto].subject) 1307 print self.col.Color(self.col.BLUE, msg) 1308 self.PrintResultSummary(board_selected, board_dict, 1309 err_lines if show_errors else [], show_sizes, show_detail, 1310 show_bloat) 1311 1312 1313 def SetupBuild(self, board_selected, commits): 1314 """Set up ready to start a build. 1315 1316 Args: 1317 board_selected: Selected boards to build 1318 commits: Selected commits to build 1319 """ 1320 # First work out how many commits we will build 1321 count = (len(commits) + self._step - 1) / self._step 1322 self.count = len(board_selected) * count 1323 self.upto = self.warned = self.fail = 0 1324 self._timestamps = collections.deque() 1325 1326 def BuildBoardsForCommit(self, board_selected, keep_outputs): 1327 """Build all boards for a single commit""" 1328 self.SetupBuild(board_selected) 1329 self.count = len(board_selected) 1330 for brd in board_selected.itervalues(): 1331 job = BuilderJob() 1332 job.board = brd 1333 job.commits = None 1334 job.keep_outputs = keep_outputs 1335 self.queue.put(brd) 1336 1337 self.queue.join() 1338 self.out_queue.join() 1339 print 1340 self.ClearLine(0) 1341 1342 def BuildCommits(self, commits, board_selected, show_errors, keep_outputs): 1343 """Build all boards for all commits (non-incremental)""" 1344 self.commit_count = len(commits) 1345 1346 self.ResetResultSummary(board_selected) 1347 for self.commit_upto in range(self.commit_count): 1348 self.SelectCommit(commits[self.commit_upto]) 1349 self.SelectOutputDir() 1350 Mkdir(self.output_dir) 1351 1352 self.BuildBoardsForCommit(board_selected, keep_outputs) 1353 board_dict, err_lines = self.GetResultSummary() 1354 self.PrintResultSummary(board_selected, board_dict, 1355 err_lines if show_errors else []) 1356 1357 if self.already_done: 1358 print '%d builds already done' % self.already_done 1359 1360 def GetThreadDir(self, thread_num): 1361 """Get the directory path to the working dir for a thread. 1362 1363 Args: 1364 thread_num: Number of thread to check. 1365 """ 1366 return os.path.join(self._working_dir, '%02d' % thread_num) 1367 1368 def _PrepareThread(self, thread_num): 1369 """Prepare the working directory for a thread. 1370 1371 This clones or fetches the repo into the thread's work directory. 1372 1373 Args: 1374 thread_num: Thread number (0, 1, ...) 1375 """ 1376 thread_dir = self.GetThreadDir(thread_num) 1377 Mkdir(thread_dir) 1378 git_dir = os.path.join(thread_dir, '.git') 1379 1380 # Clone the repo if it doesn't already exist 1381 # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so 1382 # we have a private index but uses the origin repo's contents? 1383 if self.git_dir: 1384 src_dir = os.path.abspath(self.git_dir) 1385 if os.path.exists(git_dir): 1386 gitutil.Fetch(git_dir, thread_dir) 1387 else: 1388 print 'Cloning repo for thread %d' % thread_num 1389 gitutil.Clone(src_dir, thread_dir) 1390 1391 def _PrepareWorkingSpace(self, max_threads): 1392 """Prepare the working directory for use. 1393 1394 Set up the git repo for each thread. 1395 1396 Args: 1397 max_threads: Maximum number of threads we expect to need. 1398 """ 1399 Mkdir(self._working_dir) 1400 for thread in range(max_threads): 1401 self._PrepareThread(thread) 1402 1403 def _PrepareOutputSpace(self): 1404 """Get the output directories ready to receive files. 1405 1406 We delete any output directories which look like ones we need to 1407 create. Having left over directories is confusing when the user wants 1408 to check the output manually. 1409 """ 1410 dir_list = [] 1411 for commit_upto in range(self.commit_count): 1412 dir_list.append(self._GetOutputDir(commit_upto)) 1413 1414 for dirname in glob.glob(os.path.join(self.base_dir, '*')): 1415 if dirname not in dir_list: 1416 shutil.rmtree(dirname) 1417 1418 def BuildBoards(self, commits, board_selected, show_errors, keep_outputs): 1419 """Build all commits for a list of boards 1420 1421 Args: 1422 commits: List of commits to be build, each a Commit object 1423 boards_selected: Dict of selected boards, key is target name, 1424 value is Board object 1425 show_errors: True to show summarised error/warning info 1426 keep_outputs: True to save build output files 1427 """ 1428 self.commit_count = len(commits) 1429 self.commits = commits 1430 1431 self.ResetResultSummary(board_selected) 1432 Mkdir(self.base_dir) 1433 self._PrepareWorkingSpace(min(self.num_threads, len(board_selected))) 1434 self._PrepareOutputSpace() 1435 self.SetupBuild(board_selected, commits) 1436 self.ProcessResult(None) 1437 1438 # Create jobs to build all commits for each board 1439 for brd in board_selected.itervalues(): 1440 job = BuilderJob() 1441 job.board = brd 1442 job.commits = commits 1443 job.keep_outputs = keep_outputs 1444 job.step = self._step 1445 self.queue.put(job) 1446 1447 # Wait until all jobs are started 1448 self.queue.join() 1449 1450 # Wait until we have processed all output 1451 self.out_queue.join() 1452 print 1453 self.ClearLine(0) 1454