1# Copyright (c) 2013 The Chromium OS Authors. 2# 3# Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com> 4# 5# SPDX-License-Identifier: GPL-2.0+ 6# 7 8import collections 9import errno 10from datetime import datetime, timedelta 11import glob 12import os 13import re 14import Queue 15import shutil 16import string 17import sys 18import threading 19import time 20 21import command 22import gitutil 23import terminal 24import toolchain 25 26 27""" 28Theory of Operation 29 30Please see README for user documentation, and you should be familiar with 31that before trying to make sense of this. 32 33Buildman works by keeping the machine as busy as possible, building different 34commits for different boards on multiple CPUs at once. 35 36The source repo (self.git_dir) contains all the commits to be built. Each 37thread works on a single board at a time. It checks out the first commit, 38configures it for that board, then builds it. Then it checks out the next 39commit and builds it (typically without re-configuring). When it runs out 40of commits, it gets another job from the builder and starts again with that 41board. 42 43Clearly the builder threads could work either way - they could check out a 44commit and then built it for all boards. Using separate directories for each 45commit/board pair they could leave their build product around afterwards 46also. 47 48The intent behind building a single board for multiple commits, is to make 49use of incremental builds. Since each commit is built incrementally from 50the previous one, builds are faster. Reconfiguring for a different board 51removes all intermediate object files. 52 53Many threads can be working at once, but each has its own working directory. 54When a thread finishes a build, it puts the output files into a result 55directory. 56 57The base directory used by buildman is normally '../<branch>', i.e. 58a directory higher than the source repository and named after the branch 59being built. 60 61Within the base directory, we have one subdirectory for each commit. Within 62that is one subdirectory for each board. Within that is the build output for 63that commit/board combination. 64 65Buildman also create working directories for each thread, in a .bm-work/ 66subdirectory in the base dir. 67 68As an example, say we are building branch 'us-net' for boards 'sandbox' and 69'seaboard', and say that us-net has two commits. We will have directories 70like this: 71 72us-net/ base directory 73 01_of_02_g4ed4ebc_net--Add-tftp-speed-/ 74 sandbox/ 75 u-boot.bin 76 seaboard/ 77 u-boot.bin 78 02_of_02_g4ed4ebc_net--Check-tftp-comp/ 79 sandbox/ 80 u-boot.bin 81 seaboard/ 82 u-boot.bin 83 .bm-work/ 84 00/ working directory for thread 0 (contains source checkout) 85 build/ build output 86 01/ working directory for thread 1 87 build/ build output 88 ... 89u-boot/ source directory 90 .git/ repository 91""" 92 93# Possible build outcomes 94OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4) 95 96# Translate a commit subject into a valid filename 97trans_valid_chars = string.maketrans("/: ", "---") 98 99 100def Mkdir(dirname): 101 """Make a directory if it doesn't already exist. 102 103 Args: 104 dirname: Directory to create 105 """ 106 try: 107 os.mkdir(dirname) 108 except OSError as err: 109 if err.errno == errno.EEXIST: 110 pass 111 else: 112 raise 113 114class BuilderJob: 115 """Holds information about a job to be performed by a thread 116 117 Members: 118 board: Board object to build 119 commits: List of commit options to build. 120 """ 121 def __init__(self): 122 self.board = None 123 self.commits = [] 124 125 126class ResultThread(threading.Thread): 127 """This thread processes results from builder threads. 128 129 It simply passes the results on to the builder. There is only one 130 result thread, and this helps to serialise the build output. 131 """ 132 def __init__(self, builder): 133 """Set up a new result thread 134 135 Args: 136 builder: Builder which will be sent each result 137 """ 138 threading.Thread.__init__(self) 139 self.builder = builder 140 141 def run(self): 142 """Called to start up the result thread. 143 144 We collect the next result job and pass it on to the build. 145 """ 146 while True: 147 result = self.builder.out_queue.get() 148 self.builder.ProcessResult(result) 149 self.builder.out_queue.task_done() 150 151 152class BuilderThread(threading.Thread): 153 """This thread builds U-Boot for a particular board. 154 155 An input queue provides each new job. We run 'make' to build U-Boot 156 and then pass the results on to the output queue. 157 158 Members: 159 builder: The builder which contains information we might need 160 thread_num: Our thread number (0-n-1), used to decide on a 161 temporary directory 162 """ 163 def __init__(self, builder, thread_num): 164 """Set up a new builder thread""" 165 threading.Thread.__init__(self) 166 self.builder = builder 167 self.thread_num = thread_num 168 169 def Make(self, commit, brd, stage, cwd, *args, **kwargs): 170 """Run 'make' on a particular commit and board. 171 172 The source code will already be checked out, so the 'commit' 173 argument is only for information. 174 175 Args: 176 commit: Commit object that is being built 177 brd: Board object that is being built 178 stage: Stage of the build. Valid stages are: 179 distclean - can be called to clean source 180 config - called to configure for a board 181 build - the main make invocation - it does the build 182 args: A list of arguments to pass to 'make' 183 kwargs: A list of keyword arguments to pass to command.RunPipe() 184 185 Returns: 186 CommandResult object 187 """ 188 return self.builder.do_make(commit, brd, stage, cwd, *args, 189 **kwargs) 190 191 def RunCommit(self, commit_upto, brd, work_dir, do_config, force_build, 192 force_build_failures): 193 """Build a particular commit. 194 195 If the build is already done, and we are not forcing a build, we skip 196 the build and just return the previously-saved results. 197 198 Args: 199 commit_upto: Commit number to build (0...n-1) 200 brd: Board object to build 201 work_dir: Directory to which the source will be checked out 202 do_config: True to run a make <board>_config on the source 203 force_build: Force a build even if one was previously done 204 force_build_failures: Force a bulid if the previous result showed 205 failure 206 207 Returns: 208 tuple containing: 209 - CommandResult object containing the results of the build 210 - boolean indicating whether 'make config' is still needed 211 """ 212 # Create a default result - it will be overwritte by the call to 213 # self.Make() below, in the event that we do a build. 214 result = command.CommandResult() 215 result.return_code = 0 216 if self.builder.in_tree: 217 out_dir = work_dir 218 else: 219 out_dir = os.path.join(work_dir, 'build') 220 221 # Check if the job was already completed last time 222 done_file = self.builder.GetDoneFile(commit_upto, brd.target) 223 result.already_done = os.path.exists(done_file) 224 will_build = (force_build or force_build_failures or 225 not result.already_done) 226 if result.already_done and will_build: 227 # Get the return code from that build and use it 228 with open(done_file, 'r') as fd: 229 result.return_code = int(fd.readline()) 230 err_file = self.builder.GetErrFile(commit_upto, brd.target) 231 if os.path.exists(err_file) and os.stat(err_file).st_size: 232 result.stderr = 'bad' 233 elif not force_build: 234 # The build passed, so no need to build it again 235 will_build = False 236 237 if will_build: 238 # We are going to have to build it. First, get a toolchain 239 if not self.toolchain: 240 try: 241 self.toolchain = self.builder.toolchains.Select(brd.arch) 242 except ValueError as err: 243 result.return_code = 10 244 result.stdout = '' 245 result.stderr = str(err) 246 # TODO(sjg@chromium.org): This gets swallowed, but needs 247 # to be reported. 248 249 if self.toolchain: 250 # Checkout the right commit 251 if commit_upto is not None: 252 commit = self.builder.commits[commit_upto] 253 if self.builder.checkout: 254 git_dir = os.path.join(work_dir, '.git') 255 gitutil.Checkout(commit.hash, git_dir, work_dir, 256 force=True) 257 else: 258 commit = self.builder.commit # Ick, fix this for BuildCommits() 259 260 # Set up the environment and command line 261 env = self.toolchain.MakeEnvironment() 262 Mkdir(out_dir) 263 args = [] 264 if not self.builder.in_tree: 265 args.append('O=build') 266 args.append('-s') 267 if self.builder.num_jobs is not None: 268 args.extend(['-j', str(self.builder.num_jobs)]) 269 config_args = ['%s_config' % brd.target] 270 config_out = '' 271 args.extend(self.builder.toolchains.GetMakeArguments(brd)) 272 273 # If we need to reconfigure, do that now 274 if do_config: 275 result = self.Make(commit, brd, 'distclean', work_dir, 276 'distclean', *args, env=env) 277 result = self.Make(commit, brd, 'config', work_dir, 278 *(args + config_args), env=env) 279 config_out = result.combined 280 do_config = False # No need to configure next time 281 if result.return_code == 0: 282 result = self.Make(commit, brd, 'build', work_dir, *args, 283 env=env) 284 result.stdout = config_out + result.stdout 285 else: 286 result.return_code = 1 287 result.stderr = 'No tool chain for %s\n' % brd.arch 288 result.already_done = False 289 290 result.toolchain = self.toolchain 291 result.brd = brd 292 result.commit_upto = commit_upto 293 result.out_dir = out_dir 294 return result, do_config 295 296 def _WriteResult(self, result, keep_outputs): 297 """Write a built result to the output directory. 298 299 Args: 300 result: CommandResult object containing result to write 301 keep_outputs: True to store the output binaries, False 302 to delete them 303 """ 304 # Fatal error 305 if result.return_code < 0: 306 return 307 308 # Aborted? 309 if result.stderr and 'No child processes' in result.stderr: 310 return 311 312 if result.already_done: 313 return 314 315 # Write the output and stderr 316 output_dir = self.builder._GetOutputDir(result.commit_upto) 317 Mkdir(output_dir) 318 build_dir = self.builder.GetBuildDir(result.commit_upto, 319 result.brd.target) 320 Mkdir(build_dir) 321 322 outfile = os.path.join(build_dir, 'log') 323 with open(outfile, 'w') as fd: 324 if result.stdout: 325 fd.write(result.stdout) 326 327 errfile = self.builder.GetErrFile(result.commit_upto, 328 result.brd.target) 329 if result.stderr: 330 with open(errfile, 'w') as fd: 331 fd.write(result.stderr) 332 elif os.path.exists(errfile): 333 os.remove(errfile) 334 335 if result.toolchain: 336 # Write the build result and toolchain information. 337 done_file = self.builder.GetDoneFile(result.commit_upto, 338 result.brd.target) 339 with open(done_file, 'w') as fd: 340 fd.write('%s' % result.return_code) 341 with open(os.path.join(build_dir, 'toolchain'), 'w') as fd: 342 print >>fd, 'gcc', result.toolchain.gcc 343 print >>fd, 'path', result.toolchain.path 344 print >>fd, 'cross', result.toolchain.cross 345 print >>fd, 'arch', result.toolchain.arch 346 fd.write('%s' % result.return_code) 347 348 with open(os.path.join(build_dir, 'toolchain'), 'w') as fd: 349 print >>fd, 'gcc', result.toolchain.gcc 350 print >>fd, 'path', result.toolchain.path 351 352 # Write out the image and function size information and an objdump 353 env = result.toolchain.MakeEnvironment() 354 lines = [] 355 for fname in ['u-boot', 'spl/u-boot-spl']: 356 cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname] 357 nm_result = command.RunPipe([cmd], capture=True, 358 capture_stderr=True, cwd=result.out_dir, 359 raise_on_error=False, env=env) 360 if nm_result.stdout: 361 nm = self.builder.GetFuncSizesFile(result.commit_upto, 362 result.brd.target, fname) 363 with open(nm, 'w') as fd: 364 print >>fd, nm_result.stdout, 365 366 cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname] 367 dump_result = command.RunPipe([cmd], capture=True, 368 capture_stderr=True, cwd=result.out_dir, 369 raise_on_error=False, env=env) 370 rodata_size = '' 371 if dump_result.stdout: 372 objdump = self.builder.GetObjdumpFile(result.commit_upto, 373 result.brd.target, fname) 374 with open(objdump, 'w') as fd: 375 print >>fd, dump_result.stdout, 376 for line in dump_result.stdout.splitlines(): 377 fields = line.split() 378 if len(fields) > 5 and fields[1] == '.rodata': 379 rodata_size = fields[2] 380 381 cmd = ['%ssize' % self.toolchain.cross, fname] 382 size_result = command.RunPipe([cmd], capture=True, 383 capture_stderr=True, cwd=result.out_dir, 384 raise_on_error=False, env=env) 385 if size_result.stdout: 386 lines.append(size_result.stdout.splitlines()[1] + ' ' + 387 rodata_size) 388 389 # Write out the image sizes file. This is similar to the output 390 # of binutil's 'size' utility, but it omits the header line and 391 # adds an additional hex value at the end of each line for the 392 # rodata size 393 if len(lines): 394 sizes = self.builder.GetSizesFile(result.commit_upto, 395 result.brd.target) 396 with open(sizes, 'w') as fd: 397 print >>fd, '\n'.join(lines) 398 399 # Now write the actual build output 400 if keep_outputs: 401 patterns = ['u-boot', '*.bin', 'u-boot.dtb', '*.map', 402 'include/autoconf.mk', 'spl/u-boot-spl', 403 'spl/u-boot-spl.bin'] 404 for pattern in patterns: 405 file_list = glob.glob(os.path.join(result.out_dir, pattern)) 406 for fname in file_list: 407 shutil.copy(fname, build_dir) 408 409 410 def RunJob(self, job): 411 """Run a single job 412 413 A job consists of a building a list of commits for a particular board. 414 415 Args: 416 job: Job to build 417 """ 418 brd = job.board 419 work_dir = self.builder.GetThreadDir(self.thread_num) 420 self.toolchain = None 421 if job.commits: 422 # Run 'make board_config' on the first commit 423 do_config = True 424 commit_upto = 0 425 force_build = False 426 for commit_upto in range(0, len(job.commits), job.step): 427 result, request_config = self.RunCommit(commit_upto, brd, 428 work_dir, do_config, 429 force_build or self.builder.force_build, 430 self.builder.force_build_failures) 431 failed = result.return_code or result.stderr 432 did_config = do_config 433 if failed and not do_config: 434 # If our incremental build failed, try building again 435 # with a reconfig. 436 if self.builder.force_config_on_failure: 437 result, request_config = self.RunCommit(commit_upto, 438 brd, work_dir, True, True, False) 439 did_config = True 440 if not self.builder.force_reconfig: 441 do_config = request_config 442 443 # If we built that commit, then config is done. But if we got 444 # an warning, reconfig next time to force it to build the same 445 # files that created warnings this time. Otherwise an 446 # incremental build may not build the same file, and we will 447 # think that the warning has gone away. 448 # We could avoid this by using -Werror everywhere... 449 # For errors, the problem doesn't happen, since presumably 450 # the build stopped and didn't generate output, so will retry 451 # that file next time. So we could detect warnings and deal 452 # with them specially here. For now, we just reconfigure if 453 # anything goes work. 454 # Of course this is substantially slower if there are build 455 # errors/warnings (e.g. 2-3x slower even if only 10% of builds 456 # have problems). 457 if (failed and not result.already_done and not did_config and 458 self.builder.force_config_on_failure): 459 # If this build failed, try the next one with a 460 # reconfigure. 461 # Sometimes if the board_config.h file changes it can mess 462 # with dependencies, and we get: 463 # make: *** No rule to make target `include/autoconf.mk', 464 # needed by `depend'. 465 do_config = True 466 force_build = True 467 else: 468 force_build = False 469 if self.builder.force_config_on_failure: 470 if failed: 471 do_config = True 472 result.commit_upto = commit_upto 473 if result.return_code < 0: 474 raise ValueError('Interrupt') 475 476 # We have the build results, so output the result 477 self._WriteResult(result, job.keep_outputs) 478 self.builder.out_queue.put(result) 479 else: 480 # Just build the currently checked-out build 481 result = self.RunCommit(None, True) 482 result.commit_upto = self.builder.upto 483 self.builder.out_queue.put(result) 484 485 def run(self): 486 """Our thread's run function 487 488 This thread picks a job from the queue, runs it, and then goes to the 489 next job. 490 """ 491 alive = True 492 while True: 493 job = self.builder.queue.get() 494 try: 495 if self.builder.active and alive: 496 self.RunJob(job) 497 except Exception as err: 498 alive = False 499 print err 500 self.builder.queue.task_done() 501 502 503class Builder: 504 """Class for building U-Boot for a particular commit. 505 506 Public members: (many should ->private) 507 active: True if the builder is active and has not been stopped 508 already_done: Number of builds already completed 509 base_dir: Base directory to use for builder 510 checkout: True to check out source, False to skip that step. 511 This is used for testing. 512 col: terminal.Color() object 513 count: Number of commits to build 514 do_make: Method to call to invoke Make 515 fail: Number of builds that failed due to error 516 force_build: Force building even if a build already exists 517 force_config_on_failure: If a commit fails for a board, disable 518 incremental building for the next commit we build for that 519 board, so that we will see all warnings/errors again. 520 force_build_failures: If a previously-built build (i.e. built on 521 a previous run of buildman) is marked as failed, rebuild it. 522 git_dir: Git directory containing source repository 523 last_line_len: Length of the last line we printed (used for erasing 524 it with new progress information) 525 num_jobs: Number of jobs to run at once (passed to make as -j) 526 num_threads: Number of builder threads to run 527 out_queue: Queue of results to process 528 re_make_err: Compiled regular expression for ignore_lines 529 queue: Queue of jobs to run 530 threads: List of active threads 531 toolchains: Toolchains object to use for building 532 upto: Current commit number we are building (0.count-1) 533 warned: Number of builds that produced at least one warning 534 force_reconfig: Reconfigure U-Boot on each comiit. This disables 535 incremental building, where buildman reconfigures on the first 536 commit for a baord, and then just does an incremental build for 537 the following commits. In fact buildman will reconfigure and 538 retry for any failing commits, so generally the only effect of 539 this option is to slow things down. 540 in_tree: Build U-Boot in-tree instead of specifying an output 541 directory separate from the source code. This option is really 542 only useful for testing in-tree builds. 543 544 Private members: 545 _base_board_dict: Last-summarised Dict of boards 546 _base_err_lines: Last-summarised list of errors 547 _build_period_us: Time taken for a single build (float object). 548 _complete_delay: Expected delay until completion (timedelta) 549 _next_delay_update: Next time we plan to display a progress update 550 (datatime) 551 _show_unknown: Show unknown boards (those not built) in summary 552 _timestamps: List of timestamps for the completion of the last 553 last _timestamp_count builds. Each is a datetime object. 554 _timestamp_count: Number of timestamps to keep in our list. 555 _working_dir: Base working directory containing all threads 556 """ 557 class Outcome: 558 """Records a build outcome for a single make invocation 559 560 Public Members: 561 rc: Outcome value (OUTCOME_...) 562 err_lines: List of error lines or [] if none 563 sizes: Dictionary of image size information, keyed by filename 564 - Each value is itself a dictionary containing 565 values for 'text', 'data' and 'bss', being the integer 566 size in bytes of each section. 567 func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each 568 value is itself a dictionary: 569 key: function name 570 value: Size of function in bytes 571 """ 572 def __init__(self, rc, err_lines, sizes, func_sizes): 573 self.rc = rc 574 self.err_lines = err_lines 575 self.sizes = sizes 576 self.func_sizes = func_sizes 577 578 def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, 579 checkout=True, show_unknown=True, step=1): 580 """Create a new Builder object 581 582 Args: 583 toolchains: Toolchains object to use for building 584 base_dir: Base directory to use for builder 585 git_dir: Git directory containing source repository 586 num_threads: Number of builder threads to run 587 num_jobs: Number of jobs to run at once (passed to make as -j) 588 checkout: True to check out source, False to skip that step. 589 This is used for testing. 590 show_unknown: Show unknown boards (those not built) in summary 591 step: 1 to process every commit, n to process every nth commit 592 """ 593 self.toolchains = toolchains 594 self.base_dir = base_dir 595 self._working_dir = os.path.join(base_dir, '.bm-work') 596 self.threads = [] 597 self.active = True 598 self.do_make = self.Make 599 self.checkout = checkout 600 self.num_threads = num_threads 601 self.num_jobs = num_jobs 602 self.already_done = 0 603 self.force_build = False 604 self.git_dir = git_dir 605 self._show_unknown = show_unknown 606 self._timestamp_count = 10 607 self._build_period_us = None 608 self._complete_delay = None 609 self._next_delay_update = datetime.now() 610 self.force_config_on_failure = True 611 self.force_build_failures = False 612 self.force_reconfig = False 613 self._step = step 614 self.in_tree = False 615 616 self.col = terminal.Color() 617 618 self.queue = Queue.Queue() 619 self.out_queue = Queue.Queue() 620 for i in range(self.num_threads): 621 t = BuilderThread(self, i) 622 t.setDaemon(True) 623 t.start() 624 self.threads.append(t) 625 626 self.last_line_len = 0 627 t = ResultThread(self) 628 t.setDaemon(True) 629 t.start() 630 self.threads.append(t) 631 632 ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)'] 633 self.re_make_err = re.compile('|'.join(ignore_lines)) 634 635 def __del__(self): 636 """Get rid of all threads created by the builder""" 637 for t in self.threads: 638 del t 639 640 def _AddTimestamp(self): 641 """Add a new timestamp to the list and record the build period. 642 643 The build period is the length of time taken to perform a single 644 build (one board, one commit). 645 """ 646 now = datetime.now() 647 self._timestamps.append(now) 648 count = len(self._timestamps) 649 delta = self._timestamps[-1] - self._timestamps[0] 650 seconds = delta.total_seconds() 651 652 # If we have enough data, estimate build period (time taken for a 653 # single build) and therefore completion time. 654 if count > 1 and self._next_delay_update < now: 655 self._next_delay_update = now + timedelta(seconds=2) 656 if seconds > 0: 657 self._build_period = float(seconds) / count 658 todo = self.count - self.upto 659 self._complete_delay = timedelta(microseconds= 660 self._build_period * todo * 1000000) 661 # Round it 662 self._complete_delay -= timedelta( 663 microseconds=self._complete_delay.microseconds) 664 665 if seconds > 60: 666 self._timestamps.popleft() 667 count -= 1 668 669 def ClearLine(self, length): 670 """Clear any characters on the current line 671 672 Make way for a new line of length 'length', by outputting enough 673 spaces to clear out the old line. Then remember the new length for 674 next time. 675 676 Args: 677 length: Length of new line, in characters 678 """ 679 if length < self.last_line_len: 680 print ' ' * (self.last_line_len - length), 681 print '\r', 682 self.last_line_len = length 683 sys.stdout.flush() 684 685 def SelectCommit(self, commit, checkout=True): 686 """Checkout the selected commit for this build 687 """ 688 self.commit = commit 689 if checkout and self.checkout: 690 gitutil.Checkout(commit.hash) 691 692 def Make(self, commit, brd, stage, cwd, *args, **kwargs): 693 """Run make 694 695 Args: 696 commit: Commit object that is being built 697 brd: Board object that is being built 698 stage: Stage that we are at (distclean, config, build) 699 cwd: Directory where make should be run 700 args: Arguments to pass to make 701 kwargs: Arguments to pass to command.RunPipe() 702 """ 703 cmd = ['make'] + list(args) 704 result = command.RunPipe([cmd], capture=True, capture_stderr=True, 705 cwd=cwd, raise_on_error=False, **kwargs) 706 return result 707 708 def ProcessResult(self, result): 709 """Process the result of a build, showing progress information 710 711 Args: 712 result: A CommandResult object 713 """ 714 col = terminal.Color() 715 if result: 716 target = result.brd.target 717 718 if result.return_code < 0: 719 self.active = False 720 command.StopAll() 721 return 722 723 self.upto += 1 724 if result.return_code != 0: 725 self.fail += 1 726 elif result.stderr: 727 self.warned += 1 728 if result.already_done: 729 self.already_done += 1 730 else: 731 target = '(starting)' 732 733 # Display separate counts for ok, warned and fail 734 ok = self.upto - self.warned - self.fail 735 line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok) 736 line += self.col.Color(self.col.YELLOW, '%5d' % self.warned) 737 line += self.col.Color(self.col.RED, '%5d' % self.fail) 738 739 name = ' /%-5d ' % self.count 740 741 # Add our current completion time estimate 742 self._AddTimestamp() 743 if self._complete_delay: 744 name += '%s : ' % self._complete_delay 745 # When building all boards for a commit, we can print a commit 746 # progress message. 747 if result and result.commit_upto is None: 748 name += 'commit %2d/%-3d' % (self.commit_upto + 1, 749 self.commit_count) 750 751 name += target 752 print line + name, 753 length = 13 + len(name) 754 self.ClearLine(length) 755 756 def _GetOutputDir(self, commit_upto): 757 """Get the name of the output directory for a commit number 758 759 The output directory is typically .../<branch>/<commit>. 760 761 Args: 762 commit_upto: Commit number to use (0..self.count-1) 763 """ 764 commit = self.commits[commit_upto] 765 subject = commit.subject.translate(trans_valid_chars) 766 commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1, 767 self.commit_count, commit.hash, subject[:20])) 768 output_dir = os.path.join(self.base_dir, commit_dir) 769 return output_dir 770 771 def GetBuildDir(self, commit_upto, target): 772 """Get the name of the build directory for a commit number 773 774 The build directory is typically .../<branch>/<commit>/<target>. 775 776 Args: 777 commit_upto: Commit number to use (0..self.count-1) 778 target: Target name 779 """ 780 output_dir = self._GetOutputDir(commit_upto) 781 return os.path.join(output_dir, target) 782 783 def GetDoneFile(self, commit_upto, target): 784 """Get the name of the done file for a commit number 785 786 Args: 787 commit_upto: Commit number to use (0..self.count-1) 788 target: Target name 789 """ 790 return os.path.join(self.GetBuildDir(commit_upto, target), 'done') 791 792 def GetSizesFile(self, commit_upto, target): 793 """Get the name of the sizes file for a commit number 794 795 Args: 796 commit_upto: Commit number to use (0..self.count-1) 797 target: Target name 798 """ 799 return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes') 800 801 def GetFuncSizesFile(self, commit_upto, target, elf_fname): 802 """Get the name of the funcsizes file for a commit number and ELF file 803 804 Args: 805 commit_upto: Commit number to use (0..self.count-1) 806 target: Target name 807 elf_fname: Filename of elf image 808 """ 809 return os.path.join(self.GetBuildDir(commit_upto, target), 810 '%s.sizes' % elf_fname.replace('/', '-')) 811 812 def GetObjdumpFile(self, commit_upto, target, elf_fname): 813 """Get the name of the objdump file for a commit number and ELF file 814 815 Args: 816 commit_upto: Commit number to use (0..self.count-1) 817 target: Target name 818 elf_fname: Filename of elf image 819 """ 820 return os.path.join(self.GetBuildDir(commit_upto, target), 821 '%s.objdump' % elf_fname.replace('/', '-')) 822 823 def GetErrFile(self, commit_upto, target): 824 """Get the name of the err file for a commit number 825 826 Args: 827 commit_upto: Commit number to use (0..self.count-1) 828 target: Target name 829 """ 830 output_dir = self.GetBuildDir(commit_upto, target) 831 return os.path.join(output_dir, 'err') 832 833 def FilterErrors(self, lines): 834 """Filter out errors in which we have no interest 835 836 We should probably use map(). 837 838 Args: 839 lines: List of error lines, each a string 840 Returns: 841 New list with only interesting lines included 842 """ 843 out_lines = [] 844 for line in lines: 845 if not self.re_make_err.search(line): 846 out_lines.append(line) 847 return out_lines 848 849 def ReadFuncSizes(self, fname, fd): 850 """Read function sizes from the output of 'nm' 851 852 Args: 853 fd: File containing data to read 854 fname: Filename we are reading from (just for errors) 855 856 Returns: 857 Dictionary containing size of each function in bytes, indexed by 858 function name. 859 """ 860 sym = {} 861 for line in fd.readlines(): 862 try: 863 size, type, name = line[:-1].split() 864 except: 865 print "Invalid line in file '%s': '%s'" % (fname, line[:-1]) 866 continue 867 if type in 'tTdDbB': 868 # function names begin with '.' on 64-bit powerpc 869 if '.' in name[1:]: 870 name = 'static.' + name.split('.')[0] 871 sym[name] = sym.get(name, 0) + int(size, 16) 872 return sym 873 874 def GetBuildOutcome(self, commit_upto, target, read_func_sizes): 875 """Work out the outcome of a build. 876 877 Args: 878 commit_upto: Commit number to check (0..n-1) 879 target: Target board to check 880 read_func_sizes: True to read function size information 881 882 Returns: 883 Outcome object 884 """ 885 done_file = self.GetDoneFile(commit_upto, target) 886 sizes_file = self.GetSizesFile(commit_upto, target) 887 sizes = {} 888 func_sizes = {} 889 if os.path.exists(done_file): 890 with open(done_file, 'r') as fd: 891 return_code = int(fd.readline()) 892 err_lines = [] 893 err_file = self.GetErrFile(commit_upto, target) 894 if os.path.exists(err_file): 895 with open(err_file, 'r') as fd: 896 err_lines = self.FilterErrors(fd.readlines()) 897 898 # Decide whether the build was ok, failed or created warnings 899 if return_code: 900 rc = OUTCOME_ERROR 901 elif len(err_lines): 902 rc = OUTCOME_WARNING 903 else: 904 rc = OUTCOME_OK 905 906 # Convert size information to our simple format 907 if os.path.exists(sizes_file): 908 with open(sizes_file, 'r') as fd: 909 for line in fd.readlines(): 910 values = line.split() 911 rodata = 0 912 if len(values) > 6: 913 rodata = int(values[6], 16) 914 size_dict = { 915 'all' : int(values[0]) + int(values[1]) + 916 int(values[2]), 917 'text' : int(values[0]) - rodata, 918 'data' : int(values[1]), 919 'bss' : int(values[2]), 920 'rodata' : rodata, 921 } 922 sizes[values[5]] = size_dict 923 924 if read_func_sizes: 925 pattern = self.GetFuncSizesFile(commit_upto, target, '*') 926 for fname in glob.glob(pattern): 927 with open(fname, 'r') as fd: 928 dict_name = os.path.basename(fname).replace('.sizes', 929 '') 930 func_sizes[dict_name] = self.ReadFuncSizes(fname, fd) 931 932 return Builder.Outcome(rc, err_lines, sizes, func_sizes) 933 934 return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}) 935 936 def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes): 937 """Calculate a summary of the results of building a commit. 938 939 Args: 940 board_selected: Dict containing boards to summarise 941 commit_upto: Commit number to summarize (0..self.count-1) 942 read_func_sizes: True to read function size information 943 944 Returns: 945 Tuple: 946 Dict containing boards which passed building this commit. 947 keyed by board.target 948 List containing a summary of error/warning lines 949 """ 950 board_dict = {} 951 err_lines_summary = [] 952 953 for board in boards_selected.itervalues(): 954 outcome = self.GetBuildOutcome(commit_upto, board.target, 955 read_func_sizes) 956 board_dict[board.target] = outcome 957 for err in outcome.err_lines: 958 if err and not err.rstrip() in err_lines_summary: 959 err_lines_summary.append(err.rstrip()) 960 return board_dict, err_lines_summary 961 962 def AddOutcome(self, board_dict, arch_list, changes, char, color): 963 """Add an output to our list of outcomes for each architecture 964 965 This simple function adds failing boards (changes) to the 966 relevant architecture string, so we can print the results out 967 sorted by architecture. 968 969 Args: 970 board_dict: Dict containing all boards 971 arch_list: Dict keyed by arch name. Value is a string containing 972 a list of board names which failed for that arch. 973 changes: List of boards to add to arch_list 974 color: terminal.Colour object 975 """ 976 done_arch = {} 977 for target in changes: 978 if target in board_dict: 979 arch = board_dict[target].arch 980 else: 981 arch = 'unknown' 982 str = self.col.Color(color, ' ' + target) 983 if not arch in done_arch: 984 str = self.col.Color(color, char) + ' ' + str 985 done_arch[arch] = True 986 if not arch in arch_list: 987 arch_list[arch] = str 988 else: 989 arch_list[arch] += str 990 991 992 def ColourNum(self, num): 993 color = self.col.RED if num > 0 else self.col.GREEN 994 if num == 0: 995 return '0' 996 return self.col.Color(color, str(num)) 997 998 def ResetResultSummary(self, board_selected): 999 """Reset the results summary ready for use. 1000 1001 Set up the base board list to be all those selected, and set the 1002 error lines to empty. 1003 1004 Following this, calls to PrintResultSummary() will use this 1005 information to work out what has changed. 1006 1007 Args: 1008 board_selected: Dict containing boards to summarise, keyed by 1009 board.target 1010 """ 1011 self._base_board_dict = {} 1012 for board in board_selected: 1013 self._base_board_dict[board] = Builder.Outcome(0, [], [], {}) 1014 self._base_err_lines = [] 1015 1016 def PrintFuncSizeDetail(self, fname, old, new): 1017 grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0 1018 delta, common = [], {} 1019 1020 for a in old: 1021 if a in new: 1022 common[a] = 1 1023 1024 for name in old: 1025 if name not in common: 1026 remove += 1 1027 down += old[name] 1028 delta.append([-old[name], name]) 1029 1030 for name in new: 1031 if name not in common: 1032 add += 1 1033 up += new[name] 1034 delta.append([new[name], name]) 1035 1036 for name in common: 1037 diff = new.get(name, 0) - old.get(name, 0) 1038 if diff > 0: 1039 grow, up = grow + 1, up + diff 1040 elif diff < 0: 1041 shrink, down = shrink + 1, down - diff 1042 delta.append([diff, name]) 1043 1044 delta.sort() 1045 delta.reverse() 1046 1047 args = [add, -remove, grow, -shrink, up, -down, up - down] 1048 if max(args) == 0: 1049 return 1050 args = [self.ColourNum(x) for x in args] 1051 indent = ' ' * 15 1052 print ('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' % 1053 tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args)) 1054 print '%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new', 1055 'delta') 1056 for diff, name in delta: 1057 if diff: 1058 color = self.col.RED if diff > 0 else self.col.GREEN 1059 msg = '%s %-38s %7s %7s %+7d' % (indent, name, 1060 old.get(name, '-'), new.get(name,'-'), diff) 1061 print self.col.Color(color, msg) 1062 1063 1064 def PrintSizeDetail(self, target_list, show_bloat): 1065 """Show details size information for each board 1066 1067 Args: 1068 target_list: List of targets, each a dict containing: 1069 'target': Target name 1070 'total_diff': Total difference in bytes across all areas 1071 <part_name>: Difference for that part 1072 show_bloat: Show detail for each function 1073 """ 1074 targets_by_diff = sorted(target_list, reverse=True, 1075 key=lambda x: x['_total_diff']) 1076 for result in targets_by_diff: 1077 printed_target = False 1078 for name in sorted(result): 1079 diff = result[name] 1080 if name.startswith('_'): 1081 continue 1082 if diff != 0: 1083 color = self.col.RED if diff > 0 else self.col.GREEN 1084 msg = ' %s %+d' % (name, diff) 1085 if not printed_target: 1086 print '%10s %-15s:' % ('', result['_target']), 1087 printed_target = True 1088 print self.col.Color(color, msg), 1089 if printed_target: 1090 print 1091 if show_bloat: 1092 target = result['_target'] 1093 outcome = result['_outcome'] 1094 base_outcome = self._base_board_dict[target] 1095 for fname in outcome.func_sizes: 1096 self.PrintFuncSizeDetail(fname, 1097 base_outcome.func_sizes[fname], 1098 outcome.func_sizes[fname]) 1099 1100 1101 def PrintSizeSummary(self, board_selected, board_dict, show_detail, 1102 show_bloat): 1103 """Print a summary of image sizes broken down by section. 1104 1105 The summary takes the form of one line per architecture. The 1106 line contains deltas for each of the sections (+ means the section 1107 got bigger, - means smaller). The nunmbers are the average number 1108 of bytes that a board in this section increased by. 1109 1110 For example: 1111 powerpc: (622 boards) text -0.0 1112 arm: (285 boards) text -0.0 1113 nds32: (3 boards) text -8.0 1114 1115 Args: 1116 board_selected: Dict containing boards to summarise, keyed by 1117 board.target 1118 board_dict: Dict containing boards for which we built this 1119 commit, keyed by board.target. The value is an Outcome object. 1120 show_detail: Show detail for each board 1121 show_bloat: Show detail for each function 1122 """ 1123 arch_list = {} 1124 arch_count = {} 1125 1126 # Calculate changes in size for different image parts 1127 # The previous sizes are in Board.sizes, for each board 1128 for target in board_dict: 1129 if target not in board_selected: 1130 continue 1131 base_sizes = self._base_board_dict[target].sizes 1132 outcome = board_dict[target] 1133 sizes = outcome.sizes 1134 1135 # Loop through the list of images, creating a dict of size 1136 # changes for each image/part. We end up with something like 1137 # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4} 1138 # which means that U-Boot data increased by 5 bytes and SPL 1139 # text decreased by 4. 1140 err = {'_target' : target} 1141 for image in sizes: 1142 if image in base_sizes: 1143 base_image = base_sizes[image] 1144 # Loop through the text, data, bss parts 1145 for part in sorted(sizes[image]): 1146 diff = sizes[image][part] - base_image[part] 1147 col = None 1148 if diff: 1149 if image == 'u-boot': 1150 name = part 1151 else: 1152 name = image + ':' + part 1153 err[name] = diff 1154 arch = board_selected[target].arch 1155 if not arch in arch_count: 1156 arch_count[arch] = 1 1157 else: 1158 arch_count[arch] += 1 1159 if not sizes: 1160 pass # Only add to our list when we have some stats 1161 elif not arch in arch_list: 1162 arch_list[arch] = [err] 1163 else: 1164 arch_list[arch].append(err) 1165 1166 # We now have a list of image size changes sorted by arch 1167 # Print out a summary of these 1168 for arch, target_list in arch_list.iteritems(): 1169 # Get total difference for each type 1170 totals = {} 1171 for result in target_list: 1172 total = 0 1173 for name, diff in result.iteritems(): 1174 if name.startswith('_'): 1175 continue 1176 total += diff 1177 if name in totals: 1178 totals[name] += diff 1179 else: 1180 totals[name] = diff 1181 result['_total_diff'] = total 1182 result['_outcome'] = board_dict[result['_target']] 1183 1184 count = len(target_list) 1185 printed_arch = False 1186 for name in sorted(totals): 1187 diff = totals[name] 1188 if diff: 1189 # Display the average difference in this name for this 1190 # architecture 1191 avg_diff = float(diff) / count 1192 color = self.col.RED if avg_diff > 0 else self.col.GREEN 1193 msg = ' %s %+1.1f' % (name, avg_diff) 1194 if not printed_arch: 1195 print '%10s: (for %d/%d boards)' % (arch, count, 1196 arch_count[arch]), 1197 printed_arch = True 1198 print self.col.Color(color, msg), 1199 1200 if printed_arch: 1201 print 1202 if show_detail: 1203 self.PrintSizeDetail(target_list, show_bloat) 1204 1205 1206 def PrintResultSummary(self, board_selected, board_dict, err_lines, 1207 show_sizes, show_detail, show_bloat): 1208 """Compare results with the base results and display delta. 1209 1210 Only boards mentioned in board_selected will be considered. This 1211 function is intended to be called repeatedly with the results of 1212 each commit. It therefore shows a 'diff' between what it saw in 1213 the last call and what it sees now. 1214 1215 Args: 1216 board_selected: Dict containing boards to summarise, keyed by 1217 board.target 1218 board_dict: Dict containing boards for which we built this 1219 commit, keyed by board.target. The value is an Outcome object. 1220 err_lines: A list of errors for this commit, or [] if there is 1221 none, or we don't want to print errors 1222 show_sizes: Show image size deltas 1223 show_detail: Show detail for each board 1224 show_bloat: Show detail for each function 1225 """ 1226 better = [] # List of boards fixed since last commit 1227 worse = [] # List of new broken boards since last commit 1228 new = [] # List of boards that didn't exist last time 1229 unknown = [] # List of boards that were not built 1230 1231 for target in board_dict: 1232 if target not in board_selected: 1233 continue 1234 1235 # If the board was built last time, add its outcome to a list 1236 if target in self._base_board_dict: 1237 base_outcome = self._base_board_dict[target].rc 1238 outcome = board_dict[target] 1239 if outcome.rc == OUTCOME_UNKNOWN: 1240 unknown.append(target) 1241 elif outcome.rc < base_outcome: 1242 better.append(target) 1243 elif outcome.rc > base_outcome: 1244 worse.append(target) 1245 else: 1246 new.append(target) 1247 1248 # Get a list of errors that have appeared, and disappeared 1249 better_err = [] 1250 worse_err = [] 1251 for line in err_lines: 1252 if line not in self._base_err_lines: 1253 worse_err.append('+' + line) 1254 for line in self._base_err_lines: 1255 if line not in err_lines: 1256 better_err.append('-' + line) 1257 1258 # Display results by arch 1259 if better or worse or unknown or new or worse_err or better_err: 1260 arch_list = {} 1261 self.AddOutcome(board_selected, arch_list, better, '', 1262 self.col.GREEN) 1263 self.AddOutcome(board_selected, arch_list, worse, '+', 1264 self.col.RED) 1265 self.AddOutcome(board_selected, arch_list, new, '*', self.col.BLUE) 1266 if self._show_unknown: 1267 self.AddOutcome(board_selected, arch_list, unknown, '?', 1268 self.col.MAGENTA) 1269 for arch, target_list in arch_list.iteritems(): 1270 print '%10s: %s' % (arch, target_list) 1271 if better_err: 1272 print self.col.Color(self.col.GREEN, '\n'.join(better_err)) 1273 if worse_err: 1274 print self.col.Color(self.col.RED, '\n'.join(worse_err)) 1275 1276 if show_sizes: 1277 self.PrintSizeSummary(board_selected, board_dict, show_detail, 1278 show_bloat) 1279 1280 # Save our updated information for the next call to this function 1281 self._base_board_dict = board_dict 1282 self._base_err_lines = err_lines 1283 1284 # Get a list of boards that did not get built, if needed 1285 not_built = [] 1286 for board in board_selected: 1287 if not board in board_dict: 1288 not_built.append(board) 1289 if not_built: 1290 print "Boards not built (%d): %s" % (len(not_built), 1291 ', '.join(not_built)) 1292 1293 1294 def ShowSummary(self, commits, board_selected, show_errors, show_sizes, 1295 show_detail, show_bloat): 1296 """Show a build summary for U-Boot for a given board list. 1297 1298 Reset the result summary, then repeatedly call GetResultSummary on 1299 each commit's results, then display the differences we see. 1300 1301 Args: 1302 commit: Commit objects to summarise 1303 board_selected: Dict containing boards to summarise 1304 show_errors: Show errors that occured 1305 show_sizes: Show size deltas 1306 show_detail: Show detail for each board 1307 show_bloat: Show detail for each function 1308 """ 1309 self.commit_count = len(commits) 1310 self.commits = commits 1311 self.ResetResultSummary(board_selected) 1312 1313 for commit_upto in range(0, self.commit_count, self._step): 1314 board_dict, err_lines = self.GetResultSummary(board_selected, 1315 commit_upto, read_func_sizes=show_bloat) 1316 msg = '%02d: %s' % (commit_upto + 1, commits[commit_upto].subject) 1317 print self.col.Color(self.col.BLUE, msg) 1318 self.PrintResultSummary(board_selected, board_dict, 1319 err_lines if show_errors else [], show_sizes, show_detail, 1320 show_bloat) 1321 1322 1323 def SetupBuild(self, board_selected, commits): 1324 """Set up ready to start a build. 1325 1326 Args: 1327 board_selected: Selected boards to build 1328 commits: Selected commits to build 1329 """ 1330 # First work out how many commits we will build 1331 count = (len(commits) + self._step - 1) / self._step 1332 self.count = len(board_selected) * count 1333 self.upto = self.warned = self.fail = 0 1334 self._timestamps = collections.deque() 1335 1336 def BuildBoardsForCommit(self, board_selected, keep_outputs): 1337 """Build all boards for a single commit""" 1338 self.SetupBuild(board_selected) 1339 self.count = len(board_selected) 1340 for brd in board_selected.itervalues(): 1341 job = BuilderJob() 1342 job.board = brd 1343 job.commits = None 1344 job.keep_outputs = keep_outputs 1345 self.queue.put(brd) 1346 1347 self.queue.join() 1348 self.out_queue.join() 1349 print 1350 self.ClearLine(0) 1351 1352 def BuildCommits(self, commits, board_selected, show_errors, keep_outputs): 1353 """Build all boards for all commits (non-incremental)""" 1354 self.commit_count = len(commits) 1355 1356 self.ResetResultSummary(board_selected) 1357 for self.commit_upto in range(self.commit_count): 1358 self.SelectCommit(commits[self.commit_upto]) 1359 self.SelectOutputDir() 1360 Mkdir(self.output_dir) 1361 1362 self.BuildBoardsForCommit(board_selected, keep_outputs) 1363 board_dict, err_lines = self.GetResultSummary() 1364 self.PrintResultSummary(board_selected, board_dict, 1365 err_lines if show_errors else []) 1366 1367 if self.already_done: 1368 print '%d builds already done' % self.already_done 1369 1370 def GetThreadDir(self, thread_num): 1371 """Get the directory path to the working dir for a thread. 1372 1373 Args: 1374 thread_num: Number of thread to check. 1375 """ 1376 return os.path.join(self._working_dir, '%02d' % thread_num) 1377 1378 def _PrepareThread(self, thread_num): 1379 """Prepare the working directory for a thread. 1380 1381 This clones or fetches the repo into the thread's work directory. 1382 1383 Args: 1384 thread_num: Thread number (0, 1, ...) 1385 """ 1386 thread_dir = self.GetThreadDir(thread_num) 1387 Mkdir(thread_dir) 1388 git_dir = os.path.join(thread_dir, '.git') 1389 1390 # Clone the repo if it doesn't already exist 1391 # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so 1392 # we have a private index but uses the origin repo's contents? 1393 if self.git_dir: 1394 src_dir = os.path.abspath(self.git_dir) 1395 if os.path.exists(git_dir): 1396 gitutil.Fetch(git_dir, thread_dir) 1397 else: 1398 print 'Cloning repo for thread %d' % thread_num 1399 gitutil.Clone(src_dir, thread_dir) 1400 1401 def _PrepareWorkingSpace(self, max_threads): 1402 """Prepare the working directory for use. 1403 1404 Set up the git repo for each thread. 1405 1406 Args: 1407 max_threads: Maximum number of threads we expect to need. 1408 """ 1409 Mkdir(self._working_dir) 1410 for thread in range(max_threads): 1411 self._PrepareThread(thread) 1412 1413 def _PrepareOutputSpace(self): 1414 """Get the output directories ready to receive files. 1415 1416 We delete any output directories which look like ones we need to 1417 create. Having left over directories is confusing when the user wants 1418 to check the output manually. 1419 """ 1420 dir_list = [] 1421 for commit_upto in range(self.commit_count): 1422 dir_list.append(self._GetOutputDir(commit_upto)) 1423 1424 for dirname in glob.glob(os.path.join(self.base_dir, '*')): 1425 if dirname not in dir_list: 1426 shutil.rmtree(dirname) 1427 1428 def BuildBoards(self, commits, board_selected, show_errors, keep_outputs): 1429 """Build all commits for a list of boards 1430 1431 Args: 1432 commits: List of commits to be build, each a Commit object 1433 boards_selected: Dict of selected boards, key is target name, 1434 value is Board object 1435 show_errors: True to show summarised error/warning info 1436 keep_outputs: True to save build output files 1437 """ 1438 self.commit_count = len(commits) 1439 self.commits = commits 1440 1441 self.ResetResultSummary(board_selected) 1442 Mkdir(self.base_dir) 1443 self._PrepareWorkingSpace(min(self.num_threads, len(board_selected))) 1444 self._PrepareOutputSpace() 1445 self.SetupBuild(board_selected, commits) 1446 self.ProcessResult(None) 1447 1448 # Create jobs to build all commits for each board 1449 for brd in board_selected.itervalues(): 1450 job = BuilderJob() 1451 job.board = brd 1452 job.commits = commits 1453 job.keep_outputs = keep_outputs 1454 job.step = self._step 1455 self.queue.put(job) 1456 1457 # Wait until all jobs are started 1458 self.queue.join() 1459 1460 # Wait until we have processed all output 1461 self.out_queue.join() 1462 print 1463 self.ClearLine(0) 1464