1#!/usr/bin/env python2 2# 3# Author: Masahiro Yamada <yamada.masahiro@socionext.com> 4# 5# SPDX-License-Identifier: GPL-2.0+ 6# 7 8""" 9Move config options from headers to defconfig files. 10 11Since Kconfig was introduced to U-Boot, we have worked on moving 12config options from headers to Kconfig (defconfig). 13 14This tool intends to help this tremendous work. 15 16 17Usage 18----- 19 20First, you must edit the Kconfig to add the menu entries for the configs 21you are moving. 22 23And then run this tool giving CONFIG names you want to move. 24For example, if you want to move CONFIG_CMD_USB and CONFIG_SYS_TEXT_BASE, 25simply type as follows: 26 27 $ tools/moveconfig.py CONFIG_CMD_USB CONFIG_SYS_TEXT_BASE 28 29The tool walks through all the defconfig files and move the given CONFIGs. 30 31The log is also displayed on the terminal. 32 33The log is printed for each defconfig as follows: 34 35<defconfig_name> 36 <action1> 37 <action2> 38 <action3> 39 ... 40 41<defconfig_name> is the name of the defconfig. 42 43<action*> shows what the tool did for that defconfig. 44It looks like one of the followings: 45 46 - Move 'CONFIG_... ' 47 This config option was moved to the defconfig 48 49 - CONFIG_... is not defined in Kconfig. Do nothing. 50 The entry for this CONFIG was not found in Kconfig. 51 There are two common cases: 52 - You forgot to create an entry for the CONFIG before running 53 this tool, or made a typo in a CONFIG passed to this tool. 54 - The entry was hidden due to unmet 'depends on'. 55 This is correct behavior. 56 57 - 'CONFIG_...' is the same as the define in Kconfig. Do nothing. 58 The define in the config header matched the one in Kconfig. 59 We do not need to touch it. 60 61 - Undefined. Do nothing. 62 This config option was not found in the config header. 63 Nothing to do. 64 65 - Compiler is missing. Do nothing. 66 The compiler specified for this architecture was not found 67 in your PATH environment. 68 (If -e option is passed, the tool exits immediately.) 69 70 - Failed to process. 71 An error occurred during processing this defconfig. Skipped. 72 (If -e option is passed, the tool exits immediately on error.) 73 74Finally, you will be asked, Clean up headers? [y/n]: 75 76If you say 'y' here, the unnecessary config defines are removed 77from the config headers (include/configs/*.h). 78It just uses the regex method, so you should not rely on it. 79Just in case, please do 'git diff' to see what happened. 80 81 82How does it work? 83----------------- 84 85This tool runs configuration and builds include/autoconf.mk for every 86defconfig. The config options defined in Kconfig appear in the .config 87file (unless they are hidden because of unmet dependency.) 88On the other hand, the config options defined by board headers are seen 89in include/autoconf.mk. The tool looks for the specified options in both 90of them to decide the appropriate action for the options. If the given 91config option is found in the .config, but its value does not match the 92one from the board header, the config option in the .config is replaced 93with the define in the board header. Then, the .config is synced by 94"make savedefconfig" and the defconfig is updated with it. 95 96For faster processing, this tool handles multi-threading. It creates 97separate build directories where the out-of-tree build is run. The 98temporary build directories are automatically created and deleted as 99needed. The number of threads are chosen based on the number of the CPU 100cores of your system although you can change it via -j (--jobs) option. 101 102 103Toolchains 104---------- 105 106Appropriate toolchain are necessary to generate include/autoconf.mk 107for all the architectures supported by U-Boot. Most of them are available 108at the kernel.org site, some are not provided by kernel.org. 109 110The default per-arch CROSS_COMPILE used by this tool is specified by 111the list below, CROSS_COMPILE. You may wish to update the list to 112use your own. Instead of modifying the list directly, you can give 113them via environments. 114 115 116Available options 117----------------- 118 119 -c, --color 120 Surround each portion of the log with escape sequences to display it 121 in color on the terminal. 122 123 -d, --defconfigs 124 Specify a file containing a list of defconfigs to move 125 126 -n, --dry-run 127 Perform a trial run that does not make any changes. It is useful to 128 see what is going to happen before one actually runs it. 129 130 -e, --exit-on-error 131 Exit immediately if Make exits with a non-zero status while processing 132 a defconfig file. 133 134 -s, --force-sync 135 Do "make savedefconfig" forcibly for all the defconfig files. 136 If not specified, "make savedefconfig" only occurs for cases 137 where at least one CONFIG was moved. 138 139 -H, --headers-only 140 Only cleanup the headers; skip the defconfig processing 141 142 -j, --jobs 143 Specify the number of threads to run simultaneously. If not specified, 144 the number of threads is the same as the number of CPU cores. 145 146 -v, --verbose 147 Show any build errors as boards are built 148 149To see the complete list of supported options, run 150 151 $ tools/moveconfig.py -h 152 153""" 154 155import filecmp 156import fnmatch 157import multiprocessing 158import optparse 159import os 160import re 161import shutil 162import subprocess 163import sys 164import tempfile 165import time 166 167SHOW_GNU_MAKE = 'scripts/show-gnu-make' 168SLEEP_TIME=0.03 169 170# Here is the list of cross-tools I use. 171# Most of them are available at kernel.org 172# (https://www.kernel.org/pub/tools/crosstool/files/bin/), except the followings: 173# arc: https://github.com/foss-for-synopsys-dwc-arc-processors/toolchain/releases 174# blackfin: http://sourceforge.net/projects/adi-toolchain/files/ 175# nds32: http://osdk.andestech.com/packages/nds32le-linux-glibc-v1.tgz 176# nios2: https://sourcery.mentor.com/GNUToolchain/subscription42545 177# sh: http://sourcery.mentor.com/public/gnu_toolchain/sh-linux-gnu 178# 179# openrisc kernel.org toolchain is out of date, download latest one from 180# http://opencores.org/or1k/OpenRISC_GNU_tool_chain#Prebuilt_versions 181CROSS_COMPILE = { 182 'arc': 'arc-linux-', 183 'aarch64': 'aarch64-linux-', 184 'arm': 'arm-unknown-linux-gnueabi-', 185 'avr32': 'avr32-linux-', 186 'blackfin': 'bfin-elf-', 187 'm68k': 'm68k-linux-', 188 'microblaze': 'microblaze-linux-', 189 'mips': 'mips-linux-', 190 'nds32': 'nds32le-linux-', 191 'nios2': 'nios2-linux-gnu-', 192 'openrisc': 'or1k-elf-', 193 'powerpc': 'powerpc-linux-', 194 'sh': 'sh-linux-gnu-', 195 'sparc': 'sparc-linux-', 196 'x86': 'i386-linux-' 197} 198 199STATE_IDLE = 0 200STATE_DEFCONFIG = 1 201STATE_AUTOCONF = 2 202STATE_SAVEDEFCONFIG = 3 203 204ACTION_MOVE = 0 205ACTION_NO_ENTRY = 1 206ACTION_NO_CHANGE = 2 207 208COLOR_BLACK = '0;30' 209COLOR_RED = '0;31' 210COLOR_GREEN = '0;32' 211COLOR_BROWN = '0;33' 212COLOR_BLUE = '0;34' 213COLOR_PURPLE = '0;35' 214COLOR_CYAN = '0;36' 215COLOR_LIGHT_GRAY = '0;37' 216COLOR_DARK_GRAY = '1;30' 217COLOR_LIGHT_RED = '1;31' 218COLOR_LIGHT_GREEN = '1;32' 219COLOR_YELLOW = '1;33' 220COLOR_LIGHT_BLUE = '1;34' 221COLOR_LIGHT_PURPLE = '1;35' 222COLOR_LIGHT_CYAN = '1;36' 223COLOR_WHITE = '1;37' 224 225### helper functions ### 226def get_devnull(): 227 """Get the file object of '/dev/null' device.""" 228 try: 229 devnull = subprocess.DEVNULL # py3k 230 except AttributeError: 231 devnull = open(os.devnull, 'wb') 232 return devnull 233 234def check_top_directory(): 235 """Exit if we are not at the top of source directory.""" 236 for f in ('README', 'Licenses'): 237 if not os.path.exists(f): 238 sys.exit('Please run at the top of source directory.') 239 240def check_clean_directory(): 241 """Exit if the source tree is not clean.""" 242 for f in ('.config', 'include/config'): 243 if os.path.exists(f): 244 sys.exit("source tree is not clean, please run 'make mrproper'") 245 246def get_make_cmd(): 247 """Get the command name of GNU Make. 248 249 U-Boot needs GNU Make for building, but the command name is not 250 necessarily "make". (for example, "gmake" on FreeBSD). 251 Returns the most appropriate command name on your system. 252 """ 253 process = subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE) 254 ret = process.communicate() 255 if process.returncode: 256 sys.exit('GNU Make not found') 257 return ret[0].rstrip() 258 259def color_text(color_enabled, color, string): 260 """Return colored string.""" 261 if color_enabled: 262 # LF should not be surrounded by the escape sequence. 263 # Otherwise, additional whitespace or line-feed might be printed. 264 return '\n'.join([ '\033[' + color + 'm' + s + '\033[0m' if s else '' 265 for s in string.split('\n') ]) 266 else: 267 return string 268 269def update_cross_compile(color_enabled): 270 """Update per-arch CROSS_COMPILE via environment variables 271 272 The default CROSS_COMPILE values are available 273 in the CROSS_COMPILE list above. 274 275 You can override them via environment variables 276 CROSS_COMPILE_{ARCH}. 277 278 For example, if you want to override toolchain prefixes 279 for ARM and PowerPC, you can do as follows in your shell: 280 281 export CROSS_COMPILE_ARM=... 282 export CROSS_COMPILE_POWERPC=... 283 284 Then, this function checks if specified compilers really exist in your 285 PATH environment. 286 """ 287 archs = [] 288 289 for arch in os.listdir('arch'): 290 if os.path.exists(os.path.join('arch', arch, 'Makefile')): 291 archs.append(arch) 292 293 # arm64 is a special case 294 archs.append('aarch64') 295 296 for arch in archs: 297 env = 'CROSS_COMPILE_' + arch.upper() 298 cross_compile = os.environ.get(env) 299 if not cross_compile: 300 cross_compile = CROSS_COMPILE.get(arch, '') 301 302 for path in os.environ["PATH"].split(os.pathsep): 303 gcc_path = os.path.join(path, cross_compile + 'gcc') 304 if os.path.isfile(gcc_path) and os.access(gcc_path, os.X_OK): 305 break 306 else: 307 print >> sys.stderr, color_text(color_enabled, COLOR_YELLOW, 308 'warning: %sgcc: not found in PATH. %s architecture boards will be skipped' 309 % (cross_compile, arch)) 310 cross_compile = None 311 312 CROSS_COMPILE[arch] = cross_compile 313 314def cleanup_one_header(header_path, patterns, dry_run): 315 """Clean regex-matched lines away from a file. 316 317 Arguments: 318 header_path: path to the cleaned file. 319 patterns: list of regex patterns. Any lines matching to these 320 patterns are deleted. 321 dry_run: make no changes, but still display log. 322 """ 323 with open(header_path) as f: 324 lines = f.readlines() 325 326 matched = [] 327 for i, line in enumerate(lines): 328 for pattern in patterns: 329 m = pattern.search(line) 330 if m: 331 print '%s: %s: %s' % (header_path, i + 1, line), 332 matched.append(i) 333 break 334 335 if dry_run or not matched: 336 return 337 338 with open(header_path, 'w') as f: 339 for i, line in enumerate(lines): 340 if not i in matched: 341 f.write(line) 342 343def cleanup_headers(configs, dry_run): 344 """Delete config defines from board headers. 345 346 Arguments: 347 configs: A list of CONFIGs to remove. 348 dry_run: make no changes, but still display log. 349 """ 350 while True: 351 choice = raw_input('Clean up headers? [y/n]: ').lower() 352 print choice 353 if choice == 'y' or choice == 'n': 354 break 355 356 if choice == 'n': 357 return 358 359 patterns = [] 360 for config in configs: 361 patterns.append(re.compile(r'#\s*define\s+%s\W' % config)) 362 patterns.append(re.compile(r'#\s*undef\s+%s\W' % config)) 363 364 for dir in 'include', 'arch', 'board': 365 for (dirpath, dirnames, filenames) in os.walk(dir): 366 for filename in filenames: 367 if not fnmatch.fnmatch(filename, '*~'): 368 cleanup_one_header(os.path.join(dirpath, filename), 369 patterns, dry_run) 370 371### classes ### 372class Progress: 373 374 """Progress Indicator""" 375 376 def __init__(self, total): 377 """Create a new progress indicator. 378 379 Arguments: 380 total: A number of defconfig files to process. 381 """ 382 self.current = 0 383 self.total = total 384 385 def inc(self): 386 """Increment the number of processed defconfig files.""" 387 388 self.current += 1 389 390 def show(self): 391 """Display the progress.""" 392 print ' %d defconfigs out of %d\r' % (self.current, self.total), 393 sys.stdout.flush() 394 395class KconfigParser: 396 397 """A parser of .config and include/autoconf.mk.""" 398 399 re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"') 400 re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"') 401 402 def __init__(self, configs, options, build_dir): 403 """Create a new parser. 404 405 Arguments: 406 configs: A list of CONFIGs to move. 407 options: option flags. 408 build_dir: Build directory. 409 """ 410 self.configs = configs 411 self.options = options 412 self.dotconfig = os.path.join(build_dir, '.config') 413 self.autoconf = os.path.join(build_dir, 'include', 'autoconf.mk') 414 self.config_autoconf = os.path.join(build_dir, 'include', 'config', 415 'auto.conf') 416 self.defconfig = os.path.join(build_dir, 'defconfig') 417 418 def get_cross_compile(self): 419 """Parse .config file and return CROSS_COMPILE. 420 421 Returns: 422 A string storing the compiler prefix for the architecture. 423 Return a NULL string for architectures that do not require 424 compiler prefix (Sandbox and native build is the case). 425 Return None if the specified compiler is missing in your PATH. 426 Caller should distinguish '' and None. 427 """ 428 arch = '' 429 cpu = '' 430 for line in open(self.dotconfig): 431 m = self.re_arch.match(line) 432 if m: 433 arch = m.group(1) 434 continue 435 m = self.re_cpu.match(line) 436 if m: 437 cpu = m.group(1) 438 439 if not arch: 440 return None 441 442 # fix-up for aarch64 443 if arch == 'arm' and cpu == 'armv8': 444 arch = 'aarch64' 445 446 return CROSS_COMPILE.get(arch, None) 447 448 def parse_one_config(self, config, dotconfig_lines, autoconf_lines): 449 """Parse .config, defconfig, include/autoconf.mk for one config. 450 451 This function looks for the config options in the lines from 452 defconfig, .config, and include/autoconf.mk in order to decide 453 which action should be taken for this defconfig. 454 455 Arguments: 456 config: CONFIG name to parse. 457 dotconfig_lines: lines from the .config file. 458 autoconf_lines: lines from the include/autoconf.mk file. 459 460 Returns: 461 A tupple of the action for this defconfig and the line 462 matched for the config. 463 """ 464 not_set = '# %s is not set' % config 465 466 for line in dotconfig_lines: 467 line = line.rstrip() 468 if line.startswith(config + '=') or line == not_set: 469 old_val = line 470 break 471 else: 472 return (ACTION_NO_ENTRY, config) 473 474 for line in autoconf_lines: 475 line = line.rstrip() 476 if line.startswith(config + '='): 477 new_val = line 478 break 479 else: 480 new_val = not_set 481 482 if old_val == new_val: 483 return (ACTION_NO_CHANGE, new_val) 484 485 # If this CONFIG is neither bool nor trisate 486 if old_val[-2:] != '=y' and old_val[-2:] != '=m' and old_val != not_set: 487 # tools/scripts/define2mk.sed changes '1' to 'y'. 488 # This is a problem if the CONFIG is int type. 489 # Check the type in Kconfig and handle it correctly. 490 if new_val[-2:] == '=y': 491 new_val = new_val[:-1] + '1' 492 493 return (ACTION_MOVE, new_val) 494 495 def update_dotconfig(self): 496 """Parse files for the config options and update the .config. 497 498 This function parses the generated .config and include/autoconf.mk 499 searching the target options. 500 Move the config option(s) to the .config as needed. 501 502 Arguments: 503 defconfig: defconfig name. 504 505 Returns: 506 Return a tuple of (updated flag, log string). 507 The "updated flag" is True if the .config was updated, False 508 otherwise. The "log string" shows what happend to the .config. 509 """ 510 511 results = [] 512 updated = False 513 514 with open(self.dotconfig) as f: 515 dotconfig_lines = f.readlines() 516 517 with open(self.autoconf) as f: 518 autoconf_lines = f.readlines() 519 520 for config in self.configs: 521 result = self.parse_one_config(config, dotconfig_lines, 522 autoconf_lines) 523 results.append(result) 524 525 log = '' 526 527 for (action, value) in results: 528 if action == ACTION_MOVE: 529 actlog = "Move '%s'" % value 530 log_color = COLOR_LIGHT_GREEN 531 elif action == ACTION_NO_ENTRY: 532 actlog = "%s is not defined in Kconfig. Do nothing." % value 533 log_color = COLOR_LIGHT_BLUE 534 elif action == ACTION_NO_CHANGE: 535 actlog = "'%s' is the same as the define in Kconfig. Do nothing." \ 536 % value 537 log_color = COLOR_LIGHT_PURPLE 538 else: 539 sys.exit("Internal Error. This should not happen.") 540 541 log += color_text(self.options.color, log_color, actlog) + '\n' 542 543 with open(self.dotconfig, 'a') as f: 544 for (action, value) in results: 545 if action == ACTION_MOVE: 546 f.write(value + '\n') 547 updated = True 548 549 self.results = results 550 os.remove(self.config_autoconf) 551 os.remove(self.autoconf) 552 553 return (updated, log) 554 555 def check_defconfig(self): 556 """Check the defconfig after savedefconfig 557 558 Returns: 559 Return additional log if moved CONFIGs were removed again by 560 'make savedefconfig'. 561 """ 562 563 log = '' 564 565 with open(self.defconfig) as f: 566 defconfig_lines = f.readlines() 567 568 for (action, value) in self.results: 569 if action != ACTION_MOVE: 570 continue 571 if not value + '\n' in defconfig_lines: 572 log += color_text(self.options.color, COLOR_YELLOW, 573 "'%s' was removed by savedefconfig.\n" % 574 value) 575 576 return log 577 578class Slot: 579 580 """A slot to store a subprocess. 581 582 Each instance of this class handles one subprocess. 583 This class is useful to control multiple threads 584 for faster processing. 585 """ 586 587 def __init__(self, configs, options, progress, devnull, make_cmd): 588 """Create a new process slot. 589 590 Arguments: 591 configs: A list of CONFIGs to move. 592 options: option flags. 593 progress: A progress indicator. 594 devnull: A file object of '/dev/null'. 595 make_cmd: command name of GNU Make. 596 """ 597 self.options = options 598 self.progress = progress 599 self.build_dir = tempfile.mkdtemp() 600 self.devnull = devnull 601 self.make_cmd = (make_cmd, 'O=' + self.build_dir) 602 self.parser = KconfigParser(configs, options, self.build_dir) 603 self.state = STATE_IDLE 604 self.failed_boards = [] 605 606 def __del__(self): 607 """Delete the working directory 608 609 This function makes sure the temporary directory is cleaned away 610 even if Python suddenly dies due to error. It should be done in here 611 because it is guranteed the destructor is always invoked when the 612 instance of the class gets unreferenced. 613 614 If the subprocess is still running, wait until it finishes. 615 """ 616 if self.state != STATE_IDLE: 617 while self.ps.poll() == None: 618 pass 619 shutil.rmtree(self.build_dir) 620 621 def add(self, defconfig): 622 """Assign a new subprocess for defconfig and add it to the slot. 623 624 If the slot is vacant, create a new subprocess for processing the 625 given defconfig and add it to the slot. Just returns False if 626 the slot is occupied (i.e. the current subprocess is still running). 627 628 Arguments: 629 defconfig: defconfig name. 630 631 Returns: 632 Return True on success or False on failure 633 """ 634 if self.state != STATE_IDLE: 635 return False 636 637 self.defconfig = defconfig 638 self.log = '' 639 self.do_defconfig() 640 return True 641 642 def poll(self): 643 """Check the status of the subprocess and handle it as needed. 644 645 Returns True if the slot is vacant (i.e. in idle state). 646 If the configuration is successfully finished, assign a new 647 subprocess to build include/autoconf.mk. 648 If include/autoconf.mk is generated, invoke the parser to 649 parse the .config and the include/autoconf.mk, moving 650 config options to the .config as needed. 651 If the .config was updated, run "make savedefconfig" to sync 652 it, update the original defconfig, and then set the slot back 653 to the idle state. 654 655 Returns: 656 Return True if the subprocess is terminated, False otherwise 657 """ 658 if self.state == STATE_IDLE: 659 return True 660 661 if self.ps.poll() == None: 662 return False 663 664 if self.ps.poll() != 0: 665 self.handle_error() 666 elif self.state == STATE_DEFCONFIG: 667 self.do_autoconf() 668 elif self.state == STATE_AUTOCONF: 669 self.do_savedefconfig() 670 elif self.state == STATE_SAVEDEFCONFIG: 671 self.update_defconfig() 672 else: 673 sys.exit("Internal Error. This should not happen.") 674 675 return True if self.state == STATE_IDLE else False 676 677 def handle_error(self): 678 """Handle error cases.""" 679 680 self.log += color_text(self.options.color, COLOR_LIGHT_RED, 681 "Failed to process.\n") 682 if self.options.verbose: 683 self.log += color_text(self.options.color, COLOR_LIGHT_CYAN, 684 self.ps.stderr.read()) 685 self.finish(False) 686 687 def do_defconfig(self): 688 """Run 'make <board>_defconfig' to create the .config file.""" 689 690 cmd = list(self.make_cmd) 691 cmd.append(self.defconfig) 692 self.ps = subprocess.Popen(cmd, stdout=self.devnull, 693 stderr=subprocess.PIPE) 694 self.state = STATE_DEFCONFIG 695 696 def do_autoconf(self): 697 """Run 'make include/config/auto.conf'.""" 698 699 self.cross_compile = self.parser.get_cross_compile() 700 if self.cross_compile is None: 701 self.log += color_text(self.options.color, COLOR_YELLOW, 702 "Compiler is missing. Do nothing.\n") 703 self.finish(False) 704 return 705 706 cmd = list(self.make_cmd) 707 if self.cross_compile: 708 cmd.append('CROSS_COMPILE=%s' % self.cross_compile) 709 cmd.append('KCONFIG_IGNORE_DUPLICATES=1') 710 cmd.append('include/config/auto.conf') 711 self.ps = subprocess.Popen(cmd, stdout=self.devnull, 712 stderr=subprocess.PIPE) 713 self.state = STATE_AUTOCONF 714 715 def do_savedefconfig(self): 716 """Update the .config and run 'make savedefconfig'.""" 717 718 (updated, log) = self.parser.update_dotconfig() 719 self.log += log 720 721 if not self.options.force_sync and not updated: 722 self.finish(True) 723 return 724 if updated: 725 self.log += color_text(self.options.color, COLOR_LIGHT_GREEN, 726 "Syncing by savedefconfig...\n") 727 else: 728 self.log += "Syncing by savedefconfig (forced by option)...\n" 729 730 cmd = list(self.make_cmd) 731 cmd.append('savedefconfig') 732 self.ps = subprocess.Popen(cmd, stdout=self.devnull, 733 stderr=subprocess.PIPE) 734 self.state = STATE_SAVEDEFCONFIG 735 736 def update_defconfig(self): 737 """Update the input defconfig and go back to the idle state.""" 738 739 self.log += self.parser.check_defconfig() 740 orig_defconfig = os.path.join('configs', self.defconfig) 741 new_defconfig = os.path.join(self.build_dir, 'defconfig') 742 updated = not filecmp.cmp(orig_defconfig, new_defconfig) 743 744 if updated: 745 self.log += color_text(self.options.color, COLOR_LIGHT_GREEN, 746 "defconfig was updated.\n") 747 748 if not self.options.dry_run and updated: 749 shutil.move(new_defconfig, orig_defconfig) 750 self.finish(True) 751 752 def finish(self, success): 753 """Display log along with progress and go to the idle state. 754 755 Arguments: 756 success: Should be True when the defconfig was processed 757 successfully, or False when it fails. 758 """ 759 # output at least 30 characters to hide the "* defconfigs out of *". 760 log = self.defconfig.ljust(30) + '\n' 761 762 log += '\n'.join([ ' ' + s for s in self.log.split('\n') ]) 763 # Some threads are running in parallel. 764 # Print log atomically to not mix up logs from different threads. 765 print >> (sys.stdout if success else sys.stderr), log 766 767 if not success: 768 if self.options.exit_on_error: 769 sys.exit("Exit on error.") 770 # If --exit-on-error flag is not set, skip this board and continue. 771 # Record the failed board. 772 self.failed_boards.append(self.defconfig) 773 774 self.progress.inc() 775 self.progress.show() 776 self.state = STATE_IDLE 777 778 def get_failed_boards(self): 779 """Returns a list of failed boards (defconfigs) in this slot. 780 """ 781 return self.failed_boards 782 783class Slots: 784 785 """Controller of the array of subprocess slots.""" 786 787 def __init__(self, configs, options, progress): 788 """Create a new slots controller. 789 790 Arguments: 791 configs: A list of CONFIGs to move. 792 options: option flags. 793 progress: A progress indicator. 794 """ 795 self.options = options 796 self.slots = [] 797 devnull = get_devnull() 798 make_cmd = get_make_cmd() 799 for i in range(options.jobs): 800 self.slots.append(Slot(configs, options, progress, devnull, 801 make_cmd)) 802 803 def add(self, defconfig): 804 """Add a new subprocess if a vacant slot is found. 805 806 Arguments: 807 defconfig: defconfig name to be put into. 808 809 Returns: 810 Return True on success or False on failure 811 """ 812 for slot in self.slots: 813 if slot.add(defconfig): 814 return True 815 return False 816 817 def available(self): 818 """Check if there is a vacant slot. 819 820 Returns: 821 Return True if at lease one vacant slot is found, False otherwise. 822 """ 823 for slot in self.slots: 824 if slot.poll(): 825 return True 826 return False 827 828 def empty(self): 829 """Check if all slots are vacant. 830 831 Returns: 832 Return True if all the slots are vacant, False otherwise. 833 """ 834 ret = True 835 for slot in self.slots: 836 if not slot.poll(): 837 ret = False 838 return ret 839 840 def show_failed_boards(self): 841 """Display all of the failed boards (defconfigs).""" 842 failed_boards = [] 843 844 for slot in self.slots: 845 failed_boards += slot.get_failed_boards() 846 847 if len(failed_boards) > 0: 848 msg = [ "The following boards were not processed due to error:" ] 849 msg += failed_boards 850 for line in msg: 851 print >> sys.stderr, color_text(self.options.color, 852 COLOR_LIGHT_RED, line) 853 854 with open('moveconfig.failed', 'w') as f: 855 for board in failed_boards: 856 f.write(board + '\n') 857 858def move_config(configs, options): 859 """Move config options to defconfig files. 860 861 Arguments: 862 configs: A list of CONFIGs to move. 863 options: option flags 864 """ 865 if len(configs) == 0: 866 if options.force_sync: 867 print 'No CONFIG is specified. You are probably syncing defconfigs.', 868 else: 869 print 'Neither CONFIG nor --force-sync is specified. Nothing will happen.', 870 else: 871 print 'Move ' + ', '.join(configs), 872 print '(jobs: %d)\n' % options.jobs 873 874 if options.defconfigs: 875 defconfigs = [line.strip() for line in open(options.defconfigs)] 876 for i, defconfig in enumerate(defconfigs): 877 if not defconfig.endswith('_defconfig'): 878 defconfigs[i] = defconfig + '_defconfig' 879 if not os.path.exists(os.path.join('configs', defconfigs[i])): 880 sys.exit('%s - defconfig does not exist. Stopping.' % 881 defconfigs[i]) 882 else: 883 # All the defconfig files to be processed 884 defconfigs = [] 885 for (dirpath, dirnames, filenames) in os.walk('configs'): 886 dirpath = dirpath[len('configs') + 1:] 887 for filename in fnmatch.filter(filenames, '*_defconfig'): 888 defconfigs.append(os.path.join(dirpath, filename)) 889 890 progress = Progress(len(defconfigs)) 891 slots = Slots(configs, options, progress) 892 893 # Main loop to process defconfig files: 894 # Add a new subprocess into a vacant slot. 895 # Sleep if there is no available slot. 896 for defconfig in defconfigs: 897 while not slots.add(defconfig): 898 while not slots.available(): 899 # No available slot: sleep for a while 900 time.sleep(SLEEP_TIME) 901 902 # wait until all the subprocesses finish 903 while not slots.empty(): 904 time.sleep(SLEEP_TIME) 905 906 print '' 907 slots.show_failed_boards() 908 909def main(): 910 try: 911 cpu_count = multiprocessing.cpu_count() 912 except NotImplementedError: 913 cpu_count = 1 914 915 parser = optparse.OptionParser() 916 # Add options here 917 parser.add_option('-c', '--color', action='store_true', default=False, 918 help='display the log in color') 919 parser.add_option('-d', '--defconfigs', type='string', 920 help='a file containing a list of defconfigs to move') 921 parser.add_option('-n', '--dry-run', action='store_true', default=False, 922 help='perform a trial run (show log with no changes)') 923 parser.add_option('-e', '--exit-on-error', action='store_true', 924 default=False, 925 help='exit immediately on any error') 926 parser.add_option('-s', '--force-sync', action='store_true', default=False, 927 help='force sync by savedefconfig') 928 parser.add_option('-H', '--headers-only', dest='cleanup_headers_only', 929 action='store_true', default=False, 930 help='only cleanup the headers') 931 parser.add_option('-j', '--jobs', type='int', default=cpu_count, 932 help='the number of jobs to run simultaneously') 933 parser.add_option('-v', '--verbose', action='store_true', default=False, 934 help='show any build errors as boards are built') 935 parser.usage += ' CONFIG ...' 936 937 (options, configs) = parser.parse_args() 938 939 if len(configs) == 0 and not options.force_sync: 940 parser.print_usage() 941 sys.exit(1) 942 943 # prefix the option name with CONFIG_ if missing 944 configs = [ config if config.startswith('CONFIG_') else 'CONFIG_' + config 945 for config in configs ] 946 947 check_top_directory() 948 949 check_clean_directory() 950 951 update_cross_compile(options.color) 952 953 if not options.cleanup_headers_only: 954 move_config(configs, options) 955 956 if configs: 957 cleanup_headers(configs, options.dry_run) 958 959if __name__ == '__main__': 960 main() 961