1#!/usr/bin/env python2 2# 3# Author: Masahiro Yamada <yamada.masahiro@socionext.com> 4# 5# SPDX-License-Identifier: GPL-2.0+ 6# 7 8""" 9Move config options from headers to defconfig files. 10 11Since Kconfig was introduced to U-Boot, we have worked on moving 12config options from headers to Kconfig (defconfig). 13 14This tool intends to help this tremendous work. 15 16 17Usage 18----- 19 20This tool takes one input file. (let's say 'recipe' file here.) 21The recipe describes the list of config options you want to move. 22Each line takes the form: 23<config_name> <type> <default> 24(the fields must be separated with whitespaces.) 25 26<config_name> is the name of config option. 27 28<type> is the type of the option. It must be one of bool, tristate, 29string, int, and hex. 30 31<default> is the default value of the option. It must be appropriate 32value corresponding to the option type. It must be either y or n for 33the bool type. Tristate options can also take m (although U-Boot has 34not supported the module feature). 35 36You can add two or more lines in the recipe file, so you can move 37multiple options at once. 38 39Let's say, for example, you want to move CONFIG_CMD_USB and 40CONFIG_SYS_TEXT_BASE. 41 42The type should be bool, hex, respectively. So, the recipe file 43should look like this: 44 45 $ cat recipe 46 CONFIG_CMD_USB bool n 47 CONFIG_SYS_TEXT_BASE hex 0x00000000 48 49Next you must edit the Kconfig to add the menu entries for the configs 50you are moving. 51 52And then run this tool giving the file name of the recipe 53 54 $ tools/moveconfig.py recipe 55 56The tool walks through all the defconfig files to move the config 57options specified by the recipe file. 58 59The log is also displayed on the terminal. 60 61Each line is printed in the format 62<defconfig_name> : <action> 63 64<defconfig_name> is the name of the defconfig 65(without the suffix _defconfig). 66 67<action> shows what the tool did for that defconfig. 68It looks like one of the followings: 69 70 - Move 'CONFIG_... ' 71 This config option was moved to the defconfig 72 73 - CONFIG_... is not defined in Kconfig. Do nothing. 74 The entry for this CONFIG was not found in Kconfig. 75 There are two common cases: 76 - You forgot to create an entry for the CONFIG before running 77 this tool, or made a typo in a CONFIG passed to this tool. 78 - The entry was hidden due to unmet 'depends on'. 79 This is correct behavior. 80 81 - 'CONFIG_...' is the same as the define in Kconfig. Do nothing. 82 The define in the config header matched the one in Kconfig. 83 We do not need to touch it. 84 85 - Undefined. Do nothing. 86 This config option was not found in the config header. 87 Nothing to do. 88 89 - Compiler is missing. Do nothing. 90 The compiler specified for this architecture was not found 91 in your PATH environment. 92 (If -e option is passed, the tool exits immediately.) 93 94 - Failed to process. 95 An error occurred during processing this defconfig. Skipped. 96 (If -e option is passed, the tool exits immediately on error.) 97 98Finally, you will be asked, Clean up headers? [y/n]: 99 100If you say 'y' here, the unnecessary config defines are removed 101from the config headers (include/configs/*.h). 102It just uses the regex method, so you should not rely on it. 103Just in case, please do 'git diff' to see what happened. 104 105 106How does it works? 107------------------ 108 109This tool runs configuration and builds include/autoconf.mk for every 110defconfig. The config options defined in Kconfig appear in the .config 111file (unless they are hidden because of unmet dependency.) 112On the other hand, the config options defined by board headers are seen 113in include/autoconf.mk. The tool looks for the specified options in both 114of them to decide the appropriate action for the options. If the option 115is found in the .config or the value is the same as the specified default, 116the option does not need to be touched. If the option is found in 117include/autoconf.mk, but not in the .config, and the value is different 118from the default, the tools adds the option to the defconfig. 119 120For faster processing, this tool handles multi-threading. It creates 121separate build directories where the out-of-tree build is run. The 122temporary build directories are automatically created and deleted as 123needed. The number of threads are chosen based on the number of the CPU 124cores of your system although you can change it via -j (--jobs) option. 125 126 127Toolchains 128---------- 129 130Appropriate toolchain are necessary to generate include/autoconf.mk 131for all the architectures supported by U-Boot. Most of them are available 132at the kernel.org site, some are not provided by kernel.org. 133 134The default per-arch CROSS_COMPILE used by this tool is specified by 135the list below, CROSS_COMPILE. You may wish to update the list to 136use your own. Instead of modifying the list directly, you can give 137them via environments. 138 139 140Available options 141----------------- 142 143 -c, --color 144 Surround each portion of the log with escape sequences to display it 145 in color on the terminal. 146 147 -d, --defconfigs 148 Specify a file containing a list of defconfigs to move 149 150 -n, --dry-run 151 Peform a trial run that does not make any changes. It is useful to 152 see what is going to happen before one actually runs it. 153 154 -e, --exit-on-error 155 Exit immediately if Make exits with a non-zero status while processing 156 a defconfig file. 157 158 -H, --headers-only 159 Only cleanup the headers; skip the defconfig processing 160 161 -j, --jobs 162 Specify the number of threads to run simultaneously. If not specified, 163 the number of threads is the same as the number of CPU cores. 164 165 -v, --verbose 166 Show any build errors as boards are built 167 168To see the complete list of supported options, run 169 170 $ tools/moveconfig.py -h 171 172""" 173 174import fnmatch 175import multiprocessing 176import optparse 177import os 178import re 179import shutil 180import subprocess 181import sys 182import tempfile 183import time 184 185SHOW_GNU_MAKE = 'scripts/show-gnu-make' 186SLEEP_TIME=0.03 187 188# Here is the list of cross-tools I use. 189# Most of them are available at kernel.org 190# (https://www.kernel.org/pub/tools/crosstool/files/bin/), except the followings: 191# arc: https://github.com/foss-for-synopsys-dwc-arc-processors/toolchain/releases 192# blackfin: http://sourceforge.net/projects/adi-toolchain/files/ 193# nds32: http://osdk.andestech.com/packages/nds32le-linux-glibc-v1.tgz 194# nios2: https://sourcery.mentor.com/GNUToolchain/subscription42545 195# sh: http://sourcery.mentor.com/public/gnu_toolchain/sh-linux-gnu 196# 197# openrisc kernel.org toolchain is out of date, download latest one from 198# http://opencores.org/or1k/OpenRISC_GNU_tool_chain#Prebuilt_versions 199CROSS_COMPILE = { 200 'arc': 'arc-linux-', 201 'aarch64': 'aarch64-linux-', 202 'arm': 'arm-unknown-linux-gnueabi-', 203 'avr32': 'avr32-linux-', 204 'blackfin': 'bfin-elf-', 205 'm68k': 'm68k-linux-', 206 'microblaze': 'microblaze-linux-', 207 'mips': 'mips-linux-', 208 'nds32': 'nds32le-linux-', 209 'nios2': 'nios2-linux-gnu-', 210 'openrisc': 'or1k-elf-', 211 'powerpc': 'powerpc-linux-', 212 'sh': 'sh-linux-gnu-', 213 'sparc': 'sparc-linux-', 214 'x86': 'i386-linux-' 215} 216 217STATE_IDLE = 0 218STATE_DEFCONFIG = 1 219STATE_AUTOCONF = 2 220STATE_SAVEDEFCONFIG = 3 221 222ACTION_MOVE = 0 223ACTION_NO_ENTRY = 1 224ACTION_NO_CHANGE = 2 225 226COLOR_BLACK = '0;30' 227COLOR_RED = '0;31' 228COLOR_GREEN = '0;32' 229COLOR_BROWN = '0;33' 230COLOR_BLUE = '0;34' 231COLOR_PURPLE = '0;35' 232COLOR_CYAN = '0;36' 233COLOR_LIGHT_GRAY = '0;37' 234COLOR_DARK_GRAY = '1;30' 235COLOR_LIGHT_RED = '1;31' 236COLOR_LIGHT_GREEN = '1;32' 237COLOR_YELLOW = '1;33' 238COLOR_LIGHT_BLUE = '1;34' 239COLOR_LIGHT_PURPLE = '1;35' 240COLOR_LIGHT_CYAN = '1;36' 241COLOR_WHITE = '1;37' 242 243### helper functions ### 244def get_devnull(): 245 """Get the file object of '/dev/null' device.""" 246 try: 247 devnull = subprocess.DEVNULL # py3k 248 except AttributeError: 249 devnull = open(os.devnull, 'wb') 250 return devnull 251 252def check_top_directory(): 253 """Exit if we are not at the top of source directory.""" 254 for f in ('README', 'Licenses'): 255 if not os.path.exists(f): 256 sys.exit('Please run at the top of source directory.') 257 258def check_clean_directory(): 259 """Exit if the source tree is not clean.""" 260 for f in ('.config', 'include/config'): 261 if os.path.exists(f): 262 sys.exit("source tree is not clean, please run 'make mrproper'") 263 264def get_make_cmd(): 265 """Get the command name of GNU Make. 266 267 U-Boot needs GNU Make for building, but the command name is not 268 necessarily "make". (for example, "gmake" on FreeBSD). 269 Returns the most appropriate command name on your system. 270 """ 271 process = subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE) 272 ret = process.communicate() 273 if process.returncode: 274 sys.exit('GNU Make not found') 275 return ret[0].rstrip() 276 277def color_text(color_enabled, color, string): 278 """Return colored string.""" 279 if color_enabled: 280 return '\033[' + color + 'm' + string + '\033[0m' 281 else: 282 return string 283 284def log_msg(color_enabled, color, defconfig, msg): 285 """Return the formated line for the log.""" 286 return defconfig[:-len('_defconfig')].ljust(37) + ': ' + \ 287 color_text(color_enabled, color, msg) + '\n' 288 289def update_cross_compile(color_enabled): 290 """Update per-arch CROSS_COMPILE via environment variables 291 292 The default CROSS_COMPILE values are available 293 in the CROSS_COMPILE list above. 294 295 You can override them via environment variables 296 CROSS_COMPILE_{ARCH}. 297 298 For example, if you want to override toolchain prefixes 299 for ARM and PowerPC, you can do as follows in your shell: 300 301 export CROSS_COMPILE_ARM=... 302 export CROSS_COMPILE_POWERPC=... 303 304 Then, this function checks if specified compilers really exist in your 305 PATH environment. 306 """ 307 archs = [] 308 309 for arch in os.listdir('arch'): 310 if os.path.exists(os.path.join('arch', arch, 'Makefile')): 311 archs.append(arch) 312 313 # arm64 is a special case 314 archs.append('aarch64') 315 316 for arch in archs: 317 env = 'CROSS_COMPILE_' + arch.upper() 318 cross_compile = os.environ.get(env) 319 if not cross_compile: 320 cross_compile = CROSS_COMPILE.get(arch, '') 321 322 for path in os.environ["PATH"].split(os.pathsep): 323 gcc_path = os.path.join(path, cross_compile + 'gcc') 324 if os.path.isfile(gcc_path) and os.access(gcc_path, os.X_OK): 325 break 326 else: 327 print >> sys.stderr, color_text(color_enabled, COLOR_YELLOW, 328 'warning: %sgcc: not found in PATH. %s architecture boards will be skipped' 329 % (cross_compile, arch)) 330 cross_compile = None 331 332 CROSS_COMPILE[arch] = cross_compile 333 334def cleanup_one_header(header_path, patterns, dry_run): 335 """Clean regex-matched lines away from a file. 336 337 Arguments: 338 header_path: path to the cleaned file. 339 patterns: list of regex patterns. Any lines matching to these 340 patterns are deleted. 341 dry_run: make no changes, but still display log. 342 """ 343 with open(header_path) as f: 344 lines = f.readlines() 345 346 matched = [] 347 for i, line in enumerate(lines): 348 for pattern in patterns: 349 m = pattern.search(line) 350 if m: 351 print '%s: %s: %s' % (header_path, i + 1, line), 352 matched.append(i) 353 break 354 355 if dry_run or not matched: 356 return 357 358 with open(header_path, 'w') as f: 359 for i, line in enumerate(lines): 360 if not i in matched: 361 f.write(line) 362 363def cleanup_headers(configs, dry_run): 364 """Delete config defines from board headers. 365 366 Arguments: 367 configs: A list of CONFIGs to remove. 368 dry_run: make no changes, but still display log. 369 """ 370 while True: 371 choice = raw_input('Clean up headers? [y/n]: ').lower() 372 print choice 373 if choice == 'y' or choice == 'n': 374 break 375 376 if choice == 'n': 377 return 378 379 patterns = [] 380 for config in configs: 381 patterns.append(re.compile(r'#\s*define\s+%s\W' % config)) 382 patterns.append(re.compile(r'#\s*undef\s+%s\W' % config)) 383 384 for dir in 'include', 'arch', 'board': 385 for (dirpath, dirnames, filenames) in os.walk(dir): 386 for filename in filenames: 387 if not fnmatch.fnmatch(filename, '*~'): 388 cleanup_one_header(os.path.join(dirpath, filename), 389 patterns, dry_run) 390 391### classes ### 392class Progress: 393 394 """Progress Indicator""" 395 396 def __init__(self, total): 397 """Create a new progress indicator. 398 399 Arguments: 400 total: A number of defconfig files to process. 401 """ 402 self.current = 0 403 self.total = total 404 405 def inc(self): 406 """Increment the number of processed defconfig files.""" 407 408 self.current += 1 409 410 def show(self): 411 """Display the progress.""" 412 print ' %d defconfigs out of %d\r' % (self.current, self.total), 413 sys.stdout.flush() 414 415class KconfigParser: 416 417 """A parser of .config and include/autoconf.mk.""" 418 419 re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"') 420 re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"') 421 422 def __init__(self, configs, options, progress, build_dir): 423 """Create a new parser. 424 425 Arguments: 426 configs: A list of CONFIGs to move. 427 options: option flags. 428 progress: A progress indicator 429 build_dir: Build directory. 430 """ 431 self.configs = configs 432 self.options = options 433 self.progress = progress 434 self.build_dir = build_dir 435 436 def get_cross_compile(self): 437 """Parse .config file and return CROSS_COMPILE. 438 439 Returns: 440 A string storing the compiler prefix for the architecture. 441 Return a NULL string for architectures that do not require 442 compiler prefix (Sandbox and native build is the case). 443 Return None if the specified compiler is missing in your PATH. 444 Caller should distinguish '' and None. 445 """ 446 arch = '' 447 cpu = '' 448 dotconfig = os.path.join(self.build_dir, '.config') 449 for line in open(dotconfig): 450 m = self.re_arch.match(line) 451 if m: 452 arch = m.group(1) 453 continue 454 m = self.re_cpu.match(line) 455 if m: 456 cpu = m.group(1) 457 458 if not arch: 459 return None 460 461 # fix-up for aarch64 462 if arch == 'arm' and cpu == 'armv8': 463 arch = 'aarch64' 464 465 return CROSS_COMPILE.get(arch, None) 466 467 def parse_one_config(self, config, dotconfig_lines, autoconf_lines): 468 """Parse .config, defconfig, include/autoconf.mk for one config. 469 470 This function looks for the config options in the lines from 471 defconfig, .config, and include/autoconf.mk in order to decide 472 which action should be taken for this defconfig. 473 474 Arguments: 475 config: CONFIG name to parse. 476 dotconfig_lines: lines from the .config file. 477 autoconf_lines: lines from the include/autoconf.mk file. 478 479 Returns: 480 A tupple of the action for this defconfig and the line 481 matched for the config. 482 """ 483 not_set = '# %s is not set' % config 484 485 for line in dotconfig_lines: 486 line = line.rstrip() 487 if line.startswith(config + '=') or line == not_set: 488 old_val = line 489 break 490 else: 491 return (ACTION_NO_ENTRY, config) 492 493 for line in autoconf_lines: 494 line = line.rstrip() 495 if line.startswith(config + '='): 496 new_val = line 497 break 498 else: 499 new_val = not_set 500 501 if old_val == new_val: 502 return (ACTION_NO_CHANGE, new_val) 503 504 # If this CONFIG is neither bool nor trisate 505 if old_val[-2:] != '=y' and old_val[-2:] != '=m' and old_val != not_set: 506 # tools/scripts/define2mk.sed changes '1' to 'y'. 507 # This is a problem if the CONFIG is int type. 508 # Check the type in Kconfig and handle it correctly. 509 if new_val[-2:] == '=y': 510 new_val = new_val[:-1] + '1' 511 512 return (ACTION_MOVE, new_val) 513 514 def update_dotconfig(self, defconfig): 515 """Parse files for the config options and update the .config. 516 517 This function parses the generated .config and include/autoconf.mk 518 searching the target options. 519 Move the config option(s) to the .config as needed. 520 Also, display the log to show what happened to the .config. 521 522 Arguments: 523 defconfig: defconfig name. 524 """ 525 526 dotconfig_path = os.path.join(self.build_dir, '.config') 527 autoconf_path = os.path.join(self.build_dir, 'include', 'autoconf.mk') 528 results = [] 529 530 with open(dotconfig_path) as f: 531 dotconfig_lines = f.readlines() 532 533 with open(autoconf_path) as f: 534 autoconf_lines = f.readlines() 535 536 for config in self.configs: 537 result = self.parse_one_config(config, dotconfig_lines, 538 autoconf_lines) 539 results.append(result) 540 541 log = '' 542 543 for (action, value) in results: 544 if action == ACTION_MOVE: 545 actlog = "Move '%s'" % value 546 log_color = COLOR_LIGHT_GREEN 547 elif action == ACTION_NO_ENTRY: 548 actlog = "%s is not defined in Kconfig. Do nothing." % value 549 log_color = COLOR_LIGHT_BLUE 550 elif action == ACTION_NO_CHANGE: 551 actlog = "'%s' is the same as the define in Kconfig. Do nothing." \ 552 % value 553 log_color = COLOR_LIGHT_PURPLE 554 else: 555 sys.exit("Internal Error. This should not happen.") 556 557 log += log_msg(self.options.color, log_color, defconfig, actlog) 558 559 # Some threads are running in parallel. 560 # Print log in one shot to not mix up logs from different threads. 561 print log, 562 self.progress.show() 563 564 with open(dotconfig_path, 'a') as f: 565 for (action, value) in results: 566 if action == ACTION_MOVE: 567 f.write(value + '\n') 568 569 os.remove(os.path.join(self.build_dir, 'include', 'config', 'auto.conf')) 570 os.remove(autoconf_path) 571 572class Slot: 573 574 """A slot to store a subprocess. 575 576 Each instance of this class handles one subprocess. 577 This class is useful to control multiple threads 578 for faster processing. 579 """ 580 581 def __init__(self, configs, options, progress, devnull, make_cmd): 582 """Create a new process slot. 583 584 Arguments: 585 configs: A list of CONFIGs to move. 586 options: option flags. 587 progress: A progress indicator. 588 devnull: A file object of '/dev/null'. 589 make_cmd: command name of GNU Make. 590 """ 591 self.options = options 592 self.progress = progress 593 self.build_dir = tempfile.mkdtemp() 594 self.devnull = devnull 595 self.make_cmd = (make_cmd, 'O=' + self.build_dir) 596 self.parser = KconfigParser(configs, options, progress, self.build_dir) 597 self.state = STATE_IDLE 598 self.failed_boards = [] 599 600 def __del__(self): 601 """Delete the working directory 602 603 This function makes sure the temporary directory is cleaned away 604 even if Python suddenly dies due to error. It should be done in here 605 because it is guranteed the destructor is always invoked when the 606 instance of the class gets unreferenced. 607 608 If the subprocess is still running, wait until it finishes. 609 """ 610 if self.state != STATE_IDLE: 611 while self.ps.poll() == None: 612 pass 613 shutil.rmtree(self.build_dir) 614 615 def add(self, defconfig): 616 """Assign a new subprocess for defconfig and add it to the slot. 617 618 If the slot is vacant, create a new subprocess for processing the 619 given defconfig and add it to the slot. Just returns False if 620 the slot is occupied (i.e. the current subprocess is still running). 621 622 Arguments: 623 defconfig: defconfig name. 624 625 Returns: 626 Return True on success or False on failure 627 """ 628 if self.state != STATE_IDLE: 629 return False 630 cmd = list(self.make_cmd) 631 cmd.append(defconfig) 632 self.ps = subprocess.Popen(cmd, stdout=self.devnull, 633 stderr=subprocess.PIPE) 634 self.defconfig = defconfig 635 self.state = STATE_DEFCONFIG 636 return True 637 638 def poll(self): 639 """Check the status of the subprocess and handle it as needed. 640 641 Returns True if the slot is vacant (i.e. in idle state). 642 If the configuration is successfully finished, assign a new 643 subprocess to build include/autoconf.mk. 644 If include/autoconf.mk is generated, invoke the parser to 645 parse the .config and the include/autoconf.mk, and then set the 646 slot back to the idle state. 647 648 Returns: 649 Return True if the subprocess is terminated, False otherwise 650 """ 651 if self.state == STATE_IDLE: 652 return True 653 654 if self.ps.poll() == None: 655 return False 656 657 if self.ps.poll() != 0: 658 print >> sys.stderr, log_msg(self.options.color, COLOR_LIGHT_RED, 659 self.defconfig, "Failed to process."), 660 if self.options.verbose: 661 print >> sys.stderr, color_text(self.options.color, 662 COLOR_LIGHT_CYAN, 663 self.ps.stderr.read()) 664 self.progress.inc() 665 self.progress.show() 666 if self.options.exit_on_error: 667 sys.exit("Exit on error.") 668 # If --exit-on-error flag is not set, skip this board and continue. 669 # Record the failed board. 670 self.failed_boards.append(self.defconfig) 671 self.state = STATE_IDLE 672 return True 673 674 if self.state == STATE_AUTOCONF: 675 self.parser.update_dotconfig(self.defconfig) 676 677 """Save off the defconfig in a consistent way""" 678 cmd = list(self.make_cmd) 679 cmd.append('savedefconfig') 680 self.ps = subprocess.Popen(cmd, stdout=self.devnull, 681 stderr=subprocess.PIPE) 682 self.state = STATE_SAVEDEFCONFIG 683 return False 684 685 if self.state == STATE_SAVEDEFCONFIG: 686 if not self.options.dry_run: 687 shutil.move(os.path.join(self.build_dir, 'defconfig'), 688 os.path.join('configs', self.defconfig)) 689 self.progress.inc() 690 self.state = STATE_IDLE 691 return True 692 693 self.cross_compile = self.parser.get_cross_compile() 694 if self.cross_compile is None: 695 print >> sys.stderr, log_msg(self.options.color, COLOR_YELLOW, 696 self.defconfig, 697 "Compiler is missing. Do nothing."), 698 self.progress.inc() 699 self.progress.show() 700 if self.options.exit_on_error: 701 sys.exit("Exit on error.") 702 # If --exit-on-error flag is not set, skip this board and continue. 703 # Record the failed board. 704 self.failed_boards.append(self.defconfig) 705 self.state = STATE_IDLE 706 return True 707 708 cmd = list(self.make_cmd) 709 if self.cross_compile: 710 cmd.append('CROSS_COMPILE=%s' % self.cross_compile) 711 cmd.append('KCONFIG_IGNORE_DUPLICATES=1') 712 cmd.append('include/config/auto.conf') 713 self.ps = subprocess.Popen(cmd, stdout=self.devnull, 714 stderr=subprocess.PIPE) 715 self.state = STATE_AUTOCONF 716 return False 717 718 def get_failed_boards(self): 719 """Returns a list of failed boards (defconfigs) in this slot. 720 """ 721 return self.failed_boards 722 723class Slots: 724 725 """Controller of the array of subprocess slots.""" 726 727 def __init__(self, configs, options, progress): 728 """Create a new slots controller. 729 730 Arguments: 731 configs: A list of CONFIGs to move. 732 options: option flags. 733 progress: A progress indicator. 734 """ 735 self.options = options 736 self.slots = [] 737 devnull = get_devnull() 738 make_cmd = get_make_cmd() 739 for i in range(options.jobs): 740 self.slots.append(Slot(configs, options, progress, devnull, 741 make_cmd)) 742 743 def add(self, defconfig): 744 """Add a new subprocess if a vacant slot is found. 745 746 Arguments: 747 defconfig: defconfig name to be put into. 748 749 Returns: 750 Return True on success or False on failure 751 """ 752 for slot in self.slots: 753 if slot.add(defconfig): 754 return True 755 return False 756 757 def available(self): 758 """Check if there is a vacant slot. 759 760 Returns: 761 Return True if at lease one vacant slot is found, False otherwise. 762 """ 763 for slot in self.slots: 764 if slot.poll(): 765 return True 766 return False 767 768 def empty(self): 769 """Check if all slots are vacant. 770 771 Returns: 772 Return True if all the slots are vacant, False otherwise. 773 """ 774 ret = True 775 for slot in self.slots: 776 if not slot.poll(): 777 ret = False 778 return ret 779 780 def show_failed_boards(self): 781 """Display all of the failed boards (defconfigs).""" 782 failed_boards = [] 783 784 for slot in self.slots: 785 failed_boards += slot.get_failed_boards() 786 787 if len(failed_boards) > 0: 788 msg = [ "The following boards were not processed due to error:" ] 789 msg += failed_boards 790 for line in msg: 791 print >> sys.stderr, color_text(self.options.color, 792 COLOR_LIGHT_RED, line) 793 794 with open('moveconfig.failed', 'w') as f: 795 for board in failed_boards: 796 f.write(board + '\n') 797 798def move_config(configs, options): 799 """Move config options to defconfig files. 800 801 Arguments: 802 configs: A list of CONFIGs to move. 803 options: option flags 804 """ 805 if len(configs) == 0: 806 print 'Nothing to do. exit.' 807 sys.exit(0) 808 809 print 'Move %s (jobs: %d)' % (', '.join(configs), options.jobs) 810 811 if options.defconfigs: 812 defconfigs = [line.strip() for line in open(options.defconfigs)] 813 for i, defconfig in enumerate(defconfigs): 814 if not defconfig.endswith('_defconfig'): 815 defconfigs[i] = defconfig + '_defconfig' 816 if not os.path.exists(os.path.join('configs', defconfigs[i])): 817 sys.exit('%s - defconfig does not exist. Stopping.' % 818 defconfigs[i]) 819 else: 820 # All the defconfig files to be processed 821 defconfigs = [] 822 for (dirpath, dirnames, filenames) in os.walk('configs'): 823 dirpath = dirpath[len('configs') + 1:] 824 for filename in fnmatch.filter(filenames, '*_defconfig'): 825 defconfigs.append(os.path.join(dirpath, filename)) 826 827 progress = Progress(len(defconfigs)) 828 slots = Slots(configs, options, progress) 829 830 # Main loop to process defconfig files: 831 # Add a new subprocess into a vacant slot. 832 # Sleep if there is no available slot. 833 for defconfig in defconfigs: 834 while not slots.add(defconfig): 835 while not slots.available(): 836 # No available slot: sleep for a while 837 time.sleep(SLEEP_TIME) 838 839 # wait until all the subprocesses finish 840 while not slots.empty(): 841 time.sleep(SLEEP_TIME) 842 843 progress.show() 844 print '' 845 slots.show_failed_boards() 846 847def bad_recipe(filename, linenum, msg): 848 """Print error message with the file name and the line number and exit.""" 849 sys.exit("%s: line %d: error : " % (filename, linenum) + msg) 850 851def parse_recipe(filename): 852 """Parse the recipe file and retrieve CONFIGs to move. 853 854 This function parses the given recipe file and gets the name, 855 the type, and the default value of the target config options. 856 857 Arguments: 858 filename: path to file to be parsed. 859 Returns: 860 A list of CONFIGs to move. 861 """ 862 configs = [] 863 linenum = 1 864 865 for line in open(filename): 866 tokens = line.split() 867 if len(tokens) != 3: 868 bad_recipe(filename, linenum, 869 "%d fields in this line. Each line must contain 3 fields" 870 % len(tokens)) 871 872 (config, type, default) = tokens 873 874 # prefix the option name with CONFIG_ if missing 875 if not config.startswith('CONFIG_'): 876 config = 'CONFIG_' + config 877 878 configs.append(config) 879 linenum += 1 880 881 return configs 882 883def main(): 884 try: 885 cpu_count = multiprocessing.cpu_count() 886 except NotImplementedError: 887 cpu_count = 1 888 889 parser = optparse.OptionParser() 890 # Add options here 891 parser.add_option('-c', '--color', action='store_true', default=False, 892 help='display the log in color') 893 parser.add_option('-d', '--defconfigs', type='string', 894 help='a file containing a list of defconfigs to move') 895 parser.add_option('-n', '--dry-run', action='store_true', default=False, 896 help='perform a trial run (show log with no changes)') 897 parser.add_option('-e', '--exit-on-error', action='store_true', 898 default=False, 899 help='exit immediately on any error') 900 parser.add_option('-H', '--headers-only', dest='cleanup_headers_only', 901 action='store_true', default=False, 902 help='only cleanup the headers') 903 parser.add_option('-j', '--jobs', type='int', default=cpu_count, 904 help='the number of jobs to run simultaneously') 905 parser.add_option('-v', '--verbose', action='store_true', default=False, 906 help='show any build errors as boards are built') 907 parser.usage += ' recipe_file\n\n' + \ 908 'The recipe_file should describe config options you want to move.\n' + \ 909 'Each line should contain config_name, type, default_value\n\n' + \ 910 'Example:\n' + \ 911 'CONFIG_FOO bool n\n' + \ 912 'CONFIG_BAR int 100\n' + \ 913 'CONFIG_BAZ string "hello"\n' 914 915 (options, args) = parser.parse_args() 916 917 if len(args) != 1: 918 parser.print_usage() 919 sys.exit(1) 920 921 configs = parse_recipe(args[0]) 922 923 check_top_directory() 924 925 check_clean_directory() 926 927 update_cross_compile(options.color) 928 929 if not options.cleanup_headers_only: 930 move_config(configs, options) 931 932 cleanup_headers(configs, options.dry_run) 933 934if __name__ == '__main__': 935 main() 936