1#!/usr/bin/env python2 2# 3# Author: Masahiro Yamada <yamada.masahiro@socionext.com> 4# 5# SPDX-License-Identifier: GPL-2.0+ 6# 7 8""" 9Move config options from headers to defconfig files. 10 11Since Kconfig was introduced to U-Boot, we have worked on moving 12config options from headers to Kconfig (defconfig). 13 14This tool intends to help this tremendous work. 15 16 17Usage 18----- 19 20This tool takes one input file. (let's say 'recipe' file here.) 21The recipe describes the list of config options you want to move. 22Each line takes the form: 23<config_name> <type> <default> 24(the fields must be separated with whitespaces.) 25 26<config_name> is the name of config option. 27 28<type> is the type of the option. It must be one of bool, tristate, 29string, int, and hex. 30 31<default> is the default value of the option. It must be appropriate 32value corresponding to the option type. It must be either y or n for 33the bool type. Tristate options can also take m (although U-Boot has 34not supported the module feature). 35 36You can add two or more lines in the recipe file, so you can move 37multiple options at once. 38 39Let's say, for example, you want to move CONFIG_CMD_USB and 40CONFIG_SYS_TEXT_BASE. 41 42The type should be bool, hex, respectively. So, the recipe file 43should look like this: 44 45 $ cat recipe 46 CONFIG_CMD_USB bool n 47 CONFIG_SYS_TEXT_BASE hex 0x00000000 48 49Next you must edit the Kconfig to add the menu entries for the configs 50you are moving. 51 52And then run this tool giving the file name of the recipe 53 54 $ tools/moveconfig.py recipe 55 56The tool walks through all the defconfig files to move the config 57options specified by the recipe file. 58 59The log is also displayed on the terminal. 60 61Each line is printed in the format 62<defconfig_name> : <action> 63 64<defconfig_name> is the name of the defconfig 65(without the suffix _defconfig). 66 67<action> shows what the tool did for that defconfig. 68It looks like one of the followings: 69 70 - Move 'CONFIG_... ' 71 This config option was moved to the defconfig 72 73 - CONFIG_... is not defined in Kconfig. Do nothing. 74 The entry for this CONFIG was not found in Kconfig. 75 There are two common cases: 76 - You forgot to create an entry for the CONFIG before running 77 this tool, or made a typo in a CONFIG passed to this tool. 78 - The entry was hidden due to unmet 'depends on'. 79 This is correct behavior. 80 81 - 'CONFIG_...' is the same as the define in Kconfig. Do nothing. 82 The define in the config header matched the one in Kconfig. 83 We do not need to touch it. 84 85 - Undefined. Do nothing. 86 This config option was not found in the config header. 87 Nothing to do. 88 89 - Compiler is missing. Do nothing. 90 The compiler specified for this architecture was not found 91 in your PATH environment. 92 (If -e option is passed, the tool exits immediately.) 93 94 - Failed to process. 95 An error occurred during processing this defconfig. Skipped. 96 (If -e option is passed, the tool exits immediately on error.) 97 98Finally, you will be asked, Clean up headers? [y/n]: 99 100If you say 'y' here, the unnecessary config defines are removed 101from the config headers (include/configs/*.h). 102It just uses the regex method, so you should not rely on it. 103Just in case, please do 'git diff' to see what happened. 104 105 106How does it works? 107------------------ 108 109This tool runs configuration and builds include/autoconf.mk for every 110defconfig. The config options defined in Kconfig appear in the .config 111file (unless they are hidden because of unmet dependency.) 112On the other hand, the config options defined by board headers are seen 113in include/autoconf.mk. The tool looks for the specified options in both 114of them to decide the appropriate action for the options. If the option 115is found in the .config or the value is the same as the specified default, 116the option does not need to be touched. If the option is found in 117include/autoconf.mk, but not in the .config, and the value is different 118from the default, the tools adds the option to the defconfig. 119 120For faster processing, this tool handles multi-threading. It creates 121separate build directories where the out-of-tree build is run. The 122temporary build directories are automatically created and deleted as 123needed. The number of threads are chosen based on the number of the CPU 124cores of your system although you can change it via -j (--jobs) option. 125 126 127Toolchains 128---------- 129 130Appropriate toolchain are necessary to generate include/autoconf.mk 131for all the architectures supported by U-Boot. Most of them are available 132at the kernel.org site, some are not provided by kernel.org. 133 134The default per-arch CROSS_COMPILE used by this tool is specified by 135the list below, CROSS_COMPILE. You may wish to update the list to 136use your own. Instead of modifying the list directly, you can give 137them via environments. 138 139 140Available options 141----------------- 142 143 -c, --color 144 Surround each portion of the log with escape sequences to display it 145 in color on the terminal. 146 147 -d, --defconfigs 148 Specify a file containing a list of defconfigs to move 149 150 -n, --dry-run 151 Peform a trial run that does not make any changes. It is useful to 152 see what is going to happen before one actually runs it. 153 154 -e, --exit-on-error 155 Exit immediately if Make exits with a non-zero status while processing 156 a defconfig file. 157 158 -H, --headers-only 159 Only cleanup the headers; skip the defconfig processing 160 161 -j, --jobs 162 Specify the number of threads to run simultaneously. If not specified, 163 the number of threads is the same as the number of CPU cores. 164 165 -v, --verbose 166 Show any build errors as boards are built 167 168To see the complete list of supported options, run 169 170 $ tools/moveconfig.py -h 171 172""" 173 174import fnmatch 175import multiprocessing 176import optparse 177import os 178import re 179import shutil 180import subprocess 181import sys 182import tempfile 183import time 184 185SHOW_GNU_MAKE = 'scripts/show-gnu-make' 186SLEEP_TIME=0.03 187 188# Here is the list of cross-tools I use. 189# Most of them are available at kernel.org 190# (https://www.kernel.org/pub/tools/crosstool/files/bin/), except the followings: 191# arc: https://github.com/foss-for-synopsys-dwc-arc-processors/toolchain/releases 192# blackfin: http://sourceforge.net/projects/adi-toolchain/files/ 193# nds32: http://osdk.andestech.com/packages/nds32le-linux-glibc-v1.tgz 194# nios2: https://sourcery.mentor.com/GNUToolchain/subscription42545 195# sh: http://sourcery.mentor.com/public/gnu_toolchain/sh-linux-gnu 196# 197# openrisc kernel.org toolchain is out of date, download latest one from 198# http://opencores.org/or1k/OpenRISC_GNU_tool_chain#Prebuilt_versions 199CROSS_COMPILE = { 200 'arc': 'arc-linux-', 201 'aarch64': 'aarch64-linux-', 202 'arm': 'arm-unknown-linux-gnueabi-', 203 'avr32': 'avr32-linux-', 204 'blackfin': 'bfin-elf-', 205 'm68k': 'm68k-linux-', 206 'microblaze': 'microblaze-linux-', 207 'mips': 'mips-linux-', 208 'nds32': 'nds32le-linux-', 209 'nios2': 'nios2-linux-gnu-', 210 'openrisc': 'or1k-elf-', 211 'powerpc': 'powerpc-linux-', 212 'sh': 'sh-linux-gnu-', 213 'sparc': 'sparc-linux-', 214 'x86': 'i386-linux-' 215} 216 217STATE_IDLE = 0 218STATE_DEFCONFIG = 1 219STATE_AUTOCONF = 2 220STATE_SAVEDEFCONFIG = 3 221 222ACTION_MOVE = 0 223ACTION_NO_ENTRY = 1 224ACTION_NO_CHANGE = 2 225 226COLOR_BLACK = '0;30' 227COLOR_RED = '0;31' 228COLOR_GREEN = '0;32' 229COLOR_BROWN = '0;33' 230COLOR_BLUE = '0;34' 231COLOR_PURPLE = '0;35' 232COLOR_CYAN = '0;36' 233COLOR_LIGHT_GRAY = '0;37' 234COLOR_DARK_GRAY = '1;30' 235COLOR_LIGHT_RED = '1;31' 236COLOR_LIGHT_GREEN = '1;32' 237COLOR_YELLOW = '1;33' 238COLOR_LIGHT_BLUE = '1;34' 239COLOR_LIGHT_PURPLE = '1;35' 240COLOR_LIGHT_CYAN = '1;36' 241COLOR_WHITE = '1;37' 242 243### helper functions ### 244def get_devnull(): 245 """Get the file object of '/dev/null' device.""" 246 try: 247 devnull = subprocess.DEVNULL # py3k 248 except AttributeError: 249 devnull = open(os.devnull, 'wb') 250 return devnull 251 252def check_top_directory(): 253 """Exit if we are not at the top of source directory.""" 254 for f in ('README', 'Licenses'): 255 if not os.path.exists(f): 256 sys.exit('Please run at the top of source directory.') 257 258def check_clean_directory(): 259 """Exit if the source tree is not clean.""" 260 for f in ('.config', 'include/config'): 261 if os.path.exists(f): 262 sys.exit("source tree is not clean, please run 'make mrproper'") 263 264def get_make_cmd(): 265 """Get the command name of GNU Make. 266 267 U-Boot needs GNU Make for building, but the command name is not 268 necessarily "make". (for example, "gmake" on FreeBSD). 269 Returns the most appropriate command name on your system. 270 """ 271 process = subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE) 272 ret = process.communicate() 273 if process.returncode: 274 sys.exit('GNU Make not found') 275 return ret[0].rstrip() 276 277def color_text(color_enabled, color, string): 278 """Return colored string.""" 279 if color_enabled: 280 return '\033[' + color + 'm' + string + '\033[0m' 281 else: 282 return string 283 284def log_msg(color_enabled, color, defconfig, msg): 285 """Return the formated line for the log.""" 286 return defconfig[:-len('_defconfig')].ljust(37) + ': ' + \ 287 color_text(color_enabled, color, msg) + '\n' 288 289def update_cross_compile(color_enabled): 290 """Update per-arch CROSS_COMPILE via environment variables 291 292 The default CROSS_COMPILE values are available 293 in the CROSS_COMPILE list above. 294 295 You can override them via environment variables 296 CROSS_COMPILE_{ARCH}. 297 298 For example, if you want to override toolchain prefixes 299 for ARM and PowerPC, you can do as follows in your shell: 300 301 export CROSS_COMPILE_ARM=... 302 export CROSS_COMPILE_POWERPC=... 303 304 Then, this function checks if specified compilers really exist in your 305 PATH environment. 306 """ 307 archs = [] 308 309 for arch in os.listdir('arch'): 310 if os.path.exists(os.path.join('arch', arch, 'Makefile')): 311 archs.append(arch) 312 313 # arm64 is a special case 314 archs.append('aarch64') 315 316 for arch in archs: 317 env = 'CROSS_COMPILE_' + arch.upper() 318 cross_compile = os.environ.get(env) 319 if not cross_compile: 320 cross_compile = CROSS_COMPILE.get(arch, '') 321 322 for path in os.environ["PATH"].split(os.pathsep): 323 gcc_path = os.path.join(path, cross_compile + 'gcc') 324 if os.path.isfile(gcc_path) and os.access(gcc_path, os.X_OK): 325 break 326 else: 327 print >> sys.stderr, color_text(color_enabled, COLOR_YELLOW, 328 'warning: %sgcc: not found in PATH. %s architecture boards will be skipped' 329 % (cross_compile, arch)) 330 cross_compile = None 331 332 CROSS_COMPILE[arch] = cross_compile 333 334def cleanup_one_header(header_path, patterns, dry_run): 335 """Clean regex-matched lines away from a file. 336 337 Arguments: 338 header_path: path to the cleaned file. 339 patterns: list of regex patterns. Any lines matching to these 340 patterns are deleted. 341 dry_run: make no changes, but still display log. 342 """ 343 with open(header_path) as f: 344 lines = f.readlines() 345 346 matched = [] 347 for i, line in enumerate(lines): 348 for pattern in patterns: 349 m = pattern.search(line) 350 if m: 351 print '%s: %s: %s' % (header_path, i + 1, line), 352 matched.append(i) 353 break 354 355 if dry_run or not matched: 356 return 357 358 with open(header_path, 'w') as f: 359 for i, line in enumerate(lines): 360 if not i in matched: 361 f.write(line) 362 363def cleanup_headers(config_attrs, dry_run): 364 """Delete config defines from board headers. 365 366 Arguments: 367 config_attrs: A list of dictionaris, each of them includes the name, 368 the type, and the default value of the target config. 369 dry_run: make no changes, but still display log. 370 """ 371 while True: 372 choice = raw_input('Clean up headers? [y/n]: ').lower() 373 print choice 374 if choice == 'y' or choice == 'n': 375 break 376 377 if choice == 'n': 378 return 379 380 patterns = [] 381 for config_attr in config_attrs: 382 config = config_attr['config'] 383 patterns.append(re.compile(r'#\s*define\s+%s\W' % config)) 384 patterns.append(re.compile(r'#\s*undef\s+%s\W' % config)) 385 386 for dir in 'include', 'arch', 'board': 387 for (dirpath, dirnames, filenames) in os.walk(dir): 388 for filename in filenames: 389 if not fnmatch.fnmatch(filename, '*~'): 390 cleanup_one_header(os.path.join(dirpath, filename), 391 patterns, dry_run) 392 393### classes ### 394class Progress: 395 396 """Progress Indicator""" 397 398 def __init__(self, total): 399 """Create a new progress indicator. 400 401 Arguments: 402 total: A number of defconfig files to process. 403 """ 404 self.current = 0 405 self.total = total 406 407 def inc(self): 408 """Increment the number of processed defconfig files.""" 409 410 self.current += 1 411 412 def show(self): 413 """Display the progress.""" 414 print ' %d defconfigs out of %d\r' % (self.current, self.total), 415 sys.stdout.flush() 416 417class KconfigParser: 418 419 """A parser of .config and include/autoconf.mk.""" 420 421 re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"') 422 re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"') 423 424 def __init__(self, config_attrs, options, progress, build_dir): 425 """Create a new parser. 426 427 Arguments: 428 config_attrs: A list of dictionaris, each of them includes the name, 429 the type, and the default value of the target config. 430 options: option flags. 431 progress: A progress indicator 432 build_dir: Build directory. 433 """ 434 self.config_attrs = config_attrs 435 self.options = options 436 self.progress = progress 437 self.build_dir = build_dir 438 439 def get_cross_compile(self): 440 """Parse .config file and return CROSS_COMPILE. 441 442 Returns: 443 A string storing the compiler prefix for the architecture. 444 Return a NULL string for architectures that do not require 445 compiler prefix (Sandbox and native build is the case). 446 Return None if the specified compiler is missing in your PATH. 447 Caller should distinguish '' and None. 448 """ 449 arch = '' 450 cpu = '' 451 dotconfig = os.path.join(self.build_dir, '.config') 452 for line in open(dotconfig): 453 m = self.re_arch.match(line) 454 if m: 455 arch = m.group(1) 456 continue 457 m = self.re_cpu.match(line) 458 if m: 459 cpu = m.group(1) 460 461 if not arch: 462 return None 463 464 # fix-up for aarch64 465 if arch == 'arm' and cpu == 'armv8': 466 arch = 'aarch64' 467 468 return CROSS_COMPILE.get(arch, None) 469 470 def parse_one_config(self, config_attr, dotconfig_lines, autoconf_lines): 471 """Parse .config, defconfig, include/autoconf.mk for one config. 472 473 This function looks for the config options in the lines from 474 defconfig, .config, and include/autoconf.mk in order to decide 475 which action should be taken for this defconfig. 476 477 Arguments: 478 config_attr: A dictionary including the name, the type, 479 and the default value of the target config. 480 dotconfig_lines: lines from the .config file. 481 autoconf_lines: lines from the include/autoconf.mk file. 482 483 Returns: 484 A tupple of the action for this defconfig and the line 485 matched for the config. 486 """ 487 config = config_attr['config'] 488 not_set = '# %s is not set' % config 489 490 for line in dotconfig_lines: 491 line = line.rstrip() 492 if line.startswith(config + '=') or line == not_set: 493 old_val = line 494 break 495 else: 496 return (ACTION_NO_ENTRY, config) 497 498 for line in autoconf_lines: 499 line = line.rstrip() 500 if line.startswith(config + '='): 501 new_val = line 502 break 503 else: 504 new_val = not_set 505 506 if old_val == new_val: 507 return (ACTION_NO_CHANGE, new_val) 508 509 # If this CONFIG is neither bool nor trisate 510 if old_val[-2:] != '=y' and old_val[-2:] != '=m' and old_val != not_set: 511 # tools/scripts/define2mk.sed changes '1' to 'y'. 512 # This is a problem if the CONFIG is int type. 513 # Check the type in Kconfig and handle it correctly. 514 if new_val[-2:] == '=y': 515 new_val = new_val[:-1] + '1' 516 517 return (ACTION_MOVE, new_val) 518 519 def update_dotconfig(self, defconfig): 520 """Parse files for the config options and update the .config. 521 522 This function parses the generated .config and include/autoconf.mk 523 searching the target options. 524 Move the config option(s) to the .config as needed. 525 Also, display the log to show what happened to the .config. 526 527 Arguments: 528 defconfig: defconfig name. 529 """ 530 531 dotconfig_path = os.path.join(self.build_dir, '.config') 532 autoconf_path = os.path.join(self.build_dir, 'include', 'autoconf.mk') 533 results = [] 534 535 with open(dotconfig_path) as f: 536 dotconfig_lines = f.readlines() 537 538 with open(autoconf_path) as f: 539 autoconf_lines = f.readlines() 540 541 for config_attr in self.config_attrs: 542 result = self.parse_one_config(config_attr, dotconfig_lines, 543 autoconf_lines) 544 results.append(result) 545 546 log = '' 547 548 for (action, value) in results: 549 if action == ACTION_MOVE: 550 actlog = "Move '%s'" % value 551 log_color = COLOR_LIGHT_GREEN 552 elif action == ACTION_NO_ENTRY: 553 actlog = "%s is not defined in Kconfig. Do nothing." % value 554 log_color = COLOR_LIGHT_BLUE 555 elif action == ACTION_NO_CHANGE: 556 actlog = "'%s' is the same as the define in Kconfig. Do nothing." \ 557 % value 558 log_color = COLOR_LIGHT_PURPLE 559 else: 560 sys.exit("Internal Error. This should not happen.") 561 562 log += log_msg(self.options.color, log_color, defconfig, actlog) 563 564 # Some threads are running in parallel. 565 # Print log in one shot to not mix up logs from different threads. 566 print log, 567 self.progress.show() 568 569 with open(dotconfig_path, 'a') as f: 570 for (action, value) in results: 571 if action == ACTION_MOVE: 572 f.write(value + '\n') 573 574 os.remove(os.path.join(self.build_dir, 'include', 'config', 'auto.conf')) 575 os.remove(autoconf_path) 576 577class Slot: 578 579 """A slot to store a subprocess. 580 581 Each instance of this class handles one subprocess. 582 This class is useful to control multiple threads 583 for faster processing. 584 """ 585 586 def __init__(self, config_attrs, options, progress, devnull, make_cmd): 587 """Create a new process slot. 588 589 Arguments: 590 config_attrs: A list of dictionaris, each of them includes the name, 591 the type, and the default value of the target config. 592 options: option flags. 593 progress: A progress indicator. 594 devnull: A file object of '/dev/null'. 595 make_cmd: command name of GNU Make. 596 """ 597 self.options = options 598 self.progress = progress 599 self.build_dir = tempfile.mkdtemp() 600 self.devnull = devnull 601 self.make_cmd = (make_cmd, 'O=' + self.build_dir) 602 self.parser = KconfigParser(config_attrs, options, progress, 603 self.build_dir) 604 self.state = STATE_IDLE 605 self.failed_boards = [] 606 607 def __del__(self): 608 """Delete the working directory 609 610 This function makes sure the temporary directory is cleaned away 611 even if Python suddenly dies due to error. It should be done in here 612 because it is guranteed the destructor is always invoked when the 613 instance of the class gets unreferenced. 614 615 If the subprocess is still running, wait until it finishes. 616 """ 617 if self.state != STATE_IDLE: 618 while self.ps.poll() == None: 619 pass 620 shutil.rmtree(self.build_dir) 621 622 def add(self, defconfig): 623 """Assign a new subprocess for defconfig and add it to the slot. 624 625 If the slot is vacant, create a new subprocess for processing the 626 given defconfig and add it to the slot. Just returns False if 627 the slot is occupied (i.e. the current subprocess is still running). 628 629 Arguments: 630 defconfig: defconfig name. 631 632 Returns: 633 Return True on success or False on failure 634 """ 635 if self.state != STATE_IDLE: 636 return False 637 cmd = list(self.make_cmd) 638 cmd.append(defconfig) 639 self.ps = subprocess.Popen(cmd, stdout=self.devnull, 640 stderr=subprocess.PIPE) 641 self.defconfig = defconfig 642 self.state = STATE_DEFCONFIG 643 return True 644 645 def poll(self): 646 """Check the status of the subprocess and handle it as needed. 647 648 Returns True if the slot is vacant (i.e. in idle state). 649 If the configuration is successfully finished, assign a new 650 subprocess to build include/autoconf.mk. 651 If include/autoconf.mk is generated, invoke the parser to 652 parse the .config and the include/autoconf.mk, and then set the 653 slot back to the idle state. 654 655 Returns: 656 Return True if the subprocess is terminated, False otherwise 657 """ 658 if self.state == STATE_IDLE: 659 return True 660 661 if self.ps.poll() == None: 662 return False 663 664 if self.ps.poll() != 0: 665 print >> sys.stderr, log_msg(self.options.color, COLOR_LIGHT_RED, 666 self.defconfig, "Failed to process."), 667 if self.options.verbose: 668 print >> sys.stderr, color_text(self.options.color, 669 COLOR_LIGHT_CYAN, 670 self.ps.stderr.read()) 671 self.progress.inc() 672 self.progress.show() 673 if self.options.exit_on_error: 674 sys.exit("Exit on error.") 675 # If --exit-on-error flag is not set, skip this board and continue. 676 # Record the failed board. 677 self.failed_boards.append(self.defconfig) 678 self.state = STATE_IDLE 679 return True 680 681 if self.state == STATE_AUTOCONF: 682 self.parser.update_dotconfig(self.defconfig) 683 684 """Save off the defconfig in a consistent way""" 685 cmd = list(self.make_cmd) 686 cmd.append('savedefconfig') 687 self.ps = subprocess.Popen(cmd, stdout=self.devnull, 688 stderr=subprocess.PIPE) 689 self.state = STATE_SAVEDEFCONFIG 690 return False 691 692 if self.state == STATE_SAVEDEFCONFIG: 693 if not self.options.dry_run: 694 shutil.move(os.path.join(self.build_dir, 'defconfig'), 695 os.path.join('configs', self.defconfig)) 696 self.progress.inc() 697 self.state = STATE_IDLE 698 return True 699 700 self.cross_compile = self.parser.get_cross_compile() 701 if self.cross_compile is None: 702 print >> sys.stderr, log_msg(self.options.color, COLOR_YELLOW, 703 self.defconfig, 704 "Compiler is missing. Do nothing."), 705 self.progress.inc() 706 self.progress.show() 707 if self.options.exit_on_error: 708 sys.exit("Exit on error.") 709 # If --exit-on-error flag is not set, skip this board and continue. 710 # Record the failed board. 711 self.failed_boards.append(self.defconfig) 712 self.state = STATE_IDLE 713 return True 714 715 cmd = list(self.make_cmd) 716 if self.cross_compile: 717 cmd.append('CROSS_COMPILE=%s' % self.cross_compile) 718 cmd.append('KCONFIG_IGNORE_DUPLICATES=1') 719 cmd.append('include/config/auto.conf') 720 self.ps = subprocess.Popen(cmd, stdout=self.devnull, 721 stderr=subprocess.PIPE) 722 self.state = STATE_AUTOCONF 723 return False 724 725 def get_failed_boards(self): 726 """Returns a list of failed boards (defconfigs) in this slot. 727 """ 728 return self.failed_boards 729 730class Slots: 731 732 """Controller of the array of subprocess slots.""" 733 734 def __init__(self, config_attrs, options, progress): 735 """Create a new slots controller. 736 737 Arguments: 738 config_attrs: A list of dictionaris containing the name, the type, 739 and the default value of the target CONFIG. 740 options: option flags. 741 progress: A progress indicator. 742 """ 743 self.options = options 744 self.slots = [] 745 devnull = get_devnull() 746 make_cmd = get_make_cmd() 747 for i in range(options.jobs): 748 self.slots.append(Slot(config_attrs, options, progress, devnull, 749 make_cmd)) 750 751 def add(self, defconfig): 752 """Add a new subprocess if a vacant slot is found. 753 754 Arguments: 755 defconfig: defconfig name to be put into. 756 757 Returns: 758 Return True on success or False on failure 759 """ 760 for slot in self.slots: 761 if slot.add(defconfig): 762 return True 763 return False 764 765 def available(self): 766 """Check if there is a vacant slot. 767 768 Returns: 769 Return True if at lease one vacant slot is found, False otherwise. 770 """ 771 for slot in self.slots: 772 if slot.poll(): 773 return True 774 return False 775 776 def empty(self): 777 """Check if all slots are vacant. 778 779 Returns: 780 Return True if all the slots are vacant, False otherwise. 781 """ 782 ret = True 783 for slot in self.slots: 784 if not slot.poll(): 785 ret = False 786 return ret 787 788 def show_failed_boards(self): 789 """Display all of the failed boards (defconfigs).""" 790 failed_boards = [] 791 792 for slot in self.slots: 793 failed_boards += slot.get_failed_boards() 794 795 if len(failed_boards) > 0: 796 msg = [ "The following boards were not processed due to error:" ] 797 msg += failed_boards 798 for line in msg: 799 print >> sys.stderr, color_text(self.options.color, 800 COLOR_LIGHT_RED, line) 801 802 with open('moveconfig.failed', 'w') as f: 803 for board in failed_boards: 804 f.write(board + '\n') 805 806def move_config(config_attrs, options): 807 """Move config options to defconfig files. 808 809 Arguments: 810 config_attrs: A list of dictionaris, each of them includes the name, 811 the type, and the default value of the target config. 812 options: option flags 813 """ 814 if len(config_attrs) == 0: 815 print 'Nothing to do. exit.' 816 sys.exit(0) 817 818 print 'Move the following CONFIG options (jobs: %d)' % options.jobs 819 for config_attr in config_attrs: 820 print ' %s (type: %s, default: %s)' % (config_attr['config'], 821 config_attr['type'], 822 config_attr['default']) 823 824 if options.defconfigs: 825 defconfigs = [line.strip() for line in open(options.defconfigs)] 826 for i, defconfig in enumerate(defconfigs): 827 if not defconfig.endswith('_defconfig'): 828 defconfigs[i] = defconfig + '_defconfig' 829 if not os.path.exists(os.path.join('configs', defconfigs[i])): 830 sys.exit('%s - defconfig does not exist. Stopping.' % 831 defconfigs[i]) 832 else: 833 # All the defconfig files to be processed 834 defconfigs = [] 835 for (dirpath, dirnames, filenames) in os.walk('configs'): 836 dirpath = dirpath[len('configs') + 1:] 837 for filename in fnmatch.filter(filenames, '*_defconfig'): 838 defconfigs.append(os.path.join(dirpath, filename)) 839 840 progress = Progress(len(defconfigs)) 841 slots = Slots(config_attrs, options, progress) 842 843 # Main loop to process defconfig files: 844 # Add a new subprocess into a vacant slot. 845 # Sleep if there is no available slot. 846 for defconfig in defconfigs: 847 while not slots.add(defconfig): 848 while not slots.available(): 849 # No available slot: sleep for a while 850 time.sleep(SLEEP_TIME) 851 852 # wait until all the subprocesses finish 853 while not slots.empty(): 854 time.sleep(SLEEP_TIME) 855 856 progress.show() 857 print '' 858 slots.show_failed_boards() 859 860def bad_recipe(filename, linenum, msg): 861 """Print error message with the file name and the line number and exit.""" 862 sys.exit("%s: line %d: error : " % (filename, linenum) + msg) 863 864def parse_recipe(filename): 865 """Parse the recipe file and retrieve the config attributes. 866 867 This function parses the given recipe file and gets the name, 868 the type, and the default value of the target config options. 869 870 Arguments: 871 filename: path to file to be parsed. 872 Returns: 873 A list of dictionaris, each of them includes the name, 874 the type, and the default value of the target config. 875 """ 876 config_attrs = [] 877 linenum = 1 878 879 for line in open(filename): 880 tokens = line.split() 881 if len(tokens) != 3: 882 bad_recipe(filename, linenum, 883 "%d fields in this line. Each line must contain 3 fields" 884 % len(tokens)) 885 886 (config, type, default) = tokens 887 888 # prefix the option name with CONFIG_ if missing 889 if not config.startswith('CONFIG_'): 890 config = 'CONFIG_' + config 891 892 # sanity check of default values 893 if type == 'bool': 894 if not default in ('y', 'n'): 895 bad_recipe(filename, linenum, 896 "default for bool type must be either y or n") 897 elif type == 'tristate': 898 if not default in ('y', 'm', 'n'): 899 bad_recipe(filename, linenum, 900 "default for tristate type must be y, m, or n") 901 elif type == 'string': 902 if default[0] != '"' or default[-1] != '"': 903 bad_recipe(filename, linenum, 904 "default for string type must be surrounded by double-quotations") 905 elif type == 'int': 906 try: 907 int(default) 908 except: 909 bad_recipe(filename, linenum, 910 "type is int, but default value is not decimal") 911 elif type == 'hex': 912 if len(default) < 2 or default[:2] != '0x': 913 bad_recipe(filename, linenum, 914 "default for hex type must be prefixed with 0x") 915 try: 916 int(default, 16) 917 except: 918 bad_recipe(filename, linenum, 919 "type is hex, but default value is not hexadecimal") 920 else: 921 bad_recipe(filename, linenum, 922 "unsupported type '%s'. type must be one of bool, tristate, string, int, hex" 923 % type) 924 925 config_attrs.append({'config': config, 'type': type, 'default': default}) 926 linenum += 1 927 928 return config_attrs 929 930def main(): 931 try: 932 cpu_count = multiprocessing.cpu_count() 933 except NotImplementedError: 934 cpu_count = 1 935 936 parser = optparse.OptionParser() 937 # Add options here 938 parser.add_option('-c', '--color', action='store_true', default=False, 939 help='display the log in color') 940 parser.add_option('-d', '--defconfigs', type='string', 941 help='a file containing a list of defconfigs to move') 942 parser.add_option('-n', '--dry-run', action='store_true', default=False, 943 help='perform a trial run (show log with no changes)') 944 parser.add_option('-e', '--exit-on-error', action='store_true', 945 default=False, 946 help='exit immediately on any error') 947 parser.add_option('-H', '--headers-only', dest='cleanup_headers_only', 948 action='store_true', default=False, 949 help='only cleanup the headers') 950 parser.add_option('-j', '--jobs', type='int', default=cpu_count, 951 help='the number of jobs to run simultaneously') 952 parser.add_option('-v', '--verbose', action='store_true', default=False, 953 help='show any build errors as boards are built') 954 parser.usage += ' recipe_file\n\n' + \ 955 'The recipe_file should describe config options you want to move.\n' + \ 956 'Each line should contain config_name, type, default_value\n\n' + \ 957 'Example:\n' + \ 958 'CONFIG_FOO bool n\n' + \ 959 'CONFIG_BAR int 100\n' + \ 960 'CONFIG_BAZ string "hello"\n' 961 962 (options, args) = parser.parse_args() 963 964 if len(args) != 1: 965 parser.print_usage() 966 sys.exit(1) 967 968 config_attrs = parse_recipe(args[0]) 969 970 check_top_directory() 971 972 check_clean_directory() 973 974 update_cross_compile(options.color) 975 976 if not options.cleanup_headers_only: 977 move_config(config_attrs, options) 978 979 cleanup_headers(config_attrs, options.dry_run) 980 981if __name__ == '__main__': 982 main() 983