1#!/usr/bin/env python2 2# 3# Author: Masahiro Yamada <yamada.masahiro@socionext.com> 4# 5# SPDX-License-Identifier: GPL-2.0+ 6# 7 8""" 9Move config options from headers to defconfig files. 10 11Since Kconfig was introduced to U-Boot, we have worked on moving 12config options from headers to Kconfig (defconfig). 13 14This tool intends to help this tremendous work. 15 16 17Usage 18----- 19 20This tool takes one input file. (let's say 'recipe' file here.) 21The recipe describes the list of config options you want to move. 22Each line takes the form: 23<config_name> <type> <default> 24(the fields must be separated with whitespaces.) 25 26<config_name> is the name of config option. 27 28<type> is the type of the option. It must be one of bool, tristate, 29string, int, and hex. 30 31<default> is the default value of the option. It must be appropriate 32value corresponding to the option type. It must be either y or n for 33the bool type. Tristate options can also take m (although U-Boot has 34not supported the module feature). 35 36You can add two or more lines in the recipe file, so you can move 37multiple options at once. 38 39Let's say, for example, you want to move CONFIG_CMD_USB and 40CONFIG_SYS_TEXT_BASE. 41 42The type should be bool, hex, respectively. So, the recipe file 43should look like this: 44 45 $ cat recipe 46 CONFIG_CMD_USB bool n 47 CONFIG_SYS_TEXT_BASE hex 0x00000000 48 49Next you must edit the Kconfig to add the menu entries for the configs 50you are moving. 51 52And then run this tool giving the file name of the recipe 53 54 $ tools/moveconfig.py recipe 55 56The tool walks through all the defconfig files to move the config 57options specified by the recipe file. 58 59The log is also displayed on the terminal. 60 61Each line is printed in the format 62<defconfig_name> : <action> 63 64<defconfig_name> is the name of the defconfig 65(without the suffix _defconfig). 66 67<action> shows what the tool did for that defconfig. 68It looks like one of the followings: 69 70 - Move 'CONFIG_... ' 71 This config option was moved to the defconfig 72 73 - Default value 'CONFIG_...'. Do nothing. 74 The value of this option is the same as default. 75 We do not have to add it to the defconfig. 76 77 - 'CONFIG_...' already exists in Kconfig. Do nothing. 78 This config option is already defined in Kconfig. 79 We do not need/want to touch it. 80 81 - Undefined. Do nothing. 82 This config option was not found in the config header. 83 Nothing to do. 84 85 - Compiler is missing. Do nothing. 86 The compiler specified for this architecture was not found 87 in your PATH environment. 88 (If -e option is passed, the tool exits immediately.) 89 90 - Failed to process. 91 An error occurred during processing this defconfig. Skipped. 92 (If -e option is passed, the tool exits immediately on error.) 93 94Finally, you will be asked, Clean up headers? [y/n]: 95 96If you say 'y' here, the unnecessary config defines are removed 97from the config headers (include/configs/*.h). 98It just uses the regex method, so you should not rely on it. 99Just in case, please do 'git diff' to see what happened. 100 101 102How does it works? 103------------------ 104 105This tool runs configuration and builds include/autoconf.mk for every 106defconfig. The config options defined in Kconfig appear in the .config 107file (unless they are hidden because of unmet dependency.) 108On the other hand, the config options defined by board headers are seen 109in include/autoconf.mk. The tool looks for the specified options in both 110of them to decide the appropriate action for the options. If the option 111is found in the .config or the value is the same as the specified default, 112the option does not need to be touched. If the option is found in 113include/autoconf.mk, but not in the .config, and the value is different 114from the default, the tools adds the option to the defconfig. 115 116For faster processing, this tool handles multi-threading. It creates 117separate build directories where the out-of-tree build is run. The 118temporary build directories are automatically created and deleted as 119needed. The number of threads are chosen based on the number of the CPU 120cores of your system although you can change it via -j (--jobs) option. 121 122 123Toolchains 124---------- 125 126Appropriate toolchain are necessary to generate include/autoconf.mk 127for all the architectures supported by U-Boot. Most of them are available 128at the kernel.org site, some are not provided by kernel.org. 129 130The default per-arch CROSS_COMPILE used by this tool is specified by 131the list below, CROSS_COMPILE. You may wish to update the list to 132use your own. Instead of modifying the list directly, you can give 133them via environments. 134 135 136Available options 137----------------- 138 139 -c, --color 140 Surround each portion of the log with escape sequences to display it 141 in color on the terminal. 142 143 -d, --defconfigs 144 Specify a file containing a list of defconfigs to move 145 146 -n, --dry-run 147 Peform a trial run that does not make any changes. It is useful to 148 see what is going to happen before one actually runs it. 149 150 -e, --exit-on-error 151 Exit immediately if Make exits with a non-zero status while processing 152 a defconfig file. 153 154 -H, --headers-only 155 Only cleanup the headers; skip the defconfig processing 156 157 -j, --jobs 158 Specify the number of threads to run simultaneously. If not specified, 159 the number of threads is the same as the number of CPU cores. 160 161 -v, --verbose 162 Show any build errors as boards are built 163 164To see the complete list of supported options, run 165 166 $ tools/moveconfig.py -h 167 168""" 169 170import fnmatch 171import multiprocessing 172import optparse 173import os 174import re 175import shutil 176import subprocess 177import sys 178import tempfile 179import time 180 181SHOW_GNU_MAKE = 'scripts/show-gnu-make' 182SLEEP_TIME=0.03 183 184# Here is the list of cross-tools I use. 185# Most of them are available at kernel.org 186# (https://www.kernel.org/pub/tools/crosstool/files/bin/), except the followings: 187# arc: https://github.com/foss-for-synopsys-dwc-arc-processors/toolchain/releases 188# blackfin: http://sourceforge.net/projects/adi-toolchain/files/ 189# nds32: http://osdk.andestech.com/packages/nds32le-linux-glibc-v1.tgz 190# nios2: https://sourcery.mentor.com/GNUToolchain/subscription42545 191# sh: http://sourcery.mentor.com/public/gnu_toolchain/sh-linux-gnu 192# 193# openrisc kernel.org toolchain is out of date, download latest one from 194# http://opencores.org/or1k/OpenRISC_GNU_tool_chain#Prebuilt_versions 195CROSS_COMPILE = { 196 'arc': 'arc-linux-', 197 'aarch64': 'aarch64-linux-', 198 'arm': 'arm-unknown-linux-gnueabi-', 199 'avr32': 'avr32-linux-', 200 'blackfin': 'bfin-elf-', 201 'm68k': 'm68k-linux-', 202 'microblaze': 'microblaze-linux-', 203 'mips': 'mips-linux-', 204 'nds32': 'nds32le-linux-', 205 'nios2': 'nios2-linux-gnu-', 206 'openrisc': 'or1k-elf-', 207 'powerpc': 'powerpc-linux-', 208 'sh': 'sh-linux-gnu-', 209 'sparc': 'sparc-linux-', 210 'x86': 'i386-linux-' 211} 212 213STATE_IDLE = 0 214STATE_DEFCONFIG = 1 215STATE_AUTOCONF = 2 216STATE_SAVEDEFCONFIG = 3 217 218ACTION_MOVE = 0 219ACTION_DEFAULT_VALUE = 1 220ACTION_ALREADY_EXIST = 2 221ACTION_UNDEFINED = 3 222 223COLOR_BLACK = '0;30' 224COLOR_RED = '0;31' 225COLOR_GREEN = '0;32' 226COLOR_BROWN = '0;33' 227COLOR_BLUE = '0;34' 228COLOR_PURPLE = '0;35' 229COLOR_CYAN = '0;36' 230COLOR_LIGHT_GRAY = '0;37' 231COLOR_DARK_GRAY = '1;30' 232COLOR_LIGHT_RED = '1;31' 233COLOR_LIGHT_GREEN = '1;32' 234COLOR_YELLOW = '1;33' 235COLOR_LIGHT_BLUE = '1;34' 236COLOR_LIGHT_PURPLE = '1;35' 237COLOR_LIGHT_CYAN = '1;36' 238COLOR_WHITE = '1;37' 239 240### helper functions ### 241def get_devnull(): 242 """Get the file object of '/dev/null' device.""" 243 try: 244 devnull = subprocess.DEVNULL # py3k 245 except AttributeError: 246 devnull = open(os.devnull, 'wb') 247 return devnull 248 249def check_top_directory(): 250 """Exit if we are not at the top of source directory.""" 251 for f in ('README', 'Licenses'): 252 if not os.path.exists(f): 253 sys.exit('Please run at the top of source directory.') 254 255def check_clean_directory(): 256 """Exit if the source tree is not clean.""" 257 for f in ('.config', 'include/config'): 258 if os.path.exists(f): 259 sys.exit("source tree is not clean, please run 'make mrproper'") 260 261def get_make_cmd(): 262 """Get the command name of GNU Make. 263 264 U-Boot needs GNU Make for building, but the command name is not 265 necessarily "make". (for example, "gmake" on FreeBSD). 266 Returns the most appropriate command name on your system. 267 """ 268 process = subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE) 269 ret = process.communicate() 270 if process.returncode: 271 sys.exit('GNU Make not found') 272 return ret[0].rstrip() 273 274def color_text(color_enabled, color, string): 275 """Return colored string.""" 276 if color_enabled: 277 return '\033[' + color + 'm' + string + '\033[0m' 278 else: 279 return string 280 281def log_msg(color_enabled, color, defconfig, msg): 282 """Return the formated line for the log.""" 283 return defconfig[:-len('_defconfig')].ljust(37) + ': ' + \ 284 color_text(color_enabled, color, msg) + '\n' 285 286def update_cross_compile(color_enabled): 287 """Update per-arch CROSS_COMPILE via environment variables 288 289 The default CROSS_COMPILE values are available 290 in the CROSS_COMPILE list above. 291 292 You can override them via environment variables 293 CROSS_COMPILE_{ARCH}. 294 295 For example, if you want to override toolchain prefixes 296 for ARM and PowerPC, you can do as follows in your shell: 297 298 export CROSS_COMPILE_ARM=... 299 export CROSS_COMPILE_POWERPC=... 300 301 Then, this function checks if specified compilers really exist in your 302 PATH environment. 303 """ 304 archs = [] 305 306 for arch in os.listdir('arch'): 307 if os.path.exists(os.path.join('arch', arch, 'Makefile')): 308 archs.append(arch) 309 310 # arm64 is a special case 311 archs.append('aarch64') 312 313 for arch in archs: 314 env = 'CROSS_COMPILE_' + arch.upper() 315 cross_compile = os.environ.get(env) 316 if not cross_compile: 317 cross_compile = CROSS_COMPILE.get(arch, '') 318 319 for path in os.environ["PATH"].split(os.pathsep): 320 gcc_path = os.path.join(path, cross_compile + 'gcc') 321 if os.path.isfile(gcc_path) and os.access(gcc_path, os.X_OK): 322 break 323 else: 324 print >> sys.stderr, color_text(color_enabled, COLOR_YELLOW, 325 'warning: %sgcc: not found in PATH. %s architecture boards will be skipped' 326 % (cross_compile, arch)) 327 cross_compile = None 328 329 CROSS_COMPILE[arch] = cross_compile 330 331def cleanup_one_header(header_path, patterns, dry_run): 332 """Clean regex-matched lines away from a file. 333 334 Arguments: 335 header_path: path to the cleaned file. 336 patterns: list of regex patterns. Any lines matching to these 337 patterns are deleted. 338 dry_run: make no changes, but still display log. 339 """ 340 with open(header_path) as f: 341 lines = f.readlines() 342 343 matched = [] 344 for i, line in enumerate(lines): 345 for pattern in patterns: 346 m = pattern.search(line) 347 if m: 348 print '%s: %s: %s' % (header_path, i + 1, line), 349 matched.append(i) 350 break 351 352 if dry_run or not matched: 353 return 354 355 with open(header_path, 'w') as f: 356 for i, line in enumerate(lines): 357 if not i in matched: 358 f.write(line) 359 360def cleanup_headers(config_attrs, dry_run): 361 """Delete config defines from board headers. 362 363 Arguments: 364 config_attrs: A list of dictionaris, each of them includes the name, 365 the type, and the default value of the target config. 366 dry_run: make no changes, but still display log. 367 """ 368 while True: 369 choice = raw_input('Clean up headers? [y/n]: ').lower() 370 print choice 371 if choice == 'y' or choice == 'n': 372 break 373 374 if choice == 'n': 375 return 376 377 patterns = [] 378 for config_attr in config_attrs: 379 config = config_attr['config'] 380 patterns.append(re.compile(r'#\s*define\s+%s\W' % config)) 381 patterns.append(re.compile(r'#\s*undef\s+%s\W' % config)) 382 383 for dir in 'include', 'arch', 'board': 384 for (dirpath, dirnames, filenames) in os.walk(dir): 385 for filename in filenames: 386 if not fnmatch.fnmatch(filename, '*~'): 387 cleanup_one_header(os.path.join(dirpath, filename), 388 patterns, dry_run) 389 390### classes ### 391class Progress: 392 393 """Progress Indicator""" 394 395 def __init__(self, total): 396 """Create a new progress indicator. 397 398 Arguments: 399 total: A number of defconfig files to process. 400 """ 401 self.current = 0 402 self.total = total 403 404 def inc(self): 405 """Increment the number of processed defconfig files.""" 406 407 self.current += 1 408 409 def show(self): 410 """Display the progress.""" 411 print ' %d defconfigs out of %d\r' % (self.current, self.total), 412 sys.stdout.flush() 413 414class KconfigParser: 415 416 """A parser of .config and include/autoconf.mk.""" 417 418 re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"') 419 re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"') 420 421 def __init__(self, config_attrs, options, progress, build_dir): 422 """Create a new parser. 423 424 Arguments: 425 config_attrs: A list of dictionaris, each of them includes the name, 426 the type, and the default value of the target config. 427 options: option flags. 428 progress: A progress indicator 429 build_dir: Build directory. 430 """ 431 self.config_attrs = config_attrs 432 self.options = options 433 self.progress = progress 434 self.build_dir = build_dir 435 436 def get_cross_compile(self): 437 """Parse .config file and return CROSS_COMPILE. 438 439 Returns: 440 A string storing the compiler prefix for the architecture. 441 Return a NULL string for architectures that do not require 442 compiler prefix (Sandbox and native build is the case). 443 Return None if the specified compiler is missing in your PATH. 444 Caller should distinguish '' and None. 445 """ 446 arch = '' 447 cpu = '' 448 dotconfig = os.path.join(self.build_dir, '.config') 449 for line in open(dotconfig): 450 m = self.re_arch.match(line) 451 if m: 452 arch = m.group(1) 453 continue 454 m = self.re_cpu.match(line) 455 if m: 456 cpu = m.group(1) 457 458 if not arch: 459 return None 460 461 # fix-up for aarch64 462 if arch == 'arm' and cpu == 'armv8': 463 arch = 'aarch64' 464 465 return CROSS_COMPILE.get(arch, None) 466 467 def parse_one_config(self, config_attr, defconfig_lines, autoconf_lines): 468 """Parse .config, defconfig, include/autoconf.mk for one config. 469 470 This function looks for the config options in the lines from 471 defconfig, .config, and include/autoconf.mk in order to decide 472 which action should be taken for this defconfig. 473 474 Arguments: 475 config_attr: A dictionary including the name, the type, 476 and the default value of the target config. 477 defconfig_lines: lines from the original defconfig file. 478 autoconf_lines: lines from the include/autoconf.mk file. 479 480 Returns: 481 A tupple of the action for this defconfig and the line 482 matched for the config. 483 """ 484 config = config_attr['config'] 485 not_set = '# %s is not set' % config 486 487 if config_attr['type'] in ('bool', 'tristate') and \ 488 config_attr['default'] == 'n': 489 default = not_set 490 else: 491 default = config + '=' + config_attr['default'] 492 493 for line in defconfig_lines: 494 line = line.rstrip() 495 if line.startswith(config + '=') or line == not_set: 496 return (ACTION_ALREADY_EXIST, line) 497 498 if config_attr['type'] in ('bool', 'tristate'): 499 value = not_set 500 else: 501 value = '(undefined)' 502 503 for line in autoconf_lines: 504 line = line.rstrip() 505 if line.startswith(config + '='): 506 value = line 507 break 508 509 if value == default: 510 action = ACTION_DEFAULT_VALUE 511 elif value == '(undefined)': 512 action = ACTION_UNDEFINED 513 else: 514 action = ACTION_MOVE 515 516 return (action, value) 517 518 def update_dotconfig(self, defconfig): 519 """Parse files for the config options and update the .config. 520 521 This function parses the given defconfig, the generated .config 522 and include/autoconf.mk searching the target options. 523 Move the config option(s) to the .config as needed. 524 Also, display the log to show what happened to the .config. 525 526 Arguments: 527 defconfig: defconfig name. 528 """ 529 530 defconfig_path = os.path.join('configs', defconfig) 531 dotconfig_path = os.path.join(self.build_dir, '.config') 532 autoconf_path = os.path.join(self.build_dir, 'include', 'autoconf.mk') 533 results = [] 534 535 with open(defconfig_path) as f: 536 defconfig_lines = f.readlines() 537 538 with open(autoconf_path) as f: 539 autoconf_lines = f.readlines() 540 541 for config_attr in self.config_attrs: 542 result = self.parse_one_config(config_attr, defconfig_lines, 543 autoconf_lines) 544 results.append(result) 545 546 log = '' 547 548 for (action, value) in results: 549 if action == ACTION_MOVE: 550 actlog = "Move '%s'" % value 551 log_color = COLOR_LIGHT_GREEN 552 elif action == ACTION_DEFAULT_VALUE: 553 actlog = "Default value '%s'. Do nothing." % value 554 log_color = COLOR_LIGHT_BLUE 555 elif action == ACTION_ALREADY_EXIST: 556 actlog = "'%s' already defined in Kconfig. Do nothing." % value 557 log_color = COLOR_LIGHT_PURPLE 558 elif action == ACTION_UNDEFINED: 559 actlog = "Undefined. Do nothing." 560 log_color = COLOR_DARK_GRAY 561 else: 562 sys.exit("Internal Error. This should not happen.") 563 564 log += log_msg(self.options.color, log_color, defconfig, actlog) 565 566 # Some threads are running in parallel. 567 # Print log in one shot to not mix up logs from different threads. 568 print log, 569 self.progress.show() 570 571 with open(dotconfig_path, 'a') as f: 572 for (action, value) in results: 573 if action == ACTION_MOVE: 574 f.write(value + '\n') 575 576 os.remove(os.path.join(self.build_dir, 'include', 'config', 'auto.conf')) 577 os.remove(autoconf_path) 578 579class Slot: 580 581 """A slot to store a subprocess. 582 583 Each instance of this class handles one subprocess. 584 This class is useful to control multiple threads 585 for faster processing. 586 """ 587 588 def __init__(self, config_attrs, options, progress, devnull, make_cmd): 589 """Create a new process slot. 590 591 Arguments: 592 config_attrs: A list of dictionaris, each of them includes the name, 593 the type, and the default value of the target config. 594 options: option flags. 595 progress: A progress indicator. 596 devnull: A file object of '/dev/null'. 597 make_cmd: command name of GNU Make. 598 """ 599 self.options = options 600 self.progress = progress 601 self.build_dir = tempfile.mkdtemp() 602 self.devnull = devnull 603 self.make_cmd = (make_cmd, 'O=' + self.build_dir) 604 self.parser = KconfigParser(config_attrs, options, progress, 605 self.build_dir) 606 self.state = STATE_IDLE 607 self.failed_boards = [] 608 609 def __del__(self): 610 """Delete the working directory 611 612 This function makes sure the temporary directory is cleaned away 613 even if Python suddenly dies due to error. It should be done in here 614 because it is guranteed the destructor is always invoked when the 615 instance of the class gets unreferenced. 616 617 If the subprocess is still running, wait until it finishes. 618 """ 619 if self.state != STATE_IDLE: 620 while self.ps.poll() == None: 621 pass 622 shutil.rmtree(self.build_dir) 623 624 def add(self, defconfig): 625 """Assign a new subprocess for defconfig and add it to the slot. 626 627 If the slot is vacant, create a new subprocess for processing the 628 given defconfig and add it to the slot. Just returns False if 629 the slot is occupied (i.e. the current subprocess is still running). 630 631 Arguments: 632 defconfig: defconfig name. 633 634 Returns: 635 Return True on success or False on failure 636 """ 637 if self.state != STATE_IDLE: 638 return False 639 cmd = list(self.make_cmd) 640 cmd.append(defconfig) 641 self.ps = subprocess.Popen(cmd, stdout=self.devnull, 642 stderr=subprocess.PIPE) 643 self.defconfig = defconfig 644 self.state = STATE_DEFCONFIG 645 return True 646 647 def poll(self): 648 """Check the status of the subprocess and handle it as needed. 649 650 Returns True if the slot is vacant (i.e. in idle state). 651 If the configuration is successfully finished, assign a new 652 subprocess to build include/autoconf.mk. 653 If include/autoconf.mk is generated, invoke the parser to 654 parse the .config and the include/autoconf.mk, and then set the 655 slot back to the idle state. 656 657 Returns: 658 Return True if the subprocess is terminated, False otherwise 659 """ 660 if self.state == STATE_IDLE: 661 return True 662 663 if self.ps.poll() == None: 664 return False 665 666 if self.ps.poll() != 0: 667 print >> sys.stderr, log_msg(self.options.color, COLOR_LIGHT_RED, 668 self.defconfig, "Failed to process."), 669 if self.options.verbose: 670 print >> sys.stderr, color_text(self.options.color, 671 COLOR_LIGHT_CYAN, 672 self.ps.stderr.read()) 673 self.progress.inc() 674 self.progress.show() 675 if self.options.exit_on_error: 676 sys.exit("Exit on error.") 677 # If --exit-on-error flag is not set, skip this board and continue. 678 # Record the failed board. 679 self.failed_boards.append(self.defconfig) 680 self.state = STATE_IDLE 681 return True 682 683 if self.state == STATE_AUTOCONF: 684 self.parser.update_dotconfig(self.defconfig) 685 686 """Save off the defconfig in a consistent way""" 687 cmd = list(self.make_cmd) 688 cmd.append('savedefconfig') 689 self.ps = subprocess.Popen(cmd, stdout=self.devnull, 690 stderr=subprocess.PIPE) 691 self.state = STATE_SAVEDEFCONFIG 692 return False 693 694 if self.state == STATE_SAVEDEFCONFIG: 695 if not self.options.dry_run: 696 shutil.move(os.path.join(self.build_dir, 'defconfig'), 697 os.path.join('configs', self.defconfig)) 698 self.progress.inc() 699 self.state = STATE_IDLE 700 return True 701 702 self.cross_compile = self.parser.get_cross_compile() 703 if self.cross_compile is None: 704 print >> sys.stderr, log_msg(self.options.color, COLOR_YELLOW, 705 self.defconfig, 706 "Compiler is missing. Do nothing."), 707 self.progress.inc() 708 self.progress.show() 709 if self.options.exit_on_error: 710 sys.exit("Exit on error.") 711 # If --exit-on-error flag is not set, skip this board and continue. 712 # Record the failed board. 713 self.failed_boards.append(self.defconfig) 714 self.state = STATE_IDLE 715 return True 716 717 cmd = list(self.make_cmd) 718 if self.cross_compile: 719 cmd.append('CROSS_COMPILE=%s' % self.cross_compile) 720 cmd.append('KCONFIG_IGNORE_DUPLICATES=1') 721 cmd.append('include/config/auto.conf') 722 self.ps = subprocess.Popen(cmd, stdout=self.devnull, 723 stderr=subprocess.PIPE) 724 self.state = STATE_AUTOCONF 725 return False 726 727 def get_failed_boards(self): 728 """Returns a list of failed boards (defconfigs) in this slot. 729 """ 730 return self.failed_boards 731 732class Slots: 733 734 """Controller of the array of subprocess slots.""" 735 736 def __init__(self, config_attrs, options, progress): 737 """Create a new slots controller. 738 739 Arguments: 740 config_attrs: A list of dictionaris containing the name, the type, 741 and the default value of the target CONFIG. 742 options: option flags. 743 progress: A progress indicator. 744 """ 745 self.options = options 746 self.slots = [] 747 devnull = get_devnull() 748 make_cmd = get_make_cmd() 749 for i in range(options.jobs): 750 self.slots.append(Slot(config_attrs, options, progress, devnull, 751 make_cmd)) 752 753 def add(self, defconfig): 754 """Add a new subprocess if a vacant slot is found. 755 756 Arguments: 757 defconfig: defconfig name to be put into. 758 759 Returns: 760 Return True on success or False on failure 761 """ 762 for slot in self.slots: 763 if slot.add(defconfig): 764 return True 765 return False 766 767 def available(self): 768 """Check if there is a vacant slot. 769 770 Returns: 771 Return True if at lease one vacant slot is found, False otherwise. 772 """ 773 for slot in self.slots: 774 if slot.poll(): 775 return True 776 return False 777 778 def empty(self): 779 """Check if all slots are vacant. 780 781 Returns: 782 Return True if all the slots are vacant, False otherwise. 783 """ 784 ret = True 785 for slot in self.slots: 786 if not slot.poll(): 787 ret = False 788 return ret 789 790 def show_failed_boards(self): 791 """Display all of the failed boards (defconfigs).""" 792 failed_boards = [] 793 794 for slot in self.slots: 795 failed_boards += slot.get_failed_boards() 796 797 if len(failed_boards) > 0: 798 msg = [ "The following boards were not processed due to error:" ] 799 msg += failed_boards 800 for line in msg: 801 print >> sys.stderr, color_text(self.options.color, 802 COLOR_LIGHT_RED, line) 803 804 with open('moveconfig.failed', 'w') as f: 805 for board in failed_boards: 806 f.write(board + '\n') 807 808def move_config(config_attrs, options): 809 """Move config options to defconfig files. 810 811 Arguments: 812 config_attrs: A list of dictionaris, each of them includes the name, 813 the type, and the default value of the target config. 814 options: option flags 815 """ 816 if len(config_attrs) == 0: 817 print 'Nothing to do. exit.' 818 sys.exit(0) 819 820 print 'Move the following CONFIG options (jobs: %d)' % options.jobs 821 for config_attr in config_attrs: 822 print ' %s (type: %s, default: %s)' % (config_attr['config'], 823 config_attr['type'], 824 config_attr['default']) 825 826 if options.defconfigs: 827 defconfigs = [line.strip() for line in open(options.defconfigs)] 828 for i, defconfig in enumerate(defconfigs): 829 if not defconfig.endswith('_defconfig'): 830 defconfigs[i] = defconfig + '_defconfig' 831 if not os.path.exists(os.path.join('configs', defconfigs[i])): 832 sys.exit('%s - defconfig does not exist. Stopping.' % 833 defconfigs[i]) 834 else: 835 # All the defconfig files to be processed 836 defconfigs = [] 837 for (dirpath, dirnames, filenames) in os.walk('configs'): 838 dirpath = dirpath[len('configs') + 1:] 839 for filename in fnmatch.filter(filenames, '*_defconfig'): 840 defconfigs.append(os.path.join(dirpath, filename)) 841 842 progress = Progress(len(defconfigs)) 843 slots = Slots(config_attrs, options, progress) 844 845 # Main loop to process defconfig files: 846 # Add a new subprocess into a vacant slot. 847 # Sleep if there is no available slot. 848 for defconfig in defconfigs: 849 while not slots.add(defconfig): 850 while not slots.available(): 851 # No available slot: sleep for a while 852 time.sleep(SLEEP_TIME) 853 854 # wait until all the subprocesses finish 855 while not slots.empty(): 856 time.sleep(SLEEP_TIME) 857 858 progress.show() 859 print '' 860 slots.show_failed_boards() 861 862def bad_recipe(filename, linenum, msg): 863 """Print error message with the file name and the line number and exit.""" 864 sys.exit("%s: line %d: error : " % (filename, linenum) + msg) 865 866def parse_recipe(filename): 867 """Parse the recipe file and retrieve the config attributes. 868 869 This function parses the given recipe file and gets the name, 870 the type, and the default value of the target config options. 871 872 Arguments: 873 filename: path to file to be parsed. 874 Returns: 875 A list of dictionaris, each of them includes the name, 876 the type, and the default value of the target config. 877 """ 878 config_attrs = [] 879 linenum = 1 880 881 for line in open(filename): 882 tokens = line.split() 883 if len(tokens) != 3: 884 bad_recipe(filename, linenum, 885 "%d fields in this line. Each line must contain 3 fields" 886 % len(tokens)) 887 888 (config, type, default) = tokens 889 890 # prefix the option name with CONFIG_ if missing 891 if not config.startswith('CONFIG_'): 892 config = 'CONFIG_' + config 893 894 # sanity check of default values 895 if type == 'bool': 896 if not default in ('y', 'n'): 897 bad_recipe(filename, linenum, 898 "default for bool type must be either y or n") 899 elif type == 'tristate': 900 if not default in ('y', 'm', 'n'): 901 bad_recipe(filename, linenum, 902 "default for tristate type must be y, m, or n") 903 elif type == 'string': 904 if default[0] != '"' or default[-1] != '"': 905 bad_recipe(filename, linenum, 906 "default for string type must be surrounded by double-quotations") 907 elif type == 'int': 908 try: 909 int(default) 910 except: 911 bad_recipe(filename, linenum, 912 "type is int, but default value is not decimal") 913 elif type == 'hex': 914 if len(default) < 2 or default[:2] != '0x': 915 bad_recipe(filename, linenum, 916 "default for hex type must be prefixed with 0x") 917 try: 918 int(default, 16) 919 except: 920 bad_recipe(filename, linenum, 921 "type is hex, but default value is not hexadecimal") 922 else: 923 bad_recipe(filename, linenum, 924 "unsupported type '%s'. type must be one of bool, tristate, string, int, hex" 925 % type) 926 927 config_attrs.append({'config': config, 'type': type, 'default': default}) 928 linenum += 1 929 930 return config_attrs 931 932def main(): 933 try: 934 cpu_count = multiprocessing.cpu_count() 935 except NotImplementedError: 936 cpu_count = 1 937 938 parser = optparse.OptionParser() 939 # Add options here 940 parser.add_option('-c', '--color', action='store_true', default=False, 941 help='display the log in color') 942 parser.add_option('-d', '--defconfigs', type='string', 943 help='a file containing a list of defconfigs to move') 944 parser.add_option('-n', '--dry-run', action='store_true', default=False, 945 help='perform a trial run (show log with no changes)') 946 parser.add_option('-e', '--exit-on-error', action='store_true', 947 default=False, 948 help='exit immediately on any error') 949 parser.add_option('-H', '--headers-only', dest='cleanup_headers_only', 950 action='store_true', default=False, 951 help='only cleanup the headers') 952 parser.add_option('-j', '--jobs', type='int', default=cpu_count, 953 help='the number of jobs to run simultaneously') 954 parser.add_option('-v', '--verbose', action='store_true', default=False, 955 help='show any build errors as boards are built') 956 parser.usage += ' recipe_file\n\n' + \ 957 'The recipe_file should describe config options you want to move.\n' + \ 958 'Each line should contain config_name, type, default_value\n\n' + \ 959 'Example:\n' + \ 960 'CONFIG_FOO bool n\n' + \ 961 'CONFIG_BAR int 100\n' + \ 962 'CONFIG_BAZ string "hello"\n' 963 964 (options, args) = parser.parse_args() 965 966 if len(args) != 1: 967 parser.print_usage() 968 sys.exit(1) 969 970 config_attrs = parse_recipe(args[0]) 971 972 check_top_directory() 973 974 check_clean_directory() 975 976 update_cross_compile(options.color) 977 978 if not options.cleanup_headers_only: 979 move_config(config_attrs, options) 980 981 cleanup_headers(config_attrs, options.dry_run) 982 983if __name__ == '__main__': 984 main() 985