1#!/usr/bin/env python2 2# 3# Author: Masahiro Yamada <yamada.masahiro@socionext.com> 4# 5# SPDX-License-Identifier: GPL-2.0+ 6# 7 8""" 9Move config options from headers to defconfig files. 10 11Since Kconfig was introduced to U-Boot, we have worked on moving 12config options from headers to Kconfig (defconfig). 13 14This tool intends to help this tremendous work. 15 16 17Usage 18----- 19 20First, you must edit the Kconfig to add the menu entries for the configs 21you are moving. 22 23And then run this tool giving CONFIG names you want to move. 24For example, if you want to move CONFIG_CMD_USB and CONFIG_SYS_TEXT_BASE, 25simply type as follows: 26 27 $ tools/moveconfig.py CONFIG_CMD_USB CONFIG_SYS_TEXT_BASE 28 29The tool walks through all the defconfig files and move the given CONFIGs. 30 31The log is also displayed on the terminal. 32 33The log is printed for each defconfig as follows: 34 35<defconfig_name> 36 <action1> 37 <action2> 38 <action3> 39 ... 40 41<defconfig_name> is the name of the defconfig. 42 43<action*> shows what the tool did for that defconfig. 44It looks like one of the following: 45 46 - Move 'CONFIG_... ' 47 This config option was moved to the defconfig 48 49 - CONFIG_... is not defined in Kconfig. Do nothing. 50 The entry for this CONFIG was not found in Kconfig. The option is not 51 defined in the config header, either. So, this case can be just skipped. 52 53 - CONFIG_... is not defined in Kconfig (suspicious). Do nothing. 54 This option is defined in the config header, but its entry was not found 55 in Kconfig. 56 There are two common cases: 57 - You forgot to create an entry for the CONFIG before running 58 this tool, or made a typo in a CONFIG passed to this tool. 59 - The entry was hidden due to unmet 'depends on'. 60 The tool does not know if the result is reasonable, so please check it 61 manually. 62 63 - 'CONFIG_...' is the same as the define in Kconfig. Do nothing. 64 The define in the config header matched the one in Kconfig. 65 We do not need to touch it. 66 67 - Compiler is missing. Do nothing. 68 The compiler specified for this architecture was not found 69 in your PATH environment. 70 (If -e option is passed, the tool exits immediately.) 71 72 - Failed to process. 73 An error occurred during processing this defconfig. Skipped. 74 (If -e option is passed, the tool exits immediately on error.) 75 76Finally, you will be asked, Clean up headers? [y/n]: 77 78If you say 'y' here, the unnecessary config defines are removed 79from the config headers (include/configs/*.h). 80It just uses the regex method, so you should not rely on it. 81Just in case, please do 'git diff' to see what happened. 82 83 84How does it work? 85----------------- 86 87This tool runs configuration and builds include/autoconf.mk for every 88defconfig. The config options defined in Kconfig appear in the .config 89file (unless they are hidden because of unmet dependency.) 90On the other hand, the config options defined by board headers are seen 91in include/autoconf.mk. The tool looks for the specified options in both 92of them to decide the appropriate action for the options. If the given 93config option is found in the .config, but its value does not match the 94one from the board header, the config option in the .config is replaced 95with the define in the board header. Then, the .config is synced by 96"make savedefconfig" and the defconfig is updated with it. 97 98For faster processing, this tool handles multi-threading. It creates 99separate build directories where the out-of-tree build is run. The 100temporary build directories are automatically created and deleted as 101needed. The number of threads are chosen based on the number of the CPU 102cores of your system although you can change it via -j (--jobs) option. 103 104 105Toolchains 106---------- 107 108Appropriate toolchain are necessary to generate include/autoconf.mk 109for all the architectures supported by U-Boot. Most of them are available 110at the kernel.org site, some are not provided by kernel.org. 111 112The default per-arch CROSS_COMPILE used by this tool is specified by 113the list below, CROSS_COMPILE. You may wish to update the list to 114use your own. Instead of modifying the list directly, you can give 115them via environments. 116 117 118Available options 119----------------- 120 121 -c, --color 122 Surround each portion of the log with escape sequences to display it 123 in color on the terminal. 124 125 -d, --defconfigs 126 Specify a file containing a list of defconfigs to move 127 128 -n, --dry-run 129 Perform a trial run that does not make any changes. It is useful to 130 see what is going to happen before one actually runs it. 131 132 -e, --exit-on-error 133 Exit immediately if Make exits with a non-zero status while processing 134 a defconfig file. 135 136 -s, --force-sync 137 Do "make savedefconfig" forcibly for all the defconfig files. 138 If not specified, "make savedefconfig" only occurs for cases 139 where at least one CONFIG was moved. 140 141 -S, --spl 142 Look for moved config options in spl/include/autoconf.mk instead of 143 include/autoconf.mk. This is useful for moving options for SPL build 144 because SPL related options (mostly prefixed with CONFIG_SPL_) are 145 sometimes blocked by CONFIG_SPL_BUILD ifdef conditionals. 146 147 -H, --headers-only 148 Only cleanup the headers; skip the defconfig processing 149 150 -j, --jobs 151 Specify the number of threads to run simultaneously. If not specified, 152 the number of threads is the same as the number of CPU cores. 153 154 -r, --git-ref 155 Specify the git ref to clone for building the autoconf.mk. If unspecified 156 use the CWD. This is useful for when changes to the Kconfig affect the 157 default values and you want to capture the state of the defconfig from 158 before that change was in effect. If in doubt, specify a ref pre-Kconfig 159 changes (use HEAD if Kconfig changes are not committed). Worst case it will 160 take a bit longer to run, but will always do the right thing. 161 162 -v, --verbose 163 Show any build errors as boards are built 164 165 -y, --yes 166 Instead of prompting, automatically go ahead with all operations. This 167 includes cleaning up headers and CONFIG_SYS_EXTRA_OPTIONS. 168 169To see the complete list of supported options, run 170 171 $ tools/moveconfig.py -h 172 173""" 174 175import copy 176import difflib 177import filecmp 178import fnmatch 179import multiprocessing 180import optparse 181import os 182import re 183import shutil 184import subprocess 185import sys 186import tempfile 187import time 188 189SHOW_GNU_MAKE = 'scripts/show-gnu-make' 190SLEEP_TIME=0.03 191 192# Here is the list of cross-tools I use. 193# Most of them are available at kernel.org 194# (https://www.kernel.org/pub/tools/crosstool/files/bin/), except the following: 195# arc: https://github.com/foss-for-synopsys-dwc-arc-processors/toolchain/releases 196# blackfin: http://sourceforge.net/projects/adi-toolchain/files/ 197# nds32: http://osdk.andestech.com/packages/nds32le-linux-glibc-v1.tgz 198# nios2: https://sourcery.mentor.com/GNUToolchain/subscription42545 199# sh: http://sourcery.mentor.com/public/gnu_toolchain/sh-linux-gnu 200# 201# openrisc kernel.org toolchain is out of date, download latest one from 202# http://opencores.org/or1k/OpenRISC_GNU_tool_chain#Prebuilt_versions 203CROSS_COMPILE = { 204 'arc': 'arc-linux-', 205 'aarch64': 'aarch64-linux-', 206 'arm': 'arm-unknown-linux-gnueabi-', 207 'avr32': 'avr32-linux-', 208 'blackfin': 'bfin-elf-', 209 'm68k': 'm68k-linux-', 210 'microblaze': 'microblaze-linux-', 211 'mips': 'mips-linux-', 212 'nds32': 'nds32le-linux-', 213 'nios2': 'nios2-linux-gnu-', 214 'openrisc': 'or1k-elf-', 215 'powerpc': 'powerpc-linux-', 216 'sh': 'sh-linux-gnu-', 217 'sparc': 'sparc-linux-', 218 'x86': 'i386-linux-', 219 'xtensa': 'xtensa-linux-' 220} 221 222STATE_IDLE = 0 223STATE_DEFCONFIG = 1 224STATE_AUTOCONF = 2 225STATE_SAVEDEFCONFIG = 3 226 227ACTION_MOVE = 0 228ACTION_NO_ENTRY = 1 229ACTION_NO_ENTRY_WARN = 2 230ACTION_NO_CHANGE = 3 231 232COLOR_BLACK = '0;30' 233COLOR_RED = '0;31' 234COLOR_GREEN = '0;32' 235COLOR_BROWN = '0;33' 236COLOR_BLUE = '0;34' 237COLOR_PURPLE = '0;35' 238COLOR_CYAN = '0;36' 239COLOR_LIGHT_GRAY = '0;37' 240COLOR_DARK_GRAY = '1;30' 241COLOR_LIGHT_RED = '1;31' 242COLOR_LIGHT_GREEN = '1;32' 243COLOR_YELLOW = '1;33' 244COLOR_LIGHT_BLUE = '1;34' 245COLOR_LIGHT_PURPLE = '1;35' 246COLOR_LIGHT_CYAN = '1;36' 247COLOR_WHITE = '1;37' 248 249### helper functions ### 250def get_devnull(): 251 """Get the file object of '/dev/null' device.""" 252 try: 253 devnull = subprocess.DEVNULL # py3k 254 except AttributeError: 255 devnull = open(os.devnull, 'wb') 256 return devnull 257 258def check_top_directory(): 259 """Exit if we are not at the top of source directory.""" 260 for f in ('README', 'Licenses'): 261 if not os.path.exists(f): 262 sys.exit('Please run at the top of source directory.') 263 264def check_clean_directory(): 265 """Exit if the source tree is not clean.""" 266 for f in ('.config', 'include/config'): 267 if os.path.exists(f): 268 sys.exit("source tree is not clean, please run 'make mrproper'") 269 270def get_make_cmd(): 271 """Get the command name of GNU Make. 272 273 U-Boot needs GNU Make for building, but the command name is not 274 necessarily "make". (for example, "gmake" on FreeBSD). 275 Returns the most appropriate command name on your system. 276 """ 277 process = subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE) 278 ret = process.communicate() 279 if process.returncode: 280 sys.exit('GNU Make not found') 281 return ret[0].rstrip() 282 283def get_all_defconfigs(): 284 """Get all the defconfig files under the configs/ directory.""" 285 defconfigs = [] 286 for (dirpath, dirnames, filenames) in os.walk('configs'): 287 dirpath = dirpath[len('configs') + 1:] 288 for filename in fnmatch.filter(filenames, '*_defconfig'): 289 defconfigs.append(os.path.join(dirpath, filename)) 290 291 return defconfigs 292 293def color_text(color_enabled, color, string): 294 """Return colored string.""" 295 if color_enabled: 296 # LF should not be surrounded by the escape sequence. 297 # Otherwise, additional whitespace or line-feed might be printed. 298 return '\n'.join([ '\033[' + color + 'm' + s + '\033[0m' if s else '' 299 for s in string.split('\n') ]) 300 else: 301 return string 302 303def show_diff(a, b, file_path, color_enabled): 304 """Show unidified diff. 305 306 Arguments: 307 a: A list of lines (before) 308 b: A list of lines (after) 309 file_path: Path to the file 310 color_enabled: Display the diff in color 311 """ 312 313 diff = difflib.unified_diff(a, b, 314 fromfile=os.path.join('a', file_path), 315 tofile=os.path.join('b', file_path)) 316 317 for line in diff: 318 if line[0] == '-' and line[1] != '-': 319 print color_text(color_enabled, COLOR_RED, line), 320 elif line[0] == '+' and line[1] != '+': 321 print color_text(color_enabled, COLOR_GREEN, line), 322 else: 323 print line, 324 325def update_cross_compile(color_enabled): 326 """Update per-arch CROSS_COMPILE via environment variables 327 328 The default CROSS_COMPILE values are available 329 in the CROSS_COMPILE list above. 330 331 You can override them via environment variables 332 CROSS_COMPILE_{ARCH}. 333 334 For example, if you want to override toolchain prefixes 335 for ARM and PowerPC, you can do as follows in your shell: 336 337 export CROSS_COMPILE_ARM=... 338 export CROSS_COMPILE_POWERPC=... 339 340 Then, this function checks if specified compilers really exist in your 341 PATH environment. 342 """ 343 archs = [] 344 345 for arch in os.listdir('arch'): 346 if os.path.exists(os.path.join('arch', arch, 'Makefile')): 347 archs.append(arch) 348 349 # arm64 is a special case 350 archs.append('aarch64') 351 352 for arch in archs: 353 env = 'CROSS_COMPILE_' + arch.upper() 354 cross_compile = os.environ.get(env) 355 if not cross_compile: 356 cross_compile = CROSS_COMPILE.get(arch, '') 357 358 for path in os.environ["PATH"].split(os.pathsep): 359 gcc_path = os.path.join(path, cross_compile + 'gcc') 360 if os.path.isfile(gcc_path) and os.access(gcc_path, os.X_OK): 361 break 362 else: 363 print >> sys.stderr, color_text(color_enabled, COLOR_YELLOW, 364 'warning: %sgcc: not found in PATH. %s architecture boards will be skipped' 365 % (cross_compile, arch)) 366 cross_compile = None 367 368 CROSS_COMPILE[arch] = cross_compile 369 370def extend_matched_lines(lines, matched, pre_patterns, post_patterns, extend_pre, 371 extend_post): 372 """Extend matched lines if desired patterns are found before/after already 373 matched lines. 374 375 Arguments: 376 lines: A list of lines handled. 377 matched: A list of line numbers that have been already matched. 378 (will be updated by this function) 379 pre_patterns: A list of regular expression that should be matched as 380 preamble. 381 post_patterns: A list of regular expression that should be matched as 382 postamble. 383 extend_pre: Add the line number of matched preamble to the matched list. 384 extend_post: Add the line number of matched postamble to the matched list. 385 """ 386 extended_matched = [] 387 388 j = matched[0] 389 390 for i in matched: 391 if i == 0 or i < j: 392 continue 393 j = i 394 while j in matched: 395 j += 1 396 if j >= len(lines): 397 break 398 399 for p in pre_patterns: 400 if p.search(lines[i - 1]): 401 break 402 else: 403 # not matched 404 continue 405 406 for p in post_patterns: 407 if p.search(lines[j]): 408 break 409 else: 410 # not matched 411 continue 412 413 if extend_pre: 414 extended_matched.append(i - 1) 415 if extend_post: 416 extended_matched.append(j) 417 418 matched += extended_matched 419 matched.sort() 420 421def cleanup_one_header(header_path, patterns, options): 422 """Clean regex-matched lines away from a file. 423 424 Arguments: 425 header_path: path to the cleaned file. 426 patterns: list of regex patterns. Any lines matching to these 427 patterns are deleted. 428 options: option flags. 429 """ 430 with open(header_path) as f: 431 lines = f.readlines() 432 433 matched = [] 434 for i, line in enumerate(lines): 435 if i - 1 in matched and lines[i - 1][-2:] == '\\\n': 436 matched.append(i) 437 continue 438 for pattern in patterns: 439 if pattern.search(line): 440 matched.append(i) 441 break 442 443 if not matched: 444 return 445 446 # remove empty #ifdef ... #endif, successive blank lines 447 pattern_if = re.compile(r'#\s*if(def|ndef)?\W') # #if, #ifdef, #ifndef 448 pattern_elif = re.compile(r'#\s*el(if|se)\W') # #elif, #else 449 pattern_endif = re.compile(r'#\s*endif\W') # #endif 450 pattern_blank = re.compile(r'^\s*$') # empty line 451 452 while True: 453 old_matched = copy.copy(matched) 454 extend_matched_lines(lines, matched, [pattern_if], 455 [pattern_endif], True, True) 456 extend_matched_lines(lines, matched, [pattern_elif], 457 [pattern_elif, pattern_endif], True, False) 458 extend_matched_lines(lines, matched, [pattern_if, pattern_elif], 459 [pattern_blank], False, True) 460 extend_matched_lines(lines, matched, [pattern_blank], 461 [pattern_elif, pattern_endif], True, False) 462 extend_matched_lines(lines, matched, [pattern_blank], 463 [pattern_blank], True, False) 464 if matched == old_matched: 465 break 466 467 tolines = copy.copy(lines) 468 469 for i in reversed(matched): 470 tolines.pop(i) 471 472 show_diff(lines, tolines, header_path, options.color) 473 474 if options.dry_run: 475 return 476 477 with open(header_path, 'w') as f: 478 for line in tolines: 479 f.write(line) 480 481def cleanup_headers(configs, options): 482 """Delete config defines from board headers. 483 484 Arguments: 485 configs: A list of CONFIGs to remove. 486 options: option flags. 487 """ 488 if not options.yes: 489 while True: 490 choice = raw_input('Clean up headers? [y/n]: ').lower() 491 print choice 492 if choice == 'y' or choice == 'n': 493 break 494 495 if choice == 'n': 496 return 497 498 patterns = [] 499 for config in configs: 500 patterns.append(re.compile(r'#\s*define\s+%s\W' % config)) 501 patterns.append(re.compile(r'#\s*undef\s+%s\W' % config)) 502 503 for dir in 'include', 'arch', 'board': 504 for (dirpath, dirnames, filenames) in os.walk(dir): 505 if dirpath == os.path.join('include', 'generated'): 506 continue 507 for filename in filenames: 508 if not fnmatch.fnmatch(filename, '*~'): 509 cleanup_one_header(os.path.join(dirpath, filename), 510 patterns, options) 511 512def cleanup_one_extra_option(defconfig_path, configs, options): 513 """Delete config defines in CONFIG_SYS_EXTRA_OPTIONS in one defconfig file. 514 515 Arguments: 516 defconfig_path: path to the cleaned defconfig file. 517 configs: A list of CONFIGs to remove. 518 options: option flags. 519 """ 520 521 start = 'CONFIG_SYS_EXTRA_OPTIONS="' 522 end = '"\n' 523 524 with open(defconfig_path) as f: 525 lines = f.readlines() 526 527 for i, line in enumerate(lines): 528 if line.startswith(start) and line.endswith(end): 529 break 530 else: 531 # CONFIG_SYS_EXTRA_OPTIONS was not found in this defconfig 532 return 533 534 old_tokens = line[len(start):-len(end)].split(',') 535 new_tokens = [] 536 537 for token in old_tokens: 538 pos = token.find('=') 539 if not (token[:pos] if pos >= 0 else token) in configs: 540 new_tokens.append(token) 541 542 if new_tokens == old_tokens: 543 return 544 545 tolines = copy.copy(lines) 546 547 if new_tokens: 548 tolines[i] = start + ','.join(new_tokens) + end 549 else: 550 tolines.pop(i) 551 552 show_diff(lines, tolines, defconfig_path, options.color) 553 554 if options.dry_run: 555 return 556 557 with open(defconfig_path, 'w') as f: 558 for line in tolines: 559 f.write(line) 560 561def cleanup_extra_options(configs, options): 562 """Delete config defines in CONFIG_SYS_EXTRA_OPTIONS in defconfig files. 563 564 Arguments: 565 configs: A list of CONFIGs to remove. 566 options: option flags. 567 """ 568 if not options.yes: 569 while True: 570 choice = (raw_input('Clean up CONFIG_SYS_EXTRA_OPTIONS? [y/n]: '). 571 lower()) 572 print choice 573 if choice == 'y' or choice == 'n': 574 break 575 576 if choice == 'n': 577 return 578 579 configs = [ config[len('CONFIG_'):] for config in configs ] 580 581 defconfigs = get_all_defconfigs() 582 583 for defconfig in defconfigs: 584 cleanup_one_extra_option(os.path.join('configs', defconfig), configs, 585 options) 586 587### classes ### 588class Progress: 589 590 """Progress Indicator""" 591 592 def __init__(self, total): 593 """Create a new progress indicator. 594 595 Arguments: 596 total: A number of defconfig files to process. 597 """ 598 self.current = 0 599 self.total = total 600 601 def inc(self): 602 """Increment the number of processed defconfig files.""" 603 604 self.current += 1 605 606 def show(self): 607 """Display the progress.""" 608 print ' %d defconfigs out of %d\r' % (self.current, self.total), 609 sys.stdout.flush() 610 611class KconfigParser: 612 613 """A parser of .config and include/autoconf.mk.""" 614 615 re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"') 616 re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"') 617 618 def __init__(self, configs, options, build_dir): 619 """Create a new parser. 620 621 Arguments: 622 configs: A list of CONFIGs to move. 623 options: option flags. 624 build_dir: Build directory. 625 """ 626 self.configs = configs 627 self.options = options 628 self.dotconfig = os.path.join(build_dir, '.config') 629 self.autoconf = os.path.join(build_dir, 'include', 'autoconf.mk') 630 self.spl_autoconf = os.path.join(build_dir, 'spl', 'include', 631 'autoconf.mk') 632 self.config_autoconf = os.path.join(build_dir, 'include', 'config', 633 'auto.conf') 634 self.defconfig = os.path.join(build_dir, 'defconfig') 635 636 def get_cross_compile(self): 637 """Parse .config file and return CROSS_COMPILE. 638 639 Returns: 640 A string storing the compiler prefix for the architecture. 641 Return a NULL string for architectures that do not require 642 compiler prefix (Sandbox and native build is the case). 643 Return None if the specified compiler is missing in your PATH. 644 Caller should distinguish '' and None. 645 """ 646 arch = '' 647 cpu = '' 648 for line in open(self.dotconfig): 649 m = self.re_arch.match(line) 650 if m: 651 arch = m.group(1) 652 continue 653 m = self.re_cpu.match(line) 654 if m: 655 cpu = m.group(1) 656 657 if not arch: 658 return None 659 660 # fix-up for aarch64 661 if arch == 'arm' and cpu == 'armv8': 662 arch = 'aarch64' 663 664 return CROSS_COMPILE.get(arch, None) 665 666 def parse_one_config(self, config, dotconfig_lines, autoconf_lines): 667 """Parse .config, defconfig, include/autoconf.mk for one config. 668 669 This function looks for the config options in the lines from 670 defconfig, .config, and include/autoconf.mk in order to decide 671 which action should be taken for this defconfig. 672 673 Arguments: 674 config: CONFIG name to parse. 675 dotconfig_lines: lines from the .config file. 676 autoconf_lines: lines from the include/autoconf.mk file. 677 678 Returns: 679 A tupple of the action for this defconfig and the line 680 matched for the config. 681 """ 682 not_set = '# %s is not set' % config 683 684 for line in autoconf_lines: 685 line = line.rstrip() 686 if line.startswith(config + '='): 687 new_val = line 688 break 689 else: 690 new_val = not_set 691 692 for line in dotconfig_lines: 693 line = line.rstrip() 694 if line.startswith(config + '=') or line == not_set: 695 old_val = line 696 break 697 else: 698 if new_val == not_set: 699 return (ACTION_NO_ENTRY, config) 700 else: 701 return (ACTION_NO_ENTRY_WARN, config) 702 703 # If this CONFIG is neither bool nor trisate 704 if old_val[-2:] != '=y' and old_val[-2:] != '=m' and old_val != not_set: 705 # tools/scripts/define2mk.sed changes '1' to 'y'. 706 # This is a problem if the CONFIG is int type. 707 # Check the type in Kconfig and handle it correctly. 708 if new_val[-2:] == '=y': 709 new_val = new_val[:-1] + '1' 710 711 return (ACTION_NO_CHANGE if old_val == new_val else ACTION_MOVE, 712 new_val) 713 714 def update_dotconfig(self): 715 """Parse files for the config options and update the .config. 716 717 This function parses the generated .config and include/autoconf.mk 718 searching the target options. 719 Move the config option(s) to the .config as needed. 720 721 Arguments: 722 defconfig: defconfig name. 723 724 Returns: 725 Return a tuple of (updated flag, log string). 726 The "updated flag" is True if the .config was updated, False 727 otherwise. The "log string" shows what happend to the .config. 728 """ 729 730 results = [] 731 updated = False 732 suspicious = False 733 rm_files = [self.config_autoconf, self.autoconf] 734 735 if self.options.spl: 736 if os.path.exists(self.spl_autoconf): 737 autoconf_path = self.spl_autoconf 738 rm_files.append(self.spl_autoconf) 739 else: 740 for f in rm_files: 741 os.remove(f) 742 return (updated, suspicious, 743 color_text(self.options.color, COLOR_BROWN, 744 "SPL is not enabled. Skipped.") + '\n') 745 else: 746 autoconf_path = self.autoconf 747 748 with open(self.dotconfig) as f: 749 dotconfig_lines = f.readlines() 750 751 with open(autoconf_path) as f: 752 autoconf_lines = f.readlines() 753 754 for config in self.configs: 755 result = self.parse_one_config(config, dotconfig_lines, 756 autoconf_lines) 757 results.append(result) 758 759 log = '' 760 761 for (action, value) in results: 762 if action == ACTION_MOVE: 763 actlog = "Move '%s'" % value 764 log_color = COLOR_LIGHT_GREEN 765 elif action == ACTION_NO_ENTRY: 766 actlog = "%s is not defined in Kconfig. Do nothing." % value 767 log_color = COLOR_LIGHT_BLUE 768 elif action == ACTION_NO_ENTRY_WARN: 769 actlog = "%s is not defined in Kconfig (suspicious). Do nothing." % value 770 log_color = COLOR_YELLOW 771 suspicious = True 772 elif action == ACTION_NO_CHANGE: 773 actlog = "'%s' is the same as the define in Kconfig. Do nothing." \ 774 % value 775 log_color = COLOR_LIGHT_PURPLE 776 elif action == ACTION_SPL_NOT_EXIST: 777 actlog = "SPL is not enabled for this defconfig. Skip." 778 log_color = COLOR_PURPLE 779 else: 780 sys.exit("Internal Error. This should not happen.") 781 782 log += color_text(self.options.color, log_color, actlog) + '\n' 783 784 with open(self.dotconfig, 'a') as f: 785 for (action, value) in results: 786 if action == ACTION_MOVE: 787 f.write(value + '\n') 788 updated = True 789 790 self.results = results 791 for f in rm_files: 792 os.remove(f) 793 794 return (updated, suspicious, log) 795 796 def check_defconfig(self): 797 """Check the defconfig after savedefconfig 798 799 Returns: 800 Return additional log if moved CONFIGs were removed again by 801 'make savedefconfig'. 802 """ 803 804 log = '' 805 806 with open(self.defconfig) as f: 807 defconfig_lines = f.readlines() 808 809 for (action, value) in self.results: 810 if action != ACTION_MOVE: 811 continue 812 if not value + '\n' in defconfig_lines: 813 log += color_text(self.options.color, COLOR_YELLOW, 814 "'%s' was removed by savedefconfig.\n" % 815 value) 816 817 return log 818 819class Slot: 820 821 """A slot to store a subprocess. 822 823 Each instance of this class handles one subprocess. 824 This class is useful to control multiple threads 825 for faster processing. 826 """ 827 828 def __init__(self, configs, options, progress, devnull, make_cmd, reference_src_dir): 829 """Create a new process slot. 830 831 Arguments: 832 configs: A list of CONFIGs to move. 833 options: option flags. 834 progress: A progress indicator. 835 devnull: A file object of '/dev/null'. 836 make_cmd: command name of GNU Make. 837 reference_src_dir: Determine the true starting config state from this 838 source tree. 839 """ 840 self.options = options 841 self.progress = progress 842 self.build_dir = tempfile.mkdtemp() 843 self.devnull = devnull 844 self.make_cmd = (make_cmd, 'O=' + self.build_dir) 845 self.reference_src_dir = reference_src_dir 846 self.parser = KconfigParser(configs, options, self.build_dir) 847 self.state = STATE_IDLE 848 self.failed_boards = set() 849 self.suspicious_boards = set() 850 851 def __del__(self): 852 """Delete the working directory 853 854 This function makes sure the temporary directory is cleaned away 855 even if Python suddenly dies due to error. It should be done in here 856 because it is guaranteed the destructor is always invoked when the 857 instance of the class gets unreferenced. 858 859 If the subprocess is still running, wait until it finishes. 860 """ 861 if self.state != STATE_IDLE: 862 while self.ps.poll() == None: 863 pass 864 shutil.rmtree(self.build_dir) 865 866 def add(self, defconfig): 867 """Assign a new subprocess for defconfig and add it to the slot. 868 869 If the slot is vacant, create a new subprocess for processing the 870 given defconfig and add it to the slot. Just returns False if 871 the slot is occupied (i.e. the current subprocess is still running). 872 873 Arguments: 874 defconfig: defconfig name. 875 876 Returns: 877 Return True on success or False on failure 878 """ 879 if self.state != STATE_IDLE: 880 return False 881 882 self.defconfig = defconfig 883 self.log = '' 884 self.current_src_dir = self.reference_src_dir 885 self.do_defconfig() 886 return True 887 888 def poll(self): 889 """Check the status of the subprocess and handle it as needed. 890 891 Returns True if the slot is vacant (i.e. in idle state). 892 If the configuration is successfully finished, assign a new 893 subprocess to build include/autoconf.mk. 894 If include/autoconf.mk is generated, invoke the parser to 895 parse the .config and the include/autoconf.mk, moving 896 config options to the .config as needed. 897 If the .config was updated, run "make savedefconfig" to sync 898 it, update the original defconfig, and then set the slot back 899 to the idle state. 900 901 Returns: 902 Return True if the subprocess is terminated, False otherwise 903 """ 904 if self.state == STATE_IDLE: 905 return True 906 907 if self.ps.poll() == None: 908 return False 909 910 if self.ps.poll() != 0: 911 self.handle_error() 912 elif self.state == STATE_DEFCONFIG: 913 if self.reference_src_dir and not self.current_src_dir: 914 self.do_savedefconfig() 915 else: 916 self.do_autoconf() 917 elif self.state == STATE_AUTOCONF: 918 if self.current_src_dir: 919 self.current_src_dir = None 920 self.do_defconfig() 921 else: 922 self.do_savedefconfig() 923 elif self.state == STATE_SAVEDEFCONFIG: 924 self.update_defconfig() 925 else: 926 sys.exit("Internal Error. This should not happen.") 927 928 return True if self.state == STATE_IDLE else False 929 930 def handle_error(self): 931 """Handle error cases.""" 932 933 self.log += color_text(self.options.color, COLOR_LIGHT_RED, 934 "Failed to process.\n") 935 if self.options.verbose: 936 self.log += color_text(self.options.color, COLOR_LIGHT_CYAN, 937 self.ps.stderr.read()) 938 self.finish(False) 939 940 def do_defconfig(self): 941 """Run 'make <board>_defconfig' to create the .config file.""" 942 943 cmd = list(self.make_cmd) 944 cmd.append(self.defconfig) 945 self.ps = subprocess.Popen(cmd, stdout=self.devnull, 946 stderr=subprocess.PIPE, 947 cwd=self.current_src_dir) 948 self.state = STATE_DEFCONFIG 949 950 def do_autoconf(self): 951 """Run 'make include/config/auto.conf'.""" 952 953 self.cross_compile = self.parser.get_cross_compile() 954 if self.cross_compile is None: 955 self.log += color_text(self.options.color, COLOR_YELLOW, 956 "Compiler is missing. Do nothing.\n") 957 self.finish(False) 958 return 959 960 cmd = list(self.make_cmd) 961 if self.cross_compile: 962 cmd.append('CROSS_COMPILE=%s' % self.cross_compile) 963 cmd.append('KCONFIG_IGNORE_DUPLICATES=1') 964 cmd.append('include/config/auto.conf') 965 self.ps = subprocess.Popen(cmd, stdout=self.devnull, 966 stderr=subprocess.PIPE, 967 cwd=self.current_src_dir) 968 self.state = STATE_AUTOCONF 969 970 def do_savedefconfig(self): 971 """Update the .config and run 'make savedefconfig'.""" 972 973 (updated, suspicious, log) = self.parser.update_dotconfig() 974 if suspicious: 975 self.suspicious_boards.add(self.defconfig) 976 self.log += log 977 978 if not self.options.force_sync and not updated: 979 self.finish(True) 980 return 981 if updated: 982 self.log += color_text(self.options.color, COLOR_LIGHT_GREEN, 983 "Syncing by savedefconfig...\n") 984 else: 985 self.log += "Syncing by savedefconfig (forced by option)...\n" 986 987 cmd = list(self.make_cmd) 988 cmd.append('savedefconfig') 989 self.ps = subprocess.Popen(cmd, stdout=self.devnull, 990 stderr=subprocess.PIPE) 991 self.state = STATE_SAVEDEFCONFIG 992 993 def update_defconfig(self): 994 """Update the input defconfig and go back to the idle state.""" 995 996 log = self.parser.check_defconfig() 997 if log: 998 self.suspicious_boards.add(self.defconfig) 999 self.log += log 1000 orig_defconfig = os.path.join('configs', self.defconfig) 1001 new_defconfig = os.path.join(self.build_dir, 'defconfig') 1002 updated = not filecmp.cmp(orig_defconfig, new_defconfig) 1003 1004 if updated: 1005 self.log += color_text(self.options.color, COLOR_LIGHT_BLUE, 1006 "defconfig was updated.\n") 1007 1008 if not self.options.dry_run and updated: 1009 shutil.move(new_defconfig, orig_defconfig) 1010 self.finish(True) 1011 1012 def finish(self, success): 1013 """Display log along with progress and go to the idle state. 1014 1015 Arguments: 1016 success: Should be True when the defconfig was processed 1017 successfully, or False when it fails. 1018 """ 1019 # output at least 30 characters to hide the "* defconfigs out of *". 1020 log = self.defconfig.ljust(30) + '\n' 1021 1022 log += '\n'.join([ ' ' + s for s in self.log.split('\n') ]) 1023 # Some threads are running in parallel. 1024 # Print log atomically to not mix up logs from different threads. 1025 print >> (sys.stdout if success else sys.stderr), log 1026 1027 if not success: 1028 if self.options.exit_on_error: 1029 sys.exit("Exit on error.") 1030 # If --exit-on-error flag is not set, skip this board and continue. 1031 # Record the failed board. 1032 self.failed_boards.add(self.defconfig) 1033 1034 self.progress.inc() 1035 self.progress.show() 1036 self.state = STATE_IDLE 1037 1038 def get_failed_boards(self): 1039 """Returns a set of failed boards (defconfigs) in this slot. 1040 """ 1041 return self.failed_boards 1042 1043 def get_suspicious_boards(self): 1044 """Returns a set of boards (defconfigs) with possible misconversion. 1045 """ 1046 return self.suspicious_boards - self.failed_boards 1047 1048class Slots: 1049 1050 """Controller of the array of subprocess slots.""" 1051 1052 def __init__(self, configs, options, progress, reference_src_dir): 1053 """Create a new slots controller. 1054 1055 Arguments: 1056 configs: A list of CONFIGs to move. 1057 options: option flags. 1058 progress: A progress indicator. 1059 reference_src_dir: Determine the true starting config state from this 1060 source tree. 1061 """ 1062 self.options = options 1063 self.slots = [] 1064 devnull = get_devnull() 1065 make_cmd = get_make_cmd() 1066 for i in range(options.jobs): 1067 self.slots.append(Slot(configs, options, progress, devnull, 1068 make_cmd, reference_src_dir)) 1069 1070 def add(self, defconfig): 1071 """Add a new subprocess if a vacant slot is found. 1072 1073 Arguments: 1074 defconfig: defconfig name to be put into. 1075 1076 Returns: 1077 Return True on success or False on failure 1078 """ 1079 for slot in self.slots: 1080 if slot.add(defconfig): 1081 return True 1082 return False 1083 1084 def available(self): 1085 """Check if there is a vacant slot. 1086 1087 Returns: 1088 Return True if at lease one vacant slot is found, False otherwise. 1089 """ 1090 for slot in self.slots: 1091 if slot.poll(): 1092 return True 1093 return False 1094 1095 def empty(self): 1096 """Check if all slots are vacant. 1097 1098 Returns: 1099 Return True if all the slots are vacant, False otherwise. 1100 """ 1101 ret = True 1102 for slot in self.slots: 1103 if not slot.poll(): 1104 ret = False 1105 return ret 1106 1107 def show_failed_boards(self): 1108 """Display all of the failed boards (defconfigs).""" 1109 boards = set() 1110 output_file = 'moveconfig.failed' 1111 1112 for slot in self.slots: 1113 boards |= slot.get_failed_boards() 1114 1115 if boards: 1116 boards = '\n'.join(boards) + '\n' 1117 msg = "The following boards were not processed due to error:\n" 1118 msg += boards 1119 msg += "(the list has been saved in %s)\n" % output_file 1120 print >> sys.stderr, color_text(self.options.color, COLOR_LIGHT_RED, 1121 msg) 1122 1123 with open(output_file, 'w') as f: 1124 f.write(boards) 1125 1126 def show_suspicious_boards(self): 1127 """Display all boards (defconfigs) with possible misconversion.""" 1128 boards = set() 1129 output_file = 'moveconfig.suspicious' 1130 1131 for slot in self.slots: 1132 boards |= slot.get_suspicious_boards() 1133 1134 if boards: 1135 boards = '\n'.join(boards) + '\n' 1136 msg = "The following boards might have been converted incorrectly.\n" 1137 msg += "It is highly recommended to check them manually:\n" 1138 msg += boards 1139 msg += "(the list has been saved in %s)\n" % output_file 1140 print >> sys.stderr, color_text(self.options.color, COLOR_YELLOW, 1141 msg) 1142 1143 with open(output_file, 'w') as f: 1144 f.write(boards) 1145 1146class ReferenceSource: 1147 1148 """Reference source against which original configs should be parsed.""" 1149 1150 def __init__(self, commit): 1151 """Create a reference source directory based on a specified commit. 1152 1153 Arguments: 1154 commit: commit to git-clone 1155 """ 1156 self.src_dir = tempfile.mkdtemp() 1157 print "Cloning git repo to a separate work directory..." 1158 subprocess.check_output(['git', 'clone', os.getcwd(), '.'], 1159 cwd=self.src_dir) 1160 print "Checkout '%s' to build the original autoconf.mk." % \ 1161 subprocess.check_output(['git', 'rev-parse', '--short', commit]).strip() 1162 subprocess.check_output(['git', 'checkout', commit], 1163 stderr=subprocess.STDOUT, cwd=self.src_dir) 1164 1165 def __del__(self): 1166 """Delete the reference source directory 1167 1168 This function makes sure the temporary directory is cleaned away 1169 even if Python suddenly dies due to error. It should be done in here 1170 because it is guaranteed the destructor is always invoked when the 1171 instance of the class gets unreferenced. 1172 """ 1173 shutil.rmtree(self.src_dir) 1174 1175 def get_dir(self): 1176 """Return the absolute path to the reference source directory.""" 1177 1178 return self.src_dir 1179 1180def move_config(configs, options): 1181 """Move config options to defconfig files. 1182 1183 Arguments: 1184 configs: A list of CONFIGs to move. 1185 options: option flags 1186 """ 1187 if len(configs) == 0: 1188 if options.force_sync: 1189 print 'No CONFIG is specified. You are probably syncing defconfigs.', 1190 else: 1191 print 'Neither CONFIG nor --force-sync is specified. Nothing will happen.', 1192 else: 1193 print 'Move ' + ', '.join(configs), 1194 print '(jobs: %d)\n' % options.jobs 1195 1196 if options.git_ref: 1197 reference_src = ReferenceSource(options.git_ref) 1198 reference_src_dir = reference_src.get_dir() 1199 else: 1200 reference_src_dir = None 1201 1202 if options.defconfigs: 1203 defconfigs = [line.strip() for line in open(options.defconfigs)] 1204 for i, defconfig in enumerate(defconfigs): 1205 if not defconfig.endswith('_defconfig'): 1206 defconfigs[i] = defconfig + '_defconfig' 1207 if not os.path.exists(os.path.join('configs', defconfigs[i])): 1208 sys.exit('%s - defconfig does not exist. Stopping.' % 1209 defconfigs[i]) 1210 else: 1211 defconfigs = get_all_defconfigs() 1212 1213 progress = Progress(len(defconfigs)) 1214 slots = Slots(configs, options, progress, reference_src_dir) 1215 1216 # Main loop to process defconfig files: 1217 # Add a new subprocess into a vacant slot. 1218 # Sleep if there is no available slot. 1219 for defconfig in defconfigs: 1220 while not slots.add(defconfig): 1221 while not slots.available(): 1222 # No available slot: sleep for a while 1223 time.sleep(SLEEP_TIME) 1224 1225 # wait until all the subprocesses finish 1226 while not slots.empty(): 1227 time.sleep(SLEEP_TIME) 1228 1229 print '' 1230 slots.show_failed_boards() 1231 slots.show_suspicious_boards() 1232 1233def main(): 1234 try: 1235 cpu_count = multiprocessing.cpu_count() 1236 except NotImplementedError: 1237 cpu_count = 1 1238 1239 parser = optparse.OptionParser() 1240 # Add options here 1241 parser.add_option('-c', '--color', action='store_true', default=False, 1242 help='display the log in color') 1243 parser.add_option('-d', '--defconfigs', type='string', 1244 help='a file containing a list of defconfigs to move') 1245 parser.add_option('-n', '--dry-run', action='store_true', default=False, 1246 help='perform a trial run (show log with no changes)') 1247 parser.add_option('-e', '--exit-on-error', action='store_true', 1248 default=False, 1249 help='exit immediately on any error') 1250 parser.add_option('-s', '--force-sync', action='store_true', default=False, 1251 help='force sync by savedefconfig') 1252 parser.add_option('-S', '--spl', action='store_true', default=False, 1253 help='parse config options defined for SPL build') 1254 parser.add_option('-H', '--headers-only', dest='cleanup_headers_only', 1255 action='store_true', default=False, 1256 help='only cleanup the headers') 1257 parser.add_option('-j', '--jobs', type='int', default=cpu_count, 1258 help='the number of jobs to run simultaneously') 1259 parser.add_option('-r', '--git-ref', type='string', 1260 help='the git ref to clone for building the autoconf.mk') 1261 parser.add_option('-y', '--yes', action='store_true', default=False, 1262 help="respond 'yes' to any prompts") 1263 parser.add_option('-v', '--verbose', action='store_true', default=False, 1264 help='show any build errors as boards are built') 1265 parser.usage += ' CONFIG ...' 1266 1267 (options, configs) = parser.parse_args() 1268 1269 if len(configs) == 0 and not options.force_sync: 1270 parser.print_usage() 1271 sys.exit(1) 1272 1273 # prefix the option name with CONFIG_ if missing 1274 configs = [ config if config.startswith('CONFIG_') else 'CONFIG_' + config 1275 for config in configs ] 1276 1277 check_top_directory() 1278 1279 if not options.cleanup_headers_only: 1280 check_clean_directory() 1281 update_cross_compile(options.color) 1282 move_config(configs, options) 1283 1284 if configs: 1285 cleanup_headers(configs, options) 1286 cleanup_extra_options(configs, options) 1287 1288if __name__ == '__main__': 1289 main() 1290