1#!/usr/bin/env python2 2# 3# Author: Masahiro Yamada <yamada.masahiro@socionext.com> 4# 5# SPDX-License-Identifier: GPL-2.0+ 6# 7 8""" 9Move config options from headers to defconfig files. 10 11Since Kconfig was introduced to U-Boot, we have worked on moving 12config options from headers to Kconfig (defconfig). 13 14This tool intends to help this tremendous work. 15 16 17Usage 18----- 19 20First, you must edit the Kconfig to add the menu entries for the configs 21you are moving. 22 23And then run this tool giving CONFIG names you want to move. 24For example, if you want to move CONFIG_CMD_USB and CONFIG_SYS_TEXT_BASE, 25simply type as follows: 26 27 $ tools/moveconfig.py CONFIG_CMD_USB CONFIG_SYS_TEXT_BASE 28 29The tool walks through all the defconfig files and move the given CONFIGs. 30 31The log is also displayed on the terminal. 32 33The log is printed for each defconfig as follows: 34 35<defconfig_name> 36 <action1> 37 <action2> 38 <action3> 39 ... 40 41<defconfig_name> is the name of the defconfig. 42 43<action*> shows what the tool did for that defconfig. 44It looks like one of the following: 45 46 - Move 'CONFIG_... ' 47 This config option was moved to the defconfig 48 49 - CONFIG_... is not defined in Kconfig. Do nothing. 50 The entry for this CONFIG was not found in Kconfig. The option is not 51 defined in the config header, either. So, this case can be just skipped. 52 53 - CONFIG_... is not defined in Kconfig (suspicious). Do nothing. 54 This option is defined in the config header, but its entry was not found 55 in Kconfig. 56 There are two common cases: 57 - You forgot to create an entry for the CONFIG before running 58 this tool, or made a typo in a CONFIG passed to this tool. 59 - The entry was hidden due to unmet 'depends on'. 60 The tool does not know if the result is reasonable, so please check it 61 manually. 62 63 - 'CONFIG_...' is the same as the define in Kconfig. Do nothing. 64 The define in the config header matched the one in Kconfig. 65 We do not need to touch it. 66 67 - Compiler is missing. Do nothing. 68 The compiler specified for this architecture was not found 69 in your PATH environment. 70 (If -e option is passed, the tool exits immediately.) 71 72 - Failed to process. 73 An error occurred during processing this defconfig. Skipped. 74 (If -e option is passed, the tool exits immediately on error.) 75 76Finally, you will be asked, Clean up headers? [y/n]: 77 78If you say 'y' here, the unnecessary config defines are removed 79from the config headers (include/configs/*.h). 80It just uses the regex method, so you should not rely on it. 81Just in case, please do 'git diff' to see what happened. 82 83 84How does it work? 85----------------- 86 87This tool runs configuration and builds include/autoconf.mk for every 88defconfig. The config options defined in Kconfig appear in the .config 89file (unless they are hidden because of unmet dependency.) 90On the other hand, the config options defined by board headers are seen 91in include/autoconf.mk. The tool looks for the specified options in both 92of them to decide the appropriate action for the options. If the given 93config option is found in the .config, but its value does not match the 94one from the board header, the config option in the .config is replaced 95with the define in the board header. Then, the .config is synced by 96"make savedefconfig" and the defconfig is updated with it. 97 98For faster processing, this tool handles multi-threading. It creates 99separate build directories where the out-of-tree build is run. The 100temporary build directories are automatically created and deleted as 101needed. The number of threads are chosen based on the number of the CPU 102cores of your system although you can change it via -j (--jobs) option. 103 104 105Toolchains 106---------- 107 108Appropriate toolchain are necessary to generate include/autoconf.mk 109for all the architectures supported by U-Boot. Most of them are available 110at the kernel.org site, some are not provided by kernel.org. 111 112The default per-arch CROSS_COMPILE used by this tool is specified by 113the list below, CROSS_COMPILE. You may wish to update the list to 114use your own. Instead of modifying the list directly, you can give 115them via environments. 116 117 118Available options 119----------------- 120 121 -c, --color 122 Surround each portion of the log with escape sequences to display it 123 in color on the terminal. 124 125 -C, --commit 126 Create a git commit with the changes when the operation is complete. A 127 standard commit message is used which may need to be edited. 128 129 -d, --defconfigs 130 Specify a file containing a list of defconfigs to move. The defconfig 131 files can be given with shell-style wildcards. 132 133 -n, --dry-run 134 Perform a trial run that does not make any changes. It is useful to 135 see what is going to happen before one actually runs it. 136 137 -e, --exit-on-error 138 Exit immediately if Make exits with a non-zero status while processing 139 a defconfig file. 140 141 -s, --force-sync 142 Do "make savedefconfig" forcibly for all the defconfig files. 143 If not specified, "make savedefconfig" only occurs for cases 144 where at least one CONFIG was moved. 145 146 -S, --spl 147 Look for moved config options in spl/include/autoconf.mk instead of 148 include/autoconf.mk. This is useful for moving options for SPL build 149 because SPL related options (mostly prefixed with CONFIG_SPL_) are 150 sometimes blocked by CONFIG_SPL_BUILD ifdef conditionals. 151 152 -H, --headers-only 153 Only cleanup the headers; skip the defconfig processing 154 155 -j, --jobs 156 Specify the number of threads to run simultaneously. If not specified, 157 the number of threads is the same as the number of CPU cores. 158 159 -r, --git-ref 160 Specify the git ref to clone for building the autoconf.mk. If unspecified 161 use the CWD. This is useful for when changes to the Kconfig affect the 162 default values and you want to capture the state of the defconfig from 163 before that change was in effect. If in doubt, specify a ref pre-Kconfig 164 changes (use HEAD if Kconfig changes are not committed). Worst case it will 165 take a bit longer to run, but will always do the right thing. 166 167 -v, --verbose 168 Show any build errors as boards are built 169 170 -y, --yes 171 Instead of prompting, automatically go ahead with all operations. This 172 includes cleaning up headers and CONFIG_SYS_EXTRA_OPTIONS. 173 174To see the complete list of supported options, run 175 176 $ tools/moveconfig.py -h 177 178""" 179 180import copy 181import difflib 182import filecmp 183import fnmatch 184import glob 185import multiprocessing 186import optparse 187import os 188import re 189import shutil 190import subprocess 191import sys 192import tempfile 193import time 194 195SHOW_GNU_MAKE = 'scripts/show-gnu-make' 196SLEEP_TIME=0.03 197 198# Here is the list of cross-tools I use. 199# Most of them are available at kernel.org 200# (https://www.kernel.org/pub/tools/crosstool/files/bin/), except the following: 201# arc: https://github.com/foss-for-synopsys-dwc-arc-processors/toolchain/releases 202# nds32: http://osdk.andestech.com/packages/nds32le-linux-glibc-v1.tgz 203# nios2: https://sourcery.mentor.com/GNUToolchain/subscription42545 204# sh: http://sourcery.mentor.com/public/gnu_toolchain/sh-linux-gnu 205CROSS_COMPILE = { 206 'arc': 'arc-linux-', 207 'aarch64': 'aarch64-linux-', 208 'arm': 'arm-unknown-linux-gnueabi-', 209 'm68k': 'm68k-linux-', 210 'microblaze': 'microblaze-linux-', 211 'mips': 'mips-linux-', 212 'nds32': 'nds32le-linux-', 213 'nios2': 'nios2-linux-gnu-', 214 'powerpc': 'powerpc-linux-', 215 'sh': 'sh-linux-gnu-', 216 'x86': 'i386-linux-', 217 'xtensa': 'xtensa-linux-' 218} 219 220STATE_IDLE = 0 221STATE_DEFCONFIG = 1 222STATE_AUTOCONF = 2 223STATE_SAVEDEFCONFIG = 3 224 225ACTION_MOVE = 0 226ACTION_NO_ENTRY = 1 227ACTION_NO_ENTRY_WARN = 2 228ACTION_NO_CHANGE = 3 229 230COLOR_BLACK = '0;30' 231COLOR_RED = '0;31' 232COLOR_GREEN = '0;32' 233COLOR_BROWN = '0;33' 234COLOR_BLUE = '0;34' 235COLOR_PURPLE = '0;35' 236COLOR_CYAN = '0;36' 237COLOR_LIGHT_GRAY = '0;37' 238COLOR_DARK_GRAY = '1;30' 239COLOR_LIGHT_RED = '1;31' 240COLOR_LIGHT_GREEN = '1;32' 241COLOR_YELLOW = '1;33' 242COLOR_LIGHT_BLUE = '1;34' 243COLOR_LIGHT_PURPLE = '1;35' 244COLOR_LIGHT_CYAN = '1;36' 245COLOR_WHITE = '1;37' 246 247### helper functions ### 248def get_devnull(): 249 """Get the file object of '/dev/null' device.""" 250 try: 251 devnull = subprocess.DEVNULL # py3k 252 except AttributeError: 253 devnull = open(os.devnull, 'wb') 254 return devnull 255 256def check_top_directory(): 257 """Exit if we are not at the top of source directory.""" 258 for f in ('README', 'Licenses'): 259 if not os.path.exists(f): 260 sys.exit('Please run at the top of source directory.') 261 262def check_clean_directory(): 263 """Exit if the source tree is not clean.""" 264 for f in ('.config', 'include/config'): 265 if os.path.exists(f): 266 sys.exit("source tree is not clean, please run 'make mrproper'") 267 268def get_make_cmd(): 269 """Get the command name of GNU Make. 270 271 U-Boot needs GNU Make for building, but the command name is not 272 necessarily "make". (for example, "gmake" on FreeBSD). 273 Returns the most appropriate command name on your system. 274 """ 275 process = subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE) 276 ret = process.communicate() 277 if process.returncode: 278 sys.exit('GNU Make not found') 279 return ret[0].rstrip() 280 281def get_matched_defconfig(line): 282 """Get the defconfig files that match a pattern 283 284 Args: 285 line: Path or filename to match, e.g. 'configs/snow_defconfig' or 286 'k2*_defconfig'. If no directory is provided, 'configs/' is 287 prepended 288 289 Returns: 290 a list of matching defconfig files 291 """ 292 dirname = os.path.dirname(line) 293 if dirname: 294 pattern = line 295 else: 296 pattern = os.path.join('configs', line) 297 return glob.glob(pattern) + glob.glob(pattern + '_defconfig') 298 299def get_matched_defconfigs(defconfigs_file): 300 """Get all the defconfig files that match the patterns in a file. 301 302 Args: 303 defconfigs_file: File containing a list of defconfigs to process, or 304 '-' to read the list from stdin 305 306 Returns: 307 A list of paths to defconfig files, with no duplicates 308 """ 309 defconfigs = [] 310 if defconfigs_file == '-': 311 fd = sys.stdin 312 defconfigs_file = 'stdin' 313 else: 314 fd = open(defconfigs_file) 315 for i, line in enumerate(fd): 316 line = line.strip() 317 if not line: 318 continue # skip blank lines silently 319 matched = get_matched_defconfig(line) 320 if not matched: 321 print >> sys.stderr, "warning: %s:%d: no defconfig matched '%s'" % \ 322 (defconfigs_file, i + 1, line) 323 324 defconfigs += matched 325 326 # use set() to drop multiple matching 327 return [ defconfig[len('configs') + 1:] for defconfig in set(defconfigs) ] 328 329def get_all_defconfigs(): 330 """Get all the defconfig files under the configs/ directory.""" 331 defconfigs = [] 332 for (dirpath, dirnames, filenames) in os.walk('configs'): 333 dirpath = dirpath[len('configs') + 1:] 334 for filename in fnmatch.filter(filenames, '*_defconfig'): 335 defconfigs.append(os.path.join(dirpath, filename)) 336 337 return defconfigs 338 339def color_text(color_enabled, color, string): 340 """Return colored string.""" 341 if color_enabled: 342 # LF should not be surrounded by the escape sequence. 343 # Otherwise, additional whitespace or line-feed might be printed. 344 return '\n'.join([ '\033[' + color + 'm' + s + '\033[0m' if s else '' 345 for s in string.split('\n') ]) 346 else: 347 return string 348 349def show_diff(a, b, file_path, color_enabled): 350 """Show unidified diff. 351 352 Arguments: 353 a: A list of lines (before) 354 b: A list of lines (after) 355 file_path: Path to the file 356 color_enabled: Display the diff in color 357 """ 358 359 diff = difflib.unified_diff(a, b, 360 fromfile=os.path.join('a', file_path), 361 tofile=os.path.join('b', file_path)) 362 363 for line in diff: 364 if line[0] == '-' and line[1] != '-': 365 print color_text(color_enabled, COLOR_RED, line), 366 elif line[0] == '+' and line[1] != '+': 367 print color_text(color_enabled, COLOR_GREEN, line), 368 else: 369 print line, 370 371def update_cross_compile(color_enabled): 372 """Update per-arch CROSS_COMPILE via environment variables 373 374 The default CROSS_COMPILE values are available 375 in the CROSS_COMPILE list above. 376 377 You can override them via environment variables 378 CROSS_COMPILE_{ARCH}. 379 380 For example, if you want to override toolchain prefixes 381 for ARM and PowerPC, you can do as follows in your shell: 382 383 export CROSS_COMPILE_ARM=... 384 export CROSS_COMPILE_POWERPC=... 385 386 Then, this function checks if specified compilers really exist in your 387 PATH environment. 388 """ 389 archs = [] 390 391 for arch in os.listdir('arch'): 392 if os.path.exists(os.path.join('arch', arch, 'Makefile')): 393 archs.append(arch) 394 395 # arm64 is a special case 396 archs.append('aarch64') 397 398 for arch in archs: 399 env = 'CROSS_COMPILE_' + arch.upper() 400 cross_compile = os.environ.get(env) 401 if not cross_compile: 402 cross_compile = CROSS_COMPILE.get(arch, '') 403 404 for path in os.environ["PATH"].split(os.pathsep): 405 gcc_path = os.path.join(path, cross_compile + 'gcc') 406 if os.path.isfile(gcc_path) and os.access(gcc_path, os.X_OK): 407 break 408 else: 409 print >> sys.stderr, color_text(color_enabled, COLOR_YELLOW, 410 'warning: %sgcc: not found in PATH. %s architecture boards will be skipped' 411 % (cross_compile, arch)) 412 cross_compile = None 413 414 CROSS_COMPILE[arch] = cross_compile 415 416def extend_matched_lines(lines, matched, pre_patterns, post_patterns, extend_pre, 417 extend_post): 418 """Extend matched lines if desired patterns are found before/after already 419 matched lines. 420 421 Arguments: 422 lines: A list of lines handled. 423 matched: A list of line numbers that have been already matched. 424 (will be updated by this function) 425 pre_patterns: A list of regular expression that should be matched as 426 preamble. 427 post_patterns: A list of regular expression that should be matched as 428 postamble. 429 extend_pre: Add the line number of matched preamble to the matched list. 430 extend_post: Add the line number of matched postamble to the matched list. 431 """ 432 extended_matched = [] 433 434 j = matched[0] 435 436 for i in matched: 437 if i == 0 or i < j: 438 continue 439 j = i 440 while j in matched: 441 j += 1 442 if j >= len(lines): 443 break 444 445 for p in pre_patterns: 446 if p.search(lines[i - 1]): 447 break 448 else: 449 # not matched 450 continue 451 452 for p in post_patterns: 453 if p.search(lines[j]): 454 break 455 else: 456 # not matched 457 continue 458 459 if extend_pre: 460 extended_matched.append(i - 1) 461 if extend_post: 462 extended_matched.append(j) 463 464 matched += extended_matched 465 matched.sort() 466 467def confirm(options, prompt): 468 if not options.yes: 469 while True: 470 choice = raw_input('{} [y/n]: '.format(prompt)) 471 choice = choice.lower() 472 print choice 473 if choice == 'y' or choice == 'n': 474 break 475 476 if choice == 'n': 477 return False 478 479 return True 480 481def cleanup_one_header(header_path, patterns, options): 482 """Clean regex-matched lines away from a file. 483 484 Arguments: 485 header_path: path to the cleaned file. 486 patterns: list of regex patterns. Any lines matching to these 487 patterns are deleted. 488 options: option flags. 489 """ 490 with open(header_path) as f: 491 lines = f.readlines() 492 493 matched = [] 494 for i, line in enumerate(lines): 495 if i - 1 in matched and lines[i - 1][-2:] == '\\\n': 496 matched.append(i) 497 continue 498 for pattern in patterns: 499 if pattern.search(line): 500 matched.append(i) 501 break 502 503 if not matched: 504 return 505 506 # remove empty #ifdef ... #endif, successive blank lines 507 pattern_if = re.compile(r'#\s*if(def|ndef)?\W') # #if, #ifdef, #ifndef 508 pattern_elif = re.compile(r'#\s*el(if|se)\W') # #elif, #else 509 pattern_endif = re.compile(r'#\s*endif\W') # #endif 510 pattern_blank = re.compile(r'^\s*$') # empty line 511 512 while True: 513 old_matched = copy.copy(matched) 514 extend_matched_lines(lines, matched, [pattern_if], 515 [pattern_endif], True, True) 516 extend_matched_lines(lines, matched, [pattern_elif], 517 [pattern_elif, pattern_endif], True, False) 518 extend_matched_lines(lines, matched, [pattern_if, pattern_elif], 519 [pattern_blank], False, True) 520 extend_matched_lines(lines, matched, [pattern_blank], 521 [pattern_elif, pattern_endif], True, False) 522 extend_matched_lines(lines, matched, [pattern_blank], 523 [pattern_blank], True, False) 524 if matched == old_matched: 525 break 526 527 tolines = copy.copy(lines) 528 529 for i in reversed(matched): 530 tolines.pop(i) 531 532 show_diff(lines, tolines, header_path, options.color) 533 534 if options.dry_run: 535 return 536 537 with open(header_path, 'w') as f: 538 for line in tolines: 539 f.write(line) 540 541def cleanup_headers(configs, options): 542 """Delete config defines from board headers. 543 544 Arguments: 545 configs: A list of CONFIGs to remove. 546 options: option flags. 547 """ 548 if not confirm(options, 'Clean up headers?'): 549 return 550 551 patterns = [] 552 for config in configs: 553 patterns.append(re.compile(r'#\s*define\s+%s\W' % config)) 554 patterns.append(re.compile(r'#\s*undef\s+%s\W' % config)) 555 556 for dir in 'include', 'arch', 'board': 557 for (dirpath, dirnames, filenames) in os.walk(dir): 558 if dirpath == os.path.join('include', 'generated'): 559 continue 560 for filename in filenames: 561 if not fnmatch.fnmatch(filename, '*~'): 562 cleanup_one_header(os.path.join(dirpath, filename), 563 patterns, options) 564 565def cleanup_one_extra_option(defconfig_path, configs, options): 566 """Delete config defines in CONFIG_SYS_EXTRA_OPTIONS in one defconfig file. 567 568 Arguments: 569 defconfig_path: path to the cleaned defconfig file. 570 configs: A list of CONFIGs to remove. 571 options: option flags. 572 """ 573 574 start = 'CONFIG_SYS_EXTRA_OPTIONS="' 575 end = '"\n' 576 577 with open(defconfig_path) as f: 578 lines = f.readlines() 579 580 for i, line in enumerate(lines): 581 if line.startswith(start) and line.endswith(end): 582 break 583 else: 584 # CONFIG_SYS_EXTRA_OPTIONS was not found in this defconfig 585 return 586 587 old_tokens = line[len(start):-len(end)].split(',') 588 new_tokens = [] 589 590 for token in old_tokens: 591 pos = token.find('=') 592 if not (token[:pos] if pos >= 0 else token) in configs: 593 new_tokens.append(token) 594 595 if new_tokens == old_tokens: 596 return 597 598 tolines = copy.copy(lines) 599 600 if new_tokens: 601 tolines[i] = start + ','.join(new_tokens) + end 602 else: 603 tolines.pop(i) 604 605 show_diff(lines, tolines, defconfig_path, options.color) 606 607 if options.dry_run: 608 return 609 610 with open(defconfig_path, 'w') as f: 611 for line in tolines: 612 f.write(line) 613 614def cleanup_extra_options(configs, options): 615 """Delete config defines in CONFIG_SYS_EXTRA_OPTIONS in defconfig files. 616 617 Arguments: 618 configs: A list of CONFIGs to remove. 619 options: option flags. 620 """ 621 if not confirm(options, 'Clean up CONFIG_SYS_EXTRA_OPTIONS?'): 622 return 623 624 configs = [ config[len('CONFIG_'):] for config in configs ] 625 626 defconfigs = get_all_defconfigs() 627 628 for defconfig in defconfigs: 629 cleanup_one_extra_option(os.path.join('configs', defconfig), configs, 630 options) 631 632def cleanup_whitelist(configs, options): 633 """Delete config whitelist entries 634 635 Arguments: 636 configs: A list of CONFIGs to remove. 637 options: option flags. 638 """ 639 if not confirm(options, 'Clean up whitelist entries?'): 640 return 641 642 with open(os.path.join('scripts', 'config_whitelist.txt')) as f: 643 lines = f.readlines() 644 645 lines = [x for x in lines if x.strip() not in configs] 646 647 with open(os.path.join('scripts', 'config_whitelist.txt'), 'w') as f: 648 f.write(''.join(lines)) 649 650def find_matching(patterns, line): 651 for pat in patterns: 652 if pat.search(line): 653 return True 654 return False 655 656def cleanup_readme(configs, options): 657 """Delete config description in README 658 659 Arguments: 660 configs: A list of CONFIGs to remove. 661 options: option flags. 662 """ 663 if not confirm(options, 'Clean up README?'): 664 return 665 666 patterns = [] 667 for config in configs: 668 patterns.append(re.compile(r'^\s+%s' % config)) 669 670 with open('README') as f: 671 lines = f.readlines() 672 673 found = False 674 newlines = [] 675 for line in lines: 676 if not found: 677 found = find_matching(patterns, line) 678 if found: 679 continue 680 681 if found and re.search(r'^\s+CONFIG', line): 682 found = False 683 684 if not found: 685 newlines.append(line) 686 687 with open('README', 'w') as f: 688 f.write(''.join(newlines)) 689 690 691### classes ### 692class Progress: 693 694 """Progress Indicator""" 695 696 def __init__(self, total): 697 """Create a new progress indicator. 698 699 Arguments: 700 total: A number of defconfig files to process. 701 """ 702 self.current = 0 703 self.total = total 704 705 def inc(self): 706 """Increment the number of processed defconfig files.""" 707 708 self.current += 1 709 710 def show(self): 711 """Display the progress.""" 712 print ' %d defconfigs out of %d\r' % (self.current, self.total), 713 sys.stdout.flush() 714 715class KconfigParser: 716 717 """A parser of .config and include/autoconf.mk.""" 718 719 re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"') 720 re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"') 721 722 def __init__(self, configs, options, build_dir): 723 """Create a new parser. 724 725 Arguments: 726 configs: A list of CONFIGs to move. 727 options: option flags. 728 build_dir: Build directory. 729 """ 730 self.configs = configs 731 self.options = options 732 self.dotconfig = os.path.join(build_dir, '.config') 733 self.autoconf = os.path.join(build_dir, 'include', 'autoconf.mk') 734 self.spl_autoconf = os.path.join(build_dir, 'spl', 'include', 735 'autoconf.mk') 736 self.config_autoconf = os.path.join(build_dir, 'include', 'config', 737 'auto.conf') 738 self.defconfig = os.path.join(build_dir, 'defconfig') 739 740 def get_cross_compile(self): 741 """Parse .config file and return CROSS_COMPILE. 742 743 Returns: 744 A string storing the compiler prefix for the architecture. 745 Return a NULL string for architectures that do not require 746 compiler prefix (Sandbox and native build is the case). 747 Return None if the specified compiler is missing in your PATH. 748 Caller should distinguish '' and None. 749 """ 750 arch = '' 751 cpu = '' 752 for line in open(self.dotconfig): 753 m = self.re_arch.match(line) 754 if m: 755 arch = m.group(1) 756 continue 757 m = self.re_cpu.match(line) 758 if m: 759 cpu = m.group(1) 760 761 if not arch: 762 return None 763 764 # fix-up for aarch64 765 if arch == 'arm' and cpu == 'armv8': 766 arch = 'aarch64' 767 768 return CROSS_COMPILE.get(arch, None) 769 770 def parse_one_config(self, config, dotconfig_lines, autoconf_lines): 771 """Parse .config, defconfig, include/autoconf.mk for one config. 772 773 This function looks for the config options in the lines from 774 defconfig, .config, and include/autoconf.mk in order to decide 775 which action should be taken for this defconfig. 776 777 Arguments: 778 config: CONFIG name to parse. 779 dotconfig_lines: lines from the .config file. 780 autoconf_lines: lines from the include/autoconf.mk file. 781 782 Returns: 783 A tupple of the action for this defconfig and the line 784 matched for the config. 785 """ 786 not_set = '# %s is not set' % config 787 788 for line in autoconf_lines: 789 line = line.rstrip() 790 if line.startswith(config + '='): 791 new_val = line 792 break 793 else: 794 new_val = not_set 795 796 for line in dotconfig_lines: 797 line = line.rstrip() 798 if line.startswith(config + '=') or line == not_set: 799 old_val = line 800 break 801 else: 802 if new_val == not_set: 803 return (ACTION_NO_ENTRY, config) 804 else: 805 return (ACTION_NO_ENTRY_WARN, config) 806 807 # If this CONFIG is neither bool nor trisate 808 if old_val[-2:] != '=y' and old_val[-2:] != '=m' and old_val != not_set: 809 # tools/scripts/define2mk.sed changes '1' to 'y'. 810 # This is a problem if the CONFIG is int type. 811 # Check the type in Kconfig and handle it correctly. 812 if new_val[-2:] == '=y': 813 new_val = new_val[:-1] + '1' 814 815 return (ACTION_NO_CHANGE if old_val == new_val else ACTION_MOVE, 816 new_val) 817 818 def update_dotconfig(self): 819 """Parse files for the config options and update the .config. 820 821 This function parses the generated .config and include/autoconf.mk 822 searching the target options. 823 Move the config option(s) to the .config as needed. 824 825 Arguments: 826 defconfig: defconfig name. 827 828 Returns: 829 Return a tuple of (updated flag, log string). 830 The "updated flag" is True if the .config was updated, False 831 otherwise. The "log string" shows what happend to the .config. 832 """ 833 834 results = [] 835 updated = False 836 suspicious = False 837 rm_files = [self.config_autoconf, self.autoconf] 838 839 if self.options.spl: 840 if os.path.exists(self.spl_autoconf): 841 autoconf_path = self.spl_autoconf 842 rm_files.append(self.spl_autoconf) 843 else: 844 for f in rm_files: 845 os.remove(f) 846 return (updated, suspicious, 847 color_text(self.options.color, COLOR_BROWN, 848 "SPL is not enabled. Skipped.") + '\n') 849 else: 850 autoconf_path = self.autoconf 851 852 with open(self.dotconfig) as f: 853 dotconfig_lines = f.readlines() 854 855 with open(autoconf_path) as f: 856 autoconf_lines = f.readlines() 857 858 for config in self.configs: 859 result = self.parse_one_config(config, dotconfig_lines, 860 autoconf_lines) 861 results.append(result) 862 863 log = '' 864 865 for (action, value) in results: 866 if action == ACTION_MOVE: 867 actlog = "Move '%s'" % value 868 log_color = COLOR_LIGHT_GREEN 869 elif action == ACTION_NO_ENTRY: 870 actlog = "%s is not defined in Kconfig. Do nothing." % value 871 log_color = COLOR_LIGHT_BLUE 872 elif action == ACTION_NO_ENTRY_WARN: 873 actlog = "%s is not defined in Kconfig (suspicious). Do nothing." % value 874 log_color = COLOR_YELLOW 875 suspicious = True 876 elif action == ACTION_NO_CHANGE: 877 actlog = "'%s' is the same as the define in Kconfig. Do nothing." \ 878 % value 879 log_color = COLOR_LIGHT_PURPLE 880 elif action == ACTION_SPL_NOT_EXIST: 881 actlog = "SPL is not enabled for this defconfig. Skip." 882 log_color = COLOR_PURPLE 883 else: 884 sys.exit("Internal Error. This should not happen.") 885 886 log += color_text(self.options.color, log_color, actlog) + '\n' 887 888 with open(self.dotconfig, 'a') as f: 889 for (action, value) in results: 890 if action == ACTION_MOVE: 891 f.write(value + '\n') 892 updated = True 893 894 self.results = results 895 for f in rm_files: 896 os.remove(f) 897 898 return (updated, suspicious, log) 899 900 def check_defconfig(self): 901 """Check the defconfig after savedefconfig 902 903 Returns: 904 Return additional log if moved CONFIGs were removed again by 905 'make savedefconfig'. 906 """ 907 908 log = '' 909 910 with open(self.defconfig) as f: 911 defconfig_lines = f.readlines() 912 913 for (action, value) in self.results: 914 if action != ACTION_MOVE: 915 continue 916 if not value + '\n' in defconfig_lines: 917 log += color_text(self.options.color, COLOR_YELLOW, 918 "'%s' was removed by savedefconfig.\n" % 919 value) 920 921 return log 922 923class Slot: 924 925 """A slot to store a subprocess. 926 927 Each instance of this class handles one subprocess. 928 This class is useful to control multiple threads 929 for faster processing. 930 """ 931 932 def __init__(self, configs, options, progress, devnull, make_cmd, reference_src_dir): 933 """Create a new process slot. 934 935 Arguments: 936 configs: A list of CONFIGs to move. 937 options: option flags. 938 progress: A progress indicator. 939 devnull: A file object of '/dev/null'. 940 make_cmd: command name of GNU Make. 941 reference_src_dir: Determine the true starting config state from this 942 source tree. 943 """ 944 self.options = options 945 self.progress = progress 946 self.build_dir = tempfile.mkdtemp() 947 self.devnull = devnull 948 self.make_cmd = (make_cmd, 'O=' + self.build_dir) 949 self.reference_src_dir = reference_src_dir 950 self.parser = KconfigParser(configs, options, self.build_dir) 951 self.state = STATE_IDLE 952 self.failed_boards = set() 953 self.suspicious_boards = set() 954 955 def __del__(self): 956 """Delete the working directory 957 958 This function makes sure the temporary directory is cleaned away 959 even if Python suddenly dies due to error. It should be done in here 960 because it is guaranteed the destructor is always invoked when the 961 instance of the class gets unreferenced. 962 963 If the subprocess is still running, wait until it finishes. 964 """ 965 if self.state != STATE_IDLE: 966 while self.ps.poll() == None: 967 pass 968 shutil.rmtree(self.build_dir) 969 970 def add(self, defconfig): 971 """Assign a new subprocess for defconfig and add it to the slot. 972 973 If the slot is vacant, create a new subprocess for processing the 974 given defconfig and add it to the slot. Just returns False if 975 the slot is occupied (i.e. the current subprocess is still running). 976 977 Arguments: 978 defconfig: defconfig name. 979 980 Returns: 981 Return True on success or False on failure 982 """ 983 if self.state != STATE_IDLE: 984 return False 985 986 self.defconfig = defconfig 987 self.log = '' 988 self.current_src_dir = self.reference_src_dir 989 self.do_defconfig() 990 return True 991 992 def poll(self): 993 """Check the status of the subprocess and handle it as needed. 994 995 Returns True if the slot is vacant (i.e. in idle state). 996 If the configuration is successfully finished, assign a new 997 subprocess to build include/autoconf.mk. 998 If include/autoconf.mk is generated, invoke the parser to 999 parse the .config and the include/autoconf.mk, moving 1000 config options to the .config as needed. 1001 If the .config was updated, run "make savedefconfig" to sync 1002 it, update the original defconfig, and then set the slot back 1003 to the idle state. 1004 1005 Returns: 1006 Return True if the subprocess is terminated, False otherwise 1007 """ 1008 if self.state == STATE_IDLE: 1009 return True 1010 1011 if self.ps.poll() == None: 1012 return False 1013 1014 if self.ps.poll() != 0: 1015 self.handle_error() 1016 elif self.state == STATE_DEFCONFIG: 1017 if self.reference_src_dir and not self.current_src_dir: 1018 self.do_savedefconfig() 1019 else: 1020 self.do_autoconf() 1021 elif self.state == STATE_AUTOCONF: 1022 if self.current_src_dir: 1023 self.current_src_dir = None 1024 self.do_defconfig() 1025 else: 1026 self.do_savedefconfig() 1027 elif self.state == STATE_SAVEDEFCONFIG: 1028 self.update_defconfig() 1029 else: 1030 sys.exit("Internal Error. This should not happen.") 1031 1032 return True if self.state == STATE_IDLE else False 1033 1034 def handle_error(self): 1035 """Handle error cases.""" 1036 1037 self.log += color_text(self.options.color, COLOR_LIGHT_RED, 1038 "Failed to process.\n") 1039 if self.options.verbose: 1040 self.log += color_text(self.options.color, COLOR_LIGHT_CYAN, 1041 self.ps.stderr.read()) 1042 self.finish(False) 1043 1044 def do_defconfig(self): 1045 """Run 'make <board>_defconfig' to create the .config file.""" 1046 1047 cmd = list(self.make_cmd) 1048 cmd.append(self.defconfig) 1049 self.ps = subprocess.Popen(cmd, stdout=self.devnull, 1050 stderr=subprocess.PIPE, 1051 cwd=self.current_src_dir) 1052 self.state = STATE_DEFCONFIG 1053 1054 def do_autoconf(self): 1055 """Run 'make include/config/auto.conf'.""" 1056 1057 self.cross_compile = self.parser.get_cross_compile() 1058 if self.cross_compile is None: 1059 self.log += color_text(self.options.color, COLOR_YELLOW, 1060 "Compiler is missing. Do nothing.\n") 1061 self.finish(False) 1062 return 1063 1064 cmd = list(self.make_cmd) 1065 if self.cross_compile: 1066 cmd.append('CROSS_COMPILE=%s' % self.cross_compile) 1067 cmd.append('KCONFIG_IGNORE_DUPLICATES=1') 1068 cmd.append('include/config/auto.conf') 1069 self.ps = subprocess.Popen(cmd, stdout=self.devnull, 1070 stderr=subprocess.PIPE, 1071 cwd=self.current_src_dir) 1072 self.state = STATE_AUTOCONF 1073 1074 def do_savedefconfig(self): 1075 """Update the .config and run 'make savedefconfig'.""" 1076 1077 (updated, suspicious, log) = self.parser.update_dotconfig() 1078 if suspicious: 1079 self.suspicious_boards.add(self.defconfig) 1080 self.log += log 1081 1082 if not self.options.force_sync and not updated: 1083 self.finish(True) 1084 return 1085 if updated: 1086 self.log += color_text(self.options.color, COLOR_LIGHT_GREEN, 1087 "Syncing by savedefconfig...\n") 1088 else: 1089 self.log += "Syncing by savedefconfig (forced by option)...\n" 1090 1091 cmd = list(self.make_cmd) 1092 cmd.append('savedefconfig') 1093 self.ps = subprocess.Popen(cmd, stdout=self.devnull, 1094 stderr=subprocess.PIPE) 1095 self.state = STATE_SAVEDEFCONFIG 1096 1097 def update_defconfig(self): 1098 """Update the input defconfig and go back to the idle state.""" 1099 1100 log = self.parser.check_defconfig() 1101 if log: 1102 self.suspicious_boards.add(self.defconfig) 1103 self.log += log 1104 orig_defconfig = os.path.join('configs', self.defconfig) 1105 new_defconfig = os.path.join(self.build_dir, 'defconfig') 1106 updated = not filecmp.cmp(orig_defconfig, new_defconfig) 1107 1108 if updated: 1109 self.log += color_text(self.options.color, COLOR_LIGHT_BLUE, 1110 "defconfig was updated.\n") 1111 1112 if not self.options.dry_run and updated: 1113 shutil.move(new_defconfig, orig_defconfig) 1114 self.finish(True) 1115 1116 def finish(self, success): 1117 """Display log along with progress and go to the idle state. 1118 1119 Arguments: 1120 success: Should be True when the defconfig was processed 1121 successfully, or False when it fails. 1122 """ 1123 # output at least 30 characters to hide the "* defconfigs out of *". 1124 log = self.defconfig.ljust(30) + '\n' 1125 1126 log += '\n'.join([ ' ' + s for s in self.log.split('\n') ]) 1127 # Some threads are running in parallel. 1128 # Print log atomically to not mix up logs from different threads. 1129 print >> (sys.stdout if success else sys.stderr), log 1130 1131 if not success: 1132 if self.options.exit_on_error: 1133 sys.exit("Exit on error.") 1134 # If --exit-on-error flag is not set, skip this board and continue. 1135 # Record the failed board. 1136 self.failed_boards.add(self.defconfig) 1137 1138 self.progress.inc() 1139 self.progress.show() 1140 self.state = STATE_IDLE 1141 1142 def get_failed_boards(self): 1143 """Returns a set of failed boards (defconfigs) in this slot. 1144 """ 1145 return self.failed_boards 1146 1147 def get_suspicious_boards(self): 1148 """Returns a set of boards (defconfigs) with possible misconversion. 1149 """ 1150 return self.suspicious_boards - self.failed_boards 1151 1152class Slots: 1153 1154 """Controller of the array of subprocess slots.""" 1155 1156 def __init__(self, configs, options, progress, reference_src_dir): 1157 """Create a new slots controller. 1158 1159 Arguments: 1160 configs: A list of CONFIGs to move. 1161 options: option flags. 1162 progress: A progress indicator. 1163 reference_src_dir: Determine the true starting config state from this 1164 source tree. 1165 """ 1166 self.options = options 1167 self.slots = [] 1168 devnull = get_devnull() 1169 make_cmd = get_make_cmd() 1170 for i in range(options.jobs): 1171 self.slots.append(Slot(configs, options, progress, devnull, 1172 make_cmd, reference_src_dir)) 1173 1174 def add(self, defconfig): 1175 """Add a new subprocess if a vacant slot is found. 1176 1177 Arguments: 1178 defconfig: defconfig name to be put into. 1179 1180 Returns: 1181 Return True on success or False on failure 1182 """ 1183 for slot in self.slots: 1184 if slot.add(defconfig): 1185 return True 1186 return False 1187 1188 def available(self): 1189 """Check if there is a vacant slot. 1190 1191 Returns: 1192 Return True if at lease one vacant slot is found, False otherwise. 1193 """ 1194 for slot in self.slots: 1195 if slot.poll(): 1196 return True 1197 return False 1198 1199 def empty(self): 1200 """Check if all slots are vacant. 1201 1202 Returns: 1203 Return True if all the slots are vacant, False otherwise. 1204 """ 1205 ret = True 1206 for slot in self.slots: 1207 if not slot.poll(): 1208 ret = False 1209 return ret 1210 1211 def show_failed_boards(self): 1212 """Display all of the failed boards (defconfigs).""" 1213 boards = set() 1214 output_file = 'moveconfig.failed' 1215 1216 for slot in self.slots: 1217 boards |= slot.get_failed_boards() 1218 1219 if boards: 1220 boards = '\n'.join(boards) + '\n' 1221 msg = "The following boards were not processed due to error:\n" 1222 msg += boards 1223 msg += "(the list has been saved in %s)\n" % output_file 1224 print >> sys.stderr, color_text(self.options.color, COLOR_LIGHT_RED, 1225 msg) 1226 1227 with open(output_file, 'w') as f: 1228 f.write(boards) 1229 1230 def show_suspicious_boards(self): 1231 """Display all boards (defconfigs) with possible misconversion.""" 1232 boards = set() 1233 output_file = 'moveconfig.suspicious' 1234 1235 for slot in self.slots: 1236 boards |= slot.get_suspicious_boards() 1237 1238 if boards: 1239 boards = '\n'.join(boards) + '\n' 1240 msg = "The following boards might have been converted incorrectly.\n" 1241 msg += "It is highly recommended to check them manually:\n" 1242 msg += boards 1243 msg += "(the list has been saved in %s)\n" % output_file 1244 print >> sys.stderr, color_text(self.options.color, COLOR_YELLOW, 1245 msg) 1246 1247 with open(output_file, 'w') as f: 1248 f.write(boards) 1249 1250class ReferenceSource: 1251 1252 """Reference source against which original configs should be parsed.""" 1253 1254 def __init__(self, commit): 1255 """Create a reference source directory based on a specified commit. 1256 1257 Arguments: 1258 commit: commit to git-clone 1259 """ 1260 self.src_dir = tempfile.mkdtemp() 1261 print "Cloning git repo to a separate work directory..." 1262 subprocess.check_output(['git', 'clone', os.getcwd(), '.'], 1263 cwd=self.src_dir) 1264 print "Checkout '%s' to build the original autoconf.mk." % \ 1265 subprocess.check_output(['git', 'rev-parse', '--short', commit]).strip() 1266 subprocess.check_output(['git', 'checkout', commit], 1267 stderr=subprocess.STDOUT, cwd=self.src_dir) 1268 1269 def __del__(self): 1270 """Delete the reference source directory 1271 1272 This function makes sure the temporary directory is cleaned away 1273 even if Python suddenly dies due to error. It should be done in here 1274 because it is guaranteed the destructor is always invoked when the 1275 instance of the class gets unreferenced. 1276 """ 1277 shutil.rmtree(self.src_dir) 1278 1279 def get_dir(self): 1280 """Return the absolute path to the reference source directory.""" 1281 1282 return self.src_dir 1283 1284def move_config(configs, options): 1285 """Move config options to defconfig files. 1286 1287 Arguments: 1288 configs: A list of CONFIGs to move. 1289 options: option flags 1290 """ 1291 if len(configs) == 0: 1292 if options.force_sync: 1293 print 'No CONFIG is specified. You are probably syncing defconfigs.', 1294 else: 1295 print 'Neither CONFIG nor --force-sync is specified. Nothing will happen.', 1296 else: 1297 print 'Move ' + ', '.join(configs), 1298 print '(jobs: %d)\n' % options.jobs 1299 1300 if options.git_ref: 1301 reference_src = ReferenceSource(options.git_ref) 1302 reference_src_dir = reference_src.get_dir() 1303 else: 1304 reference_src_dir = None 1305 1306 if options.defconfigs: 1307 defconfigs = get_matched_defconfigs(options.defconfigs) 1308 else: 1309 defconfigs = get_all_defconfigs() 1310 1311 progress = Progress(len(defconfigs)) 1312 slots = Slots(configs, options, progress, reference_src_dir) 1313 1314 # Main loop to process defconfig files: 1315 # Add a new subprocess into a vacant slot. 1316 # Sleep if there is no available slot. 1317 for defconfig in defconfigs: 1318 while not slots.add(defconfig): 1319 while not slots.available(): 1320 # No available slot: sleep for a while 1321 time.sleep(SLEEP_TIME) 1322 1323 # wait until all the subprocesses finish 1324 while not slots.empty(): 1325 time.sleep(SLEEP_TIME) 1326 1327 print '' 1328 slots.show_failed_boards() 1329 slots.show_suspicious_boards() 1330 1331def main(): 1332 try: 1333 cpu_count = multiprocessing.cpu_count() 1334 except NotImplementedError: 1335 cpu_count = 1 1336 1337 parser = optparse.OptionParser() 1338 # Add options here 1339 parser.add_option('-c', '--color', action='store_true', default=False, 1340 help='display the log in color') 1341 parser.add_option('-C', '--commit', action='store_true', default=False, 1342 help='Create a git commit for the operation') 1343 parser.add_option('-d', '--defconfigs', type='string', 1344 help='a file containing a list of defconfigs to move, ' 1345 "one per line (for example 'snow_defconfig') " 1346 "or '-' to read from stdin") 1347 parser.add_option('-n', '--dry-run', action='store_true', default=False, 1348 help='perform a trial run (show log with no changes)') 1349 parser.add_option('-e', '--exit-on-error', action='store_true', 1350 default=False, 1351 help='exit immediately on any error') 1352 parser.add_option('-s', '--force-sync', action='store_true', default=False, 1353 help='force sync by savedefconfig') 1354 parser.add_option('-S', '--spl', action='store_true', default=False, 1355 help='parse config options defined for SPL build') 1356 parser.add_option('-H', '--headers-only', dest='cleanup_headers_only', 1357 action='store_true', default=False, 1358 help='only cleanup the headers') 1359 parser.add_option('-j', '--jobs', type='int', default=cpu_count, 1360 help='the number of jobs to run simultaneously') 1361 parser.add_option('-r', '--git-ref', type='string', 1362 help='the git ref to clone for building the autoconf.mk') 1363 parser.add_option('-y', '--yes', action='store_true', default=False, 1364 help="respond 'yes' to any prompts") 1365 parser.add_option('-v', '--verbose', action='store_true', default=False, 1366 help='show any build errors as boards are built') 1367 parser.usage += ' CONFIG ...' 1368 1369 (options, configs) = parser.parse_args() 1370 1371 if len(configs) == 0 and not options.force_sync: 1372 parser.print_usage() 1373 sys.exit(1) 1374 1375 # prefix the option name with CONFIG_ if missing 1376 configs = [ config if config.startswith('CONFIG_') else 'CONFIG_' + config 1377 for config in configs ] 1378 1379 check_top_directory() 1380 1381 if not options.cleanup_headers_only: 1382 check_clean_directory() 1383 update_cross_compile(options.color) 1384 move_config(configs, options) 1385 1386 if configs: 1387 cleanup_headers(configs, options) 1388 cleanup_extra_options(configs, options) 1389 cleanup_whitelist(configs, options) 1390 cleanup_readme(configs, options) 1391 1392 if options.commit: 1393 subprocess.call(['git', 'add', '-u']) 1394 if configs: 1395 msg = 'Convert %s %sto Kconfig' % (configs[0], 1396 'et al ' if len(configs) > 1 else '') 1397 msg += ('\n\nThis converts the following to Kconfig:\n %s\n' % 1398 '\n '.join(configs)) 1399 else: 1400 msg = 'configs: Resync with savedefconfig' 1401 msg += '\n\nRsync all defconfig files using moveconfig.py' 1402 subprocess.call(['git', 'commit', '-s', '-m', msg]) 1403 1404if __name__ == '__main__': 1405 main() 1406