1#!/usr/bin/env python2 2# 3# Author: Masahiro Yamada <yamada.masahiro@socionext.com> 4# 5# SPDX-License-Identifier: GPL-2.0+ 6# 7 8""" 9Move config options from headers to defconfig files. 10 11Since Kconfig was introduced to U-Boot, we have worked on moving 12config options from headers to Kconfig (defconfig). 13 14This tool intends to help this tremendous work. 15 16 17Usage 18----- 19 20First, you must edit the Kconfig to add the menu entries for the configs 21you are moving. 22 23And then run this tool giving CONFIG names you want to move. 24For example, if you want to move CONFIG_CMD_USB and CONFIG_SYS_TEXT_BASE, 25simply type as follows: 26 27 $ tools/moveconfig.py CONFIG_CMD_USB CONFIG_SYS_TEXT_BASE 28 29The tool walks through all the defconfig files and move the given CONFIGs. 30 31The log is also displayed on the terminal. 32 33The log is printed for each defconfig as follows: 34 35<defconfig_name> 36 <action1> 37 <action2> 38 <action3> 39 ... 40 41<defconfig_name> is the name of the defconfig. 42 43<action*> shows what the tool did for that defconfig. 44It looks like one of the following: 45 46 - Move 'CONFIG_... ' 47 This config option was moved to the defconfig 48 49 - CONFIG_... is not defined in Kconfig. Do nothing. 50 The entry for this CONFIG was not found in Kconfig. The option is not 51 defined in the config header, either. So, this case can be just skipped. 52 53 - CONFIG_... is not defined in Kconfig (suspicious). Do nothing. 54 This option is defined in the config header, but its entry was not found 55 in Kconfig. 56 There are two common cases: 57 - You forgot to create an entry for the CONFIG before running 58 this tool, or made a typo in a CONFIG passed to this tool. 59 - The entry was hidden due to unmet 'depends on'. 60 The tool does not know if the result is reasonable, so please check it 61 manually. 62 63 - 'CONFIG_...' is the same as the define in Kconfig. Do nothing. 64 The define in the config header matched the one in Kconfig. 65 We do not need to touch it. 66 67 - Compiler is missing. Do nothing. 68 The compiler specified for this architecture was not found 69 in your PATH environment. 70 (If -e option is passed, the tool exits immediately.) 71 72 - Failed to process. 73 An error occurred during processing this defconfig. Skipped. 74 (If -e option is passed, the tool exits immediately on error.) 75 76Finally, you will be asked, Clean up headers? [y/n]: 77 78If you say 'y' here, the unnecessary config defines are removed 79from the config headers (include/configs/*.h). 80It just uses the regex method, so you should not rely on it. 81Just in case, please do 'git diff' to see what happened. 82 83 84How does it work? 85----------------- 86 87This tool runs configuration and builds include/autoconf.mk for every 88defconfig. The config options defined in Kconfig appear in the .config 89file (unless they are hidden because of unmet dependency.) 90On the other hand, the config options defined by board headers are seen 91in include/autoconf.mk. The tool looks for the specified options in both 92of them to decide the appropriate action for the options. If the given 93config option is found in the .config, but its value does not match the 94one from the board header, the config option in the .config is replaced 95with the define in the board header. Then, the .config is synced by 96"make savedefconfig" and the defconfig is updated with it. 97 98For faster processing, this tool handles multi-threading. It creates 99separate build directories where the out-of-tree build is run. The 100temporary build directories are automatically created and deleted as 101needed. The number of threads are chosen based on the number of the CPU 102cores of your system although you can change it via -j (--jobs) option. 103 104 105Toolchains 106---------- 107 108Appropriate toolchain are necessary to generate include/autoconf.mk 109for all the architectures supported by U-Boot. Most of them are available 110at the kernel.org site, some are not provided by kernel.org. 111 112The default per-arch CROSS_COMPILE used by this tool is specified by 113the list below, CROSS_COMPILE. You may wish to update the list to 114use your own. Instead of modifying the list directly, you can give 115them via environments. 116 117 118Tips and trips 119-------------- 120 121To sync only X86 defconfigs: 122 123 ./tools/moveconfig.py -s -d <(grep -l X86 configs/*) 124 125or: 126 127 grep -l X86 configs/* | ./tools/moveconfig.py -s -d - 128 129To process CONFIG_CMD_FPGAD only for a subset of configs based on path match: 130 131 ls configs/{hrcon*,iocon*,strider*} | \ 132 ./tools/moveconfig.py -Cy CONFIG_CMD_FPGAD -d - 133 134 135Available options 136----------------- 137 138 -c, --color 139 Surround each portion of the log with escape sequences to display it 140 in color on the terminal. 141 142 -C, --commit 143 Create a git commit with the changes when the operation is complete. A 144 standard commit message is used which may need to be edited. 145 146 -d, --defconfigs 147 Specify a file containing a list of defconfigs to move. The defconfig 148 files can be given with shell-style wildcards. Use '-' to read from stdin. 149 150 -n, --dry-run 151 Perform a trial run that does not make any changes. It is useful to 152 see what is going to happen before one actually runs it. 153 154 -e, --exit-on-error 155 Exit immediately if Make exits with a non-zero status while processing 156 a defconfig file. 157 158 -s, --force-sync 159 Do "make savedefconfig" forcibly for all the defconfig files. 160 If not specified, "make savedefconfig" only occurs for cases 161 where at least one CONFIG was moved. 162 163 -S, --spl 164 Look for moved config options in spl/include/autoconf.mk instead of 165 include/autoconf.mk. This is useful for moving options for SPL build 166 because SPL related options (mostly prefixed with CONFIG_SPL_) are 167 sometimes blocked by CONFIG_SPL_BUILD ifdef conditionals. 168 169 -H, --headers-only 170 Only cleanup the headers; skip the defconfig processing 171 172 -j, --jobs 173 Specify the number of threads to run simultaneously. If not specified, 174 the number of threads is the same as the number of CPU cores. 175 176 -r, --git-ref 177 Specify the git ref to clone for building the autoconf.mk. If unspecified 178 use the CWD. This is useful for when changes to the Kconfig affect the 179 default values and you want to capture the state of the defconfig from 180 before that change was in effect. If in doubt, specify a ref pre-Kconfig 181 changes (use HEAD if Kconfig changes are not committed). Worst case it will 182 take a bit longer to run, but will always do the right thing. 183 184 -v, --verbose 185 Show any build errors as boards are built 186 187 -y, --yes 188 Instead of prompting, automatically go ahead with all operations. This 189 includes cleaning up headers, CONFIG_SYS_EXTRA_OPTIONS, the config whitelist 190 and the README. 191 192To see the complete list of supported options, run 193 194 $ tools/moveconfig.py -h 195 196""" 197 198import copy 199import difflib 200import filecmp 201import fnmatch 202import glob 203import multiprocessing 204import optparse 205import os 206import re 207import shutil 208import subprocess 209import sys 210import tempfile 211import time 212 213SHOW_GNU_MAKE = 'scripts/show-gnu-make' 214SLEEP_TIME=0.03 215 216# Here is the list of cross-tools I use. 217# Most of them are available at kernel.org 218# (https://www.kernel.org/pub/tools/crosstool/files/bin/), except the following: 219# arc: https://github.com/foss-for-synopsys-dwc-arc-processors/toolchain/releases 220# nds32: http://osdk.andestech.com/packages/nds32le-linux-glibc-v1.tgz 221# nios2: https://sourcery.mentor.com/GNUToolchain/subscription42545 222# sh: http://sourcery.mentor.com/public/gnu_toolchain/sh-linux-gnu 223CROSS_COMPILE = { 224 'arc': 'arc-linux-', 225 'aarch64': 'aarch64-linux-', 226 'arm': 'arm-unknown-linux-gnueabi-', 227 'm68k': 'm68k-linux-', 228 'microblaze': 'microblaze-linux-', 229 'mips': 'mips-linux-', 230 'nds32': 'nds32le-linux-', 231 'nios2': 'nios2-linux-gnu-', 232 'powerpc': 'powerpc-linux-', 233 'sh': 'sh-linux-gnu-', 234 'x86': 'i386-linux-', 235 'xtensa': 'xtensa-linux-' 236} 237 238STATE_IDLE = 0 239STATE_DEFCONFIG = 1 240STATE_AUTOCONF = 2 241STATE_SAVEDEFCONFIG = 3 242 243ACTION_MOVE = 0 244ACTION_NO_ENTRY = 1 245ACTION_NO_ENTRY_WARN = 2 246ACTION_NO_CHANGE = 3 247 248COLOR_BLACK = '0;30' 249COLOR_RED = '0;31' 250COLOR_GREEN = '0;32' 251COLOR_BROWN = '0;33' 252COLOR_BLUE = '0;34' 253COLOR_PURPLE = '0;35' 254COLOR_CYAN = '0;36' 255COLOR_LIGHT_GRAY = '0;37' 256COLOR_DARK_GRAY = '1;30' 257COLOR_LIGHT_RED = '1;31' 258COLOR_LIGHT_GREEN = '1;32' 259COLOR_YELLOW = '1;33' 260COLOR_LIGHT_BLUE = '1;34' 261COLOR_LIGHT_PURPLE = '1;35' 262COLOR_LIGHT_CYAN = '1;36' 263COLOR_WHITE = '1;37' 264 265### helper functions ### 266def get_devnull(): 267 """Get the file object of '/dev/null' device.""" 268 try: 269 devnull = subprocess.DEVNULL # py3k 270 except AttributeError: 271 devnull = open(os.devnull, 'wb') 272 return devnull 273 274def check_top_directory(): 275 """Exit if we are not at the top of source directory.""" 276 for f in ('README', 'Licenses'): 277 if not os.path.exists(f): 278 sys.exit('Please run at the top of source directory.') 279 280def check_clean_directory(): 281 """Exit if the source tree is not clean.""" 282 for f in ('.config', 'include/config'): 283 if os.path.exists(f): 284 sys.exit("source tree is not clean, please run 'make mrproper'") 285 286def get_make_cmd(): 287 """Get the command name of GNU Make. 288 289 U-Boot needs GNU Make for building, but the command name is not 290 necessarily "make". (for example, "gmake" on FreeBSD). 291 Returns the most appropriate command name on your system. 292 """ 293 process = subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE) 294 ret = process.communicate() 295 if process.returncode: 296 sys.exit('GNU Make not found') 297 return ret[0].rstrip() 298 299def get_matched_defconfig(line): 300 """Get the defconfig files that match a pattern 301 302 Args: 303 line: Path or filename to match, e.g. 'configs/snow_defconfig' or 304 'k2*_defconfig'. If no directory is provided, 'configs/' is 305 prepended 306 307 Returns: 308 a list of matching defconfig files 309 """ 310 dirname = os.path.dirname(line) 311 if dirname: 312 pattern = line 313 else: 314 pattern = os.path.join('configs', line) 315 return glob.glob(pattern) + glob.glob(pattern + '_defconfig') 316 317def get_matched_defconfigs(defconfigs_file): 318 """Get all the defconfig files that match the patterns in a file. 319 320 Args: 321 defconfigs_file: File containing a list of defconfigs to process, or 322 '-' to read the list from stdin 323 324 Returns: 325 A list of paths to defconfig files, with no duplicates 326 """ 327 defconfigs = [] 328 if defconfigs_file == '-': 329 fd = sys.stdin 330 defconfigs_file = 'stdin' 331 else: 332 fd = open(defconfigs_file) 333 for i, line in enumerate(fd): 334 line = line.strip() 335 if not line: 336 continue # skip blank lines silently 337 matched = get_matched_defconfig(line) 338 if not matched: 339 print >> sys.stderr, "warning: %s:%d: no defconfig matched '%s'" % \ 340 (defconfigs_file, i + 1, line) 341 342 defconfigs += matched 343 344 # use set() to drop multiple matching 345 return [ defconfig[len('configs') + 1:] for defconfig in set(defconfigs) ] 346 347def get_all_defconfigs(): 348 """Get all the defconfig files under the configs/ directory.""" 349 defconfigs = [] 350 for (dirpath, dirnames, filenames) in os.walk('configs'): 351 dirpath = dirpath[len('configs') + 1:] 352 for filename in fnmatch.filter(filenames, '*_defconfig'): 353 defconfigs.append(os.path.join(dirpath, filename)) 354 355 return defconfigs 356 357def color_text(color_enabled, color, string): 358 """Return colored string.""" 359 if color_enabled: 360 # LF should not be surrounded by the escape sequence. 361 # Otherwise, additional whitespace or line-feed might be printed. 362 return '\n'.join([ '\033[' + color + 'm' + s + '\033[0m' if s else '' 363 for s in string.split('\n') ]) 364 else: 365 return string 366 367def show_diff(a, b, file_path, color_enabled): 368 """Show unidified diff. 369 370 Arguments: 371 a: A list of lines (before) 372 b: A list of lines (after) 373 file_path: Path to the file 374 color_enabled: Display the diff in color 375 """ 376 377 diff = difflib.unified_diff(a, b, 378 fromfile=os.path.join('a', file_path), 379 tofile=os.path.join('b', file_path)) 380 381 for line in diff: 382 if line[0] == '-' and line[1] != '-': 383 print color_text(color_enabled, COLOR_RED, line), 384 elif line[0] == '+' and line[1] != '+': 385 print color_text(color_enabled, COLOR_GREEN, line), 386 else: 387 print line, 388 389def update_cross_compile(color_enabled): 390 """Update per-arch CROSS_COMPILE via environment variables 391 392 The default CROSS_COMPILE values are available 393 in the CROSS_COMPILE list above. 394 395 You can override them via environment variables 396 CROSS_COMPILE_{ARCH}. 397 398 For example, if you want to override toolchain prefixes 399 for ARM and PowerPC, you can do as follows in your shell: 400 401 export CROSS_COMPILE_ARM=... 402 export CROSS_COMPILE_POWERPC=... 403 404 Then, this function checks if specified compilers really exist in your 405 PATH environment. 406 """ 407 archs = [] 408 409 for arch in os.listdir('arch'): 410 if os.path.exists(os.path.join('arch', arch, 'Makefile')): 411 archs.append(arch) 412 413 # arm64 is a special case 414 archs.append('aarch64') 415 416 for arch in archs: 417 env = 'CROSS_COMPILE_' + arch.upper() 418 cross_compile = os.environ.get(env) 419 if not cross_compile: 420 cross_compile = CROSS_COMPILE.get(arch, '') 421 422 for path in os.environ["PATH"].split(os.pathsep): 423 gcc_path = os.path.join(path, cross_compile + 'gcc') 424 if os.path.isfile(gcc_path) and os.access(gcc_path, os.X_OK): 425 break 426 else: 427 print >> sys.stderr, color_text(color_enabled, COLOR_YELLOW, 428 'warning: %sgcc: not found in PATH. %s architecture boards will be skipped' 429 % (cross_compile, arch)) 430 cross_compile = None 431 432 CROSS_COMPILE[arch] = cross_compile 433 434def extend_matched_lines(lines, matched, pre_patterns, post_patterns, extend_pre, 435 extend_post): 436 """Extend matched lines if desired patterns are found before/after already 437 matched lines. 438 439 Arguments: 440 lines: A list of lines handled. 441 matched: A list of line numbers that have been already matched. 442 (will be updated by this function) 443 pre_patterns: A list of regular expression that should be matched as 444 preamble. 445 post_patterns: A list of regular expression that should be matched as 446 postamble. 447 extend_pre: Add the line number of matched preamble to the matched list. 448 extend_post: Add the line number of matched postamble to the matched list. 449 """ 450 extended_matched = [] 451 452 j = matched[0] 453 454 for i in matched: 455 if i == 0 or i < j: 456 continue 457 j = i 458 while j in matched: 459 j += 1 460 if j >= len(lines): 461 break 462 463 for p in pre_patterns: 464 if p.search(lines[i - 1]): 465 break 466 else: 467 # not matched 468 continue 469 470 for p in post_patterns: 471 if p.search(lines[j]): 472 break 473 else: 474 # not matched 475 continue 476 477 if extend_pre: 478 extended_matched.append(i - 1) 479 if extend_post: 480 extended_matched.append(j) 481 482 matched += extended_matched 483 matched.sort() 484 485def confirm(options, prompt): 486 if not options.yes: 487 while True: 488 choice = raw_input('{} [y/n]: '.format(prompt)) 489 choice = choice.lower() 490 print choice 491 if choice == 'y' or choice == 'n': 492 break 493 494 if choice == 'n': 495 return False 496 497 return True 498 499def cleanup_one_header(header_path, patterns, options): 500 """Clean regex-matched lines away from a file. 501 502 Arguments: 503 header_path: path to the cleaned file. 504 patterns: list of regex patterns. Any lines matching to these 505 patterns are deleted. 506 options: option flags. 507 """ 508 with open(header_path) as f: 509 lines = f.readlines() 510 511 matched = [] 512 for i, line in enumerate(lines): 513 if i - 1 in matched and lines[i - 1][-2:] == '\\\n': 514 matched.append(i) 515 continue 516 for pattern in patterns: 517 if pattern.search(line): 518 matched.append(i) 519 break 520 521 if not matched: 522 return 523 524 # remove empty #ifdef ... #endif, successive blank lines 525 pattern_if = re.compile(r'#\s*if(def|ndef)?\W') # #if, #ifdef, #ifndef 526 pattern_elif = re.compile(r'#\s*el(if|se)\W') # #elif, #else 527 pattern_endif = re.compile(r'#\s*endif\W') # #endif 528 pattern_blank = re.compile(r'^\s*$') # empty line 529 530 while True: 531 old_matched = copy.copy(matched) 532 extend_matched_lines(lines, matched, [pattern_if], 533 [pattern_endif], True, True) 534 extend_matched_lines(lines, matched, [pattern_elif], 535 [pattern_elif, pattern_endif], True, False) 536 extend_matched_lines(lines, matched, [pattern_if, pattern_elif], 537 [pattern_blank], False, True) 538 extend_matched_lines(lines, matched, [pattern_blank], 539 [pattern_elif, pattern_endif], True, False) 540 extend_matched_lines(lines, matched, [pattern_blank], 541 [pattern_blank], True, False) 542 if matched == old_matched: 543 break 544 545 tolines = copy.copy(lines) 546 547 for i in reversed(matched): 548 tolines.pop(i) 549 550 show_diff(lines, tolines, header_path, options.color) 551 552 if options.dry_run: 553 return 554 555 with open(header_path, 'w') as f: 556 for line in tolines: 557 f.write(line) 558 559def cleanup_headers(configs, options): 560 """Delete config defines from board headers. 561 562 Arguments: 563 configs: A list of CONFIGs to remove. 564 options: option flags. 565 """ 566 if not confirm(options, 'Clean up headers?'): 567 return 568 569 patterns = [] 570 for config in configs: 571 patterns.append(re.compile(r'#\s*define\s+%s\W' % config)) 572 patterns.append(re.compile(r'#\s*undef\s+%s\W' % config)) 573 574 for dir in 'include', 'arch', 'board': 575 for (dirpath, dirnames, filenames) in os.walk(dir): 576 if dirpath == os.path.join('include', 'generated'): 577 continue 578 for filename in filenames: 579 if not fnmatch.fnmatch(filename, '*~'): 580 cleanup_one_header(os.path.join(dirpath, filename), 581 patterns, options) 582 583def cleanup_one_extra_option(defconfig_path, configs, options): 584 """Delete config defines in CONFIG_SYS_EXTRA_OPTIONS in one defconfig file. 585 586 Arguments: 587 defconfig_path: path to the cleaned defconfig file. 588 configs: A list of CONFIGs to remove. 589 options: option flags. 590 """ 591 592 start = 'CONFIG_SYS_EXTRA_OPTIONS="' 593 end = '"\n' 594 595 with open(defconfig_path) as f: 596 lines = f.readlines() 597 598 for i, line in enumerate(lines): 599 if line.startswith(start) and line.endswith(end): 600 break 601 else: 602 # CONFIG_SYS_EXTRA_OPTIONS was not found in this defconfig 603 return 604 605 old_tokens = line[len(start):-len(end)].split(',') 606 new_tokens = [] 607 608 for token in old_tokens: 609 pos = token.find('=') 610 if not (token[:pos] if pos >= 0 else token) in configs: 611 new_tokens.append(token) 612 613 if new_tokens == old_tokens: 614 return 615 616 tolines = copy.copy(lines) 617 618 if new_tokens: 619 tolines[i] = start + ','.join(new_tokens) + end 620 else: 621 tolines.pop(i) 622 623 show_diff(lines, tolines, defconfig_path, options.color) 624 625 if options.dry_run: 626 return 627 628 with open(defconfig_path, 'w') as f: 629 for line in tolines: 630 f.write(line) 631 632def cleanup_extra_options(configs, options): 633 """Delete config defines in CONFIG_SYS_EXTRA_OPTIONS in defconfig files. 634 635 Arguments: 636 configs: A list of CONFIGs to remove. 637 options: option flags. 638 """ 639 if not confirm(options, 'Clean up CONFIG_SYS_EXTRA_OPTIONS?'): 640 return 641 642 configs = [ config[len('CONFIG_'):] for config in configs ] 643 644 defconfigs = get_all_defconfigs() 645 646 for defconfig in defconfigs: 647 cleanup_one_extra_option(os.path.join('configs', defconfig), configs, 648 options) 649 650def cleanup_whitelist(configs, options): 651 """Delete config whitelist entries 652 653 Arguments: 654 configs: A list of CONFIGs to remove. 655 options: option flags. 656 """ 657 if not confirm(options, 'Clean up whitelist entries?'): 658 return 659 660 with open(os.path.join('scripts', 'config_whitelist.txt')) as f: 661 lines = f.readlines() 662 663 lines = [x for x in lines if x.strip() not in configs] 664 665 with open(os.path.join('scripts', 'config_whitelist.txt'), 'w') as f: 666 f.write(''.join(lines)) 667 668def find_matching(patterns, line): 669 for pat in patterns: 670 if pat.search(line): 671 return True 672 return False 673 674def cleanup_readme(configs, options): 675 """Delete config description in README 676 677 Arguments: 678 configs: A list of CONFIGs to remove. 679 options: option flags. 680 """ 681 if not confirm(options, 'Clean up README?'): 682 return 683 684 patterns = [] 685 for config in configs: 686 patterns.append(re.compile(r'^\s+%s' % config)) 687 688 with open('README') as f: 689 lines = f.readlines() 690 691 found = False 692 newlines = [] 693 for line in lines: 694 if not found: 695 found = find_matching(patterns, line) 696 if found: 697 continue 698 699 if found and re.search(r'^\s+CONFIG', line): 700 found = False 701 702 if not found: 703 newlines.append(line) 704 705 with open('README', 'w') as f: 706 f.write(''.join(newlines)) 707 708 709### classes ### 710class Progress: 711 712 """Progress Indicator""" 713 714 def __init__(self, total): 715 """Create a new progress indicator. 716 717 Arguments: 718 total: A number of defconfig files to process. 719 """ 720 self.current = 0 721 self.total = total 722 723 def inc(self): 724 """Increment the number of processed defconfig files.""" 725 726 self.current += 1 727 728 def show(self): 729 """Display the progress.""" 730 print ' %d defconfigs out of %d\r' % (self.current, self.total), 731 sys.stdout.flush() 732 733class KconfigParser: 734 735 """A parser of .config and include/autoconf.mk.""" 736 737 re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"') 738 re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"') 739 740 def __init__(self, configs, options, build_dir): 741 """Create a new parser. 742 743 Arguments: 744 configs: A list of CONFIGs to move. 745 options: option flags. 746 build_dir: Build directory. 747 """ 748 self.configs = configs 749 self.options = options 750 self.dotconfig = os.path.join(build_dir, '.config') 751 self.autoconf = os.path.join(build_dir, 'include', 'autoconf.mk') 752 self.spl_autoconf = os.path.join(build_dir, 'spl', 'include', 753 'autoconf.mk') 754 self.config_autoconf = os.path.join(build_dir, 'include', 'config', 755 'auto.conf') 756 self.defconfig = os.path.join(build_dir, 'defconfig') 757 758 def get_cross_compile(self): 759 """Parse .config file and return CROSS_COMPILE. 760 761 Returns: 762 A string storing the compiler prefix for the architecture. 763 Return a NULL string for architectures that do not require 764 compiler prefix (Sandbox and native build is the case). 765 Return None if the specified compiler is missing in your PATH. 766 Caller should distinguish '' and None. 767 """ 768 arch = '' 769 cpu = '' 770 for line in open(self.dotconfig): 771 m = self.re_arch.match(line) 772 if m: 773 arch = m.group(1) 774 continue 775 m = self.re_cpu.match(line) 776 if m: 777 cpu = m.group(1) 778 779 if not arch: 780 return None 781 782 # fix-up for aarch64 783 if arch == 'arm' and cpu == 'armv8': 784 arch = 'aarch64' 785 786 return CROSS_COMPILE.get(arch, None) 787 788 def parse_one_config(self, config, dotconfig_lines, autoconf_lines): 789 """Parse .config, defconfig, include/autoconf.mk for one config. 790 791 This function looks for the config options in the lines from 792 defconfig, .config, and include/autoconf.mk in order to decide 793 which action should be taken for this defconfig. 794 795 Arguments: 796 config: CONFIG name to parse. 797 dotconfig_lines: lines from the .config file. 798 autoconf_lines: lines from the include/autoconf.mk file. 799 800 Returns: 801 A tupple of the action for this defconfig and the line 802 matched for the config. 803 """ 804 not_set = '# %s is not set' % config 805 806 for line in autoconf_lines: 807 line = line.rstrip() 808 if line.startswith(config + '='): 809 new_val = line 810 break 811 else: 812 new_val = not_set 813 814 for line in dotconfig_lines: 815 line = line.rstrip() 816 if line.startswith(config + '=') or line == not_set: 817 old_val = line 818 break 819 else: 820 if new_val == not_set: 821 return (ACTION_NO_ENTRY, config) 822 else: 823 return (ACTION_NO_ENTRY_WARN, config) 824 825 # If this CONFIG is neither bool nor trisate 826 if old_val[-2:] != '=y' and old_val[-2:] != '=m' and old_val != not_set: 827 # tools/scripts/define2mk.sed changes '1' to 'y'. 828 # This is a problem if the CONFIG is int type. 829 # Check the type in Kconfig and handle it correctly. 830 if new_val[-2:] == '=y': 831 new_val = new_val[:-1] + '1' 832 833 return (ACTION_NO_CHANGE if old_val == new_val else ACTION_MOVE, 834 new_val) 835 836 def update_dotconfig(self): 837 """Parse files for the config options and update the .config. 838 839 This function parses the generated .config and include/autoconf.mk 840 searching the target options. 841 Move the config option(s) to the .config as needed. 842 843 Arguments: 844 defconfig: defconfig name. 845 846 Returns: 847 Return a tuple of (updated flag, log string). 848 The "updated flag" is True if the .config was updated, False 849 otherwise. The "log string" shows what happend to the .config. 850 """ 851 852 results = [] 853 updated = False 854 suspicious = False 855 rm_files = [self.config_autoconf, self.autoconf] 856 857 if self.options.spl: 858 if os.path.exists(self.spl_autoconf): 859 autoconf_path = self.spl_autoconf 860 rm_files.append(self.spl_autoconf) 861 else: 862 for f in rm_files: 863 os.remove(f) 864 return (updated, suspicious, 865 color_text(self.options.color, COLOR_BROWN, 866 "SPL is not enabled. Skipped.") + '\n') 867 else: 868 autoconf_path = self.autoconf 869 870 with open(self.dotconfig) as f: 871 dotconfig_lines = f.readlines() 872 873 with open(autoconf_path) as f: 874 autoconf_lines = f.readlines() 875 876 for config in self.configs: 877 result = self.parse_one_config(config, dotconfig_lines, 878 autoconf_lines) 879 results.append(result) 880 881 log = '' 882 883 for (action, value) in results: 884 if action == ACTION_MOVE: 885 actlog = "Move '%s'" % value 886 log_color = COLOR_LIGHT_GREEN 887 elif action == ACTION_NO_ENTRY: 888 actlog = "%s is not defined in Kconfig. Do nothing." % value 889 log_color = COLOR_LIGHT_BLUE 890 elif action == ACTION_NO_ENTRY_WARN: 891 actlog = "%s is not defined in Kconfig (suspicious). Do nothing." % value 892 log_color = COLOR_YELLOW 893 suspicious = True 894 elif action == ACTION_NO_CHANGE: 895 actlog = "'%s' is the same as the define in Kconfig. Do nothing." \ 896 % value 897 log_color = COLOR_LIGHT_PURPLE 898 elif action == ACTION_SPL_NOT_EXIST: 899 actlog = "SPL is not enabled for this defconfig. Skip." 900 log_color = COLOR_PURPLE 901 else: 902 sys.exit("Internal Error. This should not happen.") 903 904 log += color_text(self.options.color, log_color, actlog) + '\n' 905 906 with open(self.dotconfig, 'a') as f: 907 for (action, value) in results: 908 if action == ACTION_MOVE: 909 f.write(value + '\n') 910 updated = True 911 912 self.results = results 913 for f in rm_files: 914 os.remove(f) 915 916 return (updated, suspicious, log) 917 918 def check_defconfig(self): 919 """Check the defconfig after savedefconfig 920 921 Returns: 922 Return additional log if moved CONFIGs were removed again by 923 'make savedefconfig'. 924 """ 925 926 log = '' 927 928 with open(self.defconfig) as f: 929 defconfig_lines = f.readlines() 930 931 for (action, value) in self.results: 932 if action != ACTION_MOVE: 933 continue 934 if not value + '\n' in defconfig_lines: 935 log += color_text(self.options.color, COLOR_YELLOW, 936 "'%s' was removed by savedefconfig.\n" % 937 value) 938 939 return log 940 941class Slot: 942 943 """A slot to store a subprocess. 944 945 Each instance of this class handles one subprocess. 946 This class is useful to control multiple threads 947 for faster processing. 948 """ 949 950 def __init__(self, configs, options, progress, devnull, make_cmd, reference_src_dir): 951 """Create a new process slot. 952 953 Arguments: 954 configs: A list of CONFIGs to move. 955 options: option flags. 956 progress: A progress indicator. 957 devnull: A file object of '/dev/null'. 958 make_cmd: command name of GNU Make. 959 reference_src_dir: Determine the true starting config state from this 960 source tree. 961 """ 962 self.options = options 963 self.progress = progress 964 self.build_dir = tempfile.mkdtemp() 965 self.devnull = devnull 966 self.make_cmd = (make_cmd, 'O=' + self.build_dir) 967 self.reference_src_dir = reference_src_dir 968 self.parser = KconfigParser(configs, options, self.build_dir) 969 self.state = STATE_IDLE 970 self.failed_boards = set() 971 self.suspicious_boards = set() 972 973 def __del__(self): 974 """Delete the working directory 975 976 This function makes sure the temporary directory is cleaned away 977 even if Python suddenly dies due to error. It should be done in here 978 because it is guaranteed the destructor is always invoked when the 979 instance of the class gets unreferenced. 980 981 If the subprocess is still running, wait until it finishes. 982 """ 983 if self.state != STATE_IDLE: 984 while self.ps.poll() == None: 985 pass 986 shutil.rmtree(self.build_dir) 987 988 def add(self, defconfig): 989 """Assign a new subprocess for defconfig and add it to the slot. 990 991 If the slot is vacant, create a new subprocess for processing the 992 given defconfig and add it to the slot. Just returns False if 993 the slot is occupied (i.e. the current subprocess is still running). 994 995 Arguments: 996 defconfig: defconfig name. 997 998 Returns: 999 Return True on success or False on failure 1000 """ 1001 if self.state != STATE_IDLE: 1002 return False 1003 1004 self.defconfig = defconfig 1005 self.log = '' 1006 self.current_src_dir = self.reference_src_dir 1007 self.do_defconfig() 1008 return True 1009 1010 def poll(self): 1011 """Check the status of the subprocess and handle it as needed. 1012 1013 Returns True if the slot is vacant (i.e. in idle state). 1014 If the configuration is successfully finished, assign a new 1015 subprocess to build include/autoconf.mk. 1016 If include/autoconf.mk is generated, invoke the parser to 1017 parse the .config and the include/autoconf.mk, moving 1018 config options to the .config as needed. 1019 If the .config was updated, run "make savedefconfig" to sync 1020 it, update the original defconfig, and then set the slot back 1021 to the idle state. 1022 1023 Returns: 1024 Return True if the subprocess is terminated, False otherwise 1025 """ 1026 if self.state == STATE_IDLE: 1027 return True 1028 1029 if self.ps.poll() == None: 1030 return False 1031 1032 if self.ps.poll() != 0: 1033 self.handle_error() 1034 elif self.state == STATE_DEFCONFIG: 1035 if self.reference_src_dir and not self.current_src_dir: 1036 self.do_savedefconfig() 1037 else: 1038 self.do_autoconf() 1039 elif self.state == STATE_AUTOCONF: 1040 if self.current_src_dir: 1041 self.current_src_dir = None 1042 self.do_defconfig() 1043 else: 1044 self.do_savedefconfig() 1045 elif self.state == STATE_SAVEDEFCONFIG: 1046 self.update_defconfig() 1047 else: 1048 sys.exit("Internal Error. This should not happen.") 1049 1050 return True if self.state == STATE_IDLE else False 1051 1052 def handle_error(self): 1053 """Handle error cases.""" 1054 1055 self.log += color_text(self.options.color, COLOR_LIGHT_RED, 1056 "Failed to process.\n") 1057 if self.options.verbose: 1058 self.log += color_text(self.options.color, COLOR_LIGHT_CYAN, 1059 self.ps.stderr.read()) 1060 self.finish(False) 1061 1062 def do_defconfig(self): 1063 """Run 'make <board>_defconfig' to create the .config file.""" 1064 1065 cmd = list(self.make_cmd) 1066 cmd.append(self.defconfig) 1067 self.ps = subprocess.Popen(cmd, stdout=self.devnull, 1068 stderr=subprocess.PIPE, 1069 cwd=self.current_src_dir) 1070 self.state = STATE_DEFCONFIG 1071 1072 def do_autoconf(self): 1073 """Run 'make include/config/auto.conf'.""" 1074 1075 self.cross_compile = self.parser.get_cross_compile() 1076 if self.cross_compile is None: 1077 self.log += color_text(self.options.color, COLOR_YELLOW, 1078 "Compiler is missing. Do nothing.\n") 1079 self.finish(False) 1080 return 1081 1082 cmd = list(self.make_cmd) 1083 if self.cross_compile: 1084 cmd.append('CROSS_COMPILE=%s' % self.cross_compile) 1085 cmd.append('KCONFIG_IGNORE_DUPLICATES=1') 1086 cmd.append('include/config/auto.conf') 1087 self.ps = subprocess.Popen(cmd, stdout=self.devnull, 1088 stderr=subprocess.PIPE, 1089 cwd=self.current_src_dir) 1090 self.state = STATE_AUTOCONF 1091 1092 def do_savedefconfig(self): 1093 """Update the .config and run 'make savedefconfig'.""" 1094 1095 (updated, suspicious, log) = self.parser.update_dotconfig() 1096 if suspicious: 1097 self.suspicious_boards.add(self.defconfig) 1098 self.log += log 1099 1100 if not self.options.force_sync and not updated: 1101 self.finish(True) 1102 return 1103 if updated: 1104 self.log += color_text(self.options.color, COLOR_LIGHT_GREEN, 1105 "Syncing by savedefconfig...\n") 1106 else: 1107 self.log += "Syncing by savedefconfig (forced by option)...\n" 1108 1109 cmd = list(self.make_cmd) 1110 cmd.append('savedefconfig') 1111 self.ps = subprocess.Popen(cmd, stdout=self.devnull, 1112 stderr=subprocess.PIPE) 1113 self.state = STATE_SAVEDEFCONFIG 1114 1115 def update_defconfig(self): 1116 """Update the input defconfig and go back to the idle state.""" 1117 1118 log = self.parser.check_defconfig() 1119 if log: 1120 self.suspicious_boards.add(self.defconfig) 1121 self.log += log 1122 orig_defconfig = os.path.join('configs', self.defconfig) 1123 new_defconfig = os.path.join(self.build_dir, 'defconfig') 1124 updated = not filecmp.cmp(orig_defconfig, new_defconfig) 1125 1126 if updated: 1127 self.log += color_text(self.options.color, COLOR_LIGHT_BLUE, 1128 "defconfig was updated.\n") 1129 1130 if not self.options.dry_run and updated: 1131 shutil.move(new_defconfig, orig_defconfig) 1132 self.finish(True) 1133 1134 def finish(self, success): 1135 """Display log along with progress and go to the idle state. 1136 1137 Arguments: 1138 success: Should be True when the defconfig was processed 1139 successfully, or False when it fails. 1140 """ 1141 # output at least 30 characters to hide the "* defconfigs out of *". 1142 log = self.defconfig.ljust(30) + '\n' 1143 1144 log += '\n'.join([ ' ' + s for s in self.log.split('\n') ]) 1145 # Some threads are running in parallel. 1146 # Print log atomically to not mix up logs from different threads. 1147 print >> (sys.stdout if success else sys.stderr), log 1148 1149 if not success: 1150 if self.options.exit_on_error: 1151 sys.exit("Exit on error.") 1152 # If --exit-on-error flag is not set, skip this board and continue. 1153 # Record the failed board. 1154 self.failed_boards.add(self.defconfig) 1155 1156 self.progress.inc() 1157 self.progress.show() 1158 self.state = STATE_IDLE 1159 1160 def get_failed_boards(self): 1161 """Returns a set of failed boards (defconfigs) in this slot. 1162 """ 1163 return self.failed_boards 1164 1165 def get_suspicious_boards(self): 1166 """Returns a set of boards (defconfigs) with possible misconversion. 1167 """ 1168 return self.suspicious_boards - self.failed_boards 1169 1170class Slots: 1171 1172 """Controller of the array of subprocess slots.""" 1173 1174 def __init__(self, configs, options, progress, reference_src_dir): 1175 """Create a new slots controller. 1176 1177 Arguments: 1178 configs: A list of CONFIGs to move. 1179 options: option flags. 1180 progress: A progress indicator. 1181 reference_src_dir: Determine the true starting config state from this 1182 source tree. 1183 """ 1184 self.options = options 1185 self.slots = [] 1186 devnull = get_devnull() 1187 make_cmd = get_make_cmd() 1188 for i in range(options.jobs): 1189 self.slots.append(Slot(configs, options, progress, devnull, 1190 make_cmd, reference_src_dir)) 1191 1192 def add(self, defconfig): 1193 """Add a new subprocess if a vacant slot is found. 1194 1195 Arguments: 1196 defconfig: defconfig name to be put into. 1197 1198 Returns: 1199 Return True on success or False on failure 1200 """ 1201 for slot in self.slots: 1202 if slot.add(defconfig): 1203 return True 1204 return False 1205 1206 def available(self): 1207 """Check if there is a vacant slot. 1208 1209 Returns: 1210 Return True if at lease one vacant slot is found, False otherwise. 1211 """ 1212 for slot in self.slots: 1213 if slot.poll(): 1214 return True 1215 return False 1216 1217 def empty(self): 1218 """Check if all slots are vacant. 1219 1220 Returns: 1221 Return True if all the slots are vacant, False otherwise. 1222 """ 1223 ret = True 1224 for slot in self.slots: 1225 if not slot.poll(): 1226 ret = False 1227 return ret 1228 1229 def show_failed_boards(self): 1230 """Display all of the failed boards (defconfigs).""" 1231 boards = set() 1232 output_file = 'moveconfig.failed' 1233 1234 for slot in self.slots: 1235 boards |= slot.get_failed_boards() 1236 1237 if boards: 1238 boards = '\n'.join(boards) + '\n' 1239 msg = "The following boards were not processed due to error:\n" 1240 msg += boards 1241 msg += "(the list has been saved in %s)\n" % output_file 1242 print >> sys.stderr, color_text(self.options.color, COLOR_LIGHT_RED, 1243 msg) 1244 1245 with open(output_file, 'w') as f: 1246 f.write(boards) 1247 1248 def show_suspicious_boards(self): 1249 """Display all boards (defconfigs) with possible misconversion.""" 1250 boards = set() 1251 output_file = 'moveconfig.suspicious' 1252 1253 for slot in self.slots: 1254 boards |= slot.get_suspicious_boards() 1255 1256 if boards: 1257 boards = '\n'.join(boards) + '\n' 1258 msg = "The following boards might have been converted incorrectly.\n" 1259 msg += "It is highly recommended to check them manually:\n" 1260 msg += boards 1261 msg += "(the list has been saved in %s)\n" % output_file 1262 print >> sys.stderr, color_text(self.options.color, COLOR_YELLOW, 1263 msg) 1264 1265 with open(output_file, 'w') as f: 1266 f.write(boards) 1267 1268class ReferenceSource: 1269 1270 """Reference source against which original configs should be parsed.""" 1271 1272 def __init__(self, commit): 1273 """Create a reference source directory based on a specified commit. 1274 1275 Arguments: 1276 commit: commit to git-clone 1277 """ 1278 self.src_dir = tempfile.mkdtemp() 1279 print "Cloning git repo to a separate work directory..." 1280 subprocess.check_output(['git', 'clone', os.getcwd(), '.'], 1281 cwd=self.src_dir) 1282 print "Checkout '%s' to build the original autoconf.mk." % \ 1283 subprocess.check_output(['git', 'rev-parse', '--short', commit]).strip() 1284 subprocess.check_output(['git', 'checkout', commit], 1285 stderr=subprocess.STDOUT, cwd=self.src_dir) 1286 1287 def __del__(self): 1288 """Delete the reference source directory 1289 1290 This function makes sure the temporary directory is cleaned away 1291 even if Python suddenly dies due to error. It should be done in here 1292 because it is guaranteed the destructor is always invoked when the 1293 instance of the class gets unreferenced. 1294 """ 1295 shutil.rmtree(self.src_dir) 1296 1297 def get_dir(self): 1298 """Return the absolute path to the reference source directory.""" 1299 1300 return self.src_dir 1301 1302def move_config(configs, options): 1303 """Move config options to defconfig files. 1304 1305 Arguments: 1306 configs: A list of CONFIGs to move. 1307 options: option flags 1308 """ 1309 if len(configs) == 0: 1310 if options.force_sync: 1311 print 'No CONFIG is specified. You are probably syncing defconfigs.', 1312 else: 1313 print 'Neither CONFIG nor --force-sync is specified. Nothing will happen.', 1314 else: 1315 print 'Move ' + ', '.join(configs), 1316 print '(jobs: %d)\n' % options.jobs 1317 1318 if options.git_ref: 1319 reference_src = ReferenceSource(options.git_ref) 1320 reference_src_dir = reference_src.get_dir() 1321 else: 1322 reference_src_dir = None 1323 1324 if options.defconfigs: 1325 defconfigs = get_matched_defconfigs(options.defconfigs) 1326 else: 1327 defconfigs = get_all_defconfigs() 1328 1329 progress = Progress(len(defconfigs)) 1330 slots = Slots(configs, options, progress, reference_src_dir) 1331 1332 # Main loop to process defconfig files: 1333 # Add a new subprocess into a vacant slot. 1334 # Sleep if there is no available slot. 1335 for defconfig in defconfigs: 1336 while not slots.add(defconfig): 1337 while not slots.available(): 1338 # No available slot: sleep for a while 1339 time.sleep(SLEEP_TIME) 1340 1341 # wait until all the subprocesses finish 1342 while not slots.empty(): 1343 time.sleep(SLEEP_TIME) 1344 1345 print '' 1346 slots.show_failed_boards() 1347 slots.show_suspicious_boards() 1348 1349def main(): 1350 try: 1351 cpu_count = multiprocessing.cpu_count() 1352 except NotImplementedError: 1353 cpu_count = 1 1354 1355 parser = optparse.OptionParser() 1356 # Add options here 1357 parser.add_option('-c', '--color', action='store_true', default=False, 1358 help='display the log in color') 1359 parser.add_option('-C', '--commit', action='store_true', default=False, 1360 help='Create a git commit for the operation') 1361 parser.add_option('-d', '--defconfigs', type='string', 1362 help='a file containing a list of defconfigs to move, ' 1363 "one per line (for example 'snow_defconfig') " 1364 "or '-' to read from stdin") 1365 parser.add_option('-n', '--dry-run', action='store_true', default=False, 1366 help='perform a trial run (show log with no changes)') 1367 parser.add_option('-e', '--exit-on-error', action='store_true', 1368 default=False, 1369 help='exit immediately on any error') 1370 parser.add_option('-s', '--force-sync', action='store_true', default=False, 1371 help='force sync by savedefconfig') 1372 parser.add_option('-S', '--spl', action='store_true', default=False, 1373 help='parse config options defined for SPL build') 1374 parser.add_option('-H', '--headers-only', dest='cleanup_headers_only', 1375 action='store_true', default=False, 1376 help='only cleanup the headers') 1377 parser.add_option('-j', '--jobs', type='int', default=cpu_count, 1378 help='the number of jobs to run simultaneously') 1379 parser.add_option('-r', '--git-ref', type='string', 1380 help='the git ref to clone for building the autoconf.mk') 1381 parser.add_option('-y', '--yes', action='store_true', default=False, 1382 help="respond 'yes' to any prompts") 1383 parser.add_option('-v', '--verbose', action='store_true', default=False, 1384 help='show any build errors as boards are built') 1385 parser.usage += ' CONFIG ...' 1386 1387 (options, configs) = parser.parse_args() 1388 1389 if len(configs) == 0 and not options.force_sync: 1390 parser.print_usage() 1391 sys.exit(1) 1392 1393 # prefix the option name with CONFIG_ if missing 1394 configs = [ config if config.startswith('CONFIG_') else 'CONFIG_' + config 1395 for config in configs ] 1396 1397 check_top_directory() 1398 1399 if not options.cleanup_headers_only: 1400 check_clean_directory() 1401 update_cross_compile(options.color) 1402 move_config(configs, options) 1403 1404 if configs: 1405 cleanup_headers(configs, options) 1406 cleanup_extra_options(configs, options) 1407 cleanup_whitelist(configs, options) 1408 cleanup_readme(configs, options) 1409 1410 if options.commit: 1411 subprocess.call(['git', 'add', '-u']) 1412 if configs: 1413 msg = 'Convert %s %sto Kconfig' % (configs[0], 1414 'et al ' if len(configs) > 1 else '') 1415 msg += ('\n\nThis converts the following to Kconfig:\n %s\n' % 1416 '\n '.join(configs)) 1417 else: 1418 msg = 'configs: Resync with savedefconfig' 1419 msg += '\n\nRsync all defconfig files using moveconfig.py' 1420 subprocess.call(['git', 'commit', '-s', '-m', msg]) 1421 1422if __name__ == '__main__': 1423 main() 1424