1#!/usr/bin/env python2 2# 3# Author: Masahiro Yamada <yamada.masahiro@socionext.com> 4# 5# SPDX-License-Identifier: GPL-2.0+ 6# 7 8""" 9Move config options from headers to defconfig files. 10 11Since Kconfig was introduced to U-Boot, we have worked on moving 12config options from headers to Kconfig (defconfig). 13 14This tool intends to help this tremendous work. 15 16 17Usage 18----- 19 20First, you must edit the Kconfig to add the menu entries for the configs 21you are moving. 22 23And then run this tool giving CONFIG names you want to move. 24For example, if you want to move CONFIG_CMD_USB and CONFIG_SYS_TEXT_BASE, 25simply type as follows: 26 27 $ tools/moveconfig.py CONFIG_CMD_USB CONFIG_SYS_TEXT_BASE 28 29The tool walks through all the defconfig files and move the given CONFIGs. 30 31The log is also displayed on the terminal. 32 33The log is printed for each defconfig as follows: 34 35<defconfig_name> 36 <action1> 37 <action2> 38 <action3> 39 ... 40 41<defconfig_name> is the name of the defconfig. 42 43<action*> shows what the tool did for that defconfig. 44It looks like one of the following: 45 46 - Move 'CONFIG_... ' 47 This config option was moved to the defconfig 48 49 - CONFIG_... is not defined in Kconfig. Do nothing. 50 The entry for this CONFIG was not found in Kconfig. The option is not 51 defined in the config header, either. So, this case can be just skipped. 52 53 - CONFIG_... is not defined in Kconfig (suspicious). Do nothing. 54 This option is defined in the config header, but its entry was not found 55 in Kconfig. 56 There are two common cases: 57 - You forgot to create an entry for the CONFIG before running 58 this tool, or made a typo in a CONFIG passed to this tool. 59 - The entry was hidden due to unmet 'depends on'. 60 The tool does not know if the result is reasonable, so please check it 61 manually. 62 63 - 'CONFIG_...' is the same as the define in Kconfig. Do nothing. 64 The define in the config header matched the one in Kconfig. 65 We do not need to touch it. 66 67 - Compiler is missing. Do nothing. 68 The compiler specified for this architecture was not found 69 in your PATH environment. 70 (If -e option is passed, the tool exits immediately.) 71 72 - Failed to process. 73 An error occurred during processing this defconfig. Skipped. 74 (If -e option is passed, the tool exits immediately on error.) 75 76Finally, you will be asked, Clean up headers? [y/n]: 77 78If you say 'y' here, the unnecessary config defines are removed 79from the config headers (include/configs/*.h). 80It just uses the regex method, so you should not rely on it. 81Just in case, please do 'git diff' to see what happened. 82 83 84How does it work? 85----------------- 86 87This tool runs configuration and builds include/autoconf.mk for every 88defconfig. The config options defined in Kconfig appear in the .config 89file (unless they are hidden because of unmet dependency.) 90On the other hand, the config options defined by board headers are seen 91in include/autoconf.mk. The tool looks for the specified options in both 92of them to decide the appropriate action for the options. If the given 93config option is found in the .config, but its value does not match the 94one from the board header, the config option in the .config is replaced 95with the define in the board header. Then, the .config is synced by 96"make savedefconfig" and the defconfig is updated with it. 97 98For faster processing, this tool handles multi-threading. It creates 99separate build directories where the out-of-tree build is run. The 100temporary build directories are automatically created and deleted as 101needed. The number of threads are chosen based on the number of the CPU 102cores of your system although you can change it via -j (--jobs) option. 103 104 105Toolchains 106---------- 107 108Appropriate toolchain are necessary to generate include/autoconf.mk 109for all the architectures supported by U-Boot. Most of them are available 110at the kernel.org site, some are not provided by kernel.org. 111 112The default per-arch CROSS_COMPILE used by this tool is specified by 113the list below, CROSS_COMPILE. You may wish to update the list to 114use your own. Instead of modifying the list directly, you can give 115them via environments. 116 117 118Tips and trips 119-------------- 120 121To sync only X86 defconfigs: 122 123 ./tools/moveconfig.py -s -d <(grep -l X86 configs/*) 124 125or: 126 127 grep -l X86 configs/* | ./tools/moveconfig.py -s -d - 128 129To process CONFIG_CMD_FPGAD only for a subset of configs based on path match: 130 131 ls configs/{hrcon*,iocon*,strider*} | \ 132 ./tools/moveconfig.py -Cy CONFIG_CMD_FPGAD -d - 133 134 135Finding implied CONFIGs 136----------------------- 137 138Some CONFIG options can be implied by others and this can help to reduce 139the size of the defconfig files. For example, CONFIG_X86 implies 140CONFIG_CMD_IRQ, so we can put 'imply CMD_IRQ' under 'config X86' and 141all x86 boards will have that option, avoiding adding CONFIG_CMD_IRQ to 142each of the x86 defconfig files. 143 144This tool can help find such configs. To use it, first build a database: 145 146 ./tools/moveconfig.py -b 147 148Then try to query it: 149 150 ./tools/moveconfig.py -i CONFIG_CMD_IRQ 151 CONFIG_CMD_IRQ found in 311/2384 defconfigs 152 44 : CONFIG_SYS_FSL_ERRATUM_IFC_A002769 153 41 : CONFIG_SYS_FSL_ERRATUM_A007075 154 31 : CONFIG_SYS_FSL_DDR_VER_44 155 28 : CONFIG_ARCH_P1010 156 28 : CONFIG_SYS_FSL_ERRATUM_P1010_A003549 157 28 : CONFIG_SYS_FSL_ERRATUM_SEC_A003571 158 28 : CONFIG_SYS_FSL_ERRATUM_IFC_A003399 159 25 : CONFIG_SYS_FSL_ERRATUM_A008044 160 22 : CONFIG_ARCH_P1020 161 21 : CONFIG_SYS_FSL_DDR_VER_46 162 20 : CONFIG_MAX_PIRQ_LINKS 163 20 : CONFIG_HPET_ADDRESS 164 20 : CONFIG_X86 165 20 : CONFIG_PCIE_ECAM_SIZE 166 20 : CONFIG_IRQ_SLOT_COUNT 167 20 : CONFIG_I8259_PIC 168 20 : CONFIG_CPU_ADDR_BITS 169 20 : CONFIG_RAMBASE 170 20 : CONFIG_SYS_FSL_ERRATUM_A005871 171 20 : CONFIG_PCIE_ECAM_BASE 172 20 : CONFIG_X86_TSC_TIMER 173 20 : CONFIG_I8254_TIMER 174 20 : CONFIG_CMD_GETTIME 175 19 : CONFIG_SYS_FSL_ERRATUM_A005812 176 18 : CONFIG_X86_RUN_32BIT 177 17 : CONFIG_CMD_CHIP_CONFIG 178 ... 179 180This shows a list of config options which might imply CONFIG_CMD_EEPROM along 181with how many defconfigs they cover. From this you can see that CONFIG_X86 182implies CONFIG_CMD_EEPROM. Therefore, instead of adding CONFIG_CMD_EEPROM to 183the defconfig of every x86 board, you could add a single imply line to the 184Kconfig file: 185 186 config X86 187 bool "x86 architecture" 188 ... 189 imply CMD_EEPROM 190 191That will cover 20 defconfigs. Many of the options listed are not suitable as 192they are not related. E.g. it would be odd for CONFIG_CMD_GETTIME to imply 193CMD_EEPROM. 194 195Using this search you can reduce the size of moveconfig patches. 196 197 198Available options 199----------------- 200 201 -c, --color 202 Surround each portion of the log with escape sequences to display it 203 in color on the terminal. 204 205 -C, --commit 206 Create a git commit with the changes when the operation is complete. A 207 standard commit message is used which may need to be edited. 208 209 -d, --defconfigs 210 Specify a file containing a list of defconfigs to move. The defconfig 211 files can be given with shell-style wildcards. Use '-' to read from stdin. 212 213 -n, --dry-run 214 Perform a trial run that does not make any changes. It is useful to 215 see what is going to happen before one actually runs it. 216 217 -e, --exit-on-error 218 Exit immediately if Make exits with a non-zero status while processing 219 a defconfig file. 220 221 -s, --force-sync 222 Do "make savedefconfig" forcibly for all the defconfig files. 223 If not specified, "make savedefconfig" only occurs for cases 224 where at least one CONFIG was moved. 225 226 -S, --spl 227 Look for moved config options in spl/include/autoconf.mk instead of 228 include/autoconf.mk. This is useful for moving options for SPL build 229 because SPL related options (mostly prefixed with CONFIG_SPL_) are 230 sometimes blocked by CONFIG_SPL_BUILD ifdef conditionals. 231 232 -H, --headers-only 233 Only cleanup the headers; skip the defconfig processing 234 235 -j, --jobs 236 Specify the number of threads to run simultaneously. If not specified, 237 the number of threads is the same as the number of CPU cores. 238 239 -r, --git-ref 240 Specify the git ref to clone for building the autoconf.mk. If unspecified 241 use the CWD. This is useful for when changes to the Kconfig affect the 242 default values and you want to capture the state of the defconfig from 243 before that change was in effect. If in doubt, specify a ref pre-Kconfig 244 changes (use HEAD if Kconfig changes are not committed). Worst case it will 245 take a bit longer to run, but will always do the right thing. 246 247 -v, --verbose 248 Show any build errors as boards are built 249 250 -y, --yes 251 Instead of prompting, automatically go ahead with all operations. This 252 includes cleaning up headers, CONFIG_SYS_EXTRA_OPTIONS, the config whitelist 253 and the README. 254 255To see the complete list of supported options, run 256 257 $ tools/moveconfig.py -h 258 259""" 260 261import collections 262import copy 263import difflib 264import filecmp 265import fnmatch 266import glob 267import multiprocessing 268import optparse 269import os 270import Queue 271import re 272import shutil 273import subprocess 274import sys 275import tempfile 276import threading 277import time 278 279SHOW_GNU_MAKE = 'scripts/show-gnu-make' 280SLEEP_TIME=0.03 281 282# Here is the list of cross-tools I use. 283# Most of them are available at kernel.org 284# (https://www.kernel.org/pub/tools/crosstool/files/bin/), except the following: 285# arc: https://github.com/foss-for-synopsys-dwc-arc-processors/toolchain/releases 286# nds32: http://osdk.andestech.com/packages/nds32le-linux-glibc-v1.tgz 287# nios2: https://sourcery.mentor.com/GNUToolchain/subscription42545 288# sh: http://sourcery.mentor.com/public/gnu_toolchain/sh-linux-gnu 289CROSS_COMPILE = { 290 'arc': 'arc-linux-', 291 'aarch64': 'aarch64-linux-', 292 'arm': 'arm-unknown-linux-gnueabi-', 293 'm68k': 'm68k-linux-', 294 'microblaze': 'microblaze-linux-', 295 'mips': 'mips-linux-', 296 'nds32': 'nds32le-linux-', 297 'nios2': 'nios2-linux-gnu-', 298 'powerpc': 'powerpc-linux-', 299 'sh': 'sh-linux-gnu-', 300 'x86': 'i386-linux-', 301 'xtensa': 'xtensa-linux-' 302} 303 304STATE_IDLE = 0 305STATE_DEFCONFIG = 1 306STATE_AUTOCONF = 2 307STATE_SAVEDEFCONFIG = 3 308 309ACTION_MOVE = 0 310ACTION_NO_ENTRY = 1 311ACTION_NO_ENTRY_WARN = 2 312ACTION_NO_CHANGE = 3 313 314COLOR_BLACK = '0;30' 315COLOR_RED = '0;31' 316COLOR_GREEN = '0;32' 317COLOR_BROWN = '0;33' 318COLOR_BLUE = '0;34' 319COLOR_PURPLE = '0;35' 320COLOR_CYAN = '0;36' 321COLOR_LIGHT_GRAY = '0;37' 322COLOR_DARK_GRAY = '1;30' 323COLOR_LIGHT_RED = '1;31' 324COLOR_LIGHT_GREEN = '1;32' 325COLOR_YELLOW = '1;33' 326COLOR_LIGHT_BLUE = '1;34' 327COLOR_LIGHT_PURPLE = '1;35' 328COLOR_LIGHT_CYAN = '1;36' 329COLOR_WHITE = '1;37' 330 331AUTO_CONF_PATH = 'include/config/auto.conf' 332CONFIG_DATABASE = 'moveconfig.db' 333 334 335### helper functions ### 336def get_devnull(): 337 """Get the file object of '/dev/null' device.""" 338 try: 339 devnull = subprocess.DEVNULL # py3k 340 except AttributeError: 341 devnull = open(os.devnull, 'wb') 342 return devnull 343 344def check_top_directory(): 345 """Exit if we are not at the top of source directory.""" 346 for f in ('README', 'Licenses'): 347 if not os.path.exists(f): 348 sys.exit('Please run at the top of source directory.') 349 350def check_clean_directory(): 351 """Exit if the source tree is not clean.""" 352 for f in ('.config', 'include/config'): 353 if os.path.exists(f): 354 sys.exit("source tree is not clean, please run 'make mrproper'") 355 356def get_make_cmd(): 357 """Get the command name of GNU Make. 358 359 U-Boot needs GNU Make for building, but the command name is not 360 necessarily "make". (for example, "gmake" on FreeBSD). 361 Returns the most appropriate command name on your system. 362 """ 363 process = subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE) 364 ret = process.communicate() 365 if process.returncode: 366 sys.exit('GNU Make not found') 367 return ret[0].rstrip() 368 369def get_matched_defconfig(line): 370 """Get the defconfig files that match a pattern 371 372 Args: 373 line: Path or filename to match, e.g. 'configs/snow_defconfig' or 374 'k2*_defconfig'. If no directory is provided, 'configs/' is 375 prepended 376 377 Returns: 378 a list of matching defconfig files 379 """ 380 dirname = os.path.dirname(line) 381 if dirname: 382 pattern = line 383 else: 384 pattern = os.path.join('configs', line) 385 return glob.glob(pattern) + glob.glob(pattern + '_defconfig') 386 387def get_matched_defconfigs(defconfigs_file): 388 """Get all the defconfig files that match the patterns in a file. 389 390 Args: 391 defconfigs_file: File containing a list of defconfigs to process, or 392 '-' to read the list from stdin 393 394 Returns: 395 A list of paths to defconfig files, with no duplicates 396 """ 397 defconfigs = [] 398 if defconfigs_file == '-': 399 fd = sys.stdin 400 defconfigs_file = 'stdin' 401 else: 402 fd = open(defconfigs_file) 403 for i, line in enumerate(fd): 404 line = line.strip() 405 if not line: 406 continue # skip blank lines silently 407 if ' ' in line: 408 line = line.split(' ')[0] # handle 'git log' input 409 matched = get_matched_defconfig(line) 410 if not matched: 411 print >> sys.stderr, "warning: %s:%d: no defconfig matched '%s'" % \ 412 (defconfigs_file, i + 1, line) 413 414 defconfigs += matched 415 416 # use set() to drop multiple matching 417 return [ defconfig[len('configs') + 1:] for defconfig in set(defconfigs) ] 418 419def get_all_defconfigs(): 420 """Get all the defconfig files under the configs/ directory.""" 421 defconfigs = [] 422 for (dirpath, dirnames, filenames) in os.walk('configs'): 423 dirpath = dirpath[len('configs') + 1:] 424 for filename in fnmatch.filter(filenames, '*_defconfig'): 425 defconfigs.append(os.path.join(dirpath, filename)) 426 427 return defconfigs 428 429def color_text(color_enabled, color, string): 430 """Return colored string.""" 431 if color_enabled: 432 # LF should not be surrounded by the escape sequence. 433 # Otherwise, additional whitespace or line-feed might be printed. 434 return '\n'.join([ '\033[' + color + 'm' + s + '\033[0m' if s else '' 435 for s in string.split('\n') ]) 436 else: 437 return string 438 439def show_diff(a, b, file_path, color_enabled): 440 """Show unidified diff. 441 442 Arguments: 443 a: A list of lines (before) 444 b: A list of lines (after) 445 file_path: Path to the file 446 color_enabled: Display the diff in color 447 """ 448 449 diff = difflib.unified_diff(a, b, 450 fromfile=os.path.join('a', file_path), 451 tofile=os.path.join('b', file_path)) 452 453 for line in diff: 454 if line[0] == '-' and line[1] != '-': 455 print color_text(color_enabled, COLOR_RED, line), 456 elif line[0] == '+' and line[1] != '+': 457 print color_text(color_enabled, COLOR_GREEN, line), 458 else: 459 print line, 460 461def update_cross_compile(color_enabled): 462 """Update per-arch CROSS_COMPILE via environment variables 463 464 The default CROSS_COMPILE values are available 465 in the CROSS_COMPILE list above. 466 467 You can override them via environment variables 468 CROSS_COMPILE_{ARCH}. 469 470 For example, if you want to override toolchain prefixes 471 for ARM and PowerPC, you can do as follows in your shell: 472 473 export CROSS_COMPILE_ARM=... 474 export CROSS_COMPILE_POWERPC=... 475 476 Then, this function checks if specified compilers really exist in your 477 PATH environment. 478 """ 479 archs = [] 480 481 for arch in os.listdir('arch'): 482 if os.path.exists(os.path.join('arch', arch, 'Makefile')): 483 archs.append(arch) 484 485 # arm64 is a special case 486 archs.append('aarch64') 487 488 for arch in archs: 489 env = 'CROSS_COMPILE_' + arch.upper() 490 cross_compile = os.environ.get(env) 491 if not cross_compile: 492 cross_compile = CROSS_COMPILE.get(arch, '') 493 494 for path in os.environ["PATH"].split(os.pathsep): 495 gcc_path = os.path.join(path, cross_compile + 'gcc') 496 if os.path.isfile(gcc_path) and os.access(gcc_path, os.X_OK): 497 break 498 else: 499 print >> sys.stderr, color_text(color_enabled, COLOR_YELLOW, 500 'warning: %sgcc: not found in PATH. %s architecture boards will be skipped' 501 % (cross_compile, arch)) 502 cross_compile = None 503 504 CROSS_COMPILE[arch] = cross_compile 505 506def extend_matched_lines(lines, matched, pre_patterns, post_patterns, extend_pre, 507 extend_post): 508 """Extend matched lines if desired patterns are found before/after already 509 matched lines. 510 511 Arguments: 512 lines: A list of lines handled. 513 matched: A list of line numbers that have been already matched. 514 (will be updated by this function) 515 pre_patterns: A list of regular expression that should be matched as 516 preamble. 517 post_patterns: A list of regular expression that should be matched as 518 postamble. 519 extend_pre: Add the line number of matched preamble to the matched list. 520 extend_post: Add the line number of matched postamble to the matched list. 521 """ 522 extended_matched = [] 523 524 j = matched[0] 525 526 for i in matched: 527 if i == 0 or i < j: 528 continue 529 j = i 530 while j in matched: 531 j += 1 532 if j >= len(lines): 533 break 534 535 for p in pre_patterns: 536 if p.search(lines[i - 1]): 537 break 538 else: 539 # not matched 540 continue 541 542 for p in post_patterns: 543 if p.search(lines[j]): 544 break 545 else: 546 # not matched 547 continue 548 549 if extend_pre: 550 extended_matched.append(i - 1) 551 if extend_post: 552 extended_matched.append(j) 553 554 matched += extended_matched 555 matched.sort() 556 557def confirm(options, prompt): 558 if not options.yes: 559 while True: 560 choice = raw_input('{} [y/n]: '.format(prompt)) 561 choice = choice.lower() 562 print choice 563 if choice == 'y' or choice == 'n': 564 break 565 566 if choice == 'n': 567 return False 568 569 return True 570 571def cleanup_one_header(header_path, patterns, options): 572 """Clean regex-matched lines away from a file. 573 574 Arguments: 575 header_path: path to the cleaned file. 576 patterns: list of regex patterns. Any lines matching to these 577 patterns are deleted. 578 options: option flags. 579 """ 580 with open(header_path) as f: 581 lines = f.readlines() 582 583 matched = [] 584 for i, line in enumerate(lines): 585 if i - 1 in matched and lines[i - 1][-2:] == '\\\n': 586 matched.append(i) 587 continue 588 for pattern in patterns: 589 if pattern.search(line): 590 matched.append(i) 591 break 592 593 if not matched: 594 return 595 596 # remove empty #ifdef ... #endif, successive blank lines 597 pattern_if = re.compile(r'#\s*if(def|ndef)?\W') # #if, #ifdef, #ifndef 598 pattern_elif = re.compile(r'#\s*el(if|se)\W') # #elif, #else 599 pattern_endif = re.compile(r'#\s*endif\W') # #endif 600 pattern_blank = re.compile(r'^\s*$') # empty line 601 602 while True: 603 old_matched = copy.copy(matched) 604 extend_matched_lines(lines, matched, [pattern_if], 605 [pattern_endif], True, True) 606 extend_matched_lines(lines, matched, [pattern_elif], 607 [pattern_elif, pattern_endif], True, False) 608 extend_matched_lines(lines, matched, [pattern_if, pattern_elif], 609 [pattern_blank], False, True) 610 extend_matched_lines(lines, matched, [pattern_blank], 611 [pattern_elif, pattern_endif], True, False) 612 extend_matched_lines(lines, matched, [pattern_blank], 613 [pattern_blank], True, False) 614 if matched == old_matched: 615 break 616 617 tolines = copy.copy(lines) 618 619 for i in reversed(matched): 620 tolines.pop(i) 621 622 show_diff(lines, tolines, header_path, options.color) 623 624 if options.dry_run: 625 return 626 627 with open(header_path, 'w') as f: 628 for line in tolines: 629 f.write(line) 630 631def cleanup_headers(configs, options): 632 """Delete config defines from board headers. 633 634 Arguments: 635 configs: A list of CONFIGs to remove. 636 options: option flags. 637 """ 638 if not confirm(options, 'Clean up headers?'): 639 return 640 641 patterns = [] 642 for config in configs: 643 patterns.append(re.compile(r'#\s*define\s+%s\W' % config)) 644 patterns.append(re.compile(r'#\s*undef\s+%s\W' % config)) 645 646 for dir in 'include', 'arch', 'board': 647 for (dirpath, dirnames, filenames) in os.walk(dir): 648 if dirpath == os.path.join('include', 'generated'): 649 continue 650 for filename in filenames: 651 if not fnmatch.fnmatch(filename, '*~'): 652 cleanup_one_header(os.path.join(dirpath, filename), 653 patterns, options) 654 655def cleanup_one_extra_option(defconfig_path, configs, options): 656 """Delete config defines in CONFIG_SYS_EXTRA_OPTIONS in one defconfig file. 657 658 Arguments: 659 defconfig_path: path to the cleaned defconfig file. 660 configs: A list of CONFIGs to remove. 661 options: option flags. 662 """ 663 664 start = 'CONFIG_SYS_EXTRA_OPTIONS="' 665 end = '"\n' 666 667 with open(defconfig_path) as f: 668 lines = f.readlines() 669 670 for i, line in enumerate(lines): 671 if line.startswith(start) and line.endswith(end): 672 break 673 else: 674 # CONFIG_SYS_EXTRA_OPTIONS was not found in this defconfig 675 return 676 677 old_tokens = line[len(start):-len(end)].split(',') 678 new_tokens = [] 679 680 for token in old_tokens: 681 pos = token.find('=') 682 if not (token[:pos] if pos >= 0 else token) in configs: 683 new_tokens.append(token) 684 685 if new_tokens == old_tokens: 686 return 687 688 tolines = copy.copy(lines) 689 690 if new_tokens: 691 tolines[i] = start + ','.join(new_tokens) + end 692 else: 693 tolines.pop(i) 694 695 show_diff(lines, tolines, defconfig_path, options.color) 696 697 if options.dry_run: 698 return 699 700 with open(defconfig_path, 'w') as f: 701 for line in tolines: 702 f.write(line) 703 704def cleanup_extra_options(configs, options): 705 """Delete config defines in CONFIG_SYS_EXTRA_OPTIONS in defconfig files. 706 707 Arguments: 708 configs: A list of CONFIGs to remove. 709 options: option flags. 710 """ 711 if not confirm(options, 'Clean up CONFIG_SYS_EXTRA_OPTIONS?'): 712 return 713 714 configs = [ config[len('CONFIG_'):] for config in configs ] 715 716 defconfigs = get_all_defconfigs() 717 718 for defconfig in defconfigs: 719 cleanup_one_extra_option(os.path.join('configs', defconfig), configs, 720 options) 721 722def cleanup_whitelist(configs, options): 723 """Delete config whitelist entries 724 725 Arguments: 726 configs: A list of CONFIGs to remove. 727 options: option flags. 728 """ 729 if not confirm(options, 'Clean up whitelist entries?'): 730 return 731 732 with open(os.path.join('scripts', 'config_whitelist.txt')) as f: 733 lines = f.readlines() 734 735 lines = [x for x in lines if x.strip() not in configs] 736 737 with open(os.path.join('scripts', 'config_whitelist.txt'), 'w') as f: 738 f.write(''.join(lines)) 739 740def find_matching(patterns, line): 741 for pat in patterns: 742 if pat.search(line): 743 return True 744 return False 745 746def cleanup_readme(configs, options): 747 """Delete config description in README 748 749 Arguments: 750 configs: A list of CONFIGs to remove. 751 options: option flags. 752 """ 753 if not confirm(options, 'Clean up README?'): 754 return 755 756 patterns = [] 757 for config in configs: 758 patterns.append(re.compile(r'^\s+%s' % config)) 759 760 with open('README') as f: 761 lines = f.readlines() 762 763 found = False 764 newlines = [] 765 for line in lines: 766 if not found: 767 found = find_matching(patterns, line) 768 if found: 769 continue 770 771 if found and re.search(r'^\s+CONFIG', line): 772 found = False 773 774 if not found: 775 newlines.append(line) 776 777 with open('README', 'w') as f: 778 f.write(''.join(newlines)) 779 780 781### classes ### 782class Progress: 783 784 """Progress Indicator""" 785 786 def __init__(self, total): 787 """Create a new progress indicator. 788 789 Arguments: 790 total: A number of defconfig files to process. 791 """ 792 self.current = 0 793 self.total = total 794 795 def inc(self): 796 """Increment the number of processed defconfig files.""" 797 798 self.current += 1 799 800 def show(self): 801 """Display the progress.""" 802 print ' %d defconfigs out of %d\r' % (self.current, self.total), 803 sys.stdout.flush() 804 805class KconfigParser: 806 807 """A parser of .config and include/autoconf.mk.""" 808 809 re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"') 810 re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"') 811 812 def __init__(self, configs, options, build_dir): 813 """Create a new parser. 814 815 Arguments: 816 configs: A list of CONFIGs to move. 817 options: option flags. 818 build_dir: Build directory. 819 """ 820 self.configs = configs 821 self.options = options 822 self.dotconfig = os.path.join(build_dir, '.config') 823 self.autoconf = os.path.join(build_dir, 'include', 'autoconf.mk') 824 self.spl_autoconf = os.path.join(build_dir, 'spl', 'include', 825 'autoconf.mk') 826 self.config_autoconf = os.path.join(build_dir, AUTO_CONF_PATH) 827 self.defconfig = os.path.join(build_dir, 'defconfig') 828 829 def get_cross_compile(self): 830 """Parse .config file and return CROSS_COMPILE. 831 832 Returns: 833 A string storing the compiler prefix for the architecture. 834 Return a NULL string for architectures that do not require 835 compiler prefix (Sandbox and native build is the case). 836 Return None if the specified compiler is missing in your PATH. 837 Caller should distinguish '' and None. 838 """ 839 arch = '' 840 cpu = '' 841 for line in open(self.dotconfig): 842 m = self.re_arch.match(line) 843 if m: 844 arch = m.group(1) 845 continue 846 m = self.re_cpu.match(line) 847 if m: 848 cpu = m.group(1) 849 850 if not arch: 851 return None 852 853 # fix-up for aarch64 854 if arch == 'arm' and cpu == 'armv8': 855 arch = 'aarch64' 856 857 return CROSS_COMPILE.get(arch, None) 858 859 def parse_one_config(self, config, dotconfig_lines, autoconf_lines): 860 """Parse .config, defconfig, include/autoconf.mk for one config. 861 862 This function looks for the config options in the lines from 863 defconfig, .config, and include/autoconf.mk in order to decide 864 which action should be taken for this defconfig. 865 866 Arguments: 867 config: CONFIG name to parse. 868 dotconfig_lines: lines from the .config file. 869 autoconf_lines: lines from the include/autoconf.mk file. 870 871 Returns: 872 A tupple of the action for this defconfig and the line 873 matched for the config. 874 """ 875 not_set = '# %s is not set' % config 876 877 for line in autoconf_lines: 878 line = line.rstrip() 879 if line.startswith(config + '='): 880 new_val = line 881 break 882 else: 883 new_val = not_set 884 885 for line in dotconfig_lines: 886 line = line.rstrip() 887 if line.startswith(config + '=') or line == not_set: 888 old_val = line 889 break 890 else: 891 if new_val == not_set: 892 return (ACTION_NO_ENTRY, config) 893 else: 894 return (ACTION_NO_ENTRY_WARN, config) 895 896 # If this CONFIG is neither bool nor trisate 897 if old_val[-2:] != '=y' and old_val[-2:] != '=m' and old_val != not_set: 898 # tools/scripts/define2mk.sed changes '1' to 'y'. 899 # This is a problem if the CONFIG is int type. 900 # Check the type in Kconfig and handle it correctly. 901 if new_val[-2:] == '=y': 902 new_val = new_val[:-1] + '1' 903 904 return (ACTION_NO_CHANGE if old_val == new_val else ACTION_MOVE, 905 new_val) 906 907 def update_dotconfig(self): 908 """Parse files for the config options and update the .config. 909 910 This function parses the generated .config and include/autoconf.mk 911 searching the target options. 912 Move the config option(s) to the .config as needed. 913 914 Arguments: 915 defconfig: defconfig name. 916 917 Returns: 918 Return a tuple of (updated flag, log string). 919 The "updated flag" is True if the .config was updated, False 920 otherwise. The "log string" shows what happend to the .config. 921 """ 922 923 results = [] 924 updated = False 925 suspicious = False 926 rm_files = [self.config_autoconf, self.autoconf] 927 928 if self.options.spl: 929 if os.path.exists(self.spl_autoconf): 930 autoconf_path = self.spl_autoconf 931 rm_files.append(self.spl_autoconf) 932 else: 933 for f in rm_files: 934 os.remove(f) 935 return (updated, suspicious, 936 color_text(self.options.color, COLOR_BROWN, 937 "SPL is not enabled. Skipped.") + '\n') 938 else: 939 autoconf_path = self.autoconf 940 941 with open(self.dotconfig) as f: 942 dotconfig_lines = f.readlines() 943 944 with open(autoconf_path) as f: 945 autoconf_lines = f.readlines() 946 947 for config in self.configs: 948 result = self.parse_one_config(config, dotconfig_lines, 949 autoconf_lines) 950 results.append(result) 951 952 log = '' 953 954 for (action, value) in results: 955 if action == ACTION_MOVE: 956 actlog = "Move '%s'" % value 957 log_color = COLOR_LIGHT_GREEN 958 elif action == ACTION_NO_ENTRY: 959 actlog = "%s is not defined in Kconfig. Do nothing." % value 960 log_color = COLOR_LIGHT_BLUE 961 elif action == ACTION_NO_ENTRY_WARN: 962 actlog = "%s is not defined in Kconfig (suspicious). Do nothing." % value 963 log_color = COLOR_YELLOW 964 suspicious = True 965 elif action == ACTION_NO_CHANGE: 966 actlog = "'%s' is the same as the define in Kconfig. Do nothing." \ 967 % value 968 log_color = COLOR_LIGHT_PURPLE 969 elif action == ACTION_SPL_NOT_EXIST: 970 actlog = "SPL is not enabled for this defconfig. Skip." 971 log_color = COLOR_PURPLE 972 else: 973 sys.exit("Internal Error. This should not happen.") 974 975 log += color_text(self.options.color, log_color, actlog) + '\n' 976 977 with open(self.dotconfig, 'a') as f: 978 for (action, value) in results: 979 if action == ACTION_MOVE: 980 f.write(value + '\n') 981 updated = True 982 983 self.results = results 984 for f in rm_files: 985 os.remove(f) 986 987 return (updated, suspicious, log) 988 989 def check_defconfig(self): 990 """Check the defconfig after savedefconfig 991 992 Returns: 993 Return additional log if moved CONFIGs were removed again by 994 'make savedefconfig'. 995 """ 996 997 log = '' 998 999 with open(self.defconfig) as f: 1000 defconfig_lines = f.readlines() 1001 1002 for (action, value) in self.results: 1003 if action != ACTION_MOVE: 1004 continue 1005 if not value + '\n' in defconfig_lines: 1006 log += color_text(self.options.color, COLOR_YELLOW, 1007 "'%s' was removed by savedefconfig.\n" % 1008 value) 1009 1010 return log 1011 1012 1013class DatabaseThread(threading.Thread): 1014 """This thread processes results from Slot threads. 1015 1016 It collects the data in the master config directary. There is only one 1017 result thread, and this helps to serialise the build output. 1018 """ 1019 def __init__(self, config_db, db_queue): 1020 """Set up a new result thread 1021 1022 Args: 1023 builder: Builder which will be sent each result 1024 """ 1025 threading.Thread.__init__(self) 1026 self.config_db = config_db 1027 self.db_queue= db_queue 1028 1029 def run(self): 1030 """Called to start up the result thread. 1031 1032 We collect the next result job and pass it on to the build. 1033 """ 1034 while True: 1035 defconfig, configs = self.db_queue.get() 1036 self.config_db[defconfig] = configs 1037 self.db_queue.task_done() 1038 1039 1040class Slot: 1041 1042 """A slot to store a subprocess. 1043 1044 Each instance of this class handles one subprocess. 1045 This class is useful to control multiple threads 1046 for faster processing. 1047 """ 1048 1049 def __init__(self, configs, options, progress, devnull, make_cmd, 1050 reference_src_dir, db_queue): 1051 """Create a new process slot. 1052 1053 Arguments: 1054 configs: A list of CONFIGs to move. 1055 options: option flags. 1056 progress: A progress indicator. 1057 devnull: A file object of '/dev/null'. 1058 make_cmd: command name of GNU Make. 1059 reference_src_dir: Determine the true starting config state from this 1060 source tree. 1061 db_queue: output queue to write config info for the database 1062 """ 1063 self.options = options 1064 self.progress = progress 1065 self.build_dir = tempfile.mkdtemp() 1066 self.devnull = devnull 1067 self.make_cmd = (make_cmd, 'O=' + self.build_dir) 1068 self.reference_src_dir = reference_src_dir 1069 self.db_queue = db_queue 1070 self.parser = KconfigParser(configs, options, self.build_dir) 1071 self.state = STATE_IDLE 1072 self.failed_boards = set() 1073 self.suspicious_boards = set() 1074 1075 def __del__(self): 1076 """Delete the working directory 1077 1078 This function makes sure the temporary directory is cleaned away 1079 even if Python suddenly dies due to error. It should be done in here 1080 because it is guaranteed the destructor is always invoked when the 1081 instance of the class gets unreferenced. 1082 1083 If the subprocess is still running, wait until it finishes. 1084 """ 1085 if self.state != STATE_IDLE: 1086 while self.ps.poll() == None: 1087 pass 1088 shutil.rmtree(self.build_dir) 1089 1090 def add(self, defconfig): 1091 """Assign a new subprocess for defconfig and add it to the slot. 1092 1093 If the slot is vacant, create a new subprocess for processing the 1094 given defconfig and add it to the slot. Just returns False if 1095 the slot is occupied (i.e. the current subprocess is still running). 1096 1097 Arguments: 1098 defconfig: defconfig name. 1099 1100 Returns: 1101 Return True on success or False on failure 1102 """ 1103 if self.state != STATE_IDLE: 1104 return False 1105 1106 self.defconfig = defconfig 1107 self.log = '' 1108 self.current_src_dir = self.reference_src_dir 1109 self.do_defconfig() 1110 return True 1111 1112 def poll(self): 1113 """Check the status of the subprocess and handle it as needed. 1114 1115 Returns True if the slot is vacant (i.e. in idle state). 1116 If the configuration is successfully finished, assign a new 1117 subprocess to build include/autoconf.mk. 1118 If include/autoconf.mk is generated, invoke the parser to 1119 parse the .config and the include/autoconf.mk, moving 1120 config options to the .config as needed. 1121 If the .config was updated, run "make savedefconfig" to sync 1122 it, update the original defconfig, and then set the slot back 1123 to the idle state. 1124 1125 Returns: 1126 Return True if the subprocess is terminated, False otherwise 1127 """ 1128 if self.state == STATE_IDLE: 1129 return True 1130 1131 if self.ps.poll() == None: 1132 return False 1133 1134 if self.ps.poll() != 0: 1135 self.handle_error() 1136 elif self.state == STATE_DEFCONFIG: 1137 if self.reference_src_dir and not self.current_src_dir: 1138 self.do_savedefconfig() 1139 else: 1140 self.do_autoconf() 1141 elif self.state == STATE_AUTOCONF: 1142 if self.current_src_dir: 1143 self.current_src_dir = None 1144 self.do_defconfig() 1145 elif self.options.build_db: 1146 self.do_build_db() 1147 else: 1148 self.do_savedefconfig() 1149 elif self.state == STATE_SAVEDEFCONFIG: 1150 self.update_defconfig() 1151 else: 1152 sys.exit("Internal Error. This should not happen.") 1153 1154 return True if self.state == STATE_IDLE else False 1155 1156 def handle_error(self): 1157 """Handle error cases.""" 1158 1159 self.log += color_text(self.options.color, COLOR_LIGHT_RED, 1160 "Failed to process.\n") 1161 if self.options.verbose: 1162 self.log += color_text(self.options.color, COLOR_LIGHT_CYAN, 1163 self.ps.stderr.read()) 1164 self.finish(False) 1165 1166 def do_defconfig(self): 1167 """Run 'make <board>_defconfig' to create the .config file.""" 1168 1169 cmd = list(self.make_cmd) 1170 cmd.append(self.defconfig) 1171 self.ps = subprocess.Popen(cmd, stdout=self.devnull, 1172 stderr=subprocess.PIPE, 1173 cwd=self.current_src_dir) 1174 self.state = STATE_DEFCONFIG 1175 1176 def do_autoconf(self): 1177 """Run 'make AUTO_CONF_PATH'.""" 1178 1179 self.cross_compile = self.parser.get_cross_compile() 1180 if self.cross_compile is None: 1181 self.log += color_text(self.options.color, COLOR_YELLOW, 1182 "Compiler is missing. Do nothing.\n") 1183 self.finish(False) 1184 return 1185 1186 cmd = list(self.make_cmd) 1187 if self.cross_compile: 1188 cmd.append('CROSS_COMPILE=%s' % self.cross_compile) 1189 cmd.append('KCONFIG_IGNORE_DUPLICATES=1') 1190 cmd.append(AUTO_CONF_PATH) 1191 self.ps = subprocess.Popen(cmd, stdout=self.devnull, 1192 stderr=subprocess.PIPE, 1193 cwd=self.current_src_dir) 1194 self.state = STATE_AUTOCONF 1195 1196 def do_build_db(self): 1197 """Add the board to the database""" 1198 configs = {} 1199 with open(os.path.join(self.build_dir, AUTO_CONF_PATH)) as fd: 1200 for line in fd.readlines(): 1201 if line.startswith('CONFIG'): 1202 config, value = line.split('=', 1) 1203 configs[config] = value.rstrip() 1204 self.db_queue.put([self.defconfig, configs]) 1205 self.finish(True) 1206 1207 def do_savedefconfig(self): 1208 """Update the .config and run 'make savedefconfig'.""" 1209 1210 (updated, suspicious, log) = self.parser.update_dotconfig() 1211 if suspicious: 1212 self.suspicious_boards.add(self.defconfig) 1213 self.log += log 1214 1215 if not self.options.force_sync and not updated: 1216 self.finish(True) 1217 return 1218 if updated: 1219 self.log += color_text(self.options.color, COLOR_LIGHT_GREEN, 1220 "Syncing by savedefconfig...\n") 1221 else: 1222 self.log += "Syncing by savedefconfig (forced by option)...\n" 1223 1224 cmd = list(self.make_cmd) 1225 cmd.append('savedefconfig') 1226 self.ps = subprocess.Popen(cmd, stdout=self.devnull, 1227 stderr=subprocess.PIPE) 1228 self.state = STATE_SAVEDEFCONFIG 1229 1230 def update_defconfig(self): 1231 """Update the input defconfig and go back to the idle state.""" 1232 1233 log = self.parser.check_defconfig() 1234 if log: 1235 self.suspicious_boards.add(self.defconfig) 1236 self.log += log 1237 orig_defconfig = os.path.join('configs', self.defconfig) 1238 new_defconfig = os.path.join(self.build_dir, 'defconfig') 1239 updated = not filecmp.cmp(orig_defconfig, new_defconfig) 1240 1241 if updated: 1242 self.log += color_text(self.options.color, COLOR_LIGHT_BLUE, 1243 "defconfig was updated.\n") 1244 1245 if not self.options.dry_run and updated: 1246 shutil.move(new_defconfig, orig_defconfig) 1247 self.finish(True) 1248 1249 def finish(self, success): 1250 """Display log along with progress and go to the idle state. 1251 1252 Arguments: 1253 success: Should be True when the defconfig was processed 1254 successfully, or False when it fails. 1255 """ 1256 # output at least 30 characters to hide the "* defconfigs out of *". 1257 log = self.defconfig.ljust(30) + '\n' 1258 1259 log += '\n'.join([ ' ' + s for s in self.log.split('\n') ]) 1260 # Some threads are running in parallel. 1261 # Print log atomically to not mix up logs from different threads. 1262 print >> (sys.stdout if success else sys.stderr), log 1263 1264 if not success: 1265 if self.options.exit_on_error: 1266 sys.exit("Exit on error.") 1267 # If --exit-on-error flag is not set, skip this board and continue. 1268 # Record the failed board. 1269 self.failed_boards.add(self.defconfig) 1270 1271 self.progress.inc() 1272 self.progress.show() 1273 self.state = STATE_IDLE 1274 1275 def get_failed_boards(self): 1276 """Returns a set of failed boards (defconfigs) in this slot. 1277 """ 1278 return self.failed_boards 1279 1280 def get_suspicious_boards(self): 1281 """Returns a set of boards (defconfigs) with possible misconversion. 1282 """ 1283 return self.suspicious_boards - self.failed_boards 1284 1285class Slots: 1286 1287 """Controller of the array of subprocess slots.""" 1288 1289 def __init__(self, configs, options, progress, reference_src_dir, db_queue): 1290 """Create a new slots controller. 1291 1292 Arguments: 1293 configs: A list of CONFIGs to move. 1294 options: option flags. 1295 progress: A progress indicator. 1296 reference_src_dir: Determine the true starting config state from this 1297 source tree. 1298 db_queue: output queue to write config info for the database 1299 """ 1300 self.options = options 1301 self.slots = [] 1302 devnull = get_devnull() 1303 make_cmd = get_make_cmd() 1304 for i in range(options.jobs): 1305 self.slots.append(Slot(configs, options, progress, devnull, 1306 make_cmd, reference_src_dir, db_queue)) 1307 1308 def add(self, defconfig): 1309 """Add a new subprocess if a vacant slot is found. 1310 1311 Arguments: 1312 defconfig: defconfig name to be put into. 1313 1314 Returns: 1315 Return True on success or False on failure 1316 """ 1317 for slot in self.slots: 1318 if slot.add(defconfig): 1319 return True 1320 return False 1321 1322 def available(self): 1323 """Check if there is a vacant slot. 1324 1325 Returns: 1326 Return True if at lease one vacant slot is found, False otherwise. 1327 """ 1328 for slot in self.slots: 1329 if slot.poll(): 1330 return True 1331 return False 1332 1333 def empty(self): 1334 """Check if all slots are vacant. 1335 1336 Returns: 1337 Return True if all the slots are vacant, False otherwise. 1338 """ 1339 ret = True 1340 for slot in self.slots: 1341 if not slot.poll(): 1342 ret = False 1343 return ret 1344 1345 def show_failed_boards(self): 1346 """Display all of the failed boards (defconfigs).""" 1347 boards = set() 1348 output_file = 'moveconfig.failed' 1349 1350 for slot in self.slots: 1351 boards |= slot.get_failed_boards() 1352 1353 if boards: 1354 boards = '\n'.join(boards) + '\n' 1355 msg = "The following boards were not processed due to error:\n" 1356 msg += boards 1357 msg += "(the list has been saved in %s)\n" % output_file 1358 print >> sys.stderr, color_text(self.options.color, COLOR_LIGHT_RED, 1359 msg) 1360 1361 with open(output_file, 'w') as f: 1362 f.write(boards) 1363 1364 def show_suspicious_boards(self): 1365 """Display all boards (defconfigs) with possible misconversion.""" 1366 boards = set() 1367 output_file = 'moveconfig.suspicious' 1368 1369 for slot in self.slots: 1370 boards |= slot.get_suspicious_boards() 1371 1372 if boards: 1373 boards = '\n'.join(boards) + '\n' 1374 msg = "The following boards might have been converted incorrectly.\n" 1375 msg += "It is highly recommended to check them manually:\n" 1376 msg += boards 1377 msg += "(the list has been saved in %s)\n" % output_file 1378 print >> sys.stderr, color_text(self.options.color, COLOR_YELLOW, 1379 msg) 1380 1381 with open(output_file, 'w') as f: 1382 f.write(boards) 1383 1384class ReferenceSource: 1385 1386 """Reference source against which original configs should be parsed.""" 1387 1388 def __init__(self, commit): 1389 """Create a reference source directory based on a specified commit. 1390 1391 Arguments: 1392 commit: commit to git-clone 1393 """ 1394 self.src_dir = tempfile.mkdtemp() 1395 print "Cloning git repo to a separate work directory..." 1396 subprocess.check_output(['git', 'clone', os.getcwd(), '.'], 1397 cwd=self.src_dir) 1398 print "Checkout '%s' to build the original autoconf.mk." % \ 1399 subprocess.check_output(['git', 'rev-parse', '--short', commit]).strip() 1400 subprocess.check_output(['git', 'checkout', commit], 1401 stderr=subprocess.STDOUT, cwd=self.src_dir) 1402 1403 def __del__(self): 1404 """Delete the reference source directory 1405 1406 This function makes sure the temporary directory is cleaned away 1407 even if Python suddenly dies due to error. It should be done in here 1408 because it is guaranteed the destructor is always invoked when the 1409 instance of the class gets unreferenced. 1410 """ 1411 shutil.rmtree(self.src_dir) 1412 1413 def get_dir(self): 1414 """Return the absolute path to the reference source directory.""" 1415 1416 return self.src_dir 1417 1418def move_config(configs, options, db_queue): 1419 """Move config options to defconfig files. 1420 1421 Arguments: 1422 configs: A list of CONFIGs to move. 1423 options: option flags 1424 """ 1425 if len(configs) == 0: 1426 if options.force_sync: 1427 print 'No CONFIG is specified. You are probably syncing defconfigs.', 1428 elif options.build_db: 1429 print 'Building %s database' % CONFIG_DATABASE 1430 else: 1431 print 'Neither CONFIG nor --force-sync is specified. Nothing will happen.', 1432 else: 1433 print 'Move ' + ', '.join(configs), 1434 print '(jobs: %d)\n' % options.jobs 1435 1436 if options.git_ref: 1437 reference_src = ReferenceSource(options.git_ref) 1438 reference_src_dir = reference_src.get_dir() 1439 else: 1440 reference_src_dir = None 1441 1442 if options.defconfigs: 1443 defconfigs = get_matched_defconfigs(options.defconfigs) 1444 else: 1445 defconfigs = get_all_defconfigs() 1446 1447 progress = Progress(len(defconfigs)) 1448 slots = Slots(configs, options, progress, reference_src_dir, db_queue) 1449 1450 # Main loop to process defconfig files: 1451 # Add a new subprocess into a vacant slot. 1452 # Sleep if there is no available slot. 1453 for defconfig in defconfigs: 1454 while not slots.add(defconfig): 1455 while not slots.available(): 1456 # No available slot: sleep for a while 1457 time.sleep(SLEEP_TIME) 1458 1459 # wait until all the subprocesses finish 1460 while not slots.empty(): 1461 time.sleep(SLEEP_TIME) 1462 1463 print '' 1464 slots.show_failed_boards() 1465 slots.show_suspicious_boards() 1466 1467def imply_config(config_list, find_superset=False): 1468 """Find CONFIG options which imply those in the list 1469 1470 Some CONFIG options can be implied by others and this can help to reduce 1471 the size of the defconfig files. For example, CONFIG_X86 implies 1472 CONFIG_CMD_IRQ, so we can put 'imply CMD_IRQ' under 'config X86' and 1473 all x86 boards will have that option, avoiding adding CONFIG_CMD_IRQ to 1474 each of the x86 defconfig files. 1475 1476 This function uses the moveconfig database to find such options. It 1477 displays a list of things that could possibly imply those in the list. 1478 The algorithm ignores any that start with CONFIG_TARGET since these 1479 typically refer to only a few defconfigs (often one). It also does not 1480 display a config with less than 5 defconfigs. 1481 1482 The algorithm works using sets. For each target config in config_list: 1483 - Get the set 'defconfigs' which use that target config 1484 - For each config (from a list of all configs): 1485 - Get the set 'imply_defconfig' of defconfigs which use that config 1486 - 1487 - If imply_defconfigs contains anything not in defconfigs then 1488 this config does not imply the target config 1489 1490 Params: 1491 config_list: List of CONFIG options to check (each a string) 1492 find_superset: True to look for configs which are a superset of those 1493 already found. So for example if CONFIG_EXYNOS5 implies an option, 1494 but CONFIG_EXYNOS covers a larger set of defconfigs and also 1495 implies that option, this will drop the former in favour of the 1496 latter. In practice this option has not proved very used. 1497 1498 Note the terminoloy: 1499 config - a CONFIG_XXX options (a string, e.g. 'CONFIG_CMD_EEPROM') 1500 defconfig - a defconfig file (a string, e.g. 'configs/snow_defconfig') 1501 """ 1502 # key is defconfig name, value is dict of (CONFIG_xxx, value) 1503 config_db = {} 1504 1505 # Holds a dict containing the set of defconfigs that contain each config 1506 # key is config, value is set of defconfigs using that config 1507 defconfig_db = collections.defaultdict(set) 1508 1509 # Set of all config options we have seen 1510 all_configs = set() 1511 1512 # Set of all defconfigs we have seen 1513 all_defconfigs = set() 1514 1515 # Read in the database 1516 configs = {} 1517 with open(CONFIG_DATABASE) as fd: 1518 for line in fd.readlines(): 1519 line = line.rstrip() 1520 if not line: # Separator between defconfigs 1521 config_db[defconfig] = configs 1522 all_defconfigs.add(defconfig) 1523 configs = {} 1524 elif line[0] == ' ': # CONFIG line 1525 config, value = line.strip().split('=', 1) 1526 configs[config] = value 1527 defconfig_db[config].add(defconfig) 1528 all_configs.add(config) 1529 else: # New defconfig 1530 defconfig = line 1531 1532 # Work through each target config option in tern, independently 1533 for config in config_list: 1534 defconfigs = defconfig_db.get(config) 1535 if not defconfigs: 1536 print '%s not found in any defconfig' % config 1537 continue 1538 1539 # Get the set of defconfigs without this one (since a config cannot 1540 # imply itself) 1541 non_defconfigs = all_defconfigs - defconfigs 1542 num_defconfigs = len(defconfigs) 1543 print '%s found in %d/%d defconfigs' % (config, num_defconfigs, 1544 len(all_configs)) 1545 1546 # This will hold the results: key=config, value=defconfigs containing it 1547 imply_configs = {} 1548 rest_configs = all_configs - set([config]) 1549 1550 # Look at every possible config, except the target one 1551 for imply_config in rest_configs: 1552 if 'CONFIG_TARGET' in imply_config: 1553 continue 1554 1555 # Find set of defconfigs that have this config 1556 imply_defconfig = defconfig_db[imply_config] 1557 1558 # Get the intersection of this with defconfigs containing the 1559 # target config 1560 common_defconfigs = imply_defconfig & defconfigs 1561 1562 # Get the set of defconfigs containing this config which DO NOT 1563 # also contain the taret config. If this set is non-empty it means 1564 # that this config affects other defconfigs as well as (possibly) 1565 # the ones affected by the target config. This means it implies 1566 # things we don't want to imply. 1567 not_common_defconfigs = imply_defconfig & non_defconfigs 1568 if not_common_defconfigs: 1569 continue 1570 1571 # If there are common defconfigs, imply_config may be useful 1572 if common_defconfigs: 1573 skip = False 1574 if find_superset: 1575 for prev in imply_configs.keys(): 1576 prev_count = len(imply_configs[prev]) 1577 count = len(common_defconfigs) 1578 if (prev_count > count and 1579 (imply_configs[prev] & common_defconfigs == 1580 common_defconfigs)): 1581 # skip imply_config because prev is a superset 1582 skip = True 1583 break 1584 elif count > prev_count: 1585 # delete prev because imply_config is a superset 1586 del imply_configs[prev] 1587 if not skip: 1588 imply_configs[imply_config] = common_defconfigs 1589 1590 # Now we have a dict imply_configs of configs which imply each config 1591 # The value of each dict item is the set of defconfigs containing that 1592 # config. Rank them so that we print the configs that imply the largest 1593 # number of defconfigs first. 1594 ranked_configs = sorted(imply_configs, 1595 key=lambda k: len(imply_configs[k]), reverse=True) 1596 for config in ranked_configs: 1597 num_common = len(imply_configs[config]) 1598 1599 # Don't bother if there are less than 5 defconfigs affected. 1600 if num_common < 5: 1601 continue 1602 missing = defconfigs - imply_configs[config] 1603 missing_str = ', '.join(missing) if missing else 'all' 1604 missing_str = '' 1605 print ' %d : %-30s%s' % (num_common, config.ljust(30), 1606 missing_str) 1607 1608 1609def main(): 1610 try: 1611 cpu_count = multiprocessing.cpu_count() 1612 except NotImplementedError: 1613 cpu_count = 1 1614 1615 parser = optparse.OptionParser() 1616 # Add options here 1617 parser.add_option('-b', '--build-db', action='store_true', default=False, 1618 help='build a CONFIG database') 1619 parser.add_option('-c', '--color', action='store_true', default=False, 1620 help='display the log in color') 1621 parser.add_option('-C', '--commit', action='store_true', default=False, 1622 help='Create a git commit for the operation') 1623 parser.add_option('-d', '--defconfigs', type='string', 1624 help='a file containing a list of defconfigs to move, ' 1625 "one per line (for example 'snow_defconfig') " 1626 "or '-' to read from stdin") 1627 parser.add_option('-i', '--imply', action='store_true', default=False, 1628 help='find options which imply others') 1629 parser.add_option('-n', '--dry-run', action='store_true', default=False, 1630 help='perform a trial run (show log with no changes)') 1631 parser.add_option('-e', '--exit-on-error', action='store_true', 1632 default=False, 1633 help='exit immediately on any error') 1634 parser.add_option('-s', '--force-sync', action='store_true', default=False, 1635 help='force sync by savedefconfig') 1636 parser.add_option('-S', '--spl', action='store_true', default=False, 1637 help='parse config options defined for SPL build') 1638 parser.add_option('-H', '--headers-only', dest='cleanup_headers_only', 1639 action='store_true', default=False, 1640 help='only cleanup the headers') 1641 parser.add_option('-j', '--jobs', type='int', default=cpu_count, 1642 help='the number of jobs to run simultaneously') 1643 parser.add_option('-r', '--git-ref', type='string', 1644 help='the git ref to clone for building the autoconf.mk') 1645 parser.add_option('-y', '--yes', action='store_true', default=False, 1646 help="respond 'yes' to any prompts") 1647 parser.add_option('-v', '--verbose', action='store_true', default=False, 1648 help='show any build errors as boards are built') 1649 parser.usage += ' CONFIG ...' 1650 1651 (options, configs) = parser.parse_args() 1652 1653 if len(configs) == 0 and not any((options.force_sync, options.build_db, 1654 options.imply)): 1655 parser.print_usage() 1656 sys.exit(1) 1657 1658 # prefix the option name with CONFIG_ if missing 1659 configs = [ config if config.startswith('CONFIG_') else 'CONFIG_' + config 1660 for config in configs ] 1661 1662 check_top_directory() 1663 1664 if options.imply: 1665 imply_config(configs) 1666 return 1667 1668 config_db = {} 1669 db_queue = Queue.Queue() 1670 t = DatabaseThread(config_db, db_queue) 1671 t.setDaemon(True) 1672 t.start() 1673 1674 if not options.cleanup_headers_only: 1675 check_clean_directory() 1676 update_cross_compile(options.color) 1677 move_config(configs, options, db_queue) 1678 db_queue.join() 1679 1680 if configs: 1681 cleanup_headers(configs, options) 1682 cleanup_extra_options(configs, options) 1683 cleanup_whitelist(configs, options) 1684 cleanup_readme(configs, options) 1685 1686 if options.commit: 1687 subprocess.call(['git', 'add', '-u']) 1688 if configs: 1689 msg = 'Convert %s %sto Kconfig' % (configs[0], 1690 'et al ' if len(configs) > 1 else '') 1691 msg += ('\n\nThis converts the following to Kconfig:\n %s\n' % 1692 '\n '.join(configs)) 1693 else: 1694 msg = 'configs: Resync with savedefconfig' 1695 msg += '\n\nRsync all defconfig files using moveconfig.py' 1696 subprocess.call(['git', 'commit', '-s', '-m', msg]) 1697 1698 if options.build_db: 1699 with open(CONFIG_DATABASE, 'w') as fd: 1700 for defconfig, configs in config_db.iteritems(): 1701 print >>fd, '%s' % defconfig 1702 for config in sorted(configs.keys()): 1703 print >>fd, ' %s=%s' % (config, configs[config]) 1704 print >>fd 1705 1706if __name__ == '__main__': 1707 main() 1708