1#!/usr/bin/env python2 2# 3# Author: Masahiro Yamada <yamada.masahiro@socionext.com> 4# 5# SPDX-License-Identifier: GPL-2.0+ 6# 7 8""" 9Move config options from headers to defconfig files. 10 11Since Kconfig was introduced to U-Boot, we have worked on moving 12config options from headers to Kconfig (defconfig). 13 14This tool intends to help this tremendous work. 15 16 17Usage 18----- 19 20First, you must edit the Kconfig to add the menu entries for the configs 21you are moving. 22 23And then run this tool giving CONFIG names you want to move. 24For example, if you want to move CONFIG_CMD_USB and CONFIG_SYS_TEXT_BASE, 25simply type as follows: 26 27 $ tools/moveconfig.py CONFIG_CMD_USB CONFIG_SYS_TEXT_BASE 28 29The tool walks through all the defconfig files and move the given CONFIGs. 30 31The log is also displayed on the terminal. 32 33The log is printed for each defconfig as follows: 34 35<defconfig_name> 36 <action1> 37 <action2> 38 <action3> 39 ... 40 41<defconfig_name> is the name of the defconfig. 42 43<action*> shows what the tool did for that defconfig. 44It looks like one of the followings: 45 46 - Move 'CONFIG_... ' 47 This config option was moved to the defconfig 48 49 - CONFIG_... is not defined in Kconfig. Do nothing. 50 The entry for this CONFIG was not found in Kconfig. 51 There are two common cases: 52 - You forgot to create an entry for the CONFIG before running 53 this tool, or made a typo in a CONFIG passed to this tool. 54 - The entry was hidden due to unmet 'depends on'. 55 This is correct behavior. 56 57 - 'CONFIG_...' is the same as the define in Kconfig. Do nothing. 58 The define in the config header matched the one in Kconfig. 59 We do not need to touch it. 60 61 - Undefined. Do nothing. 62 This config option was not found in the config header. 63 Nothing to do. 64 65 - Compiler is missing. Do nothing. 66 The compiler specified for this architecture was not found 67 in your PATH environment. 68 (If -e option is passed, the tool exits immediately.) 69 70 - Failed to process. 71 An error occurred during processing this defconfig. Skipped. 72 (If -e option is passed, the tool exits immediately on error.) 73 74Finally, you will be asked, Clean up headers? [y/n]: 75 76If you say 'y' here, the unnecessary config defines are removed 77from the config headers (include/configs/*.h). 78It just uses the regex method, so you should not rely on it. 79Just in case, please do 'git diff' to see what happened. 80 81 82How does it work? 83----------------- 84 85This tool runs configuration and builds include/autoconf.mk for every 86defconfig. The config options defined in Kconfig appear in the .config 87file (unless they are hidden because of unmet dependency.) 88On the other hand, the config options defined by board headers are seen 89in include/autoconf.mk. The tool looks for the specified options in both 90of them to decide the appropriate action for the options. If the given 91config option is found in the .config, but its value does not match the 92one from the board header, the config option in the .config is replaced 93with the define in the board header. Then, the .config is synced by 94"make savedefconfig" and the defconfig is updated with it. 95 96For faster processing, this tool handles multi-threading. It creates 97separate build directories where the out-of-tree build is run. The 98temporary build directories are automatically created and deleted as 99needed. The number of threads are chosen based on the number of the CPU 100cores of your system although you can change it via -j (--jobs) option. 101 102 103Toolchains 104---------- 105 106Appropriate toolchain are necessary to generate include/autoconf.mk 107for all the architectures supported by U-Boot. Most of them are available 108at the kernel.org site, some are not provided by kernel.org. 109 110The default per-arch CROSS_COMPILE used by this tool is specified by 111the list below, CROSS_COMPILE. You may wish to update the list to 112use your own. Instead of modifying the list directly, you can give 113them via environments. 114 115 116Available options 117----------------- 118 119 -c, --color 120 Surround each portion of the log with escape sequences to display it 121 in color on the terminal. 122 123 -d, --defconfigs 124 Specify a file containing a list of defconfigs to move 125 126 -n, --dry-run 127 Perform a trial run that does not make any changes. It is useful to 128 see what is going to happen before one actually runs it. 129 130 -e, --exit-on-error 131 Exit immediately if Make exits with a non-zero status while processing 132 a defconfig file. 133 134 -s, --force-sync 135 Do "make savedefconfig" forcibly for all the defconfig files. 136 If not specified, "make savedefconfig" only occurs for cases 137 where at least one CONFIG was moved. 138 139 -H, --headers-only 140 Only cleanup the headers; skip the defconfig processing 141 142 -j, --jobs 143 Specify the number of threads to run simultaneously. If not specified, 144 the number of threads is the same as the number of CPU cores. 145 146 -v, --verbose 147 Show any build errors as boards are built 148 149To see the complete list of supported options, run 150 151 $ tools/moveconfig.py -h 152 153""" 154 155import filecmp 156import fnmatch 157import multiprocessing 158import optparse 159import os 160import re 161import shutil 162import subprocess 163import sys 164import tempfile 165import time 166 167SHOW_GNU_MAKE = 'scripts/show-gnu-make' 168SLEEP_TIME=0.03 169 170# Here is the list of cross-tools I use. 171# Most of them are available at kernel.org 172# (https://www.kernel.org/pub/tools/crosstool/files/bin/), except the followings: 173# arc: https://github.com/foss-for-synopsys-dwc-arc-processors/toolchain/releases 174# blackfin: http://sourceforge.net/projects/adi-toolchain/files/ 175# nds32: http://osdk.andestech.com/packages/nds32le-linux-glibc-v1.tgz 176# nios2: https://sourcery.mentor.com/GNUToolchain/subscription42545 177# sh: http://sourcery.mentor.com/public/gnu_toolchain/sh-linux-gnu 178# 179# openrisc kernel.org toolchain is out of date, download latest one from 180# http://opencores.org/or1k/OpenRISC_GNU_tool_chain#Prebuilt_versions 181CROSS_COMPILE = { 182 'arc': 'arc-linux-', 183 'aarch64': 'aarch64-linux-', 184 'arm': 'arm-unknown-linux-gnueabi-', 185 'avr32': 'avr32-linux-', 186 'blackfin': 'bfin-elf-', 187 'm68k': 'm68k-linux-', 188 'microblaze': 'microblaze-linux-', 189 'mips': 'mips-linux-', 190 'nds32': 'nds32le-linux-', 191 'nios2': 'nios2-linux-gnu-', 192 'openrisc': 'or1k-elf-', 193 'powerpc': 'powerpc-linux-', 194 'sh': 'sh-linux-gnu-', 195 'sparc': 'sparc-linux-', 196 'x86': 'i386-linux-' 197} 198 199STATE_IDLE = 0 200STATE_DEFCONFIG = 1 201STATE_AUTOCONF = 2 202STATE_SAVEDEFCONFIG = 3 203 204ACTION_MOVE = 0 205ACTION_NO_ENTRY = 1 206ACTION_NO_CHANGE = 2 207 208COLOR_BLACK = '0;30' 209COLOR_RED = '0;31' 210COLOR_GREEN = '0;32' 211COLOR_BROWN = '0;33' 212COLOR_BLUE = '0;34' 213COLOR_PURPLE = '0;35' 214COLOR_CYAN = '0;36' 215COLOR_LIGHT_GRAY = '0;37' 216COLOR_DARK_GRAY = '1;30' 217COLOR_LIGHT_RED = '1;31' 218COLOR_LIGHT_GREEN = '1;32' 219COLOR_YELLOW = '1;33' 220COLOR_LIGHT_BLUE = '1;34' 221COLOR_LIGHT_PURPLE = '1;35' 222COLOR_LIGHT_CYAN = '1;36' 223COLOR_WHITE = '1;37' 224 225### helper functions ### 226def get_devnull(): 227 """Get the file object of '/dev/null' device.""" 228 try: 229 devnull = subprocess.DEVNULL # py3k 230 except AttributeError: 231 devnull = open(os.devnull, 'wb') 232 return devnull 233 234def check_top_directory(): 235 """Exit if we are not at the top of source directory.""" 236 for f in ('README', 'Licenses'): 237 if not os.path.exists(f): 238 sys.exit('Please run at the top of source directory.') 239 240def check_clean_directory(): 241 """Exit if the source tree is not clean.""" 242 for f in ('.config', 'include/config'): 243 if os.path.exists(f): 244 sys.exit("source tree is not clean, please run 'make mrproper'") 245 246def get_make_cmd(): 247 """Get the command name of GNU Make. 248 249 U-Boot needs GNU Make for building, but the command name is not 250 necessarily "make". (for example, "gmake" on FreeBSD). 251 Returns the most appropriate command name on your system. 252 """ 253 process = subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE) 254 ret = process.communicate() 255 if process.returncode: 256 sys.exit('GNU Make not found') 257 return ret[0].rstrip() 258 259def color_text(color_enabled, color, string): 260 """Return colored string.""" 261 if color_enabled: 262 # LF should not be surrounded by the escape sequence. 263 # Otherwise, additional whitespace or line-feed might be printed. 264 return '\n'.join([ '\033[' + color + 'm' + s + '\033[0m' if s else '' 265 for s in string.split('\n') ]) 266 else: 267 return string 268 269def update_cross_compile(color_enabled): 270 """Update per-arch CROSS_COMPILE via environment variables 271 272 The default CROSS_COMPILE values are available 273 in the CROSS_COMPILE list above. 274 275 You can override them via environment variables 276 CROSS_COMPILE_{ARCH}. 277 278 For example, if you want to override toolchain prefixes 279 for ARM and PowerPC, you can do as follows in your shell: 280 281 export CROSS_COMPILE_ARM=... 282 export CROSS_COMPILE_POWERPC=... 283 284 Then, this function checks if specified compilers really exist in your 285 PATH environment. 286 """ 287 archs = [] 288 289 for arch in os.listdir('arch'): 290 if os.path.exists(os.path.join('arch', arch, 'Makefile')): 291 archs.append(arch) 292 293 # arm64 is a special case 294 archs.append('aarch64') 295 296 for arch in archs: 297 env = 'CROSS_COMPILE_' + arch.upper() 298 cross_compile = os.environ.get(env) 299 if not cross_compile: 300 cross_compile = CROSS_COMPILE.get(arch, '') 301 302 for path in os.environ["PATH"].split(os.pathsep): 303 gcc_path = os.path.join(path, cross_compile + 'gcc') 304 if os.path.isfile(gcc_path) and os.access(gcc_path, os.X_OK): 305 break 306 else: 307 print >> sys.stderr, color_text(color_enabled, COLOR_YELLOW, 308 'warning: %sgcc: not found in PATH. %s architecture boards will be skipped' 309 % (cross_compile, arch)) 310 cross_compile = None 311 312 CROSS_COMPILE[arch] = cross_compile 313 314def cleanup_one_header(header_path, patterns, dry_run): 315 """Clean regex-matched lines away from a file. 316 317 Arguments: 318 header_path: path to the cleaned file. 319 patterns: list of regex patterns. Any lines matching to these 320 patterns are deleted. 321 dry_run: make no changes, but still display log. 322 """ 323 with open(header_path) as f: 324 lines = f.readlines() 325 326 matched = [] 327 for i, line in enumerate(lines): 328 for pattern in patterns: 329 m = pattern.search(line) 330 if m: 331 print '%s: %s: %s' % (header_path, i + 1, line), 332 matched.append(i) 333 break 334 335 if dry_run or not matched: 336 return 337 338 with open(header_path, 'w') as f: 339 for i, line in enumerate(lines): 340 if not i in matched: 341 f.write(line) 342 343def cleanup_headers(configs, dry_run): 344 """Delete config defines from board headers. 345 346 Arguments: 347 configs: A list of CONFIGs to remove. 348 dry_run: make no changes, but still display log. 349 """ 350 while True: 351 choice = raw_input('Clean up headers? [y/n]: ').lower() 352 print choice 353 if choice == 'y' or choice == 'n': 354 break 355 356 if choice == 'n': 357 return 358 359 patterns = [] 360 for config in configs: 361 patterns.append(re.compile(r'#\s*define\s+%s\W' % config)) 362 patterns.append(re.compile(r'#\s*undef\s+%s\W' % config)) 363 364 for dir in 'include', 'arch', 'board': 365 for (dirpath, dirnames, filenames) in os.walk(dir): 366 for filename in filenames: 367 if not fnmatch.fnmatch(filename, '*~'): 368 cleanup_one_header(os.path.join(dirpath, filename), 369 patterns, dry_run) 370 371### classes ### 372class Progress: 373 374 """Progress Indicator""" 375 376 def __init__(self, total): 377 """Create a new progress indicator. 378 379 Arguments: 380 total: A number of defconfig files to process. 381 """ 382 self.current = 0 383 self.total = total 384 385 def inc(self): 386 """Increment the number of processed defconfig files.""" 387 388 self.current += 1 389 390 def show(self): 391 """Display the progress.""" 392 print ' %d defconfigs out of %d\r' % (self.current, self.total), 393 sys.stdout.flush() 394 395class KconfigParser: 396 397 """A parser of .config and include/autoconf.mk.""" 398 399 re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"') 400 re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"') 401 402 def __init__(self, configs, options, build_dir): 403 """Create a new parser. 404 405 Arguments: 406 configs: A list of CONFIGs to move. 407 options: option flags. 408 build_dir: Build directory. 409 """ 410 self.configs = configs 411 self.options = options 412 self.dotconfig = os.path.join(build_dir, '.config') 413 self.autoconf = os.path.join(build_dir, 'include', 'autoconf.mk') 414 self.config_autoconf = os.path.join(build_dir, 'include', 'config', 415 'auto.conf') 416 self.defconfig = os.path.join(build_dir, 'defconfig') 417 418 def get_cross_compile(self): 419 """Parse .config file and return CROSS_COMPILE. 420 421 Returns: 422 A string storing the compiler prefix for the architecture. 423 Return a NULL string for architectures that do not require 424 compiler prefix (Sandbox and native build is the case). 425 Return None if the specified compiler is missing in your PATH. 426 Caller should distinguish '' and None. 427 """ 428 arch = '' 429 cpu = '' 430 for line in open(self.dotconfig): 431 m = self.re_arch.match(line) 432 if m: 433 arch = m.group(1) 434 continue 435 m = self.re_cpu.match(line) 436 if m: 437 cpu = m.group(1) 438 439 if not arch: 440 return None 441 442 # fix-up for aarch64 443 if arch == 'arm' and cpu == 'armv8': 444 arch = 'aarch64' 445 446 return CROSS_COMPILE.get(arch, None) 447 448 def parse_one_config(self, config, dotconfig_lines, autoconf_lines): 449 """Parse .config, defconfig, include/autoconf.mk for one config. 450 451 This function looks for the config options in the lines from 452 defconfig, .config, and include/autoconf.mk in order to decide 453 which action should be taken for this defconfig. 454 455 Arguments: 456 config: CONFIG name to parse. 457 dotconfig_lines: lines from the .config file. 458 autoconf_lines: lines from the include/autoconf.mk file. 459 460 Returns: 461 A tupple of the action for this defconfig and the line 462 matched for the config. 463 """ 464 not_set = '# %s is not set' % config 465 466 for line in dotconfig_lines: 467 line = line.rstrip() 468 if line.startswith(config + '=') or line == not_set: 469 old_val = line 470 break 471 else: 472 return (ACTION_NO_ENTRY, config) 473 474 for line in autoconf_lines: 475 line = line.rstrip() 476 if line.startswith(config + '='): 477 new_val = line 478 break 479 else: 480 new_val = not_set 481 482 if old_val == new_val: 483 return (ACTION_NO_CHANGE, new_val) 484 485 # If this CONFIG is neither bool nor trisate 486 if old_val[-2:] != '=y' and old_val[-2:] != '=m' and old_val != not_set: 487 # tools/scripts/define2mk.sed changes '1' to 'y'. 488 # This is a problem if the CONFIG is int type. 489 # Check the type in Kconfig and handle it correctly. 490 if new_val[-2:] == '=y': 491 new_val = new_val[:-1] + '1' 492 493 return (ACTION_MOVE, new_val) 494 495 def update_dotconfig(self): 496 """Parse files for the config options and update the .config. 497 498 This function parses the generated .config and include/autoconf.mk 499 searching the target options. 500 Move the config option(s) to the .config as needed. 501 502 Arguments: 503 defconfig: defconfig name. 504 505 Returns: 506 Return a tuple of (updated flag, log string). 507 The "updated flag" is True if the .config was updated, False 508 otherwise. The "log string" shows what happend to the .config. 509 """ 510 511 results = [] 512 updated = False 513 514 with open(self.dotconfig) as f: 515 dotconfig_lines = f.readlines() 516 517 with open(self.autoconf) as f: 518 autoconf_lines = f.readlines() 519 520 for config in self.configs: 521 result = self.parse_one_config(config, dotconfig_lines, 522 autoconf_lines) 523 results.append(result) 524 525 log = '' 526 527 for (action, value) in results: 528 if action == ACTION_MOVE: 529 actlog = "Move '%s'" % value 530 log_color = COLOR_LIGHT_GREEN 531 elif action == ACTION_NO_ENTRY: 532 actlog = "%s is not defined in Kconfig. Do nothing." % value 533 log_color = COLOR_LIGHT_BLUE 534 elif action == ACTION_NO_CHANGE: 535 actlog = "'%s' is the same as the define in Kconfig. Do nothing." \ 536 % value 537 log_color = COLOR_LIGHT_PURPLE 538 else: 539 sys.exit("Internal Error. This should not happen.") 540 541 log += color_text(self.options.color, log_color, actlog) + '\n' 542 543 with open(self.dotconfig, 'a') as f: 544 for (action, value) in results: 545 if action == ACTION_MOVE: 546 f.write(value + '\n') 547 updated = True 548 549 self.results = results 550 os.remove(self.config_autoconf) 551 os.remove(self.autoconf) 552 553 return (updated, log) 554 555 def check_defconfig(self): 556 """Check the defconfig after savedefconfig 557 558 Returns: 559 Return additional log if moved CONFIGs were removed again by 560 'make savedefconfig'. 561 """ 562 563 log = '' 564 565 with open(self.defconfig) as f: 566 defconfig_lines = f.readlines() 567 568 for (action, value) in self.results: 569 if action != ACTION_MOVE: 570 continue 571 if not value + '\n' in defconfig_lines: 572 log += color_text(self.options.color, COLOR_YELLOW, 573 "'%s' was removed by savedefconfig.\n" % 574 value) 575 576 return log 577 578class Slot: 579 580 """A slot to store a subprocess. 581 582 Each instance of this class handles one subprocess. 583 This class is useful to control multiple threads 584 for faster processing. 585 """ 586 587 def __init__(self, configs, options, progress, devnull, make_cmd): 588 """Create a new process slot. 589 590 Arguments: 591 configs: A list of CONFIGs to move. 592 options: option flags. 593 progress: A progress indicator. 594 devnull: A file object of '/dev/null'. 595 make_cmd: command name of GNU Make. 596 """ 597 self.options = options 598 self.progress = progress 599 self.build_dir = tempfile.mkdtemp() 600 self.devnull = devnull 601 self.make_cmd = (make_cmd, 'O=' + self.build_dir) 602 self.parser = KconfigParser(configs, options, self.build_dir) 603 self.state = STATE_IDLE 604 self.failed_boards = [] 605 606 def __del__(self): 607 """Delete the working directory 608 609 This function makes sure the temporary directory is cleaned away 610 even if Python suddenly dies due to error. It should be done in here 611 because it is guranteed the destructor is always invoked when the 612 instance of the class gets unreferenced. 613 614 If the subprocess is still running, wait until it finishes. 615 """ 616 if self.state != STATE_IDLE: 617 while self.ps.poll() == None: 618 pass 619 shutil.rmtree(self.build_dir) 620 621 def add(self, defconfig): 622 """Assign a new subprocess for defconfig and add it to the slot. 623 624 If the slot is vacant, create a new subprocess for processing the 625 given defconfig and add it to the slot. Just returns False if 626 the slot is occupied (i.e. the current subprocess is still running). 627 628 Arguments: 629 defconfig: defconfig name. 630 631 Returns: 632 Return True on success or False on failure 633 """ 634 if self.state != STATE_IDLE: 635 return False 636 cmd = list(self.make_cmd) 637 cmd.append(defconfig) 638 self.ps = subprocess.Popen(cmd, stdout=self.devnull, 639 stderr=subprocess.PIPE) 640 self.defconfig = defconfig 641 self.state = STATE_DEFCONFIG 642 self.log = '' 643 return True 644 645 def poll(self): 646 """Check the status of the subprocess and handle it as needed. 647 648 Returns True if the slot is vacant (i.e. in idle state). 649 If the configuration is successfully finished, assign a new 650 subprocess to build include/autoconf.mk. 651 If include/autoconf.mk is generated, invoke the parser to 652 parse the .config and the include/autoconf.mk, moving 653 config options to the .config as needed. 654 If the .config was updated, run "make savedefconfig" to sync 655 it, update the original defconfig, and then set the slot back 656 to the idle state. 657 658 Returns: 659 Return True if the subprocess is terminated, False otherwise 660 """ 661 if self.state == STATE_IDLE: 662 return True 663 664 if self.ps.poll() == None: 665 return False 666 667 if self.ps.poll() != 0: 668 self.log += color_text(self.options.color, COLOR_LIGHT_RED, 669 "Failed to process.\n") 670 if self.options.verbose: 671 self.log += color_text(self.options.color, COLOR_LIGHT_CYAN, 672 self.ps.stderr.read()) 673 self.finish(False) 674 return True 675 676 if self.state == STATE_AUTOCONF: 677 (updated, log) = self.parser.update_dotconfig() 678 self.log += log 679 680 if not self.options.force_sync and not updated: 681 self.finish(True) 682 return True 683 if updated: 684 self.log += color_text(self.options.color, COLOR_LIGHT_GREEN, 685 "Syncing by savedefconfig...\n") 686 else: 687 self.log += "Syncing by savedefconfig (forced by option)...\n" 688 689 cmd = list(self.make_cmd) 690 cmd.append('savedefconfig') 691 self.ps = subprocess.Popen(cmd, stdout=self.devnull, 692 stderr=subprocess.PIPE) 693 self.state = STATE_SAVEDEFCONFIG 694 return False 695 696 if self.state == STATE_SAVEDEFCONFIG: 697 self.log += self.parser.check_defconfig() 698 orig_defconfig = os.path.join('configs', self.defconfig) 699 new_defconfig = os.path.join(self.build_dir, 'defconfig') 700 updated = not filecmp.cmp(orig_defconfig, new_defconfig) 701 702 if updated: 703 self.log += color_text(self.options.color, COLOR_LIGHT_GREEN, 704 "defconfig was updated.\n") 705 706 if not self.options.dry_run and updated: 707 shutil.move(new_defconfig, orig_defconfig) 708 self.finish(True) 709 return True 710 711 self.cross_compile = self.parser.get_cross_compile() 712 if self.cross_compile is None: 713 self.log += color_text(self.options.color, COLOR_YELLOW, 714 "Compiler is missing. Do nothing.\n") 715 self.finish(False) 716 return True 717 718 cmd = list(self.make_cmd) 719 if self.cross_compile: 720 cmd.append('CROSS_COMPILE=%s' % self.cross_compile) 721 cmd.append('KCONFIG_IGNORE_DUPLICATES=1') 722 cmd.append('include/config/auto.conf') 723 self.ps = subprocess.Popen(cmd, stdout=self.devnull, 724 stderr=subprocess.PIPE) 725 self.state = STATE_AUTOCONF 726 return False 727 728 def finish(self, success): 729 """Display log along with progress and go to the idle state. 730 731 Arguments: 732 success: Should be True when the defconfig was processed 733 successfully, or False when it fails. 734 """ 735 # output at least 30 characters to hide the "* defconfigs out of *". 736 log = self.defconfig.ljust(30) + '\n' 737 738 log += '\n'.join([ ' ' + s for s in self.log.split('\n') ]) 739 # Some threads are running in parallel. 740 # Print log atomically to not mix up logs from different threads. 741 print >> (sys.stdout if success else sys.stderr), log 742 743 if not success: 744 if self.options.exit_on_error: 745 sys.exit("Exit on error.") 746 # If --exit-on-error flag is not set, skip this board and continue. 747 # Record the failed board. 748 self.failed_boards.append(self.defconfig) 749 750 self.progress.inc() 751 self.progress.show() 752 self.state = STATE_IDLE 753 754 def get_failed_boards(self): 755 """Returns a list of failed boards (defconfigs) in this slot. 756 """ 757 return self.failed_boards 758 759class Slots: 760 761 """Controller of the array of subprocess slots.""" 762 763 def __init__(self, configs, options, progress): 764 """Create a new slots controller. 765 766 Arguments: 767 configs: A list of CONFIGs to move. 768 options: option flags. 769 progress: A progress indicator. 770 """ 771 self.options = options 772 self.slots = [] 773 devnull = get_devnull() 774 make_cmd = get_make_cmd() 775 for i in range(options.jobs): 776 self.slots.append(Slot(configs, options, progress, devnull, 777 make_cmd)) 778 779 def add(self, defconfig): 780 """Add a new subprocess if a vacant slot is found. 781 782 Arguments: 783 defconfig: defconfig name to be put into. 784 785 Returns: 786 Return True on success or False on failure 787 """ 788 for slot in self.slots: 789 if slot.add(defconfig): 790 return True 791 return False 792 793 def available(self): 794 """Check if there is a vacant slot. 795 796 Returns: 797 Return True if at lease one vacant slot is found, False otherwise. 798 """ 799 for slot in self.slots: 800 if slot.poll(): 801 return True 802 return False 803 804 def empty(self): 805 """Check if all slots are vacant. 806 807 Returns: 808 Return True if all the slots are vacant, False otherwise. 809 """ 810 ret = True 811 for slot in self.slots: 812 if not slot.poll(): 813 ret = False 814 return ret 815 816 def show_failed_boards(self): 817 """Display all of the failed boards (defconfigs).""" 818 failed_boards = [] 819 820 for slot in self.slots: 821 failed_boards += slot.get_failed_boards() 822 823 if len(failed_boards) > 0: 824 msg = [ "The following boards were not processed due to error:" ] 825 msg += failed_boards 826 for line in msg: 827 print >> sys.stderr, color_text(self.options.color, 828 COLOR_LIGHT_RED, line) 829 830 with open('moveconfig.failed', 'w') as f: 831 for board in failed_boards: 832 f.write(board + '\n') 833 834def move_config(configs, options): 835 """Move config options to defconfig files. 836 837 Arguments: 838 configs: A list of CONFIGs to move. 839 options: option flags 840 """ 841 if len(configs) == 0: 842 if options.force_sync: 843 print 'No CONFIG is specified. You are probably syncing defconfigs.', 844 else: 845 print 'Neither CONFIG nor --force-sync is specified. Nothing will happen.', 846 else: 847 print 'Move ' + ', '.join(configs), 848 print '(jobs: %d)\n' % options.jobs 849 850 if options.defconfigs: 851 defconfigs = [line.strip() for line in open(options.defconfigs)] 852 for i, defconfig in enumerate(defconfigs): 853 if not defconfig.endswith('_defconfig'): 854 defconfigs[i] = defconfig + '_defconfig' 855 if not os.path.exists(os.path.join('configs', defconfigs[i])): 856 sys.exit('%s - defconfig does not exist. Stopping.' % 857 defconfigs[i]) 858 else: 859 # All the defconfig files to be processed 860 defconfigs = [] 861 for (dirpath, dirnames, filenames) in os.walk('configs'): 862 dirpath = dirpath[len('configs') + 1:] 863 for filename in fnmatch.filter(filenames, '*_defconfig'): 864 defconfigs.append(os.path.join(dirpath, filename)) 865 866 progress = Progress(len(defconfigs)) 867 slots = Slots(configs, options, progress) 868 869 # Main loop to process defconfig files: 870 # Add a new subprocess into a vacant slot. 871 # Sleep if there is no available slot. 872 for defconfig in defconfigs: 873 while not slots.add(defconfig): 874 while not slots.available(): 875 # No available slot: sleep for a while 876 time.sleep(SLEEP_TIME) 877 878 # wait until all the subprocesses finish 879 while not slots.empty(): 880 time.sleep(SLEEP_TIME) 881 882 print '' 883 slots.show_failed_boards() 884 885def main(): 886 try: 887 cpu_count = multiprocessing.cpu_count() 888 except NotImplementedError: 889 cpu_count = 1 890 891 parser = optparse.OptionParser() 892 # Add options here 893 parser.add_option('-c', '--color', action='store_true', default=False, 894 help='display the log in color') 895 parser.add_option('-d', '--defconfigs', type='string', 896 help='a file containing a list of defconfigs to move') 897 parser.add_option('-n', '--dry-run', action='store_true', default=False, 898 help='perform a trial run (show log with no changes)') 899 parser.add_option('-e', '--exit-on-error', action='store_true', 900 default=False, 901 help='exit immediately on any error') 902 parser.add_option('-s', '--force-sync', action='store_true', default=False, 903 help='force sync by savedefconfig') 904 parser.add_option('-H', '--headers-only', dest='cleanup_headers_only', 905 action='store_true', default=False, 906 help='only cleanup the headers') 907 parser.add_option('-j', '--jobs', type='int', default=cpu_count, 908 help='the number of jobs to run simultaneously') 909 parser.add_option('-v', '--verbose', action='store_true', default=False, 910 help='show any build errors as boards are built') 911 parser.usage += ' CONFIG ...' 912 913 (options, configs) = parser.parse_args() 914 915 if len(configs) == 0 and not options.force_sync: 916 parser.print_usage() 917 sys.exit(1) 918 919 # prefix the option name with CONFIG_ if missing 920 configs = [ config if config.startswith('CONFIG_') else 'CONFIG_' + config 921 for config in configs ] 922 923 check_top_directory() 924 925 check_clean_directory() 926 927 update_cross_compile(options.color) 928 929 if not options.cleanup_headers_only: 930 move_config(configs, options) 931 932 if configs: 933 cleanup_headers(configs, options.dry_run) 934 935if __name__ == '__main__': 936 main() 937