1#!/usr/bin/env python2 2# 3# Author: Masahiro Yamada <yamada.masahiro@socionext.com> 4# 5# SPDX-License-Identifier: GPL-2.0+ 6# 7 8""" 9Move config options from headers to defconfig files. 10 11Since Kconfig was introduced to U-Boot, we have worked on moving 12config options from headers to Kconfig (defconfig). 13 14This tool intends to help this tremendous work. 15 16 17Usage 18----- 19 20First, you must edit the Kconfig to add the menu entries for the configs 21you are moving. 22 23And then run this tool giving CONFIG names you want to move. 24For example, if you want to move CONFIG_CMD_USB and CONFIG_SYS_TEXT_BASE, 25simply type as follows: 26 27 $ tools/moveconfig.py CONFIG_CMD_USB CONFIG_SYS_TEXT_BASE 28 29The tool walks through all the defconfig files and move the given CONFIGs. 30 31The log is also displayed on the terminal. 32 33The log is printed for each defconfig as follows: 34 35<defconfig_name> 36 <action1> 37 <action2> 38 <action3> 39 ... 40 41<defconfig_name> is the name of the defconfig. 42 43<action*> shows what the tool did for that defconfig. 44It looks like one of the followings: 45 46 - Move 'CONFIG_... ' 47 This config option was moved to the defconfig 48 49 - CONFIG_... is not defined in Kconfig. Do nothing. 50 The entry for this CONFIG was not found in Kconfig. 51 There are two common cases: 52 - You forgot to create an entry for the CONFIG before running 53 this tool, or made a typo in a CONFIG passed to this tool. 54 - The entry was hidden due to unmet 'depends on'. 55 This is correct behavior. 56 57 - 'CONFIG_...' is the same as the define in Kconfig. Do nothing. 58 The define in the config header matched the one in Kconfig. 59 We do not need to touch it. 60 61 - Undefined. Do nothing. 62 This config option was not found in the config header. 63 Nothing to do. 64 65 - Compiler is missing. Do nothing. 66 The compiler specified for this architecture was not found 67 in your PATH environment. 68 (If -e option is passed, the tool exits immediately.) 69 70 - Failed to process. 71 An error occurred during processing this defconfig. Skipped. 72 (If -e option is passed, the tool exits immediately on error.) 73 74Finally, you will be asked, Clean up headers? [y/n]: 75 76If you say 'y' here, the unnecessary config defines are removed 77from the config headers (include/configs/*.h). 78It just uses the regex method, so you should not rely on it. 79Just in case, please do 'git diff' to see what happened. 80 81 82How does it work? 83----------------- 84 85This tool runs configuration and builds include/autoconf.mk for every 86defconfig. The config options defined in Kconfig appear in the .config 87file (unless they are hidden because of unmet dependency.) 88On the other hand, the config options defined by board headers are seen 89in include/autoconf.mk. The tool looks for the specified options in both 90of them to decide the appropriate action for the options. If the given 91config option is found in the .config, but its value does not match the 92one from the board header, the config option in the .config is replaced 93with the define in the board header. Then, the .config is synced by 94"make savedefconfig" and the defconfig is updated with it. 95 96For faster processing, this tool handles multi-threading. It creates 97separate build directories where the out-of-tree build is run. The 98temporary build directories are automatically created and deleted as 99needed. The number of threads are chosen based on the number of the CPU 100cores of your system although you can change it via -j (--jobs) option. 101 102 103Toolchains 104---------- 105 106Appropriate toolchain are necessary to generate include/autoconf.mk 107for all the architectures supported by U-Boot. Most of them are available 108at the kernel.org site, some are not provided by kernel.org. 109 110The default per-arch CROSS_COMPILE used by this tool is specified by 111the list below, CROSS_COMPILE. You may wish to update the list to 112use your own. Instead of modifying the list directly, you can give 113them via environments. 114 115 116Available options 117----------------- 118 119 -c, --color 120 Surround each portion of the log with escape sequences to display it 121 in color on the terminal. 122 123 -d, --defconfigs 124 Specify a file containing a list of defconfigs to move 125 126 -n, --dry-run 127 Perform a trial run that does not make any changes. It is useful to 128 see what is going to happen before one actually runs it. 129 130 -e, --exit-on-error 131 Exit immediately if Make exits with a non-zero status while processing 132 a defconfig file. 133 134 -H, --headers-only 135 Only cleanup the headers; skip the defconfig processing 136 137 -j, --jobs 138 Specify the number of threads to run simultaneously. If not specified, 139 the number of threads is the same as the number of CPU cores. 140 141 -v, --verbose 142 Show any build errors as boards are built 143 144To see the complete list of supported options, run 145 146 $ tools/moveconfig.py -h 147 148""" 149 150import filecmp 151import fnmatch 152import multiprocessing 153import optparse 154import os 155import re 156import shutil 157import subprocess 158import sys 159import tempfile 160import time 161 162SHOW_GNU_MAKE = 'scripts/show-gnu-make' 163SLEEP_TIME=0.03 164 165# Here is the list of cross-tools I use. 166# Most of them are available at kernel.org 167# (https://www.kernel.org/pub/tools/crosstool/files/bin/), except the followings: 168# arc: https://github.com/foss-for-synopsys-dwc-arc-processors/toolchain/releases 169# blackfin: http://sourceforge.net/projects/adi-toolchain/files/ 170# nds32: http://osdk.andestech.com/packages/nds32le-linux-glibc-v1.tgz 171# nios2: https://sourcery.mentor.com/GNUToolchain/subscription42545 172# sh: http://sourcery.mentor.com/public/gnu_toolchain/sh-linux-gnu 173# 174# openrisc kernel.org toolchain is out of date, download latest one from 175# http://opencores.org/or1k/OpenRISC_GNU_tool_chain#Prebuilt_versions 176CROSS_COMPILE = { 177 'arc': 'arc-linux-', 178 'aarch64': 'aarch64-linux-', 179 'arm': 'arm-unknown-linux-gnueabi-', 180 'avr32': 'avr32-linux-', 181 'blackfin': 'bfin-elf-', 182 'm68k': 'm68k-linux-', 183 'microblaze': 'microblaze-linux-', 184 'mips': 'mips-linux-', 185 'nds32': 'nds32le-linux-', 186 'nios2': 'nios2-linux-gnu-', 187 'openrisc': 'or1k-elf-', 188 'powerpc': 'powerpc-linux-', 189 'sh': 'sh-linux-gnu-', 190 'sparc': 'sparc-linux-', 191 'x86': 'i386-linux-' 192} 193 194STATE_IDLE = 0 195STATE_DEFCONFIG = 1 196STATE_AUTOCONF = 2 197STATE_SAVEDEFCONFIG = 3 198 199ACTION_MOVE = 0 200ACTION_NO_ENTRY = 1 201ACTION_NO_CHANGE = 2 202 203COLOR_BLACK = '0;30' 204COLOR_RED = '0;31' 205COLOR_GREEN = '0;32' 206COLOR_BROWN = '0;33' 207COLOR_BLUE = '0;34' 208COLOR_PURPLE = '0;35' 209COLOR_CYAN = '0;36' 210COLOR_LIGHT_GRAY = '0;37' 211COLOR_DARK_GRAY = '1;30' 212COLOR_LIGHT_RED = '1;31' 213COLOR_LIGHT_GREEN = '1;32' 214COLOR_YELLOW = '1;33' 215COLOR_LIGHT_BLUE = '1;34' 216COLOR_LIGHT_PURPLE = '1;35' 217COLOR_LIGHT_CYAN = '1;36' 218COLOR_WHITE = '1;37' 219 220### helper functions ### 221def get_devnull(): 222 """Get the file object of '/dev/null' device.""" 223 try: 224 devnull = subprocess.DEVNULL # py3k 225 except AttributeError: 226 devnull = open(os.devnull, 'wb') 227 return devnull 228 229def check_top_directory(): 230 """Exit if we are not at the top of source directory.""" 231 for f in ('README', 'Licenses'): 232 if not os.path.exists(f): 233 sys.exit('Please run at the top of source directory.') 234 235def check_clean_directory(): 236 """Exit if the source tree is not clean.""" 237 for f in ('.config', 'include/config'): 238 if os.path.exists(f): 239 sys.exit("source tree is not clean, please run 'make mrproper'") 240 241def get_make_cmd(): 242 """Get the command name of GNU Make. 243 244 U-Boot needs GNU Make for building, but the command name is not 245 necessarily "make". (for example, "gmake" on FreeBSD). 246 Returns the most appropriate command name on your system. 247 """ 248 process = subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE) 249 ret = process.communicate() 250 if process.returncode: 251 sys.exit('GNU Make not found') 252 return ret[0].rstrip() 253 254def color_text(color_enabled, color, string): 255 """Return colored string.""" 256 if color_enabled: 257 # LF should not be surrounded by the escape sequence. 258 # Otherwise, additional whitespace or line-feed might be printed. 259 return '\n'.join([ '\033[' + color + 'm' + s + '\033[0m' if s else '' 260 for s in string.split('\n') ]) 261 else: 262 return string 263 264def update_cross_compile(color_enabled): 265 """Update per-arch CROSS_COMPILE via environment variables 266 267 The default CROSS_COMPILE values are available 268 in the CROSS_COMPILE list above. 269 270 You can override them via environment variables 271 CROSS_COMPILE_{ARCH}. 272 273 For example, if you want to override toolchain prefixes 274 for ARM and PowerPC, you can do as follows in your shell: 275 276 export CROSS_COMPILE_ARM=... 277 export CROSS_COMPILE_POWERPC=... 278 279 Then, this function checks if specified compilers really exist in your 280 PATH environment. 281 """ 282 archs = [] 283 284 for arch in os.listdir('arch'): 285 if os.path.exists(os.path.join('arch', arch, 'Makefile')): 286 archs.append(arch) 287 288 # arm64 is a special case 289 archs.append('aarch64') 290 291 for arch in archs: 292 env = 'CROSS_COMPILE_' + arch.upper() 293 cross_compile = os.environ.get(env) 294 if not cross_compile: 295 cross_compile = CROSS_COMPILE.get(arch, '') 296 297 for path in os.environ["PATH"].split(os.pathsep): 298 gcc_path = os.path.join(path, cross_compile + 'gcc') 299 if os.path.isfile(gcc_path) and os.access(gcc_path, os.X_OK): 300 break 301 else: 302 print >> sys.stderr, color_text(color_enabled, COLOR_YELLOW, 303 'warning: %sgcc: not found in PATH. %s architecture boards will be skipped' 304 % (cross_compile, arch)) 305 cross_compile = None 306 307 CROSS_COMPILE[arch] = cross_compile 308 309def cleanup_one_header(header_path, patterns, dry_run): 310 """Clean regex-matched lines away from a file. 311 312 Arguments: 313 header_path: path to the cleaned file. 314 patterns: list of regex patterns. Any lines matching to these 315 patterns are deleted. 316 dry_run: make no changes, but still display log. 317 """ 318 with open(header_path) as f: 319 lines = f.readlines() 320 321 matched = [] 322 for i, line in enumerate(lines): 323 for pattern in patterns: 324 m = pattern.search(line) 325 if m: 326 print '%s: %s: %s' % (header_path, i + 1, line), 327 matched.append(i) 328 break 329 330 if dry_run or not matched: 331 return 332 333 with open(header_path, 'w') as f: 334 for i, line in enumerate(lines): 335 if not i in matched: 336 f.write(line) 337 338def cleanup_headers(configs, dry_run): 339 """Delete config defines from board headers. 340 341 Arguments: 342 configs: A list of CONFIGs to remove. 343 dry_run: make no changes, but still display log. 344 """ 345 while True: 346 choice = raw_input('Clean up headers? [y/n]: ').lower() 347 print choice 348 if choice == 'y' or choice == 'n': 349 break 350 351 if choice == 'n': 352 return 353 354 patterns = [] 355 for config in configs: 356 patterns.append(re.compile(r'#\s*define\s+%s\W' % config)) 357 patterns.append(re.compile(r'#\s*undef\s+%s\W' % config)) 358 359 for dir in 'include', 'arch', 'board': 360 for (dirpath, dirnames, filenames) in os.walk(dir): 361 for filename in filenames: 362 if not fnmatch.fnmatch(filename, '*~'): 363 cleanup_one_header(os.path.join(dirpath, filename), 364 patterns, dry_run) 365 366### classes ### 367class Progress: 368 369 """Progress Indicator""" 370 371 def __init__(self, total): 372 """Create a new progress indicator. 373 374 Arguments: 375 total: A number of defconfig files to process. 376 """ 377 self.current = 0 378 self.total = total 379 380 def inc(self): 381 """Increment the number of processed defconfig files.""" 382 383 self.current += 1 384 385 def show(self): 386 """Display the progress.""" 387 print ' %d defconfigs out of %d\r' % (self.current, self.total), 388 sys.stdout.flush() 389 390class KconfigParser: 391 392 """A parser of .config and include/autoconf.mk.""" 393 394 re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"') 395 re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"') 396 397 def __init__(self, configs, options, build_dir): 398 """Create a new parser. 399 400 Arguments: 401 configs: A list of CONFIGs to move. 402 options: option flags. 403 build_dir: Build directory. 404 """ 405 self.configs = configs 406 self.options = options 407 self.dotconfig = os.path.join(build_dir, '.config') 408 self.autoconf = os.path.join(build_dir, 'include', 'autoconf.mk') 409 self.config_autoconf = os.path.join(build_dir, 'include', 'config', 410 'auto.conf') 411 self.defconfig = os.path.join(build_dir, 'defconfig') 412 413 def get_cross_compile(self): 414 """Parse .config file and return CROSS_COMPILE. 415 416 Returns: 417 A string storing the compiler prefix for the architecture. 418 Return a NULL string for architectures that do not require 419 compiler prefix (Sandbox and native build is the case). 420 Return None if the specified compiler is missing in your PATH. 421 Caller should distinguish '' and None. 422 """ 423 arch = '' 424 cpu = '' 425 for line in open(self.dotconfig): 426 m = self.re_arch.match(line) 427 if m: 428 arch = m.group(1) 429 continue 430 m = self.re_cpu.match(line) 431 if m: 432 cpu = m.group(1) 433 434 if not arch: 435 return None 436 437 # fix-up for aarch64 438 if arch == 'arm' and cpu == 'armv8': 439 arch = 'aarch64' 440 441 return CROSS_COMPILE.get(arch, None) 442 443 def parse_one_config(self, config, dotconfig_lines, autoconf_lines): 444 """Parse .config, defconfig, include/autoconf.mk for one config. 445 446 This function looks for the config options in the lines from 447 defconfig, .config, and include/autoconf.mk in order to decide 448 which action should be taken for this defconfig. 449 450 Arguments: 451 config: CONFIG name to parse. 452 dotconfig_lines: lines from the .config file. 453 autoconf_lines: lines from the include/autoconf.mk file. 454 455 Returns: 456 A tupple of the action for this defconfig and the line 457 matched for the config. 458 """ 459 not_set = '# %s is not set' % config 460 461 for line in dotconfig_lines: 462 line = line.rstrip() 463 if line.startswith(config + '=') or line == not_set: 464 old_val = line 465 break 466 else: 467 return (ACTION_NO_ENTRY, config) 468 469 for line in autoconf_lines: 470 line = line.rstrip() 471 if line.startswith(config + '='): 472 new_val = line 473 break 474 else: 475 new_val = not_set 476 477 if old_val == new_val: 478 return (ACTION_NO_CHANGE, new_val) 479 480 # If this CONFIG is neither bool nor trisate 481 if old_val[-2:] != '=y' and old_val[-2:] != '=m' and old_val != not_set: 482 # tools/scripts/define2mk.sed changes '1' to 'y'. 483 # This is a problem if the CONFIG is int type. 484 # Check the type in Kconfig and handle it correctly. 485 if new_val[-2:] == '=y': 486 new_val = new_val[:-1] + '1' 487 488 return (ACTION_MOVE, new_val) 489 490 def update_dotconfig(self): 491 """Parse files for the config options and update the .config. 492 493 This function parses the generated .config and include/autoconf.mk 494 searching the target options. 495 Move the config option(s) to the .config as needed. 496 497 Arguments: 498 defconfig: defconfig name. 499 500 Returns: 501 Return a tuple of (updated flag, log string). 502 The "updated flag" is True if the .config was updated, False 503 otherwise. The "log string" shows what happend to the .config. 504 """ 505 506 results = [] 507 updated = False 508 509 with open(self.dotconfig) as f: 510 dotconfig_lines = f.readlines() 511 512 with open(self.autoconf) as f: 513 autoconf_lines = f.readlines() 514 515 for config in self.configs: 516 result = self.parse_one_config(config, dotconfig_lines, 517 autoconf_lines) 518 results.append(result) 519 520 log = '' 521 522 for (action, value) in results: 523 if action == ACTION_MOVE: 524 actlog = "Move '%s'" % value 525 log_color = COLOR_LIGHT_GREEN 526 elif action == ACTION_NO_ENTRY: 527 actlog = "%s is not defined in Kconfig. Do nothing." % value 528 log_color = COLOR_LIGHT_BLUE 529 elif action == ACTION_NO_CHANGE: 530 actlog = "'%s' is the same as the define in Kconfig. Do nothing." \ 531 % value 532 log_color = COLOR_LIGHT_PURPLE 533 else: 534 sys.exit("Internal Error. This should not happen.") 535 536 log += color_text(self.options.color, log_color, actlog) + '\n' 537 538 with open(self.dotconfig, 'a') as f: 539 for (action, value) in results: 540 if action == ACTION_MOVE: 541 f.write(value + '\n') 542 updated = True 543 544 self.results = results 545 os.remove(self.config_autoconf) 546 os.remove(self.autoconf) 547 548 return (updated, log) 549 550 def check_defconfig(self): 551 """Check the defconfig after savedefconfig 552 553 Returns: 554 Return additional log if moved CONFIGs were removed again by 555 'make savedefconfig'. 556 """ 557 558 log = '' 559 560 with open(self.defconfig) as f: 561 defconfig_lines = f.readlines() 562 563 for (action, value) in self.results: 564 if action != ACTION_MOVE: 565 continue 566 if not value + '\n' in defconfig_lines: 567 log += color_text(self.options.color, COLOR_YELLOW, 568 "'%s' was removed by savedefconfig.\n" % 569 value) 570 571 return log 572 573class Slot: 574 575 """A slot to store a subprocess. 576 577 Each instance of this class handles one subprocess. 578 This class is useful to control multiple threads 579 for faster processing. 580 """ 581 582 def __init__(self, configs, options, progress, devnull, make_cmd): 583 """Create a new process slot. 584 585 Arguments: 586 configs: A list of CONFIGs to move. 587 options: option flags. 588 progress: A progress indicator. 589 devnull: A file object of '/dev/null'. 590 make_cmd: command name of GNU Make. 591 """ 592 self.options = options 593 self.progress = progress 594 self.build_dir = tempfile.mkdtemp() 595 self.devnull = devnull 596 self.make_cmd = (make_cmd, 'O=' + self.build_dir) 597 self.parser = KconfigParser(configs, options, self.build_dir) 598 self.state = STATE_IDLE 599 self.failed_boards = [] 600 601 def __del__(self): 602 """Delete the working directory 603 604 This function makes sure the temporary directory is cleaned away 605 even if Python suddenly dies due to error. It should be done in here 606 because it is guranteed the destructor is always invoked when the 607 instance of the class gets unreferenced. 608 609 If the subprocess is still running, wait until it finishes. 610 """ 611 if self.state != STATE_IDLE: 612 while self.ps.poll() == None: 613 pass 614 shutil.rmtree(self.build_dir) 615 616 def add(self, defconfig): 617 """Assign a new subprocess for defconfig and add it to the slot. 618 619 If the slot is vacant, create a new subprocess for processing the 620 given defconfig and add it to the slot. Just returns False if 621 the slot is occupied (i.e. the current subprocess is still running). 622 623 Arguments: 624 defconfig: defconfig name. 625 626 Returns: 627 Return True on success or False on failure 628 """ 629 if self.state != STATE_IDLE: 630 return False 631 cmd = list(self.make_cmd) 632 cmd.append(defconfig) 633 self.ps = subprocess.Popen(cmd, stdout=self.devnull, 634 stderr=subprocess.PIPE) 635 self.defconfig = defconfig 636 self.state = STATE_DEFCONFIG 637 self.log = '' 638 return True 639 640 def poll(self): 641 """Check the status of the subprocess and handle it as needed. 642 643 Returns True if the slot is vacant (i.e. in idle state). 644 If the configuration is successfully finished, assign a new 645 subprocess to build include/autoconf.mk. 646 If include/autoconf.mk is generated, invoke the parser to 647 parse the .config and the include/autoconf.mk, moving 648 config options to the .config as needed. 649 If the .config was updated, run "make savedefconfig" to sync 650 it, update the original defconfig, and then set the slot back 651 to the idle state. 652 653 Returns: 654 Return True if the subprocess is terminated, False otherwise 655 """ 656 if self.state == STATE_IDLE: 657 return True 658 659 if self.ps.poll() == None: 660 return False 661 662 if self.ps.poll() != 0: 663 self.log += color_text(self.options.color, COLOR_LIGHT_RED, 664 "Failed to process.\n") 665 if self.options.verbose: 666 self.log += color_text(self.options.color, COLOR_LIGHT_CYAN, 667 self.ps.stderr.read()) 668 self.finish(False) 669 return True 670 671 if self.state == STATE_AUTOCONF: 672 (updated, log) = self.parser.update_dotconfig() 673 self.log += log 674 675 if not updated: 676 self.finish(True) 677 return True 678 self.log += color_text(self.options.color, COLOR_LIGHT_GREEN, 679 "Syncing by savedefconfig...\n") 680 cmd = list(self.make_cmd) 681 cmd.append('savedefconfig') 682 self.ps = subprocess.Popen(cmd, stdout=self.devnull, 683 stderr=subprocess.PIPE) 684 self.state = STATE_SAVEDEFCONFIG 685 return False 686 687 if self.state == STATE_SAVEDEFCONFIG: 688 self.log += self.parser.check_defconfig() 689 orig_defconfig = os.path.join('configs', self.defconfig) 690 new_defconfig = os.path.join(self.build_dir, 'defconfig') 691 updated = not filecmp.cmp(orig_defconfig, new_defconfig) 692 693 if updated: 694 self.log += color_text(self.options.color, COLOR_LIGHT_GREEN, 695 "defconfig was updated.\n") 696 697 if not self.options.dry_run and updated: 698 shutil.move(new_defconfig, orig_defconfig) 699 self.finish(True) 700 return True 701 702 self.cross_compile = self.parser.get_cross_compile() 703 if self.cross_compile is None: 704 self.log += color_text(self.options.color, COLOR_YELLOW, 705 "Compiler is missing. Do nothing.\n") 706 self.finish(False) 707 return True 708 709 cmd = list(self.make_cmd) 710 if self.cross_compile: 711 cmd.append('CROSS_COMPILE=%s' % self.cross_compile) 712 cmd.append('KCONFIG_IGNORE_DUPLICATES=1') 713 cmd.append('include/config/auto.conf') 714 self.ps = subprocess.Popen(cmd, stdout=self.devnull, 715 stderr=subprocess.PIPE) 716 self.state = STATE_AUTOCONF 717 return False 718 719 def finish(self, success): 720 """Display log along with progress and go to the idle state. 721 722 Arguments: 723 success: Should be True when the defconfig was processed 724 successfully, or False when it fails. 725 """ 726 # output at least 30 characters to hide the "* defconfigs out of *". 727 log = self.defconfig.ljust(30) + '\n' 728 729 log += '\n'.join([ ' ' + s for s in self.log.split('\n') ]) 730 # Some threads are running in parallel. 731 # Print log atomically to not mix up logs from different threads. 732 print >> (sys.stdout if success else sys.stderr), log 733 734 if not success: 735 if self.options.exit_on_error: 736 sys.exit("Exit on error.") 737 # If --exit-on-error flag is not set, skip this board and continue. 738 # Record the failed board. 739 self.failed_boards.append(self.defconfig) 740 741 self.progress.inc() 742 self.progress.show() 743 self.state = STATE_IDLE 744 745 def get_failed_boards(self): 746 """Returns a list of failed boards (defconfigs) in this slot. 747 """ 748 return self.failed_boards 749 750class Slots: 751 752 """Controller of the array of subprocess slots.""" 753 754 def __init__(self, configs, options, progress): 755 """Create a new slots controller. 756 757 Arguments: 758 configs: A list of CONFIGs to move. 759 options: option flags. 760 progress: A progress indicator. 761 """ 762 self.options = options 763 self.slots = [] 764 devnull = get_devnull() 765 make_cmd = get_make_cmd() 766 for i in range(options.jobs): 767 self.slots.append(Slot(configs, options, progress, devnull, 768 make_cmd)) 769 770 def add(self, defconfig): 771 """Add a new subprocess if a vacant slot is found. 772 773 Arguments: 774 defconfig: defconfig name to be put into. 775 776 Returns: 777 Return True on success or False on failure 778 """ 779 for slot in self.slots: 780 if slot.add(defconfig): 781 return True 782 return False 783 784 def available(self): 785 """Check if there is a vacant slot. 786 787 Returns: 788 Return True if at lease one vacant slot is found, False otherwise. 789 """ 790 for slot in self.slots: 791 if slot.poll(): 792 return True 793 return False 794 795 def empty(self): 796 """Check if all slots are vacant. 797 798 Returns: 799 Return True if all the slots are vacant, False otherwise. 800 """ 801 ret = True 802 for slot in self.slots: 803 if not slot.poll(): 804 ret = False 805 return ret 806 807 def show_failed_boards(self): 808 """Display all of the failed boards (defconfigs).""" 809 failed_boards = [] 810 811 for slot in self.slots: 812 failed_boards += slot.get_failed_boards() 813 814 if len(failed_boards) > 0: 815 msg = [ "The following boards were not processed due to error:" ] 816 msg += failed_boards 817 for line in msg: 818 print >> sys.stderr, color_text(self.options.color, 819 COLOR_LIGHT_RED, line) 820 821 with open('moveconfig.failed', 'w') as f: 822 for board in failed_boards: 823 f.write(board + '\n') 824 825def move_config(configs, options): 826 """Move config options to defconfig files. 827 828 Arguments: 829 configs: A list of CONFIGs to move. 830 options: option flags 831 """ 832 if len(configs) == 0: 833 print 'Nothing to do. exit.' 834 sys.exit(0) 835 836 print 'Move %s (jobs: %d)' % (', '.join(configs), options.jobs) 837 838 if options.defconfigs: 839 defconfigs = [line.strip() for line in open(options.defconfigs)] 840 for i, defconfig in enumerate(defconfigs): 841 if not defconfig.endswith('_defconfig'): 842 defconfigs[i] = defconfig + '_defconfig' 843 if not os.path.exists(os.path.join('configs', defconfigs[i])): 844 sys.exit('%s - defconfig does not exist. Stopping.' % 845 defconfigs[i]) 846 else: 847 # All the defconfig files to be processed 848 defconfigs = [] 849 for (dirpath, dirnames, filenames) in os.walk('configs'): 850 dirpath = dirpath[len('configs') + 1:] 851 for filename in fnmatch.filter(filenames, '*_defconfig'): 852 defconfigs.append(os.path.join(dirpath, filename)) 853 854 progress = Progress(len(defconfigs)) 855 slots = Slots(configs, options, progress) 856 857 # Main loop to process defconfig files: 858 # Add a new subprocess into a vacant slot. 859 # Sleep if there is no available slot. 860 for defconfig in defconfigs: 861 while not slots.add(defconfig): 862 while not slots.available(): 863 # No available slot: sleep for a while 864 time.sleep(SLEEP_TIME) 865 866 # wait until all the subprocesses finish 867 while not slots.empty(): 868 time.sleep(SLEEP_TIME) 869 870 print '' 871 slots.show_failed_boards() 872 873def main(): 874 try: 875 cpu_count = multiprocessing.cpu_count() 876 except NotImplementedError: 877 cpu_count = 1 878 879 parser = optparse.OptionParser() 880 # Add options here 881 parser.add_option('-c', '--color', action='store_true', default=False, 882 help='display the log in color') 883 parser.add_option('-d', '--defconfigs', type='string', 884 help='a file containing a list of defconfigs to move') 885 parser.add_option('-n', '--dry-run', action='store_true', default=False, 886 help='perform a trial run (show log with no changes)') 887 parser.add_option('-e', '--exit-on-error', action='store_true', 888 default=False, 889 help='exit immediately on any error') 890 parser.add_option('-H', '--headers-only', dest='cleanup_headers_only', 891 action='store_true', default=False, 892 help='only cleanup the headers') 893 parser.add_option('-j', '--jobs', type='int', default=cpu_count, 894 help='the number of jobs to run simultaneously') 895 parser.add_option('-v', '--verbose', action='store_true', default=False, 896 help='show any build errors as boards are built') 897 parser.usage += ' CONFIG ...' 898 899 (options, configs) = parser.parse_args() 900 901 if len(configs) == 0: 902 parser.print_usage() 903 sys.exit(1) 904 905 # prefix the option name with CONFIG_ if missing 906 configs = [ config if config.startswith('CONFIG_') else 'CONFIG_' + config 907 for config in configs ] 908 909 check_top_directory() 910 911 check_clean_directory() 912 913 update_cross_compile(options.color) 914 915 if not options.cleanup_headers_only: 916 move_config(configs, options) 917 918 cleanup_headers(configs, options.dry_run) 919 920if __name__ == '__main__': 921 main() 922