1#!/usr/bin/env python 2# 3# Author: Masahiro Yamada <yamada.m@jp.panasonic.com> 4# 5# SPDX-License-Identifier: GPL-2.0+ 6# 7 8""" 9Converter from Kconfig and MAINTAINERS to boards.cfg 10 11Run 'tools/genboardscfg.py' to create boards.cfg file. 12 13Run 'tools/genboardscfg.py -h' for available options. 14""" 15 16import errno 17import fnmatch 18import glob 19import optparse 20import os 21import re 22import shutil 23import subprocess 24import sys 25import tempfile 26import time 27 28BOARD_FILE = 'boards.cfg' 29CONFIG_DIR = 'configs' 30REFORMAT_CMD = [os.path.join('tools', 'reformat.py'), 31 '-i', '-d', '-', '-s', '8'] 32SHOW_GNU_MAKE = 'scripts/show-gnu-make' 33SLEEP_TIME=0.003 34 35COMMENT_BLOCK = '''# 36# List of boards 37# Automatically generated by %s: don't edit 38# 39# Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers 40 41''' % __file__ 42 43### helper functions ### 44def get_terminal_columns(): 45 """Get the width of the terminal. 46 47 Returns: 48 The width of the terminal, or zero if the stdout is not 49 associated with tty. 50 """ 51 try: 52 return shutil.get_terminal_size().columns # Python 3.3~ 53 except AttributeError: 54 import fcntl 55 import termios 56 import struct 57 arg = struct.pack('hhhh', 0, 0, 0, 0) 58 try: 59 ret = fcntl.ioctl(sys.stdout.fileno(), termios.TIOCGWINSZ, arg) 60 except IOError as exception: 61 # If 'Inappropriate ioctl for device' error occurs, 62 # stdout is probably redirected. Return 0. 63 return 0 64 return struct.unpack('hhhh', ret)[1] 65 66def get_devnull(): 67 """Get the file object of '/dev/null' device.""" 68 try: 69 devnull = subprocess.DEVNULL # py3k 70 except AttributeError: 71 devnull = open(os.devnull, 'wb') 72 return devnull 73 74def check_top_directory(): 75 """Exit if we are not at the top of source directory.""" 76 for f in ('README', 'Licenses'): 77 if not os.path.exists(f): 78 sys.exit('Please run at the top of source directory.') 79 80def get_make_cmd(): 81 """Get the command name of GNU Make.""" 82 process = subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE) 83 ret = process.communicate() 84 if process.returncode: 85 sys.exit('GNU Make not found') 86 return ret[0].rstrip() 87 88def output_is_new(): 89 """Check if the boards.cfg file is up to date. 90 91 Returns: 92 True if the boards.cfg file exists and is newer than any of 93 *_defconfig, MAINTAINERS and Kconfig*. False otherwise. 94 """ 95 try: 96 ctime = os.path.getctime(BOARD_FILE) 97 except OSError as exception: 98 if exception.errno == errno.ENOENT: 99 # return False on 'No such file or directory' error 100 return False 101 else: 102 raise 103 104 for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR): 105 for filename in fnmatch.filter(filenames, '*_defconfig'): 106 if fnmatch.fnmatch(filename, '.*'): 107 continue 108 filepath = os.path.join(dirpath, filename) 109 if ctime < os.path.getctime(filepath): 110 return False 111 112 for (dirpath, dirnames, filenames) in os.walk('.'): 113 for filename in filenames: 114 if (fnmatch.fnmatch(filename, '*~') or 115 not fnmatch.fnmatch(filename, 'Kconfig*') and 116 not filename == 'MAINTAINERS'): 117 continue 118 filepath = os.path.join(dirpath, filename) 119 if ctime < os.path.getctime(filepath): 120 return False 121 122 # Detect a board that has been removed since the current boards.cfg 123 # was generated 124 with open(BOARD_FILE) as f: 125 for line in f: 126 if line[0] == '#' or line == '\n': 127 continue 128 defconfig = line.split()[6] + '_defconfig' 129 if not os.path.exists(os.path.join(CONFIG_DIR, defconfig)): 130 return False 131 132 return True 133 134### classes ### 135class MaintainersDatabase: 136 137 """The database of board status and maintainers.""" 138 139 def __init__(self): 140 """Create an empty database.""" 141 self.database = {} 142 143 def get_status(self, target): 144 """Return the status of the given board. 145 146 Returns: 147 Either 'Active' or 'Orphan' 148 """ 149 if not target in self.database: 150 print >> sys.stderr, "WARNING: no status info for '%s'" % target 151 return '-' 152 153 tmp = self.database[target][0] 154 if tmp.startswith('Maintained'): 155 return 'Active' 156 elif tmp.startswith('Orphan'): 157 return 'Orphan' 158 else: 159 print >> sys.stderr, ("WARNING: %s: unknown status for '%s'" % 160 (tmp, target)) 161 return '-' 162 163 def get_maintainers(self, target): 164 """Return the maintainers of the given board. 165 166 If the board has two or more maintainers, they are separated 167 with colons. 168 """ 169 if not target in self.database: 170 print >> sys.stderr, "WARNING: no maintainers for '%s'" % target 171 return '' 172 173 return ':'.join(self.database[target][1]) 174 175 def parse_file(self, file): 176 """Parse the given MAINTAINERS file. 177 178 This method parses MAINTAINERS and add board status and 179 maintainers information to the database. 180 181 Arguments: 182 file: MAINTAINERS file to be parsed 183 """ 184 targets = [] 185 maintainers = [] 186 status = '-' 187 for line in open(file): 188 tag, rest = line[:2], line[2:].strip() 189 if tag == 'M:': 190 maintainers.append(rest) 191 elif tag == 'F:': 192 # expand wildcard and filter by 'configs/*_defconfig' 193 for f in glob.glob(rest): 194 front, match, rear = f.partition('configs/') 195 if not front and match: 196 front, match, rear = rear.rpartition('_defconfig') 197 if match and not rear: 198 targets.append(front) 199 elif tag == 'S:': 200 status = rest 201 elif line == '\n': 202 for target in targets: 203 self.database[target] = (status, maintainers) 204 targets = [] 205 maintainers = [] 206 status = '-' 207 if targets: 208 for target in targets: 209 self.database[target] = (status, maintainers) 210 211class DotConfigParser: 212 213 """A parser of .config file. 214 215 Each line of the output should have the form of: 216 Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers 217 Most of them are extracted from .config file. 218 MAINTAINERS files are also consulted for Status and Maintainers fields. 219 """ 220 221 re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"') 222 re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"') 223 re_soc = re.compile(r'CONFIG_SYS_SOC="(.*)"') 224 re_vendor = re.compile(r'CONFIG_SYS_VENDOR="(.*)"') 225 re_board = re.compile(r'CONFIG_SYS_BOARD="(.*)"') 226 re_config = re.compile(r'CONFIG_SYS_CONFIG_NAME="(.*)"') 227 re_options = re.compile(r'CONFIG_SYS_EXTRA_OPTIONS="(.*)"') 228 re_list = (('arch', re_arch), ('cpu', re_cpu), ('soc', re_soc), 229 ('vendor', re_vendor), ('board', re_board), 230 ('config', re_config), ('options', re_options)) 231 must_fields = ('arch', 'config') 232 233 def __init__(self, build_dir, output, maintainers_database): 234 """Create a new .config perser. 235 236 Arguments: 237 build_dir: Build directory where .config is located 238 output: File object which the result is written to 239 maintainers_database: An instance of class MaintainersDatabase 240 """ 241 self.dotconfig = os.path.join(build_dir, '.config') 242 self.output = output 243 self.database = maintainers_database 244 245 def parse(self, defconfig): 246 """Parse .config file and output one-line database for the given board. 247 248 Arguments: 249 defconfig: Board (defconfig) name 250 """ 251 fields = {} 252 for line in open(self.dotconfig): 253 if not line.startswith('CONFIG_SYS_'): 254 continue 255 for (key, pattern) in self.re_list: 256 m = pattern.match(line) 257 if m and m.group(1): 258 fields[key] = m.group(1) 259 break 260 261 # sanity check of '.config' file 262 for field in self.must_fields: 263 if not field in fields: 264 print >> sys.stderr, ( 265 "WARNING: '%s' is not defined in '%s'. Skip." % 266 (field, defconfig)) 267 return 268 269 # fix-up for aarch64 270 if fields['arch'] == 'arm' and 'cpu' in fields: 271 if fields['cpu'] == 'armv8': 272 fields['arch'] = 'aarch64' 273 274 target, match, rear = defconfig.partition('_defconfig') 275 assert match and not rear, \ 276 '%s : invalid defconfig file name' % defconfig 277 278 fields['status'] = self.database.get_status(target) 279 fields['maintainers'] = self.database.get_maintainers(target) 280 281 if 'options' in fields: 282 options = fields['config'] + ':' + \ 283 fields['options'].replace(r'\"', '"') 284 elif fields['config'] != target: 285 options = fields['config'] 286 else: 287 options = '-' 288 289 self.output.write((' '.join(['%s'] * 9) + '\n') % 290 (fields['status'], 291 fields['arch'], 292 fields.get('cpu', '-'), 293 fields.get('soc', '-'), 294 fields.get('vendor', '-'), 295 fields.get('board', '-'), 296 target, 297 options, 298 fields['maintainers'])) 299 300class Slot: 301 302 """A slot to store a subprocess. 303 304 Each instance of this class handles one subprocess. 305 This class is useful to control multiple processes 306 for faster processing. 307 """ 308 309 def __init__(self, output, maintainers_database, devnull, make_cmd): 310 """Create a new slot. 311 312 Arguments: 313 output: File object which the result is written to 314 maintainers_database: An instance of class MaintainersDatabase 315 devnull: file object of 'dev/null' 316 make_cmd: the command name of Make 317 """ 318 self.build_dir = tempfile.mkdtemp() 319 self.devnull = devnull 320 self.ps = subprocess.Popen([make_cmd, 'O=' + self.build_dir, 321 'allnoconfig'], stdout=devnull) 322 self.occupied = True 323 self.parser = DotConfigParser(self.build_dir, output, 324 maintainers_database) 325 self.env = os.environ.copy() 326 self.env['srctree'] = os.getcwd() 327 self.env['UBOOTVERSION'] = 'dummy' 328 self.env['KCONFIG_OBJDIR'] = '' 329 330 def __del__(self): 331 """Delete the working directory""" 332 if not self.occupied: 333 while self.ps.poll() == None: 334 pass 335 shutil.rmtree(self.build_dir) 336 337 def add(self, defconfig): 338 """Add a new subprocess to the slot. 339 340 Fails if the slot is occupied, that is, the current subprocess 341 is still running. 342 343 Arguments: 344 defconfig: Board (defconfig) name 345 346 Returns: 347 Return True on success or False on fail 348 """ 349 if self.occupied: 350 return False 351 352 with open(os.path.join(self.build_dir, '.tmp_defconfig'), 'w') as f: 353 for line in open(os.path.join(CONFIG_DIR, defconfig)): 354 colon = line.find(':CONFIG_') 355 if colon == -1: 356 f.write(line) 357 else: 358 f.write(line[colon + 1:]) 359 360 self.ps = subprocess.Popen([os.path.join('scripts', 'kconfig', 'conf'), 361 '--defconfig=.tmp_defconfig', 'Kconfig'], 362 stdout=self.devnull, 363 cwd=self.build_dir, 364 env=self.env) 365 366 self.defconfig = defconfig 367 self.occupied = True 368 return True 369 370 def wait(self): 371 """Wait until the current subprocess finishes.""" 372 while self.occupied and self.ps.poll() == None: 373 time.sleep(SLEEP_TIME) 374 self.occupied = False 375 376 def poll(self): 377 """Check if the subprocess is running and invoke the .config 378 parser if the subprocess is terminated. 379 380 Returns: 381 Return True if the subprocess is terminated, False otherwise 382 """ 383 if not self.occupied: 384 return True 385 if self.ps.poll() == None: 386 return False 387 if self.ps.poll() == 0: 388 self.parser.parse(self.defconfig) 389 else: 390 print >> sys.stderr, ("WARNING: failed to process '%s'. skip." % 391 self.defconfig) 392 self.occupied = False 393 return True 394 395class Slots: 396 397 """Controller of the array of subprocess slots.""" 398 399 def __init__(self, jobs, output, maintainers_database): 400 """Create a new slots controller. 401 402 Arguments: 403 jobs: A number of slots to instantiate 404 output: File object which the result is written to 405 maintainers_database: An instance of class MaintainersDatabase 406 """ 407 self.slots = [] 408 devnull = get_devnull() 409 make_cmd = get_make_cmd() 410 for i in range(jobs): 411 self.slots.append(Slot(output, maintainers_database, 412 devnull, make_cmd)) 413 for slot in self.slots: 414 slot.wait() 415 416 def add(self, defconfig): 417 """Add a new subprocess if a vacant slot is available. 418 419 Arguments: 420 defconfig: Board (defconfig) name 421 422 Returns: 423 Return True on success or False on fail 424 """ 425 for slot in self.slots: 426 if slot.add(defconfig): 427 return True 428 return False 429 430 def available(self): 431 """Check if there is a vacant slot. 432 433 Returns: 434 Return True if a vacant slot is found, False if all slots are full 435 """ 436 for slot in self.slots: 437 if slot.poll(): 438 return True 439 return False 440 441 def empty(self): 442 """Check if all slots are vacant. 443 444 Returns: 445 Return True if all slots are vacant, False if at least one slot 446 is running 447 """ 448 ret = True 449 for slot in self.slots: 450 if not slot.poll(): 451 ret = False 452 return ret 453 454class Indicator: 455 456 """A class to control the progress indicator.""" 457 458 MIN_WIDTH = 15 459 MAX_WIDTH = 70 460 461 def __init__(self, total): 462 """Create an instance. 463 464 Arguments: 465 total: A number of boards 466 """ 467 self.total = total 468 self.cur = 0 469 width = get_terminal_columns() 470 width = min(width, self.MAX_WIDTH) 471 width -= self.MIN_WIDTH 472 if width > 0: 473 self.enabled = True 474 else: 475 self.enabled = False 476 self.width = width 477 478 def inc(self): 479 """Increment the counter and show the progress bar.""" 480 if not self.enabled: 481 return 482 self.cur += 1 483 arrow_len = self.width * self.cur // self.total 484 msg = '%4d/%d [' % (self.cur, self.total) 485 msg += '=' * arrow_len + '>' + ' ' * (self.width - arrow_len) + ']' 486 sys.stdout.write('\r' + msg) 487 sys.stdout.flush() 488 489class BoardsFileGenerator: 490 491 """Generator of boards.cfg.""" 492 493 def __init__(self): 494 """Prepare basic things for generating boards.cfg.""" 495 # All the defconfig files to be processed 496 defconfigs = [] 497 for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR): 498 dirpath = dirpath[len(CONFIG_DIR) + 1:] 499 for filename in fnmatch.filter(filenames, '*_defconfig'): 500 if fnmatch.fnmatch(filename, '.*'): 501 continue 502 defconfigs.append(os.path.join(dirpath, filename)) 503 self.defconfigs = defconfigs 504 self.indicator = Indicator(len(defconfigs)) 505 506 # Parse all the MAINTAINERS files 507 maintainers_database = MaintainersDatabase() 508 for (dirpath, dirnames, filenames) in os.walk('.'): 509 if 'MAINTAINERS' in filenames: 510 maintainers_database.parse_file(os.path.join(dirpath, 511 'MAINTAINERS')) 512 self.maintainers_database = maintainers_database 513 514 def __del__(self): 515 """Delete the incomplete boards.cfg 516 517 This destructor deletes boards.cfg if the private member 'in_progress' 518 is defined as True. The 'in_progress' member is set to True at the 519 beginning of the generate() method and set to False at its end. 520 So, in_progress==True means generating boards.cfg was terminated 521 on the way. 522 """ 523 524 if hasattr(self, 'in_progress') and self.in_progress: 525 try: 526 os.remove(BOARD_FILE) 527 except OSError as exception: 528 # Ignore 'No such file or directory' error 529 if exception.errno != errno.ENOENT: 530 raise 531 print 'Removed incomplete %s' % BOARD_FILE 532 533 def generate(self, jobs): 534 """Generate boards.cfg 535 536 This method sets the 'in_progress' member to True at the beginning 537 and sets it to False on success. The boards.cfg should not be 538 touched before/after this method because 'in_progress' is used 539 to detect the incomplete boards.cfg. 540 541 Arguments: 542 jobs: The number of jobs to run simultaneously 543 """ 544 545 self.in_progress = True 546 print 'Generating %s ... (jobs: %d)' % (BOARD_FILE, jobs) 547 548 # Output lines should be piped into the reformat tool 549 reformat_process = subprocess.Popen(REFORMAT_CMD, 550 stdin=subprocess.PIPE, 551 stdout=open(BOARD_FILE, 'w')) 552 pipe = reformat_process.stdin 553 pipe.write(COMMENT_BLOCK) 554 555 slots = Slots(jobs, pipe, self.maintainers_database) 556 557 # Main loop to process defconfig files: 558 # Add a new subprocess into a vacant slot. 559 # Sleep if there is no available slot. 560 for defconfig in self.defconfigs: 561 while not slots.add(defconfig): 562 while not slots.available(): 563 # No available slot: sleep for a while 564 time.sleep(SLEEP_TIME) 565 self.indicator.inc() 566 567 # wait until all the subprocesses finish 568 while not slots.empty(): 569 time.sleep(SLEEP_TIME) 570 print '' 571 572 # wait until the reformat tool finishes 573 reformat_process.communicate() 574 if reformat_process.returncode != 0: 575 sys.exit('"%s" failed' % REFORMAT_CMD[0]) 576 577 self.in_progress = False 578 579def gen_boards_cfg(jobs=1, force=False): 580 """Generate boards.cfg file. 581 582 The incomplete boards.cfg is deleted if an error (including 583 the termination by the keyboard interrupt) occurs on the halfway. 584 585 Arguments: 586 jobs: The number of jobs to run simultaneously 587 """ 588 check_top_directory() 589 if not force and output_is_new(): 590 print "%s is up to date. Nothing to do." % BOARD_FILE 591 sys.exit(0) 592 593 generator = BoardsFileGenerator() 594 generator.generate(jobs) 595 596def main(): 597 parser = optparse.OptionParser() 598 # Add options here 599 parser.add_option('-j', '--jobs', 600 help='the number of jobs to run simultaneously') 601 parser.add_option('-f', '--force', action="store_true", default=False, 602 help='regenerate the output even if it is new') 603 (options, args) = parser.parse_args() 604 605 if options.jobs: 606 try: 607 jobs = int(options.jobs) 608 except ValueError: 609 sys.exit('Option -j (--jobs) takes a number') 610 else: 611 try: 612 jobs = int(subprocess.Popen(['getconf', '_NPROCESSORS_ONLN'], 613 stdout=subprocess.PIPE).communicate()[0]) 614 except (OSError, ValueError): 615 print 'info: failed to get the number of CPUs. Set jobs to 1' 616 jobs = 1 617 618 gen_boards_cfg(jobs, force=options.force) 619 620if __name__ == '__main__': 621 main() 622