1#!/usr/bin/env python 2# 3# Author: Masahiro Yamada <yamada.m@jp.panasonic.com> 4# 5# SPDX-License-Identifier: GPL-2.0+ 6# 7 8""" 9Converter from Kconfig and MAINTAINERS to boards.cfg 10 11Run 'tools/genboardscfg.py' to create boards.cfg file. 12 13Run 'tools/genboardscfg.py -h' for available options. 14""" 15 16import errno 17import fnmatch 18import glob 19import optparse 20import os 21import re 22import shutil 23import subprocess 24import sys 25import tempfile 26import time 27 28BOARD_FILE = 'boards.cfg' 29CONFIG_DIR = 'configs' 30REFORMAT_CMD = [os.path.join('tools', 'reformat.py'), 31 '-i', '-d', '-', '-s', '8'] 32SHOW_GNU_MAKE = 'scripts/show-gnu-make' 33SLEEP_TIME=0.03 34 35COMMENT_BLOCK = '''# 36# List of boards 37# Automatically generated by %s: don't edit 38# 39# Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers 40 41''' % __file__ 42 43### helper functions ### 44def get_terminal_columns(): 45 """Get the width of the terminal. 46 47 Returns: 48 The width of the terminal, or zero if the stdout is not 49 associated with tty. 50 """ 51 try: 52 return shutil.get_terminal_size().columns # Python 3.3~ 53 except AttributeError: 54 import fcntl 55 import termios 56 import struct 57 arg = struct.pack('hhhh', 0, 0, 0, 0) 58 try: 59 ret = fcntl.ioctl(sys.stdout.fileno(), termios.TIOCGWINSZ, arg) 60 except IOError as exception: 61 # If 'Inappropriate ioctl for device' error occurs, 62 # stdout is probably redirected. Return 0. 63 return 0 64 return struct.unpack('hhhh', ret)[1] 65 66def get_devnull(): 67 """Get the file object of '/dev/null' device.""" 68 try: 69 devnull = subprocess.DEVNULL # py3k 70 except AttributeError: 71 devnull = open(os.devnull, 'wb') 72 return devnull 73 74def check_top_directory(): 75 """Exit if we are not at the top of source directory.""" 76 for f in ('README', 'Licenses'): 77 if not os.path.exists(f): 78 sys.exit('Please run at the top of source directory.') 79 80def get_make_cmd(): 81 """Get the command name of GNU Make.""" 82 process = subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE) 83 ret = process.communicate() 84 if process.returncode: 85 sys.exit('GNU Make not found') 86 return ret[0].rstrip() 87 88def output_is_new(): 89 """Check if the boards.cfg file is up to date. 90 91 Returns: 92 True if the boards.cfg file exists and is newer than any of 93 *_defconfig, MAINTAINERS and Kconfig*. False otherwise. 94 """ 95 try: 96 ctime = os.path.getctime(BOARD_FILE) 97 except OSError as exception: 98 if exception.errno == errno.ENOENT: 99 # return False on 'No such file or directory' error 100 return False 101 else: 102 raise 103 104 for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR): 105 for filename in fnmatch.filter(filenames, '*_defconfig'): 106 if fnmatch.fnmatch(filename, '.*'): 107 continue 108 filepath = os.path.join(dirpath, filename) 109 if ctime < os.path.getctime(filepath): 110 return False 111 112 for (dirpath, dirnames, filenames) in os.walk('.'): 113 for filename in filenames: 114 if (fnmatch.fnmatch(filename, '*~') or 115 not fnmatch.fnmatch(filename, 'Kconfig*') and 116 not filename == 'MAINTAINERS'): 117 continue 118 filepath = os.path.join(dirpath, filename) 119 if ctime < os.path.getctime(filepath): 120 return False 121 122 # Detect a board that has been removed since the current boards.cfg 123 # was generated 124 with open(BOARD_FILE) as f: 125 for line in f: 126 if line[0] == '#' or line == '\n': 127 continue 128 defconfig = line.split()[6] + '_defconfig' 129 if not os.path.exists(os.path.join(CONFIG_DIR, defconfig)): 130 return False 131 132 return True 133 134### classes ### 135class MaintainersDatabase: 136 137 """The database of board status and maintainers.""" 138 139 def __init__(self): 140 """Create an empty database.""" 141 self.database = {} 142 143 def get_status(self, target): 144 """Return the status of the given board. 145 146 Returns: 147 Either 'Active' or 'Orphan' 148 """ 149 if not target in self.database: 150 print >> sys.stderr, "WARNING: no status info for '%s'" % target 151 return '-' 152 153 tmp = self.database[target][0] 154 if tmp.startswith('Maintained'): 155 return 'Active' 156 elif tmp.startswith('Orphan'): 157 return 'Orphan' 158 else: 159 print >> sys.stderr, ("WARNING: %s: unknown status for '%s'" % 160 (tmp, target)) 161 return '-' 162 163 def get_maintainers(self, target): 164 """Return the maintainers of the given board. 165 166 If the board has two or more maintainers, they are separated 167 with colons. 168 """ 169 if not target in self.database: 170 print >> sys.stderr, "WARNING: no maintainers for '%s'" % target 171 return '' 172 173 return ':'.join(self.database[target][1]) 174 175 def parse_file(self, file): 176 """Parse the given MAINTAINERS file. 177 178 This method parses MAINTAINERS and add board status and 179 maintainers information to the database. 180 181 Arguments: 182 file: MAINTAINERS file to be parsed 183 """ 184 targets = [] 185 maintainers = [] 186 status = '-' 187 for line in open(file): 188 tag, rest = line[:2], line[2:].strip() 189 if tag == 'M:': 190 maintainers.append(rest) 191 elif tag == 'F:': 192 # expand wildcard and filter by 'configs/*_defconfig' 193 for f in glob.glob(rest): 194 front, match, rear = f.partition('configs/') 195 if not front and match: 196 front, match, rear = rear.rpartition('_defconfig') 197 if match and not rear: 198 targets.append(front) 199 elif tag == 'S:': 200 status = rest 201 elif line == '\n': 202 for target in targets: 203 self.database[target] = (status, maintainers) 204 targets = [] 205 maintainers = [] 206 status = '-' 207 if targets: 208 for target in targets: 209 self.database[target] = (status, maintainers) 210 211class DotConfigParser: 212 213 """A parser of .config file. 214 215 Each line of the output should have the form of: 216 Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers 217 Most of them are extracted from .config file. 218 MAINTAINERS files are also consulted for Status and Maintainers fields. 219 """ 220 221 re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"') 222 re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"') 223 re_soc = re.compile(r'CONFIG_SYS_SOC="(.*)"') 224 re_vendor = re.compile(r'CONFIG_SYS_VENDOR="(.*)"') 225 re_board = re.compile(r'CONFIG_SYS_BOARD="(.*)"') 226 re_config = re.compile(r'CONFIG_SYS_CONFIG_NAME="(.*)"') 227 re_options = re.compile(r'CONFIG_SYS_EXTRA_OPTIONS="(.*)"') 228 re_list = (('arch', re_arch), ('cpu', re_cpu), ('soc', re_soc), 229 ('vendor', re_vendor), ('board', re_board), 230 ('config', re_config), ('options', re_options)) 231 must_fields = ('arch', 'config') 232 233 def __init__(self, build_dir, output, maintainers_database): 234 """Create a new .config perser. 235 236 Arguments: 237 build_dir: Build directory where .config is located 238 output: File object which the result is written to 239 maintainers_database: An instance of class MaintainersDatabase 240 """ 241 self.dotconfig = os.path.join(build_dir, '.config') 242 self.output = output 243 self.database = maintainers_database 244 245 def parse(self, defconfig): 246 """Parse .config file and output one-line database for the given board. 247 248 Arguments: 249 defconfig: Board (defconfig) name 250 """ 251 fields = {} 252 for line in open(self.dotconfig): 253 if not line.startswith('CONFIG_SYS_'): 254 continue 255 for (key, pattern) in self.re_list: 256 m = pattern.match(line) 257 if m and m.group(1): 258 fields[key] = m.group(1) 259 break 260 261 # sanity check of '.config' file 262 for field in self.must_fields: 263 if not field in fields: 264 print >> sys.stderr, ( 265 "WARNING: '%s' is not defined in '%s'. Skip." % 266 (field, defconfig)) 267 return 268 269 # fix-up for aarch64 270 if fields['arch'] == 'arm' and 'cpu' in fields: 271 if fields['cpu'] == 'armv8': 272 fields['arch'] = 'aarch64' 273 274 target, match, rear = defconfig.partition('_defconfig') 275 assert match and not rear, \ 276 '%s : invalid defconfig file name' % defconfig 277 278 fields['status'] = self.database.get_status(target) 279 fields['maintainers'] = self.database.get_maintainers(target) 280 281 if 'options' in fields: 282 options = fields['config'] + ':' + \ 283 fields['options'].replace(r'\"', '"') 284 elif fields['config'] != target: 285 options = fields['config'] 286 else: 287 options = '-' 288 289 self.output.write((' '.join(['%s'] * 9) + '\n') % 290 (fields['status'], 291 fields['arch'], 292 fields.get('cpu', '-'), 293 fields.get('soc', '-'), 294 fields.get('vendor', '-'), 295 fields.get('board', '-'), 296 target, 297 options, 298 fields['maintainers'])) 299 300class Slot: 301 302 """A slot to store a subprocess. 303 304 Each instance of this class handles one subprocess. 305 This class is useful to control multiple processes 306 for faster processing. 307 """ 308 309 def __init__(self, output, maintainers_database, devnull, make_cmd): 310 """Create a new slot. 311 312 Arguments: 313 output: File object which the result is written to 314 maintainers_database: An instance of class MaintainersDatabase 315 """ 316 self.occupied = False 317 self.build_dir = tempfile.mkdtemp() 318 self.devnull = devnull 319 self.make_cmd = make_cmd 320 self.parser = DotConfigParser(self.build_dir, output, 321 maintainers_database) 322 323 def __del__(self): 324 """Delete the working directory""" 325 if not self.occupied: 326 while self.ps.poll() == None: 327 pass 328 shutil.rmtree(self.build_dir) 329 330 def add(self, defconfig): 331 """Add a new subprocess to the slot. 332 333 Fails if the slot is occupied, that is, the current subprocess 334 is still running. 335 336 Arguments: 337 defconfig: Board (defconfig) name 338 339 Returns: 340 Return True on success or False on fail 341 """ 342 if self.occupied: 343 return False 344 o = 'O=' + self.build_dir 345 self.ps = subprocess.Popen([self.make_cmd, o, defconfig], 346 stdout=self.devnull) 347 self.defconfig = defconfig 348 self.occupied = True 349 return True 350 351 def poll(self): 352 """Check if the subprocess is running and invoke the .config 353 parser if the subprocess is terminated. 354 355 Returns: 356 Return True if the subprocess is terminated, False otherwise 357 """ 358 if not self.occupied: 359 return True 360 if self.ps.poll() == None: 361 return False 362 if self.ps.poll() == 0: 363 self.parser.parse(self.defconfig) 364 else: 365 print >> sys.stderr, ("WARNING: failed to process '%s'. skip." % 366 self.defconfig) 367 self.occupied = False 368 return True 369 370class Slots: 371 372 """Controller of the array of subprocess slots.""" 373 374 def __init__(self, jobs, output, maintainers_database): 375 """Create a new slots controller. 376 377 Arguments: 378 jobs: A number of slots to instantiate 379 output: File object which the result is written to 380 maintainers_database: An instance of class MaintainersDatabase 381 """ 382 self.slots = [] 383 devnull = get_devnull() 384 make_cmd = get_make_cmd() 385 for i in range(jobs): 386 self.slots.append(Slot(output, maintainers_database, 387 devnull, make_cmd)) 388 389 def add(self, defconfig): 390 """Add a new subprocess if a vacant slot is available. 391 392 Arguments: 393 defconfig: Board (defconfig) name 394 395 Returns: 396 Return True on success or False on fail 397 """ 398 for slot in self.slots: 399 if slot.add(defconfig): 400 return True 401 return False 402 403 def available(self): 404 """Check if there is a vacant slot. 405 406 Returns: 407 Return True if a vacant slot is found, False if all slots are full 408 """ 409 for slot in self.slots: 410 if slot.poll(): 411 return True 412 return False 413 414 def empty(self): 415 """Check if all slots are vacant. 416 417 Returns: 418 Return True if all slots are vacant, False if at least one slot 419 is running 420 """ 421 ret = True 422 for slot in self.slots: 423 if not slot.poll(): 424 ret = False 425 return ret 426 427class Indicator: 428 429 """A class to control the progress indicator.""" 430 431 MIN_WIDTH = 15 432 MAX_WIDTH = 70 433 434 def __init__(self, total): 435 """Create an instance. 436 437 Arguments: 438 total: A number of boards 439 """ 440 self.total = total 441 self.cur = 0 442 width = get_terminal_columns() 443 width = min(width, self.MAX_WIDTH) 444 width -= self.MIN_WIDTH 445 if width > 0: 446 self.enabled = True 447 else: 448 self.enabled = False 449 self.width = width 450 451 def inc(self): 452 """Increment the counter and show the progress bar.""" 453 if not self.enabled: 454 return 455 self.cur += 1 456 arrow_len = self.width * self.cur // self.total 457 msg = '%4d/%d [' % (self.cur, self.total) 458 msg += '=' * arrow_len + '>' + ' ' * (self.width - arrow_len) + ']' 459 sys.stdout.write('\r' + msg) 460 sys.stdout.flush() 461 462class BoardsFileGenerator: 463 464 """Generator of boards.cfg.""" 465 466 def __init__(self): 467 """Prepare basic things for generating boards.cfg.""" 468 # All the defconfig files to be processed 469 defconfigs = [] 470 for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR): 471 dirpath = dirpath[len(CONFIG_DIR) + 1:] 472 for filename in fnmatch.filter(filenames, '*_defconfig'): 473 if fnmatch.fnmatch(filename, '.*'): 474 continue 475 defconfigs.append(os.path.join(dirpath, filename)) 476 self.defconfigs = defconfigs 477 self.indicator = Indicator(len(defconfigs)) 478 479 # Parse all the MAINTAINERS files 480 maintainers_database = MaintainersDatabase() 481 for (dirpath, dirnames, filenames) in os.walk('.'): 482 if 'MAINTAINERS' in filenames: 483 maintainers_database.parse_file(os.path.join(dirpath, 484 'MAINTAINERS')) 485 self.maintainers_database = maintainers_database 486 487 def __del__(self): 488 """Delete the incomplete boards.cfg 489 490 This destructor deletes boards.cfg if the private member 'in_progress' 491 is defined as True. The 'in_progress' member is set to True at the 492 beginning of the generate() method and set to False at its end. 493 So, in_progress==True means generating boards.cfg was terminated 494 on the way. 495 """ 496 497 if hasattr(self, 'in_progress') and self.in_progress: 498 try: 499 os.remove(BOARD_FILE) 500 except OSError as exception: 501 # Ignore 'No such file or directory' error 502 if exception.errno != errno.ENOENT: 503 raise 504 print 'Removed incomplete %s' % BOARD_FILE 505 506 def generate(self, jobs): 507 """Generate boards.cfg 508 509 This method sets the 'in_progress' member to True at the beginning 510 and sets it to False on success. The boards.cfg should not be 511 touched before/after this method because 'in_progress' is used 512 to detect the incomplete boards.cfg. 513 514 Arguments: 515 jobs: The number of jobs to run simultaneously 516 """ 517 518 self.in_progress = True 519 print 'Generating %s ... (jobs: %d)' % (BOARD_FILE, jobs) 520 521 # Output lines should be piped into the reformat tool 522 reformat_process = subprocess.Popen(REFORMAT_CMD, 523 stdin=subprocess.PIPE, 524 stdout=open(BOARD_FILE, 'w')) 525 pipe = reformat_process.stdin 526 pipe.write(COMMENT_BLOCK) 527 528 slots = Slots(jobs, pipe, self.maintainers_database) 529 530 # Main loop to process defconfig files: 531 # Add a new subprocess into a vacant slot. 532 # Sleep if there is no available slot. 533 for defconfig in self.defconfigs: 534 while not slots.add(defconfig): 535 while not slots.available(): 536 # No available slot: sleep for a while 537 time.sleep(SLEEP_TIME) 538 self.indicator.inc() 539 540 # wait until all the subprocesses finish 541 while not slots.empty(): 542 time.sleep(SLEEP_TIME) 543 print '' 544 545 # wait until the reformat tool finishes 546 reformat_process.communicate() 547 if reformat_process.returncode != 0: 548 sys.exit('"%s" failed' % REFORMAT_CMD[0]) 549 550 self.in_progress = False 551 552def gen_boards_cfg(jobs=1, force=False): 553 """Generate boards.cfg file. 554 555 The incomplete boards.cfg is deleted if an error (including 556 the termination by the keyboard interrupt) occurs on the halfway. 557 558 Arguments: 559 jobs: The number of jobs to run simultaneously 560 """ 561 check_top_directory() 562 if not force and output_is_new(): 563 print "%s is up to date. Nothing to do." % BOARD_FILE 564 sys.exit(0) 565 566 generator = BoardsFileGenerator() 567 generator.generate(jobs) 568 569def main(): 570 parser = optparse.OptionParser() 571 # Add options here 572 parser.add_option('-j', '--jobs', 573 help='the number of jobs to run simultaneously') 574 parser.add_option('-f', '--force', action="store_true", default=False, 575 help='regenerate the output even if it is new') 576 (options, args) = parser.parse_args() 577 578 if options.jobs: 579 try: 580 jobs = int(options.jobs) 581 except ValueError: 582 sys.exit('Option -j (--jobs) takes a number') 583 else: 584 try: 585 jobs = int(subprocess.Popen(['getconf', '_NPROCESSORS_ONLN'], 586 stdout=subprocess.PIPE).communicate()[0]) 587 except (OSError, ValueError): 588 print 'info: failed to get the number of CPUs. Set jobs to 1' 589 jobs = 1 590 591 gen_boards_cfg(jobs, force=options.force) 592 593if __name__ == '__main__': 594 main() 595