1#!/usr/bin/env python 2# 3# Author: Masahiro Yamada <yamada.m@jp.panasonic.com> 4# 5# SPDX-License-Identifier: GPL-2.0+ 6# 7 8""" 9Converter from Kconfig and MAINTAINERS to boards.cfg 10 11Run 'tools/genboardscfg.py' to create boards.cfg file. 12 13Run 'tools/genboardscfg.py -h' for available options. 14""" 15 16import errno 17import fnmatch 18import glob 19import optparse 20import os 21import re 22import shutil 23import subprocess 24import sys 25import tempfile 26import time 27 28BOARD_FILE = 'boards.cfg' 29CONFIG_DIR = 'configs' 30REFORMAT_CMD = [os.path.join('tools', 'reformat.py'), 31 '-i', '-d', '-', '-s', '8'] 32SHOW_GNU_MAKE = 'scripts/show-gnu-make' 33SLEEP_TIME=0.03 34 35COMMENT_BLOCK = '''# 36# List of boards 37# Automatically generated by %s: don't edit 38# 39# Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers 40 41''' % __file__ 42 43### helper functions ### 44def get_terminal_columns(): 45 """Get the width of the terminal. 46 47 Returns: 48 The width of the terminal, or zero if the stdout is not 49 associated with tty. 50 """ 51 try: 52 return shutil.get_terminal_size().columns # Python 3.3~ 53 except AttributeError: 54 import fcntl 55 import termios 56 import struct 57 arg = struct.pack('hhhh', 0, 0, 0, 0) 58 try: 59 ret = fcntl.ioctl(sys.stdout.fileno(), termios.TIOCGWINSZ, arg) 60 except IOError as exception: 61 # If 'Inappropriate ioctl for device' error occurs, 62 # stdout is probably redirected. Return 0. 63 return 0 64 return struct.unpack('hhhh', ret)[1] 65 66def get_devnull(): 67 """Get the file object of '/dev/null' device.""" 68 try: 69 devnull = subprocess.DEVNULL # py3k 70 except AttributeError: 71 devnull = open(os.devnull, 'wb') 72 return devnull 73 74def check_top_directory(): 75 """Exit if we are not at the top of source directory.""" 76 for f in ('README', 'Licenses'): 77 if not os.path.exists(f): 78 sys.exit('Please run at the top of source directory.') 79 80def get_make_cmd(): 81 """Get the command name of GNU Make.""" 82 process = subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE) 83 ret = process.communicate() 84 if process.returncode: 85 sys.exit('GNU Make not found') 86 return ret[0].rstrip() 87 88### classes ### 89class MaintainersDatabase: 90 91 """The database of board status and maintainers.""" 92 93 def __init__(self): 94 """Create an empty database.""" 95 self.database = {} 96 97 def get_status(self, target): 98 """Return the status of the given board. 99 100 Returns: 101 Either 'Active' or 'Orphan' 102 """ 103 if not target in self.database: 104 print >> sys.stderr, "WARNING: no status info for '%s'" % target 105 return '-' 106 107 tmp = self.database[target][0] 108 if tmp.startswith('Maintained'): 109 return 'Active' 110 elif tmp.startswith('Orphan'): 111 return 'Orphan' 112 else: 113 print >> sys.stderr, ("WARNING: %s: unknown status for '%s'" % 114 (tmp, target)) 115 return '-' 116 117 def get_maintainers(self, target): 118 """Return the maintainers of the given board. 119 120 If the board has two or more maintainers, they are separated 121 with colons. 122 """ 123 if not target in self.database: 124 print >> sys.stderr, "WARNING: no maintainers for '%s'" % target 125 return '' 126 127 return ':'.join(self.database[target][1]) 128 129 def parse_file(self, file): 130 """Parse the given MAINTAINERS file. 131 132 This method parses MAINTAINERS and add board status and 133 maintainers information to the database. 134 135 Arguments: 136 file: MAINTAINERS file to be parsed 137 """ 138 targets = [] 139 maintainers = [] 140 status = '-' 141 for line in open(file): 142 tag, rest = line[:2], line[2:].strip() 143 if tag == 'M:': 144 maintainers.append(rest) 145 elif tag == 'F:': 146 # expand wildcard and filter by 'configs/*_defconfig' 147 for f in glob.glob(rest): 148 front, match, rear = f.partition('configs/') 149 if not front and match: 150 front, match, rear = rear.rpartition('_defconfig') 151 if match and not rear: 152 targets.append(front) 153 elif tag == 'S:': 154 status = rest 155 elif line == '\n': 156 for target in targets: 157 self.database[target] = (status, maintainers) 158 targets = [] 159 maintainers = [] 160 status = '-' 161 if targets: 162 for target in targets: 163 self.database[target] = (status, maintainers) 164 165class DotConfigParser: 166 167 """A parser of .config file. 168 169 Each line of the output should have the form of: 170 Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers 171 Most of them are extracted from .config file. 172 MAINTAINERS files are also consulted for Status and Maintainers fields. 173 """ 174 175 re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"') 176 re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"') 177 re_soc = re.compile(r'CONFIG_SYS_SOC="(.*)"') 178 re_vendor = re.compile(r'CONFIG_SYS_VENDOR="(.*)"') 179 re_board = re.compile(r'CONFIG_SYS_BOARD="(.*)"') 180 re_config = re.compile(r'CONFIG_SYS_CONFIG_NAME="(.*)"') 181 re_options = re.compile(r'CONFIG_SYS_EXTRA_OPTIONS="(.*)"') 182 re_list = (('arch', re_arch), ('cpu', re_cpu), ('soc', re_soc), 183 ('vendor', re_vendor), ('board', re_board), 184 ('config', re_config), ('options', re_options)) 185 must_fields = ('arch', 'config') 186 187 def __init__(self, build_dir, output, maintainers_database): 188 """Create a new .config perser. 189 190 Arguments: 191 build_dir: Build directory where .config is located 192 output: File object which the result is written to 193 maintainers_database: An instance of class MaintainersDatabase 194 """ 195 self.dotconfig = os.path.join(build_dir, '.config') 196 self.output = output 197 self.database = maintainers_database 198 199 def parse(self, defconfig): 200 """Parse .config file and output one-line database for the given board. 201 202 Arguments: 203 defconfig: Board (defconfig) name 204 """ 205 fields = {} 206 for line in open(self.dotconfig): 207 if not line.startswith('CONFIG_SYS_'): 208 continue 209 for (key, pattern) in self.re_list: 210 m = pattern.match(line) 211 if m and m.group(1): 212 fields[key] = m.group(1) 213 break 214 215 # sanity check of '.config' file 216 for field in self.must_fields: 217 if not field in fields: 218 print >> sys.stderr, ( 219 "WARNING: '%s' is not defined in '%s'. Skip." % 220 (field, defconfig)) 221 return 222 223 # fix-up for aarch64 224 if fields['arch'] == 'arm' and 'cpu' in fields: 225 if fields['cpu'] == 'armv8': 226 fields['arch'] = 'aarch64' 227 228 target, match, rear = defconfig.partition('_defconfig') 229 assert match and not rear, \ 230 '%s : invalid defconfig file name' % defconfig 231 232 fields['status'] = self.database.get_status(target) 233 fields['maintainers'] = self.database.get_maintainers(target) 234 235 if 'options' in fields: 236 options = fields['config'] + ':' + \ 237 fields['options'].replace(r'\"', '"') 238 elif fields['config'] != target: 239 options = fields['config'] 240 else: 241 options = '-' 242 243 self.output.write((' '.join(['%s'] * 9) + '\n') % 244 (fields['status'], 245 fields['arch'], 246 fields.get('cpu', '-'), 247 fields.get('soc', '-'), 248 fields.get('vendor', '-'), 249 fields.get('board', '-'), 250 target, 251 options, 252 fields['maintainers'])) 253 254class Slot: 255 256 """A slot to store a subprocess. 257 258 Each instance of this class handles one subprocess. 259 This class is useful to control multiple processes 260 for faster processing. 261 """ 262 263 def __init__(self, output, maintainers_database, devnull, make_cmd): 264 """Create a new slot. 265 266 Arguments: 267 output: File object which the result is written to 268 maintainers_database: An instance of class MaintainersDatabase 269 """ 270 self.occupied = False 271 self.build_dir = tempfile.mkdtemp() 272 self.devnull = devnull 273 self.make_cmd = make_cmd 274 self.parser = DotConfigParser(self.build_dir, output, 275 maintainers_database) 276 277 def __del__(self): 278 """Delete the working directory""" 279 if not self.occupied: 280 while self.ps.poll() == None: 281 pass 282 shutil.rmtree(self.build_dir) 283 284 def add(self, defconfig): 285 """Add a new subprocess to the slot. 286 287 Fails if the slot is occupied, that is, the current subprocess 288 is still running. 289 290 Arguments: 291 defconfig: Board (defconfig) name 292 293 Returns: 294 Return True on success or False on fail 295 """ 296 if self.occupied: 297 return False 298 o = 'O=' + self.build_dir 299 self.ps = subprocess.Popen([self.make_cmd, o, defconfig], 300 stdout=self.devnull) 301 self.defconfig = defconfig 302 self.occupied = True 303 return True 304 305 def poll(self): 306 """Check if the subprocess is running and invoke the .config 307 parser if the subprocess is terminated. 308 309 Returns: 310 Return True if the subprocess is terminated, False otherwise 311 """ 312 if not self.occupied: 313 return True 314 if self.ps.poll() == None: 315 return False 316 if self.ps.poll() == 0: 317 self.parser.parse(self.defconfig) 318 else: 319 print >> sys.stderr, ("WARNING: failed to process '%s'. skip." % 320 self.defconfig) 321 self.occupied = False 322 return True 323 324class Slots: 325 326 """Controller of the array of subprocess slots.""" 327 328 def __init__(self, jobs, output, maintainers_database): 329 """Create a new slots controller. 330 331 Arguments: 332 jobs: A number of slots to instantiate 333 output: File object which the result is written to 334 maintainers_database: An instance of class MaintainersDatabase 335 """ 336 self.slots = [] 337 devnull = get_devnull() 338 make_cmd = get_make_cmd() 339 for i in range(jobs): 340 self.slots.append(Slot(output, maintainers_database, 341 devnull, make_cmd)) 342 343 def add(self, defconfig): 344 """Add a new subprocess if a vacant slot is available. 345 346 Arguments: 347 defconfig: Board (defconfig) name 348 349 Returns: 350 Return True on success or False on fail 351 """ 352 for slot in self.slots: 353 if slot.add(defconfig): 354 return True 355 return False 356 357 def available(self): 358 """Check if there is a vacant slot. 359 360 Returns: 361 Return True if a vacant slot is found, False if all slots are full 362 """ 363 for slot in self.slots: 364 if slot.poll(): 365 return True 366 return False 367 368 def empty(self): 369 """Check if all slots are vacant. 370 371 Returns: 372 Return True if all slots are vacant, False if at least one slot 373 is running 374 """ 375 ret = True 376 for slot in self.slots: 377 if not slot.poll(): 378 ret = False 379 return ret 380 381class Indicator: 382 383 """A class to control the progress indicator.""" 384 385 MIN_WIDTH = 15 386 MAX_WIDTH = 70 387 388 def __init__(self, total): 389 """Create an instance. 390 391 Arguments: 392 total: A number of boards 393 """ 394 self.total = total 395 self.cur = 0 396 width = get_terminal_columns() 397 width = min(width, self.MAX_WIDTH) 398 width -= self.MIN_WIDTH 399 if width > 0: 400 self.enabled = True 401 else: 402 self.enabled = False 403 self.width = width 404 405 def inc(self): 406 """Increment the counter and show the progress bar.""" 407 if not self.enabled: 408 return 409 self.cur += 1 410 arrow_len = self.width * self.cur // self.total 411 msg = '%4d/%d [' % (self.cur, self.total) 412 msg += '=' * arrow_len + '>' + ' ' * (self.width - arrow_len) + ']' 413 sys.stdout.write('\r' + msg) 414 sys.stdout.flush() 415 416class BoardsFileGenerator: 417 418 """Generator of boards.cfg.""" 419 420 def __init__(self): 421 """Prepare basic things for generating boards.cfg.""" 422 # All the defconfig files to be processed 423 defconfigs = [] 424 for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR): 425 dirpath = dirpath[len(CONFIG_DIR) + 1:] 426 for filename in fnmatch.filter(filenames, '*_defconfig'): 427 if fnmatch.fnmatch(filename, '.*'): 428 continue 429 defconfigs.append(os.path.join(dirpath, filename)) 430 self.defconfigs = defconfigs 431 self.indicator = Indicator(len(defconfigs)) 432 433 # Parse all the MAINTAINERS files 434 maintainers_database = MaintainersDatabase() 435 for (dirpath, dirnames, filenames) in os.walk('.'): 436 if 'MAINTAINERS' in filenames: 437 maintainers_database.parse_file(os.path.join(dirpath, 438 'MAINTAINERS')) 439 self.maintainers_database = maintainers_database 440 441 def __del__(self): 442 """Delete the incomplete boards.cfg 443 444 This destructor deletes boards.cfg if the private member 'in_progress' 445 is defined as True. The 'in_progress' member is set to True at the 446 beginning of the generate() method and set to False at its end. 447 So, in_progress==True means generating boards.cfg was terminated 448 on the way. 449 """ 450 451 if hasattr(self, 'in_progress') and self.in_progress: 452 try: 453 os.remove(BOARD_FILE) 454 except OSError as exception: 455 # Ignore 'No such file or directory' error 456 if exception.errno != errno.ENOENT: 457 raise 458 print 'Removed incomplete %s' % BOARD_FILE 459 460 def generate(self, jobs): 461 """Generate boards.cfg 462 463 This method sets the 'in_progress' member to True at the beginning 464 and sets it to False on success. The boards.cfg should not be 465 touched before/after this method because 'in_progress' is used 466 to detect the incomplete boards.cfg. 467 468 Arguments: 469 jobs: The number of jobs to run simultaneously 470 """ 471 472 self.in_progress = True 473 print 'Generating %s ... (jobs: %d)' % (BOARD_FILE, jobs) 474 475 # Output lines should be piped into the reformat tool 476 reformat_process = subprocess.Popen(REFORMAT_CMD, 477 stdin=subprocess.PIPE, 478 stdout=open(BOARD_FILE, 'w')) 479 pipe = reformat_process.stdin 480 pipe.write(COMMENT_BLOCK) 481 482 slots = Slots(jobs, pipe, self.maintainers_database) 483 484 # Main loop to process defconfig files: 485 # Add a new subprocess into a vacant slot. 486 # Sleep if there is no available slot. 487 for defconfig in self.defconfigs: 488 while not slots.add(defconfig): 489 while not slots.available(): 490 # No available slot: sleep for a while 491 time.sleep(SLEEP_TIME) 492 self.indicator.inc() 493 494 # wait until all the subprocesses finish 495 while not slots.empty(): 496 time.sleep(SLEEP_TIME) 497 print '' 498 499 # wait until the reformat tool finishes 500 reformat_process.communicate() 501 if reformat_process.returncode != 0: 502 sys.exit('"%s" failed' % REFORMAT_CMD[0]) 503 504 self.in_progress = False 505 506def gen_boards_cfg(jobs): 507 """Generate boards.cfg file. 508 509 The incomplete boards.cfg is deleted if an error (including 510 the termination by the keyboard interrupt) occurs on the halfway. 511 512 Arguments: 513 jobs: The number of jobs to run simultaneously 514 """ 515 check_top_directory() 516 generator = BoardsFileGenerator() 517 generator.generate(jobs) 518 519def main(): 520 parser = optparse.OptionParser() 521 # Add options here 522 parser.add_option('-j', '--jobs', 523 help='the number of jobs to run simultaneously') 524 (options, args) = parser.parse_args() 525 if options.jobs: 526 try: 527 jobs = int(options.jobs) 528 except ValueError: 529 sys.exit('Option -j (--jobs) takes a number') 530 else: 531 try: 532 jobs = int(subprocess.Popen(['getconf', '_NPROCESSORS_ONLN'], 533 stdout=subprocess.PIPE).communicate()[0]) 534 except (OSError, ValueError): 535 print 'info: failed to get the number of CPUs. Set jobs to 1' 536 jobs = 1 537 gen_boards_cfg(jobs) 538 539if __name__ == '__main__': 540 main() 541