1#!/usr/bin/env python 2# 3# Author: Masahiro Yamada <yamada.m@jp.panasonic.com> 4# 5# SPDX-License-Identifier: GPL-2.0+ 6# 7 8""" 9Converter from Kconfig and MAINTAINERS to boards.cfg 10 11Run 'tools/genboardscfg.py' to create boards.cfg file. 12 13Run 'tools/genboardscfg.py -h' for available options. 14""" 15 16import errno 17import fnmatch 18import glob 19import optparse 20import os 21import re 22import shutil 23import subprocess 24import sys 25import tempfile 26import time 27 28BOARD_FILE = 'boards.cfg' 29CONFIG_DIR = 'configs' 30REFORMAT_CMD = [os.path.join('tools', 'reformat.py'), 31 '-i', '-d', '-', '-s', '8'] 32SHOW_GNU_MAKE = 'scripts/show-gnu-make' 33SLEEP_TIME=0.03 34 35COMMENT_BLOCK = '''# 36# List of boards 37# Automatically generated by %s: don't edit 38# 39# Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers 40 41''' % __file__ 42 43### helper functions ### 44def get_terminal_columns(): 45 """Get the width of the terminal. 46 47 Returns: 48 The width of the terminal, or zero if the stdout is not 49 associated with tty. 50 """ 51 try: 52 return shutil.get_terminal_size().columns # Python 3.3~ 53 except AttributeError: 54 import fcntl 55 import termios 56 import struct 57 arg = struct.pack('hhhh', 0, 0, 0, 0) 58 try: 59 ret = fcntl.ioctl(sys.stdout.fileno(), termios.TIOCGWINSZ, arg) 60 except IOError as exception: 61 # If 'Inappropriate ioctl for device' error occurs, 62 # stdout is probably redirected. Return 0. 63 return 0 64 return struct.unpack('hhhh', ret)[1] 65 66def get_devnull(): 67 """Get the file object of '/dev/null' device.""" 68 try: 69 devnull = subprocess.DEVNULL # py3k 70 except AttributeError: 71 devnull = open(os.devnull, 'wb') 72 return devnull 73 74def check_top_directory(): 75 """Exit if we are not at the top of source directory.""" 76 for f in ('README', 'Licenses'): 77 if not os.path.exists(f): 78 sys.exit('Please run at the top of source directory.') 79 80def get_make_cmd(): 81 """Get the command name of GNU Make.""" 82 process = subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE) 83 ret = process.communicate() 84 if process.returncode: 85 sys.exit('GNU Make not found') 86 return ret[0].rstrip() 87 88### classes ### 89class MaintainersDatabase: 90 91 """The database of board status and maintainers.""" 92 93 def __init__(self): 94 """Create an empty database.""" 95 self.database = {} 96 97 def get_status(self, target): 98 """Return the status of the given board. 99 100 Returns: 101 Either 'Active' or 'Orphan' 102 """ 103 if not target in self.database: 104 print >> sys.stderr, "WARNING: no status info for '%s'" % target 105 return '-' 106 107 tmp = self.database[target][0] 108 if tmp.startswith('Maintained'): 109 return 'Active' 110 elif tmp.startswith('Orphan'): 111 return 'Orphan' 112 else: 113 print >> sys.stderr, ("WARNING: %s: unknown status for '%s'" % 114 (tmp, target)) 115 return '-' 116 117 def get_maintainers(self, target): 118 """Return the maintainers of the given board. 119 120 If the board has two or more maintainers, they are separated 121 with colons. 122 """ 123 if not target in self.database: 124 print >> sys.stderr, "WARNING: no maintainers for '%s'" % target 125 return '' 126 127 return ':'.join(self.database[target][1]) 128 129 def parse_file(self, file): 130 """Parse the given MAINTAINERS file. 131 132 This method parses MAINTAINERS and add board status and 133 maintainers information to the database. 134 135 Arguments: 136 file: MAINTAINERS file to be parsed 137 """ 138 targets = [] 139 maintainers = [] 140 status = '-' 141 for line in open(file): 142 tag, rest = line[:2], line[2:].strip() 143 if tag == 'M:': 144 maintainers.append(rest) 145 elif tag == 'F:': 146 # expand wildcard and filter by 'configs/*_defconfig' 147 for f in glob.glob(rest): 148 front, match, rear = f.partition('configs/') 149 if not front and match: 150 front, match, rear = rear.rpartition('_defconfig') 151 if match and not rear: 152 targets.append(front) 153 elif tag == 'S:': 154 status = rest 155 elif line == '\n': 156 for target in targets: 157 self.database[target] = (status, maintainers) 158 targets = [] 159 maintainers = [] 160 status = '-' 161 if targets: 162 for target in targets: 163 self.database[target] = (status, maintainers) 164 165class DotConfigParser: 166 167 """A parser of .config file. 168 169 Each line of the output should have the form of: 170 Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers 171 Most of them are extracted from .config file. 172 MAINTAINERS files are also consulted for Status and Maintainers fields. 173 """ 174 175 re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"') 176 re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"') 177 re_soc = re.compile(r'CONFIG_SYS_SOC="(.*)"') 178 re_vendor = re.compile(r'CONFIG_SYS_VENDOR="(.*)"') 179 re_board = re.compile(r'CONFIG_SYS_BOARD="(.*)"') 180 re_config = re.compile(r'CONFIG_SYS_CONFIG_NAME="(.*)"') 181 re_options = re.compile(r'CONFIG_SYS_EXTRA_OPTIONS="(.*)"') 182 re_list = (('arch', re_arch), ('cpu', re_cpu), ('soc', re_soc), 183 ('vendor', re_vendor), ('board', re_board), 184 ('config', re_config), ('options', re_options)) 185 must_fields = ('arch', 'config') 186 187 def __init__(self, build_dir, output, maintainers_database): 188 """Create a new .config perser. 189 190 Arguments: 191 build_dir: Build directory where .config is located 192 output: File object which the result is written to 193 maintainers_database: An instance of class MaintainersDatabase 194 """ 195 self.dotconfig = os.path.join(build_dir, '.config') 196 self.output = output 197 self.database = maintainers_database 198 199 def parse(self, defconfig): 200 """Parse .config file and output one-line database for the given board. 201 202 Arguments: 203 defconfig: Board (defconfig) name 204 """ 205 fields = {} 206 for line in open(self.dotconfig): 207 if not line.startswith('CONFIG_SYS_'): 208 continue 209 for (key, pattern) in self.re_list: 210 m = pattern.match(line) 211 if m and m.group(1): 212 fields[key] = m.group(1) 213 break 214 215 # sanity check of '.config' file 216 for field in self.must_fields: 217 if not field in fields: 218 print >> sys.stderr, ( 219 "WARNING: '%s' is not defined in '%s'. Skip." % 220 (field, defconfig)) 221 return 222 223 # fix-up for aarch64 224 if fields['arch'] == 'arm' and 'cpu' in fields: 225 if fields['cpu'] == 'armv8': 226 fields['arch'] = 'aarch64' 227 228 target, match, rear = defconfig.partition('_defconfig') 229 assert match and not rear, \ 230 '%s : invalid defconfig file name' % defconfig 231 232 fields['status'] = self.database.get_status(target) 233 fields['maintainers'] = self.database.get_maintainers(target) 234 235 if 'options' in fields: 236 options = fields['config'] + ':' + \ 237 fields['options'].replace(r'\"', '"') 238 elif fields['config'] != target: 239 options = fields['config'] 240 else: 241 options = '-' 242 243 self.output.write((' '.join(['%s'] * 9) + '\n') % 244 (fields['status'], 245 fields['arch'], 246 fields.get('cpu', '-'), 247 fields.get('soc', '-'), 248 fields.get('vendor', '-'), 249 fields.get('board', '-'), 250 target, 251 options, 252 fields['maintainers'])) 253 254class Slot: 255 256 """A slot to store a subprocess. 257 258 Each instance of this class handles one subprocess. 259 This class is useful to control multiple processes 260 for faster processing. 261 """ 262 263 def __init__(self, output, maintainers_database, devnull, make_cmd): 264 """Create a new slot. 265 266 Arguments: 267 output: File object which the result is written to 268 maintainers_database: An instance of class MaintainersDatabase 269 """ 270 self.occupied = False 271 self.build_dir = tempfile.mkdtemp() 272 self.devnull = devnull 273 self.make_cmd = make_cmd 274 self.parser = DotConfigParser(self.build_dir, output, 275 maintainers_database) 276 277 def __del__(self): 278 """Delete the working directory""" 279 if not self.occupied: 280 while self.ps.poll() == None: 281 pass 282 shutil.rmtree(self.build_dir) 283 284 def add(self, defconfig): 285 """Add a new subprocess to the slot. 286 287 Fails if the slot is occupied, that is, the current subprocess 288 is still running. 289 290 Arguments: 291 defconfig: Board (defconfig) name 292 293 Returns: 294 Return True on success or False on fail 295 """ 296 if self.occupied: 297 return False 298 o = 'O=' + self.build_dir 299 self.ps = subprocess.Popen([self.make_cmd, o, defconfig], 300 stdout=self.devnull) 301 self.defconfig = defconfig 302 self.occupied = True 303 return True 304 305 def poll(self): 306 """Check if the subprocess is running and invoke the .config 307 parser if the subprocess is terminated. 308 309 Returns: 310 Return True if the subprocess is terminated, False otherwise 311 """ 312 if not self.occupied: 313 return True 314 if self.ps.poll() == None: 315 return False 316 if self.ps.poll() == 0: 317 self.parser.parse(self.defconfig) 318 else: 319 print >> sys.stderr, ("WARNING: failed to process '%s'. skip." % 320 self.defconfig) 321 self.occupied = False 322 return True 323 324class Slots: 325 326 """Controller of the array of subprocess slots.""" 327 328 def __init__(self, jobs, output, maintainers_database): 329 """Create a new slots controller. 330 331 Arguments: 332 jobs: A number of slots to instantiate 333 output: File object which the result is written to 334 maintainers_database: An instance of class MaintainersDatabase 335 """ 336 self.slots = [] 337 devnull = get_devnull() 338 make_cmd = get_make_cmd() 339 for i in range(jobs): 340 self.slots.append(Slot(output, maintainers_database, 341 devnull, make_cmd)) 342 343 def add(self, defconfig): 344 """Add a new subprocess if a vacant slot is available. 345 346 Arguments: 347 defconfig: Board (defconfig) name 348 349 Returns: 350 Return True on success or False on fail 351 """ 352 for slot in self.slots: 353 if slot.add(defconfig): 354 return True 355 return False 356 357 def available(self): 358 """Check if there is a vacant slot. 359 360 Returns: 361 Return True if a vacant slot is found, False if all slots are full 362 """ 363 for slot in self.slots: 364 if slot.poll(): 365 return True 366 return False 367 368 def empty(self): 369 """Check if all slots are vacant. 370 371 Returns: 372 Return True if all slots are vacant, False if at least one slot 373 is running 374 """ 375 ret = True 376 for slot in self.slots: 377 if not slot.poll(): 378 ret = False 379 return ret 380 381class Indicator: 382 383 """A class to control the progress indicator.""" 384 385 MIN_WIDTH = 15 386 MAX_WIDTH = 70 387 388 def __init__(self, total): 389 """Create an instance. 390 391 Arguments: 392 total: A number of boards 393 """ 394 self.total = total 395 self.cur = 0 396 width = get_terminal_columns() 397 width = min(width, self.MAX_WIDTH) 398 width -= self.MIN_WIDTH 399 if width > 0: 400 self.enabled = True 401 else: 402 self.enabled = False 403 self.width = width 404 405 def inc(self): 406 """Increment the counter and show the progress bar.""" 407 if not self.enabled: 408 return 409 self.cur += 1 410 arrow_len = self.width * self.cur // self.total 411 msg = '%4d/%d [' % (self.cur, self.total) 412 msg += '=' * arrow_len + '>' + ' ' * (self.width - arrow_len) + ']' 413 sys.stdout.write('\r' + msg) 414 sys.stdout.flush() 415 416def __gen_boards_cfg(jobs): 417 """Generate boards.cfg file. 418 419 Arguments: 420 jobs: The number of jobs to run simultaneously 421 422 Note: 423 The incomplete boards.cfg is left over when an error (including 424 the termination by the keyboard interrupt) occurs on the halfway. 425 """ 426 check_top_directory() 427 print 'Generating %s ... (jobs: %d)' % (BOARD_FILE, jobs) 428 429 # All the defconfig files to be processed 430 defconfigs = [] 431 for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR): 432 dirpath = dirpath[len(CONFIG_DIR) + 1:] 433 for filename in fnmatch.filter(filenames, '*_defconfig'): 434 if fnmatch.fnmatch(filename, '.*'): 435 continue 436 defconfigs.append(os.path.join(dirpath, filename)) 437 438 # Parse all the MAINTAINERS files 439 maintainers_database = MaintainersDatabase() 440 for (dirpath, dirnames, filenames) in os.walk('.'): 441 if 'MAINTAINERS' in filenames: 442 maintainers_database.parse_file(os.path.join(dirpath, 443 'MAINTAINERS')) 444 445 # Output lines should be piped into the reformat tool 446 reformat_process = subprocess.Popen(REFORMAT_CMD, stdin=subprocess.PIPE, 447 stdout=open(BOARD_FILE, 'w')) 448 pipe = reformat_process.stdin 449 pipe.write(COMMENT_BLOCK) 450 451 indicator = Indicator(len(defconfigs)) 452 slots = Slots(jobs, pipe, maintainers_database) 453 454 # Main loop to process defconfig files: 455 # Add a new subprocess into a vacant slot. 456 # Sleep if there is no available slot. 457 for defconfig in defconfigs: 458 while not slots.add(defconfig): 459 while not slots.available(): 460 # No available slot: sleep for a while 461 time.sleep(SLEEP_TIME) 462 indicator.inc() 463 464 # wait until all the subprocesses finish 465 while not slots.empty(): 466 time.sleep(SLEEP_TIME) 467 print '' 468 469 # wait until the reformat tool finishes 470 reformat_process.communicate() 471 if reformat_process.returncode != 0: 472 sys.exit('"%s" failed' % REFORMAT_CMD[0]) 473 474def gen_boards_cfg(jobs): 475 """Generate boards.cfg file. 476 477 The incomplete boards.cfg is deleted if an error (including 478 the termination by the keyboard interrupt) occurs on the halfway. 479 480 Arguments: 481 jobs: The number of jobs to run simultaneously 482 """ 483 try: 484 __gen_boards_cfg(jobs) 485 except: 486 # We should remove incomplete boards.cfg 487 try: 488 os.remove(BOARD_FILE) 489 except OSError as exception: 490 # Ignore 'No such file or directory' error 491 if exception.errno != errno.ENOENT: 492 raise 493 raise 494 495def main(): 496 parser = optparse.OptionParser() 497 # Add options here 498 parser.add_option('-j', '--jobs', 499 help='the number of jobs to run simultaneously') 500 (options, args) = parser.parse_args() 501 if options.jobs: 502 try: 503 jobs = int(options.jobs) 504 except ValueError: 505 sys.exit('Option -j (--jobs) takes a number') 506 else: 507 try: 508 jobs = int(subprocess.Popen(['getconf', '_NPROCESSORS_ONLN'], 509 stdout=subprocess.PIPE).communicate()[0]) 510 except (OSError, ValueError): 511 print 'info: failed to get the number of CPUs. Set jobs to 1' 512 jobs = 1 513 gen_boards_cfg(jobs) 514 515if __name__ == '__main__': 516 main() 517