1#!/usr/bin/env python 2# 3# Author: Masahiro Yamada <yamada.m@jp.panasonic.com> 4# 5# SPDX-License-Identifier: GPL-2.0+ 6# 7 8""" 9Converter from Kconfig and MAINTAINERS to boards.cfg 10 11Run 'tools/genboardscfg.py' to create boards.cfg file. 12 13Run 'tools/genboardscfg.py -h' for available options. 14""" 15 16import errno 17import fnmatch 18import glob 19import optparse 20import os 21import re 22import shutil 23import subprocess 24import sys 25import tempfile 26import time 27 28BOARD_FILE = 'boards.cfg' 29CONFIG_DIR = 'configs' 30REFORMAT_CMD = [os.path.join('tools', 'reformat.py'), 31 '-i', '-d', '-', '-s', '8'] 32SHOW_GNU_MAKE = 'scripts/show-gnu-make' 33SLEEP_TIME=0.03 34 35COMMENT_BLOCK = '''# 36# List of boards 37# Automatically generated by %s: don't edit 38# 39# Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers 40 41''' % __file__ 42 43### helper functions ### 44def get_terminal_columns(): 45 """Get the width of the terminal. 46 47 Returns: 48 The width of the terminal, or zero if the stdout is not 49 associated with tty. 50 """ 51 try: 52 return shutil.get_terminal_size().columns # Python 3.3~ 53 except AttributeError: 54 import fcntl 55 import termios 56 import struct 57 arg = struct.pack('hhhh', 0, 0, 0, 0) 58 try: 59 ret = fcntl.ioctl(sys.stdout.fileno(), termios.TIOCGWINSZ, arg) 60 except IOError as exception: 61 # If 'Inappropriate ioctl for device' error occurs, 62 # stdout is probably redirected. Return 0. 63 return 0 64 return struct.unpack('hhhh', ret)[1] 65 66def get_devnull(): 67 """Get the file object of '/dev/null' device.""" 68 try: 69 devnull = subprocess.DEVNULL # py3k 70 except AttributeError: 71 devnull = open(os.devnull, 'wb') 72 return devnull 73 74def check_top_directory(): 75 """Exit if we are not at the top of source directory.""" 76 for f in ('README', 'Licenses'): 77 if not os.path.exists(f): 78 sys.exit('Please run at the top of source directory.') 79 80def get_make_cmd(): 81 """Get the command name of GNU Make.""" 82 process = subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE) 83 ret = process.communicate() 84 if process.returncode: 85 sys.exit('GNU Make not found') 86 return ret[0].rstrip() 87 88### classes ### 89class MaintainersDatabase: 90 91 """The database of board status and maintainers.""" 92 93 def __init__(self): 94 """Create an empty database.""" 95 self.database = {} 96 97 def get_status(self, target): 98 """Return the status of the given board. 99 100 Returns: 101 Either 'Active' or 'Orphan' 102 """ 103 if not target in self.database: 104 print >> sys.stderr, "WARNING: no status info for '%s'" % target 105 return '-' 106 107 tmp = self.database[target][0] 108 if tmp.startswith('Maintained'): 109 return 'Active' 110 elif tmp.startswith('Orphan'): 111 return 'Orphan' 112 else: 113 print >> sys.stderr, ("WARNING: %s: unknown status for '%s'" % 114 (tmp, target)) 115 return '-' 116 117 def get_maintainers(self, target): 118 """Return the maintainers of the given board. 119 120 If the board has two or more maintainers, they are separated 121 with colons. 122 """ 123 if not target in self.database: 124 print >> sys.stderr, "WARNING: no maintainers for '%s'" % target 125 return '' 126 127 return ':'.join(self.database[target][1]) 128 129 def parse_file(self, file): 130 """Parse the given MAINTAINERS file. 131 132 This method parses MAINTAINERS and add board status and 133 maintainers information to the database. 134 135 Arguments: 136 file: MAINTAINERS file to be parsed 137 """ 138 targets = [] 139 maintainers = [] 140 status = '-' 141 for line in open(file): 142 tag, rest = line[:2], line[2:].strip() 143 if tag == 'M:': 144 maintainers.append(rest) 145 elif tag == 'F:': 146 # expand wildcard and filter by 'configs/*_defconfig' 147 for f in glob.glob(rest): 148 front, match, rear = f.partition('configs/') 149 if not front and match: 150 front, match, rear = rear.rpartition('_defconfig') 151 if match and not rear: 152 targets.append(front) 153 elif tag == 'S:': 154 status = rest 155 elif line == '\n': 156 for target in targets: 157 self.database[target] = (status, maintainers) 158 targets = [] 159 maintainers = [] 160 status = '-' 161 if targets: 162 for target in targets: 163 self.database[target] = (status, maintainers) 164 165class DotConfigParser: 166 167 """A parser of .config file. 168 169 Each line of the output should have the form of: 170 Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers 171 Most of them are extracted from .config file. 172 MAINTAINERS files are also consulted for Status and Maintainers fields. 173 """ 174 175 re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"') 176 re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"') 177 re_soc = re.compile(r'CONFIG_SYS_SOC="(.*)"') 178 re_vendor = re.compile(r'CONFIG_SYS_VENDOR="(.*)"') 179 re_board = re.compile(r'CONFIG_SYS_BOARD="(.*)"') 180 re_config = re.compile(r'CONFIG_SYS_CONFIG_NAME="(.*)"') 181 re_options = re.compile(r'CONFIG_SYS_EXTRA_OPTIONS="(.*)"') 182 re_list = (('arch', re_arch), ('cpu', re_cpu), ('soc', re_soc), 183 ('vendor', re_vendor), ('board', re_board), 184 ('config', re_config), ('options', re_options)) 185 must_fields = ('arch', 'config') 186 187 def __init__(self, build_dir, output, maintainers_database): 188 """Create a new .config perser. 189 190 Arguments: 191 build_dir: Build directory where .config is located 192 output: File object which the result is written to 193 maintainers_database: An instance of class MaintainersDatabase 194 """ 195 self.dotconfig = os.path.join(build_dir, '.config') 196 self.output = output 197 self.database = maintainers_database 198 199 def parse(self, defconfig): 200 """Parse .config file and output one-line database for the given board. 201 202 Arguments: 203 defconfig: Board (defconfig) name 204 """ 205 fields = {} 206 for line in open(self.dotconfig): 207 if not line.startswith('CONFIG_SYS_'): 208 continue 209 for (key, pattern) in self.re_list: 210 m = pattern.match(line) 211 if m and m.group(1): 212 fields[key] = m.group(1) 213 break 214 215 # sanity check of '.config' file 216 for field in self.must_fields: 217 if not field in fields: 218 sys.exit('Error: %s is not defined in %s' % (field, defconfig)) 219 220 # fix-up for aarch64 221 if fields['arch'] == 'arm' and 'cpu' in fields: 222 if fields['cpu'] == 'armv8': 223 fields['arch'] = 'aarch64' 224 225 target, match, rear = defconfig.partition('_defconfig') 226 assert match and not rear, \ 227 '%s : invalid defconfig file name' % defconfig 228 229 fields['status'] = self.database.get_status(target) 230 fields['maintainers'] = self.database.get_maintainers(target) 231 232 if 'options' in fields: 233 options = fields['config'] + ':' + \ 234 fields['options'].replace(r'\"', '"') 235 elif fields['config'] != target: 236 options = fields['config'] 237 else: 238 options = '-' 239 240 self.output.write((' '.join(['%s'] * 9) + '\n') % 241 (fields['status'], 242 fields['arch'], 243 fields.get('cpu', '-'), 244 fields.get('soc', '-'), 245 fields.get('vendor', '-'), 246 fields.get('board', '-'), 247 target, 248 options, 249 fields['maintainers'])) 250 251class Slot: 252 253 """A slot to store a subprocess. 254 255 Each instance of this class handles one subprocess. 256 This class is useful to control multiple processes 257 for faster processing. 258 """ 259 260 def __init__(self, output, maintainers_database, devnull, make_cmd): 261 """Create a new slot. 262 263 Arguments: 264 output: File object which the result is written to 265 maintainers_database: An instance of class MaintainersDatabase 266 """ 267 self.occupied = False 268 self.build_dir = tempfile.mkdtemp() 269 self.devnull = devnull 270 self.make_cmd = make_cmd 271 self.parser = DotConfigParser(self.build_dir, output, 272 maintainers_database) 273 274 def __del__(self): 275 """Delete the working directory""" 276 shutil.rmtree(self.build_dir) 277 278 def add(self, defconfig): 279 """Add a new subprocess to the slot. 280 281 Fails if the slot is occupied, that is, the current subprocess 282 is still running. 283 284 Arguments: 285 defconfig: Board (defconfig) name 286 287 Returns: 288 Return True on success or False on fail 289 """ 290 if self.occupied: 291 return False 292 o = 'O=' + self.build_dir 293 self.ps = subprocess.Popen([self.make_cmd, o, defconfig], 294 stdout=self.devnull) 295 self.defconfig = defconfig 296 self.occupied = True 297 return True 298 299 def poll(self): 300 """Check if the subprocess is running and invoke the .config 301 parser if the subprocess is terminated. 302 303 Returns: 304 Return True if the subprocess is terminated, False otherwise 305 """ 306 if not self.occupied: 307 return True 308 if self.ps.poll() == None: 309 return False 310 self.parser.parse(self.defconfig) 311 self.occupied = False 312 return True 313 314class Slots: 315 316 """Controller of the array of subprocess slots.""" 317 318 def __init__(self, jobs, output, maintainers_database): 319 """Create a new slots controller. 320 321 Arguments: 322 jobs: A number of slots to instantiate 323 output: File object which the result is written to 324 maintainers_database: An instance of class MaintainersDatabase 325 """ 326 self.slots = [] 327 devnull = get_devnull() 328 make_cmd = get_make_cmd() 329 for i in range(jobs): 330 self.slots.append(Slot(output, maintainers_database, 331 devnull, make_cmd)) 332 333 def add(self, defconfig): 334 """Add a new subprocess if a vacant slot is available. 335 336 Arguments: 337 defconfig: Board (defconfig) name 338 339 Returns: 340 Return True on success or False on fail 341 """ 342 for slot in self.slots: 343 if slot.add(defconfig): 344 return True 345 return False 346 347 def available(self): 348 """Check if there is a vacant slot. 349 350 Returns: 351 Return True if a vacant slot is found, False if all slots are full 352 """ 353 for slot in self.slots: 354 if slot.poll(): 355 return True 356 return False 357 358 def empty(self): 359 """Check if all slots are vacant. 360 361 Returns: 362 Return True if all slots are vacant, False if at least one slot 363 is running 364 """ 365 ret = True 366 for slot in self.slots: 367 if not slot.poll(): 368 ret = False 369 return ret 370 371class Indicator: 372 373 """A class to control the progress indicator.""" 374 375 MIN_WIDTH = 15 376 MAX_WIDTH = 70 377 378 def __init__(self, total): 379 """Create an instance. 380 381 Arguments: 382 total: A number of boards 383 """ 384 self.total = total 385 self.cur = 0 386 width = get_terminal_columns() 387 width = min(width, self.MAX_WIDTH) 388 width -= self.MIN_WIDTH 389 if width > 0: 390 self.enabled = True 391 else: 392 self.enabled = False 393 self.width = width 394 395 def inc(self): 396 """Increment the counter and show the progress bar.""" 397 if not self.enabled: 398 return 399 self.cur += 1 400 arrow_len = self.width * self.cur // self.total 401 msg = '%4d/%d [' % (self.cur, self.total) 402 msg += '=' * arrow_len + '>' + ' ' * (self.width - arrow_len) + ']' 403 sys.stdout.write('\r' + msg) 404 sys.stdout.flush() 405 406def __gen_boards_cfg(jobs): 407 """Generate boards.cfg file. 408 409 Arguments: 410 jobs: The number of jobs to run simultaneously 411 412 Note: 413 The incomplete boards.cfg is left over when an error (including 414 the termination by the keyboard interrupt) occurs on the halfway. 415 """ 416 check_top_directory() 417 print 'Generating %s ... (jobs: %d)' % (BOARD_FILE, jobs) 418 419 # All the defconfig files to be processed 420 defconfigs = [] 421 for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR): 422 dirpath = dirpath[len(CONFIG_DIR) + 1:] 423 for filename in fnmatch.filter(filenames, '*_defconfig'): 424 if fnmatch.fnmatch(filename, '.*'): 425 continue 426 defconfigs.append(os.path.join(dirpath, filename)) 427 428 # Parse all the MAINTAINERS files 429 maintainers_database = MaintainersDatabase() 430 for (dirpath, dirnames, filenames) in os.walk('.'): 431 if 'MAINTAINERS' in filenames: 432 maintainers_database.parse_file(os.path.join(dirpath, 433 'MAINTAINERS')) 434 435 # Output lines should be piped into the reformat tool 436 reformat_process = subprocess.Popen(REFORMAT_CMD, stdin=subprocess.PIPE, 437 stdout=open(BOARD_FILE, 'w')) 438 pipe = reformat_process.stdin 439 pipe.write(COMMENT_BLOCK) 440 441 indicator = Indicator(len(defconfigs)) 442 slots = Slots(jobs, pipe, maintainers_database) 443 444 # Main loop to process defconfig files: 445 # Add a new subprocess into a vacant slot. 446 # Sleep if there is no available slot. 447 for defconfig in defconfigs: 448 while not slots.add(defconfig): 449 while not slots.available(): 450 # No available slot: sleep for a while 451 time.sleep(SLEEP_TIME) 452 indicator.inc() 453 454 # wait until all the subprocesses finish 455 while not slots.empty(): 456 time.sleep(SLEEP_TIME) 457 print '' 458 459 # wait until the reformat tool finishes 460 reformat_process.communicate() 461 if reformat_process.returncode != 0: 462 sys.exit('"%s" failed' % REFORMAT_CMD[0]) 463 464def gen_boards_cfg(jobs): 465 """Generate boards.cfg file. 466 467 The incomplete boards.cfg is deleted if an error (including 468 the termination by the keyboard interrupt) occurs on the halfway. 469 470 Arguments: 471 jobs: The number of jobs to run simultaneously 472 """ 473 try: 474 __gen_boards_cfg(jobs) 475 except: 476 # We should remove incomplete boards.cfg 477 try: 478 os.remove(BOARD_FILE) 479 except OSError as exception: 480 # Ignore 'No such file or directory' error 481 if exception.errno != errno.ENOENT: 482 raise 483 raise 484 485def main(): 486 parser = optparse.OptionParser() 487 # Add options here 488 parser.add_option('-j', '--jobs', 489 help='the number of jobs to run simultaneously') 490 (options, args) = parser.parse_args() 491 if options.jobs: 492 try: 493 jobs = int(options.jobs) 494 except ValueError: 495 sys.exit('Option -j (--jobs) takes a number') 496 else: 497 try: 498 jobs = int(subprocess.Popen(['getconf', '_NPROCESSORS_ONLN'], 499 stdout=subprocess.PIPE).communicate()[0]) 500 except (OSError, ValueError): 501 print 'info: failed to get the number of CPUs. Set jobs to 1' 502 jobs = 1 503 gen_boards_cfg(jobs) 504 505if __name__ == '__main__': 506 main() 507