1#!/usr/bin/env python 2# 3# Author: Masahiro Yamada <yamada.m@jp.panasonic.com> 4# 5# SPDX-License-Identifier: GPL-2.0+ 6# 7 8""" 9Converter from Kconfig and MAINTAINERS to boards.cfg 10 11Run 'tools/genboardscfg.py' to create boards.cfg file. 12 13Run 'tools/genboardscfg.py -h' for available options. 14""" 15 16import errno 17import fnmatch 18import glob 19import optparse 20import os 21import re 22import shutil 23import subprocess 24import sys 25import tempfile 26import time 27 28BOARD_FILE = 'boards.cfg' 29CONFIG_DIR = 'configs' 30REFORMAT_CMD = [os.path.join('tools', 'reformat.py'), 31 '-i', '-d', '-', '-s', '8'] 32SHOW_GNU_MAKE = 'scripts/show-gnu-make' 33SLEEP_TIME=0.03 34 35COMMENT_BLOCK = '''# 36# List of boards 37# Automatically generated by %s: don't edit 38# 39# Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers 40 41''' % __file__ 42 43### helper functions ### 44def get_terminal_columns(): 45 """Get the width of the terminal. 46 47 Returns: 48 The width of the terminal, or zero if the stdout is not 49 associated with tty. 50 """ 51 try: 52 return shutil.get_terminal_size().columns # Python 3.3~ 53 except AttributeError: 54 import fcntl 55 import termios 56 import struct 57 arg = struct.pack('hhhh', 0, 0, 0, 0) 58 try: 59 ret = fcntl.ioctl(sys.stdout.fileno(), termios.TIOCGWINSZ, arg) 60 except IOError as exception: 61 if exception.errno != errno.ENOTTY: 62 raise 63 # If 'Inappropriate ioctl for device' error occurs, 64 # stdout is probably redirected. Return 0. 65 return 0 66 return struct.unpack('hhhh', ret)[1] 67 68def get_devnull(): 69 """Get the file object of '/dev/null' device.""" 70 try: 71 devnull = subprocess.DEVNULL # py3k 72 except AttributeError: 73 devnull = open(os.devnull, 'wb') 74 return devnull 75 76def check_top_directory(): 77 """Exit if we are not at the top of source directory.""" 78 for f in ('README', 'Licenses'): 79 if not os.path.exists(f): 80 sys.exit('Please run at the top of source directory.') 81 82def get_make_cmd(): 83 """Get the command name of GNU Make.""" 84 process = subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE) 85 ret = process.communicate() 86 if process.returncode: 87 sys.exit('GNU Make not found') 88 return ret[0].rstrip() 89 90### classes ### 91class MaintainersDatabase: 92 93 """The database of board status and maintainers.""" 94 95 def __init__(self): 96 """Create an empty database.""" 97 self.database = {} 98 99 def get_status(self, target): 100 """Return the status of the given board. 101 102 Returns: 103 Either 'Active' or 'Orphan' 104 """ 105 tmp = self.database[target][0] 106 if tmp.startswith('Maintained'): 107 return 'Active' 108 elif tmp.startswith('Orphan'): 109 return 'Orphan' 110 else: 111 print >> sys.stderr, 'Error: %s: unknown status' % tmp 112 113 def get_maintainers(self, target): 114 """Return the maintainers of the given board. 115 116 If the board has two or more maintainers, they are separated 117 with colons. 118 """ 119 return ':'.join(self.database[target][1]) 120 121 def parse_file(self, file): 122 """Parse the given MAINTAINERS file. 123 124 This method parses MAINTAINERS and add board status and 125 maintainers information to the database. 126 127 Arguments: 128 file: MAINTAINERS file to be parsed 129 """ 130 targets = [] 131 maintainers = [] 132 status = '-' 133 for line in open(file): 134 tag, rest = line[:2], line[2:].strip() 135 if tag == 'M:': 136 maintainers.append(rest) 137 elif tag == 'F:': 138 # expand wildcard and filter by 'configs/*_defconfig' 139 for f in glob.glob(rest): 140 front, match, rear = f.partition('configs/') 141 if not front and match: 142 front, match, rear = rear.rpartition('_defconfig') 143 if match and not rear: 144 targets.append(front) 145 elif tag == 'S:': 146 status = rest 147 elif line == '\n' and targets: 148 for target in targets: 149 self.database[target] = (status, maintainers) 150 targets = [] 151 maintainers = [] 152 status = '-' 153 if targets: 154 for target in targets: 155 self.database[target] = (status, maintainers) 156 157class DotConfigParser: 158 159 """A parser of .config file. 160 161 Each line of the output should have the form of: 162 Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers 163 Most of them are extracted from .config file. 164 MAINTAINERS files are also consulted for Status and Maintainers fields. 165 """ 166 167 re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"') 168 re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"') 169 re_soc = re.compile(r'CONFIG_SYS_SOC="(.*)"') 170 re_vendor = re.compile(r'CONFIG_SYS_VENDOR="(.*)"') 171 re_board = re.compile(r'CONFIG_SYS_BOARD="(.*)"') 172 re_config = re.compile(r'CONFIG_SYS_CONFIG_NAME="(.*)"') 173 re_options = re.compile(r'CONFIG_SYS_EXTRA_OPTIONS="(.*)"') 174 re_list = (('arch', re_arch), ('cpu', re_cpu), ('soc', re_soc), 175 ('vendor', re_vendor), ('board', re_board), 176 ('config', re_config), ('options', re_options)) 177 must_fields = ('arch', 'config') 178 179 def __init__(self, build_dir, output, maintainers_database): 180 """Create a new .config perser. 181 182 Arguments: 183 build_dir: Build directory where .config is located 184 output: File object which the result is written to 185 maintainers_database: An instance of class MaintainersDatabase 186 """ 187 self.dotconfig = os.path.join(build_dir, '.config') 188 self.output = output 189 self.database = maintainers_database 190 191 def parse(self, defconfig): 192 """Parse .config file and output one-line database for the given board. 193 194 Arguments: 195 defconfig: Board (defconfig) name 196 """ 197 fields = {} 198 for line in open(self.dotconfig): 199 if not line.startswith('CONFIG_SYS_'): 200 continue 201 for (key, pattern) in self.re_list: 202 m = pattern.match(line) 203 if m and m.group(1): 204 fields[key] = m.group(1) 205 break 206 207 # sanity check of '.config' file 208 for field in self.must_fields: 209 if not field in fields: 210 sys.exit('Error: %s is not defined in %s' % (field, defconfig)) 211 212 # fix-up for aarch64 213 if fields['arch'] == 'arm' and 'cpu' in fields: 214 if fields['cpu'] == 'armv8': 215 fields['arch'] = 'aarch64' 216 217 target, match, rear = defconfig.partition('_defconfig') 218 assert match and not rear, \ 219 '%s : invalid defconfig file name' % defconfig 220 221 fields['status'] = self.database.get_status(target) 222 fields['maintainers'] = self.database.get_maintainers(target) 223 224 if 'options' in fields: 225 options = fields['config'] + ':' + \ 226 fields['options'].replace(r'\"', '"') 227 elif fields['config'] != target: 228 options = fields['config'] 229 else: 230 options = '-' 231 232 self.output.write((' '.join(['%s'] * 9) + '\n') % 233 (fields['status'], 234 fields['arch'], 235 fields.get('cpu', '-'), 236 fields.get('soc', '-'), 237 fields.get('vendor', '-'), 238 fields.get('board', '-'), 239 target, 240 options, 241 fields['maintainers'])) 242 243class Slot: 244 245 """A slot to store a subprocess. 246 247 Each instance of this class handles one subprocess. 248 This class is useful to control multiple processes 249 for faster processing. 250 """ 251 252 def __init__(self, output, maintainers_database, devnull, make_cmd): 253 """Create a new slot. 254 255 Arguments: 256 output: File object which the result is written to 257 maintainers_database: An instance of class MaintainersDatabase 258 """ 259 self.occupied = False 260 self.build_dir = tempfile.mkdtemp() 261 self.devnull = devnull 262 self.make_cmd = make_cmd 263 self.parser = DotConfigParser(self.build_dir, output, 264 maintainers_database) 265 266 def __del__(self): 267 """Delete the working directory""" 268 shutil.rmtree(self.build_dir) 269 270 def add(self, defconfig): 271 """Add a new subprocess to the slot. 272 273 Fails if the slot is occupied, that is, the current subprocess 274 is still running. 275 276 Arguments: 277 defconfig: Board (defconfig) name 278 279 Returns: 280 Return True on success or False on fail 281 """ 282 if self.occupied: 283 return False 284 o = 'O=' + self.build_dir 285 self.ps = subprocess.Popen([self.make_cmd, o, defconfig], 286 stdout=self.devnull) 287 self.defconfig = defconfig 288 self.occupied = True 289 return True 290 291 def poll(self): 292 """Check if the subprocess is running and invoke the .config 293 parser if the subprocess is terminated. 294 295 Returns: 296 Return True if the subprocess is terminated, False otherwise 297 """ 298 if not self.occupied: 299 return True 300 if self.ps.poll() == None: 301 return False 302 self.parser.parse(self.defconfig) 303 self.occupied = False 304 return True 305 306class Slots: 307 308 """Controller of the array of subprocess slots.""" 309 310 def __init__(self, jobs, output, maintainers_database): 311 """Create a new slots controller. 312 313 Arguments: 314 jobs: A number of slots to instantiate 315 output: File object which the result is written to 316 maintainers_database: An instance of class MaintainersDatabase 317 """ 318 self.slots = [] 319 devnull = get_devnull() 320 make_cmd = get_make_cmd() 321 for i in range(jobs): 322 self.slots.append(Slot(output, maintainers_database, 323 devnull, make_cmd)) 324 325 def add(self, defconfig): 326 """Add a new subprocess if a vacant slot is available. 327 328 Arguments: 329 defconfig: Board (defconfig) name 330 331 Returns: 332 Return True on success or False on fail 333 """ 334 for slot in self.slots: 335 if slot.add(defconfig): 336 return True 337 return False 338 339 def available(self): 340 """Check if there is a vacant slot. 341 342 Returns: 343 Return True if a vacant slot is found, False if all slots are full 344 """ 345 for slot in self.slots: 346 if slot.poll(): 347 return True 348 return False 349 350 def empty(self): 351 """Check if all slots are vacant. 352 353 Returns: 354 Return True if all slots are vacant, False if at least one slot 355 is running 356 """ 357 ret = True 358 for slot in self.slots: 359 if not slot.poll(): 360 ret = False 361 return ret 362 363class Indicator: 364 365 """A class to control the progress indicator.""" 366 367 MIN_WIDTH = 15 368 MAX_WIDTH = 70 369 370 def __init__(self, total): 371 """Create an instance. 372 373 Arguments: 374 total: A number of boards 375 """ 376 self.total = total 377 self.cur = 0 378 width = get_terminal_columns() 379 width = min(width, self.MAX_WIDTH) 380 width -= self.MIN_WIDTH 381 if width > 0: 382 self.enabled = True 383 else: 384 self.enabled = False 385 self.width = width 386 387 def inc(self): 388 """Increment the counter and show the progress bar.""" 389 if not self.enabled: 390 return 391 self.cur += 1 392 arrow_len = self.width * self.cur // self.total 393 msg = '%4d/%d [' % (self.cur, self.total) 394 msg += '=' * arrow_len + '>' + ' ' * (self.width - arrow_len) + ']' 395 sys.stdout.write('\r' + msg) 396 sys.stdout.flush() 397 398def __gen_boards_cfg(jobs): 399 """Generate boards.cfg file. 400 401 Arguments: 402 jobs: The number of jobs to run simultaneously 403 404 Note: 405 The incomplete boards.cfg is left over when an error (including 406 the termination by the keyboard interrupt) occurs on the halfway. 407 """ 408 check_top_directory() 409 print 'Generating %s ... (jobs: %d)' % (BOARD_FILE, jobs) 410 411 # All the defconfig files to be processed 412 defconfigs = [] 413 for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR): 414 dirpath = dirpath[len(CONFIG_DIR) + 1:] 415 for filename in fnmatch.filter(filenames, '*_defconfig'): 416 defconfigs.append(os.path.join(dirpath, filename)) 417 418 # Parse all the MAINTAINERS files 419 maintainers_database = MaintainersDatabase() 420 for (dirpath, dirnames, filenames) in os.walk('.'): 421 if 'MAINTAINERS' in filenames: 422 maintainers_database.parse_file(os.path.join(dirpath, 423 'MAINTAINERS')) 424 425 # Output lines should be piped into the reformat tool 426 reformat_process = subprocess.Popen(REFORMAT_CMD, stdin=subprocess.PIPE, 427 stdout=open(BOARD_FILE, 'w')) 428 pipe = reformat_process.stdin 429 pipe.write(COMMENT_BLOCK) 430 431 indicator = Indicator(len(defconfigs)) 432 slots = Slots(jobs, pipe, maintainers_database) 433 434 # Main loop to process defconfig files: 435 # Add a new subprocess into a vacant slot. 436 # Sleep if there is no available slot. 437 for defconfig in defconfigs: 438 while not slots.add(defconfig): 439 while not slots.available(): 440 # No available slot: sleep for a while 441 time.sleep(SLEEP_TIME) 442 indicator.inc() 443 444 # wait until all the subprocesses finish 445 while not slots.empty(): 446 time.sleep(SLEEP_TIME) 447 print '' 448 449 # wait until the reformat tool finishes 450 reformat_process.communicate() 451 if reformat_process.returncode != 0: 452 sys.exit('"%s" failed' % REFORMAT_CMD[0]) 453 454def gen_boards_cfg(jobs): 455 """Generate boards.cfg file. 456 457 The incomplete boards.cfg is deleted if an error (including 458 the termination by the keyboard interrupt) occurs on the halfway. 459 460 Arguments: 461 jobs: The number of jobs to run simultaneously 462 """ 463 try: 464 __gen_boards_cfg(jobs) 465 except: 466 # We should remove incomplete boards.cfg 467 try: 468 os.remove(BOARD_FILE) 469 except OSError as exception: 470 # Ignore 'No such file or directory' error 471 if exception.errno != errno.ENOENT: 472 raise 473 raise 474 475def main(): 476 parser = optparse.OptionParser() 477 # Add options here 478 parser.add_option('-j', '--jobs', 479 help='the number of jobs to run simultaneously') 480 (options, args) = parser.parse_args() 481 if options.jobs: 482 try: 483 jobs = int(options.jobs) 484 except ValueError: 485 sys.exit('Option -j (--jobs) takes a number') 486 else: 487 try: 488 jobs = int(subprocess.Popen(['getconf', '_NPROCESSORS_ONLN'], 489 stdout=subprocess.PIPE).communicate()[0]) 490 except (OSError, ValueError): 491 print 'info: failed to get the number of CPUs. Set jobs to 1' 492 jobs = 1 493 gen_boards_cfg(jobs) 494 495if __name__ == '__main__': 496 main() 497