xref: /rk3399_rockchip-uboot/tools/genboardscfg.py (revision 13246f48610757d1e5d7f4d92f35378e749235ec) 
1#!/usr/bin/env python
2#
3# Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
4#
5# SPDX-License-Identifier:	GPL-2.0+
6#
7
8"""
9Converter from Kconfig and MAINTAINERS to boards.cfg
10
11Run 'tools/genboardscfg.py' to create boards.cfg file.
12
13Run 'tools/genboardscfg.py -h' for available options.
14"""
15
16import errno
17import fnmatch
18import glob
19import optparse
20import os
21import re
22import shutil
23import subprocess
24import sys
25import tempfile
26import time
27
28BOARD_FILE = 'boards.cfg'
29CONFIG_DIR = 'configs'
30REFORMAT_CMD = [os.path.join('tools', 'reformat.py'),
31                '-i', '-d', '-', '-s', '8']
32SHOW_GNU_MAKE = 'scripts/show-gnu-make'
33SLEEP_TIME=0.03
34
35COMMENT_BLOCK = '''#
36# List of boards
37#   Automatically generated by %s: don't edit
38#
39# Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers
40
41''' % __file__
42
43### helper functions ###
44def get_terminal_columns():
45    """Get the width of the terminal.
46
47    Returns:
48      The width of the terminal, or zero if the stdout is not
49      associated with tty.
50    """
51    try:
52        return shutil.get_terminal_size().columns # Python 3.3~
53    except AttributeError:
54        import fcntl
55        import termios
56        import struct
57        arg = struct.pack('hhhh', 0, 0, 0, 0)
58        try:
59            ret = fcntl.ioctl(sys.stdout.fileno(), termios.TIOCGWINSZ, arg)
60        except IOError as exception:
61            # If 'Inappropriate ioctl for device' error occurs,
62            # stdout is probably redirected. Return 0.
63            return 0
64        return struct.unpack('hhhh', ret)[1]
65
66def get_devnull():
67    """Get the file object of '/dev/null' device."""
68    try:
69        devnull = subprocess.DEVNULL # py3k
70    except AttributeError:
71        devnull = open(os.devnull, 'wb')
72    return devnull
73
74def check_top_directory():
75    """Exit if we are not at the top of source directory."""
76    for f in ('README', 'Licenses'):
77        if not os.path.exists(f):
78            sys.exit('Please run at the top of source directory.')
79
80def get_make_cmd():
81    """Get the command name of GNU Make."""
82    process = subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE)
83    ret = process.communicate()
84    if process.returncode:
85        sys.exit('GNU Make not found')
86    return ret[0].rstrip()
87
88### classes ###
89class MaintainersDatabase:
90
91    """The database of board status and maintainers."""
92
93    def __init__(self):
94        """Create an empty database."""
95        self.database = {}
96
97    def get_status(self, target):
98        """Return the status of the given board.
99
100        Returns:
101          Either 'Active' or 'Orphan'
102        """
103        if not target in self.database:
104            print >> sys.stderr, "WARNING: no status info for '%s'" % target
105            return '-'
106
107        tmp = self.database[target][0]
108        if tmp.startswith('Maintained'):
109            return 'Active'
110        elif tmp.startswith('Orphan'):
111            return 'Orphan'
112        else:
113            print >> sys.stderr, ("WARNING: %s: unknown status for '%s'" %
114                                  (tmp, target))
115            return '-'
116
117    def get_maintainers(self, target):
118        """Return the maintainers of the given board.
119
120        If the board has two or more maintainers, they are separated
121        with colons.
122        """
123        if not target in self.database:
124            print >> sys.stderr, "WARNING: no maintainers for '%s'" % target
125            return ''
126
127        return ':'.join(self.database[target][1])
128
129    def parse_file(self, file):
130        """Parse the given MAINTAINERS file.
131
132        This method parses MAINTAINERS and add board status and
133        maintainers information to the database.
134
135        Arguments:
136          file: MAINTAINERS file to be parsed
137        """
138        targets = []
139        maintainers = []
140        status = '-'
141        for line in open(file):
142            tag, rest = line[:2], line[2:].strip()
143            if tag == 'M:':
144                maintainers.append(rest)
145            elif tag == 'F:':
146                # expand wildcard and filter by 'configs/*_defconfig'
147                for f in glob.glob(rest):
148                    front, match, rear = f.partition('configs/')
149                    if not front and match:
150                        front, match, rear = rear.rpartition('_defconfig')
151                        if match and not rear:
152                            targets.append(front)
153            elif tag == 'S:':
154                status = rest
155            elif line == '\n':
156                for target in targets:
157                    self.database[target] = (status, maintainers)
158                targets = []
159                maintainers = []
160                status = '-'
161        if targets:
162            for target in targets:
163                self.database[target] = (status, maintainers)
164
165class DotConfigParser:
166
167    """A parser of .config file.
168
169    Each line of the output should have the form of:
170    Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers
171    Most of them are extracted from .config file.
172    MAINTAINERS files are also consulted for Status and Maintainers fields.
173    """
174
175    re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"')
176    re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"')
177    re_soc = re.compile(r'CONFIG_SYS_SOC="(.*)"')
178    re_vendor = re.compile(r'CONFIG_SYS_VENDOR="(.*)"')
179    re_board = re.compile(r'CONFIG_SYS_BOARD="(.*)"')
180    re_config = re.compile(r'CONFIG_SYS_CONFIG_NAME="(.*)"')
181    re_options = re.compile(r'CONFIG_SYS_EXTRA_OPTIONS="(.*)"')
182    re_list = (('arch', re_arch), ('cpu', re_cpu), ('soc', re_soc),
183               ('vendor', re_vendor), ('board', re_board),
184               ('config', re_config), ('options', re_options))
185    must_fields = ('arch', 'config')
186
187    def __init__(self, build_dir, output, maintainers_database):
188        """Create a new .config perser.
189
190        Arguments:
191          build_dir: Build directory where .config is located
192          output: File object which the result is written to
193          maintainers_database: An instance of class MaintainersDatabase
194        """
195        self.dotconfig = os.path.join(build_dir, '.config')
196        self.output = output
197        self.database = maintainers_database
198
199    def parse(self, defconfig):
200        """Parse .config file and output one-line database for the given board.
201
202        Arguments:
203          defconfig: Board (defconfig) name
204        """
205        fields = {}
206        for line in open(self.dotconfig):
207            if not line.startswith('CONFIG_SYS_'):
208                continue
209            for (key, pattern) in self.re_list:
210                m = pattern.match(line)
211                if m and m.group(1):
212                    fields[key] = m.group(1)
213                    break
214
215        # sanity check of '.config' file
216        for field in self.must_fields:
217            if not field in fields:
218                print >> sys.stderr, (
219                    "WARNING: '%s' is not defined in '%s'. Skip." %
220                    (field, defconfig))
221                return
222
223        # fix-up for aarch64
224        if fields['arch'] == 'arm' and 'cpu' in fields:
225            if fields['cpu'] == 'armv8':
226                fields['arch'] = 'aarch64'
227
228        target, match, rear = defconfig.partition('_defconfig')
229        assert match and not rear, \
230                                '%s : invalid defconfig file name' % defconfig
231
232        fields['status'] = self.database.get_status(target)
233        fields['maintainers'] = self.database.get_maintainers(target)
234
235        if 'options' in fields:
236            options = fields['config'] + ':' + \
237                      fields['options'].replace(r'\"', '"')
238        elif fields['config'] != target:
239            options = fields['config']
240        else:
241            options = '-'
242
243        self.output.write((' '.join(['%s'] * 9) + '\n')  %
244                          (fields['status'],
245                           fields['arch'],
246                           fields.get('cpu', '-'),
247                           fields.get('soc', '-'),
248                           fields.get('vendor', '-'),
249                           fields.get('board', '-'),
250                           target,
251                           options,
252                           fields['maintainers']))
253
254class Slot:
255
256    """A slot to store a subprocess.
257
258    Each instance of this class handles one subprocess.
259    This class is useful to control multiple processes
260    for faster processing.
261    """
262
263    def __init__(self, output, maintainers_database, devnull, make_cmd):
264        """Create a new slot.
265
266        Arguments:
267          output: File object which the result is written to
268          maintainers_database: An instance of class MaintainersDatabase
269        """
270        self.occupied = False
271        self.build_dir = tempfile.mkdtemp()
272        self.devnull = devnull
273        self.make_cmd = make_cmd
274        self.parser = DotConfigParser(self.build_dir, output,
275                                      maintainers_database)
276
277    def __del__(self):
278        """Delete the working directory"""
279        shutil.rmtree(self.build_dir)
280
281    def add(self, defconfig):
282        """Add a new subprocess to the slot.
283
284        Fails if the slot is occupied, that is, the current subprocess
285        is still running.
286
287        Arguments:
288          defconfig: Board (defconfig) name
289
290        Returns:
291          Return True on success or False on fail
292        """
293        if self.occupied:
294            return False
295        o = 'O=' + self.build_dir
296        self.ps = subprocess.Popen([self.make_cmd, o, defconfig],
297                                   stdout=self.devnull)
298        self.defconfig = defconfig
299        self.occupied = True
300        return True
301
302    def poll(self):
303        """Check if the subprocess is running and invoke the .config
304        parser if the subprocess is terminated.
305
306        Returns:
307          Return True if the subprocess is terminated, False otherwise
308        """
309        if not self.occupied:
310            return True
311        if self.ps.poll() == None:
312            return False
313        if self.ps.poll() == 0:
314            self.parser.parse(self.defconfig)
315        else:
316            print >> sys.stderr, ("WARNING: failed to process '%s'. skip." %
317                                  self.defconfig)
318        self.occupied = False
319        return True
320
321class Slots:
322
323    """Controller of the array of subprocess slots."""
324
325    def __init__(self, jobs, output, maintainers_database):
326        """Create a new slots controller.
327
328        Arguments:
329          jobs: A number of slots to instantiate
330          output: File object which the result is written to
331          maintainers_database: An instance of class MaintainersDatabase
332        """
333        self.slots = []
334        devnull = get_devnull()
335        make_cmd = get_make_cmd()
336        for i in range(jobs):
337            self.slots.append(Slot(output, maintainers_database,
338                                   devnull, make_cmd))
339
340    def add(self, defconfig):
341        """Add a new subprocess if a vacant slot is available.
342
343        Arguments:
344          defconfig: Board (defconfig) name
345
346        Returns:
347          Return True on success or False on fail
348        """
349        for slot in self.slots:
350            if slot.add(defconfig):
351                return True
352        return False
353
354    def available(self):
355        """Check if there is a vacant slot.
356
357        Returns:
358          Return True if a vacant slot is found, False if all slots are full
359        """
360        for slot in self.slots:
361            if slot.poll():
362                return True
363        return False
364
365    def empty(self):
366        """Check if all slots are vacant.
367
368        Returns:
369          Return True if all slots are vacant, False if at least one slot
370          is running
371        """
372        ret = True
373        for slot in self.slots:
374            if not slot.poll():
375                ret = False
376        return ret
377
378class Indicator:
379
380    """A class to control the progress indicator."""
381
382    MIN_WIDTH = 15
383    MAX_WIDTH = 70
384
385    def __init__(self, total):
386        """Create an instance.
387
388        Arguments:
389          total: A number of boards
390        """
391        self.total = total
392        self.cur = 0
393        width = get_terminal_columns()
394        width = min(width, self.MAX_WIDTH)
395        width -= self.MIN_WIDTH
396        if width > 0:
397            self.enabled = True
398        else:
399            self.enabled = False
400        self.width = width
401
402    def inc(self):
403        """Increment the counter and show the progress bar."""
404        if not self.enabled:
405            return
406        self.cur += 1
407        arrow_len = self.width * self.cur // self.total
408        msg = '%4d/%d [' % (self.cur, self.total)
409        msg += '=' * arrow_len + '>' + ' ' * (self.width - arrow_len) + ']'
410        sys.stdout.write('\r' + msg)
411        sys.stdout.flush()
412
413def __gen_boards_cfg(jobs):
414    """Generate boards.cfg file.
415
416    Arguments:
417      jobs: The number of jobs to run simultaneously
418
419    Note:
420      The incomplete boards.cfg is left over when an error (including
421      the termination by the keyboard interrupt) occurs on the halfway.
422    """
423    check_top_directory()
424    print 'Generating %s ...  (jobs: %d)' % (BOARD_FILE, jobs)
425
426    # All the defconfig files to be processed
427    defconfigs = []
428    for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
429        dirpath = dirpath[len(CONFIG_DIR) + 1:]
430        for filename in fnmatch.filter(filenames, '*_defconfig'):
431            if fnmatch.fnmatch(filename, '.*'):
432                continue
433            defconfigs.append(os.path.join(dirpath, filename))
434
435    # Parse all the MAINTAINERS files
436    maintainers_database = MaintainersDatabase()
437    for (dirpath, dirnames, filenames) in os.walk('.'):
438        if 'MAINTAINERS' in filenames:
439            maintainers_database.parse_file(os.path.join(dirpath,
440                                                         'MAINTAINERS'))
441
442    # Output lines should be piped into the reformat tool
443    reformat_process = subprocess.Popen(REFORMAT_CMD, stdin=subprocess.PIPE,
444                                        stdout=open(BOARD_FILE, 'w'))
445    pipe = reformat_process.stdin
446    pipe.write(COMMENT_BLOCK)
447
448    indicator = Indicator(len(defconfigs))
449    slots = Slots(jobs, pipe, maintainers_database)
450
451    # Main loop to process defconfig files:
452    #  Add a new subprocess into a vacant slot.
453    #  Sleep if there is no available slot.
454    for defconfig in defconfigs:
455        while not slots.add(defconfig):
456            while not slots.available():
457                # No available slot: sleep for a while
458                time.sleep(SLEEP_TIME)
459        indicator.inc()
460
461    # wait until all the subprocesses finish
462    while not slots.empty():
463        time.sleep(SLEEP_TIME)
464    print ''
465
466    # wait until the reformat tool finishes
467    reformat_process.communicate()
468    if reformat_process.returncode != 0:
469        sys.exit('"%s" failed' % REFORMAT_CMD[0])
470
471def gen_boards_cfg(jobs):
472    """Generate boards.cfg file.
473
474    The incomplete boards.cfg is deleted if an error (including
475    the termination by the keyboard interrupt) occurs on the halfway.
476
477    Arguments:
478      jobs: The number of jobs to run simultaneously
479    """
480    try:
481        __gen_boards_cfg(jobs)
482    except:
483        # We should remove incomplete boards.cfg
484        try:
485            os.remove(BOARD_FILE)
486        except OSError as exception:
487            # Ignore 'No such file or directory' error
488            if exception.errno != errno.ENOENT:
489                raise
490        raise
491
492def main():
493    parser = optparse.OptionParser()
494    # Add options here
495    parser.add_option('-j', '--jobs',
496                      help='the number of jobs to run simultaneously')
497    (options, args) = parser.parse_args()
498    if options.jobs:
499        try:
500            jobs = int(options.jobs)
501        except ValueError:
502            sys.exit('Option -j (--jobs) takes a number')
503    else:
504        try:
505            jobs = int(subprocess.Popen(['getconf', '_NPROCESSORS_ONLN'],
506                                     stdout=subprocess.PIPE).communicate()[0])
507        except (OSError, ValueError):
508            print 'info: failed to get the number of CPUs. Set jobs to 1'
509            jobs = 1
510    gen_boards_cfg(jobs)
511
512if __name__ == '__main__':
513    main()
514