xref: /rk3399_rockchip-uboot/tools/genboardscfg.py (revision 31e2141d5ad76b55e4e34ee7241adc1c3d9ef099) 
1#!/usr/bin/env python
2#
3# Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
4#
5# SPDX-License-Identifier:	GPL-2.0+
6#
7
8"""
9Converter from Kconfig and MAINTAINERS to boards.cfg
10
11Run 'tools/genboardscfg.py' to create boards.cfg file.
12
13Run 'tools/genboardscfg.py -h' for available options.
14"""
15
16import errno
17import fnmatch
18import glob
19import optparse
20import os
21import re
22import shutil
23import subprocess
24import sys
25import tempfile
26import time
27
28BOARD_FILE = 'boards.cfg'
29CONFIG_DIR = 'configs'
30REFORMAT_CMD = [os.path.join('tools', 'reformat.py'),
31                '-i', '-d', '-', '-s', '8']
32SHOW_GNU_MAKE = 'scripts/show-gnu-make'
33SLEEP_TIME=0.03
34
35COMMENT_BLOCK = '''#
36# List of boards
37#   Automatically generated by %s: don't edit
38#
39# Status, Arch, CPU(:SPLCPU), SoC, Vendor, Board, Target, Options, Maintainers
40
41''' % __file__
42
43### helper functions ###
44def get_terminal_columns():
45    """Get the width of the terminal.
46
47    Returns:
48      The width of the terminal, or zero if the stdout is not
49      associated with tty.
50    """
51    try:
52        return shutil.get_terminal_size().columns # Python 3.3~
53    except AttributeError:
54        import fcntl
55        import termios
56        import struct
57        arg = struct.pack('hhhh', 0, 0, 0, 0)
58        try:
59            ret = fcntl.ioctl(sys.stdout.fileno(), termios.TIOCGWINSZ, arg)
60        except IOError as exception:
61            if exception.errno != errno.ENOTTY:
62                raise
63            # If 'Inappropriate ioctl for device' error occurs,
64            # stdout is probably redirected. Return 0.
65            return 0
66        return struct.unpack('hhhh', ret)[1]
67
68def get_devnull():
69    """Get the file object of '/dev/null' device."""
70    try:
71        devnull = subprocess.DEVNULL # py3k
72    except AttributeError:
73        devnull = open(os.devnull, 'wb')
74    return devnull
75
76def check_top_directory():
77    """Exit if we are not at the top of source directory."""
78    for f in ('README', 'Licenses'):
79        if not os.path.exists(f):
80            sys.exit('Please run at the top of source directory.')
81
82def get_make_cmd():
83    """Get the command name of GNU Make."""
84    process = subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE)
85    ret = process.communicate()
86    if process.returncode:
87        sys.exit('GNU Make not found')
88    return ret[0].rstrip()
89
90### classes ###
91class MaintainersDatabase:
92
93    """The database of board status and maintainers."""
94
95    def __init__(self):
96        """Create an empty database."""
97        self.database = {}
98
99    def get_status(self, target):
100        """Return the status of the given board.
101
102        Returns:
103          Either 'Active' or 'Orphan'
104        """
105        tmp = self.database[target][0]
106        if tmp.startswith('Maintained'):
107            return 'Active'
108        elif tmp.startswith('Orphan'):
109            return 'Orphan'
110        else:
111            print >> sys.stderr, 'Error: %s: unknown status' % tmp
112
113    def get_maintainers(self, target):
114        """Return the maintainers of the given board.
115
116        If the board has two or more maintainers, they are separated
117        with colons.
118        """
119        return ':'.join(self.database[target][1])
120
121    def parse_file(self, file):
122        """Parse the given MAINTAINERS file.
123
124        This method parses MAINTAINERS and add board status and
125        maintainers information to the database.
126
127        Arguments:
128          file: MAINTAINERS file to be parsed
129        """
130        targets = []
131        maintainers = []
132        status = '-'
133        for line in open(file):
134            tag, rest = line[:2], line[2:].strip()
135            if tag == 'M:':
136                maintainers.append(rest)
137            elif tag == 'F:':
138                # expand wildcard and filter by 'configs/*_defconfig'
139                for f in glob.glob(rest):
140                    front, match, rear = f.partition('configs/')
141                    if not front and match:
142                        front, match, rear = rear.rpartition('_defconfig')
143                        if match and not rear:
144                            targets.append(front)
145            elif tag == 'S:':
146                status = rest
147            elif line == '\n' and targets:
148                for target in targets:
149                    self.database[target] = (status, maintainers)
150                targets = []
151                maintainers = []
152                status = '-'
153        if targets:
154            for target in targets:
155                self.database[target] = (status, maintainers)
156
157class DotConfigParser:
158
159    """A parser of .config file.
160
161    Each line of the output should have the form of:
162    Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers
163    Most of them are extracted from .config file.
164    MAINTAINERS files are also consulted for Status and Maintainers fields.
165    """
166
167    re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"')
168    re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"')
169    re_soc = re.compile(r'CONFIG_SYS_SOC="(.*)"')
170    re_vendor = re.compile(r'CONFIG_SYS_VENDOR="(.*)"')
171    re_board = re.compile(r'CONFIG_SYS_BOARD="(.*)"')
172    re_config = re.compile(r'CONFIG_SYS_CONFIG_NAME="(.*)"')
173    re_options = re.compile(r'CONFIG_SYS_EXTRA_OPTIONS="(.*)"')
174    re_list = (('arch', re_arch), ('cpu', re_cpu), ('soc', re_soc),
175               ('vendor', re_vendor), ('board', re_board),
176               ('config', re_config), ('options', re_options))
177    must_fields = ('arch', 'config')
178
179    def __init__(self, build_dir, output, maintainers_database):
180        """Create a new .config perser.
181
182        Arguments:
183          build_dir: Build directory where .config is located
184          output: File object which the result is written to
185          maintainers_database: An instance of class MaintainersDatabase
186        """
187        self.dotconfig = os.path.join(build_dir, '.config')
188        self.output = output
189        self.database = maintainers_database
190
191    def parse(self, defconfig):
192        """Parse .config file and output one-line database for the given board.
193
194        Arguments:
195          defconfig: Board (defconfig) name
196        """
197        fields = {}
198        for line in open(self.dotconfig):
199            if not line.startswith('CONFIG_SYS_'):
200                continue
201            for (key, pattern) in self.re_list:
202                m = pattern.match(line)
203                if m and m.group(1):
204                    fields[key] = m.group(1)
205                    break
206
207        # sanity check of '.config' file
208        for field in self.must_fields:
209            if not field in fields:
210                sys.exit('Error: %s is not defined in %s' % (field, defconfig))
211
212        # fix-up for aarch64 and tegra
213        if fields['arch'] == 'arm' and 'cpu' in fields:
214            if fields['cpu'] == 'armv8':
215                fields['arch'] = 'aarch64'
216            if 'soc' in fields and re.match('tegra[0-9]*$', fields['soc']):
217                fields['cpu'] += ':arm720t'
218
219        target, match, rear = defconfig.partition('_defconfig')
220        assert match and not rear, \
221                                '%s : invalid defconfig file name' % defconfig
222
223        fields['status'] = self.database.get_status(target)
224        fields['maintainers'] = self.database.get_maintainers(target)
225
226        if 'options' in fields:
227            options = fields['config'] + ':' + \
228                      fields['options'].replace(r'\"', '"')
229        elif fields['config'] != target:
230            options = fields['config']
231        else:
232            options = '-'
233
234        self.output.write((' '.join(['%s'] * 9) + '\n')  %
235                          (fields['status'],
236                           fields['arch'],
237                           fields.get('cpu', '-'),
238                           fields.get('soc', '-'),
239                           fields.get('vendor', '-'),
240                           fields.get('board', '-'),
241                           target,
242                           options,
243                           fields['maintainers']))
244
245class Slot:
246
247    """A slot to store a subprocess.
248
249    Each instance of this class handles one subprocess.
250    This class is useful to control multiple processes
251    for faster processing.
252    """
253
254    def __init__(self, output, maintainers_database, devnull, make_cmd):
255        """Create a new slot.
256
257        Arguments:
258          output: File object which the result is written to
259          maintainers_database: An instance of class MaintainersDatabase
260        """
261        self.occupied = False
262        self.build_dir = tempfile.mkdtemp()
263        self.devnull = devnull
264        self.make_cmd = make_cmd
265        self.parser = DotConfigParser(self.build_dir, output,
266                                      maintainers_database)
267
268    def __del__(self):
269        """Delete the working directory"""
270        shutil.rmtree(self.build_dir)
271
272    def add(self, defconfig):
273        """Add a new subprocess to the slot.
274
275        Fails if the slot is occupied, that is, the current subprocess
276        is still running.
277
278        Arguments:
279          defconfig: Board (defconfig) name
280
281        Returns:
282          Return True on success or False on fail
283        """
284        if self.occupied:
285            return False
286        o = 'O=' + self.build_dir
287        self.ps = subprocess.Popen([self.make_cmd, o, defconfig],
288                                   stdout=self.devnull)
289        self.defconfig = defconfig
290        self.occupied = True
291        return True
292
293    def poll(self):
294        """Check if the subprocess is running and invoke the .config
295        parser if the subprocess is terminated.
296
297        Returns:
298          Return True if the subprocess is terminated, False otherwise
299        """
300        if not self.occupied:
301            return True
302        if self.ps.poll() == None:
303            return False
304        self.parser.parse(self.defconfig)
305        self.occupied = False
306        return True
307
308class Slots:
309
310    """Controller of the array of subprocess slots."""
311
312    def __init__(self, jobs, output, maintainers_database):
313        """Create a new slots controller.
314
315        Arguments:
316          jobs: A number of slots to instantiate
317          output: File object which the result is written to
318          maintainers_database: An instance of class MaintainersDatabase
319        """
320        self.slots = []
321        devnull = get_devnull()
322        make_cmd = get_make_cmd()
323        for i in range(jobs):
324            self.slots.append(Slot(output, maintainers_database,
325                                   devnull, make_cmd))
326
327    def add(self, defconfig):
328        """Add a new subprocess if a vacant slot is available.
329
330        Arguments:
331          defconfig: Board (defconfig) name
332
333        Returns:
334          Return True on success or False on fail
335        """
336        for slot in self.slots:
337            if slot.add(defconfig):
338                return True
339        return False
340
341    def available(self):
342        """Check if there is a vacant slot.
343
344        Returns:
345          Return True if a vacant slot is found, False if all slots are full
346        """
347        for slot in self.slots:
348            if slot.poll():
349                return True
350        return False
351
352    def empty(self):
353        """Check if all slots are vacant.
354
355        Returns:
356          Return True if all slots are vacant, False if at least one slot
357          is running
358        """
359        ret = True
360        for slot in self.slots:
361            if not slot.poll():
362                ret = False
363        return ret
364
365class Indicator:
366
367    """A class to control the progress indicator."""
368
369    MIN_WIDTH = 15
370    MAX_WIDTH = 70
371
372    def __init__(self, total):
373        """Create an instance.
374
375        Arguments:
376          total: A number of boards
377        """
378        self.total = total
379        self.cur = 0
380        width = get_terminal_columns()
381        width = min(width, self.MAX_WIDTH)
382        width -= self.MIN_WIDTH
383        if width > 0:
384            self.enabled = True
385        else:
386            self.enabled = False
387        self.width = width
388
389    def inc(self):
390        """Increment the counter and show the progress bar."""
391        if not self.enabled:
392            return
393        self.cur += 1
394        arrow_len = self.width * self.cur // self.total
395        msg = '%4d/%d [' % (self.cur, self.total)
396        msg += '=' * arrow_len + '>' + ' ' * (self.width - arrow_len) + ']'
397        sys.stdout.write('\r' + msg)
398        sys.stdout.flush()
399
400def __gen_boards_cfg(jobs):
401    """Generate boards.cfg file.
402
403    Arguments:
404      jobs: The number of jobs to run simultaneously
405
406    Note:
407      The incomplete boards.cfg is left over when an error (including
408      the termination by the keyboard interrupt) occurs on the halfway.
409    """
410    check_top_directory()
411    print 'Generating %s ...  (jobs: %d)' % (BOARD_FILE, jobs)
412
413    # All the defconfig files to be processed
414    defconfigs = []
415    for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
416        dirpath = dirpath[len(CONFIG_DIR) + 1:]
417        for filename in fnmatch.filter(filenames, '*_defconfig'):
418            defconfigs.append(os.path.join(dirpath, filename))
419
420    # Parse all the MAINTAINERS files
421    maintainers_database = MaintainersDatabase()
422    for (dirpath, dirnames, filenames) in os.walk('.'):
423        if 'MAINTAINERS' in filenames:
424            maintainers_database.parse_file(os.path.join(dirpath,
425                                                         'MAINTAINERS'))
426
427    # Output lines should be piped into the reformat tool
428    reformat_process = subprocess.Popen(REFORMAT_CMD, stdin=subprocess.PIPE,
429                                        stdout=open(BOARD_FILE, 'w'))
430    pipe = reformat_process.stdin
431    pipe.write(COMMENT_BLOCK)
432
433    indicator = Indicator(len(defconfigs))
434    slots = Slots(jobs, pipe, maintainers_database)
435
436    # Main loop to process defconfig files:
437    #  Add a new subprocess into a vacant slot.
438    #  Sleep if there is no available slot.
439    for defconfig in defconfigs:
440        while not slots.add(defconfig):
441            while not slots.available():
442                # No available slot: sleep for a while
443                time.sleep(SLEEP_TIME)
444        indicator.inc()
445
446    # wait until all the subprocesses finish
447    while not slots.empty():
448        time.sleep(SLEEP_TIME)
449    print ''
450
451    # wait until the reformat tool finishes
452    reformat_process.communicate()
453    if reformat_process.returncode != 0:
454        sys.exit('"%s" failed' % REFORMAT_CMD[0])
455
456def gen_boards_cfg(jobs):
457    """Generate boards.cfg file.
458
459    The incomplete boards.cfg is deleted if an error (including
460    the termination by the keyboard interrupt) occurs on the halfway.
461
462    Arguments:
463      jobs: The number of jobs to run simultaneously
464    """
465    try:
466        __gen_boards_cfg(jobs)
467    except:
468        # We should remove incomplete boards.cfg
469        try:
470            os.remove(BOARD_FILE)
471        except OSError as exception:
472            # Ignore 'No such file or directory' error
473            if exception.errno != errno.ENOENT:
474                raise
475        raise
476
477def main():
478    parser = optparse.OptionParser()
479    # Add options here
480    parser.add_option('-j', '--jobs',
481                      help='the number of jobs to run simultaneously')
482    (options, args) = parser.parse_args()
483    if options.jobs:
484        try:
485            jobs = int(options.jobs)
486        except ValueError:
487            sys.exit('Option -j (--jobs) takes a number')
488    else:
489        try:
490            jobs = int(subprocess.Popen(['getconf', '_NPROCESSORS_ONLN'],
491                                     stdout=subprocess.PIPE).communicate()[0])
492        except (OSError, ValueError):
493            print 'info: failed to get the number of CPUs. Set jobs to 1'
494            jobs = 1
495    gen_boards_cfg(jobs)
496
497if __name__ == '__main__':
498    main()
499