1#!/usr/bin/env python3 2 3# Copyright (C) 2014 by Thomas Petazzoni <thomas.petazzoni@free-electrons.com> 4# 5# This program is free software; you can redistribute it and/or modify 6# it under the terms of the GNU General Public License as published by 7# the Free Software Foundation; either version 2 of the License, or 8# (at your option) any later version. 9# 10# This program is distributed in the hope that it will be useful, 11# but WITHOUT ANY WARRANTY; without even the implied warranty of 12# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13# General Public License for more details. 14# 15# You should have received a copy of the GNU General Public License 16# along with this program; if not, write to the Free Software 17# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 18 19# This script generates a random configuration for testing Buildroot. 20 21import contextlib 22import csv 23import os 24from random import randint 25import subprocess 26import sys 27from distutils.version import StrictVersion 28import platform 29 30if sys.hexversion >= 0x3000000: 31 import urllib.request as _urllib 32else: 33 import urllib2 as _urllib 34 35 36def urlopen_closing(uri): 37 return contextlib.closing(_urllib.urlopen(uri)) 38 39 40class SystemInfo: 41 DEFAULT_NEEDED_PROGS = ["make", "git", "gcc", "timeout"] 42 DEFAULT_OPTIONAL_PROGS = ["bzr", "java", "javac", "jar", "diffoscope"] 43 44 def __init__(self): 45 self.needed_progs = list(self.__class__.DEFAULT_NEEDED_PROGS) 46 self.optional_progs = list(self.__class__.DEFAULT_OPTIONAL_PROGS) 47 self.progs = {} 48 49 def find_prog(self, name, flags=os.X_OK, env=os.environ): 50 if not name or name[0] == os.sep: 51 raise ValueError(name) 52 53 prog_path = env.get("PATH", None) 54 # for windows compatibility, we'd need to take PATHEXT into account 55 56 if prog_path: 57 for prog_dir in filter(None, prog_path.split(os.pathsep)): 58 # os.join() not necessary: non-empty prog_dir 59 # and name[0] != os.sep 60 prog = prog_dir + os.sep + name 61 if os.access(prog, flags): 62 return prog 63 # -- 64 return None 65 66 def has(self, prog): 67 """Checks whether a program is available. 68 Lazily evaluates missing entries. 69 70 Returns: None if prog not found, else path to the program [evaluates 71 to True] 72 """ 73 try: 74 return self.progs[prog] 75 except KeyError: 76 pass 77 78 have_it = self.find_prog(prog) 79 # java[c] needs special care 80 if have_it and prog in ('java', 'javac'): 81 with open(os.devnull, "w") as devnull: 82 if subprocess.call("%s -version | grep gcj" % prog, 83 shell=True, 84 stdout=devnull, stderr=devnull) != 1: 85 have_it = False 86 # -- 87 self.progs[prog] = have_it 88 return have_it 89 90 def check_requirements(self): 91 """Checks program dependencies. 92 93 Returns: True if all mandatory programs are present, else False. 94 """ 95 do_check_has_prog = self.has 96 97 missing_requirements = False 98 for prog in self.needed_progs: 99 if not do_check_has_prog(prog): 100 print("ERROR: your system lacks the '%s' program" % prog) 101 missing_requirements = True 102 103 # check optional programs here, 104 # else they'd get checked by each worker instance 105 for prog in self.optional_progs: 106 do_check_has_prog(prog) 107 108 return not missing_requirements 109 110 111def get_toolchain_configs(toolchains_csv, buildrootdir): 112 """Fetch and return the possible toolchain configurations 113 114 This function returns an array of toolchain configurations. Each 115 toolchain configuration is itself an array of lines of the defconfig. 116 """ 117 118 with open(toolchains_csv) as r: 119 # filter empty lines and comments 120 lines = [t for t in r.readlines() if len(t.strip()) > 0 and t[0] != '#'] 121 toolchains = lines 122 configs = [] 123 124 (_, _, _, _, hostarch) = os.uname() 125 # ~2015 distros report x86 when on a 32bit install 126 if hostarch == 'i686' or hostarch == 'i386' or hostarch == 'x86': 127 hostarch = 'x86' 128 129 for row in csv.reader(toolchains): 130 config = {} 131 configfile = row[0] 132 config_hostarch = row[1] 133 keep = False 134 135 # Keep all toolchain configs that work regardless of the host 136 # architecture 137 if config_hostarch == "any": 138 keep = True 139 140 # Keep all toolchain configs that can work on the current host 141 # architecture 142 if hostarch == config_hostarch: 143 keep = True 144 145 # Assume that x86 32 bits toolchains work on x86_64 build 146 # machines 147 if hostarch == 'x86_64' and config_hostarch == "x86": 148 keep = True 149 150 if not keep: 151 continue 152 153 if not os.path.isabs(configfile): 154 configfile = os.path.join(buildrootdir, configfile) 155 156 with open(configfile) as r: 157 config = r.readlines() 158 configs.append(config) 159 return configs 160 161 162def is_toolchain_usable(configfile, config): 163 """Check if the toolchain is actually usable.""" 164 165 with open(configfile) as configf: 166 configlines = configf.readlines() 167 168 # Check that the toolchain configuration is still present 169 for toolchainline in config: 170 if toolchainline not in configlines: 171 print("WARN: toolchain can't be used", file=sys.stderr) 172 print(" Missing: %s" % toolchainline.strip(), file=sys.stderr) 173 return False 174 175 # The latest Linaro toolchains on x86-64 hosts requires glibc 176 # 2.14+ on the host. 177 if platform.machine() == 'x86_64': 178 if 'BR2_TOOLCHAIN_EXTERNAL_LINARO_ARM=y\n' in configlines or \ 179 'BR2_TOOLCHAIN_EXTERNAL_LINARO_AARCH64=y\n' in configlines or \ 180 'BR2_TOOLCHAIN_EXTERNAL_LINARO_AARCH64_BE=y\n' in configlines or \ 181 'BR2_TOOLCHAIN_EXTERNAL_LINARO_ARMEB=y\n' in configlines: 182 ldd_version_output = subprocess.check_output(['ldd', '--version']) 183 glibc_version = ldd_version_output.splitlines()[0].split()[-1] 184 if StrictVersion('2.14') > StrictVersion(glibc_version): 185 print("WARN: ignoring the Linaro ARM toolchains because too old host glibc", file=sys.stderr) 186 return False 187 188 return True 189 190 191def fixup_config(sysinfo, configfile): 192 """Finalize the configuration and reject any problematic combinations 193 194 This function returns 'True' when the configuration has been 195 accepted, and 'False' when the configuration has not been accepted because 196 it is known to fail (in which case another random configuration will be 197 generated). 198 """ 199 200 with open(configfile) as configf: 201 configlines = configf.readlines() 202 203 BR2_TOOLCHAIN_EXTERNAL_URL = 'BR2_TOOLCHAIN_EXTERNAL_URL="http://autobuild.buildroot.org/toolchains/tarballs/' 204 205 if "BR2_NEEDS_HOST_JAVA=y\n" in configlines and not sysinfo.has("java"): 206 return False 207 # The ctng toolchain is affected by PR58854 208 if 'BR2_PACKAGE_LTTNG_TOOLS=y\n' in configlines and \ 209 BR2_TOOLCHAIN_EXTERNAL_URL + 'armv5-ctng-linux-gnueabi.tar.xz"\n' in configlines: 210 return False 211 # The ctng toolchain tigger an assembler error with guile package when compiled with -Os (same issue as for CS ARM 2014.05-29) 212 if 'BR2_PACKAGE_GUILE=y\n' in configlines and \ 213 'BR2_OPTIMIZE_S=y\n' in configlines and \ 214 BR2_TOOLCHAIN_EXTERNAL_URL + 'armv5-ctng-linux-gnueabi.tar.xz"\n' in configlines: 215 return False 216 # The ctng toolchain is affected by PR58854 217 if 'BR2_PACKAGE_LTTNG_TOOLS=y\n' in configlines and \ 218 BR2_TOOLCHAIN_EXTERNAL_URL + 'armv6-ctng-linux-uclibcgnueabi.tar.xz"\n' in configlines: 219 return False 220 # The ctng toolchain is affected by PR58854 221 if 'BR2_PACKAGE_LTTNG_TOOLS=y\n' in configlines and \ 222 BR2_TOOLCHAIN_EXTERNAL_URL + 'armv7-ctng-linux-gnueabihf.tar.xz"\n' in configlines: 223 return False 224 # The ctng toolchain is affected by PR60155 225 if 'BR2_PACKAGE_SDL=y\n' in configlines and \ 226 BR2_TOOLCHAIN_EXTERNAL_URL + 'powerpc-ctng-linux-uclibc.tar.xz"\n' in configlines: 227 return False 228 # The ctng toolchain is affected by PR60155 229 if 'BR2_PACKAGE_LIBMPEG2=y\n' in configlines and \ 230 BR2_TOOLCHAIN_EXTERNAL_URL + 'powerpc-ctng-linux-uclibc.tar.xz"\n' in configlines: 231 return False 232 # This MIPS toolchain uses eglibc-2.18 which lacks SYS_getdents64 233 if 'BR2_PACKAGE_STRONGSWAN=y\n' in configlines and \ 234 BR2_TOOLCHAIN_EXTERNAL_URL + 'mips64el-ctng_n64-linux-gnu.tar.xz"\n' in configlines: 235 return False 236 # This MIPS toolchain uses eglibc-2.18 which lacks SYS_getdents64 237 if 'BR2_PACKAGE_PYTHON3=y\n' in configlines and \ 238 BR2_TOOLCHAIN_EXTERNAL_URL + 'mips64el-ctng_n64-linux-gnu.tar.xz"\n' in configlines: 239 return False 240 # libffi not available on sh2a and ARMv7-M, but propagating libffi 241 # arch dependencies in Buildroot is really too much work, so we 242 # handle this here. 243 if 'BR2_sh2a=y\n' in configlines and \ 244 'BR2_PACKAGE_LIBFFI=y\n' in configlines: 245 return False 246 if 'BR2_ARM_CPU_ARMV7M=y\n' in configlines and \ 247 'BR2_PACKAGE_LIBFFI=y\n' in configlines: 248 return False 249 if 'BR2_nds32=y\n' in configlines and \ 250 'BR2_PACKAGE_LIBFFI=y\n' in configlines: 251 return False 252 if 'BR2_PACKAGE_SUNXI_BOARDS=y\n' in configlines: 253 configlines.remove('BR2_PACKAGE_SUNXI_BOARDS_FEX_FILE=""\n') 254 configlines.append('BR2_PACKAGE_SUNXI_BOARDS_FEX_FILE="a10/hackberry.fex"\n') 255 # This MIPS uClibc toolchain fails to build the gdb package 256 if 'BR2_PACKAGE_GDB=y\n' in configlines and \ 257 BR2_TOOLCHAIN_EXTERNAL_URL + 'mipsel-ctng-linux-uclibc.tar.xz"\n' in configlines: 258 return False 259 # This MIPS uClibc toolchain fails to build the rt-tests package 260 if 'BR2_PACKAGE_RT_TESTS=y\n' in configlines and \ 261 BR2_TOOLCHAIN_EXTERNAL_URL + 'mipsel-ctng-linux-uclibc.tar.xz"\n' in configlines: 262 return False 263 # This MIPS uClibc toolchain fails to build the civetweb package 264 if 'BR2_PACKAGE_CIVETWEB=y\n' in configlines and \ 265 BR2_TOOLCHAIN_EXTERNAL_URL + 'mipsel-ctng-linux-uclibc.tar.xz"\n' in configlines: 266 return False 267 # This MIPS ctng toolchain fails to build the python3 package 268 if 'BR2_PACKAGE_PYTHON3=y\n' in configlines and \ 269 BR2_TOOLCHAIN_EXTERNAL_URL + 'mips64el-ctng_n64-linux-gnu.tar.xz"\n' in configlines: 270 return False 271 # This MIPS uClibc toolchain fails to build the strace package 272 if 'BR2_PACKAGE_STRACE=y\n' in configlines and \ 273 BR2_TOOLCHAIN_EXTERNAL_URL + 'mipsel-ctng-linux-uclibc.tar.xz"\n' in configlines: 274 return False 275 # This MIPS uClibc toolchain fails to build the cdrkit package 276 if 'BR2_PACKAGE_CDRKIT=y\n' in configlines and \ 277 'BR2_STATIC_LIBS=y\n' in configlines and \ 278 BR2_TOOLCHAIN_EXTERNAL_URL + 'mipsel-ctng-linux-uclibc.tar.xz"\n' in configlines: 279 return False 280 # uClibc vfork static linking issue 281 if 'BR2_PACKAGE_ALSA_LIB=y\n' in configlines and \ 282 'BR2_STATIC_LIBS=y\n' in configlines and \ 283 BR2_TOOLCHAIN_EXTERNAL_URL + 'i486-ctng-linux-uclibc.tar.xz"\n' in configlines: 284 return False 285 # This MIPS uClibc toolchain fails to build the weston package 286 if 'BR2_PACKAGE_WESTON=y\n' in configlines and \ 287 BR2_TOOLCHAIN_EXTERNAL_URL + 'mipsel-ctng-linux-uclibc.tar.xz"\n' in configlines: 288 return False 289 # The cs nios2 2017.02 toolchain is affected by binutils PR19405 290 if 'BR2_TOOLCHAIN_EXTERNAL_CODESOURCERY_NIOSII=y\n' in configlines and \ 291 'BR2_PACKAGE_BOOST=y\n' in configlines: 292 return False 293 # The cs nios2 2017.02 toolchain is affected by binutils PR19405 294 if 'BR2_TOOLCHAIN_EXTERNAL_CODESOURCERY_NIOSII=y\n' in configlines and \ 295 'BR2_PACKAGE_QT5BASE_GUI=y\n' in configlines: 296 return False 297 # The cs nios2 2017.02 toolchain is affected by binutils PR19405 298 if 'BR2_TOOLCHAIN_EXTERNAL_CODESOURCERY_NIOSII=y\n' in configlines and \ 299 'BR2_PACKAGE_FLANN=y\n' in configlines: 300 return False 301 302 if 'BR2_PACKAGE_HOST_UBOOT_TOOLS_ENVIMAGE=y\n' in configlines: 303 bootenv = os.path.join(args.outputdir, "boot_env.txt") 304 with open(bootenv, "w+") as bootenvf: 305 bootenvf.write("prop=value") 306 configlines.remove('BR2_PACKAGE_HOST_UBOOT_TOOLS_ENVIMAGE_SOURCE=""\n') 307 configlines.append('BR2_PACKAGE_HOST_UBOOT_TOOLS_ENVIMAGE_SOURCE="%s"\n' % bootenv) 308 configlines.remove('BR2_PACKAGE_HOST_UBOOT_TOOLS_ENVIMAGE_SIZE=""\n') 309 configlines.append('BR2_PACKAGE_HOST_UBOOT_TOOLS_ENVIMAGE_SIZE="0x1000"\n') 310 311 if 'BR2_PACKAGE_HOST_UBOOT_TOOLS_BOOT_SCRIPT=y\n' in configlines: 312 bootscr = os.path.join(args.outputdir, "boot_script.txt") 313 with open(bootscr, "w+") as bootscrf: 314 bootscrf.write("prop=value") 315 configlines.remove('BR2_PACKAGE_HOST_UBOOT_TOOLS_BOOT_SCRIPT_SOURCE=""\n') 316 configlines.append('BR2_PACKAGE_HOST_UBOOT_TOOLS_BOOT_SCRIPT_SOURCE="%s"\n' % bootscr) 317 318 with open(configfile, "w+") as configf: 319 configf.writelines(configlines) 320 321 return True 322 323 324def gen_config(args): 325 """Generate a new random configuration 326 327 This function generates the configuration, by choosing a random 328 toolchain configuration and then generating a random selection of 329 packages. 330 """ 331 332 sysinfo = SystemInfo() 333 334 # Select a random toolchain configuration 335 configs = get_toolchain_configs(args.toolchains_csv, args.buildrootdir) 336 337 i = randint(0, len(configs) - 1) 338 toolchainconfig = configs[i] 339 340 configlines = list(toolchainconfig) 341 342 # Combine with the minimal configuration 343 minimalconfigfile = os.path.join(args.buildrootdir, 344 'support/config-fragments/minimal.config') 345 with open(minimalconfigfile) as minimalf: 346 configlines += minimalf.readlines() 347 348 # Allow hosts with old certificates to download over https 349 configlines.append("BR2_WGET=\"wget --passive-ftp -nd -t 3 --no-check-certificate\"\n") 350 351 # Per-package folder 352 if randint(0, 15) == 0: 353 configlines.append("BR2_PER_PACKAGE_DIRECTORIES=y\n") 354 355 # Amend the configuration with a few things. 356 if randint(0, 20) == 0: 357 configlines.append("BR2_ENABLE_DEBUG=y\n") 358 if randint(0, 20) == 0: 359 configlines.append("BR2_ENABLE_RUNTIME_DEBUG=y\n") 360 if randint(0, 1) == 0: 361 configlines.append("BR2_INIT_BUSYBOX=y\n") 362 elif randint(0, 15) == 0: 363 configlines.append("BR2_INIT_SYSTEMD=y\n") 364 elif randint(0, 10) == 0: 365 configlines.append("BR2_ROOTFS_DEVICE_CREATION_DYNAMIC_EUDEV=y\n") 366 if randint(0, 20) == 0: 367 configlines.append("BR2_STATIC_LIBS=y\n") 368 if randint(0, 20) == 0: 369 configlines.append("BR2_PACKAGE_PYTHON_PY_ONLY=y\n") 370 if randint(0, 20) == 0: 371 configlines.append("BR2_PACKAGE_PYTHON3_PY_ONLY=y\n") 372 if randint(0, 5) == 0: 373 configlines.append("BR2_OPTIMIZE_2=y\n") 374 if randint(0, 4) == 0: 375 configlines.append("BR2_SYSTEM_ENABLE_NLS=y\n") 376 if randint(0, 4) == 0: 377 configlines.append("BR2_FORTIFY_SOURCE_2=y\n") 378 379 # Randomly enable BR2_REPRODUCIBLE 10% of times 380 # also enable tar filesystem images for testing 381 if sysinfo.has("diffoscope") and randint(0, 10) == 0: 382 configlines.append("BR2_REPRODUCIBLE=y\n") 383 configlines.append("BR2_TARGET_ROOTFS_TAR=y\n") 384 385 # Write out the configuration file 386 if not os.path.exists(args.outputdir): 387 os.makedirs(args.outputdir) 388 if args.outputdir == os.path.abspath(os.path.join(args.buildrootdir, "output")): 389 configfile = os.path.join(args.buildrootdir, ".config") 390 else: 391 configfile = os.path.join(args.outputdir, ".config") 392 with open(configfile, "w+") as configf: 393 configf.writelines(configlines) 394 395 subprocess.check_call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir, 396 "olddefconfig"]) 397 398 if not is_toolchain_usable(configfile, toolchainconfig): 399 return 2 400 401 # Now, generate the random selection of packages, and fixup 402 # things if needed. 403 # Safe-guard, in case we can not quickly come to a valid 404 # configuration: allow at most 100 (arbitrary) iterations. 405 bounded_loop = 100 406 while True: 407 if bounded_loop == 0: 408 print("ERROR: cannot generate random configuration after 100 iterations", 409 file=sys.stderr) 410 return 1 411 bounded_loop -= 1 412 subprocess.check_call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir, 413 "KCONFIG_PROBABILITY=%d" % randint(1, 20), 414 "randpackageconfig"]) 415 416 if fixup_config(sysinfo, configfile): 417 break 418 419 subprocess.check_call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir, 420 "olddefconfig"]) 421 422 subprocess.check_call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir, 423 "savedefconfig"]) 424 425 return subprocess.call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir, 426 "dependencies"]) 427 428 429if __name__ == '__main__': 430 import argparse 431 parser = argparse.ArgumentParser(description="Generate a random configuration") 432 parser.add_argument("--outputdir", "-o", 433 help="Output directory (relative to current directory)", 434 type=str, default='output') 435 parser.add_argument("--buildrootdir", "-b", 436 help="Buildroot directory (relative to current directory)", 437 type=str, default='.') 438 parser.add_argument("--toolchains-csv", 439 help="Path of the toolchain configuration file", 440 type=str, 441 default="support/config-fragments/autobuild/toolchain-configs.csv") 442 args = parser.parse_args() 443 444 # We need the absolute path to use with O=, because the relative 445 # path to the output directory here is not relative to the 446 # Buildroot sources, but to the current directory. 447 args.outputdir = os.path.abspath(args.outputdir) 448 449 try: 450 ret = gen_config(args) 451 except Exception as e: 452 print(str(e), file=sys.stderr) 453 parser.exit(1) 454 parser.exit(ret) 455