1#!/usr/bin/env python3 2 3import os.path 4import re 5import requests 6import textwrap 7 8BASE_URL = "https://toolchains.bootlin.com/downloads/releases/toolchains" 9 10AUTOGENERATED_COMMENT = """# This file was auto-generated by support/scripts/gen-bootlin-toolchains 11# Do not edit 12""" 13 14# In the below dict: 15 16# - 'conditions' indicate the cumulative conditions under which the 17# toolchain will be made available. In several situations, a given 18# toolchain is usable on several architectures variants (for 19# example, an ARMv6 toolchain can be used on ARMv7) 20# - 'test_options' indicate one specific configuration where the 21# toolchain can be used. It is used to create the runtime test 22# cases. If 'test_options' does not exist, the code assumes it can 23# be made equal to 'conditions' 24# - 'prefix' is the prefix of the cross-compilation toolchain tools 25 26arches = { 27 'aarch64': { 28 'conditions': ['BR2_aarch64'], 29 'prefix': 'aarch64', 30 }, 31 'aarch64be': { 32 'conditions': ['BR2_aarch64_be'], 33 'prefix': 'aarch64_be', 34 }, 35 'arcle-750d': { 36 'conditions': ['BR2_arcle', 'BR2_arc750d'], 37 'prefix': 'arc', 38 }, 39 'arcle-hs38': { 40 'conditions': ['BR2_arcle', 'BR2_archs38'], 41 'prefix': 'arc', 42 }, 43 'armv5-eabi': { 44 'conditions': ['BR2_ARM_CPU_ARMV5', 'BR2_ARM_EABI'], 45 'test_options': ['BR2_arm', 'BR2_arm926t', 'BR2_ARM_EABI'], 46 'prefix': 'arm', 47 }, 48 'armv6-eabihf': { 49 'conditions': ['BR2_ARM_CPU_ARMV6', 'BR2_ARM_EABIHF'], 50 'test_options': ['BR2_arm', 'BR2_arm1176jzf_s', 'BR2_ARM_EABIHF'], 51 'prefix': 'arm', 52 }, 53 'armv7-eabihf': { 54 'conditions': ['BR2_ARM_CPU_ARMV7A', 'BR2_ARM_EABIHF'], 55 'test_options': ['BR2_arm', 'BR2_cortex_a8', 'BR2_ARM_EABIHF'], 56 'prefix': 'arm', 57 }, 58 'armv7m': { 59 'conditions': ['BR2_ARM_CPU_ARMV7M'], 60 'test_options': ['BR2_arm', 'BR2_cortex_m4'], 61 'prefix': 'arm', 62 }, 63 'm68k-68xxx': { 64 'conditions': ['BR2_m68k_m68k'], 65 'test_options': ['BR2_m68k', 'BR2_m68k_68040'], 66 'prefix': 'm68k', 67 }, 68 'm68k-coldfire': { 69 'conditions': ['BR2_m68k_cf'], 70 'test_options': ['BR2_m68k', 'BR2_m68k_cf5208'], 71 'prefix': 'm68k', 72 }, 73 'microblazebe': { 74 'conditions': ['BR2_microblazebe'], 75 'prefix': 'microblaze', 76 }, 77 'microblazeel': { 78 'conditions': ['BR2_microblazeel'], 79 'prefix': 'microblazeel', 80 }, 81 'mips32': { 82 # Not sure it could be used by other mips32 variants? 83 'conditions': ['BR2_mips', 'BR2_mips_32', '!BR2_MIPS_SOFT_FLOAT'], 84 'prefix': 'mips', 85 }, 86 'mips32el': { 87 # Not sure it could be used by other mips32el variants? 88 'conditions': ['BR2_mipsel', 'BR2_mips_32', '!BR2_MIPS_SOFT_FLOAT'], 89 'prefix': 'mipsel', 90 }, 91 'mips32r5el': { 92 'conditions': ['BR2_mipsel', 'BR2_mips_32r5', '!BR2_MIPS_SOFT_FLOAT'], 93 'prefix': 'mipsel', 94 }, 95 'mips32r6el': { 96 'conditions': ['BR2_mipsel', 'BR2_mips_32r6', '!BR2_MIPS_SOFT_FLOAT'], 97 'prefix': 'mipsel', 98 }, 99 'mips64': { 100 # Not sure it could be used by other mips64 variants? 101 'conditions': ['BR2_mips64', 'BR2_mips_64', '!BR2_MIPS_SOFT_FLOAT'], 102 'prefix': 'mips64', 103 }, 104 'mips64-n32': { 105 # Not sure it could be used by other mips64 variants? 106 'conditions': ['BR2_mips64', 'BR2_mips_64', 'BR2_MIPS_NABI32', '!BR2_MIPS_SOFT_FLOAT'], 107 'prefix': 'mips64', 108 }, 109 'mips64el-n32': { 110 # Not sure it could be used by other mips64el variants? 111 'conditions': ['BR2_mips64el', 'BR2_mips_64', 'BR2_MIPS_NABI32', '!BR2_MIPS_SOFT_FLOAT'], 112 'prefix': 'mips64el', 113 }, 114 'mips64r6el-n32': { 115 'conditions': ['BR2_mips64el', 'BR2_mips_64r6', 'BR2_MIPS_NABI32', '!BR2_MIPS_SOFT_FLOAT'], 116 'prefix': 'mips64el', 117 }, 118 'nios2': { 119 'conditions': ['BR2_nios2'], 120 'prefix': 'nios2', 121 }, 122 'openrisc': { 123 'conditions': ['BR2_or1k'], 124 'prefix': 'or1k', 125 }, 126 'powerpc-440fp': { 127 # Not sure it could be used by other powerpc variants? 128 'conditions': ['BR2_powerpc', 'BR2_powerpc_440fp'], 129 'prefix': 'powerpc', 130 }, 131 'powerpc-e300c3': { 132 # Not sure it could be used by other powerpc variants? 133 'conditions': ['BR2_powerpc', 'BR2_powerpc_e300c3'], 134 'prefix': 'powerpc', 135 }, 136 'powerpc-e500mc': { 137 # Not sure it could be used by other powerpc variants? 138 'conditions': ['BR2_powerpc', 'BR2_powerpc_e500mc'], 139 'prefix': 'powerpc', 140 }, 141 'powerpc64-e5500': { 142 'conditions': ['BR2_powerpc64', 'BR2_powerpc_e5500'], 143 'prefix': 'powerpc64', 144 }, 145 'powerpc64-e6500': { 146 'conditions': ['BR2_powerpc64', 'BR2_powerpc_e6500'], 147 'prefix': 'powerpc64', 148 }, 149 'powerpc64-power8': { 150 'conditions': ['BR2_powerpc64', 'BR2_powerpc_power8'], 151 'prefix': 'powerpc64', 152 }, 153 'powerpc64le-power8': { 154 'conditions': ['BR2_powerpc64le', 'BR2_powerpc_power8'], 155 'prefix': 'powerpc64le', 156 }, 157 'riscv32-ilp32d': { 158 'conditions': ['BR2_riscv', 'BR2_riscv_g', 'BR2_RISCV_32', 'BR2_RISCV_ABI_ILP32D'], 159 'prefix': 'riscv32', 160 }, 161 'riscv64': { 162 'conditions': ['BR2_riscv', 'BR2_riscv_g', 'BR2_RISCV_64', 'BR2_RISCV_ABI_LP64'], 163 'prefix': 'riscv64', 164 }, 165 'sh-sh4': { 166 'conditions': ['BR2_sh', 'BR2_sh4'], 167 'prefix': 'sh4', 168 }, 169 'sh-sh4aeb': { 170 'conditions': ['BR2_sh', 'BR2_sh4aeb'], 171 'prefix': 'sh4aeb', 172 }, 173 'sparc64': { 174 'conditions': ['BR2_sparc64', 'BR2_sparc_v9'], 175 'prefix': 'sparc64', 176 }, 177 'sparcv8': { 178 'conditions': ['BR2_sparc', 'BR2_sparc_v8'], 179 'prefix': 'sparc', 180 }, 181 'x86-64-core-i7': { 182 'conditions': ['BR2_x86_64', 183 'BR2_X86_CPU_HAS_MMX', 184 'BR2_X86_CPU_HAS_SSE', 185 'BR2_X86_CPU_HAS_SSE2', 186 'BR2_X86_CPU_HAS_SSE3', 187 'BR2_X86_CPU_HAS_SSSE3', 188 'BR2_X86_CPU_HAS_SSE4', 189 'BR2_X86_CPU_HAS_SSE42'], 190 'test_options': ['BR2_x86_64', 'BR2_x86_corei7'], 191 'prefix': 'x86_64', 192 }, 193 'x86-core2': { 194 'conditions': ['BR2_i386', 195 'BR2_X86_CPU_HAS_MMX', 196 'BR2_X86_CPU_HAS_SSE', 197 'BR2_X86_CPU_HAS_SSE2', 198 'BR2_X86_CPU_HAS_SSE3', 199 'BR2_X86_CPU_HAS_SSSE3'], 200 'test_options': ['BR2_i386', 'BR2_x86_core2'], 201 'prefix': 'i686', 202 }, 203 'x86-i686': { 204 'conditions': ['BR2_i386', 205 '!BR2_x86_i486', 206 '!BR2_x86_i586', 207 '!BR2_x86_x1000'], 208 'test_options': ['BR2_i386', 209 'BR2_x86_i686'], 210 'prefix': 'i686', 211 }, 212 'xtensa-lx60': { 213 'conditions': ['BR2_xtensa', 'BR2_XTENSA_CUSTOM', 'BR2_XTENSA_LITTLE_ENDIAN'], 214 'prefix': 'xtensa', 215 }, 216} 217 218 219class Toolchain: 220 def __init__(self, arch, libc, variant, version): 221 self.arch = arch 222 self.libc = libc 223 self.variant = variant 224 self.version = version 225 self.fname_prefix = "%s--%s--%s-%s" % (self.arch, self.libc, self.variant, self.version) 226 self.option_name = "BR2_TOOLCHAIN_EXTERNAL_BOOTLIN_%s_%s_%s" % \ 227 (self.arch.replace("-", "_").upper(), self.libc.upper(), self.variant.replace("-", "_").upper()) 228 self.fragment = requests.get(self.fragment_url).text.split("\n") 229 self.sha256 = requests.get(self.hash_url).text.split(" ")[0] 230 231 @property 232 def tarball_url(self): 233 return os.path.join(BASE_URL, self.arch, "tarballs", 234 self.fname_prefix + ".tar.bz2") 235 236 @property 237 def hash_url(self): 238 return os.path.join(BASE_URL, self.arch, "tarballs", 239 self.fname_prefix + ".sha256") 240 241 @property 242 def fragment_url(self): 243 return os.path.join(BASE_URL, self.arch, "fragments", 244 self.fname_prefix + ".frag") 245 246 def gen_config_in_options(self, f): 247 f.write("config %s\n" % self.option_name) 248 f.write("\tbool \"%s %s %s %s\"\n" % 249 (self.arch, self.libc, self.variant, self.version)) 250 depends = [] 251 selects = [] 252 253 for c in arches[self.arch]['conditions']: 254 depends.append(c) 255 256 for frag in self.fragment: 257 # libc type 258 if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_UCLIBC"): 259 selects.append("BR2_TOOLCHAIN_EXTERNAL_UCLIBC") 260 elif frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_GLIBC"): 261 # glibc needs mmu support 262 depends.append("BR2_USE_MMU") 263 # glibc doesn't support static only configuration 264 depends.append("!BR2_STATIC_LIBS") 265 selects.append("BR2_TOOLCHAIN_EXTERNAL_GLIBC") 266 elif frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_MUSL"): 267 # musl needs mmu support 268 depends.append("BR2_USE_MMU") 269 selects.append("BR2_TOOLCHAIN_EXTERNAL_MUSL") 270 271 # gcc version 272 if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_GCC_"): 273 m = re.match("^BR2_TOOLCHAIN_EXTERNAL_GCC_([0-9_]*)=y$", frag) 274 assert m, "Cannot get gcc version for toolchain %s" % self.fname_prefix 275 selects.append("BR2_TOOLCHAIN_GCC_AT_LEAST_%s" % m[1]) 276 277 # kernel headers version 278 if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HEADERS_"): 279 m = re.match("^BR2_TOOLCHAIN_EXTERNAL_HEADERS_([0-9_]*)=y$", frag) 280 assert m, "Cannot get kernel headers version for toolchain %s" % self.fname_prefix 281 selects.append("BR2_TOOLCHAIN_HEADERS_AT_LEAST_%s" % m[1]) 282 283 # C++ 284 if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CXX"): 285 selects.append("BR2_INSTALL_LIBSTDCPP") 286 287 # SSP 288 if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_SSP"): 289 selects.append("BR2_TOOLCHAIN_HAS_SSP") 290 291 # wchar 292 if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_WCHAR"): 293 selects.append("BR2_USE_WCHAR") 294 295 # locale 296 if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_LOCALE"): 297 # locale implies the availability of wchar 298 selects.append("BR2_USE_WCHAR") 299 selects.append("BR2_ENABLE_LOCALE") 300 301 # thread support 302 if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_THREADS"): 303 selects.append("BR2_TOOLCHAIN_HAS_THREADS") 304 305 if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_THREADS_DEBUG"): 306 selects.append("BR2_TOOLCHAIN_HAS_THREADS_DEBUG") 307 308 if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_THREADS_NPTL"): 309 selects.append("BR2_TOOLCHAIN_HAS_THREADS_NPTL") 310 311 # RPC 312 if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_INET_RPC"): 313 selects.append("BR2_TOOLCHAIN_HAS_NATIVE_RPC") 314 315 # D language 316 if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_DLANG"): 317 selects.append("BR2_TOOLCHAIN_HAS_DLANG") 318 319 # fortran 320 if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_FORTRAN"): 321 selects.append("BR2_TOOLCHAIN_HAS_FORTRAN") 322 323 # OpenMP 324 if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_OPENMP"): 325 selects.append("BR2_TOOLCHAIN_HAS_OPENMP") 326 327 for depend in depends: 328 f.write("\tdepends on %s\n" % depend) 329 330 for select in selects: 331 f.write("\tselect %s\n" % select) 332 333 f.write("\thelp\n") 334 335 desc = "Bootlin toolchain for the %s architecture, using the %s C library. " % \ 336 (self.arch, self.libc) 337 338 if self.variant == "stable": 339 desc += "This is a stable version, which means it is using stable and proven versions of gcc, gdb and binutils." 340 else: 341 desc += "This is a bleeding-edge version, which means it is using the latest versions of gcc, gdb and binutils." 342 343 f.write(textwrap.fill(desc, width=62, initial_indent="\t ", subsequent_indent="\t ") + "\n") 344 f.write("\n") 345 f.write("\t https://toolchains.bootlin.com/\n") 346 347 f.write("\n") 348 349 def gen_mk(self, f): 350 f.write("ifeq ($(%s),y)\n" % self.option_name) 351 f.write("TOOLCHAIN_EXTERNAL_BOOTLIN_VERSION = %s\n" % self.version) 352 f.write("TOOLCHAIN_EXTERNAL_BOOTLIN_SOURCE = %s--%s--%s-$(TOOLCHAIN_EXTERNAL_BOOTLIN_VERSION).tar.bz2\n" % 353 (self.arch, self.libc, self.variant)) 354 f.write("TOOLCHAIN_EXTERNAL_BOOTLIN_SITE = %s\n" % 355 os.path.join(BASE_URL, self.arch, "tarballs")) 356 f.write("endif\n\n") 357 pass 358 359 def gen_hash(self, f): 360 f.write("# From %s\n" % self.hash_url) 361 f.write("sha256 %s %s\n" % (self.sha256, os.path.basename(self.tarball_url))) 362 363 def gen_test(self, f): 364 if self.variant == "stable": 365 variant = "Stable" 366 else: 367 variant = "BleedingEdge" 368 testname = "TestExternalToolchainBootlin" + \ 369 self.arch.replace("-", "").capitalize() + \ 370 self.libc.capitalize() + variant 371 f.write("\n\n") 372 f.write("class %s(TestExternalToolchain):\n" % testname) 373 f.write(" config = \"\"\"\n") 374 if 'test_options' in arches[self.arch]: 375 test_options = arches[self.arch]['test_options'] 376 else: 377 test_options = arches[self.arch]['conditions'] 378 for opt in test_options: 379 if opt.startswith("!"): 380 f.write(" # %s is not set\n" % opt[1:]) 381 else: 382 f.write(" %s=y\n" % opt) 383 f.write(" BR2_TOOLCHAIN_EXTERNAL=y\n") 384 f.write(" BR2_TOOLCHAIN_EXTERNAL_BOOTLIN=y\n") 385 f.write(" %s=y\n" % self.option_name) 386 f.write(" # BR2_TARGET_ROOTFS_TAR is not set\n") 387 f.write(" \"\"\"\n") 388 f.write(" toolchain_prefix = \"%s-linux\"\n" % arches[self.arch]['prefix']) 389 f.write("\n") 390 f.write(" def test_run(self):\n") 391 f.write(" TestExternalToolchain.common_check(self)\n") 392 393 def __repr__(self): 394 return "Toolchain(arch=%s libc=%s variant=%s version=%s, option=%s)" % \ 395 (self.arch, self.libc, self.variant, self.version, self.option_name) 396 397 398def get_toolchains(): 399 toolchains = list() 400 for arch, details in arches.items(): 401 print(arch) 402 url = os.path.join(BASE_URL, arch, "available_toolchains") 403 page = requests.get(url).text 404 fnames = sorted(re.findall(r'<td><a href="(\w[^"]+)"', page)) 405 # This dict will allow us to keep only the latest version for 406 # each toolchain. 407 tmp = dict() 408 for fname in fnames: 409 parts = fname.split('--') 410 assert parts[0] == arch, "Arch does not match: %s vs. %s" % (parts[0], arch) 411 libc = parts[1] 412 if parts[2].startswith("stable-"): 413 variant = "stable" 414 version = parts[2][len("stable-"):] 415 elif parts[2].startswith("bleeding-edge-"): 416 variant = "bleeding-edge" 417 version = parts[2][len("bleeding-edge-"):] 418 tmp[(arch, libc, variant)] = version 419 420 toolchains += [Toolchain(k[0], k[1], k[2], v) for k, v in tmp.items()] 421 422 return toolchains 423 424 425def gen_config_in_options(toolchains, fpath): 426 with open(fpath, "w") as f: 427 f.write(AUTOGENERATED_COMMENT) 428 429 f.write("config BR2_TOOLCHAIN_EXTERNAL_BOOTLIN_ARCH_SUPPORTS\n") 430 f.write("\tbool\n") 431 for arch, details in arches.items(): 432 f.write("\tdefault y if %s\n" % " && ".join(details['conditions'])) 433 f.write("\n") 434 435 f.write("if BR2_TOOLCHAIN_EXTERNAL_BOOTLIN\n\n") 436 437 f.write("config BR2_TOOLCHAIN_EXTERNAL_PREFIX\n") 438 f.write("\tdefault \"$(ARCH)-linux\"\n") 439 440 f.write("\n") 441 442 f.write("config BR2_PACKAGE_PROVIDES_TOOLCHAIN_EXTERNAL\n") 443 f.write("\tdefault \"toolchain-external-bootlin\"\n") 444 445 f.write("\n") 446 447 f.write("choice\n") 448 f.write("\tprompt \"Bootlin toolchain variant\"\n") 449 450 for toolchain in toolchains: 451 toolchain.gen_config_in_options(f) 452 453 f.write("endchoice\n") 454 f.write("endif\n") 455 456 457def gen_mk(toolchains, fpath): 458 with open(fpath, "w") as f: 459 f.write("#" * 80 + "\n") 460 f.write("#\n") 461 f.write("# toolchain-external-bootlin\n") 462 f.write("#\n") 463 f.write("#" * 80 + "\n") 464 f.write("\n") 465 f.write(AUTOGENERATED_COMMENT) 466 for toolchain in toolchains: 467 toolchain.gen_mk(f) 468 f.write("$(eval $(toolchain-external-package))\n") 469 470 471def gen_hash(toolchains, fpath): 472 with open(fpath, "w") as f: 473 f.write(AUTOGENERATED_COMMENT) 474 for toolchain in toolchains: 475 toolchain.gen_hash(f) 476 477 478def gen_runtime_test(toolchains, fpath): 479 with open(fpath, "w") as f: 480 f.write(AUTOGENERATED_COMMENT) 481 f.write("from tests.toolchain.test_external import TestExternalToolchain\n") 482 for toolchain in toolchains: 483 toolchain.gen_test(f) 484 485 486def gen_toolchains(toolchains): 487 maindir = "toolchain/toolchain-external/toolchain-external-bootlin" 488 gen_config_in_options(toolchains, os.path.join(maindir, "Config.in.options")) 489 gen_mk(toolchains, os.path.join(maindir, "toolchain-external-bootlin.mk")) 490 gen_hash(toolchains, os.path.join(maindir, "toolchain-external-bootlin.hash")) 491 gen_runtime_test(toolchains, 492 os.path.join("support", "testing", "tests", "toolchain", "test_external_bootlin.py")) 493 494 495toolchains = get_toolchains() 496gen_toolchains(toolchains) 497