1# BB Class inspired by ebuild.sh 2# 3# This class will test files after installation for certain 4# security issues and other kind of issues. 5# 6# Checks we do: 7# -Check the ownership and permissions 8# -Check the RUNTIME path for the $TMPDIR 9# -Check if .la files wrongly point to workdir 10# -Check if .pc files wrongly point to workdir 11# -Check if packages contains .debug directories or .so files 12# where they should be in -dev or -dbg 13# -Check if config.log contains traces to broken autoconf tests 14# -Check invalid characters (non-utf8) on some package metadata 15# -Ensure that binaries in base_[bindir|sbindir|libdir] do not link 16# into exec_prefix 17# -Check that scripts in base_[bindir|sbindir|libdir] do not reference 18# files under exec_prefix 19# -Check if the package name is upper case 20 21# Elect whether a given type of error is a warning or error, they may 22# have been set by other files. 23WARN_QA ?= " libdir xorg-driver-abi \ 24 textrel incompatible-license files-invalid \ 25 infodir build-deps src-uri-bad symlink-to-sysroot multilib \ 26 invalid-packageconfig host-user-contaminated uppercase-pn patch-fuzz \ 27 mime mime-xdg unlisted-pkg-lics unhandled-features-check \ 28 missing-update-alternatives native-last missing-ptest \ 29 license-exists license-no-generic license-syntax license-format \ 30 license-incompatible license-file-missing obsolete-license \ 31 " 32ERROR_QA ?= "dev-so debug-deps dev-deps debug-files arch pkgconfig la \ 33 perms dep-cmp pkgvarcheck perm-config perm-line perm-link \ 34 split-strip packages-list pkgv-undefined var-undefined \ 35 version-going-backwards expanded-d invalid-chars \ 36 license-checksum dev-elf file-rdeps configure-unsafe \ 37 configure-gettext perllocalpod shebang-size \ 38 already-stripped installed-vs-shipped ldflags compile-host-path \ 39 install-host-path pn-overrides unknown-configure-option \ 40 useless-rpaths rpaths staticdev empty-dirs \ 41 " 42# Add usrmerge QA check based on distro feature 43ERROR_QA:append = "${@bb.utils.contains('DISTRO_FEATURES', 'usrmerge', ' usrmerge', '', d)}" 44 45FAKEROOT_QA = "host-user-contaminated" 46FAKEROOT_QA[doc] = "QA tests which need to run under fakeroot. If any \ 47enabled tests are listed here, the do_package_qa task will run under fakeroot." 48 49ALL_QA = "${WARN_QA} ${ERROR_QA}" 50 51UNKNOWN_CONFIGURE_OPT_IGNORE ?= "--enable-nls --disable-nls --disable-silent-rules --disable-dependency-tracking --with-libtool-sysroot --disable-static" 52 53# This is a list of directories that are expected to be empty. 54QA_EMPTY_DIRS ?= " \ 55 /dev/pts \ 56 /media \ 57 /proc \ 58 /run \ 59 /tmp \ 60 ${localstatedir}/run \ 61 ${localstatedir}/volatile \ 62" 63# It is possible to specify why a directory is expected to be empty by defining 64# QA_EMPTY_DIRS_RECOMMENDATION:<path>, which will then be included in the error 65# message if the directory is not empty. If it is not specified for a directory, 66# then "but it is expected to be empty" will be used. 67 68def package_qa_clean_path(path, d, pkg=None): 69 """ 70 Remove redundant paths from the path for display. If pkg isn't set then 71 TMPDIR is stripped, otherwise PKGDEST/pkg is stripped. 72 """ 73 if pkg: 74 path = path.replace(os.path.join(d.getVar("PKGDEST"), pkg), "/") 75 return path.replace(d.getVar("TMPDIR"), "/").replace("//", "/") 76 77QAPATHTEST[shebang-size] = "package_qa_check_shebang_size" 78def package_qa_check_shebang_size(path, name, d, elf, messages): 79 import stat 80 if os.path.islink(path) or stat.S_ISFIFO(os.stat(path).st_mode) or elf: 81 return 82 83 try: 84 with open(path, 'rb') as f: 85 stanza = f.readline(130) 86 except IOError: 87 return 88 89 if stanza.startswith(b'#!'): 90 #Shebang not found 91 try: 92 stanza = stanza.decode("utf-8") 93 except UnicodeDecodeError: 94 #If it is not a text file, it is not a script 95 return 96 97 if len(stanza) > 129: 98 oe.qa.add_message(messages, "shebang-size", "%s: %s maximum shebang size exceeded, the maximum size is 128." % (name, package_qa_clean_path(path, d))) 99 return 100 101QAPATHTEST[libexec] = "package_qa_check_libexec" 102def package_qa_check_libexec(path,name, d, elf, messages): 103 104 # Skip the case where the default is explicitly /usr/libexec 105 libexec = d.getVar('libexecdir') 106 if libexec == "/usr/libexec": 107 return True 108 109 if 'libexec' in path.split(os.path.sep): 110 oe.qa.add_message(messages, "libexec", "%s: %s is using libexec please relocate to %s" % (name, package_qa_clean_path(path, d), libexec)) 111 return False 112 113 return True 114 115QAPATHTEST[rpaths] = "package_qa_check_rpath" 116def package_qa_check_rpath(file,name, d, elf, messages): 117 """ 118 Check for dangerous RPATHs 119 """ 120 if not elf: 121 return 122 123 if os.path.islink(file): 124 return 125 126 bad_dirs = [d.getVar('BASE_WORKDIR'), d.getVar('STAGING_DIR_TARGET')] 127 128 phdrs = elf.run_objdump("-p", d) 129 130 import re 131 rpath_re = re.compile(r"\s+RPATH\s+(.*)") 132 for line in phdrs.split("\n"): 133 m = rpath_re.match(line) 134 if m: 135 rpath = m.group(1) 136 for dir in bad_dirs: 137 if dir in rpath: 138 oe.qa.add_message(messages, "rpaths", "package %s contains bad RPATH %s in file %s" % (name, rpath, file)) 139 140QAPATHTEST[useless-rpaths] = "package_qa_check_useless_rpaths" 141def package_qa_check_useless_rpaths(file, name, d, elf, messages): 142 """ 143 Check for RPATHs that are useless but not dangerous 144 """ 145 def rpath_eq(a, b): 146 return os.path.normpath(a) == os.path.normpath(b) 147 148 if not elf: 149 return 150 151 if os.path.islink(file): 152 return 153 154 libdir = d.getVar("libdir") 155 base_libdir = d.getVar("base_libdir") 156 157 phdrs = elf.run_objdump("-p", d) 158 159 import re 160 rpath_re = re.compile(r"\s+RPATH\s+(.*)") 161 for line in phdrs.split("\n"): 162 m = rpath_re.match(line) 163 if m: 164 rpath = m.group(1) 165 if rpath_eq(rpath, libdir) or rpath_eq(rpath, base_libdir): 166 # The dynamic linker searches both these places anyway. There is no point in 167 # looking there again. 168 oe.qa.add_message(messages, "useless-rpaths", "%s: %s contains probably-redundant RPATH %s" % (name, package_qa_clean_path(file, d, name), rpath)) 169 170QAPATHTEST[dev-so] = "package_qa_check_dev" 171def package_qa_check_dev(path, name, d, elf, messages): 172 """ 173 Check for ".so" library symlinks in non-dev packages 174 """ 175 176 if not name.endswith("-dev") and not name.endswith("-dbg") and not name.endswith("-ptest") and not name.startswith("nativesdk-") and path.endswith(".so") and os.path.islink(path): 177 oe.qa.add_message(messages, "dev-so", "non -dev/-dbg/nativesdk- package %s contains symlink .so '%s'" % \ 178 (name, package_qa_clean_path(path, d, name))) 179 180QAPATHTEST[dev-elf] = "package_qa_check_dev_elf" 181def package_qa_check_dev_elf(path, name, d, elf, messages): 182 """ 183 Check that -dev doesn't contain real shared libraries. The test has to 184 check that the file is not a link and is an ELF object as some recipes 185 install link-time .so files that are linker scripts. 186 """ 187 if name.endswith("-dev") and path.endswith(".so") and not os.path.islink(path) and elf: 188 oe.qa.add_message(messages, "dev-elf", "-dev package %s contains non-symlink .so '%s'" % \ 189 (name, package_qa_clean_path(path, d, name))) 190 191QAPATHTEST[staticdev] = "package_qa_check_staticdev" 192def package_qa_check_staticdev(path, name, d, elf, messages): 193 """ 194 Check for ".a" library in non-staticdev packages 195 There are a number of exceptions to this rule, -pic packages can contain 196 static libraries, the _nonshared.a belong with their -dev packages and 197 libgcc.a, libgcov.a will be skipped in their packages 198 """ 199 200 if not name.endswith("-pic") and not name.endswith("-staticdev") and not name.endswith("-ptest") and path.endswith(".a") and not path.endswith("_nonshared.a") and not '/usr/lib/debug-static/' in path and not '/.debug-static/' in path: 201 oe.qa.add_message(messages, "staticdev", "non -staticdev package contains static .a library: %s path '%s'" % \ 202 (name, package_qa_clean_path(path,d, name))) 203 204QAPATHTEST[mime] = "package_qa_check_mime" 205def package_qa_check_mime(path, name, d, elf, messages): 206 """ 207 Check if package installs mime types to /usr/share/mime/packages 208 while no inheriting mime.bbclass 209 """ 210 211 if d.getVar("datadir") + "/mime/packages" in path and path.endswith('.xml') and not bb.data.inherits_class("mime", d): 212 oe.qa.add_message(messages, "mime", "package contains mime types but does not inherit mime: %s path '%s'" % \ 213 (name, package_qa_clean_path(path,d))) 214 215QAPATHTEST[mime-xdg] = "package_qa_check_mime_xdg" 216def package_qa_check_mime_xdg(path, name, d, elf, messages): 217 """ 218 Check if package installs desktop file containing MimeType and requires 219 mime-types.bbclass to create /usr/share/applications/mimeinfo.cache 220 """ 221 222 if d.getVar("datadir") + "/applications" in path and path.endswith('.desktop') and not bb.data.inherits_class("mime-xdg", d): 223 mime_type_found = False 224 try: 225 with open(path, 'r') as f: 226 for line in f.read().split('\n'): 227 if 'MimeType' in line: 228 mime_type_found = True 229 break; 230 except: 231 # At least libreoffice installs symlinks with absolute paths that are dangling here. 232 # We could implement some magic but for few (one) recipes it is not worth the effort so just warn: 233 wstr = "%s cannot open %s - is it a symlink with absolute path?\n" % (name, package_qa_clean_path(path,d)) 234 wstr += "Please check if (linked) file contains key 'MimeType'.\n" 235 pkgname = name 236 if name == d.getVar('PN'): 237 pkgname = '${PN}' 238 wstr += "If yes: add \'inhert mime-xdg\' and \'MIME_XDG_PACKAGES += \"%s\"\' / if no add \'INSANE_SKIP:%s += \"mime-xdg\"\' to recipe." % (pkgname, pkgname) 239 oe.qa.add_message(messages, "mime-xdg", wstr) 240 if mime_type_found: 241 oe.qa.add_message(messages, "mime-xdg", "package contains desktop file with key 'MimeType' but does not inhert mime-xdg: %s path '%s'" % \ 242 (name, package_qa_clean_path(path,d))) 243 244def package_qa_check_libdir(d): 245 """ 246 Check for wrong library installation paths. For instance, catch 247 recipes installing /lib/bar.so when ${base_libdir}="lib32" or 248 installing in /usr/lib64 when ${libdir}="/usr/lib" 249 """ 250 import re 251 252 pkgdest = d.getVar('PKGDEST') 253 base_libdir = d.getVar("base_libdir") + os.sep 254 libdir = d.getVar("libdir") + os.sep 255 libexecdir = d.getVar("libexecdir") + os.sep 256 exec_prefix = d.getVar("exec_prefix") + os.sep 257 258 messages = [] 259 260 # The re's are purposely fuzzy, as some there are some .so.x.y.z files 261 # that don't follow the standard naming convention. It checks later 262 # that they are actual ELF files 263 lib_re = re.compile(r"^/lib.+\.so(\..+)?$") 264 exec_re = re.compile(r"^%s.*/lib.+\.so(\..+)?$" % exec_prefix) 265 266 for root, dirs, files in os.walk(pkgdest): 267 if root == pkgdest: 268 # Skip subdirectories for any packages with libdir in INSANE_SKIP 269 skippackages = [] 270 for package in dirs: 271 if 'libdir' in (d.getVar('INSANE_SKIP:' + package) or "").split(): 272 bb.note("Package %s skipping libdir QA test" % (package)) 273 skippackages.append(package) 274 elif d.getVar('PACKAGE_DEBUG_SPLIT_STYLE') == 'debug-file-directory' and package.endswith("-dbg"): 275 bb.note("Package %s skipping libdir QA test for PACKAGE_DEBUG_SPLIT_STYLE equals debug-file-directory" % (package)) 276 skippackages.append(package) 277 for package in skippackages: 278 dirs.remove(package) 279 for file in files: 280 full_path = os.path.join(root, file) 281 rel_path = os.path.relpath(full_path, pkgdest) 282 if os.sep in rel_path: 283 package, rel_path = rel_path.split(os.sep, 1) 284 rel_path = os.sep + rel_path 285 if lib_re.match(rel_path): 286 if base_libdir not in rel_path: 287 # make sure it's an actual ELF file 288 elf = oe.qa.ELFFile(full_path) 289 try: 290 elf.open() 291 messages.append("%s: found library in wrong location: %s" % (package, rel_path)) 292 except (oe.qa.NotELFFileError): 293 pass 294 if exec_re.match(rel_path): 295 if libdir not in rel_path and libexecdir not in rel_path: 296 # make sure it's an actual ELF file 297 elf = oe.qa.ELFFile(full_path) 298 try: 299 elf.open() 300 messages.append("%s: found library in wrong location: %s" % (package, rel_path)) 301 except (oe.qa.NotELFFileError): 302 pass 303 304 if messages: 305 oe.qa.handle_error("libdir", "\n".join(messages), d) 306 307QAPATHTEST[debug-files] = "package_qa_check_dbg" 308def package_qa_check_dbg(path, name, d, elf, messages): 309 """ 310 Check for ".debug" files or directories outside of the dbg package 311 """ 312 313 if not "-dbg" in name and not "-ptest" in name: 314 if '.debug' in path.split(os.path.sep): 315 oe.qa.add_message(messages, "debug-files", "non debug package contains .debug directory: %s path %s" % \ 316 (name, package_qa_clean_path(path,d))) 317 318QAPATHTEST[arch] = "package_qa_check_arch" 319def package_qa_check_arch(path,name,d, elf, messages): 320 """ 321 Check if archs are compatible 322 """ 323 import re, oe.elf 324 325 if not elf: 326 return 327 328 target_os = d.getVar('HOST_OS') 329 target_arch = d.getVar('HOST_ARCH') 330 provides = d.getVar('PROVIDES') 331 bpn = d.getVar('BPN') 332 333 if target_arch == "allarch": 334 pn = d.getVar('PN') 335 oe.qa.add_message(messages, "arch", pn + ": Recipe inherits the allarch class, but has packaged architecture-specific binaries") 336 return 337 338 # FIXME: Cross package confuse this check, so just skip them 339 for s in ['cross', 'nativesdk', 'cross-canadian']: 340 if bb.data.inherits_class(s, d): 341 return 342 343 # avoid following links to /usr/bin (e.g. on udev builds) 344 # we will check the files pointed to anyway... 345 if os.path.islink(path): 346 return 347 348 #if this will throw an exception, then fix the dict above 349 (machine, osabi, abiversion, littleendian, bits) \ 350 = oe.elf.machine_dict(d)[target_os][target_arch] 351 352 # Check the architecture and endiannes of the binary 353 is_32 = (("virtual/kernel" in provides) or bb.data.inherits_class("module", d)) and \ 354 (target_os == "linux-gnux32" or target_os == "linux-muslx32" or \ 355 target_os == "linux-gnu_ilp32" or re.match(r'mips64.*32', d.getVar('DEFAULTTUNE'))) 356 is_bpf = (oe.qa.elf_machine_to_string(elf.machine()) == "BPF") 357 if not ((machine == elf.machine()) or is_32 or is_bpf): 358 oe.qa.add_message(messages, "arch", "Architecture did not match (%s, expected %s) in %s" % \ 359 (oe.qa.elf_machine_to_string(elf.machine()), oe.qa.elf_machine_to_string(machine), package_qa_clean_path(path, d, name))) 360 elif not ((bits == elf.abiSize()) or is_32 or is_bpf): 361 oe.qa.add_message(messages, "arch", "Bit size did not match (%d, expected %d) in %s" % \ 362 (elf.abiSize(), bits, package_qa_clean_path(path, d, name))) 363 elif not ((littleendian == elf.isLittleEndian()) or is_bpf): 364 oe.qa.add_message(messages, "arch", "Endiannes did not match (%d, expected %d) in %s" % \ 365 (elf.isLittleEndian(), littleendian, package_qa_clean_path(path,d, name))) 366 367QAPATHTEST[desktop] = "package_qa_check_desktop" 368def package_qa_check_desktop(path, name, d, elf, messages): 369 """ 370 Run all desktop files through desktop-file-validate. 371 """ 372 if path.endswith(".desktop"): 373 desktop_file_validate = os.path.join(d.getVar('STAGING_BINDIR_NATIVE'),'desktop-file-validate') 374 output = os.popen("%s %s" % (desktop_file_validate, path)) 375 # This only produces output on errors 376 for l in output: 377 oe.qa.add_message(messages, "desktop", "Desktop file issue: " + l.strip()) 378 379QAPATHTEST[textrel] = "package_qa_textrel" 380def package_qa_textrel(path, name, d, elf, messages): 381 """ 382 Check if the binary contains relocations in .text 383 """ 384 385 if not elf: 386 return 387 388 if os.path.islink(path): 389 return 390 391 phdrs = elf.run_objdump("-p", d) 392 sane = True 393 394 import re 395 textrel_re = re.compile(r"\s+TEXTREL\s+") 396 for line in phdrs.split("\n"): 397 if textrel_re.match(line): 398 sane = False 399 break 400 401 if not sane: 402 path = package_qa_clean_path(path, d, name) 403 oe.qa.add_message(messages, "textrel", "%s: ELF binary %s has relocations in .text" % (name, path)) 404 405QAPATHTEST[ldflags] = "package_qa_hash_style" 406def package_qa_hash_style(path, name, d, elf, messages): 407 """ 408 Check if the binary has the right hash style... 409 """ 410 411 if not elf: 412 return 413 414 if os.path.islink(path): 415 return 416 417 gnu_hash = "--hash-style=gnu" in d.getVar('LDFLAGS') 418 if not gnu_hash: 419 gnu_hash = "--hash-style=both" in d.getVar('LDFLAGS') 420 if not gnu_hash: 421 return 422 423 sane = False 424 has_syms = False 425 426 phdrs = elf.run_objdump("-p", d) 427 428 # If this binary has symbols, we expect it to have GNU_HASH too. 429 for line in phdrs.split("\n"): 430 if "SYMTAB" in line: 431 has_syms = True 432 if "GNU_HASH" in line or "MIPS_XHASH" in line: 433 sane = True 434 if ("[mips32]" in line or "[mips64]" in line) and d.getVar('TCLIBC') == "musl": 435 sane = True 436 if has_syms and not sane: 437 path = package_qa_clean_path(path, d, name) 438 oe.qa.add_message(messages, "ldflags", "File %s in package %s doesn't have GNU_HASH (didn't pass LDFLAGS?)" % (path, name)) 439 440 441QAPATHTEST[buildpaths] = "package_qa_check_buildpaths" 442def package_qa_check_buildpaths(path, name, d, elf, messages): 443 """ 444 Check for build paths inside target files and error if paths are not 445 explicitly ignored. 446 """ 447 import stat 448 # Ignore .debug files, not interesting 449 if path.find(".debug") != -1: 450 return 451 452 # Ignore symlinks/devs/fifos 453 mode = os.lstat(path).st_mode 454 if stat.S_ISLNK(mode) or stat.S_ISBLK(mode) or stat.S_ISFIFO(mode) or stat.S_ISCHR(mode) or stat.S_ISSOCK(mode): 455 return 456 457 tmpdir = bytes(d.getVar('TMPDIR'), encoding="utf-8") 458 with open(path, 'rb') as f: 459 file_content = f.read() 460 if tmpdir in file_content: 461 trimmed = path.replace(os.path.join (d.getVar("PKGDEST"), name), "") 462 oe.qa.add_message(messages, "buildpaths", "File %s in package %s contains reference to TMPDIR" % (trimmed, name)) 463 464 465QAPATHTEST[xorg-driver-abi] = "package_qa_check_xorg_driver_abi" 466def package_qa_check_xorg_driver_abi(path, name, d, elf, messages): 467 """ 468 Check that all packages containing Xorg drivers have ABI dependencies 469 """ 470 471 # Skip dev, dbg or nativesdk packages 472 if name.endswith("-dev") or name.endswith("-dbg") or name.startswith("nativesdk-"): 473 return 474 475 driverdir = d.expand("${libdir}/xorg/modules/drivers/") 476 if driverdir in path and path.endswith(".so"): 477 mlprefix = d.getVar('MLPREFIX') or '' 478 for rdep in bb.utils.explode_deps(d.getVar('RDEPENDS:' + name) or ""): 479 if rdep.startswith("%sxorg-abi-" % mlprefix): 480 return 481 oe.qa.add_message(messages, "xorg-driver-abi", "Package %s contains Xorg driver (%s) but no xorg-abi- dependencies" % (name, os.path.basename(path))) 482 483QAPATHTEST[infodir] = "package_qa_check_infodir" 484def package_qa_check_infodir(path, name, d, elf, messages): 485 """ 486 Check that /usr/share/info/dir isn't shipped in a particular package 487 """ 488 infodir = d.expand("${infodir}/dir") 489 490 if infodir in path: 491 oe.qa.add_message(messages, "infodir", "The /usr/share/info/dir file is not meant to be shipped in a particular package.") 492 493QAPATHTEST[symlink-to-sysroot] = "package_qa_check_symlink_to_sysroot" 494def package_qa_check_symlink_to_sysroot(path, name, d, elf, messages): 495 """ 496 Check that the package doesn't contain any absolute symlinks to the sysroot. 497 """ 498 if os.path.islink(path): 499 target = os.readlink(path) 500 if os.path.isabs(target): 501 tmpdir = d.getVar('TMPDIR') 502 if target.startswith(tmpdir): 503 trimmed = path.replace(os.path.join (d.getVar("PKGDEST"), name), "") 504 oe.qa.add_message(messages, "symlink-to-sysroot", "Symlink %s in %s points to TMPDIR" % (trimmed, name)) 505 506# Check license variables 507do_populate_lic[postfuncs] += "populate_lic_qa_checksum" 508python populate_lic_qa_checksum() { 509 """ 510 Check for changes in the license files. 511 """ 512 513 lic_files = d.getVar('LIC_FILES_CHKSUM') or '' 514 lic = d.getVar('LICENSE') 515 pn = d.getVar('PN') 516 517 if lic == "CLOSED": 518 return 519 520 if not lic_files and d.getVar('SRC_URI'): 521 oe.qa.handle_error("license-checksum", pn + ": Recipe file fetches files and does not have license file information (LIC_FILES_CHKSUM)", d) 522 523 srcdir = d.getVar('S') 524 corebase_licensefile = d.getVar('COREBASE') + "/LICENSE" 525 for url in lic_files.split(): 526 try: 527 (type, host, path, user, pswd, parm) = bb.fetch.decodeurl(url) 528 except bb.fetch.MalformedUrl: 529 oe.qa.handle_error("license-checksum", pn + ": LIC_FILES_CHKSUM contains an invalid URL: " + url, d) 530 continue 531 srclicfile = os.path.join(srcdir, path) 532 if not os.path.isfile(srclicfile): 533 oe.qa.handle_error("license-checksum", pn + ": LIC_FILES_CHKSUM points to an invalid file: " + srclicfile, d) 534 continue 535 536 if (srclicfile == corebase_licensefile): 537 bb.warn("${COREBASE}/LICENSE is not a valid license file, please use '${COMMON_LICENSE_DIR}/MIT' for a MIT License file in LIC_FILES_CHKSUM. This will become an error in the future") 538 539 recipemd5 = parm.get('md5', '') 540 beginline, endline = 0, 0 541 if 'beginline' in parm: 542 beginline = int(parm['beginline']) 543 if 'endline' in parm: 544 endline = int(parm['endline']) 545 546 if (not beginline) and (not endline): 547 md5chksum = bb.utils.md5_file(srclicfile) 548 with open(srclicfile, 'r', errors='replace') as f: 549 license = f.read().splitlines() 550 else: 551 with open(srclicfile, 'rb') as f: 552 import hashlib 553 lineno = 0 554 license = [] 555 try: 556 m = hashlib.new('MD5', usedforsecurity=False) 557 except TypeError: 558 m = hashlib.new('MD5') 559 for line in f: 560 lineno += 1 561 if (lineno >= beginline): 562 if ((lineno <= endline) or not endline): 563 m.update(line) 564 license.append(line.decode('utf-8', errors='replace').rstrip()) 565 else: 566 break 567 md5chksum = m.hexdigest() 568 if recipemd5 == md5chksum: 569 bb.note (pn + ": md5 checksum matched for ", url) 570 else: 571 if recipemd5: 572 msg = pn + ": The LIC_FILES_CHKSUM does not match for " + url 573 msg = msg + "\n" + pn + ": The new md5 checksum is " + md5chksum 574 max_lines = int(d.getVar('QA_MAX_LICENSE_LINES') or 20) 575 if not license or license[-1] != '': 576 # Ensure that our license text ends with a line break 577 # (will be added with join() below). 578 license.append('') 579 remove = len(license) - max_lines 580 if remove > 0: 581 start = max_lines // 2 582 end = start + remove - 1 583 del license[start:end] 584 license.insert(start, '...') 585 msg = msg + "\n" + pn + ": Here is the selected license text:" + \ 586 "\n" + \ 587 "{:v^70}".format(" beginline=%d " % beginline if beginline else "") + \ 588 "\n" + "\n".join(license) + \ 589 "{:^^70}".format(" endline=%d " % endline if endline else "") 590 if beginline: 591 if endline: 592 srcfiledesc = "%s (lines %d through to %d)" % (srclicfile, beginline, endline) 593 else: 594 srcfiledesc = "%s (beginning on line %d)" % (srclicfile, beginline) 595 elif endline: 596 srcfiledesc = "%s (ending on line %d)" % (srclicfile, endline) 597 else: 598 srcfiledesc = srclicfile 599 msg = msg + "\n" + pn + ": Check if the license information has changed in %s to verify that the LICENSE value \"%s\" remains valid" % (srcfiledesc, lic) 600 601 else: 602 msg = pn + ": LIC_FILES_CHKSUM is not specified for " + url 603 msg = msg + "\n" + pn + ": The md5 checksum is " + md5chksum 604 oe.qa.handle_error("license-checksum", msg, d) 605 606 oe.qa.exit_if_errors(d) 607} 608 609def qa_check_staged(path,d): 610 """ 611 Check staged la and pc files for common problems like references to the work 612 directory. 613 614 As this is run after every stage we should be able to find the one 615 responsible for the errors easily even if we look at every .pc and .la file. 616 """ 617 618 tmpdir = d.getVar('TMPDIR') 619 workdir = os.path.join(tmpdir, "work") 620 recipesysroot = d.getVar("RECIPE_SYSROOT") 621 622 if bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d): 623 pkgconfigcheck = workdir 624 else: 625 pkgconfigcheck = tmpdir 626 627 skip = (d.getVar('INSANE_SKIP') or "").split() 628 skip_la = False 629 if 'la' in skip: 630 bb.note("Recipe %s skipping qa checking: la" % d.getVar('PN')) 631 skip_la = True 632 633 skip_pkgconfig = False 634 if 'pkgconfig' in skip: 635 bb.note("Recipe %s skipping qa checking: pkgconfig" % d.getVar('PN')) 636 skip_pkgconfig = True 637 638 # find all .la and .pc files 639 # read the content 640 # and check for stuff that looks wrong 641 for root, dirs, files in os.walk(path): 642 for file in files: 643 path = os.path.join(root,file) 644 if file.endswith(".la") and not skip_la: 645 with open(path) as f: 646 file_content = f.read() 647 file_content = file_content.replace(recipesysroot, "") 648 if workdir in file_content: 649 error_msg = "%s failed sanity test (workdir) in path %s" % (file,root) 650 oe.qa.handle_error("la", error_msg, d) 651 elif file.endswith(".pc") and not skip_pkgconfig: 652 with open(path) as f: 653 file_content = f.read() 654 file_content = file_content.replace(recipesysroot, "") 655 if pkgconfigcheck in file_content: 656 error_msg = "%s failed sanity test (tmpdir) in path %s" % (file,root) 657 oe.qa.handle_error("pkgconfig", error_msg, d) 658 659# Run all package-wide warnfuncs and errorfuncs 660def package_qa_package(warnfuncs, errorfuncs, package, d): 661 warnings = {} 662 errors = {} 663 664 for func in warnfuncs: 665 func(package, d, warnings) 666 for func in errorfuncs: 667 func(package, d, errors) 668 669 for w in warnings: 670 oe.qa.handle_error(w, warnings[w], d) 671 for e in errors: 672 oe.qa.handle_error(e, errors[e], d) 673 674 return len(errors) == 0 675 676# Run all recipe-wide warnfuncs and errorfuncs 677def package_qa_recipe(warnfuncs, errorfuncs, pn, d): 678 warnings = {} 679 errors = {} 680 681 for func in warnfuncs: 682 func(pn, d, warnings) 683 for func in errorfuncs: 684 func(pn, d, errors) 685 686 for w in warnings: 687 oe.qa.handle_error(w, warnings[w], d) 688 for e in errors: 689 oe.qa.handle_error(e, errors[e], d) 690 691 return len(errors) == 0 692 693def prepopulate_objdump_p(elf, d): 694 output = elf.run_objdump("-p", d) 695 return (elf.name, output) 696 697# Walk over all files in a directory and call func 698def package_qa_walk(warnfuncs, errorfuncs, package, d): 699 #if this will throw an exception, then fix the dict above 700 target_os = d.getVar('HOST_OS') 701 target_arch = d.getVar('HOST_ARCH') 702 703 warnings = {} 704 errors = {} 705 elves = {} 706 for path in pkgfiles[package]: 707 elf = None 708 if os.path.isfile(path): 709 elf = oe.qa.ELFFile(path) 710 try: 711 elf.open() 712 elf.close() 713 except oe.qa.NotELFFileError: 714 elf = None 715 if elf: 716 elves[path] = elf 717 718 results = oe.utils.multiprocess_launch(prepopulate_objdump_p, elves.values(), d, extraargs=(d,)) 719 for item in results: 720 elves[item[0]].set_objdump("-p", item[1]) 721 722 for path in pkgfiles[package]: 723 if path in elves: 724 elves[path].open() 725 for func in warnfuncs: 726 func(path, package, d, elves.get(path), warnings) 727 for func in errorfuncs: 728 func(path, package, d, elves.get(path), errors) 729 if path in elves: 730 elves[path].close() 731 732 for w in warnings: 733 oe.qa.handle_error(w, warnings[w], d) 734 for e in errors: 735 oe.qa.handle_error(e, errors[e], d) 736 737def package_qa_check_rdepends(pkg, pkgdest, skip, taskdeps, packages, d): 738 # Don't do this check for kernel/module recipes, there aren't too many debug/development 739 # packages and you can get false positives e.g. on kernel-module-lirc-dev 740 if bb.data.inherits_class("kernel", d) or bb.data.inherits_class("module-base", d): 741 return 742 743 if not "-dbg" in pkg and not "packagegroup-" in pkg and not "-image" in pkg: 744 localdata = bb.data.createCopy(d) 745 localdata.setVar('OVERRIDES', localdata.getVar('OVERRIDES') + ':' + pkg) 746 747 # Now check the RDEPENDS 748 rdepends = bb.utils.explode_deps(localdata.getVar('RDEPENDS') or "") 749 750 # Now do the sanity check!!! 751 if "build-deps" not in skip: 752 for rdepend in rdepends: 753 if "-dbg" in rdepend and "debug-deps" not in skip: 754 error_msg = "%s rdepends on %s" % (pkg,rdepend) 755 oe.qa.handle_error("debug-deps", error_msg, d) 756 if (not "-dev" in pkg and not "-staticdev" in pkg) and rdepend.endswith("-dev") and "dev-deps" not in skip: 757 error_msg = "%s rdepends on %s" % (pkg, rdepend) 758 oe.qa.handle_error("dev-deps", error_msg, d) 759 if rdepend not in packages: 760 rdep_data = oe.packagedata.read_subpkgdata(rdepend, d) 761 if rdep_data and 'PN' in rdep_data and rdep_data['PN'] in taskdeps: 762 continue 763 if not rdep_data or not 'PN' in rdep_data: 764 pkgdata_dir = d.getVar("PKGDATA_DIR") 765 try: 766 possibles = os.listdir("%s/runtime-rprovides/%s/" % (pkgdata_dir, rdepend)) 767 except OSError: 768 possibles = [] 769 for p in possibles: 770 rdep_data = oe.packagedata.read_subpkgdata(p, d) 771 if rdep_data and 'PN' in rdep_data and rdep_data['PN'] in taskdeps: 772 break 773 if rdep_data and 'PN' in rdep_data and rdep_data['PN'] in taskdeps: 774 continue 775 if rdep_data and 'PN' in rdep_data: 776 error_msg = "%s rdepends on %s, but it isn't a build dependency, missing %s in DEPENDS or PACKAGECONFIG?" % (pkg, rdepend, rdep_data['PN']) 777 else: 778 error_msg = "%s rdepends on %s, but it isn't a build dependency?" % (pkg, rdepend) 779 oe.qa.handle_error("build-deps", error_msg, d) 780 781 if "file-rdeps" not in skip: 782 ignored_file_rdeps = set(['/bin/sh', '/usr/bin/env', 'rtld(GNU_HASH)']) 783 if bb.data.inherits_class('nativesdk', d): 784 ignored_file_rdeps |= set(['/bin/bash', '/usr/bin/perl', 'perl']) 785 # For Saving the FILERDEPENDS 786 filerdepends = {} 787 rdep_data = oe.packagedata.read_subpkgdata(pkg, d) 788 for key in rdep_data: 789 if key.startswith("FILERDEPENDS:"): 790 for subkey in bb.utils.explode_deps(rdep_data[key]): 791 if subkey not in ignored_file_rdeps and \ 792 not subkey.startswith('perl('): 793 # We already know it starts with FILERDEPENDS_ 794 filerdepends[subkey] = key[13:] 795 796 if filerdepends: 797 done = rdepends[:] 798 # Add the rprovides of itself 799 if pkg not in done: 800 done.insert(0, pkg) 801 802 # The python is not a package, but python-core provides it, so 803 # skip checking /usr/bin/python if python is in the rdeps, in 804 # case there is a RDEPENDS:pkg = "python" in the recipe. 805 for py in [ d.getVar('MLPREFIX') + "python", "python" ]: 806 if py in done: 807 filerdepends.pop("/usr/bin/python",None) 808 done.remove(py) 809 for rdep in done: 810 # The file dependencies may contain package names, e.g., 811 # perl 812 filerdepends.pop(rdep,None) 813 814 # For Saving the FILERPROVIDES, RPROVIDES and FILES_INFO 815 rdep_data = oe.packagedata.read_subpkgdata(rdep, d) 816 for key in rdep_data: 817 if key.startswith("FILERPROVIDES:") or key.startswith("RPROVIDES:"): 818 for subkey in bb.utils.explode_deps(rdep_data[key]): 819 filerdepends.pop(subkey,None) 820 # Add the files list to the rprovides 821 if key.startswith("FILES_INFO:"): 822 # Use eval() to make it as a dict 823 for subkey in eval(rdep_data[key]): 824 filerdepends.pop(subkey,None) 825 if not filerdepends: 826 # Break if all the file rdepends are met 827 break 828 if filerdepends: 829 for key in filerdepends: 830 error_msg = "%s contained in package %s requires %s, but no providers found in RDEPENDS:%s?" % \ 831 (filerdepends[key].replace(":%s" % pkg, "").replace("@underscore@", "_"), pkg, key, pkg) 832 oe.qa.handle_error("file-rdeps", error_msg, d) 833package_qa_check_rdepends[vardepsexclude] = "OVERRIDES" 834 835def package_qa_check_deps(pkg, pkgdest, d): 836 837 localdata = bb.data.createCopy(d) 838 localdata.setVar('OVERRIDES', pkg) 839 840 def check_valid_deps(var): 841 try: 842 rvar = bb.utils.explode_dep_versions2(localdata.getVar(var) or "") 843 except ValueError as e: 844 bb.fatal("%s:%s: %s" % (var, pkg, e)) 845 for dep in rvar: 846 for v in rvar[dep]: 847 if v and not v.startswith(('< ', '= ', '> ', '<= ', '>=')): 848 error_msg = "%s:%s is invalid: %s (%s) only comparisons <, =, >, <=, and >= are allowed" % (var, pkg, dep, v) 849 oe.qa.handle_error("dep-cmp", error_msg, d) 850 851 check_valid_deps('RDEPENDS') 852 check_valid_deps('RRECOMMENDS') 853 check_valid_deps('RSUGGESTS') 854 check_valid_deps('RPROVIDES') 855 check_valid_deps('RREPLACES') 856 check_valid_deps('RCONFLICTS') 857 858QAPKGTEST[usrmerge] = "package_qa_check_usrmerge" 859def package_qa_check_usrmerge(pkg, d, messages): 860 861 pkgdest = d.getVar('PKGDEST') 862 pkg_dir = pkgdest + os.sep + pkg + os.sep 863 merged_dirs = ['bin', 'sbin', 'lib'] + d.getVar('MULTILIB_VARIANTS').split() 864 for f in merged_dirs: 865 if os.path.exists(pkg_dir + f) and not os.path.islink(pkg_dir + f): 866 msg = "%s package is not obeying usrmerge distro feature. /%s should be relocated to /usr." % (pkg, f) 867 oe.qa.add_message(messages, "usrmerge", msg) 868 return False 869 return True 870 871QAPKGTEST[perllocalpod] = "package_qa_check_perllocalpod" 872def package_qa_check_perllocalpod(pkg, d, messages): 873 """ 874 Check that the recipe didn't ship a perlocal.pod file, which shouldn't be 875 installed in a distribution package. cpan.bbclass sets NO_PERLLOCAL=1 to 876 handle this for most recipes. 877 """ 878 import glob 879 pkgd = oe.path.join(d.getVar('PKGDEST'), pkg) 880 podpath = oe.path.join(pkgd, d.getVar("libdir"), "perl*", "*", "*", "perllocal.pod") 881 882 matches = glob.glob(podpath) 883 if matches: 884 matches = [package_qa_clean_path(path, d, pkg) for path in matches] 885 msg = "%s contains perllocal.pod (%s), should not be installed" % (pkg, " ".join(matches)) 886 oe.qa.add_message(messages, "perllocalpod", msg) 887 888QAPKGTEST[expanded-d] = "package_qa_check_expanded_d" 889def package_qa_check_expanded_d(package, d, messages): 890 """ 891 Check for the expanded D (${D}) value in pkg_* and FILES 892 variables, warn the user to use it correctly. 893 """ 894 sane = True 895 expanded_d = d.getVar('D') 896 897 for var in 'FILES','pkg_preinst', 'pkg_postinst', 'pkg_prerm', 'pkg_postrm': 898 bbvar = d.getVar(var + ":" + package) or "" 899 if expanded_d in bbvar: 900 if var == 'FILES': 901 oe.qa.add_message(messages, "expanded-d", "FILES in %s recipe should not contain the ${D} variable as it references the local build directory not the target filesystem, best solution is to remove the ${D} reference" % package) 902 sane = False 903 else: 904 oe.qa.add_message(messages, "expanded-d", "%s in %s recipe contains ${D}, it should be replaced by $D instead" % (var, package)) 905 sane = False 906 return sane 907 908QAPKGTEST[unlisted-pkg-lics] = "package_qa_check_unlisted_pkg_lics" 909def package_qa_check_unlisted_pkg_lics(package, d, messages): 910 """ 911 Check that all licenses for a package are among the licenses for the recipe. 912 """ 913 pkg_lics = d.getVar('LICENSE:' + package) 914 if not pkg_lics: 915 return True 916 917 recipe_lics_set = oe.license.list_licenses(d.getVar('LICENSE')) 918 package_lics = oe.license.list_licenses(pkg_lics) 919 unlisted = package_lics - recipe_lics_set 920 if unlisted: 921 oe.qa.add_message(messages, "unlisted-pkg-lics", 922 "LICENSE:%s includes licenses (%s) that are not " 923 "listed in LICENSE" % (package, ' '.join(unlisted))) 924 return False 925 obsolete = set(oe.license.obsolete_license_list()) & package_lics - recipe_lics_set 926 if obsolete: 927 oe.qa.add_message(messages, "obsolete-license", 928 "LICENSE:%s includes obsolete licenses %s" % (package, ' '.join(obsolete))) 929 return False 930 return True 931 932QAPKGTEST[empty-dirs] = "package_qa_check_empty_dirs" 933def package_qa_check_empty_dirs(pkg, d, messages): 934 """ 935 Check for the existence of files in directories that are expected to be 936 empty. 937 """ 938 939 pkgd = oe.path.join(d.getVar('PKGDEST'), pkg) 940 for dir in (d.getVar('QA_EMPTY_DIRS') or "").split(): 941 empty_dir = oe.path.join(pkgd, dir) 942 if os.path.exists(empty_dir) and os.listdir(empty_dir): 943 recommendation = (d.getVar('QA_EMPTY_DIRS_RECOMMENDATION:' + dir) or 944 "but it is expected to be empty") 945 msg = "%s installs files in %s, %s" % (pkg, dir, recommendation) 946 oe.qa.add_message(messages, "empty-dirs", msg) 947 948def package_qa_check_encoding(keys, encode, d): 949 def check_encoding(key, enc): 950 sane = True 951 value = d.getVar(key) 952 if value: 953 try: 954 s = value.encode(enc) 955 except UnicodeDecodeError as e: 956 error_msg = "%s has non %s characters" % (key,enc) 957 sane = False 958 oe.qa.handle_error("invalid-chars", error_msg, d) 959 return sane 960 961 for key in keys: 962 sane = check_encoding(key, encode) 963 if not sane: 964 break 965 966HOST_USER_UID := "${@os.getuid()}" 967HOST_USER_GID := "${@os.getgid()}" 968 969QAPATHTEST[host-user-contaminated] = "package_qa_check_host_user" 970def package_qa_check_host_user(path, name, d, elf, messages): 971 """Check for paths outside of /home which are owned by the user running bitbake.""" 972 973 if not os.path.lexists(path): 974 return 975 976 dest = d.getVar('PKGDEST') 977 pn = d.getVar('PN') 978 home = os.path.join(dest, name, 'home') 979 if path == home or path.startswith(home + os.sep): 980 return 981 982 try: 983 stat = os.lstat(path) 984 except OSError as exc: 985 import errno 986 if exc.errno != errno.ENOENT: 987 raise 988 else: 989 check_uid = int(d.getVar('HOST_USER_UID')) 990 if stat.st_uid == check_uid: 991 oe.qa.add_message(messages, "host-user-contaminated", "%s: %s is owned by uid %d, which is the same as the user running bitbake. This may be due to host contamination" % (pn, package_qa_clean_path(path, d, name), check_uid)) 992 return False 993 994 check_gid = int(d.getVar('HOST_USER_GID')) 995 if stat.st_gid == check_gid: 996 oe.qa.add_message(messages, "host-user-contaminated", "%s: %s is owned by gid %d, which is the same as the user running bitbake. This may be due to host contamination" % (pn, package_qa_clean_path(path, d, name), check_gid)) 997 return False 998 return True 999 1000QARECIPETEST[unhandled-features-check] = "package_qa_check_unhandled_features_check" 1001def package_qa_check_unhandled_features_check(pn, d, messages): 1002 if not bb.data.inherits_class('features_check', d): 1003 var_set = False 1004 for kind in ['DISTRO', 'MACHINE', 'COMBINED']: 1005 for var in ['ANY_OF_' + kind + '_FEATURES', 'REQUIRED_' + kind + '_FEATURES', 'CONFLICT_' + kind + '_FEATURES']: 1006 if d.getVar(var) is not None or d.hasOverrides(var): 1007 var_set = True 1008 if var_set: 1009 oe.qa.handle_error("unhandled-features-check", "%s: recipe doesn't inherit features_check" % pn, d) 1010 1011QARECIPETEST[missing-update-alternatives] = "package_qa_check_missing_update_alternatives" 1012def package_qa_check_missing_update_alternatives(pn, d, messages): 1013 # Look at all packages and find out if any of those sets ALTERNATIVE variable 1014 # without inheriting update-alternatives class 1015 for pkg in (d.getVar('PACKAGES') or '').split(): 1016 if d.getVar('ALTERNATIVE:%s' % pkg) and not bb.data.inherits_class('update-alternatives', d): 1017 oe.qa.handle_error("missing-update-alternatives", "%s: recipe defines ALTERNATIVE:%s but doesn't inherit update-alternatives. This might fail during do_rootfs later!" % (pn, pkg), d) 1018 1019# The PACKAGE FUNC to scan each package 1020python do_package_qa () { 1021 import subprocess 1022 import oe.packagedata 1023 1024 bb.note("DO PACKAGE QA") 1025 1026 main_lic = d.getVar('LICENSE') 1027 1028 # Check for obsolete license references in main LICENSE (packages are checked below for any changes) 1029 main_licenses = oe.license.list_licenses(d.getVar('LICENSE')) 1030 obsolete = set(oe.license.obsolete_license_list()) & main_licenses 1031 if obsolete: 1032 oe.qa.handle_error("obsolete-license", "Recipe LICENSE includes obsolete licenses %s" % ' '.join(obsolete), d) 1033 1034 bb.build.exec_func("read_subpackage_metadata", d) 1035 1036 # Check non UTF-8 characters on recipe's metadata 1037 package_qa_check_encoding(['DESCRIPTION', 'SUMMARY', 'LICENSE', 'SECTION'], 'utf-8', d) 1038 1039 logdir = d.getVar('T') 1040 pn = d.getVar('PN') 1041 1042 # Scan the packages... 1043 pkgdest = d.getVar('PKGDEST') 1044 packages = set((d.getVar('PACKAGES') or '').split()) 1045 1046 global pkgfiles 1047 pkgfiles = {} 1048 for pkg in packages: 1049 pkgfiles[pkg] = [] 1050 pkgdir = os.path.join(pkgdest, pkg) 1051 for walkroot, dirs, files in os.walk(pkgdir): 1052 # Don't walk into top-level CONTROL or DEBIAN directories as these 1053 # are temporary directories created by do_package. 1054 if walkroot == pkgdir: 1055 for control in ("CONTROL", "DEBIAN"): 1056 if control in dirs: 1057 dirs.remove(control) 1058 for file in files: 1059 pkgfiles[pkg].append(os.path.join(walkroot, file)) 1060 1061 # no packages should be scanned 1062 if not packages: 1063 return 1064 1065 import re 1066 # The package name matches the [a-z0-9.+-]+ regular expression 1067 pkgname_pattern = re.compile(r"^[a-z0-9.+-]+$") 1068 1069 taskdepdata = d.getVar("BB_TASKDEPDATA", False) 1070 taskdeps = set() 1071 for dep in taskdepdata: 1072 taskdeps.add(taskdepdata[dep][0]) 1073 1074 def parse_test_matrix(matrix_name): 1075 testmatrix = d.getVarFlags(matrix_name) or {} 1076 g = globals() 1077 warnchecks = [] 1078 for w in (d.getVar("WARN_QA") or "").split(): 1079 if w in skip: 1080 continue 1081 if w in testmatrix and testmatrix[w] in g: 1082 warnchecks.append(g[testmatrix[w]]) 1083 1084 errorchecks = [] 1085 for e in (d.getVar("ERROR_QA") or "").split(): 1086 if e in skip: 1087 continue 1088 if e in testmatrix and testmatrix[e] in g: 1089 errorchecks.append(g[testmatrix[e]]) 1090 return warnchecks, errorchecks 1091 1092 for package in packages: 1093 skip = set((d.getVar('INSANE_SKIP') or "").split() + 1094 (d.getVar('INSANE_SKIP:' + package) or "").split()) 1095 if skip: 1096 bb.note("Package %s skipping QA tests: %s" % (package, str(skip))) 1097 1098 bb.note("Checking Package: %s" % package) 1099 # Check package name 1100 if not pkgname_pattern.match(package): 1101 oe.qa.handle_error("pkgname", 1102 "%s doesn't match the [a-z0-9.+-]+ regex" % package, d) 1103 1104 warn_checks, error_checks = parse_test_matrix("QAPATHTEST") 1105 package_qa_walk(warn_checks, error_checks, package, d) 1106 1107 warn_checks, error_checks = parse_test_matrix("QAPKGTEST") 1108 package_qa_package(warn_checks, error_checks, package, d) 1109 1110 package_qa_check_rdepends(package, pkgdest, skip, taskdeps, packages, d) 1111 package_qa_check_deps(package, pkgdest, d) 1112 1113 warn_checks, error_checks = parse_test_matrix("QARECIPETEST") 1114 package_qa_recipe(warn_checks, error_checks, pn, d) 1115 1116 if 'libdir' in d.getVar("ALL_QA").split(): 1117 package_qa_check_libdir(d) 1118 1119 oe.qa.exit_if_errors(d) 1120} 1121 1122# binutils is used for most checks, so need to set as dependency 1123# POPULATESYSROOTDEPS is defined in staging class. 1124do_package_qa[depends] += "${POPULATESYSROOTDEPS}" 1125do_package_qa[vardeps] = "${@bb.utils.contains('ERROR_QA', 'empty-dirs', 'QA_EMPTY_DIRS', '', d)}" 1126do_package_qa[vardepsexclude] = "BB_TASKDEPDATA" 1127do_package_qa[rdeptask] = "do_packagedata" 1128addtask do_package_qa after do_packagedata do_package before do_build 1129 1130# Add the package specific INSANE_SKIPs to the sstate dependencies 1131python() { 1132 pkgs = (d.getVar('PACKAGES') or '').split() 1133 for pkg in pkgs: 1134 d.appendVarFlag("do_package_qa", "vardeps", " INSANE_SKIP:{}".format(pkg)) 1135} 1136 1137SSTATETASKS += "do_package_qa" 1138do_package_qa[sstate-inputdirs] = "" 1139do_package_qa[sstate-outputdirs] = "" 1140python do_package_qa_setscene () { 1141 sstate_setscene(d) 1142} 1143addtask do_package_qa_setscene 1144 1145python do_qa_staging() { 1146 bb.note("QA checking staging") 1147 qa_check_staged(d.expand('${SYSROOT_DESTDIR}${libdir}'), d) 1148 oe.qa.exit_with_message_if_errors("QA staging was broken by the package built above", d) 1149} 1150 1151python do_qa_patch() { 1152 import subprocess 1153 1154 ########################################################################### 1155 # Check patch.log for fuzz warnings 1156 # 1157 # Further information on why we check for patch fuzz warnings: 1158 # http://lists.openembedded.org/pipermail/openembedded-core/2018-March/148675.html 1159 # https://bugzilla.yoctoproject.org/show_bug.cgi?id=10450 1160 ########################################################################### 1161 1162 logdir = d.getVar('T') 1163 patchlog = os.path.join(logdir,"log.do_patch") 1164 1165 if os.path.exists(patchlog): 1166 fuzzheader = '--- Patch fuzz start ---' 1167 fuzzfooter = '--- Patch fuzz end ---' 1168 statement = "grep -e '%s' %s > /dev/null" % (fuzzheader, patchlog) 1169 if subprocess.call(statement, shell=True) == 0: 1170 msg = "Fuzz detected:\n\n" 1171 fuzzmsg = "" 1172 inFuzzInfo = False 1173 f = open(patchlog, "r") 1174 for line in f: 1175 if fuzzheader in line: 1176 inFuzzInfo = True 1177 fuzzmsg = "" 1178 elif fuzzfooter in line: 1179 fuzzmsg = fuzzmsg.replace('\n\n', '\n') 1180 msg += fuzzmsg 1181 msg += "\n" 1182 inFuzzInfo = False 1183 elif inFuzzInfo and not 'Now at patch' in line: 1184 fuzzmsg += line 1185 f.close() 1186 msg += "The context lines in the patches can be updated with devtool:\n" 1187 msg += "\n" 1188 msg += " devtool modify %s\n" % d.getVar('PN') 1189 msg += " devtool finish --force-patch-refresh %s <layer_path>\n\n" % d.getVar('PN') 1190 msg += "Don't forget to review changes done by devtool!\n" 1191 if bb.utils.filter('ERROR_QA', 'patch-fuzz', d): 1192 bb.error(msg) 1193 elif bb.utils.filter('WARN_QA', 'patch-fuzz', d): 1194 bb.warn(msg) 1195 msg = "Patch log indicates that patches do not apply cleanly." 1196 oe.qa.handle_error("patch-fuzz", msg, d) 1197 1198 # Check if the patch contains a correctly formatted and spelled Upstream-Status 1199 import re 1200 from oe import patch 1201 1202 coremeta_path = os.path.join(d.getVar('COREBASE'), 'meta', '') 1203 for url in patch.src_patches(d): 1204 (_, _, fullpath, _, _, _) = bb.fetch.decodeurl(url) 1205 1206 # skip patches not in oe-core 1207 if not os.path.abspath(fullpath).startswith(coremeta_path): 1208 continue 1209 1210 kinda_status_re = re.compile(r"^.*upstream.*status.*$", re.IGNORECASE | re.MULTILINE) 1211 strict_status_re = re.compile(r"^Upstream-Status: (Pending|Submitted|Denied|Accepted|Inappropriate|Backport|Inactive-Upstream)( .+)?$", re.MULTILINE) 1212 guidelines = "https://www.openembedded.org/wiki/Commit_Patch_Message_Guidelines#Patch_Header_Recommendations:_Upstream-Status" 1213 1214 with open(fullpath, encoding='utf-8', errors='ignore') as f: 1215 file_content = f.read() 1216 match_kinda = kinda_status_re.search(file_content) 1217 match_strict = strict_status_re.search(file_content) 1218 1219 if not match_strict: 1220 if match_kinda: 1221 bb.error("Malformed Upstream-Status in patch\n%s\nPlease correct according to %s :\n%s" % (fullpath, guidelines, match_kinda.group(0))) 1222 else: 1223 bb.error("Missing Upstream-Status in patch\n%s\nPlease add according to %s ." % (fullpath, guidelines)) 1224} 1225 1226python do_qa_configure() { 1227 import subprocess 1228 1229 ########################################################################### 1230 # Check config.log for cross compile issues 1231 ########################################################################### 1232 1233 configs = [] 1234 workdir = d.getVar('WORKDIR') 1235 1236 skip = (d.getVar('INSANE_SKIP') or "").split() 1237 skip_configure_unsafe = False 1238 if 'configure-unsafe' in skip: 1239 bb.note("Recipe %s skipping qa checking: configure-unsafe" % d.getVar('PN')) 1240 skip_configure_unsafe = True 1241 1242 if bb.data.inherits_class('autotools', d) and not skip_configure_unsafe: 1243 bb.note("Checking autotools environment for common misconfiguration") 1244 for root, dirs, files in os.walk(workdir): 1245 statement = "grep -q -F -e 'is unsafe for cross-compilation' %s" % \ 1246 os.path.join(root,"config.log") 1247 if "config.log" in files: 1248 if subprocess.call(statement, shell=True) == 0: 1249 error_msg = """This autoconf log indicates errors, it looked at host include and/or library paths while determining system capabilities. 1250Rerun configure task after fixing this.""" 1251 oe.qa.handle_error("configure-unsafe", error_msg, d) 1252 1253 if "configure.ac" in files: 1254 configs.append(os.path.join(root,"configure.ac")) 1255 if "configure.in" in files: 1256 configs.append(os.path.join(root, "configure.in")) 1257 1258 ########################################################################### 1259 # Check gettext configuration and dependencies are correct 1260 ########################################################################### 1261 1262 skip_configure_gettext = False 1263 if 'configure-gettext' in skip: 1264 bb.note("Recipe %s skipping qa checking: configure-gettext" % d.getVar('PN')) 1265 skip_configure_gettext = True 1266 1267 cnf = d.getVar('EXTRA_OECONF') or "" 1268 if not ("gettext" in d.getVar('P') or "gcc-runtime" in d.getVar('P') or \ 1269 "--disable-nls" in cnf or skip_configure_gettext): 1270 ml = d.getVar("MLPREFIX") or "" 1271 if bb.data.inherits_class('cross-canadian', d): 1272 gt = "nativesdk-gettext" 1273 else: 1274 gt = "gettext-native" 1275 deps = bb.utils.explode_deps(d.getVar('DEPENDS') or "") 1276 if gt not in deps: 1277 for config in configs: 1278 gnu = "grep \"^[[:space:]]*AM_GNU_GETTEXT\" %s >/dev/null" % config 1279 if subprocess.call(gnu, shell=True) == 0: 1280 error_msg = "AM_GNU_GETTEXT used but no inherit gettext" 1281 oe.qa.handle_error("configure-gettext", error_msg, d) 1282 1283 ########################################################################### 1284 # Check unrecognised configure options (with a white list) 1285 ########################################################################### 1286 if bb.data.inherits_class("autotools", d): 1287 bb.note("Checking configure output for unrecognised options") 1288 try: 1289 if bb.data.inherits_class("autotools", d): 1290 flag = "WARNING: unrecognized options:" 1291 log = os.path.join(d.getVar('B'), 'config.log') 1292 output = subprocess.check_output(['grep', '-F', flag, log]).decode("utf-8").replace(', ', ' ').replace('"', '') 1293 options = set() 1294 for line in output.splitlines(): 1295 options |= set(line.partition(flag)[2].split()) 1296 ignore_opts = set(d.getVar("UNKNOWN_CONFIGURE_OPT_IGNORE").split()) 1297 options -= ignore_opts 1298 if options: 1299 pn = d.getVar('PN') 1300 error_msg = pn + ": configure was passed unrecognised options: " + " ".join(options) 1301 oe.qa.handle_error("unknown-configure-option", error_msg, d) 1302 except subprocess.CalledProcessError: 1303 pass 1304 1305 # Check invalid PACKAGECONFIG 1306 pkgconfig = (d.getVar("PACKAGECONFIG") or "").split() 1307 if pkgconfig: 1308 pkgconfigflags = d.getVarFlags("PACKAGECONFIG") or {} 1309 for pconfig in pkgconfig: 1310 if pconfig not in pkgconfigflags: 1311 pn = d.getVar('PN') 1312 error_msg = "%s: invalid PACKAGECONFIG: %s" % (pn, pconfig) 1313 oe.qa.handle_error("invalid-packageconfig", error_msg, d) 1314 1315 oe.qa.exit_if_errors(d) 1316} 1317 1318def unpack_check_src_uri(pn, d): 1319 import re 1320 1321 skip = (d.getVar('INSANE_SKIP') or "").split() 1322 if 'src-uri-bad' in skip: 1323 bb.note("Recipe %s skipping qa checking: src-uri-bad" % d.getVar('PN')) 1324 return 1325 1326 if "${PN}" in d.getVar("SRC_URI", False): 1327 oe.qa.handle_error("src-uri-bad", "%s: SRC_URI uses PN not BPN" % pn, d) 1328 1329 for url in d.getVar("SRC_URI").split(): 1330 if re.search(r"git(hu|la)b\.com/.+/.+/archive/.+", url): 1331 oe.qa.handle_error("src-uri-bad", "%s: SRC_URI uses unstable GitHub/GitLab archives, convert recipe to use git protocol" % pn, d) 1332 1333python do_qa_unpack() { 1334 src_uri = d.getVar('SRC_URI') 1335 s_dir = d.getVar('S') 1336 if src_uri and not os.path.exists(s_dir): 1337 bb.warn('%s: the directory %s (%s) pointed to by the S variable doesn\'t exist - please set S within the recipe to point to where the source has been unpacked to' % (d.getVar('PN'), d.getVar('S', False), s_dir)) 1338 1339 unpack_check_src_uri(d.getVar('PN'), d) 1340} 1341 1342# The Staging Func, to check all staging 1343#addtask qa_staging after do_populate_sysroot before do_build 1344do_populate_sysroot[postfuncs] += "do_qa_staging " 1345 1346# Check for patch fuzz 1347do_patch[postfuncs] += "do_qa_patch " 1348 1349# Check broken config.log files, for packages requiring Gettext which 1350# don't have it in DEPENDS. 1351#addtask qa_configure after do_configure before do_compile 1352do_configure[postfuncs] += "do_qa_configure " 1353 1354# Check does S exist. 1355do_unpack[postfuncs] += "do_qa_unpack" 1356 1357python () { 1358 import re 1359 1360 tests = d.getVar('ALL_QA').split() 1361 if "desktop" in tests: 1362 d.appendVar("PACKAGE_DEPENDS", " desktop-file-utils-native") 1363 1364 ########################################################################### 1365 # Check various variables 1366 ########################################################################### 1367 1368 # Checking ${FILESEXTRAPATHS} 1369 extrapaths = (d.getVar("FILESEXTRAPATHS") or "") 1370 if '__default' not in extrapaths.split(":"): 1371 msg = "FILESEXTRAPATHS-variable, must always use :prepend (or :append)\n" 1372 msg += "type of assignment, and don't forget the colon.\n" 1373 msg += "Please assign it with the format of:\n" 1374 msg += " FILESEXTRAPATHS:append := \":${THISDIR}/Your_Files_Path\" or\n" 1375 msg += " FILESEXTRAPATHS:prepend := \"${THISDIR}/Your_Files_Path:\"\n" 1376 msg += "in your bbappend file\n\n" 1377 msg += "Your incorrect assignment is:\n" 1378 msg += "%s\n" % extrapaths 1379 bb.warn(msg) 1380 1381 overrides = d.getVar('OVERRIDES').split(':') 1382 pn = d.getVar('PN') 1383 if pn in overrides: 1384 msg = 'Recipe %s has PN of "%s" which is in OVERRIDES, this can result in unexpected behaviour.' % (d.getVar("FILE"), pn) 1385 oe.qa.handle_error("pn-overrides", msg, d) 1386 prog = re.compile(r'[A-Z]') 1387 if prog.search(pn): 1388 oe.qa.handle_error("uppercase-pn", 'PN: %s is upper case, this can result in unexpected behavior.' % pn, d) 1389 1390 # Some people mistakenly use DEPENDS:${PN} instead of DEPENDS and wonder 1391 # why it doesn't work. 1392 if (d.getVar(d.expand('DEPENDS:${PN}'))): 1393 oe.qa.handle_error("pkgvarcheck", "recipe uses DEPENDS:${PN}, should use DEPENDS", d) 1394 1395 issues = [] 1396 if (d.getVar('PACKAGES') or "").split(): 1397 for dep in (d.getVar('QADEPENDS') or "").split(): 1398 d.appendVarFlag('do_package_qa', 'depends', " %s:do_populate_sysroot" % dep) 1399 for var in 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RCONFLICTS', 'RPROVIDES', 'RREPLACES', 'FILES', 'pkg_preinst', 'pkg_postinst', 'pkg_prerm', 'pkg_postrm', 'ALLOW_EMPTY': 1400 if d.getVar(var, False): 1401 issues.append(var) 1402 1403 fakeroot_tests = d.getVar('FAKEROOT_QA').split() 1404 if set(tests) & set(fakeroot_tests): 1405 d.setVarFlag('do_package_qa', 'fakeroot', '1') 1406 d.appendVarFlag('do_package_qa', 'depends', ' virtual/fakeroot-native:do_populate_sysroot') 1407 else: 1408 d.setVarFlag('do_package_qa', 'rdeptask', '') 1409 for i in issues: 1410 oe.qa.handle_error("pkgvarcheck", "%s: Variable %s is set as not being package specific, please fix this." % (d.getVar("FILE"), i), d) 1411 1412 if 'native-last' not in (d.getVar('INSANE_SKIP') or "").split(): 1413 for native_class in ['native', 'nativesdk']: 1414 if bb.data.inherits_class(native_class, d): 1415 1416 inherited_classes = d.getVar('__inherit_cache', False) or [] 1417 needle = os.path.join('classes', native_class) 1418 1419 bbclassextend = (d.getVar('BBCLASSEXTEND') or '').split() 1420 # BBCLASSEXTEND items are always added in the end 1421 skip_classes = bbclassextend 1422 if bb.data.inherits_class('native', d) or 'native' in bbclassextend: 1423 # native also inherits nopackages and relocatable bbclasses 1424 skip_classes.extend(['nopackages', 'relocatable']) 1425 1426 broken_order = [] 1427 for class_item in reversed(inherited_classes): 1428 if needle not in class_item: 1429 for extend_item in skip_classes: 1430 if os.path.join('classes', '%s.bbclass' % extend_item) in class_item: 1431 break 1432 else: 1433 pn = d.getVar('PN') 1434 broken_order.append(os.path.basename(class_item)) 1435 else: 1436 break 1437 if broken_order: 1438 oe.qa.handle_error("native-last", "%s: native/nativesdk class is not inherited last, this can result in unexpected behaviour. " 1439 "Classes inherited after native/nativesdk: %s" % (pn, " ".join(broken_order)), d) 1440 1441 oe.qa.exit_if_errors(d) 1442} 1443