1#!/usr/bin/env python3 2 3# Copyright (C) 2009 by Thomas Petazzoni <thomas.petazzoni@free-electrons.com> 4# 5# This program is free software; you can redistribute it and/or modify 6# it under the terms of the GNU General Public License as published by 7# the Free Software Foundation; either version 2 of the License, or 8# (at your option) any later version. 9# 10# This program is distributed in the hope that it will be useful, 11# but WITHOUT ANY WARRANTY; without even the implied warranty of 12# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13# General Public License for more details. 14# 15# You should have received a copy of the GNU General Public License 16# along with this program; if not, write to the Free Software 17# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 18 19import aiohttp 20import argparse 21import asyncio 22import datetime 23import fnmatch 24import os 25from collections import defaultdict 26import re 27import subprocess 28import json 29import sys 30 31brpath = os.path.normpath(os.path.join(os.path.dirname(__file__), "..", "..")) 32 33sys.path.append(os.path.join(brpath, "utils")) 34from getdeveloperlib import parse_developers # noqa: E402 35from cpedb import CPEDB # noqa: E402 36 37INFRA_RE = re.compile(r"\$\(eval \$\(([a-z-]*)-package\)\)") 38URL_RE = re.compile(r"\s*https?://\S*\s*$") 39 40RM_API_STATUS_ERROR = 1 41RM_API_STATUS_FOUND_BY_DISTRO = 2 42RM_API_STATUS_FOUND_BY_PATTERN = 3 43RM_API_STATUS_NOT_FOUND = 4 44 45 46class Defconfig: 47 def __init__(self, name, path): 48 self.name = name 49 self.path = path 50 self.developers = None 51 52 def set_developers(self, developers): 53 """ 54 Fills in the .developers field 55 """ 56 self.developers = [ 57 developer.name 58 for developer in developers 59 if developer.hasfile(self.path) 60 ] 61 62 63def get_defconfig_list(): 64 """ 65 Builds the list of Buildroot defconfigs, returning a list of Defconfig 66 objects. 67 """ 68 return [ 69 Defconfig(name[:-len('_defconfig')], os.path.join('configs', name)) 70 for name in os.listdir(os.path.join(brpath, 'configs')) 71 if name.endswith('_defconfig') 72 ] 73 74 75class Package: 76 all_licenses = dict() 77 all_license_files = list() 78 all_versions = dict() 79 all_ignored_cves = dict() 80 all_cpeids = dict() 81 # This is the list of all possible checks. Add new checks to this list so 82 # a tool that post-processeds the json output knows the checks before 83 # iterating over the packages. 84 status_checks = ['cve', 'developers', 'hash', 'license', 85 'license-files', 'patches', 'pkg-check', 'url', 'version'] 86 87 def __init__(self, name, path): 88 self.name = name 89 self.path = path 90 self.pkg_path = os.path.dirname(path) 91 self.infras = None 92 self.license = None 93 self.has_license = False 94 self.has_license_files = False 95 self.has_hash = False 96 self.patch_files = [] 97 self.warnings = 0 98 self.current_version = None 99 self.url = None 100 self.url_worker = None 101 self.cpeid = None 102 self.cves = list() 103 self.ignored_cves = list() 104 self.latest_version = {'status': RM_API_STATUS_ERROR, 'version': None, 'id': None} 105 self.status = {} 106 107 def pkgvar(self): 108 return self.name.upper().replace("-", "_") 109 110 def set_url(self): 111 """ 112 Fills in the .url field 113 """ 114 self.status['url'] = ("warning", "no Config.in") 115 pkgdir = os.path.dirname(os.path.join(brpath, self.path)) 116 for filename in os.listdir(pkgdir): 117 if fnmatch.fnmatch(filename, 'Config.*'): 118 fp = open(os.path.join(pkgdir, filename), "r") 119 for config_line in fp: 120 if URL_RE.match(config_line): 121 self.url = config_line.strip() 122 self.status['url'] = ("ok", "found") 123 fp.close() 124 return 125 self.status['url'] = ("error", "missing") 126 fp.close() 127 128 @property 129 def patch_count(self): 130 return len(self.patch_files) 131 132 @property 133 def has_valid_infra(self): 134 if self.infras is None: 135 return False 136 return len(self.infras) > 0 137 138 @property 139 def is_actual_package(self): 140 try: 141 if not self.has_valid_infra: 142 return False 143 if self.infras[0][1] == 'virtual': 144 return False 145 except IndexError: 146 return False 147 return True 148 149 def set_infra(self): 150 """ 151 Fills in the .infras field 152 """ 153 self.infras = list() 154 with open(os.path.join(brpath, self.path), 'r') as f: 155 lines = f.readlines() 156 for line in lines: 157 match = INFRA_RE.match(line) 158 if not match: 159 continue 160 infra = match.group(1) 161 if infra.startswith("host-"): 162 self.infras.append(("host", infra[5:])) 163 else: 164 self.infras.append(("target", infra)) 165 166 def set_license(self): 167 """ 168 Fills in the .status['license'] and .status['license-files'] fields 169 """ 170 if not self.is_actual_package: 171 self.status['license'] = ("na", "no valid package infra") 172 self.status['license-files'] = ("na", "no valid package infra") 173 return 174 175 var = self.pkgvar() 176 self.status['license'] = ("error", "missing") 177 self.status['license-files'] = ("error", "missing") 178 if var in self.all_licenses: 179 self.license = self.all_licenses[var] 180 self.status['license'] = ("ok", "found") 181 if var in self.all_license_files: 182 self.status['license-files'] = ("ok", "found") 183 184 def set_hash_info(self): 185 """ 186 Fills in the .status['hash'] field 187 """ 188 if not self.is_actual_package: 189 self.status['hash'] = ("na", "no valid package infra") 190 self.status['hash-license'] = ("na", "no valid package infra") 191 return 192 193 hashpath = self.path.replace(".mk", ".hash") 194 if os.path.exists(os.path.join(brpath, hashpath)): 195 self.status['hash'] = ("ok", "found") 196 else: 197 self.status['hash'] = ("error", "missing") 198 199 def set_patch_count(self): 200 """ 201 Fills in the .patch_count, .patch_files and .status['patches'] fields 202 """ 203 if not self.is_actual_package: 204 self.status['patches'] = ("na", "no valid package infra") 205 return 206 207 pkgdir = os.path.dirname(os.path.join(brpath, self.path)) 208 for subdir, _, _ in os.walk(pkgdir): 209 self.patch_files = fnmatch.filter(os.listdir(subdir), '*.patch') 210 211 if self.patch_count == 0: 212 self.status['patches'] = ("ok", "no patches") 213 elif self.patch_count < 5: 214 self.status['patches'] = ("warning", "some patches") 215 else: 216 self.status['patches'] = ("error", "lots of patches") 217 218 def set_current_version(self): 219 """ 220 Fills in the .current_version field 221 """ 222 var = self.pkgvar() 223 if var in self.all_versions: 224 self.current_version = self.all_versions[var] 225 226 def set_cpeid(self): 227 """ 228 Fills in the .cpeid field 229 """ 230 var = self.pkgvar() 231 if not self.is_actual_package: 232 self.status['cpe'] = ("na", "N/A - virtual pkg") 233 return 234 if not self.current_version: 235 self.status['cpe'] = ("na", "no version information available") 236 return 237 238 if var in self.all_cpeids: 239 self.cpeid = self.all_cpeids[var] 240 # Set a preliminary status, it might be overridden by check_package_cpes() 241 self.status['cpe'] = ("warning", "not checked against CPE dictionnary") 242 else: 243 self.status['cpe'] = ("error", "no verified CPE identifier") 244 245 def set_check_package_warnings(self): 246 """ 247 Fills in the .warnings and .status['pkg-check'] fields 248 """ 249 cmd = [os.path.join(brpath, "utils/check-package")] 250 pkgdir = os.path.dirname(os.path.join(brpath, self.path)) 251 self.status['pkg-check'] = ("error", "Missing") 252 for root, dirs, files in os.walk(pkgdir): 253 for f in files: 254 if f.endswith(".mk") or f.endswith(".hash") or f == "Config.in" or f == "Config.in.host": 255 cmd.append(os.path.join(root, f)) 256 o = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[1] 257 lines = o.splitlines() 258 for line in lines: 259 m = re.match("^([0-9]*) warnings generated", line.decode()) 260 if m: 261 self.warnings = int(m.group(1)) 262 if self.warnings == 0: 263 self.status['pkg-check'] = ("ok", "no warnings") 264 else: 265 self.status['pkg-check'] = ("error", "{} warnings".format(self.warnings)) 266 return 267 268 def set_ignored_cves(self): 269 """ 270 Give the list of CVEs ignored by the package 271 """ 272 self.ignored_cves = list(self.all_ignored_cves.get(self.pkgvar(), [])) 273 274 def set_developers(self, developers): 275 """ 276 Fills in the .developers and .status['developers'] field 277 """ 278 self.developers = [ 279 dev.name 280 for dev in developers 281 if dev.hasfile(self.path) 282 ] 283 284 if self.developers: 285 self.status['developers'] = ("ok", "{} developers".format(len(self.developers))) 286 else: 287 self.status['developers'] = ("warning", "no developers") 288 289 def is_status_ok(self, name): 290 return name in self.status and self.status[name][0] == 'ok' 291 292 def is_status_error(self, name): 293 return name in self.status and self.status[name][0] == 'error' 294 295 def is_status_na(self, name): 296 return name in self.status and self.status[name][0] == 'na' 297 298 def __eq__(self, other): 299 return self.path == other.path 300 301 def __lt__(self, other): 302 return self.path < other.path 303 304 def __str__(self): 305 return "%s (path='%s', license='%s', license_files='%s', hash='%s', patches=%d)" % \ 306 (self.name, self.path, self.is_status_ok('license'), 307 self.is_status_ok('license-files'), self.status['hash'], self.patch_count) 308 309 310def get_pkglist(npackages, package_list): 311 """ 312 Builds the list of Buildroot packages, returning a list of Package 313 objects. Only the .name and .path fields of the Package object are 314 initialized. 315 316 npackages: limit to N packages 317 package_list: limit to those packages in this list 318 """ 319 WALK_USEFUL_SUBDIRS = ["boot", "linux", "package", "toolchain"] 320 WALK_EXCLUDES = ["boot/common.mk", 321 "linux/linux-ext-.*.mk", 322 "package/freescale-imx/freescale-imx.mk", 323 "package/gcc/gcc.mk", 324 "package/gstreamer/gstreamer.mk", 325 "package/gstreamer1/gstreamer1.mk", 326 "package/gtk2-themes/gtk2-themes.mk", 327 "package/matchbox/matchbox.mk", 328 "package/opengl/opengl.mk", 329 "package/qt5/qt5.mk", 330 "package/x11r7/x11r7.mk", 331 "package/doc-asciidoc.mk", 332 "package/pkg-.*.mk", 333 "toolchain/toolchain-external/pkg-toolchain-external.mk", 334 "toolchain/toolchain-external/toolchain-external.mk", 335 "toolchain/toolchain.mk", 336 "toolchain/helpers.mk", 337 "toolchain/toolchain-wrapper.mk"] 338 packages = list() 339 count = 0 340 for root, dirs, files in os.walk(brpath): 341 root = os.path.relpath(root, brpath) 342 rootdir = root.split("/") 343 if len(rootdir) < 1: 344 continue 345 if rootdir[0] not in WALK_USEFUL_SUBDIRS: 346 continue 347 for f in files: 348 if not f.endswith(".mk"): 349 continue 350 # Strip ending ".mk" 351 pkgname = f[:-3] 352 if package_list and pkgname not in package_list: 353 continue 354 pkgpath = os.path.join(root, f) 355 skip = False 356 for exclude in WALK_EXCLUDES: 357 if re.match(exclude, pkgpath): 358 skip = True 359 continue 360 if skip: 361 continue 362 p = Package(pkgname, pkgpath) 363 packages.append(p) 364 count += 1 365 if npackages and count == npackages: 366 return packages 367 return packages 368 369 370def get_config_packages(): 371 cmd = ["make", "--no-print-directory", "show-info"] 372 js = json.loads(subprocess.check_output(cmd)) 373 return set([v["name"] for v in js.values()]) 374 375 376def package_init_make_info(): 377 # Fetch all variables at once 378 variables = subprocess.check_output(["make", "BR2_HAVE_DOT_CONFIG=y", "-s", "printvars", 379 "VARS=%_LICENSE %_LICENSE_FILES %_VERSION %_IGNORE_CVES %_CPE_ID"]) 380 variable_list = variables.decode().splitlines() 381 382 # We process first the host package VERSION, and then the target 383 # package VERSION. This means that if a package exists in both 384 # target and host variants, with different values (eg. version 385 # numbers (unlikely)), we'll report the target one. 386 variable_list = [x[5:] for x in variable_list if x.startswith("HOST_")] + \ 387 [x for x in variable_list if not x.startswith("HOST_")] 388 389 for item in variable_list: 390 # Get variable name and value 391 pkgvar, value = item.split("=", maxsplit=1) 392 393 # Strip the suffix according to the variable 394 if pkgvar.endswith("_LICENSE"): 395 # If value is "unknown", no license details available 396 if value == "unknown": 397 continue 398 pkgvar = pkgvar[:-8] 399 Package.all_licenses[pkgvar] = value 400 401 elif pkgvar.endswith("_LICENSE_FILES"): 402 if pkgvar.endswith("_MANIFEST_LICENSE_FILES"): 403 continue 404 pkgvar = pkgvar[:-14] 405 Package.all_license_files.append(pkgvar) 406 407 elif pkgvar.endswith("_VERSION"): 408 if pkgvar.endswith("_DL_VERSION"): 409 continue 410 pkgvar = pkgvar[:-8] 411 Package.all_versions[pkgvar] = value 412 413 elif pkgvar.endswith("_IGNORE_CVES"): 414 pkgvar = pkgvar[:-12] 415 Package.all_ignored_cves[pkgvar] = value.split() 416 417 elif pkgvar.endswith("_CPE_ID"): 418 pkgvar = pkgvar[:-7] 419 Package.all_cpeids[pkgvar] = value 420 421 422check_url_count = 0 423 424 425async def check_url_status(session, pkg, npkgs, retry=True): 426 global check_url_count 427 428 try: 429 async with session.get(pkg.url) as resp: 430 if resp.status >= 400: 431 pkg.status['url'] = ("error", "invalid {}".format(resp.status)) 432 check_url_count += 1 433 print("[%04d/%04d] %s" % (check_url_count, npkgs, pkg.name)) 434 return 435 except (aiohttp.ClientError, asyncio.TimeoutError): 436 if retry: 437 return await check_url_status(session, pkg, npkgs, retry=False) 438 else: 439 pkg.status['url'] = ("error", "invalid (err)") 440 check_url_count += 1 441 print("[%04d/%04d] %s" % (check_url_count, npkgs, pkg.name)) 442 return 443 444 pkg.status['url'] = ("ok", "valid") 445 check_url_count += 1 446 print("[%04d/%04d] %s" % (check_url_count, npkgs, pkg.name)) 447 448 449async def check_package_urls(packages): 450 tasks = [] 451 connector = aiohttp.TCPConnector(limit_per_host=5) 452 async with aiohttp.ClientSession(connector=connector, trust_env=True) as sess: 453 packages = [p for p in packages if p.status['url'][0] == 'ok'] 454 for pkg in packages: 455 tasks.append(asyncio.ensure_future(check_url_status(sess, pkg, len(packages)))) 456 await asyncio.wait(tasks) 457 458 459def check_package_latest_version_set_status(pkg, status, version, identifier): 460 pkg.latest_version = { 461 "status": status, 462 "version": version, 463 "id": identifier, 464 } 465 466 if pkg.latest_version['status'] == RM_API_STATUS_ERROR: 467 pkg.status['version'] = ('warning', "Release Monitoring API error") 468 elif pkg.latest_version['status'] == RM_API_STATUS_NOT_FOUND: 469 pkg.status['version'] = ('warning', "Package not found on Release Monitoring") 470 471 if pkg.latest_version['version'] is None: 472 pkg.status['version'] = ('warning', "No upstream version available on Release Monitoring") 473 elif pkg.latest_version['version'] != pkg.current_version: 474 pkg.status['version'] = ('error', "The newer version {} is available upstream".format(pkg.latest_version['version'])) 475 else: 476 pkg.status['version'] = ('ok', 'up-to-date') 477 478 479async def check_package_get_latest_version_by_distro(session, pkg, retry=True): 480 url = "https://release-monitoring.org//api/project/Buildroot/%s" % pkg.name 481 try: 482 async with session.get(url) as resp: 483 if resp.status != 200: 484 return False 485 486 data = await resp.json() 487 version = data['stable_versions'][0] if 'stable_versions' in data else data['version'] if 'version' in data else None 488 check_package_latest_version_set_status(pkg, 489 RM_API_STATUS_FOUND_BY_DISTRO, 490 version, 491 data['id']) 492 return True 493 494 except (aiohttp.ClientError, asyncio.TimeoutError): 495 if retry: 496 return await check_package_get_latest_version_by_distro(session, pkg, retry=False) 497 else: 498 return False 499 500 501async def check_package_get_latest_version_by_guess(session, pkg, retry=True): 502 url = "https://release-monitoring.org/api/projects/?pattern=%s" % pkg.name 503 try: 504 async with session.get(url) as resp: 505 if resp.status != 200: 506 return False 507 508 data = await resp.json() 509 # filter projects that have the right name and a version defined 510 projects = [p for p in data['projects'] if p['name'] == pkg.name and 'stable_versions' in p] 511 projects.sort(key=lambda x: x['id']) 512 513 if len(projects) > 0: 514 check_package_latest_version_set_status(pkg, 515 RM_API_STATUS_FOUND_BY_PATTERN, 516 projects[0]['stable_versions'][0], 517 projects[0]['id']) 518 return True 519 520 except (aiohttp.ClientError, asyncio.TimeoutError): 521 if retry: 522 return await check_package_get_latest_version_by_guess(session, pkg, retry=False) 523 else: 524 return False 525 526 527check_latest_count = 0 528 529 530async def check_package_latest_version_get(session, pkg, npkgs): 531 global check_latest_count 532 533 if await check_package_get_latest_version_by_distro(session, pkg): 534 check_latest_count += 1 535 print("[%04d/%04d] %s" % (check_latest_count, npkgs, pkg.name)) 536 return 537 538 if await check_package_get_latest_version_by_guess(session, pkg): 539 check_latest_count += 1 540 print("[%04d/%04d] %s" % (check_latest_count, npkgs, pkg.name)) 541 return 542 543 check_package_latest_version_set_status(pkg, 544 RM_API_STATUS_NOT_FOUND, 545 None, None) 546 check_latest_count += 1 547 print("[%04d/%04d] %s" % (check_latest_count, npkgs, pkg.name)) 548 549 550async def check_package_latest_version(packages): 551 """ 552 Fills in the .latest_version field of all Package objects 553 554 This field is a dict and has the following keys: 555 556 - status: one of RM_API_STATUS_ERROR, 557 RM_API_STATUS_FOUND_BY_DISTRO, RM_API_STATUS_FOUND_BY_PATTERN, 558 RM_API_STATUS_NOT_FOUND 559 - version: string containing the latest version known by 560 release-monitoring.org for this package 561 - id: string containing the id of the project corresponding to this 562 package, as known by release-monitoring.org 563 """ 564 565 for pkg in [p for p in packages if not p.is_actual_package]: 566 pkg.status['version'] = ("na", "no valid package infra") 567 568 tasks = [] 569 connector = aiohttp.TCPConnector(limit_per_host=5) 570 async with aiohttp.ClientSession(connector=connector, trust_env=True) as sess: 571 packages = [p for p in packages if p.is_actual_package] 572 for pkg in packages: 573 tasks.append(asyncio.ensure_future(check_package_latest_version_get(sess, pkg, len(packages)))) 574 await asyncio.wait(tasks) 575 576 577def check_package_cve_affects(cve, cpe_product_pkgs): 578 for product in cve.affected_products: 579 if product not in cpe_product_pkgs: 580 continue 581 for pkg in cpe_product_pkgs[product]: 582 if cve.affects(pkg.name, pkg.current_version, pkg.ignored_cves, pkg.cpeid) == cve.CVE_AFFECTS: 583 pkg.cves.append(cve.identifier) 584 585 586def check_package_cves(nvd_path, packages): 587 if not os.path.isdir(nvd_path): 588 os.makedirs(nvd_path) 589 590 cpe_product_pkgs = defaultdict(list) 591 for pkg in packages: 592 if not pkg.is_actual_package: 593 pkg.status['cve'] = ("na", "N/A") 594 continue 595 if not pkg.current_version: 596 pkg.status['cve'] = ("na", "no version information available") 597 continue 598 if pkg.cpeid: 599 cpe_product = cvecheck.cpe_product(pkg.cpeid) 600 cpe_product_pkgs[cpe_product].append(pkg) 601 else: 602 cpe_product_pkgs[pkg.name].append(pkg) 603 604 for cve in cvecheck.CVE.read_nvd_dir(nvd_path): 605 check_package_cve_affects(cve, cpe_product_pkgs) 606 607 for pkg in packages: 608 if 'cve' not in pkg.status: 609 if pkg.cves: 610 pkg.status['cve'] = ("error", "affected by CVEs") 611 else: 612 pkg.status['cve'] = ("ok", "not affected by CVEs") 613 614 615def check_package_cpes(nvd_path, packages): 616 cpedb = CPEDB(nvd_path) 617 cpedb.get_xml_dict() 618 for p in packages: 619 if not p.cpeid: 620 continue 621 if cpedb.find(p.cpeid): 622 p.status['cpe'] = ("ok", "verified CPE identifier") 623 else: 624 p.status['cpe'] = ("error", "CPE version unknown in CPE database") 625 626 627def calculate_stats(packages): 628 stats = defaultdict(int) 629 stats['packages'] = len(packages) 630 for pkg in packages: 631 # If packages have multiple infra, take the first one. For the 632 # vast majority of packages, the target and host infra are the 633 # same. There are very few packages that use a different infra 634 # for the host and target variants. 635 if len(pkg.infras) > 0: 636 infra = pkg.infras[0][1] 637 stats["infra-%s" % infra] += 1 638 else: 639 stats["infra-unknown"] += 1 640 if pkg.is_status_ok('license'): 641 stats["license"] += 1 642 else: 643 stats["no-license"] += 1 644 if pkg.is_status_ok('license-files'): 645 stats["license-files"] += 1 646 else: 647 stats["no-license-files"] += 1 648 if pkg.is_status_ok('hash'): 649 stats["hash"] += 1 650 else: 651 stats["no-hash"] += 1 652 if pkg.latest_version['status'] == RM_API_STATUS_FOUND_BY_DISTRO: 653 stats["rmo-mapping"] += 1 654 else: 655 stats["rmo-no-mapping"] += 1 656 if not pkg.latest_version['version']: 657 stats["version-unknown"] += 1 658 elif pkg.latest_version['version'] == pkg.current_version: 659 stats["version-uptodate"] += 1 660 else: 661 stats["version-not-uptodate"] += 1 662 stats["patches"] += pkg.patch_count 663 stats["total-cves"] += len(pkg.cves) 664 if len(pkg.cves) != 0: 665 stats["pkg-cves"] += 1 666 if pkg.cpeid: 667 stats["cpe-id"] += 1 668 else: 669 stats["no-cpe-id"] += 1 670 return stats 671 672 673html_header = """ 674<head> 675<script src=\"https://www.kryogenix.org/code/browser/sorttable/sorttable.js\"></script> 676<style type=\"text/css\"> 677table { 678 width: 100%; 679} 680td { 681 border: 1px solid black; 682} 683td.centered { 684 text-align: center; 685} 686td.wrong { 687 background: #ff9a69; 688} 689td.correct { 690 background: #d2ffc4; 691} 692td.nopatches { 693 background: #d2ffc4; 694} 695td.somepatches { 696 background: #ffd870; 697} 698td.lotsofpatches { 699 background: #ff9a69; 700} 701 702td.good_url { 703 background: #d2ffc4; 704} 705td.missing_url { 706 background: #ffd870; 707} 708td.invalid_url { 709 background: #ff9a69; 710} 711 712td.version-good { 713 background: #d2ffc4; 714} 715td.version-needs-update { 716 background: #ff9a69; 717} 718td.version-unknown { 719 background: #ffd870; 720} 721td.version-error { 722 background: #ccc; 723} 724 725td.cpe-ok { 726 background: #d2ffc4; 727} 728 729td.cpe-nok { 730 background: #ff9a69; 731} 732 733td.cpe-unknown { 734 background: #ffd870; 735} 736 737td.cve-ok { 738 background: #d2ffc4; 739} 740 741td.cve-nok { 742 background: #ff9a69; 743} 744 745td.cve-unknown { 746 background: #ffd870; 747} 748 749td.cve_ignored { 750 background: #ccc; 751} 752 753</style> 754<title>Statistics of Buildroot packages</title> 755</head> 756 757<a href=\"#results\">Results</a><br/> 758 759<p id=\"sortable_hint\"></p> 760""" 761 762 763html_footer = """ 764</body> 765<script> 766if (typeof sorttable === \"object\") { 767 document.getElementById(\"sortable_hint\").innerHTML = 768 \"hint: the table can be sorted by clicking the column headers\" 769} 770</script> 771</html> 772""" 773 774 775def infra_str(infra_list): 776 if not infra_list: 777 return "Unknown" 778 elif len(infra_list) == 1: 779 return "<b>%s</b><br/>%s" % (infra_list[0][1], infra_list[0][0]) 780 elif infra_list[0][1] == infra_list[1][1]: 781 return "<b>%s</b><br/>%s + %s" % \ 782 (infra_list[0][1], infra_list[0][0], infra_list[1][0]) 783 else: 784 return "<b>%s</b> (%s)<br/><b>%s</b> (%s)" % \ 785 (infra_list[0][1], infra_list[0][0], 786 infra_list[1][1], infra_list[1][0]) 787 788 789def boolean_str(b): 790 if b: 791 return "Yes" 792 else: 793 return "No" 794 795 796def dump_html_pkg(f, pkg): 797 f.write(" <tr>\n") 798 f.write(" <td>%s</td>\n" % pkg.path) 799 800 # Patch count 801 td_class = ["centered"] 802 if pkg.patch_count == 0: 803 td_class.append("nopatches") 804 elif pkg.patch_count < 5: 805 td_class.append("somepatches") 806 else: 807 td_class.append("lotsofpatches") 808 f.write(" <td class=\"%s\">%s</td>\n" % 809 (" ".join(td_class), str(pkg.patch_count))) 810 811 # Infrastructure 812 infra = infra_str(pkg.infras) 813 td_class = ["centered"] 814 if infra == "Unknown": 815 td_class.append("wrong") 816 else: 817 td_class.append("correct") 818 f.write(" <td class=\"%s\">%s</td>\n" % 819 (" ".join(td_class), infra_str(pkg.infras))) 820 821 # License 822 td_class = ["centered"] 823 if pkg.is_status_ok('license'): 824 td_class.append("correct") 825 else: 826 td_class.append("wrong") 827 f.write(" <td class=\"%s\">%s</td>\n" % 828 (" ".join(td_class), boolean_str(pkg.is_status_ok('license')))) 829 830 # License files 831 td_class = ["centered"] 832 if pkg.is_status_ok('license-files'): 833 td_class.append("correct") 834 else: 835 td_class.append("wrong") 836 f.write(" <td class=\"%s\">%s</td>\n" % 837 (" ".join(td_class), boolean_str(pkg.is_status_ok('license-files')))) 838 839 # Hash 840 td_class = ["centered"] 841 if pkg.is_status_ok('hash'): 842 td_class.append("correct") 843 else: 844 td_class.append("wrong") 845 f.write(" <td class=\"%s\">%s</td>\n" % 846 (" ".join(td_class), boolean_str(pkg.is_status_ok('hash')))) 847 848 # Current version 849 if len(pkg.current_version) > 20: 850 current_version = pkg.current_version[:20] + "..." 851 else: 852 current_version = pkg.current_version 853 f.write(" <td class=\"centered\">%s</td>\n" % current_version) 854 855 # Latest version 856 if pkg.latest_version['status'] == RM_API_STATUS_ERROR: 857 td_class.append("version-error") 858 if pkg.latest_version['version'] is None: 859 td_class.append("version-unknown") 860 elif pkg.latest_version['version'] != pkg.current_version: 861 td_class.append("version-needs-update") 862 else: 863 td_class.append("version-good") 864 865 if pkg.latest_version['status'] == RM_API_STATUS_ERROR: 866 latest_version_text = "<b>Error</b>" 867 elif pkg.latest_version['status'] == RM_API_STATUS_NOT_FOUND: 868 latest_version_text = "<b>Not found</b>" 869 else: 870 if pkg.latest_version['version'] is None: 871 latest_version_text = "<b>Found, but no version</b>" 872 else: 873 latest_version_text = "<a href=\"https://release-monitoring.org/project/%s\"><b>%s</b></a>" % \ 874 (pkg.latest_version['id'], str(pkg.latest_version['version'])) 875 876 latest_version_text += "<br/>" 877 878 if pkg.latest_version['status'] == RM_API_STATUS_FOUND_BY_DISTRO: 879 latest_version_text += "found by <a href=\"https://release-monitoring.org/distro/Buildroot/\">distro</a>" 880 else: 881 latest_version_text += "found by guess" 882 883 f.write(" <td class=\"%s\">%s</td>\n" % 884 (" ".join(td_class), latest_version_text)) 885 886 # Warnings 887 td_class = ["centered"] 888 if pkg.warnings == 0: 889 td_class.append("correct") 890 else: 891 td_class.append("wrong") 892 f.write(" <td class=\"%s\">%d</td>\n" % 893 (" ".join(td_class), pkg.warnings)) 894 895 # URL status 896 td_class = ["centered"] 897 url_str = pkg.status['url'][1] 898 if pkg.status['url'][0] in ("error", "warning"): 899 td_class.append("missing_url") 900 if pkg.status['url'][0] == "error": 901 td_class.append("invalid_url") 902 url_str = "<a href=%s>%s</a>" % (pkg.url, pkg.status['url'][1]) 903 else: 904 td_class.append("good_url") 905 url_str = "<a href=%s>Link</a>" % pkg.url 906 f.write(" <td class=\"%s\">%s</td>\n" % 907 (" ".join(td_class), url_str)) 908 909 # CVEs 910 td_class = ["centered"] 911 if pkg.is_status_ok("cve"): 912 td_class.append("cve-ok") 913 elif pkg.is_status_error("cve"): 914 td_class.append("cve-nok") 915 elif pkg.is_status_na("cve") and not pkg.is_actual_package: 916 td_class.append("cve-ok") 917 else: 918 td_class.append("cve-unknown") 919 f.write(" <td class=\"%s\">\n" % " ".join(td_class)) 920 if pkg.is_status_error("cve"): 921 for cve in pkg.cves: 922 f.write(" <a href=\"https://security-tracker.debian.org/tracker/%s\">%s<br/>\n" % (cve, cve)) 923 elif pkg.is_status_na("cve"): 924 f.write(" %s" % pkg.status['cve'][1]) 925 else: 926 f.write(" N/A\n") 927 f.write(" </td>\n") 928 929 # CVEs Ignored 930 td_class = ["centered"] 931 if pkg.ignored_cves: 932 td_class.append("cve_ignored") 933 f.write(" <td class=\"%s\">\n" % " ".join(td_class)) 934 for ignored_cve in pkg.ignored_cves: 935 f.write(" <a href=\"https://security-tracker.debian.org/tracker/%s\">%s<br/>\n" % (ignored_cve, ignored_cve)) 936 f.write(" </td>\n") 937 938 # CPE ID 939 td_class = ["left"] 940 if pkg.is_status_ok("cpe"): 941 td_class.append("cpe-ok") 942 elif pkg.is_status_error("cpe"): 943 td_class.append("cpe-nok") 944 elif pkg.is_status_na("cpe") and not pkg.is_actual_package: 945 td_class.append("cpe-ok") 946 else: 947 td_class.append("cpe-unknown") 948 f.write(" <td class=\"%s\">\n" % " ".join(td_class)) 949 if pkg.cpeid: 950 f.write(" <code>%s</code>\n" % pkg.cpeid) 951 if not pkg.is_status_ok("cpe"): 952 if pkg.is_actual_package and pkg.current_version: 953 if pkg.cpeid: 954 f.write(" <br/>%s <a href=\"https://nvd.nist.gov/products/cpe/search/results?namingFormat=2.3&keyword=%s\">(Search)</a>\n" % # noqa: E501 955 (pkg.status['cpe'][1], ":".join(pkg.cpeid.split(":")[0:5]))) 956 else: 957 f.write(" %s <a href=\"https://nvd.nist.gov/products/cpe/search/results?namingFormat=2.3&keyword=%s\">(Search)</a>\n" % # noqa: E501 958 (pkg.status['cpe'][1], pkg.name)) 959 else: 960 f.write(" %s\n" % pkg.status['cpe'][1]) 961 962 f.write(" </td>\n") 963 964 f.write(" </tr>\n") 965 966 967def dump_html_all_pkgs(f, packages): 968 f.write(""" 969<table class=\"sortable\"> 970<tr> 971<td>Package</td> 972<td class=\"centered\">Patch count</td> 973<td class=\"centered\">Infrastructure</td> 974<td class=\"centered\">License</td> 975<td class=\"centered\">License files</td> 976<td class=\"centered\">Hash file</td> 977<td class=\"centered\">Current version</td> 978<td class=\"centered\">Latest version</td> 979<td class=\"centered\">Warnings</td> 980<td class=\"centered\">Upstream URL</td> 981<td class=\"centered\">CVEs</td> 982<td class=\"centered\">CVEs Ignored</td> 983<td class=\"centered\">CPE ID</td> 984</tr> 985""") 986 for pkg in sorted(packages): 987 dump_html_pkg(f, pkg) 988 f.write("</table>") 989 990 991def dump_html_stats(f, stats): 992 f.write("<a id=\"results\"></a>\n") 993 f.write("<table>\n") 994 infras = [infra[6:] for infra in stats.keys() if infra.startswith("infra-")] 995 for infra in infras: 996 f.write(" <tr><td>Packages using the <i>%s</i> infrastructure</td><td>%s</td></tr>\n" % 997 (infra, stats["infra-%s" % infra])) 998 f.write(" <tr><td>Packages having license information</td><td>%s</td></tr>\n" % 999 stats["license"]) 1000 f.write(" <tr><td>Packages not having license information</td><td>%s</td></tr>\n" % 1001 stats["no-license"]) 1002 f.write(" <tr><td>Packages having license files information</td><td>%s</td></tr>\n" % 1003 stats["license-files"]) 1004 f.write(" <tr><td>Packages not having license files information</td><td>%s</td></tr>\n" % 1005 stats["no-license-files"]) 1006 f.write(" <tr><td>Packages having a hash file</td><td>%s</td></tr>\n" % 1007 stats["hash"]) 1008 f.write(" <tr><td>Packages not having a hash file</td><td>%s</td></tr>\n" % 1009 stats["no-hash"]) 1010 f.write(" <tr><td>Total number of patches</td><td>%s</td></tr>\n" % 1011 stats["patches"]) 1012 f.write("<tr><td>Packages having a mapping on <i>release-monitoring.org</i></td><td>%s</td></tr>\n" % 1013 stats["rmo-mapping"]) 1014 f.write("<tr><td>Packages lacking a mapping on <i>release-monitoring.org</i></td><td>%s</td></tr>\n" % 1015 stats["rmo-no-mapping"]) 1016 f.write("<tr><td>Packages that are up-to-date</td><td>%s</td></tr>\n" % 1017 stats["version-uptodate"]) 1018 f.write("<tr><td>Packages that are not up-to-date</td><td>%s</td></tr>\n" % 1019 stats["version-not-uptodate"]) 1020 f.write("<tr><td>Packages with no known upstream version</td><td>%s</td></tr>\n" % 1021 stats["version-unknown"]) 1022 f.write("<tr><td>Packages affected by CVEs</td><td>%s</td></tr>\n" % 1023 stats["pkg-cves"]) 1024 f.write("<tr><td>Total number of CVEs affecting all packages</td><td>%s</td></tr>\n" % 1025 stats["total-cves"]) 1026 f.write("<tr><td>Packages with CPE ID</td><td>%s</td></tr>\n" % 1027 stats["cpe-id"]) 1028 f.write("<tr><td>Packages without CPE ID</td><td>%s</td></tr>\n" % 1029 stats["no-cpe-id"]) 1030 f.write("</table>\n") 1031 1032 1033def dump_html_gen_info(f, date, commit): 1034 # Updated on Mon Feb 19 08:12:08 CET 2018, Git commit aa77030b8f5e41f1c53eb1c1ad664b8c814ba032 1035 f.write("<p><i>Updated on %s, git commit %s</i></p>\n" % (str(date), commit)) 1036 1037 1038def dump_html(packages, stats, date, commit, output): 1039 with open(output, 'w') as f: 1040 f.write(html_header) 1041 dump_html_all_pkgs(f, packages) 1042 dump_html_stats(f, stats) 1043 dump_html_gen_info(f, date, commit) 1044 f.write(html_footer) 1045 1046 1047def dump_json(packages, defconfigs, stats, date, commit, output): 1048 # Format packages as a dictionnary instead of a list 1049 # Exclude local field that does not contains real date 1050 excluded_fields = ['url_worker', 'name'] 1051 pkgs = { 1052 pkg.name: { 1053 k: v 1054 for k, v in pkg.__dict__.items() 1055 if k not in excluded_fields 1056 } for pkg in packages 1057 } 1058 defconfigs = { 1059 d.name: { 1060 k: v 1061 for k, v in d.__dict__.items() 1062 } for d in defconfigs 1063 } 1064 # Aggregate infrastructures into a single dict entry 1065 statistics = { 1066 k: v 1067 for k, v in stats.items() 1068 if not k.startswith('infra-') 1069 } 1070 statistics['infra'] = {k[6:]: v for k, v in stats.items() if k.startswith('infra-')} 1071 # The actual structure to dump, add commit and date to it 1072 final = {'packages': pkgs, 1073 'stats': statistics, 1074 'defconfigs': defconfigs, 1075 'package_status_checks': Package.status_checks, 1076 'commit': commit, 1077 'date': str(date)} 1078 1079 with open(output, 'w') as f: 1080 json.dump(final, f, indent=2, separators=(',', ': ')) 1081 f.write('\n') 1082 1083 1084def resolvepath(path): 1085 return os.path.abspath(os.path.expanduser(path)) 1086 1087 1088def parse_args(): 1089 parser = argparse.ArgumentParser() 1090 output = parser.add_argument_group('output', 'Output file(s)') 1091 output.add_argument('--html', dest='html', type=resolvepath, 1092 help='HTML output file') 1093 output.add_argument('--json', dest='json', type=resolvepath, 1094 help='JSON output file') 1095 packages = parser.add_mutually_exclusive_group() 1096 packages.add_argument('-c', dest='configpackages', action='store_true', 1097 help='Apply to packages enabled in current configuration') 1098 packages.add_argument('-n', dest='npackages', type=int, action='store', 1099 help='Number of packages') 1100 packages.add_argument('-p', dest='packages', action='store', 1101 help='List of packages (comma separated)') 1102 parser.add_argument('--nvd-path', dest='nvd_path', 1103 help='Path to the local NVD database', type=resolvepath) 1104 args = parser.parse_args() 1105 if not args.html and not args.json: 1106 parser.error('at least one of --html or --json (or both) is required') 1107 return args 1108 1109 1110def __main__(): 1111 global cvecheck 1112 1113 args = parse_args() 1114 1115 if args.nvd_path: 1116 import cve as cvecheck 1117 1118 if args.packages: 1119 package_list = args.packages.split(",") 1120 elif args.configpackages: 1121 package_list = get_config_packages() 1122 else: 1123 package_list = None 1124 date = datetime.datetime.utcnow() 1125 commit = subprocess.check_output(['git', '-C', brpath, 1126 'rev-parse', 1127 'HEAD']).splitlines()[0].decode() 1128 print("Build package list ...") 1129 packages = get_pkglist(args.npackages, package_list) 1130 print("Getting developers ...") 1131 developers = parse_developers() 1132 print("Build defconfig list ...") 1133 defconfigs = get_defconfig_list() 1134 for d in defconfigs: 1135 d.set_developers(developers) 1136 print("Getting package make info ...") 1137 package_init_make_info() 1138 print("Getting package details ...") 1139 for pkg in packages: 1140 pkg.set_infra() 1141 pkg.set_license() 1142 pkg.set_hash_info() 1143 pkg.set_patch_count() 1144 pkg.set_check_package_warnings() 1145 pkg.set_current_version() 1146 pkg.set_cpeid() 1147 pkg.set_url() 1148 pkg.set_ignored_cves() 1149 pkg.set_developers(developers) 1150 print("Checking URL status") 1151 loop = asyncio.get_event_loop() 1152 loop.run_until_complete(check_package_urls(packages)) 1153 print("Getting latest versions ...") 1154 loop = asyncio.get_event_loop() 1155 loop.run_until_complete(check_package_latest_version(packages)) 1156 if args.nvd_path: 1157 print("Checking packages CVEs") 1158 check_package_cves(args.nvd_path, packages) 1159 check_package_cpes(args.nvd_path, packages) 1160 print("Calculate stats") 1161 stats = calculate_stats(packages) 1162 if args.html: 1163 print("Write HTML") 1164 dump_html(packages, stats, date, commit, args.html) 1165 if args.json: 1166 print("Write JSON") 1167 dump_json(packages, defconfigs, stats, date, commit, args.json) 1168 1169 1170__main__() 1171