1# 2# Copyright BitBake Contributors 3# 4# SPDX-License-Identifier: GPL-2.0-only 5# 6 7import collections 8import fnmatch 9import logging 10import sys 11import os 12import re 13 14import bb.utils 15 16from bblayers.common import LayerPlugin 17 18logger = logging.getLogger('bitbake-layers') 19 20 21def plugin_init(plugins): 22 return QueryPlugin() 23 24 25class QueryPlugin(LayerPlugin): 26 def __init__(self): 27 super(QueryPlugin, self).__init__() 28 self.collection_res = {} 29 30 def do_show_layers(self, args): 31 """show current configured layers.""" 32 logger.plain("%s %s %s" % ("layer".ljust(20), "path".ljust(40), "priority")) 33 logger.plain('=' * 74) 34 for layer, _, regex, pri in self.tinfoil.cooker.bbfile_config_priorities: 35 layerdir = self.bbfile_collections.get(layer, None) 36 layername = self.get_layer_name(layerdir) 37 logger.plain("%s %s %d" % (layername.ljust(20), layerdir.ljust(40), pri)) 38 39 def version_str(self, pe, pv, pr = None): 40 verstr = "%s" % pv 41 if pr: 42 verstr = "%s-%s" % (verstr, pr) 43 if pe: 44 verstr = "%s:%s" % (pe, verstr) 45 return verstr 46 47 def do_show_overlayed(self, args): 48 """list overlayed recipes (where the same recipe exists in another layer) 49 50Lists the names of overlayed recipes and the available versions in each 51layer, with the preferred version first. Note that skipped recipes that 52are overlayed will also be listed, with a " (skipped)" suffix. 53""" 54 55 items_listed = self.list_recipes('Overlayed recipes', None, True, args.same_version, args.filenames, False, True, None, False, None, args.mc) 56 57 # Check for overlayed .bbclass files 58 classes = collections.defaultdict(list) 59 for layerdir in self.bblayers: 60 classdir = os.path.join(layerdir, 'classes') 61 if os.path.exists(classdir): 62 for classfile in os.listdir(classdir): 63 if os.path.splitext(classfile)[1] == '.bbclass': 64 classes[classfile].append(classdir) 65 66 # Locating classes and other files is a bit more complicated than recipes - 67 # layer priority is not a factor; instead BitBake uses the first matching 68 # file in BBPATH, which is manipulated directly by each layer's 69 # conf/layer.conf in turn, thus the order of layers in bblayers.conf is a 70 # factor - however, each layer.conf is free to either prepend or append to 71 # BBPATH (or indeed do crazy stuff with it). Thus the order in BBPATH might 72 # not be exactly the order present in bblayers.conf either. 73 bbpath = str(self.tinfoil.config_data.getVar('BBPATH')) 74 overlayed_class_found = False 75 for (classfile, classdirs) in classes.items(): 76 if len(classdirs) > 1: 77 if not overlayed_class_found: 78 logger.plain('=== Overlayed classes ===') 79 overlayed_class_found = True 80 81 mainfile = bb.utils.which(bbpath, os.path.join('classes', classfile)) 82 if args.filenames: 83 logger.plain('%s' % mainfile) 84 else: 85 # We effectively have to guess the layer here 86 logger.plain('%s:' % classfile) 87 mainlayername = '?' 88 for layerdir in self.bblayers: 89 classdir = os.path.join(layerdir, 'classes') 90 if mainfile.startswith(classdir): 91 mainlayername = self.get_layer_name(layerdir) 92 logger.plain(' %s' % mainlayername) 93 for classdir in classdirs: 94 fullpath = os.path.join(classdir, classfile) 95 if fullpath != mainfile: 96 if args.filenames: 97 print(' %s' % fullpath) 98 else: 99 print(' %s' % self.get_layer_name(os.path.dirname(classdir))) 100 101 if overlayed_class_found: 102 items_listed = True; 103 104 if not items_listed: 105 logger.plain('No overlayed files found.') 106 107 def do_show_recipes(self, args): 108 """list available recipes, showing the layer they are provided by 109 110Lists the names of recipes and the available versions in each 111layer, with the preferred version first. Optionally you may specify 112pnspec to match a specified recipe name (supports wildcards). Note that 113skipped recipes will also be listed, with a " (skipped)" suffix. 114""" 115 116 inheritlist = args.inherits.split(',') if args.inherits else [] 117 if inheritlist or args.pnspec or args.multiple: 118 title = 'Matching recipes:' 119 else: 120 title = 'Available recipes:' 121 self.list_recipes(title, args.pnspec, False, False, args.filenames, args.recipes_only, args.multiple, args.layer, args.bare, inheritlist, args.mc) 122 123 def list_recipes(self, title, pnspec, show_overlayed_only, show_same_ver_only, show_filenames, show_recipes_only, show_multi_provider_only, selected_layer, bare, inherits, mc): 124 if inherits: 125 bbpath = str(self.tinfoil.config_data.getVar('BBPATH')) 126 for classname in inherits: 127 classfile = 'classes/%s.bbclass' % classname 128 if not bb.utils.which(bbpath, classfile, history=False): 129 logger.error('No class named %s found in BBPATH', classfile) 130 sys.exit(1) 131 132 pkg_pn = self.tinfoil.cooker.recipecaches[mc].pkg_pn 133 (latest_versions, preferred_versions, required_versions) = self.tinfoil.find_providers(mc) 134 allproviders = self.tinfoil.get_all_providers(mc) 135 136 # Ensure we list skipped recipes 137 # We are largely guessing about PN, PV and the preferred version here, 138 # but we have no choice since skipped recipes are not fully parsed 139 skiplist = list(self.tinfoil.cooker.skiplist.keys()) 140 mcspec = 'mc:%s:' % mc 141 if mc: 142 skiplist = [s[len(mcspec):] for s in skiplist if s.startswith(mcspec)] 143 144 for fn in skiplist: 145 recipe_parts = os.path.splitext(os.path.basename(fn))[0].split('_') 146 p = recipe_parts[0] 147 if len(recipe_parts) > 1: 148 ver = (None, recipe_parts[1], None) 149 else: 150 ver = (None, 'unknown', None) 151 allproviders[p].append((ver, fn)) 152 if not p in pkg_pn: 153 pkg_pn[p] = 'dummy' 154 preferred_versions[p] = (ver, fn) 155 156 def print_item(f, pn, ver, layer, ispref): 157 if not selected_layer or layer == selected_layer: 158 if not bare and f in skiplist: 159 skipped = ' (skipped: %s)' % self.tinfoil.cooker.skiplist[f].skipreason 160 else: 161 skipped = '' 162 if show_filenames: 163 if ispref: 164 logger.plain("%s%s", f, skipped) 165 else: 166 logger.plain(" %s%s", f, skipped) 167 elif show_recipes_only: 168 if pn not in show_unique_pn: 169 show_unique_pn.append(pn) 170 logger.plain("%s%s", pn, skipped) 171 else: 172 if ispref: 173 logger.plain("%s:", pn) 174 logger.plain(" %s %s%s", layer.ljust(20), ver, skipped) 175 176 global_inherit = (self.tinfoil.config_data.getVar('INHERIT') or "").split() 177 cls_re = re.compile('classes/') 178 179 preffiles = [] 180 show_unique_pn = [] 181 items_listed = False 182 for p in sorted(pkg_pn): 183 if pnspec: 184 found=False 185 for pnm in pnspec: 186 if fnmatch.fnmatch(p, pnm): 187 found=True 188 break 189 if not found: 190 continue 191 192 if len(allproviders[p]) > 1 or not show_multi_provider_only: 193 pref = preferred_versions[p] 194 realfn = bb.cache.virtualfn2realfn(pref[1]) 195 preffile = realfn[0] 196 197 # We only display once per recipe, we should prefer non extended versions of the 198 # recipe if present (so e.g. in OpenEmbedded, openssl rather than nativesdk-openssl 199 # which would otherwise sort first). 200 if realfn[1] and realfn[0] in self.tinfoil.cooker.recipecaches[mc].pkg_fn: 201 continue 202 203 if inherits: 204 matchcount = 0 205 recipe_inherits = self.tinfoil.cooker_data.inherits.get(preffile, []) 206 for cls in recipe_inherits: 207 if cls_re.match(cls): 208 continue 209 classname = os.path.splitext(os.path.basename(cls))[0] 210 if classname in global_inherit: 211 continue 212 elif classname in inherits: 213 matchcount += 1 214 if matchcount != len(inherits): 215 # No match - skip this recipe 216 continue 217 218 if preffile not in preffiles: 219 preflayer = self.get_file_layer(preffile) 220 multilayer = False 221 same_ver = True 222 provs = [] 223 for prov in allproviders[p]: 224 provfile = bb.cache.virtualfn2realfn(prov[1])[0] 225 provlayer = self.get_file_layer(provfile) 226 provs.append((provfile, provlayer, prov[0])) 227 if provlayer != preflayer: 228 multilayer = True 229 if prov[0] != pref[0]: 230 same_ver = False 231 if (multilayer or not show_overlayed_only) and (same_ver or not show_same_ver_only): 232 if not items_listed: 233 logger.plain('=== %s ===' % title) 234 items_listed = True 235 print_item(preffile, p, self.version_str(pref[0][0], pref[0][1]), preflayer, True) 236 for (provfile, provlayer, provver) in provs: 237 if provfile != preffile: 238 print_item(provfile, p, self.version_str(provver[0], provver[1]), provlayer, False) 239 # Ensure we don't show two entries for BBCLASSEXTENDed recipes 240 preffiles.append(preffile) 241 242 return items_listed 243 244 def get_file_layer(self, filename): 245 layerdir = self.get_file_layerdir(filename) 246 if layerdir: 247 return self.get_layer_name(layerdir) 248 else: 249 return '?' 250 251 def get_collection_res(self): 252 if not self.collection_res: 253 self.collection_res = bb.utils.get_collection_res(self.tinfoil.config_data) 254 return self.collection_res 255 256 def get_file_layerdir(self, filename): 257 layer = bb.utils.get_file_layer(filename, self.tinfoil.config_data, self.get_collection_res()) 258 return self.bbfile_collections.get(layer, None) 259 260 def remove_layer_prefix(self, f): 261 """Remove the layer_dir prefix, e.g., f = /path/to/layer_dir/foo/blah, the 262 return value will be: layer_dir/foo/blah""" 263 f_layerdir = self.get_file_layerdir(f) 264 if not f_layerdir: 265 return f 266 prefix = os.path.join(os.path.dirname(f_layerdir), '') 267 return f[len(prefix):] if f.startswith(prefix) else f 268 269 def do_show_appends(self, args): 270 """list bbappend files and recipe files they apply to 271 272Lists recipes with the bbappends that apply to them as subitems. 273""" 274 if args.pnspec: 275 logger.plain('=== Matched appended recipes ===') 276 else: 277 logger.plain('=== Appended recipes ===') 278 279 pnlist = list(self.tinfoil.cooker_data.pkg_pn.keys()) 280 pnlist.sort() 281 appends = False 282 for pn in pnlist: 283 if args.pnspec: 284 found=False 285 for pnm in args.pnspec: 286 if fnmatch.fnmatch(pn, pnm): 287 found=True 288 break 289 if not found: 290 continue 291 292 if self.show_appends_for_pn(pn): 293 appends = True 294 295 if not args.pnspec and self.show_appends_for_skipped(): 296 appends = True 297 298 if not appends: 299 logger.plain('No append files found') 300 301 def show_appends_for_pn(self, pn): 302 filenames = self.tinfoil.cooker_data.pkg_pn[pn] 303 304 best = self.tinfoil.find_best_provider(pn) 305 best_filename = os.path.basename(best[3]) 306 307 return self.show_appends_output(filenames, best_filename) 308 309 def show_appends_for_skipped(self): 310 filenames = [os.path.basename(f) 311 for f in self.tinfoil.cooker.skiplist.keys()] 312 return self.show_appends_output(filenames, None, " (skipped)") 313 314 def show_appends_output(self, filenames, best_filename, name_suffix = ''): 315 appended, missing = self.get_appends_for_files(filenames) 316 if appended: 317 for basename, appends in appended: 318 logger.plain('%s%s:', basename, name_suffix) 319 for append in appends: 320 logger.plain(' %s', append) 321 322 if best_filename: 323 if best_filename in missing: 324 logger.warning('%s: missing append for preferred version', 325 best_filename) 326 return True 327 else: 328 return False 329 330 def get_appends_for_files(self, filenames): 331 appended, notappended = [], [] 332 for filename in filenames: 333 _, cls, mc = bb.cache.virtualfn2realfn(filename) 334 if cls: 335 continue 336 337 basename = os.path.basename(filename) 338 appends = self.tinfoil.cooker.collections[mc].get_file_appends(basename) 339 if appends: 340 appended.append((basename, list(appends))) 341 else: 342 notappended.append(basename) 343 return appended, notappended 344 345 def do_show_cross_depends(self, args): 346 """Show dependencies between recipes that cross layer boundaries. 347 348Figure out the dependencies between recipes that cross layer boundaries. 349 350NOTE: .bbappend files can impact the dependencies. 351""" 352 ignore_layers = (args.ignore or '').split(',') 353 354 pkg_fn = self.tinfoil.cooker_data.pkg_fn 355 bbpath = str(self.tinfoil.config_data.getVar('BBPATH')) 356 self.require_re = re.compile(r"require\s+(.+)") 357 self.include_re = re.compile(r"include\s+(.+)") 358 self.inherit_re = re.compile(r"inherit\s+(.+)") 359 360 global_inherit = (self.tinfoil.config_data.getVar('INHERIT') or "").split() 361 362 # The bb's DEPENDS and RDEPENDS 363 for f in pkg_fn: 364 f = bb.cache.virtualfn2realfn(f)[0] 365 # Get the layername that the file is in 366 layername = self.get_file_layer(f) 367 368 # The DEPENDS 369 deps = self.tinfoil.cooker_data.deps[f] 370 for pn in deps: 371 if pn in self.tinfoil.cooker_data.pkg_pn: 372 best = self.tinfoil.find_best_provider(pn) 373 self.check_cross_depends("DEPENDS", layername, f, best[3], args.filenames, ignore_layers) 374 375 # The RDPENDS 376 all_rdeps = self.tinfoil.cooker_data.rundeps[f].values() 377 # Remove the duplicated or null one. 378 sorted_rdeps = {} 379 # The all_rdeps is the list in list, so we need two for loops 380 for k1 in all_rdeps: 381 for k2 in k1: 382 sorted_rdeps[k2] = 1 383 all_rdeps = sorted_rdeps.keys() 384 for rdep in all_rdeps: 385 all_p, best = self.tinfoil.get_runtime_providers(rdep) 386 if all_p: 387 if f in all_p: 388 # The recipe provides this one itself, ignore 389 continue 390 self.check_cross_depends("RDEPENDS", layername, f, best, args.filenames, ignore_layers) 391 392 # The RRECOMMENDS 393 all_rrecs = self.tinfoil.cooker_data.runrecs[f].values() 394 # Remove the duplicated or null one. 395 sorted_rrecs = {} 396 # The all_rrecs is the list in list, so we need two for loops 397 for k1 in all_rrecs: 398 for k2 in k1: 399 sorted_rrecs[k2] = 1 400 all_rrecs = sorted_rrecs.keys() 401 for rrec in all_rrecs: 402 all_p, best = self.tinfoil.get_runtime_providers(rrec) 403 if all_p: 404 if f in all_p: 405 # The recipe provides this one itself, ignore 406 continue 407 self.check_cross_depends("RRECOMMENDS", layername, f, best, args.filenames, ignore_layers) 408 409 # The inherit class 410 cls_re = re.compile('classes/') 411 if f in self.tinfoil.cooker_data.inherits: 412 inherits = self.tinfoil.cooker_data.inherits[f] 413 for cls in inherits: 414 # The inherits' format is [classes/cls, /path/to/classes/cls] 415 # ignore the classes/cls. 416 if not cls_re.match(cls): 417 classname = os.path.splitext(os.path.basename(cls))[0] 418 if classname in global_inherit: 419 continue 420 inherit_layername = self.get_file_layer(cls) 421 if inherit_layername != layername and not inherit_layername in ignore_layers: 422 if not args.filenames: 423 f_short = self.remove_layer_prefix(f) 424 cls = self.remove_layer_prefix(cls) 425 else: 426 f_short = f 427 logger.plain("%s inherits %s" % (f_short, cls)) 428 429 # The 'require/include xxx' in the bb file 430 pv_re = re.compile(r"\${PV}") 431 with open(f, 'r') as fnfile: 432 line = fnfile.readline() 433 while line: 434 m, keyword = self.match_require_include(line) 435 # Found the 'require/include xxxx' 436 if m: 437 needed_file = m.group(1) 438 # Replace the ${PV} with the real PV 439 if pv_re.search(needed_file) and f in self.tinfoil.cooker_data.pkg_pepvpr: 440 pv = self.tinfoil.cooker_data.pkg_pepvpr[f][1] 441 needed_file = re.sub(r"\${PV}", pv, needed_file) 442 self.print_cross_files(bbpath, keyword, layername, f, needed_file, args.filenames, ignore_layers) 443 line = fnfile.readline() 444 445 # The "require/include xxx" in conf/machine/*.conf, .inc and .bbclass 446 conf_re = re.compile(r".*/conf/machine/[^\/]*\.conf$") 447 inc_re = re.compile(r".*\.inc$") 448 # The "inherit xxx" in .bbclass 449 bbclass_re = re.compile(r".*\.bbclass$") 450 for layerdir in self.bblayers: 451 layername = self.get_layer_name(layerdir) 452 for dirpath, dirnames, filenames in os.walk(layerdir): 453 for name in filenames: 454 f = os.path.join(dirpath, name) 455 s = conf_re.match(f) or inc_re.match(f) or bbclass_re.match(f) 456 if s: 457 with open(f, 'r') as ffile: 458 line = ffile.readline() 459 while line: 460 m, keyword = self.match_require_include(line) 461 # Only bbclass has the "inherit xxx" here. 462 bbclass="" 463 if not m and f.endswith(".bbclass"): 464 m, keyword = self.match_inherit(line) 465 bbclass=".bbclass" 466 # Find a 'require/include xxxx' 467 if m: 468 self.print_cross_files(bbpath, keyword, layername, f, m.group(1) + bbclass, args.filenames, ignore_layers) 469 line = ffile.readline() 470 471 def print_cross_files(self, bbpath, keyword, layername, f, needed_filename, show_filenames, ignore_layers): 472 """Print the depends that crosses a layer boundary""" 473 needed_file = bb.utils.which(bbpath, needed_filename) 474 if needed_file: 475 # Which layer is this file from 476 needed_layername = self.get_file_layer(needed_file) 477 if needed_layername != layername and not needed_layername in ignore_layers: 478 if not show_filenames: 479 f = self.remove_layer_prefix(f) 480 needed_file = self.remove_layer_prefix(needed_file) 481 logger.plain("%s %s %s" %(f, keyword, needed_file)) 482 483 def match_inherit(self, line): 484 """Match the inherit xxx line""" 485 return (self.inherit_re.match(line), "inherits") 486 487 def match_require_include(self, line): 488 """Match the require/include xxx line""" 489 m = self.require_re.match(line) 490 keyword = "requires" 491 if not m: 492 m = self.include_re.match(line) 493 keyword = "includes" 494 return (m, keyword) 495 496 def check_cross_depends(self, keyword, layername, f, needed_file, show_filenames, ignore_layers): 497 """Print the DEPENDS/RDEPENDS file that crosses a layer boundary""" 498 best_realfn = bb.cache.virtualfn2realfn(needed_file)[0] 499 needed_layername = self.get_file_layer(best_realfn) 500 if needed_layername != layername and not needed_layername in ignore_layers: 501 if not show_filenames: 502 f = self.remove_layer_prefix(f) 503 best_realfn = self.remove_layer_prefix(best_realfn) 504 505 logger.plain("%s %s %s" % (f, keyword, best_realfn)) 506 507 def register_commands(self, sp): 508 self.add_command(sp, 'show-layers', self.do_show_layers, parserecipes=False) 509 510 parser_show_overlayed = self.add_command(sp, 'show-overlayed', self.do_show_overlayed) 511 parser_show_overlayed.add_argument('-f', '--filenames', help='instead of the default formatting, list filenames of higher priority recipes with the ones they overlay indented underneath', action='store_true') 512 parser_show_overlayed.add_argument('-s', '--same-version', help='only list overlayed recipes where the version is the same', action='store_true') 513 parser_show_overlayed.add_argument('--mc', help='use specified multiconfig', default='') 514 515 parser_show_recipes = self.add_command(sp, 'show-recipes', self.do_show_recipes) 516 parser_show_recipes.add_argument('-f', '--filenames', help='instead of the default formatting, list filenames of higher priority recipes with the ones they overlay indented underneath', action='store_true') 517 parser_show_recipes.add_argument('-r', '--recipes-only', help='instead of the default formatting, list recipes only', action='store_true') 518 parser_show_recipes.add_argument('-m', '--multiple', help='only list where multiple recipes (in the same layer or different layers) exist for the same recipe name', action='store_true') 519 parser_show_recipes.add_argument('-i', '--inherits', help='only list recipes that inherit the named class(es) - separate multiple classes using , (without spaces)', metavar='CLASS', default='') 520 parser_show_recipes.add_argument('-l', '--layer', help='only list recipes from the selected layer', default='') 521 parser_show_recipes.add_argument('-b', '--bare', help='output just names without the "(skipped)" marker', action='store_true') 522 parser_show_recipes.add_argument('--mc', help='use specified multiconfig', default='') 523 parser_show_recipes.add_argument('pnspec', nargs='*', help='optional recipe name specification (wildcards allowed, enclose in quotes to avoid shell expansion)') 524 525 parser_show_appends = self.add_command(sp, 'show-appends', self.do_show_appends) 526 parser_show_appends.add_argument('pnspec', nargs='*', help='optional recipe name specification (wildcards allowed, enclose in quotes to avoid shell expansion)') 527 528 parser_show_cross_depends = self.add_command(sp, 'show-cross-depends', self.do_show_cross_depends) 529 parser_show_cross_depends.add_argument('-f', '--filenames', help='show full file path', action='store_true') 530 parser_show_cross_depends.add_argument('-i', '--ignore', help='ignore dependencies on items in the specified layer(s) (split multiple layer names with commas, no spaces)', metavar='LAYERNAME') 531