1# Utility functions for reading and modifying recipes 2# 3# Some code borrowed from the OE layer index 4# 5# Copyright (C) 2013-2017 Intel Corporation 6# 7# SPDX-License-Identifier: GPL-2.0-only 8# 9 10import sys 11import os 12import os.path 13import tempfile 14import textwrap 15import difflib 16from . import utils 17import shutil 18import re 19import fnmatch 20import glob 21import bb.tinfoil 22 23from collections import OrderedDict, defaultdict 24from bb.utils import vercmp_string 25 26# Help us to find places to insert values 27recipe_progression = ['SUMMARY', 'DESCRIPTION', 'AUTHOR', 'HOMEPAGE', 'BUGTRACKER', 'SECTION', 'LICENSE', 'LICENSE_FLAGS', 'LIC_FILES_CHKSUM', 'PROVIDES', 'DEPENDS', 'PR', 'PV', 'SRCREV', 'SRCPV', 'SRC_URI', 'S', 'do_fetch()', 'do_unpack()', 'do_patch()', 'EXTRA_OECONF', 'EXTRA_OECMAKE', 'EXTRA_OESCONS', 'do_configure()', 'EXTRA_OEMAKE', 'do_compile()', 'do_install()', 'do_populate_sysroot()', 'INITSCRIPT', 'USERADD', 'GROUPADD', 'PACKAGES', 'FILES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RPROVIDES', 'RREPLACES', 'RCONFLICTS', 'ALLOW_EMPTY', 'populate_packages()', 'do_package()', 'do_deploy()', 'BBCLASSEXTEND'] 28# Variables that sometimes are a bit long but shouldn't be wrapped 29nowrap_vars = ['SUMMARY', 'HOMEPAGE', 'BUGTRACKER', r'SRC_URI\[(.+\.)?md5sum\]', r'SRC_URI\[(.+\.)?sha256sum\]'] 30list_vars = ['SRC_URI', 'LIC_FILES_CHKSUM'] 31meta_vars = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION'] 32 33 34def simplify_history(history, d): 35 """ 36 Eliminate any irrelevant events from a variable history 37 """ 38 ret_history = [] 39 has_set = False 40 # Go backwards through the history and remove any immediate operations 41 # before the most recent set 42 for event in reversed(history): 43 if 'flag' in event or not 'file' in event: 44 continue 45 if event['op'] == 'set': 46 if has_set: 47 continue 48 has_set = True 49 elif event['op'] in ('append', 'prepend', 'postdot', 'predot'): 50 # Reminder: "append" and "prepend" mean += and =+ respectively, NOT :append / :prepend 51 if has_set: 52 continue 53 ret_history.insert(0, event) 54 return ret_history 55 56 57def get_var_files(fn, varlist, d): 58 """Find the file in which each of a list of variables is set. 59 Note: requires variable history to be enabled when parsing. 60 """ 61 varfiles = {} 62 for v in varlist: 63 files = [] 64 if '[' in v: 65 varsplit = v.split('[') 66 varflag = varsplit[1].split(']')[0] 67 history = d.varhistory.variable(varsplit[0]) 68 for event in history: 69 if 'file' in event and event.get('flag', '') == varflag: 70 files.append(event['file']) 71 else: 72 history = d.varhistory.variable(v) 73 for event in history: 74 if 'file' in event and not 'flag' in event: 75 files.append(event['file']) 76 if files: 77 actualfile = files[-1] 78 else: 79 actualfile = None 80 varfiles[v] = actualfile 81 82 return varfiles 83 84 85def split_var_value(value, assignment=True): 86 """ 87 Split a space-separated variable's value into a list of items, 88 taking into account that some of the items might be made up of 89 expressions containing spaces that should not be split. 90 Parameters: 91 value: 92 The string value to split 93 assignment: 94 True to assume that the value represents an assignment 95 statement, False otherwise. If True, and an assignment 96 statement is passed in the first item in 97 the returned list will be the part of the assignment 98 statement up to and including the opening quote character, 99 and the last item will be the closing quote. 100 """ 101 inexpr = 0 102 lastchar = None 103 out = [] 104 buf = '' 105 for char in value: 106 if char == '{': 107 if lastchar == '$': 108 inexpr += 1 109 elif char == '}': 110 inexpr -= 1 111 elif assignment and char in '"\'' and inexpr == 0: 112 if buf: 113 out.append(buf) 114 out.append(char) 115 char = '' 116 buf = '' 117 elif char.isspace() and inexpr == 0: 118 char = '' 119 if buf: 120 out.append(buf) 121 buf = '' 122 buf += char 123 lastchar = char 124 if buf: 125 out.append(buf) 126 127 # Join together assignment statement and opening quote 128 outlist = out 129 if assignment: 130 assigfound = False 131 for idx, item in enumerate(out): 132 if '=' in item: 133 assigfound = True 134 if assigfound: 135 if '"' in item or "'" in item: 136 outlist = [' '.join(out[:idx+1])] 137 outlist.extend(out[idx+1:]) 138 break 139 return outlist 140 141 142def patch_recipe_lines(fromlines, values, trailing_newline=True): 143 """Update or insert variable values into lines from a recipe. 144 Note that some manual inspection/intervention may be required 145 since this cannot handle all situations. 146 """ 147 148 import bb.utils 149 150 if trailing_newline: 151 newline = '\n' 152 else: 153 newline = '' 154 155 nowrap_vars_res = [] 156 for item in nowrap_vars: 157 nowrap_vars_res.append(re.compile('^%s$' % item)) 158 159 recipe_progression_res = [] 160 recipe_progression_restrs = [] 161 for item in recipe_progression: 162 if item.endswith('()'): 163 key = item[:-2] 164 else: 165 key = item 166 restr = r'%s(_[a-zA-Z0-9-_$(){}]+|\[[^\]]*\])?' % key 167 if item.endswith('()'): 168 recipe_progression_restrs.append(restr + '()') 169 else: 170 recipe_progression_restrs.append(restr) 171 recipe_progression_res.append(re.compile('^%s$' % restr)) 172 173 def get_recipe_pos(variable): 174 for i, p in enumerate(recipe_progression_res): 175 if p.match(variable): 176 return i 177 return -1 178 179 remainingnames = {} 180 for k in values.keys(): 181 remainingnames[k] = get_recipe_pos(k) 182 remainingnames = OrderedDict(sorted(remainingnames.items(), key=lambda x: x[1])) 183 184 modifying = False 185 186 def outputvalue(name, lines, rewindcomments=False): 187 if values[name] is None: 188 return 189 if isinstance(values[name], tuple): 190 op, value = values[name] 191 if op == '+=' and value.strip() == '': 192 return 193 else: 194 value = values[name] 195 op = '=' 196 rawtext = '%s %s "%s"%s' % (name, op, value, newline) 197 addlines = [] 198 nowrap = False 199 for nowrap_re in nowrap_vars_res: 200 if nowrap_re.match(name): 201 nowrap = True 202 break 203 if nowrap: 204 addlines.append(rawtext) 205 elif name in list_vars: 206 splitvalue = split_var_value(value, assignment=False) 207 if len(splitvalue) > 1: 208 linesplit = ' \\\n' + (' ' * (len(name) + 4)) 209 addlines.append('%s %s "%s%s"%s' % (name, op, linesplit.join(splitvalue), linesplit, newline)) 210 else: 211 addlines.append(rawtext) 212 else: 213 wrapped = textwrap.wrap(rawtext) 214 for wrapline in wrapped[:-1]: 215 addlines.append('%s \\%s' % (wrapline, newline)) 216 addlines.append('%s%s' % (wrapped[-1], newline)) 217 218 # Split on newlines - this isn't strictly necessary if you are only 219 # going to write the output to disk, but if you want to compare it 220 # (as patch_recipe_file() will do if patch=True) then it's important. 221 addlines = [line for l in addlines for line in l.splitlines(True)] 222 if rewindcomments: 223 # Ensure we insert the lines before any leading comments 224 # (that we'd want to ensure remain leading the next value) 225 for i, ln in reversed(list(enumerate(lines))): 226 if not ln.startswith('#'): 227 lines[i+1:i+1] = addlines 228 break 229 else: 230 lines.extend(addlines) 231 else: 232 lines.extend(addlines) 233 234 existingnames = [] 235 def patch_recipe_varfunc(varname, origvalue, op, newlines): 236 if modifying: 237 # Insert anything that should come before this variable 238 pos = get_recipe_pos(varname) 239 for k in list(remainingnames): 240 if remainingnames[k] > -1 and pos >= remainingnames[k] and not k in existingnames: 241 outputvalue(k, newlines, rewindcomments=True) 242 del remainingnames[k] 243 # Now change this variable, if it needs to be changed 244 if varname in existingnames and op in ['+=', '=', '=+']: 245 if varname in remainingnames: 246 outputvalue(varname, newlines) 247 del remainingnames[varname] 248 return None, None, 0, True 249 else: 250 if varname in values: 251 existingnames.append(varname) 252 return origvalue, None, 0, True 253 254 # First run - establish which values we want to set are already in the file 255 varlist = [re.escape(item) for item in values.keys()] 256 bb.utils.edit_metadata(fromlines, varlist, patch_recipe_varfunc) 257 # Second run - actually set everything 258 modifying = True 259 varlist.extend(recipe_progression_restrs) 260 changed, tolines = bb.utils.edit_metadata(fromlines, varlist, patch_recipe_varfunc, match_overrides=True) 261 262 if remainingnames: 263 if tolines and tolines[-1].strip() != '': 264 tolines.append('\n') 265 for k in remainingnames.keys(): 266 outputvalue(k, tolines) 267 268 return changed, tolines 269 270 271def patch_recipe_file(fn, values, patch=False, relpath='', redirect_output=None): 272 """Update or insert variable values into a recipe file (assuming you 273 have already identified the exact file you want to update.) 274 Note that some manual inspection/intervention may be required 275 since this cannot handle all situations. 276 """ 277 278 with open(fn, 'r') as f: 279 fromlines = f.readlines() 280 281 _, tolines = patch_recipe_lines(fromlines, values) 282 283 if redirect_output: 284 with open(os.path.join(redirect_output, os.path.basename(fn)), 'w') as f: 285 f.writelines(tolines) 286 return None 287 elif patch: 288 relfn = os.path.relpath(fn, relpath) 289 diff = difflib.unified_diff(fromlines, tolines, 'a/%s' % relfn, 'b/%s' % relfn) 290 return diff 291 else: 292 with open(fn, 'w') as f: 293 f.writelines(tolines) 294 return None 295 296 297def localise_file_vars(fn, varfiles, varlist): 298 """Given a list of variables and variable history (fetched with get_var_files()) 299 find where each variable should be set/changed. This handles for example where a 300 recipe includes an inc file where variables might be changed - in most cases 301 we want to update the inc file when changing the variable value rather than adding 302 it to the recipe itself. 303 """ 304 fndir = os.path.dirname(fn) + os.sep 305 306 first_meta_file = None 307 for v in meta_vars: 308 f = varfiles.get(v, None) 309 if f: 310 actualdir = os.path.dirname(f) + os.sep 311 if actualdir.startswith(fndir): 312 first_meta_file = f 313 break 314 315 filevars = defaultdict(list) 316 for v in varlist: 317 f = varfiles[v] 318 # Only return files that are in the same directory as the recipe or in some directory below there 319 # (this excludes bbclass files and common inc files that wouldn't be appropriate to set the variable 320 # in if we were going to set a value specific to this recipe) 321 if f: 322 actualfile = f 323 else: 324 # Variable isn't in a file, if it's one of the "meta" vars, use the first file with a meta var in it 325 if first_meta_file: 326 actualfile = first_meta_file 327 else: 328 actualfile = fn 329 330 actualdir = os.path.dirname(actualfile) + os.sep 331 if not actualdir.startswith(fndir): 332 actualfile = fn 333 filevars[actualfile].append(v) 334 335 return filevars 336 337def patch_recipe(d, fn, varvalues, patch=False, relpath='', redirect_output=None): 338 """Modify a list of variable values in the specified recipe. Handles inc files if 339 used by the recipe. 340 """ 341 overrides = d.getVar('OVERRIDES').split(':') 342 def override_applicable(hevent): 343 op = hevent['op'] 344 if '[' in op: 345 opoverrides = op.split('[')[1].split(']')[0].split(':') 346 for opoverride in opoverrides: 347 if not opoverride in overrides: 348 return False 349 return True 350 351 varlist = varvalues.keys() 352 fn = os.path.abspath(fn) 353 varfiles = get_var_files(fn, varlist, d) 354 locs = localise_file_vars(fn, varfiles, varlist) 355 patches = [] 356 for f,v in locs.items(): 357 vals = {k: varvalues[k] for k in v} 358 f = os.path.abspath(f) 359 if f == fn: 360 extravals = {} 361 for var, value in vals.items(): 362 if var in list_vars: 363 history = simplify_history(d.varhistory.variable(var), d) 364 recipe_set = False 365 for event in history: 366 if os.path.abspath(event['file']) == fn: 367 if event['op'] == 'set': 368 recipe_set = True 369 if not recipe_set: 370 for event in history: 371 if event['op'].startswith(':remove'): 372 continue 373 if not override_applicable(event): 374 continue 375 newvalue = value.replace(event['detail'], '') 376 if newvalue == value and os.path.abspath(event['file']) == fn and event['op'].startswith(':'): 377 op = event['op'].replace('[', ':').replace(']', '') 378 extravals[var + op] = None 379 value = newvalue 380 vals[var] = ('+=', value) 381 vals.update(extravals) 382 patchdata = patch_recipe_file(f, vals, patch, relpath, redirect_output) 383 if patch: 384 patches.append(patchdata) 385 386 if patch: 387 return patches 388 else: 389 return None 390 391 392 393def copy_recipe_files(d, tgt_dir, whole_dir=False, download=True, all_variants=False): 394 """Copy (local) recipe files, including both files included via include/require, 395 and files referred to in the SRC_URI variable.""" 396 import bb.fetch2 397 import oe.path 398 399 # FIXME need a warning if the unexpanded SRC_URI value contains variable references 400 401 uri_values = [] 402 localpaths = [] 403 def fetch_urls(rdata): 404 # Collect the local paths from SRC_URI 405 srcuri = rdata.getVar('SRC_URI') or "" 406 if srcuri not in uri_values: 407 fetch = bb.fetch2.Fetch(srcuri.split(), rdata) 408 if download: 409 fetch.download() 410 for pth in fetch.localpaths(): 411 if pth not in localpaths: 412 localpaths.append(os.path.abspath(pth)) 413 uri_values.append(srcuri) 414 415 fetch_urls(d) 416 if all_variants: 417 # Get files for other variants e.g. in the case of a SRC_URI:append 418 localdata = bb.data.createCopy(d) 419 variants = (localdata.getVar('BBCLASSEXTEND') or '').split() 420 if variants: 421 # Ensure we handle class-target if we're dealing with one of the variants 422 variants.append('target') 423 for variant in variants: 424 if variant.startswith("devupstream"): 425 localdata.setVar('SRCPV', 'git') 426 localdata.setVar('CLASSOVERRIDE', 'class-%s' % variant) 427 fetch_urls(localdata) 428 429 # Copy local files to target directory and gather any remote files 430 bb_dir = os.path.abspath(os.path.dirname(d.getVar('FILE'))) + os.sep 431 remotes = [] 432 copied = [] 433 # Need to do this in two steps since we want to check against the absolute path 434 includes = [os.path.abspath(path) for path in d.getVar('BBINCLUDED').split() if os.path.exists(path)] 435 # We also check this below, but we don't want any items in this list being considered remotes 436 includes = [path for path in includes if path.startswith(bb_dir)] 437 for path in localpaths + includes: 438 # Only import files that are under the meta directory 439 if path.startswith(bb_dir): 440 if not whole_dir: 441 relpath = os.path.relpath(path, bb_dir) 442 subdir = os.path.join(tgt_dir, os.path.dirname(relpath)) 443 if not os.path.exists(subdir): 444 os.makedirs(subdir) 445 shutil.copy2(path, os.path.join(tgt_dir, relpath)) 446 copied.append(relpath) 447 else: 448 remotes.append(path) 449 # Simply copy whole meta dir, if requested 450 if whole_dir: 451 shutil.copytree(bb_dir, tgt_dir) 452 453 return copied, remotes 454 455 456def get_recipe_local_files(d, patches=False, archives=False): 457 """Get a list of local files in SRC_URI within a recipe.""" 458 import oe.patch 459 uris = (d.getVar('SRC_URI') or "").split() 460 fetch = bb.fetch2.Fetch(uris, d) 461 # FIXME this list should be factored out somewhere else (such as the 462 # fetcher) though note that this only encompasses actual container formats 463 # i.e. that can contain multiple files as opposed to those that only 464 # contain a compressed stream (i.e. .tar.gz as opposed to just .gz) 465 archive_exts = ['.tar', '.tgz', '.tar.gz', '.tar.Z', '.tbz', '.tbz2', '.tar.bz2', '.txz', '.tar.xz', '.tar.lz', '.zip', '.jar', '.rpm', '.srpm', '.deb', '.ipk', '.tar.7z', '.7z'] 466 ret = {} 467 for uri in uris: 468 if fetch.ud[uri].type == 'file': 469 if (not patches and 470 oe.patch.patch_path(uri, fetch, '', expand=False)): 471 continue 472 # Skip files that are referenced by absolute path 473 fname = fetch.ud[uri].basepath 474 if os.path.isabs(fname): 475 continue 476 # Handle subdir= 477 subdir = fetch.ud[uri].parm.get('subdir', '') 478 if subdir: 479 if os.path.isabs(subdir): 480 continue 481 fname = os.path.join(subdir, fname) 482 localpath = fetch.localpath(uri) 483 if not archives: 484 # Ignore archives that will be unpacked 485 if localpath.endswith(tuple(archive_exts)): 486 unpack = fetch.ud[uri].parm.get('unpack', True) 487 if unpack: 488 continue 489 if os.path.isdir(localpath): 490 for root, dirs, files in os.walk(localpath): 491 for fname in files: 492 fileabspath = os.path.join(root,fname) 493 srcdir = os.path.dirname(localpath) 494 ret[os.path.relpath(fileabspath,srcdir)] = fileabspath 495 else: 496 ret[fname] = localpath 497 return ret 498 499 500def get_recipe_patches(d): 501 """Get a list of the patches included in SRC_URI within a recipe.""" 502 import oe.patch 503 patches = oe.patch.src_patches(d, expand=False) 504 patchfiles = [] 505 for patch in patches: 506 _, _, local, _, _, parm = bb.fetch.decodeurl(patch) 507 patchfiles.append(local) 508 return patchfiles 509 510 511def get_recipe_patched_files(d): 512 """ 513 Get the list of patches for a recipe along with the files each patch modifies. 514 Params: 515 d: the datastore for the recipe 516 Returns: 517 a dict mapping patch file path to a list of tuples of changed files and 518 change mode ('A' for add, 'D' for delete or 'M' for modify) 519 """ 520 import oe.patch 521 patches = oe.patch.src_patches(d, expand=False) 522 patchedfiles = {} 523 for patch in patches: 524 _, _, patchfile, _, _, parm = bb.fetch.decodeurl(patch) 525 striplevel = int(parm['striplevel']) 526 patchedfiles[patchfile] = oe.patch.PatchSet.getPatchedFiles(patchfile, striplevel, os.path.join(d.getVar('S'), parm.get('patchdir', ''))) 527 return patchedfiles 528 529 530def validate_pn(pn): 531 """Perform validation on a recipe name (PN) for a new recipe.""" 532 reserved_names = ['forcevariable', 'append', 'prepend', 'remove'] 533 if not re.match('^[0-9a-z-.+]+$', pn): 534 return 'Recipe name "%s" is invalid: only characters 0-9, a-z, -, + and . are allowed' % pn 535 elif pn in reserved_names: 536 return 'Recipe name "%s" is invalid: is a reserved keyword' % pn 537 elif pn.startswith('pn-'): 538 return 'Recipe name "%s" is invalid: names starting with "pn-" are reserved' % pn 539 elif pn.endswith(('.bb', '.bbappend', '.bbclass', '.inc', '.conf')): 540 return 'Recipe name "%s" is invalid: should be just a name, not a file name' % pn 541 return '' 542 543 544def get_bbfile_path(d, destdir, extrapathhint=None): 545 """ 546 Determine the correct path for a recipe within a layer 547 Parameters: 548 d: Recipe-specific datastore 549 destdir: destination directory. Can be the path to the base of the layer or a 550 partial path somewhere within the layer. 551 extrapathhint: a path relative to the base of the layer to try 552 """ 553 import bb.cookerdata 554 555 destdir = os.path.abspath(destdir) 556 destlayerdir = find_layerdir(destdir) 557 558 # Parse the specified layer's layer.conf file directly, in case the layer isn't in bblayers.conf 559 confdata = d.createCopy() 560 confdata.setVar('BBFILES', '') 561 confdata.setVar('LAYERDIR', destlayerdir) 562 destlayerconf = os.path.join(destlayerdir, "conf", "layer.conf") 563 confdata = bb.cookerdata.parse_config_file(destlayerconf, confdata) 564 pn = d.getVar('PN') 565 566 # Parse BBFILES_DYNAMIC and append to BBFILES 567 bbfiles_dynamic = (confdata.getVar('BBFILES_DYNAMIC') or "").split() 568 collections = (confdata.getVar('BBFILE_COLLECTIONS') or "").split() 569 invalid = [] 570 for entry in bbfiles_dynamic: 571 parts = entry.split(":", 1) 572 if len(parts) != 2: 573 invalid.append(entry) 574 continue 575 l, f = parts 576 invert = l[0] == "!" 577 if invert: 578 l = l[1:] 579 if (l in collections and not invert) or (l not in collections and invert): 580 confdata.appendVar("BBFILES", " " + f) 581 if invalid: 582 return None 583 bbfilespecs = (confdata.getVar('BBFILES') or '').split() 584 if destdir == destlayerdir: 585 for bbfilespec in bbfilespecs: 586 if not bbfilespec.endswith('.bbappend'): 587 for match in glob.glob(bbfilespec): 588 splitext = os.path.splitext(os.path.basename(match)) 589 if splitext[1] == '.bb': 590 mpn = splitext[0].split('_')[0] 591 if mpn == pn: 592 return os.path.dirname(match) 593 594 # Try to make up a path that matches BBFILES 595 # this is a little crude, but better than nothing 596 bpn = d.getVar('BPN') 597 recipefn = os.path.basename(d.getVar('FILE')) 598 pathoptions = [destdir] 599 if extrapathhint: 600 pathoptions.append(os.path.join(destdir, extrapathhint)) 601 if destdir == destlayerdir: 602 pathoptions.append(os.path.join(destdir, 'recipes-%s' % bpn, bpn)) 603 pathoptions.append(os.path.join(destdir, 'recipes', bpn)) 604 pathoptions.append(os.path.join(destdir, bpn)) 605 elif not destdir.endswith(('/' + pn, '/' + bpn)): 606 pathoptions.append(os.path.join(destdir, bpn)) 607 closepath = '' 608 for pathoption in pathoptions: 609 bbfilepath = os.path.join(pathoption, 'test.bb') 610 for bbfilespec in bbfilespecs: 611 if fnmatch.fnmatchcase(bbfilepath, bbfilespec): 612 return pathoption 613 return None 614 615def get_bbappend_path(d, destlayerdir, wildcardver=False): 616 """Determine how a bbappend for a recipe should be named and located within another layer""" 617 618 import bb.cookerdata 619 620 destlayerdir = os.path.abspath(destlayerdir) 621 recipefile = d.getVar('FILE') 622 recipefn = os.path.splitext(os.path.basename(recipefile))[0] 623 if wildcardver and '_' in recipefn: 624 recipefn = recipefn.split('_', 1)[0] + '_%' 625 appendfn = recipefn + '.bbappend' 626 627 # Parse the specified layer's layer.conf file directly, in case the layer isn't in bblayers.conf 628 confdata = d.createCopy() 629 confdata.setVar('BBFILES', '') 630 confdata.setVar('LAYERDIR', destlayerdir) 631 destlayerconf = os.path.join(destlayerdir, "conf", "layer.conf") 632 confdata = bb.cookerdata.parse_config_file(destlayerconf, confdata) 633 634 origlayerdir = find_layerdir(recipefile) 635 if not origlayerdir: 636 return (None, False) 637 # Now join this to the path where the bbappend is going and check if it is covered by BBFILES 638 appendpath = os.path.join(destlayerdir, os.path.relpath(os.path.dirname(recipefile), origlayerdir), appendfn) 639 closepath = '' 640 pathok = True 641 for bbfilespec in confdata.getVar('BBFILES').split(): 642 if fnmatch.fnmatchcase(appendpath, bbfilespec): 643 # Our append path works, we're done 644 break 645 elif bbfilespec.startswith(destlayerdir) and fnmatch.fnmatchcase('test.bbappend', os.path.basename(bbfilespec)): 646 # Try to find the longest matching path 647 if len(bbfilespec) > len(closepath): 648 closepath = bbfilespec 649 else: 650 # Unfortunately the bbappend layer and the original recipe's layer don't have the same structure 651 if closepath: 652 # bbappend layer's layer.conf at least has a spec that picks up .bbappend files 653 # Now we just need to substitute out any wildcards 654 appendsubdir = os.path.relpath(os.path.dirname(closepath), destlayerdir) 655 if 'recipes-*' in appendsubdir: 656 # Try to copy this part from the original recipe path 657 res = re.search('/recipes-[^/]+/', recipefile) 658 if res: 659 appendsubdir = appendsubdir.replace('/recipes-*/', res.group(0)) 660 # This is crude, but we have to do something 661 appendsubdir = appendsubdir.replace('*', recipefn.split('_')[0]) 662 appendsubdir = appendsubdir.replace('?', 'a') 663 appendpath = os.path.join(destlayerdir, appendsubdir, appendfn) 664 else: 665 pathok = False 666 return (appendpath, pathok) 667 668 669def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, machine=None, extralines=None, removevalues=None, redirect_output=None, params=None): 670 """ 671 Writes a bbappend file for a recipe 672 Parameters: 673 rd: data dictionary for the recipe 674 destlayerdir: base directory of the layer to place the bbappend in 675 (subdirectory path from there will be determined automatically) 676 srcfiles: dict of source files to add to SRC_URI, where the value 677 is the full path to the file to be added, and the value is the 678 original filename as it would appear in SRC_URI or None if it 679 isn't already present. You may pass None for this parameter if 680 you simply want to specify your own content via the extralines 681 parameter. 682 install: dict mapping entries in srcfiles to a tuple of two elements: 683 install path (*without* ${D} prefix) and permission value (as a 684 string, e.g. '0644'). 685 wildcardver: True to use a % wildcard in the bbappend filename, or 686 False to make the bbappend specific to the recipe version. 687 machine: 688 If specified, make the changes in the bbappend specific to this 689 machine. This will also cause PACKAGE_ARCH = "${MACHINE_ARCH}" 690 to be added to the bbappend. 691 extralines: 692 Extra lines to add to the bbappend. This may be a dict of name 693 value pairs, or simply a list of the lines. 694 removevalues: 695 Variable values to remove - a dict of names/values. 696 redirect_output: 697 If specified, redirects writing the output file to the 698 specified directory (for dry-run purposes) 699 params: 700 Parameters to use when adding entries to SRC_URI. If specified, 701 should be a list of dicts with the same length as srcfiles. 702 """ 703 704 if not removevalues: 705 removevalues = {} 706 707 # Determine how the bbappend should be named 708 appendpath, pathok = get_bbappend_path(rd, destlayerdir, wildcardver) 709 if not appendpath: 710 bb.error('Unable to determine layer directory containing %s' % recipefile) 711 return (None, None) 712 if not pathok: 713 bb.warn('Unable to determine correct subdirectory path for bbappend file - check that what %s adds to BBFILES also matches .bbappend files. Using %s for now, but until you fix this the bbappend will not be applied.' % (os.path.join(destlayerdir, 'conf', 'layer.conf'), os.path.dirname(appendpath))) 714 715 appenddir = os.path.dirname(appendpath) 716 if not redirect_output: 717 bb.utils.mkdirhier(appenddir) 718 719 # FIXME check if the bbappend doesn't get overridden by a higher priority layer? 720 721 layerdirs = [os.path.abspath(layerdir) for layerdir in rd.getVar('BBLAYERS').split()] 722 if not os.path.abspath(destlayerdir) in layerdirs: 723 bb.warn('Specified layer is not currently enabled in bblayers.conf, you will need to add it before this bbappend will be active') 724 725 bbappendlines = [] 726 if extralines: 727 if isinstance(extralines, dict): 728 for name, value in extralines.items(): 729 bbappendlines.append((name, '=', value)) 730 else: 731 # Do our best to split it 732 for line in extralines: 733 if line[-1] == '\n': 734 line = line[:-1] 735 splitline = line.split(None, 2) 736 if len(splitline) == 3: 737 bbappendlines.append(tuple(splitline)) 738 else: 739 raise Exception('Invalid extralines value passed') 740 741 def popline(varname): 742 for i in range(0, len(bbappendlines)): 743 if bbappendlines[i][0] == varname: 744 line = bbappendlines.pop(i) 745 return line 746 return None 747 748 def appendline(varname, op, value): 749 for i in range(0, len(bbappendlines)): 750 item = bbappendlines[i] 751 if item[0] == varname: 752 bbappendlines[i] = (item[0], item[1], item[2] + ' ' + value) 753 break 754 else: 755 bbappendlines.append((varname, op, value)) 756 757 destsubdir = rd.getVar('PN') 758 if srcfiles: 759 bbappendlines.append(('FILESEXTRAPATHS:prepend', ':=', '${THISDIR}/${PN}:')) 760 761 appendoverride = '' 762 if machine: 763 bbappendlines.append(('PACKAGE_ARCH', '=', '${MACHINE_ARCH}')) 764 appendoverride = ':%s' % machine 765 copyfiles = {} 766 if srcfiles: 767 instfunclines = [] 768 for i, (newfile, origsrcfile) in enumerate(srcfiles.items()): 769 srcfile = origsrcfile 770 srcurientry = None 771 if not srcfile: 772 srcfile = os.path.basename(newfile) 773 srcurientry = 'file://%s' % srcfile 774 if params and params[i]: 775 srcurientry = '%s;%s' % (srcurientry, ';'.join('%s=%s' % (k,v) for k,v in params[i].items())) 776 # Double-check it's not there already 777 # FIXME do we care if the entry is added by another bbappend that might go away? 778 if not srcurientry in rd.getVar('SRC_URI').split(): 779 if machine: 780 appendline('SRC_URI:append%s' % appendoverride, '=', ' ' + srcurientry) 781 else: 782 appendline('SRC_URI', '+=', srcurientry) 783 copyfiles[newfile] = srcfile 784 if install: 785 institem = install.pop(newfile, None) 786 if institem: 787 (destpath, perms) = institem 788 instdestpath = replace_dir_vars(destpath, rd) 789 instdirline = 'install -d ${D}%s' % os.path.dirname(instdestpath) 790 if not instdirline in instfunclines: 791 instfunclines.append(instdirline) 792 instfunclines.append('install -m %s ${WORKDIR}/%s ${D}%s' % (perms, os.path.basename(srcfile), instdestpath)) 793 if instfunclines: 794 bbappendlines.append(('do_install:append%s()' % appendoverride, '', instfunclines)) 795 796 if redirect_output: 797 bb.note('Writing append file %s (dry-run)' % appendpath) 798 outfile = os.path.join(redirect_output, os.path.basename(appendpath)) 799 # Only take a copy if the file isn't already there (this function may be called 800 # multiple times per operation when we're handling overrides) 801 if os.path.exists(appendpath) and not os.path.exists(outfile): 802 shutil.copy2(appendpath, outfile) 803 else: 804 bb.note('Writing append file %s' % appendpath) 805 outfile = appendpath 806 807 if os.path.exists(outfile): 808 # Work around lack of nonlocal in python 2 809 extvars = {'destsubdir': destsubdir} 810 811 def appendfile_varfunc(varname, origvalue, op, newlines): 812 if varname == 'FILESEXTRAPATHS:prepend': 813 if origvalue.startswith('${THISDIR}/'): 814 popline('FILESEXTRAPATHS:prepend') 815 extvars['destsubdir'] = rd.expand(origvalue.split('${THISDIR}/', 1)[1].rstrip(':')) 816 elif varname == 'PACKAGE_ARCH': 817 if machine: 818 popline('PACKAGE_ARCH') 819 return (machine, None, 4, False) 820 elif varname.startswith('do_install:append'): 821 func = popline(varname) 822 if func: 823 instfunclines = [line.strip() for line in origvalue.strip('\n').splitlines()] 824 for line in func[2]: 825 if not line in instfunclines: 826 instfunclines.append(line) 827 return (instfunclines, None, 4, False) 828 else: 829 splitval = split_var_value(origvalue, assignment=False) 830 changed = False 831 removevar = varname 832 if varname in ['SRC_URI', 'SRC_URI:append%s' % appendoverride]: 833 removevar = 'SRC_URI' 834 line = popline(varname) 835 if line: 836 if line[2] not in splitval: 837 splitval.append(line[2]) 838 changed = True 839 else: 840 line = popline(varname) 841 if line: 842 splitval = [line[2]] 843 changed = True 844 845 if removevar in removevalues: 846 remove = removevalues[removevar] 847 if isinstance(remove, str): 848 if remove in splitval: 849 splitval.remove(remove) 850 changed = True 851 else: 852 for removeitem in remove: 853 if removeitem in splitval: 854 splitval.remove(removeitem) 855 changed = True 856 857 if changed: 858 newvalue = splitval 859 if len(newvalue) == 1: 860 # Ensure it's written out as one line 861 if ':append' in varname: 862 newvalue = ' ' + newvalue[0] 863 else: 864 newvalue = newvalue[0] 865 if not newvalue and (op in ['+=', '.='] or ':append' in varname): 866 # There's no point appending nothing 867 newvalue = None 868 if varname.endswith('()'): 869 indent = 4 870 else: 871 indent = -1 872 return (newvalue, None, indent, True) 873 return (origvalue, None, 4, False) 874 875 varnames = [item[0] for item in bbappendlines] 876 if removevalues: 877 varnames.extend(list(removevalues.keys())) 878 879 with open(outfile, 'r') as f: 880 (updated, newlines) = bb.utils.edit_metadata(f, varnames, appendfile_varfunc) 881 882 destsubdir = extvars['destsubdir'] 883 else: 884 updated = False 885 newlines = [] 886 887 if bbappendlines: 888 for line in bbappendlines: 889 if line[0].endswith('()'): 890 newlines.append('%s {\n %s\n}\n' % (line[0], '\n '.join(line[2]))) 891 else: 892 newlines.append('%s %s "%s"\n\n' % line) 893 updated = True 894 895 if updated: 896 with open(outfile, 'w') as f: 897 f.writelines(newlines) 898 899 if copyfiles: 900 if machine: 901 destsubdir = os.path.join(destsubdir, machine) 902 if redirect_output: 903 outdir = redirect_output 904 else: 905 outdir = appenddir 906 for newfile, srcfile in copyfiles.items(): 907 filedest = os.path.join(outdir, destsubdir, os.path.basename(srcfile)) 908 if os.path.abspath(newfile) != os.path.abspath(filedest): 909 if newfile.startswith(tempfile.gettempdir()): 910 newfiledisp = os.path.basename(newfile) 911 else: 912 newfiledisp = newfile 913 if redirect_output: 914 bb.note('Copying %s to %s (dry-run)' % (newfiledisp, os.path.join(appenddir, destsubdir, os.path.basename(srcfile)))) 915 else: 916 bb.note('Copying %s to %s' % (newfiledisp, filedest)) 917 bb.utils.mkdirhier(os.path.dirname(filedest)) 918 shutil.copyfile(newfile, filedest) 919 920 return (appendpath, os.path.join(appenddir, destsubdir)) 921 922 923def find_layerdir(fn): 924 """ Figure out the path to the base of the layer containing a file (e.g. a recipe)""" 925 pth = os.path.abspath(fn) 926 layerdir = '' 927 while pth: 928 if os.path.exists(os.path.join(pth, 'conf', 'layer.conf')): 929 layerdir = pth 930 break 931 pth = os.path.dirname(pth) 932 if pth == '/': 933 return None 934 return layerdir 935 936 937def replace_dir_vars(path, d): 938 """Replace common directory paths with appropriate variable references (e.g. /etc becomes ${sysconfdir})""" 939 dirvars = {} 940 # Sort by length so we get the variables we're interested in first 941 for var in sorted(list(d.keys()), key=len): 942 if var.endswith('dir') and var.lower() == var: 943 value = d.getVar(var) 944 if value.startswith('/') and not '\n' in value and value not in dirvars: 945 dirvars[value] = var 946 for dirpath in sorted(list(dirvars.keys()), reverse=True): 947 path = path.replace(dirpath, '${%s}' % dirvars[dirpath]) 948 return path 949 950def get_recipe_pv_without_srcpv(pv, uri_type): 951 """ 952 Get PV without SRCPV common in SCM's for now only 953 support git. 954 955 Returns tuple with pv, prefix and suffix. 956 """ 957 pfx = '' 958 sfx = '' 959 960 if uri_type == 'git': 961 git_regex = re.compile(r"(?P<pfx>v?)(?P<ver>.*?)(?P<sfx>\+[^\+]*(git)?r?(AUTOINC\+))(?P<rev>.*)") 962 m = git_regex.match(pv) 963 964 if m: 965 pv = m.group('ver') 966 pfx = m.group('pfx') 967 sfx = m.group('sfx') 968 else: 969 regex = re.compile(r"(?P<pfx>(v|r)?)(?P<ver>.*)") 970 m = regex.match(pv) 971 if m: 972 pv = m.group('ver') 973 pfx = m.group('pfx') 974 975 return (pv, pfx, sfx) 976 977def get_recipe_upstream_version(rd): 978 """ 979 Get upstream version of recipe using bb.fetch2 methods with support for 980 http, https, ftp and git. 981 982 bb.fetch2 exceptions can be raised, 983 FetchError when don't have network access or upstream site don't response. 984 NoMethodError when uri latest_versionstring method isn't implemented. 985 986 Returns a dictonary with version, repository revision, current_version, type and datetime. 987 Type can be A for Automatic, M for Manual and U for Unknown. 988 """ 989 from bb.fetch2 import decodeurl 990 from datetime import datetime 991 992 ru = {} 993 ru['current_version'] = rd.getVar('PV') 994 ru['version'] = '' 995 ru['type'] = 'U' 996 ru['datetime'] = '' 997 ru['revision'] = '' 998 999 # XXX: If don't have SRC_URI means that don't have upstream sources so 1000 # returns the current recipe version, so that upstream version check 1001 # declares a match. 1002 src_uris = rd.getVar('SRC_URI') 1003 if not src_uris: 1004 ru['version'] = ru['current_version'] 1005 ru['type'] = 'M' 1006 ru['datetime'] = datetime.now() 1007 return ru 1008 1009 # XXX: we suppose that the first entry points to the upstream sources 1010 src_uri = src_uris.split()[0] 1011 uri_type, _, _, _, _, _ = decodeurl(src_uri) 1012 1013 (pv, pfx, sfx) = get_recipe_pv_without_srcpv(rd.getVar('PV'), uri_type) 1014 ru['current_version'] = pv 1015 1016 manual_upstream_version = rd.getVar("RECIPE_UPSTREAM_VERSION") 1017 if manual_upstream_version: 1018 # manual tracking of upstream version. 1019 ru['version'] = manual_upstream_version 1020 ru['type'] = 'M' 1021 1022 manual_upstream_date = rd.getVar("CHECK_DATE") 1023 if manual_upstream_date: 1024 date = datetime.strptime(manual_upstream_date, "%b %d, %Y") 1025 else: 1026 date = datetime.now() 1027 ru['datetime'] = date 1028 1029 elif uri_type == "file": 1030 # files are always up-to-date 1031 ru['version'] = pv 1032 ru['type'] = 'A' 1033 ru['datetime'] = datetime.now() 1034 else: 1035 ud = bb.fetch2.FetchData(src_uri, rd) 1036 if rd.getVar("UPSTREAM_CHECK_COMMITS") == "1": 1037 bb.fetch2.get_srcrev(rd) 1038 revision = ud.method.latest_revision(ud, rd, 'default') 1039 upversion = pv 1040 if revision != rd.getVar("SRCREV"): 1041 upversion = upversion + "-new-commits-available" 1042 else: 1043 pupver = ud.method.latest_versionstring(ud, rd) 1044 (upversion, revision) = pupver 1045 1046 if upversion: 1047 ru['version'] = upversion 1048 ru['type'] = 'A' 1049 1050 if revision: 1051 ru['revision'] = revision 1052 1053 ru['datetime'] = datetime.now() 1054 1055 return ru 1056 1057def _get_recipe_upgrade_status(data): 1058 uv = get_recipe_upstream_version(data) 1059 1060 pn = data.getVar('PN') 1061 cur_ver = uv['current_version'] 1062 1063 upstream_version_unknown = data.getVar('UPSTREAM_VERSION_UNKNOWN') 1064 if not uv['version']: 1065 status = "UNKNOWN" if upstream_version_unknown else "UNKNOWN_BROKEN" 1066 else: 1067 cmp = vercmp_string(uv['current_version'], uv['version']) 1068 if cmp == -1: 1069 status = "UPDATE" if not upstream_version_unknown else "KNOWN_BROKEN" 1070 elif cmp == 0: 1071 status = "MATCH" if not upstream_version_unknown else "KNOWN_BROKEN" 1072 else: 1073 status = "UNKNOWN" if upstream_version_unknown else "UNKNOWN_BROKEN" 1074 1075 next_ver = uv['version'] if uv['version'] else "N/A" 1076 revision = uv['revision'] if uv['revision'] else "N/A" 1077 maintainer = data.getVar('RECIPE_MAINTAINER') 1078 no_upgrade_reason = data.getVar('RECIPE_NO_UPDATE_REASON') 1079 1080 return (pn, status, cur_ver, next_ver, maintainer, revision, no_upgrade_reason) 1081 1082def get_recipe_upgrade_status(recipes=None): 1083 pkgs_list = [] 1084 data_copy_list = [] 1085 copy_vars = ('SRC_URI', 1086 'PV', 1087 'DL_DIR', 1088 'PN', 1089 'CACHE', 1090 'PERSISTENT_DIR', 1091 'BB_URI_HEADREVS', 1092 'UPSTREAM_CHECK_COMMITS', 1093 'UPSTREAM_CHECK_GITTAGREGEX', 1094 'UPSTREAM_CHECK_REGEX', 1095 'UPSTREAM_CHECK_URI', 1096 'UPSTREAM_VERSION_UNKNOWN', 1097 'RECIPE_MAINTAINER', 1098 'RECIPE_NO_UPDATE_REASON', 1099 'RECIPE_UPSTREAM_VERSION', 1100 'RECIPE_UPSTREAM_DATE', 1101 'CHECK_DATE', 1102 'FETCHCMD_bzr', 1103 'FETCHCMD_ccrc', 1104 'FETCHCMD_cvs', 1105 'FETCHCMD_git', 1106 'FETCHCMD_hg', 1107 'FETCHCMD_npm', 1108 'FETCHCMD_osc', 1109 'FETCHCMD_p4', 1110 'FETCHCMD_repo', 1111 'FETCHCMD_s3', 1112 'FETCHCMD_svn', 1113 'FETCHCMD_wget', 1114 ) 1115 1116 with bb.tinfoil.Tinfoil() as tinfoil: 1117 tinfoil.prepare(config_only=False) 1118 1119 if not recipes: 1120 recipes = tinfoil.all_recipe_files(variants=False) 1121 1122 for fn in recipes: 1123 try: 1124 if fn.startswith("/"): 1125 data = tinfoil.parse_recipe_file(fn) 1126 else: 1127 data = tinfoil.parse_recipe(fn) 1128 except bb.providers.NoProvider: 1129 bb.note(" No provider for %s" % fn) 1130 continue 1131 1132 unreliable = data.getVar('UPSTREAM_CHECK_UNRELIABLE') 1133 if unreliable == "1": 1134 bb.note(" Skip package %s as upstream check unreliable" % pn) 1135 continue 1136 1137 data_copy = bb.data.init() 1138 for var in copy_vars: 1139 data_copy.setVar(var, data.getVar(var)) 1140 for k in data: 1141 if k.startswith('SRCREV'): 1142 data_copy.setVar(k, data.getVar(k)) 1143 1144 data_copy_list.append(data_copy) 1145 1146 from concurrent.futures import ProcessPoolExecutor 1147 with ProcessPoolExecutor(max_workers=utils.cpu_count()) as executor: 1148 pkgs_list = executor.map(_get_recipe_upgrade_status, data_copy_list) 1149 1150 return pkgs_list 1151