1""" 2BitBake 'Data' implementations 3 4Functions for interacting with the data structure used by the 5BitBake build tools. 6 7The expandKeys and update_data are the most expensive 8operations. At night the cookie monster came by and 9suggested 'give me cookies on setting the variables and 10things will work out'. Taking this suggestion into account 11applying the skills from the not yet passed 'Entwurf und 12Analyse von Algorithmen' lecture and the cookie 13monster seems to be right. We will track setVar more carefully 14to have faster update_data and expandKeys operations. 15 16This is a trade-off between speed and memory again but 17the speed is more critical here. 18""" 19 20# Copyright (C) 2003, 2004 Chris Larson 21# Copyright (C) 2005 Holger Hans Peter Freyther 22# 23# SPDX-License-Identifier: GPL-2.0-only 24# 25# Based on functions from the base bb module, Copyright 2003 Holger Schurig 26 27import sys, os, re 28import hashlib 29if sys.argv[0][-5:] == "pydoc": 30 path = os.path.dirname(os.path.dirname(sys.argv[1])) 31else: 32 path = os.path.dirname(os.path.dirname(sys.argv[0])) 33sys.path.insert(0, path) 34from itertools import groupby 35 36from bb import data_smart 37from bb import codeparser 38import bb 39 40logger = data_smart.logger 41_dict_type = data_smart.DataSmart 42 43def init(): 44 """Return a new object representing the Bitbake data""" 45 return _dict_type() 46 47def init_db(parent = None): 48 """Return a new object representing the Bitbake data, 49 optionally based on an existing object""" 50 if parent is not None: 51 return parent.createCopy() 52 else: 53 return _dict_type() 54 55def createCopy(source): 56 """Link the source set to the destination 57 If one does not find the value in the destination set, 58 search will go on to the source set to get the value. 59 Value from source are copy-on-write. i.e. any try to 60 modify one of them will end up putting the modified value 61 in the destination set. 62 """ 63 return source.createCopy() 64 65def initVar(var, d): 66 """Non-destructive var init for data structure""" 67 d.initVar(var) 68 69def keys(d): 70 """Return a list of keys in d""" 71 return d.keys() 72 73 74__expand_var_regexp__ = re.compile(r"\${[^{}]+}") 75__expand_python_regexp__ = re.compile(r"\${@.+?}") 76 77def expand(s, d, varname = None): 78 """Variable expansion using the data store""" 79 return d.expand(s, varname) 80 81def expandKeys(alterdata, readdata = None): 82 if readdata is None: 83 readdata = alterdata 84 85 todolist = {} 86 for key in alterdata: 87 if not '${' in key: 88 continue 89 90 ekey = expand(key, readdata) 91 if key == ekey: 92 continue 93 todolist[key] = ekey 94 95 # These two for loops are split for performance to maximise the 96 # usefulness of the expand cache 97 for key in sorted(todolist): 98 ekey = todolist[key] 99 newval = alterdata.getVar(ekey, False) 100 if newval is not None: 101 val = alterdata.getVar(key, False) 102 if val is not None: 103 bb.warn("Variable key %s (%s) replaces original key %s (%s)." % (key, val, ekey, newval)) 104 alterdata.renameVar(key, ekey) 105 106def inheritFromOS(d, savedenv, permitted): 107 """Inherit variables from the initial environment.""" 108 exportlist = bb.utils.preserved_envvars_exported() 109 for s in savedenv.keys(): 110 if s in permitted: 111 try: 112 d.setVar(s, savedenv.getVar(s), op = 'from env') 113 if s in exportlist: 114 d.setVarFlag(s, "export", True, op = 'auto env export') 115 except TypeError: 116 pass 117 118def emit_var(var, o=sys.__stdout__, d = init(), all=False): 119 """Emit a variable to be sourced by a shell.""" 120 func = d.getVarFlag(var, "func", False) 121 if d.getVarFlag(var, 'python', False) and func: 122 return False 123 124 export = d.getVarFlag(var, "export", False) 125 unexport = d.getVarFlag(var, "unexport", False) 126 if not all and not export and not unexport and not func: 127 return False 128 129 try: 130 if all: 131 oval = d.getVar(var, False) 132 val = d.getVar(var) 133 except (KeyboardInterrupt): 134 raise 135 except Exception as exc: 136 o.write('# expansion of %s threw %s: %s\n' % (var, exc.__class__.__name__, str(exc))) 137 return False 138 139 if all: 140 d.varhistory.emit(var, oval, val, o, d) 141 142 if (var.find("-") != -1 or var.find(".") != -1 or var.find('{') != -1 or var.find('}') != -1 or var.find('+') != -1) and not all: 143 return False 144 145 varExpanded = d.expand(var) 146 147 if unexport: 148 o.write('unset %s\n' % varExpanded) 149 return False 150 151 if val is None: 152 return False 153 154 val = str(val) 155 156 if varExpanded.startswith("BASH_FUNC_"): 157 varExpanded = varExpanded[10:-2] 158 val = val[3:] # Strip off "() " 159 o.write("%s() %s\n" % (varExpanded, val)) 160 o.write("export -f %s\n" % (varExpanded)) 161 return True 162 163 if func: 164 # Write a comment indicating where the shell function came from (line number and filename) to make it easier 165 # for the user to diagnose task failures. This comment is also used by build.py to determine the metadata 166 # location of shell functions. 167 o.write("# line: {0}, file: {1}\n".format( 168 d.getVarFlag(var, "lineno", False), 169 d.getVarFlag(var, "filename", False))) 170 # NOTE: should probably check for unbalanced {} within the var 171 val = val.rstrip('\n') 172 o.write("%s() {\n%s\n}\n" % (varExpanded, val)) 173 return 1 174 175 if export: 176 o.write('export ') 177 178 # if we're going to output this within doublequotes, 179 # to a shell, we need to escape the quotes in the var 180 alter = re.sub('"', '\\"', val) 181 alter = re.sub('\n', ' \\\n', alter) 182 alter = re.sub('\\$', '\\\\$', alter) 183 o.write('%s="%s"\n' % (varExpanded, alter)) 184 return False 185 186def emit_env(o=sys.__stdout__, d = init(), all=False): 187 """Emits all items in the data store in a format such that it can be sourced by a shell.""" 188 189 isfunc = lambda key: bool(d.getVarFlag(key, "func", False)) 190 keys = sorted((key for key in d.keys() if not key.startswith("__")), key=isfunc) 191 grouped = groupby(keys, isfunc) 192 for isfunc, keys in grouped: 193 for key in sorted(keys): 194 emit_var(key, o, d, all and not isfunc) and o.write('\n') 195 196def exported_keys(d): 197 return (key for key in d.keys() if not key.startswith('__') and 198 d.getVarFlag(key, 'export', False) and 199 not d.getVarFlag(key, 'unexport', False)) 200 201def exported_vars(d): 202 k = list(exported_keys(d)) 203 for key in k: 204 try: 205 value = d.getVar(key) 206 except Exception as err: 207 bb.warn("%s: Unable to export ${%s}: %s" % (d.getVar("FILE"), key, err)) 208 continue 209 210 if value is not None: 211 yield key, str(value) 212 213def emit_func(func, o=sys.__stdout__, d = init()): 214 """Emits all items in the data store in a format such that it can be sourced by a shell.""" 215 216 keys = (key for key in d.keys() if not key.startswith("__") and not d.getVarFlag(key, "func", False)) 217 for key in sorted(keys): 218 emit_var(key, o, d, False) 219 220 o.write('\n') 221 emit_var(func, o, d, False) and o.write('\n') 222 newdeps = bb.codeparser.ShellParser(func, logger).parse_shell(d.getVar(func)) 223 newdeps |= set((d.getVarFlag(func, "vardeps") or "").split()) 224 seen = set() 225 while newdeps: 226 deps = newdeps 227 seen |= deps 228 newdeps = set() 229 for dep in sorted(deps): 230 if d.getVarFlag(dep, "func", False) and not d.getVarFlag(dep, "python", False): 231 emit_var(dep, o, d, False) and o.write('\n') 232 newdeps |= bb.codeparser.ShellParser(dep, logger).parse_shell(d.getVar(dep)) 233 newdeps |= set((d.getVarFlag(dep, "vardeps") or "").split()) 234 newdeps -= seen 235 236_functionfmt = """ 237def {function}(d): 238{body}""" 239 240def emit_func_python(func, o=sys.__stdout__, d = init()): 241 """Emits all items in the data store in a format such that it can be sourced by a shell.""" 242 243 def write_func(func, o, call = False): 244 body = d.getVar(func, False) 245 if not body.startswith("def"): 246 body = _functionfmt.format(function=func, body=body) 247 248 o.write(body.strip() + "\n\n") 249 if call: 250 o.write(func + "(d)" + "\n\n") 251 252 write_func(func, o, True) 253 pp = bb.codeparser.PythonParser(func, logger) 254 pp.parse_python(d.getVar(func, False)) 255 newdeps = pp.execs 256 newdeps |= set((d.getVarFlag(func, "vardeps") or "").split()) 257 seen = set() 258 while newdeps: 259 deps = newdeps 260 seen |= deps 261 newdeps = set() 262 for dep in deps: 263 if d.getVarFlag(dep, "func", False) and d.getVarFlag(dep, "python", False): 264 write_func(dep, o) 265 pp = bb.codeparser.PythonParser(dep, logger) 266 pp.parse_python(d.getVar(dep, False)) 267 newdeps |= pp.execs 268 newdeps |= set((d.getVarFlag(dep, "vardeps") or "").split()) 269 newdeps -= seen 270 271def update_data(d): 272 """Performs final steps upon the datastore, including application of overrides""" 273 d.finalize(parent = True) 274 275def build_dependencies(key, keys, shelldeps, varflagsexcl, ignored_vars, d): 276 deps = set() 277 try: 278 if key[-1] == ']': 279 vf = key[:-1].split('[') 280 if vf[1] == "vardepvalueexclude": 281 return deps, "" 282 value, parser = d.getVarFlag(vf[0], vf[1], False, retparser=True) 283 deps |= parser.references 284 deps = deps | (keys & parser.execs) 285 return deps, value 286 varflags = d.getVarFlags(key, ["vardeps", "vardepvalue", "vardepsexclude", "exports", "postfuncs", "prefuncs", "lineno", "filename"]) or {} 287 vardeps = varflags.get("vardeps") 288 exclusions = varflags.get("vardepsexclude", "").split() 289 290 def handle_contains(value, contains, exclusions, d): 291 newvalue = [] 292 if value: 293 newvalue.append(str(value)) 294 for k in sorted(contains): 295 if k in exclusions or k in ignored_vars: 296 continue 297 l = (d.getVar(k) or "").split() 298 for item in sorted(contains[k]): 299 for word in item.split(): 300 if not word in l: 301 newvalue.append("\n%s{%s} = Unset" % (k, item)) 302 break 303 else: 304 newvalue.append("\n%s{%s} = Set" % (k, item)) 305 return "".join(newvalue) 306 307 def handle_remove(value, deps, removes, d): 308 for r in sorted(removes): 309 r2 = d.expandWithRefs(r, None) 310 value += "\n_remove of %s" % r 311 deps |= r2.references 312 deps = deps | (keys & r2.execs) 313 return value 314 315 if "vardepvalue" in varflags: 316 value = varflags.get("vardepvalue") 317 elif varflags.get("func"): 318 if varflags.get("python"): 319 value = d.getVarFlag(key, "_content", False) 320 parser = bb.codeparser.PythonParser(key, logger) 321 parser.parse_python(value, filename=varflags.get("filename"), lineno=varflags.get("lineno")) 322 deps = deps | parser.references 323 deps = deps | (keys & parser.execs) 324 value = handle_contains(value, parser.contains, exclusions, d) 325 else: 326 value, parsedvar = d.getVarFlag(key, "_content", False, retparser=True) 327 parser = bb.codeparser.ShellParser(key, logger) 328 parser.parse_shell(parsedvar.value) 329 deps = deps | shelldeps 330 deps = deps | parsedvar.references 331 deps = deps | (keys & parser.execs) | (keys & parsedvar.execs) 332 value = handle_contains(value, parsedvar.contains, exclusions, d) 333 if hasattr(parsedvar, "removes"): 334 value = handle_remove(value, deps, parsedvar.removes, d) 335 if vardeps is None: 336 parser.log.flush() 337 if "prefuncs" in varflags: 338 deps = deps | set(varflags["prefuncs"].split()) 339 if "postfuncs" in varflags: 340 deps = deps | set(varflags["postfuncs"].split()) 341 if "exports" in varflags: 342 deps = deps | set(varflags["exports"].split()) 343 else: 344 value, parser = d.getVarFlag(key, "_content", False, retparser=True) 345 deps |= parser.references 346 deps = deps | (keys & parser.execs) 347 value = handle_contains(value, parser.contains, exclusions, d) 348 if hasattr(parser, "removes"): 349 value = handle_remove(value, deps, parser.removes, d) 350 351 if "vardepvalueexclude" in varflags: 352 exclude = varflags.get("vardepvalueexclude") 353 for excl in exclude.split('|'): 354 if excl: 355 value = value.replace(excl, '') 356 357 # Add varflags, assuming an exclusion list is set 358 if varflagsexcl: 359 varfdeps = [] 360 for f in varflags: 361 if f not in varflagsexcl: 362 varfdeps.append('%s[%s]' % (key, f)) 363 if varfdeps: 364 deps |= set(varfdeps) 365 366 deps |= set((vardeps or "").split()) 367 deps -= set(exclusions) 368 except bb.parse.SkipRecipe: 369 raise 370 except Exception as e: 371 bb.warn("Exception during build_dependencies for %s" % key) 372 raise 373 return deps, value 374 #bb.note("Variable %s references %s and calls %s" % (key, str(deps), str(execs))) 375 #d.setVarFlag(key, "vardeps", deps) 376 377def generate_dependencies(d, ignored_vars): 378 379 keys = set(key for key in d if not key.startswith("__")) 380 shelldeps = set(key for key in d.getVar("__exportlist", False) if d.getVarFlag(key, "export", False) and not d.getVarFlag(key, "unexport", False)) 381 varflagsexcl = d.getVar('BB_SIGNATURE_EXCLUDE_FLAGS') 382 383 deps = {} 384 values = {} 385 386 tasklist = d.getVar('__BBTASKS', False) or [] 387 for task in tasklist: 388 deps[task], values[task] = build_dependencies(task, keys, shelldeps, varflagsexcl, ignored_vars, d) 389 newdeps = deps[task] 390 seen = set() 391 while newdeps: 392 nextdeps = newdeps - ignored_vars 393 seen |= nextdeps 394 newdeps = set() 395 for dep in nextdeps: 396 if dep not in deps: 397 deps[dep], values[dep] = build_dependencies(dep, keys, shelldeps, varflagsexcl, ignored_vars, d) 398 newdeps |= deps[dep] 399 newdeps -= seen 400 #print "For %s: %s" % (task, str(deps[task])) 401 return tasklist, deps, values 402 403def generate_dependency_hash(tasklist, gendeps, lookupcache, ignored_vars, fn): 404 taskdeps = {} 405 basehash = {} 406 407 for task in tasklist: 408 data = lookupcache[task] 409 410 if data is None: 411 bb.error("Task %s from %s seems to be empty?!" % (task, fn)) 412 data = [] 413 else: 414 data = [data] 415 416 gendeps[task] -= ignored_vars 417 newdeps = gendeps[task] 418 seen = set() 419 while newdeps: 420 nextdeps = newdeps 421 seen |= nextdeps 422 newdeps = set() 423 for dep in nextdeps: 424 if dep in ignored_vars: 425 continue 426 gendeps[dep] -= ignored_vars 427 newdeps |= gendeps[dep] 428 newdeps -= seen 429 430 alldeps = sorted(seen) 431 for dep in alldeps: 432 data.append(dep) 433 var = lookupcache[dep] 434 if var is not None: 435 data.append(str(var)) 436 k = fn + ":" + task 437 basehash[k] = hashlib.sha256("".join(data).encode("utf-8")).hexdigest() 438 taskdeps[task] = alldeps 439 440 return taskdeps, basehash 441 442def inherits_class(klass, d): 443 val = d.getVar('__inherit_cache', False) or [] 444 needle = os.path.join('classes', '%s.bbclass' % klass) 445 for v in val: 446 if v.endswith(needle): 447 return True 448 return False 449