1# 2# BitBake 'Build' implementation 3# 4# Core code for function execution and task handling in the 5# BitBake build tools. 6# 7# Copyright (C) 2003, 2004 Chris Larson 8# 9# Based on Gentoo's portage.py. 10# 11# SPDX-License-Identifier: GPL-2.0-only 12# 13# Based on functions from the base bb module, Copyright 2003 Holger Schurig 14 15import os 16import sys 17import logging 18import glob 19import itertools 20import time 21import re 22import stat 23import bb 24import bb.msg 25import bb.process 26import bb.progress 27from bb import data, event, utils 28 29bblogger = logging.getLogger('BitBake') 30logger = logging.getLogger('BitBake.Build') 31 32verboseShellLogging = False 33verboseStdoutLogging = False 34 35__mtime_cache = {} 36 37def cached_mtime_noerror(f): 38 if f not in __mtime_cache: 39 try: 40 __mtime_cache[f] = os.stat(f)[stat.ST_MTIME] 41 except OSError: 42 return 0 43 return __mtime_cache[f] 44 45def reset_cache(): 46 global __mtime_cache 47 __mtime_cache = {} 48 49# When we execute a Python function, we'd like certain things 50# in all namespaces, hence we add them to __builtins__. 51# If we do not do this and use the exec globals, they will 52# not be available to subfunctions. 53if hasattr(__builtins__, '__setitem__'): 54 builtins = __builtins__ 55else: 56 builtins = __builtins__.__dict__ 57 58builtins['bb'] = bb 59builtins['os'] = os 60 61class TaskBase(event.Event): 62 """Base class for task events""" 63 64 def __init__(self, t, fn, logfile, d): 65 self._task = t 66 self._fn = fn 67 self._package = d.getVar("PF") 68 self._mc = d.getVar("BB_CURRENT_MC") 69 self.taskfile = d.getVar("FILE") 70 self.taskname = self._task 71 self.logfile = logfile 72 self.time = time.time() 73 self.pn = d.getVar("PN") 74 self.pv = d.getVar("PV") 75 event.Event.__init__(self) 76 self._message = "recipe %s: task %s: %s" % (d.getVar("PF"), t, self.getDisplayName()) 77 78 def getTask(self): 79 return self._task 80 81 def setTask(self, task): 82 self._task = task 83 84 def getDisplayName(self): 85 return bb.event.getName(self)[4:] 86 87 task = property(getTask, setTask, None, "task property") 88 89class TaskStarted(TaskBase): 90 """Task execution started""" 91 def __init__(self, t, fn, logfile, taskflags, d): 92 super(TaskStarted, self).__init__(t, fn, logfile, d) 93 self.taskflags = taskflags 94 95class TaskSucceeded(TaskBase): 96 """Task execution completed""" 97 98class TaskFailed(TaskBase): 99 """Task execution failed""" 100 101 def __init__(self, task, fn, logfile, metadata, errprinted = False): 102 self.errprinted = errprinted 103 super(TaskFailed, self).__init__(task, fn, logfile, metadata) 104 105class TaskFailedSilent(TaskBase): 106 """Task execution failed (silently)""" 107 def getDisplayName(self): 108 # Don't need to tell the user it was silent 109 return "Failed" 110 111class TaskInvalid(TaskBase): 112 113 def __init__(self, task, fn, metadata): 114 super(TaskInvalid, self).__init__(task, fn, None, metadata) 115 self._message = "No such task '%s'" % task 116 117class TaskProgress(event.Event): 118 """ 119 Task made some progress that could be reported to the user, usually in 120 the form of a progress bar or similar. 121 NOTE: this class does not inherit from TaskBase since it doesn't need 122 to - it's fired within the task context itself, so we don't have any of 123 the context information that you do in the case of the other events. 124 The event PID can be used to determine which task it came from. 125 The progress value is normally 0-100, but can also be negative 126 indicating that progress has been made but we aren't able to determine 127 how much. 128 The rate is optional, this is simply an extra string to display to the 129 user if specified. 130 """ 131 def __init__(self, progress, rate=None): 132 self.progress = progress 133 self.rate = rate 134 event.Event.__init__(self) 135 136 137class LogTee(object): 138 def __init__(self, logger, outfile): 139 self.outfile = outfile 140 self.logger = logger 141 self.name = self.outfile.name 142 143 def write(self, string): 144 self.logger.plain(string) 145 self.outfile.write(string) 146 147 def __enter__(self): 148 self.outfile.__enter__() 149 return self 150 151 def __exit__(self, *excinfo): 152 self.outfile.__exit__(*excinfo) 153 154 def __repr__(self): 155 return '<LogTee {0}>'.format(self.name) 156 157 def flush(self): 158 self.outfile.flush() 159 160 161class StdoutNoopContextManager: 162 """ 163 This class acts like sys.stdout, but adds noop __enter__ and __exit__ methods. 164 """ 165 def __enter__(self): 166 return sys.stdout 167 168 def __exit__(self, *exc_info): 169 pass 170 171 def write(self, string): 172 return sys.stdout.write(string) 173 174 def flush(self): 175 sys.stdout.flush() 176 177 @property 178 def name(self): 179 return sys.stdout.name 180 181 182def exec_func(func, d, dirs = None): 183 """Execute a BB 'function'""" 184 185 try: 186 oldcwd = os.getcwd() 187 except: 188 oldcwd = None 189 190 flags = d.getVarFlags(func) 191 cleandirs = flags.get('cleandirs') if flags else None 192 if cleandirs: 193 for cdir in d.expand(cleandirs).split(): 194 bb.utils.remove(cdir, True) 195 bb.utils.mkdirhier(cdir) 196 197 if flags and dirs is None: 198 dirs = flags.get('dirs') 199 if dirs: 200 dirs = d.expand(dirs).split() 201 202 if dirs: 203 for adir in dirs: 204 bb.utils.mkdirhier(adir) 205 adir = dirs[-1] 206 else: 207 adir = None 208 209 body = d.getVar(func, False) 210 if not body: 211 if body is None: 212 logger.warning("Function %s doesn't exist", func) 213 return 214 215 ispython = flags.get('python') 216 217 lockflag = flags.get('lockfiles') 218 if lockflag: 219 lockfiles = [f for f in d.expand(lockflag).split()] 220 else: 221 lockfiles = None 222 223 tempdir = d.getVar('T') 224 225 # or func allows items to be executed outside of the normal 226 # task set, such as buildhistory 227 task = d.getVar('BB_RUNTASK') or func 228 if task == func: 229 taskfunc = task 230 else: 231 taskfunc = "%s.%s" % (task, func) 232 233 runfmt = d.getVar('BB_RUNFMT') or "run.{func}.{pid}" 234 runfn = runfmt.format(taskfunc=taskfunc, task=task, func=func, pid=os.getpid()) 235 runfile = os.path.join(tempdir, runfn) 236 bb.utils.mkdirhier(os.path.dirname(runfile)) 237 238 # Setup the courtesy link to the runfn, only for tasks 239 # we create the link 'just' before the run script is created 240 # if we create it after, and if the run script fails, then the 241 # link won't be created as an exception would be fired. 242 if task == func: 243 runlink = os.path.join(tempdir, 'run.{0}'.format(task)) 244 if runlink: 245 bb.utils.remove(runlink) 246 247 try: 248 os.symlink(runfn, runlink) 249 except OSError: 250 pass 251 252 with bb.utils.fileslocked(lockfiles): 253 if ispython: 254 exec_func_python(func, d, runfile, cwd=adir) 255 else: 256 exec_func_shell(func, d, runfile, cwd=adir) 257 258 try: 259 curcwd = os.getcwd() 260 except: 261 curcwd = None 262 263 if oldcwd and curcwd != oldcwd: 264 try: 265 bb.warn("Task %s changed cwd to %s" % (func, curcwd)) 266 os.chdir(oldcwd) 267 except: 268 pass 269 270_functionfmt = """ 271{function}(d) 272""" 273logformatter = bb.msg.BBLogFormatter("%(levelname)s: %(message)s") 274def exec_func_python(func, d, runfile, cwd=None): 275 """Execute a python BB 'function'""" 276 277 code = _functionfmt.format(function=func) 278 bb.utils.mkdirhier(os.path.dirname(runfile)) 279 with open(runfile, 'w') as script: 280 bb.data.emit_func_python(func, script, d) 281 282 if cwd: 283 try: 284 olddir = os.getcwd() 285 except OSError as e: 286 bb.warn("%s: Cannot get cwd: %s" % (func, e)) 287 olddir = None 288 os.chdir(cwd) 289 290 bb.debug(2, "Executing python function %s" % func) 291 292 try: 293 text = "def %s(d):\n%s" % (func, d.getVar(func, False)) 294 fn = d.getVarFlag(func, "filename", False) 295 lineno = int(d.getVarFlag(func, "lineno", False)) 296 bb.methodpool.insert_method(func, text, fn, lineno - 1) 297 298 comp = utils.better_compile(code, func, "exec_func_python() autogenerated") 299 utils.better_exec(comp, {"d": d}, code, "exec_func_python() autogenerated") 300 finally: 301 # We want any stdout/stderr to be printed before any other log messages to make debugging 302 # more accurate. In some cases we seem to lose stdout/stderr entirely in logging tests without this. 303 sys.stdout.flush() 304 sys.stderr.flush() 305 bb.debug(2, "Python function %s finished" % func) 306 307 if cwd and olddir: 308 try: 309 os.chdir(olddir) 310 except OSError as e: 311 bb.warn("%s: Cannot restore cwd %s: %s" % (func, olddir, e)) 312 313def shell_trap_code(): 314 return '''#!/bin/sh\n 315__BITBAKE_LAST_LINE=0 316 317# Emit a useful diagnostic if something fails: 318bb_sh_exit_handler() { 319 ret=$? 320 if [ "$ret" != 0 ]; then 321 echo "WARNING: exit code $ret from a shell command." 322 fi 323 exit $ret 324} 325 326bb_bash_exit_handler() { 327 ret=$? 328 { set +x; } > /dev/null 329 trap "" DEBUG 330 if [ "$ret" != 0 ]; then 331 echo "WARNING: ${BASH_SOURCE[0]}:${__BITBAKE_LAST_LINE} exit $ret from '$1'" 332 333 echo "WARNING: Backtrace (BB generated script): " 334 for i in $(seq 1 $((${#FUNCNAME[@]} - 1))); do 335 if [ "$i" -eq 1 ]; then 336 echo -e "\t#$((i)): ${FUNCNAME[$i]}, ${BASH_SOURCE[$((i-1))]}, line ${__BITBAKE_LAST_LINE}" 337 else 338 echo -e "\t#$((i)): ${FUNCNAME[$i]}, ${BASH_SOURCE[$((i-1))]}, line ${BASH_LINENO[$((i-1))]}" 339 fi 340 done 341 fi 342 exit $ret 343} 344 345bb_bash_debug_handler() { 346 local line=${BASH_LINENO[0]} 347 # For some reason the DEBUG trap trips with lineno=1 when scripts exit; ignore it 348 if [ "$line" -eq 1 ]; then 349 return 350 fi 351 352 # Track the line number of commands as they execute. This is so we can have access to the failing line number 353 # in the EXIT trap. See http://gnu-bash.2382.n7.nabble.com/trap-echo-quot-trap-exit-on-LINENO-quot-EXIT-gt-wrong-linenumber-td3666.html 354 if [ "${FUNCNAME[1]}" != "bb_bash_exit_handler" ]; then 355 __BITBAKE_LAST_LINE=$line 356 fi 357} 358 359case $BASH_VERSION in 360"") trap 'bb_sh_exit_handler' 0 361 set -e 362 ;; 363*) trap 'bb_bash_exit_handler "$BASH_COMMAND"' 0 364 trap '{ bb_bash_debug_handler; } 2>/dev/null' DEBUG 365 set -e 366 shopt -s extdebug 367 ;; 368esac 369''' 370 371def create_progress_handler(func, progress, logfile, d): 372 if progress == 'percent': 373 # Use default regex 374 return bb.progress.BasicProgressHandler(d, outfile=logfile) 375 elif progress.startswith('percent:'): 376 # Use specified regex 377 return bb.progress.BasicProgressHandler(d, regex=progress.split(':', 1)[1], outfile=logfile) 378 elif progress.startswith('outof:'): 379 # Use specified regex 380 return bb.progress.OutOfProgressHandler(d, regex=progress.split(':', 1)[1], outfile=logfile) 381 elif progress.startswith("custom:"): 382 # Use a custom progress handler that was injected via OE_EXTRA_IMPORTS or __builtins__ 383 import functools 384 from types import ModuleType 385 386 parts = progress.split(":", 2) 387 _, cls, otherargs = parts[0], parts[1], (parts[2] or None) if parts[2:] else None 388 if cls: 389 def resolve(x, y): 390 if not x: 391 return None 392 if isinstance(x, ModuleType): 393 return getattr(x, y, None) 394 return x.get(y) 395 cls_obj = functools.reduce(resolve, cls.split("."), bb.utils._context) 396 if not cls_obj: 397 # Fall-back on __builtins__ 398 cls_obj = functools.reduce(resolve, cls.split("."), __builtins__) 399 if cls_obj: 400 return cls_obj(d, outfile=logfile, otherargs=otherargs) 401 bb.warn('%s: unknown custom progress handler in task progress varflag value "%s", ignoring' % (func, cls)) 402 else: 403 bb.warn('%s: invalid task progress varflag value "%s", ignoring' % (func, progress)) 404 405 return logfile 406 407def exec_func_shell(func, d, runfile, cwd=None): 408 """Execute a shell function from the metadata 409 410 Note on directory behavior. The 'dirs' varflag should contain a list 411 of the directories you need created prior to execution. The last 412 item in the list is where we will chdir/cd to. 413 """ 414 415 # Don't let the emitted shell script override PWD 416 d.delVarFlag('PWD', 'export') 417 418 with open(runfile, 'w') as script: 419 script.write(shell_trap_code()) 420 421 bb.data.emit_func(func, script, d) 422 423 if verboseShellLogging or bb.utils.to_boolean(d.getVar("BB_VERBOSE_LOGS", False)): 424 script.write("set -x\n") 425 if cwd: 426 script.write("cd '%s'\n" % cwd) 427 script.write("%s\n" % func) 428 script.write(''' 429# cleanup 430ret=$? 431trap '' 0 432exit $ret 433''') 434 435 os.chmod(runfile, 0o775) 436 437 cmd = runfile 438 if d.getVarFlag(func, 'fakeroot', False): 439 fakerootcmd = d.getVar('FAKEROOT') 440 if fakerootcmd: 441 cmd = [fakerootcmd, runfile] 442 443 if verboseStdoutLogging: 444 logfile = LogTee(logger, StdoutNoopContextManager()) 445 else: 446 logfile = StdoutNoopContextManager() 447 448 progress = d.getVarFlag(func, 'progress') 449 if progress: 450 try: 451 logfile = create_progress_handler(func, progress, logfile, d) 452 except: 453 from traceback import format_exc 454 logger.error("Failed to create progress handler") 455 logger.error(format_exc()) 456 raise 457 458 fifobuffer = bytearray() 459 def readfifo(data): 460 nonlocal fifobuffer 461 fifobuffer.extend(data) 462 while fifobuffer: 463 message, token, nextmsg = fifobuffer.partition(b"\00") 464 if token: 465 splitval = message.split(b' ', 1) 466 cmd = splitval[0].decode("utf-8") 467 if len(splitval) > 1: 468 value = splitval[1].decode("utf-8") 469 else: 470 value = '' 471 if cmd == 'bbplain': 472 bb.plain(value) 473 elif cmd == 'bbnote': 474 bb.note(value) 475 elif cmd == 'bbverbnote': 476 bb.verbnote(value) 477 elif cmd == 'bbwarn': 478 bb.warn(value) 479 elif cmd == 'bberror': 480 bb.error(value) 481 elif cmd == 'bbfatal': 482 # The caller will call exit themselves, so bb.error() is 483 # what we want here rather than bb.fatal() 484 bb.error(value) 485 elif cmd == 'bbfatal_log': 486 bb.error(value, forcelog=True) 487 elif cmd == 'bbdebug': 488 splitval = value.split(' ', 1) 489 level = int(splitval[0]) 490 value = splitval[1] 491 bb.debug(level, value) 492 else: 493 bb.warn("Unrecognised command '%s' on FIFO" % cmd) 494 fifobuffer = nextmsg 495 else: 496 break 497 498 tempdir = d.getVar('T') 499 fifopath = os.path.join(tempdir, 'fifo.%s' % os.getpid()) 500 if os.path.exists(fifopath): 501 os.unlink(fifopath) 502 os.mkfifo(fifopath) 503 with open(fifopath, 'r+b', buffering=0) as fifo: 504 try: 505 bb.debug(2, "Executing shell function %s" % func) 506 with open(os.devnull, 'r+') as stdin, logfile: 507 bb.process.run(cmd, shell=False, stdin=stdin, log=logfile, extrafiles=[(fifo,readfifo)]) 508 except bb.process.ExecutionError as exe: 509 # Find the backtrace that the shell trap generated 510 backtrace_marker_regex = re.compile(r"WARNING: Backtrace \(BB generated script\)") 511 stdout_lines = (exe.stdout or "").split("\n") 512 backtrace_start_line = None 513 for i, line in enumerate(reversed(stdout_lines)): 514 if backtrace_marker_regex.search(line): 515 backtrace_start_line = len(stdout_lines) - i 516 break 517 518 # Read the backtrace frames, starting at the location we just found 519 backtrace_entry_regex = re.compile(r"#(?P<frameno>\d+): (?P<funcname>[^\s]+), (?P<file>.+?), line (" 520 r"?P<lineno>\d+)") 521 backtrace_frames = [] 522 if backtrace_start_line: 523 for line in itertools.islice(stdout_lines, backtrace_start_line, None): 524 match = backtrace_entry_regex.search(line) 525 if match: 526 backtrace_frames.append(match.groupdict()) 527 528 with open(runfile, "r") as script: 529 script_lines = [line.rstrip() for line in script.readlines()] 530 531 # For each backtrace frame, search backwards in the script (from the line number called out by the frame), 532 # to find the comment that emit_vars injected when it wrote the script. This will give us the metadata 533 # filename (e.g. .bb or .bbclass) and line number where the shell function was originally defined. 534 script_metadata_comment_regex = re.compile(r"# line: (?P<lineno>\d+), file: (?P<file>.+)") 535 better_frames = [] 536 # Skip the very last frame since it's just the call to the shell task in the body of the script 537 for frame in backtrace_frames[:-1]: 538 # Check whether the frame corresponds to a function defined in the script vs external script. 539 if os.path.samefile(frame["file"], runfile): 540 # Search backwards from the frame lineno to locate the comment that BB injected 541 i = int(frame["lineno"]) - 1 542 while i >= 0: 543 match = script_metadata_comment_regex.match(script_lines[i]) 544 if match: 545 # Calculate the relative line in the function itself 546 relative_line_in_function = int(frame["lineno"]) - i - 2 547 # Calculate line in the function as declared in the metadata 548 metadata_function_line = relative_line_in_function + int(match["lineno"]) 549 better_frames.append("#{frameno}: {funcname}, {file}, line {lineno}".format( 550 frameno=frame["frameno"], 551 funcname=frame["funcname"], 552 file=match["file"], 553 lineno=metadata_function_line 554 )) 555 break 556 i -= 1 557 else: 558 better_frames.append("#{frameno}: {funcname}, {file}, line {lineno}".format(**frame)) 559 560 if better_frames: 561 better_frames = ("\t{0}".format(frame) for frame in better_frames) 562 exe.extra_message = "\nBacktrace (metadata-relative locations):\n{0}".format("\n".join(better_frames)) 563 raise 564 finally: 565 os.unlink(fifopath) 566 567 bb.debug(2, "Shell function %s finished" % func) 568 569def _task_data(fn, task, d): 570 localdata = bb.data.createCopy(d) 571 localdata.setVar('BB_FILENAME', fn) 572 localdata.setVar('OVERRIDES', 'task-%s:%s' % 573 (task[3:].replace('_', '-'), d.getVar('OVERRIDES', False))) 574 localdata.finalize() 575 bb.data.expandKeys(localdata) 576 return localdata 577 578def _exec_task(fn, task, d, quieterr): 579 """Execute a BB 'task' 580 581 Execution of a task involves a bit more setup than executing a function, 582 running it with its own local metadata, and with some useful variables set. 583 """ 584 if not d.getVarFlag(task, 'task', False): 585 event.fire(TaskInvalid(task, fn, d), d) 586 logger.error("No such task: %s" % task) 587 return 1 588 589 logger.debug("Executing task %s", task) 590 591 localdata = _task_data(fn, task, d) 592 tempdir = localdata.getVar('T') 593 if not tempdir: 594 bb.fatal("T variable not set, unable to build") 595 596 # Change nice level if we're asked to 597 nice = localdata.getVar("BB_TASK_NICE_LEVEL") 598 if nice: 599 curnice = os.nice(0) 600 nice = int(nice) - curnice 601 newnice = os.nice(nice) 602 logger.debug("Renice to %s " % newnice) 603 ionice = localdata.getVar("BB_TASK_IONICE_LEVEL") 604 if ionice: 605 try: 606 cls, prio = ionice.split(".", 1) 607 bb.utils.ioprio_set(os.getpid(), int(cls), int(prio)) 608 except: 609 bb.warn("Invalid ionice level %s" % ionice) 610 611 bb.utils.mkdirhier(tempdir) 612 613 # Determine the logfile to generate 614 logfmt = localdata.getVar('BB_LOGFMT') or 'log.{task}.{pid}' 615 logbase = logfmt.format(task=task, pid=os.getpid()) 616 617 # Document the order of the tasks... 618 logorder = os.path.join(tempdir, 'log.task_order') 619 try: 620 with open(logorder, 'a') as logorderfile: 621 logorderfile.write('{0} ({1}): {2}\n'.format(task, os.getpid(), logbase)) 622 except OSError: 623 logger.exception("Opening log file '%s'", logorder) 624 pass 625 626 # Setup the courtesy link to the logfn 627 loglink = os.path.join(tempdir, 'log.{0}'.format(task)) 628 logfn = os.path.join(tempdir, logbase) 629 if loglink: 630 bb.utils.remove(loglink) 631 632 try: 633 os.symlink(logbase, loglink) 634 except OSError: 635 pass 636 637 prefuncs = localdata.getVarFlag(task, 'prefuncs', expand=True) 638 postfuncs = localdata.getVarFlag(task, 'postfuncs', expand=True) 639 640 class ErrorCheckHandler(logging.Handler): 641 def __init__(self): 642 self.triggered = False 643 logging.Handler.__init__(self, logging.ERROR) 644 def emit(self, record): 645 if getattr(record, 'forcelog', False): 646 self.triggered = False 647 else: 648 self.triggered = True 649 650 # Handle logfiles 651 try: 652 bb.utils.mkdirhier(os.path.dirname(logfn)) 653 logfile = open(logfn, 'w') 654 except OSError: 655 logger.exception("Opening log file '%s'", logfn) 656 pass 657 658 # Dup the existing fds so we dont lose them 659 osi = [os.dup(sys.stdin.fileno()), sys.stdin.fileno()] 660 oso = [os.dup(sys.stdout.fileno()), sys.stdout.fileno()] 661 ose = [os.dup(sys.stderr.fileno()), sys.stderr.fileno()] 662 663 # Replace those fds with our own 664 with open('/dev/null', 'r') as si: 665 os.dup2(si.fileno(), osi[1]) 666 os.dup2(logfile.fileno(), oso[1]) 667 os.dup2(logfile.fileno(), ose[1]) 668 669 # Ensure Python logging goes to the logfile 670 handler = logging.StreamHandler(logfile) 671 handler.setFormatter(logformatter) 672 # Always enable full debug output into task logfiles 673 handler.setLevel(logging.DEBUG - 2) 674 bblogger.addHandler(handler) 675 676 errchk = ErrorCheckHandler() 677 bblogger.addHandler(errchk) 678 679 localdata.setVar('BB_LOGFILE', logfn) 680 localdata.setVar('BB_RUNTASK', task) 681 localdata.setVar('BB_TASK_LOGGER', bblogger) 682 683 flags = localdata.getVarFlags(task) 684 685 try: 686 try: 687 event.fire(TaskStarted(task, fn, logfn, flags, localdata), localdata) 688 689 for func in (prefuncs or '').split(): 690 exec_func(func, localdata) 691 exec_func(task, localdata) 692 for func in (postfuncs or '').split(): 693 exec_func(func, localdata) 694 finally: 695 # Need to flush and close the logs before sending events where the 696 # UI may try to look at the logs. 697 sys.stdout.flush() 698 sys.stderr.flush() 699 700 bblogger.removeHandler(handler) 701 702 # Restore the backup fds 703 os.dup2(osi[0], osi[1]) 704 os.dup2(oso[0], oso[1]) 705 os.dup2(ose[0], ose[1]) 706 707 # Close the backup fds 708 os.close(osi[0]) 709 os.close(oso[0]) 710 os.close(ose[0]) 711 712 logfile.close() 713 if os.path.exists(logfn) and os.path.getsize(logfn) == 0: 714 logger.debug2("Zero size logfn %s, removing", logfn) 715 bb.utils.remove(logfn) 716 bb.utils.remove(loglink) 717 except (Exception, SystemExit) as exc: 718 handled = False 719 if isinstance(exc, bb.BBHandledException): 720 handled = True 721 722 if quieterr: 723 if not handled: 724 logger.warning(repr(exc)) 725 event.fire(TaskFailedSilent(task, fn, logfn, localdata), localdata) 726 else: 727 errprinted = errchk.triggered 728 # If the output is already on stdout, we've printed the information in the 729 # logs once already so don't duplicate 730 if verboseStdoutLogging or handled: 731 errprinted = True 732 if not handled: 733 logger.error(repr(exc)) 734 event.fire(TaskFailed(task, fn, logfn, localdata, errprinted), localdata) 735 return 1 736 737 event.fire(TaskSucceeded(task, fn, logfn, localdata), localdata) 738 739 if not localdata.getVarFlag(task, 'nostamp', False) and not localdata.getVarFlag(task, 'selfstamp', False): 740 make_stamp(task, localdata) 741 742 return 0 743 744def exec_task(fn, task, d, profile = False): 745 try: 746 quieterr = False 747 if d.getVarFlag(task, "quieterrors", False) is not None: 748 quieterr = True 749 750 if profile: 751 profname = "profile-%s.log" % (d.getVar("PN") + "-" + task) 752 try: 753 import cProfile as profile 754 except: 755 import profile 756 prof = profile.Profile() 757 ret = profile.Profile.runcall(prof, _exec_task, fn, task, d, quieterr) 758 prof.dump_stats(profname) 759 bb.utils.process_profilelog(profname) 760 761 return ret 762 else: 763 return _exec_task(fn, task, d, quieterr) 764 765 except Exception: 766 from traceback import format_exc 767 if not quieterr: 768 logger.error("Build of %s failed" % (task)) 769 logger.error(format_exc()) 770 failedevent = TaskFailed(task, None, d, True) 771 event.fire(failedevent, d) 772 return 1 773 774def stamp_internal(taskname, d, file_name, baseonly=False, noextra=False): 775 """ 776 Internal stamp helper function 777 Makes sure the stamp directory exists 778 Returns the stamp path+filename 779 780 In the bitbake core, d can be a CacheData and file_name will be set. 781 When called in task context, d will be a data store, file_name will not be set 782 """ 783 taskflagname = taskname 784 if taskname.endswith("_setscene") and taskname != "do_setscene": 785 taskflagname = taskname.replace("_setscene", "") 786 787 if file_name: 788 stamp = d.stamp[file_name] 789 extrainfo = d.stamp_extrainfo[file_name].get(taskflagname) or "" 790 else: 791 stamp = d.getVar('STAMP') 792 file_name = d.getVar('BB_FILENAME') 793 extrainfo = d.getVarFlag(taskflagname, 'stamp-extra-info') or "" 794 795 if baseonly: 796 return stamp 797 if noextra: 798 extrainfo = "" 799 800 if not stamp: 801 return 802 803 stamp = bb.parse.siggen.stampfile(stamp, file_name, taskname, extrainfo) 804 805 stampdir = os.path.dirname(stamp) 806 if cached_mtime_noerror(stampdir) == 0: 807 bb.utils.mkdirhier(stampdir) 808 809 return stamp 810 811def stamp_cleanmask_internal(taskname, d, file_name): 812 """ 813 Internal stamp helper function to generate stamp cleaning mask 814 Returns the stamp path+filename 815 816 In the bitbake core, d can be a CacheData and file_name will be set. 817 When called in task context, d will be a data store, file_name will not be set 818 """ 819 taskflagname = taskname 820 if taskname.endswith("_setscene") and taskname != "do_setscene": 821 taskflagname = taskname.replace("_setscene", "") 822 823 if file_name: 824 stamp = d.stampclean[file_name] 825 extrainfo = d.stamp_extrainfo[file_name].get(taskflagname) or "" 826 else: 827 stamp = d.getVar('STAMPCLEAN') 828 file_name = d.getVar('BB_FILENAME') 829 extrainfo = d.getVarFlag(taskflagname, 'stamp-extra-info') or "" 830 831 if not stamp: 832 return [] 833 834 cleanmask = bb.parse.siggen.stampcleanmask(stamp, file_name, taskname, extrainfo) 835 836 return [cleanmask, cleanmask.replace(taskflagname, taskflagname + "_setscene")] 837 838def clean_stamp(task, d, file_name = None): 839 cleanmask = stamp_cleanmask_internal(task, d, file_name) 840 for mask in cleanmask: 841 for name in glob.glob(mask): 842 # Preserve sigdata files in the stamps directory 843 if "sigdata" in name or "sigbasedata" in name: 844 continue 845 # Preserve taint files in the stamps directory 846 if name.endswith('.taint'): 847 continue 848 os.unlink(name) 849 return 850 851def make_stamp(task, d, file_name = None): 852 """ 853 Creates/updates a stamp for a given task 854 (d can be a data dict or dataCache) 855 """ 856 clean_stamp(task, d, file_name) 857 858 stamp = stamp_internal(task, d, file_name) 859 # Remove the file and recreate to force timestamp 860 # change on broken NFS filesystems 861 if stamp: 862 bb.utils.remove(stamp) 863 open(stamp, "w").close() 864 865 # If we're in task context, write out a signature file for each task 866 # as it completes 867 if not task.endswith("_setscene") and task != "do_setscene" and not file_name: 868 stampbase = stamp_internal(task, d, None, True) 869 file_name = d.getVar('BB_FILENAME') 870 bb.parse.siggen.dump_sigtask(file_name, task, stampbase, True) 871 872def find_stale_stamps(task, d, file_name=None): 873 current = stamp_internal(task, d, file_name) 874 current2 = stamp_internal(task + "_setscene", d, file_name) 875 cleanmask = stamp_cleanmask_internal(task, d, file_name) 876 found = [] 877 for mask in cleanmask: 878 for name in glob.glob(mask): 879 if "sigdata" in name or "sigbasedata" in name: 880 continue 881 if name.endswith('.taint'): 882 continue 883 if name == current or name == current2: 884 continue 885 logger.debug2("Stampfile %s does not match %s or %s" % (name, current, current2)) 886 found.append(name) 887 return found 888 889def del_stamp(task, d, file_name = None): 890 """ 891 Removes a stamp for a given task 892 (d can be a data dict or dataCache) 893 """ 894 stamp = stamp_internal(task, d, file_name) 895 bb.utils.remove(stamp) 896 897def write_taint(task, d, file_name = None): 898 """ 899 Creates a "taint" file which will force the specified task and its 900 dependents to be re-run the next time by influencing the value of its 901 taskhash. 902 (d can be a data dict or dataCache) 903 """ 904 import uuid 905 if file_name: 906 taintfn = d.stamp[file_name] + '.' + task + '.taint' 907 else: 908 taintfn = d.getVar('STAMP') + '.' + task + '.taint' 909 bb.utils.mkdirhier(os.path.dirname(taintfn)) 910 # The specific content of the taint file is not really important, 911 # we just need it to be random, so a random UUID is used 912 with open(taintfn, 'w') as taintf: 913 taintf.write(str(uuid.uuid4())) 914 915def stampfile(taskname, d, file_name = None, noextra=False): 916 """ 917 Return the stamp for a given task 918 (d can be a data dict or dataCache) 919 """ 920 return stamp_internal(taskname, d, file_name, noextra=noextra) 921 922def add_tasks(tasklist, d): 923 task_deps = d.getVar('_task_deps', False) 924 if not task_deps: 925 task_deps = {} 926 if not 'tasks' in task_deps: 927 task_deps['tasks'] = [] 928 if not 'parents' in task_deps: 929 task_deps['parents'] = {} 930 931 for task in tasklist: 932 task = d.expand(task) 933 934 d.setVarFlag(task, 'task', 1) 935 936 if not task in task_deps['tasks']: 937 task_deps['tasks'].append(task) 938 939 flags = d.getVarFlags(task) 940 def getTask(name): 941 if not name in task_deps: 942 task_deps[name] = {} 943 if name in flags: 944 deptask = d.expand(flags[name]) 945 if name in ['noexec', 'fakeroot', 'nostamp']: 946 if deptask != '1': 947 bb.warn("In a future version of BitBake, setting the '{}' flag to something other than '1' " 948 "will result in the flag not being set. See YP bug #13808.".format(name)) 949 950 task_deps[name][task] = deptask 951 getTask('mcdepends') 952 getTask('depends') 953 getTask('rdepends') 954 getTask('deptask') 955 getTask('rdeptask') 956 getTask('recrdeptask') 957 getTask('recideptask') 958 getTask('nostamp') 959 getTask('fakeroot') 960 getTask('noexec') 961 getTask('umask') 962 task_deps['parents'][task] = [] 963 if 'deps' in flags: 964 for dep in flags['deps']: 965 # Check and warn for "addtask task after foo" while foo does not exist 966 #if not dep in tasklist: 967 # bb.warn('%s: dependent task %s for %s does not exist' % (d.getVar('PN'), dep, task)) 968 dep = d.expand(dep) 969 task_deps['parents'][task].append(dep) 970 971 # don't assume holding a reference 972 d.setVar('_task_deps', task_deps) 973 974def addtask(task, before, after, d): 975 if task[:3] != "do_": 976 task = "do_" + task 977 978 d.setVarFlag(task, "task", 1) 979 bbtasks = d.getVar('__BBTASKS', False) or [] 980 if task not in bbtasks: 981 bbtasks.append(task) 982 d.setVar('__BBTASKS', bbtasks) 983 984 existing = d.getVarFlag(task, "deps", False) or [] 985 if after is not None: 986 # set up deps for function 987 for entry in after.split(): 988 if entry not in existing: 989 existing.append(entry) 990 d.setVarFlag(task, "deps", existing) 991 if before is not None: 992 # set up things that depend on this func 993 for entry in before.split(): 994 existing = d.getVarFlag(entry, "deps", False) or [] 995 if task not in existing: 996 d.setVarFlag(entry, "deps", [task] + existing) 997 998def deltask(task, d): 999 if task[:3] != "do_": 1000 task = "do_" + task 1001 1002 bbtasks = d.getVar('__BBTASKS', False) or [] 1003 if task in bbtasks: 1004 bbtasks.remove(task) 1005 d.delVarFlag(task, 'task') 1006 d.setVar('__BBTASKS', bbtasks) 1007 1008 d.delVarFlag(task, 'deps') 1009 for bbtask in d.getVar('__BBTASKS', False) or []: 1010 deps = d.getVarFlag(bbtask, 'deps', False) or [] 1011 if task in deps: 1012 deps.remove(task) 1013 d.setVarFlag(bbtask, 'deps', deps) 1014 1015def preceedtask(task, with_recrdeptasks, d): 1016 """ 1017 Returns a set of tasks in the current recipe which were specified as 1018 precondition by the task itself ("after") or which listed themselves 1019 as precondition ("before"). Preceeding tasks specified via the 1020 "recrdeptask" are included in the result only if requested. Beware 1021 that this may lead to the task itself being listed. 1022 """ 1023 preceed = set() 1024 1025 # Ignore tasks which don't exist 1026 tasks = d.getVar('__BBTASKS', False) 1027 if task not in tasks: 1028 return preceed 1029 1030 preceed.update(d.getVarFlag(task, 'deps') or []) 1031 if with_recrdeptasks: 1032 recrdeptask = d.getVarFlag(task, 'recrdeptask') 1033 if recrdeptask: 1034 preceed.update(recrdeptask.split()) 1035 return preceed 1036 1037def tasksbetween(task_start, task_end, d): 1038 """ 1039 Return the list of tasks between two tasks in the current recipe, 1040 where task_start is to start at and task_end is the task to end at 1041 (and task_end has a dependency chain back to task_start). 1042 """ 1043 outtasks = [] 1044 tasks = list(filter(lambda k: d.getVarFlag(k, "task"), d.keys())) 1045 def follow_chain(task, endtask, chain=None): 1046 if not chain: 1047 chain = [] 1048 if task in chain: 1049 bb.fatal("Circular task dependencies as %s depends on itself via the chain %s" % (task, " -> ".join(chain))) 1050 chain.append(task) 1051 for othertask in tasks: 1052 if othertask == task: 1053 continue 1054 if task == endtask: 1055 for ctask in chain: 1056 if ctask not in outtasks: 1057 outtasks.append(ctask) 1058 else: 1059 deps = d.getVarFlag(othertask, 'deps', False) 1060 if task in deps: 1061 follow_chain(othertask, endtask, chain) 1062 chain.pop() 1063 follow_chain(task_start, task_end) 1064 return outtasks 1065