1# 2# Copyright (C) 2003, 2004 Chris Larson 3# Copyright (C) 2003, 2004 Phil Blundell 4# Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer 5# Copyright (C) 2005 Holger Hans Peter Freyther 6# Copyright (C) 2005 ROAD GmbH 7# Copyright (C) 2006 Richard Purdie 8# 9# SPDX-License-Identifier: GPL-2.0-only 10# 11 12import os 13import sys 14import logging 15import optparse 16import warnings 17import fcntl 18import time 19import traceback 20 21import bb 22from bb import event 23import bb.msg 24from bb import cooker 25from bb import ui 26from bb import server 27from bb import cookerdata 28 29import bb.server.process 30import bb.server.xmlrpcclient 31 32logger = logging.getLogger("BitBake") 33 34class BBMainException(Exception): 35 pass 36 37class BBMainFatal(bb.BBHandledException): 38 pass 39 40def present_options(optionlist): 41 if len(optionlist) > 1: 42 return ' or '.join([', '.join(optionlist[:-1]), optionlist[-1]]) 43 else: 44 return optionlist[0] 45 46class BitbakeHelpFormatter(optparse.IndentedHelpFormatter): 47 def format_option(self, option): 48 # We need to do this here rather than in the text we supply to 49 # add_option() because we don't want to call list_extension_modules() 50 # on every execution (since it imports all of the modules) 51 # Note also that we modify option.help rather than the returned text 52 # - this is so that we don't have to re-format the text ourselves 53 if option.dest == 'ui': 54 valid_uis = list_extension_modules(bb.ui, 'main') 55 option.help = option.help.replace('@CHOICES@', present_options(valid_uis)) 56 57 return optparse.IndentedHelpFormatter.format_option(self, option) 58 59def list_extension_modules(pkg, checkattr): 60 """ 61 Lists extension modules in a specific Python package 62 (e.g. UIs, servers). NOTE: Calling this function will import all of the 63 submodules of the specified module in order to check for the specified 64 attribute; this can have unusual side-effects. As a result, this should 65 only be called when displaying help text or error messages. 66 Parameters: 67 pkg: previously imported Python package to list 68 checkattr: attribute to look for in module to determine if it's valid 69 as the type of extension you are looking for 70 """ 71 import pkgutil 72 pkgdir = os.path.dirname(pkg.__file__) 73 74 modules = [] 75 for _, modulename, _ in pkgutil.iter_modules([pkgdir]): 76 if os.path.isdir(os.path.join(pkgdir, modulename)): 77 # ignore directories 78 continue 79 try: 80 module = __import__(pkg.__name__, fromlist=[modulename]) 81 except: 82 # If we can't import it, it's not valid 83 continue 84 module_if = getattr(module, modulename) 85 if getattr(module_if, 'hidden_extension', False): 86 continue 87 if not checkattr or hasattr(module_if, checkattr): 88 modules.append(modulename) 89 return modules 90 91def import_extension_module(pkg, modulename, checkattr): 92 try: 93 # Dynamically load the UI based on the ui name. Although we 94 # suggest a fixed set this allows you to have flexibility in which 95 # ones are available. 96 module = __import__(pkg.__name__, fromlist=[modulename]) 97 return getattr(module, modulename) 98 except AttributeError: 99 modules = present_options(list_extension_modules(pkg, checkattr)) 100 raise BBMainException('FATAL: Unable to import extension module "%s" from %s. ' 101 'Valid extension modules: %s' % (modulename, pkg.__name__, modules)) 102 103# Display bitbake/OE warnings via the BitBake.Warnings logger, ignoring others""" 104warnlog = logging.getLogger("BitBake.Warnings") 105_warnings_showwarning = warnings.showwarning 106def _showwarning(message, category, filename, lineno, file=None, line=None): 107 if file is not None: 108 if _warnings_showwarning is not None: 109 _warnings_showwarning(message, category, filename, lineno, file, line) 110 else: 111 s = warnings.formatwarning(message, category, filename, lineno) 112 warnlog.warning(s) 113 114warnings.showwarning = _showwarning 115 116def create_bitbake_parser(): 117 parser = optparse.OptionParser( 118 formatter=BitbakeHelpFormatter(), 119 version="BitBake Build Tool Core version %s" % bb.__version__, 120 usage="""%prog [options] [recipename/target recipe:do_task ...] 121 122 Executes the specified task (default is 'build') for a given set of target recipes (.bb files). 123 It is assumed there is a conf/bblayers.conf available in cwd or in BBPATH which 124 will provide the layer, BBFILES and other configuration information.""") 125 126 parser.add_option("-b", "--buildfile", action="store", dest="buildfile", default=None, 127 help="Execute tasks from a specific .bb recipe directly. WARNING: Does " 128 "not handle any dependencies from other recipes.") 129 130 parser.add_option("-k", "--continue", action="store_false", dest="halt", default=True, 131 help="Continue as much as possible after an error. While the target that " 132 "failed and anything depending on it cannot be built, as much as " 133 "possible will be built before stopping.") 134 135 parser.add_option("-f", "--force", action="store_true", dest="force", default=False, 136 help="Force the specified targets/task to run (invalidating any " 137 "existing stamp file).") 138 139 parser.add_option("-c", "--cmd", action="store", dest="cmd", 140 help="Specify the task to execute. The exact options available " 141 "depend on the metadata. Some examples might be 'compile'" 142 " or 'populate_sysroot' or 'listtasks' may give a list of " 143 "the tasks available.") 144 145 parser.add_option("-C", "--clear-stamp", action="store", dest="invalidate_stamp", 146 help="Invalidate the stamp for the specified task such as 'compile' " 147 "and then run the default task for the specified target(s).") 148 149 parser.add_option("-r", "--read", action="append", dest="prefile", default=[], 150 help="Read the specified file before bitbake.conf.") 151 152 parser.add_option("-R", "--postread", action="append", dest="postfile", default=[], 153 help="Read the specified file after bitbake.conf.") 154 155 parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False, 156 help="Enable tracing of shell tasks (with 'set -x'). " 157 "Also print bb.note(...) messages to stdout (in " 158 "addition to writing them to ${T}/log.do_<task>).") 159 160 parser.add_option("-D", "--debug", action="count", dest="debug", default=0, 161 help="Increase the debug level. You can specify this " 162 "more than once. -D sets the debug level to 1, " 163 "where only bb.debug(1, ...) messages are printed " 164 "to stdout; -DD sets the debug level to 2, where " 165 "both bb.debug(1, ...) and bb.debug(2, ...) " 166 "messages are printed; etc. Without -D, no debug " 167 "messages are printed. Note that -D only affects " 168 "output to stdout. All debug messages are written " 169 "to ${T}/log.do_taskname, regardless of the debug " 170 "level.") 171 172 parser.add_option("-q", "--quiet", action="count", dest="quiet", default=0, 173 help="Output less log message data to the terminal. You can specify this more than once.") 174 175 parser.add_option("-n", "--dry-run", action="store_true", dest="dry_run", default=False, 176 help="Don't execute, just go through the motions.") 177 178 parser.add_option("-S", "--dump-signatures", action="append", dest="dump_signatures", 179 default=[], metavar="SIGNATURE_HANDLER", 180 help="Dump out the signature construction information, with no task " 181 "execution. The SIGNATURE_HANDLER parameter is passed to the " 182 "handler. Two common values are none and printdiff but the handler " 183 "may define more/less. none means only dump the signature, printdiff" 184 " means compare the dumped signature with the cached one.") 185 186 parser.add_option("-p", "--parse-only", action="store_true", 187 dest="parse_only", default=False, 188 help="Quit after parsing the BB recipes.") 189 190 parser.add_option("-s", "--show-versions", action="store_true", 191 dest="show_versions", default=False, 192 help="Show current and preferred versions of all recipes.") 193 194 parser.add_option("-e", "--environment", action="store_true", 195 dest="show_environment", default=False, 196 help="Show the global or per-recipe environment complete with information" 197 " about where variables were set/changed.") 198 199 parser.add_option("-g", "--graphviz", action="store_true", dest="dot_graph", default=False, 200 help="Save dependency tree information for the specified " 201 "targets in the dot syntax.") 202 203 parser.add_option("-I", "--ignore-deps", action="append", 204 dest="extra_assume_provided", default=[], 205 help="Assume these dependencies don't exist and are already provided " 206 "(equivalent to ASSUME_PROVIDED). Useful to make dependency " 207 "graphs more appealing") 208 209 parser.add_option("-l", "--log-domains", action="append", dest="debug_domains", default=[], 210 help="Show debug logging for the specified logging domains") 211 212 parser.add_option("-P", "--profile", action="store_true", dest="profile", default=False, 213 help="Profile the command and save reports.") 214 215 # @CHOICES@ is substituted out by BitbakeHelpFormatter above 216 parser.add_option("-u", "--ui", action="store", dest="ui", 217 default=os.environ.get('BITBAKE_UI', 'knotty'), 218 help="The user interface to use (@CHOICES@ - default %default).") 219 220 parser.add_option("", "--token", action="store", dest="xmlrpctoken", 221 default=os.environ.get("BBTOKEN"), 222 help="Specify the connection token to be used when connecting " 223 "to a remote server.") 224 225 parser.add_option("", "--revisions-changed", action="store_true", 226 dest="revisions_changed", default=False, 227 help="Set the exit code depending on whether upstream floating " 228 "revisions have changed or not.") 229 230 parser.add_option("", "--server-only", action="store_true", 231 dest="server_only", default=False, 232 help="Run bitbake without a UI, only starting a server " 233 "(cooker) process.") 234 235 parser.add_option("-B", "--bind", action="store", dest="bind", default=False, 236 help="The name/address for the bitbake xmlrpc server to bind to.") 237 238 parser.add_option("-T", "--idle-timeout", type=float, dest="server_timeout", 239 default=os.getenv("BB_SERVER_TIMEOUT"), 240 help="Set timeout to unload bitbake server due to inactivity, " 241 "set to -1 means no unload, " 242 "default: Environment variable BB_SERVER_TIMEOUT.") 243 244 parser.add_option("", "--no-setscene", action="store_true", 245 dest="nosetscene", default=False, 246 help="Do not run any setscene tasks. sstate will be ignored and " 247 "everything needed, built.") 248 249 parser.add_option("", "--skip-setscene", action="store_true", 250 dest="skipsetscene", default=False, 251 help="Skip setscene tasks if they would be executed. Tasks previously " 252 "restored from sstate will be kept, unlike --no-setscene") 253 254 parser.add_option("", "--setscene-only", action="store_true", 255 dest="setsceneonly", default=False, 256 help="Only run setscene tasks, don't run any real tasks.") 257 258 parser.add_option("", "--remote-server", action="store", dest="remote_server", 259 default=os.environ.get("BBSERVER"), 260 help="Connect to the specified server.") 261 262 parser.add_option("-m", "--kill-server", action="store_true", 263 dest="kill_server", default=False, 264 help="Terminate any running bitbake server.") 265 266 parser.add_option("", "--observe-only", action="store_true", 267 dest="observe_only", default=False, 268 help="Connect to a server as an observing-only client.") 269 270 parser.add_option("", "--status-only", action="store_true", 271 dest="status_only", default=False, 272 help="Check the status of the remote bitbake server.") 273 274 parser.add_option("-w", "--write-log", action="store", dest="writeeventlog", 275 default=os.environ.get("BBEVENTLOG"), 276 help="Writes the event log of the build to a bitbake event json file. " 277 "Use '' (empty string) to assign the name automatically.") 278 279 parser.add_option("", "--runall", action="append", dest="runall", 280 help="Run the specified task for any recipe in the taskgraph of the specified target (even if it wouldn't otherwise have run).") 281 282 parser.add_option("", "--runonly", action="append", dest="runonly", 283 help="Run only the specified task within the taskgraph of the specified targets (and any task dependencies those tasks may have).") 284 return parser 285 286 287class BitBakeConfigParameters(cookerdata.ConfigParameters): 288 def parseCommandLine(self, argv=sys.argv): 289 parser = create_bitbake_parser() 290 options, targets = parser.parse_args(argv) 291 292 if options.quiet and options.verbose: 293 parser.error("options --quiet and --verbose are mutually exclusive") 294 295 if options.quiet and options.debug: 296 parser.error("options --quiet and --debug are mutually exclusive") 297 298 # use configuration files from environment variables 299 if "BBPRECONF" in os.environ: 300 options.prefile.append(os.environ["BBPRECONF"]) 301 302 if "BBPOSTCONF" in os.environ: 303 options.postfile.append(os.environ["BBPOSTCONF"]) 304 305 # fill in proper log name if not supplied 306 if options.writeeventlog is not None and len(options.writeeventlog) == 0: 307 from datetime import datetime 308 eventlog = "bitbake_eventlog_%s.json" % datetime.now().strftime("%Y%m%d%H%M%S") 309 options.writeeventlog = eventlog 310 311 if options.bind: 312 try: 313 #Checking that the port is a number and is a ':' delimited value 314 (host, port) = options.bind.split(':') 315 port = int(port) 316 except (ValueError,IndexError): 317 raise BBMainException("FATAL: Malformed host:port bind parameter") 318 options.xmlrpcinterface = (host, port) 319 else: 320 options.xmlrpcinterface = (None, 0) 321 322 return options, targets[1:] 323 324 325def bitbake_main(configParams, configuration): 326 327 # Python multiprocessing requires /dev/shm on Linux 328 if sys.platform.startswith('linux') and not os.access('/dev/shm', os.W_OK | os.X_OK): 329 raise BBMainException("FATAL: /dev/shm does not exist or is not writable") 330 331 # Unbuffer stdout to avoid log truncation in the event 332 # of an unorderly exit as well as to provide timely 333 # updates to log files for use with tail 334 try: 335 if sys.stdout.name == '<stdout>': 336 # Reopen with O_SYNC (unbuffered) 337 fl = fcntl.fcntl(sys.stdout.fileno(), fcntl.F_GETFL) 338 fl |= os.O_SYNC 339 fcntl.fcntl(sys.stdout.fileno(), fcntl.F_SETFL, fl) 340 except: 341 pass 342 343 if configParams.server_only and configParams.remote_server: 344 raise BBMainException("FATAL: The '--server-only' option conflicts with %s.\n" % 345 ("the BBSERVER environment variable" if "BBSERVER" in os.environ \ 346 else "the '--remote-server' option")) 347 348 if configParams.observe_only and not (configParams.remote_server or configParams.bind): 349 raise BBMainException("FATAL: '--observe-only' can only be used by UI clients " 350 "connecting to a server.\n") 351 352 if "BBDEBUG" in os.environ: 353 level = int(os.environ["BBDEBUG"]) 354 if level > configParams.debug: 355 configParams.debug = level 356 357 bb.msg.init_msgconfig(configParams.verbose, configParams.debug, 358 configParams.debug_domains) 359 360 server_connection, ui_module = setup_bitbake(configParams) 361 # No server connection 362 if server_connection is None: 363 if configParams.status_only: 364 return 1 365 if configParams.kill_server: 366 return 0 367 368 if not configParams.server_only: 369 if configParams.status_only: 370 server_connection.terminate() 371 return 0 372 373 try: 374 for event in bb.event.ui_queue: 375 server_connection.events.queue_event(event) 376 bb.event.ui_queue = [] 377 378 return ui_module.main(server_connection.connection, server_connection.events, 379 configParams) 380 finally: 381 server_connection.terminate() 382 else: 383 return 0 384 385 return 1 386 387def setup_bitbake(configParams, extrafeatures=None): 388 # Ensure logging messages get sent to the UI as events 389 handler = bb.event.LogHandler() 390 if not configParams.status_only: 391 # In status only mode there are no logs and no UI 392 logger.addHandler(handler) 393 394 if configParams.server_only: 395 featureset = [] 396 ui_module = None 397 else: 398 ui_module = import_extension_module(bb.ui, configParams.ui, 'main') 399 # Collect the feature set for the UI 400 featureset = getattr(ui_module, "featureSet", []) 401 402 if extrafeatures: 403 for feature in extrafeatures: 404 if not feature in featureset: 405 featureset.append(feature) 406 407 server_connection = None 408 409 # Clear away any spurious environment variables while we stoke up the cooker 410 # (done after import_extension_module() above since for example import gi triggers env var usage) 411 cleanedvars = bb.utils.clean_environment() 412 413 if configParams.remote_server: 414 # Connect to a remote XMLRPC server 415 server_connection = bb.server.xmlrpcclient.connectXMLRPC(configParams.remote_server, featureset, 416 configParams.observe_only, configParams.xmlrpctoken) 417 else: 418 retries = 8 419 while retries: 420 try: 421 topdir, lock = lockBitbake() 422 sockname = topdir + "/bitbake.sock" 423 if lock: 424 if configParams.status_only or configParams.kill_server: 425 logger.info("bitbake server is not running.") 426 lock.close() 427 return None, None 428 # we start a server with a given featureset 429 logger.info("Starting bitbake server...") 430 # Clear the event queue since we already displayed messages 431 bb.event.ui_queue = [] 432 server = bb.server.process.BitBakeServer(lock, sockname, featureset, configParams.server_timeout, configParams.xmlrpcinterface) 433 434 else: 435 logger.info("Reconnecting to bitbake server...") 436 if not os.path.exists(sockname): 437 logger.info("Previous bitbake instance shutting down?, waiting to retry...") 438 i = 0 439 lock = None 440 # Wait for 5s or until we can get the lock 441 while not lock and i < 50: 442 time.sleep(0.1) 443 _, lock = lockBitbake() 444 i += 1 445 if lock: 446 bb.utils.unlockfile(lock) 447 raise bb.server.process.ProcessTimeout("Bitbake still shutting down as socket exists but no lock?") 448 if not configParams.server_only: 449 server_connection = bb.server.process.connectProcessServer(sockname, featureset) 450 451 if server_connection or configParams.server_only: 452 break 453 except BBMainFatal: 454 raise 455 except (Exception, bb.server.process.ProcessTimeout, SystemExit) as e: 456 # SystemExit does not inherit from the Exception class, needs to be included explicitly 457 if not retries: 458 raise 459 retries -= 1 460 tryno = 8 - retries 461 if isinstance(e, (bb.server.process.ProcessTimeout, BrokenPipeError, EOFError, SystemExit)): 462 logger.info("Retrying server connection (#%d)..." % tryno) 463 else: 464 logger.info("Retrying server connection (#%d)... (%s)" % (tryno, traceback.format_exc())) 465 466 if not retries: 467 bb.fatal("Unable to connect to bitbake server, or start one (server startup failures would be in bitbake-cookerdaemon.log).") 468 bb.event.print_ui_queue() 469 if retries < 5: 470 time.sleep(5) 471 472 if configParams.kill_server: 473 server_connection.connection.terminateServer() 474 server_connection.terminate() 475 bb.event.ui_queue = [] 476 logger.info("Terminated bitbake server.") 477 return None, None 478 479 # Restore the environment in case the UI needs it 480 for k in cleanedvars: 481 os.environ[k] = cleanedvars[k] 482 483 logger.removeHandler(handler) 484 485 return server_connection, ui_module 486 487def lockBitbake(): 488 topdir = bb.cookerdata.findTopdir() 489 if not topdir: 490 bb.error("Unable to find conf/bblayers.conf or conf/bitbake.conf. BBPATH is unset and/or not in a build directory?") 491 raise BBMainFatal 492 lockfile = topdir + "/bitbake.lock" 493 return topdir, bb.utils.lockfile(lockfile, False, False) 494 495