1""" 2BitBake 'Command' module 3 4Provide an interface to interact with the bitbake server through 'commands' 5""" 6 7# Copyright (C) 2006-2007 Richard Purdie 8# 9# SPDX-License-Identifier: GPL-2.0-only 10# 11 12""" 13The bitbake server takes 'commands' from its UI/commandline. 14Commands are either synchronous or asynchronous. 15Async commands return data to the client in the form of events. 16Sync commands must only return data through the function return value 17and must not trigger events, directly or indirectly. 18Commands are queued in a CommandQueue 19""" 20 21from collections import OrderedDict, defaultdict 22 23import io 24import bb.event 25import bb.cooker 26import bb.remotedata 27 28class DataStoreConnectionHandle(object): 29 def __init__(self, dsindex=0): 30 self.dsindex = dsindex 31 32class CommandCompleted(bb.event.Event): 33 pass 34 35class CommandExit(bb.event.Event): 36 def __init__(self, exitcode): 37 bb.event.Event.__init__(self) 38 self.exitcode = int(exitcode) 39 40class CommandFailed(CommandExit): 41 def __init__(self, message): 42 self.error = message 43 CommandExit.__init__(self, 1) 44 def __str__(self): 45 return "Command execution failed: %s" % self.error 46 47class CommandError(Exception): 48 pass 49 50class Command: 51 """ 52 A queue of asynchronous commands for bitbake 53 """ 54 def __init__(self, cooker): 55 self.cooker = cooker 56 self.cmds_sync = CommandsSync() 57 self.cmds_async = CommandsAsync() 58 self.remotedatastores = None 59 60 # FIXME Add lock for this 61 self.currentAsyncCommand = None 62 63 def runCommand(self, commandline, ro_only = False): 64 command = commandline.pop(0) 65 66 # Ensure cooker is ready for commands 67 if command != "updateConfig" and command != "setFeatures": 68 try: 69 self.cooker.init_configdata() 70 if not self.remotedatastores: 71 self.remotedatastores = bb.remotedata.RemoteDatastores(self.cooker) 72 except (Exception, SystemExit) as exc: 73 import traceback 74 if isinstance(exc, bb.BBHandledException): 75 # We need to start returning real exceptions here. Until we do, we can't 76 # tell if an exception is an instance of bb.BBHandledException 77 return None, "bb.BBHandledException()\n" + traceback.format_exc() 78 return None, traceback.format_exc() 79 80 if hasattr(CommandsSync, command): 81 # Can run synchronous commands straight away 82 command_method = getattr(self.cmds_sync, command) 83 if ro_only: 84 if not hasattr(command_method, 'readonly') or not getattr(command_method, 'readonly'): 85 return None, "Not able to execute not readonly commands in readonly mode" 86 try: 87 self.cooker.process_inotify_updates() 88 if getattr(command_method, 'needconfig', True): 89 self.cooker.updateCacheSync() 90 result = command_method(self, commandline) 91 except CommandError as exc: 92 return None, exc.args[0] 93 except (Exception, SystemExit) as exc: 94 import traceback 95 if isinstance(exc, bb.BBHandledException): 96 # We need to start returning real exceptions here. Until we do, we can't 97 # tell if an exception is an instance of bb.BBHandledException 98 return None, "bb.BBHandledException()\n" + traceback.format_exc() 99 return None, traceback.format_exc() 100 else: 101 return result, None 102 if self.currentAsyncCommand is not None: 103 return None, "Busy (%s in progress)" % self.currentAsyncCommand[0] 104 if command not in CommandsAsync.__dict__: 105 return None, "No such command" 106 self.currentAsyncCommand = (command, commandline) 107 self.cooker.idleCallBackRegister(self.cooker.runCommands, self.cooker) 108 return True, None 109 110 def runAsyncCommand(self): 111 try: 112 self.cooker.process_inotify_updates() 113 if self.cooker.state in (bb.cooker.state.error, bb.cooker.state.shutdown, bb.cooker.state.forceshutdown): 114 # updateCache will trigger a shutdown of the parser 115 # and then raise BBHandledException triggering an exit 116 self.cooker.updateCache() 117 return False 118 if self.currentAsyncCommand is not None: 119 (command, options) = self.currentAsyncCommand 120 commandmethod = getattr(CommandsAsync, command) 121 needcache = getattr( commandmethod, "needcache" ) 122 if needcache and self.cooker.state != bb.cooker.state.running: 123 self.cooker.updateCache() 124 return True 125 else: 126 commandmethod(self.cmds_async, self, options) 127 return False 128 else: 129 return False 130 except KeyboardInterrupt as exc: 131 self.finishAsyncCommand("Interrupted") 132 return False 133 except SystemExit as exc: 134 arg = exc.args[0] 135 if isinstance(arg, str): 136 self.finishAsyncCommand(arg) 137 else: 138 self.finishAsyncCommand("Exited with %s" % arg) 139 return False 140 except Exception as exc: 141 import traceback 142 if isinstance(exc, bb.BBHandledException): 143 self.finishAsyncCommand("") 144 else: 145 self.finishAsyncCommand(traceback.format_exc()) 146 return False 147 148 def finishAsyncCommand(self, msg=None, code=None): 149 if msg or msg == "": 150 bb.event.fire(CommandFailed(msg), self.cooker.data) 151 elif code: 152 bb.event.fire(CommandExit(code), self.cooker.data) 153 else: 154 bb.event.fire(CommandCompleted(), self.cooker.data) 155 self.currentAsyncCommand = None 156 self.cooker.finishcommand() 157 158 def reset(self): 159 if self.remotedatastores: 160 self.remotedatastores = bb.remotedata.RemoteDatastores(self.cooker) 161 162class CommandsSync: 163 """ 164 A class of synchronous commands 165 These should run quickly so as not to hurt interactive performance. 166 These must not influence any running synchronous command. 167 """ 168 169 def stateShutdown(self, command, params): 170 """ 171 Trigger cooker 'shutdown' mode 172 """ 173 command.cooker.shutdown(False) 174 175 def stateForceShutdown(self, command, params): 176 """ 177 Stop the cooker 178 """ 179 command.cooker.shutdown(True) 180 181 def getAllKeysWithFlags(self, command, params): 182 """ 183 Returns a dump of the global state. Call with 184 variable flags to be retrieved as params. 185 """ 186 flaglist = params[0] 187 return command.cooker.getAllKeysWithFlags(flaglist) 188 getAllKeysWithFlags.readonly = True 189 190 def getVariable(self, command, params): 191 """ 192 Read the value of a variable from data 193 """ 194 varname = params[0] 195 expand = True 196 if len(params) > 1: 197 expand = (params[1] == "True") 198 199 return command.cooker.data.getVar(varname, expand) 200 getVariable.readonly = True 201 202 def setVariable(self, command, params): 203 """ 204 Set the value of variable in data 205 """ 206 varname = params[0] 207 value = str(params[1]) 208 command.cooker.extraconfigdata[varname] = value 209 command.cooker.data.setVar(varname, value) 210 211 def getSetVariable(self, command, params): 212 """ 213 Read the value of a variable from data and set it into the datastore 214 which effectively expands and locks the value. 215 """ 216 varname = params[0] 217 result = self.getVariable(command, params) 218 command.cooker.data.setVar(varname, result) 219 return result 220 221 def setConfig(self, command, params): 222 """ 223 Set the value of variable in configuration 224 """ 225 varname = params[0] 226 value = str(params[1]) 227 setattr(command.cooker.configuration, varname, value) 228 229 def enableDataTracking(self, command, params): 230 """ 231 Enable history tracking for variables 232 """ 233 command.cooker.enableDataTracking() 234 235 def disableDataTracking(self, command, params): 236 """ 237 Disable history tracking for variables 238 """ 239 command.cooker.disableDataTracking() 240 241 def setPrePostConfFiles(self, command, params): 242 prefiles = params[0].split() 243 postfiles = params[1].split() 244 command.cooker.configuration.prefile = prefiles 245 command.cooker.configuration.postfile = postfiles 246 setPrePostConfFiles.needconfig = False 247 248 def matchFile(self, command, params): 249 fMatch = params[0] 250 try: 251 mc = params[0] 252 except IndexError: 253 mc = '' 254 return command.cooker.matchFile(fMatch, mc) 255 matchFile.needconfig = False 256 257 def getUIHandlerNum(self, command, params): 258 return bb.event.get_uihandler() 259 getUIHandlerNum.needconfig = False 260 getUIHandlerNum.readonly = True 261 262 def setEventMask(self, command, params): 263 handlerNum = params[0] 264 llevel = params[1] 265 debug_domains = params[2] 266 mask = params[3] 267 return bb.event.set_UIHmask(handlerNum, llevel, debug_domains, mask) 268 setEventMask.needconfig = False 269 setEventMask.readonly = True 270 271 def setFeatures(self, command, params): 272 """ 273 Set the cooker features to include the passed list of features 274 """ 275 features = params[0] 276 command.cooker.setFeatures(features) 277 setFeatures.needconfig = False 278 # although we change the internal state of the cooker, this is transparent since 279 # we always take and leave the cooker in state.initial 280 setFeatures.readonly = True 281 282 def updateConfig(self, command, params): 283 options = params[0] 284 environment = params[1] 285 cmdline = params[2] 286 command.cooker.updateConfigOpts(options, environment, cmdline) 287 updateConfig.needconfig = False 288 289 def parseConfiguration(self, command, params): 290 """Instruct bitbake to parse its configuration 291 NOTE: it is only necessary to call this if you aren't calling any normal action 292 (otherwise parsing is taken care of automatically) 293 """ 294 command.cooker.parseConfiguration() 295 parseConfiguration.needconfig = False 296 297 def getLayerPriorities(self, command, params): 298 command.cooker.parseConfiguration() 299 ret = [] 300 # regex objects cannot be marshalled by xmlrpc 301 for collection, pattern, regex, pri in command.cooker.bbfile_config_priorities: 302 ret.append((collection, pattern, regex.pattern, pri)) 303 return ret 304 getLayerPriorities.readonly = True 305 306 def getRecipes(self, command, params): 307 try: 308 mc = params[0] 309 except IndexError: 310 mc = '' 311 return list(command.cooker.recipecaches[mc].pkg_pn.items()) 312 getRecipes.readonly = True 313 314 def getRecipeDepends(self, command, params): 315 try: 316 mc = params[0] 317 except IndexError: 318 mc = '' 319 return list(command.cooker.recipecaches[mc].deps.items()) 320 getRecipeDepends.readonly = True 321 322 def getRecipeVersions(self, command, params): 323 try: 324 mc = params[0] 325 except IndexError: 326 mc = '' 327 return command.cooker.recipecaches[mc].pkg_pepvpr 328 getRecipeVersions.readonly = True 329 330 def getRecipeProvides(self, command, params): 331 try: 332 mc = params[0] 333 except IndexError: 334 mc = '' 335 return command.cooker.recipecaches[mc].fn_provides 336 getRecipeProvides.readonly = True 337 338 def getRecipePackages(self, command, params): 339 try: 340 mc = params[0] 341 except IndexError: 342 mc = '' 343 return command.cooker.recipecaches[mc].packages 344 getRecipePackages.readonly = True 345 346 def getRecipePackagesDynamic(self, command, params): 347 try: 348 mc = params[0] 349 except IndexError: 350 mc = '' 351 return command.cooker.recipecaches[mc].packages_dynamic 352 getRecipePackagesDynamic.readonly = True 353 354 def getRProviders(self, command, params): 355 try: 356 mc = params[0] 357 except IndexError: 358 mc = '' 359 return command.cooker.recipecaches[mc].rproviders 360 getRProviders.readonly = True 361 362 def getRuntimeDepends(self, command, params): 363 ret = [] 364 try: 365 mc = params[0] 366 except IndexError: 367 mc = '' 368 rundeps = command.cooker.recipecaches[mc].rundeps 369 for key, value in rundeps.items(): 370 if isinstance(value, defaultdict): 371 value = dict(value) 372 ret.append((key, value)) 373 return ret 374 getRuntimeDepends.readonly = True 375 376 def getRuntimeRecommends(self, command, params): 377 ret = [] 378 try: 379 mc = params[0] 380 except IndexError: 381 mc = '' 382 runrecs = command.cooker.recipecaches[mc].runrecs 383 for key, value in runrecs.items(): 384 if isinstance(value, defaultdict): 385 value = dict(value) 386 ret.append((key, value)) 387 return ret 388 getRuntimeRecommends.readonly = True 389 390 def getRecipeInherits(self, command, params): 391 try: 392 mc = params[0] 393 except IndexError: 394 mc = '' 395 return command.cooker.recipecaches[mc].inherits 396 getRecipeInherits.readonly = True 397 398 def getBbFilePriority(self, command, params): 399 try: 400 mc = params[0] 401 except IndexError: 402 mc = '' 403 return command.cooker.recipecaches[mc].bbfile_priority 404 getBbFilePriority.readonly = True 405 406 def getDefaultPreference(self, command, params): 407 try: 408 mc = params[0] 409 except IndexError: 410 mc = '' 411 return command.cooker.recipecaches[mc].pkg_dp 412 getDefaultPreference.readonly = True 413 414 def getSkippedRecipes(self, command, params): 415 # Return list sorted by reverse priority order 416 import bb.cache 417 def sortkey(x): 418 vfn, _ = x 419 realfn, _, mc = bb.cache.virtualfn2realfn(vfn) 420 return (-command.cooker.collections[mc].calc_bbfile_priority(realfn)[0], vfn) 421 422 skipdict = OrderedDict(sorted(command.cooker.skiplist.items(), key=sortkey)) 423 return list(skipdict.items()) 424 getSkippedRecipes.readonly = True 425 426 def getOverlayedRecipes(self, command, params): 427 try: 428 mc = params[0] 429 except IndexError: 430 mc = '' 431 return list(command.cooker.collections[mc].overlayed.items()) 432 getOverlayedRecipes.readonly = True 433 434 def getFileAppends(self, command, params): 435 fn = params[0] 436 try: 437 mc = params[1] 438 except IndexError: 439 mc = '' 440 return command.cooker.collections[mc].get_file_appends(fn) 441 getFileAppends.readonly = True 442 443 def getAllAppends(self, command, params): 444 try: 445 mc = params[0] 446 except IndexError: 447 mc = '' 448 return command.cooker.collections[mc].bbappends 449 getAllAppends.readonly = True 450 451 def findProviders(self, command, params): 452 try: 453 mc = params[0] 454 except IndexError: 455 mc = '' 456 return command.cooker.findProviders(mc) 457 findProviders.readonly = True 458 459 def findBestProvider(self, command, params): 460 (mc, pn) = bb.runqueue.split_mc(params[0]) 461 return command.cooker.findBestProvider(pn, mc) 462 findBestProvider.readonly = True 463 464 def allProviders(self, command, params): 465 try: 466 mc = params[0] 467 except IndexError: 468 mc = '' 469 return list(bb.providers.allProviders(command.cooker.recipecaches[mc]).items()) 470 allProviders.readonly = True 471 472 def getRuntimeProviders(self, command, params): 473 rprovide = params[0] 474 try: 475 mc = params[1] 476 except IndexError: 477 mc = '' 478 all_p = bb.providers.getRuntimeProviders(command.cooker.recipecaches[mc], rprovide) 479 if all_p: 480 best = bb.providers.filterProvidersRunTime(all_p, rprovide, 481 command.cooker.data, 482 command.cooker.recipecaches[mc])[0][0] 483 else: 484 best = None 485 return all_p, best 486 getRuntimeProviders.readonly = True 487 488 def dataStoreConnectorCmd(self, command, params): 489 dsindex = params[0] 490 method = params[1] 491 args = params[2] 492 kwargs = params[3] 493 494 d = command.remotedatastores[dsindex] 495 ret = getattr(d, method)(*args, **kwargs) 496 497 if isinstance(ret, bb.data_smart.DataSmart): 498 idx = command.remotedatastores.store(ret) 499 return DataStoreConnectionHandle(idx) 500 501 return ret 502 503 def dataStoreConnectorVarHistCmd(self, command, params): 504 dsindex = params[0] 505 method = params[1] 506 args = params[2] 507 kwargs = params[3] 508 509 d = command.remotedatastores[dsindex].varhistory 510 return getattr(d, method)(*args, **kwargs) 511 512 def dataStoreConnectorVarHistCmdEmit(self, command, params): 513 dsindex = params[0] 514 var = params[1] 515 oval = params[2] 516 val = params[3] 517 d = command.remotedatastores[params[4]] 518 519 o = io.StringIO() 520 command.remotedatastores[dsindex].varhistory.emit(var, oval, val, o, d) 521 return o.getvalue() 522 523 def dataStoreConnectorIncHistCmd(self, command, params): 524 dsindex = params[0] 525 method = params[1] 526 args = params[2] 527 kwargs = params[3] 528 529 d = command.remotedatastores[dsindex].inchistory 530 return getattr(d, method)(*args, **kwargs) 531 532 def dataStoreConnectorRelease(self, command, params): 533 dsindex = params[0] 534 if dsindex <= 0: 535 raise CommandError('dataStoreConnectorRelease: invalid index %d' % dsindex) 536 command.remotedatastores.release(dsindex) 537 538 def parseRecipeFile(self, command, params): 539 """ 540 Parse the specified recipe file (with or without bbappends) 541 and return a datastore object representing the environment 542 for the recipe. 543 """ 544 fn = params[0] 545 mc = bb.runqueue.mc_from_tid(fn) 546 appends = params[1] 547 appendlist = params[2] 548 if len(params) > 3: 549 config_data = command.remotedatastores[params[3]] 550 else: 551 config_data = None 552 553 if appends: 554 if appendlist is not None: 555 appendfiles = appendlist 556 else: 557 appendfiles = command.cooker.collections[mc].get_file_appends(fn) 558 else: 559 appendfiles = [] 560 # We are calling bb.cache locally here rather than on the server, 561 # but that's OK because it doesn't actually need anything from 562 # the server barring the global datastore (which we have a remote 563 # version of) 564 if config_data: 565 # We have to use a different function here if we're passing in a datastore 566 # NOTE: we took a copy above, so we don't do it here again 567 envdata = bb.cache.parse_recipe(config_data, fn, appendfiles, mc)[''] 568 else: 569 # Use the standard path 570 parser = bb.cache.NoCache(command.cooker.databuilder) 571 envdata = parser.loadDataFull(fn, appendfiles) 572 idx = command.remotedatastores.store(envdata) 573 return DataStoreConnectionHandle(idx) 574 parseRecipeFile.readonly = True 575 576class CommandsAsync: 577 """ 578 A class of asynchronous commands 579 These functions communicate via generated events. 580 Any function that requires metadata parsing should be here. 581 """ 582 583 def buildFile(self, command, params): 584 """ 585 Build a single specified .bb file 586 """ 587 bfile = params[0] 588 task = params[1] 589 if len(params) > 2: 590 internal = params[2] 591 else: 592 internal = False 593 594 if internal: 595 command.cooker.buildFileInternal(bfile, task, fireevents=False, quietlog=True) 596 else: 597 command.cooker.buildFile(bfile, task) 598 buildFile.needcache = False 599 600 def buildTargets(self, command, params): 601 """ 602 Build a set of targets 603 """ 604 pkgs_to_build = params[0] 605 task = params[1] 606 607 command.cooker.buildTargets(pkgs_to_build, task) 608 buildTargets.needcache = True 609 610 def generateDepTreeEvent(self, command, params): 611 """ 612 Generate an event containing the dependency information 613 """ 614 pkgs_to_build = params[0] 615 task = params[1] 616 617 command.cooker.generateDepTreeEvent(pkgs_to_build, task) 618 command.finishAsyncCommand() 619 generateDepTreeEvent.needcache = True 620 621 def generateDotGraph(self, command, params): 622 """ 623 Dump dependency information to disk as .dot files 624 """ 625 pkgs_to_build = params[0] 626 task = params[1] 627 628 command.cooker.generateDotGraphFiles(pkgs_to_build, task) 629 command.finishAsyncCommand() 630 generateDotGraph.needcache = True 631 632 def generateTargetsTree(self, command, params): 633 """ 634 Generate a tree of buildable targets. 635 If klass is provided ensure all recipes that inherit the class are 636 included in the package list. 637 If pkg_list provided use that list (plus any extras brought in by 638 klass) rather than generating a tree for all packages. 639 """ 640 klass = params[0] 641 pkg_list = params[1] 642 643 command.cooker.generateTargetsTree(klass, pkg_list) 644 command.finishAsyncCommand() 645 generateTargetsTree.needcache = True 646 647 def findConfigFiles(self, command, params): 648 """ 649 Find config files which provide appropriate values 650 for the passed configuration variable. i.e. MACHINE 651 """ 652 varname = params[0] 653 654 command.cooker.findConfigFiles(varname) 655 command.finishAsyncCommand() 656 findConfigFiles.needcache = False 657 658 def findFilesMatchingInDir(self, command, params): 659 """ 660 Find implementation files matching the specified pattern 661 in the requested subdirectory of a BBPATH 662 """ 663 pattern = params[0] 664 directory = params[1] 665 666 command.cooker.findFilesMatchingInDir(pattern, directory) 667 command.finishAsyncCommand() 668 findFilesMatchingInDir.needcache = False 669 670 def testCookerCommandEvent(self, command, params): 671 """ 672 Dummy command used by OEQA selftest to test tinfoil without IO 673 """ 674 pattern = params[0] 675 676 command.cooker.testCookerCommandEvent(pattern) 677 command.finishAsyncCommand() 678 testCookerCommandEvent.needcache = False 679 680 def findConfigFilePath(self, command, params): 681 """ 682 Find the path of the requested configuration file 683 """ 684 configfile = params[0] 685 686 command.cooker.findConfigFilePath(configfile) 687 command.finishAsyncCommand() 688 findConfigFilePath.needcache = False 689 690 def showVersions(self, command, params): 691 """ 692 Show the currently selected versions 693 """ 694 command.cooker.showVersions() 695 command.finishAsyncCommand() 696 showVersions.needcache = True 697 698 def showEnvironmentTarget(self, command, params): 699 """ 700 Print the environment of a target recipe 701 (needs the cache to work out which recipe to use) 702 """ 703 pkg = params[0] 704 705 command.cooker.showEnvironment(None, pkg) 706 command.finishAsyncCommand() 707 showEnvironmentTarget.needcache = True 708 709 def showEnvironment(self, command, params): 710 """ 711 Print the standard environment 712 or if specified the environment for a specified recipe 713 """ 714 bfile = params[0] 715 716 command.cooker.showEnvironment(bfile) 717 command.finishAsyncCommand() 718 showEnvironment.needcache = False 719 720 def parseFiles(self, command, params): 721 """ 722 Parse the .bb files 723 """ 724 command.cooker.updateCache() 725 command.finishAsyncCommand() 726 parseFiles.needcache = True 727 728 def compareRevisions(self, command, params): 729 """ 730 Parse the .bb files 731 """ 732 if bb.fetch.fetcher_compare_revisions(command.cooker.data): 733 command.finishAsyncCommand(code=1) 734 else: 735 command.finishAsyncCommand() 736 compareRevisions.needcache = True 737 738 def triggerEvent(self, command, params): 739 """ 740 Trigger a certain event 741 """ 742 event = params[0] 743 bb.event.fire(eval(event), command.cooker.data) 744 command.currentAsyncCommand = None 745 triggerEvent.needcache = False 746 747 def resetCooker(self, command, params): 748 """ 749 Reset the cooker to its initial state, thus forcing a reparse for 750 any async command that has the needcache property set to True 751 """ 752 command.cooker.reset() 753 command.finishAsyncCommand() 754 resetCooker.needcache = False 755 756 def clientComplete(self, command, params): 757 """ 758 Do the right thing when the controlling client exits 759 """ 760 command.cooker.clientComplete() 761 command.finishAsyncCommand() 762 clientComplete.needcache = False 763 764 def findSigInfo(self, command, params): 765 """ 766 Find signature info files via the signature generator 767 """ 768 (mc, pn) = bb.runqueue.split_mc(params[0]) 769 taskname = params[1] 770 sigs = params[2] 771 res = bb.siggen.find_siginfo(pn, taskname, sigs, command.cooker.databuilder.mcdata[mc]) 772 bb.event.fire(bb.event.FindSigInfoResult(res), command.cooker.databuilder.mcdata[mc]) 773 command.finishAsyncCommand() 774 findSigInfo.needcache = False 775