1# Extensible SDK 2 3inherit populate_sdk_base 4 5# Used to override TOOLCHAIN_HOST_TASK in the eSDK case 6TOOLCHAIN_HOST_TASK_ESDK = " \ 7 meta-environment-extsdk-${MACHINE} \ 8 " 9 10SDK_RELOCATE_AFTER_INSTALL:task-populate-sdk-ext = "0" 11 12SDK_EXT = "" 13SDK_EXT:task-populate-sdk-ext = "-ext" 14 15# Options are full or minimal 16SDK_EXT_TYPE ?= "full" 17SDK_INCLUDE_PKGDATA ?= "0" 18SDK_INCLUDE_TOOLCHAIN ?= "${@'1' if d.getVar('SDK_EXT_TYPE') == 'full' else '0'}" 19SDK_INCLUDE_NATIVESDK ?= "0" 20SDK_INCLUDE_BUILDTOOLS ?= '1' 21 22SDK_RECRDEP_TASKS ?= "" 23SDK_CUSTOM_TEMPLATECONF ?= "0" 24 25ESDK_LOCALCONF_ALLOW ?= "" 26ESDK_LOCALCONF_REMOVE ?= "CONF_VERSION \ 27 BB_NUMBER_THREADS \ 28 BB_NUMBER_PARSE_THREADS \ 29 PARALLEL_MAKE \ 30 PRSERV_HOST \ 31 SSTATE_MIRRORS \ 32 DL_DIR \ 33 SSTATE_DIR \ 34 TMPDIR \ 35 BB_SERVER_TIMEOUT \ 36 " 37ESDK_CLASS_INHERIT_DISABLE ?= "buildhistory icecc" 38SDK_UPDATE_URL ?= "" 39 40SDK_TARGETS ?= "${PN}" 41 42def get_sdk_install_targets(d, images_only=False): 43 sdk_install_targets = '' 44 if images_only or d.getVar('SDK_EXT_TYPE') != 'minimal': 45 sdk_install_targets = d.getVar('SDK_TARGETS') 46 47 depd = d.getVar('BB_TASKDEPDATA', False) 48 tasklist = bb.build.tasksbetween('do_image_complete', 'do_build', d) 49 tasklist.remove('do_build') 50 for v in depd.values(): 51 if v[1] in tasklist: 52 if v[0] not in sdk_install_targets: 53 sdk_install_targets += ' {}'.format(v[0]) 54 55 if not images_only: 56 if d.getVar('SDK_INCLUDE_PKGDATA') == '1': 57 sdk_install_targets += ' meta-world-pkgdata:do_allpackagedata' 58 if d.getVar('SDK_INCLUDE_TOOLCHAIN') == '1': 59 sdk_install_targets += ' meta-extsdk-toolchain:do_populate_sysroot' 60 61 return sdk_install_targets 62 63get_sdk_install_targets[vardepsexclude] = "BB_TASKDEPDATA" 64 65OE_INIT_ENV_SCRIPT ?= "oe-init-build-env" 66 67# The files from COREBASE that you want preserved in the COREBASE copied 68# into the sdk. This allows someone to have their own setup scripts in 69# COREBASE be preserved as well as untracked files. 70COREBASE_FILES ?= " \ 71 oe-init-build-env \ 72 scripts \ 73 LICENSE \ 74 .templateconf \ 75" 76 77SDK_DIR:task-populate-sdk-ext = "${WORKDIR}/sdk-ext" 78B:task-populate-sdk-ext = "${SDK_DIR}" 79TOOLCHAINEXT_OUTPUTNAME ?= "${SDK_NAME}-toolchain-ext-${SDK_VERSION}" 80TOOLCHAIN_OUTPUTNAME:task-populate-sdk-ext = "${TOOLCHAINEXT_OUTPUTNAME}" 81 82SDK_EXT_TARGET_MANIFEST = "${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.target.manifest" 83SDK_EXT_HOST_MANIFEST = "${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.host.manifest" 84 85python write_target_sdk_ext_manifest () { 86 from oe.sdk import get_extra_sdkinfo 87 sstate_dir = d.expand('${SDK_OUTPUT}/${SDKPATH}/sstate-cache') 88 extra_info = get_extra_sdkinfo(sstate_dir) 89 90 target = d.getVar('TARGET_SYS') 91 target_multimach = d.getVar('MULTIMACH_TARGET_SYS') 92 real_target_multimach = d.getVar('REAL_MULTIMACH_TARGET_SYS') 93 94 pkgs = {} 95 os.makedirs(os.path.dirname(d.getVar('SDK_EXT_TARGET_MANIFEST')), exist_ok=True) 96 with open(d.getVar('SDK_EXT_TARGET_MANIFEST'), 'w') as f: 97 for fn in extra_info['filesizes']: 98 info = fn.split(':') 99 if info[2] in (target, target_multimach, real_target_multimach) \ 100 or info[5] == 'allarch': 101 if not info[1] in pkgs: 102 f.write("%s %s %s\n" % (info[1], info[2], info[3])) 103 pkgs[info[1]] = {} 104} 105python write_host_sdk_ext_manifest () { 106 from oe.sdk import get_extra_sdkinfo 107 sstate_dir = d.expand('${SDK_OUTPUT}/${SDKPATH}/sstate-cache') 108 extra_info = get_extra_sdkinfo(sstate_dir) 109 host = d.getVar('BUILD_SYS') 110 with open(d.getVar('SDK_EXT_HOST_MANIFEST'), 'w') as f: 111 for fn in extra_info['filesizes']: 112 info = fn.split(':') 113 if info[2] == host: 114 f.write("%s %s %s\n" % (info[1], info[2], info[3])) 115} 116 117SDK_POSTPROCESS_COMMAND:append:task-populate-sdk-ext = " write_target_sdk_ext_manifest; write_host_sdk_ext_manifest; " 118 119SDK_TITLE:task-populate-sdk-ext = "${@d.getVar('DISTRO_NAME') or d.getVar('DISTRO')} Extensible SDK" 120 121def clean_esdk_builddir(d, sdkbasepath): 122 """Clean up traces of the fake build for create_filtered_tasklist()""" 123 import shutil 124 cleanpaths = ['cache', 'tmp'] 125 for pth in cleanpaths: 126 fullpth = os.path.join(sdkbasepath, pth) 127 if os.path.isdir(fullpth): 128 shutil.rmtree(fullpth) 129 elif os.path.isfile(fullpth): 130 os.remove(fullpth) 131 132def create_filtered_tasklist(d, sdkbasepath, tasklistfile, conf_initpath): 133 """ 134 Create a filtered list of tasks. Also double-checks that the build system 135 within the SDK basically works and required sstate artifacts are available. 136 """ 137 import tempfile 138 import shutil 139 import oe.copy_buildsystem 140 141 # Create a temporary build directory that we can pass to the env setup script 142 shutil.copyfile(sdkbasepath + '/conf/local.conf', sdkbasepath + '/conf/local.conf.bak') 143 try: 144 with open(sdkbasepath + '/conf/local.conf', 'a') as f: 145 # Force the use of sstate from the build system 146 f.write('\nSSTATE_DIR:forcevariable = "%s"\n' % d.getVar('SSTATE_DIR')) 147 f.write('SSTATE_MIRRORS:forcevariable = "file://universal/(.*) file://universal-4.9/\\1 file://universal-4.9/(.*) file://universal-4.8/\\1"\n') 148 # Ensure TMPDIR is the default so that clean_esdk_builddir() can delete it 149 f.write('TMPDIR:forcevariable = "${TOPDIR}/tmp"\n') 150 f.write('TCLIBCAPPEND:forcevariable = ""\n') 151 # Drop uninative if the build isn't using it (or else NATIVELSBSTRING will 152 # be different and we won't be able to find our native sstate) 153 if not bb.data.inherits_class('uninative', d): 154 f.write('INHERIT:remove = "uninative"\n') 155 156 # Unfortunately the default SDKPATH (or even a custom value) may contain characters that bitbake 157 # will not allow in its COREBASE path, so we need to rename the directory temporarily 158 temp_sdkbasepath = d.getVar('SDK_OUTPUT') + '/tmp-renamed-sdk' 159 # Delete any existing temp dir 160 try: 161 shutil.rmtree(temp_sdkbasepath) 162 except FileNotFoundError: 163 pass 164 bb.utils.rename(sdkbasepath, temp_sdkbasepath) 165 cmdprefix = '. %s .; ' % conf_initpath 166 logfile = d.getVar('WORKDIR') + '/tasklist_bb_log.txt' 167 try: 168 oe.copy_buildsystem.check_sstate_task_list(d, get_sdk_install_targets(d), tasklistfile, cmdprefix=cmdprefix, cwd=temp_sdkbasepath, logfile=logfile) 169 except bb.process.ExecutionError as e: 170 msg = 'Failed to generate filtered task list for extensible SDK:\n%s' % e.stdout.rstrip() 171 if 'attempted to execute unexpectedly and should have been setscened' in e.stdout: 172 msg += '\n----------\n\nNOTE: "attempted to execute unexpectedly and should have been setscened" errors indicate this may be caused by missing sstate artifacts that were likely produced in earlier builds, but have been subsequently deleted for some reason.\n' 173 bb.fatal(msg) 174 bb.utils.rename(temp_sdkbasepath, sdkbasepath) 175 # Clean out residue of running bitbake, which check_sstate_task_list() 176 # will effectively do 177 clean_esdk_builddir(d, sdkbasepath) 178 finally: 179 localconf = sdkbasepath + '/conf/local.conf' 180 if os.path.exists(localconf + '.bak'): 181 os.replace(localconf + '.bak', localconf) 182 183python copy_buildsystem () { 184 import re 185 import shutil 186 import glob 187 import oe.copy_buildsystem 188 189 oe_init_env_script = d.getVar('OE_INIT_ENV_SCRIPT') 190 191 conf_bbpath = '' 192 conf_initpath = '' 193 core_meta_subdir = '' 194 195 # Copy in all metadata layers + bitbake (as repositories) 196 buildsystem = oe.copy_buildsystem.BuildSystem('extensible SDK', d) 197 baseoutpath = d.getVar('SDK_OUTPUT') + '/' + d.getVar('SDKPATH') 198 199 #check if custome templateconf path is set 200 use_custom_templateconf = d.getVar('SDK_CUSTOM_TEMPLATECONF') 201 202 # Determine if we're building a derivative extensible SDK (from devtool build-sdk) 203 derivative = (d.getVar('SDK_DERIVATIVE') or '') == '1' 204 if derivative: 205 workspace_name = 'orig-workspace' 206 else: 207 workspace_name = None 208 209 corebase, sdkbblayers = buildsystem.copy_bitbake_and_layers(baseoutpath + '/layers', workspace_name) 210 conf_bbpath = os.path.join('layers', corebase, 'bitbake') 211 212 for path in os.listdir(baseoutpath + '/layers'): 213 relpath = os.path.join('layers', path, oe_init_env_script) 214 if os.path.exists(os.path.join(baseoutpath, relpath)): 215 conf_initpath = relpath 216 217 relpath = os.path.join('layers', path, 'scripts', 'devtool') 218 if os.path.exists(os.path.join(baseoutpath, relpath)): 219 scriptrelpath = os.path.dirname(relpath) 220 221 relpath = os.path.join('layers', path, 'meta') 222 if os.path.exists(os.path.join(baseoutpath, relpath, 'lib', 'oe')): 223 core_meta_subdir = relpath 224 225 d.setVar('oe_init_build_env_path', conf_initpath) 226 d.setVar('scriptrelpath', scriptrelpath) 227 228 # Write out config file for devtool 229 import configparser 230 config = configparser.SafeConfigParser() 231 config.add_section('General') 232 config.set('General', 'bitbake_subdir', conf_bbpath) 233 config.set('General', 'init_path', conf_initpath) 234 config.set('General', 'core_meta_subdir', core_meta_subdir) 235 config.add_section('SDK') 236 config.set('SDK', 'sdk_targets', d.getVar('SDK_TARGETS')) 237 updateurl = d.getVar('SDK_UPDATE_URL') 238 if updateurl: 239 config.set('SDK', 'updateserver', updateurl) 240 bb.utils.mkdirhier(os.path.join(baseoutpath, 'conf')) 241 with open(os.path.join(baseoutpath, 'conf', 'devtool.conf'), 'w') as f: 242 config.write(f) 243 244 unlockedsigs = os.path.join(baseoutpath, 'conf', 'unlocked-sigs.inc') 245 with open(unlockedsigs, 'w') as f: 246 pass 247 248 # Create a layer for new recipes / appends 249 bbpath = d.getVar('BBPATH') 250 env = os.environ.copy() 251 env['PYTHONDONTWRITEBYTECODE'] = '1' 252 bb.process.run(['devtool', '--bbpath', bbpath, '--basepath', baseoutpath, 'create-workspace', '--create-only', os.path.join(baseoutpath, 'workspace')], env=env) 253 254 # Create bblayers.conf 255 bb.utils.mkdirhier(baseoutpath + '/conf') 256 with open(baseoutpath + '/conf/bblayers.conf', 'w') as f: 257 f.write('# WARNING: this configuration has been automatically generated and in\n') 258 f.write('# most cases should not be edited. If you need more flexibility than\n') 259 f.write('# this configuration provides, it is strongly suggested that you set\n') 260 f.write('# up a proper instance of the full build system and use that instead.\n\n') 261 262 # LCONF_VERSION may not be set, for example when using meta-poky 263 # so don't error if it isn't found 264 lconf_version = d.getVar('LCONF_VERSION', False) 265 if lconf_version is not None: 266 f.write('LCONF_VERSION = "%s"\n\n' % lconf_version) 267 268 f.write('BBPATH = "$' + '{TOPDIR}"\n') 269 f.write('SDKBASEMETAPATH = "$' + '{TOPDIR}"\n') 270 f.write('BBLAYERS := " \\\n') 271 for layerrelpath in sdkbblayers: 272 f.write(' $' + '{SDKBASEMETAPATH}/layers/%s \\\n' % layerrelpath) 273 f.write(' $' + '{SDKBASEMETAPATH}/workspace \\\n') 274 f.write(' "\n') 275 276 # Copy uninative tarball 277 # For now this is where uninative.bbclass expects the tarball 278 if bb.data.inherits_class('uninative', d): 279 uninative_file = d.expand('${UNINATIVE_DLDIR}/' + d.getVarFlag("UNINATIVE_CHECKSUM", d.getVar("BUILD_ARCH")) + '/${UNINATIVE_TARBALL}') 280 uninative_checksum = bb.utils.sha256_file(uninative_file) 281 uninative_outdir = '%s/downloads/uninative/%s' % (baseoutpath, uninative_checksum) 282 bb.utils.mkdirhier(uninative_outdir) 283 shutil.copy(uninative_file, uninative_outdir) 284 285 env_passthrough = (d.getVar('BB_ENV_PASSTHROUGH_ADDITIONS') or '').split() 286 env_passthrough_values = {} 287 288 # Create local.conf 289 builddir = d.getVar('TOPDIR') 290 if derivative and os.path.exists(builddir + '/conf/site.conf'): 291 shutil.copyfile(builddir + '/conf/site.conf', baseoutpath + '/conf/site.conf') 292 if derivative and os.path.exists(builddir + '/conf/auto.conf'): 293 shutil.copyfile(builddir + '/conf/auto.conf', baseoutpath + '/conf/auto.conf') 294 if derivative: 295 shutil.copyfile(builddir + '/conf/local.conf', baseoutpath + '/conf/local.conf') 296 else: 297 local_conf_allowed = (d.getVar('ESDK_LOCALCONF_ALLOW') or '').split() 298 local_conf_remove = (d.getVar('ESDK_LOCALCONF_REMOVE') or '').split() 299 def handle_var(varname, origvalue, op, newlines): 300 if varname in local_conf_remove or (origvalue.strip().startswith('/') and not varname in local_conf_allowed): 301 newlines.append('# Removed original setting of %s\n' % varname) 302 return None, op, 0, True 303 else: 304 if varname in env_passthrough: 305 env_passthrough_values[varname] = origvalue 306 return origvalue, op, 0, True 307 varlist = ['[^#=+ ]*'] 308 oldlines = [] 309 if os.path.exists(builddir + '/conf/site.conf'): 310 with open(builddir + '/conf/site.conf', 'r') as f: 311 oldlines += f.readlines() 312 if os.path.exists(builddir + '/conf/auto.conf'): 313 with open(builddir + '/conf/auto.conf', 'r') as f: 314 oldlines += f.readlines() 315 if os.path.exists(builddir + '/conf/local.conf'): 316 with open(builddir + '/conf/local.conf', 'r') as f: 317 oldlines += f.readlines() 318 (updated, newlines) = bb.utils.edit_metadata(oldlines, varlist, handle_var) 319 320 with open(baseoutpath + '/conf/local.conf', 'w') as f: 321 f.write('# WARNING: this configuration has been automatically generated and in\n') 322 f.write('# most cases should not be edited. If you need more flexibility than\n') 323 f.write('# this configuration provides, it is strongly suggested that you set\n') 324 f.write('# up a proper instance of the full build system and use that instead.\n\n') 325 for line in newlines: 326 if line.strip() and not line.startswith('#'): 327 f.write(line) 328 # Write a newline just in case there's none at the end of the original 329 f.write('\n') 330 331 f.write('TMPDIR = "${TOPDIR}/tmp"\n') 332 f.write('TCLIBCAPPEND = ""\n') 333 f.write('DL_DIR = "${TOPDIR}/downloads"\n') 334 335 if bb.data.inherits_class('uninative', d): 336 f.write('INHERIT += "%s"\n' % 'uninative') 337 f.write('UNINATIVE_CHECKSUM[%s] = "%s"\n\n' % (d.getVar('BUILD_ARCH'), uninative_checksum)) 338 f.write('CONF_VERSION = "%s"\n\n' % d.getVar('CONF_VERSION', False)) 339 340 # Some classes are not suitable for SDK, remove them from INHERIT 341 f.write('INHERIT:remove = "%s"\n' % d.getVar('ESDK_CLASS_INHERIT_DISABLE', False)) 342 343 # Bypass the default connectivity check if any 344 f.write('CONNECTIVITY_CHECK_URIS = ""\n\n') 345 346 # This warning will come out if reverse dependencies for a task 347 # don't have sstate as well as the task itself. We already know 348 # this will be the case for the extensible sdk, so turn off the 349 # warning. 350 f.write('SIGGEN_LOCKEDSIGS_SSTATE_EXISTS_CHECK = "none"\n\n') 351 352 # Warn if the sigs in the locked-signature file don't match 353 # the sig computed from the metadata. 354 f.write('SIGGEN_LOCKEDSIGS_TASKSIG_CHECK = "warn"\n\n') 355 356 # We want to be able to set this without a full reparse 357 f.write('BB_HASHCONFIG_IGNORE_VARS:append = " SIGGEN_UNLOCKED_RECIPES"\n\n') 358 359 # Set up which tasks are ignored for run on install 360 f.write('BB_SETSCENE_ENFORCE_IGNORE_TASKS = "%:* *:do_shared_workdir *:do_rm_work wic-tools:* *:do_addto_recipe_sysroot"\n\n') 361 362 # Hide the config information from bitbake output (since it's fixed within the SDK) 363 f.write('BUILDCFG_HEADER = ""\n\n') 364 365 # Write METADATA_REVISION 366 f.write('METADATA_REVISION = "%s"\n\n' % d.getVar('METADATA_REVISION')) 367 368 f.write('# Provide a flag to indicate we are in the EXT_SDK Context\n') 369 f.write('WITHIN_EXT_SDK = "1"\n\n') 370 371 # Map gcc-dependent uninative sstate cache for installer usage 372 f.write('SSTATE_MIRRORS += " file://universal/(.*) file://universal-4.9/\\1 file://universal-4.9/(.*) file://universal-4.8/\\1"\n\n') 373 374 if d.getVar("PRSERV_HOST"): 375 # Override this, we now include PR data, so it should only point ot the local database 376 f.write('PRSERV_HOST = "localhost:0"\n\n') 377 378 # Allow additional config through sdk-extra.conf 379 fn = bb.cookerdata.findConfigFile('sdk-extra.conf', d) 380 if fn: 381 with open(fn, 'r') as xf: 382 for line in xf: 383 f.write(line) 384 385 # If you define a sdk_extraconf() function then it can contain additional config 386 # (Though this is awkward; sdk-extra.conf should probably be used instead) 387 extraconf = (d.getVar('sdk_extraconf') or '').strip() 388 if extraconf: 389 # Strip off any leading / trailing spaces 390 for line in extraconf.splitlines(): 391 f.write(line.strip() + '\n') 392 393 f.write('require conf/locked-sigs.inc\n') 394 f.write('require conf/unlocked-sigs.inc\n') 395 396 # Copy multiple configurations if they exist in the users config directory 397 if d.getVar('BBMULTICONFIG') is not None: 398 bb.utils.mkdirhier(os.path.join(baseoutpath, 'conf', 'multiconfig')) 399 for mc in d.getVar('BBMULTICONFIG').split(): 400 dest_stub = "/conf/multiconfig/%s.conf" % (mc,) 401 if os.path.exists(builddir + dest_stub): 402 shutil.copyfile(builddir + dest_stub, baseoutpath + dest_stub) 403 404 if os.path.exists(builddir + '/cache/bb_unihashes.dat'): 405 bb.parse.siggen.save_unitaskhashes() 406 bb.utils.mkdirhier(os.path.join(baseoutpath, 'cache')) 407 shutil.copyfile(builddir + '/cache/bb_unihashes.dat', baseoutpath + '/cache/bb_unihashes.dat') 408 409 # If PR Service is in use, we need to export this as well 410 bb.note('Do we have a pr database?') 411 if d.getVar("PRSERV_HOST"): 412 bb.note('Writing PR database...') 413 # Based on the code in classes/prexport.bbclass 414 import oe.prservice 415 #dump meta info of tables 416 localdata = d.createCopy() 417 localdata.setVar('PRSERV_DUMPOPT_COL', "1") 418 localdata.setVar('PRSERV_DUMPDIR', os.path.join(baseoutpath, 'conf')) 419 localdata.setVar('PRSERV_DUMPFILE', '${PRSERV_DUMPDIR}/prserv.inc') 420 421 bb.note('PR Database write to %s' % (localdata.getVar('PRSERV_DUMPFILE'))) 422 423 retval = oe.prservice.prserv_dump_db(localdata) 424 if not retval: 425 bb.error("prexport_handler: export failed!") 426 return 427 (metainfo, datainfo) = retval 428 oe.prservice.prserv_export_tofile(localdata, metainfo, datainfo, True) 429 430 # Use templateconf.cfg file from builddir if exists 431 if os.path.exists(builddir + '/conf/templateconf.cfg') and use_custom_templateconf == '1': 432 shutil.copyfile(builddir + '/conf/templateconf.cfg', baseoutpath + '/conf/templateconf.cfg') 433 else: 434 # Write a templateconf.cfg 435 with open(baseoutpath + '/conf/templateconf.cfg', 'w') as f: 436 f.write('meta/conf\n') 437 438 # Ensure any variables set from the external environment (by way of 439 # BB_ENV_PASSTHROUGH_ADDITIONS) are set in the SDK's configuration 440 extralines = [] 441 for name, value in env_passthrough_values.items(): 442 actualvalue = d.getVar(name) or '' 443 if value != actualvalue: 444 extralines.append('%s = "%s"\n' % (name, actualvalue)) 445 if extralines: 446 with open(baseoutpath + '/conf/local.conf', 'a') as f: 447 f.write('\n') 448 f.write('# Extra settings from environment:\n') 449 for line in extralines: 450 f.write(line) 451 f.write('\n') 452 453 # Filter the locked signatures file to just the sstate tasks we are interested in 454 excluded_targets = get_sdk_install_targets(d, images_only=True) 455 sigfile = d.getVar('WORKDIR') + '/locked-sigs.inc' 456 lockedsigs_pruned = baseoutpath + '/conf/locked-sigs.inc' 457 #nativesdk-only sigfile to merge into locked-sigs.inc 458 sdk_include_nativesdk = (d.getVar("SDK_INCLUDE_NATIVESDK") == '1') 459 nativesigfile = d.getVar('WORKDIR') + '/locked-sigs_nativesdk.inc' 460 nativesigfile_pruned = d.getVar('WORKDIR') + '/locked-sigs_nativesdk_pruned.inc' 461 462 if sdk_include_nativesdk: 463 oe.copy_buildsystem.prune_lockedsigs([], 464 excluded_targets.split(), 465 nativesigfile, 466 True, 467 nativesigfile_pruned) 468 469 oe.copy_buildsystem.merge_lockedsigs([], 470 sigfile, 471 nativesigfile_pruned, 472 sigfile) 473 474 oe.copy_buildsystem.prune_lockedsigs([], 475 excluded_targets.split(), 476 sigfile, 477 False, 478 lockedsigs_pruned) 479 480 sstate_out = baseoutpath + '/sstate-cache' 481 bb.utils.remove(sstate_out, True) 482 483 # uninative.bbclass sets NATIVELSBSTRING to 'universal%s' % oe.utils.host_gcc_version(d) 484 fixedlsbstring = "universal%s" % oe.utils.host_gcc_version(d) 485 486 sdk_include_toolchain = (d.getVar('SDK_INCLUDE_TOOLCHAIN') == '1') 487 sdk_ext_type = d.getVar('SDK_EXT_TYPE') 488 if (sdk_ext_type != 'minimal' or sdk_include_toolchain or derivative) and not sdk_include_nativesdk: 489 # Create the filtered task list used to generate the sstate cache shipped with the SDK 490 tasklistfn = d.getVar('WORKDIR') + '/tasklist.txt' 491 create_filtered_tasklist(d, baseoutpath, tasklistfn, conf_initpath) 492 else: 493 tasklistfn = None 494 495 if os.path.exists(builddir + '/cache/bb_unihashes.dat'): 496 bb.parse.siggen.save_unitaskhashes() 497 bb.utils.mkdirhier(os.path.join(baseoutpath, 'cache')) 498 shutil.copyfile(builddir + '/cache/bb_unihashes.dat', baseoutpath + '/cache/bb_unihashes.dat') 499 500 # Add packagedata if enabled 501 if d.getVar('SDK_INCLUDE_PKGDATA') == '1': 502 lockedsigs_base = d.getVar('WORKDIR') + '/locked-sigs-base.inc' 503 lockedsigs_copy = d.getVar('WORKDIR') + '/locked-sigs-copy.inc' 504 shutil.move(lockedsigs_pruned, lockedsigs_base) 505 oe.copy_buildsystem.merge_lockedsigs(['do_packagedata'], 506 lockedsigs_base, 507 d.getVar('STAGING_DIR_HOST') + '/world-pkgdata/locked-sigs-pkgdata.inc', 508 lockedsigs_pruned, 509 lockedsigs_copy) 510 511 if sdk_include_toolchain: 512 lockedsigs_base = d.getVar('WORKDIR') + '/locked-sigs-base2.inc' 513 lockedsigs_toolchain = d.expand("${STAGING_DIR}/${TUNE_PKGARCH}/meta-extsdk-toolchain/locked-sigs/locked-sigs-extsdk-toolchain.inc") 514 shutil.move(lockedsigs_pruned, lockedsigs_base) 515 oe.copy_buildsystem.merge_lockedsigs([], 516 lockedsigs_base, 517 lockedsigs_toolchain, 518 lockedsigs_pruned) 519 oe.copy_buildsystem.create_locked_sstate_cache(lockedsigs_toolchain, 520 d.getVar('SSTATE_DIR'), 521 sstate_out, d, 522 fixedlsbstring, 523 filterfile=tasklistfn) 524 525 if sdk_ext_type == 'minimal': 526 if derivative: 527 # Assume the user is not going to set up an additional sstate 528 # mirror, thus we need to copy the additional artifacts (from 529 # workspace recipes) into the derivative SDK 530 lockedsigs_orig = d.getVar('TOPDIR') + '/conf/locked-sigs.inc' 531 if os.path.exists(lockedsigs_orig): 532 lockedsigs_extra = d.getVar('WORKDIR') + '/locked-sigs-extra.inc' 533 oe.copy_buildsystem.merge_lockedsigs(None, 534 lockedsigs_orig, 535 lockedsigs_pruned, 536 None, 537 lockedsigs_extra) 538 oe.copy_buildsystem.create_locked_sstate_cache(lockedsigs_extra, 539 d.getVar('SSTATE_DIR'), 540 sstate_out, d, 541 fixedlsbstring, 542 filterfile=tasklistfn) 543 else: 544 oe.copy_buildsystem.create_locked_sstate_cache(lockedsigs_pruned, 545 d.getVar('SSTATE_DIR'), 546 sstate_out, d, 547 fixedlsbstring, 548 filterfile=tasklistfn) 549 550 # We don't need sstate do_package files 551 for root, dirs, files in os.walk(sstate_out): 552 for name in files: 553 if name.endswith("_package.tar.zst"): 554 f = os.path.join(root, name) 555 os.remove(f) 556 557 # Write manifest file 558 # Note: at the moment we cannot include the env setup script here to keep 559 # it updated, since it gets modified during SDK installation (see 560 # sdk_ext_postinst() below) thus the checksum we take here would always 561 # be different. 562 manifest_file_list = ['conf/*'] 563 if d.getVar('BBMULTICONFIG') is not None: 564 manifest_file_list.append('conf/multiconfig/*') 565 566 esdk_manifest_excludes = (d.getVar('ESDK_MANIFEST_EXCLUDES') or '').split() 567 esdk_manifest_excludes_list = [] 568 for exclude_item in esdk_manifest_excludes: 569 esdk_manifest_excludes_list += glob.glob(os.path.join(baseoutpath, exclude_item)) 570 manifest_file = os.path.join(baseoutpath, 'conf', 'sdk-conf-manifest') 571 with open(manifest_file, 'w') as f: 572 for item in manifest_file_list: 573 for fn in glob.glob(os.path.join(baseoutpath, item)): 574 if fn == manifest_file or os.path.isdir(fn): 575 continue 576 if fn in esdk_manifest_excludes_list: 577 continue 578 chksum = bb.utils.sha256_file(fn) 579 f.write('%s\t%s\n' % (chksum, os.path.relpath(fn, baseoutpath))) 580} 581 582def get_current_buildtools(d): 583 """Get the file name of the current buildtools installer""" 584 import glob 585 btfiles = glob.glob(os.path.join(d.getVar('SDK_DEPLOY'), '*-buildtools-nativesdk-standalone-*.sh')) 586 btfiles.sort(key=os.path.getctime) 587 return os.path.basename(btfiles[-1]) 588 589def get_sdk_required_utilities(buildtools_fn, d): 590 """Find required utilities that aren't provided by the buildtools""" 591 sanity_required_utilities = (d.getVar('SANITY_REQUIRED_UTILITIES') or '').split() 592 sanity_required_utilities.append(d.expand('${BUILD_PREFIX}gcc')) 593 sanity_required_utilities.append(d.expand('${BUILD_PREFIX}g++')) 594 if buildtools_fn: 595 buildtools_installer = os.path.join(d.getVar('SDK_DEPLOY'), buildtools_fn) 596 filelist, _ = bb.process.run('%s -l' % buildtools_installer) 597 else: 598 buildtools_installer = None 599 filelist = "" 600 localdata = bb.data.createCopy(d) 601 localdata.setVar('SDKPATH', '.') 602 sdkpathnative = localdata.getVar('SDKPATHNATIVE') 603 sdkbindirs = [localdata.getVar('bindir_nativesdk'), 604 localdata.getVar('sbindir_nativesdk'), 605 localdata.getVar('base_bindir_nativesdk'), 606 localdata.getVar('base_sbindir_nativesdk')] 607 for line in filelist.splitlines(): 608 splitline = line.split() 609 if len(splitline) > 5: 610 fn = splitline[5] 611 if not fn.startswith('./'): 612 fn = './%s' % fn 613 if fn.startswith(sdkpathnative): 614 relpth = '/' + os.path.relpath(fn, sdkpathnative) 615 for bindir in sdkbindirs: 616 if relpth.startswith(bindir): 617 relpth = os.path.relpath(relpth, bindir) 618 if relpth in sanity_required_utilities: 619 sanity_required_utilities.remove(relpth) 620 break 621 return ' '.join(sanity_required_utilities) 622 623install_tools() { 624 install -d ${SDK_OUTPUT}/${SDKPATHNATIVE}${bindir_nativesdk} 625 scripts="devtool recipetool oe-find-native-sysroot runqemu* wic" 626 for script in $scripts; do 627 for scriptfn in `find ${SDK_OUTPUT}/${SDKPATH}/${scriptrelpath} -maxdepth 1 -executable -name "$script"`; do 628 targetscriptfn="${SDK_OUTPUT}/${SDKPATHNATIVE}${bindir_nativesdk}/$(basename $scriptfn)" 629 test -e ${targetscriptfn} || ln -rs ${scriptfn} ${targetscriptfn} 630 done 631 done 632 # We can't use the same method as above because files in the sysroot won't exist at this point 633 # (they get populated from sstate on installation) 634 unfsd_path="${SDK_OUTPUT}/${SDKPATHNATIVE}${bindir_nativesdk}/unfsd" 635 if [ "${SDK_INCLUDE_TOOLCHAIN}" = "1" -a ! -e $unfsd_path ] ; then 636 binrelpath=${@os.path.relpath(d.getVar('STAGING_BINDIR_NATIVE'), d.getVar('TMPDIR'))} 637 ln -rs ${SDK_OUTPUT}/${SDKPATH}/tmp/$binrelpath/unfsd $unfsd_path 638 fi 639 touch ${SDK_OUTPUT}/${SDKPATH}/.devtoolbase 640 641 # find latest buildtools-tarball and install it 642 if [ -n "${SDK_BUILDTOOLS_INSTALLER}" ]; then 643 install ${SDK_DEPLOY}/${SDK_BUILDTOOLS_INSTALLER} ${SDK_OUTPUT}/${SDKPATH} 644 fi 645 646 install -m 0644 ${COREBASE}/meta/files/ext-sdk-prepare.py ${SDK_OUTPUT}/${SDKPATH} 647} 648do_populate_sdk_ext[file-checksums] += "${COREBASE}/meta/files/ext-sdk-prepare.py:True" 649 650sdk_ext_preinst() { 651 # Since bitbake won't run as root it doesn't make sense to try and install 652 # the extensible sdk as root. 653 if [ "`id -u`" = "0" ]; then 654 echo "ERROR: The extensible sdk cannot be installed as root." 655 exit 1 656 fi 657 if ! command -v locale > /dev/null; then 658 echo "ERROR: The installer requires the locale command, please install it first" 659 exit 1 660 fi 661 # Check setting of LC_ALL set above 662 canonicalised_locale=`echo $LC_ALL | sed 's/UTF-8/utf8/'` 663 if ! locale -a | grep -q $canonicalised_locale ; then 664 echo "ERROR: the installer requires the $LC_ALL locale to be installed (but not selected), please install it first" 665 exit 1 666 fi 667 # The relocation script used by buildtools installer requires python 668 if ! command -v python3 > /dev/null; then 669 echo "ERROR: The installer requires python3, please install it first" 670 exit 1 671 fi 672 missing_utils="" 673 for util in ${SDK_REQUIRED_UTILITIES}; do 674 if ! command -v $util > /dev/null; then 675 missing_utils="$missing_utils $util" 676 fi 677 done 678 if [ -n "$missing_utils" ] ; then 679 echo "ERROR: the SDK requires the following missing utilities, please install them: $missing_utils" 680 exit 1 681 fi 682 SDK_EXTENSIBLE="1" 683 if [ "$publish" = "1" ] && [ "${SDK_EXT_TYPE}" = "minimal" ] ; then 684 EXTRA_TAR_OPTIONS="$EXTRA_TAR_OPTIONS --exclude=sstate-cache" 685 fi 686} 687SDK_PRE_INSTALL_COMMAND:task-populate-sdk-ext = "${sdk_ext_preinst}" 688 689# FIXME this preparation should be done as part of the SDK construction 690sdk_ext_postinst() { 691 printf "\nExtracting buildtools...\n" 692 cd $target_sdk_dir 693 env_setup_script="$target_sdk_dir/environment-setup-${REAL_MULTIMACH_TARGET_SYS}" 694 if [ -n "${SDK_BUILDTOOLS_INSTALLER}" ]; then 695 printf "buildtools\ny" | ./${SDK_BUILDTOOLS_INSTALLER} > buildtools.log || { printf 'ERROR: buildtools installation failed:\n' ; cat buildtools.log ; echo "printf 'ERROR: this SDK was not fully installed and needs reinstalling\n'" >> $env_setup_script ; exit 1 ; } 696 697 # Delete the buildtools tar file since it won't be used again 698 rm -f ./${SDK_BUILDTOOLS_INSTALLER} 699 # We don't need the log either since it succeeded 700 rm -f buildtools.log 701 702 # Make sure when the user sets up the environment, they also get 703 # the buildtools-tarball tools in their path. 704 echo "# Save and reset OECORE_NATIVE_SYSROOT as buildtools may change it" >> $env_setup_script 705 echo "SAVED=\"\$OECORE_NATIVE_SYSROOT\"" >> $env_setup_script 706 echo ". $target_sdk_dir/buildtools/environment-setup*" >> $env_setup_script 707 echo "OECORE_NATIVE_SYSROOT=\"\$SAVED\"" >> $env_setup_script 708 fi 709 710 # Allow bitbake environment setup to be ran as part of this sdk. 711 echo "export OE_SKIP_SDK_CHECK=1" >> $env_setup_script 712 # Work around runqemu not knowing how to get this information within the eSDK 713 echo "export DEPLOY_DIR_IMAGE=$target_sdk_dir/tmp/${@os.path.relpath(d.getVar('DEPLOY_DIR_IMAGE'), d.getVar('TMPDIR'))}" >> $env_setup_script 714 715 # A bit of another hack, but we need this in the path only for devtool 716 # so put it at the end of $PATH. 717 echo "export PATH=\"$target_sdk_dir/sysroots/${SDK_SYS}${bindir_nativesdk}:\$PATH\"" >> $env_setup_script 718 719 echo "printf 'SDK environment now set up; additionally you may now run devtool to perform development tasks.\nRun devtool --help for further details.\n'" >> $env_setup_script 720 721 # Warn if trying to use external bitbake and the ext SDK together 722 echo "(which bitbake > /dev/null 2>&1 && echo 'WARNING: attempting to use the extensible SDK in an environment set up to run bitbake - this may lead to unexpected results. Please source this script in a new shell session instead.') || true" >> $env_setup_script 723 724 if [ "$prepare_buildsystem" != "no" -a -n "${SDK_BUILDTOOLS_INSTALLER}" ]; then 725 printf "Preparing build system...\n" 726 # dash which is /bin/sh on Ubuntu will not preserve the 727 # current working directory when first ran, nor will it set $1 when 728 # sourcing a script. That is why this has to look so ugly. 729 LOGFILE="$target_sdk_dir/preparing_build_system.log" 730 sh -c ". buildtools/environment-setup* > $LOGFILE && cd $target_sdk_dir/`dirname ${oe_init_build_env_path}` && set $target_sdk_dir && . $target_sdk_dir/${oe_init_build_env_path} $target_sdk_dir >> $LOGFILE && python3 $target_sdk_dir/ext-sdk-prepare.py $LOGFILE '${SDK_INSTALL_TARGETS}'" || { echo "printf 'ERROR: this SDK was not fully installed and needs reinstalling\n'" >> $env_setup_script ; exit 1 ; } 731 fi 732 if [ -e $target_sdk_dir/ext-sdk-prepare.py ]; then 733 rm $target_sdk_dir/ext-sdk-prepare.py 734 fi 735 echo done 736} 737 738SDK_POST_INSTALL_COMMAND:task-populate-sdk-ext = "${sdk_ext_postinst}" 739 740SDK_POSTPROCESS_COMMAND:prepend:task-populate-sdk-ext = "copy_buildsystem; install_tools; " 741 742SDK_INSTALL_TARGETS = "" 743fakeroot python do_populate_sdk_ext() { 744 # FIXME hopefully we can remove this restriction at some point, but uninative 745 # currently forces this upon us 746 if d.getVar('SDK_ARCH') != d.getVar('BUILD_ARCH'): 747 bb.fatal('The extensible SDK can currently only be built for the same architecture as the machine being built on - SDK_ARCH is set to %s (likely via setting SDKMACHINE) which is different from the architecture of the build machine (%s). Unable to continue.' % (d.getVar('SDK_ARCH'), d.getVar('BUILD_ARCH'))) 748 749 # FIXME hopefully we can remove this restriction at some point, but the eSDK 750 # can only be built for the primary (default) multiconfig 751 if d.getVar('BB_CURRENT_MC') != 'default': 752 bb.fatal('The extensible SDK can currently only be built for the default multiconfig. Currently trying to build for %s.' % d.getVar('BB_CURRENT_MC')) 753 754 # eSDK dependencies don't use the traditional variables and things don't work properly if they are set 755 d.setVar("TOOLCHAIN_HOST_TASK", "${TOOLCHAIN_HOST_TASK_ESDK}") 756 d.setVar("TOOLCHAIN_TARGET_TASK", "") 757 758 d.setVar('SDK_INSTALL_TARGETS', get_sdk_install_targets(d)) 759 if d.getVar('SDK_INCLUDE_BUILDTOOLS') == '1': 760 buildtools_fn = get_current_buildtools(d) 761 else: 762 buildtools_fn = None 763 d.setVar('SDK_REQUIRED_UTILITIES', get_sdk_required_utilities(buildtools_fn, d)) 764 d.setVar('SDK_BUILDTOOLS_INSTALLER', buildtools_fn) 765 d.setVar('SDKDEPLOYDIR', '${SDKEXTDEPLOYDIR}') 766 # ESDKs have a libc from the buildtools so ensure we don't ship linguas twice 767 d.delVar('SDKIMAGE_LINGUAS') 768 if d.getVar("SDK_INCLUDE_NATIVESDK") == '1': 769 generate_nativesdk_lockedsigs(d) 770 populate_sdk_common(d) 771} 772 773def generate_nativesdk_lockedsigs(d): 774 import oe.copy_buildsystem 775 sigfile = d.getVar('WORKDIR') + '/locked-sigs_nativesdk.inc' 776 oe.copy_buildsystem.generate_locked_sigs(sigfile, d) 777 778def get_ext_sdk_depends(d): 779 # Note: the deps varflag is a list not a string, so we need to specify expand=False 780 deps = d.getVarFlag('do_image_complete', 'deps', False) 781 pn = d.getVar('PN') 782 deplist = ['%s:%s' % (pn, dep) for dep in deps] 783 tasklist = bb.build.tasksbetween('do_image_complete', 'do_build', d) 784 tasklist.append('do_rootfs') 785 for task in tasklist: 786 deplist.extend((d.getVarFlag(task, 'depends') or '').split()) 787 return ' '.join(deplist) 788 789python do_sdk_depends() { 790 # We have to do this separately in its own task so we avoid recursing into 791 # dependencies we don't need to (e.g. buildtools-tarball) and bringing those 792 # into the SDK's sstate-cache 793 import oe.copy_buildsystem 794 sigfile = d.getVar('WORKDIR') + '/locked-sigs.inc' 795 oe.copy_buildsystem.generate_locked_sigs(sigfile, d) 796} 797addtask sdk_depends 798 799do_sdk_depends[dirs] = "${WORKDIR}" 800do_sdk_depends[depends] = "${@get_ext_sdk_depends(d)} meta-extsdk-toolchain:do_populate_sysroot" 801do_sdk_depends[recrdeptask] = "${@d.getVarFlag('do_populate_sdk', 'recrdeptask', False)}" 802do_sdk_depends[recrdeptask] += "do_populate_lic do_package_qa do_populate_sysroot do_deploy ${SDK_RECRDEP_TASKS}" 803do_sdk_depends[rdepends] = "${@' '.join([x + ':do_package_write_${IMAGE_PKGTYPE} ' + x + ':do_packagedata' for x in d.getVar('TOOLCHAIN_HOST_TASK_ESDK').split()])}" 804 805do_populate_sdk_ext[dirs] = "${@d.getVarFlag('do_populate_sdk', 'dirs', False)}" 806 807do_populate_sdk_ext[depends] = "${@d.getVarFlag('do_populate_sdk', 'depends', False)} \ 808 ${@'buildtools-tarball:do_populate_sdk' if d.getVar('SDK_INCLUDE_BUILDTOOLS') == '1' else ''} \ 809 ${@'meta-world-pkgdata:do_collect_packagedata' if d.getVar('SDK_INCLUDE_PKGDATA') == '1' else ''} \ 810 ${@'meta-extsdk-toolchain:do_locked_sigs' if d.getVar('SDK_INCLUDE_TOOLCHAIN') == '1' else ''}" 811 812# We must avoid depending on do_build here if rm_work.bbclass is active, 813# because otherwise do_rm_work may run before do_populate_sdk_ext itself. 814# We can't mark do_populate_sdk_ext and do_sdk_depends as having to 815# run before do_rm_work, because then they would also run as part 816# of normal builds. 817do_populate_sdk_ext[rdepends] += "${@' '.join([x + ':' + (d.getVar('RM_WORK_BUILD_WITHOUT') or 'do_build') for x in d.getVar('SDK_TARGETS').split()])}" 818 819# Make sure code changes can result in rebuild 820do_populate_sdk_ext[vardeps] += "copy_buildsystem \ 821 sdk_ext_postinst" 822 823# Since any change in the metadata of any layer should cause a rebuild of the 824# sdk(since the layers are put in the sdk) set the task to nostamp so it 825# always runs. 826do_populate_sdk_ext[nostamp] = "1" 827 828SDKEXTDEPLOYDIR = "${WORKDIR}/deploy-${PN}-populate-sdk-ext" 829 830SSTATETASKS += "do_populate_sdk_ext" 831SSTATE_SKIP_CREATION:task-populate-sdk-ext = '1' 832do_populate_sdk_ext[cleandirs] = "${SDKEXTDEPLOYDIR}" 833do_populate_sdk_ext[sstate-inputdirs] = "${SDKEXTDEPLOYDIR}" 834do_populate_sdk_ext[sstate-outputdirs] = "${SDK_DEPLOY}" 835do_populate_sdk_ext[stamp-extra-info] = "${MACHINE_ARCH}" 836 837addtask populate_sdk_ext after do_sdk_depends 838