1# Recipe creation tool - append plugin 2# 3# Copyright (C) 2015 Intel Corporation 4# 5# SPDX-License-Identifier: GPL-2.0-only 6# 7 8import sys 9import os 10import argparse 11import glob 12import fnmatch 13import re 14import subprocess 15import logging 16import stat 17import shutil 18import scriptutils 19import errno 20from collections import defaultdict 21 22logger = logging.getLogger('recipetool') 23 24tinfoil = None 25 26def tinfoil_init(instance): 27 global tinfoil 28 tinfoil = instance 29 30 31# FIXME guessing when we don't have pkgdata? 32# FIXME mode to create patch rather than directly substitute 33 34class InvalidTargetFileError(Exception): 35 pass 36 37def find_target_file(targetpath, d, pkglist=None): 38 """Find the recipe installing the specified target path, optionally limited to a select list of packages""" 39 import json 40 41 pkgdata_dir = d.getVar('PKGDATA_DIR') 42 43 # The mix between /etc and ${sysconfdir} here may look odd, but it is just 44 # being consistent with usage elsewhere 45 invalidtargets = {'${sysconfdir}/version': '${sysconfdir}/version is written out at image creation time', 46 '/etc/timestamp': '/etc/timestamp is written out at image creation time', 47 '/dev/*': '/dev is handled by udev (or equivalent) and the kernel (devtmpfs)', 48 '/etc/passwd': '/etc/passwd should be managed through the useradd and extrausers classes', 49 '/etc/group': '/etc/group should be managed through the useradd and extrausers classes', 50 '/etc/shadow': '/etc/shadow should be managed through the useradd and extrausers classes', 51 '/etc/gshadow': '/etc/gshadow should be managed through the useradd and extrausers classes', 52 '${sysconfdir}/hostname': '${sysconfdir}/hostname contents should be set by setting hostname:pn-base-files = "value" in configuration',} 53 54 for pthspec, message in invalidtargets.items(): 55 if fnmatch.fnmatchcase(targetpath, d.expand(pthspec)): 56 raise InvalidTargetFileError(d.expand(message)) 57 58 targetpath_re = re.compile(r'\s+(\$D)?%s(\s|$)' % targetpath) 59 60 recipes = defaultdict(list) 61 for root, dirs, files in os.walk(os.path.join(pkgdata_dir, 'runtime')): 62 if pkglist: 63 filelist = pkglist 64 else: 65 filelist = files 66 for fn in filelist: 67 pkgdatafile = os.path.join(root, fn) 68 if pkglist and not os.path.exists(pkgdatafile): 69 continue 70 with open(pkgdatafile, 'r') as f: 71 pn = '' 72 # This does assume that PN comes before other values, but that's a fairly safe assumption 73 for line in f: 74 if line.startswith('PN:'): 75 pn = line.split(': ', 1)[1].strip() 76 elif line.startswith('FILES_INFO'): 77 val = line.split(': ', 1)[1].strip() 78 dictval = json.loads(val) 79 for fullpth in dictval.keys(): 80 if fnmatch.fnmatchcase(fullpth, targetpath): 81 recipes[targetpath].append(pn) 82 elif line.startswith('pkg_preinst:') or line.startswith('pkg_postinst:'): 83 scriptval = line.split(': ', 1)[1].strip().encode('utf-8').decode('unicode_escape') 84 if 'update-alternatives --install %s ' % targetpath in scriptval: 85 recipes[targetpath].append('?%s' % pn) 86 elif targetpath_re.search(scriptval): 87 recipes[targetpath].append('!%s' % pn) 88 return recipes 89 90def _parse_recipe(pn, tinfoil): 91 try: 92 rd = tinfoil.parse_recipe(pn) 93 except bb.providers.NoProvider as e: 94 logger.error(str(e)) 95 return None 96 return rd 97 98def determine_file_source(targetpath, rd): 99 """Assuming we know a file came from a specific recipe, figure out exactly where it came from""" 100 import oe.recipeutils 101 102 # See if it's in do_install for the recipe 103 workdir = rd.getVar('WORKDIR') 104 src_uri = rd.getVar('SRC_URI') 105 srcfile = '' 106 modpatches = [] 107 elements = check_do_install(rd, targetpath) 108 if elements: 109 logger.debug('do_install line:\n%s' % ' '.join(elements)) 110 srcpath = get_source_path(elements) 111 logger.debug('source path: %s' % srcpath) 112 if not srcpath.startswith('/'): 113 # Handle non-absolute path 114 srcpath = os.path.abspath(os.path.join(rd.getVarFlag('do_install', 'dirs').split()[-1], srcpath)) 115 if srcpath.startswith(workdir): 116 # OK, now we have the source file name, look for it in SRC_URI 117 workdirfile = os.path.relpath(srcpath, workdir) 118 # FIXME this is where we ought to have some code in the fetcher, because this is naive 119 for item in src_uri.split(): 120 localpath = bb.fetch2.localpath(item, rd) 121 # Source path specified in do_install might be a glob 122 if fnmatch.fnmatch(os.path.basename(localpath), workdirfile): 123 srcfile = 'file://%s' % localpath 124 elif '/' in workdirfile: 125 if item == 'file://%s' % workdirfile: 126 srcfile = 'file://%s' % localpath 127 128 # Check patches 129 srcpatches = [] 130 patchedfiles = oe.recipeutils.get_recipe_patched_files(rd) 131 for patch, filelist in patchedfiles.items(): 132 for fileitem in filelist: 133 if fileitem[0] == srcpath: 134 srcpatches.append((patch, fileitem[1])) 135 if srcpatches: 136 addpatch = None 137 for patch in srcpatches: 138 if patch[1] == 'A': 139 addpatch = patch[0] 140 else: 141 modpatches.append(patch[0]) 142 if addpatch: 143 srcfile = 'patch://%s' % addpatch 144 145 return (srcfile, elements, modpatches) 146 147def get_source_path(cmdelements): 148 """Find the source path specified within a command""" 149 command = cmdelements[0] 150 if command in ['install', 'cp']: 151 helptext = subprocess.check_output('LC_ALL=C %s --help' % command, shell=True).decode('utf-8') 152 argopts = '' 153 argopt_line_re = re.compile('^-([a-zA-Z0-9]), --[a-z-]+=') 154 for line in helptext.splitlines(): 155 line = line.lstrip() 156 res = argopt_line_re.search(line) 157 if res: 158 argopts += res.group(1) 159 if not argopts: 160 # Fallback 161 if command == 'install': 162 argopts = 'gmoSt' 163 elif command == 'cp': 164 argopts = 't' 165 else: 166 raise Exception('No fallback arguments for command %s' % command) 167 168 skipnext = False 169 for elem in cmdelements[1:-1]: 170 if elem.startswith('-'): 171 if len(elem) > 1 and elem[1] in argopts: 172 skipnext = True 173 continue 174 if skipnext: 175 skipnext = False 176 continue 177 return elem 178 else: 179 raise Exception('get_source_path: no handling for command "%s"') 180 181def get_func_deps(func, d): 182 """Find the function dependencies of a shell function""" 183 deps = bb.codeparser.ShellParser(func, logger).parse_shell(d.getVar(func)) 184 deps |= set((d.getVarFlag(func, "vardeps") or "").split()) 185 funcdeps = [] 186 for dep in deps: 187 if d.getVarFlag(dep, 'func'): 188 funcdeps.append(dep) 189 return funcdeps 190 191def check_do_install(rd, targetpath): 192 """Look at do_install for a command that installs/copies the specified target path""" 193 instpath = os.path.abspath(os.path.join(rd.getVar('D'), targetpath.lstrip('/'))) 194 do_install = rd.getVar('do_install') 195 # Handle where do_install calls other functions (somewhat crudely, but good enough for this purpose) 196 deps = get_func_deps('do_install', rd) 197 for dep in deps: 198 do_install = do_install.replace(dep, rd.getVar(dep)) 199 200 # Look backwards through do_install as we want to catch where a later line (perhaps 201 # from a bbappend) is writing over the top 202 for line in reversed(do_install.splitlines()): 203 line = line.strip() 204 if (line.startswith('install ') and ' -m' in line) or line.startswith('cp '): 205 elements = line.split() 206 destpath = os.path.abspath(elements[-1]) 207 if destpath == instpath: 208 return elements 209 elif destpath.rstrip('/') == os.path.dirname(instpath): 210 # FIXME this doesn't take recursive copy into account; unsure if it's practical to do so 211 srcpath = get_source_path(elements) 212 if fnmatch.fnmatchcase(os.path.basename(instpath), os.path.basename(srcpath)): 213 return elements 214 return None 215 216 217def appendfile(args): 218 import oe.recipeutils 219 220 stdout = '' 221 try: 222 (stdout, _) = bb.process.run('LANG=C file -b %s' % args.newfile, shell=True) 223 if 'cannot open' in stdout: 224 raise bb.process.ExecutionError(stdout) 225 except bb.process.ExecutionError as err: 226 logger.debug('file command returned error: %s' % err) 227 stdout = '' 228 if stdout: 229 logger.debug('file command output: %s' % stdout.rstrip()) 230 if ('executable' in stdout and not 'shell script' in stdout) or 'shared object' in stdout: 231 logger.warning('This file looks like it is a binary or otherwise the output of compilation. If it is, you should consider building it properly instead of substituting a binary file directly.') 232 233 if args.recipe: 234 recipes = {args.targetpath: [args.recipe],} 235 else: 236 try: 237 recipes = find_target_file(args.targetpath, tinfoil.config_data) 238 except InvalidTargetFileError as e: 239 logger.error('%s cannot be handled by this tool: %s' % (args.targetpath, e)) 240 return 1 241 if not recipes: 242 logger.error('Unable to find any package producing path %s - this may be because the recipe packaging it has not been built yet' % args.targetpath) 243 return 1 244 245 alternative_pns = [] 246 postinst_pns = [] 247 248 selectpn = None 249 for targetpath, pnlist in recipes.items(): 250 for pn in pnlist: 251 if pn.startswith('?'): 252 alternative_pns.append(pn[1:]) 253 elif pn.startswith('!'): 254 postinst_pns.append(pn[1:]) 255 elif selectpn: 256 # hit here with multilibs 257 continue 258 else: 259 selectpn = pn 260 261 if not selectpn and len(alternative_pns) == 1: 262 selectpn = alternative_pns[0] 263 logger.error('File %s is an alternative possibly provided by recipe %s but seemingly no other, selecting it by default - you should double check other recipes' % (args.targetpath, selectpn)) 264 265 if selectpn: 266 logger.debug('Selecting recipe %s for file %s' % (selectpn, args.targetpath)) 267 if postinst_pns: 268 logger.warning('%s be modified by postinstall scripts for the following recipes:\n %s\nThis may or may not be an issue depending on what modifications these postinstall scripts make.' % (args.targetpath, '\n '.join(postinst_pns))) 269 rd = _parse_recipe(selectpn, tinfoil) 270 if not rd: 271 # Error message already shown 272 return 1 273 sourcefile, instelements, modpatches = determine_file_source(args.targetpath, rd) 274 sourcepath = None 275 if sourcefile: 276 sourcetype, sourcepath = sourcefile.split('://', 1) 277 logger.debug('Original source file is %s (%s)' % (sourcepath, sourcetype)) 278 if sourcetype == 'patch': 279 logger.warning('File %s is added by the patch %s - you may need to remove or replace this patch in order to replace the file.' % (args.targetpath, sourcepath)) 280 sourcepath = None 281 else: 282 logger.debug('Unable to determine source file, proceeding anyway') 283 if modpatches: 284 logger.warning('File %s is modified by the following patches:\n %s' % (args.targetpath, '\n '.join(modpatches))) 285 286 if instelements and sourcepath: 287 install = None 288 else: 289 # Auto-determine permissions 290 # Check destination 291 binpaths = '${bindir}:${sbindir}:${base_bindir}:${base_sbindir}:${libexecdir}:${sysconfdir}/init.d' 292 perms = '0644' 293 if os.path.abspath(os.path.dirname(args.targetpath)) in rd.expand(binpaths).split(':'): 294 # File is going into a directory normally reserved for executables, so it should be executable 295 perms = '0755' 296 else: 297 # Check source 298 st = os.stat(args.newfile) 299 if st.st_mode & stat.S_IXUSR: 300 perms = '0755' 301 install = {args.newfile: (args.targetpath, perms)} 302 oe.recipeutils.bbappend_recipe(rd, args.destlayer, {args.newfile: sourcepath}, install, wildcardver=args.wildcard_version, machine=args.machine) 303 return 0 304 else: 305 if alternative_pns: 306 logger.error('File %s is an alternative possibly provided by the following recipes:\n %s\nPlease select recipe with -r/--recipe' % (targetpath, '\n '.join(alternative_pns))) 307 elif postinst_pns: 308 logger.error('File %s may be written out in a pre/postinstall script of the following recipes:\n %s\nPlease select recipe with -r/--recipe' % (targetpath, '\n '.join(postinst_pns))) 309 return 3 310 311 312def appendsrc(args, files, rd, extralines=None): 313 import oe.recipeutils 314 315 srcdir = rd.getVar('S') 316 workdir = rd.getVar('WORKDIR') 317 318 import bb.fetch 319 simplified = {} 320 src_uri = rd.getVar('SRC_URI').split() 321 for uri in src_uri: 322 if uri.endswith(';'): 323 uri = uri[:-1] 324 simple_uri = bb.fetch.URI(uri) 325 simple_uri.params = {} 326 simplified[str(simple_uri)] = uri 327 328 copyfiles = {} 329 extralines = extralines or [] 330 for newfile, srcfile in files.items(): 331 src_destdir = os.path.dirname(srcfile) 332 if not args.use_workdir: 333 if rd.getVar('S') == rd.getVar('STAGING_KERNEL_DIR'): 334 srcdir = os.path.join(workdir, 'git') 335 if not bb.data.inherits_class('kernel-yocto', rd): 336 logger.warning('S == STAGING_KERNEL_DIR and non-kernel-yocto, unable to determine path to srcdir, defaulting to ${WORKDIR}/git') 337 src_destdir = os.path.join(os.path.relpath(srcdir, workdir), src_destdir) 338 src_destdir = os.path.normpath(src_destdir) 339 340 source_uri = 'file://{0}'.format(os.path.basename(srcfile)) 341 if src_destdir and src_destdir != '.': 342 source_uri += ';subdir={0}'.format(src_destdir) 343 344 simple = bb.fetch.URI(source_uri) 345 simple.params = {} 346 simple_str = str(simple) 347 if simple_str in simplified: 348 existing = simplified[simple_str] 349 if source_uri != existing: 350 logger.warning('{0!r} is already in SRC_URI, with different parameters: {1!r}, not adding'.format(source_uri, existing)) 351 else: 352 logger.warning('{0!r} is already in SRC_URI, not adding'.format(source_uri)) 353 else: 354 extralines.append('SRC_URI += {0}'.format(source_uri)) 355 copyfiles[newfile] = srcfile 356 357 oe.recipeutils.bbappend_recipe(rd, args.destlayer, copyfiles, None, wildcardver=args.wildcard_version, machine=args.machine, extralines=extralines) 358 359 360def appendsrcfiles(parser, args): 361 recipedata = _parse_recipe(args.recipe, tinfoil) 362 if not recipedata: 363 parser.error('RECIPE must be a valid recipe name') 364 365 files = dict((f, os.path.join(args.destdir, os.path.basename(f))) 366 for f in args.files) 367 return appendsrc(args, files, recipedata) 368 369 370def appendsrcfile(parser, args): 371 recipedata = _parse_recipe(args.recipe, tinfoil) 372 if not recipedata: 373 parser.error('RECIPE must be a valid recipe name') 374 375 if not args.destfile: 376 args.destfile = os.path.basename(args.file) 377 elif args.destfile.endswith('/'): 378 args.destfile = os.path.join(args.destfile, os.path.basename(args.file)) 379 380 return appendsrc(args, {args.file: args.destfile}, recipedata) 381 382 383def layer(layerpath): 384 if not os.path.exists(os.path.join(layerpath, 'conf', 'layer.conf')): 385 raise argparse.ArgumentTypeError('{0!r} must be a path to a valid layer'.format(layerpath)) 386 return layerpath 387 388 389def existing_path(filepath): 390 if not os.path.exists(filepath): 391 raise argparse.ArgumentTypeError('{0!r} must be an existing path'.format(filepath)) 392 return filepath 393 394 395def existing_file(filepath): 396 filepath = existing_path(filepath) 397 if os.path.isdir(filepath): 398 raise argparse.ArgumentTypeError('{0!r} must be a file, not a directory'.format(filepath)) 399 return filepath 400 401 402def destination_path(destpath): 403 if os.path.isabs(destpath): 404 raise argparse.ArgumentTypeError('{0!r} must be a relative path, not absolute'.format(destpath)) 405 return destpath 406 407 408def target_path(targetpath): 409 if not os.path.isabs(targetpath): 410 raise argparse.ArgumentTypeError('{0!r} must be an absolute path, not relative'.format(targetpath)) 411 return targetpath 412 413 414def register_commands(subparsers): 415 common = argparse.ArgumentParser(add_help=False) 416 common.add_argument('-m', '--machine', help='Make bbappend changes specific to a machine only', metavar='MACHINE') 417 common.add_argument('-w', '--wildcard-version', help='Use wildcard to make the bbappend apply to any recipe version', action='store_true') 418 common.add_argument('destlayer', metavar='DESTLAYER', help='Base directory of the destination layer to write the bbappend to', type=layer) 419 420 parser_appendfile = subparsers.add_parser('appendfile', 421 parents=[common], 422 help='Create/update a bbappend to replace a target file', 423 description='Creates a bbappend (or updates an existing one) to replace the specified file that appears in the target system, determining the recipe that packages the file and the required path and name for the bbappend automatically. Note that the ability to determine the recipe packaging a particular file depends upon the recipe\'s do_packagedata task having already run prior to running this command (which it will have when the recipe has been built successfully, which in turn will have happened if one or more of the recipe\'s packages is included in an image that has been built successfully).') 424 parser_appendfile.add_argument('targetpath', help='Path to the file to be replaced (as it would appear within the target image, e.g. /etc/motd)', type=target_path) 425 parser_appendfile.add_argument('newfile', help='Custom file to replace the target file with', type=existing_file) 426 parser_appendfile.add_argument('-r', '--recipe', help='Override recipe to apply to (default is to find which recipe already packages the file)') 427 parser_appendfile.set_defaults(func=appendfile, parserecipes=True) 428 429 common_src = argparse.ArgumentParser(add_help=False, parents=[common]) 430 common_src.add_argument('-W', '--workdir', help='Unpack file into WORKDIR rather than S', dest='use_workdir', action='store_true') 431 common_src.add_argument('recipe', metavar='RECIPE', help='Override recipe to apply to') 432 433 parser = subparsers.add_parser('appendsrcfiles', 434 parents=[common_src], 435 help='Create/update a bbappend to add or replace source files', 436 description='Creates a bbappend (or updates an existing one) to add or replace the specified file in the recipe sources, either those in WORKDIR or those in the source tree. This command lets you specify multiple files with a destination directory, so cannot specify the destination filename. See the `appendsrcfile` command for the other behavior.') 437 parser.add_argument('-D', '--destdir', help='Destination directory (relative to S or WORKDIR, defaults to ".")', default='', type=destination_path) 438 parser.add_argument('files', nargs='+', metavar='FILE', help='File(s) to be added to the recipe sources (WORKDIR or S)', type=existing_path) 439 parser.set_defaults(func=lambda a: appendsrcfiles(parser, a), parserecipes=True) 440 441 parser = subparsers.add_parser('appendsrcfile', 442 parents=[common_src], 443 help='Create/update a bbappend to add or replace a source file', 444 description='Creates a bbappend (or updates an existing one) to add or replace the specified files in the recipe sources, either those in WORKDIR or those in the source tree. This command lets you specify the destination filename, not just destination directory, but only works for one file. See the `appendsrcfiles` command for the other behavior.') 445 parser.add_argument('file', metavar='FILE', help='File to be added to the recipe sources (WORKDIR or S)', type=existing_path) 446 parser.add_argument('destfile', metavar='DESTFILE', nargs='?', help='Destination path (relative to S or WORKDIR, optional)', type=destination_path) 447 parser.set_defaults(func=lambda a: appendsrcfile(parser, a), parserecipes=True) 448