1# 2# Copyright BitBake Contributors 3# 4# SPDX-License-Identifier: GPL-2.0-only 5# 6 7import fnmatch 8import logging 9import os 10import shutil 11import sys 12import tempfile 13 14import bb.utils 15 16from bblayers.common import LayerPlugin 17 18logger = logging.getLogger('bitbake-layers') 19 20 21def plugin_init(plugins): 22 return ActionPlugin() 23 24 25class ActionPlugin(LayerPlugin): 26 def do_add_layer(self, args): 27 """Add one or more layers to bblayers.conf.""" 28 layerdirs = [os.path.abspath(ldir) for ldir in args.layerdir] 29 30 for layerdir in layerdirs: 31 if not os.path.exists(layerdir): 32 sys.stderr.write("Specified layer directory %s doesn't exist\n" % layerdir) 33 return 1 34 35 layer_conf = os.path.join(layerdir, 'conf', 'layer.conf') 36 if not os.path.exists(layer_conf): 37 sys.stderr.write("Specified layer directory %s doesn't contain a conf/layer.conf file\n" % layerdir) 38 return 1 39 40 bblayers_conf = os.path.join('conf', 'bblayers.conf') 41 if not os.path.exists(bblayers_conf): 42 sys.stderr.write("Unable to find bblayers.conf\n") 43 return 1 44 45 # Back up bblayers.conf to tempdir before we add layers 46 tempdir = tempfile.mkdtemp() 47 backup = tempdir + "/bblayers.conf.bak" 48 shutil.copy2(bblayers_conf, backup) 49 50 try: 51 notadded, _ = bb.utils.edit_bblayers_conf(bblayers_conf, layerdirs, None) 52 if not (args.force or notadded): 53 try: 54 self.tinfoil.run_command('parseConfiguration') 55 except (bb.tinfoil.TinfoilUIException, bb.BBHandledException): 56 # Restore the back up copy of bblayers.conf 57 shutil.copy2(backup, bblayers_conf) 58 bb.fatal("Parse failure with the specified layer added, exiting.") 59 else: 60 for item in notadded: 61 sys.stderr.write("Specified layer %s is already in BBLAYERS\n" % item) 62 finally: 63 # Remove the back up copy of bblayers.conf 64 shutil.rmtree(tempdir) 65 66 def do_remove_layer(self, args): 67 """Remove one or more layers from bblayers.conf.""" 68 bblayers_conf = os.path.join('conf', 'bblayers.conf') 69 if not os.path.exists(bblayers_conf): 70 sys.stderr.write("Unable to find bblayers.conf\n") 71 return 1 72 73 layerdirs = [] 74 for item in args.layerdir: 75 if item.startswith('*'): 76 layerdir = item 77 elif not '/' in item: 78 layerdir = '*/%s' % item 79 else: 80 layerdir = os.path.abspath(item) 81 layerdirs.append(layerdir) 82 (_, notremoved) = bb.utils.edit_bblayers_conf(bblayers_conf, None, layerdirs) 83 if notremoved: 84 for item in notremoved: 85 sys.stderr.write("No layers matching %s found in BBLAYERS\n" % item) 86 return 1 87 88 def do_flatten(self, args): 89 """flatten layer configuration into a separate output directory. 90 91Takes the specified layers (or all layers in the current layer 92configuration if none are specified) and builds a "flattened" directory 93containing the contents of all layers, with any overlayed recipes removed 94and bbappends appended to the corresponding recipes. Note that some manual 95cleanup may still be necessary afterwards, in particular: 96 97* where non-recipe files (such as patches) are overwritten (the flatten 98 command will show a warning for these) 99* where anything beyond the normal layer setup has been added to 100 layer.conf (only the lowest priority number layer's layer.conf is used) 101* overridden/appended items from bbappends will need to be tidied up 102* when the flattened layers do not have the same directory structure (the 103 flatten command should show a warning when this will cause a problem) 104 105Warning: if you flatten several layers where another layer is intended to 106be used "inbetween" them (in layer priority order) such that recipes / 107bbappends in the layers interact, and then attempt to use the new output 108layer together with that other layer, you may no longer get the same 109build results (as the layer priority order has effectively changed). 110""" 111 if len(args.layer) == 1: 112 logger.error('If you specify layers to flatten you must specify at least two') 113 return 1 114 115 outputdir = args.outputdir 116 if os.path.exists(outputdir) and os.listdir(outputdir): 117 logger.error('Directory %s exists and is non-empty, please clear it out first' % outputdir) 118 return 1 119 120 layers = self.bblayers 121 if len(args.layer) > 2: 122 layernames = args.layer 123 found_layernames = [] 124 found_layerdirs = [] 125 for layerdir in layers: 126 layername = self.get_layer_name(layerdir) 127 if layername in layernames: 128 found_layerdirs.append(layerdir) 129 found_layernames.append(layername) 130 131 for layername in layernames: 132 if not layername in found_layernames: 133 logger.error('Unable to find layer %s in current configuration, please run "%s show-layers" to list configured layers' % (layername, os.path.basename(sys.argv[0]))) 134 return 135 layers = found_layerdirs 136 else: 137 layernames = [] 138 139 # Ensure a specified path matches our list of layers 140 def layer_path_match(path): 141 for layerdir in layers: 142 if path.startswith(os.path.join(layerdir, '')): 143 return layerdir 144 return None 145 146 applied_appends = [] 147 for layer in layers: 148 overlayed = set() 149 for mc in self.tinfoil.cooker.multiconfigs: 150 for f in self.tinfoil.cooker.collections[mc].overlayed.keys(): 151 for of in self.tinfoil.cooker.collections[mc].overlayed[f]: 152 if of.startswith(layer): 153 overlayed.add(of) 154 155 logger.plain('Copying files from %s...' % layer ) 156 for root, dirs, files in os.walk(layer): 157 if '.git' in dirs: 158 dirs.remove('.git') 159 if '.hg' in dirs: 160 dirs.remove('.hg') 161 162 for f1 in files: 163 f1full = os.sep.join([root, f1]) 164 if f1full in overlayed: 165 logger.plain(' Skipping overlayed file %s' % f1full ) 166 else: 167 ext = os.path.splitext(f1)[1] 168 if ext != '.bbappend': 169 fdest = f1full[len(layer):] 170 fdest = os.path.normpath(os.sep.join([outputdir,fdest])) 171 bb.utils.mkdirhier(os.path.dirname(fdest)) 172 if os.path.exists(fdest): 173 if f1 == 'layer.conf' and root.endswith('/conf'): 174 logger.plain(' Skipping layer config file %s' % f1full ) 175 continue 176 else: 177 logger.warning('Overwriting file %s', fdest) 178 bb.utils.copyfile(f1full, fdest) 179 if ext == '.bb': 180 appends = set() 181 for mc in self.tinfoil.cooker.multiconfigs: 182 appends |= set(self.tinfoil.cooker.collections[mc].get_file_appends(f1full)) 183 for append in appends: 184 if layer_path_match(append): 185 logger.plain(' Applying append %s to %s' % (append, fdest)) 186 self.apply_append(append, fdest) 187 applied_appends.append(append) 188 189 # Take care of when some layers are excluded and yet we have included bbappends for those recipes 190 bbappends = set() 191 for mc in self.tinfoil.cooker.multiconfigs: 192 bbappends |= set(self.tinfoil.cooker.collections[mc].bbappends) 193 194 for b in bbappends: 195 (recipename, appendname) = b 196 if appendname not in applied_appends: 197 first_append = None 198 layer = layer_path_match(appendname) 199 if layer: 200 if first_append: 201 self.apply_append(appendname, first_append) 202 else: 203 fdest = appendname[len(layer):] 204 fdest = os.path.normpath(os.sep.join([outputdir,fdest])) 205 bb.utils.mkdirhier(os.path.dirname(fdest)) 206 bb.utils.copyfile(appendname, fdest) 207 first_append = fdest 208 209 # Get the regex for the first layer in our list (which is where the conf/layer.conf file will 210 # have come from) 211 first_regex = None 212 layerdir = layers[0] 213 for layername, pattern, regex, _ in self.tinfoil.cooker.bbfile_config_priorities: 214 if regex.match(os.path.join(layerdir, 'test')): 215 first_regex = regex 216 break 217 218 if first_regex: 219 # Find the BBFILES entries that match (which will have come from this conf/layer.conf file) 220 bbfiles = str(self.tinfoil.config_data.getVar('BBFILES')).split() 221 bbfiles_layer = [] 222 for item in bbfiles: 223 if first_regex.match(item): 224 newpath = os.path.join(outputdir, item[len(layerdir)+1:]) 225 bbfiles_layer.append(newpath) 226 227 if bbfiles_layer: 228 # Check that all important layer files match BBFILES 229 for root, dirs, files in os.walk(outputdir): 230 for f1 in files: 231 ext = os.path.splitext(f1)[1] 232 if ext in ['.bb', '.bbappend']: 233 f1full = os.sep.join([root, f1]) 234 entry_found = False 235 for item in bbfiles_layer: 236 if fnmatch.fnmatch(f1full, item): 237 entry_found = True 238 break 239 if not entry_found: 240 logger.warning("File %s does not match the flattened layer's BBFILES setting, you may need to edit conf/layer.conf or move the file elsewhere" % f1full) 241 242 def get_file_layer(self, filename): 243 layerdir = self.get_file_layerdir(filename) 244 if layerdir: 245 return self.get_layer_name(layerdir) 246 else: 247 return '?' 248 249 def get_file_layerdir(self, filename): 250 layer = bb.utils.get_file_layer(filename, self.tinfoil.config_data) 251 return self.bbfile_collections.get(layer, None) 252 253 def apply_append(self, appendname, recipename): 254 with open(appendname, 'r') as appendfile: 255 with open(recipename, 'a') as recipefile: 256 recipefile.write('\n') 257 recipefile.write('##### bbappended from %s #####\n' % self.get_file_layer(appendname)) 258 recipefile.writelines(appendfile.readlines()) 259 260 def register_commands(self, sp): 261 parser_add_layer = self.add_command(sp, 'add-layer', self.do_add_layer, parserecipes=False) 262 parser_add_layer.add_argument('layerdir', nargs='+', help='Layer directory/directories to add') 263 264 parser_remove_layer = self.add_command(sp, 'remove-layer', self.do_remove_layer, parserecipes=False) 265 parser_remove_layer.add_argument('layerdir', nargs='+', help='Layer directory/directories to remove (wildcards allowed, enclose in quotes to avoid shell expansion)') 266 parser_remove_layer.set_defaults(func=self.do_remove_layer) 267 268 parser_flatten = self.add_command(sp, 'flatten', self.do_flatten) 269 parser_flatten.add_argument('layer', nargs='*', help='Optional layer(s) to flatten (otherwise all are flattened)') 270 parser_flatten.add_argument('outputdir', help='Output directory') 271