1# 2# SPDX-License-Identifier: GPL-2.0-only 3# 4from abc import ABCMeta, abstractmethod 5from oe.utils import execute_pre_post_process 6from oe.package_manager import * 7from oe.manifest import * 8import oe.path 9import shutil 10import os 11import subprocess 12import re 13 14class Rootfs(object, metaclass=ABCMeta): 15 """ 16 This is an abstract class. Do not instantiate this directly. 17 """ 18 19 def __init__(self, d, progress_reporter=None, logcatcher=None): 20 self.d = d 21 self.pm = None 22 self.image_rootfs = self.d.getVar('IMAGE_ROOTFS') 23 self.deploydir = self.d.getVar('IMGDEPLOYDIR') 24 self.progress_reporter = progress_reporter 25 self.logcatcher = logcatcher 26 27 self.install_order = Manifest.INSTALL_ORDER 28 29 @abstractmethod 30 def _create(self): 31 pass 32 33 @abstractmethod 34 def _get_delayed_postinsts(self): 35 pass 36 37 @abstractmethod 38 def _save_postinsts(self): 39 pass 40 41 @abstractmethod 42 def _log_check(self): 43 pass 44 45 def _log_check_common(self, type, match): 46 # Ignore any lines containing log_check to avoid recursion, and ignore 47 # lines beginning with a + since sh -x may emit code which isn't 48 # actually executed, but may contain error messages 49 excludes = [ 'log_check', r'^\+' ] 50 if hasattr(self, 'log_check_expected_regexes'): 51 excludes.extend(self.log_check_expected_regexes) 52 # Insert custom log_check excludes 53 excludes += [x for x in (self.d.getVar("IMAGE_LOG_CHECK_EXCLUDES") or "").split(" ") if x] 54 excludes = [re.compile(x) for x in excludes] 55 r = re.compile(match) 56 log_path = self.d.expand("${T}/log.do_rootfs") 57 messages = [] 58 with open(log_path, 'r') as log: 59 for line in log: 60 if self.logcatcher and self.logcatcher.contains(line.rstrip()): 61 continue 62 for ee in excludes: 63 m = ee.search(line) 64 if m: 65 break 66 if m: 67 continue 68 69 m = r.search(line) 70 if m: 71 messages.append('[log_check] %s' % line) 72 if messages: 73 if len(messages) == 1: 74 msg = '1 %s message' % type 75 else: 76 msg = '%d %s messages' % (len(messages), type) 77 msg = '[log_check] %s: found %s in the logfile:\n%s' % \ 78 (self.d.getVar('PN'), msg, ''.join(messages)) 79 if type == 'error': 80 bb.fatal(msg) 81 else: 82 bb.warn(msg) 83 84 def _log_check_warn(self): 85 self._log_check_common('warning', '^(warn|Warn|WARNING:)') 86 87 def _log_check_error(self): 88 self._log_check_common('error', self.log_check_regex) 89 90 def _insert_feed_uris(self): 91 if bb.utils.contains("IMAGE_FEATURES", "package-management", 92 True, False, self.d): 93 self.pm.insert_feeds_uris(self.d.getVar('PACKAGE_FEED_URIS') or "", 94 self.d.getVar('PACKAGE_FEED_BASE_PATHS') or "", 95 self.d.getVar('PACKAGE_FEED_ARCHS')) 96 97 98 """ 99 The _cleanup() method should be used to clean-up stuff that we don't really 100 want to end up on target. For example, in the case of RPM, the DB locks. 101 The method is called, once, at the end of create() method. 102 """ 103 @abstractmethod 104 def _cleanup(self): 105 pass 106 107 def _setup_dbg_rootfs(self, dirs): 108 gen_debugfs = self.d.getVar('IMAGE_GEN_DEBUGFS') or '0' 109 if gen_debugfs != '1': 110 return 111 112 bb.note(" Renaming the original rootfs...") 113 try: 114 shutil.rmtree(self.image_rootfs + '-orig') 115 except: 116 pass 117 bb.utils.rename(self.image_rootfs, self.image_rootfs + '-orig') 118 119 bb.note(" Creating debug rootfs...") 120 bb.utils.mkdirhier(self.image_rootfs) 121 122 bb.note(" Copying back package database...") 123 for dir in dirs: 124 if not os.path.isdir(self.image_rootfs + '-orig' + dir): 125 continue 126 bb.utils.mkdirhier(self.image_rootfs + os.path.dirname(dir)) 127 shutil.copytree(self.image_rootfs + '-orig' + dir, self.image_rootfs + dir, symlinks=True) 128 129 # Copy files located in /usr/lib/debug or /usr/src/debug 130 for dir in ["/usr/lib/debug", "/usr/src/debug"]: 131 src = self.image_rootfs + '-orig' + dir 132 if os.path.exists(src): 133 dst = self.image_rootfs + dir 134 bb.utils.mkdirhier(os.path.dirname(dst)) 135 shutil.copytree(src, dst) 136 137 # Copy files with suffix '.debug' or located in '.debug' dir. 138 for root, dirs, files in os.walk(self.image_rootfs + '-orig'): 139 relative_dir = root[len(self.image_rootfs + '-orig'):] 140 for f in files: 141 if f.endswith('.debug') or '/.debug' in relative_dir: 142 bb.utils.mkdirhier(self.image_rootfs + relative_dir) 143 shutil.copy(os.path.join(root, f), 144 self.image_rootfs + relative_dir) 145 146 bb.note(" Install complementary '*-dbg' packages...") 147 self.pm.install_complementary('*-dbg') 148 149 if self.d.getVar('PACKAGE_DEBUG_SPLIT_STYLE') == 'debug-with-srcpkg': 150 bb.note(" Install complementary '*-src' packages...") 151 self.pm.install_complementary('*-src') 152 153 """ 154 Install additional debug packages. Possibility to install additional packages, 155 which are not automatically installed as complementary package of 156 standard one, e.g. debug package of static libraries. 157 """ 158 extra_debug_pkgs = self.d.getVar('IMAGE_INSTALL_DEBUGFS') 159 if extra_debug_pkgs: 160 bb.note(" Install extra debug packages...") 161 self.pm.install(extra_debug_pkgs.split(), True) 162 163 bb.note(" Rename debug rootfs...") 164 try: 165 shutil.rmtree(self.image_rootfs + '-dbg') 166 except: 167 pass 168 bb.utils.rename(self.image_rootfs, self.image_rootfs + '-dbg') 169 170 bb.note(" Restoring original rootfs...") 171 bb.utils.rename(self.image_rootfs + '-orig', self.image_rootfs) 172 173 def _exec_shell_cmd(self, cmd): 174 fakerootcmd = self.d.getVar('FAKEROOT') 175 if fakerootcmd is not None: 176 exec_cmd = [fakerootcmd, cmd] 177 else: 178 exec_cmd = cmd 179 180 try: 181 subprocess.check_output(exec_cmd, stderr=subprocess.STDOUT) 182 except subprocess.CalledProcessError as e: 183 return("Command '%s' returned %d:\n%s" % (e.cmd, e.returncode, e.output)) 184 185 return None 186 187 def create(self): 188 bb.note("###### Generate rootfs #######") 189 pre_process_cmds = self.d.getVar("ROOTFS_PREPROCESS_COMMAND") 190 post_process_cmds = self.d.getVar("ROOTFS_POSTPROCESS_COMMAND") 191 rootfs_post_install_cmds = self.d.getVar('ROOTFS_POSTINSTALL_COMMAND') 192 193 execute_pre_post_process(self.d, pre_process_cmds) 194 195 if self.progress_reporter: 196 self.progress_reporter.next_stage() 197 198 # call the package manager dependent create method 199 self._create() 200 201 sysconfdir = self.image_rootfs + self.d.getVar('sysconfdir') 202 bb.utils.mkdirhier(sysconfdir) 203 with open(sysconfdir + "/version", "w+") as ver: 204 ver.write(self.d.getVar('BUILDNAME') + "\n") 205 206 execute_pre_post_process(self.d, rootfs_post_install_cmds) 207 208 self.pm.run_intercepts() 209 210 execute_pre_post_process(self.d, post_process_cmds) 211 212 if self.progress_reporter: 213 self.progress_reporter.next_stage() 214 215 if bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs", 216 True, False, self.d) and \ 217 not bb.utils.contains("IMAGE_FEATURES", 218 "read-only-rootfs-delayed-postinsts", 219 True, False, self.d): 220 delayed_postinsts = self._get_delayed_postinsts() 221 if delayed_postinsts is not None: 222 bb.fatal("The following packages could not be configured " 223 "offline and rootfs is read-only: %s" % 224 delayed_postinsts) 225 226 if self.d.getVar('USE_DEVFS') != "1": 227 self._create_devfs() 228 229 self._uninstall_unneeded() 230 231 if self.progress_reporter: 232 self.progress_reporter.next_stage() 233 234 self._insert_feed_uris() 235 236 self._run_ldconfig() 237 238 if self.d.getVar('USE_DEPMOD') != "0": 239 self._generate_kernel_module_deps() 240 241 self._cleanup() 242 self._log_check() 243 244 if self.progress_reporter: 245 self.progress_reporter.next_stage() 246 247 248 def _uninstall_unneeded(self): 249 # Remove the run-postinsts package if no delayed postinsts are found 250 delayed_postinsts = self._get_delayed_postinsts() 251 if delayed_postinsts is None: 252 if os.path.exists(self.d.expand("${IMAGE_ROOTFS}${sysconfdir}/init.d/run-postinsts")) or os.path.exists(self.d.expand("${IMAGE_ROOTFS}${systemd_system_unitdir}/run-postinsts.service")): 253 self.pm.remove(["run-postinsts"]) 254 255 image_rorfs = bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs", 256 True, False, self.d) 257 image_rorfs_force = self.d.getVar('FORCE_RO_REMOVE') 258 259 if image_rorfs or image_rorfs_force == "1": 260 # Remove components that we don't need if it's a read-only rootfs 261 unneeded_pkgs = self.d.getVar("ROOTFS_RO_UNNEEDED").split() 262 pkgs_installed = image_list_installed_packages(self.d) 263 # Make sure update-alternatives is removed last. This is 264 # because its database has to available while uninstalling 265 # other packages, allowing alternative symlinks of packages 266 # to be uninstalled or to be managed correctly otherwise. 267 provider = self.d.getVar("VIRTUAL-RUNTIME_update-alternatives") 268 pkgs_to_remove = sorted([pkg for pkg in pkgs_installed if pkg in unneeded_pkgs], key=lambda x: x == provider) 269 270 # update-alternatives provider is removed in its own remove() 271 # call because all package managers do not guarantee the packages 272 # are removed in the order they given in the list (which is 273 # passed to the command line). The sorting done earlier is 274 # utilized to implement the 2-stage removal. 275 if len(pkgs_to_remove) > 1: 276 self.pm.remove(pkgs_to_remove[:-1], False) 277 if len(pkgs_to_remove) > 0: 278 self.pm.remove([pkgs_to_remove[-1]], False) 279 280 if delayed_postinsts: 281 self._save_postinsts() 282 if image_rorfs: 283 bb.warn("There are post install scripts " 284 "in a read-only rootfs") 285 286 post_uninstall_cmds = self.d.getVar("ROOTFS_POSTUNINSTALL_COMMAND") 287 execute_pre_post_process(self.d, post_uninstall_cmds) 288 289 runtime_pkgmanage = bb.utils.contains("IMAGE_FEATURES", "package-management", 290 True, False, self.d) 291 if not runtime_pkgmanage: 292 # Remove the package manager data files 293 self.pm.remove_packaging_data() 294 295 def _run_ldconfig(self): 296 if self.d.getVar('LDCONFIGDEPEND'): 297 bb.note("Executing: ldconfig -r " + self.image_rootfs + " -c new -v -X") 298 self._exec_shell_cmd(['ldconfig', '-r', self.image_rootfs, '-c', 299 'new', '-v', '-X']) 300 301 image_rorfs = bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs", 302 True, False, self.d) 303 ldconfig_in_features = bb.utils.contains("DISTRO_FEATURES", "ldconfig", 304 True, False, self.d) 305 if image_rorfs or not ldconfig_in_features: 306 ldconfig_cache_dir = os.path.join(self.image_rootfs, "var/cache/ldconfig") 307 if os.path.exists(ldconfig_cache_dir): 308 bb.note("Removing ldconfig auxiliary cache...") 309 shutil.rmtree(ldconfig_cache_dir) 310 311 def _check_for_kernel_modules(self, modules_dir): 312 for root, dirs, files in os.walk(modules_dir, topdown=True): 313 for name in files: 314 found_ko = name.endswith((".ko", ".ko.gz", ".ko.xz", ".ko.zst")) 315 if found_ko: 316 return found_ko 317 return False 318 319 def _generate_kernel_module_deps(self): 320 modules_dir = os.path.join(self.image_rootfs, 'lib', 'modules') 321 # if we don't have any modules don't bother to do the depmod 322 if not self._check_for_kernel_modules(modules_dir): 323 bb.note("No Kernel Modules found, not running depmod") 324 return 325 326 kernel_abi_ver_file = oe.path.join(self.d.getVar('PKGDATA_DIR'), "kernel-depmod", 327 'kernel-abiversion') 328 if not os.path.exists(kernel_abi_ver_file): 329 bb.fatal("No kernel-abiversion file found (%s), cannot run depmod, aborting" % kernel_abi_ver_file) 330 331 with open(kernel_abi_ver_file) as f: 332 kernel_ver = f.read().strip(' \n') 333 334 versioned_modules_dir = os.path.join(self.image_rootfs, modules_dir, kernel_ver) 335 336 bb.utils.mkdirhier(versioned_modules_dir) 337 338 self._exec_shell_cmd(['depmodwrapper', '-a', '-b', self.image_rootfs, kernel_ver]) 339 340 """ 341 Create devfs: 342 * IMAGE_DEVICE_TABLE is the old name to an absolute path to a device table file 343 * IMAGE_DEVICE_TABLES is a new name for a file, or list of files, seached 344 for in the BBPATH 345 If neither are specified then the default name of files/device_table-minimal.txt 346 is searched for in the BBPATH (same as the old version.) 347 """ 348 def _create_devfs(self): 349 devtable_list = [] 350 devtable = self.d.getVar('IMAGE_DEVICE_TABLE') 351 if devtable is not None: 352 devtable_list.append(devtable) 353 else: 354 devtables = self.d.getVar('IMAGE_DEVICE_TABLES') 355 if devtables is None: 356 devtables = 'files/device_table-minimal.txt' 357 for devtable in devtables.split(): 358 devtable_list.append("%s" % bb.utils.which(self.d.getVar('BBPATH'), devtable)) 359 360 for devtable in devtable_list: 361 self._exec_shell_cmd(["makedevs", "-r", 362 self.image_rootfs, "-D", devtable]) 363 364 365def get_class_for_type(imgtype): 366 import importlib 367 mod = importlib.import_module('oe.package_manager.' + imgtype + '.rootfs') 368 return mod.PkgRootfs 369 370def variable_depends(d, manifest_dir=None): 371 img_type = d.getVar('IMAGE_PKGTYPE') 372 cls = get_class_for_type(img_type) 373 return cls._depends_list() 374 375def create_rootfs(d, manifest_dir=None, progress_reporter=None, logcatcher=None): 376 env_bkp = os.environ.copy() 377 378 img_type = d.getVar('IMAGE_PKGTYPE') 379 380 cls = get_class_for_type(img_type) 381 cls(d, manifest_dir, progress_reporter, logcatcher).create() 382 os.environ.clear() 383 os.environ.update(env_bkp) 384 385 386def image_list_installed_packages(d, rootfs_dir=None): 387 # Theres no rootfs for baremetal images 388 if bb.data.inherits_class('baremetal-image', d): 389 return "" 390 391 if not rootfs_dir: 392 rootfs_dir = d.getVar('IMAGE_ROOTFS') 393 394 img_type = d.getVar('IMAGE_PKGTYPE') 395 396 import importlib 397 cls = importlib.import_module('oe.package_manager.' + img_type) 398 return cls.PMPkgsList(d, rootfs_dir).list_pkgs() 399 400if __name__ == "__main__": 401 """ 402 We should be able to run this as a standalone script, from outside bitbake 403 environment. 404 """ 405 """ 406 TBD 407 """ 408