1# 2# SPDX-License-Identifier: GPL-2.0-only 3# 4 5import re 6import subprocess 7from oe.package_manager import * 8 9class DpkgIndexer(Indexer): 10 def _create_configs(self): 11 bb.utils.mkdirhier(self.apt_conf_dir) 12 bb.utils.mkdirhier(os.path.join(self.apt_conf_dir, "lists", "partial")) 13 bb.utils.mkdirhier(os.path.join(self.apt_conf_dir, "apt.conf.d")) 14 bb.utils.mkdirhier(os.path.join(self.apt_conf_dir, "preferences.d")) 15 16 with open(os.path.join(self.apt_conf_dir, "preferences"), 17 "w") as prefs_file: 18 pass 19 with open(os.path.join(self.apt_conf_dir, "sources.list"), 20 "w+") as sources_file: 21 pass 22 23 with open(self.apt_conf_file, "w") as apt_conf: 24 with open(os.path.join(self.d.expand("${STAGING_ETCDIR_NATIVE}"), 25 "apt", "apt.conf.sample")) as apt_conf_sample: 26 for line in apt_conf_sample.read().split("\n"): 27 line = re.sub(r"#ROOTFS#", "/dev/null", line) 28 line = re.sub(r"#APTCONF#", self.apt_conf_dir, line) 29 apt_conf.write(line + "\n") 30 31 def write_index(self): 32 self.apt_conf_dir = os.path.join(self.d.expand("${APTCONF_TARGET}"), 33 "apt-ftparchive") 34 self.apt_conf_file = os.path.join(self.apt_conf_dir, "apt.conf") 35 self._create_configs() 36 37 os.environ['APT_CONFIG'] = self.apt_conf_file 38 39 pkg_archs = self.d.getVar('PACKAGE_ARCHS') 40 if pkg_archs is not None: 41 arch_list = pkg_archs.split() 42 sdk_pkg_archs = self.d.getVar('SDK_PACKAGE_ARCHS') 43 if sdk_pkg_archs is not None: 44 for a in sdk_pkg_archs.split(): 45 if a not in pkg_archs: 46 arch_list.append(a) 47 48 all_mlb_pkg_arch_list = (self.d.getVar('ALL_MULTILIB_PACKAGE_ARCHS') or "").split() 49 arch_list.extend(arch for arch in all_mlb_pkg_arch_list if arch not in arch_list) 50 51 apt_ftparchive = bb.utils.which(os.getenv('PATH'), "apt-ftparchive") 52 gzip = bb.utils.which(os.getenv('PATH'), "gzip") 53 54 index_cmds = [] 55 deb_dirs_found = False 56 index_sign_files = set() 57 for arch in arch_list: 58 arch_dir = os.path.join(self.deploy_dir, arch) 59 if not os.path.isdir(arch_dir): 60 continue 61 62 cmd = "cd %s; PSEUDO_UNLOAD=1 %s packages . > Packages;" % (arch_dir, apt_ftparchive) 63 64 cmd += "%s -fcn Packages > Packages.gz;" % gzip 65 66 release_file = os.path.join(arch_dir, "Release") 67 index_sign_files.add(release_file) 68 69 with open(release_file, "w+") as release: 70 release.write("Label: %s\n" % arch) 71 72 cmd += "PSEUDO_UNLOAD=1 %s release . >> Release" % apt_ftparchive 73 74 index_cmds.append(cmd) 75 76 deb_dirs_found = True 77 78 if not deb_dirs_found: 79 bb.note("There are no packages in %s" % self.deploy_dir) 80 return 81 82 oe.utils.multiprocess_launch(create_index, index_cmds, self.d) 83 if self.d.getVar('PACKAGE_FEED_SIGN') == '1': 84 signer = get_signer(self.d, self.d.getVar('PACKAGE_FEED_GPG_BACKEND')) 85 else: 86 signer = None 87 if signer: 88 for f in index_sign_files: 89 signer.detach_sign(f, 90 self.d.getVar('PACKAGE_FEED_GPG_NAME'), 91 self.d.getVar('PACKAGE_FEED_GPG_PASSPHRASE_FILE'), 92 output_suffix="gpg", 93 use_sha256=True) 94 95class PMPkgsList(PkgsList): 96 97 def list_pkgs(self): 98 cmd = [bb.utils.which(os.getenv('PATH'), "dpkg-query"), 99 "--admindir=%s/var/lib/dpkg" % self.rootfs_dir, 100 "-W"] 101 102 cmd.append("-f=Package: ${Package}\nArchitecture: ${PackageArch}\nVersion: ${Version}\nFile: ${Package}_${Version}_${Architecture}.deb\nDepends: ${Depends}\nRecommends: ${Recommends}\nProvides: ${Provides}\n\n") 103 104 try: 105 cmd_output = subprocess.check_output(cmd, stderr=subprocess.STDOUT).strip().decode("utf-8") 106 except subprocess.CalledProcessError as e: 107 bb.fatal("Cannot get the installed packages list. Command '%s' " 108 "returned %d:\n%s" % (' '.join(cmd), e.returncode, e.output.decode("utf-8"))) 109 110 return opkg_query(cmd_output) 111 112class OpkgDpkgPM(PackageManager): 113 def __init__(self, d, target_rootfs): 114 """ 115 This is an abstract class. Do not instantiate this directly. 116 """ 117 super(OpkgDpkgPM, self).__init__(d, target_rootfs) 118 119 def package_info(self, pkg, cmd): 120 """ 121 Returns a dictionary with the package info. 122 123 This method extracts the common parts for Opkg and Dpkg 124 """ 125 126 try: 127 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).decode("utf-8") 128 except subprocess.CalledProcessError as e: 129 bb.fatal("Unable to list available packages. Command '%s' " 130 "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) 131 return opkg_query(output) 132 133 def extract(self, pkg, pkg_info): 134 """ 135 Returns the path to a tmpdir where resides the contents of a package. 136 137 Deleting the tmpdir is responsability of the caller. 138 139 This method extracts the common parts for Opkg and Dpkg 140 """ 141 142 ar_cmd = bb.utils.which(os.getenv("PATH"), "ar") 143 tar_cmd = bb.utils.which(os.getenv("PATH"), "tar") 144 pkg_path = pkg_info[pkg]["filepath"] 145 146 if not os.path.isfile(pkg_path): 147 bb.fatal("Unable to extract package for '%s'." 148 "File %s doesn't exists" % (pkg, pkg_path)) 149 150 tmp_dir = tempfile.mkdtemp() 151 current_dir = os.getcwd() 152 os.chdir(tmp_dir) 153 data_tar = 'data.tar.xz' 154 155 try: 156 cmd = [ar_cmd, 'x', pkg_path] 157 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) 158 cmd = [tar_cmd, 'xf', data_tar] 159 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) 160 except subprocess.CalledProcessError as e: 161 bb.utils.remove(tmp_dir, recurse=True) 162 bb.fatal("Unable to extract %s package. Command '%s' " 163 "returned %d:\n%s" % (pkg_path, ' '.join(cmd), e.returncode, e.output.decode("utf-8"))) 164 except OSError as e: 165 bb.utils.remove(tmp_dir, recurse=True) 166 bb.fatal("Unable to extract %s package. Command '%s' " 167 "returned %d:\n%s at %s" % (pkg_path, ' '.join(cmd), e.errno, e.strerror, e.filename)) 168 169 bb.note("Extracted %s to %s" % (pkg_path, tmp_dir)) 170 bb.utils.remove(os.path.join(tmp_dir, "debian-binary")) 171 bb.utils.remove(os.path.join(tmp_dir, "control.tar.gz")) 172 os.chdir(current_dir) 173 174 return tmp_dir 175 176 def _handle_intercept_failure(self, registered_pkgs): 177 self.mark_packages("unpacked", registered_pkgs.split()) 178 179class DpkgPM(OpkgDpkgPM): 180 def __init__(self, d, target_rootfs, archs, base_archs, apt_conf_dir=None, deb_repo_workdir="oe-rootfs-repo", filterbydependencies=True): 181 super(DpkgPM, self).__init__(d, target_rootfs) 182 self.deploy_dir = oe.path.join(self.d.getVar('WORKDIR'), deb_repo_workdir) 183 184 create_packages_dir(self.d, self.deploy_dir, d.getVar("DEPLOY_DIR_DEB"), "package_write_deb", filterbydependencies) 185 186 if apt_conf_dir is None: 187 self.apt_conf_dir = self.d.expand("${APTCONF_TARGET}/apt") 188 else: 189 self.apt_conf_dir = apt_conf_dir 190 self.apt_conf_file = os.path.join(self.apt_conf_dir, "apt.conf") 191 self.apt_get_cmd = bb.utils.which(os.getenv('PATH'), "apt-get") 192 self.apt_cache_cmd = bb.utils.which(os.getenv('PATH'), "apt-cache") 193 194 self.apt_args = d.getVar("APT_ARGS") 195 196 self.all_arch_list = archs.split() 197 all_mlb_pkg_arch_list = (self.d.getVar('ALL_MULTILIB_PACKAGE_ARCHS') or "").split() 198 self.all_arch_list.extend(arch for arch in all_mlb_pkg_arch_list if arch not in self.all_arch_list) 199 200 self._create_configs(archs, base_archs) 201 202 self.indexer = DpkgIndexer(self.d, self.deploy_dir) 203 204 def mark_packages(self, status_tag, packages=None): 205 """ 206 This function will change a package's status in /var/lib/dpkg/status file. 207 If 'packages' is None then the new_status will be applied to all 208 packages 209 """ 210 status_file = self.target_rootfs + "/var/lib/dpkg/status" 211 212 with open(status_file, "r") as sf: 213 with open(status_file + ".tmp", "w+") as tmp_sf: 214 if packages is None: 215 tmp_sf.write(re.sub(r"Package: (.*?)\n((?:[^\n]+\n)*?)Status: (.*)(?:unpacked|installed)", 216 r"Package: \1\n\2Status: \3%s" % status_tag, 217 sf.read())) 218 else: 219 if type(packages).__name__ != "list": 220 raise TypeError("'packages' should be a list object") 221 222 status = sf.read() 223 for pkg in packages: 224 status = re.sub(r"Package: %s\n((?:[^\n]+\n)*?)Status: (.*)(?:unpacked|installed)" % pkg, 225 r"Package: %s\n\1Status: \2%s" % (pkg, status_tag), 226 status) 227 228 tmp_sf.write(status) 229 230 bb.utils.rename(status_file + ".tmp", status_file) 231 232 def run_pre_post_installs(self, package_name=None): 233 """ 234 Run the pre/post installs for package "package_name". If package_name is 235 None, then run all pre/post install scriptlets. 236 """ 237 info_dir = self.target_rootfs + "/var/lib/dpkg/info" 238 ControlScript = collections.namedtuple("ControlScript", ["suffix", "name", "argument"]) 239 control_scripts = [ 240 ControlScript(".preinst", "Preinstall", "install"), 241 ControlScript(".postinst", "Postinstall", "configure")] 242 status_file = self.target_rootfs + "/var/lib/dpkg/status" 243 installed_pkgs = [] 244 245 with open(status_file, "r") as status: 246 for line in status.read().split('\n'): 247 m = re.match(r"^Package: (.*)", line) 248 if m is not None: 249 installed_pkgs.append(m.group(1)) 250 251 if package_name is not None and not package_name in installed_pkgs: 252 return 253 254 os.environ['D'] = self.target_rootfs 255 os.environ['OFFLINE_ROOT'] = self.target_rootfs 256 os.environ['IPKG_OFFLINE_ROOT'] = self.target_rootfs 257 os.environ['OPKG_OFFLINE_ROOT'] = self.target_rootfs 258 os.environ['INTERCEPT_DIR'] = self.intercepts_dir 259 os.environ['NATIVE_ROOT'] = self.d.getVar('STAGING_DIR_NATIVE') 260 261 for pkg_name in installed_pkgs: 262 for control_script in control_scripts: 263 p_full = os.path.join(info_dir, pkg_name + control_script.suffix) 264 if os.path.exists(p_full): 265 try: 266 bb.note("Executing %s for package: %s ..." % 267 (control_script.name.lower(), pkg_name)) 268 output = subprocess.check_output([p_full, control_script.argument], 269 stderr=subprocess.STDOUT).decode("utf-8") 270 bb.note(output) 271 except subprocess.CalledProcessError as e: 272 bb.warn("%s for package %s failed with %d:\n%s" % 273 (control_script.name, pkg_name, e.returncode, 274 e.output.decode("utf-8"))) 275 failed_postinsts_abort([pkg_name], self.d.expand("${T}/log.do_${BB_CURRENTTASK}")) 276 277 def update(self): 278 os.environ['APT_CONFIG'] = self.apt_conf_file 279 280 self.deploy_dir_lock() 281 282 cmd = "%s update" % self.apt_get_cmd 283 284 try: 285 subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) 286 except subprocess.CalledProcessError as e: 287 bb.fatal("Unable to update the package index files. Command '%s' " 288 "returned %d:\n%s" % (e.cmd, e.returncode, e.output.decode("utf-8"))) 289 290 self.deploy_dir_unlock() 291 292 def install(self, pkgs, attempt_only=False): 293 if attempt_only and len(pkgs) == 0: 294 return 295 296 os.environ['APT_CONFIG'] = self.apt_conf_file 297 298 cmd = "%s %s install --allow-downgrades --allow-remove-essential --allow-change-held-packages --allow-unauthenticated --no-remove %s" % \ 299 (self.apt_get_cmd, self.apt_args, ' '.join(pkgs)) 300 301 try: 302 bb.note("Installing the following packages: %s" % ' '.join(pkgs)) 303 output = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) 304 bb.note(output.decode("utf-8")) 305 except subprocess.CalledProcessError as e: 306 (bb.fatal, bb.warn)[attempt_only]("Unable to install packages. " 307 "Command '%s' returned %d:\n%s" % 308 (cmd, e.returncode, e.output.decode("utf-8"))) 309 310 # rename *.dpkg-new files/dirs 311 for root, dirs, files in os.walk(self.target_rootfs): 312 for dir in dirs: 313 new_dir = re.sub(r"\.dpkg-new", "", dir) 314 if dir != new_dir: 315 bb.utils.rename(os.path.join(root, dir), 316 os.path.join(root, new_dir)) 317 318 for file in files: 319 new_file = re.sub(r"\.dpkg-new", "", file) 320 if file != new_file: 321 bb.utils.rename(os.path.join(root, file), 322 os.path.join(root, new_file)) 323 324 325 def remove(self, pkgs, with_dependencies=True): 326 if not pkgs: 327 return 328 329 os.environ['D'] = self.target_rootfs 330 os.environ['OFFLINE_ROOT'] = self.target_rootfs 331 os.environ['IPKG_OFFLINE_ROOT'] = self.target_rootfs 332 os.environ['OPKG_OFFLINE_ROOT'] = self.target_rootfs 333 os.environ['INTERCEPT_DIR'] = self.intercepts_dir 334 335 if with_dependencies: 336 os.environ['APT_CONFIG'] = self.apt_conf_file 337 cmd = "%s purge %s" % (self.apt_get_cmd, ' '.join(pkgs)) 338 else: 339 cmd = "%s --admindir=%s/var/lib/dpkg --instdir=%s" \ 340 " -P --force-depends %s" % \ 341 (bb.utils.which(os.getenv('PATH'), "dpkg"), 342 self.target_rootfs, self.target_rootfs, ' '.join(pkgs)) 343 344 try: 345 subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) 346 except subprocess.CalledProcessError as e: 347 bb.fatal("Unable to remove packages. Command '%s' " 348 "returned %d:\n%s" % (e.cmd, e.returncode, e.output.decode("utf-8"))) 349 350 def write_index(self): 351 self.deploy_dir_lock() 352 353 result = self.indexer.write_index() 354 355 self.deploy_dir_unlock() 356 357 if result is not None: 358 bb.fatal(result) 359 360 def insert_feeds_uris(self, feed_uris, feed_base_paths, feed_archs): 361 if feed_uris == "": 362 return 363 364 365 sources_conf = os.path.join("%s/etc/apt/sources.list" 366 % self.target_rootfs) 367 if not os.path.exists(os.path.dirname(sources_conf)): 368 return 369 370 arch_list = [] 371 372 if feed_archs is None: 373 for arch in self.all_arch_list: 374 if not os.path.exists(os.path.join(self.deploy_dir, arch)): 375 continue 376 arch_list.append(arch) 377 else: 378 arch_list = feed_archs.split() 379 380 feed_uris = self.construct_uris(feed_uris.split(), feed_base_paths.split()) 381 382 with open(sources_conf, "w+") as sources_file: 383 for uri in feed_uris: 384 if arch_list: 385 for arch in arch_list: 386 bb.note('Adding dpkg channel at (%s)' % uri) 387 sources_file.write("deb [trusted=yes] %s/%s ./\n" % 388 (uri, arch)) 389 else: 390 bb.note('Adding dpkg channel at (%s)' % uri) 391 sources_file.write("deb [trusted=yes] %s ./\n" % uri) 392 393 def _create_configs(self, archs, base_archs): 394 base_archs = re.sub(r"_", r"-", base_archs) 395 396 if os.path.exists(self.apt_conf_dir): 397 bb.utils.remove(self.apt_conf_dir, True) 398 399 bb.utils.mkdirhier(self.apt_conf_dir) 400 bb.utils.mkdirhier(self.apt_conf_dir + "/lists/partial/") 401 bb.utils.mkdirhier(self.apt_conf_dir + "/apt.conf.d/") 402 bb.utils.mkdirhier(self.apt_conf_dir + "/preferences.d/") 403 404 arch_list = [] 405 for arch in self.all_arch_list: 406 if not os.path.exists(os.path.join(self.deploy_dir, arch)): 407 continue 408 arch_list.append(arch) 409 410 with open(os.path.join(self.apt_conf_dir, "preferences"), "w+") as prefs_file: 411 priority = 801 412 for arch in arch_list: 413 prefs_file.write( 414 "Package: *\n" 415 "Pin: release l=%s\n" 416 "Pin-Priority: %d\n\n" % (arch, priority)) 417 418 priority += 5 419 420 pkg_exclude = self.d.getVar('PACKAGE_EXCLUDE') or "" 421 for pkg in pkg_exclude.split(): 422 prefs_file.write( 423 "Package: %s\n" 424 "Pin: release *\n" 425 "Pin-Priority: -1\n\n" % pkg) 426 427 arch_list.reverse() 428 429 with open(os.path.join(self.apt_conf_dir, "sources.list"), "w+") as sources_file: 430 for arch in arch_list: 431 sources_file.write("deb [trusted=yes] file:%s/ ./\n" % 432 os.path.join(self.deploy_dir, arch)) 433 434 base_arch_list = base_archs.split() 435 multilib_variants = self.d.getVar("MULTILIB_VARIANTS"); 436 for variant in multilib_variants.split(): 437 localdata = bb.data.createCopy(self.d) 438 variant_tune = localdata.getVar("DEFAULTTUNE:virtclass-multilib-" + variant, False) 439 orig_arch = localdata.getVar("DPKG_ARCH") 440 localdata.setVar("DEFAULTTUNE", variant_tune) 441 variant_arch = localdata.getVar("DPKG_ARCH") 442 if variant_arch not in base_arch_list: 443 base_arch_list.append(variant_arch) 444 445 with open(self.apt_conf_file, "w+") as apt_conf: 446 with open(self.d.expand("${STAGING_ETCDIR_NATIVE}/apt/apt.conf.sample")) as apt_conf_sample: 447 for line in apt_conf_sample.read().split("\n"): 448 match_arch = re.match(r" Architecture \".*\";$", line) 449 architectures = "" 450 if match_arch: 451 for base_arch in base_arch_list: 452 architectures += "\"%s\";" % base_arch 453 apt_conf.write(" Architectures {%s};\n" % architectures); 454 apt_conf.write(" Architecture \"%s\";\n" % base_archs) 455 else: 456 line = re.sub(r"#ROOTFS#", self.target_rootfs, line) 457 line = re.sub(r"#APTCONF#", self.apt_conf_dir, line) 458 apt_conf.write(line + "\n") 459 460 target_dpkg_dir = "%s/var/lib/dpkg" % self.target_rootfs 461 bb.utils.mkdirhier(os.path.join(target_dpkg_dir, "info")) 462 463 bb.utils.mkdirhier(os.path.join(target_dpkg_dir, "updates")) 464 465 if not os.path.exists(os.path.join(target_dpkg_dir, "status")): 466 open(os.path.join(target_dpkg_dir, "status"), "w+").close() 467 if not os.path.exists(os.path.join(target_dpkg_dir, "available")): 468 open(os.path.join(target_dpkg_dir, "available"), "w+").close() 469 470 def remove_packaging_data(self): 471 bb.utils.remove(self.target_rootfs + self.d.getVar('opkglibdir'), True) 472 bb.utils.remove(self.target_rootfs + "/var/lib/dpkg/", True) 473 474 def fix_broken_dependencies(self): 475 os.environ['APT_CONFIG'] = self.apt_conf_file 476 477 cmd = "%s %s --allow-unauthenticated -f install" % (self.apt_get_cmd, self.apt_args) 478 479 try: 480 subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) 481 except subprocess.CalledProcessError as e: 482 bb.fatal("Cannot fix broken dependencies. Command '%s' " 483 "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) 484 485 def list_installed(self): 486 return PMPkgsList(self.d, self.target_rootfs).list_pkgs() 487 488 def package_info(self, pkg): 489 """ 490 Returns a dictionary with the package info. 491 """ 492 cmd = "%s show %s" % (self.apt_cache_cmd, pkg) 493 pkg_info = super(DpkgPM, self).package_info(pkg, cmd) 494 495 pkg_arch = pkg_info[pkg]["pkgarch"] 496 pkg_filename = pkg_info[pkg]["filename"] 497 pkg_info[pkg]["filepath"] = \ 498 os.path.join(self.deploy_dir, pkg_arch, pkg_filename) 499 500 return pkg_info 501 502 def extract(self, pkg): 503 """ 504 Returns the path to a tmpdir where resides the contents of a package. 505 506 Deleting the tmpdir is responsability of the caller. 507 """ 508 pkg_info = self.package_info(pkg) 509 if not pkg_info: 510 bb.fatal("Unable to get information for package '%s' while " 511 "trying to extract the package." % pkg) 512 513 tmp_dir = super(DpkgPM, self).extract(pkg, pkg_info) 514 bb.utils.remove(os.path.join(tmp_dir, "data.tar.xz")) 515 516 return tmp_dir 517