1# 2# SPDX-License-Identifier: GPL-2.0-only 3# 4 5import re 6import shutil 7import subprocess 8from oe.package_manager import * 9 10class OpkgIndexer(Indexer): 11 def write_index(self): 12 arch_vars = ["ALL_MULTILIB_PACKAGE_ARCHS", 13 "SDK_PACKAGE_ARCHS", 14 ] 15 16 opkg_index_cmd = bb.utils.which(os.getenv('PATH'), "opkg-make-index") 17 if self.d.getVar('PACKAGE_FEED_SIGN') == '1': 18 signer = get_signer(self.d, self.d.getVar('PACKAGE_FEED_GPG_BACKEND')) 19 else: 20 signer = None 21 22 if not os.path.exists(os.path.join(self.deploy_dir, "Packages")): 23 open(os.path.join(self.deploy_dir, "Packages"), "w").close() 24 25 index_cmds = set() 26 index_sign_files = set() 27 for arch_var in arch_vars: 28 archs = self.d.getVar(arch_var) 29 if archs is None: 30 continue 31 32 for arch in archs.split(): 33 pkgs_dir = os.path.join(self.deploy_dir, arch) 34 pkgs_file = os.path.join(pkgs_dir, "Packages") 35 36 if not os.path.isdir(pkgs_dir): 37 continue 38 39 if not os.path.exists(pkgs_file): 40 open(pkgs_file, "w").close() 41 42 index_cmds.add('%s --checksum md5 --checksum sha256 -r %s -p %s -m %s' % 43 (opkg_index_cmd, pkgs_file, pkgs_file, pkgs_dir)) 44 45 index_sign_files.add(pkgs_file) 46 47 if len(index_cmds) == 0: 48 bb.note("There are no packages in %s!" % self.deploy_dir) 49 return 50 51 oe.utils.multiprocess_launch(create_index, index_cmds, self.d) 52 53 if signer: 54 feed_sig_type = self.d.getVar('PACKAGE_FEED_GPG_SIGNATURE_TYPE') 55 is_ascii_sig = (feed_sig_type.upper() != "BIN") 56 for f in index_sign_files: 57 signer.detach_sign(f, 58 self.d.getVar('PACKAGE_FEED_GPG_NAME'), 59 self.d.getVar('PACKAGE_FEED_GPG_PASSPHRASE_FILE'), 60 armor=is_ascii_sig) 61 62class PMPkgsList(PkgsList): 63 def __init__(self, d, rootfs_dir): 64 super(PMPkgsList, self).__init__(d, rootfs_dir) 65 config_file = d.getVar("IPKGCONF_TARGET") 66 67 self.opkg_cmd = bb.utils.which(os.getenv('PATH'), "opkg") 68 self.opkg_args = "-f %s -o %s " % (config_file, rootfs_dir) 69 self.opkg_args += self.d.getVar("OPKG_ARGS") 70 71 def list_pkgs(self, format=None): 72 cmd = "%s %s status" % (self.opkg_cmd, self.opkg_args) 73 74 # opkg returns success even when it printed some 75 # "Collected errors:" report to stderr. Mixing stderr into 76 # stdout then leads to random failures later on when 77 # parsing the output. To avoid this we need to collect both 78 # output streams separately and check for empty stderr. 79 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) 80 cmd_output, cmd_stderr = p.communicate() 81 cmd_output = cmd_output.decode("utf-8") 82 cmd_stderr = cmd_stderr.decode("utf-8") 83 if p.returncode or cmd_stderr: 84 bb.fatal("Cannot get the installed packages list. Command '%s' " 85 "returned %d and stderr:\n%s" % (cmd, p.returncode, cmd_stderr)) 86 87 return opkg_query(cmd_output) 88 89 90 91class OpkgDpkgPM(PackageManager): 92 def __init__(self, d, target_rootfs): 93 """ 94 This is an abstract class. Do not instantiate this directly. 95 """ 96 super(OpkgDpkgPM, self).__init__(d, target_rootfs) 97 98 def package_info(self, pkg, cmd): 99 """ 100 Returns a dictionary with the package info. 101 102 This method extracts the common parts for Opkg and Dpkg 103 """ 104 105 proc = subprocess.run(cmd, capture_output=True, encoding="utf-8", shell=True) 106 if proc.returncode: 107 bb.fatal("Unable to list available packages. Command '%s' " 108 "returned %d:\n%s" % (cmd, proc.returncode, proc.stderr)) 109 elif proc.stderr: 110 bb.note("Command '%s' returned stderr: %s" % (cmd, proc.stderr)) 111 112 return opkg_query(proc.stdout) 113 114 def extract(self, pkg, pkg_info): 115 """ 116 Returns the path to a tmpdir where resides the contents of a package. 117 118 Deleting the tmpdir is responsability of the caller. 119 120 This method extracts the common parts for Opkg and Dpkg 121 """ 122 123 ar_cmd = bb.utils.which(os.getenv("PATH"), "ar") 124 tar_cmd = bb.utils.which(os.getenv("PATH"), "tar") 125 pkg_path = pkg_info[pkg]["filepath"] 126 127 if not os.path.isfile(pkg_path): 128 bb.fatal("Unable to extract package for '%s'." 129 "File %s doesn't exists" % (pkg, pkg_path)) 130 131 tmp_dir = tempfile.mkdtemp() 132 current_dir = os.getcwd() 133 os.chdir(tmp_dir) 134 data_tar = 'data.tar.xz' 135 136 try: 137 cmd = [ar_cmd, 'x', pkg_path] 138 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) 139 cmd = [tar_cmd, 'xf', data_tar] 140 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) 141 except subprocess.CalledProcessError as e: 142 bb.utils.remove(tmp_dir, recurse=True) 143 bb.fatal("Unable to extract %s package. Command '%s' " 144 "returned %d:\n%s" % (pkg_path, ' '.join(cmd), e.returncode, e.output.decode("utf-8"))) 145 except OSError as e: 146 bb.utils.remove(tmp_dir, recurse=True) 147 bb.fatal("Unable to extract %s package. Command '%s' " 148 "returned %d:\n%s at %s" % (pkg_path, ' '.join(cmd), e.errno, e.strerror, e.filename)) 149 150 bb.note("Extracted %s to %s" % (pkg_path, tmp_dir)) 151 bb.utils.remove(os.path.join(tmp_dir, "debian-binary")) 152 bb.utils.remove(os.path.join(tmp_dir, "control.tar.gz")) 153 os.chdir(current_dir) 154 155 return tmp_dir 156 157 def _handle_intercept_failure(self, registered_pkgs): 158 self.mark_packages("unpacked", registered_pkgs.split()) 159 160class OpkgPM(OpkgDpkgPM): 161 def __init__(self, d, target_rootfs, config_file, archs, task_name='target', ipk_repo_workdir="oe-rootfs-repo", filterbydependencies=True, prepare_index=True): 162 super(OpkgPM, self).__init__(d, target_rootfs) 163 164 self.config_file = config_file 165 self.pkg_archs = archs 166 self.task_name = task_name 167 168 self.deploy_dir = oe.path.join(self.d.getVar('WORKDIR'), ipk_repo_workdir) 169 self.deploy_lock_file = os.path.join(self.deploy_dir, "deploy.lock") 170 self.opkg_cmd = bb.utils.which(os.getenv('PATH'), "opkg") 171 self.opkg_args = "--volatile-cache -f %s -t %s -o %s " % (self.config_file, self.d.expand('${T}/ipktemp/') ,target_rootfs) 172 self.opkg_args += self.d.getVar("OPKG_ARGS") 173 174 if prepare_index: 175 create_packages_dir(self.d, self.deploy_dir, d.getVar("DEPLOY_DIR_IPK"), "package_write_ipk", filterbydependencies) 176 177 self.opkg_dir = oe.path.join(target_rootfs, self.d.getVar('OPKGLIBDIR'), "opkg") 178 bb.utils.mkdirhier(self.opkg_dir) 179 180 self.saved_opkg_dir = self.d.expand('${T}/saved/%s' % self.task_name) 181 if not os.path.exists(self.d.expand('${T}/saved')): 182 bb.utils.mkdirhier(self.d.expand('${T}/saved')) 183 184 self.from_feeds = (self.d.getVar('BUILD_IMAGES_FROM_FEEDS') or "") == "1" 185 if self.from_feeds: 186 self._create_custom_config() 187 else: 188 self._create_config() 189 190 self.indexer = OpkgIndexer(self.d, self.deploy_dir) 191 192 def mark_packages(self, status_tag, packages=None): 193 """ 194 This function will change a package's status in /var/lib/opkg/status file. 195 If 'packages' is None then the new_status will be applied to all 196 packages 197 """ 198 status_file = os.path.join(self.opkg_dir, "status") 199 200 with open(status_file, "r") as sf: 201 with open(status_file + ".tmp", "w+") as tmp_sf: 202 if packages is None: 203 tmp_sf.write(re.sub(r"Package: (.*?)\n((?:[^\n]+\n)*?)Status: (.*)(?:unpacked|installed)", 204 r"Package: \1\n\2Status: \3%s" % status_tag, 205 sf.read())) 206 else: 207 if type(packages).__name__ != "list": 208 raise TypeError("'packages' should be a list object") 209 210 status = sf.read() 211 for pkg in packages: 212 status = re.sub(r"Package: %s\n((?:[^\n]+\n)*?)Status: (.*)(?:unpacked|installed)" % pkg, 213 r"Package: %s\n\1Status: \2%s" % (pkg, status_tag), 214 status) 215 216 tmp_sf.write(status) 217 218 bb.utils.rename(status_file + ".tmp", status_file) 219 220 def _create_custom_config(self): 221 bb.note("Building from feeds activated!") 222 223 with open(self.config_file, "w+") as config_file: 224 priority = 1 225 for arch in self.pkg_archs.split(): 226 config_file.write("arch %s %d\n" % (arch, priority)) 227 priority += 5 228 229 for line in (self.d.getVar('IPK_FEED_URIS') or "").split(): 230 feed_match = re.match(r"^[ \t]*(.*)##([^ \t]*)[ \t]*$", line) 231 232 if feed_match is not None: 233 feed_name = feed_match.group(1) 234 feed_uri = feed_match.group(2) 235 236 bb.note("Add %s feed with URL %s" % (feed_name, feed_uri)) 237 238 config_file.write("src/gz %s %s\n" % (feed_name, feed_uri)) 239 240 """ 241 Allow to use package deploy directory contents as quick devel-testing 242 feed. This creates individual feed configs for each arch subdir of those 243 specified as compatible for the current machine. 244 NOTE: Development-helper feature, NOT a full-fledged feed. 245 """ 246 if (self.d.getVar('FEED_DEPLOYDIR_BASE_URI') or "") != "": 247 for arch in self.pkg_archs.split(): 248 cfg_file_name = os.path.join(self.target_rootfs, 249 self.d.getVar("sysconfdir"), 250 "opkg", 251 "local-%s-feed.conf" % arch) 252 253 with open(cfg_file_name, "w+") as cfg_file: 254 cfg_file.write("src/gz local-%s %s/%s" % 255 (arch, 256 self.d.getVar('FEED_DEPLOYDIR_BASE_URI'), 257 arch)) 258 259 if self.d.getVar('OPKGLIBDIR') != '/var/lib': 260 # There is no command line option for this anymore, we need to add 261 # info_dir and status_file to config file, if OPKGLIBDIR doesn't have 262 # the default value of "/var/lib" as defined in opkg: 263 # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_LISTS_DIR VARDIR "/lib/opkg/lists" 264 # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_INFO_DIR VARDIR "/lib/opkg/info" 265 # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_STATUS_FILE VARDIR "/lib/opkg/status" 266 cfg_file.write("option info_dir %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'info')) 267 cfg_file.write("option lists_dir %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'lists')) 268 cfg_file.write("option status_file %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'status')) 269 270 271 def _create_config(self): 272 with open(self.config_file, "w+") as config_file: 273 priority = 1 274 for arch in self.pkg_archs.split(): 275 config_file.write("arch %s %d\n" % (arch, priority)) 276 priority += 5 277 278 config_file.write("src oe file:%s\n" % self.deploy_dir) 279 280 for arch in self.pkg_archs.split(): 281 pkgs_dir = os.path.join(self.deploy_dir, arch) 282 if os.path.isdir(pkgs_dir): 283 config_file.write("src oe-%s file:%s\n" % 284 (arch, pkgs_dir)) 285 286 if self.d.getVar('OPKGLIBDIR') != '/var/lib': 287 # There is no command line option for this anymore, we need to add 288 # info_dir and status_file to config file, if OPKGLIBDIR doesn't have 289 # the default value of "/var/lib" as defined in opkg: 290 # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_LISTS_DIR VARDIR "/lib/opkg/lists" 291 # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_INFO_DIR VARDIR "/lib/opkg/info" 292 # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_STATUS_FILE VARDIR "/lib/opkg/status" 293 config_file.write("option info_dir %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'info')) 294 config_file.write("option lists_dir %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'lists')) 295 config_file.write("option status_file %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'status')) 296 297 def insert_feeds_uris(self, feed_uris, feed_base_paths, feed_archs): 298 if feed_uris == "": 299 return 300 301 rootfs_config = os.path.join('%s/etc/opkg/base-feeds.conf' 302 % self.target_rootfs) 303 304 os.makedirs('%s/etc/opkg' % self.target_rootfs, exist_ok=True) 305 306 feed_uris = self.construct_uris(feed_uris.split(), feed_base_paths.split()) 307 archs = self.pkg_archs.split() if feed_archs is None else feed_archs.split() 308 309 with open(rootfs_config, "w+") as config_file: 310 uri_iterator = 0 311 for uri in feed_uris: 312 if archs: 313 for arch in archs: 314 if (feed_archs is None) and (not os.path.exists(oe.path.join(self.deploy_dir, arch))): 315 continue 316 bb.note('Adding opkg feed url-%s-%d (%s)' % 317 (arch, uri_iterator, uri)) 318 config_file.write("src/gz uri-%s-%d %s/%s\n" % 319 (arch, uri_iterator, uri, arch)) 320 else: 321 bb.note('Adding opkg feed url-%d (%s)' % 322 (uri_iterator, uri)) 323 config_file.write("src/gz uri-%d %s\n" % 324 (uri_iterator, uri)) 325 326 uri_iterator += 1 327 328 def update(self): 329 self.deploy_dir_lock() 330 331 cmd = "%s %s update" % (self.opkg_cmd, self.opkg_args) 332 333 try: 334 subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) 335 except subprocess.CalledProcessError as e: 336 self.deploy_dir_unlock() 337 bb.fatal("Unable to update the package index files. Command '%s' " 338 "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) 339 340 self.deploy_dir_unlock() 341 342 def install(self, pkgs, attempt_only=False): 343 if not pkgs: 344 return 345 346 cmd = "%s %s" % (self.opkg_cmd, self.opkg_args) 347 for exclude in (self.d.getVar("PACKAGE_EXCLUDE") or "").split(): 348 cmd += " --add-exclude %s" % exclude 349 for bad_recommendation in (self.d.getVar("BAD_RECOMMENDATIONS") or "").split(): 350 cmd += " --add-ignore-recommends %s" % bad_recommendation 351 cmd += " install " 352 cmd += " ".join(pkgs) 353 354 os.environ['D'] = self.target_rootfs 355 os.environ['OFFLINE_ROOT'] = self.target_rootfs 356 os.environ['IPKG_OFFLINE_ROOT'] = self.target_rootfs 357 os.environ['OPKG_OFFLINE_ROOT'] = self.target_rootfs 358 os.environ['INTERCEPT_DIR'] = self.intercepts_dir 359 os.environ['NATIVE_ROOT'] = self.d.getVar('STAGING_DIR_NATIVE') 360 361 try: 362 bb.note("Installing the following packages: %s" % ' '.join(pkgs)) 363 bb.note(cmd) 364 output = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT).decode("utf-8") 365 bb.note(output) 366 failed_pkgs = [] 367 for line in output.split('\n'): 368 if line.endswith("configuration required on target."): 369 bb.warn(line) 370 failed_pkgs.append(line.split(".")[0]) 371 if failed_pkgs: 372 failed_postinsts_abort(failed_pkgs, self.d.expand("${T}/log.do_${BB_CURRENTTASK}")) 373 except subprocess.CalledProcessError as e: 374 (bb.fatal, bb.warn)[attempt_only]("Unable to install packages. " 375 "Command '%s' returned %d:\n%s" % 376 (cmd, e.returncode, e.output.decode("utf-8"))) 377 378 def remove(self, pkgs, with_dependencies=True): 379 if not pkgs: 380 return 381 382 if with_dependencies: 383 cmd = "%s %s --force-remove --force-removal-of-dependent-packages remove %s" % \ 384 (self.opkg_cmd, self.opkg_args, ' '.join(pkgs)) 385 else: 386 cmd = "%s %s --force-depends remove %s" % \ 387 (self.opkg_cmd, self.opkg_args, ' '.join(pkgs)) 388 389 try: 390 bb.note(cmd) 391 output = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT).decode("utf-8") 392 bb.note(output) 393 except subprocess.CalledProcessError as e: 394 bb.fatal("Unable to remove packages. Command '%s' " 395 "returned %d:\n%s" % (e.cmd, e.returncode, e.output.decode("utf-8"))) 396 397 def write_index(self): 398 self.deploy_dir_lock() 399 400 result = self.indexer.write_index() 401 402 self.deploy_dir_unlock() 403 404 if result is not None: 405 bb.fatal(result) 406 407 def remove_packaging_data(self): 408 cachedir = oe.path.join(self.target_rootfs, self.d.getVar("localstatedir"), "cache", "opkg") 409 bb.utils.remove(self.opkg_dir, True) 410 bb.utils.remove(cachedir, True) 411 412 def remove_lists(self): 413 if not self.from_feeds: 414 bb.utils.remove(os.path.join(self.opkg_dir, "lists"), True) 415 416 def list_installed(self): 417 return PMPkgsList(self.d, self.target_rootfs).list_pkgs() 418 419 def dummy_install(self, pkgs): 420 """ 421 The following function dummy installs pkgs and returns the log of output. 422 """ 423 if len(pkgs) == 0: 424 return 425 426 # Create an temp dir as opkg root for dummy installation 427 temp_rootfs = self.d.expand('${T}/opkg') 428 opkg_lib_dir = self.d.getVar('OPKGLIBDIR') 429 if opkg_lib_dir[0] == "/": 430 opkg_lib_dir = opkg_lib_dir[1:] 431 temp_opkg_dir = os.path.join(temp_rootfs, opkg_lib_dir, 'opkg') 432 bb.utils.mkdirhier(temp_opkg_dir) 433 434 opkg_args = "-f %s -o %s " % (self.config_file, temp_rootfs) 435 opkg_args += self.d.getVar("OPKG_ARGS") 436 437 cmd = "%s %s update" % (self.opkg_cmd, opkg_args) 438 try: 439 subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) 440 except subprocess.CalledProcessError as e: 441 bb.fatal("Unable to update. Command '%s' " 442 "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) 443 444 # Dummy installation 445 cmd = "%s %s --noaction install %s " % (self.opkg_cmd, 446 opkg_args, 447 ' '.join(pkgs)) 448 proc = subprocess.run(cmd, capture_output=True, encoding="utf-8", shell=True) 449 if proc.returncode: 450 bb.fatal("Unable to dummy install packages. Command '%s' " 451 "returned %d:\n%s" % (cmd, proc.returncode, proc.stderr)) 452 elif proc.stderr: 453 bb.note("Command '%s' returned stderr: %s" % (cmd, proc.stderr)) 454 455 bb.utils.remove(temp_rootfs, True) 456 457 return proc.stdout 458 459 def backup_packaging_data(self): 460 # Save the opkglib for increment ipk image generation 461 if os.path.exists(self.saved_opkg_dir): 462 bb.utils.remove(self.saved_opkg_dir, True) 463 shutil.copytree(self.opkg_dir, 464 self.saved_opkg_dir, 465 symlinks=True) 466 467 def recover_packaging_data(self): 468 # Move the opkglib back 469 if os.path.exists(self.saved_opkg_dir): 470 if os.path.exists(self.opkg_dir): 471 bb.utils.remove(self.opkg_dir, True) 472 473 bb.note('Recover packaging data') 474 shutil.copytree(self.saved_opkg_dir, 475 self.opkg_dir, 476 symlinks=True) 477 478 def package_info(self, pkg): 479 """ 480 Returns a dictionary with the package info. 481 """ 482 cmd = "%s %s info %s" % (self.opkg_cmd, self.opkg_args, pkg) 483 pkg_info = super(OpkgPM, self).package_info(pkg, cmd) 484 485 pkg_arch = pkg_info[pkg]["arch"] 486 pkg_filename = pkg_info[pkg]["filename"] 487 pkg_info[pkg]["filepath"] = \ 488 os.path.join(self.deploy_dir, pkg_arch, pkg_filename) 489 490 return pkg_info 491 492 def extract(self, pkg): 493 """ 494 Returns the path to a tmpdir where resides the contents of a package. 495 496 Deleting the tmpdir is responsability of the caller. 497 """ 498 pkg_info = self.package_info(pkg) 499 if not pkg_info: 500 bb.fatal("Unable to get information for package '%s' while " 501 "trying to extract the package." % pkg) 502 503 tmp_dir = super(OpkgPM, self).extract(pkg, pkg_info) 504 bb.utils.remove(os.path.join(tmp_dir, "data.tar.xz")) 505 506 return tmp_dir 507