1# 2# Copyright (c) 2013, Intel Corporation. 3# 4# SPDX-License-Identifier: GPL-2.0-only 5# 6# DESCRIPTION 7# This implements the 'direct' imager plugin class for 'wic' 8# 9# AUTHORS 10# Tom Zanussi <tom.zanussi (at] linux.intel.com> 11# 12 13import logging 14import os 15import random 16import shutil 17import tempfile 18import uuid 19 20from time import strftime 21 22from oe.path import copyhardlinktree 23 24from wic import WicError 25from wic.filemap import sparse_copy 26from wic.ksparser import KickStart, KickStartError 27from wic.pluginbase import PluginMgr, ImagerPlugin 28from wic.misc import get_bitbake_var, exec_cmd, exec_native_cmd 29 30logger = logging.getLogger('wic') 31 32class DirectPlugin(ImagerPlugin): 33 """ 34 Install a system into a file containing a partitioned disk image. 35 36 An image file is formatted with a partition table, each partition 37 created from a rootfs or other OpenEmbedded build artifact and dd'ed 38 into the virtual disk. The disk image can subsequently be dd'ed onto 39 media and used on actual hardware. 40 """ 41 name = 'direct' 42 43 def __init__(self, wks_file, rootfs_dir, bootimg_dir, kernel_dir, 44 native_sysroot, oe_builddir, options): 45 try: 46 self.ks = KickStart(wks_file) 47 except KickStartError as err: 48 raise WicError(str(err)) 49 50 # parse possible 'rootfs=name' items 51 self.rootfs_dir = dict(rdir.split('=') for rdir in rootfs_dir.split(' ')) 52 self.bootimg_dir = bootimg_dir 53 self.kernel_dir = kernel_dir 54 self.native_sysroot = native_sysroot 55 self.oe_builddir = oe_builddir 56 57 self.debug = options.debug 58 self.outdir = options.outdir 59 self.compressor = options.compressor 60 self.bmap = options.bmap 61 self.no_fstab_update = options.no_fstab_update 62 self.updated_fstab_path = None 63 64 self.name = "%s-%s" % (os.path.splitext(os.path.basename(wks_file))[0], 65 strftime("%Y%m%d%H%M")) 66 self.workdir = self.setup_workdir(options.workdir) 67 self._image = None 68 self.ptable_format = self.ks.bootloader.ptable 69 self.parts = self.ks.partitions 70 71 # as a convenience, set source to the boot partition source 72 # instead of forcing it to be set via bootloader --source 73 for part in self.parts: 74 if not self.ks.bootloader.source and part.mountpoint == "/boot": 75 self.ks.bootloader.source = part.source 76 break 77 78 image_path = self._full_path(self.workdir, self.parts[0].disk, "direct") 79 self._image = PartitionedImage(image_path, self.ptable_format, 80 self.parts, self.native_sysroot, 81 options.extra_space) 82 83 def setup_workdir(self, workdir): 84 if workdir: 85 if os.path.exists(workdir): 86 raise WicError("Internal workdir '%s' specified in wic arguments already exists!" % (workdir)) 87 88 os.makedirs(workdir) 89 return workdir 90 else: 91 return tempfile.mkdtemp(dir=self.outdir, prefix='tmp.wic.') 92 93 def do_create(self): 94 """ 95 Plugin entry point. 96 """ 97 try: 98 self.create() 99 self.assemble() 100 self.finalize() 101 self.print_info() 102 finally: 103 self.cleanup() 104 105 def update_fstab(self, image_rootfs): 106 """Assume partition order same as in wks""" 107 if not image_rootfs: 108 return 109 110 fstab_path = image_rootfs + "/etc/fstab" 111 if not os.path.isfile(fstab_path): 112 return 113 114 with open(fstab_path) as fstab: 115 fstab_lines = fstab.readlines() 116 117 updated = False 118 for part in self.parts: 119 if not part.realnum or not part.mountpoint \ 120 or part.mountpoint == "/" or not (part.mountpoint.startswith('/') or part.mountpoint == "swap"): 121 continue 122 123 if part.use_uuid: 124 if part.fsuuid: 125 # FAT UUID is different from others 126 if len(part.fsuuid) == 10: 127 device_name = "UUID=%s-%s" % \ 128 (part.fsuuid[2:6], part.fsuuid[6:]) 129 else: 130 device_name = "UUID=%s" % part.fsuuid 131 else: 132 device_name = "PARTUUID=%s" % part.uuid 133 elif part.use_label: 134 device_name = "LABEL=%s" % part.label 135 else: 136 # mmc device partitions are named mmcblk0p1, mmcblk0p2.. 137 prefix = 'p' if part.disk.startswith('mmcblk') else '' 138 device_name = "/dev/%s%s%d" % (part.disk, prefix, part.realnum) 139 140 opts = part.fsopts if part.fsopts else "defaults" 141 line = "\t".join([device_name, part.mountpoint, part.fstype, 142 opts, "0", "0"]) + "\n" 143 144 fstab_lines.append(line) 145 updated = True 146 147 if updated: 148 self.updated_fstab_path = os.path.join(self.workdir, "fstab") 149 with open(self.updated_fstab_path, "w") as f: 150 f.writelines(fstab_lines) 151 if os.getenv('SOURCE_DATE_EPOCH'): 152 fstab_time = int(os.getenv('SOURCE_DATE_EPOCH')) 153 os.utime(self.updated_fstab_path, (fstab_time, fstab_time)) 154 155 def _full_path(self, path, name, extention): 156 """ Construct full file path to a file we generate. """ 157 return os.path.join(path, "%s-%s.%s" % (self.name, name, extention)) 158 159 # 160 # Actual implemention 161 # 162 def create(self): 163 """ 164 For 'wic', we already have our build artifacts - we just create 165 filesystems from the artifacts directly and combine them into 166 a partitioned image. 167 """ 168 if not self.no_fstab_update: 169 self.update_fstab(self.rootfs_dir.get("ROOTFS_DIR")) 170 171 for part in self.parts: 172 # get rootfs size from bitbake variable if it's not set in .ks file 173 if not part.size: 174 # and if rootfs name is specified for the partition 175 image_name = self.rootfs_dir.get(part.rootfs_dir) 176 if image_name and os.path.sep not in image_name: 177 # Bitbake variable ROOTFS_SIZE is calculated in 178 # Image._get_rootfs_size method from meta/lib/oe/image.py 179 # using IMAGE_ROOTFS_SIZE, IMAGE_ROOTFS_ALIGNMENT, 180 # IMAGE_OVERHEAD_FACTOR and IMAGE_ROOTFS_EXTRA_SPACE 181 rsize_bb = get_bitbake_var('ROOTFS_SIZE', image_name) 182 if rsize_bb: 183 part.size = int(round(float(rsize_bb))) 184 185 self._image.prepare(self) 186 self._image.layout_partitions() 187 self._image.create() 188 189 def assemble(self): 190 """ 191 Assemble partitions into disk image 192 """ 193 self._image.assemble() 194 195 def finalize(self): 196 """ 197 Finalize the disk image. 198 199 For example, prepare the image to be bootable by e.g. 200 creating and installing a bootloader configuration. 201 """ 202 source_plugin = self.ks.bootloader.source 203 disk_name = self.parts[0].disk 204 if source_plugin: 205 plugin = PluginMgr.get_plugins('source')[source_plugin] 206 plugin.do_install_disk(self._image, disk_name, self, self.workdir, 207 self.oe_builddir, self.bootimg_dir, 208 self.kernel_dir, self.native_sysroot) 209 210 full_path = self._image.path 211 # Generate .bmap 212 if self.bmap: 213 logger.debug("Generating bmap file for %s", disk_name) 214 python = os.path.join(self.native_sysroot, 'usr/bin/python3-native/python3') 215 bmaptool = os.path.join(self.native_sysroot, 'usr/bin/bmaptool') 216 exec_native_cmd("%s %s create %s -o %s.bmap" % \ 217 (python, bmaptool, full_path, full_path), self.native_sysroot) 218 # Compress the image 219 if self.compressor: 220 logger.debug("Compressing disk %s with %s", disk_name, self.compressor) 221 exec_cmd("%s %s" % (self.compressor, full_path)) 222 223 def print_info(self): 224 """ 225 Print the image(s) and artifacts used, for the user. 226 """ 227 msg = "The new image(s) can be found here:\n" 228 229 extension = "direct" + {"gzip": ".gz", 230 "bzip2": ".bz2", 231 "xz": ".xz", 232 None: ""}.get(self.compressor) 233 full_path = self._full_path(self.outdir, self.parts[0].disk, extension) 234 msg += ' %s\n\n' % full_path 235 236 msg += 'The following build artifacts were used to create the image(s):\n' 237 for part in self.parts: 238 if part.rootfs_dir is None: 239 continue 240 if part.mountpoint == '/': 241 suffix = ':' 242 else: 243 suffix = '["%s"]:' % (part.mountpoint or part.label) 244 rootdir = part.rootfs_dir 245 msg += ' ROOTFS_DIR%s%s\n' % (suffix.ljust(20), rootdir) 246 247 msg += ' BOOTIMG_DIR: %s\n' % self.bootimg_dir 248 msg += ' KERNEL_DIR: %s\n' % self.kernel_dir 249 msg += ' NATIVE_SYSROOT: %s\n' % self.native_sysroot 250 251 logger.info(msg) 252 253 @property 254 def rootdev(self): 255 """ 256 Get root device name to use as a 'root' parameter 257 in kernel command line. 258 259 Assume partition order same as in wks 260 """ 261 for part in self.parts: 262 if part.mountpoint == "/": 263 if part.uuid: 264 return "PARTUUID=%s" % part.uuid 265 elif part.label and self.ptable_format != 'msdos': 266 return "PARTLABEL=%s" % part.label 267 else: 268 suffix = 'p' if part.disk.startswith('mmcblk') else '' 269 return "/dev/%s%s%-d" % (part.disk, suffix, part.realnum) 270 271 def cleanup(self): 272 if self._image: 273 self._image.cleanup() 274 275 # Move results to the output dir 276 if not os.path.exists(self.outdir): 277 os.makedirs(self.outdir) 278 279 for fname in os.listdir(self.workdir): 280 path = os.path.join(self.workdir, fname) 281 if os.path.isfile(path): 282 shutil.move(path, os.path.join(self.outdir, fname)) 283 284 # remove work directory when it is not in debugging mode 285 if not self.debug: 286 shutil.rmtree(self.workdir, ignore_errors=True) 287 288# Overhead of the MBR partitioning scheme (just one sector) 289MBR_OVERHEAD = 1 290 291# Overhead of the GPT partitioning scheme 292GPT_OVERHEAD = 34 293 294# Size of a sector in bytes 295SECTOR_SIZE = 512 296 297class PartitionedImage(): 298 """ 299 Partitioned image in a file. 300 """ 301 302 def __init__(self, path, ptable_format, partitions, native_sysroot=None, extra_space=0): 303 self.path = path # Path to the image file 304 self.numpart = 0 # Number of allocated partitions 305 self.realpart = 0 # Number of partitions in the partition table 306 self.primary_part_num = 0 # Number of primary partitions (msdos) 307 self.extendedpart = 0 # Create extended partition before this logical partition (msdos) 308 self.extended_size_sec = 0 # Size of exteded partition (msdos) 309 self.logical_part_cnt = 0 # Number of total logical paritions (msdos) 310 self.offset = 0 # Offset of next partition (in sectors) 311 self.min_size = 0 # Minimum required disk size to fit 312 # all partitions (in bytes) 313 self.ptable_format = ptable_format # Partition table format 314 # Disk system identifier 315 self.identifier = random.SystemRandom().randint(1, 0xffffffff) 316 317 self.partitions = partitions 318 self.partimages = [] 319 # Size of a sector used in calculations 320 self.sector_size = SECTOR_SIZE 321 self.native_sysroot = native_sysroot 322 num_real_partitions = len([p for p in self.partitions if not p.no_table]) 323 self.extra_space = extra_space 324 325 # calculate the real partition number, accounting for partitions not 326 # in the partition table and logical partitions 327 realnum = 0 328 for part in self.partitions: 329 if part.no_table: 330 part.realnum = 0 331 else: 332 realnum += 1 333 if self.ptable_format == 'msdos' and realnum > 3 and num_real_partitions > 4: 334 part.realnum = realnum + 1 335 continue 336 part.realnum = realnum 337 338 # generate parition and filesystem UUIDs 339 for part in self.partitions: 340 if not part.uuid and part.use_uuid: 341 if self.ptable_format == 'gpt': 342 part.uuid = str(uuid.uuid4()) 343 else: # msdos partition table 344 part.uuid = '%08x-%02d' % (self.identifier, part.realnum) 345 if not part.fsuuid: 346 if part.fstype == 'vfat' or part.fstype == 'msdos': 347 part.fsuuid = '0x' + str(uuid.uuid4())[:8].upper() 348 else: 349 part.fsuuid = str(uuid.uuid4()) 350 else: 351 #make sure the fsuuid for vfat/msdos align with format 0xYYYYYYYY 352 if part.fstype == 'vfat' or part.fstype == 'msdos': 353 if part.fsuuid.upper().startswith("0X"): 354 part.fsuuid = '0x' + part.fsuuid.upper()[2:].rjust(8,"0") 355 else: 356 part.fsuuid = '0x' + part.fsuuid.upper().rjust(8,"0") 357 358 def prepare(self, imager): 359 """Prepare an image. Call prepare method of all image partitions.""" 360 for part in self.partitions: 361 # need to create the filesystems in order to get their 362 # sizes before we can add them and do the layout. 363 part.prepare(imager, imager.workdir, imager.oe_builddir, 364 imager.rootfs_dir, imager.bootimg_dir, 365 imager.kernel_dir, imager.native_sysroot, 366 imager.updated_fstab_path) 367 368 # Converting kB to sectors for parted 369 part.size_sec = part.disk_size * 1024 // self.sector_size 370 371 def layout_partitions(self): 372 """ Layout the partitions, meaning calculate the position of every 373 partition on the disk. The 'ptable_format' parameter defines the 374 partition table format and may be "msdos". """ 375 376 logger.debug("Assigning %s partitions to disks", self.ptable_format) 377 378 # The number of primary and logical partitions. Extended partition and 379 # partitions not listed in the table are not included. 380 num_real_partitions = len([p for p in self.partitions if not p.no_table]) 381 382 # Go through partitions in the order they are added in .ks file 383 for num in range(len(self.partitions)): 384 part = self.partitions[num] 385 386 if self.ptable_format == 'msdos' and part.part_name: 387 raise WicError("setting custom partition name is not " \ 388 "implemented for msdos partitions") 389 390 if self.ptable_format == 'msdos' and part.part_type: 391 # The --part-type can also be implemented for MBR partitions, 392 # in which case it would map to the 1-byte "partition type" 393 # filed at offset 3 of the partition entry. 394 raise WicError("setting custom partition type is not " \ 395 "implemented for msdos partitions") 396 397 # Get the disk where the partition is located 398 self.numpart += 1 399 if not part.no_table: 400 self.realpart += 1 401 402 if self.numpart == 1: 403 if self.ptable_format == "msdos": 404 overhead = MBR_OVERHEAD 405 elif self.ptable_format == "gpt": 406 overhead = GPT_OVERHEAD 407 408 # Skip one sector required for the partitioning scheme overhead 409 self.offset += overhead 410 411 if self.ptable_format == "msdos": 412 if self.primary_part_num > 3 or \ 413 (self.extendedpart == 0 and self.primary_part_num >= 3 and num_real_partitions > 4): 414 part.type = 'logical' 415 # Reserve a sector for EBR for every logical partition 416 # before alignment is performed. 417 if part.type == 'logical': 418 self.offset += 2 419 420 align_sectors = 0 421 if part.align: 422 # If not first partition and we do have alignment set we need 423 # to align the partition. 424 # FIXME: This leaves a empty spaces to the disk. To fill the 425 # gaps we could enlargea the previous partition? 426 427 # Calc how much the alignment is off. 428 align_sectors = self.offset % (part.align * 1024 // self.sector_size) 429 430 if align_sectors: 431 # If partition is not aligned as required, we need 432 # to move forward to the next alignment point 433 align_sectors = (part.align * 1024 // self.sector_size) - align_sectors 434 435 logger.debug("Realignment for %s%s with %s sectors, original" 436 " offset %s, target alignment is %sK.", 437 part.disk, self.numpart, align_sectors, 438 self.offset, part.align) 439 440 # increase the offset so we actually start the partition on right alignment 441 self.offset += align_sectors 442 443 if part.offset is not None: 444 offset = part.offset // self.sector_size 445 446 if offset * self.sector_size != part.offset: 447 raise WicError("Could not place %s%s at offset %d with sector size %d" % (part.disk, self.numpart, part.offset, self.sector_size)) 448 449 delta = offset - self.offset 450 if delta < 0: 451 raise WicError("Could not place %s%s at offset %d: next free sector is %d (delta: %d)" % (part.disk, self.numpart, part.offset, self.offset, delta)) 452 453 logger.debug("Skipping %d sectors to place %s%s at offset %dK", 454 delta, part.disk, self.numpart, part.offset) 455 456 self.offset = offset 457 458 part.start = self.offset 459 self.offset += part.size_sec 460 461 if not part.no_table: 462 part.num = self.realpart 463 else: 464 part.num = 0 465 466 if self.ptable_format == "msdos" and not part.no_table: 467 if part.type == 'logical': 468 self.logical_part_cnt += 1 469 part.num = self.logical_part_cnt + 4 470 if self.extendedpart == 0: 471 # Create extended partition as a primary partition 472 self.primary_part_num += 1 473 self.extendedpart = part.num 474 else: 475 self.extended_size_sec += align_sectors 476 self.extended_size_sec += part.size_sec + 2 477 else: 478 self.primary_part_num += 1 479 part.num = self.primary_part_num 480 481 logger.debug("Assigned %s to %s%d, sectors range %d-%d size %d " 482 "sectors (%d bytes).", part.mountpoint, part.disk, 483 part.num, part.start, self.offset - 1, part.size_sec, 484 part.size_sec * self.sector_size) 485 486 # Once all the partitions have been layed out, we can calculate the 487 # minumim disk size 488 self.min_size = self.offset 489 if self.ptable_format == "gpt": 490 self.min_size += GPT_OVERHEAD 491 492 self.min_size *= self.sector_size 493 self.min_size += self.extra_space 494 495 def _create_partition(self, device, parttype, fstype, start, size): 496 """ Create a partition on an image described by the 'device' object. """ 497 498 # Start is included to the size so we need to substract one from the end. 499 end = start + size - 1 500 logger.debug("Added '%s' partition, sectors %d-%d, size %d sectors", 501 parttype, start, end, size) 502 503 cmd = "parted -s %s unit s mkpart %s" % (device, parttype) 504 if fstype: 505 cmd += " %s" % fstype 506 cmd += " %d %d" % (start, end) 507 508 return exec_native_cmd(cmd, self.native_sysroot) 509 510 def create(self): 511 logger.debug("Creating sparse file %s", self.path) 512 with open(self.path, 'w') as sparse: 513 os.ftruncate(sparse.fileno(), self.min_size) 514 515 logger.debug("Initializing partition table for %s", self.path) 516 exec_native_cmd("parted -s %s mklabel %s" % 517 (self.path, self.ptable_format), self.native_sysroot) 518 519 logger.debug("Set disk identifier %x", self.identifier) 520 with open(self.path, 'r+b') as img: 521 img.seek(0x1B8) 522 img.write(self.identifier.to_bytes(4, 'little')) 523 524 logger.debug("Creating partitions") 525 526 for part in self.partitions: 527 if part.num == 0: 528 continue 529 530 if self.ptable_format == "msdos" and part.num == self.extendedpart: 531 # Create an extended partition (note: extended 532 # partition is described in MBR and contains all 533 # logical partitions). The logical partitions save a 534 # sector for an EBR just before the start of a 535 # partition. The extended partition must start one 536 # sector before the start of the first logical 537 # partition. This way the first EBR is inside of the 538 # extended partition. Since the extended partitions 539 # starts a sector before the first logical partition, 540 # add a sector at the back, so that there is enough 541 # room for all logical partitions. 542 self._create_partition(self.path, "extended", 543 None, part.start - 2, 544 self.extended_size_sec) 545 546 if part.fstype == "swap": 547 parted_fs_type = "linux-swap" 548 elif part.fstype == "vfat": 549 parted_fs_type = "fat32" 550 elif part.fstype == "msdos": 551 parted_fs_type = "fat16" 552 if not part.system_id: 553 part.system_id = '0x6' # FAT16 554 else: 555 # Type for ext2/ext3/ext4/btrfs 556 parted_fs_type = "ext2" 557 558 # Boot ROM of OMAP boards require vfat boot partition to have an 559 # even number of sectors. 560 if part.mountpoint == "/boot" and part.fstype in ["vfat", "msdos"] \ 561 and part.size_sec % 2: 562 logger.debug("Subtracting one sector from '%s' partition to " 563 "get even number of sectors for the partition", 564 part.mountpoint) 565 part.size_sec -= 1 566 567 self._create_partition(self.path, part.type, 568 parted_fs_type, part.start, part.size_sec) 569 570 if part.part_name: 571 logger.debug("partition %d: set name to %s", 572 part.num, part.part_name) 573 exec_native_cmd("sgdisk --change-name=%d:%s %s" % \ 574 (part.num, part.part_name, 575 self.path), self.native_sysroot) 576 577 if part.part_type: 578 logger.debug("partition %d: set type UID to %s", 579 part.num, part.part_type) 580 exec_native_cmd("sgdisk --typecode=%d:%s %s" % \ 581 (part.num, part.part_type, 582 self.path), self.native_sysroot) 583 584 if part.uuid and self.ptable_format == "gpt": 585 logger.debug("partition %d: set UUID to %s", 586 part.num, part.uuid) 587 exec_native_cmd("sgdisk --partition-guid=%d:%s %s" % \ 588 (part.num, part.uuid, self.path), 589 self.native_sysroot) 590 591 if part.label and self.ptable_format == "gpt": 592 logger.debug("partition %d: set name to %s", 593 part.num, part.label) 594 exec_native_cmd("parted -s %s name %d %s" % \ 595 (self.path, part.num, part.label), 596 self.native_sysroot) 597 598 if part.active: 599 flag_name = "legacy_boot" if self.ptable_format == 'gpt' else "boot" 600 logger.debug("Set '%s' flag for partition '%s' on disk '%s'", 601 flag_name, part.num, self.path) 602 exec_native_cmd("parted -s %s set %d %s on" % \ 603 (self.path, part.num, flag_name), 604 self.native_sysroot) 605 if part.system_id: 606 exec_native_cmd("sfdisk --part-type %s %s %s" % \ 607 (self.path, part.num, part.system_id), 608 self.native_sysroot) 609 610 def cleanup(self): 611 pass 612 613 def assemble(self): 614 logger.debug("Installing partitions") 615 616 for part in self.partitions: 617 source = part.source_file 618 if source: 619 # install source_file contents into a partition 620 sparse_copy(source, self.path, seek=part.start * self.sector_size) 621 622 logger.debug("Installed %s in partition %d, sectors %d-%d, " 623 "size %d sectors", source, part.num, part.start, 624 part.start + part.size_sec - 1, part.size_sec) 625 626 partimage = self.path + '.p%d' % part.num 627 os.rename(source, partimage) 628 self.partimages.append(partimage) 629