1# 2# Copyright (c) 2013, Intel Corporation. 3# 4# SPDX-License-Identifier: GPL-2.0-only 5# 6# DESCRIPTION 7# This module provides a place to collect various wic-related utils 8# for the OpenEmbedded Image Tools. 9# 10# AUTHORS 11# Tom Zanussi <tom.zanussi (at] linux.intel.com> 12# 13"""Miscellaneous functions.""" 14 15import logging 16import os 17import re 18import subprocess 19import shutil 20 21from collections import defaultdict 22 23from wic import WicError 24 25logger = logging.getLogger('wic') 26 27# executable -> recipe pairs for exec_native_cmd 28NATIVE_RECIPES = {"bmaptool": "bmap-tools", 29 "dumpe2fs": "e2fsprogs", 30 "grub-mkimage": "grub-efi", 31 "isohybrid": "syslinux", 32 "mcopy": "mtools", 33 "mdel" : "mtools", 34 "mdeltree" : "mtools", 35 "mdir" : "mtools", 36 "mkdosfs": "dosfstools", 37 "mkisofs": "cdrtools", 38 "mkfs.btrfs": "btrfs-tools", 39 "mkfs.ext2": "e2fsprogs", 40 "mkfs.ext3": "e2fsprogs", 41 "mkfs.ext4": "e2fsprogs", 42 "mkfs.vfat": "dosfstools", 43 "mksquashfs": "squashfs-tools", 44 "mkswap": "util-linux", 45 "mmd": "mtools", 46 "parted": "parted", 47 "sfdisk": "util-linux", 48 "sgdisk": "gptfdisk", 49 "syslinux": "syslinux", 50 "tar": "tar" 51 } 52 53def runtool(cmdln_or_args): 54 """ wrapper for most of the subprocess calls 55 input: 56 cmdln_or_args: can be both args and cmdln str (shell=True) 57 return: 58 rc, output 59 """ 60 if isinstance(cmdln_or_args, list): 61 cmd = cmdln_or_args[0] 62 shell = False 63 else: 64 import shlex 65 cmd = shlex.split(cmdln_or_args)[0] 66 shell = True 67 68 sout = subprocess.PIPE 69 serr = subprocess.STDOUT 70 71 try: 72 process = subprocess.Popen(cmdln_or_args, stdout=sout, 73 stderr=serr, shell=shell) 74 sout, serr = process.communicate() 75 # combine stdout and stderr, filter None out and decode 76 out = ''.join([out.decode('utf-8') for out in [sout, serr] if out]) 77 except OSError as err: 78 if err.errno == 2: 79 # [Errno 2] No such file or directory 80 raise WicError('Cannot run command: %s, lost dependency?' % cmd) 81 else: 82 raise # relay 83 84 return process.returncode, out 85 86def _exec_cmd(cmd_and_args, as_shell=False): 87 """ 88 Execute command, catching stderr, stdout 89 90 Need to execute as_shell if the command uses wildcards 91 """ 92 logger.debug("_exec_cmd: %s", cmd_and_args) 93 args = cmd_and_args.split() 94 logger.debug(args) 95 96 if as_shell: 97 ret, out = runtool(cmd_and_args) 98 else: 99 ret, out = runtool(args) 100 out = out.strip() 101 if ret != 0: 102 raise WicError("_exec_cmd: %s returned '%s' instead of 0\noutput: %s" % \ 103 (cmd_and_args, ret, out)) 104 105 logger.debug("_exec_cmd: output for %s (rc = %d): %s", 106 cmd_and_args, ret, out) 107 108 return ret, out 109 110 111def exec_cmd(cmd_and_args, as_shell=False): 112 """ 113 Execute command, return output 114 """ 115 return _exec_cmd(cmd_and_args, as_shell)[1] 116 117def find_executable(cmd, paths): 118 recipe = cmd 119 if recipe in NATIVE_RECIPES: 120 recipe = NATIVE_RECIPES[recipe] 121 provided = get_bitbake_var("ASSUME_PROVIDED") 122 if provided and "%s-native" % recipe in provided: 123 return True 124 125 return shutil.which(cmd, path=paths) 126 127def exec_native_cmd(cmd_and_args, native_sysroot, pseudo=""): 128 """ 129 Execute native command, catching stderr, stdout 130 131 Need to execute as_shell if the command uses wildcards 132 133 Always need to execute native commands as_shell 134 """ 135 # The reason -1 is used is because there may be "export" commands. 136 args = cmd_and_args.split(';')[-1].split() 137 logger.debug(args) 138 139 if pseudo: 140 cmd_and_args = pseudo + cmd_and_args 141 142 hosttools_dir = get_bitbake_var("HOSTTOOLS_DIR") 143 target_sys = get_bitbake_var("TARGET_SYS") 144 145 native_paths = "%s/sbin:%s/usr/sbin:%s/usr/bin:%s/usr/bin/%s:%s/bin:%s" % \ 146 (native_sysroot, native_sysroot, 147 native_sysroot, native_sysroot, target_sys, 148 native_sysroot, hosttools_dir) 149 150 native_cmd_and_args = "export PATH=%s:$PATH;%s" % \ 151 (native_paths, cmd_and_args) 152 logger.debug("exec_native_cmd: %s", native_cmd_and_args) 153 154 # If the command isn't in the native sysroot say we failed. 155 if find_executable(args[0], native_paths): 156 ret, out = _exec_cmd(native_cmd_and_args, True) 157 else: 158 ret = 127 159 out = "can't find native executable %s in %s" % (args[0], native_paths) 160 161 prog = args[0] 162 # shell command-not-found 163 if ret == 127 \ 164 or (pseudo and ret == 1 and out == "Can't find '%s' in $PATH." % prog): 165 msg = "A native program %s required to build the image "\ 166 "was not found (see details above).\n\n" % prog 167 recipe = NATIVE_RECIPES.get(prog) 168 if recipe: 169 msg += "Please make sure wic-tools have %s-native in its DEPENDS, "\ 170 "build it with 'bitbake wic-tools' and try again.\n" % recipe 171 else: 172 msg += "Wic failed to find a recipe to build native %s. Please "\ 173 "file a bug against wic.\n" % prog 174 raise WicError(msg) 175 176 return ret, out 177 178BOOTDD_EXTRA_SPACE = 16384 179 180class BitbakeVars(defaultdict): 181 """ 182 Container for Bitbake variables. 183 """ 184 def __init__(self): 185 defaultdict.__init__(self, dict) 186 187 # default_image and vars_dir attributes should be set from outside 188 self.default_image = None 189 self.vars_dir = None 190 191 def _parse_line(self, line, image, matcher=re.compile(r"^([a-zA-Z0-9\-_+./~]+)=(.*)")): 192 """ 193 Parse one line from bitbake -e output or from .env file. 194 Put result key-value pair into the storage. 195 """ 196 if "=" not in line: 197 return 198 match = matcher.match(line) 199 if not match: 200 return 201 key, val = match.groups() 202 self[image][key] = val.strip('"') 203 204 def get_var(self, var, image=None, cache=True): 205 """ 206 Get bitbake variable from 'bitbake -e' output or from .env file. 207 This is a lazy method, i.e. it runs bitbake or parses file only when 208 only when variable is requested. It also caches results. 209 """ 210 if not image: 211 image = self.default_image 212 213 if image not in self: 214 if image and self.vars_dir: 215 fname = os.path.join(self.vars_dir, image + '.env') 216 if os.path.isfile(fname): 217 # parse .env file 218 with open(fname) as varsfile: 219 for line in varsfile: 220 self._parse_line(line, image) 221 else: 222 print("Couldn't get bitbake variable from %s." % fname) 223 print("File %s doesn't exist." % fname) 224 return 225 else: 226 # Get bitbake -e output 227 cmd = "bitbake -e" 228 if image: 229 cmd += " %s" % image 230 231 log_level = logger.getEffectiveLevel() 232 logger.setLevel(logging.INFO) 233 ret, lines = _exec_cmd(cmd) 234 logger.setLevel(log_level) 235 236 if ret: 237 logger.error("Couldn't get '%s' output.", cmd) 238 logger.error("Bitbake failed with error:\n%s\n", lines) 239 return 240 241 # Parse bitbake -e output 242 for line in lines.split('\n'): 243 self._parse_line(line, image) 244 245 # Make first image a default set of variables 246 if cache: 247 images = [key for key in self if key] 248 if len(images) == 1: 249 self[None] = self[image] 250 251 result = self[image].get(var) 252 if not cache: 253 self.pop(image, None) 254 255 return result 256 257# Create BB_VARS singleton 258BB_VARS = BitbakeVars() 259 260def get_bitbake_var(var, image=None, cache=True): 261 """ 262 Provide old get_bitbake_var API by wrapping 263 get_var method of BB_VARS singleton. 264 """ 265 return BB_VARS.get_var(var, image, cache) 266