1# Populates LICENSE_DIRECTORY as set in distro config with the license files as set by 2# LIC_FILES_CHKSUM. 3# TODO: 4# - There is a real issue revolving around license naming standards. 5 6LICENSE_DIRECTORY ??= "${DEPLOY_DIR}/licenses" 7LICSSTATEDIR = "${WORKDIR}/license-destdir/" 8 9# Create extra package with license texts and add it to RRECOMMENDS:${PN} 10LICENSE_CREATE_PACKAGE[type] = "boolean" 11LICENSE_CREATE_PACKAGE ??= "0" 12LICENSE_PACKAGE_SUFFIX ??= "-lic" 13LICENSE_FILES_DIRECTORY ??= "${datadir}/licenses/" 14 15addtask populate_lic after do_patch before do_build 16do_populate_lic[dirs] = "${LICSSTATEDIR}/${PN}" 17do_populate_lic[cleandirs] = "${LICSSTATEDIR}" 18 19python do_populate_lic() { 20 """ 21 Populate LICENSE_DIRECTORY with licenses. 22 """ 23 lic_files_paths = find_license_files(d) 24 25 # The base directory we wrangle licenses to 26 destdir = os.path.join(d.getVar('LICSSTATEDIR'), d.getVar('PN')) 27 copy_license_files(lic_files_paths, destdir) 28 info = get_recipe_info(d) 29 with open(os.path.join(destdir, "recipeinfo"), "w") as f: 30 for key in sorted(info.keys()): 31 f.write("%s: %s\n" % (key, info[key])) 32 oe.qa.exit_if_errors(d) 33} 34 35PSEUDO_IGNORE_PATHS .= ",${@','.join(((d.getVar('COMMON_LICENSE_DIR') or '') + ' ' + (d.getVar('LICENSE_PATH') or '') + ' ' + d.getVar('COREBASE') + '/meta/COPYING').split())}" 36# it would be better to copy them in do_install:append, but find_license_filesa is python 37python perform_packagecopy:prepend () { 38 enabled = oe.data.typed_value('LICENSE_CREATE_PACKAGE', d) 39 if d.getVar('CLASSOVERRIDE') == 'class-target' and enabled: 40 lic_files_paths = find_license_files(d) 41 42 # LICENSE_FILES_DIRECTORY starts with '/' so os.path.join cannot be used to join D and LICENSE_FILES_DIRECTORY 43 destdir = d.getVar('D') + os.path.join(d.getVar('LICENSE_FILES_DIRECTORY'), d.getVar('PN')) 44 copy_license_files(lic_files_paths, destdir) 45 add_package_and_files(d) 46} 47perform_packagecopy[vardeps] += "LICENSE_CREATE_PACKAGE" 48 49def get_recipe_info(d): 50 info = {} 51 info["PV"] = d.getVar("PV") 52 info["PR"] = d.getVar("PR") 53 info["LICENSE"] = d.getVar("LICENSE") 54 return info 55 56def add_package_and_files(d): 57 packages = d.getVar('PACKAGES') 58 files = d.getVar('LICENSE_FILES_DIRECTORY') 59 pn = d.getVar('PN') 60 pn_lic = "%s%s" % (pn, d.getVar('LICENSE_PACKAGE_SUFFIX', False)) 61 if pn_lic in packages.split(): 62 bb.warn("%s package already existed in %s." % (pn_lic, pn)) 63 else: 64 # first in PACKAGES to be sure that nothing else gets LICENSE_FILES_DIRECTORY 65 d.setVar('PACKAGES', "%s %s" % (pn_lic, packages)) 66 d.setVar('FILES:' + pn_lic, files) 67 68def copy_license_files(lic_files_paths, destdir): 69 import shutil 70 import errno 71 72 bb.utils.mkdirhier(destdir) 73 for (basename, path, beginline, endline) in lic_files_paths: 74 try: 75 src = path 76 dst = os.path.join(destdir, basename) 77 if os.path.exists(dst): 78 os.remove(dst) 79 if os.path.islink(src): 80 src = os.path.realpath(src) 81 canlink = os.access(src, os.W_OK) and (os.stat(src).st_dev == os.stat(destdir).st_dev) and beginline is None and endline is None 82 if canlink: 83 try: 84 os.link(src, dst) 85 except OSError as err: 86 if err.errno == errno.EXDEV: 87 # Copy license files if hardlink is not possible even if st_dev is the 88 # same on source and destination (docker container with device-mapper?) 89 canlink = False 90 else: 91 raise 92 # Only chown if we did hardlink and we're running under pseudo 93 if canlink and os.environ.get('PSEUDO_DISABLED') == '0': 94 os.chown(dst,0,0) 95 if not canlink: 96 begin_idx = max(0, int(beginline) - 1) if beginline is not None else None 97 end_idx = max(0, int(endline)) if endline is not None else None 98 if begin_idx is None and end_idx is None: 99 shutil.copyfile(src, dst) 100 else: 101 with open(src, 'rb') as src_f: 102 with open(dst, 'wb') as dst_f: 103 dst_f.write(b''.join(src_f.readlines()[begin_idx:end_idx])) 104 105 except Exception as e: 106 bb.warn("Could not copy license file %s to %s: %s" % (src, dst, e)) 107 108def find_license_files(d): 109 """ 110 Creates list of files used in LIC_FILES_CHKSUM and generic LICENSE files. 111 """ 112 import shutil 113 import oe.license 114 from collections import defaultdict, OrderedDict 115 116 # All the license files for the package 117 lic_files = d.getVar('LIC_FILES_CHKSUM') or "" 118 pn = d.getVar('PN') 119 # The license files are located in S/LIC_FILE_CHECKSUM. 120 srcdir = d.getVar('S') 121 # Directory we store the generic licenses as set in the distro configuration 122 generic_directory = d.getVar('COMMON_LICENSE_DIR') 123 # List of basename, path tuples 124 lic_files_paths = [] 125 # hash for keep track generic lics mappings 126 non_generic_lics = {} 127 # Entries from LIC_FILES_CHKSUM 128 lic_chksums = {} 129 license_source_dirs = [] 130 license_source_dirs.append(generic_directory) 131 try: 132 additional_lic_dirs = d.getVar('LICENSE_PATH').split() 133 for lic_dir in additional_lic_dirs: 134 license_source_dirs.append(lic_dir) 135 except: 136 pass 137 138 class FindVisitor(oe.license.LicenseVisitor): 139 def visit_Str(self, node): 140 # 141 # Until I figure out what to do with 142 # the two modifiers I support (or greater = + 143 # and "with exceptions" being * 144 # we'll just strip out the modifier and put 145 # the base license. 146 find_license(node.s.replace("+", "").replace("*", "")) 147 self.generic_visit(node) 148 149 def visit_Constant(self, node): 150 find_license(node.value.replace("+", "").replace("*", "")) 151 self.generic_visit(node) 152 153 def find_license(license_type): 154 try: 155 bb.utils.mkdirhier(gen_lic_dest) 156 except: 157 pass 158 spdx_generic = None 159 license_source = None 160 # If the generic does not exist we need to check to see if there is an SPDX mapping to it, 161 # unless NO_GENERIC_LICENSE is set. 162 for lic_dir in license_source_dirs: 163 if not os.path.isfile(os.path.join(lic_dir, license_type)): 164 if d.getVarFlag('SPDXLICENSEMAP', license_type) != None: 165 # Great, there is an SPDXLICENSEMAP. We can copy! 166 bb.debug(1, "We need to use a SPDXLICENSEMAP for %s" % (license_type)) 167 spdx_generic = d.getVarFlag('SPDXLICENSEMAP', license_type) 168 license_source = lic_dir 169 break 170 elif os.path.isfile(os.path.join(lic_dir, license_type)): 171 spdx_generic = license_type 172 license_source = lic_dir 173 break 174 175 non_generic_lic = d.getVarFlag('NO_GENERIC_LICENSE', license_type) 176 if spdx_generic and license_source: 177 # we really should copy to generic_ + spdx_generic, however, that ends up messing the manifest 178 # audit up. This should be fixed in emit_pkgdata (or, we actually got and fix all the recipes) 179 180 lic_files_paths.append(("generic_" + license_type, os.path.join(license_source, spdx_generic), 181 None, None)) 182 183 # The user may attempt to use NO_GENERIC_LICENSE for a generic license which doesn't make sense 184 # and should not be allowed, warn the user in this case. 185 if d.getVarFlag('NO_GENERIC_LICENSE', license_type): 186 oe.qa.handle_error("license-no-generic", 187 "%s: %s is a generic license, please don't use NO_GENERIC_LICENSE for it." % (pn, license_type), d) 188 189 elif non_generic_lic and non_generic_lic in lic_chksums: 190 # if NO_GENERIC_LICENSE is set, we copy the license files from the fetched source 191 # of the package rather than the license_source_dirs. 192 lic_files_paths.append(("generic_" + license_type, 193 os.path.join(srcdir, non_generic_lic), None, None)) 194 non_generic_lics[non_generic_lic] = license_type 195 else: 196 # Explicitly avoid the CLOSED license because this isn't generic 197 if license_type != 'CLOSED': 198 # And here is where we warn people that their licenses are lousy 199 oe.qa.handle_error("license-exists", 200 "%s: No generic license file exists for: %s in any provider" % (pn, license_type), d) 201 pass 202 203 if not generic_directory: 204 bb.fatal("COMMON_LICENSE_DIR is unset. Please set this in your distro config") 205 206 for url in lic_files.split(): 207 try: 208 (method, host, path, user, pswd, parm) = bb.fetch.decodeurl(url) 209 if method != "file" or not path: 210 raise bb.fetch.MalformedUrl() 211 except bb.fetch.MalformedUrl: 212 bb.fatal("%s: LIC_FILES_CHKSUM contains an invalid URL: %s" % (d.getVar('PF'), url)) 213 # We want the license filename and path 214 chksum = parm.get('md5', None) 215 beginline = parm.get('beginline') 216 endline = parm.get('endline') 217 lic_chksums[path] = (chksum, beginline, endline) 218 219 v = FindVisitor() 220 try: 221 v.visit_string(d.getVar('LICENSE')) 222 except oe.license.InvalidLicense as exc: 223 bb.fatal('%s: %s' % (d.getVar('PF'), exc)) 224 except SyntaxError: 225 oe.qa.handle_error("license-syntax", 226 "%s: Failed to parse it's LICENSE field." % (d.getVar('PF')), d) 227 # Add files from LIC_FILES_CHKSUM to list of license files 228 lic_chksum_paths = defaultdict(OrderedDict) 229 for path, data in sorted(lic_chksums.items()): 230 lic_chksum_paths[os.path.basename(path)][data] = (os.path.join(srcdir, path), data[1], data[2]) 231 for basename, files in lic_chksum_paths.items(): 232 if len(files) == 1: 233 # Don't copy again a LICENSE already handled as non-generic 234 if basename in non_generic_lics: 235 continue 236 data = list(files.values())[0] 237 lic_files_paths.append(tuple([basename] + list(data))) 238 else: 239 # If there are multiple different license files with identical 240 # basenames we rename them to <file>.0, <file>.1, ... 241 for i, data in enumerate(files.values()): 242 lic_files_paths.append(tuple(["%s.%d" % (basename, i)] + list(data))) 243 244 return lic_files_paths 245 246def return_spdx(d, license): 247 """ 248 This function returns the spdx mapping of a license if it exists. 249 """ 250 return d.getVarFlag('SPDXLICENSEMAP', license) 251 252def canonical_license(d, license): 253 """ 254 Return the canonical (SPDX) form of the license if available (so GPLv3 255 becomes GPL-3.0-only) or the passed license if there is no canonical form. 256 """ 257 return d.getVarFlag('SPDXLICENSEMAP', license) or license 258 259def expand_wildcard_licenses(d, wildcard_licenses): 260 """ 261 There are some common wildcard values users may want to use. Support them 262 here. 263 """ 264 licenses = set(wildcard_licenses) 265 mapping = { 266 "AGPL-3.0*" : ["AGPL-3.0-only", "AGPL-3.0-or-later"], 267 "GPL-3.0*" : ["GPL-3.0-only", "GPL-3.0-or-later"], 268 "LGPL-3.0*" : ["LGPL-3.0-only", "LGPL-3.0-or-later"], 269 } 270 for k in mapping: 271 if k in wildcard_licenses: 272 licenses.remove(k) 273 for item in mapping[k]: 274 licenses.add(item) 275 276 for l in licenses: 277 if l in oe.license.obsolete_license_list(): 278 bb.fatal("Error, %s is an obsolete license, please use an SPDX reference in INCOMPATIBLE_LICENSE" % l) 279 if "*" in l: 280 bb.fatal("Error, %s is an invalid license wildcard entry" % l) 281 282 return list(licenses) 283 284def incompatible_license_contains(license, truevalue, falsevalue, d): 285 license = canonical_license(d, license) 286 bad_licenses = (d.getVar('INCOMPATIBLE_LICENSE') or "").split() 287 bad_licenses = expand_wildcard_licenses(d, bad_licenses) 288 return truevalue if license in bad_licenses else falsevalue 289 290def incompatible_pkg_license(d, dont_want_licenses, license): 291 # Handles an "or" or two license sets provided by 292 # flattened_licenses(), pick one that works if possible. 293 def choose_lic_set(a, b): 294 return a if all(oe.license.license_ok(canonical_license(d, lic), 295 dont_want_licenses) for lic in a) else b 296 297 try: 298 licenses = oe.license.flattened_licenses(license, choose_lic_set) 299 except oe.license.LicenseError as exc: 300 bb.fatal('%s: %s' % (d.getVar('P'), exc)) 301 302 incompatible_lic = [] 303 for l in licenses: 304 license = canonical_license(d, l) 305 if not oe.license.license_ok(license, dont_want_licenses): 306 incompatible_lic.append(license) 307 308 return sorted(incompatible_lic) 309 310def incompatible_license(d, dont_want_licenses, package=None): 311 """ 312 This function checks if a recipe has only incompatible licenses. It also 313 take into consideration 'or' operand. dont_want_licenses should be passed 314 as canonical (SPDX) names. 315 """ 316 import oe.license 317 license = d.getVar("LICENSE:%s" % package) if package else None 318 if not license: 319 license = d.getVar('LICENSE') 320 321 return incompatible_pkg_license(d, dont_want_licenses, license) 322 323def check_license_flags(d): 324 """ 325 This function checks if a recipe has any LICENSE_FLAGS that 326 aren't acceptable. 327 328 If it does, it returns the all LICENSE_FLAGS missing from the list 329 of acceptable license flags, or all of the LICENSE_FLAGS if there 330 is no list of acceptable flags. 331 332 If everything is is acceptable, it returns None. 333 """ 334 335 def license_flag_matches(flag, acceptlist, pn): 336 """ 337 Return True if flag matches something in acceptlist, None if not. 338 339 Before we test a flag against the acceptlist, we append _${PN} 340 to it. We then try to match that string against the 341 acceptlist. This covers the normal case, where we expect 342 LICENSE_FLAGS to be a simple string like 'commercial', which 343 the user typically matches exactly in the acceptlist by 344 explicitly appending the package name e.g 'commercial_foo'. 345 If we fail the match however, we then split the flag across 346 '_' and append each fragment and test until we either match or 347 run out of fragments. 348 """ 349 flag_pn = ("%s_%s" % (flag, pn)) 350 for candidate in acceptlist: 351 if flag_pn == candidate: 352 return True 353 354 flag_cur = "" 355 flagments = flag_pn.split("_") 356 flagments.pop() # we've already tested the full string 357 for flagment in flagments: 358 if flag_cur: 359 flag_cur += "_" 360 flag_cur += flagment 361 for candidate in acceptlist: 362 if flag_cur == candidate: 363 return True 364 return False 365 366 def all_license_flags_match(license_flags, acceptlist): 367 """ Return all unmatched flags, None if all flags match """ 368 pn = d.getVar('PN') 369 split_acceptlist = acceptlist.split() 370 flags = [] 371 for flag in license_flags.split(): 372 if not license_flag_matches(flag, split_acceptlist, pn): 373 flags.append(flag) 374 return flags if flags else None 375 376 license_flags = d.getVar('LICENSE_FLAGS') 377 if license_flags: 378 acceptlist = d.getVar('LICENSE_FLAGS_ACCEPTED') 379 if not acceptlist: 380 return license_flags.split() 381 unmatched_flags = all_license_flags_match(license_flags, acceptlist) 382 if unmatched_flags: 383 return unmatched_flags 384 return None 385 386def check_license_format(d): 387 """ 388 This function checks if LICENSE is well defined, 389 Validate operators in LICENSES. 390 No spaces are allowed between LICENSES. 391 """ 392 pn = d.getVar('PN') 393 licenses = d.getVar('LICENSE') 394 from oe.license import license_operator, license_operator_chars, license_pattern 395 396 elements = list(filter(lambda x: x.strip(), license_operator.split(licenses))) 397 for pos, element in enumerate(elements): 398 if license_pattern.match(element): 399 if pos > 0 and license_pattern.match(elements[pos - 1]): 400 oe.qa.handle_error('license-format', 401 '%s: LICENSE value "%s" has an invalid format - license names ' \ 402 'must be separated by the following characters to indicate ' \ 403 'the license selection: %s' % 404 (pn, licenses, license_operator_chars), d) 405 elif not license_operator.match(element): 406 oe.qa.handle_error('license-format', 407 '%s: LICENSE value "%s" has an invalid separator "%s" that is not ' \ 408 'in the valid list of separators (%s)' % 409 (pn, licenses, element, license_operator_chars), d) 410 411SSTATETASKS += "do_populate_lic" 412do_populate_lic[sstate-inputdirs] = "${LICSSTATEDIR}" 413do_populate_lic[sstate-outputdirs] = "${LICENSE_DIRECTORY}/" 414 415IMAGE_CLASSES:append = " license_image" 416 417python do_populate_lic_setscene () { 418 sstate_setscene(d) 419} 420addtask do_populate_lic_setscene 421