1# Copyright (C) 2016 Intel Corporation 2# Copyright (C) 2020 Savoir-Faire Linux 3# 4# SPDX-License-Identifier: GPL-2.0-only 5# 6"""Recipe creation tool - npm module support plugin""" 7 8import json 9import logging 10import os 11import re 12import sys 13import tempfile 14import bb 15from bb.fetch2.npm import NpmEnvironment 16from bb.fetch2.npmsw import foreach_dependencies 17from recipetool.create import RecipeHandler 18from recipetool.create import get_license_md5sums 19from recipetool.create import guess_license 20from recipetool.create import split_pkg_licenses 21logger = logging.getLogger('recipetool') 22 23TINFOIL = None 24 25def tinfoil_init(instance): 26 """Initialize tinfoil""" 27 global TINFOIL 28 TINFOIL = instance 29 30class NpmRecipeHandler(RecipeHandler): 31 """Class to handle the npm recipe creation""" 32 33 @staticmethod 34 def _npm_name(name): 35 """Generate a Yocto friendly npm name""" 36 name = re.sub("/", "-", name) 37 name = name.lower() 38 name = re.sub(r"[^\-a-z0-9]", "", name) 39 name = name.strip("-") 40 return name 41 42 @staticmethod 43 def _get_registry(lines): 44 """Get the registry value from the 'npm://registry' url""" 45 registry = None 46 47 def _handle_registry(varname, origvalue, op, newlines): 48 nonlocal registry 49 if origvalue.startswith("npm://"): 50 registry = re.sub(r"^npm://", "http://", origvalue.split(";")[0]) 51 return origvalue, None, 0, True 52 53 bb.utils.edit_metadata(lines, ["SRC_URI"], _handle_registry) 54 55 return registry 56 57 @staticmethod 58 def _ensure_npm(): 59 """Check if the 'npm' command is available in the recipes""" 60 if not TINFOIL.recipes_parsed: 61 TINFOIL.parse_recipes() 62 63 try: 64 d = TINFOIL.parse_recipe("nodejs-native") 65 except bb.providers.NoProvider: 66 bb.error("Nothing provides 'nodejs-native' which is required for the build") 67 bb.note("You will likely need to add a layer that provides nodejs") 68 sys.exit(14) 69 70 bindir = d.getVar("STAGING_BINDIR_NATIVE") 71 npmpath = os.path.join(bindir, "npm") 72 73 if not os.path.exists(npmpath): 74 TINFOIL.build_targets("nodejs-native", "addto_recipe_sysroot") 75 76 if not os.path.exists(npmpath): 77 bb.error("Failed to add 'npm' to sysroot") 78 sys.exit(14) 79 80 return bindir 81 82 @staticmethod 83 def _npm_global_configs(dev): 84 """Get the npm global configuration""" 85 configs = [] 86 87 if dev: 88 configs.append(("also", "development")) 89 else: 90 configs.append(("only", "production")) 91 92 configs.append(("save", "false")) 93 configs.append(("package-lock", "false")) 94 configs.append(("shrinkwrap", "false")) 95 return configs 96 97 def _run_npm_install(self, d, srctree, registry, dev): 98 """Run the 'npm install' command without building the addons""" 99 configs = self._npm_global_configs(dev) 100 configs.append(("ignore-scripts", "true")) 101 102 if registry: 103 configs.append(("registry", registry)) 104 105 bb.utils.remove(os.path.join(srctree, "node_modules"), recurse=True) 106 107 env = NpmEnvironment(d, configs=configs) 108 env.run("npm install", workdir=srctree) 109 110 def _generate_shrinkwrap(self, d, srctree, dev): 111 """Check and generate the 'npm-shrinkwrap.json' file if needed""" 112 configs = self._npm_global_configs(dev) 113 114 env = NpmEnvironment(d, configs=configs) 115 env.run("npm shrinkwrap", workdir=srctree) 116 117 return os.path.join(srctree, "npm-shrinkwrap.json") 118 119 def _handle_licenses(self, srctree, shrinkwrap_file, dev): 120 """Return the extra license files and the list of packages""" 121 licfiles = [] 122 packages = {} 123 124 # Handle the parent package 125 packages["${PN}"] = "" 126 127 def _licfiles_append_fallback_readme_files(destdir): 128 """Append README files as fallback to license files if a license files is missing""" 129 130 fallback = True 131 readmes = [] 132 basedir = os.path.join(srctree, destdir) 133 for fn in os.listdir(basedir): 134 upper = fn.upper() 135 if upper.startswith("README"): 136 fullpath = os.path.join(basedir, fn) 137 readmes.append(fullpath) 138 if upper.startswith("COPYING") or "LICENCE" in upper or "LICENSE" in upper: 139 fallback = False 140 if fallback: 141 for readme in readmes: 142 licfiles.append(os.path.relpath(readme, srctree)) 143 144 # Handle the dependencies 145 def _handle_dependency(name, params, deptree): 146 suffix = "-".join([self._npm_name(dep) for dep in deptree]) 147 destdirs = [os.path.join("node_modules", dep) for dep in deptree] 148 destdir = os.path.join(*destdirs) 149 packages["${PN}-" + suffix] = destdir 150 _licfiles_append_fallback_readme_files(destdir) 151 152 with open(shrinkwrap_file, "r") as f: 153 shrinkwrap = json.load(f) 154 155 foreach_dependencies(shrinkwrap, _handle_dependency, dev) 156 157 return licfiles, packages 158 159 def process(self, srctree, classes, lines_before, lines_after, handled, extravalues): 160 """Handle the npm recipe creation""" 161 162 if "buildsystem" in handled: 163 return False 164 165 files = RecipeHandler.checkfiles(srctree, ["package.json"]) 166 167 if not files: 168 return False 169 170 with open(files[0], "r") as f: 171 data = json.load(f) 172 173 if "name" not in data or "version" not in data: 174 return False 175 176 extravalues["PN"] = self._npm_name(data["name"]) 177 extravalues["PV"] = data["version"] 178 179 if "description" in data: 180 extravalues["SUMMARY"] = data["description"] 181 182 if "homepage" in data: 183 extravalues["HOMEPAGE"] = data["homepage"] 184 185 dev = bb.utils.to_boolean(str(extravalues.get("NPM_INSTALL_DEV", "0")), False) 186 registry = self._get_registry(lines_before) 187 188 bb.note("Checking if npm is available ...") 189 # The native npm is used here (and not the host one) to ensure that the 190 # npm version is high enough to ensure an efficient dependency tree 191 # resolution and avoid issue with the shrinkwrap file format. 192 # Moreover the native npm is mandatory for the build. 193 bindir = self._ensure_npm() 194 195 d = bb.data.createCopy(TINFOIL.config_data) 196 d.prependVar("PATH", bindir + ":") 197 d.setVar("S", srctree) 198 199 bb.note("Generating shrinkwrap file ...") 200 # To generate the shrinkwrap file the dependencies have to be installed 201 # first. During the generation process some files may be updated / 202 # deleted. By default devtool tracks the diffs in the srctree and raises 203 # errors when finishing the recipe if some diffs are found. 204 git_exclude_file = os.path.join(srctree, ".git", "info", "exclude") 205 if os.path.exists(git_exclude_file): 206 with open(git_exclude_file, "r+") as f: 207 lines = f.readlines() 208 for line in ["/node_modules/", "/npm-shrinkwrap.json"]: 209 if line not in lines: 210 f.write(line + "\n") 211 212 lock_file = os.path.join(srctree, "package-lock.json") 213 lock_copy = lock_file + ".copy" 214 if os.path.exists(lock_file): 215 bb.utils.copyfile(lock_file, lock_copy) 216 217 self._run_npm_install(d, srctree, registry, dev) 218 shrinkwrap_file = self._generate_shrinkwrap(d, srctree, dev) 219 220 with open(shrinkwrap_file, "r") as f: 221 shrinkwrap = json.load(f) 222 223 if os.path.exists(lock_copy): 224 bb.utils.movefile(lock_copy, lock_file) 225 226 # Add the shrinkwrap file as 'extrafiles' 227 shrinkwrap_copy = shrinkwrap_file + ".copy" 228 bb.utils.copyfile(shrinkwrap_file, shrinkwrap_copy) 229 extravalues.setdefault("extrafiles", {}) 230 extravalues["extrafiles"]["npm-shrinkwrap.json"] = shrinkwrap_copy 231 232 url_local = "npmsw://%s" % shrinkwrap_file 233 url_recipe= "npmsw://${THISDIR}/${BPN}/npm-shrinkwrap.json" 234 235 if dev: 236 url_local += ";dev=1" 237 url_recipe += ";dev=1" 238 239 # Add the npmsw url in the SRC_URI of the generated recipe 240 def _handle_srcuri(varname, origvalue, op, newlines): 241 """Update the version value and add the 'npmsw://' url""" 242 value = origvalue.replace("version=" + data["version"], "version=${PV}") 243 value = value.replace("version=latest", "version=${PV}") 244 values = [line.strip() for line in value.strip('\n').splitlines()] 245 if "dependencies" in shrinkwrap: 246 values.append(url_recipe) 247 return values, None, 4, False 248 249 (_, newlines) = bb.utils.edit_metadata(lines_before, ["SRC_URI"], _handle_srcuri) 250 lines_before[:] = [line.rstrip('\n') for line in newlines] 251 252 # In order to generate correct licence checksums in the recipe the 253 # dependencies have to be fetched again using the npmsw url 254 bb.note("Fetching npm dependencies ...") 255 bb.utils.remove(os.path.join(srctree, "node_modules"), recurse=True) 256 fetcher = bb.fetch2.Fetch([url_local], d) 257 fetcher.download() 258 fetcher.unpack(srctree) 259 260 bb.note("Handling licences ...") 261 (licfiles, packages) = self._handle_licenses(srctree, shrinkwrap_file, dev) 262 263 def _guess_odd_license(licfiles): 264 import bb 265 266 md5sums = get_license_md5sums(d, linenumbers=True) 267 268 chksums = [] 269 licenses = [] 270 for licfile in licfiles: 271 f = os.path.join(srctree, licfile) 272 md5value = bb.utils.md5_file(f) 273 (license, beginline, endline, md5) = md5sums.get(md5value, 274 (None, "", "", "")) 275 if not license: 276 license = "Unknown" 277 logger.info("Please add the following line for '%s' to a " 278 "'lib/recipetool/licenses.csv' and replace `Unknown`, " 279 "`X`, `Y` and `MD5` with the license, begin line, " 280 "end line and partial MD5 checksum:\n" \ 281 "%s,Unknown,X,Y,MD5" % (licfile, md5value)) 282 chksums.append("file://%s%s%s;md5=%s" % (licfile, 283 ";beginline=%s" % (beginline) if beginline else "", 284 ";endline=%s" % (endline) if endline else "", 285 md5 if md5 else md5value)) 286 licenses.append((license, licfile, md5value)) 287 return (licenses, chksums) 288 289 (licenses, extravalues["LIC_FILES_CHKSUM"]) = _guess_odd_license(licfiles) 290 split_pkg_licenses([*licenses, *guess_license(srctree, d)], packages, lines_after) 291 292 classes.append("npm") 293 handled.append("buildsystem") 294 295 return True 296 297def register_recipe_handlers(handlers): 298 """Register the npm handler""" 299 handlers.append((NpmRecipeHandler(), 60)) 300