1# Copyright (C) 2020 Savoir-Faire Linux 2# 3# SPDX-License-Identifier: GPL-2.0-only 4# 5# This bbclass builds and installs an npm package to the target. The package 6# sources files should be fetched in the calling recipe by using the SRC_URI 7# variable. The ${S} variable should be updated depending of your fetcher. 8# 9# Usage: 10# SRC_URI = "..." 11# inherit npm 12# 13# Optional variables: 14# NPM_ARCH: 15# Override the auto generated npm architecture. 16# 17# NPM_INSTALL_DEV: 18# Set to 1 to also install devDependencies. 19 20inherit python3native 21 22DEPENDS:prepend = "nodejs-native nodejs-oe-cache-native " 23RDEPENDS:${PN}:append:class-target = " nodejs" 24 25EXTRA_OENPM = "" 26 27NPM_INSTALL_DEV ?= "0" 28 29NPM_NODEDIR ?= "${RECIPE_SYSROOT_NATIVE}${prefix_native}" 30 31def npm_target_arch_map(target_arch): 32 """Maps arch names to npm arch names""" 33 import re 34 if re.match("p(pc|owerpc)(|64)", target_arch): 35 return "ppc" 36 elif re.match("i.86$", target_arch): 37 return "ia32" 38 elif re.match("x86_64$", target_arch): 39 return "x64" 40 elif re.match("arm64$", target_arch): 41 return "arm" 42 return target_arch 43 44NPM_ARCH ?= "${@npm_target_arch_map(d.getVar("TARGET_ARCH"))}" 45 46NPM_PACKAGE = "${WORKDIR}/npm-package" 47NPM_CACHE = "${WORKDIR}/npm-cache" 48NPM_BUILD = "${WORKDIR}/npm-build" 49NPM_REGISTRY = "${WORKDIR}/npm-registry" 50 51def npm_global_configs(d): 52 """Get the npm global configuration""" 53 configs = [] 54 # Ensure no network access is done 55 configs.append(("offline", "true")) 56 configs.append(("proxy", "http://invalid")) 57 # Configure the cache directory 58 configs.append(("cache", d.getVar("NPM_CACHE"))) 59 return configs 60 61## 'npm pack' runs 'prepare' and 'prepack' scripts. Support for 62## 'ignore-scripts' which prevents this behavior has been removed 63## from nodejs 16. Use simple 'tar' instead of. 64def npm_pack(env, srcdir, workdir): 65 """Emulate 'npm pack' on a specified directory""" 66 import subprocess 67 import os 68 import json 69 70 src = os.path.join(srcdir, 'package.json') 71 with open(src) as f: 72 j = json.load(f) 73 74 # base does not really matter and is for documentation purposes 75 # only. But the 'version' part must exist because other parts of 76 # the bbclass rely on it. 77 base = j['name'].split('/')[-1] 78 tarball = os.path.join(workdir, "%s-%s.tgz" % (base, j['version'])); 79 80 # TODO: real 'npm pack' does not include directories while 'tar' 81 # does. But this does not seem to matter... 82 subprocess.run(['tar', 'czf', tarball, 83 '--exclude', './node-modules', 84 '--exclude-vcs', 85 '--transform', 's,^\./,package/,', 86 '--mtime', '1985-10-26T08:15:00.000Z', 87 '.'], 88 check = True, cwd = srcdir) 89 90 return (tarball, j) 91 92python npm_do_configure() { 93 """ 94 Step one: configure the npm cache and the main npm package 95 96 Every dependencies have been fetched and patched in the source directory. 97 They have to be packed (this remove unneeded files) and added to the npm 98 cache to be available for the next step. 99 100 The main package and its associated manifest file and shrinkwrap file have 101 to be configured to take into account these cached dependencies. 102 """ 103 import base64 104 import copy 105 import json 106 import re 107 import shlex 108 import tempfile 109 from bb.fetch2.npm import NpmEnvironment 110 from bb.fetch2.npm import npm_unpack 111 from bb.fetch2.npmsw import foreach_dependencies 112 from bb.progress import OutOfProgressHandler 113 from oe.npm_registry import NpmRegistry 114 115 bb.utils.remove(d.getVar("NPM_CACHE"), recurse=True) 116 bb.utils.remove(d.getVar("NPM_PACKAGE"), recurse=True) 117 118 env = NpmEnvironment(d, configs=npm_global_configs(d)) 119 registry = NpmRegistry(d.getVar('NPM_REGISTRY'), d.getVar('NPM_CACHE')) 120 121 def _npm_cache_add(tarball, pkg): 122 """Add tarball to local registry and register it in the 123 cache""" 124 registry.add_pkg(tarball, pkg) 125 126 def _npm_integrity(tarball): 127 """Return the npm integrity of a specified tarball""" 128 sha512 = bb.utils.sha512_file(tarball) 129 return "sha512-" + base64.b64encode(bytes.fromhex(sha512)).decode() 130 131 def _npmsw_dependency_dict(orig, deptree): 132 """ 133 Return the sub dictionary in the 'orig' dictionary corresponding to the 134 'deptree' dependency tree. This function follows the shrinkwrap file 135 format. 136 """ 137 ptr = orig 138 for dep in deptree: 139 if "dependencies" not in ptr: 140 ptr["dependencies"] = {} 141 ptr = ptr["dependencies"] 142 if dep not in ptr: 143 ptr[dep] = {} 144 ptr = ptr[dep] 145 return ptr 146 147 # Manage the manifest file and shrinkwrap files 148 orig_manifest_file = d.expand("${S}/package.json") 149 orig_shrinkwrap_file = d.expand("${S}/npm-shrinkwrap.json") 150 cached_manifest_file = d.expand("${NPM_PACKAGE}/package.json") 151 cached_shrinkwrap_file = d.expand("${NPM_PACKAGE}/npm-shrinkwrap.json") 152 153 with open(orig_manifest_file, "r") as f: 154 orig_manifest = json.load(f) 155 156 cached_manifest = copy.deepcopy(orig_manifest) 157 cached_manifest.pop("dependencies", None) 158 cached_manifest.pop("devDependencies", None) 159 160 has_shrinkwrap_file = True 161 162 try: 163 with open(orig_shrinkwrap_file, "r") as f: 164 orig_shrinkwrap = json.load(f) 165 except IOError: 166 has_shrinkwrap_file = False 167 168 if has_shrinkwrap_file: 169 cached_shrinkwrap = copy.deepcopy(orig_shrinkwrap) 170 cached_shrinkwrap.pop("dependencies", None) 171 172 # Manage the dependencies 173 progress = OutOfProgressHandler(d, r"^(\d+)/(\d+)$") 174 progress_total = 1 # also count the main package 175 progress_done = 0 176 177 def _count_dependency(name, params, deptree): 178 nonlocal progress_total 179 progress_total += 1 180 181 def _cache_dependency(name, params, deptree): 182 destsubdirs = [os.path.join("node_modules", dep) for dep in deptree] 183 destsuffix = os.path.join(*destsubdirs) 184 with tempfile.TemporaryDirectory() as tmpdir: 185 # Add the dependency to the npm cache 186 destdir = os.path.join(d.getVar("S"), destsuffix) 187 (tarball, pkg) = npm_pack(env, destdir, tmpdir) 188 _npm_cache_add(tarball, pkg) 189 # Add its signature to the cached shrinkwrap 190 dep = _npmsw_dependency_dict(cached_shrinkwrap, deptree) 191 dep["version"] = pkg['version'] 192 dep["integrity"] = _npm_integrity(tarball) 193 if params.get("dev", False): 194 dep["dev"] = True 195 # Display progress 196 nonlocal progress_done 197 progress_done += 1 198 progress.write("%d/%d" % (progress_done, progress_total)) 199 200 dev = bb.utils.to_boolean(d.getVar("NPM_INSTALL_DEV"), False) 201 202 if has_shrinkwrap_file: 203 foreach_dependencies(orig_shrinkwrap, _count_dependency, dev) 204 foreach_dependencies(orig_shrinkwrap, _cache_dependency, dev) 205 206 # Configure the main package 207 with tempfile.TemporaryDirectory() as tmpdir: 208 (tarball, _) = npm_pack(env, d.getVar("S"), tmpdir) 209 npm_unpack(tarball, d.getVar("NPM_PACKAGE"), d) 210 211 # Configure the cached manifest file and cached shrinkwrap file 212 def _update_manifest(depkey): 213 for name in orig_manifest.get(depkey, {}): 214 version = cached_shrinkwrap["dependencies"][name]["version"] 215 if depkey not in cached_manifest: 216 cached_manifest[depkey] = {} 217 cached_manifest[depkey][name] = version 218 219 if has_shrinkwrap_file: 220 _update_manifest("dependencies") 221 222 if dev: 223 if has_shrinkwrap_file: 224 _update_manifest("devDependencies") 225 226 with open(cached_manifest_file, "w") as f: 227 json.dump(cached_manifest, f, indent=2) 228 229 if has_shrinkwrap_file: 230 with open(cached_shrinkwrap_file, "w") as f: 231 json.dump(cached_shrinkwrap, f, indent=2) 232} 233 234python npm_do_compile() { 235 """ 236 Step two: install the npm package 237 238 Use the configured main package and the cached dependencies to run the 239 installation process. The installation is done in a directory which is 240 not the destination directory yet. 241 242 A combination of 'npm pack' and 'npm install' is used to ensure that the 243 installed files are actual copies instead of symbolic links (which is the 244 default npm behavior). 245 """ 246 import shlex 247 import tempfile 248 from bb.fetch2.npm import NpmEnvironment 249 250 bb.utils.remove(d.getVar("NPM_BUILD"), recurse=True) 251 252 with tempfile.TemporaryDirectory() as tmpdir: 253 args = [] 254 configs = npm_global_configs(d) 255 256 if bb.utils.to_boolean(d.getVar("NPM_INSTALL_DEV"), False): 257 configs.append(("also", "development")) 258 else: 259 configs.append(("only", "production")) 260 261 # Report as many logs as possible for debugging purpose 262 configs.append(("loglevel", "silly")) 263 264 # Configure the installation to be done globally in the build directory 265 configs.append(("global", "true")) 266 configs.append(("prefix", d.getVar("NPM_BUILD"))) 267 268 # Add node-gyp configuration 269 configs.append(("arch", d.getVar("NPM_ARCH"))) 270 configs.append(("release", "true")) 271 configs.append(("nodedir", d.getVar("NPM_NODEDIR"))) 272 configs.append(("python", d.getVar("PYTHON"))) 273 274 env = NpmEnvironment(d, configs) 275 276 # Add node-pre-gyp configuration 277 args.append(("target_arch", d.getVar("NPM_ARCH"))) 278 args.append(("build-from-source", "true")) 279 280 # Pack and install the main package 281 (tarball, _) = npm_pack(env, d.getVar("NPM_PACKAGE"), tmpdir) 282 cmd = "npm install %s %s" % (shlex.quote(tarball), d.getVar("EXTRA_OENPM")) 283 env.run(cmd, args=args) 284} 285 286npm_do_install() { 287 # Step three: final install 288 # 289 # The previous installation have to be filtered to remove some extra files. 290 291 rm -rf ${D} 292 293 # Copy the entire lib and bin directories 294 install -d ${D}/${nonarch_libdir} 295 cp --no-preserve=ownership --recursive ${NPM_BUILD}/lib/. ${D}/${nonarch_libdir} 296 297 if [ -d "${NPM_BUILD}/bin" ] 298 then 299 install -d ${D}/${bindir} 300 cp --no-preserve=ownership --recursive ${NPM_BUILD}/bin/. ${D}/${bindir} 301 fi 302 303 # If the package (or its dependencies) uses node-gyp to build native addons, 304 # object files, static libraries or other temporary files can be hidden in 305 # the lib directory. To reduce the package size and to avoid QA issues 306 # (staticdev with static library files) these files must be removed. 307 local GYP_REGEX=".*/build/Release/[^/]*.node" 308 309 # Remove any node-gyp directory in ${D} to remove temporary build files 310 for GYP_D_FILE in $(find ${D} -regex "${GYP_REGEX}") 311 do 312 local GYP_D_DIR=${GYP_D_FILE%/Release/*} 313 314 rm --recursive --force ${GYP_D_DIR} 315 done 316 317 # Copy only the node-gyp release files 318 for GYP_B_FILE in $(find ${NPM_BUILD} -regex "${GYP_REGEX}") 319 do 320 local GYP_D_FILE=${D}/${prefix}/${GYP_B_FILE#${NPM_BUILD}} 321 322 install -d ${GYP_D_FILE%/*} 323 install -m 755 ${GYP_B_FILE} ${GYP_D_FILE} 324 done 325 326 # Remove the shrinkwrap file which does not need to be packed 327 rm -f ${D}/${nonarch_libdir}/node_modules/*/npm-shrinkwrap.json 328 rm -f ${D}/${nonarch_libdir}/node_modules/@*/*/npm-shrinkwrap.json 329 330 # node(1) is using /usr/lib/node as default include directory and npm(1) is 331 # using /usr/lib/node_modules as install directory. Let's make both happy. 332 ln -fs node_modules ${D}/${nonarch_libdir}/node 333} 334 335FILES:${PN} += " \ 336 ${bindir} \ 337 ${nonarch_libdir} \ 338" 339 340EXPORT_FUNCTIONS do_configure do_compile do_install 341