|
@@ -0,0 +1,255 @@
|
|
|
+# Copyright (C) 2020 Savoir-Faire Linux
|
|
|
+#
|
|
|
+# SPDX-License-Identifier: GPL-2.0-only
|
|
|
+#
|
|
|
+"""
|
|
|
+BitBake 'Fetch' npm shrinkwrap implementation
|
|
|
+
|
|
|
+npm fetcher support the SRC_URI with format of:
|
|
|
+SRC_URI = "npmsw://some.registry.url;OptionA=xxx;OptionB=xxx;..."
|
|
|
+
|
|
|
+Supported SRC_URI options are:
|
|
|
+
|
|
|
+- dev
|
|
|
+ Set to 1 to also install devDependencies.
|
|
|
+
|
|
|
+- destsuffix
|
|
|
+ Specifies the directory to use to unpack the dependencies (default: ${S}).
|
|
|
+"""
|
|
|
+
|
|
|
+import json
|
|
|
+import os
|
|
|
+import re
|
|
|
+import bb
|
|
|
+from bb.fetch2 import Fetch
|
|
|
+from bb.fetch2 import FetchMethod
|
|
|
+from bb.fetch2 import ParameterError
|
|
|
+from bb.fetch2 import URI
|
|
|
+from bb.fetch2.npm import npm_integrity
|
|
|
+from bb.fetch2.npm import npm_localfile
|
|
|
+from bb.fetch2.npm import npm_unpack
|
|
|
+from bb.utils import is_semver
|
|
|
+
|
|
|
+def foreach_dependencies(shrinkwrap, callback=None, dev=False):
|
|
|
+ """
|
|
|
+ Run a callback for each dependencies of a shrinkwrap file.
|
|
|
+ The callback is using the format:
|
|
|
+ callback(name, params, deptree)
|
|
|
+ with:
|
|
|
+ name = the package name (string)
|
|
|
+ params = the package parameters (dictionary)
|
|
|
+ deptree = the package dependency tree (array of strings)
|
|
|
+ """
|
|
|
+ def _walk_deps(deps, deptree):
|
|
|
+ for name in deps:
|
|
|
+ subtree = [*deptree, name]
|
|
|
+ _walk_deps(deps[name].get("dependencies", {}), subtree)
|
|
|
+ if callback is not None:
|
|
|
+ if deps[name].get("dev", False) and not dev:
|
|
|
+ continue
|
|
|
+ elif deps[name].get("bundled", False):
|
|
|
+ continue
|
|
|
+ callback(name, deps[name], subtree)
|
|
|
+
|
|
|
+ _walk_deps(shrinkwrap.get("dependencies", {}), [])
|
|
|
+
|
|
|
+class NpmShrinkWrap(FetchMethod):
|
|
|
+ """Class to fetch all package from a shrinkwrap file"""
|
|
|
+
|
|
|
+ def supports(self, ud, d):
|
|
|
+ """Check if a given url can be fetched with npmsw"""
|
|
|
+ return ud.type in ["npmsw"]
|
|
|
+
|
|
|
+ def urldata_init(self, ud, d):
|
|
|
+ """Init npmsw specific variables within url data"""
|
|
|
+
|
|
|
+ # Get the 'shrinkwrap' parameter
|
|
|
+ ud.shrinkwrap_file = re.sub(r"^npmsw://", "", ud.url.split(";")[0])
|
|
|
+
|
|
|
+ # Get the 'dev' parameter
|
|
|
+ ud.dev = bb.utils.to_boolean(ud.parm.get("dev"), False)
|
|
|
+
|
|
|
+ # Resolve the dependencies
|
|
|
+ ud.deps = []
|
|
|
+
|
|
|
+ def _resolve_dependency(name, params, deptree):
|
|
|
+ url = None
|
|
|
+ localpath = None
|
|
|
+ extrapaths = []
|
|
|
+ destsubdirs = [os.path.join("node_modules", dep) for dep in deptree]
|
|
|
+ destsuffix = os.path.join(*destsubdirs)
|
|
|
+
|
|
|
+ integrity = params.get("integrity", None)
|
|
|
+ resolved = params.get("resolved", None)
|
|
|
+ version = params.get("version", None)
|
|
|
+
|
|
|
+ # Handle registry sources
|
|
|
+ if is_semver(version) and resolved and integrity:
|
|
|
+ localfile = npm_localfile(name, version)
|
|
|
+
|
|
|
+ uri = URI(resolved)
|
|
|
+ uri.params["downloadfilename"] = localfile
|
|
|
+
|
|
|
+ checksum_name, checksum_expected = npm_integrity(integrity)
|
|
|
+ uri.params[checksum_name] = checksum_expected
|
|
|
+
|
|
|
+ url = str(uri)
|
|
|
+
|
|
|
+ localpath = os.path.join(d.getVar("DL_DIR"), localfile)
|
|
|
+
|
|
|
+ # Create a resolve file to mimic the npm fetcher and allow
|
|
|
+ # re-usability of the downloaded file.
|
|
|
+ resolvefile = localpath + ".resolved"
|
|
|
+
|
|
|
+ bb.utils.mkdirhier(os.path.dirname(resolvefile))
|
|
|
+ with open(resolvefile, "w") as f:
|
|
|
+ f.write(url)
|
|
|
+
|
|
|
+ extrapaths.append(resolvefile)
|
|
|
+
|
|
|
+ # Handle http tarball sources
|
|
|
+ elif version.startswith("http") and integrity:
|
|
|
+ localfile = os.path.join("npm2", os.path.basename(version))
|
|
|
+
|
|
|
+ uri = URI(version)
|
|
|
+ uri.params["downloadfilename"] = localfile
|
|
|
+
|
|
|
+ checksum_name, checksum_expected = npm_integrity(integrity)
|
|
|
+ uri.params[checksum_name] = checksum_expected
|
|
|
+
|
|
|
+ url = str(uri)
|
|
|
+
|
|
|
+ localpath = os.path.join(d.getVar("DL_DIR"), localfile)
|
|
|
+
|
|
|
+ # Handle git sources
|
|
|
+ elif version.startswith("git"):
|
|
|
+ regex = re.compile(r"""
|
|
|
+ ^
|
|
|
+ git\+
|
|
|
+ (?P<protocol>[a-z]+)
|
|
|
+ ://
|
|
|
+ (?P<url>[^#]+)
|
|
|
+ \#
|
|
|
+ (?P<rev>[0-9a-f]+)
|
|
|
+ $
|
|
|
+ """, re.VERBOSE)
|
|
|
+
|
|
|
+ match = regex.match(version)
|
|
|
+
|
|
|
+ if not match:
|
|
|
+ raise ParameterError("Invalid git url: %s" % version, ud.url)
|
|
|
+
|
|
|
+ groups = match.groupdict()
|
|
|
+
|
|
|
+ uri = URI("git://" + str(groups["url"]))
|
|
|
+ uri.params["protocol"] = str(groups["protocol"])
|
|
|
+ uri.params["rev"] = str(groups["rev"])
|
|
|
+ uri.params["destsuffix"] = destsuffix
|
|
|
+
|
|
|
+ url = str(uri)
|
|
|
+
|
|
|
+ # local tarball sources and local link sources are unsupported
|
|
|
+ else:
|
|
|
+ raise ParameterError("Unsupported dependency: %s" % name, ud.url)
|
|
|
+
|
|
|
+ ud.deps.append({
|
|
|
+ "url": url,
|
|
|
+ "localpath": localpath,
|
|
|
+ "extrapaths": extrapaths,
|
|
|
+ "destsuffix": destsuffix,
|
|
|
+ })
|
|
|
+
|
|
|
+ try:
|
|
|
+ with open(ud.shrinkwrap_file, "r") as f:
|
|
|
+ shrinkwrap = json.load(f)
|
|
|
+ except Exception as e:
|
|
|
+ raise ParameterError("Invalid shrinkwrap file: %s" % str(e), ud.url)
|
|
|
+
|
|
|
+ foreach_dependencies(shrinkwrap, _resolve_dependency, ud.dev)
|
|
|
+
|
|
|
+ # Avoid conflicts between the environment data and:
|
|
|
+ # - the proxy url revision
|
|
|
+ # - the proxy url checksum
|
|
|
+ data = bb.data.createCopy(d)
|
|
|
+ data.delVar("SRCREV")
|
|
|
+ data.delVarFlags("SRC_URI")
|
|
|
+
|
|
|
+ # This fetcher resolves multiple URIs from a shrinkwrap file and then
|
|
|
+ # forwards it to a proxy fetcher. The management of the donestamp file,
|
|
|
+ # the lockfile and the checksums are forwarded to the proxy fetcher.
|
|
|
+ ud.proxy = Fetch([dep["url"] for dep in ud.deps], data)
|
|
|
+ ud.needdonestamp = False
|
|
|
+
|
|
|
+ @staticmethod
|
|
|
+ def _foreach_proxy_method(ud, handle):
|
|
|
+ returns = []
|
|
|
+ for proxy_url in ud.proxy.urls:
|
|
|
+ proxy_ud = ud.proxy.ud[proxy_url]
|
|
|
+ proxy_d = ud.proxy.d
|
|
|
+ proxy_ud.setup_localpath(proxy_d)
|
|
|
+ returns.append(handle(proxy_ud.method, proxy_ud, proxy_d))
|
|
|
+ return returns
|
|
|
+
|
|
|
+ def verify_donestamp(self, ud, d):
|
|
|
+ """Verify the donestamp file"""
|
|
|
+ def _handle(m, ud, d):
|
|
|
+ return m.verify_donestamp(ud, d)
|
|
|
+ return all(self._foreach_proxy_method(ud, _handle))
|
|
|
+
|
|
|
+ def update_donestamp(self, ud, d):
|
|
|
+ """Update the donestamp file"""
|
|
|
+ def _handle(m, ud, d):
|
|
|
+ m.update_donestamp(ud, d)
|
|
|
+ self._foreach_proxy_method(ud, _handle)
|
|
|
+
|
|
|
+ def need_update(self, ud, d):
|
|
|
+ """Force a fetch, even if localpath exists ?"""
|
|
|
+ def _handle(m, ud, d):
|
|
|
+ return m.need_update(ud, d)
|
|
|
+ return all(self._foreach_proxy_method(ud, _handle))
|
|
|
+
|
|
|
+ def try_mirrors(self, fetch, ud, d, mirrors):
|
|
|
+ """Try to use a mirror"""
|
|
|
+ def _handle(m, ud, d):
|
|
|
+ return m.try_mirrors(fetch, ud, d, mirrors)
|
|
|
+ return all(self._foreach_proxy_method(ud, _handle))
|
|
|
+
|
|
|
+ def download(self, ud, d):
|
|
|
+ """Fetch url"""
|
|
|
+ ud.proxy.download()
|
|
|
+
|
|
|
+ def unpack(self, ud, rootdir, d):
|
|
|
+ """Unpack the downloaded dependencies"""
|
|
|
+ destdir = d.getVar("S")
|
|
|
+ destsuffix = ud.parm.get("destsuffix")
|
|
|
+ if destsuffix:
|
|
|
+ destdir = os.path.join(rootdir, destsuffix)
|
|
|
+
|
|
|
+ bb.utils.mkdirhier(destdir)
|
|
|
+ bb.utils.copyfile(ud.shrinkwrap_file,
|
|
|
+ os.path.join(destdir, "npm-shrinkwrap.json"))
|
|
|
+
|
|
|
+ auto = [dep["url"] for dep in ud.deps if not dep["localpath"]]
|
|
|
+ manual = [dep for dep in ud.deps if dep["localpath"]]
|
|
|
+
|
|
|
+ if auto:
|
|
|
+ ud.proxy.unpack(destdir, auto)
|
|
|
+
|
|
|
+ for dep in manual:
|
|
|
+ depdestdir = os.path.join(destdir, dep["destsuffix"])
|
|
|
+ npm_unpack(dep["localpath"], depdestdir, d)
|
|
|
+
|
|
|
+ def clean(self, ud, d):
|
|
|
+ """Clean any existing full or partial download"""
|
|
|
+ ud.proxy.clean()
|
|
|
+
|
|
|
+ # Clean extra files
|
|
|
+ for dep in ud.deps:
|
|
|
+ for path in dep["extrapaths"]:
|
|
|
+ bb.utils.remove(path)
|
|
|
+
|
|
|
+ def done(self, ud, d):
|
|
|
+ """Is the download done ?"""
|
|
|
+ def _handle(m, ud, d):
|
|
|
+ return m.done(ud, d)
|
|
|
+ return all(self._foreach_proxy_method(ud, _handle))
|