fetch2/npm: Add npm fetcher

npm fetcher with support for shrinkwrap files and lockdown files to easily
download and install an npm package with strict dependency resolution.

The SRC_URI should be in the format of:
SRC_URI = "npm://registry.npmjs.org/;name=${PN};version=${PV}"

To add a shrinkwrap and lockdown file use:
NPM_SHRINKWRAP := "${THISDIR}/${PN}/npm-shrinkwrap.json"
NPM_LOCKDOWN := "${THISDIR}/${PN}/lockdown.json"

Signed-off-by: Brendan Le Foll <brendan.le.foll@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

1 files changed, 226 insertions, 0 deletions

diff --git a/lib/bb/fetch2/npm.py b/lib/bb/fetch2/npm.py
new file mode 100644
index 000000000..54cf76df0
--- /dev/null
+++ b/lib/bb/fetch2/npm.py
@@ -0,0 +1,226 @@
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+"""
+BitBake 'Fetch' NPM implementation
+
+The NPM fetcher is used to retrieve files from the npmjs repository
+
+Usage in the recipe:
+
+    SRC_URI = "npm://registry.npmjs.org/;name=${PN};version=${PV}"
+    Suported SRC_URI options are:
+
+    - name
+    - version
+
+    npm://registry.npmjs.org/${PN}/-/${PN}-${PV}.tgz  would become npm://registry.npmjs.org;name=${PN};ver=${PV}
+    The fetcher all triggers off the existence of ud.localpath. If that exists and has the ".done" stamp, its assumed the fetch is good/done
+
+"""
+
+import os
+import sys
+import urllib
+import json
+import subprocess
+import signal
+import bb
+from   bb import data
+from   bb.fetch2 import FetchMethod
+from   bb.fetch2 import FetchError
+from   bb.fetch2 import ChecksumError
+from   bb.fetch2 import runfetchcmd
+from   bb.fetch2 import logger
+from   bb.fetch2 import UnpackError
+from   distutils import spawn
+
+def subprocess_setup():
+    # Python installs a SIGPIPE handler by default. This is usually not what
+    # non-Python subprocesses expect.
+    # SIGPIPE errors are known issues with gzip/bash
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+
+class Npm(FetchMethod):
+
+    """Class to fetch urls via 'npm'"""
+    def init(self, d):
+        pass
+
+    def supports(self, ud, d):
+        """
+        Check to see if a given url can be fetched with npm
+        """
+        return ud.type in ['npm']
+
+    def debug(self, msg):
+        logger.debug(1, "NpmFetch: %s", msg)
+
+    def clean(self, ud, d):
+         logger.debug(2, "Calling cleanup %s" % ud.pkgname)
+         bb.utils.remove(ud.localpath, False)
+         bb.utils.remove(ud.pkgdatadir, True)
+
+    def urldata_init(self, ud, d):
+        """
+        init NPM specific variable within url data
+        """
+        if 'downloadfilename' in ud.parm:
+            ud.basename = ud.parm['downloadfilename']
+        else:
+            ud.basename = os.path.basename(ud.path)
+
+        # can't call it ud.name otherwise fetcher base class will start doing sha1stuff
+        # TODO: find a way to get an sha1/sha256 manifest of pkg & all deps
+        ud.pkgname = ud.parm.get("name", None)
+        if not ud.pkgname:
+            raise ParameterError("NPM fetcher requires a name parameter")
+        ud.version = ud.parm.get("version", None)
+        if not ud.version:
+            raise ParameterError("NPM fetcher requires a version parameter")
+        ud.bbnpmmanifest = "%s-%s.deps.json" % (ud.pkgname, ud.version)
+        ud.registry = "http://%s" % ud.basename
+        prefixdir = "npm/%s" % ud.pkgname
+        ud.pkgdatadir = d.expand("${DL_DIR}/%s" % prefixdir)
+        if not os.path.exists(ud.pkgdatadir):
+            bb.utils.mkdirhiet(ud.pkgdatadir)
+        ud.localpath = d.expand("${DL_DIR}/npm/%s" % ud.bbnpmmanifest)
+
+        self.basecmd = d.getVar("FETCHCMD_wget", True) or "/usr/bin/env wget -O -t 2 -T 30 -nv --passive-ftp --no-check-certificate "
+        self.basecmd += " --directory-prefix=%s " % prefixdir
+
+    def need_update(self, ud, d):
+        if os.path.exists(ud.localpath):
+            return False
+        return True
+
+    def _runwget(self, ud, d, command, quiet):
+        logger.debug(2, "Fetching %s using command '%s'" % (ud.url, command))
+        bb.fetch2.check_network_access(d, command)
+        runfetchcmd(command, d, quiet)
+
+    def _unpackdep(self, ud, pkg, data, destdir, dldir, d):
+       file = data[pkg]['tgz']
+       logger.debug(2, "file to extract is %s" % file)
+       if file.endswith('.tgz') or file.endswith('.tar.gz') or file.endswith('.tar.Z'):
+            cmd = 'tar xz --strip 1 --no-same-owner -f %s/%s' % (dldir, file)
+       else:
+            bb.fatal("NPM package %s downloaded not a tarball!" % file)
+
+       # Change to subdir before executing command
+       save_cwd = os.getcwd()
+       if not os.path.exists(destdir):
+           os.makedirs(destdir)
+       os.chdir(destdir)
+       path = d.getVar('PATH', True)
+       if path:
+            cmd = "PATH=\"%s\" %s" % (path, cmd)
+       bb.note("Unpacking %s to %s/" % (file, os.getcwd()))
+       ret = subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True)
+       os.chdir(save_cwd)
+
+       if ret != 0:
+            raise UnpackError("Unpack command %s failed with return value %s" % (cmd, ret), ud.url)
+
+       if 'deps' not in data[pkg]:
+            return
+       for dep in data[pkg]['deps']:
+           self._unpackdep(ud, dep, data[pkg]['deps'], "%s/node_modules/%s" % (destdir, dep), dldir, d)
+
+
+    def unpack(self, ud, destdir, d):
+        dldir = d.getVar("DL_DIR", True)
+        depdumpfile = "%s-%s.deps.json" % (ud.pkgname, ud.version)
+        with open("%s/npm/%s" % (dldir, depdumpfile)) as datafile:
+            workobj = json.load(datafile)
+        dldir = "%s/%s" % (os.path.dirname(ud.localpath), ud.pkgname)
+
+        self._unpackdep(ud, ud.pkgname, workobj,  "%s/npmpkg" % destdir, dldir, d)
+
+    def _getdependencies(self, pkg, data, version, d, ud):
+        pkgfullname = pkg
+        if version:
+            pkgfullname += "@%s" % version
+        logger.debug(2, "Calling getdeps on %s" % pkg)
+        fetchcmd = "npm view %s dist.tarball --registry %s" % (pkgfullname, ud.registry)
+        output = runfetchcmd(fetchcmd, d, True)
+        # npm may resolve multiple versions
+        outputarray = output.strip().splitlines()
+        # we just take the latest version npm resolved
+        #logger.debug(2, "Output URL is %s - %s - %s" % (ud.basepath, ud.basename, ud.localfile))
+        outputurl = outputarray[len(outputarray)-1].rstrip()
+        if (len(outputarray) > 1):
+            # remove the preceding version/name from npm output and then the
+            # first and last quotes
+            outputurl = outputurl.split(" ")[1][1:-1]
+        data[pkg] = {}
+        data[pkg]['tgz'] = os.path.basename(outputurl)
+        self._runwget(ud, d, "%s %s" % (self.basecmd, outputurl), False)
+        #fetchcmd = "npm view %s@%s dependencies --json" % (pkg, version)
+        fetchcmd = "npm view %s dependencies --json --registry %s" % (pkgfullname, ud.registry)
+        output = runfetchcmd(fetchcmd, d, True)
+        try:
+          depsfound = json.loads(output)
+        except:
+            # just assume there is no deps to be loaded here
+            return
+        data[pkg]['deps'] = {}
+        for dep, version in depsfound.iteritems():
+            self._getdependencies(dep, data[pkg]['deps'], version, d, ud)
+
+    def _getshrinkeddependencies(self, pkg, data, version, d, ud, lockdown, manifest):
+        logger.debug(2, "NPM shrinkwrap file is %s" % data)
+        outputurl = "invalid"
+        if ('resolved' not in data):
+            # will be the case for ${PN}
+            fetchcmd = "npm view %s@%s dist.tarball --registry %s" % (pkg, version, ud.registry)
+            logger.debug(2, "Found this matching URL: %s" % str(fetchcmd))
+            outputurl = runfetchcmd(fetchcmd, d, True)
+        else:
+            outputurl = data['resolved']
+        self._runwget(ud, d, "%s %s" % (self.basecmd, outputurl), False)
+        manifest[pkg] = {}
+        manifest[pkg]['tgz'] = os.path.basename(outputurl).rstrip()
+        manifest[pkg]['deps'] = {}
+
+        if pkg in lockdown:
+            sha1_expected = lockdown[pkg][version]
+            sha1_data = bb.utils.sha1_file("npm/%s/%s" % (ud.pkgname, manifest[pkg]['tgz']))
+            if sha1_expected != sha1_data:
+                msg = "\nFile: '%s' has %s checksum %s when %s was expected" % (manifest[pkg]['tgz'], 'sha1', sha1_data, sha1_expected)
+                raise ChecksumError('Checksum mismatch!%s' % msg)
+        else:
+            logger.debug(2, "No lockdown data for %s@%s" % (pkg, version))
+
+        if 'dependencies' in data:
+            for obj in data['dependencies']:
+                logger.debug(2, "Found dep is %s" % str(obj))
+                self._getshrinkeddependencies(obj, data['dependencies'][obj], data['dependencies'][obj]['version'], d, ud, lockdown, manifest[pkg]['deps'])
+
+    def download(self, ud, d):
+        """Fetch url"""
+        jsondepobj = {}
+        shrinkobj = {}
+        lockdown = {}
+
+        shwrf = d.getVar('NPM_SHRINKWRAP', True)
+        logger.debug(2, "NPM shrinkwrap file is %s" % shwrf)
+        try:
+            with open(shwrf) as datafile:
+                shrinkobj = json.load(datafile)
+        except:
+            logger.warn('Missing shrinkwrap file in NPM_SHRINKWRAP for %s, this will lead to unreliable builds!' % ud.pkgname)
+        lckdf = d.getVar('NPM_LOCKDOWN', True)
+        logger.debug(2, "NPM lockdown file is %s" % lckdf)
+        try:
+            with open(lckdf) as datafile:
+                lockdown = json.load(datafile)
+        except:
+            logger.warn('Missing lockdown file in NPM_LOCKDOWN for %s, this will lead to unreproducible builds!' % ud.pkgname)
+
+        if ('name' not in shrinkobj):
+            self._getdependencies(ud.pkgname, jsondepobj, ud.version, d, ud)
+        else:
+            self._getshrinkeddependencies(ud.pkgname, shrinkobj, ud.version, d, ud, lockdown, jsondepobj)
+
+        with open(ud.localpath, 'w') as outfile:
+            json.dump(jsondepobj, outfile)