Remove use of npm ls in grunt tasks (#11965)

* [grunt/build] refactor _build:notice task to not depend on npm

The _build:notice task used to rely on the output of `npm ls` to determine where modules were defined, but the task now just asks `license-checker` to include the `realPath` of the modules it describes in it's output, which is ultimately the same thing but works with `yarn` too.

* [grunt/licenses] convert to use lib/packages/getInstalledPackages()

* [grunt/notice/generate] test generateNoticeText()

* [grunt/licenses] tested assertLicensesValid()

* [npm] remove npm dev dep

* [tasks/lib/packages] do not include kibana in "installed packages"

* [tasks/lib/notice] join all notices with the same separator

(cherry picked from commit 5c04ff65fb)

package.json

@@ -265,7 +265,6 @@
     "ncp": "2.0.0",
     "nock": "8.0.0",
     "node-sass": "3.8.0",
-    "npm": "3.10.10",
     "portscanner": "1.0.0",
     "proxyquire": "1.7.10",
     "sass-loader": "4.0.0",

tasks/build/notice.js

@@ -1,94 +1,35 @@
-import _ from 'lodash';
-import npmLicense from 'license-checker';
-import glob from 'glob';
-import path from 'path';
-import fs from 'fs';
-import { execSync } from 'child_process';
+import { resolve } from 'path';
 
-export default function licenses(grunt) {
-  grunt.registerTask('_build:notice', 'Adds a notice', function () {
-    const done = this.async();
-    const buildPath = path.join(grunt.config.get('buildDir'), 'kibana');
+import {
+  getInstalledPackages,
+  generateNoticeText,
+} from '../lib';
 
-    function getPackagePaths() {
-      const packagePaths = {};
-      const installedPackages = execSync(`npm ls --parseable --long`, {
-        cwd: buildPath
-      });
-      installedPackages.toString().trim().split('\n').forEach(pkg => {
-        let modulePath;
-        let dirPath;
-        let packageName;
-        let drive;
-        const packageDetails = pkg.split(':');
-        if (/^win/.test(process.platform)) {
-          [drive, dirPath, packageName] = packageDetails;
-          modulePath = `${drive}:${dirPath}`;
-        } else {
-          [modulePath, packageName] = packageDetails;
-        }
-        const licenses = glob.sync(path.join(modulePath, '*LICENSE*'));
-        const notices = glob.sync(path.join(modulePath, '*NOTICE*'));
-        packagePaths[packageName] = {
-          relative: modulePath.replace(/.*(\/|\\)kibana(\/|\\)/, ''),
-          licenses,
-          notices
-        };
-      });
-      return packagePaths;
-    }
-
-    function combineFiles(filePaths) {
-      let content = '';
-      filePaths.forEach(filePath => {
-        content += fs.readFileSync(filePath) + '\n';
-      });
-      return content;
-    }
-
-    function getNodeInfo() {
-      const nodeVersion = grunt.config.get('nodeVersion');
-      const nodeDir = path.join(grunt.config.get('root'), '.node_binaries', nodeVersion);
-      const licensePath = path.join(nodeDir, 'linux-x64', 'LICENSE');
-      const license = fs.readFileSync(licensePath);
-      return `This product bundles Node.js.\n\n${license}`;
-    }
-
-    function getPackageInfo(packages) {
-      const packagePaths = getPackagePaths();
-      const overrides = grunt.config.get('licenses.options.overrides');
-      let content = '';
-      _.forOwn(packages, (value, key) => {
-        const licenses = [].concat(overrides.hasOwnProperty(key) ? overrides[key] : value.licenses);
-        if (!licenses.length || licenses.includes('UNKNOWN')) return grunt.fail.fatal(`Unknown license for ${key}`);
-        const packagePath = packagePaths[key];
-        const readLicenseAndNotice = combineFiles([].concat(packagePath.licenses, packagePath.notices));
-        const licenseOverview = licenses.length > 1 ? `the\n"${licenses.join('", ')} licenses` : `a\n"${licenses[0]}" license`;
-        const licenseAndNotice = readLicenseAndNotice ? `\n${readLicenseAndNotice}` : `  For details, see ${packagePath.relative}/.`;
-        const combinedText = `This product bundles ${key} which is available under ${licenseOverview}.${licenseAndNotice}\n---\n`;
-
-        content += combinedText;
-      });
-      return content;
-    }
-
-    function getBaseNotice() {
-      return fs.readFileSync(path.join(__dirname, 'notice', 'base_notice.txt'));
-    }
-
-    npmLicense.init({
-      start: buildPath,
-      production: true,
-      json: true
-    }, (result, error) => {
-      if (error) return grunt.fail.fatal(error);
-      const noticePath = path.join(buildPath, 'NOTICE.txt');
-      const fd = fs.openSync(noticePath, 'w');
-      fs.appendFileSync(fd, getBaseNotice());
-      fs.appendFileSync(fd, getPackageInfo(result));
-      fs.appendFileSync(fd, getNodeInfo());
-      fs.closeSync(fd);
-      done();
-    });
+async function generate(grunt, directory) {
+  return await generateNoticeText({
+    packages: await getInstalledPackages({
+      directory,
+      licenseOverrides: grunt.config.get('licenses.options.overrides')
+    }),
+    nodeDir: grunt.config.get('platforms')[0].nodeDir
+  });
+}
+
+export default function (grunt) {
+  grunt.registerTask('_build:notice', 'Adds a notice', function () {
+    const done = this.async();
+    const kibanaDir = resolve(grunt.config.get('buildDir'), 'kibana');
+    const noticePath = resolve(kibanaDir, 'NOTICE.txt');
+
+    generate(grunt, kibanaDir).then(
+      (noticeText) => {
+        grunt.file.write(noticePath, noticeText);
+        done();
+      },
+      (error) => {
+        grunt.fail.fatal(error);
+        done(error);
+      }
+    );
   });
 }

tasks/config/simplemocha.js

@@ -11,6 +11,7 @@ module.exports = {
       'test/mocha_setup.js',
       'test/**/__tests__/**/*.js',
       'src/**/__tests__/**/*.js',
+      'tasks/**/__tests__/**/*.js',
       'test/fixtures/__tests__/*.js',
       '!src/**/public/**',
       '!**/_*.js'

tasks/lib/index.js

@@ -0,0 +1,3 @@
+export { generateNoticeText } from './notice';
+export { getInstalledPackages } from './packages';
+export { assertLicensesValid } from './licenses';

tasks/lib/licenses/__tests__/valid.js

@@ -0,0 +1,62 @@
+import { resolve } from 'path';
+
+import expect from 'expect.js';
+
+import { assertLicensesValid } from '../valid';
+
+const NODE_MODULES = resolve(__dirname, '../../../../node_modules');
+
+const PACKAGE = {
+  name: '@elastic/httpolyglot',
+  version: '0.1.2-elasticpatch1',
+  licenses: ['MIT'],
+  directory: resolve(NODE_MODULES, '@elastic/httpolyglot'),
+  relative: 'node_modules/@elastic/httpolyglot',
+};
+
+describe('tasks/lib/licenses', () => {
+  describe('assertLicensesValid()', () => {
+    it('returns undefined when package has valid license', () => {
+      expect(assertLicensesValid({
+        packages: [PACKAGE],
+        validLicenses: [...PACKAGE.licenses]
+      })).to.be(undefined);
+    });
+
+    it('throw an error when the packages license is invalid', () => {
+      expect(() => {
+        assertLicensesValid({
+          packages: [PACKAGE],
+          validLicenses: [`not ${PACKAGE.licenses[0]}`]
+        });
+      }).to.throwError(PACKAGE.name);
+    });
+
+    it('throws an error when the package has no licenses', () => {
+      expect(() => {
+        assertLicensesValid({
+          packages: [
+            {
+              ...PACKAGE,
+              licenses: []
+            }
+          ],
+          validLicenses: [...PACKAGE.licenses]
+        });
+      }).to.throwError(PACKAGE.name);
+    });
+
+    it('includes the relative path to packages in error message', () => {
+      try {
+        assertLicensesValid({
+          packages: [PACKAGE],
+          validLicenses: ['none']
+        });
+        throw new Error('expected assertLicensesValid() to throw');
+      } catch (error) {
+        expect(error.message).to.contain(PACKAGE.relative);
+        expect(error.message).to.not.contain(PACKAGE.directory);
+      }
+    });
+  });
+});

tasks/lib/licenses/index.js

@@ -0,0 +1 @@
+export { assertLicensesValid } from './valid';

tasks/lib/licenses/valid.js

@@ -0,0 +1,47 @@
+const describeInvalidLicenses = getInvalid => pkg => (
+`
+  ${pkg.name}
+    version: ${pkg.version}
+    all licenses: ${pkg.licenses}
+    invalid licenses: ${getInvalid(pkg.licenses).join(', ')}
+    path: ${pkg.relative}
+`
+);
+
+/**
+ *  When given a list of packages and the valid license
+ *  options, either throws an error with details about
+ *  violations or returns undefined.
+ *
+ *  @param  {Object} [options={}]
+ *  @property {Array<Package>} options.packages List of packages to check, see
+ *                                              getInstalledPackages() in ../packages
+ *  @property {Array<string>} options.validLicenses
+ *  @return {undefined}
+ */
+export function assertLicensesValid(options = {}) {
+  const {
+    packages,
+    validLicenses
+  } = options;
+
+  if (!packages || !validLicenses) {
+    throw new Error('packages and validLicenses options are required');
+  }
+
+  const getInvalid = licenses => (
+    licenses.filter(license => !validLicenses.includes(license))
+  );
+
+  const isPackageInvalid = pkg => (
+    !pkg.licenses.length || getInvalid(pkg.licenses).length > 0
+  );
+
+  const invalidMsgs = packages
+    .filter(isPackageInvalid)
+    .map(describeInvalidLicenses(getInvalid));
+
+  if (invalidMsgs.length) {
+    throw new Error(`Non-confirming licenses: ${invalidMsgs.join('')}`);
+  }
+}

tasks/lib/notice/__tests__/notice.js

@@ -0,0 +1,55 @@
+import { resolve } from 'path';
+import { readFileSync } from 'fs';
+
+import expect from 'expect.js';
+
+import { generateNoticeText } from '../notice';
+
+const NODE_MODULES = resolve(__dirname, '../../../../node_modules');
+const NODE_DIR = resolve(process.execPath, '../..');
+const PACKAGES = [
+  {
+    name: '@elastic/httpolyglot',
+    version: '0.1.2-elasticpatch1',
+    licenses: ['MIT'],
+    directory: resolve(NODE_MODULES, '@elastic/httpolyglot'),
+    relative: 'node_modules/@elastic/httpolyglot',
+  },
+  {
+    name: 'aws-sdk',
+    version: '2.0.31',
+    licenses: ['Apache 2.0'],
+    directory: resolve(NODE_MODULES, 'aws-sdk'),
+    relative: 'node_modules/aws-sdk',
+  }
+];
+
+describe('tasks/lib/notice', () => {
+  describe('generateNoticeText()', () => {
+    let notice;
+    before(async () => notice = await generateNoticeText({
+      packages: PACKAGES,
+      nodeDir: NODE_DIR
+    }));
+
+    it('returns a string', () => {
+      expect(notice).to.be.a('string');
+    });
+
+    it('includes *NOTICE* files from packages', () => {
+      expect(notice).to.contain(readFileSync(resolve(NODE_MODULES, 'aws-sdk/NOTICE.txt'), 'utf8'));
+    });
+
+    it('includes *LICENSE* files from packages', () => {
+      expect(notice).to.contain(readFileSync(resolve(NODE_MODULES, '@elastic/httpolyglot/LICENSE'), 'utf8'));
+    });
+
+    it('includes the LICENSE file from node', () => {
+      expect(notice).to.contain(readFileSync(resolve(NODE_DIR, 'LICENSE'), 'utf8'));
+    });
+
+    it('includes the base_notice.txt file', () => {
+      expect(notice).to.contain(readFileSync(resolve(__dirname, '../base_notice.txt'), 'utf8'));
+    });
+  });
+});

tasks/lib/notice/base_notice.txt

@@ -56,4 +56,3 @@ THE SOFTWARE.
 ---
 This product bundles geohash.js which is available under a
 "MIT" license.  For details, see src/ui/public/utils/decode_geo_hash.js.
----

tasks/lib/notice/bundled_notices.js

@@ -0,0 +1,14 @@
+import { resolve } from 'path';
+import { readFile } from 'fs';
+
+import { fromNode as fcb } from 'bluebird';
+import glob from 'glob';
+
+export async function getBundledNotices(packageDirectory) {
+  const pattern = resolve(packageDirectory, '*{LICENSE,NOTICE}*');
+  const paths = await fcb(cb => glob(pattern, cb));
+  return Promise.all(paths.map(async path => ({
+    path,
+    text: await fcb(cb => readFile(path, 'utf8', cb))
+  })));
+}

tasks/lib/notice/index.js

@@ -0,0 +1 @@
+export { generateNoticeText } from './notice';

tasks/lib/notice/node_notice.js

@@ -0,0 +1,8 @@
+import { resolve } from 'path';
+import { readFileSync } from 'fs';
+
+export function generateNodeNoticeText(nodeDir) {
+  const licensePath = resolve(nodeDir, 'LICENSE');
+  const license = readFileSync(licensePath, 'utf8');
+  return `This product bundles Node.js.\n\n${license}`;
+}

tasks/lib/notice/notice.js

@@ -0,0 +1,33 @@
+import { resolve } from 'path';
+import { readFileSync } from 'fs';
+
+import { generatePackageNoticeText } from './package_notice';
+import { generateNodeNoticeText } from './node_notice';
+
+const BASE_NOTICE = resolve(__dirname, './base_notice.txt');
+
+/**
+ *  When given a list of packages and the directory to the
+ *  node distribution that will be shipping with Kibana,
+ *  generates the text for NOTICE.txt
+ *
+ *  @param  {Object} [options={}]
+ *  @property {Array<Package>} options.packages List of packages to check, see
+ *                                              getInstalledPackages() in ../packages
+ *  @property {string} options.nodeDir The directory containing the version of node.js
+ *                                     that will ship with Kibana
+ *  @return {undefined}
+ */
+export async function generateNoticeText(options = {}) {
+  const { packages, nodeDir } = options;
+
+  const packageNotices = await Promise.all(
+    packages.map(generatePackageNoticeText)
+  );
+
+  return [
+    readFileSync(BASE_NOTICE, 'utf8'),
+    ...packageNotices,
+    generateNodeNoticeText(nodeDir),
+  ].join('\n---\n');
+}

tasks/lib/notice/package_notice.js

@@ -0,0 +1,22 @@
+import { getBundledNotices } from './bundled_notices';
+
+const concatNotices = notices => (
+  notices.map(notice => notice.text).join('\n')
+);
+
+export async function generatePackageNoticeText(pkg) {
+  const bundledNotices = concatNotices(await getBundledNotices(pkg.directory));
+
+  const intro = `This product bundles ${pkg.name}@${pkg.version}`;
+  const license = ` which is available under ${
+    pkg.licenses.length > 1
+      ? `the\n"${pkg.licenses.join('", ')} licenses.`
+      : `a\n"${pkg.licenses[0]}" license.`
+  }`;
+
+  const moreInfo = bundledNotices
+    ? `\n${bundledNotices}\n`
+    : `  For details, see ${pkg.relative}/.`;
+
+  return `${intro}${license}${moreInfo}`;
+}

tasks/lib/packages/__tests__/fixtures/fixture1/index.js

@@ -0,0 +1 @@
+console.log('I am fixture 1');

tasks/lib/packages/__tests__/fixtures/fixture1/node_modules/dep1/index.js

@@ -0,0 +1 @@
+console.log('I am dep 1');

tasks/lib/packages/__tests__/fixtures/fixture1/node_modules/dep1/package.json

@@ -0,0 +1,5 @@
+{
+  "name": "dep1",
+  "version": "0.0.2",
+  "license": "Apache-2.0"
+}

tasks/lib/packages/__tests__/fixtures/fixture1/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "fixture1",
+  "version": "0.0.1",
+  "license": "MIT",
+  "dependencies": {
+    "dep1": "0.0.2"
+  }
+}

tasks/lib/packages/__tests__/installed_packages.js

@@ -0,0 +1,59 @@
+import { resolve } from 'path';
+
+import { uniq } from 'lodash';
+import expect from 'expect.js';
+
+import { getInstalledPackages } from '../installed_packages';
+
+const KIBANA_ROOT = resolve(__dirname, '../../../../');
+const FIXTURE1_ROOT = resolve(__dirname, 'fixtures/fixture1');
+
+describe('tasks/lib/packages', () => {
+  describe('getInstalledPackages()', function () {
+
+    let kibanaPackages;
+    let fixture1Packages;
+    before(async function () {
+      this.timeout(30 * 1000);
+      [kibanaPackages, fixture1Packages] = await Promise.all([
+        getInstalledPackages({
+          directory: KIBANA_ROOT
+        }),
+        getInstalledPackages({
+          directory: FIXTURE1_ROOT
+        }),
+      ]);
+    });
+
+    it('requires a directory', async () => {
+      try {
+        await getInstalledPackages({});
+        throw new Error('expected getInstalledPackages() to reject');
+      } catch (err) {
+        expect(err.message).to.contain('directory');
+      }
+    });
+
+    it('reads all installed packages of a module', () => {
+      expect(fixture1Packages).to.eql([
+        {
+          name: 'dep1',
+          version: '0.0.2',
+          licenses: [ 'Apache-2.0' ],
+          directory: resolve(FIXTURE1_ROOT, 'node_modules/dep1'),
+          relative: 'node_modules/dep1',
+        }
+      ]);
+    });
+
+    it('returns a single entry for every package/version combo', () => {
+      const tags = kibanaPackages.map(pkg => `${pkg.name}@${pkg.version}`);
+      expect(tags).to.eql(uniq(tags));
+    });
+
+    it('does not include root package in the list', async () => {
+      expect(kibanaPackages.find(pkg => pkg.name === 'kibana')).to.be(undefined);
+      expect(fixture1Packages.find(pkg => pkg.name === 'fixture1')).to.be(undefined);
+    });
+  });
+});

tasks/lib/packages/index.js

@@ -0,0 +1 @@
+export { getInstalledPackages } from './installed_packages';

tasks/lib/packages/installed_packages.js

@@ -0,0 +1,55 @@
+import { relative } from 'path';
+
+import { callLicenseChecker } from './license_checker';
+
+/**
+ *  Get a list of objects with details about each installed
+ *  NPM package.
+ *
+ *  @param  {Object} [options={}]
+ *  @property {String} options.directory root of the project to read
+ *  @property {Boolean} [options.dev=false] should development dependencies be included?
+ *  @property {Object} [options.licenseOverrides] map of `${name}@${version}` to a list of
+ *                                                license ids to override the automatically
+ *                                                detected ones
+ *  @return {Array<Object>}
+ */
+export async function getInstalledPackages(options = {}) {
+  const {
+    directory,
+    dev = false,
+    licenseOverrides = {}
+  } = options;
+
+  if (!directory) {
+    throw new Error('You must specify a directory to read installed packages from');
+  }
+
+  const licenseInfo = await callLicenseChecker({ directory, dev });
+  return Object
+    .keys(licenseInfo)
+    .map(key => {
+      const keyParts = key.split('@');
+      const name = keyParts.slice(0, -1).join('@');
+      const version = keyParts[keyParts.length - 1];
+      const {
+        licenses: detectedLicenses,
+        realPath,
+      } = licenseInfo[key];
+
+      const licenses = [].concat(
+        licenseOverrides[key]
+          ? licenseOverrides[key]
+          : detectedLicenses
+      );
+
+      return {
+        name,
+        version,
+        licenses,
+        directory: realPath,
+        relative: relative(directory, realPath)
+      };
+    })
+    .filter(pkg => pkg.directory !== directory);
+}

tasks/lib/packages/license_checker.js

@@ -0,0 +1,26 @@
+import licenseChecker from 'license-checker';
+
+export function callLicenseChecker(options = {}) {
+  const {
+    directory,
+    dev = false
+  } = options;
+
+  if (!directory) {
+    throw new Error('You must specify the directory where license checker should start');
+  }
+
+  return new Promise((resolve, reject) => {
+    licenseChecker.init({
+      start: directory,
+      production: !dev,
+      json: true,
+      customFormat: {
+        realPath: true
+      }
+    }, (licenseInfo, err) => {
+      if (err) reject(err);
+      else resolve(licenseInfo);
+    });
+  });
+}

tasks/licenses.js

@@ -1,74 +1,29 @@
-import _ from 'lodash';
-import { fromNode } from 'bluebird';
-import npmLicense from 'license-checker';
+import {
+  getInstalledPackages,
+  assertLicensesValid
+} from './lib';
 
 export default function licenses(grunt) {
   grunt.registerTask('licenses', 'Checks dependency licenses', async function () {
-    const config = this.options();
     const done = this.async();
 
-    const options = {
-      start: process.cwd(),
-      production: true,
-      json: true
-    };
-
-    const packages = await fromNode(cb => {
-      npmLicense.init(options, result => {
-        cb(undefined, result);
+    try {
+      const options = this.options({
+        licenses: [],
+        overrides: {}
       });
-    });
 
-    /**
-     * Licenses for a package by name with overrides
-     *
-     * @param {String} name
-     * @return {Array}
-     */
-
-    function licensesForPackage(name) {
-      let licenses = packages[name].licenses;
-
-      if (config.overrides.hasOwnProperty(name)) {
-        licenses = config.overrides[name];
-      }
-
-      return typeof licenses === 'string' ? [licenses] : licenses;
+      assertLicensesValid({
+        packages: await getInstalledPackages({
+          directory: grunt.config.get('root'),
+          licenseOverrides: options.overrides
+        }),
+        validLicenses: options.licenses
+      });
+      done();
+    } catch (err) {
+      grunt.fail.fatal(err);
+      done(err);
     }
-
-    /**
-     * Determine if a package has a valid license
-     *
-     * @param {String} name
-     * @return {Boolean}
-     */
-
-    function isInvalidLicense(name) {
-      const licenses = licensesForPackage(name);
-
-      // verify all licenses for the package are in the config
-      return _.intersection(licenses, config.licenses).length < licenses.length;
-    }
-
-    // Build object containing only invalid packages
-    const invalidPackages = _.pick(packages, (pkg, name) => {
-      return isInvalidLicense(name);
-    });
-
-    if (Object.keys(invalidPackages).length) {
-      const execSync = require('child_process').execSync;
-      const names = Object.keys(invalidPackages);
-
-      // Uses npm ls to create tree for package locations
-      const tree = execSync(`npm ls ${names.join(' ')}`);
-
-      grunt.log.debug(JSON.stringify(invalidPackages, null, 2));
-      grunt.fail.warn(
-        `Non-confirming licenses:\n ${names.join('\n ')}\n\n${tree}`,
-        invalidPackages.length
-      );
-    }
-
-    done();
   });
 }