[build] finish updates

Gruntfile.js

@@ -63,8 +63,9 @@ module.exports = function (grunt) {
   grunt.config.merge(config);
 
   config.userScriptsDir = __dirname + '/build/userScripts';
+  // ensure that these run first, other configs need them
   config.services = require('./tasks/config/services')(grunt);
-  config.platforms = require('./tasks/config/packages')(grunt);
+  config.platforms = require('./tasks/config/platforms')(grunt);
 
   grunt.config.merge(config);
 

tasks/build/archives.js

@@ -4,25 +4,27 @@ module.exports = function createPackages(grunt) {
   let { execFile } = require('child_process');
   let { all, fromNode } = require('bluebird');
 
-  let version = config.get('pkg.version');
   let rootDir = config.get('root');
-  let targetDir = resolve(rootDir, 'target');
-  let buildDir = resolve(rootDir, 'build/kibana');
+  let buildDir = resolve(rootDir, 'build');
 
   let exec = async (cmd, args) => {
     grunt.log.writeln(` > ${cmd} ${args.join(' ')}`);
-    await fromNode(cb => execFile(cmd, args, { cwd: rootDir }, cb));
+    await fromNode(cb => execFile(cmd, args, { cwd: buildDir }, cb));
   };
 
+
   let archives = async (platform) => {
+    let tarPath = resolve(rootDir, platform.tarPath);
+    let zipPath = resolve(rootDir, platform.zipPath);
+
     // kibana.tar.gz
-    await exec('tar', ['-zchf', platform.tarPath, platform.buildDir]);
+    await exec('tar', ['-zchf', tarPath, platform.buildName]);
 
     // kibana.zip
     if (/windows/.test(platform.name)) {
-      await exec('zip', ['-rq', '-ll', platform.zipPath, platform.buildDir]);
+      await exec('zip', ['-rq', '-ll', zipPath, platform.buildName]);
     } else {
-      await exec('zip', ['-rq', platform.zipPath, platform.buildDir]);
+      await exec('zip', ['-rq', zipPath, platform.buildName]);
     }
   };
 

tasks/build/downloadNodes.js

@@ -1,86 +1,88 @@
 module.exports = function (grunt) {
-  let { map, fromNode, promisify } = require('bluebird');
+  let { map, fromNode } = require('bluebird');
   let { resolve } = require('path');
   let { pluck } = require('lodash');
   let { createWriteStream } = require('fs');
   let { createGunzip } = require('zlib');
   let { Extract } = require('tar');
-
-  let mkdirp = promisify(require('mkdirp'));
-  let rename = promisify(require('fs').rename);
-  let get = (uri) => fromNode(cb => require('wreck').request('GET', uri, null, cb));
-
-  let exists = (path) => fromNode(cb => {
-    require('fs').stat(path, err => cb(null, !err));
-  });
+  let { rename } = require('fs');
+  let wreck = require('wreck');
 
   let platforms = grunt.config.get('platforms');
-  let nodeVersion = grunt.config.get('nodeVersion');
-  let baseUri = `https://iojs.org/dist/v${nodeVersion}`;
+  let activeDownloads = [];
 
-  let writeTar = async (to, from) => {
-    await fromNode(cb => {
-      from
-      .pipe(createGunzip())
-      .on('error', cb)
-      .pipe(new Extract({ path: to, strip: 1 }))
-      .on('error', cb)
-      .on('end', cb);
+  let start = async (platform) => {
+    let finalDir = platform.nodeDir;
+    if (!grunt.file.isPathAbsolute(finalDir)) {
+      // since we are using fs module function we need absolute paths
+      finalDir = resolve(grunt.config.get('root'), finalDir);
+    }
+
+    let downloadDir = `${finalDir}.temp`;
+
+    if (grunt.file.isDir(platform.nodeDir)) {
+      grunt.log.ok(`${platform.name} exists`);
+      return;
+    }
+
+    let resp = await fromNode(cb => {
+      let req = wreck.request('GET', platform.nodeUrl, null, function (err, resp) {
+        if (err) {
+          return cb(err);
+        }
+
+        if (resp.statusCode !== 200) {
+          return cb(new Error(`${platform.nodeUrl} failed with a ${resp.statusCode} response`));
+        }
+
+        return cb(null, resp);
+      });
     });
-  };
 
-  let writeExe = async (to, from) => {
-    let winBinDir = resolve(to, 'bin');
-    await mkdirp(winBinDir);
-    await fromNode(cb => {
-      from
-      .pipe(createWriteStream(resolve(winBinDir, 'node.exe')))
-      .on('error', cb)
-      .on('finish', cb);
-    });
-  };
+    // use an async iife to store promise for download
+    // then store platform in active downloads list
+    // which we will read from in the finish task
+    platform.downloadPromise = (async () => {
+      grunt.file.mkdir(downloadDir);
 
-  let run = () => {
-    grunt.log.ok(`downloading node binaries`);
-
-    return map(platforms, async function (platform) {
-      grunt.file.mkdir(platform.nodeDir);
-
-      let finalDir = resolve(grunt.config.get('root'), platform.nodeDir);
-      let downloadDir = `${finalDir}.temp`;
-
-      if (await exists(finalDir)) {
-        grunt.log.debug(`${platform} download exists`);
-        return;
-      }
-
-      let uri;
-      if (platform === 'windows') {
-        uri = `${baseUri}/win-x64/iojs.exe`;
+      if (platform.name === 'windows') {
+        await fromNode(cb => {
+          resp
+          .pipe(createWriteStream(resolve(downloadDir, 'node.exe')))
+          .on('error', cb)
+          .on('finish', cb);
+        });
       } else {
-        uri = `${baseUri}/iojs-v${nodeVersion}-${platform}.tar.gz`;
+        await fromNode(cb => {
+          resp
+          .pipe(createGunzip())
+          .on('error', cb)
+          .pipe(new Extract({ path: downloadDir, strip: 1 }))
+          .on('error', cb)
+          .on('end', cb);
+        });
       }
 
-      let resp = await get('GET', uri);
-      if (resp.statusCode !== 200) {
-        throw new Error(uri + ' failed with a ' + resp.statusCode);
-      }
+      await fromNode(cb => {
+        rename(downloadDir, finalDir, cb);
+      });
+    }());
 
-      let write = platform === 'windows' ? writeExe : writeTar;
-      await write(downloadDir, resp);
-      await rename(downloadDir, finalDir);
+    activeDownloads.push(platform);
 
-      grunt.log.debug(`${platform} download complete`);
-    });
+    var bytes = parseInt(resp.headers['content-length'], 10) || 'unknown number of';
+    var mb = ((bytes / 1024) / 1024).toFixed(2);
+    grunt.log.ok(`downloading ${platform.name} - ${mb} mb`);
   };
 
-  let current = null;
-  let start = () => current || (current = run());
+  grunt.registerTask('build:downloadNodes:start', function () {
+    map(platforms, start).nodeify(this.async());
+  });
 
-  grunt.registerTask('build:downloadNodes:start', start);
   grunt.registerTask('build:downloadNodes:finish', function () {
-    start().then(() => {
-      grunt.log.ok(`node binaries for ${pluck(platforms, 'name').join(', ')} downloaded`);
+    map(activeDownloads, async (platform) => {
+      await platform.downloadPromise;
+      grunt.log.ok(`${platform.name} download complete`);
     })
     .nodeify(this.async());
   });

tasks/build/index.js

@@ -13,10 +13,11 @@ module.exports = function (grunt) {
     'build:packageJson',
     'build:readme',
     'build:installNpmDeps',
+    'clean:testsFromModules',
+    'clean:deepModules',
     'run:optimizeBuild',
     'stop:optimizeBuild',
     'build:downloadNodes:finish',
-    'clean:testsFromModules',
     'build:versionedLinks',
     'build:archives',
     !grunt.option('os-packages') ? [] : [

tasks/build/installNpmDeps.js

@@ -1,15 +1,13 @@
 module.exports = function (grunt) {
-  var { execSync } = require('child_process');
+  var { exec } = require('child_process');
   var { resolve } = require('path');
 
   grunt.registerTask('build:installNpmDeps', function () {
     grunt.file.mkdir('build/kibana/node_modules');
 
-    execSync('npm install  --production --no-optional', {
+    exec('npm install  --production --no-optional', {
       cwd: grunt.config.process('<%= root %>/build/kibana')
-    });
-
-    grunt.log.ok('done');
+    }, this.async());
   });
 };
 

tasks/config/packages.js

@@ -1,22 +0,0 @@
-module.exports = function (grunt) {
-  let version = grunt.config.get('pkg.version');
-  let nodeVersion = grunt.config.get('nodeVersion');
-
-  return [
-    'darwin-x64',
-    'linux-x64',
-    'linux-x86',
-    'windows'
-  ].map(function (name) {
-    var filename = `kibana-${version}-${name}`;
-
-    return {
-      name,
-      filename,
-      tarPath: `target/${filename}.tar.gz`,
-      zipPath: `target/${filename}.zip`,
-      buildDir: `build/${filename}`,
-      nodeDir: `.node_binaries/${nodeVersion}/${name}`,
-    };
-  });
-};

tasks/config/platforms.js

@@ -0,0 +1,26 @@
+module.exports = function (grunt) {
+  let version = grunt.config.get('pkg.version');
+  let nodeVersion = grunt.config.get('nodeVersion');
+  let baseUri = `https://iojs.org/dist/v${nodeVersion}`;
+
+  return [
+    'darwin-x64',
+    'linux-x64',
+    'linux-x86',
+    'windows'
+  ].map(function (name) {
+    let win = name === 'windows';
+
+    let nodeUrl = win ? `${baseUri}/win-x64/iojs.exe` : `${baseUri}/iojs-v${nodeVersion}-${name}.tar.gz`;
+    let nodeDir = `.node_binaries/${nodeVersion}/${name}`;
+
+    let buildName = `kibana-${version}-${name}`;
+    let buildDir = `build/${buildName}`;
+    let tarName = `${buildName}.tar.gz`;
+    let tarPath = `target/${tarName}`;
+    let zipName = `${buildName}.zip`;
+    let zipPath = `target/${zipName}`;
+
+    return { name, buildName, nodeUrl, tarName, tarPath, zipName, zipPath, buildDir, nodeDir };
+  });
+};

tasks/config/run.js

@@ -23,7 +23,7 @@ module.exports = function (grunt) {
       options: {
         wait: false,
         ready: /Optimization .+ complete/,
-        quiet: false
+        quiet: true
       },
       cmd: './build/kibana/bin/kibana',
       args: [

tasks/config/s3.js

@@ -6,14 +6,14 @@ module.exports = function (grunt) {
     release: {
       bucket: 'download.elasticsearch.org',
       access: 'private',
-      // debug: true, // uncommment to prevent actual upload
+      debug: !grunt.option('upload-to-s3'),
       upload: config.get('platforms')
       .reduce(function (files, platform) {
         return files.concat(
-          platform.tarPath,
-          platform.tarPath + '.sha1.txt',
-          platform.zipPath,
-          platform.zipPath + '.sha1.txt'
+          platform.tarName,
+          platform.tarName + '.sha1.txt',
+          platform.zipName,
+          platform.zipName + '.sha1.txt'
         );
       }, [])
       .map(function (filename) {

tasks/jenkins.js

@@ -2,7 +2,8 @@ module.exports = function (grunt) {
 
   grunt.registerTask('jenkins', 'Jenkins build script', [
     'esvm:dev',
-    'test'
+    'test',
+    'build'
   ]);
 
 };

tasks/release.js

@@ -5,6 +5,7 @@ module.exports = function (grunt) {
     'distribute:load_s3_config',
     'build',
     's3:release',
+    'distribute:complete'
   ]);
 
   // collect the key and secret from the .aws-config.json file, finish configuring the s3 task
@@ -15,4 +16,19 @@ module.exports = function (grunt) {
       secret: config.secret
     });
   });
+
+  grunt.registerTask('distribute:complete', function () {
+    grunt.log.ok('Builds released');
+    grunt.log.write(
+`
+${grunt.config.get('platforms').reduce((t, p) => {
+  return (
+`${t}https://download.elastic.co/kibana/kibana/${p.buildName}.tar.gz
+https://download.elastic.co/kibana/kibana/${p.buildName}.zip
+`
+  );
+}, '')}
+`
+);
+  });
 };