Merge branch 'master' into feature/ingest

CONTRIBUTING.md

@@ -55,7 +55,7 @@ Please make sure you have signed the [Contributor License Agreement](http://www.
   npm run elasticsearch
   ```
 
-- Start the development server. _On Windows, you'll need you use Git Bash, Cygwin, or a similar shell that exposes the `sh` command._
+- Start the development server. _On Windows, you'll need you use Git Bash, Cygwin, or a similar shell that exposes the `sh` command.  And to successfully build you'll need Cygwin optional packages zip, tar, and shasum._
 
   ```sh
   npm start

src/plugins/kbn_doc_views/public/__tests__/doc_views.js

@@ -106,6 +106,8 @@ describe('docViews', function () {
         expect($scope.filter.calledOnce).to.be(true);
         cell.find('.fa-search-minus').first().click();
         expect($scope.filter.calledTwice).to.be(true);
+        cell.find('.fa-asterisk').first().click();
+        expect($scope.filter.calledThrice).to.be(true);
       });
 
       it('should NOT apply a filter when clicking non-filterable fields', function () {
@@ -115,6 +117,8 @@ describe('docViews', function () {
         expect($scope.filter.calledOnce).to.be(false);
         cell.find('.fa-search-minus').first().click();
         expect($scope.filter.calledTwice).to.be(false);
+        cell.find('.fa-asterisk').first().click();
+        expect($scope.filter.calledOnce).to.be(true);
       });
     });
 

src/plugins/kbn_doc_views/public/views/table.html

@@ -27,6 +27,15 @@
             tooltip-append-to-body="1"
             class="fa fa-columns"></i>
         </span>
+        <span ng-if="!indexPattern.metaFields.includes(field)">
+          <i ng-click="filter('_exists_', field, '+')"
+            tooltip="Filter for field present"
+            tooltip-append-to-body="1"
+            class="fa fa-asterisk"></i>
+        </span>
+        <span ng-if="indexPattern.metaFields.includes(field)" tooltip="Unable to filter for presence of meta fields">
+          <i class="fa fa-asterisk text-muted"></i>
+        </span>
       </td>
 
       <td>

src/plugins/kibana/public/settings/sections/objects/_objects.js

@@ -1,13 +1,13 @@
 import { saveAs } from '@spalger/filesaver';
-import _ from 'lodash';
+import { extend, find, flattenDeep, partialRight, pick, pluck, sortBy } from 'lodash';
 import angular from 'angular';
 import registry from 'plugins/kibana/settings/saved_object_registry';
 import objectIndexHTML from 'plugins/kibana/settings/sections/objects/_objects.html';
 import 'ui/directives/file_upload';
 import uiRoutes from 'ui/routes';
 import uiModules from 'ui/modules';
-const MAX_SIZE = Math.pow(2, 31) - 1;
 
+const MAX_SIZE = Math.pow(2, 31) - 1;
 
 uiRoutes
 .when('/settings/objects', {
@@ -41,9 +41,9 @@ uiModules.get('apps/settings')
         });
 
         $q.all(services).then(function (data) {
-          $scope.services = _.sortBy(data, 'title');
+          $scope.services = sortBy(data, 'title');
           let tab = $scope.services[0];
-          if ($state.tab) $scope.currentTab = tab = _.find($scope.services, {title: $state.tab});
+          if ($state.tab) $scope.currentTab = tab = find($scope.services, {title: $state.tab});
 
           $scope.$watch('state.tab', function (tab) {
             if (!tab) $scope.changeTab($scope.services[0]);
@@ -83,23 +83,23 @@ uiModules.get('apps/settings')
       };
 
       $scope.bulkDelete = function () {
-        $scope.currentTab.service.delete(_.pluck($scope.selectedItems, 'id')).then(refreshData).then(function () {
+        $scope.currentTab.service.delete(pluck($scope.selectedItems, 'id')).then(refreshData).then(function () {
           $scope.selectedItems.length = 0;
         });
       };
 
       $scope.bulkExport = function () {
-        const objs = $scope.selectedItems.map(_.partialRight(_.extend, {type: $scope.currentTab.type}));
+        const objs = $scope.selectedItems.map(partialRight(extend, {type: $scope.currentTab.type}));
         retrieveAndExportDocs(objs);
       };
 
-      $scope.exportAll = () => {
-        Promise.map($scope.services, (service) =>
-          service.service.scanAll('').then((results) =>
-            results.hits.map((hit) => _.extend(hit, {type: service.type}))
-          )
-        ).then((results) => retrieveAndExportDocs(_.flattenDeep(results)));
-      };
+      $scope.exportAll = () => Promise
+        .map($scope.services, service => service.service
+          .scanAll('')
+          .then(result => result.hits.map(hit => extend(hit, { type: service.type })))
+        )
+        .then(results => retrieveAndExportDocs(flattenDeep(results)))
+        .catch(error => notify.error(error));
 
       function retrieveAndExportDocs(objs) {
         if (!objs.length) return notify.error('No saved objects to export.');
@@ -108,7 +108,7 @@ uiModules.get('apps/settings')
           body: {docs: objs.map(transformToMget)}
         })
         .then(function (response) {
-          saveToFile(response.docs.map(_.partialRight(_.pick, '_id', '_type', '_source')));
+          saveToFile(response.docs.map(partialRight(pick, '_id', '_type', '_source')));
         });
       }
 
@@ -131,7 +131,7 @@ uiModules.get('apps/settings')
         }
 
         return Promise.map(docs, function (doc) {
-          const service = _.find($scope.services, {type: doc._type}).service;
+          const service = find($scope.services, {type: doc._type}).service;
           return service.get().then(function (obj) {
             obj.id = doc._id;
             return obj.applyESResp(doc).then(function () {

src/ui/public/agg_types/__tests__/buckets/create_filter/date_histogram.js

@@ -75,6 +75,9 @@ describe('AggConfig Filters', function () {
       expect(fieldParams).to.have.property('lte');
       expect(fieldParams.lte).to.be.a('number');
 
+      expect(fieldParams).to.have.property('format');
+      expect(fieldParams.format).to.be('epoch_millis');
+
       expect(fieldParams.gte).to.be.lessThan(fieldParams.lte);
 
       expect(filter).to.have.property('meta');

src/ui/public/agg_types/__tests__/metrics/percentile_ranks.js

@@ -0,0 +1,40 @@
+import expect from 'expect.js';
+import ngMock from 'ng_mock';
+import AggTypeMetricPercentileRanksProvider from 'ui/agg_types/metrics/percentile_ranks';
+import VisProvider from 'ui/vis';
+import FixturesStubbedLogstashIndexPatternProvider from 'fixtures/stubbed_logstash_index_pattern';
+
+describe('AggTypeMetricPercentileRanksProvider class', function () {
+
+  let Vis;
+  let indexPattern;
+  let aggTypeMetricPercentileRanks;
+
+  beforeEach(ngMock.module('kibana'));
+  beforeEach(ngMock.inject(function (Private) {
+    Vis = Private(VisProvider);
+    indexPattern = Private(FixturesStubbedLogstashIndexPatternProvider);
+    aggTypeMetricPercentileRanks = Private(AggTypeMetricPercentileRanksProvider);
+  }));
+
+  it('uses the custom label if it is set', function () {
+    const vis = new Vis(indexPattern, {});
+
+    // Grab the aggConfig off the vis (we don't actually use the vis for
+    // anything else)
+    const aggConfig = vis.aggs[0];
+    aggConfig.params.customLabel = 'my custom field label';
+    aggConfig.params.values = [ 5000, 10000 ];
+    aggConfig.params.field = {
+      displayName: 'bytes'
+    };
+
+    const responseAggs = aggTypeMetricPercentileRanks.getResponseAggs(aggConfig);
+    const percentileRankLabelFor5kBytes = responseAggs[0].makeLabel();
+    const percentileRankLabelFor10kBytes = responseAggs[1].makeLabel();
+
+    expect(percentileRankLabelFor5kBytes).to.be('Percentile rank 5,000 of "my custom field label"');
+    expect(percentileRankLabelFor10kBytes).to.be('Percentile rank 10,000 of "my custom field label"');
+  });
+
+});

src/ui/public/agg_types/buckets/create_filter/date_histogram.js

@@ -8,7 +8,8 @@ export default function createDateHistogramFilterProvider(Private) {
 
     return buildRangeFilter(agg.params.field, {
       gte: start.valueOf(),
-      lte: start.add(interval).subtract(1, 'ms').valueOf()
+      lte: start.add(interval).subtract(1, 'ms').valueOf(),
+      format: 'epoch_millis'
     }, agg.vis.indexPattern);
   };
 

src/ui/public/agg_types/metrics/percentile_ranks.js

@@ -17,8 +17,9 @@ export default function AggTypeMetricPercentileRanksProvider(Private) {
     makeLabel: function () {
       let field = this.field();
       let format = (field && field.format) || fieldFormats.getDefaultInstance('number');
+      const label = this.params.customLabel || this.fieldDisplayName();
 
-      return 'Percentile rank ' + format.convert(this.key, 'text') + ' of "' + this.fieldDisplayName() + '"';
+      return 'Percentile rank ' + format.convert(this.key, 'text') + ' of "' + label + '"';
     }
   };
 

src/ui/public/state_management/state.js

@@ -28,10 +28,14 @@ export default function StateProvider(Private, $rootScope, $location) {
       // beginning of full route update, new app will be initialized before
       // $routeChangeSuccess or $routeChangeError
       $rootScope.$on('$routeChangeStart', function () {
+        if (!self._persistAcrossApps) {
+          self.destroy();
+        }
+      }),
+
+      $rootScope.$on('$routeChangeSuccess', function () {
         if (self._persistAcrossApps) {
           self.fetch();
-        } else {
-          self.destroy();
         }
       })
     ]);

src/ui/public/utils/__tests__/scanner.js

@@ -53,9 +53,22 @@ describe('Scanner', function () {
       scroll = sinon.stub(scanner.client, 'scroll', (req, cb) => cb(null, mockScroll));
     });
 
+    it('should reject when an error occurs', function (done) {
+      search.restore();
+      search = sinon.stub(scanner.client, 'search', (req, cb) => cb(new Error('fail.')));
+      return scanner.scanAndMap('')
+      .then(function (response) {
+        done(new Error('should reject'));
+      })
+      .catch(function (error) {
+        expect(error.message).to.be('fail.');
+        done();
+      });
+    });
+
     it('should search and then scroll for results', function () {
       return scanner.scanAndMap('')
-      .then(function (error, response) {
+      .then(function (response) {
         expect(search.called).to.be(true);
         expect(scroll.called).to.be(true);
       });

src/ui/public/utils/scanner.js

@@ -38,6 +38,10 @@ Scanner.prototype.scanAndMap = function (searchString, options, mapFn) {
 
   return new Promise((resolve, reject) => {
     const getMoreUntilDone = (error, response) => {
+      if (error) {
+        reject(error);
+        return;
+      }
       const scanAllResults = opts.docCount === Infinity;
       allResults.total = scanAllResults ? response.hits.total : Math.min(response.hits.total, opts.docCount);
       scrollId = response._scroll_id || scrollId;

tasks/build/archives.js

@@ -1,6 +1,6 @@
 module.exports = function createPackages(grunt) {
   let { config } = grunt;
-  let { resolve } = require('path');
+  let { resolve, relative } = require('path');
   let { execFile } = require('child_process');
   let { all, fromNode } = require('bluebird');
 
@@ -13,13 +13,13 @@ module.exports = function createPackages(grunt) {
 
   let archives = async (platform) => {
     // kibana.tar.gz
-    await exec('tar', ['-zchf', platform.tarPath, platform.buildName]);
+    await exec('tar', ['-zchf', relative(buildPath, platform.tarPath), platform.buildName]);
 
     // kibana.zip
     if (/windows/.test(platform.name)) {
-      await exec('zip', ['-rq', '-ll', platform.zipPath, platform.buildName]);
+      await exec('zip', ['-rq', '-ll', relative(buildPath, platform.zipPath), platform.buildName]);
     } else {
-      await exec('zip', ['-rq', platform.zipPath, platform.buildName]);
+      await exec('zip', ['-rq', relative(buildPath, platform.zipPath), platform.buildName]);
     }
   };
 

tasks/build/shasums.js

@@ -1,6 +1,8 @@
 var { promisify } = require('bluebird');
 var readdir = promisify(require('fs').readdir);
 var exec = promisify(require('child_process').exec);
+var platform = require('os').platform();
+var cmd = /^win/.test(platform) ? 'sha1sum ' : 'shasum ';
 
 module.exports = function (grunt) {
   grunt.registerTask('_build:shasums', function () {
@@ -11,7 +13,7 @@ module.exports = function (grunt) {
       // only sha the archives
       if (!archive.match(/\.zip$|\.tar.gz$/)) return;
 
-      return exec('shasum ' + archive + ' > ' + archive + '.sha1.txt', {
+      return exec(cmd + archive + ' > ' + archive + '.sha1.txt', {
         cwd: targetDir
       });
     })

tasks/config/licenses.js

@@ -6,14 +6,18 @@ module.exports = function (grunt) {
         'MIT*',
         'MIT License',
         'MIT/X11',
+        'OFL-1.1 AND MIT',
         'new BSD, and MIT',
+        '(MIT OR Apache-2.0)',
         'BSD',
         'BSD*',
         'BSD New',
         'BSD-like',
         'BSD-2-Clause',
+        '(BSD-2-Clause OR MIT OR Apache-2.0)',
         'BSD-3-Clause',
         'CC-BY',
+        'CC-BY-4.0',
         'Apache',
         'Apache*',
         'Apache v2',
@@ -29,9 +33,10 @@ module.exports = function (grunt) {
       ],
       overrides: {
         'amdefine@1.0.0': ['BSD-3-Clause', 'MIT'],
-        'angular-bootstrap@0.10.0': ['MIT'],
+        '@spalger/angular-bootstrap@0.12.1': ['MIT'],
         'angular-ui-ace@0.2.3': ['MIT'],
         'assert-plus@0.1.5': ['MIT'],
+        'buffers@0.1.1': ['MIT/X11'],
         'color-name@1.0.0': ['UNLICENSE'],
         'commander@2.2.0': ['MIT'],
         'css-color-names@0.0.1': ['MIT'],
@@ -39,15 +44,23 @@ module.exports = function (grunt) {
         'css-stringify@1.0.5': ['MIT'],
         'css@1.0.8': ['MIT'],
         'cycle@1.0.3': ['Public-Domain'],
+        'delegate@3.0.1': ['MIT'],
         'FileSaver@1.1.0': ['MIT'],
         'flatten@0.0.1': ['MIT'],
         'indexof@0.0.1': ['MIT'],
         'inherits@1.0.0': ['ISC'],
+        'is-buffer@1.1.2': ['MIT'],
         'jsonpointer@1.1.0': ['MIT'],
+        'lazy-cache@1.0.3': ['MIT'],
         'leaflet@0.7.2': ['BSD-2-Clause'],
         'Nonsense@0.1.2': ['Public-Domain'],
         'pkginfo@0.2.3': ['MIT'],
+        'readable-stream@2.0.5': ['MIT'],
+        'repeat-string@1.5.2': ['MIT'],
+        'ripemd160@0.2.0': ['MIT'],
+        'select@1.0.6': ['MIT'],
         'uglify-js@2.2.5': ['BSD'],
+        'tweetnacl@0.14.3': ['Public-Domain'],
       }
     }
   };

tasks/config/run.js

@@ -4,6 +4,7 @@ module.exports = function (grunt) {
   let {resolve} = require('path');
   let root = p => resolve(__dirname, '../../', p);
   let binScript =  /^win/.test(platform) ? '.\\bin\\kibana.bat' : './bin/kibana';
+  let buildScript =  /^win/.test(platform) ? '.\\build\\kibana\\bin\\kibana.bat' : './build/kibana/bin/kibana';
   let uiConfig = require(root('test/server_config'));
 
   const stdDevArgs = [
@@ -150,7 +151,7 @@ module.exports = function (grunt) {
         ready: /Optimization .+ complete/,
         quiet: true
       },
-      cmd: './build/kibana/bin/kibana',
+      cmd: buildScript,
       args: [
         '--env.name=production',
         '--logging.json=false',

tasks/licenses.js

@@ -13,11 +13,15 @@ module.exports = function (grunt) {
     var options = { start: process.cwd(), json: true };
     var checkQueueLength = 2;
 
+    function getLicenses(info, dependency) {
+      if (config.overrides[dependency]) return config.overrides[dependency];
+      if (info && info.licenses) return _.flatten([info.licenses]);
+    }
+
     function processPackage(info, dependency) {
       var pkgInfo = {};
       pkgInfo.name = dependency;
-      pkgInfo.licenses = config.overrides[dependency] || (info && info.licenses);
-      pkgInfo.licenses = _.isArray(pkgInfo.licenses) ? pkgInfo.licenses : [pkgInfo.licenses];
+      pkgInfo.licenses = getLicenses(info, dependency);
       pkgInfo.valid = (function () {
         if (_.intersection(pkgInfo.licenses, config.licenses).length > 0) {
           return true;

tasks/test.js

@@ -60,6 +60,7 @@ module.exports = function (grunt) {
 
     grunt.task.run(_.compact([
       !grunt.option('quick') && 'eslint:source',
+      'licenses',
       'test:quick'
     ]));
   });

test/functional/apps/console/_console.js

@@ -0,0 +1,68 @@
+import {
+  bdd,
+  scenarioManager,
+  common,
+  // settingsPage,
+  // headerPage,
+  consolePage
+} from '../../../support';
+
+(function () {
+  var expect = require('expect.js');
+
+  (function () {
+    bdd.describe('console app', function describeIndexTests() {
+      bdd.before(function () {
+        common.debug('navigateTo console');
+        return common.navigateToApp('console', false)
+        .catch(common.handleError(this));
+      });
+
+
+      bdd.it('should show the default request', function () {
+        var expectedRequest = [
+          'GET _search',
+          '{',
+          '  "query": {',
+          '    "match_all": {}',
+          '  }',
+          '}',
+          ''
+        ];
+        // collapse the help pane because we only get the VISIBLE TEXT, not the part that is scrolled
+        return consolePage.collapseHelp()
+        .then(function () {
+          return common.try(function () {
+            return consolePage.getRequest()
+            .then(function (actualRequest) {
+              expect(actualRequest).to.eql(expectedRequest);
+            });
+          });
+        })
+        .catch(common.handleError(this));
+      });
+
+      bdd.it('default request reponse should contain .kibana' , function () {
+        var expectedResponseContains = '"_index": ".kibana",';
+        var elasticsearch = common.getEsHostPort();
+        return consolePage.setServer(elasticsearch)
+        .then(function () {
+          return consolePage.clickPlay();
+        })
+        .then(function () {
+          return common.try(function () {
+            return consolePage.getResponse()
+            .then(function (actualResponse) {
+              common.debug(actualResponse);
+              expect(actualResponse).to.contain(expectedResponseContains);
+            });
+          });
+        })
+        .catch(common.handleError(this));
+      });
+
+
+
+    });
+  }());
+}());

test/functional/apps/console/index.js

@@ -0,0 +1,17 @@
+import { bdd, remote, scenarioManager, defaultTimeout } from '../../../support';
+
+(function () {
+  bdd.describe('console app', function () {
+    this.timeout = defaultTimeout;
+
+    bdd.before(function () {
+      return remote.setWindowSize(1200,800);
+    });
+
+    bdd.after(function unloadMakelogs() {
+      return scenarioManager.unload('logstashFunctional');
+    });
+
+    require('./_console');
+  });
+}());

test/functional/apps/discover/_collapse_expand.js

@@ -12,10 +12,8 @@ import {
 
   (function () {
     bdd.describe('discover tab', function describeIndexTests() {
-      var baseUrl;
 
       bdd.before(function () {
-        baseUrl = common.getHostPort();
 
         var fromTime = '2015-09-19 06:31:44.000';
         var toTime = '2015-09-23 18:31:44.000';
@@ -26,7 +24,7 @@ import {
         .then(function loadIfEmptyMakelogs() {
           return scenarioManager.loadIfEmpty('logstashFunctional');
         })
-        .then(function (navigateTo) {
+        .then(function () {
           common.debug('navigateTo');
           return settingsPage.navigateTo().then(settingsPage.clickExistingIndicesAddDataLink);
         })

test/functional/index.js

@@ -23,7 +23,8 @@ define(function (require) {
       'intern/dojo/node!./apps/discover',
       'intern/dojo/node!./status_page',
       'intern/dojo/node!./apps/settings',
-      'intern/dojo/node!./apps/visualize'
+      'intern/dojo/node!./apps/visualize',
+      'intern/dojo/node!./apps/console'
     ], function () {});
   });
 });

test/server_config.js

@@ -41,6 +41,9 @@ module.exports = {
     settings: {
       pathname: kibanaURL,
       hash: '/settings'
+    },
+    console: {
+      pathname: 'app/console',
     }
   }
 };

test/support/index.js

@@ -6,6 +6,7 @@ import SettingsPage from './pages/settings_page';
 import HeaderPage from './pages/header_page';
 import VisualizePage from './pages/visualize_page';
 import ShieldPage from './pages/shield_page';
+import ConsolePage from './pages/console_page';
 
 const kbnInternVars = global.__kibana__intern__;
 
@@ -23,6 +24,7 @@ defineDelayedExport('headerPage', () => new HeaderPage());
 defineDelayedExport('settingsPage', () => new SettingsPage());
 defineDelayedExport('visualizePage', () => new VisualizePage());
 defineDelayedExport('shieldPage', () => new ShieldPage());
+defineDelayedExport('consolePage', () => new ConsolePage());
 
 // creates an export for values that aren't actually avaialable until
 // until tests start to run. These getters will throw errors if the export

test/support/pages/common.js

@@ -9,6 +9,7 @@ export default (function () {
   var _ = require('lodash');
   var parse = require('url').parse;
   var format = require('url').format;
+  var util = require('util');
   var path = require('path');
 
   function injectTimestampQuery(func, url) {
@@ -52,6 +53,10 @@ export default (function () {
       return getUrl.baseUrl(config.servers.kibana);
     },
 
+    getEsHostPort: function getHostPort() {
+      return getUrl.baseUrl(config.servers.elasticsearch);
+    },
+
     navigateToApp: function (appName, testStatusPage) {
       var self = this;
       // navUrl includes user:password@ for use with Shield
@@ -223,12 +228,12 @@ export default (function () {
       return this.tryForTime(defaultFindTimeout, block);
     },
 
-    log: function log(logString) {
-      console.log(moment().format('HH:mm:ss.SSS') + ': ' + logString);
+    log(...args) {
+      console.log(moment().format('HH:mm:ss.SSS') + ':', util.format(...args));
     },
 
-    debug: function debug(logString) {
-      if (config.debug) this.log(logString);
+    debug(...args) {
+      if (config.debug) this.log(...args);
     },
 
     sleep: function sleep(sleepMilliseconds) {

test/support/pages/console_page.js

@@ -0,0 +1,69 @@
+import { remote, defaultFindTimeout } from '../';
+
+// in test/support/pages/shield_page.js
+export default (function (require) {
+  // the page object is created as a constructor
+  // so we can provide the remote Command object
+  // at runtime
+  var thisTime;
+
+  function ConsolePage() {
+    this.remote = remote;
+    thisTime = this.remote.setFindTimeout(defaultFindTimeout);
+  }
+
+  ConsolePage.prototype = {
+    constructor: ConsolePage,
+
+
+    getServer: function getServer() {
+      return thisTime
+      .findByCssSelector('#kibana-body > div.content > div > div')
+      .getVisibleText();
+    },
+
+    setServer: function setServer(server) {
+      return thisTime
+      .findByCssSelector('input[aria-label="Server Name"]')
+      .clearValue()
+      .type(server);
+    },
+
+    getRequest: function getRequest() {
+      return thisTime
+      .findAllByCssSelector('div.ace_line_group')
+      .then(function (editorData) {
+
+        function getEditorData(line) {
+          return line.getVisibleText();
+        }
+
+        var getEditorDataPromises = editorData.map(getEditorData);
+        return Promise.all(getEditorDataPromises);
+      });
+    },
+
+    getResponse: function getResponse() {
+      return thisTime
+      .findByCssSelector('#output > div.ace_scroller > div')
+      .getVisibleText();
+    },
+
+    clickPlay: function clickPlay() {
+      return thisTime
+      .findByCssSelector('#editor_actions > span.ng-scope > a > i')
+      .click();
+    },
+
+    collapseHelp: function collapseHelp() {
+      return thisTime
+      .findByCssSelector('div.config-close.remove > i')
+      .click();
+
+    }
+
+
+  };
+
+  return ConsolePage;
+}());