Merge pull request #8462 from elastic/jasper/backport/8421/5.0

[backport] PR #8421 to 5.0

src/core_plugins/kibana/public/management/sections/indices/_indexed_fields.js

@@ -23,8 +23,9 @@ uiModules.get('apps/management')
         { title: 'name' },
         { title: 'type' },
         { title: 'format' },
+        { title: 'searchable', info: 'These fields can be used in the filter bar' },
+        { title: 'aggregatable' , info: 'These fields can be used in visualization aggregations' },
         { title: 'analyzed', info: 'Analyzed fields may require extra memory to visualize' },
-        { title: 'indexed', info: 'Fields that are not indexed are unavailable for search' },
         { title: 'controls', sortable: false }
       ];
 
@@ -54,12 +55,16 @@ uiModules.get('apps/management')
             },
             _.get($scope.indexPattern, ['fieldFormatMap', field.name, 'type', 'title']),
             {
-              markup: field.analyzed ? yesTemplate : noTemplate,
-              value: field.analyzed
+              markup: field.searchable ? yesTemplate : noTemplate,
+              value: field.searchable
             },
             {
-              markup: field.indexed ? yesTemplate : noTemplate,
-              value: field.indexed
+              markup: field.aggregatable ? yesTemplate : noTemplate,
+              value: field.aggregatable
+            },
+            {
+              markup: field.analyzed ? yesTemplate : noTemplate,
+              value: field.analyzed
             },
             {
               markup: controlsHtml,

src/core_plugins/kibana/public/visualize/editor/agg_params.js

@@ -138,7 +138,7 @@ uiModules
       }
 
       function getIndexedFields(param) {
-        let fields = $scope.agg.vis.indexPattern.fields.raw;
+        let fields = _.filter($scope.agg.vis.indexPattern.fields.raw, 'aggregatable');
         const fieldTypes = param.filterFieldTypes;
 
         if (fieldTypes) {

src/core_plugins/kibana/server/routes/api/ingest/index.js

@@ -3,6 +3,7 @@ import { registerDelete } from './register_delete';
 import { registerProcessors } from './register_processors';
 import { registerSimulate } from './register_simulate';
 import { registerData } from './register_data';
+import { registerFieldCapabilities } from './register_field_capabilities';
 
 export default function (server) {
   registerPost(server);
@@ -10,4 +11,5 @@ export default function (server) {
   registerProcessors(server);
   registerSimulate(server);
   registerData(server);
+  registerFieldCapabilities(server);
 }

src/core_plugins/kibana/server/routes/api/ingest/register_field_capabilities.js

@@ -0,0 +1,31 @@
+import _ from 'lodash';
+import handleESError from '../../../lib/handle_es_error';
+
+export function registerFieldCapabilities(server) {
+  server.route({
+    path: '/api/kibana/{indices}/field_capabilities',
+    method: ['GET'],
+    handler: function (req, reply) {
+      const callWithRequest = server.plugins.elasticsearch.callWithRequest;
+      const indices = req.params.indices || '';
+
+      return callWithRequest(req, 'fieldStats', {
+        fields: '*',
+        level: 'cluster',
+        index: indices,
+        allowNoIndices: false
+      })
+      .catch((error) => {
+        reply(handleESError(error));
+      })
+      .then((res) => {
+        const fields = _.get(res, 'indices._all.fields', {});
+        const fieldsFilteredValues = _.mapValues(fields, (value) => {
+          return _.pick(value, ['searchable', 'aggregatable']);
+        });
+
+        reply({fields: fieldsFilteredValues});
+      });
+    }
+  });
+}

src/ui/public/index_patterns/__tests__/_index_pattern.js

@@ -43,6 +43,9 @@ describe('index pattern', function () {
     sinon.stub(mapper, 'getFieldsForIndexPattern', function () {
       return Promise.resolve(_.filter(mockLogstashFields, { scripted: false }));
     });
+    sinon.stub(mapper, 'clearCache', function () {
+      return Promise.resolve();
+    });
 
     // stub mappingSetup
     mappingSetup = Private(UtilsMappingSetupProvider);
@@ -151,15 +154,34 @@ describe('index pattern', function () {
     const indexPatternId = 'test-pattern';
     let indexPattern;
     let fieldLength;
-    let truncatedFields;
+    let customFields;
 
     beforeEach(function () {
       fieldLength = mockLogstashFields.length;
-      truncatedFields = mockLogstashFields.slice(3);
+      customFields = [{
+        analyzed: true,
+        count: 30,
+        filterable: true,
+        indexed: true,
+        name: 'foo',
+        scripted: false,
+        sortable: true,
+        type: 'number',
+        aggregatable: true,
+        searchable: false
+      },
+      {
+        name: 'script number',
+        type: 'number',
+        scripted: true,
+        script: '1234',
+        lang: 'expression'
+      }];
+
       return create(indexPatternId, {
         _source: {
           customFormats: '{}',
-          fields: JSON.stringify(truncatedFields)
+          fields: JSON.stringify(customFields)
         }
       }).then(function (pattern) {
         indexPattern = pattern;
@@ -168,8 +190,8 @@ describe('index pattern', function () {
 
     it('should fetch fields from the doc source', function () {
       // ensure that we don't have all the fields
-      expect(truncatedFields.length).to.not.equal(mockLogstashFields.length);
-      expect(indexPattern.fields).to.have.length(truncatedFields.length);
+      expect(customFields.length).to.not.equal(mockLogstashFields.length);
+      expect(indexPattern.fields).to.have.length(customFields.length);
 
       // ensure that all fields will be included in the returned docSource
       setDocsourcePayload(docSourceResponse(indexPatternId));
@@ -201,9 +223,8 @@ describe('index pattern', function () {
         // called to append scripted fields to the response from mapper.getFieldsForIndexPattern
         expect(scriptedFieldsSpy.callCount).to.equal(1);
 
-        const scripted = _.where(mockLogstashFields, { scripted: true });
-        const expected = _.filter(indexPattern.fields, { scripted: true });
-        expect(_.pluck(expected, 'name')).to.eql(_.pluck(scripted, 'name'));
+        const expected = _.filter(indexPattern.fields, {scripted: true});
+        expect(_.pluck(expected, 'name')).to.eql(['script number']);
       });
     });
   });

src/ui/public/index_patterns/_enhance_fields_with_capabilities.js

@@ -0,0 +1,16 @@
+import chrome from 'ui/chrome';
+import _ from 'lodash';
+
+export default function ($http) {
+
+  return function (fields, indices) {
+    return $http.get(chrome.addBasePath(`/api/kibana/${indices}/field_capabilities`))
+    .then((res) => {
+      const stats = _.get(res, 'data.fields', {});
+
+      return _.map(fields, (field) => {
+        return _.assign(field, stats[field.name]);
+      });
+    });
+  };
+}

src/ui/public/index_patterns/_field.js

@@ -57,6 +57,10 @@ export default function FieldObjectProvider(Private, shortDotsFilter, $rootScope
     obj.fact('analyzed', !!spec.analyzed);
     obj.fact('doc_values', !!spec.doc_values);
 
+    // stats
+    obj.fact('searchable', !!spec.searchable);
+    obj.fact('aggregatable', !!spec.aggregatable);
+
     // usage flags, read-only and won't be saved
     obj.comp('format', format);
     obj.comp('sortable', sortable);

src/ui/public/index_patterns/_index_pattern.js

@@ -82,23 +82,34 @@ export default function IndexPatternFactory(Private, Notifier, config, kbnIndex,
     // give index pattern all of the values in _source
     _.assign(indexPattern, response._source);
 
-    indexFields(indexPattern);
+    const promise = indexFields(indexPattern);
 
     // any time index pattern in ES is updated, update index pattern object
     docSources
     .get(indexPattern)
     .onUpdate()
     .then(response => updateFromElasticSearch(indexPattern, response), notify.fatal);
+
+    return promise;
+  }
+
+  function containsFieldCapabilities(fields) {
+    return _.any(fields, (field) => {
+      return _.has(field, 'aggregatable') && _.has(field, 'searchable');
+    });
   }
 
   function indexFields(indexPattern) {
+    let promise = Promise.resolve();
+
     if (!indexPattern.id) {
-      return;
+      return promise;
     }
-    if (!indexPattern.fields) {
-      return indexPattern.refreshFields();
+
+    if (!indexPattern.fields || !containsFieldCapabilities(indexPattern.fields)) {
+      promise = indexPattern.refreshFields();
     }
-    initFields(indexPattern);
+    return promise.then(() => {initFields(indexPattern);});
   }
 
   function setId(indexPattern, id) {

src/ui/public/index_patterns/_mapper.js

@@ -1,12 +1,14 @@
 import { IndexPatternMissingIndices } from 'ui/errors';
 import _ from 'lodash';
 import moment from 'moment';
+import EnhanceFieldsWithCapabilitiesProvider from 'ui/index_patterns/_enhance_fields_with_capabilities';
 import IndexPatternsTransformMappingIntoFieldsProvider from 'ui/index_patterns/_transform_mapping_into_fields';
 import IndexPatternsIntervalsProvider from 'ui/index_patterns/_intervals';
 import IndexPatternsPatternToWildcardProvider from 'ui/index_patterns/_pattern_to_wildcard';
 import IndexPatternsLocalCacheProvider from 'ui/index_patterns/_local_cache';
 export default function MapperService(Private, Promise, es, config, kbnIndex) {
 
+  let enhanceFieldsWithCapabilities = Private(EnhanceFieldsWithCapabilitiesProvider);
   let transformMappingIntoFields = Private(IndexPatternsTransformMappingIntoFieldsProvider);
   let intervals = Private(IndexPatternsIntervalsProvider);
   let patternToWildcard = Private(IndexPatternsPatternToWildcardProvider);
@@ -49,16 +51,17 @@ export default function MapperService(Private, Promise, es, config, kbnIndex) {
         });
       }
 
-      let promise = Promise.resolve(id);
+      let indexList = id;
+      let promise = Promise.resolve();
       if (indexPattern.intervalName) {
         promise = self.getIndicesForIndexPattern(indexPattern)
         .then(function (existing) {
           if (existing.matches.length === 0) throw new IndexPatternMissingIndices();
-          return existing.matches.slice(-config.get('indexPattern:fieldMapping:lookBack')); // Grab the most recent
+          indexList = existing.matches.slice(-config.get('indexPattern:fieldMapping:lookBack')); // Grab the most recent
         });
       }
 
-      return promise.then(function (indexList) {
+      return promise.then(function () {
         return es.indices.getFieldMapping({
           index: indexList,
           fields: '*',
@@ -69,6 +72,7 @@ export default function MapperService(Private, Promise, es, config, kbnIndex) {
       })
       .catch(handleMissingIndexPattern)
       .then(transformMappingIntoFields)
+      .then(fields => enhanceFieldsWithCapabilities(fields, indexList))
       .then(function (fields) {
         fieldCache.set(id, fields);
         return fieldCache.get(id);

test/unit/api/ingest/_field_capabilities.js

@@ -0,0 +1,87 @@
+define(function (require) {
+  var Promise = require('bluebird');
+  var _ = require('intern/dojo/node!lodash');
+  var expect = require('intern/dojo/node!expect.js');
+
+  return function (bdd, scenarioManager, request) {
+    bdd.describe('field_capabilities API', function postIngest() {
+
+      bdd.before(function () {
+        return scenarioManager.client.create({
+          index: 'foo-1',
+          type: 'bar',
+          id: '1',
+          body: {
+            foo: 'bar'
+          }
+        })
+        .then(function () {
+          return scenarioManager.client.create({
+            index: 'foo-2',
+            type: 'bar',
+            id: '2',
+            body: {
+              baz: 'bar'
+            }
+          });
+        })
+        .then(function () {
+          return scenarioManager.client.indices.refresh({
+            index: ['foo-1', 'foo-2']
+          });
+        });
+      });
+
+      bdd.after(function () {
+        return scenarioManager.reload('emptyKibana')
+        .then(function () {
+          scenarioManager.client.indices.delete({
+            index: 'foo*'
+          });
+        });
+      });
+
+      bdd.it('should return searchable/aggregatable flags for fields in the indices specified', function () {
+        return request.get('/kibana/foo-1/field_capabilities')
+        .expect(200)
+        .then(function (response) {
+          var fields = response.body.fields;
+          expect(fields.foo).to.eql({searchable: true, aggregatable: false});
+          expect(fields['foo.keyword']).to.eql({searchable: true, aggregatable: true});
+          expect(fields).to.not.have.property('baz');
+        });
+      });
+
+      bdd.it('should accept wildcards in the index name', function () {
+        return request.get('/kibana/foo-*/field_capabilities')
+        .expect(200)
+        .then(function (response) {
+          var fields = response.body.fields;
+          expect(fields.foo).to.eql({searchable: true, aggregatable: false});
+          expect(fields.baz).to.eql({searchable: true, aggregatable: false});
+        });
+      });
+
+      bdd.it('should accept comma delimited lists of indices', function () {
+        return request.get('/kibana/foo-1,foo-2/field_capabilities')
+        .expect(200)
+        .then(function (response) {
+          var fields = response.body.fields;
+          expect(fields.foo).to.eql({searchable: true, aggregatable: false});
+          expect(fields.baz).to.eql({searchable: true, aggregatable: false});
+        });
+      });
+
+      bdd.it('should return 404 if a pattern matches no indices', function () {
+        return request.post('/kibana/doesnotexist-*/field_capabilities')
+        .expect(404);
+      });
+
+      bdd.it('should return 404 if a concrete index does not exist', function () {
+        return request.post('/kibana/concrete/field_capabilities')
+        .expect(404);
+      });
+
+    });
+  };
+});

test/unit/api/ingest/index.js

@@ -12,6 +12,7 @@ define(function (require) {
   var simulate = require('./_simulate');
   var processors = require('./_processors');
   var processorTypes = require('./processors/index');
+  var fieldCapabilities = require('./_field_capabilities');
 
   bdd.describe('ingest API', function () {
     var scenarioManager = new ScenarioManager(url.format(serverConfig.servers.elasticsearch));
@@ -31,5 +32,6 @@ define(function (require) {
     simulate(bdd, scenarioManager, request);
     processors(bdd, scenarioManager, request);
     processorTypes(bdd, scenarioManager, request);
+    fieldCapabilities(bdd, scenarioManager, request);
   });
 });