Backport PR #8421

---------

**Commit 1:**
Add field_capabilities API

This adds a simple API for getting the searchable/aggregatable status of
a list of fields in a given index, list of indices, or index pattern. In
the future this will probably evolve into a full blown fields info API
that we can use when removing the index pattern mapping cache. For now
though it's built to provide the minimum info needed to fix
https://github.com/elastic/kibana/issues/6769

Usage:

The API exposes a single GET endpoint.

```
GET /api/kibana/{indices}/field_capabilities
```

`indices` can be a single index, a comma delimited list, or a wildcard
pattern

Example response:

```
{
  "fields": {
    "imsearchable": {
      "searchable": true,
      "aggregatable": false
    },
    "imaggregatable": {
      "searchable": true,
      "aggregatable": true
    },
  }
}
```

* Original sha: 1af6b76bd4
* Authored by Matthew Bargar <mbargar@gmail.com> on 2016-09-21T18:38:34Z

**Commit 2:**
Filter non-aggregatable fields from vis editor UI

Using the field_capabilities API added in the previous commit, this commit enhances
the client side index pattern object with information about the
searchable and aggregatable status of each field in the index pattern.
We then use this information to filter out non-aggregatable fields from
the vis editor so that users won't accidentally select them and get
nasty errors. An example of a non-aggregatable field would be a `text`
field without fielddata enabled (which is the default).

I also added the searchable and aggregatable flags to the index pattern
page so users can see the status of their fields. I removed the `indexed`
column because it was mostly redundant with `searchable` and I needed
the horizontal space.

The addition of the searchable and aggregatable properties for index
pattern fields would require users to manually refresh their field list
when upgrading to 5.0. This commit also adds a check for those properties and
if they're missing it automatically refreshes the field list for the
user in a seamless manner.

* Original sha: 4a906f30c7
* Authored by Matthew Bargar <mbargar@gmail.com> on 2016-09-21T19:18:10Z

src/core_plugins/kibana/public/management/sections/indices/_indexed_fields.js

@@ -23,8 +23,9 @@ uiModules.get('apps/management')
         { title: 'name' },
         { title: 'type' },
         { title: 'format' },
+        { title: 'searchable', info: 'These fields can be used in the filter bar' },
+        { title: 'aggregatable' , info: 'These fields can be used in visualization aggregations' },
         { title: 'analyzed', info: 'Analyzed fields may require extra memory to visualize' },
-        { title: 'indexed', info: 'Fields that are not indexed are unavailable for search' },
         { title: 'controls', sortable: false }
       ];
 
@@ -54,12 +55,16 @@ uiModules.get('apps/management')
             },
             _.get($scope.indexPattern, ['fieldFormatMap', field.name, 'type', 'title']),
             {
-              markup: field.analyzed ? yesTemplate : noTemplate,
-              value: field.analyzed
+              markup: field.searchable ? yesTemplate : noTemplate,
+              value: field.searchable
             },
             {
-              markup: field.indexed ? yesTemplate : noTemplate,
-              value: field.indexed
+              markup: field.aggregatable ? yesTemplate : noTemplate,
+              value: field.aggregatable
+            },
+            {
+              markup: field.analyzed ? yesTemplate : noTemplate,
+              value: field.analyzed
             },
             {
               markup: controlsHtml,

src/core_plugins/kibana/public/visualize/editor/agg_params.js

@@ -138,7 +138,7 @@ uiModules
       }
 
       function getIndexedFields(param) {
-        let fields = $scope.agg.vis.indexPattern.fields.raw;
+        let fields = _.filter($scope.agg.vis.indexPattern.fields.raw, 'aggregatable');
         const fieldTypes = param.filterFieldTypes;
 
         if (fieldTypes) {

src/core_plugins/kibana/server/routes/api/ingest/index.js

@@ -3,6 +3,7 @@ import { registerDelete } from './register_delete';
 import { registerProcessors } from './register_processors';
 import { registerSimulate } from './register_simulate';
 import { registerData } from './register_data';
+import { registerFieldCapabilities } from './register_field_capabilities';
 
 export default function (server) {
   registerPost(server);
@@ -10,4 +11,5 @@ export default function (server) {
   registerProcessors(server);
   registerSimulate(server);
   registerData(server);
+  registerFieldCapabilities(server);
 }

src/core_plugins/kibana/server/routes/api/ingest/register_field_capabilities.js

@@ -0,0 +1,31 @@
+import _ from 'lodash';
+import handleESError from '../../../lib/handle_es_error';
+
+export function registerFieldCapabilities(server) {
+  server.route({
+    path: '/api/kibana/{indices}/field_capabilities',
+    method: ['GET'],
+    handler: function (req, reply) {
+      const callWithRequest = server.plugins.elasticsearch.callWithRequest;
+      const indices = req.params.indices || '';
+
+      return callWithRequest(req, 'fieldStats', {
+        fields: '*',
+        level: 'cluster',
+        index: indices,
+        allowNoIndices: false
+      })
+      .catch((error) => {
+        reply(handleESError(error));
+      })
+      .then((res) => {
+        const fields = _.get(res, 'indices._all.fields', {});
+        const fieldsFilteredValues = _.mapValues(fields, (value) => {
+          return _.pick(value, ['searchable', 'aggregatable']);
+        });
+
+        reply({fields: fieldsFilteredValues});
+      });
+    }
+  });
+}

src/ui/public/index_patterns/__tests__/_index_pattern.js

@@ -43,6 +43,9 @@ describe('index pattern', function () {
     sinon.stub(mapper, 'getFieldsForIndexPattern', function () {
       return Promise.resolve(_.filter(mockLogstashFields, { scripted: false }));
     });
+    sinon.stub(mapper, 'clearCache', function () {
+      return Promise.resolve();
+    });
 
     // stub mappingSetup
     mappingSetup = Private(UtilsMappingSetupProvider);
@@ -151,15 +154,34 @@ describe('index pattern', function () {
     const indexPatternId = 'test-pattern';
     let indexPattern;
     let fieldLength;
-    let truncatedFields;
+    let customFields;
 
     beforeEach(function () {
       fieldLength = mockLogstashFields.length;
-      truncatedFields = mockLogstashFields.slice(3);
+      customFields = [{
+        analyzed: true,
+        count: 30,
+        filterable: true,
+        indexed: true,
+        name: 'foo',
+        scripted: false,
+        sortable: true,
+        type: 'number',
+        aggregatable: true,
+        searchable: false
+      },
+      {
+        name: 'script number',
+        type: 'number',
+        scripted: true,
+        script: '1234',
+        lang: 'expression'
+      }];
+
       return create(indexPatternId, {
         _source: {
           customFormats: '{}',
-          fields: JSON.stringify(truncatedFields)
+          fields: JSON.stringify(customFields)
         }
       }).then(function (pattern) {
         indexPattern = pattern;
@@ -168,8 +190,8 @@ describe('index pattern', function () {
 
     it('should fetch fields from the doc source', function () {
       // ensure that we don't have all the fields
-      expect(truncatedFields.length).to.not.equal(mockLogstashFields.length);
-      expect(indexPattern.fields).to.have.length(truncatedFields.length);
+      expect(customFields.length).to.not.equal(mockLogstashFields.length);
+      expect(indexPattern.fields).to.have.length(customFields.length);
 
       // ensure that all fields will be included in the returned docSource
       setDocsourcePayload(docSourceResponse(indexPatternId));
@@ -201,9 +223,8 @@ describe('index pattern', function () {
         // called to append scripted fields to the response from mapper.getFieldsForIndexPattern
         expect(scriptedFieldsSpy.callCount).to.equal(1);
 
-        const scripted = _.where(mockLogstashFields, { scripted: true });
-        const expected = _.filter(indexPattern.fields, { scripted: true });
-        expect(_.pluck(expected, 'name')).to.eql(_.pluck(scripted, 'name'));
+        const expected = _.filter(indexPattern.fields, {scripted: true});
+        expect(_.pluck(expected, 'name')).to.eql(['script number']);
       });
     });
   });

src/ui/public/index_patterns/_enhance_fields_with_capabilities.js

@@ -0,0 +1,16 @@
+import chrome from 'ui/chrome';
+import _ from 'lodash';
+
+export default function ($http) {
+
+  return function (fields, indices) {
+    return $http.get(chrome.addBasePath(`/api/kibana/${indices}/field_capabilities`))
+    .then((res) => {
+      const stats = _.get(res, 'data.fields', {});
+
+      return _.map(fields, (field) => {
+        return _.assign(field, stats[field.name]);
+      });
+    });
+  };
+}

src/ui/public/index_patterns/_field.js

@@ -57,6 +57,10 @@ export default function FieldObjectProvider(Private, shortDotsFilter, $rootScope
     obj.fact('analyzed', !!spec.analyzed);
     obj.fact('doc_values', !!spec.doc_values);
 
+    // stats
+    obj.fact('searchable', !!spec.searchable);
+    obj.fact('aggregatable', !!spec.aggregatable);
+
     // usage flags, read-only and won't be saved
     obj.comp('format', format);
     obj.comp('sortable', sortable);

src/ui/public/index_patterns/_index_pattern.js

@@ -82,23 +82,34 @@ export default function IndexPatternFactory(Private, Notifier, config, kbnIndex,
     // give index pattern all of the values in _source
     _.assign(indexPattern, response._source);
 
-    indexFields(indexPattern);
+    const promise = indexFields(indexPattern);
 
     // any time index pattern in ES is updated, update index pattern object
     docSources
     .get(indexPattern)
     .onUpdate()
     .then(response => updateFromElasticSearch(indexPattern, response), notify.fatal);
+
+    return promise;
+  }
+
+  function containsFieldCapabilities(fields) {
+    return _.any(fields, (field) => {
+      return _.has(field, 'aggregatable') && _.has(field, 'searchable');
+    });
   }
 
   function indexFields(indexPattern) {
+    let promise = Promise.resolve();
+
     if (!indexPattern.id) {
-      return;
+      return promise;
     }
-    if (!indexPattern.fields) {
-      return indexPattern.refreshFields();
+
+    if (!indexPattern.fields || !containsFieldCapabilities(indexPattern.fields)) {
+      promise = indexPattern.refreshFields();
     }
-    initFields(indexPattern);
+    return promise.then(() => {initFields(indexPattern);});
   }
 
   function setId(indexPattern, id) {

src/ui/public/index_patterns/_mapper.js

@@ -1,12 +1,14 @@
 import { IndexPatternMissingIndices } from 'ui/errors';
 import _ from 'lodash';
 import moment from 'moment';
+import EnhanceFieldsWithCapabilitiesProvider from 'ui/index_patterns/_enhance_fields_with_capabilities';
 import IndexPatternsTransformMappingIntoFieldsProvider from 'ui/index_patterns/_transform_mapping_into_fields';
 import IndexPatternsIntervalsProvider from 'ui/index_patterns/_intervals';
 import IndexPatternsPatternToWildcardProvider from 'ui/index_patterns/_pattern_to_wildcard';
 import IndexPatternsLocalCacheProvider from 'ui/index_patterns/_local_cache';
 export default function MapperService(Private, Promise, es, config, kbnIndex) {
 
+  let enhanceFieldsWithCapabilities = Private(EnhanceFieldsWithCapabilitiesProvider);
   let transformMappingIntoFields = Private(IndexPatternsTransformMappingIntoFieldsProvider);
   let intervals = Private(IndexPatternsIntervalsProvider);
   let patternToWildcard = Private(IndexPatternsPatternToWildcardProvider);
@@ -49,16 +51,17 @@ export default function MapperService(Private, Promise, es, config, kbnIndex) {
         });
       }
 
-      let promise = Promise.resolve(id);
+      let indexList = id;
+      let promise = Promise.resolve();
       if (indexPattern.intervalName) {
         promise = self.getIndicesForIndexPattern(indexPattern)
         .then(function (existing) {
           if (existing.matches.length === 0) throw new IndexPatternMissingIndices();
-          return existing.matches.slice(-config.get('indexPattern:fieldMapping:lookBack')); // Grab the most recent
+          indexList = existing.matches.slice(-config.get('indexPattern:fieldMapping:lookBack')); // Grab the most recent
         });
       }
 
-      return promise.then(function (indexList) {
+      return promise.then(function () {
         return es.indices.getFieldMapping({
           index: indexList,
           fields: '*',
@@ -69,6 +72,7 @@ export default function MapperService(Private, Promise, es, config, kbnIndex) {
       })
       .catch(handleMissingIndexPattern)
       .then(transformMappingIntoFields)
+      .then(fields => enhanceFieldsWithCapabilities(fields, indexList))
       .then(function (fields) {
         fieldCache.set(id, fields);
         return fieldCache.get(id);

test/unit/api/ingest/_field_capabilities.js

@@ -0,0 +1,87 @@
+define(function (require) {
+  var Promise = require('bluebird');
+  var _ = require('intern/dojo/node!lodash');
+  var expect = require('intern/dojo/node!expect.js');
+
+  return function (bdd, scenarioManager, request) {
+    bdd.describe('field_capabilities API', function postIngest() {
+
+      bdd.before(function () {
+        return scenarioManager.client.create({
+          index: 'foo-1',
+          type: 'bar',
+          id: '1',
+          body: {
+            foo: 'bar'
+          }
+        })
+        .then(function () {
+          return scenarioManager.client.create({
+            index: 'foo-2',
+            type: 'bar',
+            id: '2',
+            body: {
+              baz: 'bar'
+            }
+          });
+        })
+        .then(function () {
+          return scenarioManager.client.indices.refresh({
+            index: ['foo-1', 'foo-2']
+          });
+        });
+      });
+
+      bdd.after(function () {
+        return scenarioManager.reload('emptyKibana')
+        .then(function () {
+          scenarioManager.client.indices.delete({
+            index: 'foo*'
+          });
+        });
+      });
+
+      bdd.it('should return searchable/aggregatable flags for fields in the indices specified', function () {
+        return request.get('/kibana/foo-1/field_capabilities')
+        .expect(200)
+        .then(function (response) {
+          var fields = response.body.fields;
+          expect(fields.foo).to.eql({searchable: true, aggregatable: false});
+          expect(fields['foo.keyword']).to.eql({searchable: true, aggregatable: true});
+          expect(fields).to.not.have.property('baz');
+        });
+      });
+
+      bdd.it('should accept wildcards in the index name', function () {
+        return request.get('/kibana/foo-*/field_capabilities')
+        .expect(200)
+        .then(function (response) {
+          var fields = response.body.fields;
+          expect(fields.foo).to.eql({searchable: true, aggregatable: false});
+          expect(fields.baz).to.eql({searchable: true, aggregatable: false});
+        });
+      });
+
+      bdd.it('should accept comma delimited lists of indices', function () {
+        return request.get('/kibana/foo-1,foo-2/field_capabilities')
+        .expect(200)
+        .then(function (response) {
+          var fields = response.body.fields;
+          expect(fields.foo).to.eql({searchable: true, aggregatable: false});
+          expect(fields.baz).to.eql({searchable: true, aggregatable: false});
+        });
+      });
+
+      bdd.it('should return 404 if a pattern matches no indices', function () {
+        return request.post('/kibana/doesnotexist-*/field_capabilities')
+        .expect(404);
+      });
+
+      bdd.it('should return 404 if a concrete index does not exist', function () {
+        return request.post('/kibana/concrete/field_capabilities')
+        .expect(404);
+      });
+
+    });
+  };
+});

test/unit/api/ingest/index.js

@@ -12,6 +12,7 @@ define(function (require) {
   var simulate = require('./_simulate');
   var processors = require('./_processors');
   var processorTypes = require('./processors/index');
+  var fieldCapabilities = require('./_field_capabilities');
 
   bdd.describe('ingest API', function () {
     var scenarioManager = new ScenarioManager(url.format(serverConfig.servers.elasticsearch));
@@ -31,5 +32,6 @@ define(function (require) {
     simulate(bdd, scenarioManager, request);
     processors(bdd, scenarioManager, request);
     processorTypes(bdd, scenarioManager, request);
+    fieldCapabilities(bdd, scenarioManager, request);
   });
 });