Add mapping info to index pattern when POST includes a template

src/fixtures/index_pattern.json

@@ -0,0 +1,16 @@
+{
+  "title": "logstash-*",
+  "timeFieldName": "@timestamp",
+  "fields": [
+    {
+      "name": "geo.coordinates",
+      "count": 0,
+      "scripted": false
+    },
+    {
+      "name": "ip",
+      "count": 2,
+      "scripted": false
+    }
+  ]
+}

src/fixtures/index_pattern_with_mappings.json

@@ -1,26 +0,0 @@
-{
-  "title": "logstash-*",
-  "timeFieldName": "@timestamp",
-  "fields": [
-    {
-      "name": "geo.coordinates",
-      "count": 0,
-      "scripted": false,
-      "mapping": {
-        "index": "not_analyzed",
-        "type": "geo_point",
-        "doc_values": false
-      }
-    },
-    {
-      "name": "ip",
-      "count": 2,
-      "scripted": false,
-      "mapping": {
-        "index": "not_analyzed",
-        "type": "ip",
-        "doc_values": true
-      }
-    }
-  ]
-}

src/plugins/kibana/server/lib/__tests__/add_mapping_info_to_pattern_fields.js

@@ -0,0 +1,62 @@
+const expect = require('expect.js');
+const addMappingInfoToPatternFields = require('../add_mapping_info_to_pattern_fields');
+const indexPattern = require('../../../../../fixtures/index_pattern.json');
+const indexTemplate = require('../../../../../fixtures/index_template.json')['kibana-logstash-*'];
+const _ = require('lodash');
+
+describe('addMappingInfoToPatternFields', function () {
+  let testPattern;
+  let testTemplate;
+
+  beforeEach(function () {
+    testPattern = _.cloneDeep(indexPattern);
+    testTemplate = _.cloneDeep(indexTemplate);
+  });
+
+  it('should throw an error if either param is undefined', function () {
+    expect(addMappingInfoToPatternFields).withArgs()
+      .to.throwException(/indexPattern and template are required arguments/);
+    expect(addMappingInfoToPatternFields).withArgs(testPattern)
+      .to.throwException(/indexPattern and template are required arguments/);
+    expect(addMappingInfoToPatternFields).withArgs(undefined, testTemplate)
+      .to.throwException(/indexPattern and template are required arguments/);
+  });
+
+  it('should add mapping info from the provided template to the matching fields in the index pattern', function () {
+    addMappingInfoToPatternFields(testPattern, testTemplate);
+    expect(_.get(testPattern, 'fields[0]')).to.be.ok();
+    expect(_.get(testPattern, 'fields[1]')).to.be.ok();
+    expect(testPattern.fields[0]).to.have.property('type', 'geo_point');
+    expect(testPattern.fields[0]).to.have.property('indexed', true);
+    expect(testPattern.fields[0]).to.have.property('analyzed', false);
+    expect(testPattern.fields[0]).to.have.property('doc_values', false);
+
+    expect(testPattern.fields[1]).to.have.property('type', 'ip');
+    expect(testPattern.fields[1]).to.have.property('indexed', true);
+    expect(testPattern.fields[1]).to.have.property('analyzed', false);
+    expect(testPattern.fields[1]).to.have.property('doc_values', true);
+  });
+
+  it('should detect conflicts in the template mappings and note them in the index pattern', function () {
+    _.set(testTemplate, 'mappings.apache.properties', {
+      ip: {
+        index: 'not_analyzed',
+        type: 'string',
+        doc_values: true
+      }
+    });
+
+    addMappingInfoToPatternFields(testPattern, testTemplate);
+
+    expect(testPattern.fields[1]).to.have.property('type', 'conflict');
+    expect(testPattern.fields[1]).to.have.property('indexed', false);
+  });
+
+  it('should override some elasticsearch metadata mappings', function () {
+    _.set(testTemplate, 'mappings._default_.properties._source', {type: 'string'});
+    _.set(testPattern, 'fields[2]', {name: '_source', type: 'string'});
+
+    addMappingInfoToPatternFields(testPattern, testTemplate);
+    expect(testPattern.fields[2]).to.have.property('type', '_source');
+  });
+});

src/plugins/kibana/server/lib/__tests__/remove_deprecated_field_props.js

@@ -1,7 +1,7 @@
 const expect = require('expect.js');
 const _ = require('lodash');
 const removeDeprecatedFieldProps = require('../remove_deprecated_field_props');
-const indexPattern = require('../../../../../fixtures/index_pattern_with_mappings.json');
+const indexPattern = require('../../../../../fixtures/index_pattern.json');
 
 indexPattern.fields[0].type = 'geo_point';
 indexPattern.fields[0].indexed = true;

src/plugins/kibana/server/lib/add_mapping_info_to_pattern_fields.js

@@ -0,0 +1,59 @@
+const castMappingType = require('./cast_mapping_type');
+const _ = require('lodash');
+
+module.exports = function addMappingInfoToPatternFields(indexPattern, template) {
+  if (!indexPattern || !template) {
+    throw new Error('indexPattern and template are required arguments');
+  }
+
+  // Override the mapping, even if elasticsearch says otherwise
+  var mappingOverrides = {
+    _source: {type: '_source'},
+    _index: {type: 'string'},
+    _type: {type: 'string'},
+    _id: {type: 'string'},
+    _timestamp: {
+      type: 'date',
+      indexed: true
+    },
+    _score: {
+      type: 'number',
+      indexed: false
+    }
+  };
+
+  const dedupedMappings = _.reduce(template.mappings, function (acc, typeMappings) {
+    return _.assign(acc, typeMappings.properties, function (mergedValue, propertyValue) {
+      if (mergedValue === undefined) {
+        return propertyValue;
+      }
+      if (mergedValue.type !== propertyValue.type) {
+        mergedValue.type = 'conflict';
+        mergedValue.index = false;
+      }
+      return mergedValue;
+    });
+  }, {});
+
+  indexPattern.fields.forEach(function (field) {
+    const mapping = dedupedMappings[field.name];
+
+    if (mapping) {
+      field.type = castMappingType(mapping.type);
+
+      if (!mapping.index || mapping.index === 'no') {
+        // elasticsearch responds with false sometimes and 'no' others
+        field.indexed = false;
+      } else {
+        field.indexed = true;
+      }
+
+      field.analyzed = mapping.index === 'analyzed';
+      field.doc_values = mapping.doc_values;
+
+      if (mappingOverrides[field.name]) {
+        _.merge(field, mappingOverrides[field.name]);
+      }
+    }
+  });
+};

src/plugins/kibana/server/lib/cast_mapping_type.js

@@ -0,0 +1,31 @@
+const types = {
+  string: {type: 'string', group: 'base'},
+  date: {type: 'date', group: 'base'},
+  boolean: {type: 'boolean', group: 'base'},
+  float: {type: 'number', group: 'number'},
+  double: {type: 'number', group: 'number'},
+  integer: {type: 'number', group: 'number'},
+  long: {type: 'number', group: 'number'},
+  short: {type: 'number', group: 'number'},
+  byte: {type: 'number', group: 'number'},
+  token_count: {type: 'number', group: 'number'},
+  geo_point: {type: 'geo_point', group: 'geo'},
+  geo_shape: {type: 'geo_shape', group: 'geo'},
+  ip: {type: 'ip', group: 'other'},
+  attachment: {type: 'attachment', group: 'other'},
+  murmur3: {type: 'murmur3', group: 'hash'},
+  conflict: {type: 'conflict', group: 'other'}
+};
+
+/**
+ * Based on _cast_mapping_type from ui/index_patterns
+ * Accepts a mapping type, and converts it into its js equivilent
+ * @param  {String} type - the type from the mapping's 'type' field
+ * @return {String} - the most specific type that we care for
+ */
+module.exports = function castMappingType(name) {
+  if (!name) return 'unknown';
+
+  var match = types[name];
+  return match ? match.type : 'string';
+};

src/plugins/kibana/server/routes/api/index_patterns/register_post.js

@@ -3,6 +3,7 @@ const _ = require('lodash');
 const {templateToPattern, patternToTemplate} = require('../../../lib/convert_pattern_and_template_name');
 const indexPatternSchema = require('../../../lib/schemas/index_pattern_schema');
 const handleESError = require('../../../lib/handle_es_error');
+const addMappingInfoToPatternFields = require('../../../lib/add_mapping_info_to_pattern_fields');
 
 module.exports = function registerPost(server) {
   server.route({
@@ -26,6 +27,9 @@ module.exports = function registerPost(server) {
       const isWildcard = _.contains(indexPattern.title, '*');
       const templateResource = _.isEmpty(included) ? null : included[0];
 
+      if (!_.isEmpty(templateResource)) {
+        addMappingInfoToPatternFields(indexPattern, templateResource.attributes);
+      }
       indexPattern.fields = JSON.stringify(indexPattern.fields);
 
       const patternCreateParams = {