[5.x] Disable dynamic/Implement static mappings (#10638) (#11417)

* Disable dynamic/Implement static mappings (#10638)

* Disable dynamic mappings

* Disable automatic type creation

* Centralize .kibana mappings

* Use dynamic strict instead of false

* Register mappings per plugin

* Check for registered mapping conflicts

* Expose mappings on server scope instead of module

* Stub elasticsearch mappings class

* Add tests for KibanaMappings class

* [mappings tests] to.an -> to.be.an

* Return on mapping conflict and include plugin id if provided

* [uiExports] Add mappings uiExport type

* Use uiExports config in mappings tests

* Clean up ui_mappings test

* Revert plugin require ordering

* Pull uiExports mappings from kibana object isntead of kbnServer

* [test] Update index mapping to look for keyword

src/core_plugins/elasticsearch/index.js

@@ -13,10 +13,9 @@ import createProxy, { createPath } from './lib/create_proxy';
 
 const DEFAULT_REQUEST_HEADERS = [ 'authorization' ];
 
-module.exports = function ({ Plugin }) {
-  return new Plugin({
+module.exports = function (kibana) {
+  return new kibana.Plugin({
     require: ['kibana'],
-
     config(Joi) {
       const { array, boolean, number, object, string, ref } = Joi;
 
@@ -159,9 +158,9 @@ module.exports = function ({ Plugin }) {
           pre: [ noDirectIndex, noBulkCheck ]
         }
       );
-
       // Set up the health check service and start it.
-      const { start, waitUntilReady } = healthCheck(this, server);
+      const mappings = kibana.uiExports.mappings.getCombined();
+      const { start, waitUntilReady } = healthCheck(this, server, { mappings });
       server.expose('waitUntilReady', waitUntilReady);
       start();
     }

src/core_plugins/elasticsearch/lib/__tests__/create_kibana_index.js

@@ -2,7 +2,7 @@ import _ from 'lodash';
 import sinon from 'sinon';
 import expect from 'expect.js';
 import Promise from 'bluebird';
-
+import mappings from './fixtures/mappings';
 import createKibanaIndex from '../create_kibana_index';
 import SetupError from '../setup_error';
 
@@ -23,13 +23,12 @@ describe('plugins/elasticsearch', function () {
       get.withArgs('kibana.index').returns(config.kibana.index);
       config = function () { return { get: get }; };
 
-      _.set(server, 'plugins.elasticsearch', {});
       _.set(server, 'config', config);
 
       callWithInternalUser = sinon.stub();
       cluster = { callWithInternalUser: callWithInternalUser };
 
-      server.plugins.elasticsearch.getCluster = sinon.stub().withArgs('admin').returns(cluster);
+      _.set(server, 'plugins.elasticsearch.getCluster', sinon.stub().withArgs('admin').returns(cluster));
     });
 
     describe('successful requests', function () {
@@ -39,14 +38,14 @@ describe('plugins/elasticsearch', function () {
       });
 
       it('should check cluster.health upon successful index creation', function () {
-        const fn = createKibanaIndex(server);
+        const fn = createKibanaIndex(server, mappings);
         return fn.then(function () {
           sinon.assert.calledOnce(callWithInternalUser.withArgs('cluster.health', sinon.match.any));
         });
       });
 
       it('should be created with mappings for config.buildNum', function () {
-        const fn = createKibanaIndex(server);
+        const fn = createKibanaIndex(server, mappings);
         return fn.then(function () {
           const params = callWithInternalUser.args[0][1];
           expect(params)
@@ -60,14 +59,12 @@ describe('plugins/elasticsearch', function () {
           expect(params.body.mappings.config.properties)
             .to.have.property('buildNum');
           expect(params.body.mappings.config.properties.buildNum)
-            .to.have.property('type', 'string');
-          expect(params.body.mappings.config.properties.buildNum)
-            .to.have.property('index', 'not_analyzed');
+            .to.have.property('type', 'keyword');
         });
       });
 
       it('should be created with 1 shard and default replica', function () {
-        const fn = createKibanaIndex(server);
+        const fn = createKibanaIndex(server, mappings);
         return fn.then(function () {
           const params = callWithInternalUser.args[0][1];
           expect(params)
@@ -82,7 +79,7 @@ describe('plugins/elasticsearch', function () {
       });
 
       it('should be created with index name set in the config', function () {
-        const fn = createKibanaIndex(server);
+        const fn = createKibanaIndex(server, mappings);
         return fn.then(function () {
           const params = callWithInternalUser.args[0][1];
           expect(params)

src/core_plugins/elasticsearch/lib/__tests__/fixtures/mappings.js

@@ -0,0 +1,13 @@
+module.exports = {
+  '_default_': {
+    'dynamic': 'strict'
+  },
+  config: {
+    dynamic: true,
+    properties: {
+      buildNum: {
+        type: 'keyword'
+      }
+    }
+  }
+};

src/core_plugins/elasticsearch/lib/__tests__/health_check.js

@@ -5,6 +5,7 @@ import url from 'url';
 
 const NoConnections = require('elasticsearch').errors.NoConnections;
 
+import mappings from './fixtures/mappings';
 import healthCheck from '../health_check';
 import kibanaVersion from '../kibana_version';
 import serverConfig from '../../../../../test/server_config';
@@ -70,7 +71,7 @@ describe('plugins/elasticsearch', () => {
         }
       };
 
-      health = healthCheck(plugin, server);
+      health = healthCheck(plugin, server, { mappings });
     });
 
     afterEach(() => {

src/core_plugins/elasticsearch/lib/create_kibana_index.js

@@ -1,7 +1,6 @@
 import SetupError from './setup_error';
-import { mappings } from './kibana_index_mappings';
 
-module.exports = function (server) {
+module.exports = function (server, mappings) {
   const { callWithInternalUser } = server.plugins.elasticsearch.getCluster('admin');
   const index = server.config().get('kibana.index');
 
@@ -15,7 +14,8 @@ module.exports = function (server) {
     index: index,
     body: {
       settings: {
-        number_of_shards: 1
+        number_of_shards: 1,
+        'index.mapper.dynamic': false,
       },
       mappings
     }

src/core_plugins/elasticsearch/lib/health_check.js

@@ -16,7 +16,7 @@ const NO_INDEX = 'no_index';
 const INITIALIZING = 'initializing';
 const READY = 'ready';
 
-module.exports = function (plugin, server) {
+module.exports = function (plugin, server, { mappings }) {
   const config = server.config();
   const callAdminAsKibanaUser = server.plugins.elasticsearch.getCluster('admin').callWithInternalUser;
   const callDataAsKibanaUser = server.plugins.elasticsearch.getCluster('data').callWithInternalUser;
@@ -70,7 +70,7 @@ module.exports = function (plugin, server) {
     .then(function (health) {
       if (health === NO_INDEX) {
         plugin.status.yellow('No existing Kibana index found');
-        return createKibanaIndex(server);
+        return createKibanaIndex(server, mappings);
       }
 
       if (health === INITIALIZING) {
@@ -98,7 +98,7 @@ module.exports = function (plugin, server) {
       .then(() => ensureNotTribe(callAdminAsKibanaUser))
       .then(() => ensureAllowExplicitIndex(callAdminAsKibanaUser, config))
       .then(waitForShards)
-      .then(_.partial(migrateConfig, server))
+      .then(_.partial(migrateConfig, server, { mappings }))
       .then(() => {
         const tribeUrl = config.get('elasticsearch.tribe.url');
         if (tribeUrl) {

src/core_plugins/elasticsearch/lib/kibana_index_mappings.js

@@ -1,17 +0,0 @@
-export const mappings = {
-  config: {
-    properties: {
-      buildNum: {
-        type: 'string',
-        index: 'not_analyzed'
-      }
-    }
-  },
-  server: {
-    properties: {
-      uuid: {
-        type: 'keyword'
-      }
-    }
-  }
-};

src/core_plugins/elasticsearch/lib/migrate_config.js

@@ -1,10 +1,9 @@
+import { get } from 'lodash';
 import upgrade from './upgrade_config';
-import { mappings } from './kibana_index_mappings';
 
-module.exports = function (server) {
+module.exports = function (server, { mappings }) {
   const config = server.config();
   const { callWithInternalUser } = server.plugins.elasticsearch.getCluster('admin');
-
   const options =  {
     index: config.get('kibana.index'),
     type: 'config',
@@ -14,7 +13,7 @@ module.exports = function (server) {
         {
           buildNum: {
             order: 'desc',
-            unmapped_type: mappings.config.properties.buildNum.type
+            unmapped_type: get(mappings, 'config.properties.buildNum.type') || 'keyword'
           }
         }
       ]

src/core_plugins/kibana/index.js

@@ -9,6 +9,7 @@ import search from './server/routes/api/search';
 import settings from './server/routes/api/settings';
 import scripts from './server/routes/api/scripts';
 import * as systemApi from './server/lib/system_api';
+import mappings from './mappings.json';
 
 const mkdirp = Promise.promisify(mkdirpNode);
 
@@ -16,7 +17,6 @@ module.exports = function (kibana) {
   const kbnBaseUrl = '/app/kibana';
   return new kibana.Plugin({
     id: 'kibana',
-
     config: function (Joi) {
       return Joi.object({
         enabled: Joi.boolean().default(true),
@@ -42,7 +42,6 @@ module.exports = function (kibana) {
           'devTools',
           'docViews'
         ],
-
         injectVars: function (server) {
           const serverConfig = server.config();
 
@@ -120,7 +119,8 @@ module.exports = function (kibana) {
 
       translations: [
         resolve(__dirname, './translations/en.json')
-      ]
+      ],
+      mappings
     },
 
     preInit: async function (server) {

src/core_plugins/kibana/mappings.json

@@ -0,0 +1,171 @@
+{
+  "index-pattern": {
+    "properties": {
+      "fieldFormatMap": {
+        "type": "text"
+      },
+      "fields": {
+        "type": "text"
+      },
+      "intervalName": {
+        "type": "keyword"
+      },
+      "notExpandable": {
+        "type": "boolean"
+      },
+      "sourceFilters": {
+        "type": "text"
+      },
+      "timeFieldName": {
+        "type": "keyword"
+      },
+      "title": {
+        "type": "text"
+      }
+    }
+  },
+  "visualization": {
+    "properties": {
+      "description": {
+        "type": "text"
+      },
+      "kibanaSavedObjectMeta": {
+        "properties": {
+          "searchSourceJSON": {
+            "type": "text"
+          }
+        }
+      },
+      "savedSearchId": {
+        "type": "keyword"
+      },
+      "title": {
+        "type": "text"
+      },
+      "uiStateJSON": {
+        "type": "text"
+      },
+      "version": {
+        "type": "integer"
+      },
+      "visState": {
+        "type": "text"
+      }
+    }
+  },
+  "search": {
+    "properties": {
+      "columns": {
+        "type": "keyword"
+      },
+      "description": {
+        "type": "text"
+      },
+      "hits": {
+        "type": "integer"
+      },
+      "kibanaSavedObjectMeta": {
+        "properties": {
+          "searchSourceJSON": {
+            "type": "text"
+          }
+        }
+      },
+      "sort": {
+        "type": "keyword"
+      },
+      "title": {
+        "type": "text"
+      },
+      "version": {
+        "type": "integer"
+      }
+    }
+  },
+  "dashboard": {
+    "properties": {
+      "description": {
+        "type": "text"
+      },
+      "hits": {
+        "type": "integer"
+      },
+      "kibanaSavedObjectMeta": {
+        "properties": {
+          "searchSourceJSON": {
+            "type": "text"
+          }
+        }
+      },
+      "optionsJSON": {
+        "type": "text"
+      },
+      "panelsJSON": {
+        "type": "text"
+      },
+      "refreshInterval": {
+        "properties": {
+          "display": {
+            "type": "keyword"
+          },
+          "pause": {
+            "type": "boolean"
+          },
+          "section": {
+            "type": "integer"
+          },
+          "value": {
+            "type": "integer"
+          }
+        }
+      },
+      "timeFrom": {
+        "type": "keyword"
+      },
+      "timeRestore": {
+        "type": "boolean"
+      },
+      "timeTo": {
+        "type": "keyword"
+      },
+      "title": {
+        "type": "text"
+      },
+      "uiStateJSON": {
+        "type": "text"
+      },
+      "version": {
+        "type": "integer"
+      }
+    }
+  },
+  "url": {
+    "properties": {
+      "accessCount": {
+        "type": "long"
+      },
+      "accessDate": {
+        "type": "date"
+      },
+      "createDate": {
+        "type": "date"
+      },
+      "url": {
+        "type": "text",
+        "fields": {
+          "keyword": {
+            "type": "keyword",
+            "ignore_above": 2048
+          }
+        }
+      }
+    }
+  },
+  "server": {
+    "properties": {
+      "uuid": {
+        "type": "keyword"
+      }
+    }
+  }
+}

src/core_plugins/timelion/index.js

@@ -35,7 +35,8 @@ module.exports = function (kibana) {
       ],
       visTypes: [
         'plugins/timelion/vis'
-      ]
+      ],
+      mappings: require('./mappings.json')
     },
     init: require('./init.js'),
   });

src/core_plugins/timelion/init.js

@@ -3,7 +3,6 @@ import processFunctionDefinition from './server/lib/process_function_definition'
 
 module.exports = function (server) {
   //var config = server.config();
-
   require('./server/routes/run.js')(server);
   require('./server/routes/functions.js')(server);
   require('./server/routes/validate_es.js')(server);

src/core_plugins/timelion/mappings.json

@@ -0,0 +1,43 @@
+{
+  "timelion-sheet": {
+    "properties": {
+      "description": {
+        "type": "text"
+      },
+      "hits": {
+        "type": "integer"
+      },
+      "kibanaSavedObjectMeta": {
+        "properties": {
+          "searchSourceJSON": {
+            "type": "text"
+          }
+        }
+      },
+      "timelion_chart_height": {
+        "type": "integer"
+      },
+      "timelion_columns": {
+        "type": "integer"
+      },
+      "timelion_interval": {
+        "type": "keyword"
+      },
+      "timelion_other_interval": {
+        "type": "keyword"
+      },
+      "timelion_rows": {
+        "type": "integer"
+      },
+      "timelion_sheet": {
+        "type": "text"
+      },
+      "title": {
+        "type": "text"
+      },
+      "version": {
+        "type": "integer"
+      }
+    }
+  }
+}

src/ui/__tests__/ui_mappings.js

@@ -0,0 +1,40 @@
+import expect from 'expect.js';
+import { has } from 'lodash';
+import { MappingsCollection } from '../ui_mappings';
+
+
+describe('UiExports', function () {
+  describe('MappingsCollection', function () {
+    let mappingsCollection;
+    beforeEach(() => {
+      mappingsCollection = new MappingsCollection();
+    });
+
+    it('provides default mappings', function () {
+      expect(mappingsCollection.getCombined()).to.be.an('object');
+    });
+
+    it('registers new mappings', () => {
+      mappingsCollection.register({
+        foo: {
+          'properties': {
+            'bar': {
+              'type': 'text'
+            }
+          }
+        }
+      });
+
+      const mappings = mappingsCollection.getCombined();
+      expect(has(mappings, 'foo.properties.bar')).to.be(true);
+    });
+
+    it('throws and includes the plugin id in the mapping conflict message', () => {
+      const mappings = { foo: 'bar' };
+      const plugin = { plugin: 'abc123' };
+      mappingsCollection.register(mappings, plugin);
+      expect(mappingsCollection.register).withArgs(mappings, plugin).to.throwException(/abc123/);
+    });
+  });
+
+});

src/ui/ui_exports.js

@@ -3,6 +3,7 @@ import minimatch from 'minimatch';
 
 import UiAppCollection from './ui_app_collection';
 import UiNavLinkCollection from './ui_nav_link_collection';
+import { MappingsCollection } from './ui_mappings';
 
 class UiExports {
   constructor({ urlBasePath }) {
@@ -15,6 +16,7 @@ class UiExports {
     this.bundleProviders = [];
     this.defaultInjectedVars = {};
     this.injectedVarsReplacers = [];
+    this.mappings = new MappingsCollection();
   }
 
   consumePlugin(plugin) {
@@ -124,6 +126,11 @@ class UiExports {
           });
         };
 
+      case 'mappings':
+        return (plugin, mappings) => {
+          this.mappings.register(mappings, { plugin: plugin.id });
+        };
+
       case 'replaceInjectedVars':
         return (plugin, replacer) => {
           this.injectedVarsReplacers.push(replacer);

src/ui/ui_mappings.js

@@ -0,0 +1,37 @@
+import _ from 'lodash';
+
+class MappingsCollection {
+  constructor() {
+    this._defaultMappings = {
+      '_default_': {
+        'dynamic': 'strict'
+      },
+      config: {
+        dynamic: true,
+        properties: {
+          buildNum: {
+            type: 'keyword'
+          }
+        }
+      },
+    };
+    this._currentMappings = _.cloneDeep(this._defaultMappings);
+  }
+
+  getCombined = () => {
+    return this._currentMappings;
+  }
+
+  register = (newMappings, options = {}) => {
+    Object.keys(this._currentMappings).forEach(key => {
+      if (newMappings.hasOwnProperty(key)) {
+        const pluginPartial = options.plugin ? `registered by plugin ${options.plugin} ` : '';
+        throw new Error(`Mappings for ${key} ${pluginPartial}have already been defined`);
+        return;
+      }
+    });
+    Object.assign(this._currentMappings, newMappings);
+  }
+}
+
+export { MappingsCollection };