[6.7] Allow for compatibility with ES 7.x (#30636)

Make Kibana 6.7 compatible with 7.0, at least enough to unblock manual testing.

packages/kbn-es/src/integration_tests/cluster.test.js

@@ -139,10 +139,14 @@ describe('#installSnapshot()', () => {
     const cluster = new Cluster(log);
     await cluster.installSnapshot({ foo: 'bar' });
     expect(installSnapshot).toHaveBeenCalledTimes(1);
-    expect(installSnapshot).toHaveBeenCalledWith({
-      log,
-      foo: 'bar',
-    });
+    expect(installSnapshot).toHaveBeenCalledWith(
+      // In es6.7, this is the literal object. In 7.0, it also
+      // contains "version": "7.0.0"
+      expect.objectContaining({
+        log,
+        foo: 'bar',
+      })
+    );
   });
 
   it('rejects if installSnapshot() rejects', async () => {

src/es_archiver/lib/indices/create_index_stream.js

@@ -53,16 +53,20 @@ export function createCreateIndexStream({ client, stats, skipExisting, log, kiba
         await client.indices.create({
           method: 'PUT',
           index,
+          include_type_name: true,
           body: { settings, mappings },
         });
 
-        if (index.startsWith('.kibana') && await isSpacesEnabled({ kibanaUrl })) {
+        if (index.startsWith('.kibana') && (await isSpacesEnabled({ kibanaUrl }))) {
           await createDefaultSpace({ index, client });
         }
 
         stats.createdIndex(index, { settings });
       } catch (err) {
-        if (get(err, 'body.error.type') !== 'resource_already_exists_exception' || attemptNumber >= 3) {
+        if (
+          get(err, 'body.error.type') !== 'resource_already_exists_exception' ||
+          attemptNumber >= 3
+        ) {
           throw err;
         }
 
@@ -104,6 +108,6 @@ export function createCreateIndexStream({ client, stats, skipExisting, log, kiba
       } catch (err) {
         callback(err);
       }
-    }
+    },
   });
 }

src/server/sample_data/routes/install.js

@@ -118,7 +118,8 @@ export const createInstallRoute = () => ({
                   properties: dataIndexConfig.fields
                 }
               }
-            }
+            },
+            include_type_name: true
           };
           await callWithRequest(request, 'indices.create', createIndexParams);
         } catch (err) {

src/server/saved_objects/migrations/core/call_cluster.ts

@@ -83,10 +83,12 @@ export interface PutTemplateOpts {
 
 export interface IndexOpts {
   index: string;
+  include_type_name?: boolean;
 }
 
 export interface IndexCreationOpts {
   index: string;
+  include_type_name?: boolean;
   body?: {
     mappings?: IndexMapping;
     settings?: {

src/server/saved_objects/migrations/core/elastic_index.test.ts

@@ -641,7 +641,9 @@ describe('ElasticIndex', () => {
       });
 
       expect(hasMigrations).toBeFalsy();
-      expect(callCluster.args).toEqual([['indices.get', { ignore: [404], index: '.myalias' }]]);
+      expect(callCluster.args).toEqual([
+        ['indices.get', { ignore: [404], index: '.myalias', include_type_name: true }],
+      ]);
     });
 
     test('is true if there are no migrations defined', async () => {

src/server/saved_objects/migrations/core/elastic_index.ts

@@ -52,7 +52,11 @@ export interface FullIndexInfo {
  * index mappings are somewhat what we expect.
  */
 export async function fetchInfo(callCluster: CallCluster, index: string): Promise<FullIndexInfo> {
-  const result = await callCluster('indices.get', { ignore: [404], index });
+  const result = await callCluster('indices.get', {
+    ignore: [404],
+    index,
+    include_type_name: true,
+  });
 
   if ((result as NotFound).status === 404) {
     return {
@@ -226,7 +230,11 @@ export async function createIndex(
   index: string,
   mappings?: IndexMapping
 ) {
-  await callCluster('indices.create', { body: { mappings, settings }, index });
+  await callCluster('indices.create', {
+    body: { mappings, settings },
+    index,
+    include_type_name: true,
+  });
 }
 
 export async function deleteIndex(callCluster: CallCluster, index: string) {
@@ -251,6 +259,7 @@ export async function convertToAlias(
   await callCluster('indices.create', {
     body: { mappings: info.mappings, settings },
     index: info.indexName,
+    include_type_name: true,
   });
 
   await reindex(callCluster, alias, info.indexName, batchSize);

src/server/saved_objects/migrations/core/index_migrator.test.ts

@@ -89,6 +89,7 @@ describe('IndexMigrator', () => {
         settings: { number_of_shards: 1, auto_expand_replicas: '0-1' },
       },
       index: '.kibana_1',
+      include_type_name: true,
     });
   });
 
@@ -180,6 +181,7 @@ describe('IndexMigrator', () => {
     await new IndexMigrator(opts).migrate();
 
     sinon.assert.calledWith(callCluster, 'indices.create', {
+      include_type_name: true,
       body: {
         mappings: {
           doc: {

src/server/saved_objects/service/create_saved_objects_service.js

@@ -38,6 +38,7 @@ export function createSavedObjectsService(server, schema, serializer, migrator)
       const index = server.config().get('kibana.index');
       await adminCluster.callWithInternalUser('indices.putTemplate', {
         name: `kibana_index_template:${index}`,
+        include_type_name: true,
         body: {
           template: index,
           settings: {

src/server/saved_objects/service/lib/repository.test.js

@@ -1210,7 +1210,7 @@ describe('SavedObjectsRepository', () => {
         },
         ignore: [404],
         refresh: 'wait_for',
-        index: '.kibana-test'
+        index: '.kibana-test',
       });
 
       sinon.assert.calledOnce(onBeforeWrite);
@@ -1229,7 +1229,7 @@ describe('SavedObjectsRepository', () => {
         },
         ignore: [404],
         refresh: 'wait_for',
-        index: '.kibana-test'
+        index: '.kibana-test',
       });
 
       sinon.assert.calledOnce(onBeforeWrite);
@@ -1252,7 +1252,7 @@ describe('SavedObjectsRepository', () => {
         },
         ignore: [404],
         refresh: 'wait_for',
-        index: '.kibana-test'
+        index: '.kibana-test',
       });
 
       sinon.assert.calledOnce(onBeforeWrite);

src/ui/ui_settings/routes/__tests__/index_missing.js

@@ -20,11 +20,7 @@
 import expect from 'expect.js';
 import sinon from 'sinon';
 
-import {
-  getServices,
-  chance,
-  assertSinonMatch,
-} from './lib';
+import { getServices, chance, assertSinonMatch } from './lib';
 
 export function indexMissingSuite() {
   async function setup() {
@@ -41,17 +37,18 @@ export function indexMissingSuite() {
       // but automatically by writing to es when index didn't exist
       async assertValidKibanaIndex() {
         const resp = await callCluster('indices.get', {
-          index: indexName
+          index: indexName,
+          include_type_name: true,
         });
 
-        expect(resp[indexName].mappings).to.have.property('doc');
-        expect(resp[indexName].mappings.doc.properties).to.have.keys(
+        const mappings = resp[indexName].mappings.doc;
+        expect(mappings.properties).to.have.keys(
           'index-pattern',
           'visualization',
           'search',
           'dashboard'
         );
-      }
+      },
     };
   }
 
@@ -61,7 +58,7 @@ export function indexMissingSuite() {
 
       const { statusCode, result } = await kbnServer.inject({
         method: 'GET',
-        url: '/api/kibana/settings'
+        url: '/api/kibana/settings',
       });
 
       expect(statusCode).to.be(200);
@@ -72,9 +69,9 @@ export function indexMissingSuite() {
           },
           foo: {
             userValue: 'bar',
-            isOverridden: true
-          }
-        }
+            isOverridden: true,
+          },
+        },
       });
     });
   });
@@ -88,24 +85,24 @@ export function indexMissingSuite() {
         method: 'POST',
         url: '/api/kibana/settings/defaultIndex',
         payload: {
-          value: defaultIndex
-        }
+          value: defaultIndex,
+        },
       });
 
       expect(statusCode).to.be(200);
       assertSinonMatch(result, {
         settings: {
           buildNum: {
-            userValue: sinon.match.number
+            userValue: sinon.match.number,
           },
           defaultIndex: {
-            userValue: defaultIndex
+            userValue: defaultIndex,
           },
           foo: {
             userValue: 'bar',
-            isOverridden: true
-          }
-        }
+            isOverridden: true,
+          },
+        },
       });
 
       await assertValidKibanaIndex();
@@ -121,24 +118,24 @@ export function indexMissingSuite() {
         method: 'POST',
         url: '/api/kibana/settings',
         payload: {
-          changes: { defaultIndex }
-        }
+          changes: { defaultIndex },
+        },
       });
 
       expect(statusCode).to.be(200);
       assertSinonMatch(result, {
         settings: {
           buildNum: {
-            userValue: sinon.match.number
+            userValue: sinon.match.number,
           },
           defaultIndex: {
-            userValue: defaultIndex
+            userValue: defaultIndex,
           },
           foo: {
             userValue: 'bar',
-            isOverridden: true
-          }
-        }
+            isOverridden: true,
+          },
+        },
       });
 
       await assertValidKibanaIndex();
@@ -151,20 +148,20 @@ export function indexMissingSuite() {
 
       const { statusCode, result } = await kbnServer.inject({
         method: 'DELETE',
-        url: '/api/kibana/settings/defaultIndex'
+        url: '/api/kibana/settings/defaultIndex',
       });
 
       expect(statusCode).to.be(200);
       assertSinonMatch(result, {
         settings: {
           buildNum: {
-            userValue: sinon.match.number
+            userValue: sinon.match.number,
           },
           foo: {
             userValue: 'bar',
-            isOverridden: true
-          }
-        }
+            isOverridden: true,
+          },
+        },
       });
 
       await assertValidKibanaIndex();

test/api_integration/apis/saved_objects/migrations.js

@@ -219,6 +219,7 @@ async function createIndex({ callCluster, index }) {
   };
   await callCluster('indices.create', {
     index,
+    include_type_name: true,
     body: { mappings: { doc: { dynamic: 'strict', properties } } },
   });
 }

x-pack/plugins/beats_management/server/lib/adapters/database/kibana_database_adapter.ts

@@ -108,6 +108,7 @@ export class KibanaDatabaseAdapter implements DatabaseAdapter {
     const result = await this.callWithUser({ kind: 'internal' }, 'indices.putTemplate', {
       name,
       body: template,
+      include_type_name: true,
     });
 
     return result;

x-pack/plugins/index_lifecycle_management/server/routes/api/index/register_bootstrap_route.js

@@ -12,6 +12,7 @@ import { licensePreRoutingFactory } from '../../../lib/license_pre_routing_facto
 async function bootstrap(callWithRequest, payload) {
   await callWithRequest('indices.create', {
     index: payload.indexName,
+    include_type_name: true,
     body: {
       aliases: {
         [payload.aliasName]: {

x-pack/plugins/index_lifecycle_management/server/routes/api/templates/register_add_policy_route.js

@@ -14,7 +14,7 @@ import { licensePreRoutingFactory } from'../../../lib/license_pre_routing_factor
 import { merge } from 'lodash';
 
 async function getIndexTemplate(callWithRequest, templateName) {
-  const response = await callWithRequest('indices.getTemplate', { name: templateName });
+  const response = await callWithRequest('indices.getTemplate', { name: templateName, include_type_name: true });
   return response[templateName];
 }
 

x-pack/plugins/index_management/server/routes/api/mapping/register_mapping_route.js

@@ -20,6 +20,7 @@ function formatHit(hit, indexName) {
 async function fetchMapping(callWithRequest, indexName) {
   const params = {
     expand_wildcards: 'none',
+    include_type_name: true,
     index: indexName,
   };
 

x-pack/plugins/logstash/server/routes/api/upgrade/register_execute_route.js

@@ -25,6 +25,7 @@ async function executeUpgrade(callWithRequest) {
   return callWithRequest('indices.putMapping', {
     index: INDEX_NAMES.PIPELINES,
     type: TYPE_NAMES.PIPELINES,
+    include_type_name: true,
     body: {
       properties: {
         pipeline_settings: {

x-pack/plugins/ml/server/models/file_data_visualizer/import_data.js

@@ -93,7 +93,7 @@ export function importDataProvider(callWithRequest) {
       body.settings = settings;
     }
 
-    await callWithRequest('indices.create', { index, body });
+    await callWithRequest('indices.create', { index, body, include_type_name: true });
   }
 
   async function indexData(index, pipelineId, data) {

x-pack/plugins/reporting/server/lib/esqueue/helpers/create_index.js

@@ -88,7 +88,8 @@ export function createIndex(client, indexName,
       if (!exists) {
         return client.indices.create({
           index: indexName,
-          body: body
+          body: body,
+          include_type_name: true,
         })
           .then(() => true)
           .catch(err => {

x-pack/plugins/task_manager/task_store.test.ts

@@ -43,6 +43,7 @@ describe('TaskStore', () => {
           },
         },
         name: 'tasky',
+        include_type_name: true,
       });
     });
 

x-pack/plugins/task_manager/task_store.ts

@@ -129,6 +129,7 @@ export class TaskStore {
       // check if template exists
       const templateCheck = await this.callCluster('indices.getTemplate', {
         name: templateName,
+        include_type_name: true,
         filter_path: '*.version',
       });
       // extract the existing version

x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_actions.ts

@@ -239,7 +239,10 @@ export const reindexActionsFactory = (
     },
 
     async getBooleanFieldPaths(indexName: string) {
-      const results = await callCluster('indices.getMapping', { index: indexName });
+      const results = await callCluster('indices.getMapping', {
+        index: indexName,
+        include_type_name: true,
+      });
       const mapping = getSingleMappingType(results[indexName].mappings);
 
       // It's possible an index doesn't have a mapping.
@@ -248,8 +251,7 @@ export const reindexActionsFactory = (
 
     async getFlatSettings(indexName: string) {
       const flatSettings = (await callCluster('transport.request', {
-        // TODO: set `&include_type_name=true` to false in 7.0
-        path: `/${encodeURIComponent(indexName)}?flat_settings=true`,
+        path: `/${encodeURIComponent(indexName)}?include_type_name=true&flat_settings=true`,
       })) as { [indexName: string]: FlatSettings };
 
       if (!flatSettings[indexName]) {

x-pack/test/api_integration/apis/management/cross_cluster_replication/lib/es_index.js

@@ -15,7 +15,7 @@ export const initElasticsearchIndicesHelpers = (es) => {
 
   const createIndex = (index = getRandomString()) => {
     indicesCreated.push(index);
-    return es.indices.create({ index }).then(() => index);
+    return es.indices.create({ index, include_type_name: true }).then(() => index);
   };
 
   const deleteIndex = (index) => {

x-pack/test/api_integration/apis/monitoring/common/mappings_exist.js

@@ -48,7 +48,7 @@ export default function ({ getService }) {
       let mappings;
 
       before('load mappings', async () => {
-        const template = await es.indices.getTemplate({ name: indexTemplate });
+        const template = await es.indices.getTemplate({ name: indexTemplate, include_type_name: true });
         mappings = get(template, [indexTemplate, 'mappings', 'doc', 'properties']);
       });