[7.5] [Index template] Fix editor should support mappings types (#55804) (#56279)

x-pack/legacy/plugins/index_management/public/components/mappings_editor/index.ts

@@ -0,0 +1,7 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export { doMappingsHaveType } from './lib';

x-pack/legacy/plugins/index_management/public/components/mappings_editor/lib/error_reporter.ts

@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import { ValidationError } from 'io-ts';
+import { fold } from 'fp-ts/lib/Either';
+import { Reporter } from 'io-ts/lib/Reporter';
+
+export type ReporterResult = Array<{ path: string[]; message: string }>;
+
+const failure = (validation: ValidationError[]): ReporterResult => {
+  return validation.map(e => {
+    const path: string[] = [];
+    let validationName = '';
+
+    e.context.forEach((ctx, idx) => {
+      if (ctx.key) {
+        path.push(ctx.key);
+      }
+
+      if (idx === e.context.length - 1) {
+        validationName = ctx.type.name;
+      }
+    });
+    const lastItemName = path[path.length - 1];
+    return {
+      path,
+      message:
+        'Invalid value ' +
+        JSON.stringify(e.value) +
+        ` supplied to ${lastItemName}(${validationName})`,
+    };
+  });
+};
+
+const empty: never[] = [];
+const success = () => empty;
+
+export const errorReporter: Reporter<ReporterResult> = {
+  report: fold(failure, success),
+};

x-pack/legacy/plugins/index_management/public/components/mappings_editor/lib/extract_mappings_definition.ts

@@ -0,0 +1,56 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import { isPlainObject } from 'lodash';
+
+import { GenericObject } from '../types';
+import { validateMappingsConfiguration, VALID_MAPPINGS_PARAMETERS } from './mappings_validator';
+
+interface MappingsWithType {
+  type?: string;
+  mappings: GenericObject;
+}
+
+const isMappingDefinition = (obj: GenericObject): boolean => {
+  const areAllKeysValid = Object.keys(obj).every(key => VALID_MAPPINGS_PARAMETERS.includes(key));
+
+  if (!areAllKeysValid) {
+    return false;
+  }
+
+  const { properties, dynamic_templates: dynamicTemplates, ...mappingsConfiguration } = obj;
+
+  const { errors } = validateMappingsConfiguration(mappingsConfiguration);
+  const isConfigurationValid = errors.length === 0;
+  const isPropertiesValid = properties === undefined || isPlainObject(properties);
+  const isDynamicTemplatesValid = dynamicTemplates === undefined || Array.isArray(dynamicTemplates);
+
+  // If the configuration, the properties and the dynamic templates are valid
+  // we can assume that the mapping is declared at root level (no types)
+  return isConfigurationValid && isPropertiesValid && isDynamicTemplatesValid;
+};
+
+const getMappingsDefinitionWithType = (mappings: GenericObject): MappingsWithType[] => {
+  if (isMappingDefinition(mappings)) {
+    // No need to go any further
+    return [{ mappings }];
+  }
+
+  // At this point there must be one or more type mappings
+  const typedMappings = Object.entries(mappings).reduce(
+    (acc: Array<{ type: string; mappings: GenericObject }>, [type, value]) => {
+      if (isMappingDefinition(value)) {
+        acc.push({ type, mappings: value as GenericObject });
+      }
+      return acc;
+    },
+    []
+  );
+
+  return typedMappings;
+};
+
+export const doMappingsHaveType = (mappings: GenericObject = {}): boolean =>
+  getMappingsDefinitionWithType(mappings).filter(({ type }) => type !== undefined).length > 0;

x-pack/legacy/plugins/index_management/public/components/mappings_editor/lib/index.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export * from './mappings_validator';
+
+export * from './extract_mappings_definition';

x-pack/legacy/plugins/index_management/public/components/mappings_editor/lib/mappings_validator.ts

@@ -0,0 +1,106 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import { pick } from 'lodash';
+import * as t from 'io-ts';
+import { ordString } from 'fp-ts/lib/Ord';
+import { toArray } from 'fp-ts/lib/Set';
+import { isLeft } from 'fp-ts/lib/Either';
+
+import { errorReporter } from './error_reporter';
+
+type MappingsValidationError =
+  | { code: 'ERR_CONFIG'; configName: string }
+  | { code: 'ERR_FIELD'; fieldPath: string }
+  | { code: 'ERR_PARAMETER'; paramName: string; fieldPath: string };
+
+/**
+ * Single source of truth to validate the *configuration* of the mappings.
+ * Whenever a user loads a JSON object it will be validate against this Joi schema.
+ */
+const mappingsConfigurationSchema = t.exact(
+  t.partial({
+    dynamic: t.union([t.literal(true), t.literal(false), t.literal('strict')]),
+    date_detection: t.boolean,
+    numeric_detection: t.boolean,
+    dynamic_date_formats: t.array(t.string),
+    _source: t.exact(
+      t.partial({
+        enabled: t.boolean,
+        includes: t.array(t.string),
+        excludes: t.array(t.string),
+      })
+    ),
+    _meta: t.UnknownRecord,
+    _routing: t.interface({
+      required: t.boolean,
+    }),
+  })
+);
+
+const mappingsConfigurationSchemaKeys = Object.keys(mappingsConfigurationSchema.type.props);
+const sourceConfigurationSchemaKeys = Object.keys(
+  mappingsConfigurationSchema.type.props._source.type.props
+);
+
+export const validateMappingsConfiguration = (
+  mappingsConfiguration: any
+): { value: any; errors: MappingsValidationError[] } => {
+  // Set to keep track of invalid configuration parameters.
+  const configurationRemoved: Set<string> = new Set();
+
+  let copyOfMappingsConfig = { ...mappingsConfiguration };
+  const result = mappingsConfigurationSchema.decode(mappingsConfiguration);
+  const isSchemaInvalid = isLeft(result);
+
+  const unknownConfigurationParameters = Object.keys(mappingsConfiguration).filter(
+    key => mappingsConfigurationSchemaKeys.includes(key) === false
+  );
+
+  const unknownSourceConfigurationParameters =
+    mappingsConfiguration._source !== undefined
+      ? Object.keys(mappingsConfiguration._source).filter(
+          key => sourceConfigurationSchemaKeys.includes(key) === false
+        )
+      : [];
+
+  if (isSchemaInvalid) {
+    /**
+     * To keep the logic simple we will strip out the parameters that contain errors
+     */
+    const errors = errorReporter.report(result);
+    errors.forEach(error => {
+      const configurationName = error.path[0];
+      configurationRemoved.add(configurationName);
+      delete copyOfMappingsConfig[configurationName];
+    });
+  }
+
+  if (unknownConfigurationParameters.length > 0) {
+    unknownConfigurationParameters.forEach(configName => configurationRemoved.add(configName));
+  }
+
+  if (unknownSourceConfigurationParameters.length > 0) {
+    configurationRemoved.add('_source');
+    delete copyOfMappingsConfig._source;
+  }
+
+  copyOfMappingsConfig = pick(copyOfMappingsConfig, mappingsConfigurationSchemaKeys);
+
+  const errors: MappingsValidationError[] = toArray<string>(ordString)(configurationRemoved)
+    .map(configName => ({
+      code: 'ERR_CONFIG',
+      configName,
+    }))
+    .sort((a, b) => a.configName.localeCompare(b.configName)) as MappingsValidationError[];
+
+  return { value: copyOfMappingsConfig, errors };
+};
+
+export const VALID_MAPPINGS_PARAMETERS = [
+  ...mappingsConfigurationSchemaKeys,
+  'dynamic_templates',
+  'properties',
+];

x-pack/legacy/plugins/index_management/public/components/mappings_editor/types.ts

@@ -0,0 +1,142 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import { ReactNode } from 'react';
+
+export interface DataTypeDefinition {
+  label: string;
+  value: DataType;
+  documentation?: {
+    main: string;
+    [key: string]: string;
+  };
+  subTypes?: { label: string; types: SubType[] };
+  description?: () => ReactNode;
+}
+
+export type MainType =
+  | 'text'
+  | 'keyword'
+  | 'numeric'
+  | 'binary'
+  | 'boolean'
+  | 'range'
+  | 'object'
+  | 'nested'
+  | 'alias'
+  | 'completion'
+  | 'dense_vector'
+  | 'flattened'
+  | 'ip'
+  | 'join'
+  | 'percolator'
+  | 'rank_feature'
+  | 'rank_features'
+  | 'shape'
+  | 'search_as_you_type'
+  | 'date'
+  | 'date_nanos'
+  | 'geo_point'
+  | 'geo_shape'
+  | 'token_count';
+
+export type SubType = NumericType | RangeType;
+
+export type DataType = MainType | SubType;
+
+export type NumericType =
+  | 'long'
+  | 'integer'
+  | 'short'
+  | 'byte'
+  | 'double'
+  | 'float'
+  | 'half_float'
+  | 'scaled_float';
+
+export type RangeType =
+  | 'integer_range'
+  | 'float_range'
+  | 'long_range'
+  | 'ip_range'
+  | 'double_range'
+  | 'date_range';
+
+export type ParameterName =
+  | 'name'
+  | 'type'
+  | 'store'
+  | 'index'
+  | 'fielddata'
+  | 'fielddata_frequency_filter'
+  | 'fielddata_frequency_filter_percentage'
+  | 'fielddata_frequency_filter_absolute'
+  | 'doc_values'
+  | 'doc_values_binary'
+  | 'coerce'
+  | 'coerce_shape'
+  | 'ignore_malformed'
+  | 'null_value'
+  | 'null_value_numeric'
+  | 'null_value_boolean'
+  | 'null_value_geo_point'
+  | 'null_value_ip'
+  | 'copy_to'
+  | 'dynamic'
+  | 'dynamic_toggle'
+  | 'dynamic_strict'
+  | 'enabled'
+  | 'boost'
+  | 'locale'
+  | 'format'
+  | 'analyzer'
+  | 'search_analyzer'
+  | 'search_quote_analyzer'
+  | 'index_options'
+  | 'index_options_flattened'
+  | 'index_options_keyword'
+  | 'eager_global_ordinals'
+  | 'eager_global_ordinals_join'
+  | 'index_prefixes'
+  | 'index_phrases'
+  | 'norms'
+  | 'norms_keyword'
+  | 'term_vector'
+  | 'position_increment_gap'
+  | 'similarity'
+  | 'normalizer'
+  | 'ignore_above'
+  | 'split_queries_on_whitespace'
+  | 'scaling_factor'
+  | 'max_input_length'
+  | 'preserve_separators'
+  | 'preserve_position_increments'
+  | 'ignore_z_value'
+  | 'enable_position_increments'
+  | 'orientation'
+  | 'points_only'
+  | 'path'
+  | 'dims'
+  | 'depth_limit'
+  | 'relations'
+  | 'max_shingle_size';
+
+interface FieldBasic {
+  name: string;
+  type: DataType;
+  subType?: SubType;
+  properties?: { [key: string]: Omit<Field, 'name'> };
+  fields?: { [key: string]: Omit<Field, 'name'> };
+}
+
+type FieldParams = {
+  [K in ParameterName]: unknown;
+};
+
+export type Field = FieldBasic & Partial<FieldParams>;
+
+export interface GenericObject {
+  [key: string]: any;
+}

x-pack/legacy/plugins/index_management/public/services/api.ts

@@ -38,6 +38,7 @@ import { TAB_SETTINGS, TAB_MAPPING, TAB_STATS } from '../constants';
 import { trackUiMetric, METRIC_TYPE } from './track_ui_metric';
 import { useRequest, sendRequest } from './use_request';
 import { Template } from '../../common/types';
+import { doMappingsHaveType } from '../components/mappings_editor';
 
 let httpClient: ng.IHttpService;
 
@@ -225,8 +226,9 @@ export function loadIndexTemplate(name: Template['name']) {
 }
 
 export async function saveTemplate(template: Template, isClone?: boolean) {
+  const includeTypeName = doMappingsHaveType(template.mappings);
   const result = sendRequest({
-    path: `${apiPrefix}/templates`,
+    path: `${apiPrefix}/templates?include_type_name=${includeTypeName}`,
     method: 'put',
     body: template,
   });
@@ -239,9 +241,10 @@ export async function saveTemplate(template: Template, isClone?: boolean) {
 }
 
 export async function updateTemplate(template: Template) {
+  const includeTypeName = doMappingsHaveType(template.mappings);
   const { name } = template;
   const result = sendRequest({
-    path: `${apiPrefix}/templates/${encodeURIComponent(name)}`,
+    path: `${apiPrefix}/templates/${encodeURIComponent(name)}?include_type_name=${includeTypeName}`,
     method: 'put',
     body: template,
   });

x-pack/legacy/plugins/index_management/server/routes/api/mapping/register_mapping_route.js

@@ -15,6 +15,7 @@ const handler = async (request, callWithRequest) => {
   const params = {
     expand_wildcards: 'none',
     index: indexName,
+    include_type_name: true,
   };
 
   const hit = await callWithRequest('indices.getMapping', params);

x-pack/legacy/plugins/index_management/server/routes/api/templates/register_create_route.ts

@@ -15,6 +15,7 @@ import { serializeTemplate } from '../../../../common/lib';
 
 const handler: RouterRouteHandler = async (req, callWithRequest) => {
   const template = req.payload as Template;
+  const { include_type_name } = req.query as any;
   const serializedTemplate = serializeTemplate(template) as TemplateEs;
 
   const { name, order, index_patterns, version, settings, mappings, aliases } = serializedTemplate;
@@ -49,6 +50,7 @@ const handler: RouterRouteHandler = async (req, callWithRequest) => {
   return await callWithRequest('indices.putTemplate', {
     name,
     order,
+    include_type_name,
     body: {
       index_patterns,
       version,

x-pack/legacy/plugins/index_management/server/routes/api/templates/register_get_routes.ts

@@ -13,7 +13,9 @@ let callWithInternalUser: any;
 const allHandler: RouterRouteHandler = async (_req, callWithRequest) => {
   const managedTemplatePrefix = await getManagedTemplatePrefix(callWithInternalUser);
 
-  const indexTemplatesByName = await callWithRequest('indices.getTemplate');
+  const indexTemplatesByName = await callWithRequest('indices.getTemplate', {
+    include_type_name: true,
+  });
 
   return deserializeTemplateList(indexTemplatesByName, managedTemplatePrefix);
 };
@@ -21,7 +23,10 @@ const allHandler: RouterRouteHandler = async (_req, callWithRequest) => {
 const oneHandler: RouterRouteHandler = async (req, callWithRequest) => {
   const { name } = req.params;
   const managedTemplatePrefix = await getManagedTemplatePrefix(callWithInternalUser);
-  const indexTemplateByName = await callWithRequest('indices.getTemplate', { name });
+  const indexTemplateByName = await callWithRequest('indices.getTemplate', {
+    name,
+    include_type_name: true,
+  });
 
   if (indexTemplateByName[name]) {
     return deserializeTemplate({ ...indexTemplateByName[name], name }, managedTemplatePrefix);

x-pack/legacy/plugins/index_management/server/routes/api/templates/register_update_route.ts

@@ -10,6 +10,7 @@ import { serializeTemplate } from '../../../../common/lib';
 
 const handler: RouterRouteHandler = async (req, callWithRequest) => {
   const { name } = req.params;
+  const { include_type_name } = req.query as any;
   const template = req.payload as Template;
   const serializedTemplate = serializeTemplate(template) as TemplateEs;
 
@@ -22,6 +23,7 @@ const handler: RouterRouteHandler = async (req, callWithRequest) => {
   return await callWithRequest('indices.putTemplate', {
     name,
     order,
+    include_type_name,
     body: {
       index_patterns,
       version,

x-pack/test/api_integration/apis/management/index_management/mapping.js

@@ -32,7 +32,8 @@ export default function({ getService }) {
 
       const { body } = await getIndexMapping(index).expect(200);
 
-      expect(body.mapping).to.eql(mappings);
+      // As, on 7.x we require the mappings with type (include_type_name), the default "_doc" type is returned
+      expect(body.mapping).to.eql({ _doc: mappings });
     });
   });
 }