[maps][file upload] remove number_of_shards index setting (#165390)

closes https://github.com/elastic/kibana/issues/165366
closes https://github.com/elastic/kibana/issues/165367
close https://github.com/elastic/kibana/issues/165697
replaces https://github.com/elastic/kibana/pull/165337

Serverless elasticsearch does not support index setting
`number_of_shards`

PR resolves issue be removing `number_of_shards`. When
`number_of_shards` was introduced way back when, the default value was
5. Now the default value is one. Therefore there is no point providing
the setting since its the same as the default. Just removing so code
works across both serverless and traditional deployments

PR also cleans up some types that are duplicative of elasticsearch types

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Tiago Costa <tiago.costa@elastic.co>

x-pack/plugins/data_visualizer/public/application/common/components/combined_fields/utils.test.ts

@@ -24,10 +24,10 @@ test('addCombinedFieldsToMappings', () => {
     },
     properties: {
       lat: {
-        type: 'number',
+        type: 'float' as 'float',
       },
       lon: {
-        type: 'number',
+        type: 'float' as 'float',
       },
     },
   };
@@ -37,10 +37,10 @@ test('addCombinedFieldsToMappings', () => {
     },
     properties: {
       lat: {
-        type: 'number',
+        type: 'float',
       },
       lon: {
-        type: 'number',
+        type: 'float',
       },
       location: {
         type: 'geo_point',
@@ -56,13 +56,13 @@ test('removeCombinedFieldsFromMappings', () => {
     },
     properties: {
       lat: {
-        type: 'number',
+        type: 'float' as 'float',
       },
       lon: {
-        type: 'number',
+        type: 'float' as 'float',
       },
       location: {
-        type: 'geo_point',
+        type: 'geo_point' as 'geo_point',
       },
     },
   };
@@ -72,10 +72,10 @@ test('removeCombinedFieldsFromMappings', () => {
     },
     properties: {
       lat: {
-        type: 'number',
+        type: 'float',
       },
       lon: {
-        type: 'number',
+        type: 'float',
       },
     },
   });

x-pack/plugins/data_visualizer/public/application/common/components/combined_fields/utils.ts

@@ -8,11 +8,8 @@
 import { i18n } from '@kbn/i18n';
 import { cloneDeep } from 'lodash';
 import { v4 as uuidv4 } from 'uuid';
-import type {
-  FindFileStructureResponse,
-  IngestPipeline,
-  Mappings,
-} from '@kbn/file-upload-plugin/common';
+import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import type { FindFileStructureResponse, IngestPipeline } from '@kbn/file-upload-plugin/common';
 import { CombinedField } from './types';
 
 const COMMON_LAT_NAMES = ['latitude', 'lat'];
@@ -28,23 +25,23 @@ export function getDefaultCombinedFields(results: FindFileStructureResponse) {
 }
 
 export function addCombinedFieldsToMappings(
-  mappings: Mappings,
+  mappings: MappingTypeMapping,
   combinedFields: CombinedField[]
-): Mappings {
-  const updatedMappings = { ...mappings };
+): MappingTypeMapping {
+  const updatedMappings = { properties: {}, ...mappings };
   combinedFields.forEach((combinedField) => {
     updatedMappings.properties[combinedField.combinedFieldName] = {
-      type: combinedField.mappingType,
+      type: combinedField.mappingType as any,
     };
   });
   return updatedMappings;
 }
 
 export function removeCombinedFieldsFromMappings(
-  mappings: Mappings,
+  mappings: MappingTypeMapping,
   combinedFields: CombinedField[]
 ) {
-  const updatedMappings = { ...mappings };
+  const updatedMappings = { properties: {}, ...mappings };
   combinedFields.forEach((combinedField) => {
     delete updatedMappings.properties[combinedField.combinedFieldName];
   });

x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/import_view.js

@@ -37,7 +37,7 @@ import {
 import { MODE as DATAVISUALIZER_MODE } from '../file_data_visualizer_view/constants';
 
 const DEFAULT_TIME_FIELD = '@timestamp';
-const DEFAULT_INDEX_SETTINGS = { number_of_shards: 1 };
+const DEFAULT_INDEX_SETTINGS = {};
 const CONFIG_MODE = { SIMPLE: 0, ADVANCED: 1 };
 
 const DEFAULT_STATE = {

x-pack/plugins/file_upload/common/index.ts

@@ -14,5 +14,4 @@ export type {
   FindFileStructureResponse,
   InputOverrides,
   IngestPipeline,
-  Mappings,
 } from './types';

x-pack/plugins/file_upload/common/types.ts

@@ -124,22 +124,6 @@ export interface ImportDocMessage {
 
 export type ImportDoc = ImportDocMessage | string | object;
 
-export interface Settings {
-  pipeline?: string;
-  index: string;
-  body: any[];
-  [key: string]: any;
-}
-
-export interface Mappings {
-  _meta?: {
-    created_by: string;
-  };
-  properties: {
-    [key: string]: any;
-  };
-}
-
 export interface IngestPipelineWrapper {
   id: string;
   pipeline: IngestPipeline;

x-pack/plugins/file_upload/public/components/geo_upload_wizard.tsx

@@ -15,7 +15,6 @@ import { ImportCompleteView } from './import_complete_view';
 import type { FileUploadComponentProps, FileUploadGeoResults } from '../lazy_load_bundle';
 import { ImportResults } from '../importer';
 import { GeoFileImporter } from '../importer/geo';
-import type { Settings } from '../../common/types';
 import { hasImportPermission } from '../api';
 import { getPartialImportMessage } from './utils';
 
@@ -103,9 +102,6 @@ export class GeoUploadWizard extends Component<FileUploadComponentProps, State>
     //
     // create index
     //
-    const settings = {
-      number_of_shards: 1,
-    } as unknown as Settings;
     const mappings = {
       properties: {
         geometry: {
@@ -127,7 +123,7 @@ export class GeoUploadWizard extends Component<FileUploadComponentProps, State>
     this._geoFileImporter.setGeoFieldType(this.state.geoFieldType);
     const initializeImportResp = await this._geoFileImporter.initializeImport(
       this.state.indexName,
-      settings,
+      {},
       mappings,
       ingestPipeline
     );

x-pack/plugins/file_upload/public/importer/importer.ts

@@ -7,18 +7,15 @@
 
 import { chunk, intersection } from 'lodash';
 import moment from 'moment';
+import type {
+  IndicesIndexSettings,
+  MappingTypeMapping,
+} from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { i18n } from '@kbn/i18n';
 import { isPopulatedObject } from '@kbn/ml-is-populated-object';
 import { getHttp } from '../kibana_services';
 import { MB } from '../../common/constants';
-import type {
-  ImportDoc,
-  ImportFailure,
-  ImportResponse,
-  Mappings,
-  Settings,
-  IngestPipeline,
-} from '../../common/types';
+import type { ImportDoc, ImportFailure, ImportResponse, IngestPipeline } from '../../common/types';
 import { CreateDocsResponse, IImporter, ImportResults } from './types';
 
 const CHUNK_SIZE = 5000;
@@ -63,8 +60,8 @@ export abstract class Importer implements IImporter {
 
   public async initializeImport(
     index: string,
-    settings: Settings,
-    mappings: Mappings,
+    settings: IndicesIndexSettings,
+    mappings: MappingTypeMapping,
     pipeline: IngestPipeline
   ) {
     updatePipelineTimezone(pipeline);
@@ -279,8 +276,8 @@ export function callImportRoute({
   id: string | undefined;
   index: string;
   data: ImportDoc[];
-  settings: Settings | unknown;
-  mappings: Mappings | unknown;
+  settings: IndicesIndexSettings;
+  mappings: MappingTypeMapping;
   ingestPipeline: {
     id?: string;
     pipeline?: IngestPipeline;

x-pack/plugins/file_upload/public/importer/types.ts

@@ -6,17 +6,15 @@
  */
 
 import type {
-  ImportFailure,
-  IngestPipeline,
-  ImportDoc,
-  ImportResponse,
-  Mappings,
-  Settings,
-} from '../../common/types';
+  IndicesIndexSettings,
+  MappingTypeMapping,
+} from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+
+import type { ImportFailure, IngestPipeline, ImportDoc, ImportResponse } from '../../common/types';
 
 export interface ImportConfig {
-  settings: Settings;
-  mappings: Mappings;
+  settings: IndicesIndexSettings;
+  mappings: MappingTypeMapping;
   pipeline: IngestPipeline;
 }
 
@@ -44,8 +42,8 @@ export interface IImporter {
   read(data: ArrayBuffer): { success: boolean };
   initializeImport(
     index: string,
-    settings: Settings,
-    mappings: Mappings,
+    settings: IndicesIndexSettings,
+    mappings: MappingTypeMapping,
     pipeline: IngestPipeline
   ): Promise<ImportResponse>;
   import(

x-pack/plugins/file_upload/server/import_data.ts

@@ -6,22 +6,21 @@
  */
 
 import { IScopedClusterClient } from '@kbn/core/server';
+import type {
+  BulkRequest,
+  IndicesCreateRequest,
+  IndicesIndexSettings,
+  MappingTypeMapping,
+} from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { INDEX_META_DATA_CREATED_BY } from '../common/constants';
-import {
-  ImportResponse,
-  ImportFailure,
-  InputData,
-  Settings,
-  Mappings,
-  IngestPipelineWrapper,
-} from '../common/types';
+import { ImportResponse, ImportFailure, InputData, IngestPipelineWrapper } from '../common/types';
 
 export function importDataProvider({ asCurrentUser }: IScopedClusterClient) {
   async function importData(
     id: string | undefined,
     index: string,
-    settings: Settings,
-    mappings: Mappings,
+    settings: IndicesIndexSettings,
+    mappings: MappingTypeMapping,
     ingestPipeline: IngestPipelineWrapper,
     data: InputData
   ): Promise<ImportResponse> {
@@ -89,8 +88,12 @@ export function importDataProvider({ asCurrentUser }: IScopedClusterClient) {
     }
   }
 
-  async function createIndex(index: string, settings: Settings, mappings: Mappings) {
-    const body: { mappings: Mappings; settings?: Settings } = {
+  async function createIndex(
+    index: string,
+    settings: IndicesIndexSettings,
+    mappings: MappingTypeMapping
+  ) {
+    const body: IndicesCreateRequest['body'] = {
       mappings: {
         _meta: {
           created_by: INDEX_META_DATA_CREATED_BY,
@@ -103,7 +106,6 @@ export function importDataProvider({ asCurrentUser }: IScopedClusterClient) {
       body.settings = settings;
     }
 
-    // @ts-expect-error settings.index is not compatible
     await asCurrentUser.indices.create({ index, body }, { maxRetries: 0 });
   }
 
@@ -115,12 +117,12 @@ export function importDataProvider({ asCurrentUser }: IScopedClusterClient) {
         body.push(data[i]);
       }
 
-      const settings: Settings = { index, body };
+      const bulkRequest: BulkRequest = { index, body };
       if (pipelineId !== undefined) {
-        settings.pipeline = pipelineId;
+        bulkRequest.pipeline = pipelineId;
       }
 
-      const resp = await asCurrentUser.bulk(settings, { maxRetries: 0 });
+      const resp = await asCurrentUser.bulk(bulkRequest, { maxRetries: 0 });
       if (resp.errors) {
         throw resp;
       } else {

x-pack/plugins/file_upload/server/routes.ts

@@ -8,8 +8,12 @@
 import { schema } from '@kbn/config-schema';
 import { IScopedClusterClient } from '@kbn/core/server';
 import { CoreSetup, Logger } from '@kbn/core/server';
+import type {
+  IndicesIndexSettings,
+  MappingTypeMapping,
+} from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { MAX_FILE_SIZE_BYTES } from '../common/constants';
-import type { IngestPipelineWrapper, InputData, Mappings, Settings } from '../common/types';
+import type { IngestPipelineWrapper, InputData } from '../common/types';
 import { wrapError } from './error_wrapper';
 import { importDataProvider } from './import_data';
 import { getTimeFieldRange } from './get_time_field_range';
@@ -29,8 +33,8 @@ function importData(
   client: IScopedClusterClient,
   id: string | undefined,
   index: string,
-  settings: Settings,
-  mappings: Mappings,
+  settings: IndicesIndexSettings,
+  mappings: MappingTypeMapping,
   ingestPipeline: IngestPipelineWrapper,
   data: InputData
 ) {

x-pack/plugins/maps/common/types.ts

@@ -17,19 +17,6 @@ export interface MatchingIndexesResp {
   error?: Error;
 }
 
-export interface IndexSourceMappings {
-  _meta?: {
-    created_by: string;
-  };
-  properties: {
-    [key: string]: any;
-  };
-}
-
-export interface BodySettings {
-  [key: string]: any;
-}
-
 export interface WriteSettings {
   index: string;
   body: object;

x-pack/plugins/maps/public/classes/layers/wizards/new_vector_layer_wizard/create_new_index_pattern.ts

@@ -5,8 +5,9 @@
  * 2.0.
  */
 
+import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { getHttp } from '../../../../kibana_services';
-import { CreateDocSourceResp, IndexSourceMappings } from '../../../../../common/types';
+import { CreateDocSourceResp } from '../../../../../common/types';
 import { INDEX_SOURCE_API_PATH } from '../../../../../common/constants';
 
 export const createNewIndexAndPattern = async ({
@@ -14,7 +15,7 @@ export const createNewIndexAndPattern = async ({
   defaultMappings = {},
 }: {
   indexName: string;
-  defaultMappings: IndexSourceMappings | {};
+  defaultMappings: MappingTypeMapping | {};
 }) => {
   return await getHttp().fetch<CreateDocSourceResp>({
     path: `/${INDEX_SOURCE_API_PATH}`,

x-pack/plugins/maps/server/data_indexing/create_doc_source.ts

@@ -5,12 +5,15 @@
  * 2.0.
  */
 
+import type {
+  IndicesCreateRequest,
+  MappingTypeMapping,
+} from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { ElasticsearchClient, IScopedClusterClient } from '@kbn/core/server';
 import { DataViewsCommonService } from '@kbn/data-plugin/server';
-import { CreateDocSourceResp, IndexSourceMappings, BodySettings } from '../../common/types';
+import { CreateDocSourceResp } from '../../common/types';
 import { MAPS_NEW_VECTOR_LAYER_META_CREATED_BY } from '../../common/constants';
 
-const DEFAULT_SETTINGS = { number_of_shards: 1 };
 const DEFAULT_META = {
   _meta: {
     created_by: MAPS_NEW_VECTOR_LAYER_META_CREATED_BY,
@@ -19,7 +22,7 @@ const DEFAULT_META = {
 
 export async function createDocSource(
   index: string,
-  mappings: IndexSourceMappings,
+  mappings: MappingTypeMapping,
   { asCurrentUser }: IScopedClusterClient,
   indexPatternsService: DataViewsCommonService
 ): Promise<CreateDocSourceResp> {
@@ -41,15 +44,14 @@ export async function createDocSource(
 
 async function createIndex(
   indexName: string,
-  mappings: IndexSourceMappings,
+  mappings: MappingTypeMapping,
   asCurrentUser: ElasticsearchClient
 ) {
-  const body: { mappings: IndexSourceMappings; settings: BodySettings } = {
+  const body: IndicesCreateRequest['body'] = {
     mappings: {
       ...DEFAULT_META,
       ...mappings,
     },
-    settings: DEFAULT_SETTINGS,
   };
 
   await asCurrentUser.indices.create({ index: indexName, body });