[Security Assistant] Switch to use default inference endpoint (#208668)

## Summary

In 8.17 we have introduced `semantic_text`
https://github.com/elastic/kibana/pull/197007 which required dedicated
inference endpoint.
As we now have default `.elser-2-elasticsearch` inference endpoint
available we want to migrate it out, but it's not possible to just
override `inference_id` mapping for the Knowledge Base data stream, so
instead we decided to first update the mapping by adding
`search_inference_id` pointing to the `.elser-2-elasticsearch` (to make
sure the data is queryable without the dedicated endpoint). Then we
update the Data Stream mapping to use the default endpoint and after
that we rollover the DS index to make sure new index is created and new
inference endpoint is used for new Knowledge Base data ingestion.

Will add testing steps soon

x-pack/solutions/security/packages/data-stream-adapter/src/create_or_update_data_stream.ts

@@ -16,6 +16,7 @@ interface UpdateIndexMappingsOpts {
   esClient: ElasticsearchClient;
   indexNames: string[];
   totalFieldsLimit: number;
+  writeIndexOnly?: boolean;
 }
 
 interface UpdateIndexOpts {
@@ -23,6 +24,7 @@ interface UpdateIndexOpts {
   esClient: ElasticsearchClient;
   indexName: string;
   totalFieldsLimit: number;
+  writeIndexOnly?: boolean;
 }
 
 const updateTotalFieldLimitSetting = async ({
@@ -50,7 +52,7 @@ const updateTotalFieldLimitSetting = async ({
 // is due to the fact settings can be classed as dynamic and static, and static
 // updates will fail on an index that isn't closed. New settings *will* be applied as part
 // of the ILM policy rollovers. More info: https://github.com/elastic/kibana/pull/113389#issuecomment-940152654
-const updateMapping = async ({ logger, esClient, indexName }: UpdateIndexOpts) => {
+const updateMapping = async ({ logger, esClient, indexName, writeIndexOnly }: UpdateIndexOpts) => {
   logger.debug(`Updating mappings for ${indexName} data stream.`);
 
   let simulatedIndexMapping: IndicesSimulateIndexTemplateResponse;
@@ -75,7 +77,12 @@ const updateMapping = async ({ logger, esClient, indexName }: UpdateIndexOpts) =
 
   try {
     await retryTransientEsErrors(
-      () => esClient.indices.putMapping({ index: indexName, body: simulatedMapping }),
+      () =>
+        esClient.indices.putMapping({
+          index: indexName,
+          body: simulatedMapping,
+          write_index_only: writeIndexOnly,
+        }),
       { logger }
     );
   } catch (err) {
@@ -91,6 +98,7 @@ const updateDataStreamMappings = async ({
   esClient,
   totalFieldsLimit,
   indexNames,
+  writeIndexOnly,
 }: UpdateIndexMappingsOpts) => {
   // Update total field limit setting of found indices
   // Other index setting changes are not updated at this time
@@ -101,7 +109,9 @@ const updateDataStreamMappings = async ({
   );
   // Update mappings of the found indices.
   await Promise.all(
-    indexNames.map((indexName) => updateMapping({ logger, esClient, totalFieldsLimit, indexName }))
+    indexNames.map((indexName) =>
+      updateMapping({ logger, esClient, totalFieldsLimit, indexName, writeIndexOnly })
+    )
   );
 };
 
@@ -110,6 +120,7 @@ export interface CreateOrUpdateDataStreamParams {
   logger: Logger;
   esClient: ElasticsearchClient;
   totalFieldsLimit: number;
+  writeIndexOnly?: boolean;
 }
 
 export async function createOrUpdateDataStream({
@@ -117,6 +128,7 @@ export async function createOrUpdateDataStream({
   esClient,
   name,
   totalFieldsLimit,
+  writeIndexOnly,
 }: CreateOrUpdateDataStreamParams): Promise<void> {
   logger.info(`Creating data stream - ${name}`);
 
@@ -142,6 +154,7 @@ export async function createOrUpdateDataStream({
       esClient,
       indexNames: [name],
       totalFieldsLimit,
+      writeIndexOnly,
     });
   } else {
     try {
@@ -204,6 +217,7 @@ export interface CreateOrUpdateSpacesDataStreamParams {
   logger: Logger;
   esClient: ElasticsearchClient;
   totalFieldsLimit: number;
+  writeIndexOnly?: boolean;
 }
 
 export async function updateDataStreams({
@@ -211,6 +225,7 @@ export async function updateDataStreams({
   esClient,
   name,
   totalFieldsLimit,
+  writeIndexOnly,
 }: CreateOrUpdateSpacesDataStreamParams): Promise<void> {
   logger.info(`Updating data streams - ${name}`);
 
@@ -234,6 +249,7 @@ export async function updateDataStreams({
       esClient,
       totalFieldsLimit,
       indexNames: dataStreams.map((dataStream) => dataStream.name),
+      writeIndexOnly,
     });
   }
 }

x-pack/solutions/security/packages/data-stream-adapter/src/data_stream_adapter.ts

@@ -30,6 +30,7 @@ export class DataStreamAdapter extends IndexAdapter {
         esClient,
         logger,
         totalFieldsLimit: this.totalFieldsLimit,
+        writeIndexOnly: this.writeIndexOnly,
       }),
       `${this.name} data stream`
     );

x-pack/solutions/security/packages/data-stream-adapter/src/data_stream_spaces_adapter.ts

@@ -33,6 +33,7 @@ export class DataStreamSpacesAdapter extends IndexPatternAdapter {
         esClient,
         logger,
         totalFieldsLimit: this.totalFieldsLimit,
+        writeIndexOnly: this.writeIndexOnly,
       }),
       `update space data streams`
     );

x-pack/solutions/security/packages/index-adapter/src/create_or_update_index.ts

@@ -16,6 +16,7 @@ interface UpdateIndexMappingsOpts {
   esClient: ElasticsearchClient;
   indexNames: string[];
   totalFieldsLimit: number;
+  writeIndexOnly?: boolean;
 }
 
 interface UpdateIndexOpts {
@@ -23,6 +24,7 @@ interface UpdateIndexOpts {
   esClient: ElasticsearchClient;
   indexName: string;
   totalFieldsLimit: number;
+  writeIndexOnly?: boolean;
 }
 
 const updateTotalFieldLimitSetting = async ({
@@ -50,7 +52,7 @@ const updateTotalFieldLimitSetting = async ({
 // is due to the fact settings can be classed as dynamic and static, and static
 // updates will fail on an index that isn't closed. New settings *will* be applied as part
 // of the ILM policy rollovers. More info: https://github.com/elastic/kibana/pull/113389#issuecomment-940152654
-const updateMapping = async ({ logger, esClient, indexName }: UpdateIndexOpts) => {
+const updateMapping = async ({ logger, esClient, indexName, writeIndexOnly }: UpdateIndexOpts) => {
   logger.debug(`Updating mappings for ${indexName} data stream.`);
 
   let simulatedIndexMapping: IndicesSimulateIndexTemplateResponse;
@@ -75,7 +77,12 @@ const updateMapping = async ({ logger, esClient, indexName }: UpdateIndexOpts) =
 
   try {
     await retryTransientEsErrors(
-      () => esClient.indices.putMapping({ index: indexName, body: simulatedMapping }),
+      () =>
+        esClient.indices.putMapping({
+          index: indexName,
+          body: simulatedMapping,
+          write_index_only: writeIndexOnly,
+        }),
       { logger }
     );
   } catch (err) {
@@ -91,6 +98,7 @@ const updateIndexMappings = async ({
   esClient,
   totalFieldsLimit,
   indexNames,
+  writeIndexOnly,
 }: UpdateIndexMappingsOpts) => {
   // Update total field limit setting of found indices
   // Other index setting changes are not updated at this time
@@ -101,7 +109,9 @@ const updateIndexMappings = async ({
   );
   // Update mappings of the found indices.
   await Promise.all(
-    indexNames.map((indexName) => updateMapping({ logger, esClient, totalFieldsLimit, indexName }))
+    indexNames.map((indexName) =>
+      updateMapping({ logger, esClient, totalFieldsLimit, indexName, writeIndexOnly })
+    )
   );
 };
 
@@ -200,6 +210,7 @@ export interface CreateOrUpdateSpacesIndexParams {
   logger: Logger;
   esClient: ElasticsearchClient;
   totalFieldsLimit: number;
+  writeIndexOnly?: boolean;
 }
 
 export async function updateIndices({
@@ -207,6 +218,7 @@ export async function updateIndices({
   esClient,
   name,
   totalFieldsLimit,
+  writeIndexOnly,
 }: CreateOrUpdateSpacesIndexParams): Promise<void> {
   logger.info(`Updating indices - ${name}`);
 
@@ -230,6 +242,7 @@ export async function updateIndices({
       esClient,
       totalFieldsLimit,
       indexNames: indices,
+      writeIndexOnly,
     });
   }
 }

x-pack/solutions/security/packages/index-adapter/src/field_maps/types.ts

@@ -54,6 +54,7 @@ export type FieldMap<T extends string = string> = Record<
     dynamic?: boolean | 'strict';
     properties?: Record<string, { type: string }>;
     inference_id?: string;
+    search_inference_id?: string;
     copy_to?: string;
   }
 >;

x-pack/solutions/security/packages/index-adapter/src/index_adapter.ts

@@ -25,6 +25,7 @@ import {
 export interface IndexAdapterParams {
   kibanaVersion: string;
   totalFieldsLimit?: number;
+  writeIndexOnly?: boolean;
 }
 export type SetComponentTemplateParams = GetComponentTemplateOpts;
 export type SetIndexTemplateParams = Omit<
@@ -51,11 +52,13 @@ export class IndexAdapter {
   protected componentTemplates: ClusterPutComponentTemplateRequest[] = [];
   protected indexTemplates: IndicesPutIndexTemplateRequest[] = [];
   protected installed: boolean;
+  protected writeIndexOnly: boolean;
 
-  constructor(protected readonly name: string, options: IndexAdapterParams) {
+  constructor(public readonly name: string, options: IndexAdapterParams) {
     this.installed = false;
     this.kibanaVersion = options.kibanaVersion;
     this.totalFieldsLimit = options.totalFieldsLimit ?? DEFAULT_FIELDS_LIMIT;
+    this.writeIndexOnly = options.writeIndexOnly ?? false;
   }
 
   public setComponentTemplate(params: SetComponentTemplateParams) {
@@ -98,7 +101,7 @@ export class IndexAdapter {
     };
   }
 
-  protected async installTemplates(params: InstallParams) {
+  public async installTemplates(params: InstallParams) {
     const { logger, pluginStop$, tasksTimeoutMs } = params;
     const esClient = await params.esClient;
     const installFn = this.getInstallFn({ logger, pluginStop$, tasksTimeoutMs });

x-pack/solutions/security/packages/index-adapter/src/index_pattern_adapter.ts

@@ -40,6 +40,7 @@ export class IndexPatternAdapter extends IndexAdapter {
         esClient,
         logger,
         totalFieldsLimit: this.totalFieldsLimit,
+        writeIndexOnly: this.writeIndexOnly,
       }),
       `update specific indices`
     );

x-pack/solutions/security/plugins/elastic_assistant/server/__mocks__/data_clients.mock.ts

@@ -69,6 +69,7 @@ const createKnowledgeBaseDataClientMock = () => {
     getRequiredKnowledgeBaseDocumentEntries: jest.fn(),
     getWriter: jest.fn().mockResolvedValue({ bulk: jest.fn() }),
     isInferenceEndpointExists: jest.fn(),
+    getInferenceEndpointId: jest.fn(),
     isModelInstalled: jest.fn(),
     isSecurityLabsDocsLoaded: jest.fn(),
     isSetupAvailable: jest.fn(),

x-pack/solutions/security/plugins/elastic_assistant/server/ai_assistant_data_clients/knowledge_base/field_maps_configuration.ts

@@ -7,6 +7,7 @@
 import { FieldMap } from '@kbn/data-stream-adapter';
 
 export const ASSISTANT_ELSER_INFERENCE_ID = 'elastic-security-ai-assistant-elser2';
+export const ELASTICSEARCH_ELSER_INFERENCE_ID = '.elser-2-elasticsearch';
 
 export const knowledgeBaseFieldMap: FieldMap = {
   // Base fields

x-pack/solutions/security/plugins/elastic_assistant/server/ai_assistant_data_clients/knowledge_base/index.test.ts

@@ -73,6 +73,7 @@ describe('AIAssistantKnowledgeBaseDataClient', () => {
       ingestPipelineResourceName: 'something',
       setIsKBSetupInProgress: jest.fn().mockImplementation(() => {}),
       manageGlobalKnowledgeBaseAIAssistant: true,
+      assistantDefaultInferenceEndpoint: false,
     };
     esClientMock.search.mockReturnValue(
       // @ts-expect-error not full response interface

x-pack/solutions/security/plugins/elastic_assistant/server/ai_assistant_data_clients/knowledge_base/index.ts

@@ -59,7 +59,10 @@ import {
   loadSecurityLabs,
   getSecurityLabsDocsCount,
 } from '../../lib/langchain/content_loaders/security_labs_loader';
-import { ASSISTANT_ELSER_INFERENCE_ID } from './field_maps_configuration';
+import {
+  ASSISTANT_ELSER_INFERENCE_ID,
+  ELASTICSEARCH_ELSER_INFERENCE_ID,
+} from './field_maps_configuration';
 import { BulkOperationError } from '../../lib/data_stream/documents_data_writer';
 import { AUDIT_OUTCOME, KnowledgeBaseAuditAction, knowledgeBaseAuditEvent } from './audit_events';
 
@@ -79,6 +82,7 @@ export interface KnowledgeBaseDataClientParams extends AIAssistantDataClientPara
   ingestPipelineResourceName: string;
   setIsKBSetupInProgress: (isInProgress: boolean) => void;
   manageGlobalKnowledgeBaseAIAssistant: boolean;
+  assistantDefaultInferenceEndpoint: boolean;
 }
 export class AIAssistantKnowledgeBaseDataClient extends AIAssistantDataClient {
   constructor(public readonly options: KnowledgeBaseDataClientParams) {
@@ -150,17 +154,44 @@ export class AIAssistantKnowledgeBaseDataClient extends AIAssistantDataClient {
     }
   };
 
+  public getInferenceEndpointId = async () => {
+    if (!this.options.assistantDefaultInferenceEndpoint) {
+      return ASSISTANT_ELSER_INFERENCE_ID;
+    }
+    const esClient = await this.options.elasticsearchClientPromise;
+
+    try {
+      const elasticsearchInference = await esClient.inference.get({
+        inference_id: ASSISTANT_ELSER_INFERENCE_ID,
+        task_type: 'sparse_embedding',
+      });
+
+      if (elasticsearchInference) {
+        return ASSISTANT_ELSER_INFERENCE_ID;
+      }
+    } catch (error) {
+      this.options.logger.debug(
+        `Error checking if Inference endpoint ${ASSISTANT_ELSER_INFERENCE_ID} exists: ${error}`
+      );
+    }
+
+    // Fallback to the dedicated inference endpoint
+    return ELASTICSEARCH_ELSER_INFERENCE_ID;
+  };
+
   /**
    * Checks if the inference endpoint is deployed and allocated in Elasticsearch
    *
    * @returns Promise<boolean> indicating whether the model is deployed
    */
-  public isInferenceEndpointExists = async (): Promise<boolean> => {
+  public isInferenceEndpointExists = async (inferenceEndpointId?: string): Promise<boolean> => {
+    const inferenceId = inferenceEndpointId || (await this.getInferenceEndpointId());
+
     try {
       const esClient = await this.options.elasticsearchClientPromise;
 
       const inferenceExists = !!(await esClient.inference.get({
-        inference_id: ASSISTANT_ELSER_INFERENCE_ID,
+        inference_id: inferenceId,
         task_type: 'sparse_embedding',
       }));
       if (!inferenceExists) {
@@ -181,7 +212,7 @@ export class AIAssistantKnowledgeBaseDataClient extends AIAssistantDataClient {
           (node) => node.routing_state.routing_state === 'started'
         );
 
-      return getResponse.trained_model_stats?.some(
+      return !!getResponse.trained_model_stats?.some(
         (stats) => isReadyESS(stats) || isReadyServerless(stats)
       );
     } catch (error) {
@@ -196,46 +227,57 @@ export class AIAssistantKnowledgeBaseDataClient extends AIAssistantDataClient {
     const elserId = await this.options.getElserId();
     this.options.logger.debug(`Deploying ELSER model '${elserId}'...`);
     const esClient = await this.options.elasticsearchClientPromise;
+    const inferenceId = await this.getInferenceEndpointId();
+    const inferenceExists = await this.isInferenceEndpointExists(inferenceId);
 
-    try {
-      await esClient.inference.delete({
-        inference_id: ASSISTANT_ELSER_INFERENCE_ID,
-        // it's being used in the mapping so we need to force delete
-        force: true,
-      });
-      this.options.logger.debug(`Deleted existing inference endpoint for ELSER model '${elserId}'`);
-    } catch (error) {
-      this.options.logger.error(
-        `Error deleting inference endpoint for ELSER model '${elserId}':\n${error}`
-      );
-    }
+    // Don't try to create the inference endpoint for ELASTICSEARCH_ELSER_INFERENCE_ID
+    if (inferenceId === ASSISTANT_ELSER_INFERENCE_ID) {
+      if (inferenceExists) {
+        try {
+          await esClient.inference.delete({
+            inference_id: ASSISTANT_ELSER_INFERENCE_ID,
+            // it's being used in the mapping so we need to force delete
+            force: true,
+          });
+          this.options.logger.debug(
+            `Deleted existing inference endpoint for ELSER model '${elserId}'`
+          );
+        } catch (error) {
+          this.options.logger.error(
+            `Error deleting inference endpoint for ELSER model '${elserId}':\n${error}`
+          );
+        }
+      }
 
-    try {
-      await esClient.inference.put({
-        task_type: 'sparse_embedding',
-        inference_id: ASSISTANT_ELSER_INFERENCE_ID,
-        inference_config: {
-          service: 'elasticsearch',
-          service_settings: {
-            adaptive_allocations: {
-              enabled: true,
-              min_number_of_allocations: 0,
-              max_number_of_allocations: 8,
+      try {
+        await esClient.inference.put({
+          task_type: 'sparse_embedding',
+          inference_id: ASSISTANT_ELSER_INFERENCE_ID,
+          inference_config: {
+            service: 'elasticsearch',
+            service_settings: {
+              adaptive_allocations: {
+                enabled: true,
+                min_number_of_allocations: 0,
+                max_number_of_allocations: 8,
+              },
+              num_threads: 1,
+              model_id: elserId,
             },
-            num_threads: 1,
-            model_id: elserId,
+            task_settings: {},
           },
-          task_settings: {},
-        },
-      });
+        });
 
-      // await for the model to be deployed
-      await this.isInferenceEndpointExists();
-    } catch (error) {
-      this.options.logger.error(
-        `Error creating inference endpoint for ELSER model '${elserId}':\n${error}`
-      );
-      throw new Error(`Error creating inference endpoint for ELSER model '${elserId}':\n${error}`);
+        // await for the model to be deployed
+        await this.isInferenceEndpointExists(inferenceId);
+      } catch (error) {
+        this.options.logger.error(
+          `Error creating inference endpoint for ELSER model '${elserId}':\n${error}`
+        );
+        throw new Error(
+          `Error creating inference endpoint for ELSER model '${elserId}':\n${error}`
+        );
+      }
     }
   };
 

x-pack/solutions/security/plugins/elastic_assistant/server/ai_assistant_service/index.ts

@@ -13,6 +13,11 @@ import type { MlPluginSetup } from '@kbn/ml-plugin/server';
 import { Subject } from 'rxjs';
 import { LicensingApiRequestHandlerContext } from '@kbn/licensing-plugin/server';
 import { ProductDocBaseStartContract } from '@kbn/product-doc-base-plugin/server';
+import {
+  IndicesGetFieldMappingResponse,
+  IndicesIndexSettings,
+} from '@elastic/elasticsearch/lib/api/types';
+import { omit } from 'lodash';
 import { attackDiscoveryFieldMap } from '../lib/attack_discovery/persistence/field_maps_configuration/field_maps_configuration';
 import { defendInsightsFieldMap } from '../ai_assistant_data_clients/defend_insights/field_maps_configuration';
 import { getDefaultAnonymizationFields } from '../../common/anonymization';
@@ -35,19 +40,18 @@ import {
 import { assistantPromptsFieldMap } from '../ai_assistant_data_clients/prompts/field_maps_configuration';
 import { assistantAnonymizationFieldsFieldMap } from '../ai_assistant_data_clients/anonymization_fields/field_maps_configuration';
 import { AIAssistantDataClient } from '../ai_assistant_data_clients';
-import { knowledgeBaseFieldMap } from '../ai_assistant_data_clients/knowledge_base/field_maps_configuration';
+import {
+  ASSISTANT_ELSER_INFERENCE_ID,
+  ELASTICSEARCH_ELSER_INFERENCE_ID,
+  knowledgeBaseFieldMap,
+} from '../ai_assistant_data_clients/knowledge_base/field_maps_configuration';
 import {
   AIAssistantKnowledgeBaseDataClient,
   GetAIAssistantKnowledgeBaseDataClientParams,
 } from '../ai_assistant_data_clients/knowledge_base';
 import { AttackDiscoveryDataClient } from '../lib/attack_discovery/persistence';
 import { DefendInsightsDataClient } from '../ai_assistant_data_clients/defend_insights';
-import {
-  createGetElserId,
-  createPipeline,
-  ensureProductDocumentationInstalled,
-  pipelineExists,
-} from './helpers';
+import { createGetElserId, ensureProductDocumentationInstalled } from './helpers';
 import { hasAIAssistantLicense } from '../routes/helpers';
 
 const TOTAL_FIELDS_LIMIT = 2500;
@@ -84,6 +88,8 @@ export type CreateDataStream = (params: {
   fieldMap: FieldMap;
   kibanaVersion: string;
   spaceId?: string;
+  settings?: IndicesIndexSettings;
+  writeIndexOnly?: boolean;
 }) => DataStreamSpacesAdapter;
 
 export class AIAssistantService {
@@ -104,6 +110,8 @@ export class AIAssistantService {
   // Temporary 'feature flag' to determine if we should initialize the new message metadata mappings, toggled when citations should be enabled.
   private contentReferencesEnabled: boolean = false;
   private hasInitializedContentReferences: boolean = false;
+  // Temporary 'feature flag' to determine if we should initialize the new knowledge base mappings
+  private assistantDefaultInferenceEndpoint: boolean = false;
 
   constructor(private readonly options: AIAssistantServiceOpts) {
     this.initialized = false;
@@ -167,15 +175,23 @@ export class AIAssistantService {
     this.isKBSetupInProgress = isInProgress;
   }
 
-  private createDataStream: CreateDataStream = ({ resource, kibanaVersion, fieldMap }) => {
+  private createDataStream: CreateDataStream = ({
+    resource,
+    kibanaVersion,
+    fieldMap,
+    settings,
+    writeIndexOnly,
+  }) => {
     const newDataStream = new DataStreamSpacesAdapter(this.resourceNames.aliases[resource], {
       kibanaVersion,
       totalFieldsLimit: TOTAL_FIELDS_LIMIT,
+      writeIndexOnly,
     });
 
     newDataStream.setComponentTemplate({
       name: this.resourceNames.componentTemplate[resource],
       fieldMap,
+      settings,
     });
 
     newDataStream.setIndexTemplate({
@@ -229,29 +245,124 @@ export class AIAssistantService {
         pluginStop$: this.options.pluginStop$,
       });
 
-      await this.knowledgeBaseDataStream.install({
-        esClient,
-        logger: this.options.logger,
-        pluginStop$: this.options.pluginStop$,
-      });
+      if (this.assistantDefaultInferenceEndpoint) {
+        const knowledgeBaseDataStreamExists = (
+          await esClient.indices.getDataStream({
+            name: this.knowledgeBaseDataStream.name,
+          })
+        )?.data_streams?.length;
 
-      // Note: Pipeline creation can be removed in favor of semantic_text
-      const pipelineCreated = await pipelineExists({
-        esClient,
-        id: this.resourceNames.pipelines.knowledgeBase,
-      });
-      // ensure pipeline is re-created for those upgrading
-      // pipeline is noop now, so if one does not exist we do not need one
-      if (pipelineCreated) {
-        this.options.logger.debug(
-          `Installing ingest pipeline - ${this.resourceNames.pipelines.knowledgeBase}`
-        );
-        const response = await createPipeline({
+        // update component template for semantic_text field
+        // rollover
+        let mappings: IndicesGetFieldMappingResponse = {};
+        try {
+          mappings = await esClient.indices.getFieldMapping({
+            index: '.kibana-elastic-ai-assistant-knowledge-base-default',
+            fields: ['semantic_text'],
+          });
+        } catch (error) {
+          /* empty */
+        }
+
+        const isUsingDedicatedInferenceEndpoint =
+          (
+            Object.values(mappings)[0]?.mappings?.semantic_text?.mapping?.semantic_text as {
+              inference_id: string;
+            }
+          )?.inference_id === ASSISTANT_ELSER_INFERENCE_ID;
+
+        if (knowledgeBaseDataStreamExists && isUsingDedicatedInferenceEndpoint) {
+          const currentDataStream = this.createDataStream({
+            resource: 'knowledgeBase',
+            kibanaVersion: this.options.kibanaVersion,
+            fieldMap: {
+              ...omit(knowledgeBaseFieldMap, 'semantic_text'),
+              semantic_text: {
+                type: 'semantic_text',
+                array: false,
+                required: false,
+                inference_id: ASSISTANT_ELSER_INFERENCE_ID,
+                search_inference_id: ELASTICSEARCH_ELSER_INFERENCE_ID,
+              },
+            },
+          });
+
+          // Add `search_inference_id` to the existing mappings
+          await currentDataStream.install({
+            esClient,
+            logger: this.options.logger,
+            pluginStop$: this.options.pluginStop$,
+          });
+
+          // Migrate data stream mapping to the default inference_id
+          const newDS = this.createDataStream({
+            resource: 'knowledgeBase',
+            kibanaVersion: this.options.kibanaVersion,
+            fieldMap: {
+              ...omit(knowledgeBaseFieldMap, 'semantic_text'),
+              semantic_text: {
+                type: 'semantic_text',
+                array: false,
+                required: false,
+              },
+            },
+            settings: {
+              // force new semantic_text field behavior
+              'index.mapping.semantic_text.use_legacy_format': false,
+            },
+            writeIndexOnly: true,
+          });
+
+          // We need to first install the templates and then rollover the indices
+          await newDS.installTemplates({
+            esClient,
+            logger: this.options.logger,
+            pluginStop$: this.options.pluginStop$,
+          });
+
+          const indexNames = (
+            await esClient.indices.getDataStream({ name: newDS.name })
+          ).data_streams.map((ds) => ds.name);
+
+          try {
+            await Promise.all(
+              indexNames.map((indexName) => esClient.indices.rollover({ alias: indexName }))
+            );
+          } catch (e) {
+            /* empty */
+          }
+        } else {
+          // We need to make sure that the data stream is created with the correct mappings
+          this.knowledgeBaseDataStream = this.createDataStream({
+            resource: 'knowledgeBase',
+            kibanaVersion: this.options.kibanaVersion,
+            fieldMap: {
+              ...omit(knowledgeBaseFieldMap, 'semantic_text'),
+              semantic_text: {
+                type: 'semantic_text',
+                array: false,
+                required: false,
+              },
+            },
+            settings: {
+              // force new semantic_text field behavior
+              'index.mapping.semantic_text.use_legacy_format': false,
+            },
+            writeIndexOnly: true,
+          });
+          await this.knowledgeBaseDataStream.install({
+            esClient,
+            logger: this.options.logger,
+            pluginStop$: this.options.pluginStop$,
+          });
+        }
+      } else {
+        // Legacy path
+        await this.knowledgeBaseDataStream.install({
           esClient,
-          id: this.resourceNames.pipelines.knowledgeBase,
+          logger: this.options.logger,
+          pluginStop$: this.options.pluginStop$,
         });
-
-        this.options.logger.debug(`Installed ingest pipeline: ${response}`);
       }
 
       await this.promptsDataStream.install({
@@ -443,6 +554,7 @@ export class AIAssistantService {
       setIsKBSetupInProgress: this.setIsKBSetupInProgress.bind(this),
       spaceId: opts.spaceId,
       manageGlobalKnowledgeBaseAIAssistant: opts.manageGlobalKnowledgeBaseAIAssistant ?? false,
+      assistantDefaultInferenceEndpoint: this.assistantDefaultInferenceEndpoint,
     });
   }
 

x-pack/test/functional/es_archives/security_solution/semantic_text_fields/data.json

@@ -5,77 +5,7 @@
     "index": "semantic_text_fields",
     "source": {
       "@timestamp": "2024-11-01T15:52:16.648Z",
-      "content": {
-        "text": "my favorite color is green",
-        "inference": {
-          "inference_id": ".elser-2-elasticsearch",
-          "model_settings": {
-            "task_type": "sparse_embedding"
-          },
-          "chunks": [
-            {
-              "text": "my favorite color is green",
-              "embeddings": {
-                "green": 2.8714406,
-                "favorite": 2.5192127,
-                "color": 2.499853,
-                "favourite": 1.7829537,
-                "colors": 1.2280636,
-                "my": 1.1292906,
-                "friend": 0.87358737,
-                "rainbow": 0.8518238,
-                "love": 0.8146304,
-                "choice": 0.7517174,
-                "nature": 0.62242556,
-                "beautiful": 0.6110072,
-                "personality": 0.5559894,
-                "dr": 0.5296162,
-                "your": 0.51745296,
-                "art": 0.45324937,
-                "colour": 0.44607934,
-                "theme": 0.4360909,
-                "mood": 0.43253413,
-                "personal": 0.4201024,
-                "style": 0.39435387,
-                "blue": 0.38090202,
-                "nickname": 0.37952134,
-                "design": 0.37043664,
-                "dream": 0.3620103,
-                "desire": 0.35553402,
-                "best": 0.32577398,
-                "favorites": 0.30795538,
-                "humor": 0.30244058,
-                "popular": 0.2957705,
-                "brand": 0.28912684,
-                "neutral": 0.28545624,
-                "passion": 0.28457505,
-                "i": 0.27936152,
-                "preference": 0.24133624,
-                "inspiration": 0.24008423,
-                "purple": 0.23559056,
-                "culture": 0.23260204,
-                "flower": 0.21190192,
-                "bright": 0.20443156,
-                "beauty": 0.20076275,
-                "aura": 0.19355631,
-                "palette": 0.17414959,
-                "wonder": 0.16287619,
-                "photo": 0.16179858,
-                "orange": 0.14167522,
-                "dress": 0.12800644,
-                "camouflage": 0.061010167,
-                "grass": 0.05907971,
-                "tone": 0.028165601,
-                "painting": 0.026917756,
-                "cartoon": 0.019969255,
-                "always": 0.013872984,
-                "yellow": 0.0113299545,
-                "colorful": 0.0036836881
-              }
-            }
-          ]
-        }
-      },
+      "content": "my favorite color is green",
       "text": "my favorite color is green"
     }
   }