[Observability][SecuritySolution] Update entity manager to support extension of mappings and ingest pipeline (#188410)

## Summary


### Acceptance Criteria

- [x] When starting Kibana, the global entity index templates are no
longer created
- [x] When installing a definition, an index template is generated and
installed scoped to the definition ID
- [x] When deleting a definition, the related index template is also
deleted
- [x] The index template composes the current component templates (base,
entity, event) as well as the new custom component templates with the
setting ignore_missing_component_templates set to true
- [x] The new component templates should be named:
<definition_id>@platform, <definition_id>-history@platform,
<definition_id>-latest@platform, <definition_id>@custom,
<definition_id>-history@custom and <definition_id>-latest@custom
- [x] The ingest pipelines include a pipeline processor that calls out
the pipelines named <definition_id>@platform and
<definition_id>-history@platform or <definition_id>-latest@platform,
<definition_id>@custom and <definition_id>-history@custom or
<definition_id>-latest@custom if they exist
- [x] The index template should have a priority of 200 and be set to
managed
- [x] The @custom component template should take precedence over the
@platform component template, allowing users to override things we have
set if they so wish
- [x] set managed_by to 'elastic_entity_model',


### Checklist


- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

---------

Co-authored-by: Kevin Lacabane <kevin.lacabane@elastic.co>
Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

x-pack/plugins/observability_solution/entity_manager/common/constants_entities.ts

@@ -18,8 +18,6 @@ export const ENTITY_EVENT_COMPONENT_TEMPLATE_V1 =
 
 // History constants
 export const ENTITY_HISTORY = 'history' as const;
-export const ENTITY_HISTORY_INDEX_TEMPLATE_V1 =
-  `${ENTITY_BASE_PREFIX}_${ENTITY_SCHEMA_VERSION_V1}_${ENTITY_HISTORY}_index_template` as const;
 export const ENTITY_HISTORY_BASE_COMPONENT_TEMPLATE_V1 =
   `${ENTITY_BASE_PREFIX}_${ENTITY_SCHEMA_VERSION_V1}_${ENTITY_HISTORY}_base` as const;
 export const ENTITY_HISTORY_PREFIX_V1 =
@@ -29,8 +27,6 @@ export const ENTITY_HISTORY_INDEX_PREFIX_V1 =
 
 // Latest constants
 export const ENTITY_LATEST = 'latest' as const;
-export const ENTITY_LATEST_INDEX_TEMPLATE_V1 =
-  `${ENTITY_BASE_PREFIX}_${ENTITY_SCHEMA_VERSION_V1}_${ENTITY_LATEST}_index_template` as const;
 export const ENTITY_LATEST_BASE_COMPONENT_TEMPLATE_V1 =
   `${ENTITY_BASE_PREFIX}_${ENTITY_SCHEMA_VERSION_V1}_${ENTITY_LATEST}_base` as const;
 export const ENTITY_LATEST_PREFIX_V1 =

x-pack/plugins/observability_solution/entity_manager/common/helpers.test.ts

@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { getEntityHistoryIndexTemplateV1, getEntityLatestIndexTemplateV1 } from './helpers';
+
+describe('helpers', () => {
+  it('getEntityHistoryIndexTemplateV1 should return the correct value', () => {
+    const definitionId = 'test';
+    const result = getEntityHistoryIndexTemplateV1(definitionId);
+    expect(result).toEqual('entities_v1_history_test_index_template');
+  });
+
+  it('getEntityLatestIndexTemplateV1 should return the correct value', () => {
+    const definitionId = 'test';
+    const result = getEntityLatestIndexTemplateV1(definitionId);
+    expect(result).toEqual('entities_v1_latest_test_index_template');
+  });
+});

x-pack/plugins/observability_solution/entity_manager/common/helpers.ts

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+  ENTITY_BASE_PREFIX,
+  ENTITY_HISTORY,
+  ENTITY_LATEST,
+  ENTITY_SCHEMA_VERSION_V1,
+} from './constants_entities';
+
+export const getEntityHistoryIndexTemplateV1 = (definitionId: string) =>
+  `${ENTITY_BASE_PREFIX}_${ENTITY_SCHEMA_VERSION_V1}_${ENTITY_HISTORY}_${definitionId}_index_template` as const;
+
+export const getEntityLatestIndexTemplateV1 = (definitionId: string) =>
+  `${ENTITY_BASE_PREFIX}_${ENTITY_SCHEMA_VERSION_V1}_${ENTITY_LATEST}_${definitionId}_index_template` as const;

x-pack/plugins/observability_solution/entity_manager/server/lib/auth/privileges.ts

@@ -21,7 +21,13 @@ export const requiredRunTimePrivileges = {
       privileges: ['read', 'view_index_metadata'],
     },
   ],
-  cluster: ['manage_transform', 'monitor_transform', 'manage_ingest_pipelines', 'monitor'],
+  cluster: [
+    'manage_transform',
+    'monitor_transform',
+    'manage_ingest_pipelines',
+    'monitor',
+    'manage_index_templates',
+  ],
   application: [
     {
       application: 'kibana-.kibana',

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/__snapshots__/generate_history_processors.test.ts.snap

@@ -148,5 +148,29 @@ if (ctx.entity?.metadata?.sourceIndex != null) {
       "index_name_prefix": ".entities.v1.history.admin-console-services.",
     },
   },
+  Object {
+    "pipeline": Object {
+      "ignore_missing_pipeline": true,
+      "name": "admin-console-services@platform",
+    },
+  },
+  Object {
+    "pipeline": Object {
+      "ignore_missing_pipeline": true,
+      "name": "admin-console-services-history@platform",
+    },
+  },
+  Object {
+    "pipeline": Object {
+      "ignore_missing_pipeline": true,
+      "name": "admin-console-services@custom",
+    },
+  },
+  Object {
+    "pipeline": Object {
+      "ignore_missing_pipeline": true,
+      "name": "admin-console-services-history@custom",
+    },
+  },
 ]
 `;

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/__snapshots__/generate_latest_processors.test.ts.snap

@@ -108,5 +108,29 @@ ctx.event.category = ctx.entity.identity.event.category.keySet().toArray()[0];",
       "value": ".entities.v1.latest.admin-console-services",
     },
   },
+  Object {
+    "pipeline": Object {
+      "ignore_missing_pipeline": true,
+      "name": "admin-console-services@platform",
+    },
+  },
+  Object {
+    "pipeline": Object {
+      "ignore_missing_pipeline": true,
+      "name": "admin-console-services-latest@platform",
+    },
+  },
+  Object {
+    "pipeline": Object {
+      "ignore_missing_pipeline": true,
+      "name": "admin-console-services@custom",
+    },
+  },
+  Object {
+    "pipeline": Object {
+      "ignore_missing_pipeline": true,
+      "name": "admin-console-services-latest@custom",
+    },
+  },
 ]
 `;

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts

@@ -163,5 +163,30 @@ export function generateHistoryProcessors(definition: EntityDefinition) {
         date_formats: ['UNIX_MS', 'ISO8601', "yyyy-MM-dd'T'HH:mm:ss.SSSXX"],
       },
     },
+    {
+      pipeline: {
+        ignore_missing_pipeline: true,
+        name: `${definition.id}@platform`,
+      },
+    },
+    {
+      pipeline: {
+        ignore_missing_pipeline: true,
+        name: `${definition.id}-history@platform`,
+      },
+    },
+
+    {
+      pipeline: {
+        ignore_missing_pipeline: true,
+        name: `${definition.id}@custom`,
+      },
+    },
+    {
+      pipeline: {
+        ignore_missing_pipeline: true,
+        name: `${definition.id}-history@custom`,
+      },
+    },
   ];
 }

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts

@@ -122,5 +122,30 @@ export function generateLatestProcessors(definition: EntityDefinition) {
         value: `${generateLatestIndexName(definition)}`,
       },
     },
+    {
+      pipeline: {
+        ignore_missing_pipeline: true,
+        name: `${definition.id}@platform`,
+      },
+    },
+    {
+      pipeline: {
+        ignore_missing_pipeline: true,
+        name: `${definition.id}-latest@platform`,
+      },
+    },
+    {
+      pipeline: {
+        ignore_missing_pipeline: true,
+        name: `${definition.id}@custom`,
+      },
+    },
+
+    {
+      pipeline: {
+        ignore_missing_pipeline: true,
+        name: `${definition.id}-latest@custom`,
+      },
+    },
   ];
 }

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.test.ts

@@ -34,6 +34,18 @@ const assertHasCreatedDefinition = (
     overwrite: true,
   });
 
+  expect(esClient.indices.putIndexTemplate).toBeCalledTimes(2);
+  expect(esClient.indices.putIndexTemplate).toBeCalledWith(
+    expect.objectContaining({
+      name: `entities_v1_history_${definition.id}_index_template`,
+    })
+  );
+  expect(esClient.indices.putIndexTemplate).toBeCalledWith(
+    expect.objectContaining({
+      name: `entities_v1_latest_${definition.id}_index_template`,
+    })
+  );
+
   expect(esClient.ingest.putPipeline).toBeCalledTimes(2);
   expect(esClient.ingest.putPipeline).toBeCalledWith({
     id: generateHistoryIngestPipelineId(builtInServicesFromLogsEntityDefinition),
@@ -111,6 +123,20 @@ const assertHasUninstalledDefinition = (
   expect(esClient.transform.deleteTransform).toBeCalledTimes(2);
   expect(esClient.ingest.deletePipeline).toBeCalledTimes(2);
   expect(soClient.delete).toBeCalledTimes(1);
+
+  expect(esClient.indices.deleteIndexTemplate).toBeCalledTimes(2);
+  expect(esClient.indices.deleteIndexTemplate).toBeCalledWith(
+    {
+      name: `entities_v1_history_${definition.id}_index_template`,
+    },
+    { ignore: [404] }
+  );
+  expect(esClient.indices.deleteIndexTemplate).toBeCalledWith(
+    {
+      name: `entities_v1_latest_${definition.id}_index_template`,
+    },
+    { ignore: [404] }
+  );
 };
 
 describe('install_entity_definition', () => {

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.ts

@@ -9,6 +9,10 @@ import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
 import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server';
 import { EntityDefinition } from '@kbn/entities-schema';
 import { Logger } from '@kbn/logging';
+import {
+  getEntityHistoryIndexTemplateV1,
+  getEntityLatestIndexTemplateV1,
+} from '../../../common/helpers';
 import {
   createAndInstallHistoryIngestPipeline,
   createAndInstallLatestIngestPipeline,
@@ -28,6 +32,9 @@ import {
   stopAndDeleteLatestTransform,
 } from './stop_and_delete_transform';
 import { uninstallEntityDefinition } from './uninstall_entity_definition';
+import { deleteTemplate, upsertTemplate } from '../manage_index_templates';
+import { getEntitiesLatestIndexTemplateConfig } from '../../templates/entities_latest_template';
+import { getEntitiesHistoryIndexTemplateConfig } from '../../templates/entities_history_template';
 
 export interface InstallDefinitionParams {
   esClient: ElasticsearchClient;
@@ -52,6 +59,10 @@ export async function installEntityDefinition({
       latest: false,
     },
     definition: false,
+    indexTemplates: {
+      history: false,
+      latest: false,
+    },
   };
 
   try {
@@ -62,6 +73,20 @@ export async function installEntityDefinition({
     const entityDefinition = await saveEntityDefinition(soClient, definition);
     installState.definition = true;
 
+    // install scoped index template
+    await upsertTemplate({
+      esClient,
+      logger,
+      template: getEntitiesHistoryIndexTemplateConfig(definition.id),
+    });
+    installState.indexTemplates.history = true;
+    await upsertTemplate({
+      esClient,
+      logger,
+      template: getEntitiesLatestIndexTemplateConfig(definition.id),
+    });
+    installState.indexTemplates.latest = true;
+
     // install ingest pipelines
     logger.debug(`Installing ingest pipelines for definition ${definition.id}`);
     await createAndInstallHistoryIngestPipeline(esClient, entityDefinition, logger);
@@ -99,6 +124,21 @@ export async function installEntityDefinition({
       await stopAndDeleteLatestTransform(esClient, definition, logger);
     }
 
+    if (installState.indexTemplates.history) {
+      await deleteTemplate({
+        esClient,
+        logger,
+        name: getEntityHistoryIndexTemplateV1(definition.id),
+      });
+    }
+    if (installState.indexTemplates.latest) {
+      await deleteTemplate({
+        esClient,
+        logger,
+        name: getEntityLatestIndexTemplateV1(definition.id),
+      });
+    }
+
     throw e;
   }
 }

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/uninstall_entity_definition.ts

@@ -9,6 +9,10 @@ import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
 import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server';
 import { EntityDefinition } from '@kbn/entities-schema';
 import { Logger } from '@kbn/logging';
+import {
+  getEntityHistoryIndexTemplateV1,
+  getEntityLatestIndexTemplateV1,
+} from '../../../common/helpers';
 import { deleteEntityDefinition } from './delete_entity_definition';
 import { deleteIndices } from './delete_index';
 import { deleteHistoryIngestPipeline, deleteLatestIngestPipeline } from './delete_ingest_pipeline';
@@ -17,6 +21,7 @@ import {
   stopAndDeleteHistoryTransform,
   stopAndDeleteLatestTransform,
 } from './stop_and_delete_transform';
+import { deleteTemplate } from '../manage_index_templates';
 
 export async function uninstallEntityDefinition({
   definition,
@@ -36,6 +41,9 @@ export async function uninstallEntityDefinition({
   await deleteHistoryIngestPipeline(esClient, definition, logger);
   await deleteLatestIngestPipeline(esClient, definition, logger);
   await deleteEntityDefinition(soClient, definition, logger);
+  await deleteTemplate({ esClient, logger, name: getEntityHistoryIndexTemplateV1(definition.id) });
+  await deleteTemplate({ esClient, logger, name: getEntityLatestIndexTemplateV1(definition.id) });
+
   if (deleteData) {
     await deleteIndices(esClient, definition, logger);
   }

x-pack/plugins/observability_solution/entity_manager/server/lib/manage_index_templates.ts

@@ -10,6 +10,7 @@ import {
   IndicesPutIndexTemplateRequest,
 } from '@elastic/elasticsearch/lib/api/types';
 import { ElasticsearchClient, Logger } from '@kbn/core/server';
+import { retryTransientEsErrors } from './entities/helpers/retry';
 
 interface TemplateManagementOptions {
   esClient: ElasticsearchClient;
@@ -23,12 +24,18 @@ interface ComponentManagementOptions {
   logger: Logger;
 }
 
+interface DeleteTemplateOptions {
+  esClient: ElasticsearchClient;
+  name: string;
+  logger: Logger;
+}
+
 export async function upsertTemplate({ esClient, template, logger }: TemplateManagementOptions) {
   try {
-    await esClient.indices.putIndexTemplate(template);
+    await retryTransientEsErrors(() => esClient.indices.putIndexTemplate(template), { logger });
   } catch (error: any) {
     logger.error(`Error updating entity manager index template: ${error.message}`);
-    return;
+    throw error;
   }
 
   logger.info(
@@ -37,12 +44,26 @@ export async function upsertTemplate({ esClient, template, logger }: TemplateMan
   logger.debug(() => `Entity manager index template: ${JSON.stringify(template)}`);
 }
 
+export async function deleteTemplate({ esClient, name, logger }: DeleteTemplateOptions) {
+  try {
+    await retryTransientEsErrors(
+      () => esClient.indices.deleteIndexTemplate({ name }, { ignore: [404] }),
+      { logger }
+    );
+  } catch (error: any) {
+    logger.error(`Error deleting entity manager index template: ${error.message}`);
+    throw error;
+  }
+}
+
 export async function upsertComponent({ esClient, component, logger }: ComponentManagementOptions) {
   try {
-    await esClient.cluster.putComponentTemplate(component);
+    await retryTransientEsErrors(() => esClient.cluster.putComponentTemplate(component), {
+      logger,
+    });
   } catch (error: any) {
     logger.error(`Error updating entity manager component template: ${error.message}`);
-    return;
+    throw error;
   }
 
   logger.info(

x-pack/plugins/observability_solution/entity_manager/server/plugin.ts

@@ -14,7 +14,7 @@ import {
   PluginConfigDescriptor,
   Logger,
 } from '@kbn/core/server';
-import { upsertComponent, upsertTemplate } from './lib/manage_index_templates';
+import { upsertComponent } from './lib/manage_index_templates';
 import { setupRoutes } from './routes';
 import {
   EntityManagerPluginSetupDependencies,
@@ -27,8 +27,6 @@ import { entityDefinition, EntityDiscoveryApiKeyType } from './saved_objects';
 import { entitiesEntityComponentTemplateConfig } from './templates/components/entity';
 import { entitiesLatestBaseComponentTemplateConfig } from './templates/components/base_latest';
 import { entitiesHistoryBaseComponentTemplateConfig } from './templates/components/base_history';
-import { entitiesHistoryIndexTemplateConfig } from './templates/entities_history_template';
-import { entitiesLatestIndexTemplateConfig } from './templates/entities_latest_template';
 
 export type EntityManagerServerPluginSetup = ReturnType<EntityManagerServerPlugin['setup']>;
 export type EntityManagerServerPluginStart = ReturnType<EntityManagerServerPlugin['start']>;
@@ -113,22 +111,7 @@ export class EntityManagerServerPlugin
         logger: this.logger,
         component: entitiesEntityComponentTemplateConfig,
       }),
-    ])
-      .then(() =>
-        upsertTemplate({
-          esClient,
-          logger: this.logger,
-          template: entitiesHistoryIndexTemplateConfig,
-        })
-      )
-      .then(() =>
-        upsertTemplate({
-          esClient,
-          logger: this.logger,
-          template: entitiesLatestIndexTemplateConfig,
-        })
-      )
-      .catch(() => {});
+    ]).catch(() => {});
 
     return {};
   }

x-pack/plugins/observability_solution/entity_manager/server/templates/components/helpers.test.ts

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { getCustomHistoryTemplateComponents, getCustomLatestTemplateComponents } from './helpers';
+
+describe('helpers', () => {
+  it('getCustomLatestTemplateComponents should return template component in the right sort order', () => {
+    const definitionId = 'test';
+    const result = getCustomLatestTemplateComponents(definitionId);
+    expect(result).toEqual([
+      'test@platform',
+      'test-latest@platform',
+      'test@custom',
+      'test-latest@custom',
+    ]);
+  });
+
+  it('getCustomHistoryTemplateComponents should return template component in the right sort order', () => {
+    const definitionId = 'test';
+    const result = getCustomHistoryTemplateComponents(definitionId);
+    expect(result).toEqual([
+      'test@platform',
+      'test-history@platform',
+      'test@custom',
+      'test-history@custom',
+    ]);
+  });
+});

x-pack/plugins/observability_solution/entity_manager/server/templates/components/helpers.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const getCustomLatestTemplateComponents = (definitionId: string) => [
+  `${definitionId}@platform`, // @platform goes before so it can be overwritten by custom
+  `${definitionId}-latest@platform`,
+  `${definitionId}@custom`,
+  `${definitionId}-latest@custom`,
+];
+
+export const getCustomHistoryTemplateComponents = (definitionId: string) => [
+  `${definitionId}@platform`, // @platform goes before so it can be overwritten by custom
+  `${definitionId}-history@platform`,
+  `${definitionId}@custom`,
+  `${definitionId}-history@custom`,
+];

x-pack/plugins/observability_solution/entity_manager/server/templates/entities_history_template.ts

@@ -6,29 +6,35 @@
  */
 
 import { IndicesPutIndexTemplateRequest } from '@elastic/elasticsearch/lib/api/types';
+import { getEntityHistoryIndexTemplateV1 } from '../../common/helpers';
 import {
   ENTITY_ENTITY_COMPONENT_TEMPLATE_V1,
   ENTITY_EVENT_COMPONENT_TEMPLATE_V1,
   ENTITY_HISTORY_BASE_COMPONENT_TEMPLATE_V1,
   ENTITY_HISTORY_INDEX_PREFIX_V1,
-  ENTITY_HISTORY_INDEX_TEMPLATE_V1,
 } from '../../common/constants_entities';
+import { getCustomHistoryTemplateComponents } from './components/helpers';
 
-export const entitiesHistoryIndexTemplateConfig: IndicesPutIndexTemplateRequest = {
-  name: ENTITY_HISTORY_INDEX_TEMPLATE_V1,
+export const getEntitiesHistoryIndexTemplateConfig = (
+  definitionId: string
+): IndicesPutIndexTemplateRequest => ({
+  name: getEntityHistoryIndexTemplateV1(definitionId),
   _meta: {
     description:
       "Index template for indices managed by the Elastic Entity Model's entity discovery framework for the history dataset",
     ecs_version: '8.0.0',
     managed: true,
+    managed_by: 'elastic_entity_model',
   },
+  ignore_missing_component_templates: getCustomHistoryTemplateComponents(definitionId),
   composed_of: [
     ENTITY_HISTORY_BASE_COMPONENT_TEMPLATE_V1,
     ENTITY_ENTITY_COMPONENT_TEMPLATE_V1,
     ENTITY_EVENT_COMPONENT_TEMPLATE_V1,
+    ...getCustomHistoryTemplateComponents(definitionId),
   ],
   index_patterns: [`${ENTITY_HISTORY_INDEX_PREFIX_V1}.*`],
-  priority: 1,
+  priority: 200,
   template: {
     mappings: {
       _meta: {
@@ -72,4 +78,4 @@ export const entitiesHistoryIndexTemplateConfig: IndicesPutIndexTemplateRequest
       },
     },
   },
-};
+});

x-pack/plugins/observability_solution/entity_manager/server/templates/entities_latest_template.ts

@@ -6,26 +6,32 @@
  */
 
 import { IndicesPutIndexTemplateRequest } from '@elastic/elasticsearch/lib/api/types';
+import { getEntityLatestIndexTemplateV1 } from '../../common/helpers';
 import {
   ENTITY_ENTITY_COMPONENT_TEMPLATE_V1,
   ENTITY_EVENT_COMPONENT_TEMPLATE_V1,
   ENTITY_LATEST_BASE_COMPONENT_TEMPLATE_V1,
   ENTITY_LATEST_INDEX_PREFIX_V1,
-  ENTITY_LATEST_INDEX_TEMPLATE_V1,
 } from '../../common/constants_entities';
+import { getCustomLatestTemplateComponents } from './components/helpers';
 
-export const entitiesLatestIndexTemplateConfig: IndicesPutIndexTemplateRequest = {
-  name: ENTITY_LATEST_INDEX_TEMPLATE_V1,
+export const getEntitiesLatestIndexTemplateConfig = (
+  definitionId: string
+): IndicesPutIndexTemplateRequest => ({
+  name: getEntityLatestIndexTemplateV1(definitionId),
   _meta: {
     description:
       "Index template for indices managed by the Elastic Entity Model's entity discovery framework for the latest dataset",
     ecs_version: '8.0.0',
     managed: true,
+    managed_by: 'elastic_entity_model',
   },
+  ignore_missing_component_templates: getCustomLatestTemplateComponents(definitionId),
   composed_of: [
     ENTITY_LATEST_BASE_COMPONENT_TEMPLATE_V1,
     ENTITY_ENTITY_COMPONENT_TEMPLATE_V1,
     ENTITY_EVENT_COMPONENT_TEMPLATE_V1,
+    ...getCustomLatestTemplateComponents(definitionId),
   ],
   index_patterns: [`${ENTITY_LATEST_INDEX_PREFIX_V1}.*`],
   priority: 1,
@@ -72,4 +78,4 @@ export const entitiesLatestIndexTemplateConfig: IndicesPutIndexTemplateRequest =
       },
     },
   },
-};
+});