[Fleet] Add global component template to all fleet index templates (#102225)

x-pack/plugins/fleet/common/types/index.ts

@@ -25,6 +25,7 @@ export interface FleetConfigType {
   };
   agentPolicies?: PreconfiguredAgentPolicy[];
   packages?: PreconfiguredPackage[];
+  agentIdVerificationEnabled?: boolean;
 }
 
 // Calling Object.entries(PackagesGroupedByStatus) gave `status: string`

x-pack/plugins/fleet/public/mock/plugin_configuration.ts

@@ -12,6 +12,7 @@ export const createConfigurationMock = (): FleetConfigType => {
     enabled: true,
     registryUrl: '',
     registryProxyUrl: '',
+    agentIdVerificationEnabled: true,
     agents: {
       enabled: true,
       elasticsearch: {

x-pack/plugins/fleet/server/constants/fleet_es_assets.ts

@@ -5,9 +5,37 @@
  * 2.0.
  */
 
-export const FINAL_PIPELINE_ID = '.fleet_final_pipeline';
+export const FLEET_FINAL_PIPELINE_ID = '.fleet_final_pipeline-1';
 
-export const FINAL_PIPELINE = `---
+export const FLEET_GLOBAL_COMPONENT_TEMPLATE_NAME = '.fleet_component_template-1';
+
+export const FLEET_GLOBAL_COMPONENT_TEMPLATE_CONTENT = {
+  _meta: {},
+  template: {
+    settings: {
+      index: {
+        final_pipeline: FLEET_FINAL_PIPELINE_ID,
+      },
+    },
+    mappings: {
+      properties: {
+        event: {
+          properties: {
+            ingested: {
+              type: 'date',
+            },
+            agent_id_status: {
+              ignore_above: 1024,
+              type: 'keyword',
+            },
+          },
+        },
+      },
+    },
+  },
+};
+
+export const FLEET_FINAL_PIPELINE_CONTENT = `---
 description: >
   Final pipeline for processing all incoming Fleet Agent documents.
 processors:

x-pack/plugins/fleet/server/constants/index.ts

@@ -57,3 +57,10 @@ export {
   PRECONFIGURATION_DELETION_RECORD_SAVED_OBJECT_TYPE,
   PRECONFIGURATION_LATEST_KEYWORD,
 } from '../../common';
+
+export {
+  FLEET_GLOBAL_COMPONENT_TEMPLATE_NAME,
+  FLEET_GLOBAL_COMPONENT_TEMPLATE_CONTENT,
+  FLEET_FINAL_PIPELINE_ID,
+  FLEET_FINAL_PIPELINE_CONTENT,
+} from './fleet_es_assets';

x-pack/plugins/fleet/server/index.ts

@@ -77,6 +77,7 @@ export const config: PluginConfigDescriptor = {
     }),
     packages: PreconfiguredPackagesSchema,
     agentPolicies: PreconfiguredAgentPoliciesSchema,
+    agentIdVerificationEnabled: schema.boolean({ defaultValue: true }),
   }),
 };
 

x-pack/plugins/fleet/server/mocks/index.ts

@@ -4,6 +4,8 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
+import { of } from 'rxjs';
+
 import {
   elasticsearchServiceMock,
   loggingSystemMock,
@@ -22,6 +24,14 @@ import type { FleetAppContext } from '../plugin';
 export * from '../services/artifacts/mocks';
 
 export const createAppContextStartContractMock = (): FleetAppContext => {
+  const config = {
+    agents: { enabled: true, elasticsearch: {} },
+    enabled: true,
+    agentIdVerificationEnabled: true,
+  };
+
+  const config$ = of(config);
+
   return {
     elasticsearch: elasticsearchServiceMock.createStart(),
     data: dataPluginMock.createStartContract(),
@@ -33,7 +43,9 @@ export const createAppContextStartContractMock = (): FleetAppContext => {
     configInitialValue: {
       agents: { enabled: true, elasticsearch: {} },
       enabled: true,
+      agentIdVerificationEnabled: true,
     },
+    config$,
     kibanaVersion: '8.0.0',
     kibanaBranch: 'master',
   };

x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts

@@ -14,9 +14,9 @@ import { getAsset, getPathParts } from '../../archive';
 import type { ArchiveEntry } from '../../archive';
 import { saveInstalledEsRefs } from '../../packages/install';
 import { getInstallationObject } from '../../packages';
+import { FLEET_FINAL_PIPELINE_CONTENT, FLEET_FINAL_PIPELINE_ID } from '../../../../constants';
 
 import { deletePipelineRefs } from './remove';
-import { FINAL_PIPELINE, FINAL_PIPELINE_ID } from './final_pipeline';
 
 interface RewriteSubstitution {
   source: string;
@@ -190,22 +190,24 @@ export async function ensureFleetFinalPipelineIsInstalled(esClient: Elasticsearc
   const esClientRequestOptions: TransportRequestOptions = {
     ignore: [404],
   };
-  const res = await esClient.ingest.getPipeline({ id: FINAL_PIPELINE_ID }, esClientRequestOptions);
+  const res = await esClient.ingest.getPipeline(
+    { id: FLEET_FINAL_PIPELINE_ID },
+    esClientRequestOptions
+  );
 
   if (res.statusCode === 404) {
-    await esClient.ingest.putPipeline(
-      // @ts-ignore pipeline is define in yaml
-      { id: FINAL_PIPELINE_ID, body: FINAL_PIPELINE },
-      {
-        headers: {
-          // pipeline is YAML
-          'Content-Type': 'application/yaml',
-          // but we want JSON responses (to extract error messages, status code, or other metadata)
-          Accept: 'application/json',
-        },
-      }
-    );
+    await installPipeline({
+      esClient,
+      pipeline: {
+        nameForInstallation: FLEET_FINAL_PIPELINE_ID,
+        contentForInstallation: FLEET_FINAL_PIPELINE_CONTENT,
+        extension: 'yml',
+      },
+    });
+    return { isCreated: true };
   }
+
+  return { isCreated: false };
 }
 
 const isDirectory = ({ path }: ArchiveEntry) => path.endsWith('/');

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap

@@ -25,8 +25,7 @@ exports[`EPM template tests loading base.yml: base.yml 1`] = `
           "default_field": [
             "long.nested.foo"
           ]
-        },
-        "final_pipeline": ".fleet_final_pipeline"
+        }
       }
     },
     "mappings": {
@@ -99,7 +98,9 @@ exports[`EPM template tests loading base.yml: base.yml 1`] = `
     }
   },
   "data_stream": {},
-  "composed_of": [],
+  "composed_of": [
+    ".fleet_component_template-1"
+  ],
   "_meta": {
     "package": {
       "name": "nginx"
@@ -140,8 +141,7 @@ exports[`EPM template tests loading coredns.logs.yml: coredns.logs.yml 1`] = `
             "coredns.response.code",
             "coredns.response.flags"
           ]
-        },
-        "final_pipeline": ".fleet_final_pipeline"
+        }
       }
     },
     "mappings": {
@@ -214,7 +214,9 @@ exports[`EPM template tests loading coredns.logs.yml: coredns.logs.yml 1`] = `
     }
   },
   "data_stream": {},
-  "composed_of": [],
+  "composed_of": [
+    ".fleet_component_template-1"
+  ],
   "_meta": {
     "package": {
       "name": "coredns"
@@ -283,8 +285,7 @@ exports[`EPM template tests loading system.yml: system.yml 1`] = `
             "system.users.scope",
             "system.users.remote_host"
           ]
-        },
-        "final_pipeline": ".fleet_final_pipeline"
+        }
       }
     },
     "mappings": {
@@ -1741,7 +1742,9 @@ exports[`EPM template tests loading system.yml: system.yml 1`] = `
     }
   },
   "data_stream": {},
-  "composed_of": [],
+  "composed_of": [
+    ".fleet_component_template-1"
+  ],
   "_meta": {
     "package": {
       "name": "system"

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts

@@ -20,6 +20,10 @@ import type { Field } from '../../fields/field';
 import { getPipelineNameForInstallation } from '../ingest_pipeline/install';
 import { getAsset, getPathParts } from '../../archive';
 import { removeAssetTypesFromInstalledEs, saveInstalledEsRefs } from '../../packages/install';
+import {
+  FLEET_GLOBAL_COMPONENT_TEMPLATE_NAME,
+  FLEET_GLOBAL_COMPONENT_TEMPLATE_CONTENT,
+} from '../../../../constants';
 
 import {
   generateMappings,
@@ -164,7 +168,7 @@ export async function installTemplateForDataStream({
 }
 
 interface TemplateMapEntry {
-  _meta: { package: { name: string } };
+  _meta: { package?: { name: string } };
   template:
     | {
         mappings: NonNullable<RegistryElasticsearch['index_template.mappings']>;
@@ -277,6 +281,28 @@ async function installDataStreamComponentTemplates(params: {
   return templateNames;
 }
 
+export async function ensureDefaultComponentTemplate(esClient: ElasticsearchClient) {
+  const { body: getTemplateRes } = await esClient.cluster.getComponentTemplate(
+    {
+      name: FLEET_GLOBAL_COMPONENT_TEMPLATE_NAME,
+    },
+    {
+      ignore: [404],
+    }
+  );
+
+  const existingTemplate = getTemplateRes?.component_templates?.[0];
+  if (!existingTemplate) {
+    await putComponentTemplate(esClient, {
+      name: FLEET_GLOBAL_COMPONENT_TEMPLATE_NAME,
+      body: FLEET_GLOBAL_COMPONENT_TEMPLATE_CONTENT,
+      create: true,
+    });
+  }
+
+  return { isCreated: !existingTemplate };
+}
+
 export async function installTemplate({
   esClient,
   fields,
@@ -378,12 +404,13 @@ export function getAllTemplateRefs(installedTemplates: IndexTemplateEntry[]) {
         type: ElasticsearchAssetType.indexTemplate,
       },
     ];
-    const componentTemplates = installedTemplate.indexTemplate.composed_of.map(
-      (componentTemplateId) => ({
+    const componentTemplates = installedTemplate.indexTemplate.composed_of
+      // Filter global component template shared between integrations
+      .filter((componentTemplateId) => componentTemplateId !== FLEET_GLOBAL_COMPONENT_TEMPLATE_NAME)
+      .map((componentTemplateId) => ({
         id: componentTemplateId,
         type: ElasticsearchAssetType.componentTemplate,
-      })
-    );
+      }));
     return indexTemplates.concat(componentTemplates);
   });
 }

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts

@@ -24,6 +24,8 @@ import {
   generateTemplateIndexPattern,
 } from './template';
 
+const FLEET_COMPONENT_TEMPLATE = '.fleet_component_template-1';
+
 // Add our own serialiser to just do JSON.stringify
 expect.addSnapshotSerializer({
   print(val) {
@@ -67,7 +69,7 @@ describe('EPM template', () => {
       composedOfTemplates,
       templatePriority: 200,
     });
-    expect(template.composed_of).toStrictEqual(composedOfTemplates);
+    expect(template.composed_of).toStrictEqual([...composedOfTemplates, FLEET_COMPONENT_TEMPLATE]);
   });
 
   it('adds empty composed_of correctly', () => {
@@ -82,7 +84,7 @@ describe('EPM template', () => {
       composedOfTemplates,
       templatePriority: 200,
     });
-    expect(template.composed_of).toStrictEqual(composedOfTemplates);
+    expect(template.composed_of).toStrictEqual([FLEET_COMPONENT_TEMPLATE]);
   });
 
   it('adds hidden field correctly', () => {

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts

@@ -16,7 +16,7 @@ import type {
 } from '../../../../types';
 import { appContextService } from '../../../';
 import { getRegistryDataStreamAssetBaseName } from '../index';
-import { FINAL_PIPELINE_ID } from '../ingest_pipeline/final_pipeline';
+import { FLEET_GLOBAL_COMPONENT_TEMPLATE_NAME } from '../../../../constants';
 
 interface Properties {
   [key: string]: any;
@@ -90,7 +90,11 @@ export function getTemplate({
   if (template.template.settings.index.final_pipeline) {
     throw new Error(`Error template for ${templateIndexPattern} contains a final_pipeline`);
   }
-  template.template.settings.index.final_pipeline = FINAL_PIPELINE_ID;
+
+  if (appContextService.getConfig()?.agentIdVerificationEnabled) {
+    // Add fleet global assets
+    template.composed_of = [...(template.composed_of || []), FLEET_GLOBAL_COMPONENT_TEMPLATE_NAME];
+  }
 
   return template;
 }

x-pack/plugins/fleet/server/services/epm/packages/get.ts

@@ -101,6 +101,8 @@ export async function getPackageSavedObjects(
   });
 }
 
+export const getInstallations = getPackageSavedObjects;
+
 export async function getPackageInfo(options: {
   savedObjectsClient: SavedObjectsClientContract;
   pkgName: string;

x-pack/plugins/fleet/server/services/epm/packages/index.ts

@@ -17,6 +17,7 @@ export {
   getFile,
   getInstallationObject,
   getInstallation,
+  getInstallations,
   getPackageInfo,
   getPackages,
   getLimitedPackages,

x-pack/plugins/fleet/server/services/setup.ts

@@ -24,7 +24,10 @@ import { awaitIfPending } from './setup_utils';
 import { ensureAgentActionPolicyChangeExists } from './agents';
 import { awaitIfFleetServerSetupPending } from './fleet_server';
 import { ensureFleetFinalPipelineIsInstalled } from './epm/elasticsearch/ingest_pipeline/install';
+import { ensureDefaultComponentTemplate } from './epm/elasticsearch/template/install';
+import { getInstallations, installPackage } from './epm/packages';
 import { isPackageInstalled } from './epm/packages/install';
+import { pkgToPkgKey } from './epm/registry';
 
 export interface SetupStatus {
   isInitialized: boolean;
@@ -47,9 +50,10 @@ async function createSetupSideEffects(
     settingsService.settingsSetup(soClient),
   ]);
 
-  await ensureFleetFinalPipelineIsInstalled(esClient);
-
   await awaitIfFleetServerSetupPending();
+  if (appContextService.getConfig()?.agentIdVerificationEnabled) {
+    await ensureFleetGlobalEsAssets(soClient, esClient);
+  }
 
   const { agentPolicies: policiesOrUndefined, packages: packagesOrUndefined } =
     appContextService.getConfig() ?? {};
@@ -95,6 +99,49 @@ async function createSetupSideEffects(
   };
 }
 
+/**
+ * Ensure ES assets shared by all Fleet index template are installed
+ */
+export async function ensureFleetGlobalEsAssets(
+  soClient: SavedObjectsClientContract,
+  esClient: ElasticsearchClient
+) {
+  const logger = appContextService.getLogger();
+  // Ensure Global Fleet ES assets are installed
+  const globalAssetsRes = await Promise.all([
+    ensureDefaultComponentTemplate(esClient),
+    ensureFleetFinalPipelineIsInstalled(esClient),
+  ]);
+
+  if (globalAssetsRes.some((asset) => asset.isCreated)) {
+    // Update existing index template
+    const packages = await getInstallations(soClient);
+
+    await Promise.all(
+      packages.saved_objects.map(async ({ attributes: installation }) => {
+        if (installation.install_source !== 'registry') {
+          logger.error(
+            `Package needs to be manually reinstalled ${installation.name} after installing Fleet global assets`
+          );
+          return;
+        }
+        await installPackage({
+          installSource: installation.install_source,
+          savedObjectsClient: soClient,
+          pkgkey: pkgToPkgKey({ name: installation.name, version: installation.version }),
+          esClient,
+          // Force install the pacakge will update the index template and the datastream write indices
+          force: true,
+        }).catch((err) => {
+          logger.error(
+            `Package needs to be manually reinstalled ${installation.name} after installing Fleet global assets: ${err.message}`
+          );
+        });
+      })
+    );
+  }
+}
+
 export async function ensureDefaultEnrollmentAPIKeysExists(
   soClient: SavedObjectsClientContract,
   esClient: ElasticsearchClient,

x-pack/test/api_integration/apis/ml/modules/index.ts

@@ -9,11 +9,14 @@ import { FtrProviderContext } from '../../../ftr_provider_context';
 
 export default function ({ getService, loadTestFile }: FtrProviderContext) {
   const ml = getService('ml');
+  const supertest = getService('supertest');
 
   const fleetPackages = ['apache', 'nginx'];
 
   describe('modules', function () {
     before(async () => {
+      // Fleet need to be setup to be able to setup packages
+      await supertest.post(`/api/fleet/setup`).set({ 'kbn-xsrf': 'some-xsrf-token' }).expect(200);
       for (const fleetPackage of fleetPackages) {
         await ml.testResources.installFleetPackage(fleetPackage);
       }

x-pack/test/fleet_api_integration/apis/epm/final_pipeline.ts

@@ -12,7 +12,7 @@ import { skipIfNoDockerRegistry } from '../../helpers';
 
 const TEST_INDEX = 'logs-log.log-test';
 
-const FINAL_PIPELINE_ID = '.fleet_final_pipeline';
+const FINAL_PIPELINE_ID = '.fleet_final_pipeline-1';
 
 let pkgKey: string;
 
@@ -43,7 +43,6 @@ export default function (providerContext: FtrProviderContext) {
       const { body: getPackagesRes } = await supertest.get(
         `/api/fleet/epm/packages?experimental=true`
       );
-
       const logPackage = getPackagesRes.response.find((p: any) => p.name === 'log');
       if (!logPackage) {
         throw new Error('No log package');
@@ -85,12 +84,11 @@ export default function (providerContext: FtrProviderContext) {
     it('should correctly setup the final pipeline and apply to fleet managed index template', async () => {
       const pipelineRes = await es.ingest.getPipeline({ id: FINAL_PIPELINE_ID });
       expect(pipelineRes.body).to.have.property(FINAL_PIPELINE_ID);
-
       const res = await es.indices.getIndexTemplate({ name: 'logs-log.log' });
       expect(res.body.index_templates.length).to.be(1);
-      expect(
-        res.body.index_templates[0]?.index_template?.template?.settings?.index?.final_pipeline
-      ).to.be(FINAL_PIPELINE_ID);
+      expect(res.body.index_templates[0]?.index_template?.composed_of).to.contain(
+        '.fleet_component_template-1'
+      );
     });
 
     it('For a doc written without api key should write the correct api key status', async () => {

x-pack/test/fleet_api_integration/apis/epm/install_overrides.ts

@@ -49,6 +49,7 @@ export default function (providerContext: FtrProviderContext) {
         `${templateName}@mappings`,
         `${templateName}@settings`,
         `${templateName}@custom`,
+        '.fleet_component_template-1',
       ]);
 
       ({ body } = await es.transport.request({