[Fleet] Remove legacy component templates on package install (#130758)

* remove legacy component templates as part of package install

* re-work unit tests

* remove unnecessary await

* check if component templates are in use before deleting

* add integration tests

* PR feedback

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/remove_legacy.test.ts

@@ -0,0 +1,262 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type {
+  ClusterComponentTemplate,
+  IndicesGetIndexTemplateIndexTemplateItem,
+} from '@elastic/elasticsearch/lib/api/types';
+
+import type { Logger } from '@kbn/core/server';
+
+import uuid from 'uuid';
+
+import { loggingSystemMock } from '@kbn/core/server/mocks';
+
+import type { InstallablePackage, RegistryDataStream } from '../../../../types';
+
+import {
+  _getLegacyComponentTemplatesForPackage,
+  _getIndexTemplatesToUsedByMap,
+  _filterComponentTemplatesInUse,
+} from './remove_legacy';
+
+const mockLogger: Logger = loggingSystemMock.create().get();
+
+const pickRandom = (arr: any[]) => arr[Math.floor(Math.random() * arr.length)];
+const pickRandomType = pickRandom.bind(null, ['logs', 'metrics']);
+const createMockDataStream = ({
+  packageName,
+  type,
+  dataset,
+}: {
+  packageName: string;
+  type?: string;
+  dataset?: string;
+}) => {
+  return {
+    type: type || pickRandomType(),
+    dataset: dataset || uuid.v4(),
+    title: packageName,
+    package: packageName,
+    path: 'some_path',
+    release: 'ga',
+  } as RegistryDataStream;
+};
+const createMockComponentTemplate = ({
+  name = 'templateName',
+  packageName,
+}: {
+  name?: string;
+  packageName: string;
+}) => {
+  return {
+    name,
+    component_template: {
+      _meta: {
+        package: { name: packageName },
+      },
+      template: {
+        settings: {},
+      },
+    },
+  } as ClusterComponentTemplate;
+};
+
+const createMockTemplate = ({ name, composedOf = [] }: { name: string; composedOf?: string[] }) =>
+  ({
+    name,
+    index_template: {
+      composed_of: composedOf,
+    },
+  } as IndicesGetIndexTemplateIndexTemplateItem);
+
+const makeArrayOf = (arraySize: number, fn = (i: any) => i) => {
+  return [...Array(arraySize)].map(fn);
+};
+describe('_getLegacyComponentTemplatesForPackage', () => {
+  it('should handle empty templates array', () => {
+    const templates = [] as ClusterComponentTemplate[];
+    const pkg = { name: 'testPkg', data_streams: [] as RegistryDataStream[] } as InstallablePackage;
+
+    const result = _getLegacyComponentTemplatesForPackage(templates, pkg);
+    expect(result).toEqual([]);
+  });
+  it('should return empty array if no legacy templates', () => {
+    const packageName = 'testPkg';
+    const templates = makeArrayOf(1000, () => createMockComponentTemplate({ packageName }));
+    const pkg = {
+      name: packageName,
+      data_streams: makeArrayOf(100, () => createMockDataStream({ packageName })),
+    } as InstallablePackage;
+
+    const result = _getLegacyComponentTemplatesForPackage(templates, pkg);
+    expect(result).toEqual([]);
+  });
+
+  it('should find legacy templates', () => {
+    const packageName = 'testPkg';
+    const legacyTemplates = [
+      'logs-testPkg.dataset@settings',
+      'logs-testPkg.dataset@mappings',
+      'metrics-testPkg.dataset2@mappings',
+      'metrics-testPkg.dataset2@settings',
+    ];
+    const templates = [
+      ...makeArrayOf(100, () => createMockComponentTemplate({ packageName })),
+      ...legacyTemplates.map((name) => createMockComponentTemplate({ name, packageName })),
+    ];
+    const pkg = {
+      name: packageName,
+      data_streams: [
+        ...makeArrayOf(20, () => createMockDataStream({ packageName })),
+        createMockDataStream({ type: 'logs', packageName, dataset: 'testPkg.dataset' }),
+        createMockDataStream({ type: 'metrics', packageName, dataset: 'testPkg.dataset2' }),
+      ],
+    } as InstallablePackage;
+
+    const result = _getLegacyComponentTemplatesForPackage(templates, pkg);
+    expect(result).toEqual(legacyTemplates);
+  });
+
+  it('should only return templates if package name matches as well', () => {
+    const packageName = 'testPkg';
+    const legacyTemplates = [
+      'logs-testPkg.dataset@settings',
+      'logs-testPkg.dataset@mappings',
+      'metrics-testPkg.dataset2@mappings',
+      'metrics-testPkg.dataset2@settings',
+    ];
+    const templates = [
+      ...makeArrayOf(20, () => createMockComponentTemplate({ packageName })),
+      ...legacyTemplates.map((name) =>
+        createMockComponentTemplate({ name, packageName: 'someOtherPkg' })
+      ),
+    ];
+    const pkg = {
+      name: packageName,
+      data_streams: [
+        ...makeArrayOf(20, () => createMockDataStream({ packageName })),
+        createMockDataStream({ type: 'logs', packageName, dataset: 'testPkg.dataset' }),
+        createMockDataStream({ type: 'metrics', packageName, dataset: 'testPkg.dataset2' }),
+      ],
+    } as InstallablePackage;
+
+    const result = _getLegacyComponentTemplatesForPackage(templates, pkg);
+    expect(result).toEqual([]);
+  });
+});
+
+describe('_getIndexTemplatesToUsedByMap', () => {
+  it('should return empty map if no index templates provided', () => {
+    const indexTemplates = [] as IndicesGetIndexTemplateIndexTemplateItem[];
+
+    const result = _getIndexTemplatesToUsedByMap(indexTemplates);
+
+    expect(result.size).toEqual(0);
+  });
+
+  it('should return empty map if no index templates have no component templates', () => {
+    const indexTemplates = [createMockTemplate({ name: 'tmpl1' })];
+
+    const result = _getIndexTemplatesToUsedByMap(indexTemplates);
+
+    expect(result.size).toEqual(0);
+  });
+
+  it('should return correct map if templates have composedOf', () => {
+    const indexTemplates = [
+      createMockTemplate({ name: 'tmpl1' }),
+      createMockTemplate({ name: 'tmpl2', composedOf: ['ctmp1'] }),
+      createMockTemplate({ name: 'tmpl3', composedOf: ['ctmp1', 'ctmp2'] }),
+      createMockTemplate({ name: 'tmpl4', composedOf: ['ctmp3'] }),
+    ];
+
+    const expectedMap = {
+      ctmp1: ['tmpl2', 'tmpl3'],
+      ctmp2: ['tmpl3'],
+      ctmp3: ['tmpl4'],
+    };
+
+    const result = _getIndexTemplatesToUsedByMap(indexTemplates);
+
+    expect(Object.fromEntries(result)).toEqual(expectedMap);
+  });
+});
+
+describe('_filterComponentTemplatesInUse', () => {
+  it('should return empty array if provided with empty component templates', () => {
+    const componentTemplateNames = [] as string[];
+    const indexTemplates = [] as IndicesGetIndexTemplateIndexTemplateItem[];
+
+    const result = _filterComponentTemplatesInUse({
+      componentTemplateNames,
+      indexTemplates,
+      logger: mockLogger,
+    });
+
+    expect(result).toHaveLength(0);
+  });
+
+  it('should remove component template used by index template ', () => {
+    const componentTemplateNames = ['ctmp1', 'ctmp2'] as string[];
+    const indexTemplates = [
+      createMockTemplate({ name: 'tmpl1', composedOf: ['ctmp1'] }),
+    ] as IndicesGetIndexTemplateIndexTemplateItem[];
+
+    const result = _filterComponentTemplatesInUse({
+      componentTemplateNames,
+      indexTemplates,
+      logger: mockLogger,
+    });
+
+    expect(result).toEqual(['ctmp2']);
+  });
+  it('should remove component templates used by one index template ', () => {
+    const componentTemplateNames = ['ctmp1', 'ctmp2', 'ctmp3'] as string[];
+    const indexTemplates = [
+      createMockTemplate({ name: 'tmpl1', composedOf: ['ctmp1', 'ctmp2'] }),
+    ] as IndicesGetIndexTemplateIndexTemplateItem[];
+
+    const result = _filterComponentTemplatesInUse({
+      componentTemplateNames,
+      indexTemplates,
+      logger: mockLogger,
+    });
+
+    expect(result).toEqual(['ctmp3']);
+  });
+  it('should remove component templates used by different index templates ', () => {
+    const componentTemplateNames = ['ctmp1', 'ctmp2', 'ctmp3'] as string[];
+    const indexTemplates = [
+      createMockTemplate({ name: 'tmpl1', composedOf: ['ctmp1'] }),
+      createMockTemplate({ name: 'tmpl2', composedOf: ['ctmp2'] }),
+    ] as IndicesGetIndexTemplateIndexTemplateItem[];
+
+    const result = _filterComponentTemplatesInUse({
+      componentTemplateNames,
+      indexTemplates,
+      logger: mockLogger,
+    });
+
+    expect(result).toEqual(['ctmp3']);
+  });
+  it('should remove component templates used by multiple index templates ', () => {
+    const componentTemplateNames = ['ctmp1', 'ctmp2', 'ctmp3'] as string[];
+    const indexTemplates = [
+      createMockTemplate({ name: 'tmpl1', composedOf: ['ctmp1', 'ctmp2'] }),
+      createMockTemplate({ name: 'tmpl2', composedOf: ['ctmp2', 'ctmp1'] }),
+    ] as IndicesGetIndexTemplateIndexTemplateItem[];
+
+    const result = _filterComponentTemplatesInUse({
+      componentTemplateNames,
+      indexTemplates,
+      logger: mockLogger,
+    });
+
+    expect(result).toEqual(['ctmp3']);
+  });
+});

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/remove_legacy.ts

@@ -0,0 +1,161 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type {
+  ClusterComponentTemplate,
+  IndicesGetIndexTemplateIndexTemplateItem,
+} from '@elastic/elasticsearch/lib/api/types';
+import type { ElasticsearchClient, Logger } from '@kbn/core/server';
+
+import type { InstallablePackage, RegistryDataStream } from '../../../../types';
+import { getRegistryDataStreamAssetBaseName } from '..';
+const LEGACY_TEMPLATE_SUFFIXES = ['@mappings', '@settings'];
+
+const getComponentTemplateWithSuffix = (dataStream: RegistryDataStream, suffix: string) => {
+  const baseName = getRegistryDataStreamAssetBaseName(dataStream);
+
+  return baseName + suffix;
+};
+
+export const _getLegacyComponentTemplatesForPackage = (
+  componentTemplates: ClusterComponentTemplate[],
+  installablePackage: InstallablePackage
+): string[] => {
+  const legacyNamesLookup: Set<string> = new Set();
+
+  // fill a map with all possible @mappings and @settings component
+  // template names for fast lookup below.
+  installablePackage.data_streams?.forEach((ds) => {
+    LEGACY_TEMPLATE_SUFFIXES.forEach((suffix) => {
+      legacyNamesLookup.add(getComponentTemplateWithSuffix(ds, suffix));
+    });
+  });
+
+  return componentTemplates.reduce<string[]>((legacyTemplates, componentTemplate) => {
+    if (!legacyNamesLookup.has(componentTemplate.name)) return legacyTemplates;
+
+    if (componentTemplate.component_template._meta?.package?.name !== installablePackage.name) {
+      return legacyTemplates;
+    }
+
+    return legacyTemplates.concat(componentTemplate.name);
+  }, []);
+};
+
+const _deleteComponentTemplates = async (params: {
+  templateNames: string[];
+  esClient: ElasticsearchClient;
+  logger: Logger;
+}): Promise<void> => {
+  const { templateNames, esClient, logger } = params;
+  const deleteResults = await Promise.allSettled(
+    templateNames.map((name) => esClient.cluster.deleteComponentTemplate({ name }))
+  );
+
+  const errors = deleteResults.filter((r) => r.status === 'rejected') as PromiseRejectedResult[];
+
+  if (errors.length) {
+    const prettyErrors = errors.map((e) => `"${e.reason}"`).join(', ');
+    logger.debug(
+      `Encountered ${errors.length} errors deleting legacy component templates: ${prettyErrors}`
+    );
+  }
+};
+
+export const _getIndexTemplatesToUsedByMap = (
+  indexTemplates: IndicesGetIndexTemplateIndexTemplateItem[]
+) => {
+  const lookupMap: Map<string, string[]> = new Map();
+
+  indexTemplates.forEach(({ name: indexTemplateName, index_template: indexTemplate }) => {
+    const composedOf = indexTemplate?.composed_of;
+
+    if (!composedOf) return;
+
+    composedOf.forEach((componentTemplateName) => {
+      const existingEntry = lookupMap.get(componentTemplateName) || [];
+
+      lookupMap.set(componentTemplateName, existingEntry.concat(indexTemplateName));
+    });
+  });
+  return lookupMap;
+};
+
+const _getAllComponentTemplates = async (esClient: ElasticsearchClient) => {
+  const { component_templates: componentTemplates } = await esClient.cluster.getComponentTemplate();
+
+  return componentTemplates;
+};
+
+const _getAllIndexTemplatesWithComposedOf = async (esClient: ElasticsearchClient) => {
+  const { index_templates: indexTemplates } = await esClient.indices.getIndexTemplate();
+  return indexTemplates.filter((tmpl) => tmpl.index_template.composed_of?.length);
+};
+
+export const _filterComponentTemplatesInUse = ({
+  componentTemplateNames,
+  indexTemplates,
+  logger,
+}: {
+  componentTemplateNames: string[];
+  indexTemplates: IndicesGetIndexTemplateIndexTemplateItem[];
+  logger: Logger;
+}): string[] => {
+  const usedByLookup = _getIndexTemplatesToUsedByMap(indexTemplates);
+
+  return componentTemplateNames.filter((componentTemplateName) => {
+    const indexTemplatesUsingComponentTemplate = usedByLookup.get(componentTemplateName);
+
+    if (indexTemplatesUsingComponentTemplate?.length) {
+      const prettyTemplates = indexTemplatesUsingComponentTemplate.join(', ');
+      logger.debug(
+        `Not deleting legacy template ${componentTemplateName} as it is in use by index templates: ${prettyTemplates}`
+      );
+      return false;
+    }
+
+    return true;
+  });
+};
+
+export const removeLegacyTemplates = async (params: {
+  packageInfo: InstallablePackage;
+  esClient: ElasticsearchClient;
+  logger: Logger;
+}): Promise<void> => {
+  const { packageInfo, esClient, logger } = params;
+
+  const allComponentTemplates = await _getAllComponentTemplates(esClient);
+
+  const legacyComponentTemplateNames = _getLegacyComponentTemplatesForPackage(
+    allComponentTemplates,
+    packageInfo
+  );
+
+  if (!legacyComponentTemplateNames.length) return;
+
+  // all index templates that are composed of at least one component template
+  const allIndexTemplatesWithComposedOf = await _getAllIndexTemplatesWithComposedOf(esClient);
+
+  let templatesToDelete = legacyComponentTemplateNames;
+  if (allIndexTemplatesWithComposedOf.length) {
+    // get the component templates not in use by any index templates
+    templatesToDelete = _filterComponentTemplatesInUse({
+      componentTemplateNames: legacyComponentTemplateNames,
+      indexTemplates: allIndexTemplatesWithComposedOf,
+      logger,
+    });
+  }
+
+  if (!templatesToDelete.length) return;
+
+  await _deleteComponentTemplates({
+    templateNames: templatesToDelete,
+    esClient,
+    logger,
+  });
+};

x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts

@@ -23,6 +23,7 @@ import type { InstallablePackage, InstallSource, PackageAssetReference } from '.
 import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants';
 import type { AssetReference, Installation, InstallType } from '../../../types';
 import { installTemplates } from '../elasticsearch/template/install';
+import { removeLegacyTemplates } from '../elasticsearch/template/remove_legacy';
 import {
   installPipelines,
   isTopLevelPipeline,
@@ -161,6 +162,12 @@ export async function _installPackage({
       savedObjectsClient
     );
 
+    try {
+      await removeLegacyTemplates({ packageInfo, esClient, logger });
+    } catch (e) {
+      logger.warn(`Error removing legacy templates: ${e.message}`);
+    }
+
     // update current backing indices of each data stream
     await updateCurrentWriteIndices(esClient, logger, installedTemplates);
 

x-pack/test/fleet_api_integration/apis/epm/index.js

@@ -27,6 +27,7 @@ export default function loadTests({ loadTestFile }) {
     loadTestFile(require.resolve('./update_assets'));
     loadTestFile(require.resolve('./data_stream'));
     loadTestFile(require.resolve('./package_install_complete'));
+    loadTestFile(require.resolve('./remove_legacy_templates'));
     loadTestFile(require.resolve('./install_error_rollback'));
     loadTestFile(require.resolve('./final_pipeline'));
   });

x-pack/test/fleet_api_integration/apis/epm/remove_legacy_templates.ts

@@ -0,0 +1,152 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import path from 'path';
+import fs from 'fs';
+import { promisify } from 'util';
+import { FtrProviderContext } from '../../../api_integration/ftr_provider_context';
+import { skipIfNoDockerRegistry } from '../../helpers';
+import { setupFleetAndAgents } from '../agents/services';
+const sleep = promisify(setTimeout);
+
+export default function (providerContext: FtrProviderContext) {
+  const { getService } = providerContext;
+  const supertest = getService('supertest');
+  const dockerServers = getService('dockerServers');
+  const server = dockerServers.get('registry');
+  const esClient = getService('es');
+
+  const uploadPkgName = 'apache';
+  const uploadPkgVersion = '0.1.4';
+
+  const installUploadPackage = async () => {
+    const buf = fs.readFileSync(testPkgArchiveZip);
+    await supertest
+      .post(`/api/fleet/epm/packages`)
+      .set('kbn-xsrf', 'xxxx')
+      .type('application/zip')
+      .send(buf)
+      .expect(200);
+  };
+
+  const testPkgArchiveZip = path.join(
+    path.dirname(__filename),
+    '../fixtures/direct_upload_packages/apache_0.1.4.zip'
+  );
+
+  const legacyComponentTemplates = [
+    {
+      name: 'logs-apache.access@settings',
+      template: {
+        settings: {
+          index: {
+            lifecycle: {
+              name: 'idontexist',
+            },
+          },
+        },
+      },
+      _meta: {
+        package: {
+          name: 'apache',
+        },
+      },
+    },
+    {
+      name: 'logs-apache.access@mappings',
+      template: {
+        mappings: {
+          dynamic: false,
+        },
+      },
+      _meta: {
+        package: {
+          name: 'apache',
+        },
+      },
+    },
+  ];
+  const createLegacyComponentTemplates = async () =>
+    Promise.all(
+      legacyComponentTemplates.map((tmpl) => esClient.cluster.putComponentTemplate(tmpl))
+    );
+
+  const deleteLegacyComponentTemplates = async () => {
+    esClient.cluster
+      .deleteComponentTemplate({ name: legacyComponentTemplates.map((t) => t.name) })
+      .catch((e) => {});
+  };
+
+  const waitUntilLegacyComponentTemplatesCreated = async () => {
+    const legacyTemplateNames = legacyComponentTemplates.map((t) => t.name);
+    for (let i = 5; i > 0; i--) {
+      const { component_templates: ctmps } = await esClient.cluster.getComponentTemplate();
+
+      const createdTemplates = ctmps.filter((tmp) => legacyTemplateNames.includes(tmp.name));
+
+      if (createdTemplates.length === legacyTemplateNames.length) return;
+
+      await sleep(500);
+    }
+
+    throw new Error('Legacy component templates not created after 5 attempts');
+  };
+  const uninstallPackage = async (pkg: string, version: string) => {
+    await supertest.delete(`/api/fleet/epm/packages/${pkg}/${version}`).set('kbn-xsrf', 'xxxx');
+  };
+
+  describe('legacy component template removal', async () => {
+    skipIfNoDockerRegistry(providerContext);
+    setupFleetAndAgents(providerContext);
+
+    afterEach(async () => {
+      if (!server.enabled) return;
+      await deleteLegacyComponentTemplates();
+      await uninstallPackage(uploadPkgName, uploadPkgVersion);
+    });
+
+    after(async () => {
+      await esClient.indices.deleteIndexTemplate({ name: 'testtemplate' });
+    });
+
+    it('should remove legacy component templates if not in use by index templates', async () => {
+      await createLegacyComponentTemplates();
+
+      await waitUntilLegacyComponentTemplatesCreated();
+      await installUploadPackage();
+
+      const { component_templates: allComponentTemplates } =
+        await esClient.cluster.getComponentTemplate();
+      const allComponentTemplateNames = allComponentTemplates.map((t) => t.name);
+
+      expect(allComponentTemplateNames.includes('logs-apache.access@settings')).to.equal(false);
+      expect(allComponentTemplateNames.includes('logs-apache.access@mappings')).to.equal(false);
+    });
+
+    it('should not remove legacy component templates if in use by index templates', async () => {
+      await createLegacyComponentTemplates();
+
+      await esClient.indices.putIndexTemplate({
+        name: 'testtemplate',
+        index_patterns: ['nonexistentindices'],
+        template: {},
+        composed_of: ['logs-apache.access@settings', 'logs-apache.access@mappings'],
+      });
+
+      await waitUntilLegacyComponentTemplatesCreated();
+      await installUploadPackage();
+
+      const { component_templates: allComponentTemplates } =
+        await esClient.cluster.getComponentTemplate();
+      const allComponentTemplateNames = allComponentTemplates.map((t) => t.name);
+
+      expect(allComponentTemplateNames.includes('logs-apache.access@settings')).to.equal(true);
+      expect(allComponentTemplateNames.includes('logs-apache.access@mappings')).to.equal(true);
+    });
+  });
+}