[Security Solution][Detection engine] refactors lists legacy templates/ILM API calls in Serverless env. (#164844)

## Summary

- moved deletion of legacy index template inside to
`migrateListIndexToDataStream` and `migrateListToDataStream`. That would
allow us not to rely on `410` error to tell if we are in Serverless
environment and `_template` API is blocked. Migrate to DS function is
called only in Stateful environment, as lists indices do not exist in
Serverless
- deletion of legacy index template during migration has also other
benefit: it will be called eventually for every instance of Kibana, and
be more efficient then just calling it during index creation

x-pack/plugins/lists/server/routes/list_index/create_list_index_route.ts

@@ -11,7 +11,7 @@ import { LIST_INDEX } from '@kbn/securitysolution-list-constants';
 
 import { createListIndexResponse } from '../../../common/api';
 import type { ListsPluginRouter } from '../../types';
-import { buildSiemResponse, removeLegacyTemplatesIfExist } from '../utils';
+import { buildSiemResponse } from '../utils';
 import { getListClient } from '..';
 
 export const createListIndexRoute = (router: ListsPluginRouter): void => {
@@ -43,8 +43,6 @@ export const createListIndexRoute = (router: ListsPluginRouter): void => {
           await lists.setListItemTemplate();
         }
 
-        await removeLegacyTemplatesIfExist(lists);
-
         if (listDataStreamExists && listItemDataStreamExists) {
           return siemResponse.error({
             body: `data stream: "${lists.getListName()}" and "${lists.getListItemName()}" already exists`,

x-pack/plugins/lists/server/routes/list_index/delete_list_index_route.ts

@@ -12,7 +12,7 @@ import { LIST_INDEX } from '@kbn/securitysolution-list-constants';
 import { ListClient } from '../../services/lists/list_client';
 import type { ListsPluginRouter } from '../../types';
 import { deleteListIndexResponse } from '../../../common/api';
-import { buildSiemResponse, removeLegacyTemplatesIfExist } from '../utils';
+import { buildSiemResponse } from '../utils';
 import { getListClient } from '..';
 
 /**
@@ -66,11 +66,17 @@ export const deleteListIndexRoute = (router: ListsPluginRouter): void => {
         // ensure data streams deleted if exist
         await deleteDataStreams(lists, listDataStreamExists, listItemDataStreamExists);
 
-        // ensure indices deleted if exist and were not migrated
-        await deleteIndices(lists, listIndexExists, listItemIndexExists);
+        // we need to call this section only if any of these indices exist
+        // to delete indices, ILM policies and legacy templates
+        // ILM policies and legacy templates do not exist on serverless,
+        // so by checking if any of index exists we ensure it is stateful
+        if (listIndexExists || listItemIndexExists) {
+          await deleteIndices(lists, listIndexExists, listItemIndexExists);
+          await lists.deleteLegacyListTemplateIfExists();
+          await lists.deleteLegacyListItemTemplateIfExists();
+        }
 
         await deleteIndexTemplates(lists);
-        await removeLegacyTemplatesIfExist(lists);
 
         const [validated, errors] = validate({ acknowledged: true }, deleteListIndexResponse);
         if (errors != null) {

x-pack/plugins/lists/server/routes/utils/index.ts

@@ -5,7 +5,6 @@
  * 2.0.
  */
 
-export * from './remove_templates_if_exist';
 export * from './get_error_message_exception_list_item';
 export * from './get_error_message_exception_list';
 export * from './get_list_client';

x-pack/plugins/lists/server/routes/utils/remove_templates_if_exist.ts

@@ -1,28 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import { ListClient } from '../../services/lists/list_client';
-
-export const removeLegacyTemplatesIfExist = async (lists: ListClient): Promise<void> => {
-  try {
-    const legacyTemplateExists = await lists.getLegacyListTemplateExists();
-    const legacyTemplateListItemsExists = await lists.getLegacyListItemTemplateExists();
-
-    // Check if the old legacy lists and items template exists and remove it
-    if (legacyTemplateExists) {
-      await lists.deleteLegacyListTemplate();
-    }
-    if (legacyTemplateListItemsExists) {
-      await lists.deleteLegacyListItemTemplate();
-    }
-  } catch (err) {
-    // 410 error is for ES serverless, this API doesn't exist there any more, so _template request returns 410 error
-    if (err.statusCode !== 404 && err.statusCode !== 410) {
-      throw err;
-    }
-  }
-};

x-pack/plugins/lists/server/services/lists/list_client.ts

@@ -331,6 +331,10 @@ export class ListClient {
     if (await this.getListPolicyExists()) {
       await this.deleteListPolicy();
     }
+
+    // as migration will be called eventually for every instance of Kibana, it's more efficient to delete
+    // legacy index template if it exists during migration
+    await this.deleteLegacyListTemplateIfExists();
   };
 
   /**
@@ -353,6 +357,10 @@ export class ListClient {
     if (await this.getListItemPolicyExists()) {
       await this.deleteListItemPolicy();
     }
+
+    // as migration will be called eventually for every instance of Kibana, it's more efficient to delete
+    // legacy index template if it exists during migration
+    await this.deleteLegacyListItemTemplateIfExists();
   };
 
   /**
@@ -588,6 +596,23 @@ export class ListClient {
     return deleteTemplate(esClient, listName);
   };
 
+  /**
+   * Checks if legacy lists template exists and delete it
+   */
+  public deleteLegacyListTemplateIfExists = async (): Promise<void> => {
+    try {
+      const legacyTemplateExists = await this.getLegacyListTemplateExists();
+
+      if (legacyTemplateExists) {
+        await this.deleteLegacyListTemplate();
+      }
+    } catch (err) {
+      if (err.statusCode !== 404) {
+        throw err;
+      }
+    }
+  };
+
   /**
    * Delete the list item boot strap index for ILM policies.
    * @returns The contents of the bootstrap response from Elasticsearch
@@ -598,6 +623,23 @@ export class ListClient {
     return deleteTemplate(esClient, listItemName);
   };
 
+  /**
+   * Checks if legacy list item template exists and delete it
+   */
+  public deleteLegacyListItemTemplateIfExists = async (): Promise<void> => {
+    try {
+      const legacyTemplateListItemsExists = await this.getLegacyListItemTemplateExists();
+
+      if (legacyTemplateListItemsExists) {
+        await this.deleteLegacyListItemTemplate();
+      }
+    } catch (err) {
+      if (err.statusCode !== 404) {
+        throw err;
+      }
+    }
+  };
+
   /**
    * Given a list item id, this will delete the single list item
    * @returns The list item if found, otherwise null