[8.8] [Response Ops][Alerting] Adding null checks when iterating through index template list (#158742) (#158820)

# Backport

This will backport the following commits from `main` to `8.8`:
- [[Response Ops][Alerting] Adding null checks when iterating through
index template list
(#158742)](https://github.com/elastic/kibana/pull/158742)

<!--- Backport version: 8.9.7 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Ying
Mao","email":"ying.mao@elastic.co"},"sourceCommit":{"committedDate":"2023-06-01T11:56:51Z","message":"[Response
Ops][Alerting] Adding null checks when iterating through index template
list (#158742)\n\n## Summary\r\n\r\nWhen updating common component
templates during AAD resource\r\ninstallation, we occassionally run into
errors where the index templates\r\nusing the common component template
has a total field limit that is less\r\nthan the new total number of
fields with the updated component template.\r\nWhen this occurs, we
query the ES index template API to get the list of\r\nindex templates in
order to check their `composed_of` field to see if\r\nthey reference the
specific component template and act accordingly. In\r\ntheory,
`composed_of` is (and is typed as) a required array. In\r\npractice, it
seems that this field can be missing from the response.\r\nSince we're
doing a `.includes` check on this array, this can lead to\r\nnull
dereference errors that can halt alerts as data
resource\r\ninstallation.\r\n\r\nThis PR adds a check to make sure the
`composed_of` field exists before\r\nusing it.\r\n\r\n## To Verify\r\n1.
Run Kibana 8.6 locally and create a metric threshold rule &
detection\r\nrule that generates alerts. Make sure the alert index
templates for\r\nthese rule types have been created and the rule runs
successfully.\r\n2. Update to Kibana 8.7. Make sure the rules run
successfully and\r\ngenerate alerts. Create a data view with no
component templates. To do\r\nthis, go to `Stack Management > Index
Management > Index Templates` and\r\nclick `Create template`. Fill in a
name and index pattern (`test*` is\r\nfine) and make sure the `Create
data stream` switch is checked. Click\r\nthrough all the remaining
options without setting anything else. Use\r\n`GET
/_index_template/<name>` to verify that this index template has
no\r\n`composed_of` field\r\n3. Update to this branch. All alert
resources should be installed\r\nsuccessfully and there should be no
errors on
startup.","sha":"0f02b9e968f43bdae1c26c0149f47e6266e85827","branchLabelMapping":{"^v8.9.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["bug","release_note:fix","Feature:Alerting","Team:ResponseOps","v8.9.0","v8.8.1"],"number":158742,"url":"https://github.com/elastic/kibana/pull/158742","mergeCommit":{"message":"[Response
Ops][Alerting] Adding null checks when iterating through index template
list (#158742)\n\n## Summary\r\n\r\nWhen updating common component
templates during AAD resource\r\ninstallation, we occassionally run into
errors where the index templates\r\nusing the common component template
has a total field limit that is less\r\nthan the new total number of
fields with the updated component template.\r\nWhen this occurs, we
query the ES index template API to get the list of\r\nindex templates in
order to check their `composed_of` field to see if\r\nthey reference the
specific component template and act accordingly. In\r\ntheory,
`composed_of` is (and is typed as) a required array. In\r\npractice, it
seems that this field can be missing from the response.\r\nSince we're
doing a `.includes` check on this array, this can lead to\r\nnull
dereference errors that can halt alerts as data
resource\r\ninstallation.\r\n\r\nThis PR adds a check to make sure the
`composed_of` field exists before\r\nusing it.\r\n\r\n## To Verify\r\n1.
Run Kibana 8.6 locally and create a metric threshold rule &
detection\r\nrule that generates alerts. Make sure the alert index
templates for\r\nthese rule types have been created and the rule runs
successfully.\r\n2. Update to Kibana 8.7. Make sure the rules run
successfully and\r\ngenerate alerts. Create a data view with no
component templates. To do\r\nthis, go to `Stack Management > Index
Management > Index Templates` and\r\nclick `Create template`. Fill in a
name and index pattern (`test*` is\r\nfine) and make sure the `Create
data stream` switch is checked. Click\r\nthrough all the remaining
options without setting anything else. Use\r\n`GET
/_index_template/<name>` to verify that this index template has
no\r\n`composed_of` field\r\n3. Update to this branch. All alert
resources should be installed\r\nsuccessfully and there should be no
errors on
startup.","sha":"0f02b9e968f43bdae1c26c0149f47e6266e85827"}},"sourceBranch":"main","suggestedTargetBranches":["8.8"],"targetPullRequestStates":[{"branch":"main","label":"v8.9.0","labelRegex":"^v8.9.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/158742","number":158742,"mergeCommit":{"message":"[Response
Ops][Alerting] Adding null checks when iterating through index template
list (#158742)\n\n## Summary\r\n\r\nWhen updating common component
templates during AAD resource\r\ninstallation, we occassionally run into
errors where the index templates\r\nusing the common component template
has a total field limit that is less\r\nthan the new total number of
fields with the updated component template.\r\nWhen this occurs, we
query the ES index template API to get the list of\r\nindex templates in
order to check their `composed_of` field to see if\r\nthey reference the
specific component template and act accordingly. In\r\ntheory,
`composed_of` is (and is typed as) a required array. In\r\npractice, it
seems that this field can be missing from the response.\r\nSince we're
doing a `.includes` check on this array, this can lead to\r\nnull
dereference errors that can halt alerts as data
resource\r\ninstallation.\r\n\r\nThis PR adds a check to make sure the
`composed_of` field exists before\r\nusing it.\r\n\r\n## To Verify\r\n1.
Run Kibana 8.6 locally and create a metric threshold rule &
detection\r\nrule that generates alerts. Make sure the alert index
templates for\r\nthese rule types have been created and the rule runs
successfully.\r\n2. Update to Kibana 8.7. Make sure the rules run
successfully and\r\ngenerate alerts. Create a data view with no
component templates. To do\r\nthis, go to `Stack Management > Index
Management > Index Templates` and\r\nclick `Create template`. Fill in a
name and index pattern (`test*` is\r\nfine) and make sure the `Create
data stream` switch is checked. Click\r\nthrough all the remaining
options without setting anything else. Use\r\n`GET
/_index_template/<name>` to verify that this index template has
no\r\n`composed_of` field\r\n3. Update to this branch. All alert
resources should be installed\r\nsuccessfully and there should be no
errors on
startup.","sha":"0f02b9e968f43bdae1c26c0149f47e6266e85827"}},{"branch":"8.8","label":"v8.8.1","labelRegex":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->

Co-authored-by: Ying Mao <ying.mao@elastic.co>

x-pack/plugins/alerting/server/alerts_service/lib/create_or_update_component_template.test.ts

@@ -189,6 +189,112 @@ describe('createOrUpdateComponentTemplate', () => {
     });
   });
 
+  it(`should update index template field limit and retry if putTemplate throws error with field limit error when there are malformed index templates`, async () => {
+    clusterClient.cluster.putComponentTemplate.mockRejectedValueOnce(
+      new EsErrors.ResponseError(
+        elasticsearchClientMock.createApiResponse({
+          statusCode: 400,
+          body: {
+            error: {
+              root_cause: [
+                {
+                  type: 'illegal_argument_exception',
+                  reason:
+                    'updating component template [.alerts-ecs-mappings] results in invalid composable template [.alerts-security.alerts-default-index-template] after templates are merged',
+                },
+              ],
+              type: 'illegal_argument_exception',
+              reason:
+                'updating component template [.alerts-ecs-mappings] results in invalid composable template [.alerts-security.alerts-default-index-template] after templates are merged',
+              caused_by: {
+                type: 'illegal_argument_exception',
+                reason:
+                  'composable template [.alerts-security.alerts-default-index-template] template after composition with component templates [.alerts-ecs-mappings, .alerts-security.alerts-mappings, .alerts-technical-mappings] is invalid',
+                caused_by: {
+                  type: 'illegal_argument_exception',
+                  reason:
+                    'invalid composite mappings for [.alerts-security.alerts-default-index-template]',
+                  caused_by: {
+                    type: 'illegal_argument_exception',
+                    reason: 'Limit of total fields [1900] has been exceeded',
+                  },
+                },
+              },
+            },
+          },
+        })
+      )
+    );
+    const existingIndexTemplate = {
+      name: 'test-template',
+      index_template: {
+        index_patterns: ['test*'],
+        composed_of: ['test-mappings'],
+        template: {
+          settings: {
+            auto_expand_replicas: '0-1',
+            hidden: true,
+            'index.lifecycle': {
+              name: '.alerts-ilm-policy',
+              rollover_alias: `.alerts-empty-default`,
+            },
+            'index.mapping.total_fields.limit': 1800,
+          },
+          mappings: {
+            dynamic: false,
+          },
+        },
+      },
+    };
+
+    clusterClient.indices.getIndexTemplate.mockResolvedValueOnce({
+      index_templates: [
+        existingIndexTemplate,
+        {
+          name: 'lyndon',
+          // @ts-expect-error
+          index_template: {
+            index_patterns: ['intel*'],
+          },
+        },
+        {
+          name: 'sample_ds',
+          // @ts-expect-error
+          index_template: {
+            index_patterns: ['sample_ds-*'],
+            data_stream: {
+              hidden: false,
+              allow_custom_routing: false,
+            },
+          },
+        },
+      ],
+    });
+
+    await createOrUpdateComponentTemplate({
+      logger,
+      esClient: clusterClient,
+      template: ComponentTemplate,
+      totalFieldsLimit: 2500,
+    });
+
+    expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(2);
+    expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledTimes(1);
+    expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledWith({
+      name: existingIndexTemplate.name,
+      body: {
+        ...existingIndexTemplate.index_template,
+        template: {
+          ...existingIndexTemplate.index_template.template,
+          settings: {
+            ...existingIndexTemplate.index_template.template?.settings,
+            'index.mapping.total_fields.limit': 2500,
+          },
+        },
+      },
+    });
+  });
+
   it(`should retry getIndexTemplate and putIndexTemplate on transient ES errors`, async () => {
     clusterClient.cluster.putComponentTemplate.mockRejectedValueOnce(
       new EsErrors.ResponseError(

x-pack/plugins/alerting/server/alerts_service/lib/create_or_update_component_template.ts

@@ -32,8 +32,16 @@ const getIndexTemplatesUsingComponentTemplate = async (
     { logger }
   );
   const indexTemplatesUsingComponentTemplate = (indexTemplates ?? []).filter(
-    (indexTemplate: IndicesGetIndexTemplateIndexTemplateItem) =>
-      indexTemplate.index_template.composed_of.includes(componentTemplateName)
+    (indexTemplate: IndicesGetIndexTemplateIndexTemplateItem) => {
+      if (
+        indexTemplate &&
+        indexTemplate.index_template &&
+        indexTemplate.index_template.composed_of
+      ) {
+        return indexTemplate.index_template.composed_of.includes(componentTemplateName);
+      }
+      return false;
+    }
   );
   await asyncForEach(
     indexTemplatesUsingComponentTemplate,