[Response Ops][Alerting] Revert ignore_malformed changes (#163610)

Reverting https://github.com/elastic/kibana/pull/163414 and
https://github.com/elastic/kibana/pull/163487

## Summary

@pmuellr uncovered a bug in ES with `ignore_malformed` and datastreams
while working on https://github.com/elastic/kibana/issues/154266

> Found what I hope is an ES bug yesterday w/data streams (DS). It
doesn’t like ignore_malformed on the @timestamp field
🙂. I think this is a bug since [the doc says
(https://www.elastic.co/guide/en/elasticsearch/reference/current/ignore-malformed.html#ignore-malformed-setting)
Mapping types that don’t support the setting will ignore it if set on
the index level.
I think it’s understandable - the @timestamp field is a key field for DS
(can be overridden) - so you’d not be surprised it’s treated specially.
But … why not just ignore it in that case, like the other mapping types
that are ignored.
I tried overriding ignore_malformed for just that field, and it
complained that I couldn’t use that option on that field! hahahahah
So, we’d be left having to add ignore_malformed to every mapped field in
our mappings, except for @timestamp.
For the time being, I’ve removed all the ignore_malformed stuff in my
AaD DS PR, when using DS, but left it when using alias/index.
Unless someone knows more about this special ignored_malformed /
@timestamp field / data-stream relationship, I’ll boil down a simple
test case and open an issue for ES.

In order to avoid having even more divergent code between serverless &
serverful, we will revert this change until we can confirm a bug with ES
and hopefully get a fix in.

x-pack/plugins/alerting/server/alerts_service/alerts_service.test.ts

@@ -114,7 +114,6 @@ const getIndexTemplatePutBody = (opts?: GetIndexTemplatePutBodyOpts) => {
             name: '.alerts-ilm-policy',
             rollover_alias: `.alerts-${context ? context : 'test'}.alerts-${namespace}`,
           },
-          'index.mapping.ignore_malformed': true,
           'index.mapping.total_fields.limit': 2500,
         },
         mappings: {
@@ -641,7 +640,6 @@ describe('Alerts Service', () => {
                 name: '.alerts-ilm-policy',
                 rollover_alias: `.alerts-empty.alerts-default`,
               },
-              'index.mapping.ignore_malformed': true,
               'index.mapping.total_fields.limit': 2500,
             },
             mappings: {

x-pack/plugins/alerting/server/alerts_service/lib/create_or_update_index_template.test.ts

@@ -42,7 +42,6 @@ const IndexTemplate = (namespace: string = 'default') => ({
           name: 'test-ilm-policy',
           rollover_alias: `.alerts-test.alerts-${namespace}`,
         },
-        'index.mapping.ignore_malformed': true,
         'index.mapping.total_fields.limit': 2500,
       },
     },

x-pack/plugins/alerting/server/alerts_service/lib/create_or_update_index_template.ts

@@ -54,7 +54,6 @@ export const getIndexTemplate = ({
             rollover_alias: indexPatterns.alias,
           },
           'index.mapping.total_fields.limit': totalFieldsLimit,
-          'index.mapping.ignore_malformed': true,
         },
         mappings: {
           dynamic: false,

x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/alerts_as_data/install_resources.ts

@@ -163,7 +163,6 @@ export default function createAlertsAsDataInstallResourcesTest({ getService }: F
             rollover_alias: '.alerts-test.patternfiring.alerts-default',
           },
           mapping: {
-            ignore_malformed: 'true',
             total_fields: {
               limit: '2500',
             },
@@ -197,7 +196,6 @@ export default function createAlertsAsDataInstallResourcesTest({ getService }: F
       });
 
       expect(contextIndex[indexName].settings?.index?.mapping).to.eql({
-        ignore_malformed: 'true',
         total_fields: {
           limit: '2500',
         },

x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/non_ecs_fields.ts

@@ -56,6 +56,7 @@ export default ({ getService }: FtrProviderContext) => {
     };
   };
 
+  // FAILING ES PROMOTION: https://github.com/elastic/kibana/issues/154277
   describe('Non ECS fields in alert document source', () => {
     before(async () => {
       await esArchiver.load(
@@ -258,8 +259,7 @@ export default ({ getService }: FtrProviderContext) => {
 
     // we don't validate it because geo_point is very complex type with many various representations: array, different object, string with few valid patterns
     // more on geo_point type https://www.elastic.co/guide/en/elasticsearch/reference/current/geo-point.html
-    // since .alerts-* indices allow _ignore_malformed option, alert will be created for this document
+    it('should fail creating alert when ECS field mapping is geo_point', async () => {
-    it('should not fail creating alert when ECS field mapping is geo_point', async () => {
       const document = {
         client: {
           geo: {
@@ -269,11 +269,12 @@ export default ({ getService }: FtrProviderContext) => {
         },
       };
 
-      const { errors, alertSource } = await indexAndCreatePreviewAlert(document);
+      const { errors } = await indexAndCreatePreviewAlert(document);
 
-      expect(errors).toEqual([]);
+      expect(errors[0]).toContain('Bulk Indexing of signals failed');
-
+      expect(errors[0]).toContain(
-      expect(alertSource).toHaveProperty('client.geo.location', 'test test');
+        'failed to parse field [client.geo.location] of type [geo_point]'
+      );
     });
 
     it('should strip invalid boolean values and left valid ones', async () => {