[Response Ops] add ignore_malformed to alerts mappings AGAIN (#165781)

Resolves https://github.com/elastic/kibana/issues/161465

This is a re-do of https://github.com/elastic/kibana/pull/163414, which
we had to revert since data streams do not support `ignore_malformed` on
the `@timestamp` field. We now specifically add `ignore_malformed:
false` for that field, and then use `ignore_malformed: true` at the
index level.

This ignores malformed content globally across all allowed mapping
types. For existing alerts as data indices, the new setting is not
applied directly to the existing concrete indices but will be applied
whenever the alias rolls over and a new concrete index is created.

x-pack/plugins/alerting/common/alert_schema/field_maps/mapping_from_field_map.test.ts

@@ -188,6 +188,7 @@ describe('mappingFromFieldMap', () => {
       dynamic: 'strict',
       properties: {
         '@timestamp': {
+          ignore_malformed: false,
           type: 'date',
         },
         event: {

x-pack/plugins/alerting/common/alert_schema/field_maps/mapping_from_field_map.ts

@@ -43,6 +43,10 @@ export function mappingFromFieldMap(
       : rest;
 
     set(mappings.properties, field.name.split('.').join('.properties.'), mapped);
+
+    if (name === '@timestamp') {
+      set(mappings.properties, `${name}.ignore_malformed`, false);
+    }
   });
 
   return mappings;

x-pack/plugins/alerting/server/alerts_service/alerts_service.test.ts

@@ -141,6 +141,7 @@ const getIndexTemplatePutBody = (opts?: GetIndexTemplatePutBodyOpts) => {
                   rollover_alias: `.alerts-${context ? context : 'test'}.alerts-${namespace}`,
                 },
               }),
+          'index.mapping.ignore_malformed': true,
           'index.mapping.total_fields.limit': 2500,
         },
         mappings: {
@@ -808,6 +809,7 @@ describe('Alerts Service', () => {
                           rollover_alias: `.alerts-empty.alerts-default`,
                         },
                       }),
+                  'index.mapping.ignore_malformed': true,
                   'index.mapping.total_fields.limit': 2500,
                 },
                 mappings: {

x-pack/plugins/alerting/server/alerts_service/lib/create_or_update_index_template.test.ts

@@ -48,6 +48,7 @@ const IndexTemplate = (namespace: string = 'default', useDataStream: boolean = f
                 rollover_alias: `.alerts-test.alerts-${namespace}`,
               },
             }),
+        'index.mapping.ignore_malformed': true,
         'index.mapping.total_fields.limit': 2500,
       },
     },

x-pack/plugins/alerting/server/alerts_service/lib/create_or_update_index_template.ts

@@ -68,6 +68,7 @@ export const getIndexTemplate = ({
             : {
                 'index.lifecycle': indexLifecycle,
               }),
+          'index.mapping.ignore_malformed': true,
           'index.mapping.total_fields.limit': totalFieldsLimit,
         },
         mappings: {

x-pack/plugins/security_solution/server/lib/risk_engine/risk_engine_data_client.test.ts

@@ -175,6 +175,7 @@ describe('RiskEngineDataClient', () => {
                 "dynamic": "strict",
                 "properties": Object {
                   "@timestamp": Object {
+                    "ignore_malformed": false,
                     "type": "date",
                   },
                   "host": Object {
@@ -360,6 +361,7 @@ describe('RiskEngineDataClient', () => {
                 dynamic: 'strict',
                 properties: {
                   '@timestamp': {
+                    ignore_malformed: false,
                     type: 'date',
                   },
                   host: {

x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/alerts_as_data/install_resources.ts

@@ -163,6 +163,7 @@ export default function createAlertsAsDataInstallResourcesTest({ getService }: F
             rollover_alias: '.alerts-test.patternfiring.alerts-default',
           },
           mapping: {
+            ignore_malformed: 'true',
             total_fields: {
               limit: '2500',
             },
@@ -196,6 +197,7 @@ export default function createAlertsAsDataInstallResourcesTest({ getService }: F
       });
 
       expect(contextIndex[indexName].settings?.index?.mapping).to.eql({
+        ignore_malformed: 'true',
         total_fields: {
           limit: '2500',
         },

x-pack/test/detection_engine_api_integration/security_and_spaces/group10/risk_engine/init_and_status_apis.ts

@@ -104,6 +104,7 @@ export default ({ getService }: FtrProviderContext) => {
           dynamic: 'strict',
           properties: {
             '@timestamp': {
+              ignore_malformed: false,
               type: 'date',
             },
             host: {

x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/non_ecs_fields.ts

@@ -56,7 +56,6 @@ export default ({ getService }: FtrProviderContext) => {
     };
   };
 
-  // FAILING ES PROMOTION: https://github.com/elastic/kibana/issues/154277
   describe('Non ECS fields in alert document source', () => {
     before(async () => {
       await esArchiver.load(
@@ -257,9 +256,10 @@ export default ({ getService }: FtrProviderContext) => {
       expect(alertSource).toHaveProperty('client.nat.port', '3000');
     });
 
-    // we don't validate it because geo_point is very complex type with many various representations: array, different object, string with few valid patterns
+    // We don't validate it because geo_point is very complex type with many various representations: array,
-    // more on geo_point type https://www.elastic.co/guide/en/elasticsearch/reference/current/geo-point.html
+    // different object, string with few valid patterns.
-    it('should fail creating alert when ECS field mapping is geo_point', async () => {
+    // More on geo_point type https://www.elastic.co/guide/en/elasticsearch/reference/current/geo-point.html
+    it('should not fail creating alert when ECS field mapping is geo_point', async () => {
       const document = {
         client: {
           geo: {
@@ -269,12 +269,10 @@ export default ({ getService }: FtrProviderContext) => {
         },
       };
 
-      const { errors } = await indexAndCreatePreviewAlert(document);
+      const { errors, alertSource } = await indexAndCreatePreviewAlert(document);
 
-      expect(errors[0]).toContain('Bulk Indexing of signals failed');
+      expect(errors).toEqual([]);
-      expect(errors[0]).toContain(
+      expect(alertSource).toHaveProperty('client.geo.location', 'test test');
-        'failed to parse field [client.geo.location] of type [geo_point]'
-      );
     });
 
     it('should strip invalid boolean values and left valid ones', async () => {