Setup ignore_malformed in fleet (#157184)

## Summary

Add default setting `ignore_malformed: true` to all datastream of type
`logs`. Since the field `@timestamp` needs to have `ignore_malformed:
false` in the mappings, I'm setting this setting automatically even if
not defined in the integration.

I had to fix a bug that prevented index.mappings to be copied from the
default settings into the template settings

### Checklist

Delete any items that are not applicable to this PR.

- [ ] Any text added follows [EUI's writing
guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses
sentence case text and includes [i18n
support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md)
- [ ]
[Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html)
was added for features that require explanation or tutorials
- [ ] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [ ] Any UI touched in this PR is usable by keyboard only (learn more
about [keyboard accessibility](https://webaim.org/techniques/keyboard/))
- [ ] Any UI touched in this PR does not create any new axe failures
(run axe in browser:
[FF](https://addons.mozilla.org/en-US/firefox/addon/axe-devtools/),
[Chrome](https://chrome.google.com/webstore/detail/axe-web-accessibility-tes/lhdoppojpmngadmnindnejefpokejbdd?hl=en-US))
- [ ] If a plugin configuration key changed, check if it needs to be
allowlisted in the cloud and added to the [docker
list](https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker)
- [ ] This renders correctly on smaller devices using a responsive
layout. (You can test this [in your
browser](https://www.browserstack.com/guide/responsive-testing-on-local-server))
- [ ] This was checked for [cross-browser
compatibility](https://www.elastic.co/support/matrix#matrix_browsers)


### For maintainers

- [ ] This was checked for breaking API changes and was [labeled
appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Kyle Pollich <kyle.pollich@elastic.co>

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.test.ts

@@ -69,6 +69,9 @@ describe('buildDefaultSettings', () => {
           "lifecycle": Object {
             "name": "logs",
           },
+          "mapping": Object {
+            "ignore_malformed": true,
+          },
           "query": Object {
             "default_field": Array [
               "field1Keyword",

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.ts

@@ -75,6 +75,14 @@ export function buildDefaultSettings({
           }),
       // What should be our default for the compression?
       codec: 'best_compression',
+      // setting `ignore_malformed` only for data_stream for logs
+      ...(type === 'logs'
+        ? {
+            mapping: {
+              ignore_malformed: true,
+            },
+          }
+        : {}),
       // All the default fields which should be queried have to be added here.
       // So far we add all keyword and text fields here if there are any, otherwise
       // this setting is skipped.

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts

@@ -426,6 +426,11 @@ function generateDateMapping(field: Field): IndexTemplateMapping {
   if (field.date_format) {
     mapping.format = field.date_format;
   }
+
+  if (field.name === '@timestamp') {
+    mapping.ignore_malformed = false;
+  }
+
   return mapping;
 }
 

x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transforms.test.ts

@@ -298,7 +298,12 @@ _meta:
                 },
               },
               mappings: {
-                properties: { '@timestamp': { type: 'date' } },
+                properties: {
+                  '@timestamp': {
+                    ignore_malformed: false,
+                    type: 'date',
+                  },
+                },
                 dynamic_templates: [
                   {
                     strings_as_keyword: {
@@ -581,7 +586,12 @@ _meta:
                 },
               },
               mappings: {
-                properties: { '@timestamp': { type: 'date' } },
+                properties: {
+                  '@timestamp': {
+                    ignore_malformed: false,
+                    type: 'date',
+                  },
+                },
                 dynamic_templates: [
                   {
                     strings_as_keyword: {
@@ -844,7 +854,14 @@ _meta:
           body: {
             template: {
               settings: { index: { mapping: { total_fields: { limit: '10000' } } } },
-              mappings: { properties: { '@timestamp': { type: 'date' } } },
+              mappings: {
+                properties: {
+                  '@timestamp': {
+                    ignore_malformed: false,
+                    type: 'date',
+                  },
+                },
+              },
             },
             _meta: meta,
           },

x-pack/test/fleet_api_integration/apis/epm/data_stream.ts

@@ -270,7 +270,7 @@ export default function (providerContext: FtrProviderContext) {
                 ...packagePolicyData.package,
                 experimental_data_stream_features: [
                   {
-                    data_stream: logsTemplateName,
+                    data_stream: metricsTemplateName,
                     features: {
                       synthetic_source: true,
                     },
@@ -287,7 +287,7 @@ export default function (providerContext: FtrProviderContext) {
       const pkgName = 'no_tsdb_to_tsdb';
       const pkgVersion = '0.1.0';
       const pkgUpdateVersion = '0.2.0';
-      const logsTemplateName = `logs-${pkgName}.test`;
+      const metricsTemplateName = `metrics-${pkgName}.test`;
       const namespace = 'default';
 
       skipIfNoDockerRegistry(providerContext);
@@ -300,7 +300,7 @@ export default function (providerContext: FtrProviderContext) {
         await es.transport.request(
           {
             method: 'POST',
-            path: `/${logsTemplateName}-${namespace}/_doc`,
+            path: `/${metricsTemplateName}-${namespace}/_doc`,
             body: {
               '@timestamp': '2015-01-01',
               logs_test_name: 'test',
@@ -319,7 +319,7 @@ export default function (providerContext: FtrProviderContext) {
         await es.transport.request(
           {
             method: 'DELETE',
-            path: `/_data_stream/${logsTemplateName}-${namespace}`,
+            path: `/_data_stream/${metricsTemplateName}-${namespace}`,
           },
           { meta: true }
         );
@@ -330,15 +330,15 @@ export default function (providerContext: FtrProviderContext) {
       it('rolls over data stream when index_mode: time_series is set in the updated package version', async () => {
         await installPackage(pkgName, pkgUpdateVersion);
 
-        const resLogsDatastream = await es.transport.request<any>(
+        const resMetricsDatastream = await es.transport.request<any>(
           {
             method: 'GET',
-            path: `/_data_stream/${logsTemplateName}-${namespace}`,
+            path: `/_data_stream/${metricsTemplateName}-${namespace}`,
           },
           { meta: true }
         );
 
-        expect(resLogsDatastream.body.data_streams[0].indices.length).equal(2);
+        expect(resMetricsDatastream.body.data_streams[0].indices.length).equal(2);
       });
     });
   });

x-pack/test/fleet_api_integration/apis/epm/install_overrides.ts

@@ -137,6 +137,7 @@ export default function (providerContext: FtrProviderContext) {
                 name: 'overridden by user',
               },
               mapping: {
+                ignore_malformed: `true`,
                 total_fields: {
                   limit: '10000',
                 },
@@ -148,6 +149,7 @@ export default function (providerContext: FtrProviderContext) {
             dynamic: 'false',
             properties: {
               '@timestamp': {
+                ignore_malformed: false,
                 type: 'date',
               },
               data_stream: {

x-pack/test/fleet_api_integration/apis/epm/update_assets.ts

@@ -136,6 +136,7 @@ export default function (providerContext: FtrProviderContext) {
         resPackage.body.component_templates[0].component_template.template.mappings.properties
       ).eql({
         '@timestamp': {
+          ignore_malformed: false,
           type: 'date',
         },
         test_logs2: {
@@ -222,6 +223,7 @@ export default function (providerContext: FtrProviderContext) {
             name: 'reference2',
           },
           mapping: {
+            ignore_malformed: `true`,
             total_fields: {
               limit: '10000',
             },
@@ -235,6 +237,7 @@ export default function (providerContext: FtrProviderContext) {
         dynamic: true,
         properties: {
           '@timestamp': {
+            ignore_malformed: false,
             type: 'date',
           },
           data_stream: {
@@ -301,6 +304,7 @@ export default function (providerContext: FtrProviderContext) {
         resPackage.body.component_templates[0].component_template.template.mappings.properties
       ).eql({
         '@timestamp': {
+          ignore_malformed: false,
           type: 'date',
         },
         metrics_test_name2: {

x-pack/test/fleet_api_integration/apis/fixtures/test_packages/no_tsdb_to_tsdb/0.1.0/data_stream/test/manifest.yml

@@ -1,6 +1,6 @@
 title: Test Dataset
 
-type: logs
+type: metrics
 
 elasticsearch:
   index_template.mappings:

x-pack/test/fleet_api_integration/apis/fixtures/test_packages/no_tsdb_to_tsdb/0.2.0/data_stream/test/manifest.yml

@@ -1,6 +1,6 @@
 title: Test Dataset
 
-type: logs
+type: metrics
 
 elasticsearch:
   index_mode: time_series

x-pack/test/fleet_api_integration/apis/package_policy/input_package_create_upgrade.ts

@@ -223,7 +223,10 @@ export default function (providerContext: FtrProviderContext) {
                 lifecycle: { name: 'logs' },
                 codec: 'best_compression',
                 default_pipeline: 'logs-dataset1-1.0.0',
-                mapping: { total_fields: { limit: '10000' } },
+                mapping: {
+                  total_fields: { limit: '10000' },
+                  ignore_malformed: 'true',
+                },
               },
             },
             mappings: {

x-pack/test/functional/es_archives/endpoint/resolver/api_feature/mappings.json

@@ -2689,7 +2689,7 @@
           "rollover_alias": "endgame-4.21.0"
         },
         "mapping": {
-          "ignore_malformed": "true",
+          "ignore_malformed": true,
           "total_fields": {
             "limit": "10000"
           }