[Fleet] Fix to read for Transform destination index mappings from all files under fields folder (#168499)

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts

@@ -25,7 +25,7 @@ import {
   buildComponentTemplates,
   installComponentAndIndexTemplateForDataStream,
 } from '../template/install';
-import { processFields } from '../../fields/field';
+import { isFields, processFields } from '../../fields/field';
 import { generateMappings } from '../template/template';
 import { getESAssetMetadata } from '../meta';
 import { updateEsAssetReferences } from '../../packages/install';
@@ -162,7 +162,6 @@ const processTransformAssetsPerModule = (
       installablePackage,
       path
     );
-
     // Since there can be multiple assets per transform definition
     // We want to create a unique list of assets/specifications for each transform
     if (transformsSpecifications.get(transformModuleId) === undefined) {
@@ -172,7 +171,8 @@ const processTransformAssetsPerModule = (
 
     const content = safeLoad(getAsset(path).toString('utf-8'));
 
-    if (fileName === TRANSFORM_SPECS_TYPES.FIELDS) {
+    // Handling fields.yml and all other files within 'fields' folder
+    if (fileName === TRANSFORM_SPECS_TYPES.FIELDS || isFields(path)) {
       const validFields = processFields(content);
       const mappings = generateMappings(validFields);
       const templateName = getTransformAssetNameForInstallation(
@@ -198,7 +198,14 @@ const processTransformAssetsPerModule = (
       } else {
         destinationIndexTemplates[indexToModify] = template;
       }
-      packageAssets?.set('mappings', mappings);
+
+      // If there's already mappings set previously, append it to new
+      const previousMappings =
+        transformsSpecifications.get(transformModuleId)?.get('mappings') ?? {};
+
+      transformsSpecifications.get(transformModuleId)?.set('mappings', {
+        properties: { ...previousMappings.properties, ...mappings.properties },
+      });
     }
 
     if (fileName === TRANSFORM_SPECS_TYPES.TRANSFORM) {

x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transforms.test.ts

@@ -107,7 +107,34 @@ _meta:
   type: date
 - name: updated_at
   type: alias
-  path: event.ingested`,
+  path: event.ingested
+- external: ecs
+  name: ecs.version
+- external: ecs
+  name: message`,
+      BEATS_FIELDS: `- name: input.type
+  type: keyword
+  description: Type of Filebeat input.
+- name: log.flags
+  type: keyword
+  description: Flags for the log file.
+- name: log.offset
+  type: long
+  description: Offset of the entry in the log file.
+- name: log.file.path
+  type: keyword
+  description: Path to the log file.`,
+      AGENT_FIELDS: `- name: instance.name
+  level: extended
+  type: keyword
+  ignore_above: 1024
+  description: Instance name of the host machine.
+- name: machine.type
+  level: extended
+  type: keyword
+  ignore_above: 1024
+  description: Machine type of the host machine.
+  example: t2.medium`,
     };
   };
   const getExpectedData = (transformVersion: string) => {
@@ -210,6 +237,8 @@ _meta:
       ],
     } as unknown as Installation;
     (getAsset as jest.MockedFunction<typeof getAsset>)
+      .mockReturnValueOnce(Buffer.from(sourceData.BEATS_FIELDS, 'utf8'))
+      .mockReturnValueOnce(Buffer.from(sourceData.AGENT_FIELDS, 'utf8'))
       .mockReturnValueOnce(Buffer.from(sourceData.FIELDS, 'utf8'))
       .mockReturnValueOnce(Buffer.from(sourceData.MANIFEST, 'utf8'))
       .mockReturnValueOnce(Buffer.from(sourceData.TRANSFORM, 'utf8'));
@@ -247,6 +276,8 @@ _meta:
         version: '0.16.0-dev.0',
       } as unknown as RegistryPackage,
       paths: [
+        'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/fields/beats.yml',
+        'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/fields/agent.yml',
         'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/fields/fields.yml',
         'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/manifest.yml',
         'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml',
@@ -300,10 +331,19 @@ _meta:
               },
               mappings: {
                 properties: {
-                  '@timestamp': {
-                    ignore_malformed: false,
-                    type: 'date',
+                  input: { properties: { type: { type: 'keyword', ignore_above: 1024 } } },
+                  log: {
+                    properties: {
+                      flags: { type: 'keyword', ignore_above: 1024 },
+                      offset: { type: 'long' },
+                      file: { properties: { path: { type: 'keyword', ignore_above: 1024 } } },
+                    },
                   },
+                  instance: { properties: { name: { type: 'keyword', ignore_above: 1024 } } },
+                  machine: { properties: { type: { type: 'keyword', ignore_above: 1024 } } },
+                  '@timestamp': { ignore_malformed: false, type: 'date' },
+                  ecs: { properties: { version: { type: 'keyword', ignore_above: 1024 } } },
+                  message: { type: 'keyword', ignore_above: 1024 },
                 },
                 dynamic_templates: [
                   {
@@ -582,6 +622,8 @@ _meta:
                     ignore_malformed: false,
                     type: 'date',
                   },
+                  ecs: { properties: { version: { type: 'keyword', ignore_above: 1024 } } },
+                  message: { type: 'keyword', ignore_above: 1024 },
                 },
                 dynamic_templates: [
                   {
@@ -852,6 +894,8 @@ _meta:
                     ignore_malformed: false,
                     type: 'date',
                   },
+                  ecs: { properties: { version: { type: 'keyword', ignore_above: 1024 } } },
+                  message: { type: 'keyword', ignore_above: 1024 },
                 },
               },
             },

x-pack/plugins/fleet/server/services/epm/fields/field.ts

@@ -284,7 +284,7 @@ export function processFields(fields: Fields): Fields {
   return validateFields(dedupedFields, dedupedFields);
 }
 
-const isFields = (path: string) => {
+export const isFields = (path: string) => {
   return path.includes('/fields/');
 };