[ML] Fix missing script aggs on the transform preview table (#103913)

* [ML] get field type from sampled doc for script fields

* [ML] refactor, unit tests

x-pack/plugins/transform/public/app/hooks/use_pivot_data.test.ts

@@ -0,0 +1,78 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { getCombinedProperties } from './use_pivot_data';
+import { ES_FIELD_TYPES } from '../../../../../../src/plugins/data/common';
+
+describe('getCombinedProperties', () => {
+  test('extracts missing mappings from docs', () => {
+    const mappingProps = {
+      testProp: {
+        type: ES_FIELD_TYPES.STRING,
+      },
+    };
+
+    const docs = [
+      {
+        testProp: 'test_value1',
+        scriptProp: 1,
+      },
+      {
+        testProp: 'test_value2',
+        scriptProp: 2,
+      },
+      {
+        testProp: 'test_value3',
+        scriptProp: 3,
+      },
+    ];
+
+    expect(getCombinedProperties(mappingProps, docs)).toEqual({
+      testProp: {
+        type: 'string',
+      },
+      scriptProp: {
+        type: 'number',
+      },
+    });
+  });
+
+  test('does not override defined mappings', () => {
+    const mappingProps = {
+      testProp: {
+        type: ES_FIELD_TYPES.STRING,
+      },
+      scriptProp: {
+        type: ES_FIELD_TYPES.LONG,
+      },
+    };
+
+    const docs = [
+      {
+        testProp: 'test_value1',
+        scriptProp: 1,
+      },
+      {
+        testProp: 'test_value2',
+        scriptProp: 2,
+      },
+      {
+        testProp: 'test_value3',
+        scriptProp: 3,
+      },
+    ];
+
+    expect(getCombinedProperties(mappingProps, docs)).toEqual({
+      testProp: {
+        type: 'string',
+      },
+      scriptProp: {
+        type: 'long',
+      },
+    });
+  });
+});

x-pack/plugins/transform/public/app/hooks/use_pivot_data.ts

@@ -13,6 +13,7 @@ import { EuiDataGridColumn } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
 import { getFlattenedObject } from '@kbn/std';
 
+import { sample, difference } from 'lodash';
 import { ES_FIELD_TYPES } from '../../../../../../src/plugins/data/common';
 
 import type { PreviewMappingsProperties } from '../../../common/api_schemas/transforms';
@@ -71,6 +72,25 @@ function sortColumnsForLatest(sortField: string) {
   };
 }
 
+/**
+ * Extracts missing mappings from docs.
+ */
+export function getCombinedProperties(
+  populatedProperties: PreviewMappingsProperties,
+  docs: Array<Record<string, unknown>>
+): PreviewMappingsProperties {
+  // Take a sample from docs and resolve missing mappings
+  const sampleDoc = sample(docs) ?? {};
+  const missingMappings = difference(Object.keys(sampleDoc), Object.keys(populatedProperties));
+  return {
+    ...populatedProperties,
+    ...missingMappings.reduce((acc, curr) => {
+      acc[curr] = { type: typeof sampleDoc[curr] as ES_FIELD_TYPES };
+      return acc;
+    }, {} as PreviewMappingsProperties),
+  };
+}
+
 export const usePivotData = (
   indexPatternTitle: SearchItems['indexPattern']['title'],
   query: PivotQuery,
@@ -170,7 +190,7 @@ export const usePivotData = (
     const populatedFields = [...new Set(docs.map(Object.keys).flat(1))];
 
     // 3. Filter mapping properties by populated fields
-    const populatedProperties: PreviewMappingsProperties = Object.entries(
+    let populatedProperties: PreviewMappingsProperties = Object.entries(
       resp.generated_dest_index.mappings.properties
     )
       .filter(([key]) => populatedFields.includes(key))
@@ -182,6 +202,8 @@ export const usePivotData = (
         {}
       );
 
+    populatedProperties = getCombinedProperties(populatedProperties, docs);
+
     setTableItems(docs);
     setRowCount(docs.length);
     setRowCountRelation(ES_CLIENT_TOTAL_HITS_RELATION.EQ);