[ML] Transforms: Support for terms agg in pivot configurations. (#123634)

Adds support for the terms agg for transform pivot configurations.

x-pack/plugins/transform/common/types/pivot_aggs.ts

@@ -18,6 +18,7 @@ export const PIVOT_SUPPORTED_AGGS = {
   VALUE_COUNT: 'value_count',
   FILTER: 'filter',
   TOP_METRICS: 'top_metrics',
+  TERMS: 'terms',
 } as const;
 
 export type PivotSupportedAggs = typeof PIVOT_SUPPORTED_AGGS[keyof typeof PIVOT_SUPPORTED_AGGS];

x-pack/plugins/transform/public/app/common/index.ts

@@ -39,7 +39,9 @@ export {
   getEsAggFromAggConfig,
   isPivotAggsConfigWithUiSupport,
   isPivotAggsConfigPercentiles,
+  isPivotAggsConfigTerms,
   PERCENTILES_AGG_DEFAULT_PERCENTS,
+  TERMS_AGG_DEFAULT_SIZE,
   pivotAggsFieldSupport,
 } from './pivot_aggs';
 export type {

x-pack/plugins/transform/public/app/common/pivot_aggs.ts

@@ -28,6 +28,7 @@ export function isPivotSupportedAggs(arg: unknown): arg is PivotSupportedAggs {
 }
 
 export const PERCENTILES_AGG_DEFAULT_PERCENTS = [1, 5, 25, 50, 75, 95, 99];
+export const TERMS_AGG_DEFAULT_SIZE = 10;
 
 export const pivotAggsFieldSupport = {
   [KBN_FIELD_TYPES.ATTACHMENT]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT, PIVOT_SUPPORTED_AGGS.FILTER],
@@ -45,6 +46,7 @@ export const pivotAggsFieldSupport = {
     PIVOT_SUPPORTED_AGGS.VALUE_COUNT,
     PIVOT_SUPPORTED_AGGS.FILTER,
     PIVOT_SUPPORTED_AGGS.TOP_METRICS,
+    PIVOT_SUPPORTED_AGGS.TERMS,
   ],
   [KBN_FIELD_TYPES.MURMUR3]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT, PIVOT_SUPPORTED_AGGS.FILTER],
   [KBN_FIELD_TYPES.NUMBER]: [
@@ -63,6 +65,7 @@ export const pivotAggsFieldSupport = {
     PIVOT_SUPPORTED_AGGS.VALUE_COUNT,
     PIVOT_SUPPORTED_AGGS.FILTER,
     PIVOT_SUPPORTED_AGGS.TOP_METRICS,
+    PIVOT_SUPPORTED_AGGS.TERMS,
   ],
   [KBN_FIELD_TYPES._SOURCE]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT, PIVOT_SUPPORTED_AGGS.FILTER],
   [KBN_FIELD_TYPES.UNKNOWN]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT, PIVOT_SUPPORTED_AGGS.FILTER],
@@ -226,9 +229,15 @@ interface PivotAggsConfigPercentiles extends PivotAggsConfigWithUiBase {
   percents: number[];
 }
 
+interface PivotAggsConfigTerms extends PivotAggsConfigWithUiBase {
+  agg: typeof PIVOT_SUPPORTED_AGGS.TERMS;
+  size: number;
+}
+
 export type PivotAggsConfigWithUiSupport =
   | PivotAggsConfigWithUiBase
   | PivotAggsConfigPercentiles
+  | PivotAggsConfigTerms
   | PivotAggsConfigWithExtendedForm;
 
 export function isPivotAggsConfigWithUiSupport(arg: unknown): arg is PivotAggsConfigWithUiSupport {
@@ -258,6 +267,10 @@ export function isPivotAggsConfigPercentiles(arg: unknown): arg is PivotAggsConf
   );
 }
 
+export function isPivotAggsConfigTerms(arg: unknown): arg is PivotAggsConfigTerms {
+  return isPopulatedObject(arg, ['agg', 'field', 'size']) && arg.agg === PIVOT_SUPPORTED_AGGS.TERMS;
+}
+
 export type PivotAggsConfig = PivotAggsConfigBase | PivotAggsConfigWithUiSupport;
 
 export type PivotAggsConfigWithUiSupportDict = Dictionary<PivotAggsConfigWithUiSupport>;

x-pack/plugins/transform/public/app/hooks/use_pivot_data.ts

@@ -13,7 +13,7 @@ import { EuiDataGridColumn } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
 import { getFlattenedObject } from '@kbn/std';
 
-import { sample, difference } from 'lodash';
+import { difference } from 'lodash';
 import { ES_FIELD_TYPES } from '../../../../../../src/plugins/data/common';
 
 import type { PreviewMappingsProperties } from '../../../common/api_schemas/transforms';
@@ -79,12 +79,16 @@ export function getCombinedProperties(
   populatedProperties: PreviewMappingsProperties,
   docs: Array<Record<string, unknown>>
 ): PreviewMappingsProperties {
-  // Take a sample from docs and resolve missing mappings
-  const sampleDoc = sample(docs) ?? {};
-  const missingMappings = difference(Object.keys(sampleDoc), Object.keys(populatedProperties));
+  // Identify missing mappings
+  const missingMappings = difference(
+    // Create an array of unique flattened field names across all docs
+    [...new Set(docs.flatMap(Object.keys))],
+    Object.keys(populatedProperties)
+  );
   return {
     ...populatedProperties,
     ...missingMappings.reduce((acc, curr) => {
+      const sampleDoc = docs.find((d) => typeof d[curr] !== 'undefined') ?? {};
       acc[curr] = { type: typeof sampleDoc[curr] as ES_FIELD_TYPES };
       return acc;
     }, {} as PreviewMappingsProperties),

x-pack/plugins/transform/public/app/sections/create_transform/components/aggregation_list/popover_form.tsx

@@ -13,6 +13,7 @@ import {
   EuiButton,
   EuiCodeBlock,
   EuiComboBox,
+  EuiFieldNumber,
   EuiFieldText,
   EuiForm,
   EuiFormRow,
@@ -32,9 +33,11 @@ import {
 import {
   isAggName,
   isPivotAggsConfigPercentiles,
+  isPivotAggsConfigTerms,
   isPivotAggsConfigWithUiSupport,
   getEsAggFromAggConfig,
   PERCENTILES_AGG_DEFAULT_PERCENTS,
+  TERMS_AGG_DEFAULT_SIZE,
   PivotAggsConfig,
   PivotAggsConfigWithUiSupportDict,
 } from '../../../../common';
@@ -75,6 +78,30 @@ function parsePercentsInput(inputValue: string | undefined) {
   return [];
 }
 
+// Input string should only include comma separated numbers
+function isValidPercentsInput(inputValue: string) {
+  return /^[0-9]+(,[0-9]+)*$/.test(inputValue);
+}
+
+function getDefaultSize(defaultData: PivotAggsConfig): number | undefined {
+  if (isPivotAggsConfigTerms(defaultData)) {
+    return defaultData.size;
+  }
+}
+
+function parseSizeInput(inputValue: string | undefined) {
+  if (inputValue !== undefined && isValidSizeInput(inputValue)) {
+    return parseInt(inputValue, 10);
+  }
+
+  return TERMS_AGG_DEFAULT_SIZE;
+}
+
+// Input string should only include numbers
+function isValidSizeInput(inputValue: string) {
+  return /^\d+$/.test(inputValue);
+}
+
 export const PopoverForm: React.FC<Props> = ({ defaultData, otherAggNames, onChange, options }) => {
   const [aggConfigDef, setAggConfigDef] = useState(cloneDeep(defaultData));
 
@@ -85,6 +112,9 @@ export const PopoverForm: React.FC<Props> = ({ defaultData, otherAggNames, onCha
   );
 
   const [percents, setPercents] = useState(getDefaultPercents(defaultData));
+  const [validPercents, setValidPercents] = useState(agg === PIVOT_SUPPORTED_AGGS.PERCENTILES);
+  const [size, setSize] = useState(getDefaultSize(defaultData));
+  const [validSize, setValidSize] = useState(agg === PIVOT_SUPPORTED_AGGS.TERMS);
 
   const isUnsupportedAgg = !isPivotAggsConfigWithUiSupport(defaultData);
 
@@ -118,10 +148,19 @@ export const PopoverForm: React.FC<Props> = ({ defaultData, otherAggNames, onCha
     if (aggVal === PIVOT_SUPPORTED_AGGS.PERCENTILES && percents === undefined) {
       setPercents(PERCENTILES_AGG_DEFAULT_PERCENTS);
     }
+    if (aggVal === PIVOT_SUPPORTED_AGGS.TERMS && size === undefined) {
+      setSize(TERMS_AGG_DEFAULT_SIZE);
+    }
   }
 
   function updatePercents(inputValue: string) {
     setPercents(parsePercentsInput(inputValue));
+    setValidPercents(isValidPercentsInput(inputValue));
+  }
+
+  function updateSize(inputValue: string) {
+    setSize(parseSizeInput(inputValue));
+    setValidSize(isValidSizeInput(inputValue));
   }
 
   function getUpdatedItem(): PivotAggsConfig {
@@ -137,15 +176,7 @@ export const PopoverForm: React.FC<Props> = ({ defaultData, otherAggNames, onCha
       resultField = field[0];
     }
 
-    if (agg !== PIVOT_SUPPORTED_AGGS.PERCENTILES) {
-      updatedItem = {
-        ...aggConfigDef,
-        agg,
-        aggName,
-        field: resultField,
-        dropDownName: defaultData.dropDownName,
-      };
-    } else {
+    if (agg === PIVOT_SUPPORTED_AGGS.PERCENTILES) {
       updatedItem = {
         agg,
         aggName,
@@ -153,6 +184,22 @@ export const PopoverForm: React.FC<Props> = ({ defaultData, otherAggNames, onCha
         dropDownName: defaultData.dropDownName,
         percents,
       };
+    } else if (agg === PIVOT_SUPPORTED_AGGS.TERMS) {
+      updatedItem = {
+        agg,
+        aggName,
+        field: resultField,
+        dropDownName: defaultData.dropDownName,
+        size,
+      };
+    } else {
+      updatedItem = {
+        ...aggConfigDef,
+        agg,
+        aggName,
+        field: resultField,
+        dropDownName: defaultData.dropDownName,
+      };
     }
 
     return updatedItem;
@@ -202,13 +249,18 @@ export const PopoverForm: React.FC<Props> = ({ defaultData, otherAggNames, onCha
     percentsText = percents.toString();
   }
 
-  const validPercents =
-    agg === PIVOT_SUPPORTED_AGGS.PERCENTILES && parsePercentsInput(percentsText).length > 0;
+  let sizeText;
+  if (size !== undefined) {
+    sizeText = size.toString();
+  }
 
   let formValid = validAggName;
   if (formValid && agg === PIVOT_SUPPORTED_AGGS.PERCENTILES) {
     formValid = validPercents;
   }
+  if (formValid && agg === PIVOT_SUPPORTED_AGGS.TERMS) {
+    formValid = validSize;
+  }
   if (isPivotAggsWithExtendedForm(aggConfigDef)) {
     formValid = validAggName && aggConfigDef.isValid();
   }
@@ -325,6 +377,23 @@ export const PopoverForm: React.FC<Props> = ({ defaultData, otherAggNames, onCha
           />
         </EuiFormRow>
       )}
+      {agg === PIVOT_SUPPORTED_AGGS.TERMS && (
+        <EuiFormRow
+          label={i18n.translate('xpack.transform.agg.popoverForm.sizeLabel', {
+            defaultMessage: 'Size',
+          })}
+          error={
+            !validSize && [
+              i18n.translate('xpack.transform.groupBy.popoverForm.invalidSizeErrorMessage', {
+                defaultMessage: 'Enter a valid positive number',
+              }),
+            ]
+          }
+          isInvalid={!validSize}
+        >
+          <EuiFieldNumber defaultValue={sizeText} onChange={(e) => updateSize(e.target.value)} />
+        </EuiFormRow>
+      )}
       {isUnsupportedAgg && (
         <EuiCodeBlock
           aria-label={i18n.translate('xpack.transform.agg.popoverForm.codeBlock', {

x-pack/test/functional/apps/transform/creation_index_pattern.ts

@@ -337,6 +337,64 @@ export default function ({ getService }: FtrProviderContext) {
           discoverQueryHits: '10',
         },
       } as PivotTransformTestData,
+      {
+        type: 'pivot',
+        suiteTitle: 'batch transform with terms group and terms agg',
+        source: 'ft_ecommerce',
+        groupByEntries: [
+          {
+            identifier: 'terms(customer_gender)',
+            label: 'customer_gender',
+          } as GroupByEntry,
+        ],
+        aggregationEntries: [
+          {
+            identifier: 'terms(geoip.city_name)',
+            label: 'geoip.city_name.terms',
+          },
+        ],
+        transformId: `ec_3_${Date.now()}`,
+        transformDescription:
+          'ecommerce batch transform with group by terms(customer_gender) and aggregation terms(geoip.city_name)',
+        get destinationIndex(): string {
+          return `user-${this.transformId}`;
+        },
+        discoverAdjustSuperDatePicker: false,
+        expected: {
+          pivotAdvancedEditorValueArr: ['{', '  "group_by": {', '    "customer_gender": {'],
+          pivotAdvancedEditorValue: {
+            group_by: {
+              customer_gender: {
+                terms: {
+                  field: 'customer_gender',
+                },
+              },
+            },
+            aggregations: {
+              'geoip.city_name': {
+                terms: {
+                  field: 'geoip.city_name',
+                  size: 3,
+                },
+              },
+            },
+          },
+          transformPreview: {
+            column: 0,
+            values: ['FEMALE', 'MALE'],
+          },
+          row: {
+            status: TRANSFORM_STATE.STOPPED,
+            mode: 'batch',
+            progress: '100',
+          },
+          indexPreview: {
+            columns: 10,
+            rows: 5,
+          },
+          discoverQueryHits: '2',
+        },
+      } as PivotTransformTestData,
       {
         type: 'latest',
         suiteTitle: 'batch transform with the latest function',
@@ -351,7 +409,7 @@ export default function ({ getService }: FtrProviderContext) {
           identifier: 'order_date',
           label: 'order_date',
         },
-        transformId: `ec_3_${Date.now()}`,
+        transformId: `ec_4_${Date.now()}`,
 
         transformDescription:
           'ecommerce batch transform with the latest function config, sort by order_data, country code as unique key',