[ML] Extended group-by/aggs support. (#36421) (#36678)

- Adds pivotAggsFieldSupport and pivotGroupByFieldSupport. These objects have a mapping of field_type -> array of supported_aggs and allow a better organization of available options in the dropdowns for group-by and aggs.
- The support for certain field_type/aggs combinations has been extended:
  - value_count is available for every field
  - IP fields are now available for group by and cardinality
  - number and date support all aggregations
  - _id, _type, _index fields are no longer available
- Removes unnecessary union type PivotAggSupportedAggs.
- Moves x-pack/plugins/ml/common/constants/field_types.js to field_types.ts. GitHub doesn't recognize it as a rename though because of TypeScript changes.

x-pack/plugins/ml/common/constants/field_types.js

@@ -1,58 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-
-
-export const ES_FIELD_TYPES = {
-  ATTACHMENT: 'attachment',
-  BOOLEAN: 'boolean',
-  BYTE: 'byte',
-  DATE: 'date',
-  DOUBLE: 'double',
-  FLOAT: 'float',
-  GEO_POINT: 'geo_point',
-  GEO_SHAPE: 'geo_shape',
-  HALF_FLOAT: 'half_float',
-  INTEGER: 'integer',
-  IP: 'ip',
-  KEYWORD: 'keyword',
-  LONG: 'long',
-  MURMUR3: 'murmur3',
-  SCALED_FLOAT: 'scaled_float',
-  SHORT: 'short',
-  TEXT: 'text',
-  TOKEN_COUNT: 'token_count',
-  _ID: '_id',
-  _SOURCE: '_source',
-  _TYPE: '_type',
-};
-
-export const KBN_FIELD_TYPES = {
-  ATTACHMENT: 'attachment',
-  BOOLEAN: 'boolean',
-  DATE: 'date',
-  GEO_POINT: 'geo_point',
-  GEO_SHAPE: 'geo_shape',
-  IP: 'ip',
-  MURMUR3: 'murmur3',
-  NUMBER: 'number',
-  STRING: 'string',
-  _SOURCE: '_source',
-  UNKNOWN: 'unknown',
-  CONFLICT: 'conflict',
-};
-
-export const ML_JOB_FIELD_TYPES = {
-  BOOLEAN: 'boolean',
-  DATE: 'date',
-  GEO_POINT: 'geo_point',
-  IP: 'ip',
-  KEYWORD: 'keyword',
-  NUMBER: 'number',
-  TEXT: 'text',
-  UNKNOWN: 'unknown',
-};
-

x-pack/plugins/ml/common/constants/field_types.ts

@@ -0,0 +1,55 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export enum ES_FIELD_TYPES {
+  ATTACHMENT = 'attachment',
+  BOOLEAN = 'boolean',
+  BYTE = 'byte',
+  DATE = 'date',
+  DOUBLE = 'double',
+  FLOAT = 'float',
+  GEO_POINT = 'geo_point',
+  GEO_SHAPE = 'geo_shape',
+  HALF_FLOAT = 'half_float',
+  INTEGER = 'integer',
+  IP = 'ip',
+  KEYWORD = 'keyword',
+  LONG = 'long',
+  MURMUR3 = 'murmur3',
+  SCALED_FLOAT = 'scaled_float',
+  SHORT = 'short',
+  TEXT = 'text',
+  TOKEN_COUNT = 'token_count',
+  _ID = '_id',
+  _SOURCE = '_source',
+  _TYPE = '_type',
+}
+
+export enum KBN_FIELD_TYPES {
+  ATTACHMENT = 'attachment',
+  BOOLEAN = 'boolean',
+  DATE = 'date',
+  GEO_POINT = 'geo_point',
+  GEO_SHAPE = 'geo_shape',
+  IP = 'ip',
+  MURMUR3 = 'murmur3',
+  NUMBER = 'number',
+  STRING = 'string',
+  _SOURCE = '_source',
+  UNKNOWN = 'unknown',
+  CONFLICT = 'conflict',
+}
+
+export enum ML_JOB_FIELD_TYPES {
+  BOOLEAN = 'boolean',
+  DATE = 'date',
+  GEO_POINT = 'geo_point',
+  IP = 'ip',
+  KEYWORD = 'keyword',
+  NUMBER = 'number',
+  TEXT = 'text',
+  UNKNOWN = 'unknown',
+}

x-pack/plugins/ml/public/data_frame/common/aggregations.ts

@@ -5,7 +5,6 @@
  */
 
 export type AggName = string;
-export type FieldName = string;
 
 export function isAggName(arg: any): arg is AggName {
   // allow all characters except `[]>` and must not start or end with a space.

x-pack/plugins/ml/public/data_frame/common/fields.ts

@@ -0,0 +1,7 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export type FieldName = string;

x-pack/plugins/ml/public/data_frame/common/index.ts

@@ -5,6 +5,7 @@
  */
 
 export * from './aggregations';
+export * from './fields';
 export * from './dropdown';
 export * from './kibana_context';
 export * from './navigation';

x-pack/plugins/ml/public/data_frame/common/pivot_aggs.ts

@@ -5,8 +5,10 @@
  */
 
 import { Dictionary } from '../../../common/types/common';
+import { KBN_FIELD_TYPES } from '../../../common/constants/field_types';
 
-import { AggName, FieldName } from './aggregations';
+import { AggName } from './aggregations';
+import { FieldName } from './fields';
 
 export enum PIVOT_SUPPORTED_AGGS {
   AVG = 'avg',
@@ -17,25 +19,46 @@ export enum PIVOT_SUPPORTED_AGGS {
   VALUE_COUNT = 'value_count',
 }
 
-type PivotAggSupportedAggs =
-  | PIVOT_SUPPORTED_AGGS.AVG
-  | PIVOT_SUPPORTED_AGGS.CARDINALITY
-  | PIVOT_SUPPORTED_AGGS.MAX
-  | PIVOT_SUPPORTED_AGGS.MIN
-  | PIVOT_SUPPORTED_AGGS.SUM
-  | PIVOT_SUPPORTED_AGGS.VALUE_COUNT;
-
-export const pivotSupportedAggs = [
+export const pivotSupportedAggs: PIVOT_SUPPORTED_AGGS[] = [
   PIVOT_SUPPORTED_AGGS.AVG,
   PIVOT_SUPPORTED_AGGS.CARDINALITY,
   PIVOT_SUPPORTED_AGGS.MAX,
   PIVOT_SUPPORTED_AGGS.MIN,
   PIVOT_SUPPORTED_AGGS.SUM,
   PIVOT_SUPPORTED_AGGS.VALUE_COUNT,
-] as PivotAggSupportedAggs[];
+];
+
+export const pivotAggsFieldSupport = {
+  [KBN_FIELD_TYPES.ATTACHMENT]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT],
+  [KBN_FIELD_TYPES.BOOLEAN]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT],
+  [KBN_FIELD_TYPES.DATE]: [
+    PIVOT_SUPPORTED_AGGS.AVG,
+    PIVOT_SUPPORTED_AGGS.CARDINALITY,
+    PIVOT_SUPPORTED_AGGS.MAX,
+    PIVOT_SUPPORTED_AGGS.MIN,
+    PIVOT_SUPPORTED_AGGS.SUM,
+    PIVOT_SUPPORTED_AGGS.VALUE_COUNT,
+  ],
+  [KBN_FIELD_TYPES.GEO_POINT]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT],
+  [KBN_FIELD_TYPES.GEO_SHAPE]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT],
+  [KBN_FIELD_TYPES.IP]: [PIVOT_SUPPORTED_AGGS.CARDINALITY, PIVOT_SUPPORTED_AGGS.VALUE_COUNT],
+  [KBN_FIELD_TYPES.MURMUR3]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT],
+  [KBN_FIELD_TYPES.NUMBER]: [
+    PIVOT_SUPPORTED_AGGS.AVG,
+    PIVOT_SUPPORTED_AGGS.CARDINALITY,
+    PIVOT_SUPPORTED_AGGS.MAX,
+    PIVOT_SUPPORTED_AGGS.MIN,
+    PIVOT_SUPPORTED_AGGS.SUM,
+    PIVOT_SUPPORTED_AGGS.VALUE_COUNT,
+  ],
+  [KBN_FIELD_TYPES.STRING]: [PIVOT_SUPPORTED_AGGS.CARDINALITY, PIVOT_SUPPORTED_AGGS.VALUE_COUNT],
+  [KBN_FIELD_TYPES._SOURCE]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT],
+  [KBN_FIELD_TYPES.UNKNOWN]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT],
+  [KBN_FIELD_TYPES.CONFLICT]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT],
+};
 
 type PivotAgg = {
-  [key in PivotAggSupportedAggs]?: {
+  [key in PIVOT_SUPPORTED_AGGS]?: {
     field: FieldName;
   }
 };
@@ -44,7 +67,7 @@ export type PivotAggDict = { [key in AggName]: PivotAgg };
 
 // The internal representation of an aggregation definition.
 export interface PivotAggsConfig {
-  agg: PivotAggSupportedAggs;
+  agg: PIVOT_SUPPORTED_AGGS;
   field: FieldName;
   aggName: AggName;
 }

x-pack/plugins/ml/public/data_frame/common/pivot_group_by.ts

@@ -5,8 +5,10 @@
  */
 
 import { Dictionary } from '../../../common/types/common';
+import { KBN_FIELD_TYPES } from '../../../common/constants/field_types';
 
-import { AggName, FieldName } from './aggregations';
+import { AggName } from './aggregations';
+import { FieldName } from './fields';
 
 export enum PIVOT_SUPPORTED_GROUP_BY_AGGS {
   DATE_HISTOGRAM = 'date_histogram',
@@ -19,10 +21,31 @@ export type PivotSupportedGroupByAggs =
   | PIVOT_SUPPORTED_GROUP_BY_AGGS.HISTOGRAM
   | PIVOT_SUPPORTED_GROUP_BY_AGGS.TERMS;
 
+export const pivotSupportedGroupByAggs: PivotSupportedGroupByAggs[] = [
+  PIVOT_SUPPORTED_GROUP_BY_AGGS.DATE_HISTOGRAM,
+  PIVOT_SUPPORTED_GROUP_BY_AGGS.HISTOGRAM,
+  PIVOT_SUPPORTED_GROUP_BY_AGGS.TERMS,
+];
+
 export type PivotSupportedGroupByAggsWithInterval =
   | PIVOT_SUPPORTED_GROUP_BY_AGGS.HISTOGRAM
   | PIVOT_SUPPORTED_GROUP_BY_AGGS.DATE_HISTOGRAM;
 
+export const pivotGroupByFieldSupport = {
+  [KBN_FIELD_TYPES.ATTACHMENT]: [],
+  [KBN_FIELD_TYPES.BOOLEAN]: [],
+  [KBN_FIELD_TYPES.DATE]: [PIVOT_SUPPORTED_GROUP_BY_AGGS.DATE_HISTOGRAM],
+  [KBN_FIELD_TYPES.GEO_POINT]: [],
+  [KBN_FIELD_TYPES.GEO_SHAPE]: [],
+  [KBN_FIELD_TYPES.IP]: [PIVOT_SUPPORTED_GROUP_BY_AGGS.TERMS],
+  [KBN_FIELD_TYPES.MURMUR3]: [],
+  [KBN_FIELD_TYPES.NUMBER]: [PIVOT_SUPPORTED_GROUP_BY_AGGS.HISTOGRAM],
+  [KBN_FIELD_TYPES.STRING]: [PIVOT_SUPPORTED_GROUP_BY_AGGS.TERMS],
+  [KBN_FIELD_TYPES._SOURCE]: [],
+  [KBN_FIELD_TYPES.UNKNOWN]: [],
+  [KBN_FIELD_TYPES.CONFLICT]: [],
+};
+
 interface GroupByConfigBase {
   field: FieldName;
   aggName: AggName;

x-pack/plugins/ml/public/data_frame/components/define_pivot/common.test.ts

@@ -32,6 +32,7 @@ describe('Data Frame: Define Pivot Common', () => {
           label: 'the-field',
           options: [
             { label: 'avg(the-field)' },
+            { label: 'cardinality(the-field)' },
             { label: 'max(the-field)' },
             { label: 'min(the-field)' },
             { label: 'sum(the-field)' },
@@ -41,6 +42,11 @@ describe('Data Frame: Define Pivot Common', () => {
       ],
       aggOptionsData: {
         'avg(the-field)': { agg: 'avg', field: 'the-field', aggName: 'avg(the-field)' },
+        'cardinality(the-field)': {
+          agg: 'cardinality',
+          field: 'the-field',
+          aggName: 'cardinality(the-field)',
+        },
         'max(the-field)': { agg: 'max', field: 'the-field', aggName: 'max(the-field)' },
         'min(the-field)': { agg: 'min', field: 'the-field', aggName: 'min(the-field)' },
         'sum(the-field)': { agg: 'sum', field: 'the-field', aggName: 'sum(the-field)' },

x-pack/plugins/ml/public/data_frame/components/define_pivot/common.ts

@@ -8,22 +8,52 @@ import { EuiComboBoxOptionProps } from '@elastic/eui';
 
 import { StaticIndexPattern } from 'ui/index_patterns';
 
+import { KBN_FIELD_TYPES } from '../../../../common/constants/field_types';
+
 import {
   DataFramePreviewRequest,
   DropDownLabel,
   DropDownOption,
+  FieldName,
   PivotAggsConfigDict,
+  pivotAggsFieldSupport,
   PivotGroupByConfigDict,
-  pivotSupportedAggs,
-  PIVOT_SUPPORTED_AGGS,
+  pivotGroupByFieldSupport,
   PIVOT_SUPPORTED_GROUP_BY_AGGS,
 } from '../../common';
 
-export enum FIELD_TYPE {
-  DATE = 'date',
-  IP = 'ip',
-  NUMBER = 'number',
-  STRING = 'string',
+export interface Field {
+  name: FieldName;
+  type: KBN_FIELD_TYPES;
+}
+
+function getDefaultGroupByConfig(
+  aggName: string,
+  fieldName: FieldName,
+  groupByAgg: PIVOT_SUPPORTED_GROUP_BY_AGGS
+) {
+  switch (groupByAgg) {
+    case PIVOT_SUPPORTED_GROUP_BY_AGGS.TERMS:
+      return {
+        agg: groupByAgg,
+        aggName,
+        field: fieldName,
+      };
+    case PIVOT_SUPPORTED_GROUP_BY_AGGS.HISTOGRAM:
+      return {
+        agg: groupByAgg,
+        aggName,
+        field: fieldName,
+        interval: '10',
+      };
+    case PIVOT_SUPPORTED_GROUP_BY_AGGS.DATE_HISTOGRAM:
+      return {
+        agg: groupByAgg,
+        aggName,
+        field: fieldName,
+        interval: '1m',
+      };
+  }
 }
 
 export function getPivotDropdownOptions(indexPattern: StaticIndexPattern) {
@@ -35,55 +65,28 @@ export function getPivotDropdownOptions(indexPattern: StaticIndexPattern) {
   const aggOptions: EuiComboBoxOptionProps[] = [];
   const aggOptionsData: PivotAggsConfigDict = {};
 
+  const ignoreFieldNames = ['_id', '_index', '_type'];
   const fields = indexPattern.fields
-    .filter(field => field.aggregatable === true)
-    .map(field => ({ name: field.name, type: field.type }));
+    .filter(field => field.aggregatable === true && !ignoreFieldNames.includes(field.name))
+    .map((field): Field => ({ name: field.name, type: field.type as KBN_FIELD_TYPES }));
 
   fields.forEach(field => {
-    // group by
-    if (field.type === FIELD_TYPE.STRING) {
-      const aggName = `${PIVOT_SUPPORTED_GROUP_BY_AGGS.TERMS}(${field.name})`;
+    // Group by
+    const availableGroupByAggs = pivotGroupByFieldSupport[field.type];
+    availableGroupByAggs.forEach(groupByAgg => {
+      const aggName = `${groupByAgg}(${field.name})`;
       const groupByOption: DropDownLabel = { label: aggName };
       groupByOptions.push(groupByOption);
-      groupByOptionsData[aggName] = {
-        agg: PIVOT_SUPPORTED_GROUP_BY_AGGS.TERMS,
-        field: field.name,
-        aggName,
-      };
-    } else if (field.type === FIELD_TYPE.NUMBER) {
-      const aggName = `${PIVOT_SUPPORTED_GROUP_BY_AGGS.HISTOGRAM}(${field.name})`;
-      const groupByOption: DropDownLabel = { label: aggName };
-      groupByOptions.push(groupByOption);
-      groupByOptionsData[aggName] = {
-        agg: PIVOT_SUPPORTED_GROUP_BY_AGGS.HISTOGRAM,
-        field: field.name,
-        aggName,
-        interval: '10',
-      };
-    } else if (field.type === FIELD_TYPE.DATE) {
-      const aggName = `${PIVOT_SUPPORTED_GROUP_BY_AGGS.DATE_HISTOGRAM}(${field.name})`;
-      const groupByOption: DropDownLabel = { label: aggName };
-      groupByOptions.push(groupByOption);
-      groupByOptionsData[aggName] = {
-        agg: PIVOT_SUPPORTED_GROUP_BY_AGGS.DATE_HISTOGRAM,
-        field: field.name,
-        aggName,
-        interval: '1m',
-      };
-    }
+      groupByOptionsData[aggName] = getDefaultGroupByConfig(aggName, field.name, groupByAgg);
+    });
 
-    // aggregations
+    // Aggregations
     const aggOption: DropDownOption = { label: field.name, options: [] };
-    pivotSupportedAggs.forEach(agg => {
-      if (
-        (agg === PIVOT_SUPPORTED_AGGS.CARDINALITY &&
-          (field.type === FIELD_TYPE.STRING || field.type === FIELD_TYPE.IP)) ||
-        (agg !== PIVOT_SUPPORTED_AGGS.CARDINALITY && field.type === FIELD_TYPE.NUMBER)
-      ) {
-        const aggName = `${agg}(${field.name})`;
-        aggOption.options.push({ label: aggName });
-        aggOptionsData[aggName] = { agg, field: field.name, aggName };
-      }
+    const availableAggs = pivotAggsFieldSupport[field.type];
+    availableAggs.forEach(agg => {
+      const aggName = `${agg}(${field.name})`;
+      aggOption.options.push({ label: aggName });
+      aggOptionsData[aggName] = { agg, field: field.name, aggName };
     });
     aggOptions.push(aggOption);
   });

x-pack/plugins/ml/public/data_frame/components/define_pivot/define_pivot_summary.tsx

@@ -17,6 +17,8 @@ import {
   EuiText,
 } from '@elastic/eui';
 
+import { KBN_FIELD_TYPES } from '../../../../common/constants/field_types';
+
 import { AggListSummary } from '../../components/aggregation_list';
 import { GroupByListSummary } from '../../components/group_by_list';
 import { PivotPreview } from './pivot_preview';
@@ -27,10 +29,9 @@ import {
   isKibanaContext,
   KibanaContext,
   PivotAggsConfigDict,
-  PIVOT_SUPPORTED_AGGS,
-  pivotSupportedAggs,
+  pivotAggsFieldSupport,
 } from '../../common';
-import { FIELD_TYPE } from './common';
+import { Field } from './common';
 import { DefinePivotExposedState } from './define_pivot_form';
 
 const defaultSearch = '*';
@@ -49,28 +50,23 @@ export const DefinePivotSummary: SFC<DefinePivotExposedState> = ({
 
   const indexPattern = kibanaContext.currentIndexPattern;
 
+  const ignoreFieldNames = ['_id', '_index', '_type'];
   const fields = indexPattern.fields
-    .filter(field => field.aggregatable === true)
-    .map(field => ({ name: field.name, type: field.type }));
+    .filter(field => field.aggregatable === true && !ignoreFieldNames.includes(field.name))
+    .map((field): Field => ({ name: field.name, type: field.type as KBN_FIELD_TYPES }));
 
   // The available aggregations
   const aggOptions: EuiComboBoxOptionProps[] = [];
   const aggOptionsData: PivotAggsConfigDict = {};
 
   fields.forEach(field => {
-    // aggregations
+    // Aggregations
     const aggOption: DropDownOption = { label: field.name, options: [] };
-    pivotSupportedAggs.forEach(agg => {
-      if (
-        (agg === PIVOT_SUPPORTED_AGGS.CARDINALITY &&
-          (field.type === FIELD_TYPE.STRING || field.type === FIELD_TYPE.IP)) ||
-        (agg !== PIVOT_SUPPORTED_AGGS.CARDINALITY && field.type === FIELD_TYPE.NUMBER)
-      ) {
-        const label = `${agg}(${field.name})`;
-        aggOption.options.push({ label });
-        const aggName = `${agg}_${field.name}`;
-        aggOptionsData[label] = { agg, field: field.name, aggName };
-      }
+    const availableAggs = pivotAggsFieldSupport[field.type];
+    availableAggs.forEach(agg => {
+      const aggName = `${agg}(${field.name})`;
+      aggOption.options.push({ label: aggName });
+      aggOptionsData[aggName] = { agg, field: field.name, aggName };
     });
     aggOptions.push(aggOption);
   });