[ML] Improving job wizards with datafeed aggregations (#55180)

* [ML] Improving job wizards with datafeed aggregations

* picking all agg keys for fields

* function move and rename

x-pack/legacy/plugins/ml/common/constants/field_types.ts

@@ -16,3 +16,4 @@ export enum ML_JOB_FIELD_TYPES {
 }
 
 export const MLCATEGORY = 'mlcategory';
+export const DOC_COUNT = 'doc_count';

x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/advanced_job_creator.ts

@@ -183,7 +183,7 @@ export class AdvancedJobCreator extends JobCreator {
 
   public cloneFromExistingJob(job: Job, datafeed: Datafeed) {
     this._overrideConfigs(job, datafeed);
-    const detectors = getRichDetectors(job, datafeed, this.scriptFields, true);
+    const detectors = getRichDetectors(job, datafeed, this.additionalFields, true);
 
     // keep track of the custom rules for each detector
     const customRules = this._detectors.map(d => d.custom_rules);

x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts

@@ -140,7 +140,7 @@ export class CategorizationJobCreator extends JobCreator {
   public cloneFromExistingJob(job: Job, datafeed: Datafeed) {
     this._overrideConfigs(job, datafeed);
     this.createdBy = CREATED_BY_LABEL.CATEGORIZATION;
-    const detectors = getRichDetectors(job, datafeed, this.scriptFields, false);
+    const detectors = getRichDetectors(job, datafeed, this.additionalFields, false);
 
     const dtr = detectors[0];
     if (detectors.length && dtr.agg !== null && dtr.field !== null) {

x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/job_creator.ts

@@ -19,7 +19,7 @@ import {
   CREATED_BY_LABEL,
   SHARED_RESULTS_INDEX_NAME,
 } from '../../../../../../common/constants/new_job';
-import { isSparseDataJob } from './util/general';
+import { isSparseDataJob, collectAggs } from './util/general';
 import { parseInterval } from '../../../../../../common/util/parse_interval';
 import { Calendar } from '../../../../../../common/types/calendars';
 import { mlCalendarService } from '../../../../services/calendar_service';
@@ -43,6 +43,7 @@ export class JobCreator {
   protected _aggs: Aggregation[] = [];
   protected _fields: Field[] = [];
   protected _scriptFields: Field[] = [];
+  protected _aggregationFields: Field[] = [];
   protected _sparseData: boolean = false;
   private _stopAllRefreshPolls: {
     stop: boolean;
@@ -450,6 +451,14 @@ export class JobCreator {
     return this._scriptFields;
   }
 
+  public get aggregationFields(): Field[] {
+    return this._aggregationFields;
+  }
+
+  public get additionalFields(): Field[] {
+    return [...this._scriptFields, ...this._aggregationFields];
+  }
+
   public get subscribers(): ProgressSubscriber[] {
     return this._subscribers;
   }
@@ -603,6 +612,7 @@ export class JobCreator {
     }
     this._sparseData = isSparseDataJob(job, datafeed);
 
+    this._scriptFields = [];
     if (this._datafeed_config.script_fields !== undefined) {
       this._scriptFields = Object.keys(this._datafeed_config.script_fields).map(f => ({
         id: f,
@@ -610,8 +620,11 @@ export class JobCreator {
         type: ES_FIELD_TYPES.KEYWORD,
         aggregatable: true,
       }));
-    } else {
-      this._scriptFields = [];
+    }
+
+    this._aggregationFields = [];
+    if (this._datafeed_config.aggregations?.buckets !== undefined) {
+      collectAggs(this._datafeed_config.aggregations.buckets, this._aggregationFields);
     }
   }
 }

x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/multi_metric_job_creator.ts

@@ -153,7 +153,7 @@ export class MultiMetricJobCreator extends JobCreator {
   public cloneFromExistingJob(job: Job, datafeed: Datafeed) {
     this._overrideConfigs(job, datafeed);
     this.createdBy = CREATED_BY_LABEL.MULTI_METRIC;
-    const detectors = getRichDetectors(job, datafeed, this.scriptFields, false);
+    const detectors = getRichDetectors(job, datafeed, this.additionalFields, false);
 
     if (datafeed.aggregations !== undefined) {
       // if we've converting from a single metric job,

x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/population_job_creator.ts

@@ -135,7 +135,7 @@ export class PopulationJobCreator extends JobCreator {
   public cloneFromExistingJob(job: Job, datafeed: Datafeed) {
     this._overrideConfigs(job, datafeed);
     this.createdBy = CREATED_BY_LABEL.POPULATION;
-    const detectors = getRichDetectors(job, datafeed, this.scriptFields, false);
+    const detectors = getRichDetectors(job, datafeed, this.additionalFields, false);
 
     this.removeAllDetectors();
 

x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/single_metric_job_creator.ts

@@ -190,7 +190,7 @@ export class SingleMetricJobCreator extends JobCreator {
   public cloneFromExistingJob(job: Job, datafeed: Datafeed) {
     this._overrideConfigs(job, datafeed);
     this.createdBy = CREATED_BY_LABEL.SINGLE_METRIC;
-    const detectors = getRichDetectors(job, datafeed, this.scriptFields, false);
+    const detectors = getRichDetectors(job, datafeed, this.additionalFields, false);
 
     this.removeAllDetectors();
 

x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/util/general.ts

@@ -11,7 +11,8 @@ import {
   ML_JOB_AGGREGATION,
   SPARSE_DATA_AGGREGATIONS,
 } from '../../../../../../../common/constants/aggregation_types';
-import { MLCATEGORY } from '../../../../../../../common/constants/field_types';
+import { MLCATEGORY, DOC_COUNT } from '../../../../../../../common/constants/field_types';
+import { ES_FIELD_TYPES } from '../../../../../../../../../../../src/plugins/data/public';
 import {
   EVENT_RATE_FIELD_ID,
   Field,
@@ -27,14 +28,14 @@ import {
 } from '../index';
 import { CREATED_BY_LABEL, JOB_TYPE } from '../../../../../../../common/constants/new_job';
 
-const getFieldByIdFactory = (scriptFields: Field[]) => (id: string) => {
+const getFieldByIdFactory = (additionalFields: Field[]) => (id: string) => {
   let field = newJobCapsService.getFieldById(id);
   // if no field could be found it may be a pretend field, like mlcategory or a script field
   if (field === null) {
     if (id === MLCATEGORY) {
       field = mlCategory;
-    } else if (scriptFields.length) {
-      field = scriptFields.find(f => f.id === id) || null;
+    } else if (additionalFields.length) {
+      field = additionalFields.find(f => f.id === id) || null;
     }
   }
   return field;
@@ -44,12 +45,12 @@ const getFieldByIdFactory = (scriptFields: Field[]) => (id: string) => {
 export function getRichDetectors(
   job: Job,
   datafeed: Datafeed,
-  scriptFields: Field[],
+  additionalFields: Field[],
   advanced: boolean = false
 ) {
   const detectors = advanced ? getDetectorsAdvanced(job, datafeed) : getDetectors(job, datafeed);
 
-  const getFieldById = getFieldByIdFactory(scriptFields);
+  const getFieldById = getFieldByIdFactory(additionalFields);
 
   return detectors.map(d => {
     let field = null;
@@ -82,19 +83,19 @@ export function getRichDetectors(
   });
 }
 
-export function createFieldOptions(fields: Field[]) {
-  return fields
-    .filter(f => f.id !== EVENT_RATE_FIELD_ID)
-    .map(f => ({
-      label: f.name,
-    }))
-    .sort((a, b) => a.label.localeCompare(b.label));
-}
-
-export function createScriptFieldOptions(scriptFields: Field[]) {
-  return scriptFields.map(f => ({
-    label: f.id,
-  }));
+export function createFieldOptions(fields: Field[], additionalFields: Field[]) {
+  return [
+    ...fields
+      .filter(f => f.id !== EVENT_RATE_FIELD_ID)
+      .map(f => ({
+        label: f.name,
+      })),
+    ...additionalFields
+      .filter(f => fields.some(f2 => f2.id === f.id) === false)
+      .map(f => ({
+        label: f.id,
+      })),
+  ].sort((a, b) => a.label.localeCompare(b.label));
 }
 
 export function createMlcategoryFieldOption(categorizationFieldName: string | null) {
@@ -108,6 +109,16 @@ export function createMlcategoryFieldOption(categorizationFieldName: string | nu
   ];
 }
 
+export function createDocCountFieldOption(usingAggregations: boolean) {
+  return usingAggregations
+    ? [
+        {
+          label: DOC_COUNT,
+        },
+      ]
+    : [];
+}
+
 function getDetectorsAdvanced(job: Job, datafeed: Datafeed) {
   return processFieldlessAggs(job.analysis_config.detectors);
 }
@@ -305,3 +316,26 @@ export function getJobCreatorTitle(jobCreator: JobCreatorType) {
       return '';
   }
 }
+
+// recurse through a datafeed aggregation object,
+// adding top level keys from each nested agg to an array
+// of fields
+export function collectAggs(o: any, aggFields: Field[]) {
+  for (const i in o) {
+    if (o[i] !== null && typeof o[i] === 'object') {
+      if (i === 'aggregations' || i === 'aggs') {
+        Object.keys(o[i]).forEach(k => {
+          if (k !== 'aggregations' && i !== 'aggs') {
+            aggFields.push({
+              id: k,
+              name: k,
+              type: ES_FIELD_TYPES.KEYWORD,
+              aggregatable: true,
+            });
+          }
+        });
+      }
+      collectAggs(o[i], aggFields);
+    }
+  }
+}

x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/datafeed_step/components/time_field/time_field_select.tsx

@@ -4,9 +4,10 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import React, { FC } from 'react';
+import React, { FC, useContext } from 'react';
 import { EuiComboBox, EuiComboBoxOptionProps } from '@elastic/eui';
 
+import { JobCreatorContext } from '../../../job_creator_context';
 import { Field } from '../../../../../../../../../common/types/fields';
 import { createFieldOptions } from '../../../../../common/job_creator/util/general';
 
@@ -17,7 +18,8 @@ interface Props {
 }
 
 export const TimeFieldSelect: FC<Props> = ({ fields, changeHandler, selectedField }) => {
-  const options: EuiComboBoxOptionProps[] = createFieldOptions(fields);
+  const { jobCreator } = useContext(JobCreatorContext);
+  const options: EuiComboBoxOptionProps[] = createFieldOptions(fields, jobCreator.additionalFields);
 
   const selection: EuiComboBoxOptionProps[] = [];
   if (selectedField !== null) {

x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/advanced_detector_modal/advanced_detector_modal.tsx

@@ -18,7 +18,6 @@ import { JobCreatorContext } from '../../../job_creator_context';
 import { AdvancedJobCreator } from '../../../../../common/job_creator';
 import {
   createFieldOptions,
-  createScriptFieldOptions,
   createMlcategoryFieldOption,
 } from '../../../../../common/job_creator/util/general';
 import {
@@ -88,7 +87,7 @@ export const AdvancedDetectorModal: FC<Props> = ({
   const [fieldOptionEnabled, setFieldOptionEnabled] = useState(true);
   const { descriptionPlaceholder, setDescriptionPlaceholder } = useDetectorPlaceholder(detector);
 
-  const usingScriptFields = jobCreator.scriptFields.length > 0;
+  const usingScriptFields = jobCreator.additionalFields.length > 0;
   // list of aggregation combobox options.
 
   const aggOptions: EuiComboBoxOptionProps[] = aggs
@@ -98,12 +97,12 @@ export const AdvancedDetectorModal: FC<Props> = ({
   // fields available for the selected agg
   const { currentFieldOptions, setCurrentFieldOptions } = useCurrentFieldOptions(
     detector.agg,
-    jobCreator.scriptFields
+    jobCreator.additionalFields,
+    fields
   );
 
   const allFieldOptions: EuiComboBoxOptionProps[] = [
-    ...createFieldOptions(fields),
-    ...createScriptFieldOptions(jobCreator.scriptFields),
+    ...createFieldOptions(fields, jobCreator.additionalFields),
   ].sort(comboBoxOptionsSort);
 
   const splitFieldOptions: EuiComboBoxOptionProps[] = [
@@ -127,7 +126,9 @@ export const AdvancedDetectorModal: FC<Props> = ({
       return mlCategory;
     }
     return (
-      fields.find(f => f.id === title) || jobCreator.scriptFields.find(f => f.id === title) || null
+      fields.find(f => f.id === title) ||
+      jobCreator.additionalFields.find(f => f.id === title) ||
+      null
     );
   }
 
@@ -365,21 +366,27 @@ function useDetectorPlaceholder(detector: RichDetector) {
 }
 
 // creates list of combobox options based on an aggregation's field list
-function createFieldOptionsFromAgg(agg: Aggregation | null) {
-  return createFieldOptions(agg !== null && agg.fields !== undefined ? agg.fields : []);
+function createFieldOptionsFromAgg(agg: Aggregation | null, additionalFields: Field[]) {
+  return createFieldOptions(
+    agg !== null && agg.fields !== undefined ? agg.fields : [],
+    additionalFields
+  );
 }
 
 // custom hook for storing combobox options based on an aggregation field list
-function useCurrentFieldOptions(aggregation: Aggregation | null, scriptFields: Field[]) {
+function useCurrentFieldOptions(
+  aggregation: Aggregation | null,
+  additionalFields: Field[],
+  fields: Field[]
+) {
   const [currentFieldOptions, setCurrentFieldOptions] = useState(
-    createFieldOptionsFromAgg(aggregation)
+    createFieldOptionsFromAgg(aggregation, additionalFields)
   );
-  const scriptFieldOptions = createScriptFieldOptions(scriptFields);
 
   return {
     currentFieldOptions,
     setCurrentFieldOptions: (agg: Aggregation | null) =>
-      setCurrentFieldOptions([...createFieldOptionsFromAgg(agg), ...scriptFieldOptions]),
+      setCurrentFieldOptions(createFieldOptionsFromAgg(agg, additionalFields)),
   };
 }
 

x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_field/categorization_field_select.tsx

@@ -9,10 +9,7 @@ import { EuiComboBox, EuiComboBoxOptionProps } from '@elastic/eui';
 
 import { JobCreatorContext } from '../../../job_creator_context';
 import { Field } from '../../../../../../../../../common/types/fields';
-import {
-  createFieldOptions,
-  createScriptFieldOptions,
-} from '../../../../../common/job_creator/util/general';
+import { createFieldOptions } from '../../../../../common/job_creator/util/general';
 
 interface Props {
   fields: Field[];
@@ -23,8 +20,7 @@ interface Props {
 export const CategorizationFieldSelect: FC<Props> = ({ fields, changeHandler, selectedField }) => {
   const { jobCreator } = useContext(JobCreatorContext);
   const options: EuiComboBoxOptionProps[] = [
-    ...createFieldOptions(fields),
-    ...createScriptFieldOptions(jobCreator.scriptFields),
+    ...createFieldOptions(fields, jobCreator.additionalFields),
   ];
 
   const selection: EuiComboBoxOptionProps[] = [];

x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/influencers/influencers_select.tsx

@@ -11,7 +11,6 @@ import { JobCreatorContext } from '../../../job_creator_context';
 import { Field } from '../../../../../../../../../common/types/fields';
 import {
   createFieldOptions,
-  createScriptFieldOptions,
   createMlcategoryFieldOption,
 } from '../../../../../common/job_creator/util/general';
 
@@ -24,8 +23,7 @@ interface Props {
 export const InfluencersSelect: FC<Props> = ({ fields, changeHandler, selectedInfluencers }) => {
   const { jobCreator } = useContext(JobCreatorContext);
   const options: EuiComboBoxOptionProps[] = [
-    ...createFieldOptions(fields),
-    ...createScriptFieldOptions(jobCreator.scriptFields),
+    ...createFieldOptions(fields, jobCreator.additionalFields),
     ...createMlcategoryFieldOption(jobCreator.categorizationFieldName),
   ];
 

x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/summary_count_field/summary_count_field_select.tsx

@@ -11,7 +11,7 @@ import { JobCreatorContext } from '../../../job_creator_context';
 import { Field } from '../../../../../../../../../common/types/fields';
 import {
   createFieldOptions,
-  createScriptFieldOptions,
+  createDocCountFieldOption,
 } from '../../../../../common/job_creator/util/general';
 
 interface Props {
@@ -23,8 +23,8 @@ interface Props {
 export const SummaryCountFieldSelect: FC<Props> = ({ fields, changeHandler, selectedField }) => {
   const { jobCreator } = useContext(JobCreatorContext);
   const options: EuiComboBoxOptionProps[] = [
-    ...createFieldOptions(fields),
-    ...createScriptFieldOptions(jobCreator.scriptFields),
+    ...createFieldOptions(fields, jobCreator.additionalFields),
+    ...createDocCountFieldOption(jobCreator.aggregationFields.length > 0),
   ];
 
   const selection: EuiComboBoxOptionProps[] = [];