[8.x] [ML] Single Metric Viewer: Enable cross-filtering for 'by', 'over' and 'partition' field values (#193255) (#194280)

# Backport

This will backport the following commits from `main` to `8.x`:
- [[ML] Single Metric Viewer: Enable cross-filtering for 'by',
'over' and 'partition' field values
(#193255)](https://github.com/elastic/kibana/pull/193255)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Robert
Jaszczurek","email":"92210485+rbrtj@users.noreply.github.com"},"sourceCommit":{"committedDate":"2024-09-27T10:30:53Z","message":"[ML]
Single Metric Viewer: Enable cross-filtering for 'by', 'over' and
'partition' field values (#193255)\n\n## Summary\r\n\r\nEnables
cross-filtering for 'by', 'over' and 'partition' field values in\r\nthe
Single Metric Viewer.\r\n\r\nFixes
[#171932](d86d0688-dc69-43f0-aa24-130ff38935e6","sha":"07290bfac955c7d62ba93b52d888499dd6006cf3","branchLabelMapping":{"^v9.0.0$":"main","^v8.16.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:enhancement",":ml","Feature:Anomaly
Detection","v9.0.0","Team:ML","backport:prev-minor","v8.16.0"],"title":"[ML]
Single Metric Viewer: Enable cross-filtering for 'by', 'over' and
'partition' field
values","number":193255,"url":"https://github.com/elastic/kibana/pull/193255","mergeCommit":{"message":"[ML]
Single Metric Viewer: Enable cross-filtering for 'by', 'over' and
'partition' field values (#193255)\n\n## Summary\r\n\r\nEnables
cross-filtering for 'by', 'over' and 'partition' field values in\r\nthe
Single Metric Viewer.\r\n\r\nFixes
[#171932](d86d0688-dc69-43f0-aa24-130ff38935e6","sha":"07290bfac955c7d62ba93b52d888499dd6006cf3"}},"sourceBranch":"main","suggestedTargetBranches":["8.x"],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/193255","number":193255,"mergeCommit":{"message":"[ML]
Single Metric Viewer: Enable cross-filtering for 'by', 'over' and
'partition' field values (#193255)\n\n## Summary\r\n\r\nEnables
cross-filtering for 'by', 'over' and 'partition' field values in\r\nthe
Single Metric Viewer.\r\n\r\nFixes
[#171932](d86d0688-dc69-43f0-aa24-130ff38935e6","sha":"07290bfac955c7d62ba93b52d888499dd6006cf3"}},{"branch":"8.x","label":"v8.16.0","branchLabelMappingKey":"^v8.16.0$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->

Co-authored-by: Robert Jaszczurek <92210485+rbrtj@users.noreply.github.com>

x-pack/plugins/ml/common/types/storage.ts

@@ -35,6 +35,7 @@ export type PartitionFieldConfig =
         by: 'anomaly_score' | 'name';
         order: 'asc' | 'desc';
       };
+      value: string;
     }
   | undefined;
 

x-pack/plugins/ml/public/application/timeseriesexplorer/components/entity_control/entity_control.tsx

@@ -188,7 +188,7 @@ export class EntityControl extends Component<EntityControlProps, EntityControlSt
         selectedOptions={selectedOptions}
         onChange={this.onChange}
         onSearchChange={this.onSearchChange}
-        isClearable={false}
+        isClearable={true}
         renderOption={this.renderOption}
         data-test-subj={`mlSingleMetricViewerEntitySelection ${entity.fieldName}`}
         prepend={

x-pack/plugins/ml/public/application/timeseriesexplorer/components/series_controls/series_controls.tsx

@@ -24,7 +24,7 @@ import type {
 } from '../../../../../common/types/anomaly_detection_jobs';
 import { useMlKibana } from '../../../contexts/kibana';
 import { APP_STATE_ACTION } from '../../timeseriesexplorer_constants';
-import type { ComboBoxOption, EntityControlProps } from '../entity_control/entity_control';
+import type { ComboBoxOption, Entity, EntityControlProps } from '../entity_control/entity_control';
 import { EMPTY_FIELD_VALUE_LABEL } from '../entity_control/entity_control';
 import { getControlsForDetector } from '../../get_controls_for_detector';
 import {
@@ -57,15 +57,16 @@ export type UiPartitionFieldConfig = Exclude<PartitionFieldConfig, undefined>;
  * Provides default fields configuration.
  */
 const getDefaultFieldConfig = (
-  fieldTypes: MlEntityFieldType[],
+  entities: Entity[],
   isAnomalousOnly: boolean,
   applyTimeRange: boolean
 ): UiPartitionFieldsConfig => {
-  return fieldTypes.reduce((acc, f) => {
-    acc[f] = {
+  return entities.reduce((acc, f) => {
+    acc[f.fieldType] = {
       applyTimeRange,
       anomalousOnly: isAnomalousOnly,
       sort: { by: 'anomaly_score', order: 'desc' },
+      ...(f.fieldValue && { value: f.fieldValue }),
     };
     return acc;
   }, {} as UiPartitionFieldsConfig);
@@ -141,18 +142,28 @@ export const SeriesControls: FC<PropsWithChildren<SeriesControlsProps>> = ({
 
   // Merge the default config with the one from the local storage
   const resultFieldsConfig = useMemo(() => {
-    return {
-      ...getDefaultFieldConfig(
-        entityControls.map((v) => v.fieldType),
-        !storageFieldsConfig
-          ? true
-          : Object.values(storageFieldsConfig).some((v) => !!v?.anomalousOnly),
-        !storageFieldsConfig
-          ? true
-          : Object.values(storageFieldsConfig).some((v) => !!v?.applyTimeRange)
-      ),
-      ...(!storageFieldsConfig ? {} : storageFieldsConfig),
-    };
+    const resultFieldConfig = getDefaultFieldConfig(
+      entityControls,
+      !storageFieldsConfig
+        ? true
+        : Object.values(storageFieldsConfig).some((v) => !!v?.anomalousOnly),
+      !storageFieldsConfig
+        ? true
+        : Object.values(storageFieldsConfig).some((v) => !!v?.applyTimeRange)
+    );
+
+    // Early return to prevent unnecessary looping through the default config
+    if (!storageFieldsConfig) return resultFieldConfig;
+
+    // Override only the fields properties stored in the local storage
+    for (const key of Object.keys(resultFieldConfig) as MlEntityFieldType[]) {
+      resultFieldConfig[key] = {
+        ...resultFieldConfig[key],
+        ...storageFieldsConfig[key],
+      } as UiPartitionFieldConfig;
+    }
+
+    return resultFieldConfig;
   }, [entityControls, storageFieldsConfig]);
 
   /**
@@ -286,9 +297,20 @@ export const SeriesControls: FC<PropsWithChildren<SeriesControlsProps>> = ({
         }
       }
 
+      // Remove the value from the field config to avoid storing it in the local storage
+      const { value, ...updatedFieldConfigWithoutValue } = updatedFieldConfig;
+
+      // Remove the value from the result config to avoid storing it in the local storage
+      const updatedResultConfigWithoutValues = Object.fromEntries(
+        Object.entries(updatedResultConfig).map(([key, fieldValue]) => {
+          const { value: _, ...rest } = fieldValue;
+          return [key, rest];
+        })
+      );
+
       setStorageFieldsConfig({
-        ...updatedResultConfig,
-        [fieldType]: updatedFieldConfig,
+        ...updatedResultConfigWithoutValues,
+        [fieldType]: updatedFieldConfigWithoutValue,
       });
     },
     [resultFieldsConfig, setStorageFieldsConfig]

x-pack/plugins/ml/server/models/results_service/get_partition_fields_values.ts

@@ -35,16 +35,24 @@ function getFieldAgg(
   fieldType: MlPartitionFieldsType,
   isModelPlotSearch: boolean,
   query?: string,
-  fieldConfig?: FieldConfig
+  fieldsConfig?: FieldsConfig
 ) {
   const AGG_SIZE = 100;
 
+  const fieldConfig = fieldsConfig?.[fieldType];
   const fieldNameKey = `${fieldType}_name`;
   const fieldValueKey = `${fieldType}_value`;
 
   const sortByField =
     fieldConfig?.sort?.by === 'name' || isModelPlotSearch ? '_key' : 'maxRecordScore';
 
+  const splitFieldFilterValues = Object.entries(fieldsConfig ?? {})
+    .filter(([key, field]) => key !== fieldType && field.value)
+    .map(([key, field]) => ({
+      fieldValueKey: `${key}_value`,
+      fieldValue: field.value,
+    }));
+
   return {
     [fieldNameKey]: {
       terms: {
@@ -77,6 +85,11 @@ function getFieldAgg(
                   },
                 ]
               : []),
+            ...splitFieldFilterValues.map((filterValue) => ({
+              term: {
+                [filterValue.fieldValueKey]: filterValue.fieldValue,
+              },
+            })),
           ],
         },
       },
@@ -233,7 +246,7 @@ export const getPartitionFieldsValuesFactory = (mlClient: MlClient) =>
         ...ML_PARTITION_FIELDS.reduce((acc, key) => {
           return Object.assign(
             acc,
-            getFieldAgg(key, isModelPlotSearch, searchTerm[key], fieldsConfig[key])
+            getFieldAgg(key, isModelPlotSearch, searchTerm[key], fieldsConfig)
           );
         }, {}),
       },

x-pack/plugins/ml/server/routes/schemas/results_service_schema.ts

@@ -61,6 +61,7 @@ const fieldConfig = schema.maybe(
       by: schema.string(),
       order: schema.maybe(schema.string()),
     }),
+    value: schema.maybe(schema.string()),
   })
 );
 

x-pack/test/api_integration/apis/ml/results/get_partition_fields_values.ts

@@ -38,10 +38,38 @@ export default ({ getService }: FtrProviderContext) => {
     } as Job;
   }
 
-  function getDatafeedConfig(jobId: string) {
+  function getJobConfigWithByField(jobId: string) {
+    return {
+      job_id: jobId,
+      description:
+        'count by geoip.city_name partition=day_of_week on ecommerce dataset with 1h bucket span',
+      analysis_config: {
+        bucket_span: '1h',
+        influencers: ['geoip.city_name', 'day_of_week'],
+        detectors: [
+          {
+            function: 'count',
+            by_field_name: 'geoip.city_name',
+            partition_field_name: 'day_of_week',
+          },
+        ],
+      },
+      data_description: {
+        time_field: 'order_date',
+        time_format: 'epoch_ms',
+      },
+      analysis_limits: {
+        model_memory_limit: '11mb',
+        categorization_examples_limit: 4,
+      },
+      model_plot_config: { enabled: false },
+    } as Job;
+  }
+
+  function getDatafeedConfig(jobId: string, indices: string[]) {
     return {
       datafeed_id: `datafeed-${jobId}`,
-      indices: ['ft_farequote'],
+      indices,
       job_id: jobId,
       query: { bool: { must: [{ match_all: {} }] } },
     } as Datafeed;
@@ -50,12 +78,17 @@ export default ({ getService }: FtrProviderContext) => {
   async function createMockJobs() {
     await ml.api.createAndRunAnomalyDetectionLookbackJob(
       getJobConfig('fq_multi_1_ae'),
-      getDatafeedConfig('fq_multi_1_ae')
+      getDatafeedConfig('fq_multi_1_ae', ['ft_farequote'])
     );
 
     await ml.api.createAndRunAnomalyDetectionLookbackJob(
       getJobConfig('fq_multi_2_ae', false),
-      getDatafeedConfig('fq_multi_2_ae')
+      getDatafeedConfig('fq_multi_2_ae', ['ft_farequote'])
+    );
+
+    await ml.api.createAndRunAnomalyDetectionLookbackJob(
+      getJobConfigWithByField('ecommerce_advanced_1'),
+      getDatafeedConfig('ecommerce_advanced_1', ['ft_ecommerce'])
     );
   }
 
@@ -72,6 +105,7 @@ export default ({ getService }: FtrProviderContext) => {
   describe('PartitionFieldsValues', function () {
     before(async () => {
       await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/farequote');
+      await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/ecommerce');
       await ml.testResources.setKibanaTimeZoneToUTC();
       await createMockJobs();
     });
@@ -229,5 +263,63 @@ export default ({ getService }: FtrProviderContext) => {
         expect(body.partition_field.values.length).to.eql(19);
       });
     });
+
+    describe('cross filtering', () => {
+      it('should return filtered values for by_field when partition_field is set', async () => {
+        const requestBody = {
+          jobId: 'ecommerce_advanced_1',
+          criteriaFields: [{ fieldName: 'detector_index', fieldValue: 0 }],
+          earliestMs: 1687968000000, // June 28, 2023 16:00:00 GMT
+          latestMs: 1688140800000, // June 30, 2023 16:00:00 GMT
+          searchTerm: {},
+          fieldsConfig: {
+            by_field: {
+              anomalousOnly: true,
+              applyTimeRange: true,
+              sort: { order: 'desc', by: 'anomaly_score' },
+            },
+            partition_field: {
+              anomalousOnly: true,
+              applyTimeRange: true,
+              sort: { order: 'desc', by: 'anomaly_score' },
+              value: 'Saturday',
+            },
+          },
+        };
+        const body = await runRequest(requestBody);
+
+        expect(body.by_field.values.length).to.eql(1);
+        expect(body.by_field.values[0].value).to.eql('Abu Dhabi');
+      });
+
+      it('should return filtered values for partition_field when by_field is set', async () => {
+        const requestBody = {
+          jobId: 'ecommerce_advanced_1',
+          criteriaFields: [{ fieldName: 'detector_index', fieldValue: 0 }],
+          earliestMs: 1687968000000, // June 28, 2023 16:00:00 GMT
+          latestMs: 1688140800000, // June 30, 2023 16:00:00 GMT
+          searchTerm: {},
+          fieldsConfig: {
+            by_field: {
+              anomalousOnly: true,
+              applyTimeRange: true,
+              sort: { order: 'desc', by: 'anomaly_score' },
+              value: 'Abu Dhabi',
+            },
+            partition_field: {
+              anomalousOnly: true,
+              applyTimeRange: true,
+              sort: { order: 'desc', by: 'anomaly_score' },
+            },
+          },
+        };
+
+        const body = await runRequest(requestBody);
+
+        expect(body.partition_field.values.length).to.eql(2);
+        expect(body.partition_field.values[0].value).to.eql('Saturday');
+        expect(body.partition_field.values[1].value).to.eql('Monday');
+      });
+    });
   });
 };