[Metrics UI] Add ability to filter anomaly detection datafeed (#89721) (#90689)

* Add null check for empty process data

* Add Ability to filter datafeed for ml jobs

* Merge user-defined query with default query

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

x-pack/plugins/infra/public/containers/ml/infra_ml_module.tsx

@@ -6,7 +6,6 @@
  */
 
 import { useCallback, useMemo } from 'react';
-import { DatasetFilter } from '../../../common/infra_ml';
 import { useKibanaContextForPlugin } from '../../hooks/use_kibana';
 import { useTrackedPromise } from '../../utils/use_tracked_promise';
 import { useModuleStatus } from './infra_ml_module_status';
@@ -52,7 +51,7 @@ export const useInfraMLModule = <JobType extends string>({
         selectedIndices: string[],
         start: number | undefined,
         end: number | undefined,
-        datasetFilter: DatasetFilter,
+        filter: string,
         partitionField?: string
       ) => {
         dispatchModuleStatus({ type: 'startedSetup' });
@@ -60,7 +59,7 @@ export const useInfraMLModule = <JobType extends string>({
           {
             start,
             end,
-            datasetFilter,
+            filter,
             moduleSourceConfiguration: {
               indices: selectedIndices,
               sourceId,
@@ -114,13 +113,13 @@ export const useInfraMLModule = <JobType extends string>({
       selectedIndices: string[],
       start: number | undefined,
       end: number | undefined,
-      datasetFilter: DatasetFilter,
+      filter: string,
       partitionField?: string
     ) => {
       dispatchModuleStatus({ type: 'startedSetup' });
       cleanUpModule()
         .then(() => {
-          setUpModule(selectedIndices, start, end, datasetFilter, partitionField);
+          setUpModule(selectedIndices, start, end, filter, partitionField);
         })
         .catch(() => {
           dispatchModuleStatus({ type: 'failedSetup' });

x-pack/plugins/infra/public/containers/ml/infra_ml_module_types.ts

@@ -10,7 +10,6 @@ import {
   ValidateLogEntryDatasetsResponsePayload,
   ValidationIndicesResponsePayload,
 } from '../../../common/http_api/log_analysis';
-import { DatasetFilter } from '../../../common/infra_ml';
 import { DeleteJobsResponsePayload } from './api/ml_cleanup';
 import { FetchJobStatusResponsePayload } from './api/ml_get_jobs_summary_api';
 import { GetMlModuleResponsePayload } from './api/ml_get_module';
@@ -21,7 +20,7 @@ export { JobModelSizeStats, JobSummary } from './api/ml_get_jobs_summary_api';
 export interface SetUpModuleArgs {
   start?: number | undefined;
   end?: number | undefined;
-  datasetFilter?: DatasetFilter;
+  filter?: any;
   moduleSourceConfiguration: ModuleSourceConfiguration;
   partitionField?: string;
 }

x-pack/plugins/infra/public/containers/ml/modules/metrics_hosts/module_descriptor.ts

@@ -67,6 +67,7 @@ const setUpModule = async (setUpModuleArgs: SetUpModuleArgs, fetch: HttpHandler)
   const {
     start,
     end,
+    filter,
     moduleSourceConfiguration: { spaceId, sourceId, indices, timestampField },
     partitionField,
   } = setUpModuleArgs;
@@ -107,10 +108,23 @@ const setUpModule = async (setUpModuleArgs: SetUpModuleArgs, fetch: HttpHandler)
 
   const datafeedOverrides = jobIds.map((id) => {
     const { datafeed: defaultDatafeedConfig } = getDefaultJobConfigs(id);
+    const config = { ...defaultDatafeedConfig };
+
+    if (filter) {
+      const query = JSON.parse(filter);
+
+      config.query.bool = {
+        ...config.query.bool,
+        ...query.bool,
+      };
+    }
 
     if (!partitionField || id === 'hosts_memory_usage') {
       // Since the host memory usage doesn't have custom aggs, we don't need to do anything to add a partition field
-      return defaultDatafeedConfig;
+      return {
+        ...config,
+        job_id: id,
+      };
     }
 
     // If we have a partition field, we need to change the aggregation to do a terms agg at the top level
@@ -126,7 +140,7 @@ const setUpModule = async (setUpModuleArgs: SetUpModuleArgs, fetch: HttpHandler)
     };
 
     return {
-      ...defaultDatafeedConfig,
+      ...config,
       job_id: id,
       aggregations,
     };

x-pack/plugins/infra/public/containers/ml/modules/metrics_k8s/module_descriptor.ts

@@ -68,6 +68,7 @@ const setUpModule = async (setUpModuleArgs: SetUpModuleArgs, fetch: HttpHandler)
   const {
     start,
     end,
+    filter,
     moduleSourceConfiguration: { spaceId, sourceId, indices, timestampField },
     partitionField,
   } = setUpModuleArgs;
@@ -107,10 +108,23 @@ const setUpModule = async (setUpModuleArgs: SetUpModuleArgs, fetch: HttpHandler)
 
   const datafeedOverrides = jobIds.map((id) => {
     const { datafeed: defaultDatafeedConfig } = getDefaultJobConfigs(id);
+    const config = { ...defaultDatafeedConfig };
+
+    if (filter) {
+      const query = JSON.parse(filter);
+
+      config.query.bool = {
+        ...config.query.bool,
+        ...query.bool,
+      };
+    }
 
     if (!partitionField || id === 'k8s_memory_usage') {
       // Since the host memory usage doesn't have custom aggs, we don't need to do anything to add a partition field
-      return defaultDatafeedConfig;
+      return {
+        ...config,
+        job_id: id,
+      };
     }
 
     // Because the ML K8s jobs ship with a default partition field of {kubernetes.namespace}, ignore that agg and wrap it in our own agg.
@@ -131,7 +145,7 @@ const setUpModule = async (setUpModuleArgs: SetUpModuleArgs, fetch: HttpHandler)
     };
 
     return {
-      ...defaultDatafeedConfig,
+      ...config,
       job_id: id,
       aggregations,
     };

x-pack/plugins/infra/public/pages/metrics/inventory_view/components/ml/anomaly_detection/job_setup_screen.tsx

@@ -4,7 +4,7 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-
+import { debounce } from 'lodash';
 import React, { useState, useCallback, useMemo, useEffect } from 'react';
 import { EuiForm, EuiDescribedFormGroup, EuiFormRow } from '@elastic/eui';
 import { EuiText, EuiSpacer } from '@elastic/eui';
@@ -22,6 +22,8 @@ import { useMetricK8sModuleContext } from '../../../../../../containers/ml/modul
 import { useMetricHostsModuleContext } from '../../../../../../containers/ml/modules/metrics_hosts/module';
 import { FixedDatePicker } from '../../../../../../components/fixed_datepicker';
 import { DEFAULT_K8S_PARTITION_FIELD } from '../../../../../../containers/ml/modules/metrics_k8s/module_descriptor';
+import { MetricsExplorerKueryBar } from '../../../../metrics_explorer/components/kuery_bar';
+import { convertKueryToElasticSearchQuery } from '../../../../../../utils/kuery';
 
 interface Props {
   jobType: 'hosts' | 'kubernetes';
@@ -36,6 +38,8 @@ export const JobSetupScreen = (props: Props) => {
   const [partitionField, setPartitionField] = useState<string[] | null>(null);
   const h = useMetricHostsModuleContext();
   const k = useMetricK8sModuleContext();
+  const [filter, setFilter] = useState<string>('');
+  const [filterQuery, setFilterQuery] = useState<string>('');
   const { createDerivedIndexPattern } = useSourceViaHttp({
     sourceId: 'default',
     type: 'metrics',
@@ -89,7 +93,7 @@ export const JobSetupScreen = (props: Props) => {
         indicies,
         moment(startDate).toDate().getTime(),
         undefined,
-        { type: 'includeAll' },
+        filterQuery,
         partitionField ? partitionField[0] : undefined
       );
     } else {
@@ -97,11 +101,30 @@ export const JobSetupScreen = (props: Props) => {
         indicies,
         moment(startDate).toDate().getTime(),
         undefined,
-        { type: 'includeAll' },
+        filterQuery,
         partitionField ? partitionField[0] : undefined
       );
     }
-  }, [cleanUpAndSetUpModule, setUpModule, hasSummaries, indicies, partitionField, startDate]);
+  }, [
+    cleanUpAndSetUpModule,
+    filterQuery,
+    setUpModule,
+    hasSummaries,
+    indicies,
+    partitionField,
+    startDate,
+  ]);
+
+  const onFilterChange = useCallback(
+    (f: string) => {
+      setFilter(f || '');
+      setFilterQuery(convertKueryToElasticSearchQuery(f, derivedIndexPattern) || '');
+    },
+    [derivedIndexPattern]
+  );
+
+  /* eslint-disable-next-line react-hooks/exhaustive-deps */
+  const debouncedOnFilterChange = useCallback(debounce(onFilterChange, 500), [onFilterChange]);
 
   const onPartitionFieldChange = useCallback((value: Array<{ label: string }>) => {
     setPartitionField(value.map((v) => v.label));
@@ -250,6 +273,40 @@ export const JobSetupScreen = (props: Props) => {
                   />
                 </EuiFormRow>
               </EuiDescribedFormGroup>
+
+              <EuiDescribedFormGroup
+                title={
+                  <h3>
+                    <FormattedMessage
+                      id="xpack.infra.ml.steps.setupProcess.filter.title"
+                      defaultMessage="Filter"
+                    />
+                  </h3>
+                }
+                description={
+                  <FormattedMessage
+                    id="xpack.infra.ml.steps.setupProcess.filter.description"
+                    defaultMessage="By default, machine learning jobs analyze all of your metric data."
+                  />
+                }
+              >
+                <EuiFormRow
+                  display="rowCompressed"
+                  label={
+                    <FormattedMessage
+                      id="xpack.infra.ml.steps.setupProcess.filter.label"
+                      defaultMessage="Filter (optional)"
+                    />
+                  }
+                >
+                  <MetricsExplorerKueryBar
+                    derivedIndexPattern={derivedIndexPattern}
+                    onSubmit={onFilterChange}
+                    onChange={debouncedOnFilterChange}
+                    value={filter}
+                  />
+                </EuiFormRow>
+              </EuiDescribedFormGroup>
             </EuiForm>
           </>
         )}