[ML] Explain log rate spikes: Split main chart by selected field/value pair. (#136712)

- Adds the ability to show the hovered/selected field/value pair in the main chart.
- Optional search queries now get applied to the analysis.

x-pack/plugins/aiops/common/api/explain_log_rate_spikes/schema.ts

@@ -10,7 +10,7 @@ import { schema, TypeOf } from '@kbn/config-schema';
 export const aiopsExplainLogRateSpikesSchema = schema.object({
   start: schema.number(),
   end: schema.number(),
-  kuery: schema.string(),
+  searchQuery: schema.string(),
   timeFieldName: schema.string(),
   includeFrozen: schema.maybe(schema.boolean()),
   /** Analysis selection time ranges */

x-pack/plugins/aiops/public/application/utils/query_utils.ts

@@ -8,11 +8,14 @@
 // TODO Consolidate with duplicate query utils in
 // `x-pack/plugins/data_visualizer/common/utils/query_utils.ts`
 
+import { cloneDeep } from 'lodash';
+
 import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types';
 import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+
 import { Query } from '@kbn/es-query';
-import { cloneDeep } from 'lodash';
 import { isPopulatedObject } from '@kbn/ml-is-populated-object';
+import type { ChangePoint } from '@kbn/ml-agg-utils';
 
 /*
  * Contains utility functions for building and processing queries.
@@ -24,7 +27,9 @@ export function buildBaseFilterCriteria(
   timeFieldName?: string,
   earliestMs?: number,
   latestMs?: number,
-  query?: Query['query']
+  query?: Query['query'],
+  selectedChangePoint?: ChangePoint,
+  includeSelectedChangePoint = true
 ): estypes.QueryDslQueryContainer[] {
   const filterCriteria = [];
   if (timeFieldName && earliestMs && latestMs) {
@@ -43,6 +48,22 @@ export function buildBaseFilterCriteria(
     filterCriteria.push(query);
   }
 
+  if (selectedChangePoint && includeSelectedChangePoint) {
+    filterCriteria.push({
+      term: { [selectedChangePoint.fieldName]: selectedChangePoint.fieldValue },
+    });
+  } else if (selectedChangePoint && !includeSelectedChangePoint) {
+    filterCriteria.push({
+      bool: {
+        must_not: [
+          {
+            term: { [selectedChangePoint.fieldName]: selectedChangePoint.fieldValue },
+          },
+        ],
+      },
+    });
+  }
+
   return filterCriteria;
 }
 

x-pack/plugins/aiops/public/components/document_count_content/document_count_chart/document_count_chart.tsx

@@ -27,6 +27,7 @@ import { DualBrush, DualBrushAnnotation } from '@kbn/aiops-components';
 import { getWindowParameters } from '@kbn/aiops-utils';
 import type { WindowParameters } from '@kbn/aiops-utils';
 import { MULTILAYER_TIME_AXIS_STYLE } from '@kbn/charts-plugin/common';
+import type { ChangePoint } from '@kbn/ml-agg-utils';
 
 import { useAiOpsKibana } from '../../../kibana_context';
 
@@ -39,9 +40,11 @@ interface DocumentCountChartProps {
   brushSelectionUpdateHandler: (d: WindowParameters) => void;
   width?: number;
   chartPoints: DocumentCountChartPoint[];
+  chartPointsSplit?: DocumentCountChartPoint[];
   timeRangeEarliest: number;
   timeRangeLatest: number;
   interval: number;
+  changePoint?: ChangePoint;
 }
 
 const SPEC_ID = 'document_count';
@@ -65,9 +68,11 @@ export const DocumentCountChart: FC<DocumentCountChartProps> = ({
   brushSelectionUpdateHandler,
   width,
   chartPoints,
+  chartPointsSplit,
   timeRangeEarliest,
   timeRangeLatest,
   interval,
+  changePoint,
 }) => {
   const {
     services: { data, uiSettings, fieldFormats, charts },
@@ -79,9 +84,21 @@ export const DocumentCountChart: FC<DocumentCountChartProps> = ({
   const xAxisFormatter = fieldFormats.deserialize({ id: 'date' });
   const useLegacyTimeAxis = uiSettings.get('visualization:useLegacyTimeAxis', false);
 
-  const seriesName = i18n.translate('xpack.aiops.dataGrid.field.documentCountChart.seriesLabel', {
-    defaultMessage: 'document count',
-  });
+  const overallSeriesName = i18n.translate(
+    'xpack.aiops.dataGrid.field.documentCountChart.seriesLabel',
+    {
+      defaultMessage: 'document count',
+    }
+  );
+
+  const overallSeriesNameWithSplit = i18n.translate(
+    'xpack.aiops.dataGrid.field.documentCountChartSplit.seriesLabel',
+    {
+      defaultMessage: 'other document count',
+    }
+  );
+
+  const splitSeriesName = `${changePoint?.fieldName}:${changePoint?.fieldValue}`;
 
   // TODO Let user choose between ZOOM and BRUSH mode.
   const [viewMode] = useState<VIEW_MODE>(VIEW_MODE.BRUSH);
@@ -107,6 +124,26 @@ export const DocumentCountChart: FC<DocumentCountChartProps> = ({
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [chartPoints, timeRangeEarliest, timeRangeLatest, interval]);
 
+  const adjustedChartPointsSplit = useMemo(() => {
+    // Display empty chart when no data in range
+    if (!Array.isArray(chartPointsSplit) || chartPointsSplit.length < 1)
+      return [{ time: timeRangeEarliest, value: 0 }];
+
+    // If chart has only one bucket
+    // it won't show up correctly unless we add an extra data point
+    if (chartPointsSplit.length === 1) {
+      return [
+        ...chartPointsSplit,
+        {
+          time: interval ? Number(chartPointsSplit[0].time) + interval : timeRangeEarliest,
+          value: 0,
+        },
+      ];
+    }
+    return chartPointsSplit;
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [chartPointsSplit, timeRangeEarliest, timeRangeLatest, interval]);
+
   const timefilterUpdateHandler = useCallback(
     (ranges: { from: number; to: number }) => {
       data.query.timefilter.timefilter.setTime({
@@ -223,14 +260,29 @@ export const DocumentCountChart: FC<DocumentCountChartProps> = ({
           <Axis id="left" position={Position.Left} />
           <BarSeries
             id={SPEC_ID}
-            name={seriesName}
+            name={chartPointsSplit ? overallSeriesNameWithSplit : overallSeriesName}
             xScaleType={ScaleType.Time}
             yScaleType={ScaleType.Linear}
             xAccessor="time"
             yAccessors={['value']}
             data={adjustedChartPoints}
+            stackAccessors={[0]}
             timeZone={timeZone}
           />
+          {chartPointsSplit && (
+            <BarSeries
+              id={`${SPEC_ID}_split`}
+              name={splitSeriesName}
+              xScaleType={ScaleType.Time}
+              yScaleType={ScaleType.Linear}
+              xAccessor="time"
+              yAccessors={['value']}
+              data={adjustedChartPointsSplit}
+              stackAccessors={[0]}
+              timeZone={timeZone}
+              color={['orange']}
+            />
+          )}
           {windowParameters && (
             <>
               <DualBrushAnnotation

x-pack/plugins/aiops/public/components/document_count_content/document_count_content/document_count_content.tsx

@@ -6,7 +6,8 @@
  */
 import React, { FC } from 'react';
 
-import { WindowParameters } from '@kbn/aiops-utils';
+import type { WindowParameters } from '@kbn/aiops-utils';
+import type { ChangePoint } from '@kbn/ml-agg-utils';
 
 import { DocumentCountChart, DocumentCountChartPoint } from '../document_count_chart';
 import { TotalCountHeader } from '../total_count_header';
@@ -14,13 +15,17 @@ import { DocumentCountStats } from '../../../get_document_stats';
 
 export interface DocumentCountContentProps {
   brushSelectionUpdateHandler: (d: WindowParameters) => void;
+  changePoint?: ChangePoint;
   documentCountStats?: DocumentCountStats;
+  documentCountStatsSplit?: DocumentCountStats;
   totalCount: number;
 }
 
 export const DocumentCountContent: FC<DocumentCountContentProps> = ({
   brushSelectionUpdateHandler,
+  changePoint,
   documentCountStats,
+  documentCountStatsSplit,
   totalCount,
 }) => {
   if (documentCountStats === undefined) {
@@ -37,6 +42,12 @@ export const DocumentCountContent: FC<DocumentCountContentProps> = ({
     chartPoints = Object.entries(buckets).map(([time, value]) => ({ time: +time, value }));
   }
 
+  let chartPointsSplit: DocumentCountChartPoint[] | undefined;
+  if (documentCountStatsSplit?.buckets !== undefined) {
+    const buckets: Record<string, number> = documentCountStatsSplit?.buckets;
+    chartPointsSplit = Object.entries(buckets).map(([time, value]) => ({ time: +time, value }));
+  }
+
   return (
     <>
       <TotalCountHeader totalCount={totalCount} />
@@ -44,9 +55,11 @@ export const DocumentCountContent: FC<DocumentCountContentProps> = ({
         <DocumentCountChart
           brushSelectionUpdateHandler={brushSelectionUpdateHandler}
           chartPoints={chartPoints}
+          chartPointsSplit={chartPointsSplit}
           timeRangeEarliest={timeRangeEarliest}
           timeRangeLatest={timeRangeLatest}
           interval={documentCountStats.interval}
+          changePoint={changePoint}
         />
       )}
     </>

x-pack/plugins/aiops/public/components/explain_log_rate_spikes/explain_log_rate_spikes_analysis.tsx

@@ -6,10 +6,13 @@
  */
 
 import React, { useEffect, FC } from 'react';
+
 import type { DataView } from '@kbn/data-views-plugin/public';
 import { ProgressControls } from '@kbn/aiops-components';
 import { useFetchStream } from '@kbn/aiops-utils';
 import type { WindowParameters } from '@kbn/aiops-utils';
+import type { ChangePoint } from '@kbn/ml-agg-utils';
+import type { Query } from '@kbn/es-query';
 
 import { useAiOpsKibana } from '../../kibana_context';
 import { initialState, streamReducer } from '../../../common/api/stream_reducer';
@@ -29,6 +32,10 @@ interface ExplainLogRateSpikesAnalysisProps {
   latest: number;
   /** Window parameters for the analysis */
   windowParameters: WindowParameters;
+  searchQuery: Query['query'];
+  onPinnedChangePoint?: (changePoint: ChangePoint | null) => void;
+  onSelectedChangePoint?: (changePoint: ChangePoint | null) => void;
+  selectedChangePoint?: ChangePoint;
 }
 
 export const ExplainLogRateSpikesAnalysis: FC<ExplainLogRateSpikesAnalysisProps> = ({
@@ -36,6 +43,10 @@ export const ExplainLogRateSpikesAnalysis: FC<ExplainLogRateSpikesAnalysisProps>
   earliest,
   latest,
   windowParameters,
+  searchQuery,
+  onPinnedChangePoint,
+  onSelectedChangePoint,
+  selectedChangePoint,
 }) => {
   const { services } = useAiOpsKibana();
   const basePath = services.http?.basePath.get() ?? '';
@@ -48,8 +59,7 @@ export const ExplainLogRateSpikesAnalysis: FC<ExplainLogRateSpikesAnalysisProps>
     {
       start: earliest,
       end: latest,
-      // TODO Consider an optional Kuery.
-      kuery: '',
+      searchQuery: JSON.stringify(searchQuery),
       // TODO Handle data view without time fields.
       timeFieldName: dataView.timeFieldName ?? '',
       index: dataView.title,
@@ -57,22 +67,42 @@ export const ExplainLogRateSpikesAnalysis: FC<ExplainLogRateSpikesAnalysisProps>
     },
     { reducer: streamReducer, initialState }
   );
+
   useEffect(() => {
     start();
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, []);
 
+  // Start handler clears possibly hovered or pinned
+  // change points on analysis refresh.
+  function startHandler() {
+    if (onPinnedChangePoint) {
+      onPinnedChangePoint(null);
+    }
+    if (onSelectedChangePoint) {
+      onSelectedChangePoint(null);
+    }
+    start();
+  }
+
   return (
     <>
       <ProgressControls
         progress={data.loaded}
         progressMessage={data.loadingState ?? ''}
         isRunning={isRunning}
-        onRefresh={start}
+        onRefresh={startHandler}
         onCancel={cancel}
       />
       {data?.changePoints ? (
-        <SpikeAnalysisTable changePointData={data.changePoints} loading={isRunning} error={error} />
+        <SpikeAnalysisTable
+          changePoints={data.changePoints}
+          loading={isRunning}
+          error={error}
+          onPinnedChangePoint={onPinnedChangePoint}
+          onSelectedChangePoint={onSelectedChangePoint}
+          selectedChangePoint={selectedChangePoint}
+        />
       ) : null}
     </>
   );

x-pack/plugins/aiops/public/components/explain_log_rate_spikes/explain_log_rate_spikes_app_state.tsx

@@ -15,9 +15,9 @@ import { useHistory, useLocation } from 'react-router-dom';
 import { SavedSearch } from '@kbn/discover-plugin/public';
 
 import { EuiPageBody } from '@elastic/eui';
-import { DataView } from '@kbn/data-views-plugin/public';
 
-import { ExplainLogRateSpikes } from './explain_log_rate_spikes';
+import type { DataView } from '@kbn/data-views-plugin/public';
+
 import {
   SEARCH_QUERY_LANGUAGE,
   SearchQueryLanguage,
@@ -34,7 +34,9 @@ import {
   SetUrlState,
 } from '../../hooks/url_state';
 
-export interface ExplainLogRateSpikesWrapperProps {
+import { ExplainLogRateSpikesPage } from './explain_log_rate_spikes_page';
+
+export interface ExplainLogRateSpikesAppStateProps {
   /** The data view to analyze. */
   dataView: DataView;
   /** The saved search to analyze. */
@@ -64,7 +66,7 @@ export const getDefaultAiOpsListState = (
 
 export const restorableDefaults = getDefaultAiOpsListState();
 
-export const ExplainLogRateSpikesWrapper: FC<ExplainLogRateSpikesWrapperProps> = ({
+export const ExplainLogRateSpikesAppState: FC<ExplainLogRateSpikesAppStateProps> = ({
   dataView,
   savedSearch,
 }) => {
@@ -159,7 +161,7 @@ export const ExplainLogRateSpikesWrapper: FC<ExplainLogRateSpikesWrapperProps> =
   return (
     <UrlStateContextProvider value={{ searchString: urlSearchString, setUrlState }}>
       <EuiPageBody data-test-subj="aiopsIndexPage" paddingSize="none" panelled={false}>
-        <ExplainLogRateSpikes dataView={dataView} savedSearch={savedSearch} />
+        <ExplainLogRateSpikesPage dataView={dataView} savedSearch={savedSearch} />
       </EuiPageBody>
     </UrlStateContextProvider>
   );

x-pack/plugins/aiops/public/components/explain_log_rate_spikes/explain_log_rate_spikes_page.tsx

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import React, { useCallback, useEffect, useState, FC } from 'react';
+import React, { useCallback, useEffect, useMemo, useState, FC } from 'react';
 import {
   EuiFlexGroup,
   EuiFlexItem,
@@ -19,6 +19,7 @@ import {
 
 import type { DataView } from '@kbn/data-views-plugin/public';
 import type { WindowParameters } from '@kbn/aiops-utils';
+import type { ChangePoint } from '@kbn/ml-agg-utils';
 import { Filter, Query } from '@kbn/es-query';
 import { SavedSearch } from '@kbn/discover-plugin/public';
 
@@ -26,24 +27,28 @@ import { useAiOpsKibana } from '../../kibana_context';
 import { SearchQueryLanguage, SavedSearchSavedObject } from '../../application/utils/search_utils';
 import { useUrlState, usePageUrlState, AppStateKey } from '../../hooks/url_state';
 import { useData } from '../../hooks/use_data';
-import { restorableDefaults } from './explain_log_rate_spikes_wrapper';
 import { FullTimeRangeSelector } from '../full_time_range_selector';
 import { DocumentCountContent } from '../document_count_content/document_count_content';
 import { DatePickerWrapper } from '../date_picker_wrapper';
 import { SearchPanel } from '../search_panel';
+
+import { restorableDefaults } from './explain_log_rate_spikes_app_state';
 import { ExplainLogRateSpikesAnalysis } from './explain_log_rate_spikes_analysis';
 
 /**
  * ExplainLogRateSpikes props require a data view.
  */
-interface ExplainLogRateSpikesProps {
+interface ExplainLogRateSpikesPageProps {
   /** The data view to analyze. */
   dataView: DataView;
   /** The saved search to analyze. */
   savedSearch: SavedSearch | SavedSearchSavedObject | null;
 }
 
-export const ExplainLogRateSpikes: FC<ExplainLogRateSpikesProps> = ({ dataView, savedSearch }) => {
+export const ExplainLogRateSpikesPage: FC<ExplainLogRateSpikesPageProps> = ({
+  dataView,
+  savedSearch,
+}) => {
   const { services } = useAiOpsKibana();
   const { data: dataService } = services;
 
@@ -82,8 +87,37 @@ export const ExplainLogRateSpikes: FC<ExplainLogRateSpikesProps> = ({ dataView,
     [currentSavedSearch, aiopsListState, setAiopsListState]
   );
 
-  const { docStats, timefilter, earliest, latest, searchQueryLanguage, searchString, searchQuery } =
-    useData({ currentDataView: dataView, currentSavedSearch }, aiopsListState, setGlobalState);
+  const [pinnedChangePoint, setPinnedChangePoint] = useState<ChangePoint | null>(null);
+  const [selectedChangePoint, setSelectedChangePoint] = useState<ChangePoint | null>(null);
+
+  // If a row is pinned, still overrule with a potentially hovered row.
+  const currentSelectedChangePoint = useMemo(() => {
+    if (selectedChangePoint) {
+      return selectedChangePoint;
+    } else if (pinnedChangePoint) {
+      return pinnedChangePoint;
+    }
+  }, [pinnedChangePoint, selectedChangePoint]);
+
+  const {
+    overallDocStats,
+    selectedDocStats,
+    timefilter,
+    earliest,
+    latest,
+    searchQueryLanguage,
+    searchString,
+    searchQuery,
+  } = useData(
+    { currentDataView: dataView, currentSavedSearch },
+    aiopsListState,
+    setGlobalState,
+    currentSelectedChangePoint
+  );
+
+  const totalCount = currentSelectedChangePoint
+    ? overallDocStats.totalCount + selectedDocStats.totalCount
+    : overallDocStats.totalCount;
 
   useEffect(() => {
     return () => {
@@ -169,12 +203,16 @@ export const ExplainLogRateSpikes: FC<ExplainLogRateSpikesProps> = ({ dataView,
               setSearchParams={setSearchParams}
             />
           </EuiFlexItem>
-          {docStats?.totalCount !== undefined && (
+          {overallDocStats?.totalCount !== undefined && (
             <EuiFlexItem>
               <DocumentCountContent
                 brushSelectionUpdateHandler={setWindowParameters}
-                documentCountStats={docStats.documentCountStats}
-                totalCount={docStats.totalCount}
+                documentCountStats={overallDocStats.documentCountStats}
+                documentCountStatsSplit={
+                  currentSelectedChangePoint ? selectedDocStats.documentCountStats : undefined
+                }
+                totalCount={totalCount}
+                changePoint={currentSelectedChangePoint}
               />
             </EuiFlexItem>
           )}
@@ -186,6 +224,10 @@ export const ExplainLogRateSpikes: FC<ExplainLogRateSpikesProps> = ({ dataView,
                 earliest={earliest}
                 latest={latest}
                 windowParameters={windowParameters}
+                searchQuery={searchQuery}
+                onPinnedChangePoint={setPinnedChangePoint}
+                onSelectedChangePoint={setSelectedChangePoint}
+                selectedChangePoint={currentSelectedChangePoint}
               />
             </EuiFlexItem>
           )}

x-pack/plugins/aiops/public/components/explain_log_rate_spikes/index.ts

@@ -5,9 +5,26 @@
  * 2.0.
  */
 
-export type { ExplainLogRateSpikesWrapperProps } from './explain_log_rate_spikes_wrapper';
-import { ExplainLogRateSpikesWrapper } from './explain_log_rate_spikes_wrapper';
+/**
+ * The usage of the components in this folder works like this:
+ *
+ * <ExplainLogRateSpikesAppState>
+ *   <ExplainLogRateSpikesPageProps>
+ *     <ExplainLogRateSpikesAnalysis>
+ *
+ * - `ExplainLogRateSpikesAppState`: Manages and passes down url/app state related data, e.g. search parameters.
+ * - `ExplainLogRateSpikesPageProps`: The overall page layout. Includes state management for data selection
+ *   like date range, data fetching for the document count chart, window parameters for the analysis.
+ * - `ExplainLogRateSpikesAnalysis`: Hosts the analysis results table including code to fetch its data.
+ *   While for example the earliest/latest parameter can still be `undefined` on load in the upper component,
+ *   this component expects all necessary parameters/props already to be defined. The reason is the usage of
+ *   data fetching hooks which cannot be called conditionally, so the pattern used here is to only load this
+ *   whole component conditionally on the outer level.
+ */
+
+export type { ExplainLogRateSpikesAppStateProps } from './explain_log_rate_spikes_app_state';
+import { ExplainLogRateSpikesAppState } from './explain_log_rate_spikes_app_state';
 
 // required for dynamic import using React.lazy()
 // eslint-disable-next-line import/no-default-export
-export default ExplainLogRateSpikesWrapper;
+export default ExplainLogRateSpikesAppState;

x-pack/plugins/aiops/public/components/spike_analysis_table/spike_analysis_table.tsx

@@ -19,13 +19,23 @@ const noDataText = i18n.translate('xpack.aiops.correlations.correlationsTable.no
   defaultMessage: 'No data',
 });
 
-interface Props {
-  changePointData: ChangePoint[];
+interface SpikeAnalysisTableProps {
+  changePoints: ChangePoint[];
   error?: string;
   loading: boolean;
+  onPinnedChangePoint?: (changePoint: ChangePoint | null) => void;
+  onSelectedChangePoint?: (changePoint: ChangePoint | null) => void;
+  selectedChangePoint?: ChangePoint;
 }
 
-export const SpikeAnalysisTable: FC<Props> = ({ changePointData, error, loading }) => {
+export const SpikeAnalysisTable: FC<SpikeAnalysisTableProps> = ({
+  changePoints,
+  error,
+  loading,
+  onPinnedChangePoint,
+  onSelectedChangePoint,
+  selectedChangePoint,
+}) => {
   const [pageIndex, setPageIndex] = useState(0);
   const [pageSize, setPageSize] = useState(10);
 
@@ -102,9 +112,9 @@ export const SpikeAnalysisTable: FC<Props> = ({ changePointData, error, loading
   const { pagination, pageOfItems } = useMemo(() => {
     const pageStart = pageIndex * pageSize;
 
-    const itemCount = changePointData?.length ?? 0;
+    const itemCount = changePoints?.length ?? 0;
     return {
-      pageOfItems: changePointData
+      pageOfItems: changePoints
         // Temporary default sorting by ascending pValue until we add native table sorting
         ?.sort((a, b) => {
           return (a?.pValue ?? 1) - (b?.pValue ?? 0);
@@ -117,7 +127,7 @@ export const SpikeAnalysisTable: FC<Props> = ({ changePointData, error, loading
         pageSizeOptions: PAGINATION_SIZE_OPTIONS,
       },
     };
-  }, [pageIndex, pageSize, changePointData]);
+  }, [pageIndex, pageSize, changePoints]);
 
   return (
     <EuiBasicTable
@@ -130,29 +140,35 @@ export const SpikeAnalysisTable: FC<Props> = ({ changePointData, error, loading
       loading={loading}
       error={error}
       // sorting={sorting}
-      //   rowProps={(term) => {
-      //     return {
-      //       onClick: () => {
-      //         // if (setPinnedSignificantTerm) {
-      //         //   setPinnedSignificantTerm(term);
-      //         // }
-      //       },
-      //       onMouseEnter: () => {
-      //         // setSelectedSignificantTerm(term);
-      //       },
-      //       onMouseLeave: () => {
-      //         // setSelectedSignificantTerm(null);
-      //       },
-      //       // style:
-      //       //   selectedTerm &&
-      //       //   selectedTerm.fieldValue === term.fieldValue &&
-      //       //   selectedTerm.fieldName === term.fieldName
-      //       //     ? {
-      //       //         backgroundColor: euiTheme.eui.euiColorLightestShade,
-      //       //       }
-      //       //     : null,
-      //     };
-      //   }}
+      rowProps={(changePoint) => {
+        return {
+          onClick: () => {
+            if (onPinnedChangePoint) {
+              onPinnedChangePoint(changePoint);
+            }
+          },
+          onMouseEnter: () => {
+            if (onSelectedChangePoint) {
+              onSelectedChangePoint(changePoint);
+            }
+          },
+          onMouseLeave: () => {
+            if (onSelectedChangePoint) {
+              onSelectedChangePoint(null);
+            }
+          },
+          style:
+            selectedChangePoint &&
+            selectedChangePoint.fieldValue === changePoint.fieldValue &&
+            selectedChangePoint.fieldName === changePoint.fieldName
+              ? {
+                  // TODO use euiTheme
+                  // backgroundColor: euiTheme.eui.euiColorLightestShade,
+                  backgroundColor: '#ddd',
+                }
+              : null,
+        };
+      }}
     />
   );
 };

x-pack/plugins/aiops/public/get_document_stats.ts

@@ -6,9 +6,13 @@
  */
 
 import { each, get } from 'lodash';
+
 import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+
 import { isPopulatedObject } from '@kbn/ml-is-populated-object';
-import { Query } from '@kbn/es-query';
+import type { ChangePoint } from '@kbn/ml-agg-utils';
+import type { Query } from '@kbn/es-query';
+
 import { buildBaseFilterCriteria } from './application/utils/query_utils';
 
 export interface DocumentCountStats {
@@ -28,6 +32,8 @@ export interface DocumentStatsSearchStrategyParams {
   timeFieldName?: string;
   runtimeFieldMap?: estypes.MappingRuntimeFields;
   fieldsToFetch?: string[];
+  selectedChangePoint?: ChangePoint;
+  includeSelectedChangePoint?: boolean;
 }
 
 export const getDocumentCountStatsRequest = (params: DocumentStatsSearchStrategyParams) => {
@@ -40,10 +46,19 @@ export const getDocumentCountStatsRequest = (params: DocumentStatsSearchStrategy
     searchQuery,
     intervalMs,
     fieldsToFetch,
+    selectedChangePoint,
+    includeSelectedChangePoint,
   } = params;
 
   const size = 0;
-  const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, searchQuery);
+  const filterCriteria = buildBaseFilterCriteria(
+    timeFieldName,
+    earliestMs,
+    latestMs,
+    searchQuery,
+    selectedChangePoint,
+    includeSelectedChangePoint
+  );
 
   // Don't use the sampler aggregation as this can lead to some potentially
   // confusing date histogram results depending on the date range of data amongst shards.

x-pack/plugins/aiops/public/hooks/use_data.ts

@@ -5,23 +5,29 @@
  * 2.0.
  */
 
-import { useEffect, useMemo, useState } from 'react'; //  useCallback, useRef
-import type { DataView } from '@kbn/data-views-plugin/public';
+import { useEffect, useMemo, useState } from 'react';
 import { merge } from 'rxjs';
+
+import type { DataView } from '@kbn/data-views-plugin/public';
 import { UI_SETTINGS } from '@kbn/data-plugin/common';
-import { SavedSearch } from '@kbn/discover-plugin/public';
-import { useAiOpsKibana } from '../kibana_context';
-import { useTimefilter } from './use_time_filter';
-import { aiopsRefresh$ } from '../application/services/timefilter_refresh_service';
+import type { ChangePoint } from '@kbn/ml-agg-utils';
+
+import type { SavedSearch } from '@kbn/discover-plugin/public';
+
 import { TimeBuckets } from '../../common/time_buckets';
-import { useDocumentCountStats } from './use_document_count_stats';
-import { Dictionary } from './url_state';
-import { DocumentStatsSearchStrategyParams } from '../get_document_stats';
+
+import { useAiOpsKibana } from '../kibana_context';
+import { aiopsRefresh$ } from '../application/services/timefilter_refresh_service';
+import type { DocumentStatsSearchStrategyParams } from '../get_document_stats';
+import type { AiOpsIndexBasedAppState } from '../components/explain_log_rate_spikes/explain_log_rate_spikes_app_state';
 import {
   getEsQueryFromSavedSearch,
   SavedSearchSavedObject,
 } from '../application/utils/search_utils';
-import { AiOpsIndexBasedAppState } from '../components/explain_log_rate_spikes/explain_log_rate_spikes_wrapper';
+
+import { useTimefilter } from './use_time_filter';
+import { useDocumentCountStats } from './use_document_count_stats';
+import type { Dictionary } from './url_state';
 
 export const useData = (
   {
@@ -29,7 +35,8 @@ export const useData = (
     currentSavedSearch,
   }: { currentDataView: DataView; currentSavedSearch: SavedSearch | SavedSearchSavedObject | null },
   aiopsListState: AiOpsIndexBasedAppState,
-  onUpdate: (params: Dictionary<unknown>) => void
+  onUpdate: (params: Dictionary<unknown>) => void,
+  selectedChangePoint?: ChangePoint
 ) => {
   const { services } = useAiOpsKibana();
   const { uiSettings, data } = services;
@@ -90,7 +97,23 @@ export const useData = (
     autoRefreshSelector: true,
   });
 
-  const { docStats } = useDocumentCountStats(fieldStatsRequest, lastRefresh);
+  const overallStatsRequest = useMemo(() => {
+    return fieldStatsRequest
+      ? { ...fieldStatsRequest, selectedChangePoint, includeSelectedChangePoint: false }
+      : undefined;
+  }, [fieldStatsRequest, selectedChangePoint]);
+
+  const selectedChangePointStatsRequest = useMemo(() => {
+    return fieldStatsRequest
+      ? { ...fieldStatsRequest, selectedChangePoint, includeSelectedChangePoint: true }
+      : undefined;
+  }, [fieldStatsRequest, selectedChangePoint]);
+
+  const { docStats: overallDocStats } = useDocumentCountStats(overallStatsRequest, lastRefresh);
+  const { docStats: selectedDocStats } = useDocumentCountStats(
+    selectedChangePointStatsRequest,
+    lastRefresh
+  );
 
   function updateFieldStatsRequest() {
     const timefilterActiveBounds = timefilter.getActiveBounds();
@@ -150,7 +173,8 @@ export const useData = (
   }, [searchString, JSON.stringify(searchQuery)]);
 
   return {
-    docStats,
+    overallDocStats,
+    selectedDocStats,
     timefilter,
     /** Start timestamp filter */
     earliest: fieldStatsRequest?.earliest,

x-pack/plugins/aiops/public/shared_lazy_components.tsx

@@ -8,9 +8,9 @@
 import React, { FC, Suspense } from 'react';
 
 import { EuiErrorBoundary, EuiLoadingContent } from '@elastic/eui';
-import type { ExplainLogRateSpikesWrapperProps } from './components/explain_log_rate_spikes';
+import type { ExplainLogRateSpikesAppStateProps } from './components/explain_log_rate_spikes';
 
-const ExplainLogRateSpikesWrapperLazy = React.lazy(
+const ExplainLogRateSpikesAppStateLazy = React.lazy(
   () => import('./components/explain_log_rate_spikes')
 );
 
@@ -21,11 +21,11 @@ const LazyWrapper: FC = ({ children }) => (
 );
 
 /**
- * Lazy-wrapped ExplainLogRateSpikesWrapper React component
- * @param {ExplainLogRateSpikesWrapperProps}  props - properties specifying the data on which to run the analysis.
+ * Lazy-wrapped ExplainLogRateSpikesAppState React component
+ * @param {ExplainLogRateSpikesAppStateProps}  props - properties specifying the data on which to run the analysis.
  */
-export const ExplainLogRateSpikes: FC<ExplainLogRateSpikesWrapperProps> = (props) => (
+export const ExplainLogRateSpikes: FC<ExplainLogRateSpikesAppStateProps> = (props) => (
   <LazyWrapper>
-    <ExplainLogRateSpikesWrapperLazy {...props} />
+    <ExplainLogRateSpikesAppStateLazy {...props} />
   </LazyWrapper>
 );

x-pack/plugins/aiops/server/routes/queries/fetch_field_candidates.test.ts

@@ -9,9 +9,11 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 
 import type { ElasticsearchClient } from '@kbn/core/server';
 
+import type { AiopsExplainLogRateSpikesSchema } from '../../../common/api/explain_log_rate_spikes';
+
 import { fetchFieldCandidates, getRandomDocsRequest } from './fetch_field_candidates';
 
-const params = {
+const params: AiopsExplainLogRateSpikesSchema = {
   index: 'the-index',
   timeFieldName: 'the-time-field-name',
   start: 1577836800000,
@@ -21,7 +23,7 @@ const params = {
   deviationMin: 30,
   deviationMax: 40,
   includeFrozen: false,
-  kuery: '',
+  searchQuery: '{"bool":{"filter":[],"must":[{"match_all":{}}],"must_not":[]}}',
 };
 
 describe('query_field_candidates', () => {
@@ -38,6 +40,7 @@ describe('query_field_candidates', () => {
               query: {
                 bool: {
                   filter: [
+                    { bool: { filter: [], must: [{ match_all: {} }], must_not: [] } },
                     {
                       range: {
                         'the-time-field-name': {

x-pack/plugins/aiops/server/routes/queries/get_filters.test.ts

@@ -8,11 +8,11 @@
 import { getFilters } from './get_filters';
 
 describe('getFilters', () => {
-  it('returns an empty array with no timeFieldName and kuery supplied', () => {
+  it('returns an empty array with no timeFieldName and searchQuery supplied', () => {
     const filters = getFilters({
       index: 'the-index',
       timeFieldName: '',
-      kuery: '',
+      searchQuery: '{"bool":{"filter":[],"must":[{"match_all":{}}],"must_not":[]}}',
       start: 1577836800000,
       end: 1609459200000,
       baselineMin: 10,
@@ -27,7 +27,7 @@ describe('getFilters', () => {
     const filters = getFilters({
       index: 'the-index',
       timeFieldName: 'the-time-field-name',
-      kuery: '',
+      searchQuery: '{"bool":{"filter":[],"must":[{"match_all":{}}],"must_not":[]}}',
       start: 1577836800000,
       end: 1609459200000,
       baselineMin: 10,

x-pack/plugins/aiops/server/routes/queries/get_filters.ts

@@ -8,7 +8,6 @@
 import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 
 import { ESFilter } from '@kbn/core/types/elasticsearch';
-import { fromKueryExpression, toElasticsearchQuery } from '@kbn/es-query';
 
 import type { AiopsExplainLogRateSpikesSchema } from '../../../common/api/explain_log_rate_spikes';
 
@@ -30,17 +29,7 @@ export function rangeQuery(
   ];
 }
 
-export function kqlQuery(kql: string): estypes.QueryDslQueryContainer[] {
-  if (!kql) {
-    return [];
-  }
-
-  const ast = fromKueryExpression(kql);
-  return [toElasticsearchQuery(ast)];
-}
-
 export function getFilters({
-  kuery,
   start,
   end,
   timeFieldName,
@@ -51,9 +40,5 @@ export function getFilters({
     filters.push(...rangeQuery(start, end, timeFieldName));
   }
 
-  if (kuery !== '') {
-    filters.push(...kqlQuery(kuery));
-  }
-
   return filters;
 }

x-pack/plugins/aiops/server/routes/queries/get_query_with_params.test.ts

@@ -20,12 +20,13 @@ describe('getQueryWithParams', () => {
         deviationMin: 30,
         deviationMax: 40,
         includeFrozen: false,
-        kuery: '',
+        searchQuery: '{"bool":{"filter":[],"must":[{"match_all":{}}],"must_not":[]}}',
       },
     });
     expect(query).toEqual({
       bool: {
         filter: [
+          { bool: { filter: [], must: [{ match_all: {} }], must_not: [] } },
           {
             range: {
               'the-time-field-name': {
@@ -52,7 +53,7 @@ describe('getQueryWithParams', () => {
         deviationMin: 30,
         deviationMax: 40,
         includeFrozen: false,
-        kuery: '',
+        searchQuery: '{"bool":{"filter":[],"must":[{"match_all":{}}],"must_not":[]}}',
       },
       termFilters: [
         {
@@ -64,6 +65,7 @@ describe('getQueryWithParams', () => {
     expect(query).toEqual({
       bool: {
         filter: [
+          { bool: { filter: [], must: [{ match_all: {} }], must_not: [] } },
           {
             range: {
               'the-time-field-name': {

x-pack/plugins/aiops/server/routes/queries/get_query_with_params.ts

@@ -7,7 +7,9 @@
 
 import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 
+import type { Query } from '@kbn/es-query';
 import type { FieldValuePair } from '@kbn/ml-agg-utils';
+
 import type { AiopsExplainLogRateSpikesSchema } from '../../../common/api/explain_log_rate_spikes';
 
 import { getFilters } from './get_filters';
@@ -21,9 +23,11 @@ interface QueryParams {
   termFilters?: FieldValuePair[];
 }
 export const getQueryWithParams = ({ params, termFilters }: QueryParams) => {
+  const searchQuery = JSON.parse(params.searchQuery) as Query['query'];
   return {
     bool: {
       filter: [
+        searchQuery,
         ...getFilters(params),
         ...(Array.isArray(termFilters) ? termFilters.map(getTermsQuery) : []),
       ] as estypes.QueryDslQueryContainer[],

x-pack/plugins/aiops/server/routes/queries/get_request_base.test.ts

@@ -12,7 +12,7 @@ describe('getRequestBase', () => {
     const requestBase = getRequestBase({
       index: 'the-index',
       timeFieldName: 'the-time-field-name',
-      kuery: '',
+      searchQuery: '{"bool":{"filter":[],"must":[{"match_all":{}}],"must_not":[]}}',
       start: 1577836800000,
       end: 1609459200000,
       baselineMin: 10,
@@ -28,7 +28,7 @@ describe('getRequestBase', () => {
       index: 'the-index',
       timeFieldName: 'the-time-field-name',
       includeFrozen: true,
-      kuery: '',
+      searchQuery: '{"bool":{"filter":[],"must":[{"match_all":{}}],"must_not":[]}}',
       start: 1577836800000,
       end: 1609459200000,
       baselineMin: 10,

x-pack/test/api_integration/apis/aiops/explain_log_rate_spikes.ts

@@ -28,7 +28,7 @@ export default ({ getService }: FtrProviderContext) => {
     deviationMin: 1561986810992,
     end: 2147483647000,
     index: 'ft_ecommerce',
-    kuery: '',
+    searchQuery: '{"bool":{"filter":[],"must":[{"match_all":{}}],"must_not":[]}}',
     start: 0,
     timeFieldName: 'order_date',
   };

x-pack/test/api_integration_basic/apis/aiops/permissions.ts

@@ -26,7 +26,7 @@ export default ({ getService }: FtrProviderContext) => {
     deviationMin: 1561986810992,
     end: 2147483647000,
     index: 'ft_ecommerce',
-    kuery: '',
+    searchQuery: '{"bool":{"filter":[],"must":[{"match_all":{}}],"must_not":[]}}',
     start: 0,
     timeFieldName: 'order_date',
   };