[ML] Add new Data comparison view (#161365)

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

.github/CODEOWNERS

@@ -492,6 +492,7 @@ x-pack/packages/ml/data_grid @elastic/ml-ui
 x-pack/packages/ml/date_picker @elastic/ml-ui
 x-pack/packages/ml/date_utils @elastic/ml-ui
 x-pack/packages/ml/error_utils @elastic/ml-ui
+x-pack/packages/ml/in_memory_table @elastic/ml-ui
 x-pack/packages/ml/is_defined @elastic/ml-ui
 x-pack/packages/ml/is_populated_object @elastic/ml-ui
 x-pack/packages/ml/kibana_theme @elastic/ml-ui

api_docs/kbn_aiops_components.devdocs.json

@@ -124,4 +124,4 @@
     "misc": [],
     "objects": []
   }
-}
+}

package.json

@@ -508,6 +508,7 @@
     "@kbn/ml-date-picker": "link:x-pack/packages/ml/date_picker",
     "@kbn/ml-date-utils": "link:x-pack/packages/ml/date_utils",
     "@kbn/ml-error-utils": "link:x-pack/packages/ml/error_utils",
+    "@kbn/ml-in-memory-table": "link:x-pack/packages/ml/in_memory_table",
     "@kbn/ml-is-defined": "link:x-pack/packages/ml/is_defined",
     "@kbn/ml-is-populated-object": "link:x-pack/packages/ml/is_populated_object",
     "@kbn/ml-kibana-theme": "link:x-pack/packages/ml/kibana_theme",

packages/deeplinks/ml/deep_links.ts

@@ -15,6 +15,7 @@ export type LinkId =
   | 'anomalyDetection'
   | 'anomalyExplorer'
   | 'singleMetricViewer'
+  | 'dataComparison'
   | 'dataFrameAnalytics'
   | 'resultExplorer'
   | 'analyticsMap'

tsconfig.base.json

@@ -978,6 +978,8 @@
       "@kbn/ml-date-utils/*": ["x-pack/packages/ml/date_utils/*"],
       "@kbn/ml-error-utils": ["x-pack/packages/ml/error_utils"],
       "@kbn/ml-error-utils/*": ["x-pack/packages/ml/error_utils/*"],
+      "@kbn/ml-in-memory-table": ["x-pack/packages/ml/in_memory_table"],
+      "@kbn/ml-in-memory-table/*": ["x-pack/packages/ml/in_memory_table/*"],
       "@kbn/ml-is-defined": ["x-pack/packages/ml/is_defined"],
       "@kbn/ml-is-defined/*": ["x-pack/packages/ml/is_defined/*"],
       "@kbn/ml-is-populated-object": ["x-pack/packages/ml/is_populated_object"],

x-pack/packages/ml/aiops_components/index.ts

@@ -7,3 +7,5 @@
 
 export { DualBrush, DualBrushAnnotation } from './src/dual_brush';
 export { ProgressControls } from './src/progress_controls';
+export { DocumentCountChart } from './src/document_count_chart';
+export type { DocumentCountChartPoint, DocumentCountChartProps } from './src/document_count_chart';

x-pack/packages/ml/aiops_components/src/document_count_chart/brush_badge.tsx

@@ -22,6 +22,15 @@ interface BrushBadgeProps {
   width: number;
 }
 
+/**
+ * Badge component
+ * @param label - label
+ * @param marginLeft - margin left
+ * @param timestampFrom - start timestamp
+ * @param timestampTo - ending timestamp
+ * @param width - width of badge
+ * @constructor
+ */
 export const BrushBadge: FC<BrushBadgeProps> = ({
   label,
   marginLeft,

x-pack/packages/ml/aiops_components/src/document_count_chart/document_count_chart.tsx

@@ -23,14 +23,20 @@ import {
 
 import { i18n } from '@kbn/i18n';
 import { IUiSettingsClient } from '@kbn/core/public';
-import { DualBrush, DualBrushAnnotation } from '@kbn/aiops-components';
 import { getSnappedWindowParameters, getWindowParameters } from '@kbn/aiops-utils';
 import type { WindowParameters } from '@kbn/aiops-utils';
 import { MULTILAYER_TIME_AXIS_STYLE } from '@kbn/charts-plugin/common';
 
-import { useAiopsAppContext } from '../../../hooks/use_aiops_app_context';
+import {
+  BarStyleAccessor,
+  RectAnnotationSpec,
+} from '@elastic/charts/dist/chart_types/xy_chart/utils/specs';
 
+import type { DataPublicPluginStart } from '@kbn/data-plugin/public';
+import type { ChartsPluginStart } from '@kbn/charts-plugin/public';
+import type { FieldFormatsStart } from '@kbn/field-formats-plugin/public';
 import { BrushBadge } from './brush_badge';
+import { DualBrush, DualBrushAnnotation } from '../..';
 
 declare global {
   interface Window {
@@ -46,27 +52,79 @@ interface TimeFilterRange {
   to: number;
 }
 
+/**
+ * Datum for the bar chart
+ */
 export interface DocumentCountChartPoint {
+  /**
+   * Time of bucket
+   */
   time: number | string;
+  /**
+   * Number of doc count for that time bucket
+   */
   value: number;
 }
 
-interface DocumentCountChartProps {
-  brushSelectionUpdateHandler?: (d: WindowParameters, force: boolean) => void;
+/**
+ * Brush settings
+ */
+export interface BrushSettings {
+  /**
+   * Optional label name for brush
+   */
+  label?: string;
+  /**
+   * Optional style for brush
+   */
+  annotationStyle?: RectAnnotationSpec['style'];
+  /**
+   * Optional width for brush
+   */
+  badgeWidth?: number;
+}
+
+/**
+ * Props for document count chart
+ */
+export interface DocumentCountChartProps {
+  /** List of Kibana services that are required as dependencies */
+  dependencies: {
+    data: DataPublicPluginStart;
+    charts: ChartsPluginStart;
+    fieldFormats: FieldFormatsStart;
+    uiSettings: IUiSettingsClient;
+  };
+  /** Optional callback function which gets called the brush selection has changed */
+  brushSelectionUpdateHandler?: (windowParameters: WindowParameters, force: boolean) => void;
+  /** Optional width */
   width?: number;
+  /** Data chart points */
   chartPoints: DocumentCountChartPoint[];
+  /** Data chart points split */
   chartPointsSplit?: DocumentCountChartPoint[];
+  /** Start time range for the chart */
   timeRangeEarliest: number;
+  /** Ending time range for the chart */
   timeRangeLatest: number;
+  /** Time interval for the document count buckets */
   interval: number;
+  /** Label to name the adjustedChartPointsSplit histogram */
   chartPointsSplitLabel: string;
+  /** Whether or not brush has been reset */
   isBrushCleared: boolean;
-  /* Timestamp for start of initial analysis */
+  /** Timestamp for start of initial analysis */
   autoAnalysisStart?: number | WindowParameters;
+  /** Optional style to override bar chart  */
+  barStyleAccessor?: BarStyleAccessor;
   /** Optional color override for the default bar color for charts */
   barColorOverride?: string;
   /** Optional color override for the highlighted bar color for charts */
   barHighlightColorOverride?: string;
+  /** Optional settings override for the 'deviation' brush */
+  deviationBrush?: BrushSettings;
+  /** Optional settings override for the 'baseline' brush */
+  baselineBrush?: BrushSettings;
 }
 
 const SPEC_ID = 'document_count';
@@ -102,7 +160,29 @@ function getBaselineBadgeOverflow(
     : 0;
 }
 
+/**
+ * Document count chart with draggable brushes to select time ranges
+ * by default use `Baseline` and `Deviation` for the badge names
+ * @param dependencies - List of Kibana services that are required as dependencies
+ * @param brushSelectionUpdateHandler - Optional callback function which gets called the brush selection has changed
+ * @param width - Optional width
+ * @param chartPoints - Data chart points
+ * @param chartPointsSplit - Data chart points split
+ * @param timeRangeEarliest - Start time range for the chart
+ * @param timeRangeLatest - Ending time range for the chart
+ * @param interval - Time interval for the document count buckets
+ * @param chartPointsSplitLabel - Label to name the adjustedChartPointsSplit histogram
+ * @param isBrushCleared - Whether or not brush has been reset
+ * @param autoAnalysisStart - Timestamp for start of initial analysis
+ * @param barColorOverride - Optional color override for the default bar color for charts
+ * @param barStyleAccessor - Optional style to override bar chart
+ * @param barHighlightColorOverride - Optional color override for the highlighted bar color for charts
+ * @param deviationBrush - Optional settings override for the 'deviation' brush
+ * @param baselineBrush - Optional settings override for the 'baseline' brush
+ * @constructor
+ */
 export const DocumentCountChart: FC<DocumentCountChartProps> = ({
+  dependencies,
   brushSelectionUpdateHandler,
   width,
   chartPoints,
@@ -114,9 +194,12 @@ export const DocumentCountChart: FC<DocumentCountChartProps> = ({
   isBrushCleared,
   autoAnalysisStart,
   barColorOverride,
+  barStyleAccessor,
   barHighlightColorOverride,
+  deviationBrush = {},
+  baselineBrush = {},
 }) => {
-  const { data, uiSettings, fieldFormats, charts } = useAiopsAppContext();
+  const { data, uiSettings, fieldFormats, charts } = dependencies;
 
   const chartTheme = charts.theme.useChartsTheme();
   const chartBaseTheme = charts.theme.useChartsBaseTheme();
@@ -339,22 +422,28 @@ export const DocumentCountChart: FC<DocumentCountChartProps> = ({
         <div className="aiopsHistogramBrushes" data-test-subj="aiopsHistogramBrushes">
           <div css={{ height: BADGE_HEIGHT }}>
             <BrushBadge
-              label={i18n.translate('xpack.aiops.documentCountChart.baselineBadgeLabel', {
-                defaultMessage: 'Baseline',
-              })}
+              label={
+                baselineBrush.label ??
+                i18n.translate('xpack.aiops.documentCountChart.baselineBadgeLabel', {
+                  defaultMessage: 'Baseline',
+                })
+              }
               marginLeft={baselineBadgeMarginLeft - baselineBadgeOverflow}
               timestampFrom={windowParameters.baselineMin}
               timestampTo={windowParameters.baselineMax}
-              width={BADGE_WIDTH}
+              width={baselineBrush.badgeWidth ?? BADGE_WIDTH}
             />
             <BrushBadge
-              label={i18n.translate('xpack.aiops.documentCountChart.deviationBadgeLabel', {
-                defaultMessage: 'Deviation',
-              })}
+              label={
+                deviationBrush.label ??
+                i18n.translate('xpack.aiops.documentCountChart.deviationBadgeLabel', {
+                  defaultMessage: 'Deviation',
+                })
+              }
               marginLeft={mlBrushMarginLeft + (windowParametersAsPixels?.deviationMin ?? 0)}
               timestampFrom={windowParameters.deviationMin}
               timestampTo={windowParameters.deviationMax}
-              width={BADGE_WIDTH}
+              width={deviationBrush.badgeWidth ?? BADGE_WIDTH}
             />
           </div>
           <div
@@ -416,6 +505,7 @@ export const DocumentCountChart: FC<DocumentCountChartProps> = ({
               timeZone={timeZone}
               color={barColor}
               yNice
+              styleAccessor={barStyleAccessor}
             />
           )}
           {adjustedChartPointsSplit?.length && (
@@ -438,11 +528,13 @@ export const DocumentCountChart: FC<DocumentCountChartProps> = ({
                 id="aiopsBaseline"
                 min={windowParameters.baselineMin}
                 max={windowParameters.baselineMax}
+                style={baselineBrush.annotationStyle}
               />
               <DualBrushAnnotation
                 id="aiopsDeviation"
                 min={windowParameters.deviationMin}
                 max={windowParameters.deviationMax}
+                style={deviationBrush.annotationStyle}
               />
             </>
           )}

x-pack/packages/ml/aiops_components/src/document_count_chart/index.ts

@@ -6,4 +6,4 @@
  */
 
 export { DocumentCountChart } from './document_count_chart';
-export type { DocumentCountChartPoint } from './document_count_chart';
+export type { DocumentCountChartPoint, DocumentCountChartProps } from './document_count_chart';

x-pack/packages/ml/aiops_components/src/dual_brush/dual_brush.tsx

@@ -55,22 +55,71 @@ const BRUSH_HANDLE_SIZE = 4;
 const BRUSH_HANDLE_ROUNDED_CORNER = 2;
 
 interface DualBrushProps {
+  /**
+   * Min and max numeric timestamps for the two brushes
+   */
   windowParameters: WindowParameters;
+  /**
+   * Min timestamp for x domain
+   */
   min: number;
+  /**
+   * Max timestamp for x domain
+   */
   max: number;
+  /**
+   * Callback function whenever the brush changes
+   */
   onChange?: (windowParameters: WindowParameters, windowPxParameters: WindowParameters) => void;
+  /**
+   * Margin left
+   */
   marginLeft: number;
+  /**
+   * Nearest timestamps to snap to the brushes to
+   */
   snapTimestamps?: number[];
+  /**
+   * Width
+   */
   width: number;
 }
 
+/**
+ * DualBrush React Component
+ * Dual brush component that overlays the document count chart
+ * @type {FC<DualBrushProps>}
+ * @param props - `DualBrushProps` component props
+ * @returns {React.ReactElement} The DualBrush component.
+ */
 export function DualBrush({
+  /**
+   * Min and max numeric timestamps for the two brushes
+   */
   windowParameters,
+  /**
+   * Min timestamp for x domain
+   */
   min,
+  /**
+   * Max timestamp for x domain
+   */
   max,
+  /**
+   * Callback function whenever the brush changes
+   */
   onChange,
+  /**
+   * Margin left
+   */
   marginLeft,
+  /**
+   * Nearest timestamps to snap to the brushes to
+   */
   snapTimestamps,
+  /**
+   * Width
+   */
   width,
 }: DualBrushProps) {
   const d3BrushContainer = useRef(null);

x-pack/packages/ml/aiops_components/src/dual_brush/dual_brush_annotation.tsx

@@ -9,14 +9,23 @@ import React, { FC } from 'react';
 
 import { RectAnnotation } from '@elastic/charts';
 import { useEuiTheme } from '@elastic/eui';
+import { RectAnnotationSpec } from '@elastic/charts/dist/chart_types/xy_chart/utils/specs';
 
 interface BrushAnnotationProps {
   id: string;
   min: number;
   max: number;
+  style?: RectAnnotationSpec['style'];
 }
 
-export const DualBrushAnnotation: FC<BrushAnnotationProps> = ({ id, min, max }) => {
+/**
+ * DualBrushAnnotation React Component
+ * Dual brush annotation component that overlays the document count chart
+ * @type {FC<BrushAnnotationProps>}
+ * @param props - `BrushAnnotationProps` component props
+ * @returns {React.ReactElement} The DualBrushAnnotation component.
+ */
+export const DualBrushAnnotation: FC<BrushAnnotationProps> = ({ id, min, max, style }) => {
   const { euiTheme } = useEuiTheme();
   const { colors } = euiTheme;
 
@@ -34,12 +43,14 @@ export const DualBrushAnnotation: FC<BrushAnnotationProps> = ({ id, min, max })
         },
       ]}
       id={`rect_brush_annotation_${id}`}
-      style={{
-        strokeWidth: 0,
-        stroke: colors.lightShade,
-        fill: colors.lightShade,
-        opacity: 0.5,
-      }}
+      style={
+        style ?? {
+          strokeWidth: 0,
+          stroke: colors.lightShade,
+          fill: colors.lightShade,
+          opacity: 0.5,
+        }
+      }
       hideTooltips={true}
     />
   );

x-pack/packages/ml/aiops_components/src/progress_controls/progress_controls.tsx

@@ -26,6 +26,9 @@ import { useAnimatedProgressBarBackground } from './use_animated_progress_bar_ba
 // TODO Consolidate with duplicate component `CorrelationsProgressControls` in
 // `x-pack/plugins/apm/public/components/app/correlations/progress_controls.tsx`
 
+/**
+ * Props for ProgressControlProps
+ */
 interface ProgressControlProps {
   isBrushCleared: boolean;
   progress: number;
@@ -35,8 +38,32 @@ interface ProgressControlProps {
   onReset: () => void;
   isRunning: boolean;
   shouldRerunAnalysis: boolean;
+  runAnalysisDisabled?: boolean;
 }
 
+/**
+ * ProgressControls React Component
+ * Component with ability to Run & cancel analysis
+ * by default use `Baseline` and `Deviation` for the badge name
+ * @type {FC<ProgressControlProps>}
+ * @param children - List of Kibana services that are required as dependencies
+ * @param brushSelectionUpdateHandler - Optional callback function which gets called the brush selection has changed
+ * @param width - Optional width
+ * @param chartPoints - Data chart points
+ * @param chartPointsSplit - Data chart points split
+ * @param timeRangeEarliest - Start time range for the chart
+ * @param timeRangeLatest - Ending time range for the chart
+ * @param interval - Time interval for the document count buckets
+ * @param chartPointsSplitLabel - Label to name the adjustedChartPointsSplit histogram
+ * @param isBrushCleared - Whether or not brush has been reset
+ * @param autoAnalysisStart - Timestamp for start of initial analysis
+ * @param barColorOverride - Optional color override for the default bar color for charts
+ * @param barStyleAccessor - Optional style to override bar chart
+ * @param barHighlightColorOverride - Optional color override for the highlighted bar color for charts
+ * @param deviationBrush - Optional settings override for the 'deviation' brush
+ * @param baselineBrush - Optional settings override for the 'baseline' brush
+ * @returns {React.ReactElement} The ProgressControls component.
+ */
 export const ProgressControls: FC<ProgressControlProps> = ({
   children,
   isBrushCleared,
@@ -47,6 +74,7 @@ export const ProgressControls: FC<ProgressControlProps> = ({
   onReset,
   isRunning,
   shouldRerunAnalysis,
+  runAnalysisDisabled = false,
 }) => {
   const { euiTheme } = useEuiTheme();
   const runningProgressBarStyles = useAnimatedProgressBarBackground(euiTheme.colors.success);
@@ -57,6 +85,7 @@ export const ProgressControls: FC<ProgressControlProps> = ({
       <EuiFlexItem grow={false}>
         {!isRunning && (
           <EuiButton
+            disabled={runAnalysisDisabled}
             data-test-subj={`aiopsRerunAnalysisButton${shouldRerunAnalysis ? ' shouldRerun' : ''}`}
             size="s"
             onClick={onRefresh}

x-pack/packages/ml/aiops_components/tsconfig.json

@@ -22,6 +22,10 @@
     "@kbn/i18n-react",
     "@kbn/aiops-utils",
     "@kbn/i18n",
+    "@kbn/core",
+    "@kbn/charts-plugin",
+    "@kbn/data-plugin",
+    "@kbn/field-formats-plugin",
   ],
   "exclude": [
     "target/**/*",

x-pack/packages/ml/in_memory_table/README.md

@@ -0,0 +1,3 @@
+# @kbn/ml-in-memory-table
+
+This package contains custom hooks for the EuiInMemoryTable.

x-pack/packages/ml/in_memory_table/hooks/use_table_state.ts

@@ -5,14 +5,46 @@
  * 2.0.
  */
 
-import { useState } from 'react';
+import { Dispatch, SetStateAction, useState } from 'react';
 import { EuiInMemoryTable, Direction, Pagination } from '@elastic/eui';
 
-export function useTableState<T>(items: T[], initialSortField: string) {
+/**
+ * Returned type for useTableState hook
+ */
+export interface UseTableState<T> {
+  /**
+   * Callback function which gets called whenever the pagination or sorting state of the table changed
+   */
+  onTableChange: EuiInMemoryTable<T>['onTableChange'];
+  /**
+   * Pagination object which contains pageIndex, pageSize
+   */
+  pagination: Pagination;
+  /**
+   * Sort field and sort direction
+   */
+  sorting: { sort: { field: string; direction: Direction } };
+  /**
+   * setPageIndex setter function which updates page index
+   */
+  setPageIndex: Dispatch<SetStateAction<number>>;
+}
+
+/**
+ * Hook to help with managing the pagination and sorting for EuiInMemoryTable
+ * @param {TableItem} items - data to show in the table
+ * @param {string} initialSortField - field name to sort by default
+ * @param {string} initialSortDirection - default to 'asc'
+ */
+export function useTableState<T>(
+  items: T[],
+  initialSortField: string,
+  initialSortDirection: 'asc' | 'desc' = 'asc'
+) {
   const [pageIndex, setPageIndex] = useState(0);
   const [pageSize, setPageSize] = useState(10);
   const [sortField, setSortField] = useState<string>(initialSortField);
-  const [sortDirection, setSortDirection] = useState<Direction>('asc');
+  const [sortDirection, setSortDirection] = useState<Direction>(initialSortDirection);
 
   const onTableChange: EuiInMemoryTable<T>['onTableChange'] = ({
     page = { index: 0, size: 10 },
@@ -42,5 +74,10 @@ export function useTableState<T>(items: T[], initialSortField: string) {
     },
   };
 
-  return { onTableChange, pagination, sorting, setPageIndex };
+  return {
+    onTableChange,
+    pagination,
+    sorting,
+    setPageIndex,
+  };
 }

x-pack/packages/ml/in_memory_table/index.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export { useTableState } from './hooks/use_table_state';
+export type { UseTableState } from './hooks/use_table_state';

x-pack/packages/ml/in_memory_table/jest.config.js

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+module.exports = {
+  preset: '@kbn/test',
+  rootDir: '../../../..',
+  roots: ['<rootDir>/x-pack/packages/ml/in_memory_table'],
+};

x-pack/packages/ml/in_memory_table/kibana.jsonc

@@ -0,0 +1,5 @@
+{
+  "type": "shared-common",
+  "id": "@kbn/ml-in-memory-table",
+  "owner": "@elastic/ml-ui"
+}

x-pack/packages/ml/in_memory_table/package.json

@@ -0,0 +1,6 @@
+{
+  "name": "@kbn/ml-in-memory-table",
+  "private": true,
+  "version": "1.0.0",
+  "license": "Elastic License 2.0"
+}

x-pack/packages/ml/in_memory_table/tsconfig.json

@@ -0,0 +1,23 @@
+{
+  "extends": "../../../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "target/types",
+    "types": [
+      "jest",
+      "node",
+      "react",
+      "@emotion/react/types/css-prop",
+      "@testing-library/jest-dom",
+      "@testing-library/react"
+    ]
+  },
+  "include": [
+    "**/*.ts",
+    "**/*.tsx",
+  ],
+  "exclude": [
+    "target/**/*"
+  ],
+  "kbn_references": [
+  ]
+}

x-pack/packages/ml/query_utils/index.ts

@@ -9,3 +9,6 @@ export { addExcludeFrozenToQuery } from './src/add_exclude_frozen_to_query';
 export { buildBaseFilterCriteria } from './src/build_base_filter_criteria';
 export { ES_CLIENT_TOTAL_HITS_RELATION } from './src/es_client_total_hits_relation';
 export { getSafeAggregationName } from './src/get_safe_aggregation_name';
+export { SEARCH_QUERY_LANGUAGE } from './src/types';
+export type { SearchQueryLanguage } from './src/types';
+export { getDefaultDSLQuery } from './src/get_default_query';

x-pack/packages/ml/query_utils/src/get_default_query.ts

@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { cloneDeep } from 'lodash';
+import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+
+const DEFAULT_QUERY = {
+  bool: {
+    must: [
+      {
+        match_all: {},
+      },
+    ],
+  },
+};
+
+/**
+ * Default DSL query which matches all the results
+ */
+export function getDefaultDSLQuery(): QueryDslQueryContainer {
+  return cloneDeep(DEFAULT_QUERY);
+}

x-pack/packages/ml/query_utils/src/types.ts

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/**
+ * Constant for kuery and lucene string
+ */
+export const SEARCH_QUERY_LANGUAGE = {
+  KUERY: 'kuery',
+  LUCENE: 'lucene',
+} as const;
+
+/**
+ * Type for SearchQueryLanguage
+ */
+export type SearchQueryLanguage = typeof SEARCH_QUERY_LANGUAGE[keyof typeof SEARCH_QUERY_LANGUAGE];

x-pack/packages/ml/random_sampler_utils/index.ts

@@ -10,3 +10,4 @@ export {
   createRandomSamplerWrapper,
   type RandomSamplerWrapper,
 } from './src/random_sampler_wrapper';
+export * from './src/random_sampler_manager';

x-pack/packages/ml/random_sampler_utils/src/random_sampler_manager.ts

@@ -0,0 +1,168 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { BehaviorSubject } from 'rxjs';
+import { createRandomSamplerWrapper } from './random_sampler_wrapper';
+
+/**
+ * List of default probabilities to use for random sampler
+ */
+export const RANDOM_SAMPLER_PROBABILITIES = [
+  0.00001, 0.00005, 0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5,
+].map((n) => n * 100);
+
+/**
+ * Default recommended minimum probability for default sampling
+ */
+export const MIN_SAMPLER_PROBABILITY = 0.00001;
+
+/**
+ * Default step minimum probability for default sampling
+ */
+export const RANDOM_SAMPLER_STEP = MIN_SAMPLER_PROBABILITY * 100;
+
+/**
+ * Default probability to use
+ */
+export const DEFAULT_PROBABILITY = 0.001;
+
+/**
+ * Default options for random sampler
+ */
+export const RANDOM_SAMPLER_OPTION = {
+  ON_AUTOMATIC: 'on_automatic',
+  ON_MANUAL: 'on_manual',
+  OFF: 'off',
+} as const;
+
+/**
+ * Default option for random sampler type
+ */
+export type RandomSamplerOption = typeof RANDOM_SAMPLER_OPTION[keyof typeof RANDOM_SAMPLER_OPTION];
+
+/**
+ * Type for the random sampler probability
+ */
+export type RandomSamplerProbability = number | null;
+
+/**
+ * Class that helps manage random sampling settings
+ * Automatically calculates the probability if only total doc count is provided
+ * Else, use the probability that was explicitly set
+ */
+export class RandomSampler {
+  private docCount$ = new BehaviorSubject<number>(0);
+  private mode$ = new BehaviorSubject<RandomSamplerOption>(RANDOM_SAMPLER_OPTION.ON_AUTOMATIC);
+  private probability$ = new BehaviorSubject<RandomSamplerProbability>(DEFAULT_PROBABILITY);
+  private setRandomSamplerModeInStorage: (mode: RandomSamplerOption) => void;
+  private setRandomSamplerProbabilityInStorage: (prob: RandomSamplerProbability) => void;
+
+  /**
+   * Initial values
+   * @param {RandomSamplerOption} randomSamplerMode - random sampler mode
+   * @param setRandomSamplerMode - callback to be called when random sampler mode is set
+   * @param randomSamplerProbability - initial value for random sampler probability
+   * @param setRandomSamplerProbability - initial setter for random sampler probability
+   */
+  constructor(
+    randomSamplerMode: RandomSamplerOption,
+    setRandomSamplerMode: (mode: RandomSamplerOption) => void,
+    randomSamplerProbability: RandomSamplerProbability,
+    setRandomSamplerProbability: (prob: RandomSamplerProbability) => void
+  ) {
+    this.mode$.next(randomSamplerMode);
+    this.setRandomSamplerModeInStorage = setRandomSamplerMode;
+    this.probability$.next(randomSamplerProbability);
+    this.setRandomSamplerProbabilityInStorage = setRandomSamplerProbability;
+  }
+
+  /**
+   * Set total doc count
+   * If probability is not explicitly set, this doc count is used for calculating the suggested probability for sampling
+   * @param docCount - total document count
+   */
+  setDocCount(docCount: number) {
+    return this.docCount$.next(docCount);
+  }
+
+  /**
+   * Get doc count
+   */
+  getDocCount() {
+    return this.docCount$.getValue();
+  }
+  /**
+   * Set and save in storage what mode of random sampling to use
+   * @param {RandomSamplerOption} mode - mode to use when wrapping/unwrapping random sampling aggs
+   */
+  public setMode(mode: RandomSamplerOption) {
+    this.setRandomSamplerModeInStorage(mode);
+    return this.mode$.next(mode);
+  }
+
+  /**
+   * Observable to get currently set mode of random sampling
+   */
+  public getMode$() {
+    return this.mode$.asObservable();
+  }
+
+  /**
+   * Helper to get currently set mode of random sampling
+   */
+  public getMode() {
+    return this.mode$.getValue();
+  }
+
+  /**
+   * Helper to set the probability to use for random sampling requests
+   * @param {RandomSamplerProbability} probability - numeric value 0 < probability < 1 to use for random sampling
+   */
+  public setProbability(probability: RandomSamplerProbability) {
+    this.setRandomSamplerProbabilityInStorage(probability);
+    return this.probability$.next(probability);
+  }
+
+  /**
+   * Observability to get the probability to use for random sampling requests
+   */
+  public getProbability$() {
+    return this.probability$.asObservable();
+  }
+
+  /**
+   * Observability to get the probability to use for random sampling requests
+   */
+  public getProbability() {
+    return this.probability$.getValue();
+  }
+
+  /**
+   * Helper to return factory to extend any ES aggregations with the random sampling probability
+   * Returns wrapper = {wrap, unwrap}
+   * Where {wrap} extends the ES aggregations with the random sampling probability
+   * And {unwrap} accesses the original ES aggregations directly
+   */
+  public createRandomSamplerWrapper() {
+    const mode = this.getMode();
+    const probability = this.getProbability();
+
+    let prob = {};
+    if (mode === RANDOM_SAMPLER_OPTION.ON_MANUAL) {
+      prob = { probability };
+    } else if (mode === RANDOM_SAMPLER_OPTION.OFF) {
+      prob = { probability: 1 };
+    }
+
+    const wrapper = createRandomSamplerWrapper({
+      ...prob,
+      totalNumDocs: this.getDocCount(),
+    });
+    this.setProbability(wrapper.probability);
+    return wrapper;
+  }
+}

x-pack/plugins/aiops/public/application/utils/search_utils.ts

@@ -26,27 +26,11 @@ import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWith
 import type { DataView } from '@kbn/data-views-plugin/public';
 import type { SimpleSavedObject } from '@kbn/core/public';
 import { isPopulatedObject } from '@kbn/ml-is-populated-object';
-
-const DEFAULT_QUERY = {
-  bool: {
-    must: [
-      {
-        match_all: {},
-      },
-    ],
-  },
-};
-
-export const SEARCH_QUERY_LANGUAGE = {
-  KUERY: 'kuery',
-  LUCENE: 'lucene',
-} as const;
-
-export type SearchQueryLanguage = typeof SEARCH_QUERY_LANGUAGE[keyof typeof SEARCH_QUERY_LANGUAGE];
-
-export function getDefaultQuery() {
-  return cloneDeep(DEFAULT_QUERY);
-}
+import {
+  getDefaultDSLQuery,
+  type SearchQueryLanguage,
+  SEARCH_QUERY_LANGUAGE,
+} from '@kbn/ml-query-utils';
 
 export type SavedSearchSavedObject = SimpleSavedObject<any>;
 
@@ -94,7 +78,7 @@ export function createMergedEsQuery(
   dataView?: DataView,
   uiSettings?: IUiSettingsClient
 ) {
-  let combinedQuery: QueryDslQueryContainer = getDefaultQuery();
+  let combinedQuery: QueryDslQueryContainer = getDefaultDSLQuery();
 
   // FIXME: Add support for AggregateQuery type #150091
   if (isQuery(query) && query.language === SEARCH_QUERY_LANGUAGE.KUERY) {
@@ -171,7 +155,7 @@ export function getEsQueryFromSavedSearch({
     // Flattened query from search source may contain a clause that narrows the time range
     // which might interfere with global time pickers so we need to remove
     const savedQuery =
-      cloneDeep(savedSearch.searchSource.getSearchRequestBody()?.query) ?? getDefaultQuery();
+      cloneDeep(savedSearch.searchSource.getSearchRequestBody()?.query) ?? getDefaultDSLQuery();
     const timeField = savedSearch.searchSource.getField('index')?.timeFieldName;
 
     if (Array.isArray(savedQuery.bool.filter) && timeField !== undefined) {

x-pack/plugins/aiops/public/application/utils/url_state.ts

@@ -9,7 +9,7 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 
 import type { Filter, Query } from '@kbn/es-query';
 
-import { SEARCH_QUERY_LANGUAGE, SearchQueryLanguage } from './search_utils';
+import { SEARCH_QUERY_LANGUAGE, type SearchQueryLanguage } from '@kbn/ml-query-utils';
 
 const defaultSearchQuery = {
   match_all: {},

x-pack/plugins/aiops/public/components/change_point_detection/search_bar.tsx

@@ -10,7 +10,7 @@ import { i18n } from '@kbn/i18n';
 import { type Filter, fromKueryExpression, type Query } from '@kbn/es-query';
 import { type SearchBarOwnProps } from '@kbn/unified-search-plugin/public/search_bar';
 import { EuiSpacer, EuiTextColor } from '@elastic/eui';
-import { SEARCH_QUERY_LANGUAGE } from '../../application/utils/search_utils';
+import { SEARCH_QUERY_LANGUAGE } from '@kbn/ml-query-utils';
 import { useDataSource } from '../../hooks/use_data_source';
 import { useAiopsAppContext } from '../../hooks/use_aiops_app_context';
 

x-pack/plugins/aiops/public/components/document_count_content/document_count_content/document_count_content.tsx

@@ -11,9 +11,14 @@ import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui';
 
 import type { WindowParameters } from '@kbn/aiops-utils';
 
+import {
+  BarStyleAccessor,
+  RectAnnotationSpec,
+} from '@elastic/charts/dist/chart_types/xy_chart/utils/specs';
+import { DocumentCountChart, type DocumentCountChartPoint } from '@kbn/aiops-components';
+import { useAiopsAppContext } from '../../../hooks/use_aiops_app_context';
 import { DocumentCountStats } from '../../../get_document_stats';
 
-import { DocumentCountChart, DocumentCountChartPoint } from '../document_count_chart';
 import { TotalCountHeader } from '../total_count_header';
 
 export interface DocumentCountContentProps {
@@ -29,6 +34,13 @@ export interface DocumentCountContentProps {
   barColorOverride?: string;
   /** Optional color override for the highlighted bar color for charts */
   barHighlightColorOverride?: string;
+  windowParameters?: WindowParameters;
+  incomingInitialAnalysisStart?: number | WindowParameters;
+  baselineLabel?: string;
+  deviationLabel?: string;
+  barStyleAccessor?: BarStyleAccessor;
+  baselineAnnotationStyle?: RectAnnotationSpec['style'];
+  deviationAnnotationStyle?: RectAnnotationSpec['style'];
 }
 
 export const DocumentCountContent: FC<DocumentCountContentProps> = ({
@@ -42,7 +54,12 @@ export const DocumentCountContent: FC<DocumentCountContentProps> = ({
   initialAnalysisStart,
   barColorOverride,
   barHighlightColorOverride,
+  windowParameters,
+  incomingInitialAnalysisStart,
+  ...docCountChartProps
 }) => {
+  const { data, uiSettings, fieldFormats, charts } = useAiopsAppContext();
+
   const bucketTimestamps = Object.keys(documentCountStats?.buckets ?? {}).map((time) => +time);
   const splitBucketTimestamps = Object.keys(documentCountStatsSplit?.buckets ?? {}).map(
     (time) => +time
@@ -84,6 +101,7 @@ export const DocumentCountContent: FC<DocumentCountContentProps> = ({
       {documentCountStats.interval !== undefined && (
         <EuiFlexItem>
           <DocumentCountChart
+            dependencies={{ data, uiSettings, fieldFormats, charts }}
             brushSelectionUpdateHandler={brushSelectionUpdateHandler}
             chartPoints={chartPoints}
             chartPointsSplit={chartPointsSplit}
@@ -95,6 +113,7 @@ export const DocumentCountContent: FC<DocumentCountContentProps> = ({
             autoAnalysisStart={initialAnalysisStart}
             barColorOverride={barColorOverride}
             barHighlightColorOverride={barHighlightColorOverride}
+            {...docCountChartProps}
           />
         </EuiFlexItem>
       )}

x-pack/plugins/aiops/public/components/log_categorization/category_table/category_table.tsx

@@ -21,12 +21,12 @@ import {
 
 import { DataViewField } from '@kbn/data-views-plugin/common';
 import { Filter } from '@kbn/es-query';
+import { useTableState } from '@kbn/ml-in-memory-table';
 import { useDiscoverLinks, createFilter, QueryMode, QUERY_MODE } from '../use_discover_links';
 import { MiniHistogram } from '../../mini_histogram';
 import { useEuiTheme } from '../../../hooks/use_eui_theme';
 import type { LogCategorizationAppState } from '../../../application/utils/url_state';
 import type { EventRate, Category, SparkLinesPerCategory } from '../use_categorize_request';
-import { useTableState } from './use_table_state';
 import { getLabels } from './labels';
 import { TableHeader } from './table_header';
 

x-pack/plugins/aiops/public/components/log_categorization/document_count_chart.tsx

@@ -8,7 +8,8 @@
 import React, { FC, useMemo } from 'react';
 
 import { i18n } from '@kbn/i18n';
-import { DocumentCountChart as DocumentCountChartRoot } from '../document_count_content/document_count_chart';
+import { DocumentCountChart as DocumentCountChartRoot } from '@kbn/aiops-components';
+import { useAiopsAppContext } from '../../hooks/use_aiops_app_context';
 import { TotalCountHeader } from '../document_count_content/total_count_header';
 import type { Category, SparkLinesPerCategory } from './use_categorize_request';
 import type { EventRate } from './use_categorize_request';
@@ -31,6 +32,8 @@ export const DocumentCountChart: FC<Props> = ({
   selectedCategory,
   documentCountStats,
 }) => {
+  const { data, uiSettings, fieldFormats, charts } = useAiopsAppContext();
+
   const chartPointsSplitLabel = i18n.translate(
     'xpack.aiops.logCategorization.chartPointsSplitLabel',
     {
@@ -69,6 +72,7 @@ export const DocumentCountChart: FC<Props> = ({
     <>
       <TotalCountHeader totalCount={totalCount} />
       <DocumentCountChartRoot
+        dependencies={{ data, uiSettings, fieldFormats, charts }}
         chartPoints={chartPoints}
         chartPointsSplit={chartPointsSplit}
         timeRangeEarliest={eventRate[0].key}

x-pack/plugins/aiops/public/components/log_categorization/log_categorization_page.tsx

@@ -26,10 +26,10 @@ import { FormattedMessage } from '@kbn/i18n-react';
 import { usePageUrlState, useUrlState } from '@kbn/ml-url-state';
 
 import type { FieldValidationResults } from '@kbn/ml-category-validator';
+import type { SearchQueryLanguage } from '@kbn/ml-query-utils';
 import { useDataSource } from '../../hooks/use_data_source';
 import { useData } from '../../hooks/use_data';
 import { useSearch } from '../../hooks/use_search';
-import type { SearchQueryLanguage } from '../../application/utils/search_utils';
 import { useAiopsAppContext } from '../../hooks/use_aiops_app_context';
 import {
   getDefaultLogCategorizationAppState,

x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_page.tsx

@@ -13,9 +13,9 @@ import { EuiFlexGroup, EuiFlexItem, EuiPageBody, EuiPageSection, EuiSpacer } fro
 import { Filter, FilterStateStore, Query } from '@kbn/es-query';
 import { useUrlState, usePageUrlState } from '@kbn/ml-url-state';
 
+import type { SearchQueryLanguage } from '@kbn/ml-query-utils';
 import { useDataSource } from '../../hooks/use_data_source';
 import { useAiopsAppContext } from '../../hooks/use_aiops_app_context';
-import { SearchQueryLanguage } from '../../application/utils/search_utils';
 import { useData } from '../../hooks/use_data';
 import { useSearch } from '../../hooks/use_search';
 import {

x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/use_view_in_discover_action.tsx

@@ -10,7 +10,7 @@ import React, { useMemo } from 'react';
 import { i18n } from '@kbn/i18n';
 import type { SignificantTerm } from '@kbn/ml-agg-utils';
 
-import { SEARCH_QUERY_LANGUAGE } from '../../application/utils/search_utils';
+import { SEARCH_QUERY_LANGUAGE } from '@kbn/ml-query-utils';
 import { useAiopsAppContext } from '../../hooks/use_aiops_app_context';
 
 import { TableActionButton } from './table_action_button';

x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/use_view_in_log_pattern_analysis_action.tsx

@@ -12,7 +12,7 @@ import { fromKueryExpression, toElasticsearchQuery } from '@kbn/es-query';
 import { i18n } from '@kbn/i18n';
 import type { SignificantTerm } from '@kbn/ml-agg-utils';
 
-import { SEARCH_QUERY_LANGUAGE } from '../../application/utils/search_utils';
+import { SEARCH_QUERY_LANGUAGE } from '@kbn/ml-query-utils';
 import { useAiopsAppContext } from '../../hooks/use_aiops_app_context';
 
 import { TableActionButton } from './table_action_button';

x-pack/plugins/aiops/public/components/search_panel/search_panel.tsx

@@ -11,7 +11,7 @@ import { i18n } from '@kbn/i18n';
 import { Query, Filter } from '@kbn/es-query';
 import type { TimeRange } from '@kbn/es-query';
 import { DataView, DataViewField } from '@kbn/data-views-plugin/public';
-import { SearchQueryLanguage } from '../../application/utils/search_utils';
+import type { SearchQueryLanguage } from '@kbn/ml-query-utils';
 import { useAiopsAppContext } from '../../hooks/use_aiops_app_context';
 import { createMergedEsQuery } from '../../application/utils/search_utils';
 interface Props {

x-pack/plugins/aiops/tsconfig.json

@@ -37,6 +37,7 @@
     "@kbn/ml-category-validator",
     "@kbn/ml-date-picker",
     "@kbn/ml-error-utils",
+    "@kbn/ml-in-memory-table",
     "@kbn/ml-is-defined",
     "@kbn/ml-is-populated-object",
     "@kbn/ml-kibana-theme",

x-pack/plugins/data_visualizer/common/i18n_constants.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { i18n } from '@kbn/i18n';
+
+export const EXPAND_ROW = i18n.translate('xpack.dataVisualizer.table.expandRowScreenMsg', {
+  defaultMessage: 'Expand row',
+});
+
+export const COLLAPSE_ROW = i18n.translate('xpack.dataVisualizer.table.collapseAriaLabel', {
+  defaultMessage: 'Collapse row',
+});

x-pack/plugins/data_visualizer/public/api/index.ts

@@ -6,7 +6,11 @@
  */
 
 import { lazyLoadModules } from '../lazy_load_bundle';
-import type { FileDataVisualizerSpec, IndexDataVisualizerSpec } from '../application';
+import type {
+  DataComparisonSpec,
+  FileDataVisualizerSpec,
+  IndexDataVisualizerSpec,
+} from '../application';
 
 export async function getFileDataVisualizerComponent(): Promise<() => FileDataVisualizerSpec> {
   const modules = await lazyLoadModules();
@@ -17,3 +21,8 @@ export async function getIndexDataVisualizerComponent(): Promise<() => IndexData
   const modules = await lazyLoadModules();
   return () => modules.IndexDataVisualizer;
 }
+
+export async function getDataComparisonComponent(): Promise<() => DataComparisonSpec> {
+  const modules = await lazyLoadModules();
+  return () => modules.DataComparison;
+}

x-pack/plugins/data_visualizer/public/application/common/components/document_count_content/document_count_content.tsx

@@ -20,7 +20,6 @@ import {
 } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
 import { FormattedMessage } from '@kbn/i18n-react';
-import { isDefined } from '@kbn/ml-is-defined';
 import type { DocumentCountChartPoint } from './document_count_chart';
 import {
   RandomSamplerOption,
@@ -30,7 +29,8 @@ import {
 import { TotalCountHeader } from './total_count_header';
 import type { DocumentCountStats } from '../../../../../common/types/field_stats';
 import { DocumentCountChart } from './document_count_chart';
-import { RandomSamplerRangeSlider } from './random_sampler_range_slider';
+import { RandomSamplerRangeSlider } from '../random_sampling_menu/random_sampler_range_slider';
+import { ProbabilityUsedMessage } from '../random_sampling_menu/probability_used';
 
 export interface Props {
   documentCountStats?: DocumentCountStats;
@@ -42,20 +42,6 @@ export interface Props {
   loading: boolean;
 }
 
-const ProbabilityUsedMessage = ({ samplingProbability }: Pick<Props, 'samplingProbability'>) => {
-  return isDefined(samplingProbability) ? (
-    <div data-test-subj="dvRandomSamplerProbabilityUsedMsg">
-      <EuiSpacer size="m" />
-
-      <FormattedMessage
-        id="xpack.dataVisualizer.randomSamplerSettingsPopUp.probabilityLabel"
-        defaultMessage="Probability used: {samplingProbability}%"
-        values={{ samplingProbability: samplingProbability * 100 }}
-      />
-    </div>
-  ) : null;
-};
-
 const CalculatingProbabilityMessage = (
   <div data-test-subj="dvRandomSamplerCalculatingProbabilityMsg">
     <EuiSpacer size="m" />

x-pack/plugins/data_visualizer/public/application/common/components/random_sampling_menu/probability_used.tsx

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { isDefined } from '@kbn/ml-is-defined';
+import { EuiSpacer } from '@elastic/eui';
+import { FormattedMessage } from '@kbn/i18n-react';
+import React from 'react';
+import { Props } from '../document_count_content/document_count_content';
+
+export const ProbabilityUsedMessage = ({
+  samplingProbability,
+}: Pick<Props, 'samplingProbability'>) => {
+  return isDefined(samplingProbability) ? (
+    <div data-test-subj="dvRandomSamplerProbabilityUsedMsg">
+      <EuiSpacer size="m" />
+
+      <FormattedMessage
+        id="xpack.dataVisualizer.randomSamplerSettingsPopUp.probabilityLabel"
+        defaultMessage="Probability used: {samplingProbability}%"
+        values={{ samplingProbability: samplingProbability * 100 }}
+      />
+    </div>
+  ) : null;
+};

x-pack/plugins/data_visualizer/public/application/common/components/random_sampling_menu/random_sampler_range_slider.tsx

@@ -5,12 +5,12 @@
  * 2.0.
  */
 
+import React, { useState } from 'react';
+import { isDefined } from '@kbn/ml-is-defined';
 import { EuiButton, EuiFlexItem, EuiFormRow, EuiRange, EuiSpacer } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
-import { isDefined } from '@kbn/ml-is-defined';
-import { FormattedMessage } from '@kbn/i18n-react';
-import React, { useState } from 'react';
 import { roundToDecimalPlace } from '@kbn/ml-number-utils';
+import { FormattedMessage } from '@kbn/i18n-react';
 import {
   MIN_SAMPLER_PROBABILITY,
   RANDOM_SAMPLER_PROBABILITIES,

x-pack/plugins/data_visualizer/public/application/common/components/random_sampling_menu/random_sampling_menu.tsx

@@ -0,0 +1,184 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React, { FC, useCallback, useMemo, useState } from 'react';
+import useObservable from 'react-use/lib/useObservable';
+import { i18n } from '@kbn/i18n';
+import {
+  EuiButtonEmpty,
+  EuiCallOut,
+  EuiFlexItem,
+  EuiFormRow,
+  EuiPanel,
+  EuiPopover,
+  EuiSelect,
+  EuiSpacer,
+} from '@elastic/eui';
+import { RandomSampler } from '@kbn/ml-random-sampler-utils';
+import { RandomSamplerRangeSlider } from './random_sampler_range_slider';
+import {
+  MIN_SAMPLER_PROBABILITY,
+  RANDOM_SAMPLER_OPTION,
+  RANDOM_SAMPLER_SELECT_OPTIONS,
+  RandomSamplerOption,
+} from '../../../index_data_visualizer/constants/random_sampler';
+import { ProbabilityUsedMessage } from './probability_used';
+
+interface Props {
+  randomSampler: RandomSampler;
+  reload: () => void;
+}
+
+export const SamplingMenu: FC<Props> = ({ randomSampler, reload }) => {
+  const [showSamplingOptionsPopover, setShowSamplingOptionsPopover] = useState(false);
+
+  const samplingProbability = useObservable(
+    randomSampler.getProbability$(),
+    randomSampler.getProbability()
+  );
+
+  const setSamplingProbability = useCallback(
+    (probability: number | null) => {
+      randomSampler.setProbability(probability);
+      reload();
+    },
+    [reload, randomSampler]
+  );
+
+  const randomSamplerPreference = useObservable(randomSampler.getMode$(), randomSampler.getMode());
+
+  const setRandomSamplerPreference = useCallback(
+    (nextPref: RandomSamplerOption) => {
+      if (nextPref === RANDOM_SAMPLER_OPTION.ON_MANUAL) {
+        // By default, when switching to manual, restore previously chosen probability
+        // else, default to 0.001%
+        const savedRandomSamplerProbability = randomSampler.getProbability();
+        randomSampler.setProbability(
+          savedRandomSamplerProbability &&
+            savedRandomSamplerProbability > 0 &&
+            savedRandomSamplerProbability <= 0.5
+            ? savedRandomSamplerProbability
+            : MIN_SAMPLER_PROBABILITY
+        );
+      }
+      randomSampler.setMode(nextPref);
+      reload();
+    },
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+    [setSamplingProbability, randomSampler]
+  );
+
+  const { calloutInfoMessage, buttonText } = useMemo(() => {
+    switch (randomSamplerPreference) {
+      case RANDOM_SAMPLER_OPTION.OFF:
+        return {
+          calloutInfoMessage: i18n.translate(
+            'xpack.dataVisualizer.randomSamplerSettingsPopUp.offCallout.message',
+            {
+              defaultMessage:
+                'Random sampling can be turned on to increase the speed of analysis, although some accuracy will be lost.',
+            }
+          ),
+          buttonText: i18n.translate(
+            'xpack.dataVisualizer.randomSamplerSettingsPopUp.offCallout.button',
+            {
+              defaultMessage: 'No sampling',
+            }
+          ),
+        };
+      case RANDOM_SAMPLER_OPTION.ON_AUTOMATIC:
+        return {
+          calloutInfoMessage: i18n.translate(
+            'xpack.dataVisualizer.randomSamplerSettingsPopUp.onAutomaticCallout.message',
+            {
+              defaultMessage:
+                'The view will use random sampler aggregations. The probability is automatically set to balance accuracy and speed.',
+            }
+          ),
+          buttonText: i18n.translate(
+            'xpack.dataVisualizer.randomSamplerSettingsPopUp.onAutomaticCallout.button',
+            {
+              defaultMessage: 'Auto sampling',
+            }
+          ),
+        };
+
+      case RANDOM_SAMPLER_OPTION.ON_MANUAL:
+      default:
+        return {
+          calloutInfoMessage: i18n.translate(
+            'xpack.dataVisualizer.randomSamplerSettingsPopUp.onManualCallout.message',
+            {
+              defaultMessage:
+                'The view will use random sampler aggregations. A lower percentage probability increases performance, but some accuracy is lost.',
+            }
+          ),
+          buttonText: i18n.translate(
+            'xpack.dataVisualizer.randomSamplerSettingsPopUp.onManualCallout.button',
+            {
+              defaultMessage: 'Manual sampling',
+            }
+          ),
+        };
+    }
+  }, [randomSamplerPreference]);
+
+  return (
+    <EuiPopover
+      data-test-subj="aiopsRandomSamplerOptionsPopover"
+      id="aiopsSamplingOptions"
+      button={
+        <EuiButtonEmpty
+          onClick={() => setShowSamplingOptionsPopover(!showSamplingOptionsPopover)}
+          iconSide="right"
+          iconType="arrowDown"
+        >
+          {buttonText}
+        </EuiButtonEmpty>
+      }
+      isOpen={showSamplingOptionsPopover}
+      closePopover={() => setShowSamplingOptionsPopover(false)}
+      panelPaddingSize="none"
+      anchorPosition="downLeft"
+    >
+      <EuiPanel style={{ maxWidth: 400 }}>
+        <EuiFlexItem grow={true}>
+          <EuiCallOut size="s" color={'primary'} title={calloutInfoMessage} />
+        </EuiFlexItem>
+        <EuiSpacer size="m" />
+
+        <EuiFormRow
+          data-test-subj="aiopsRandomSamplerOptionsFormRow"
+          label={i18n.translate(
+            'xpack.dataVisualizer.randomSamplerSettingsPopUp.randomSamplerRowLabel',
+            {
+              defaultMessage: 'Random sampling',
+            }
+          )}
+        >
+          <EuiSelect
+            data-test-subj="aiopsRandomSamplerOptionsSelect"
+            options={RANDOM_SAMPLER_SELECT_OPTIONS}
+            value={randomSamplerPreference}
+            onChange={(e) => setRandomSamplerPreference(e.target.value as RandomSamplerOption)}
+          />
+        </EuiFormRow>
+
+        {randomSamplerPreference === RANDOM_SAMPLER_OPTION.ON_MANUAL ? (
+          <RandomSamplerRangeSlider
+            samplingProbability={samplingProbability}
+            setSamplingProbability={setSamplingProbability}
+          />
+        ) : null}
+
+        {randomSamplerPreference === RANDOM_SAMPLER_OPTION.ON_AUTOMATIC ? (
+          <ProbabilityUsedMessage samplingProbability={samplingProbability} />
+        ) : null}
+      </EuiPanel>
+    </EuiPopover>
+  );
+};

x-pack/plugins/data_visualizer/public/application/common/hooks/data_source_context.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { createContext, useContext } from 'react';
+import type { DataView } from '@kbn/data-views-plugin/common';
+import type { SavedSearch } from '@kbn/saved-search-plugin/public';
+
+export const DataSourceContext = createContext<{
+  dataView: DataView | never;
+  savedSearch: SavedSearch | null;
+}>({
+  get dataView(): never {
+    throw new Error('DataSourceContext is not implemented');
+  },
+  savedSearch: null,
+});
+
+export function useDataSource() {
+  return useContext(DataSourceContext);
+}

x-pack/plugins/data_visualizer/public/application/common/hooks/use_data.ts

@@ -0,0 +1,113 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { DataView } from '@kbn/data-views-plugin/common';
+import * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import { Dictionary } from '@kbn/ml-url-state';
+import { Moment } from 'moment';
+import { useExecutionContext } from '@kbn/kibana-react-plugin/public';
+import { useEffect, useMemo, useState } from 'react';
+import { mlTimefilterRefresh$, useTimefilter } from '@kbn/ml-date-picker';
+import { merge } from 'rxjs';
+import { RandomSampler } from '@kbn/ml-random-sampler-utils';
+import {
+  DocumentStatsSearchStrategyParams,
+  useDocumentCountStats,
+} from './use_document_count_stats';
+import { useDataVisualizerKibana } from '../../kibana_context';
+import { useTimeBuckets } from './use_time_buckets';
+
+const DEFAULT_BAR_TARGET = 75;
+
+export const useData = (
+  selectedDataView: DataView,
+  contextId: string,
+  searchQuery: estypes.QueryDslQueryContainer,
+  randomSampler: RandomSampler,
+  onUpdate?: (params: Dictionary<unknown>) => void,
+  barTarget: number = DEFAULT_BAR_TARGET,
+  timeRange?: { min: Moment; max: Moment }
+) => {
+  const {
+    services: { executionContext },
+  } = useDataVisualizerKibana();
+
+  useExecutionContext(executionContext, {
+    name: 'data_visualizer',
+    type: 'application',
+    id: contextId,
+  });
+
+  const [lastRefresh, setLastRefresh] = useState(0);
+
+  const _timeBuckets = useTimeBuckets();
+  const timefilter = useTimefilter({
+    timeRangeSelector: selectedDataView?.timeFieldName !== undefined,
+    autoRefreshSelector: true,
+  });
+
+  const docCountRequestParams: DocumentStatsSearchStrategyParams | undefined = useMemo(() => {
+    const timefilterActiveBounds = timeRange ?? timefilter.getActiveBounds();
+    if (timefilterActiveBounds !== undefined) {
+      _timeBuckets.setInterval('auto');
+      _timeBuckets.setBounds(timefilterActiveBounds);
+      _timeBuckets.setBarTarget(barTarget);
+      return {
+        earliest: timefilterActiveBounds.min?.valueOf(),
+        latest: timefilterActiveBounds.max?.valueOf(),
+        intervalMs: _timeBuckets.getInterval()?.asMilliseconds(),
+        index: selectedDataView.getIndexPattern(),
+        searchQuery,
+        timeFieldName: selectedDataView.timeFieldName,
+        runtimeFieldMap: selectedDataView.getRuntimeMappings(),
+      };
+    }
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [lastRefresh, JSON.stringify({ searchQuery, timeRange })]);
+
+  const documentStats = useDocumentCountStats(docCountRequestParams, lastRefresh, randomSampler);
+
+  useEffect(() => {
+    const timefilterUpdateSubscription = merge(
+      timefilter.getAutoRefreshFetch$(),
+      timefilter.getTimeUpdate$(),
+      mlTimefilterRefresh$
+    ).subscribe(() => {
+      if (onUpdate) {
+        onUpdate({
+          time: timefilter.getTime(),
+          refreshInterval: timefilter.getRefreshInterval(),
+        });
+        setLastRefresh(Date.now());
+      }
+    });
+
+    // This listens just for an initial update of the timefilter to be switched on.
+    const timefilterEnabledSubscription = timefilter.getEnabledUpdated$().subscribe(() => {
+      if (docCountRequestParams === undefined) {
+        setLastRefresh(Date.now());
+      }
+    });
+
+    return () => {
+      timefilterUpdateSubscription.unsubscribe();
+      timefilterEnabledSubscription.unsubscribe();
+    };
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, []);
+
+  return {
+    documentStats,
+    timefilter,
+    /** Start timestamp filter */
+    earliest: docCountRequestParams?.earliest,
+    /** End timestamp filter */
+    latest: docCountRequestParams?.latest,
+    intervalMs: docCountRequestParams?.intervalMs,
+    forceRefresh: () => setLastRefresh(Date.now()),
+  };
+};

x-pack/plugins/data_visualizer/public/application/common/hooks/use_document_count_stats.ts

@@ -0,0 +1,290 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { useCallback, useEffect, useRef, useState } from 'react';
+import { stringHash } from '@kbn/ml-string-hash';
+import { extractErrorProperties } from '@kbn/ml-error-utils';
+import { Query } from '@kbn/es-query';
+import * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import { SignificantTerm } from '@kbn/ml-agg-utils';
+import {
+  createRandomSamplerWrapper,
+  RandomSampler,
+  RandomSamplerWrapper,
+} from '@kbn/ml-random-sampler-utils';
+import { isPopulatedObject } from '@kbn/ml-is-populated-object';
+import { each, get } from 'lodash';
+import { lastValueFrom } from 'rxjs';
+import { buildBaseFilterCriteria } from '@kbn/ml-query-utils';
+import useObservable from 'react-use/lib/useObservable';
+import { useDataVisualizerKibana } from '../../kibana_context';
+import { displayError } from '../util/display_error';
+
+export const RANDOM_SAMPLER_SEED = 3867418;
+
+export interface DocumentStats {
+  sampleProbability: number;
+  totalCount: number;
+  documentCountStats?: DocumentCountStats;
+  documentCountStatsCompare?: DocumentCountStats;
+}
+
+export interface DocumentCountStats {
+  interval?: number;
+  buckets?: { [key: string]: number };
+  timeRangeEarliest?: number;
+  timeRangeLatest?: number;
+  totalCount: number;
+}
+
+export interface DocumentStatsSearchStrategyParams {
+  earliest?: number;
+  latest?: number;
+  intervalMs?: number;
+  index: string;
+  searchQuery: Query['query'];
+  timeFieldName?: string;
+  runtimeFieldMap?: estypes.MappingRuntimeFields;
+  fieldsToFetch?: string[];
+  selectedSignificantTerm?: SignificantTerm;
+  includeSelectedSignificantTerm?: boolean;
+  trackTotalHits?: boolean;
+}
+
+export const getDocumentCountStatsRequest = (
+  params: DocumentStatsSearchStrategyParams,
+  randomSamplerWrapper?: RandomSamplerWrapper,
+  skipAggs = false
+) => {
+  const {
+    index,
+    timeFieldName,
+    earliest: earliestMs,
+    latest: latestMs,
+    runtimeFieldMap,
+    searchQuery,
+    intervalMs,
+    fieldsToFetch,
+    trackTotalHits,
+  } = params;
+
+  const size = 0;
+  const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, searchQuery);
+
+  const rawAggs: Record<string, estypes.AggregationsAggregationContainer> = {
+    eventRate: {
+      date_histogram: {
+        field: timeFieldName,
+        fixed_interval: `${intervalMs}ms`,
+        min_doc_count: 0,
+        ...(earliestMs !== undefined && latestMs !== undefined
+          ? {
+              extended_bounds: {
+                min: earliestMs,
+                max: latestMs,
+              },
+            }
+          : {}),
+      },
+    },
+  };
+
+  const aggs = randomSamplerWrapper ? randomSamplerWrapper.wrap(rawAggs) : rawAggs;
+
+  const searchBody = {
+    query: {
+      bool: {
+        filter: filterCriteria,
+      },
+    },
+    ...(!fieldsToFetch &&
+    !skipAggs &&
+    timeFieldName !== undefined &&
+    intervalMs !== undefined &&
+    intervalMs > 0
+      ? { aggs }
+      : {}),
+    ...(isPopulatedObject(runtimeFieldMap) ? { runtime_mappings: runtimeFieldMap } : {}),
+    track_total_hits: trackTotalHits === true,
+    size,
+  };
+  return {
+    index,
+    body: searchBody,
+  };
+};
+
+export const processDocumentCountStats = (
+  body: estypes.SearchResponse | undefined,
+  params: DocumentStatsSearchStrategyParams,
+  randomSamplerWrapper?: RandomSamplerWrapper
+): DocumentCountStats | undefined => {
+  if (!body) return undefined;
+
+  const totalCount = (body.hits.total as estypes.SearchTotalHits).value ?? body.hits.total ?? 0;
+
+  if (
+    params.intervalMs === undefined ||
+    params.earliest === undefined ||
+    params.latest === undefined
+  ) {
+    return {
+      totalCount,
+    };
+  }
+  const buckets: { [key: string]: number } = {};
+  const dataByTimeBucket: Array<{ key: string; doc_count: number }> = get(
+    randomSamplerWrapper && body.aggregations !== undefined
+      ? randomSamplerWrapper.unwrap(body.aggregations)
+      : body.aggregations,
+    ['eventRate', 'buckets'],
+    []
+  );
+  each(dataByTimeBucket, (dataForTime) => {
+    const time = dataForTime.key;
+    buckets[time] = dataForTime.doc_count;
+  });
+
+  return {
+    interval: params.intervalMs,
+    buckets,
+    timeRangeEarliest: params.earliest,
+    timeRangeLatest: params.latest,
+    totalCount,
+  };
+};
+
+export interface DocumentStatsSearchStrategyParams {
+  earliest?: number;
+  latest?: number;
+  intervalMs?: number;
+  index: string;
+  searchQuery: Query['query'];
+  timeFieldName?: string;
+  runtimeFieldMap?: estypes.MappingRuntimeFields;
+  fieldsToFetch?: string[];
+  selectedSignificantTerm?: SignificantTerm;
+  includeSelectedSignificantTerm?: boolean;
+  trackTotalHits?: boolean;
+}
+
+export function useDocumentCountStats<TParams extends DocumentStatsSearchStrategyParams>(
+  searchParams: TParams | undefined,
+  lastRefresh: number,
+  randomSampler: RandomSampler
+): DocumentStats {
+  const {
+    data,
+    notifications: { toasts },
+  } = useDataVisualizerKibana().services;
+
+  const abortCtrl = useRef(new AbortController());
+
+  const [documentStats, setDocumentStats] = useState<DocumentStats>({
+    sampleProbability: 1,
+    totalCount: 0,
+  });
+
+  const [documentStatsCache, setDocumentStatsCache] = useState<Record<string, DocumentStats>>({});
+  const samplingProbability = useObservable(
+    randomSampler.getProbability$(),
+    randomSampler.getProbability()
+  );
+
+  const fetchDocumentCountData = useCallback(async () => {
+    if (!searchParams) return;
+
+    const cacheKey = stringHash(
+      `${JSON.stringify(searchParams)}-${randomSampler.getProbability()}`
+    );
+
+    if (documentStatsCache[cacheKey]) {
+      setDocumentStats(documentStatsCache[cacheKey]);
+      return;
+    }
+
+    try {
+      abortCtrl.current = new AbortController();
+
+      const totalHitsParams = {
+        ...searchParams,
+        selectedSignificantTerm: undefined,
+        trackTotalHits: true,
+      };
+
+      const totalHitsResp = await lastValueFrom(
+        data.search.search(
+          {
+            params: getDocumentCountStatsRequest(totalHitsParams, undefined, true),
+          },
+          { abortSignal: abortCtrl.current.signal }
+        )
+      );
+      const totalHitsStats = processDocumentCountStats(totalHitsResp?.rawResponse, searchParams);
+      const totalCount = totalHitsStats?.totalCount ?? 0;
+
+      if (randomSampler) {
+        randomSampler.setDocCount(totalCount);
+      }
+      const randomSamplerWrapper = randomSampler
+        ? randomSampler.createRandomSamplerWrapper()
+        : createRandomSamplerWrapper({
+            totalNumDocs: totalCount,
+            seed: RANDOM_SAMPLER_SEED,
+          });
+      const resp = await lastValueFrom(
+        data.search.search(
+          {
+            params: getDocumentCountStatsRequest(
+              { ...searchParams, trackTotalHits: false },
+              randomSamplerWrapper
+            ),
+          },
+          { abortSignal: abortCtrl.current.signal }
+        )
+      );
+
+      const documentCountStats = processDocumentCountStats(
+        resp?.rawResponse,
+        searchParams,
+        randomSamplerWrapper
+      );
+
+      const newStats: DocumentStats = {
+        sampleProbability: randomSamplerWrapper.probability,
+        documentCountStats,
+        totalCount,
+      };
+
+      setDocumentStatsCache({
+        ...documentStatsCache,
+        [cacheKey]: newStats,
+      });
+    } catch (error) {
+      // An `AbortError` gets triggered when a user cancels a request by navigating away, we need to ignore these errors.
+      if (error.name !== 'AbortError') {
+        displayError(toasts, searchParams!.index, extractErrorProperties(error));
+      }
+    }
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [data?.search, documentStatsCache, searchParams, toasts, samplingProbability]);
+
+  useEffect(
+    function getDocumentCountData() {
+      fetchDocumentCountData();
+      return () => abortCtrl.current.abort();
+    },
+    [fetchDocumentCountData, lastRefresh, samplingProbability]
+  );
+
+  // Clear the document count stats cache when the outer page (date picker/search bar) triggers a refresh.
+  useEffect(() => {
+    setDocumentStatsCache({});
+  }, [lastRefresh]);
+
+  return documentStats;
+}

x-pack/plugins/data_visualizer/public/application/common/hooks/use_search.ts

@@ -0,0 +1,69 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { DataView } from '@kbn/data-views-plugin/public';
+import type { SavedSearch } from '@kbn/saved-search-plugin/public';
+
+import { useEffect } from 'react';
+import { FilterStateStore } from '@kbn/es-query';
+import { getEsQueryFromSavedSearch } from '../../index_data_visualizer/utils/saved_search_utils';
+import { useDataVisualizerKibana } from '../../kibana_context';
+import type { BasicAppState } from '../../data_comparison/types';
+
+export const useSearch = (
+  { dataView, savedSearch }: { dataView: DataView; savedSearch: SavedSearch | null | undefined },
+  appState: BasicAppState,
+  readOnly: boolean = false
+) => {
+  const {
+    uiSettings,
+    data: {
+      query: { filterManager },
+    },
+  } = useDataVisualizerKibana().services;
+
+  useEffect(
+    function clearFiltersOnLeave() {
+      return () => {
+        // We want to clear all filters that have not been pinned globally
+        // when navigating to other pages
+        filterManager
+          .getFilters()
+          .filter((f) => f.$state?.store === FilterStateStore.APP_STATE)
+          .forEach((f) => filterManager.removeFilter(f));
+      };
+    },
+    [filterManager]
+  );
+
+  const searchData = getEsQueryFromSavedSearch({
+    dataView,
+    uiSettings,
+    savedSearch,
+    filterManager,
+  });
+
+  if (searchData === undefined || (appState && appState.searchString !== '')) {
+    if (appState?.filters && readOnly === false) {
+      const globalFilters = filterManager?.getGlobalFilters();
+
+      if (filterManager) filterManager.setFilters(appState.filters);
+      if (globalFilters) filterManager?.addFilters(globalFilters);
+    }
+    return {
+      searchQuery: appState?.searchQuery,
+      searchString: appState?.searchString,
+      searchQueryLanguage: appState?.searchQueryLanguage,
+    };
+  } else {
+    return {
+      searchQuery: searchData.searchQuery,
+      searchString: searchData.searchString,
+      searchQueryLanguage: searchData.queryLanguage,
+    };
+  }
+};

x-pack/plugins/data_visualizer/public/application/common/hooks/use_time_buckets.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { useMemo } from 'react';
+import { UI_SETTINGS } from '@kbn/data-service';
+import { TimeBuckets } from '../../../../common/services/time_buckets';
+import { useDataVisualizerKibana } from '../../kibana_context';
+
+export const useTimeBuckets = () => {
+  const { uiSettings } = useDataVisualizerKibana().services;
+
+  return useMemo(() => {
+    return new TimeBuckets({
+      [UI_SETTINGS.HISTOGRAM_MAX_BARS]: uiSettings.get(UI_SETTINGS.HISTOGRAM_MAX_BARS),
+      [UI_SETTINGS.HISTOGRAM_BAR_TARGET]: uiSettings.get(UI_SETTINGS.HISTOGRAM_BAR_TARGET),
+      dateFormat: uiSettings.get('dateFormat'),
+      'dateFormat:scaled': uiSettings.get('dateFormat:scaled'),
+    });
+  }, [uiSettings]);
+};

x-pack/plugins/data_visualizer/public/application/common/util/display_error.ts

@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ToastsStart } from '@kbn/core-notifications-browser';
+import { i18n } from '@kbn/i18n';
+
+export function displayError(toastNotifications: ToastsStart, index: string, err: any) {
+  if (err.statusCode === 500) {
+    toastNotifications.addError(err, {
+      title: i18n.translate('xpack.dataVisualizer.index.dataLoader.internalServerErrorMessage', {
+        defaultMessage:
+          'Error loading data in index {index}. {message}. ' +
+          'The request may have timed out. Try using a smaller sample size or narrowing the time range.',
+        values: {
+          index,
+          message: err.error ?? err.message,
+        },
+      }),
+    });
+  } else {
+    toastNotifications.addError(err, {
+      title: i18n.translate('xpack.dataVisualizer.index.errorLoadingDataMessage', {
+        defaultMessage: 'Error loading data in index {index}. {message}.',
+        values: {
+          index,
+          message: err.error ?? err.message,
+        },
+      }),
+    });
+  }
+}

x-pack/plugins/data_visualizer/public/application/common/util/get_error_messages_from_es_shard_failures.test.ts

@@ -0,0 +1,56 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { getErrorMessagesFromEsShardFailures } from './get_error_messages_from_es_shard_failures';
+
+describe('getErrorMessagesFromEsShardFailures', () => {
+  test('returns extracted reasons if _shard.failures exist', () => {
+    const reason =
+      '[parent] Data too large, data for [<reused_arrays>] would be [1059268784/1010.1mb], which is larger than the limit of [1020054732/972.7mb], real usage: [940288176/896.7mb], new bytes reserved: [118980608/113.4mb], usages [inflight_requests=60008/58.6kb, model_inference=0/0b, eql_sequence=0/0b, fielddata=245141854/233.7mb, request=162398296/154.8mb]';
+    const resp = {
+      took: 37,
+      timed_out: false,
+      _shards: {
+        total: 2,
+        successful: 1,
+        skipped: 1,
+        failed: 1,
+        failures: [
+          {
+            shard: 0,
+            index: 'apm-7.2.0-span-2019.10.31',
+            node: 'PEyAKEkKQFql88n4oXyYMw',
+            reason: {
+              type: 'circuit_breaking_exception',
+              reason,
+              bytes_wanted: 1059268784,
+              bytes_limit: 1020054732,
+              durability: 'PERMANENT',
+            },
+          },
+        ],
+      },
+      hits: {
+        total: 0,
+        max_score: 0,
+        hits: [],
+      },
+    };
+    expect(getErrorMessagesFromEsShardFailures(resp)).toEqual([reason]);
+  });
+
+  test('returns empty array if _shard.failures not defined', () => {
+    const resp = {
+      took: 37,
+      timed_out: false,
+    };
+    expect(getErrorMessagesFromEsShardFailures(resp)).toEqual([]);
+    expect(getErrorMessagesFromEsShardFailures(null)).toEqual([]);
+    expect(getErrorMessagesFromEsShardFailures(undefined)).toEqual([]);
+    expect(getErrorMessagesFromEsShardFailures('')).toEqual([]);
+  });
+});

x-pack/plugins/data_visualizer/public/application/common/util/get_error_messages_from_es_shard_failures.ts

@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { isPopulatedObject } from '@kbn/ml-is-populated-object';
+import { isDefined } from '@kbn/ml-is-defined';
+
+export const getErrorMessagesFromEsShardFailures = (arg?: unknown): string[] => {
+  if (isPopulatedObject<string, { failures: object[] }>(arg, ['_shards'])) {
+    return (arg._shards.failures ?? [])
+      .map((failure) =>
+        isPopulatedObject<string, { reason?: string }>(failure, ['reason']) && failure.reason.reason
+          ? failure.reason.reason
+          : undefined
+      )
+      .filter(isDefined);
+  }
+  return [];
+};

x-pack/plugins/data_visualizer/public/application/data_comparison/charts/data_comparison_distribution_chart.tsx

@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { Axis, BarSeries, Chart, Position, ScaleType, Settings, Tooltip } from '@elastic/charts';
+import React from 'react';
+import { NoChartsData } from './no_charts_data';
+import { ComparisonHistogram } from '../types';
+import { DataComparisonChartTooltipBody } from '../data_comparison_chart_tooltip_body';
+import { COMPARISON_LABEL, DATA_COMPARISON_TYPE } from '../constants';
+
+export const DataComparisonDistributionChart = ({
+  featureName,
+  fieldType,
+  data,
+  colors,
+}: {
+  featureName: string;
+  fieldType: string;
+  data: ComparisonHistogram[];
+  colors: { referenceColor: string; productionColor: string };
+}) => {
+  if (data.length === 0) return <NoChartsData />;
+  return (
+    <Chart>
+      <Tooltip body={DataComparisonChartTooltipBody} />
+      <Settings />
+      <Axis id="bottom" position={Position.Bottom} />
+      <Axis id="left2" position={Position.Left} tickFormat={(d: any) => Number(d).toFixed(2)} />
+      <BarSeries
+        id="data-drift-viz"
+        name={featureName}
+        xScaleType={
+          fieldType === DATA_COMPARISON_TYPE.NUMERIC ? ScaleType.Linear : ScaleType.Ordinal
+        }
+        yScaleType={ScaleType.Linear}
+        xAccessor="key"
+        yAccessors={['percentage']}
+        splitSeriesAccessors={['g']}
+        data={data}
+        color={(identifier) => {
+          const key = identifier.seriesKeys[0];
+          return key === COMPARISON_LABEL ? colors.productionColor : colors.referenceColor;
+        }}
+      />
+    </Chart>
+  );
+};

x-pack/plugins/data_visualizer/public/application/data_comparison/charts/no_charts_data.tsx

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FormattedMessage } from '@kbn/i18n-react';
+import { EuiText, type EuiTextProps } from '@elastic/eui';
+import React from 'react';
+
+export const NoChartsData = ({ textAlign }: { textAlign?: EuiTextProps['textAlign'] }) => {
+  return (
+    <EuiText
+      color="subdued"
+      textAlign={textAlign ?? 'center'}
+      size={'s'}
+      css={{
+        display: 'flex',
+        justifyContent: 'center',
+        height: '100%',
+        width: '100%',
+        alignItems: 'center',
+      }}
+    >
+      <FormattedMessage id="xpack.dataVisualizer.noData" defaultMessage="No data" />
+    </EuiText>
+  );
+};

x-pack/plugins/data_visualizer/public/application/data_comparison/charts/overlap_distribution_chart.tsx

@@ -0,0 +1,64 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { AreaSeries, Chart, CurveType, ScaleType, Settings, Tooltip } from '@elastic/charts';
+import { i18n } from '@kbn/i18n';
+import React from 'react';
+import { NoChartsData } from './no_charts_data';
+import type { ComparisonHistogram, DataComparisonField } from '../types';
+import { DataComparisonChartTooltipBody } from '../data_comparison_chart_tooltip_body';
+import { COMPARISON_LABEL, DATA_COMPARISON_TYPE, REFERENCE_LABEL } from '../constants';
+
+export const OverlapDistributionComparison = ({
+  data,
+  colors,
+  fieldType,
+  fieldName,
+}: {
+  data: ComparisonHistogram[];
+  colors: { referenceColor: string; productionColor: string };
+  fieldType?: DataComparisonField['type'];
+  fieldName?: DataComparisonField['field'];
+}) => {
+  if (data.length === 0) return <NoChartsData textAlign="left" />;
+
+  return (
+    <Chart>
+      <Tooltip body={DataComparisonChartTooltipBody} />
+
+      <Settings showLegend={false} />
+      <AreaSeries
+        id="dataVisualizer.overlapDistributionComparisonChart"
+        name={i18n.translate(
+          'xpack.dataVisualizer.dataComparison.distributionComparisonChartName',
+          {
+            defaultMessage:
+              'Distribution comparison of {referenceLabel} and {comparisonLabel} data for {fieldName}',
+            values: {
+              referenceLabel: REFERENCE_LABEL.toLowerCase(),
+              comparisonLabel: COMPARISON_LABEL.toLowerCase(),
+              fieldName,
+            },
+          }
+        )}
+        xScaleType={
+          fieldType === DATA_COMPARISON_TYPE.NUMERIC ? ScaleType.Linear : ScaleType.Ordinal
+        }
+        yScaleType={ScaleType.Linear}
+        xAccessor="key"
+        yAccessors={['percentage']}
+        splitSeriesAccessors={['g']}
+        data={data}
+        curve={CurveType.CURVE_STEP}
+        color={(identifier) => {
+          const key = identifier.seriesKeys[0];
+          return key === COMPARISON_LABEL ? colors.productionColor : colors.referenceColor;
+        }}
+      />
+    </Chart>
+  );
+};

x-pack/plugins/data_visualizer/public/application/data_comparison/charts/single_distribution_chart.tsx

@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { SeriesColorAccessor } from '@elastic/charts/dist/chart_types/xy_chart/utils/specs';
+import { BarSeries, Chart, ScaleType, Settings } from '@elastic/charts';
+import React from 'react';
+import { NoChartsData } from './no_charts_data';
+import { DATA_COMPARISON_TYPE } from '../constants';
+import { DataComparisonField, Histogram } from '../types';
+
+export const SingleDistributionChart = ({
+  data,
+  color,
+  fieldType,
+  name,
+}: {
+  data: Histogram[];
+  name: string;
+  color?: SeriesColorAccessor;
+  fieldType?: DataComparisonField['type'];
+}) => {
+  if (data.length === 0) return <NoChartsData textAlign="left" />;
+
+  return (
+    <Chart>
+      <Settings />
+      <BarSeries
+        id={`${name}-distr-viz`}
+        name={name}
+        xScaleType={
+          fieldType === DATA_COMPARISON_TYPE.NUMERIC ? ScaleType.Linear : ScaleType.Ordinal
+        }
+        yScaleType={ScaleType.Linear}
+        xAccessor="key"
+        yAccessors={['percentage']}
+        data={data}
+        color={color}
+      />
+    </Chart>
+  );
+};

x-pack/plugins/data_visualizer/public/application/data_comparison/data_comparison_app_state.tsx

@@ -0,0 +1,101 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React, { FC } from 'react';
+import { pick } from 'lodash';
+
+import type { SavedSearch } from '@kbn/saved-search-plugin/public';
+import type { DataView } from '@kbn/data-views-plugin/public';
+import { StorageContextProvider } from '@kbn/ml-local-storage';
+import { UrlStateProvider } from '@kbn/ml-url-state';
+import { Storage } from '@kbn/kibana-utils-plugin/public';
+import { DatePickerContextProvider } from '@kbn/ml-date-picker';
+import { UI_SETTINGS } from '@kbn/data-plugin/common';
+import {
+  KibanaContextProvider,
+  KibanaThemeProvider,
+  toMountPoint,
+  wrapWithTheme,
+} from '@kbn/kibana-react-plugin/public';
+
+import { DV_STORAGE_KEYS } from '../index_data_visualizer/types/storage';
+import { getCoreStart, getPluginsStart } from '../../kibana_services';
+import { DataComparisonPage } from './data_comparison_page';
+import { DataSourceContext } from '../common/hooks/data_source_context';
+
+const localStorage = new Storage(window.localStorage);
+
+export interface DataComparisonDetectionAppStateProps {
+  /** The data view to analyze. */
+  dataView: DataView;
+  /** The saved search to analyze. */
+  savedSearch: SavedSearch | null;
+}
+
+export type DataComparisonSpec = typeof DataComparisonDetectionAppState;
+
+export const DataComparisonDetectionAppState: FC<DataComparisonDetectionAppStateProps> = ({
+  dataView,
+  savedSearch,
+}) => {
+  if (!(dataView || savedSearch)) {
+    throw Error('No data view or saved search available.');
+  }
+
+  const coreStart = getCoreStart();
+  const {
+    data,
+    maps,
+    embeddable,
+    discover,
+    share,
+    security,
+    fileUpload,
+    lens,
+    dataViewFieldEditor,
+    uiActions,
+    charts,
+    unifiedSearch,
+  } = getPluginsStart();
+  const services = {
+    data,
+    maps,
+    embeddable,
+    discover,
+    share,
+    security,
+    fileUpload,
+    lens,
+    dataViewFieldEditor,
+    uiActions,
+    charts,
+    unifiedSearch,
+    ...coreStart,
+  };
+  const datePickerDeps = {
+    ...pick(services, ['data', 'http', 'notifications', 'theme', 'uiSettings']),
+    toMountPoint,
+    wrapWithTheme,
+    uiSettingsKeys: UI_SETTINGS,
+  };
+
+  return (
+    <KibanaThemeProvider theme$={coreStart.theme.theme$}>
+      <KibanaContextProvider services={{ ...services }}>
+        <UrlStateProvider>
+          <DataSourceContext.Provider value={{ dataView, savedSearch }}>
+            <StorageContextProvider storage={localStorage} storageKeys={DV_STORAGE_KEYS}>
+              <DatePickerContextProvider {...datePickerDeps}>
+                <DataComparisonPage />
+              </DatePickerContextProvider>
+            </StorageContextProvider>
+          </DataSourceContext.Provider>
+        </UrlStateProvider>
+      </KibanaContextProvider>
+    </KibanaThemeProvider>
+  );
+};

x-pack/plugins/data_visualizer/public/application/data_comparison/data_comparison_chart_tooltip_body.tsx

@@ -0,0 +1,60 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+  TooltipCellStyle,
+  TooltipSpec,
+  TooltipTable,
+  TooltipTableBody,
+  TooltipTableCell,
+  TooltipTableColorCell,
+  TooltipTableFooter,
+  TooltipTableHeader,
+  TooltipTableRow,
+} from '@elastic/charts';
+import React from 'react';
+
+const style: TooltipCellStyle = { textAlign: 'right' };
+export const DataComparisonChartTooltipBody: TooltipSpec['body'] = ({ items }) => {
+  return (
+    <TooltipTable gridTemplateColumns={`repeat(${4}, auto)`} maxHeight={120}>
+      <TooltipTableHeader>
+        <TooltipTableRow>
+          {<TooltipTableColorCell />}
+          <TooltipTableCell style={style} />
+          <TooltipTableCell>Count</TooltipTableCell>
+          <TooltipTableCell>Percent</TooltipTableCell>
+        </TooltipTableRow>
+      </TooltipTableHeader>
+      <TooltipTableBody>
+        {items.map(({ label, datum, seriesIdentifier: { key }, color }) => (
+          <TooltipTableRow key={`${key}-${datum.x}`}>
+            {<TooltipTableColorCell color={color} />}
+            <TooltipTableCell style={style}>{label}</TooltipTableCell>
+            <TooltipTableCell style={style}>{datum.doc_count}</TooltipTableCell>
+            <TooltipTableCell style={style}>{`${(datum.percentage * 100).toFixed(
+              1
+            )}`}</TooltipTableCell>
+          </TooltipTableRow>
+        ))}
+      </TooltipTableBody>
+      <TooltipTableFooter>
+        <TooltipTableRow>
+          {<TooltipTableColorCell />}
+          <TooltipTableCell style={style}>Diff</TooltipTableCell>
+
+          <TooltipTableCell style={style}>
+            {items[1].datum.doc_count - items[0].datum.doc_count}
+          </TooltipTableCell>
+          <TooltipTableCell style={style}>
+            {`${((items[1].datum.percentage - items[0].datum.percentage) * 100).toFixed(1)}%`}
+          </TooltipTableCell>
+        </TooltipTableRow>
+      </TooltipTableFooter>
+    </TooltipTable>
+  );
+};

x-pack/plugins/data_visualizer/public/application/data_comparison/data_comparison_overview_table.tsx

@@ -0,0 +1,291 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { UseTableState } from '@kbn/ml-in-memory-table';
+import React, { ReactNode, useMemo, useState } from 'react';
+import { i18n } from '@kbn/i18n';
+import {
+  EuiBasicTableColumn,
+  EuiButtonIcon,
+  EuiIcon,
+  EuiInMemoryTable,
+  EuiScreenReaderOnly,
+  EuiTableFieldDataColumnType,
+  EuiToolTip,
+} from '@elastic/eui';
+import { FieldTypeIcon } from '../common/components/field_type_icon';
+import { COLLAPSE_ROW, EXPAND_ROW } from '../../../common/i18n_constants';
+import { COMPARISON_LABEL, REFERENCE_LABEL } from './constants';
+import { useCurrentEuiTheme } from '../common/hooks/use_current_eui_theme';
+import { DataComparisonField, Feature, FETCH_STATUS } from './types';
+import { formatSignificanceLevel } from './data_comparison_utils';
+import { SingleDistributionChart } from './charts/single_distribution_chart';
+import { OverlapDistributionComparison } from './charts/overlap_distribution_chart';
+import { DataComparisonDistributionChart } from './charts/data_comparison_distribution_chart';
+
+const dataComparisonYesLabel = i18n.translate(
+  'xpack.dataVisualizer.dataComparison.fieldTypeYesLabel',
+  {
+    defaultMessage: 'Yes',
+  }
+);
+const dataComparisonNoLabel = i18n.translate(
+  'xpack.dataVisualizer.dataComparison.driftDetectedNoLabel',
+  {
+    defaultMessage: 'No',
+  }
+);
+
+export const DataComparisonOverviewTable = ({
+  data,
+  onTableChange,
+  pagination,
+  sorting,
+  status,
+}: {
+  data: Feature[];
+  status: FETCH_STATUS;
+} & UseTableState<Feature>) => {
+  const euiTheme = useCurrentEuiTheme();
+  const colors = {
+    referenceColor: euiTheme.euiColorVis2,
+    productionColor: euiTheme.euiColorVis1,
+  };
+  const [itemIdToExpandedRowMap, setItemIdToExpandedRowMap] = useState<Record<string, ReactNode>>(
+    {}
+  );
+
+  const referenceDistributionLabel = i18n.translate(
+    'xpack.dataVisualizer.dataComparison.dataComparisonDistributionLabel',
+    {
+      defaultMessage: '{label} distribution',
+      values: { label: REFERENCE_LABEL },
+    }
+  );
+  const comparisonDistributionLabel = i18n.translate(
+    'xpack.dataVisualizer.dataComparison.dataComparisonDistributionLabel',
+    {
+      defaultMessage: '{label} distribution',
+      values: { label: COMPARISON_LABEL },
+    }
+  );
+
+  const columns: Array<EuiBasicTableColumn<Feature>> = [
+    {
+      align: 'left',
+      width: '40px',
+      isExpander: true,
+      name: (
+        <EuiScreenReaderOnly>
+          <span>{EXPAND_ROW}</span>
+        </EuiScreenReaderOnly>
+      ),
+      render: (item: Feature) => {
+        const itemIdToExpandedRowMapValues = { ...itemIdToExpandedRowMap };
+
+        return (
+          <EuiButtonIcon
+            onClick={() => toggleDetails(item)}
+            aria-label={itemIdToExpandedRowMapValues[item.featureName] ? COLLAPSE_ROW : EXPAND_ROW}
+            iconType={itemIdToExpandedRowMapValues[item.featureName] ? 'arrowDown' : 'arrowRight'}
+          />
+        );
+      },
+    },
+
+    {
+      field: 'featureName',
+      name: i18n.translate('xpack.dataVisualizer.dataComparison.fieldNameLabel', {
+        defaultMessage: 'Name',
+      }),
+      'data-test-subj': 'mlDataComparisonOverviewTableFeatureName',
+      sortable: true,
+      textOnly: true,
+    },
+    {
+      field: 'secondaryType',
+      name: i18n.translate('xpack.dataVisualizer.dataComparison.fieldTypeLabel', {
+        defaultMessage: 'Type',
+      }),
+      'data-test-subj': 'mlDataComparisonOverviewTableFeatureType',
+      sortable: true,
+      textOnly: true,
+      render: (secondaryType: DataComparisonField['secondaryType']) => {
+        return <FieldTypeIcon type={secondaryType} tooltipEnabled={true} />;
+      },
+    },
+    {
+      field: 'driftDetected',
+      name: i18n.translate('xpack.dataVisualizer.dataComparison.driftDetectedLabel', {
+        defaultMessage: 'Drift detected',
+      }),
+      'data-test-subj': 'mlDataComparisonOverviewTableDriftDetected',
+      sortable: true,
+      textOnly: true,
+      render: (driftDetected: boolean) => {
+        return <span>{driftDetected ? dataComparisonYesLabel : dataComparisonNoLabel}</span>;
+      },
+    },
+    {
+      field: 'similarityTestPValue',
+      name: (
+        <EuiToolTip
+          content={i18n.translate('xpack.dataVisualizer.dataComparison.pValueTooltip', {
+            defaultMessage:
+              'Indicates how extreme the change is. Lower values indicate greater change.',
+          })}
+        >
+          <span>
+            {i18n.translate('xpack.dataVisualizer.dataComparison.pValueLabel', {
+              defaultMessage: 'Similarity p-value',
+            })}
+            <EuiIcon size="s" color="subdued" type="questionInCircle" className="eui-alignTop" />
+          </span>
+        </EuiToolTip>
+      ),
+      'data-test-subj': 'mlDataComparisonOverviewTableSimilarityTestPValue',
+      sortable: true,
+      textOnly: true,
+      render: (similarityTestPValue: number) => {
+        return <span>{formatSignificanceLevel(similarityTestPValue)}</span>;
+      },
+    },
+    {
+      field: 'referenceHistogram',
+      name: referenceDistributionLabel,
+      'data-test-subj': 'mlDataComparisonOverviewTableReferenceDistribution',
+      sortable: false,
+      render: (referenceHistogram: Feature['referenceHistogram'], item) => {
+        return (
+          <div css={{ width: 100, height: 40 }}>
+            <SingleDistributionChart
+              fieldType={item.fieldType}
+              data={referenceHistogram}
+              color={colors.referenceColor}
+              name={referenceDistributionLabel}
+            />
+          </div>
+        );
+      },
+    },
+    {
+      field: 'productionHistogram',
+      name: comparisonDistributionLabel,
+      'data-test-subj': 'mlDataComparisonOverviewTableDataComparisonDistributionChart',
+      sortable: false,
+      render: (productionDistribution: Feature['productionHistogram'], item) => {
+        return (
+          <div css={{ width: 100, height: 40 }}>
+            <SingleDistributionChart
+              fieldType={item.fieldType}
+              data={productionDistribution}
+              color={colors.productionColor}
+              name={comparisonDistributionLabel}
+            />
+          </div>
+        );
+      },
+    },
+    {
+      field: 'comparisonDistribution',
+      name: 'Comparison',
+      'data-test-subj': 'mlDataComparisonOverviewTableDataComparisonDistributionChart',
+      sortable: false,
+      render: (comparisonDistribution: Feature['comparisonDistribution'], item) => {
+        return (
+          <div css={{ width: 100, height: 40 }}>
+            <OverlapDistributionComparison
+              fieldName={item.featureName}
+              fieldType={item.fieldType}
+              data={comparisonDistribution}
+              colors={colors}
+            />
+          </div>
+        );
+      },
+    },
+  ];
+
+  const getRowProps = (item: Feature) => {
+    return {
+      'data-test-subj': `mlDataComparisonOverviewTableRow row-${item.featureName}`,
+      className: 'mlDataComparisonOverviewTableRow',
+      onClick: () => {},
+    };
+  };
+
+  const getCellProps = (item: Feature, column: EuiTableFieldDataColumnType<Feature>) => {
+    const { field } = column;
+    return {
+      className: 'mlDataComparisonOverviewTableCell',
+      'data-test-subj': `mlDataComparisonOverviewTableCell row-${item.featureName}-column-${String(
+        field
+      )}`,
+      textOnly: true,
+    };
+  };
+
+  const toggleDetails = (item: Feature) => {
+    const itemIdToExpandedRowMapValues = { ...itemIdToExpandedRowMap };
+
+    if (itemIdToExpandedRowMapValues[item.featureName]) {
+      delete itemIdToExpandedRowMapValues[item.featureName];
+    } else {
+      const { featureName, comparisonDistribution } = item;
+      itemIdToExpandedRowMapValues[item.featureName] = (
+        <div css={{ width: '100%', height: 200 }}>
+          <DataComparisonDistributionChart
+            featureName={featureName}
+            fieldType={item.fieldType}
+            data={comparisonDistribution}
+            colors={colors}
+          />
+        </div>
+      );
+    }
+    setItemIdToExpandedRowMap(itemIdToExpandedRowMapValues);
+  };
+
+  const tableMessage = useMemo(() => {
+    switch (status) {
+      case FETCH_STATUS.NOT_INITIATED:
+        return i18n.translate('xpack.dataVisualizer.dataComparison.dataComparisonRunAnalysisMsg', {
+          defaultMessage: 'Run analysis to compare reference and comparison data',
+        });
+      case FETCH_STATUS.LOADING:
+        return i18n.translate('xpack.dataVisualizer.dataComparison.dataComparisonLoadingMsg', {
+          defaultMessage: 'Analyzing',
+        });
+      default:
+        return undefined;
+    }
+  }, [status]);
+
+  return (
+    <EuiInMemoryTable<Feature>
+      tableCaption={i18n.translate(
+        'xpack.dataVisualizer.dataComparison.dataComparisonTableCaption',
+        {
+          defaultMessage: 'Data comparison overview',
+        }
+      )}
+      items={data}
+      rowHeader="featureName"
+      columns={columns}
+      rowProps={getRowProps}
+      cellProps={getCellProps}
+      itemId="featureName"
+      itemIdToExpandedRowMap={itemIdToExpandedRowMap}
+      isExpandable={true}
+      sorting={sorting}
+      onChange={onTableChange}
+      pagination={pagination}
+      loading={status === FETCH_STATUS.LOADING}
+      message={tableMessage}
+    />
+  );
+};

x-pack/plugins/data_visualizer/public/application/data_comparison/data_comparison_page.tsx

@@ -0,0 +1,395 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React, { useCallback, useEffect, useState, FC, useMemo } from 'react';
+
+import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import {
+  EuiFlexGroup,
+  EuiFlexItem,
+  EuiPageBody,
+  EuiPageSection,
+  EuiPanel,
+  EuiSpacer,
+  EuiPageHeader,
+  EuiCallOut,
+} from '@elastic/eui';
+
+import type { WindowParameters } from '@kbn/aiops-utils';
+import type { Filter, Query } from '@kbn/es-query';
+import { useUrlState, usePageUrlState } from '@kbn/ml-url-state';
+import type { DataSeriesDatum } from '@elastic/charts/dist/chart_types/xy_chart/utils/series';
+import { useStorage } from '@kbn/ml-local-storage';
+import {
+  DatePickerWrapper,
+  FROZEN_TIER_PREFERENCE,
+  FullTimeRangeSelector,
+  FullTimeRangeSelectorProps,
+  useTimefilter,
+} from '@kbn/ml-date-picker';
+import moment from 'moment';
+import { css } from '@emotion/react';
+import type { SearchQueryLanguage } from '@kbn/ml-query-utils';
+import { i18n } from '@kbn/i18n';
+import { RANDOM_SAMPLER_OPTION, RandomSampler } from '@kbn/ml-random-sampler-utils';
+import { MIN_SAMPLER_PROBABILITY } from '../index_data_visualizer/constants/random_sampler';
+import { useData } from '../common/hooks/use_data';
+import {
+  DV_FROZEN_TIER_PREFERENCE,
+  DV_RANDOM_SAMPLER_P_VALUE,
+  DV_RANDOM_SAMPLER_PREFERENCE,
+  DVKey,
+  DVStorageMapped,
+} from '../index_data_visualizer/types/storage';
+import { useCurrentEuiTheme } from '../common/hooks/use_current_eui_theme';
+import { DataComparisonFullAppState, getDefaultDataComparisonState } from './types';
+import { useDataSource } from '../common/hooks/data_source_context';
+import { useDataVisualizerKibana } from '../kibana_context';
+import { DataComparisonView } from './data_comparison_view';
+import { COMPARISON_LABEL, REFERENCE_LABEL } from './constants';
+import { SearchPanelContent } from '../index_data_visualizer/components/search_panel/search_bar';
+import { useSearch } from '../common/hooks/use_search';
+import { DocumentCountWithDualBrush } from './document_count_with_dual_brush';
+
+const dataViewTitleHeader = css({
+  minWidth: '300px',
+});
+
+export const PageHeader: FC = () => {
+  const [, setGlobalState] = useUrlState('_g');
+  const { dataView } = useDataSource();
+
+  const [frozenDataPreference, setFrozenDataPreference] = useStorage<
+    DVKey,
+    DVStorageMapped<typeof DV_FROZEN_TIER_PREFERENCE>
+  >(
+    DV_FROZEN_TIER_PREFERENCE,
+    // By default we will exclude frozen data tier
+    FROZEN_TIER_PREFERENCE.EXCLUDE
+  );
+
+  const timefilter = useTimefilter({
+    timeRangeSelector: dataView.timeFieldName !== undefined,
+    autoRefreshSelector: true,
+  });
+
+  const updateTimeState: FullTimeRangeSelectorProps['callback'] = useCallback(
+    (update) => {
+      setGlobalState({
+        time: {
+          from: moment(update.start.epoch).toISOString(),
+          to: moment(update.end.epoch).toISOString(),
+        },
+      });
+    },
+    [setGlobalState]
+  );
+
+  const hasValidTimeField = useMemo(
+    () => dataView.timeFieldName !== undefined && dataView.timeFieldName !== '',
+    [dataView.timeFieldName]
+  );
+
+  return (
+    <EuiPageHeader
+      pageTitle={<div css={dataViewTitleHeader}>{dataView.getName()}</div>}
+      rightSideItems={[
+        <EuiFlexGroup gutterSize="s" data-test-subj="dataComparisonTimeRangeSelectorSection">
+          {hasValidTimeField ? (
+            <EuiFlexItem grow={false}>
+              <FullTimeRangeSelector
+                frozenDataPreference={frozenDataPreference}
+                setFrozenDataPreference={setFrozenDataPreference}
+                dataView={dataView}
+                query={undefined}
+                disabled={false}
+                timefilter={timefilter}
+                callback={updateTimeState}
+              />
+            </EuiFlexItem>
+          ) : null}
+          <DatePickerWrapper
+            isAutoRefreshOnly={!hasValidTimeField}
+            showRefresh={!hasValidTimeField}
+            width="full"
+            flexGroup={false}
+          />
+        </EuiFlexGroup>,
+      ]}
+    />
+  );
+};
+
+export const DataComparisonPage: FC = () => {
+  const {
+    services: { data: dataService },
+  } = useDataVisualizerKibana();
+  const { dataView, savedSearch } = useDataSource();
+
+  const [dataComparisonListState, setAiopsListState] = usePageUrlState<{
+    pageKey: 'DV_DATA_COMP';
+    pageUrlState: DataComparisonFullAppState;
+  }>('DV_DATA_COMP', getDefaultDataComparisonState());
+
+  const [randomSamplerMode, setRandomSamplerMode] = useStorage<
+    DVKey,
+    DVStorageMapped<typeof DV_RANDOM_SAMPLER_PREFERENCE>
+  >(DV_RANDOM_SAMPLER_PREFERENCE, RANDOM_SAMPLER_OPTION.ON_AUTOMATIC);
+
+  const [randomSamplerProbability, setRandomSamplerProbability] = useStorage<
+    DVKey,
+    DVStorageMapped<typeof DV_RANDOM_SAMPLER_P_VALUE>
+  >(DV_RANDOM_SAMPLER_P_VALUE, MIN_SAMPLER_PROBABILITY);
+  const [lastRefresh, setLastRefresh] = useState(0);
+
+  const forceRefresh = useCallback(() => setLastRefresh(Date.now()), [setLastRefresh]);
+
+  const randomSampler = useMemo(
+    () =>
+      new RandomSampler(
+        randomSamplerMode,
+        setRandomSamplerMode,
+        randomSamplerProbability,
+        setRandomSamplerProbability
+      ),
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+    []
+  );
+
+  const [globalState, setGlobalState] = useUrlState('_g');
+
+  const [selectedSavedSearch, setSelectedSavedSearch] = useState(savedSearch);
+
+  useEffect(() => {
+    if (savedSearch) {
+      setSelectedSavedSearch(savedSearch);
+    }
+  }, [savedSearch]);
+
+  const setSearchParams = useCallback(
+    (searchParams: {
+      searchQuery: estypes.QueryDslQueryContainer;
+      searchString: Query['query'];
+      queryLanguage: SearchQueryLanguage;
+      filters: Filter[];
+    }) => {
+      // When the user loads a saved search and then clears or modifies the query
+      // we should remove the saved search and replace it with the index pattern id
+      if (selectedSavedSearch !== null) {
+        setSelectedSavedSearch(null);
+      }
+
+      setAiopsListState({
+        ...dataComparisonListState,
+        searchQuery: searchParams.searchQuery,
+        searchString: searchParams.searchString,
+        searchQueryLanguage: searchParams.queryLanguage,
+        filters: searchParams.filters,
+      });
+    },
+    [selectedSavedSearch, dataComparisonListState, setAiopsListState]
+  );
+
+  const { searchQueryLanguage, searchString, searchQuery } = useSearch(
+    { dataView, savedSearch },
+    dataComparisonListState
+  );
+
+  const { documentStats, timefilter } = useData(
+    dataView,
+    'data_drift',
+    searchQuery,
+    randomSampler,
+    setGlobalState,
+    undefined
+  );
+
+  const { sampleProbability, totalCount, documentCountStats, documentCountStatsCompare } =
+    documentStats;
+
+  useEffect(() => {
+    randomSampler.setDocCount(totalCount);
+  }, [totalCount, randomSampler]);
+
+  useEffect(() => {
+    if (globalState?.time !== undefined) {
+      timefilter.setTime({
+        from: globalState.time.from,
+        to: globalState.time.to,
+      });
+    }
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [JSON.stringify(globalState?.time), timefilter]);
+
+  useEffect(() => {
+    if (globalState?.refreshInterval !== undefined) {
+      timefilter.setRefreshInterval(globalState.refreshInterval);
+    }
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [JSON.stringify(globalState?.refreshInterval), timefilter]);
+
+  useEffect(() => {
+    // Update data query manager if input string is updated
+    dataService?.query.queryString.setQuery({
+      query: searchString ?? '',
+      language: searchQueryLanguage,
+    });
+  }, [dataService, searchQueryLanguage, searchString]);
+
+  const euiTheme = useCurrentEuiTheme();
+  const colors = {
+    referenceColor: euiTheme.euiColorVis2,
+    productionColor: euiTheme.euiColorVis1,
+  };
+
+  const [windowParameters, setWindowParameters] = useState<WindowParameters | undefined>();
+  const [initialAnalysisStart, setInitialAnalysisStart] = useState<
+    number | WindowParameters | undefined
+  >();
+  const [isBrushCleared, setIsBrushCleared] = useState(true);
+
+  function brushSelectionUpdate(d: WindowParameters, force: boolean) {
+    if (!isBrushCleared || force) {
+      setWindowParameters(d);
+    }
+    if (force) {
+      setIsBrushCleared(false);
+    }
+  }
+
+  function clearSelection() {
+    setWindowParameters(undefined);
+    setIsBrushCleared(true);
+    setInitialAnalysisStart(undefined);
+  }
+
+  const barStyleAccessor = useCallback(
+    (datum: DataSeriesDatum) => {
+      if (!windowParameters) return null;
+
+      const start = datum.x;
+      const end =
+        (typeof datum.x === 'string' ? parseInt(datum.x, 10) : datum.x) +
+        (documentCountStats?.interval ?? 0);
+
+      if (start >= windowParameters.baselineMin && end <= windowParameters.baselineMax) {
+        return colors.referenceColor;
+      }
+      if (start >= windowParameters.deviationMin && end <= windowParameters.deviationMax) {
+        return colors.productionColor;
+      }
+
+      return null;
+    },
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+    [JSON.stringify({ windowParameters, colors })]
+  );
+
+  return (
+    <EuiPageBody
+      data-test-subj="dataComparisonDataComparisonPage"
+      paddingSize="none"
+      panelled={false}
+    >
+      <PageHeader />
+      <EuiSpacer size="m" />
+      <EuiPageSection paddingSize="none">
+        <EuiFlexGroup gutterSize="m" direction="column">
+          <EuiFlexItem>
+            <SearchPanelContent
+              dataView={dataView}
+              searchString={searchString}
+              searchQuery={searchQuery}
+              searchQueryLanguage={searchQueryLanguage}
+              setSearchParams={setSearchParams}
+            />
+          </EuiFlexItem>
+          {documentCountStats !== undefined && (
+            <EuiFlexItem>
+              <EuiPanel paddingSize="m">
+                <DocumentCountWithDualBrush
+                  randomSampler={randomSampler}
+                  reload={forceRefresh}
+                  brushSelectionUpdateHandler={brushSelectionUpdate}
+                  documentCountStats={documentCountStats}
+                  documentCountStatsSplit={documentCountStatsCompare}
+                  isBrushCleared={isBrushCleared}
+                  totalCount={totalCount}
+                  approximate={sampleProbability < 1}
+                  sampleProbability={sampleProbability}
+                  initialAnalysisStart={initialAnalysisStart}
+                  barStyleAccessor={barStyleAccessor}
+                  baselineBrush={{
+                    label: REFERENCE_LABEL,
+                    annotationStyle: {
+                      strokeWidth: 0,
+                      stroke: colors.referenceColor,
+                      fill: colors.referenceColor,
+                      opacity: 0.5,
+                    },
+                    badgeWidth: 80,
+                  }}
+                  deviationBrush={{
+                    label: COMPARISON_LABEL,
+                    annotationStyle: {
+                      strokeWidth: 0,
+                      stroke: colors.productionColor,
+                      fill: colors.productionColor,
+                      opacity: 0.5,
+                    },
+                    badgeWidth: 90,
+                  }}
+                />
+              </EuiPanel>
+            </EuiFlexItem>
+          )}
+
+          <EuiFlexItem>
+            <EuiPanel paddingSize="m">
+              {!dataView?.isTimeBased() ? (
+                <EuiCallOut
+                  title={i18n.translate(
+                    'xpack.dataVisualizer.dataViewNotBasedOnTimeSeriesWarning.title',
+                    {
+                      defaultMessage:
+                        'The data view "{dataViewTitle}" is not based on a time series.',
+                      values: { dataViewTitle: dataView.getName() },
+                    }
+                  )}
+                  color="danger"
+                  iconType="warning"
+                >
+                  <p>
+                    {i18n.translate(
+                      'xpack.dataVisualizer.dataComparisonTimeSeriesWarning.description',
+                      {
+                        defaultMessage: 'Data comparison only runs over time-based indices.',
+                      }
+                    )}
+                  </p>
+                </EuiCallOut>
+              ) : (
+                <DataComparisonView
+                  isBrushCleared={isBrushCleared}
+                  onReset={clearSelection}
+                  windowParameters={windowParameters}
+                  dataView={dataView}
+                  searchString={searchString ?? ''}
+                  searchQuery={searchQuery}
+                  searchQueryLanguage={searchQueryLanguage}
+                  lastRefresh={lastRefresh}
+                  randomSampler={randomSampler}
+                  forceRefresh={forceRefresh}
+                />
+              )}
+            </EuiPanel>
+          </EuiFlexItem>
+        </EuiFlexGroup>
+      </EuiPageSection>
+    </EuiPageBody>
+  );
+};

x-pack/plugins/data_visualizer/public/application/data_comparison/data_comparison_utils.test.ts

@@ -0,0 +1,88 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { computeChi2PValue } from './data_comparison_utils';
+import { Histogram } from './types';
+
+describe('computeChi2PValue()', () => {
+  test('should return close to 1 if datasets are both empty or nearly identical', () => {
+    const referenceTerms: Histogram[] = [
+      {
+        key: 'ap-northwest-1',
+        doc_count: 40348,
+        percentage: 0.2712470588235294,
+      },
+      {
+        key: 'us-east-1',
+        doc_count: 15134,
+        percentage: 0.10174117647058824,
+      },
+      {
+        key: 'eu-central-1',
+        doc_count: 12614,
+        percentage: 0.0848,
+      },
+      {
+        key: 'sa-east-1',
+        doc_count: 80654,
+        percentage: 0.5422117647058824,
+      },
+    ];
+    const productionTerms: Histogram[] = [
+      {
+        key: 'ap-northwest-1',
+        doc_count: 40320,
+        percentage: 0.2609691846654714,
+      },
+      {
+        key: 'us-east-1',
+        doc_count: 15127,
+        percentage: 0.09790875139966732,
+      },
+      {
+        key: 'eu-central-1',
+        doc_count: 12614,
+        percentage: 0.08164348450819088,
+      },
+      {
+        key: 'sa-east-1',
+        doc_count: 86440,
+        percentage: 0.5594785794266703,
+      },
+    ];
+    expect(computeChi2PValue([], [])).toStrictEqual(1);
+    expect(computeChi2PValue(referenceTerms, productionTerms)).toStrictEqual(0.99);
+  });
+
+  test('should return close to 0 if datasets differ', () => {
+    const referenceTerms: Histogram[] = [
+      {
+        key: 'jackson',
+        doc_count: 1,
+        percentage: 1,
+      },
+      {
+        key: 'yahya',
+        doc_count: 0,
+        percentage: 0,
+      },
+    ];
+    const productionTerms: Histogram[] = [
+      {
+        key: 'jackson',
+        doc_count: 0,
+        percentage: 0,
+      },
+      {
+        key: 'yahya',
+        doc_count: 1,
+        percentage: 1,
+      },
+    ];
+    expect(computeChi2PValue(referenceTerms, productionTerms)).toStrictEqual(0);
+  });
+});

x-pack/plugins/data_visualizer/public/application/data_comparison/data_comparison_utils.ts

@@ -0,0 +1,82 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { CRITICAL_VALUES_TABLE, SIGNIFICANCE_LEVELS } from './constants';
+import { Histogram } from './types';
+
+const criticalTableLookup = (chi2Statistic: number, df: number) => {
+  if (df < 1) return 1;
+  if (!Number.isInteger(df)) throw Error('Degrees of freedom must be a valid integer');
+
+  // Get the row index
+  const rowIndex: number = df - 1;
+
+  // Get the column index
+  let minDiff: number = Math.abs(CRITICAL_VALUES_TABLE[rowIndex][0] - chi2Statistic);
+  let columnIndex: number = 0;
+  for (let j = 1; j < CRITICAL_VALUES_TABLE[rowIndex].length; j++) {
+    const diff: number = Math.abs(CRITICAL_VALUES_TABLE[rowIndex][j] - chi2Statistic);
+    if (diff < minDiff) {
+      minDiff = diff;
+      columnIndex = j;
+    }
+  }
+
+  const significanceLevel: number = SIGNIFICANCE_LEVELS[columnIndex];
+  return significanceLevel;
+};
+
+/**
+ * Compute the p-value for how similar the datasets are.
+ * Returned value ranges from 0 to 1, with 1 meaning the datasets are identical.
+ * @param normalizedBaselineTerms
+ * @param normalizedDriftedTerms
+ */
+export const computeChi2PValue = (
+  normalizedBaselineTerms: Histogram[],
+  normalizedDriftedTerms: Histogram[]
+) => {
+  // Get all unique keys from both arrays
+  const allKeys: string[] = Array.from(
+    new Set([
+      ...normalizedBaselineTerms.map((term) => term.key.toString()),
+      ...normalizedDriftedTerms.map((term) => term.key.toString()),
+    ])
+  ).slice(0, 100);
+
+  // Calculate the chi-squared statistic and degrees of freedom
+  let chiSquared: number = 0;
+  const degreesOfFreedom: number = allKeys.length - 1;
+
+  if (degreesOfFreedom === 0) return 1;
+
+  allKeys.forEach((key) => {
+    const baselineTerm = normalizedBaselineTerms.find((term) => term.key === key);
+    const driftedTerm = normalizedDriftedTerms.find((term) => term.key === key);
+
+    const observed: number = driftedTerm?.percentage ?? 0;
+    const expected: number = baselineTerm?.percentage ?? 0;
+    chiSquared += Math.pow(observed - expected, 2) / (expected > 0 ? expected : 1e-6); // Prevent divide by zero
+  });
+
+  return criticalTableLookup(chiSquared, degreesOfFreedom);
+};
+
+/**
+ * formatSignificanceLevel
+ * @param significanceLevel
+ */
+export const formatSignificanceLevel = (significanceLevel: number) => {
+  if (typeof significanceLevel !== 'number' || isNaN(significanceLevel)) return '';
+  if (significanceLevel < 1e-6) {
+    return '< 0.000001';
+  } else if (significanceLevel < 0.01) {
+    return significanceLevel.toExponential(0);
+  } else {
+    return significanceLevel.toFixed(2);
+  }
+};

x-pack/plugins/data_visualizer/public/application/data_comparison/data_comparison_view.tsx

@@ -0,0 +1,228 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import React, { useCallback, useMemo, useState } from 'react';
+import { EuiEmptyPrompt, EuiFlexItem, EuiFormRow, EuiSwitch, EuiSpacer } from '@elastic/eui';
+import type { DataView } from '@kbn/data-views-plugin/public';
+import type { WindowParameters } from '@kbn/aiops-utils';
+import { i18n } from '@kbn/i18n';
+import type { Query } from '@kbn/es-query';
+import { ProgressControls } from '@kbn/aiops-components';
+import { isEqual } from 'lodash';
+import { FormattedMessage } from '@kbn/i18n-react';
+import { EuiSwitchEvent } from '@elastic/eui/src/components/form/switch/switch';
+import { useTableState } from '@kbn/ml-in-memory-table';
+import type { SearchQueryLanguage } from '@kbn/ml-query-utils';
+import { RandomSampler } from '@kbn/ml-random-sampler-utils';
+import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types';
+import { kbnTypeToSupportedType } from '../common/util/field_types_utils';
+import { getDataComparisonType, useFetchDataComparisonResult } from './use_data_drift_result';
+import type { DataComparisonField, Feature, TimeRange } from './types';
+import { DataComparisonOverviewTable } from './data_comparison_overview_table';
+
+const showOnlyDriftedFieldsOptionLabel = i18n.translate(
+  'xpack.dataVisualizer.dataComparison.showOnlyDriftedFieldsOptionLabel',
+  { defaultMessage: 'Show only fields with drifted data' }
+);
+
+interface DataComparisonViewProps {
+  windowParameters?: WindowParameters;
+  dataView: DataView;
+  searchString: Query['query'];
+  searchQuery: QueryDslQueryContainer;
+  searchQueryLanguage: SearchQueryLanguage;
+  isBrushCleared: boolean;
+  runAnalysisDisabled?: boolean;
+  onReset: () => void;
+  lastRefresh: number;
+  forceRefresh: () => void;
+  randomSampler: RandomSampler;
+}
+// Data drift view
+export const DataComparisonView = ({
+  windowParameters,
+  dataView,
+  searchString,
+  searchQuery,
+  searchQueryLanguage,
+  onReset,
+  isBrushCleared,
+  lastRefresh,
+  forceRefresh,
+  randomSampler,
+}: DataComparisonViewProps) => {
+  const [showDataComparisonOnly, setShowDataComparisonOnly] = useState(false);
+
+  const [currentAnalysisWindowParameters, setCurrentAnalysisWindowParameters] = useState<
+    WindowParameters | undefined
+  >(windowParameters);
+
+  const [fetchInfo, setFetchIno] = useState<
+    | {
+        fields: DataComparisonField[];
+        currentDataView: DataView;
+        timeRanges?: { reference: TimeRange; production: TimeRange };
+      }
+    | undefined
+  >();
+
+  const onRefresh = useCallback(() => {
+    setCurrentAnalysisWindowParameters(windowParameters);
+    const mergedFields: DataComparisonField[] = [];
+    if (dataView) {
+      mergedFields.push(
+        ...dataView.fields
+          .filter(
+            (f) =>
+              f.aggregatable === true &&
+              // @ts-ignore metadata does exist
+              f.spec.metadata_field! !== true &&
+              getDataComparisonType(f.type) !== 'unsupported' &&
+              mergedFields.findIndex((merged) => merged.field === f.name) === -1
+          )
+          .map((f) => ({
+            field: f.name,
+            type: getDataComparisonType(f.type),
+            secondaryType: kbnTypeToSupportedType(f),
+            displayName: f.displayName,
+          }))
+      );
+    }
+    setFetchIno({
+      fields: mergedFields,
+      currentDataView: dataView,
+      ...(windowParameters
+        ? {
+            timeRanges: {
+              reference: {
+                start: windowParameters.baselineMin,
+                end: windowParameters.baselineMax,
+              },
+              production: {
+                start: windowParameters.deviationMin,
+                end: windowParameters.deviationMax,
+              },
+            },
+          }
+        : {}),
+    });
+    if (forceRefresh) {
+      forceRefresh();
+    }
+  }, [dataView, windowParameters, forceRefresh]);
+
+  const { result, cancelRequest } = useFetchDataComparisonResult({
+    ...fetchInfo,
+    lastRefresh,
+    randomSampler,
+    searchString,
+    searchQueryLanguage,
+    searchQuery,
+  });
+
+  const filteredData = useMemo(() => {
+    if (!result?.data) return [];
+
+    switch (showDataComparisonOnly) {
+      case true:
+        return result.data.filter((d) => d.driftDetected === true);
+      default:
+        return result.data;
+    }
+  }, [result.data, showDataComparisonOnly]);
+
+  const { onTableChange, pagination, sorting, setPageIndex } = useTableState<Feature>(
+    filteredData,
+    'driftDetected',
+    'desc'
+  );
+
+  const shouldRerunAnalysis = useMemo(
+    () =>
+      currentAnalysisWindowParameters !== undefined &&
+      !isEqual(currentAnalysisWindowParameters, windowParameters),
+    [currentAnalysisWindowParameters, windowParameters]
+  );
+
+  const onShowDataComparisonOnlyToggle = (e: EuiSwitchEvent) => {
+    setShowDataComparisonOnly(e.target.checked);
+    setPageIndex(0);
+  };
+
+  return windowParameters === undefined ? (
+    <EuiEmptyPrompt
+      color="subdued"
+      hasShadow={false}
+      hasBorder={false}
+      css={{ minWidth: '100%' }}
+      title={
+        <h2>
+          <FormattedMessage
+            id="xpack.dataVisualizer.dataComparison.emptyPromptTitle"
+            defaultMessage="Select a time range for reference and comparison data in the histogram chart to compare data distribution."
+          />
+        </h2>
+      }
+      titleSize="xs"
+      body={
+        <p>
+          <FormattedMessage
+            id="xpack.dataVisualizer.dataComparison.emptyPromptBody"
+            defaultMessage="The Data Comparison View compares the statistical properties of features in the 'reference' and 'comparison' data sets.
+"
+          />
+        </p>
+      }
+      data-test-subj="dataVisualizerNoWindowParametersEmptyPrompt"
+    />
+  ) : (
+    <div>
+      <ProgressControls
+        isBrushCleared={isBrushCleared}
+        onReset={onReset}
+        progress={result.loaded}
+        progressMessage={result.progressMessage ?? ''}
+        isRunning={result.loaded > 0 && result.loaded < 1}
+        onRefresh={onRefresh}
+        onCancel={cancelRequest}
+        shouldRerunAnalysis={shouldRerunAnalysis}
+        runAnalysisDisabled={!dataView || !windowParameters}
+      >
+        <EuiFlexItem grow={false}>
+          <EuiFormRow display="columnCompressedSwitch">
+            <EuiSwitch
+              label={showOnlyDriftedFieldsOptionLabel}
+              aria-label={showOnlyDriftedFieldsOptionLabel}
+              checked={showDataComparisonOnly}
+              onChange={onShowDataComparisonOnlyToggle}
+              compressed
+            />
+          </EuiFormRow>
+        </EuiFlexItem>
+      </ProgressControls>
+      <EuiSpacer size="m" />
+
+      {result.error ? (
+        <EuiEmptyPrompt
+          css={{ minWidth: '100%' }}
+          color="danger"
+          title={<h2>{result.error}</h2>}
+          titleSize="xs"
+          body={<span>{result.errorBody}</span>}
+        />
+      ) : (
+        <DataComparisonOverviewTable
+          data={filteredData}
+          onTableChange={onTableChange}
+          pagination={pagination}
+          sorting={sorting}
+          setPageIndex={setPageIndex}
+          status={result.status}
+        />
+      )}
+    </div>
+  );
+};

x-pack/plugins/data_visualizer/public/application/data_comparison/document_count_with_dual_brush.tsx

@@ -0,0 +1,133 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { WindowParameters } from '@kbn/aiops-utils';
+import React, { FC } from 'react';
+import { DocumentCountChart, type DocumentCountChartPoint } from '@kbn/aiops-components';
+import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui';
+import type { DocumentCountChartProps } from '@kbn/aiops-components';
+import { RandomSampler } from '@kbn/ml-random-sampler-utils';
+import { useDataVisualizerKibana } from '../kibana_context';
+import { DocumentCountStats } from '../../../common/types/field_stats';
+import { TotalCountHeader } from '../common/components/document_count_content/total_count_header';
+import { SamplingMenu } from '../common/components/random_sampling_menu/random_sampling_menu';
+
+export interface DocumentCountContentProps
+  extends Omit<
+    DocumentCountChartProps,
+    | 'dependencies'
+    | 'chartPoints'
+    | 'timeRangeEarliest'
+    | 'timeRangeLatest'
+    | 'interval'
+    | 'chartPointsSplitLabel'
+  > {
+  brushSelectionUpdateHandler: (d: WindowParameters, force: boolean) => void;
+  documentCountStats?: DocumentCountStats;
+  documentCountStatsSplit?: DocumentCountStats;
+  documentCountStatsSplitLabel?: string;
+  isBrushCleared: boolean;
+  totalCount: number;
+  sampleProbability: number;
+  initialAnalysisStart?: number | WindowParameters;
+  /** Optional color override for the default bar color for charts */
+  barColorOverride?: string;
+  /** Optional color override for the highlighted bar color for charts */
+  barHighlightColorOverride?: string;
+  windowParameters?: WindowParameters;
+  incomingInitialAnalysisStart?: number | WindowParameters;
+  randomSampler: RandomSampler;
+  reload: () => void;
+  approximate: boolean;
+}
+
+export const DocumentCountWithDualBrush: FC<DocumentCountContentProps> = ({
+  randomSampler,
+  reload,
+  brushSelectionUpdateHandler,
+  documentCountStats,
+  documentCountStatsSplit,
+  documentCountStatsSplitLabel = '',
+  isBrushCleared,
+  totalCount,
+  sampleProbability,
+  initialAnalysisStart,
+  barColorOverride,
+  barHighlightColorOverride,
+  windowParameters,
+  incomingInitialAnalysisStart,
+  approximate,
+  ...docCountChartProps
+}) => {
+  const {
+    services: { data, uiSettings, fieldFormats, charts },
+  } = useDataVisualizerKibana();
+
+  const bucketTimestamps = Object.keys(documentCountStats?.buckets ?? {}).map((time) => +time);
+  const splitBucketTimestamps = Object.keys(documentCountStatsSplit?.buckets ?? {}).map(
+    (time) => +time
+  );
+  const timeRangeEarliest = Math.min(...[...bucketTimestamps, ...splitBucketTimestamps]);
+  const timeRangeLatest = Math.max(...[...bucketTimestamps, ...splitBucketTimestamps]);
+
+  if (
+    documentCountStats === undefined ||
+    documentCountStats.buckets === undefined ||
+    timeRangeEarliest === undefined ||
+    timeRangeLatest === undefined
+  ) {
+    return totalCount !== undefined ? <TotalCountHeader totalCount={totalCount} /> : null;
+  }
+
+  const chartPoints: DocumentCountChartPoint[] = Object.entries(documentCountStats.buckets).map(
+    ([time, value]) => ({
+      time: +time,
+      value,
+    })
+  );
+
+  let chartPointsSplit: DocumentCountChartPoint[] | undefined;
+  if (documentCountStatsSplit?.buckets !== undefined) {
+    chartPointsSplit = Object.entries(documentCountStatsSplit?.buckets).map(([time, value]) => ({
+      time: +time,
+      value,
+    }));
+  }
+
+  return (
+    <EuiFlexGroup gutterSize="m" direction="column">
+      <EuiFlexGroup gutterSize="m" direction="row">
+        <EuiFlexItem>
+          <TotalCountHeader totalCount={totalCount} approximate={approximate} />
+        </EuiFlexItem>
+        <EuiFlexItem grow={false}>
+          <SamplingMenu randomSampler={randomSampler} reload={reload} />
+        </EuiFlexItem>
+      </EuiFlexGroup>
+
+      {documentCountStats.interval !== undefined && (
+        <EuiFlexItem>
+          <DocumentCountChart
+            dependencies={{ data, uiSettings, fieldFormats, charts }}
+            brushSelectionUpdateHandler={brushSelectionUpdateHandler}
+            chartPoints={chartPoints}
+            chartPointsSplit={chartPointsSplit}
+            timeRangeEarliest={timeRangeEarliest}
+            timeRangeLatest={timeRangeLatest}
+            interval={documentCountStats.interval}
+            chartPointsSplitLabel={documentCountStatsSplitLabel}
+            isBrushCleared={isBrushCleared}
+            autoAnalysisStart={initialAnalysisStart}
+            barColorOverride={barColorOverride}
+            barHighlightColorOverride={barHighlightColorOverride}
+            {...docCountChartProps}
+          />
+        </EuiFlexItem>
+      )}
+    </EuiFlexGroup>
+  );
+};

x-pack/plugins/data_visualizer/public/application/data_comparison/index.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+  DataComparisonDetectionAppState,
+  type DataComparisonSpec,
+} from './data_comparison_app_state';
+export { type DataComparisonSpec };
+// required for dynamic import using React.lazy()
+// eslint-disable-next-line import/no-default-export
+export default DataComparisonDetectionAppState;

x-pack/plugins/data_visualizer/public/application/data_comparison/types.ts

@@ -0,0 +1,115 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { isPopulatedObject } from '@kbn/ml-is-populated-object';
+import { Filter, Query } from '@kbn/es-query';
+import * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import { SEARCH_QUERY_LANGUAGE, SearchQueryLanguage } from '@kbn/ml-query-utils';
+import { DATA_COMPARISON_TYPE } from './constants';
+
+export interface DataComparisonAppState {
+  searchString?: Query['query'];
+  searchQuery?: estypes.QueryDslQueryContainer;
+  searchQueryLanguage: SearchQueryLanguage;
+  filters?: Filter[];
+}
+
+export type DataComparisonFullAppState = Required<DataComparisonAppState>;
+export type BasicAppState = DataComparisonFullAppState;
+
+const defaultSearchQuery = {
+  match_all: {},
+};
+
+export const getDefaultDataComparisonState = (
+  overrides?: Partial<DataComparisonAppState>
+): DataComparisonFullAppState => ({
+  searchString: '',
+  searchQuery: defaultSearchQuery,
+  searchQueryLanguage: SEARCH_QUERY_LANGUAGE.KUERY,
+  filters: [],
+  ...overrides,
+});
+
+export interface Histogram {
+  doc_count: number;
+  key: string | number;
+  percentage?: number;
+}
+
+export interface ComparisonHistogram extends Histogram {
+  g: string;
+}
+
+// Show the overview table
+export interface Feature {
+  featureName: string;
+  fieldType: DataComparisonField['type'];
+  driftDetected: boolean;
+  similarityTestPValue: number;
+  productionHistogram: Histogram[];
+  referenceHistogram: Histogram[];
+  comparisonDistribution: ComparisonHistogram[];
+}
+
+export interface DataComparisonField {
+  field: string;
+  type: DataComparisonType;
+  secondaryType: string;
+  displayName: string;
+}
+export enum FETCH_STATUS {
+  LOADING = 'loading',
+  SUCCESS = 'success',
+  FAILURE = 'failure',
+  NOT_INITIATED = 'not_initiated',
+}
+
+export interface Result<T extends unknown> {
+  status: FETCH_STATUS;
+  data?: T;
+  error?: string;
+  errorBody?: string;
+}
+
+export interface TimeRange {
+  start: string | number;
+  end: string | number;
+}
+
+export interface Range {
+  min: number;
+  max: number;
+  interval: number;
+}
+
+export interface NumericDriftData {
+  type: 'numeric';
+  pValue: number;
+  range?: Range;
+  referenceHistogram: Histogram[];
+  productionHistogram: Histogram[];
+  secondaryType: string;
+}
+export interface CategoricalDriftData {
+  type: 'categorical';
+  driftedTerms: Histogram[];
+  driftedSumOtherDocCount: number;
+  baselineTerms: Histogram[];
+  baselineSumOtherDocCount: number;
+  secondaryType: string;
+}
+
+export const isNumericDriftData = (arg: any): arg is NumericDriftData => {
+  return isPopulatedObject(arg, ['type']) && arg.type === DATA_COMPARISON_TYPE.NUMERIC;
+};
+
+export const isCategoricalDriftData = (arg: any): arg is CategoricalDriftData => {
+  return isPopulatedObject(arg, ['type']) && arg.type === DATA_COMPARISON_TYPE.CATEGORICAL;
+};
+
+export type DataComparisonType = typeof DATA_COMPARISON_TYPE[keyof typeof DATA_COMPARISON_TYPE];

x-pack/plugins/data_visualizer/public/application/data_comparison/use_data_drift_result.ts

@@ -0,0 +1,891 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
+import type { IKibanaSearchRequest } from '@kbn/data-plugin/common';
+import { lastValueFrom } from 'rxjs';
+import * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import type { DataView } from '@kbn/data-views-plugin/public';
+import { isPopulatedObject } from '@kbn/ml-is-populated-object';
+import type { Query } from '@kbn/data-plugin/common';
+import { chunk, cloneDeep, flatten } from 'lodash';
+import type { MappingRuntimeFields } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import type { SearchQueryLanguage } from '@kbn/ml-query-utils';
+import { getDefaultDSLQuery } from '@kbn/ml-query-utils';
+import { i18n } from '@kbn/i18n';
+import { RandomSampler, RandomSamplerWrapper } from '@kbn/ml-random-sampler-utils';
+import { extractErrorMessage } from '@kbn/ml-error-utils';
+import { AggregationsAggregate } from '@elastic/elasticsearch/lib/api/types';
+import { QueryDslBoolQuery } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import { isDefined } from '@kbn/ml-is-defined';
+import { useDataVisualizerKibana } from '../kibana_context';
+import {
+  REFERENCE_LABEL,
+  COMPARISON_LABEL,
+  DRIFT_P_VALUE_THRESHOLD,
+  DATA_COMPARISON_TYPE,
+} from './constants';
+
+import {
+  Histogram,
+  NumericDriftData,
+  CategoricalDriftData,
+  Range,
+  FETCH_STATUS,
+  Result,
+  isNumericDriftData,
+  Feature,
+  DataComparisonField,
+  TimeRange,
+} from './types';
+import { computeChi2PValue } from './data_comparison_utils';
+
+export const getDataComparisonType = (kibanaType: string): DataComparisonField['type'] => {
+  switch (kibanaType) {
+    case 'number':
+      return DATA_COMPARISON_TYPE.NUMERIC;
+    case 'boolean':
+    case 'string':
+      return DATA_COMPARISON_TYPE.CATEGORICAL;
+    default:
+      return DATA_COMPARISON_TYPE.UNSUPPORTED;
+  }
+};
+
+type UseDataSearch = ReturnType<typeof useDataSearch>;
+
+export const useDataSearch = <T>() => {
+  const { data } = useDataVisualizerKibana().services;
+
+  return useCallback(
+    async (esSearchRequestParams: IKibanaSearchRequest['params'], abortSignal?: AbortSignal) => {
+      try {
+        const { rawResponse: resp } = await lastValueFrom(
+          data.search.search(
+            {
+              params: esSearchRequestParams,
+            },
+            { abortSignal }
+          )
+        );
+
+        return resp;
+      } catch (error) {
+        if (error.name === 'AbortError') {
+          // ignore abort errors
+        } else {
+          throw Error(error);
+        }
+      }
+    },
+    [data]
+  );
+};
+
+const percents = [5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75, 80, 85, 90, 95];
+
+const normalizeHistogram = (histogram: Histogram[]): Histogram[] => {
+  // Compute a total doc_count for all terms
+  const totalDocCount: number = histogram.reduce((acc, term) => acc + term.doc_count, 0);
+  // Iterate over the original array and update the doc_count of each term in the new array
+  return histogram.map((term) => ({
+    ...term,
+    percentage: totalDocCount > 0 ? term.doc_count / totalDocCount : 0,
+  }));
+};
+
+const normalizeTerms = (
+  terms: Histogram[],
+  keys: Array<{ key: string; relative_drift: number }>,
+  totalDocCount: number
+): { normalizedTerms: Histogram[]; totalDocCount: number } => {
+  // Create a new array of terms with the same keys as the given array
+  const normalizedTerms: Array<Histogram & { relative_drift?: number }> = keys.map((term) => ({
+    ...term,
+    doc_count: 0,
+    percentage: 0,
+  }));
+
+  // Iterate over the original array and update the doc_count of each term in the new array
+  terms.forEach((term) => {
+    const index: number = keys.findIndex((k) => k.key === term.key.toString());
+    if (index !== -1) {
+      normalizedTerms[index].doc_count = term.doc_count;
+      normalizedTerms[index].percentage = term.doc_count / totalDocCount;
+    }
+  });
+
+  return {
+    normalizedTerms,
+    totalDocCount,
+  };
+};
+
+const processDataComparisonResult = (
+  result: Record<string, NumericDriftData | CategoricalDriftData>
+): Feature[] => {
+  return Object.entries(result).map(([featureName, data]) => {
+    if (isNumericDriftData(data)) {
+      // normalize data.referenceHistogram and data.productionHistogram to use frequencies instead of counts
+      const referenceHistogram: Histogram[] = normalizeHistogram(data.referenceHistogram);
+      const productionHistogram: Histogram[] = normalizeHistogram(data.productionHistogram);
+
+      return {
+        featureName,
+        secondaryType: data.secondaryType,
+        fieldType: DATA_COMPARISON_TYPE.NUMERIC,
+        driftDetected: data.pValue < DRIFT_P_VALUE_THRESHOLD,
+        similarityTestPValue: data.pValue,
+        referenceHistogram: referenceHistogram ?? [],
+        productionHistogram: productionHistogram ?? [],
+        comparisonDistribution: [
+          ...referenceHistogram.map((h) => ({ ...h, g: REFERENCE_LABEL })),
+          ...productionHistogram.map((h) => ({ ...h, g: COMPARISON_LABEL })),
+        ],
+      };
+    }
+
+    // normalize data.baselineTerms and data.driftedTerms to have same keys
+    // Get all unique keys from both arrays
+    const allKeys: string[] = Array.from(
+      new Set([
+        ...data.baselineTerms.map((term) => term.key.toString()),
+        ...data.driftedTerms.map((term) => term.key.toString()),
+      ])
+    );
+
+    // Compute a total doc_count for all terms
+    const referenceTotalDocCount: number = data.baselineTerms.reduce(
+      (acc, term) => acc + term.doc_count,
+      data.baselineSumOtherDocCount
+    );
+    const productionTotalDocCount: number = data.driftedTerms.reduce(
+      (acc, term) => acc + term.doc_count,
+      data.driftedSumOtherDocCount
+    );
+
+    // Sort the categories (allKeys) by the following metric: Math.abs(productionDocCount-referenceDocCount)/referenceDocCount
+    const sortedKeys = allKeys
+      .map((k) => {
+        const key = k.toString();
+        const baselineTerm = data.baselineTerms.find((t) => t.key === key);
+        const driftedTerm = data.driftedTerms.find((t) => t.key === key);
+        if (baselineTerm && driftedTerm) {
+          const referencePercentage = baselineTerm.doc_count / referenceTotalDocCount;
+          const productionPercentage = driftedTerm.doc_count / productionTotalDocCount;
+          return {
+            key,
+            relative_drift:
+              Math.abs(productionPercentage - referencePercentage) / referencePercentage,
+          };
+        }
+        return {
+          key,
+          relative_drift: 0,
+        };
+      })
+      .sort((s1, s2) => s2.relative_drift - s1.relative_drift);
+
+    // Normalize the baseline and drifted terms arrays
+    const { normalizedTerms: normalizedBaselineTerms } = normalizeTerms(
+      data.baselineTerms,
+      sortedKeys,
+      referenceTotalDocCount
+    );
+    const { normalizedTerms: normalizedDriftedTerms } = normalizeTerms(
+      data.driftedTerms,
+      sortedKeys,
+      productionTotalDocCount
+    );
+
+    const pValue: number = computeChi2PValue(normalizedBaselineTerms, normalizedDriftedTerms);
+    return {
+      featureName,
+      secondaryType: data.secondaryType,
+      fieldType: DATA_COMPARISON_TYPE.CATEGORICAL,
+      driftDetected: pValue < DRIFT_P_VALUE_THRESHOLD,
+      similarityTestPValue: pValue,
+      referenceHistogram: normalizedBaselineTerms ?? [],
+      productionHistogram: normalizedDriftedTerms ?? [],
+      comparisonDistribution: [
+        ...normalizedBaselineTerms.map((h) => ({ ...h, g: REFERENCE_LABEL })),
+        ...normalizedDriftedTerms.map((h) => ({ ...h, g: COMPARISON_LABEL })),
+      ],
+    };
+  });
+};
+
+const getDataComparisonQuery = ({
+  runtimeFields,
+  searchQuery,
+  datetimeField,
+  timeRange,
+}: {
+  runtimeFields: MappingRuntimeFields;
+  searchQuery?: estypes.QueryDslQueryContainer;
+  datetimeField?: string;
+  timeRange?: TimeRange;
+}): NonNullable<estypes.SearchRequest['body']> => {
+  let rangeFilter;
+  if (timeRange && datetimeField !== undefined && isPopulatedObject(timeRange, ['start', 'end'])) {
+    rangeFilter = {
+      range: {
+        [datetimeField]: {
+          gte: timeRange.start,
+          lte: timeRange.end,
+          format: 'epoch_millis',
+        },
+      },
+    };
+  }
+
+  const query = cloneDeep(
+    !searchQuery || isPopulatedObject(searchQuery, ['match_all'])
+      ? getDefaultDSLQuery()
+      : searchQuery
+  );
+
+  if (rangeFilter && isPopulatedObject<string, QueryDslBoolQuery>(query, ['bool'])) {
+    if (Array.isArray(query.bool.filter)) {
+      // @ts-expect-error gte and lte can be numeric
+      query.bool.filter.push(rangeFilter);
+    } else {
+      // @ts-expect-error gte and lte can be numeric
+      query.bool.filter = [rangeFilter];
+    }
+  }
+
+  const refDataQuery: NonNullable<estypes.SearchRequest['body']> = {
+    query,
+  };
+  if (runtimeFields) {
+    refDataQuery.runtime_mappings = runtimeFields;
+  }
+  return refDataQuery;
+};
+
+const fetchReferenceBaselineData = async ({
+  baseRequest,
+  fields,
+  randomSamplerWrapper,
+  dataSearch,
+  signal,
+}: {
+  baseRequest: EsRequestParams;
+  dataSearch: UseDataSearch;
+  fields: DataComparisonField[];
+  randomSamplerWrapper: RandomSamplerWrapper;
+  signal: AbortSignal;
+}) => {
+  const baselineRequest = { ...baseRequest };
+  const baselineRequestAggs: Record<string, estypes.AggregationsAggregationContainer> = {};
+
+  // for each field with type "numeric", add a percentiles agg to the request
+  for (const { field, type } of fields) {
+    // if the field is numeric, add a percentiles and stats aggregations to the request
+    if (type === DATA_COMPARISON_TYPE.NUMERIC) {
+      baselineRequestAggs[`${field}_percentiles`] = {
+        percentiles: {
+          field,
+          percents,
+        },
+      };
+      baselineRequestAggs[`${field}_stats`] = {
+        stats: {
+          field,
+        },
+      };
+    }
+    // if the field is categorical, add a terms aggregation to the request
+    if (type === DATA_COMPARISON_TYPE.CATEGORICAL) {
+      baselineRequestAggs[`${field}_terms`] = {
+        terms: {
+          field,
+          size: 100, // also DFA can potentially handle problems with 100 categories, for visualization purposes we will use top 10
+        },
+      };
+    }
+  }
+
+  const baselineResponse = await dataSearch(
+    {
+      ...baselineRequest,
+      body: { ...baselineRequest.body, aggs: randomSamplerWrapper.wrap(baselineRequestAggs) },
+    },
+    signal
+  );
+
+  return baselineResponse;
+};
+
+const fetchComparisonDriftedData = async ({
+  dataSearch,
+  fields,
+  baselineResponseAggs,
+  baseRequest,
+  randomSamplerWrapper,
+  signal,
+}: {
+  baseRequest: EsRequestParams;
+  dataSearch: UseDataSearch;
+  fields: DataComparisonField[];
+  randomSamplerWrapper: RandomSamplerWrapper;
+  signal: AbortSignal;
+  baselineResponseAggs: object;
+}) => {
+  const driftedRequest = { ...baseRequest };
+  const driftedRequestAggs: Record<string, estypes.AggregationsAggregationContainer> = {};
+
+  for (const { field, type } of fields) {
+    if (
+      isPopulatedObject(baselineResponseAggs, [`${field}_percentiles`]) &&
+      type === DATA_COMPARISON_TYPE.NUMERIC
+    ) {
+      // create ranges based on percentiles
+      const percentiles = Object.values<number>(
+        (baselineResponseAggs[`${field}_percentiles`] as Record<string, { values: number }>).values
+      );
+      const ranges: Array<{ from?: number; to?: number }> = [];
+      percentiles.forEach((val: number, idx) => {
+        if (idx === 0) {
+          ranges.push({ to: val });
+        } else if (idx === percentiles.length - 1) {
+          ranges.push({ from: val });
+        } else {
+          ranges.push({ from: percentiles[idx - 1], to: val });
+        }
+      });
+      // add range and bucket_count_ks_test to the request
+      driftedRequestAggs[`${field}_ranges`] = {
+        range: {
+          field,
+          ranges,
+        },
+      };
+      driftedRequestAggs[`${field}_ks_test`] = {
+        bucket_count_ks_test: {
+          buckets_path: `${field}_ranges>_count`,
+          alternative: ['two_sided'],
+        },
+      };
+      // add stats aggregation to the request
+      driftedRequestAggs[`${field}_stats`] = {
+        stats: {
+          field,
+        },
+      };
+    }
+    // if feature is categoric perform terms aggregation
+    if (type === DATA_COMPARISON_TYPE.CATEGORICAL) {
+      driftedRequestAggs[`${field}_terms`] = {
+        terms: {
+          field,
+          size: 100, // also DFA can potentially handle problems with 100 categories, for visualization purposes we will use top 10
+        },
+      };
+    }
+  }
+
+  const driftedResp = await dataSearch(
+    {
+      ...driftedRequest,
+      body: { ...driftedRequest.body, aggs: randomSamplerWrapper.wrap(driftedRequestAggs) },
+    },
+    signal
+  );
+  return driftedResp;
+};
+
+const fetchHistogramData = async ({
+  dataSearch,
+  fields,
+  driftedRespAggs,
+  baselineResponseAggs,
+  baseRequest,
+  randomSamplerWrapper,
+  signal,
+}: {
+  baseRequest: EsRequestParams;
+  dataSearch: UseDataSearch;
+  fields: DataComparisonField[];
+  randomSamplerWrapper: RandomSamplerWrapper;
+  signal: AbortSignal;
+  baselineResponseAggs: Record<string, estypes.AggregationsStatsAggregate>;
+  driftedRespAggs: Record<string, estypes.AggregationsStatsAggregate>;
+}) => {
+  const histogramRequestAggs: Record<string, estypes.AggregationsAggregationContainer> = {};
+  const fieldRange: { [field: string]: Range } = {};
+
+  for (const { field, type } of fields) {
+    // add histogram aggregation with min and max from baseline
+    if (
+      type === DATA_COMPARISON_TYPE.NUMERIC &&
+      baselineResponseAggs[`${field}_stats`] &&
+      driftedRespAggs[`${field}_stats`]
+    ) {
+      const numBins = 10;
+      const min = Math.min(
+        baselineResponseAggs[`${field}_stats`].min!,
+        driftedRespAggs[`${field}_stats`].min!
+      );
+      const max = Math.max(
+        baselineResponseAggs[`${field}_stats`].max!,
+        driftedRespAggs[`${field}_stats`].max!
+      );
+      const interval = (max - min) / numBins;
+
+      if (interval === 0) {
+        continue;
+      }
+      const offset = min;
+      fieldRange[field] = { min, max, interval };
+      histogramRequestAggs[`${field}_histogram`] = {
+        histogram: {
+          field,
+          interval,
+          offset,
+          extended_bounds: {
+            min,
+            max,
+          },
+        },
+      };
+    }
+  }
+  if (isPopulatedObject(histogramRequestAggs)) {
+    const histogramRequest = {
+      ...baseRequest,
+      body: {
+        ...baseRequest.body,
+        aggs: randomSamplerWrapper.wrap(histogramRequestAggs),
+      },
+    };
+
+    return dataSearch(histogramRequest, signal);
+  }
+};
+
+const isFulfilled = <T>(
+  input: PromiseSettledResult<Awaited<T>>
+): input is PromiseFulfilledResult<Awaited<T>> => input.status === 'fulfilled';
+const isRejected = <T>(input: PromiseSettledResult<Awaited<T>>): input is PromiseRejectedResult =>
+  input.status === 'rejected';
+
+type EsRequestParams = NonNullable<
+  IKibanaSearchRequest<NonNullable<estypes.SearchRequest>>['params']
+>;
+
+interface ReturnedError {
+  error?: string;
+  errorBody?: string;
+}
+
+function isReturnedError(arg: unknown): arg is ReturnedError {
+  return isPopulatedObject(arg, ['error']);
+}
+
+/**
+ * Help split one big request into multiple requests (with max of 30 fields/request)
+ * to avoid too big of a data payload
+ * Returns a merged
+ * @param fields - list of fields to split
+ * @param randomSamplerWrapper - helper from randomSampler to pack and unpack 'sample' path from esResponse.aggregations
+ * @param asyncFetchFn - callback function with the divided fields
+ */
+export const fetchInParallelChunks = async <
+  ReturnedRespFromFetchFn extends { aggregations: Record<string, AggregationsAggregate> }
+>({
+  fields,
+  randomSamplerWrapper,
+  asyncFetchFn,
+  errorMsg,
+}: {
+  fields: DataComparisonField[];
+  randomSamplerWrapper: RandomSamplerWrapper;
+  asyncFetchFn: (chunkedFields: DataComparisonField[]) => Promise<ReturnedRespFromFetchFn>;
+  errorMsg?: string;
+}): Promise<ReturnedRespFromFetchFn | ReturnedError> => {
+  const { unwrap } = randomSamplerWrapper;
+  const results = await Promise.allSettled(
+    chunk(fields, 30).map((chunkedFields: DataComparisonField[]) => asyncFetchFn(chunkedFields))
+  );
+
+  const mergedResults = results
+    .filter(isFulfilled)
+    .filter((r) => r.value)
+    .map((r) => {
+      try {
+        return unwrap(r?.value.aggregations);
+      } catch (e) {
+        return undefined;
+      }
+    })
+    .filter(isDefined);
+
+  if (mergedResults.length === 0) {
+    const error = results.find(isRejected);
+    if (error) {
+      // eslint-disable-next-line no-console
+      console.error(error);
+      return {
+        error: errorMsg ?? 'An error occurred fetching data comparison data',
+        errorBody: error.reason.message,
+      };
+    }
+  }
+
+  const baselineResponseAggs = flatten(mergedResults).reduce(
+    (prev, acc) => ({ ...acc, ...prev }),
+    {}
+  );
+  return baselineResponseAggs;
+};
+
+const initialState = {
+  data: undefined,
+  status: FETCH_STATUS.NOT_INITIATED,
+  error: undefined,
+  errorBody: undefined,
+};
+export const useFetchDataComparisonResult = (
+  {
+    fields,
+    currentDataView,
+    timeRanges,
+    searchQuery,
+    searchString,
+    lastRefresh,
+    randomSampler,
+  }: {
+    lastRefresh: number;
+    randomSampler?: RandomSampler;
+    fields?: DataComparisonField[];
+    currentDataView?: DataView;
+    timeRanges?: { reference: TimeRange; production: TimeRange };
+    searchQuery?: estypes.QueryDslQueryContainer;
+    searchString?: Query['query'];
+    searchQueryLanguage?: SearchQueryLanguage;
+  } = { lastRefresh: 0 }
+) => {
+  const dataSearch = useDataSearch();
+  const [result, setResult] = useState<Result<Feature[]>>(initialState);
+  const [loaded, setLoaded] = useState<number>(0);
+  const [progressMessage, setProgressMessage] = useState<string | undefined>();
+  const abortController = useRef(new AbortController());
+
+  const cancelRequest = useCallback(() => {
+    abortController.current.abort();
+    abortController.current = new AbortController();
+    setResult(initialState);
+    setProgressMessage(undefined);
+    setLoaded(0);
+  }, []);
+
+  useEffect(
+    () => {
+      const doFetchEsRequest = async function () {
+        if (!randomSampler) return;
+
+        const randomSamplerWrapper = randomSampler.createRandomSamplerWrapper();
+
+        setLoaded(0);
+        setResult({
+          data: undefined,
+          status: FETCH_STATUS.NOT_INITIATED,
+          error: undefined,
+        });
+
+        setProgressMessage(
+          i18n.translate('xpack.dataVisualizer.dataComparison.progress.started', {
+            defaultMessage: `Ready to fetch data for comparison.`,
+          })
+        );
+
+        const signal = abortController.current.signal;
+        if (!fields || !currentDataView) return;
+
+        setResult({ data: undefined, status: FETCH_STATUS.LOADING, error: undefined });
+
+        // Place holder for when there might be difference data views in the future
+        const referenceIndex = currentDataView?.getIndexPattern();
+        const productionIndex = referenceIndex;
+
+        const runtimeFields = currentDataView?.getRuntimeMappings();
+
+        setProgressMessage(
+          i18n.translate('xpack.dataVisualizer.dataComparison.progress.loadedFields', {
+            defaultMessage: `Loaded fields from index '{referenceIndex}' to analyze.`,
+            values: { referenceIndex },
+          })
+        );
+        const refDataQuery = getDataComparisonQuery({
+          searchQuery,
+          datetimeField: currentDataView?.timeFieldName,
+          runtimeFields,
+          timeRange: timeRanges?.reference,
+        });
+
+        try {
+          const fieldsCount = fields.length;
+
+          setProgressMessage(
+            i18n.translate('xpack.dataVisualizer.dataComparison.progress.loadingReference', {
+              defaultMessage: `Loading reference data for {fieldsCount} fields.`,
+              values: { fieldsCount },
+            })
+          );
+
+          const baselineRequest: EsRequestParams = {
+            index: referenceIndex,
+            body: {
+              size: 0,
+              aggs: {} as Record<string, estypes.AggregationsAggregationContainer>,
+              ...refDataQuery,
+            },
+          };
+
+          const baselineResponseAggs = await fetchInParallelChunks({
+            fields,
+            randomSamplerWrapper,
+
+            asyncFetchFn: (chunkedFields) =>
+              fetchReferenceBaselineData({
+                dataSearch,
+                baseRequest: baselineRequest,
+                fields: chunkedFields,
+                randomSamplerWrapper,
+                signal,
+              }),
+          });
+
+          if (isReturnedError(baselineResponseAggs)) {
+            setResult({
+              data: undefined,
+              status: FETCH_STATUS.FAILURE,
+              error: baselineResponseAggs.error,
+              errorBody: baselineResponseAggs.errorBody,
+            });
+            return;
+          }
+
+          setProgressMessage(
+            i18n.translate('xpack.dataVisualizer.dataComparison.progress.loadedReference', {
+              defaultMessage: `Loaded reference data.`,
+            })
+          );
+          setLoaded(0.25);
+
+          const prodDataQuery = getDataComparisonQuery({
+            searchQuery,
+            datetimeField: currentDataView?.timeFieldName,
+            runtimeFields,
+            timeRange: timeRanges?.production,
+          });
+
+          setProgressMessage(
+            i18n.translate('xpack.dataVisualizer.dataComparison.progress.loadingComparison', {
+              defaultMessage: `Loading comparison data for {fieldsCount} fields.`,
+              values: { fieldsCount },
+            })
+          );
+
+          const driftedRequest: EsRequestParams = {
+            index: productionIndex,
+            body: {
+              size: 0,
+              aggs: {} as Record<string, estypes.AggregationsAggregationContainer>,
+              ...prodDataQuery,
+            },
+          };
+          const driftedRespAggs = await fetchInParallelChunks({
+            fields,
+            randomSamplerWrapper,
+
+            asyncFetchFn: (chunkedFields: DataComparisonField[]) =>
+              fetchComparisonDriftedData({
+                dataSearch,
+                baseRequest: driftedRequest,
+                baselineResponseAggs,
+                fields: chunkedFields,
+                randomSamplerWrapper,
+                signal,
+              }),
+          });
+          if (isReturnedError(driftedRespAggs)) {
+            setResult({
+              data: undefined,
+              status: FETCH_STATUS.FAILURE,
+              error: driftedRespAggs.error,
+              errorBody: driftedRespAggs.errorBody,
+            });
+            return;
+          }
+
+          setLoaded(0.5);
+          setProgressMessage(
+            i18n.translate('xpack.dataVisualizer.dataComparison.progress.loadedComparison', {
+              defaultMessage: `Loaded comparison data. Now loading histogram data.`,
+            })
+          );
+
+          const referenceHistogramRequest: EsRequestParams = {
+            index: referenceIndex,
+            body: {
+              size: 0,
+              aggs: {} as Record<string, estypes.AggregationsAggregationContainer>,
+              ...refDataQuery,
+            },
+          };
+
+          const referenceHistogramRespAggs = await fetchInParallelChunks({
+            fields,
+            randomSamplerWrapper,
+
+            asyncFetchFn: (chunkedFields: DataComparisonField[]) =>
+              fetchHistogramData({
+                dataSearch,
+                baseRequest: referenceHistogramRequest,
+                baselineResponseAggs,
+                driftedRespAggs,
+                fields: chunkedFields,
+                randomSamplerWrapper,
+                signal,
+              }),
+          });
+
+          if (isReturnedError(referenceHistogramRespAggs)) {
+            setResult({
+              data: undefined,
+              status: FETCH_STATUS.FAILURE,
+              error: referenceHistogramRespAggs.error,
+              errorBody: referenceHistogramRespAggs.errorBody,
+            });
+            return;
+          }
+
+          setLoaded(0.75);
+          setProgressMessage(
+            i18n.translate(
+              'xpack.dataVisualizer.dataComparison.progress.loadedReferenceHistogram',
+              {
+                defaultMessage: `Loaded histogram data for reference data set.`,
+              }
+            )
+          );
+
+          const productionHistogramRequest: EsRequestParams = {
+            index: productionIndex,
+            body: {
+              size: 0,
+              aggs: {} as Record<string, estypes.AggregationsAggregationContainer>,
+              ...prodDataQuery,
+            },
+          };
+
+          const productionHistogramRespAggs = await fetchInParallelChunks({
+            fields,
+            randomSamplerWrapper,
+
+            asyncFetchFn: (chunkedFields: DataComparisonField[]) =>
+              fetchHistogramData({
+                dataSearch,
+                baseRequest: productionHistogramRequest,
+                baselineResponseAggs,
+                driftedRespAggs,
+                fields: chunkedFields,
+                randomSamplerWrapper,
+                signal,
+              }),
+          });
+
+          if (isReturnedError(productionHistogramRespAggs)) {
+            setResult({
+              data: undefined,
+              status: FETCH_STATUS.FAILURE,
+              error: productionHistogramRespAggs.error,
+              errorBody: productionHistogramRespAggs.errorBody,
+            });
+            return;
+          }
+
+          const data: Record<string, NumericDriftData | CategoricalDriftData> = {};
+          for (const { field, type, secondaryType } of fields) {
+            if (
+              type === DATA_COMPARISON_TYPE.NUMERIC &&
+              driftedRespAggs[`${field}_ks_test`] &&
+              referenceHistogramRespAggs[`${field}_histogram`] &&
+              productionHistogramRespAggs[`${field}_histogram`]
+            ) {
+              data[field] = {
+                secondaryType,
+                type: DATA_COMPARISON_TYPE.NUMERIC,
+                pValue: driftedRespAggs[`${field}_ks_test`].two_sided,
+                referenceHistogram: referenceHistogramRespAggs[`${field}_histogram`].buckets,
+                productionHistogram: productionHistogramRespAggs[`${field}_histogram`].buckets,
+              };
+            }
+            if (
+              type === DATA_COMPARISON_TYPE.CATEGORICAL &&
+              driftedRespAggs[`${field}_terms`] &&
+              baselineResponseAggs[`${field}_terms`]
+            ) {
+              data[field] = {
+                secondaryType,
+                type: DATA_COMPARISON_TYPE.CATEGORICAL,
+                driftedTerms: driftedRespAggs[`${field}_terms`].buckets ?? [],
+                driftedSumOtherDocCount: driftedRespAggs[`${field}_terms`].sum_other_doc_count,
+                baselineTerms: baselineResponseAggs[`${field}_terms`].buckets ?? [],
+                baselineSumOtherDocCount:
+                  baselineResponseAggs[`${field}_terms`].sum_other_doc_count,
+              };
+            }
+          }
+
+          setProgressMessage(
+            i18n.translate('xpack.dataVisualizer.dataComparison.progress.loadedHistogramData', {
+              defaultMessage: `Loaded histogram data for comparison data set.`,
+            })
+          );
+
+          setResult({
+            data: processDataComparisonResult(data),
+            status: FETCH_STATUS.SUCCESS,
+          });
+          setLoaded(1);
+        } catch (e) {
+          // eslint-disable-next-line no-console
+          console.error(e);
+          setResult({
+            data: undefined,
+            status: FETCH_STATUS.FAILURE,
+            error: 'An error occurred while fetching data comparison data',
+            errorBody: extractErrorMessage(e),
+          });
+        }
+      };
+
+      doFetchEsRequest();
+    },
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+    [
+      dataSearch,
+      // eslint-disable-next-line react-hooks/exhaustive-deps
+      JSON.stringify({
+        fields,
+        timeRanges,
+        currentDataView: currentDataView?.id,
+        searchString,
+        lastRefresh,
+      }),
+    ]
+  );
+  const dataComparisonResult = useMemo(
+    () => ({ result: { ...result, loaded, progressMessage }, cancelRequest }),
+    [result, loaded, progressMessage, cancelRequest]
+  );
+  return dataComparisonResult;
+};

x-pack/plugins/data_visualizer/public/application/index.ts

@@ -12,3 +12,4 @@ export type {
   IndexDataVisualizerViewProps,
 } from './index_data_visualizer';
 export { IndexDataVisualizer } from './index_data_visualizer';
+export type { DataComparisonSpec } from './data_comparison';

x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx

@@ -24,7 +24,6 @@ import {
   EuiTitle,
 } from '@elastic/eui';
 
-import { i18n } from '@kbn/i18n';
 import { Filter, FilterStateStore, Query } from '@kbn/es-query';
 import { generateFilters } from '@kbn/data-plugin/public';
 import { DataView, DataViewField } from '@kbn/data-views-plugin/public';
@@ -37,6 +36,7 @@ import {
 import { useStorage } from '@kbn/ml-local-storage';
 
 import type { SavedSearch } from '@kbn/saved-search-plugin/public';
+import { SEARCH_QUERY_LANGUAGE, SearchQueryLanguage } from '@kbn/ml-query-utils';
 import { kbnTypeToSupportedType } from '../../../common/util/field_types_utils';
 import { useCurrentEuiTheme } from '../../../common/hooks/use_current_eui_theme';
 import {
@@ -59,7 +59,6 @@ import {
   DataVisualizerIndexBasedAppState,
   DataVisualizerIndexBasedPageUrlState,
 } from '../../types/index_data_visualizer_state';
-import { SEARCH_QUERY_LANGUAGE, SearchQueryLanguage } from '../../types/combined_query';
 import { useDataVisualizerKibana } from '../../../kibana_context';
 import { FieldCountPanel } from '../../../common/components/field_count_panel';
 import { DocumentCountContent } from '../../../common/components/document_count_content';
@@ -173,8 +172,7 @@ export const IndexDataVisualizerView: FC<IndexDataVisualizerViewProps> = (dataVi
   );
 
   const { services } = useDataVisualizerKibana();
-  const { notifications, uiSettings, data } = services;
-  const { toasts } = notifications;
+  const { uiSettings, data } = services;
 
   const [dataVisualizerListState, setDataVisualizerListState] =
     usePageUrlState<DataVisualizerIndexBasedPageUrlState>(
@@ -189,26 +187,6 @@ export const IndexDataVisualizerView: FC<IndexDataVisualizerViewProps> = (dataVi
 
   const { currentDataView, currentSessionId, getAdditionalLinks } = dataVisualizerProps;
 
-  useEffect(() => {
-    if (!currentDataView.isTimeBased()) {
-      toasts.addWarning({
-        title: i18n.translate(
-          'xpack.dataVisualizer.index.dataViewNotBasedOnTimeSeriesNotificationTitle',
-          {
-            defaultMessage: 'The data view {dataViewTitle} is not based on a time series',
-            values: { dataViewTitle: currentDataView.title },
-          }
-        ),
-        text: i18n.translate(
-          'xpack.dataVisualizer.index.dataViewNotBasedOnTimeSeriesNotificationDescription',
-          {
-            defaultMessage: 'Anomaly detection only runs over time-based indices',
-          }
-        ),
-      });
-    }
-  }, [currentDataView, toasts]);
-
   const dataViewFields: DataViewField[] = currentDataView.fields;
 
   const fieldTypes = useMemo(() => {

x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/search_bar.tsx

@@ -0,0 +1,115 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { Filter, Query, TimeRange } from '@kbn/es-query';
+import { i18n } from '@kbn/i18n';
+import React, { useEffect, useState } from 'react';
+import { isDefined } from '@kbn/ml-is-defined';
+import { DataView } from '@kbn/data-views-plugin/common';
+import { SearchQueryLanguage } from '@kbn/ml-query-utils';
+import { createMergedEsQuery } from '../../utils/saved_search_utils';
+import { useDataVisualizerKibana } from '../../../kibana_context';
+
+export const SearchPanelContent = ({
+  searchQuery,
+  searchString,
+  searchQueryLanguage,
+  dataView,
+  setSearchParams,
+}: {
+  dataView: DataView;
+  searchQuery: Query['query'];
+  searchString: Query['query'];
+  searchQueryLanguage: SearchQueryLanguage;
+  setSearchParams({
+    searchQuery,
+    searchString,
+    queryLanguage,
+    filters,
+  }: {
+    searchQuery: Query['query'];
+    searchString: Query['query'];
+    queryLanguage: SearchQueryLanguage;
+    filters: Filter[];
+  }): void;
+}) => {
+  const {
+    services: {
+      uiSettings,
+      notifications: { toasts },
+      data: { query: queryManager },
+      unifiedSearch: {
+        ui: { SearchBar },
+      },
+    },
+  } = useDataVisualizerKibana();
+  // The internal state of the input query bar updated on every key stroke.
+  const [searchInput, setSearchInput] = useState<Query>({
+    query: searchString || '',
+    language: searchQueryLanguage,
+  });
+
+  useEffect(() => {
+    setSearchInput({
+      query: searchString || '',
+      language: searchQueryLanguage,
+    });
+  }, [searchQueryLanguage, searchString, queryManager.filterManager]);
+
+  const searchHandler = ({ query, filters }: { query?: Query; filters?: Filter[] }) => {
+    const mergedQuery = isDefined(query) ? query : searchInput;
+    const mergedFilters = isDefined(filters) ? filters : queryManager.filterManager.getFilters();
+    try {
+      if (mergedFilters) {
+        queryManager.filterManager.setFilters(mergedFilters);
+      }
+
+      const combinedQuery = createMergedEsQuery(
+        mergedQuery,
+        queryManager.filterManager.getFilters() ?? [],
+        dataView,
+        uiSettings
+      );
+
+      setSearchParams({
+        searchQuery: combinedQuery,
+        searchString: mergedQuery.query,
+        queryLanguage: mergedQuery.language as SearchQueryLanguage,
+        filters: mergedFilters,
+      });
+    } catch (e) {
+      console.log('Invalid syntax', JSON.stringify(e, null, 2)); // eslint-disable-line no-console
+      toasts.addError(e, {
+        title: i18n.translate('xpack.dataVisualizer.searchPanel.invalidSyntax', {
+          defaultMessage: 'Invalid syntax',
+        }),
+      });
+    }
+  };
+
+  return (
+    <SearchBar
+      dataTestSubj="dataVisualizerQueryInput"
+      appName={'dataVisualizer'}
+      showFilterBar={true}
+      showDatePicker={false}
+      showQueryInput={true}
+      query={searchInput}
+      onQuerySubmit={(params: { dateRange: TimeRange; query?: Query | undefined }) =>
+        searchHandler({ query: params.query })
+      }
+      onFiltersUpdated={(filters: Filter[]) => searchHandler({ filters })}
+      indexPatterns={[dataView]}
+      placeholder={i18n.translate('xpack.dataVisualizer.searchPanel.queryBarPlaceholderText', {
+        defaultMessage: 'Search… (e.g. status:200 AND extension:"PHP")',
+      })}
+      displayStyle={'inPage'}
+      isClearable={true}
+      customSubmitButton={<div />}
+    />
+  );
+};

x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/search_panel.tsx

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import React, { FC, useEffect, useState } from 'react';
+import React, { FC } from 'react';
 import { css } from '@emotion/react';
 import {
   useEuiBreakpoint,
@@ -14,16 +14,12 @@ import {
   EuiFlexGroup,
   EuiSpacer,
 } from '@elastic/eui';
-import { i18n } from '@kbn/i18n';
 import { Query, Filter } from '@kbn/es-query';
-import type { TimeRange } from '@kbn/es-query';
 import { DataView, DataViewField } from '@kbn/data-views-plugin/public';
-import { isDefined } from '@kbn/ml-is-defined';
+import { SearchQueryLanguage } from '@kbn/ml-query-utils';
+import { SearchPanelContent } from './search_bar';
 import { DataVisualizerFieldNamesFilter } from './field_name_filter';
 import { DataVisualizerFieldTypeFilter } from './field_type_filter';
-import { SearchQueryLanguage } from '../../types/combined_query';
-import { useDataVisualizerKibana } from '../../../kibana_context';
-import { createMergedEsQuery } from '../../utils/saved_search_utils';
 import { OverallStats } from '../../types/overall_stats';
 
 interface Props {
@@ -55,6 +51,7 @@ interface Props {
 export const SearchPanel: FC<Props> = ({
   dataView,
   searchString,
+  searchQuery,
   searchQueryLanguage,
   overallStats,
   indexedFieldTypes,
@@ -65,60 +62,6 @@ export const SearchPanel: FC<Props> = ({
   setSearchParams,
   showEmptyFields,
 }) => {
-  const {
-    services: {
-      uiSettings,
-      notifications: { toasts },
-      data: { query: queryManager },
-      unifiedSearch: {
-        ui: { SearchBar },
-      },
-    },
-  } = useDataVisualizerKibana();
-  // The internal state of the input query bar updated on every key stroke.
-  const [searchInput, setSearchInput] = useState<Query>({
-    query: searchString || '',
-    language: searchQueryLanguage,
-  });
-
-  useEffect(() => {
-    setSearchInput({
-      query: searchString || '',
-      language: searchQueryLanguage,
-    });
-  }, [searchQueryLanguage, searchString, queryManager.filterManager]);
-
-  const searchHandler = ({ query, filters }: { query?: Query; filters?: Filter[] }) => {
-    const mergedQuery = isDefined(query) ? query : searchInput;
-    const mergedFilters = isDefined(filters) ? filters : queryManager.filterManager.getFilters();
-    try {
-      if (mergedFilters) {
-        queryManager.filterManager.setFilters(mergedFilters);
-      }
-
-      const combinedQuery = createMergedEsQuery(
-        mergedQuery,
-        queryManager.filterManager.getFilters() ?? [],
-        dataView,
-        uiSettings
-      );
-
-      setSearchParams({
-        searchQuery: combinedQuery,
-        searchString: mergedQuery.query,
-        queryLanguage: mergedQuery.language as SearchQueryLanguage,
-        filters: mergedFilters,
-      });
-    } catch (e) {
-      console.log('Invalid syntax', JSON.stringify(e, null, 2)); // eslint-disable-line no-console
-      toasts.addError(e, {
-        title: i18n.translate('xpack.dataVisualizer.searchPanel.invalidSyntax', {
-          defaultMessage: 'Invalid syntax',
-        }),
-      });
-    }
-  };
-
   const dvSearchPanelControls = css({
     marginLeft: '0px !important',
     paddingLeft: '0px !important',
@@ -135,7 +78,6 @@ export const SearchPanel: FC<Props> = ({
       flexDirection: 'column',
     },
   });
-
   const dvSearchBar = css({
     [useEuiBreakpoint(['xs', 's', 'm', 'l'])]: {
       minWidth: `max(100%, 300px)`,
@@ -152,24 +94,12 @@ export const SearchPanel: FC<Props> = ({
       responsive={false}
     >
       <EuiFlexItem grow={9} css={dvSearchBar}>
-        <SearchBar
-          dataTestSubj="dataVisualizerQueryInput"
-          appName={'dataVisualizer'}
-          showFilterBar={true}
-          showDatePicker={false}
-          showQueryInput={true}
-          query={searchInput}
-          onQuerySubmit={(params: { dateRange: TimeRange; query?: Query | undefined }) =>
-            searchHandler({ query: params.query })
-          }
-          onFiltersUpdated={(filters: Filter[]) => searchHandler({ filters })}
-          indexPatterns={[dataView]}
-          placeholder={i18n.translate('xpack.dataVisualizer.searchPanel.queryBarPlaceholderText', {
-            defaultMessage: 'Search… (e.g. status:200 AND extension:"PHP")',
-          })}
-          displayStyle={'inPage'}
-          isClearable={true}
-          customSubmitButton={<div />}
+        <SearchPanelContent
+          dataView={dataView}
+          setSearchParams={setSearchParams}
+          searchString={searchString}
+          searchQuery={searchQuery}
+          searchQueryLanguage={searchQueryLanguage}
         />
       </EuiFlexItem>
 

x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts

@@ -10,7 +10,7 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
 import { merge } from 'rxjs';
 import type { EuiTableActionsColumnType } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
-import { type DataViewField, UI_SETTINGS } from '@kbn/data-plugin/common';
+import { type DataViewField } from '@kbn/data-plugin/common';
 import { ES_FIELD_TYPES, KBN_FIELD_TYPES } from '@kbn/field-types';
 import seedrandom from 'seedrandom';
 import type { SamplingOption } from '@kbn/discover-plugin/public/application/main/components/field_stats_table/field_stats_table';
@@ -19,6 +19,7 @@ import { mlTimefilterRefresh$, useTimefilter } from '@kbn/ml-date-picker';
 import useObservable from 'react-use/lib/useObservable';
 import type { KibanaExecutionContext } from '@kbn/core-execution-context-common';
 import { useExecutionContext } from '@kbn/kibana-react-plugin/public';
+import { useTimeBuckets } from '../../common/hooks/use_time_buckets';
 import { DATA_VISUALIZER_GRID_EMBEDDABLE_TYPE } from '../embeddables/grid_embeddable/constants';
 import { filterFields } from '../../common/components/fields_stats_grid/filter_fields';
 import type { RandomSamplerOption } from '../constants/random_sampler';
@@ -26,7 +27,6 @@ import type { DataVisualizerIndexBasedAppState } from '../types/index_data_visua
 import { useDataVisualizerKibana } from '../../kibana_context';
 import { getEsQueryFromSavedSearch } from '../utils/saved_search_utils';
 import type { MetricFieldsStats } from '../../common/components/stats_table/components/field_count_stats';
-import { TimeBuckets } from '../../../../common/services/time_buckets';
 import type { FieldVisConfig } from '../../common/components/stats_table/types';
 import {
   NON_AGGREGATABLE_FIELD_TYPES,
@@ -168,14 +168,7 @@ export const useDataVisualizerGridData = (
     lastRefresh,
   ]);
 
-  const _timeBuckets = useMemo(() => {
-    return new TimeBuckets({
-      [UI_SETTINGS.HISTOGRAM_MAX_BARS]: uiSettings.get(UI_SETTINGS.HISTOGRAM_MAX_BARS),
-      [UI_SETTINGS.HISTOGRAM_BAR_TARGET]: uiSettings.get(UI_SETTINGS.HISTOGRAM_BAR_TARGET),
-      dateFormat: uiSettings.get('dateFormat'),
-      'dateFormat:scaled': uiSettings.get('dateFormat:scaled'),
-    });
-  }, [uiSettings]);
+  const _timeBuckets = useTimeBuckets();
 
   const timefilter = useTimefilter({
     timeRangeSelector: currentDataView?.timeFieldName !== undefined,

x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts

@@ -8,8 +8,6 @@
 import { useCallback, useEffect, useState, useRef, useMemo, useReducer } from 'react';
 import { from, Subscription, Observable } from 'rxjs';
 import { mergeMap, last, map, toArray } from 'rxjs/operators';
-import { i18n } from '@kbn/i18n';
-import type { ToastsStart } from '@kbn/core/public';
 import { chunk } from 'lodash';
 import type {
   IKibanaSearchRequest,
@@ -38,6 +36,7 @@ import {
 import { getDocumentCountStats } from '../search_strategy/requests/get_document_stats';
 import { getInitialProgress, getReducer } from '../progress_utils';
 import { MAX_CONCURRENT_REQUESTS } from '../constants/index_data_visualizer_viewer';
+import { displayError } from '../../common/util/display_error';
 
 /**
  * Helper function to run forkJoin
@@ -63,32 +62,6 @@ export function rateLimitingForkJoin<T>(
   );
 }
 
-function displayError(toastNotifications: ToastsStart, index: string, err: any) {
-  if (err.statusCode === 500) {
-    toastNotifications.addError(err, {
-      title: i18n.translate('xpack.dataVisualizer.index.dataLoader.internalServerErrorMessage', {
-        defaultMessage:
-          'Error loading data in index {index}. {message}. ' +
-          'The request may have timed out. Try using a smaller sample size or narrowing the time range.',
-        values: {
-          index,
-          message: err.error ?? err.message,
-        },
-      }),
-    });
-  } else {
-    toastNotifications.addError(err, {
-      title: i18n.translate('xpack.dataVisualizer.index.errorLoadingDataMessage', {
-        defaultMessage: 'Error loading data in index {index}. {message}.',
-        values: {
-          index,
-          message: err.error ?? err.message,
-        },
-      }),
-    });
-  }
-}
-
 export function useOverallStats<TParams extends OverallStatsSearchStrategyParams>(
   searchStrategyParams: TParams | undefined,
   lastRefresh: number,

x-pack/plugins/data_visualizer/public/application/index_data_visualizer/locator/locator.ts

@@ -12,7 +12,7 @@ import type { RefreshInterval } from '@kbn/data-plugin/common';
 import { LocatorDefinition, LocatorPublic } from '@kbn/share-plugin/common';
 import { GlobalQueryStateFromUrl } from '@kbn/data-plugin/public';
 import { type Dictionary, isRisonSerializationRequired } from '@kbn/ml-url-state';
-import { SearchQueryLanguage } from '../types/combined_query';
+import { SearchQueryLanguage } from '@kbn/ml-query-utils';
 
 export const DATA_VISUALIZER_APP_LOCATOR = 'DATA_VISUALIZER_APP_LOCATOR';
 

x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/combined_query.ts

@@ -7,13 +7,6 @@
 
 import { Query } from '@kbn/es-query';
 
-export const SEARCH_QUERY_LANGUAGE = {
-  KUERY: 'kuery',
-  LUCENE: 'lucene',
-} as const;
-
-export type SearchQueryLanguage = typeof SEARCH_QUERY_LANGUAGE[keyof typeof SEARCH_QUERY_LANGUAGE];
-
 export interface CombinedQuery {
   searchString: Query['query'];
   searchQueryLanguage: string;

x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/index_data_visualizer_state.ts

@@ -7,8 +7,8 @@
 
 import type { Filter } from '@kbn/es-query';
 import type { Query } from '@kbn/data-plugin/common/query';
+import type { SearchQueryLanguage } from '@kbn/ml-query-utils';
 import type { RandomSamplerOption } from '../constants/random_sampler';
-import type { SearchQueryLanguage } from './combined_query';
 
 import type { DATA_VISUALIZER_INDEX_VIEWER } from '../constants/index_data_visualizer_viewer';
 

x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/storage.ts

@@ -7,7 +7,7 @@
 
 import { type FrozenTierPreference } from '@kbn/ml-date-picker';
 
-import { RandomSamplerOption } from '../constants/random_sampler';
+import { type RandomSamplerOption } from '../constants/random_sampler';
 
 export const DV_FROZEN_TIER_PREFERENCE = 'dataVisualizer.frozenDataTierPreference';
 export const DV_RANDOM_SAMPLER_PREFERENCE = 'dataVisualizer.randomSamplerPreference';
@@ -26,7 +26,7 @@ export type DVStorageMapped<T extends DVKey> = T extends typeof DV_FROZEN_TIER_P
   : T extends typeof DV_RANDOM_SAMPLER_PREFERENCE
   ? RandomSamplerOption | undefined
   : T extends typeof DV_RANDOM_SAMPLER_P_VALUE
-  ? number | undefined
+  ? number | null
   : null;
 
 export const DV_STORAGE_KEYS = [

x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts

@@ -23,23 +23,10 @@ import { DataView } from '@kbn/data-views-plugin/public';
 import type { SavedSearch } from '@kbn/saved-search-plugin/public';
 import { getEsQueryConfig, isQuery, SearchSource } from '@kbn/data-plugin/common';
 import { FilterManager, mapAndFlattenFilters } from '@kbn/data-plugin/public';
-import { SEARCH_QUERY_LANGUAGE, SearchQueryLanguage } from '../types/combined_query';
+import { getDefaultDSLQuery } from '@kbn/ml-query-utils';
+import { SEARCH_QUERY_LANGUAGE, SearchQueryLanguage } from '@kbn/ml-query-utils';
 import { isSavedSearchSavedObject, SavedSearchSavedObject } from '../../../../common/types';
 
-const DEFAULT_QUERY = {
-  bool: {
-    must: [
-      {
-        match_all: {},
-      },
-    ],
-  },
-};
-
-export function getDefaultQuery() {
-  return cloneDeep(DEFAULT_QUERY);
-}
-
 /**
  * Parse the stringified searchSourceJSON
  * from a saved search or saved search object
@@ -82,7 +69,7 @@ export function createMergedEsQuery(
   dataView?: DataView,
   uiSettings?: IUiSettingsClient
 ) {
-  let combinedQuery: QueryDslQueryContainer = getDefaultQuery();
+  let combinedQuery = getDefaultDSLQuery() as QueryDslQueryContainer;
 
   if (isQuery(query) && query.language === SEARCH_QUERY_LANGUAGE.KUERY) {
     const ast = fromKueryExpression(query.query);
@@ -158,7 +145,7 @@ export function getEsQueryFromSavedSearch({
     // Flattened query from search source may contain a clause that narrows the time range
     // which might interfere with global time pickers so we need to remove
     const savedQuery =
-      cloneDeep(savedSearch.searchSource.getSearchRequestBody()?.query) ?? getDefaultQuery();
+      cloneDeep(savedSearch.searchSource.getSearchRequestBody()?.query) ?? getDefaultDSLQuery();
     const timeField = savedSearch.searchSource.getField('index')?.timeFieldName;
 
     if (Array.isArray(savedQuery.bool.filter) && timeField !== undefined) {

x-pack/plugins/data_visualizer/public/index.ts

@@ -17,6 +17,7 @@ export type {
   FileDataVisualizerSpec,
   IndexDataVisualizerSpec,
   IndexDataVisualizerViewProps,
+  DataComparisonSpec,
 } from './application';
 export type {
   GetAdditionalLinksParams,

x-pack/plugins/data_visualizer/public/lazy_load_bundle/component_wrapper.tsx

@@ -5,7 +5,15 @@
  * 2.0.
  */
 
-import React, { FC } from 'react';
+import React, { FC, Suspense } from 'react';
+import { EuiErrorBoundary, EuiSkeletonText } from '@elastic/eui';
+import type { DataComparisonDetectionAppStateProps } from '../application/data_comparison/data_comparison_app_state';
+
+const LazyWrapper: FC = ({ children }) => (
+  <EuiErrorBoundary>
+    <Suspense fallback={<EuiSkeletonText lines={3} />}>{children}</Suspense>
+  </EuiErrorBoundary>
+);
 
 const FileDataVisualizerComponent = React.lazy(
   () => import('../application/file_data_visualizer/file_data_visualizer')
@@ -18,3 +26,15 @@ export const FileDataVisualizerWrapper: FC = () => {
     </React.Suspense>
   );
 };
+
+const DataComparisonLazy = React.lazy(() => import('../application/data_comparison'));
+
+/**
+ * Lazy-wrapped ExplainLogRateSpikesAppState React component
+ * @param {ExplainLogRateSpikesAppStateProps}  props - properties specifying the data on which to run the analysis.
+ */
+export const DataComparison: FC<DataComparisonDetectionAppStateProps> = (props) => (
+  <LazyWrapper>
+    <DataComparisonLazy {...props} />
+  </LazyWrapper>
+);

x-pack/plugins/data_visualizer/public/lazy_load_bundle/index.ts

@@ -6,7 +6,11 @@
  */
 
 import { HttpSetup } from '@kbn/core/public';
-import type { FileDataVisualizerSpec, IndexDataVisualizerSpec } from '../application';
+import type {
+  DataComparisonSpec,
+  FileDataVisualizerSpec,
+  IndexDataVisualizerSpec,
+} from '../application';
 import { getCoreStart } from '../kibana_services';
 
 let loadModulesPromise: Promise<LazyLoadedModules>;
@@ -14,6 +18,7 @@ let loadModulesPromise: Promise<LazyLoadedModules>;
 interface LazyLoadedModules {
   FileDataVisualizer: FileDataVisualizerSpec;
   IndexDataVisualizer: IndexDataVisualizerSpec;
+  DataComparison: DataComparisonSpec;
   getHttp: () => HttpSetup;
 }
 

x-pack/plugins/data_visualizer/public/lazy_load_bundle/lazy/index.tsx

@@ -7,3 +7,4 @@
 
 export type { FileDataVisualizerSpec, IndexDataVisualizerSpec } from '../../application';
 export { FileDataVisualizer, IndexDataVisualizer } from '../../application';
+export { DataComparison } from '../component_wrapper';

x-pack/plugins/data_visualizer/public/plugin.ts

@@ -24,7 +24,11 @@ import type { LensPublicStart } from '@kbn/lens-plugin/public';
 import type { IndexPatternFieldEditorStart } from '@kbn/data-view-field-editor-plugin/public';
 import { FieldFormatsStart } from '@kbn/field-formats-plugin/public';
 import type { UiActionsStart } from '@kbn/ui-actions-plugin/public';
-import { getFileDataVisualizerComponent, getIndexDataVisualizerComponent } from './api';
+import {
+  getDataComparisonComponent,
+  getFileDataVisualizerComponent,
+  getIndexDataVisualizerComponent,
+} from './api';
 import { getMaxBytesFormatted } from './application/common/util/get_max_bytes';
 import { registerHomeAddData, registerHomeFeatureCatalogue } from './register_home';
 import { registerEmbeddables } from './application/index_data_visualizer/embeddables';
@@ -85,6 +89,7 @@ export class DataVisualizerPlugin
     return {
       getFileDataVisualizerComponent,
       getIndexDataVisualizerComponent,
+      getDataComparisonComponent,
       getMaxBytesFormatted,
     };
   }

x-pack/plugins/data_visualizer/tsconfig.json

@@ -13,6 +13,8 @@
   ],
   "kbn_references": [
     "@kbn/ace",
+    "@kbn/aiops-components",
+    "@kbn/aiops-utils",
     "@kbn/charts-plugin",
     "@kbn/cloud-chat-plugin",
     "@kbn/cloud-plugin",
@@ -50,6 +52,7 @@
     "@kbn/ml-data-grid",
     "@kbn/ml-error-utils",
     "@kbn/ml-kibana-theme",
+    "@kbn/ml-in-memory-table",
     "@kbn/react-field",
     "@kbn/rison",
     "@kbn/saved-search-plugin",
@@ -60,7 +63,11 @@
     "@kbn/unified-search-plugin",
     "@kbn/usage-collection-plugin",
     "@kbn/utility-types",
-    "@kbn/unified-field-list"
+    "@kbn/unified-field-list",
+    "@kbn/ml-string-hash",
+    "@kbn/ml-random-sampler-utils",
+    "@kbn/data-service",
+    "@kbn/core-notifications-browser"
   ],
   "exclude": [
     "target/**/*",

x-pack/plugins/ml/common/constants/locator.ts

@@ -15,6 +15,8 @@ export const ML_PAGES = {
   DATA_FRAME_ANALYTICS_SOURCE_SELECTION: 'data_frame_analytics/source_selection',
   DATA_FRAME_ANALYTICS_CREATE_JOB: 'data_frame_analytics/new_job',
   TRAINED_MODELS_MANAGE: 'trained_models',
+  DATA_COMPARISON_INDEX_SELECT: 'data_comparison_index_select',
+  DATA_COMPARISON: 'data_comparison',
   NODES: 'nodes',
   MEMORY_USAGE: 'memory_usage',
   DATA_FRAME_ANALYTICS_EXPLORATION: 'data_frame_analytics/exploration',

x-pack/plugins/ml/common/constants/search.ts

@@ -7,10 +7,3 @@
 
 export const ANNOTATIONS_TABLE_DEFAULT_QUERY_SIZE = 500;
 export const ANOMALIES_TABLE_DEFAULT_QUERY_SIZE = 500;
-
-export const SEARCH_QUERY_LANGUAGE = {
-  KUERY: 'kuery',
-  LUCENE: 'lucene',
-} as const;
-
-export type SearchQueryLanguage = typeof SEARCH_QUERY_LANGUAGE[keyof typeof SEARCH_QUERY_LANGUAGE];

x-pack/plugins/ml/common/types/locator.ts

@@ -8,10 +8,10 @@
 import type { SerializableRecord } from '@kbn/utility-types';
 import type { LocatorPublic } from '@kbn/share-plugin/public';
 import type { RefreshInterval, TimeRange } from '@kbn/data-plugin/common/query';
-import type { InfluencersFilterQuery } from '@kbn/ml-anomaly-utils';
 import type { DataFrameAnalysisConfigType } from '@kbn/ml-data-frame-analytics-utils';
+import type { InfluencersFilterQuery } from '@kbn/ml-anomaly-utils';
+import type { SearchQueryLanguage } from '@kbn/ml-query-utils';
 import type { JobId } from './anomaly_detection_jobs/job';
-import type { SearchQueryLanguage } from '../constants/search';
 import type { ListingPageUrlState } from './common';
 import { ML_PAGES } from '../constants/locator';
 
@@ -58,6 +58,8 @@ export type MlGenericUrlState = MLPageState<
   | typeof ML_PAGES.FILTER_LISTS_MANAGE
   | typeof ML_PAGES.FILTER_LISTS_NEW
   | typeof ML_PAGES.SETTINGS
+  | typeof ML_PAGES.DATA_COMPARISON
+  | typeof ML_PAGES.DATA_COMPARISON_INDEX_SELECT
   | typeof ML_PAGES.DATA_VISUALIZER
   | typeof ML_PAGES.DATA_VISUALIZER_FILE
   | typeof ML_PAGES.DATA_VISUALIZER_INDEX_SELECT
@@ -70,7 +72,6 @@ export type MlGenericUrlState = MLPageState<
   | typeof ML_PAGES.AIOPS_CHANGE_POINT_DETECTION,
   MlGenericUrlPageState | undefined
 >;
-
 export interface AnomalyDetectionQueryState {
   jobId?: JobId | string[];
   groupIds?: string[];

x-pack/plugins/ml/public/application/components/anomalies_table/links_menu.tsx

@@ -30,6 +30,7 @@ import {
   type MlAnomaliesTableRecord,
 } from '@kbn/ml-anomaly-utils';
 import { formatHumanReadableDateTimeSeconds, timeFormatter } from '@kbn/ml-date-utils';
+import { SEARCH_QUERY_LANGUAGE } from '@kbn/ml-query-utils';
 import { mlJobService } from '../../services/job_service';
 import { getDataViewIdFromName } from '../../util/index_utils';
 import { getInitialAnomaliesLayers, getInitialSourceIndexFieldLayers } from '../../../maps/util';
@@ -38,7 +39,6 @@ import { ml } from '../../services/ml_api_service';
 import { escapeKueryForFieldValuePair, replaceStringTokens } from '../../util/string_utils';
 import { getUrlForRecord, openCustomUrlWindow } from '../../util/custom_url_utils';
 import { ML_APP_LOCATOR, ML_PAGES } from '../../../../common/constants/locator';
-import { SEARCH_QUERY_LANGUAGE } from '../../../../common/constants/search';
 // @ts-ignore
 import {
   escapeDoubleQuotes,

x-pack/plugins/ml/public/application/components/ml_page/side_nav.tsx

@@ -234,6 +234,15 @@ export function useSideNavItems(activeRoute: MlRoute | undefined) {
             disabled: false,
             testSubj: 'mlMainTab indexDataVisualizer',
           },
+          {
+            id: 'data_comparison',
+            pathId: ML_PAGES.DATA_COMPARISON_INDEX_SELECT,
+            name: i18n.translate('xpack.ml.navMenu.dataComparisonText', {
+              defaultMessage: 'Data Comparison',
+            }),
+            disabled: disableLinks,
+            testSubj: 'mlMainTab dataComparison',
+          },
         ],
       },
     ];

x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step_form.tsx

@@ -34,6 +34,7 @@ import {
   TRAINING_PERCENT_MAX,
 } from '@kbn/ml-data-frame-analytics-utils';
 import { DataGrid } from '@kbn/ml-data-grid';
+import { SEARCH_QUERY_LANGUAGE } from '@kbn/ml-query-utils';
 import { useMlKibana } from '../../../../../contexts/kibana';
 import {
   EuiComboBoxWithFieldStats,
@@ -62,7 +63,6 @@ import { fetchExplainData } from '../shared';
 import { useIndexData } from '../../hooks';
 import { ExplorationQueryBar } from '../../../analytics_exploration/components/exploration_query_bar';
 import { useSavedSearch, SavedSearchQuery } from './use_saved_search';
-import { SEARCH_QUERY_LANGUAGE } from '../../../../../../../common/constants/search';
 import { ExplorationQueryBarProps } from '../../../analytics_exploration/components/exploration_query_bar/exploration_query_bar';
 
 import { ScatterplotMatrix } from '../../../../../components/scatterplot_matrix';

x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/use_saved_search.ts

@@ -15,9 +15,9 @@ import {
   Query,
   toElasticsearchQuery,
 } from '@kbn/es-query';
+import { SEARCH_QUERY_LANGUAGE } from '@kbn/ml-query-utils';
 import { useMlKibana } from '../../../../../contexts/kibana';
 import { useDataSource } from '../../../../../contexts/ml';
-import { SEARCH_QUERY_LANGUAGE } from '../../../../../../../common/constants/search';
 
 // `undefined` is used for a non-initialized state
 // `null` is set if no saved search is used

x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/expandable_section/expandable_section_results.tsx

@@ -43,7 +43,7 @@ import {
 } from '@kbn/ml-data-frame-analytics-utils';
 
 import * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
-import { SEARCH_QUERY_LANGUAGE } from '../../../../../../../common/constants/search';
+import { SEARCH_QUERY_LANGUAGE } from '@kbn/ml-query-utils';
 
 import { getToastNotifications } from '../../../../../util/dependency_cache';
 import { useColorRange, ColorRangeLegend } from '../../../../../components/color_range_legend';

x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_query_bar/exploration_query_bar.tsx

@@ -18,11 +18,8 @@ import type { Query } from '@kbn/es-query';
 import { QueryStringInput } from '@kbn/unified-search-plugin/public';
 import { QueryErrorMessage } from '@kbn/ml-error-utils';
 
+import { SEARCH_QUERY_LANGUAGE, SearchQueryLanguage } from '@kbn/ml-query-utils';
 import { Dictionary } from '../../../../../../../common/types/common';
-import {
-  SEARCH_QUERY_LANGUAGE,
-  SearchQueryLanguage,
-} from '../../../../../../../common/constants/search';
 import { removeFilterFromQueryString } from '../../../../../explorer/explorer_utils';
 import { useMlKibana } from '../../../../../contexts/kibana';
 

x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/hooks/use_exploration_url_state.ts

@@ -7,9 +7,9 @@
 
 import { isPopulatedObject } from '@kbn/ml-is-populated-object';
 import { usePageUrlState } from '@kbn/ml-url-state';
+import { SEARCH_QUERY_LANGUAGE } from '@kbn/ml-query-utils';
 import { ML_PAGES } from '../../../../../../common/constants/locator';
 import { ExplorationPageUrlState } from '../../../../../../common/types/locator';
-import { SEARCH_QUERY_LANGUAGE } from '../../../../../../common/constants/search';
 
 export function getDefaultExplorationPageUrlState(
   overrides?: Partial<ExplorationPageUrlState>

x-pack/plugins/ml/public/application/datavisualizer/data_comparison/data_comparison_page.tsx

@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React, { FC, useEffect, useState } from 'react';
+import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui';
+import { FormattedMessage } from '@kbn/i18n-react';
+import type { DataComparisonSpec } from '@kbn/data-visualizer-plugin/public';
+import { useMlKibana } from '../../contexts/kibana';
+import { useDataSource } from '../../contexts/ml';
+import { MlPageHeader } from '../../components/page_header';
+import { TechnicalPreviewBadge } from '../../components/technical_preview_badge';
+
+export const DataComparisonPage: FC = () => {
+  const {
+    services: { dataVisualizer },
+  } = useMlKibana();
+
+  const [DataComparisonView, setDataComparisonView] = useState<DataComparisonSpec | null>(null);
+
+  useEffect(() => {
+    if (dataVisualizer !== undefined) {
+      const { getDataComparisonComponent } = dataVisualizer;
+      getDataComparisonComponent().then(setDataComparisonView);
+    }
+  }, [dataVisualizer]);
+
+  const { selectedDataView: dataView, selectedSavedSearch: savedSearch } = useDataSource();
+
+  return (
+    <>
+      <MlPageHeader>
+        <EuiFlexGroup responsive={false} wrap={false} alignItems={'center'} gutterSize={'m'}>
+          <EuiFlexItem grow={false}>
+            <FormattedMessage
+              id="xpack.ml.dataComparisonWithDocCount.pageHeader"
+              defaultMessage="Data comparison"
+            />
+          </EuiFlexItem>
+          <EuiFlexItem grow={false}>
+            <TechnicalPreviewBadge />
+          </EuiFlexItem>
+        </EuiFlexGroup>
+      </MlPageHeader>
+      {dataView && DataComparisonView ? (
+        <DataComparisonView dataView={dataView} savedSearch={savedSearch} />
+      ) : null}
+    </>
+  );
+};

x-pack/plugins/ml/public/application/explorer/anomaly_context_menu.tsx

@@ -21,9 +21,9 @@ import useObservable from 'react-use/lib/useObservable';
 import type { Query, TimeRange } from '@kbn/es-query';
 import { isDefined } from '@kbn/ml-is-defined';
 import { useTimeRangeUpdates } from '@kbn/ml-date-picker';
+import { SEARCH_QUERY_LANGUAGE } from '@kbn/ml-query-utils';
 import { useAnomalyExplorerContext } from './anomaly_explorer_context';
 import { escapeKueryForFieldValuePair } from '../util/string_utils';
-import { SEARCH_QUERY_LANGUAGE } from '../../../common/constants/search';
 import { useCasesModal } from '../contexts/kibana/use_cases_modal';
 import { DEFAULT_MAX_SERIES_TO_PLOT } from '../services/anomaly_explorer_charts_service';
 import { ANOMALY_EXPLORER_CHARTS_EMBEDDABLE_TYPE } from '../../embeddables';

x-pack/plugins/ml/public/application/explorer/anomaly_timeline.tsx

@@ -30,7 +30,7 @@ import type { Query } from '@kbn/es-query';
 import { formatHumanReadableDateTime } from '@kbn/ml-date-utils';
 import { isDefined } from '@kbn/ml-is-defined';
 import { useTimeRangeUpdates } from '@kbn/ml-date-picker';
-import { SEARCH_QUERY_LANGUAGE } from '../../../common/constants/search';
+import { SEARCH_QUERY_LANGUAGE } from '@kbn/ml-query-utils';
 import { useCasesModal } from '../contexts/kibana/use_cases_modal';
 import { ANOMALY_SWIMLANE_EMBEDDABLE_TYPE } from '../..';
 import {

x-pack/plugins/ml/public/application/explorer/components/explorer_query_bar/explorer_query_bar.tsx

@@ -14,7 +14,7 @@ import { QueryStringInput } from '@kbn/unified-search-plugin/public';
 import type { DataView } from '@kbn/data-views-plugin/common';
 import type { QueryErrorMessage } from '@kbn/ml-error-utils';
 import type { InfluencersFilterQuery } from '@kbn/ml-anomaly-utils';
-import { SEARCH_QUERY_LANGUAGE } from '../../../../../common/constants/search';
+import { SEARCH_QUERY_LANGUAGE } from '@kbn/ml-query-utils';
 import { useAnomalyExplorerContext } from '../../anomaly_explorer_context';
 import { useMlKibana } from '../../../contexts/kibana';
 

x-pack/plugins/ml/public/application/explorer/dashboard_controls/add_anomaly_charts_to_dashboard_controls.tsx

@@ -10,10 +10,10 @@ import { EuiFieldNumber, EuiFormRow, htmlIdGenerator } from '@elastic/eui';
 import type { Query } from '@kbn/es-query';
 import useObservable from 'react-use/lib/useObservable';
 import { isDefined } from '@kbn/ml-is-defined';
+import { SEARCH_QUERY_LANGUAGE } from '@kbn/ml-query-utils';
 import { getSelectionInfluencers } from '../explorer_utils';
 import { useAnomalyExplorerContext } from '../anomaly_explorer_context';
 import { escapeKueryForFieldValuePair } from '../../util/string_utils';
-import { SEARCH_QUERY_LANGUAGE } from '../../../../common/constants/search';
 import { useDashboardTable } from './use_dashboards_table';
 import { AddToDashboardControl } from './add_to_dashboard_controls';
 import { useAddToDashboardActions } from './use_add_to_dashboard_actions';

x-pack/plugins/ml/public/application/explorer/dashboard_controls/add_swimlane_to_dashboard_controls.tsx

@@ -17,7 +17,7 @@ import { FormattedMessage } from '@kbn/i18n-react';
 import { i18n } from '@kbn/i18n';
 import { DashboardAttributes } from '@kbn/dashboard-plugin/common';
 import type { Query } from '@kbn/es-query';
-import { SEARCH_QUERY_LANGUAGE } from '../../../../common/constants/search';
+import { SEARCH_QUERY_LANGUAGE } from '@kbn/ml-query-utils';
 import { getDefaultSwimlanePanelTitle } from '../../../embeddables/anomaly_swimlane/anomaly_swimlane_embeddable';
 import { SWIMLANE_TYPE, SwimlaneType } from '../explorer_constants';
 import { JobId } from '../../../../common/types/anomaly_detection_jobs';