[ML] Match Data Visualizer/Field stats table content with the popover (#140667)

src/plugins/discover/public/application/main/components/field_stats_table/field_stats_table.tsx

@@ -23,7 +23,7 @@ import { css } from '@emotion/react';
 import { useDiscoverServices } from '../../../../hooks/use_discover_services';
 import { FIELD_STATISTICS_LOADED } from './constants';
 import type { GetStateReturn } from '../../services/discover_state';
-import { AvailableFields$, DataRefetch$ } from '../../hooks/use_saved_search';
+import { AvailableFields$, DataRefetch$, DataTotalHits$ } from '../../hooks/use_saved_search';
 
 export interface DataVisualizerGridEmbeddableInput extends EmbeddableInput {
   dataView: DataView;
@@ -38,6 +38,7 @@ export interface DataVisualizerGridEmbeddableInput extends EmbeddableInput {
   onAddFilter?: (field: DataViewField | string, value: string, type: '+' | '-') => void;
   sessionId?: string;
   fieldsToFetch?: string[];
+  totalDocuments?: number;
 }
 export interface DataVisualizerGridEmbeddableOutput extends EmbeddableOutput {
   showDistributions?: boolean;
@@ -89,6 +90,7 @@ export interface FieldStatisticsTableProps {
   savedSearchRefetch$?: DataRefetch$;
   availableFields$?: AvailableFields$;
   searchSessionId?: string;
+  savedSearchDataTotalHits$?: DataTotalHits$;
 }
 
 export const FieldStatisticsTable = (props: FieldStatisticsTableProps) => {
@@ -104,6 +106,7 @@ export const FieldStatisticsTable = (props: FieldStatisticsTableProps) => {
     trackUiMetric,
     savedSearchRefetch$,
     searchSessionId,
+    savedSearchDataTotalHits$,
   } = props;
   const services = useDiscoverServices();
   const [embeddable, setEmbeddable] = useState<
@@ -157,6 +160,9 @@ export const FieldStatisticsTable = (props: FieldStatisticsTableProps) => {
         onAddFilter,
         sessionId: searchSessionId,
         fieldsToFetch: availableFields$?.getValue().fields,
+        totalDocuments: savedSearchDataTotalHits$
+          ? savedSearchDataTotalHits$.getValue()?.result
+          : undefined,
       });
       embeddable.reload();
     }
@@ -170,6 +176,7 @@ export const FieldStatisticsTable = (props: FieldStatisticsTableProps) => {
     onAddFilter,
     searchSessionId,
     availableFields$,
+    savedSearchDataTotalHits$,
   ]);
 
   useEffect(() => {

src/plugins/discover/public/application/main/components/layout/discover_main_content.tsx

@@ -180,6 +180,7 @@ export const DiscoverMainContent = ({
               onAddFilter={!isPlainRecord ? onAddFilter : undefined}
               trackUiMetric={trackUiMetric}
               savedSearchRefetch$={savedSearchRefetch$}
+              savedSearchDataTotalHits$={savedSearchData$.totalHits$}
             />
           )}
         </EuiFlexGroup>

x-pack/plugins/data_visualizer/common/types/field_request_config.ts

@@ -40,6 +40,7 @@ export interface GeoPointExample {
 }
 
 export interface FieldVisStats {
+  totalDocuments?: number;
   error?: Error;
   cardinality?: number;
   count?: number;

x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/index_based_expanded_row.tsx

@@ -29,16 +29,18 @@ export const IndexBasedDataVisualizerExpandedRow = ({
   dataView,
   combinedQuery,
   onAddFilter,
+  totalDocuments,
 }: {
   item: FieldVisConfig;
   dataView: DataView | undefined;
   combinedQuery: CombinedQuery;
+  totalDocuments?: number;
   /**
    * Callback to add a filter to filter bar
    */
   onAddFilter?: (field: DataViewField | string, value: string, type: '+' | '-') => void;
 }) => {
-  const config = item;
+  const config = { ...item, stats: { ...item.stats, totalDocuments } };
   const { loading, type, existsInDocs, fieldName } = config;
 
   function getCardContent() {

x-pack/plugins/data_visualizer/public/application/common/components/top_values/top_values.tsx

@@ -10,9 +10,9 @@ import {
   EuiFlexGroup,
   EuiFlexItem,
   EuiProgress,
-  EuiSpacer,
   EuiText,
   EuiButtonIcon,
+  EuiSpacer,
 } from '@elastic/eui';
 
 import { FormattedMessage } from '@kbn/i18n-react';
@@ -20,6 +20,9 @@ import { FormattedMessage } from '@kbn/i18n-react';
 import classNames from 'classnames';
 import { i18n } from '@kbn/i18n';
 import { DataViewField } from '@kbn/data-views-plugin/public';
+import { ES_FIELD_TYPES, KBN_FIELD_TYPES } from '@kbn/data-plugin/common';
+import { css } from '@emotion/react';
+import { useDataVisualizerKibana } from '../../../kibana_context';
 import { roundToDecimalPlace, kibanaFieldFormat } from '../utils';
 import { ExpandedRowFieldHeader } from '../stats_table/components/expanded_row_field_header';
 import { FieldVisStats } from '../../../../../common/types';
@@ -43,17 +46,78 @@ function getPercentLabel(docCount: number, topValuesSampleSize: number): string
 }
 
 export const TopValues: FC<Props> = ({ stats, fieldFormat, barColor, compressed, onAddFilter }) => {
+  const {
+    services: { data },
+  } = useDataVisualizerKibana();
+
+  const { fieldFormats } = data;
+
   if (stats === undefined || !stats.topValues) return null;
   const {
     topValues,
     topValuesSampleSize,
-    topValuesSamplerShardSize,
     count,
     isTopValuesSampled,
     fieldName,
+    sampleCount,
+    topValuesSamplerShardSize,
   } = stats;
 
+  const totalDocuments = stats.totalDocuments;
+
   const progressBarMax = isTopValuesSampled === true ? topValuesSampleSize : count;
+
+  const topValuesOtherCount =
+    (progressBarMax ?? 0) -
+    (topValues ? topValues.map((value) => value.doc_count).reduce((v, acc) => acc + v) : 0);
+
+  const countsElement =
+    totalDocuments !== undefined ? (
+      <EuiText color="subdued" size="xs">
+        {isTopValuesSampled ? (
+          <FormattedMessage
+            id="xpack.dataVisualizer.dataGrid.field.topValues.calculatedFromSampleRecordsLabel"
+            defaultMessage="Calculated from {sampledDocumentsFormatted} sample {sampledDocuments, plural, one {record} other {records}}."
+            values={{
+              sampledDocuments: sampleCount,
+              sampledDocumentsFormatted: (
+                <strong>
+                  {fieldFormats
+                    .getDefaultInstance(KBN_FIELD_TYPES.NUMBER, [ES_FIELD_TYPES.INTEGER])
+                    .convert(sampleCount)}
+                </strong>
+              ),
+            }}
+          />
+        ) : (
+          <FormattedMessage
+            id="xpack.dataVisualizer.dataGrid.field.topValues.calculatedFromTotalRecordsLabel"
+            defaultMessage="Calculated from {totalDocumentsFormatted} {totalDocuments, plural, one {record} other {records}}."
+            values={{
+              totalDocuments,
+              totalDocumentsFormatted: (
+                <strong>
+                  {fieldFormats
+                    .getDefaultInstance(KBN_FIELD_TYPES.NUMBER, [ES_FIELD_TYPES.INTEGER])
+                    .convert(totalDocuments ?? 0)}
+                </strong>
+              ),
+            }}
+          />
+        )}
+      </EuiText>
+    ) : (
+      <EuiText size="xs" textAlign={'center'}>
+        <FormattedMessage
+          id="xpack.dataVisualizer.dataGrid.field.topValues.calculatedFromSampleDescription"
+          defaultMessage="Calculated from sample of {topValuesSamplerShardSize} documents per shard"
+          values={{
+            topValuesSamplerShardSize,
+          }}
+        />
+      </EuiText>
+    );
+
   return (
     <ExpandedRowPanel
       dataTestSubj={'dataVisualizerFieldDataTopValues'}
@@ -70,96 +134,125 @@ export const TopValues: FC<Props> = ({ stats, fieldFormat, barColor, compressed,
         data-test-subj="dataVisualizerFieldDataTopValuesContent"
         className={classNames('fieldDataTopValuesContainer', 'dvTopValues__wrapper')}
       >
-        {Array.isArray(topValues) &&
+        {Array.isArray(topValues)
-          topValues.map((value) => (
+          ? topValues.map((value) => (
-            <EuiFlexGroup gutterSize="xs" alignItems="center" key={value.key}>
+              <EuiFlexGroup gutterSize="xs" alignItems="center" key={value.key}>
-              <EuiFlexItem data-test-subj="dataVisualizerFieldDataTopValueBar">
+                <EuiFlexItem data-test-subj="dataVisualizerFieldDataTopValueBar">
-                <EuiProgress
+                  <EuiProgress
-                  value={value.doc_count}
+                    value={value.doc_count}
-                  max={progressBarMax}
+                    max={progressBarMax}
-                  color={barColor}
+                    color={barColor}
-                  size="xs"
+                    size="xs"
-                  label={kibanaFieldFormat(value.key, fieldFormat)}
+                    label={kibanaFieldFormat(value.key, fieldFormat)}
-                  className={classNames('eui-textTruncate', 'topValuesValueLabelContainer')}
+                    className={classNames('eui-textTruncate', 'topValuesValueLabelContainer')}
-                  valueText={`${value.doc_count}${
+                    valueText={`${value.doc_count}${
-                    progressBarMax !== undefined
+                      progressBarMax !== undefined
-                      ? ` (${getPercentLabel(value.doc_count, progressBarMax)})`
+                        ? ` (${getPercentLabel(value.doc_count, progressBarMax)})`
-                      : ''
+                        : ''
-                  }`}
+                    }`}
-                />
-              </EuiFlexItem>
-              {fieldName !== undefined && value.key !== undefined && onAddFilter !== undefined ? (
-                <>
-                  <EuiButtonIcon
-                    iconSize="s"
-                    iconType="plusInCircle"
-                    onClick={() =>
-                      onAddFilter(
-                        fieldName,
-                        typeof value.key === 'number' ? value.key.toString() : value.key,
-                        '+'
-                      )
-                    }
-                    aria-label={i18n.translate(
-                      'xpack.dataVisualizer.dataGrid.field.addFilterAriaLabel',
-                      {
-                        defaultMessage: 'Filter for {fieldName}: "{value}"',
-                        values: { fieldName, value: value.key },
-                      }
-                    )}
-                    data-test-subj={`dvFieldDataTopValuesAddFilterButton-${value.key}-${value.key}`}
-                    style={{
-                      minHeight: 'auto',
-                      minWidth: 'auto',
-                      paddingRight: 2,
-                      paddingLeft: 2,
-                      paddingTop: 0,
-                      paddingBottom: 0,
-                    }}
                   />
-                  <EuiButtonIcon
+                </EuiFlexItem>
-                    iconSize="s"
+                {fieldName !== undefined && value.key !== undefined && onAddFilter !== undefined ? (
-                    iconType="minusInCircle"
+                  <div
-                    onClick={() =>
+                    css={css`
-                      onAddFilter(
+                      width: 48px;
-                        fieldName,
+                    `}
-                        typeof value.key === 'number' ? value.key.toString() : value.key,
+                  >
-                        '-'
+                    <EuiButtonIcon
-                      )
+                      iconSize="s"
-                    }
+                      iconType="plusInCircle"
-                    aria-label={i18n.translate(
+                      onClick={() =>
-                      'xpack.dataVisualizer.dataGrid.field.removeFilterAriaLabel',
+                        onAddFilter(
-                      {
+                          fieldName,
-                        defaultMessage: 'Filter out {fieldName}: "{value}"',
+                          typeof value.key === 'number' ? value.key.toString() : value.key,
-                        values: { fieldName, value: value.key },
+                          '+'
+                        )
                       }
-                    )}
+                      aria-label={i18n.translate(
-                    data-test-subj={`dvFieldDataTopValuesExcludeFilterButton-${value.key}-${value.key}`}
+                        'xpack.dataVisualizer.dataGrid.field.addFilterAriaLabel',
-                    style={{
+                        {
-                      minHeight: 'auto',
+                          defaultMessage: 'Filter for {fieldName}: "{value}"',
-                      minWidth: 'auto',
+                          values: { fieldName, value: value.key },
-                      paddingTop: 0,
+                        }
-                      paddingBottom: 0,
+                      )}
-                      paddingRight: 2,
+                      data-test-subj={`dvFieldDataTopValuesAddFilterButton-${value.key}-${value.key}`}
-                      paddingLeft: 2,
+                      style={{
-                    }}
+                        minHeight: 'auto',
+                        minWidth: 'auto',
+                        paddingRight: 2,
+                        paddingLeft: 2,
+                        paddingTop: 0,
+                        paddingBottom: 0,
+                      }}
+                    />
+                    <EuiButtonIcon
+                      iconSize="s"
+                      iconType="minusInCircle"
+                      onClick={() =>
+                        onAddFilter(
+                          fieldName,
+                          typeof value.key === 'number' ? value.key.toString() : value.key,
+                          '-'
+                        )
+                      }
+                      aria-label={i18n.translate(
+                        'xpack.dataVisualizer.dataGrid.field.removeFilterAriaLabel',
+                        {
+                          defaultMessage: 'Filter out {fieldName}: "{value}"',
+                          values: { fieldName, value: value.key },
+                        }
+                      )}
+                      data-test-subj={`dvFieldDataTopValuesExcludeFilterButton-${value.key}-${value.key}`}
+                      style={{
+                        minHeight: 'auto',
+                        minWidth: 'auto',
+                        paddingTop: 0,
+                        paddingBottom: 0,
+                        paddingRight: 2,
+                        paddingLeft: 2,
+                      }}
+                    />
+                  </div>
+                ) : null}
+              </EuiFlexGroup>
+            ))
+          : null}
+        {topValuesOtherCount > 0 ? (
+          <EuiFlexGroup gutterSize="xs" alignItems="center" key="other">
+            <EuiFlexItem data-test-subj="dataVisualizerFieldDataTopValueBar">
+              <EuiProgress
+                value={topValuesOtherCount}
+                max={progressBarMax}
+                color={barColor}
+                size="xs"
+                label={
+                  <FormattedMessage
+                    id="xpack.dataVisualizer.dataGrid.field.topValuesOtherLabel"
+                    defaultMessage="Other"
                   />
-                </>
+                }
-              ) : null}
+                className={classNames('eui-textTruncate', 'topValuesValueLabelContainer')}
-            </EuiFlexGroup>
+                valueText={`${topValuesOtherCount}${
-          ))}
+                  progressBarMax !== undefined
+                    ? ` (${getPercentLabel(topValuesOtherCount, progressBarMax)})`
+                    : ''
+                }`}
+              />
+            </EuiFlexItem>
+            {onAddFilter ? (
+              <div
+                css={css`
+                  width: 48px;
+                `}
+              />
+            ) : null}
+          </EuiFlexGroup>
+        ) : null}
+
         {isTopValuesSampled === true && (
           <Fragment>
             <EuiSpacer size="xs" />
-            <EuiText size="xs" textAlign={'center'}>
+            {countsElement}
-              <FormattedMessage
-                id="xpack.dataVisualizer.dataGrid.field.topValues.calculatedFromSampleDescription"
-                defaultMessage="Calculated from sample of {topValuesSamplerShardSize} documents per shard"
-                values={{
-                  topValuesSamplerShardSize,
-                }}
-              />
-            </EuiText>
           </Fragment>
         )}
       </div>

x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx

@@ -419,13 +419,14 @@ export const IndexDataVisualizerView: FC<IndexDataVisualizerViewProps> = (dataVi
               dataView={currentDataView}
               combinedQuery={{ searchQueryLanguage, searchString }}
               onAddFilter={onAddFilter}
+              totalDocuments={overallStats.totalCount}
             />
           );
         }
         return m;
       }, {} as ItemIdToExpandedRowMap);
     },
-    [currentDataView, searchQueryLanguage, searchString, onAddFilter]
+    [currentDataView, searchQueryLanguage, searchString, onAddFilter, overallStats.totalCount]
   );
 
   // Some actions open up fly-out or popup

x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx

@@ -54,6 +54,7 @@ export interface DataVisualizerGridInput {
   onAddFilter?: (field: DataViewField | string, value: string, type: '+' | '-') => void;
   sessionId?: string;
   fieldsToFetch?: string[];
+  totalDocuments?: number;
 }
 export type DataVisualizerGridEmbeddableInput = EmbeddableInput & DataVisualizerGridInput;
 export type DataVisualizerGridEmbeddableOutput = EmbeddableOutput;
@@ -100,6 +101,7 @@ export const EmbeddableWrapper = ({
               dataView={input.dataView}
               combinedQuery={{ searchQueryLanguage, searchString }}
               onAddFilter={input.onAddFilter}
+              totalDocuments={input.totalDocuments}
             />
           );
         }

x-pack/test/functional/apps/ml/data_visualizer/index_data_visualizer_data_view_management.ts

@@ -62,7 +62,7 @@ export default function ({ getService }: FtrProviderContext) {
           existsInDocs: true,
           aggregatable: true,
           loading: false,
-          exampleCount: 10,
+          exampleCount: 11,
           docCountFormatted: '5000 (100%)',
           viewableInLens: true,
           hasActionMenu: true,
@@ -94,7 +94,7 @@ export default function ({ getService }: FtrProviderContext) {
           loading: false,
           docCountFormatted: '5000 (100%)',
           statsMaxDecimalPlaces: 3,
-          topValuesCount: 10,
+          topValuesCount: 11,
           viewableInLens: true,
           hasActionMenu: false,
         },

x-pack/test/functional/apps/ml/data_visualizer/index_test_data.ts

@@ -29,7 +29,7 @@ export const farequoteDataViewTestData: TestData = {
         loading: false,
         docCountFormatted: '5000 (100%)',
         statsMaxDecimalPlaces: 3,
-        topValuesCount: 10,
+        topValuesCount: 11,
         viewableInLens: true,
       },
     ],
@@ -70,7 +70,7 @@ export const farequoteDataViewTestData: TestData = {
         existsInDocs: true,
         aggregatable: true,
         loading: false,
-        exampleCount: 10,
+        exampleCount: 11,
         docCountFormatted: '5000 (100%)',
         viewableInLens: true,
       },
@@ -126,7 +126,7 @@ export const farequoteKQLSearchTestData: TestData = {
         loading: false,
         docCountFormatted: '5000 (100%)',
         statsMaxDecimalPlaces: 3,
-        topValuesCount: 10,
+        topValuesCount: 11,
         viewableInLens: true,
       },
     ],
@@ -224,7 +224,7 @@ export const farequoteKQLFiltersSearchTestData: TestData = {
         loading: false,
         docCountFormatted: '5000 (100%)',
         statsMaxDecimalPlaces: 3,
-        topValuesCount: 10,
+        topValuesCount: 11,
         viewableInLens: true,
       },
     ],
@@ -322,7 +322,7 @@ export const farequoteLuceneSearchTestData: TestData = {
         loading: false,
         docCountFormatted: '5000 (100%)',
         statsMaxDecimalPlaces: 3,
-        topValuesCount: 10,
+        topValuesCount: 11,
         viewableInLens: true,
       },
     ],
@@ -420,7 +420,7 @@ export const farequoteLuceneFiltersSearchTestData: TestData = {
         loading: false,
         docCountFormatted: '5000 (100%)',
         statsMaxDecimalPlaces: 3,
-        topValuesCount: 10,
+        topValuesCount: 11,
         viewableInLens: true,
       },
     ],