[ML] Data vizualizer: add choropleth map for index and file (#99434)

* wip: add choropleth map to dataviz

* add choropleth map to datavisualizer index and file based

* fix translation

* make function name more clear

x-pack/plugins/file_data_visualizer/public/application/components/stats_table/components/field_data_expanded_row/choropleth_map.tsx

@@ -0,0 +1,126 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React, { FC, useMemo } from 'react';
+import { EuiFlexItem, EuiSpacer, EuiText, htmlIdGenerator } from '@elastic/eui';
+import { i18n } from '@kbn/i18n';
+import { FormattedMessage } from '@kbn/i18n/react';
+import {
+  FIELD_ORIGIN,
+  SOURCE_TYPES,
+  STYLE_TYPE,
+  COLOR_MAP_TYPE,
+} from '../../../../../../../maps/common/constants';
+import { EMSTermJoinConfig } from '../../../../../../../maps/public';
+import { FieldVisStats } from '../../types';
+import { VectorLayerDescriptor } from '../../../../../../../maps/common/descriptor_types';
+import { EmbeddedMapComponent } from '../../../embedded_map';
+
+export const getChoroplethTopValuesLayer = (
+  fieldName: string,
+  topValues: Array<{ key: any; doc_count: number }>,
+  { layerId, field }: EMSTermJoinConfig
+): VectorLayerDescriptor => {
+  return {
+    id: htmlIdGenerator()(),
+    label: i18n.translate('xpack.fileDataVisualizer.choroplethMap.topValuesCount', {
+      defaultMessage: 'Top values count for {fieldName}',
+      values: { fieldName },
+    }),
+    joins: [
+      {
+        // Left join is the id from the type of field (e.g. world_countries)
+        leftField: field,
+        right: {
+          id: 'anomaly_count',
+          type: SOURCE_TYPES.TABLE_SOURCE,
+          __rows: topValues,
+          __columns: [
+            {
+              name: 'key',
+              type: 'string',
+            },
+            {
+              name: 'doc_count',
+              type: 'number',
+            },
+          ],
+          // Right join/term is the field in the doc you’re trying to join it to (foreign key - e.g. US)
+          term: 'key',
+        },
+      },
+    ],
+    sourceDescriptor: {
+      type: 'EMS_FILE',
+      id: layerId,
+    },
+    style: {
+      type: 'VECTOR',
+      // @ts-ignore missing style properties. Remove once 'VectorLayerDescriptor' type is updated
+      properties: {
+        icon: { type: STYLE_TYPE.STATIC, options: { value: 'marker' } },
+        fillColor: {
+          type: STYLE_TYPE.DYNAMIC,
+          options: {
+            color: 'Blue to Red',
+            colorCategory: 'palette_0',
+            fieldMetaOptions: { isEnabled: true, sigma: 3 },
+            type: COLOR_MAP_TYPE.ORDINAL,
+            field: {
+              name: 'doc_count',
+              origin: FIELD_ORIGIN.JOIN,
+            },
+            useCustomColorRamp: false,
+          },
+        },
+        lineColor: {
+          type: STYLE_TYPE.DYNAMIC,
+          options: { fieldMetaOptions: { isEnabled: true } },
+        },
+        lineWidth: { type: STYLE_TYPE.STATIC, options: { size: 1 } },
+      },
+      isTimeAware: true,
+    },
+    type: 'VECTOR',
+  };
+};
+
+interface Props {
+  stats: FieldVisStats | undefined;
+  suggestion: EMSTermJoinConfig;
+}
+
+export const ChoroplethMap: FC<Props> = ({ stats, suggestion }) => {
+  const { fieldName, isTopValuesSampled, topValues, topValuesSamplerShardSize } = stats!;
+
+  const layerList: VectorLayerDescriptor[] = useMemo(
+    () => [getChoroplethTopValuesLayer(fieldName || '', topValues || [], suggestion)],
+    [suggestion, fieldName, topValues]
+  );
+
+  return (
+    <EuiFlexItem data-test-subj={'fileDataVisualizerChoroplethMapTopValues'}>
+      <div style={{ width: '100%', minHeight: 300 }}>
+        <EmbeddedMapComponent layerList={layerList} />
+      </div>
+      {isTopValuesSampled === true && (
+        <>
+          <EuiSpacer size="xs" />
+          <EuiText size="xs" textAlign={'left'}>
+            <FormattedMessage
+              id="xpack.fileDataVisualizer.fieldDataCardExpandedRow.choroplethMapTopValues.calculatedFromSampleDescription"
+              defaultMessage="Calculated from sample of {topValuesSamplerShardSize} documents per shard"
+              values={{
+                topValuesSamplerShardSize,
+              }}
+            />
+          </EuiText>
+        </>
+      )}
+    </EuiFlexItem>
+  );
+};

x-pack/plugins/file_data_visualizer/public/application/components/stats_table/components/field_data_expanded_row/keyword_content.tsx

@@ -5,21 +5,55 @@
  * 2.0.
  */
 
-import React, { FC } from 'react';
+import React, { FC, useCallback, useEffect, useState } from 'react';
 import type { FieldDataRowProps } from '../../types/field_data_row';
 import { TopValues } from '../../../top_values';
+import { EMSTermJoinConfig } from '../../../../../../../maps/public';
+import { useFileDataVisualizerKibana } from '../../../../kibana_context';
 import { DocumentStatsTable } from './document_stats';
 import { ExpandedRowContent } from './expanded_row_content';
+import { ChoroplethMap } from './choropleth_map';
+
+const COMMON_EMS_LAYER_IDS = [
+  'world_countries',
+  'administrative_regions_lvl2',
+  'usa_zip_codes',
+  'usa_states',
+];
 
 export const KeywordContent: FC<FieldDataRowProps> = ({ config }) => {
-  const { stats } = config;
+  const [EMSSuggestion, setEMSSuggestion] = useState<EMSTermJoinConfig | null | undefined>();
+  const { stats, fieldName } = config;
   const fieldFormat = 'fieldFormat' in config ? config.fieldFormat : undefined;
+  const {
+    services: { maps: mapsPlugin },
+  } = useFileDataVisualizerKibana();
+
+  const loadEMSTermSuggestions = useCallback(async () => {
+    if (!mapsPlugin) return;
+    const suggestion: EMSTermJoinConfig | null = await mapsPlugin.suggestEMSTermJoinConfig({
+      emsLayerIds: COMMON_EMS_LAYER_IDS,
+      sampleValues: Array.isArray(stats?.topValues)
+        ? stats?.topValues.map((value) => value.key)
+        : [],
+      sampleValuesColumnName: fieldName || '',
+    });
+    setEMSSuggestion(suggestion);
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [fieldName]);
+
+  useEffect(
+    function getInitialEMSTermSuggestion() {
+      loadEMSTermSuggestions();
+    },
+    [loadEMSTermSuggestions]
+  );
 
   return (
     <ExpandedRowContent dataTestSubj={'mlDVKeywordContent'}>
       <DocumentStatsTable config={config} />
-
       <TopValues stats={stats} fieldFormat={fieldFormat} barColor="secondary" />
+      {EMSSuggestion && stats && <ChoroplethMap stats={stats} suggestion={EMSSuggestion} />}
     </ExpandedRowContent>
   );
 };

x-pack/plugins/ml/common/constants/embeddable_map.ts

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const COMMON_EMS_LAYER_IDS = [
+  'world_countries',
+  'administrative_regions_lvl2',
+  'usa_zip_codes',
+  'usa_states',
+];

x-pack/plugins/ml/public/application/datavisualizer/index_based/components/field_data_row/choropleth_map/choropleth_map.tsx

@@ -0,0 +1,126 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React, { FC, useMemo } from 'react';
+import { EuiFlexItem, EuiSpacer, EuiText, htmlIdGenerator } from '@elastic/eui';
+import { i18n } from '@kbn/i18n';
+import { FormattedMessage } from '@kbn/i18n/react';
+import {
+  FIELD_ORIGIN,
+  SOURCE_TYPES,
+  STYLE_TYPE,
+  COLOR_MAP_TYPE,
+} from '../../../../../../../../maps/common/constants';
+import { EMSTermJoinConfig } from '../../../../../../../../maps/public';
+import { FieldVisStats } from '../../../../stats_table/types';
+import { VectorLayerDescriptor } from '../../../../../../../../maps/common/descriptor_types';
+import { MlEmbeddedMapComponent } from '../../../../../components/ml_embedded_map';
+
+export const getChoroplethTopValuesLayer = (
+  fieldName: string,
+  topValues: Array<{ key: any; doc_count: number }>,
+  { layerId, field }: EMSTermJoinConfig
+): VectorLayerDescriptor => {
+  return {
+    id: htmlIdGenerator()(),
+    label: i18n.translate('xpack.ml.dataviz.choroplethMap.topValuesCount', {
+      defaultMessage: 'Top values count for {fieldName}',
+      values: { fieldName },
+    }),
+    joins: [
+      {
+        // Left join is the id from the type of field (e.g. world_countries)
+        leftField: field,
+        right: {
+          id: 'anomaly_count',
+          type: SOURCE_TYPES.TABLE_SOURCE,
+          __rows: topValues,
+          __columns: [
+            {
+              name: 'key',
+              type: 'string',
+            },
+            {
+              name: 'doc_count',
+              type: 'number',
+            },
+          ],
+          // Right join/term is the field in the doc you’re trying to join it to (foreign key - e.g. US)
+          term: 'key',
+        },
+      },
+    ],
+    sourceDescriptor: {
+      type: 'EMS_FILE',
+      id: layerId,
+    },
+    style: {
+      type: 'VECTOR',
+      // @ts-ignore missing style properties. Remove once 'VectorLayerDescriptor' type is updated
+      properties: {
+        icon: { type: STYLE_TYPE.STATIC, options: { value: 'marker' } },
+        fillColor: {
+          type: STYLE_TYPE.DYNAMIC,
+          options: {
+            color: 'Blue to Red',
+            colorCategory: 'palette_0',
+            fieldMetaOptions: { isEnabled: true, sigma: 3 },
+            type: COLOR_MAP_TYPE.ORDINAL,
+            field: {
+              name: 'doc_count',
+              origin: FIELD_ORIGIN.JOIN,
+            },
+            useCustomColorRamp: false,
+          },
+        },
+        lineColor: {
+          type: STYLE_TYPE.DYNAMIC,
+          options: { fieldMetaOptions: { isEnabled: true } },
+        },
+        lineWidth: { type: STYLE_TYPE.STATIC, options: { size: 1 } },
+      },
+      isTimeAware: true,
+    },
+    type: 'VECTOR',
+  };
+};
+
+interface Props {
+  stats: FieldVisStats | undefined;
+  suggestion: EMSTermJoinConfig;
+}
+
+export const ChoroplethMap: FC<Props> = ({ stats, suggestion }) => {
+  const { fieldName, isTopValuesSampled, topValues, topValuesSamplerShardSize } = stats!;
+
+  const layerList: VectorLayerDescriptor[] = useMemo(
+    () => [getChoroplethTopValuesLayer(fieldName || '', topValues || [], suggestion)],
+    [suggestion, stats]
+  );
+
+  return (
+    <EuiFlexItem data-test-subj={'mlChoroplethMapTopValues'}>
+      <div style={{ width: '100%', minHeight: 300 }}>
+        <MlEmbeddedMapComponent layerList={layerList} />
+      </div>
+      {isTopValuesSampled === true && (
+        <>
+          <EuiSpacer size="xs" />
+          <EuiText size="xs" textAlign={'left'}>
+            <FormattedMessage
+              id="xpack.ml.fieldDataCard.choroplethMapTopValues.calculatedFromSampleDescription"
+              defaultMessage="Calculated from sample of {topValuesSamplerShardSize} documents per shard"
+              values={{
+                topValuesSamplerShardSize,
+              }}
+            />
+          </EuiText>
+        </>
+      )}
+    </EuiFlexItem>
+  );
+};

x-pack/plugins/ml/public/application/datavisualizer/index_based/components/field_data_row/choropleth_map/index.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export { ChoroplethMap } from './choropleth_map';

x-pack/plugins/ml/public/application/datavisualizer/stats_table/components/field_data_expanded_row/keyword_content.tsx

@@ -5,21 +5,50 @@
  * 2.0.
  */
 
-import React, { FC } from 'react';
+import React, { FC, useEffect, useState } from 'react';
 import type { FieldDataRowProps } from '../../types/field_data_row';
 import { TopValues } from '../../../index_based/components/field_data_row/top_values';
+import { ChoroplethMap } from '../../../index_based/components/field_data_row/choropleth_map';
+import { useMlKibana } from '../../../../../application/contexts/kibana';
+import { EMSTermJoinConfig } from '../../../../../../../maps/public';
+import { COMMON_EMS_LAYER_IDS } from '../../../../../../common/constants/embeddable_map';
 import { DocumentStatsTable } from './document_stats';
 import { ExpandedRowContent } from './expanded_row_content';
 
 export const KeywordContent: FC<FieldDataRowProps> = ({ config }) => {
-  const { stats } = config;
+  const [EMSSuggestion, setEMSSuggestion] = useState<EMSTermJoinConfig | null | undefined>();
+  const { stats, fieldName } = config;
   const fieldFormat = 'fieldFormat' in config ? config.fieldFormat : undefined;
+  const {
+    services: { maps: mapsPlugin },
+  } = useMlKibana();
+
+  const loadEMSTermSuggestions = async () => {
+    if (!mapsPlugin) return;
+    const suggestion: EMSTermJoinConfig | null = await mapsPlugin.suggestEMSTermJoinConfig({
+      emsLayerIds: COMMON_EMS_LAYER_IDS,
+      sampleValues: Array.isArray(stats?.topValues)
+        ? stats?.topValues.map((value) => value.key)
+        : [],
+      sampleValuesColumnName: fieldName || '',
+    });
+    setEMSSuggestion(suggestion);
+  };
+
+  useEffect(
+    function getInitialEMSTermSuggestion() {
+      loadEMSTermSuggestions();
+    },
+    [config?.fieldName]
+  );
 
   return (
     <ExpandedRowContent dataTestSubj={'mlDVKeywordContent'}>
       <DocumentStatsTable config={config} />
-
-      <TopValues stats={stats} fieldFormat={fieldFormat} barColor="secondary" />
+      {EMSSuggestion && stats && <ChoroplethMap stats={stats} suggestion={EMSSuggestion} />}
+      {EMSSuggestion === null && (
+        <TopValues stats={stats} fieldFormat={fieldFormat} barColor="secondary" />
+      )}
     </ExpandedRowContent>
   );
 };

x-pack/plugins/ml/public/application/explorer/anomalies_map.tsx

@@ -28,14 +28,9 @@ import { isDefined } from '../../../common/types/guards';
 import { MlEmbeddedMapComponent } from '../components/ml_embedded_map';
 import { EMSTermJoinConfig } from '../../../../maps/public';
 import { AnomaliesTableRecord } from '../../../common/types/anomalies';
+import { COMMON_EMS_LAYER_IDS } from '../../../common/constants/embeddable_map';
 
 const MAX_ENTITY_VALUES = 3;
-const COMMON_EMS_LAYER_IDS = [
-  'world_countries',
-  'administrative_regions_lvl2',
-  'usa_zip_codes',
-  'usa_states',
-];
 
 function getAnomalyRows(anomalies: AnomaliesTableRecord[], jobId: string) {
   const anomalyRows: Record<