[ML] Geo job from dashboard map visualization: correctly include all filters/queries in job config (#148989)

## Summary

Related issue: https://github.com/elastic/kibana/issues/147567

This PR
- adds job creation compatibility check for map visualizations in
dashboard
- ensures filters/queries (dashboard, visualization, layer-level for
maps) are correctly added to datafeed
- fixes the way queries were merged


### Checklist

Delete any items that are not applicable to this PR.

- [ ] Any text added follows [EUI's writing
guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses
sentence case text and includes [i18n
support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md)
- [ ]
[Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html)
was added for features that require explanation or tutorials
- [ ] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [ ] Any UI touched in this PR is usable by keyboard only (learn more
about [keyboard accessibility](https://webaim.org/techniques/keyboard/))
- [ ] Any UI touched in this PR does not create any new axe failures
(run axe in browser:
[FF](https://addons.mozilla.org/en-US/firefox/addon/axe-devtools/),
[Chrome](https://chrome.google.com/webstore/detail/axe-web-accessibility-tes/lhdoppojpmngadmnindnejefpokejbdd?hl=en-US))
- [ ] If a plugin configuration key changed, check if it needs to be
allowlisted in the cloud and added to the [docker
list](https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker)
- [ ] This renders correctly on smaller devices using a responsive
layout. (You can test this [in your
browser](https://www.browserstack.com/guide/responsive-testing-on-local-server))
- [ ] This was checked for [cross-browser
compatibility](https://www.elastic.co/support/matrix#matrix_browsers)

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: James Gowdy <jgowdy@elastic.co>

src/plugins/dashboard/public/index.ts

@@ -15,6 +15,7 @@ export {
   LEGACY_DASHBOARD_APP_ID,
 } from './dashboard_constants';
 export { DASHBOARD_CONTAINER_TYPE } from './dashboard_container';
+export type { DashboardContainer } from './dashboard_container/embeddable/dashboard_container';
 export type { DashboardSetup, DashboardStart, DashboardFeatureFlagConfig } from './plugin';
 
 export {

x-pack/plugins/ml/public/application/jobs/new_job/job_from_dashboard/quick_create_job_base.ts

@@ -6,7 +6,7 @@
  */
 
 import { i18n } from '@kbn/i18n';
-import { mergeWith, uniqBy, isEqual } from 'lodash';
+import { mergeWith, uniqWith, isEqual } from 'lodash';
 import type { IUiSettingsClient } from '@kbn/core/public';
 import type { TimefilterContract } from '@kbn/data-plugin/public';
 import type { SharePluginStart } from '@kbn/share-plugin/public';
@@ -47,6 +47,16 @@ export interface CreateState {
   datafeedStarted: CreationState;
 }
 
+function mergeQueriesCheck(
+  objValue: estypes.QueryDslQueryContainer,
+  srcValue: estypes.QueryDslQueryContainer
+) {
+  if (Array.isArray(objValue)) {
+    const combinedQuery = objValue.concat(srcValue);
+    return uniqWith(combinedQuery, isEqual);
+  }
+}
+
 export class QuickJobCreatorBase {
   constructor(
     protected readonly kibanaConfig: IUiSettingsClient,
@@ -176,8 +186,11 @@ export class QuickJobCreatorBase {
   protected combineQueriesAndFilters(
     dashboard: { query: Query; filters: Filter[] },
     vis: { query: Query; filters: Filter[] },
-    dataView: DataViewBase
+    dataView: DataViewBase,
+    layerQuery?: { query: Query; filters: Filter[] }
   ): estypes.QueryDslQueryContainer {
+    let mergedVisAndLayerQueries;
+
     const { combinedQuery: dashboardQueries } = createQueries(
       {
         query: dashboard.query,
@@ -196,15 +209,23 @@ export class QuickJobCreatorBase {
       this.kibanaConfig
     );
 
+    if (layerQuery) {
+      const { combinedQuery: layerQueries } = createQueries(
+        {
+          query: layerQuery.query,
+          filter: layerQuery.filters,
+        },
+        dataView,
+        this.kibanaConfig
+      );
+      // combine vis and layer queries if layer-level query exists
+      mergedVisAndLayerQueries = mergeWith(visQueries, layerQueries, mergeQueriesCheck);
+    }
+
     const mergedQueries = mergeWith(
       dashboardQueries,
-      visQueries,
-      (objValue: estypes.QueryDslQueryContainer, srcValue: estypes.QueryDslQueryContainer) => {
-        if (Array.isArray(objValue)) {
-          const combinedQuery = objValue.concat(srcValue);
-          return uniqBy(combinedQuery, isEqual);
-        }
-      }
+      mergedVisAndLayerQueries ? mergedVisAndLayerQueries : visQueries,
+      mergeQueriesCheck
     );
 
     return mergedQueries;

x-pack/plugins/ml/public/application/jobs/new_job/job_from_map/index.ts

@@ -7,5 +7,9 @@
 
 export { resolver } from './route_resolver';
 export { QuickGeoJobCreator } from './quick_create_job';
-export { getJobsItemsFromEmbeddable, redirectToGeoJobWizard } from './utils';
+export {
+  getJobsItemsFromEmbeddable,
+  isCompatibleMapVisualization,
+  redirectToGeoJobWizard,
+} from './utils';
 export { type LayerResult, VisualizationExtractor } from './visualization_extractor';

x-pack/plugins/ml/public/application/jobs/new_job/job_from_map/quick_create_job.ts

@@ -35,6 +35,7 @@ interface VisDescriptor {
   geoField: string;
   splitField: string | null;
   bucketSpan?: string;
+  layerLevelQuery?: Query | null;
 }
 
 export class QuickGeoJobCreator extends QuickJobCreatorBase {
@@ -57,6 +58,7 @@ export class QuickGeoJobCreator extends QuickJobCreatorBase {
     sourceDataView,
     geoField,
     splitField,
+    layerLevelQuery,
   }: {
     jobId: string;
     bucketSpan: string;
@@ -67,6 +69,7 @@ export class QuickGeoJobCreator extends QuickJobCreatorBase {
     sourceDataView?: DataView;
     geoField: string;
     splitField: string | null;
+    layerLevelQuery: Query | null;
   }): Promise<CreateState> {
     const {
       query: dashboardQuery,
@@ -92,6 +95,7 @@ export class QuickGeoJobCreator extends QuickJobCreatorBase {
       filters: dashboardFilters,
       embeddableQuery,
       embeddableFilters,
+      layerLevelQuery,
       geoField,
       splitField,
       bucketSpan,
@@ -120,7 +124,8 @@ export class QuickGeoJobCreator extends QuickJobCreatorBase {
     embeddableQuery: Query,
     embeddableFilters: Filter[],
     geoField: string,
-    splitField: string | null = null
+    splitField: string | null = null,
+    layerLevelQuery?: Query
   ) {
     try {
       const { jobConfig, datafeedConfig, start, end, includeTimeRange } = await this.createGeoJob({
@@ -133,6 +138,7 @@ export class QuickGeoJobCreator extends QuickJobCreatorBase {
         embeddableFilters,
         geoField,
         splitField,
+        layerLevelQuery,
       });
 
       // add job config and start and end dates to the
@@ -165,6 +171,7 @@ export class QuickGeoJobCreator extends QuickJobCreatorBase {
     filters,
     embeddableQuery,
     embeddableFilters,
+    layerLevelQuery,
     geoField,
     splitField,
     bucketSpan,
@@ -177,6 +184,7 @@ export class QuickGeoJobCreator extends QuickJobCreatorBase {
     filters: Filter[];
     embeddableQuery: Query;
     embeddableFilters: Filter[];
+    layerLevelQuery?: Query | null;
     geoField: string;
     splitField: string | null;
     bucketSpan?: string;
@@ -186,6 +194,7 @@ export class QuickGeoJobCreator extends QuickJobCreatorBase {
       dataViewId,
       dashboard: { query, filters },
       embeddable: { query: embeddableQuery, filters: embeddableFilters },
+      layerLevelQuery,
       geoField,
       splitField,
       ...(bucketSpan ? { bucketSpan } : {}),
@@ -234,6 +243,7 @@ export class QuickGeoJobCreator extends QuickJobCreatorBase {
     dataViewId,
     dashboard,
     embeddable,
+    layerLevelQuery,
     bucketSpan,
     geoField,
     splitField,
@@ -244,10 +254,12 @@ export class QuickGeoJobCreator extends QuickJobCreatorBase {
 
     const jobConfig = createEmptyJob();
     const datafeedConfig = createEmptyDatafeed(dataView.getIndexPattern());
+
     const combinedFiltersAndQueries = this.combineQueriesAndFilters(
       dashboard,
       embeddable,
-      dataView!
+      dataView!,
+      layerLevelQuery ? { query: layerLevelQuery, filters: [] } : undefined
     );
 
     datafeedConfig.query = combinedFiltersAndQueries;

x-pack/plugins/ml/public/application/jobs/new_job/job_from_map/route_resolver.ts

@@ -21,10 +21,12 @@ export async function resolver(
   geoField: string,
   splitField: string,
   fromRisonString: string,
-  toRisonString: string
+  toRisonString: string,
+  layer?: string
 ) {
   let decodedDashboard;
   let decodedEmbeddable;
+  let decodedLayer;
   let splitFieldDecoded;
   let dvId;
 
@@ -46,6 +48,14 @@ export async function resolver(
     decodedEmbeddable = { query: getDefaultQuery(), filters: [] };
   }
 
+  if (layer) {
+    try {
+      decodedLayer = rison.decode(layer) as { query: Query };
+    } catch (error) {
+      decodedLayer = { query: getDefaultQuery(), filters: [] };
+    }
+  }
+
   try {
     splitFieldDecoded = rison.decode(splitField) as string;
   } catch (error) {
@@ -76,6 +86,7 @@ export async function resolver(
     decodedEmbeddable.query,
     decodedEmbeddable.filters,
     geoField,
-    splitFieldDecoded
+    splitFieldDecoded,
+    decodedLayer?.query
   );
 }

x-pack/plugins/ml/public/application/jobs/new_job/job_from_map/utils.ts

@@ -6,6 +6,7 @@
  */
 
 import { i18n } from '@kbn/i18n';
+import type { Query } from '@kbn/es-query';
 import type { MapEmbeddable } from '@kbn/maps-plugin/public';
 import type { SharePluginStart } from '@kbn/share-plugin/public';
 import { ML_PAGES, ML_APP_LOCATOR } from '../../../../../common/constants/locator';
@@ -14,6 +15,7 @@ export async function redirectToGeoJobWizard(
   embeddable: MapEmbeddable,
   dataViewId: string,
   geoField: string,
+  layerQuery: Query | null,
   splitField: string | null,
   share: SharePluginStart
 ) {
@@ -30,6 +32,7 @@ export async function redirectToGeoJobWizard(
     splitField,
     from,
     to,
+    ...(layerQuery ? { layer: { query: layerQuery } } : {}),
   };
 
   const url = await locator?.getUrl({
@@ -40,6 +43,16 @@ export async function redirectToGeoJobWizard(
   window.open(url, '_blank');
 }
 
+export function isCompatibleMapVisualization(embeddable: MapEmbeddable) {
+  return embeddable.getLayerList().some((layer) => {
+    const geoField = layer.getGeoFieldNames().length ? layer.getGeoFieldNames()[0] : undefined;
+    const dataViewId = layer.getIndexPatternIds().length
+      ? layer.getIndexPatternIds()[0]
+      : undefined;
+    return geoField && dataViewId;
+  });
+}
+
 export async function getJobsItemsFromEmbeddable(embeddable: MapEmbeddable) {
   // Get dashboard level query/filters
   const { filters, timeRange, ...input } = embeddable.getInput();

x-pack/plugins/ml/public/application/jobs/new_job/job_from_map/visualization_extractor.ts

@@ -10,6 +10,7 @@ import type { MapEmbeddable } from '@kbn/maps-plugin/public';
 import type { EuiComboBoxOptionOption } from '@elastic/eui';
 import type { DataView } from '@kbn/data-views-plugin/common';
 import type { Query } from '@kbn/es-query';
+import type { DashboardContainer } from '@kbn/dashboard-plugin/public';
 import { categoryFieldTypes } from '../../../../../common/util/fields_utils';
 
 export interface LayerResult {
@@ -27,8 +28,8 @@ export class VisualizationExtractor {
 
   public async getResultLayersFromEmbeddable(embeddable: MapEmbeddable): Promise<LayerResult[]> {
     const layers: LayerResult[] = [];
-    // @ts-ignore
-    const dataViews: DataView[] = embeddable.getRoot()?.getAllDataViews() ?? [];
+    const dataViews: DataView[] =
+      (embeddable.getRoot() as DashboardContainer)?.getAllDataViews() ?? [];
 
     // Keep track of geoFields for layers as they can be repeated
     const layerGeoFields: Record<string, boolean> = {};

x-pack/plugins/ml/public/application/routing/routes/new_job/from_map.tsx

@@ -22,13 +22,22 @@ export const fromMapRouteFactory = (): MlRoute => ({
 });
 
 const PageWrapper: FC<PageProps> = ({ location, deps }) => {
-  const { dashboard, dataViewId, embeddable, geoField, splitField, from, to }: Record<string, any> =
-    parse(location.search, {
-      sort: false,
-    });
+  const {
+    dashboard,
+    dataViewId,
+    embeddable,
+    geoField,
+    splitField,
+    from,
+    to,
+    layer,
+  }: Record<string, any> = parse(location.search, {
+    sort: false,
+  });
 
   const { context } = useResolver(undefined, undefined, deps.config, deps.dataViewsContract, {
-    redirect: () => resolver(dashboard, dataViewId, embeddable, geoField, splitField, from, to),
+    redirect: () =>
+      resolver(dashboard, dataViewId, embeddable, geoField, splitField, from, to, layer),
   });
   return <PageLoader context={context}>{<Redirect to="/jobs/new_job" />}</PageLoader>;
 };

x-pack/plugins/ml/public/embeddables/job_creation/map/map_vis_layer_selection_flyout/layer/compatible_layer.tsx

@@ -64,6 +64,7 @@ export const CompatibleLayer: FC<Props> = ({ embeddable, layer, layerIndex }) =>
       embeddable,
       layer.dataView!.id!,
       layer.geoField,
+      layer.query,
       selectedSplitField,
       share
     );
@@ -87,6 +88,7 @@ export const CompatibleLayer: FC<Props> = ({ embeddable, layer, layerIndex }) =>
           runInRealTime,
           sourceDataView: layer.dataView,
           geoField: layer.geoField,
+          layerLevelQuery: layer.query,
           splitField: selectedSplitField,
         });
 

x-pack/plugins/ml/public/ui_actions/open_vis_in_ml_action.tsx

@@ -64,6 +64,9 @@ export function createVisToADJobAction(
           import('../application/jobs/new_job/job_from_lens'),
           getStartServices(),
         ]);
+      const { isCompatibleMapVisualization } = await import(
+        '../application/jobs/new_job/job_from_map'
+      );
 
       if (
         !coreStart.application.capabilities.ml?.canCreateJob ||
@@ -76,6 +79,8 @@ export function createVisToADJobAction(
         if (embeddableType === 'lens' && lens) {
           const { chartInfo } = await getJobsItemsFromEmbeddable(context.embeddable, lens);
           return isCompatibleVisualizationType(chartInfo!);
+        } else if (isMapEmbeddable(context.embeddable)) {
+          return isCompatibleMapVisualization(context.embeddable);
         }
         return true;
       } catch (error) {