Add essql search strategy and integrate in canvas (#94754)

* Add essql search strategy, new escount temporary function, and new essql temporary function

* Move old es* functions to legacy, add esdocs to use search strategy, add parameter arg for essql

* Clean up

* cleanup

* cleanup

* Move request builder files to common

* cleanup

* add comment

* PR Feedback

* Removing old types

* update type

* Add data.search to labs and fix error messages

* Fix function help type

* Add data service to usage collector types

* Update telemetry

* remove unrelated telemetry change

* Enable multi value leniency for SQL  queries

* Display data service lab project

src/plugins/kibana_usage_collection/server/collectors/management/schema.ts

@@ -440,4 +440,8 @@ export const stackManagementSchema: MakeSchemaFrom<UsageStats> = {
     type: 'boolean',
     _meta: { description: 'Non-default value of setting.' },
   },
+  'labs:canvas:useDataService': {
+    type: 'boolean',
+    _meta: { description: 'Non-default value of setting.' },
+  },
 };

src/plugins/kibana_usage_collection/server/collectors/management/types.ts

@@ -119,5 +119,6 @@ export interface UsageStats {
   'banners:textColor': string;
   'banners:backgroundColor': string;
   'labs:canvas:enable_ui': boolean;
+  'labs:canvas:useDataService': boolean;
   'labs:presentation:timeToPresent': boolean;
 }

src/plugins/presentation_util/common/labs.ts

@@ -8,9 +8,10 @@
 
 import { i18n } from '@kbn/i18n';
 
+export const USE_DATA_SERVICE = 'labs:canvas:useDataService';
 export const TIME_TO_PRESENT = 'labs:presentation:timeToPresent';
 
-export const projectIDs = [TIME_TO_PRESENT] as const;
+export const projectIDs = [TIME_TO_PRESENT, USE_DATA_SERVICE] as const;
 export const environmentNames = ['kibana', 'browser', 'session'] as const;
 export const solutionNames = ['canvas', 'dashboard', 'presentation'] as const;
 
@@ -32,6 +33,22 @@ export const projects: { [ID in ProjectID]: ProjectConfig & { id: ID } } = {
     }),
     solutions: ['canvas'],
   },
+  [USE_DATA_SERVICE]: {
+    id: USE_DATA_SERVICE,
+    isActive: true,
+    isDisplayed: true,
+    environments: ['kibana', 'browser', 'session'],
+    name: i18n.translate('presentationUtil.experiments.enableUseDataServiceExperimentName', {
+      defaultMessage: 'Use data service',
+    }),
+    description: i18n.translate(
+      'presentationUtil.experiments.enableUseDataServiceExperimentDescription',
+      {
+        defaultMessage: 'An experiment of using the new data.search service for Canvas datasources',
+      }
+    ),
+    solutions: ['canvas'],
+  },
 };
 
 export type ProjectID = typeof projectIDs[number];

src/plugins/telemetry/schema/oss_plugins.json

@@ -8342,6 +8342,12 @@
           "_meta": {
             "description": "Non-default value of setting."
           }
+        },
+        "labs:canvas:useDataService": {
+          "type": "boolean",
+          "_meta": {
+            "description": "Non-default value of setting."
+          }
         }
       }
     },

x-pack/plugins/canvas/canvas_plugin_src/functions/browser/escount.ts

@@ -0,0 +1,94 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+  ExpressionFunctionDefinition,
+  ExpressionValueFilter,
+} from 'src/plugins/expressions/common';
+
+// @ts-expect-error untyped local
+import { buildESRequest } from '../../../common/lib/request/build_es_request';
+
+import { searchService } from '../../../public/services';
+
+import { getFunctionHelp } from '../../../i18n';
+
+interface Arguments {
+  index: string | null;
+  query: string;
+}
+
+export function escount(): ExpressionFunctionDefinition<
+  'escount',
+  ExpressionValueFilter,
+  Arguments,
+  any
+> {
+  const { help, args: argHelp } = getFunctionHelp().escount;
+
+  return {
+    name: 'escount',
+    type: 'number',
+    context: {
+      types: ['filter'],
+    },
+    help,
+    args: {
+      query: {
+        types: ['string'],
+        aliases: ['_', 'q'],
+        help: argHelp.query,
+        default: '"-_index:.kibana"',
+      },
+      index: {
+        types: ['string'],
+        default: '_all',
+        help: argHelp.index,
+      },
+    },
+    fn: (input, args, handlers) => {
+      input.and = input.and.concat([
+        {
+          type: 'filter',
+          filterType: 'luceneQueryString',
+          query: args.query,
+          and: [],
+        },
+      ]);
+
+      const esRequest = buildESRequest(
+        {
+          index: args.index,
+          body: {
+            track_total_hits: true,
+            size: 0,
+            query: {
+              bool: {
+                must: [{ match_all: {} }],
+              },
+            },
+          },
+        },
+        input
+      );
+
+      const search = searchService.getService().search;
+      const req = {
+        params: {
+          ...esRequest,
+        },
+      };
+
+      return search
+        .search(req)
+        .toPromise()
+        .then((resp: any) => {
+          return resp.rawResponse.hits.total;
+        });
+    },
+  };
+}

x-pack/plugins/canvas/canvas_plugin_src/functions/browser/esdocs.ts

@@ -0,0 +1,141 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+  ExpressionFunctionDefinition,
+  ExpressionValueFilter,
+} from 'src/plugins/expressions/common';
+
+// @ts-expect-error untyped local
+import { buildESRequest } from '../../../common/lib/request/build_es_request';
+
+import { searchService } from '../../../public/services';
+import { ESSQL_SEARCH_STRATEGY } from '../../../common/lib/constants';
+import { EssqlSearchStrategyRequest, EssqlSearchStrategyResponse } from '../../../types';
+import { getFunctionHelp } from '../../../i18n';
+
+interface Arguments {
+  index: string;
+  query: string;
+  sort: string;
+  fields: string;
+  metaFields: string;
+  count: number;
+}
+
+export function esdocs(): ExpressionFunctionDefinition<
+  'esdocs',
+  ExpressionValueFilter,
+  Arguments,
+  any
+> {
+  const { help, args: argHelp } = getFunctionHelp().esdocs;
+
+  return {
+    name: 'esdocs',
+    type: 'datatable',
+    context: {
+      types: ['filter'],
+    },
+    help,
+    args: {
+      query: {
+        types: ['string'],
+        aliases: ['_', 'q'],
+        help: argHelp.query,
+        default: '-_index:.kibana',
+      },
+      count: {
+        types: ['number'],
+        default: 1000,
+        help: argHelp.count,
+      },
+      fields: {
+        help: argHelp.fields,
+        types: ['string'],
+      },
+      index: {
+        types: ['string'],
+        default: '_all',
+        help: argHelp.index,
+      },
+      // TODO: This arg isn't being used in the function.
+      // We need to restore this functionality or remove it as an arg.
+      metaFields: {
+        help: argHelp.metaFields,
+        types: ['string'],
+      },
+      sort: {
+        types: ['string'],
+        help: argHelp.sort,
+      },
+    },
+    fn: async (input, args, handlers) => {
+      const { count, index, fields, sort } = args;
+
+      input.and = input.and.concat([
+        {
+          type: 'filter',
+          filterType: 'luceneQueryString',
+          query: args.query,
+          and: [],
+        },
+      ]);
+
+      // Load ad-hoc to avoid adding to the page load bundle size
+      const squel = await import('squel');
+
+      let query = squel.select({
+        autoQuoteTableNames: true,
+        autoQuoteFieldNames: true,
+        autoQuoteAliasNames: true,
+        nameQuoteCharacter: '"',
+      });
+
+      if (index) {
+        query.from(index);
+      }
+
+      if (fields) {
+        const allFields = fields.split(',').map((field) => field.trim());
+        allFields.forEach((field) => (query = query.field(field)));
+      }
+
+      if (sort) {
+        const [sortField, sortOrder] = sort.split(',').map((str) => str.trim());
+        if (sortField) {
+          query.order(`"${sortField}"`, sortOrder === 'asc');
+        }
+      }
+
+      const search = searchService.getService().search;
+
+      const req = {
+        count,
+        query: query.toString(),
+        filter: input.and,
+      };
+
+      // We're requesting the data using the ESSQL strategy because
+      // the SQL routes return type information with the result set
+      return search
+        .search<EssqlSearchStrategyRequest, EssqlSearchStrategyResponse>(req, {
+          strategy: ESSQL_SEARCH_STRATEGY,
+        })
+        .toPromise()
+        .then((resp: EssqlSearchStrategyResponse) => {
+          return {
+            type: 'datatable',
+            meta: {
+              type: 'essql',
+            },
+            ...resp,
+          };
+        });
+    },
+  };
+}

x-pack/plugins/canvas/canvas_plugin_src/functions/browser/essql.ts

@@ -0,0 +1,103 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+  ExpressionFunctionDefinition,
+  ExpressionValueFilter,
+} from 'src/plugins/expressions/common';
+import { searchService } from '../../../public/services';
+import { ESSQL_SEARCH_STRATEGY } from '../../../common/lib/constants';
+import { EssqlSearchStrategyRequest, EssqlSearchStrategyResponse } from '../../../types';
+import { getFunctionHelp } from '../../../i18n';
+
+interface Arguments {
+  query: string;
+  parameter: Array<string | number | boolean>;
+  count: number;
+  timezone: string;
+}
+
+export function essql(): ExpressionFunctionDefinition<
+  'essql',
+  ExpressionValueFilter,
+  Arguments,
+  any
+> {
+  const { help, args: argHelp } = getFunctionHelp().essql;
+
+  return {
+    name: 'essql',
+    type: 'datatable',
+    context: {
+      types: ['filter'],
+    },
+    help,
+    args: {
+      query: {
+        aliases: ['_', 'q'],
+        types: ['string'],
+        help: argHelp.query,
+      },
+      parameter: {
+        aliases: ['param'],
+        types: ['string', 'number', 'boolean'],
+        multi: true,
+        help: argHelp.parameter,
+      },
+      count: {
+        types: ['number'],
+        help: argHelp.count,
+        default: 1000,
+      },
+      timezone: {
+        aliases: ['tz'],
+        types: ['string'],
+        default: 'UTC',
+        help: argHelp.timezone,
+      },
+    },
+    fn: (input, args, handlers) => {
+      const search = searchService.getService().search;
+      const { parameter, ...restOfArgs } = args;
+      const req = {
+        ...restOfArgs,
+        params: parameter,
+        filter: input.and,
+      };
+
+      return search
+        .search<EssqlSearchStrategyRequest, EssqlSearchStrategyResponse>(req, {
+          strategy: ESSQL_SEARCH_STRATEGY,
+        })
+        .toPromise()
+        .then((resp: EssqlSearchStrategyResponse) => {
+          return {
+            type: 'datatable',
+            meta: {
+              type: 'essql',
+            },
+            ...resp,
+          };
+        })
+        .catch((e) => {
+          let message = `Unexpected error from Elasticsearch: ${e.message}`;
+          if (e.err) {
+            const { type, reason } = e.err.attributes;
+            if (type === 'parsing_exception') {
+              message = `Couldn't parse Elasticsearch SQL query. You may need to add double quotes to names containing special characters. Check your query and try again. Error: ${reason}`;
+            } else {
+              message = `Unexpected error from Elasticsearch: ${type} - ${reason}`;
+            }
+          }
+
+          // Re-write the error message before surfacing it up
+          e.message = message;
+          throw e;
+        });
+    },
+  };
+}

x-pack/plugins/canvas/canvas_plugin_src/functions/browser/index.ts

@@ -10,5 +10,17 @@ import { functions as externalFunctions } from '../external';
 import { location } from './location';
 import { markdown } from './markdown';
 import { urlparam } from './urlparam';
+import { escount } from './escount';
+import { esdocs } from './esdocs';
+import { essql } from './essql';
 
-export const functions = [location, markdown, urlparam, ...commonFunctions, ...externalFunctions];
+export const functions = [
+  location,
+  markdown,
+  urlparam,
+  escount,
+  esdocs,
+  essql,
+  ...commonFunctions,
+  ...externalFunctions,
+];

x-pack/plugins/canvas/canvas_plugin_src/functions/server/escount.ts

@@ -9,10 +9,9 @@ import {
   ExpressionFunctionDefinition,
   ExpressionValueFilter,
 } from 'src/plugins/expressions/common';
-/* eslint-disable */
 // @ts-expect-error untyped local
-import { buildESRequest } from '../../../server/lib/build_es_request';
-/* eslint-enable */
+import { buildESRequest } from '../../../common/lib/request/build_es_request';
+
 import { getFunctionHelp } from '../../../i18n';
 
 interface Arguments {

x-pack/plugins/canvas/common/lib/constants.ts

@@ -44,3 +44,4 @@ export const API_ROUTE_SHAREABLE_RUNTIME_DOWNLOAD = `/public/canvas/${SHAREABLE_
 export const CANVAS_EMBEDDABLE_CLASSNAME = `canvasEmbeddable`;
 export const CONTEXT_MENU_TOP_BORDER_CLASSNAME = 'canvasContextMenu--topBorder';
 export const API_ROUTE_FUNCTIONS = `${API_ROUTE}/fns`;
+export const ESSQL_SEARCH_STRATEGY = 'essql';

x-pack/plugins/canvas/common/lib/request/build_bool_array.ts

@@ -6,7 +6,7 @@
  */
 
 import { getESFilter } from './get_es_filter';
-import { ExpressionValueFilter } from '../../types';
+import { ExpressionValueFilter } from '../../../types';
 
 const compact = <T>(arr: T[]) => (Array.isArray(arr) ? arr.filter((val) => Boolean(val)) : []);
 

x-pack/plugins/canvas/common/lib/request/filters.ts

@@ -11,7 +11,7 @@ import {
   CanvasTimeFilter,
   CanvasLuceneFilter,
   CanvasExactlyFilter,
-} from '../../types';
+} from '../../../types';
 
 /*
  TODO: This could be pluggable

x-pack/plugins/canvas/common/lib/request/get_es_filter.ts

@@ -12,7 +12,7 @@
 */
 
 import { filters } from './filters';
-import { ExpressionValueFilter } from '../../types';
+import { ExpressionValueFilter } from '../../../types';
 
 export function getESFilter(filter: ExpressionValueFilter) {
   if (!filter.filterType || !filters[filter.filterType]) {

x-pack/plugins/canvas/i18n/functions/dict/escount.ts

@@ -6,7 +6,7 @@
  */
 
 import { i18n } from '@kbn/i18n';
-import { escount } from '../../../canvas_plugin_src/functions/server/escount';
+import { escount } from '../../../canvas_plugin_src/functions/browser/escount';
 import { FunctionHelp } from '../function_help';
 import { FunctionFactory } from '../../../types';
 import { ELASTICSEARCH, LUCENE } from '../../constants';

x-pack/plugins/canvas/i18n/functions/dict/esdocs.ts

@@ -6,7 +6,7 @@
  */
 
 import { i18n } from '@kbn/i18n';
-import { esdocs } from '../../../canvas_plugin_src/functions/server/esdocs';
+import { esdocs } from '../../../canvas_plugin_src/functions/browser/esdocs';
 import { FunctionHelp } from '../function_help';
 import { FunctionFactory } from '../../../types';
 import { ELASTICSEARCH, LUCENE } from '../../constants';

x-pack/plugins/canvas/i18n/functions/dict/essql.ts

@@ -6,7 +6,7 @@
  */
 
 import { i18n } from '@kbn/i18n';
-import { essql } from '../../../canvas_plugin_src/functions/server/essql';
+import { essql } from '../../../canvas_plugin_src/functions/browser/essql';
 import { FunctionHelp } from '../function_help';
 import { FunctionFactory } from '../../../types';
 import { ELASTICSEARCH, SQL, ISO8601, UTC } from '../../constants';
@@ -27,6 +27,12 @@ export const help: FunctionHelp<FunctionFactory<typeof essql>> = {
         SQL,
       },
     }),
+    parameter: i18n.translate('xpack.canvas.functions.essql.args.parameterHelpText', {
+      defaultMessage: 'A parameter to be passed to the {SQL} query.',
+      values: {
+        SQL,
+      },
+    }),
     count: i18n.translate('xpack.canvas.functions.essql.args.countHelpText', {
       defaultMessage:
         'The number of documents to retrieve. For better performance, use a smaller data set.',

x-pack/plugins/canvas/public/plugin.tsx

@@ -22,7 +22,7 @@ import { getSessionStorage } from './lib/storage';
 import { SESSIONSTORAGE_LASTPATH } from '../common/lib/constants';
 import { featureCatalogueEntry } from './feature_catalogue_entry';
 import { ExpressionsSetup, ExpressionsStart } from '../../../../src/plugins/expressions/public';
-import { DataPublicPluginSetup } from '../../../../src/plugins/data/public';
+import { DataPublicPluginSetup, DataPublicPluginStart } from '../../../../src/plugins/data/public';
 import { UiActionsStart } from '../../../../src/plugins/ui_actions/public';
 import { EmbeddableStart } from '../../../../src/plugins/embeddable/public';
 import { UsageCollectionSetup } from '../../../../src/plugins/usage_collection/public';
@@ -54,6 +54,7 @@ export interface CanvasStartDeps {
   inspector: InspectorStart;
   uiActions: UiActionsStart;
   charts: ChartsPluginStart;
+  data: DataPublicPluginStart;
   presentationUtil: PresentationUtilPluginStart;
 }
 

x-pack/plugins/canvas/public/services/context.tsx

@@ -25,6 +25,7 @@ const defaultContextValue = {
   notify: {},
   platform: {},
   navLink: {},
+  search: {},
 };
 
 const context = createContext<CanvasServices>(defaultContextValue as CanvasServices);
@@ -54,6 +55,7 @@ export const ServicesProvider: FC<{
     notify: specifiedProviders.notify.getService(),
     platform: specifiedProviders.platform.getService(),
     navLink: specifiedProviders.navLink.getService(),
+    search: specifiedProviders.search.getService(),
     reporting: specifiedProviders.reporting.getService(),
     labs: specifiedProviders.labs.getService(),
   };

x-pack/plugins/canvas/public/services/expressions.ts

@@ -24,6 +24,11 @@ export const expressionsServiceFactory: CanvasServiceFactory<ExpressionsService>
   const loadServerFunctionWrappers = async () => {
     if (!cached) {
       cached = (async () => {
+        const labService = startPlugins.presentationUtil.labsService;
+        const useDataSearchProject = labService.getProject('labs:canvas:useDataService');
+        const hasDataSearch = useDataSearchProject.status.isEnabled;
+        const dataSearchFns = ['essql', 'esdocs', 'escount'];
+
         const serverFunctionList = await coreSetup.http.get(API_ROUTE_FUNCTIONS);
         const batchedFunction = bfetch.batchedFunction({ url: API_ROUTE_FUNCTIONS });
         const { serialize } = serializeProvider(expressions.getTypes());
@@ -32,9 +37,16 @@ export const expressionsServiceFactory: CanvasServiceFactory<ExpressionsService>
         // function that matches its definition, but which simply
         // calls the server-side function endpoint.
         Object.keys(serverFunctionList).forEach((functionName) => {
-          if (expressions.getFunction(functionName)) {
+          // Allow function to be overwritten if we want to use
+          // the server-hosted essql, esdocs, and escount functions
+          if (dataSearchFns.includes(functionName)) {
+            if (hasDataSearch && expressions.getFunction(functionName)) {
+              return;
+            }
+          } else if (expressions.getFunction(functionName)) {
             return;
           }
+
           const fn = () => ({
             ...serverFunctionList[functionName],
             fn: (input: any, args: any) => {

x-pack/plugins/canvas/public/services/index.ts

@@ -13,10 +13,12 @@ import { platformServiceFactory } from './platform';
 import { navLinkServiceFactory } from './nav_link';
 import { embeddablesServiceFactory } from './embeddables';
 import { expressionsServiceFactory } from './expressions';
+import { searchServiceFactory } from './search';
 import { labsServiceFactory } from './labs';
 import { reportingServiceFactory } from './reporting';
 
 export { NotifyService } from './notify';
+export { SearchService } from './search';
 export { PlatformService } from './platform';
 export { NavLinkService } from './nav_link';
 export { EmbeddablesService } from './embeddables';
@@ -80,6 +82,7 @@ export const services = {
   notify: new CanvasServiceProvider(notifyServiceFactory),
   platform: new CanvasServiceProvider(platformServiceFactory),
   navLink: new CanvasServiceProvider(navLinkServiceFactory),
+  search: new CanvasServiceProvider(searchServiceFactory),
   reporting: new CanvasServiceProvider(reportingServiceFactory),
   labs: new CanvasServiceProvider(labsServiceFactory),
 };
@@ -92,6 +95,7 @@ export interface CanvasServices {
   notify: ServiceFromProvider<typeof services.notify>;
   platform: ServiceFromProvider<typeof services.platform>;
   navLink: ServiceFromProvider<typeof services.navLink>;
+  search: ServiceFromProvider<typeof services.search>;
   reporting: ServiceFromProvider<typeof services.reporting>;
   labs: ServiceFromProvider<typeof services.labs>;
 }
@@ -120,5 +124,6 @@ export const {
   platform: platformService,
   navLink: navLinkService,
   expressions: expressionsService,
+  search: searchService,
   reporting: reportingService,
 } = services;

x-pack/plugins/canvas/public/services/search.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { DataPublicPluginStart } from 'src/plugins/data/public';
+import { CanvasServiceFactory } from '.';
+
+export interface SearchService {
+  search: DataPublicPluginStart['search'];
+}
+
+export const searchServiceFactory: CanvasServiceFactory<SearchService> = (
+  setup,
+  start,
+  canvasSetup,
+  canvasStart
+) => {
+  return {
+    search: canvasStart.data.search,
+  };
+};

x-pack/plugins/canvas/public/services/stubs/index.ts

@@ -13,6 +13,7 @@ import { navLinkService } from './nav_link';
 import { notifyService } from './notify';
 import { labsService } from './labs';
 import { platformService } from './platform';
+import { searchService } from './search';
 
 export const stubs: CanvasServices = {
   embeddables: embeddablesService,
@@ -21,6 +22,7 @@ export const stubs: CanvasServices = {
   navLink: navLinkService,
   notify: notifyService,
   platform: platformService,
+  search: searchService,
   labs: labsService,
 };
 

x-pack/plugins/canvas/public/services/stubs/search.ts

@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+const noop = (..._args: any[]): any => {};
+
+export const searchService: any = {
+  search: noop,
+};

x-pack/plugins/canvas/server/lib/essql_strategy.ts

@@ -0,0 +1,96 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { from } from 'rxjs';
+import { map, zipObject } from 'lodash';
+
+import { ISearchStrategy, PluginStart } from 'src/plugins/data/server';
+
+import { getKbnServerError } from '../../../../../src/plugins/kibana_utils/server';
+import { EssqlSearchStrategyRequest, EssqlSearchStrategyResponse } from '../../types';
+
+import { buildBoolArray } from '../../common/lib/request/build_bool_array';
+import { sanitizeName } from '../../common/lib/request/sanitize_name';
+import { normalizeType } from '../../common/lib/request/normalize_type';
+
+export const essqlSearchStrategyProvider = (
+  data: PluginStart
+): ISearchStrategy<EssqlSearchStrategyRequest, EssqlSearchStrategyResponse> => {
+  return {
+    search: (request, options, { esClient }) => {
+      const { count, query, filter, timezone, params } = request;
+
+      const searchUntilEnd = async () => {
+        try {
+          let response = await esClient.asCurrentUser.sql.query({
+            format: 'json',
+            body: {
+              query,
+              // @ts-expect-error `params` missing from `QuerySqlRequest` type
+              params,
+              field_multi_value_leniency: true,
+              time_zone: timezone,
+              fetch_size: count,
+              client_id: 'canvas',
+              filter: {
+                bool: {
+                  must: [{ match_all: {} }, ...buildBoolArray(filter)],
+                },
+              },
+            },
+          });
+
+          let body = response.body;
+
+          const columns = body.columns!.map(({ name, type }) => {
+            return {
+              id: sanitizeName(name),
+              name: sanitizeName(name),
+              meta: { type: normalizeType(type) },
+            };
+          });
+          const columnNames = map(columns, 'name');
+          let rows = body.rows.map((row) => zipObject(columnNames, row));
+
+          // If we still have rows to retrieve, continue requesting data
+          // using the cursor until we have everything
+          while (rows.length < count && body.cursor !== undefined) {
+            response = await esClient.asCurrentUser.sql.query({
+              format: 'json',
+              body: {
+                cursor: body.cursor,
+              },
+            });
+
+            body = response.body;
+
+            rows = [...rows, ...body.rows.map((row) => zipObject(columnNames, row))];
+          }
+
+          // If we used a cursor, clean it up
+          if (body.cursor !== undefined) {
+            await esClient.asCurrentUser.sql.clearCursor({
+              body: {
+                cursor: body.cursor,
+              },
+            });
+          }
+
+          return {
+            columns,
+            rows,
+            rawResponse: response,
+          };
+        } catch (e) {
+          throw getKbnServerError(e);
+        }
+      };
+
+      return from(searchUntilEnd());
+    },
+  };
+};

x-pack/plugins/canvas/server/lib/query_es_sql.ts

@@ -6,9 +6,9 @@
  */
 
 import { map, zipObject } from 'lodash';
-import { buildBoolArray } from './build_bool_array';
-import { sanitizeName } from './sanitize_name';
-import { normalizeType } from './normalize_type';
+import { buildBoolArray } from '../../common/lib/request/build_bool_array';
+import { sanitizeName } from '../../common/lib/request/sanitize_name';
+import { normalizeType } from '../../common/lib/request/normalize_type';
 import { LegacyAPICaller } from '../../../../../src/core/server';
 import { ExpressionValueFilter } from '../../types';
 

x-pack/plugins/canvas/server/plugin.ts

@@ -6,10 +6,15 @@
  */
 
 import { CoreSetup, PluginInitializerContext, Plugin, Logger, CoreStart } from 'src/core/server';
+import {
+  PluginSetup as DataPluginSetup,
+  PluginStart as DataPluginStart,
+} from 'src/plugins/data/server';
 import { ExpressionsServerSetup } from 'src/plugins/expressions/server';
 import { BfetchServerSetup } from 'src/plugins/bfetch/server';
 import { UsageCollectionSetup } from 'src/plugins/usage_collection/server';
 import { HomeServerPluginSetup } from 'src/plugins/home/server';
+import { ESSQL_SEARCH_STRATEGY } from '../common/lib/constants';
 import { ReportingSetup } from '../../reporting/server';
 import { PluginSetupContract as FeaturesPluginSetup } from '../../features/server';
 import { getCanvasFeature } from './feature';
@@ -19,6 +24,7 @@ import { loadSampleData } from './sample_data';
 import { setupInterpreter } from './setup_interpreter';
 import { customElementType, workpadType, workpadTemplateType } from './saved_objects';
 import { initializeTemplates } from './templates';
+import { essqlSearchStrategyProvider } from './lib/essql_strategy';
 import { getUISettings } from './ui_settings';
 
 interface PluginsSetup {
@@ -26,17 +32,22 @@ interface PluginsSetup {
   features: FeaturesPluginSetup;
   home: HomeServerPluginSetup;
   bfetch: BfetchServerSetup;
+  data: DataPluginSetup;
   reporting?: ReportingSetup;
   usageCollection?: UsageCollectionSetup;
 }
 
+interface PluginsStart {
+  data: DataPluginStart;
+}
+
 export class CanvasPlugin implements Plugin {
   private readonly logger: Logger;
   constructor(public readonly initializerContext: PluginInitializerContext) {
     this.logger = initializerContext.logger.get();
   }
 
-  public setup(coreSetup: CoreSetup, plugins: PluginsSetup) {
+  public setup(coreSetup: CoreSetup<PluginsStart>, plugins: PluginsSetup) {
     coreSetup.uiSettings.register(getUISettings());
     coreSetup.savedObjects.registerType(customElementType);
     coreSetup.savedObjects.registerType(workpadType);
@@ -64,6 +75,11 @@ export class CanvasPlugin implements Plugin {
     registerCanvasUsageCollector(plugins.usageCollection, globalConfig.kibana.index);
 
     setupInterpreter(plugins.expressions);
+
+    coreSetup.getStartServices().then(([_, depsStart]) => {
+      const strategy = essqlSearchStrategyProvider(depsStart.data);
+      plugins.data.search.registerSearchStrategy(ESSQL_SEARCH_STRATEGY, strategy);
+    });
   }
 
   public start(coreStart: CoreStart) {

x-pack/plugins/canvas/server/routes/es_fields/es_fields.ts

@@ -9,7 +9,7 @@ import { mapValues, keys } from 'lodash';
 import { schema } from '@kbn/config-schema';
 import { API_ROUTE } from '../../../common/lib';
 import { catchErrorHandler } from '../catch_error_handler';
-import { normalizeType } from '../../lib/normalize_type';
+import { normalizeType } from '../../../common/lib/request/normalize_type';
 import { RouteInitializerDeps } from '..';
 
 const ESFieldsRequestSchema = schema.object({

x-pack/plugins/canvas/types/index.ts

@@ -14,5 +14,6 @@ export * from './functions';
 export * from './renderers';
 export * from './shortcuts';
 export * from './state';
+export * from './strategy';
 export * from './style';
 export * from './telemetry';

x-pack/plugins/canvas/types/strategy.ts

@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ApiResponse } from '@elastic/elasticsearch/lib/Transport';
+import { QuerySqlResponse } from '@elastic/elasticsearch/api/types';
+import { IKibanaSearchRequest } from 'src/plugins/data/common';
+import { ExpressionValueFilter } from '.';
+export interface EssqlSearchStrategyRequest extends IKibanaSearchRequest {
+  count: number;
+  query: string;
+  params?: Array<string | number | boolean>;
+  timezone?: string;
+  filter: ExpressionValueFilter[];
+}
+
+export interface EssqlSearchStrategyResponse {
+  columns: Array<{
+    id: string;
+    name: string;
+    meta: {
+      type: string;
+    };
+  }>;
+  rows: any[];
+
+  rawResponse: ApiResponse<QuerySqlResponse>;
+}