[ES|QL] Use same adhoc dataviews for queries with the same index pattern (#174736)

.github/CODEOWNERS

@@ -383,6 +383,7 @@ packages/kbn-eslint-plugin-imports @elastic/kibana-operations
 packages/kbn-eslint-plugin-telemetry @elastic/obs-knowledge-team
 examples/eso_model_version_example @elastic/kibana-security
 x-pack/test/encrypted_saved_objects_api_integration/plugins/api_consumer_plugin @elastic/kibana-security
+packages/kbn-esql-utils @elastic/kibana-visualizations
 packages/kbn-event-annotation-common @elastic/kibana-visualizations
 packages/kbn-event-annotation-components @elastic/kibana-visualizations
 src/plugins/event_annotation_listing @elastic/kibana-visualizations

package.json

@@ -422,6 +422,7 @@
     "@kbn/es-ui-shared-plugin": "link:src/plugins/es_ui_shared",
     "@kbn/eso-model-version-example": "link:examples/eso_model_version_example",
     "@kbn/eso-plugin": "link:x-pack/test/encrypted_saved_objects_api_integration/plugins/api_consumer_plugin",
+    "@kbn/esql-utils": "link:packages/kbn-esql-utils",
     "@kbn/event-annotation-common": "link:packages/kbn-event-annotation-common",
     "@kbn/event-annotation-components": "link:packages/kbn-event-annotation-components",
     "@kbn/event-annotation-listing-plugin": "link:src/plugins/event_annotation_listing",

packages/kbn-es-query/tsconfig.json

@@ -15,7 +15,7 @@
   "kbn_references": [
     "@kbn/utility-types",
     "@kbn/i18n",
-    "@kbn/safer-lodash-set",
+    "@kbn/safer-lodash-set"
   ],
   "exclude": [
     "target/**/*",

packages/kbn-esql-utils/README.md

@@ -0,0 +1,4 @@
+# @kbn/esql-utils
+
+This package contains utilities for ES|QL.
+

packages/kbn-esql-utils/index.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export { getESQLAdHocDataview } from './src';

packages/kbn-esql-utils/jest.config.js

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+module.exports = {
+  preset: '@kbn/test',
+  rootDir: '../..',
+  roots: ['<rootDir>/packages/kbn-esql-utils'],
+};

packages/kbn-esql-utils/kibana.jsonc

@@ -0,0 +1,5 @@
+{
+  "type": "shared-common",
+  "id": "@kbn/esql-utils",
+  "owner": "@elastic/kibana-visualizations"
+}

packages/kbn-esql-utils/package.json

@@ -0,0 +1,7 @@
+{
+  "name": "@kbn/esql-utils",
+  "private": true,
+  "version": "1.0.0",
+  "license": "SSPL-1.0 OR Elastic License 2.0",
+  "sideEffects": false
+}

packages/kbn-esql-utils/src/index.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export * from './utils';

packages/kbn-esql-utils/src/utils/index.ts

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+import type { DataViewsPublicPluginStart } from '@kbn/data-views-plugin/public';
+
+// uses browser sha256 method with fallback if unavailable
+async function sha256(str: string) {
+  if (crypto.subtle) {
+    const enc = new TextEncoder();
+    const hash = await crypto.subtle.digest('SHA-256', enc.encode(str));
+    return Array.from(new Uint8Array(hash))
+      .map((v) => v.toString(16).padStart(2, '0'))
+      .join('');
+  } else {
+    const { sha256: sha256fn } = await import('./sha256');
+    return sha256fn(str);
+  }
+}
+
+// Some applications need to have a dataview to work properly with ES|QL queries
+// This is a helper to create one. The id is constructed from the indexpattern.
+// As there are no runtime fields or field formatters or default time fields
+// the same adhoc dataview can be constructed/used. This comes with great advantages such
+// as solving the problem descibed here https://github.com/elastic/kibana/issues/168131
+export async function getESQLAdHocDataview(
+  indexPattern: string,
+  dataViewsService: DataViewsPublicPluginStart
+) {
+  return await dataViewsService.create({
+    title: indexPattern,
+    id: await sha256(`esql-${indexPattern}`),
+  });
+}

packages/kbn-esql-utils/src/utils/sha256.ts

@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { Sha256 } from '@kbn/crypto-browser';
+
+export const sha256 = async (str: string) => new Sha256().update(str).digest('hex');

packages/kbn-esql-utils/tsconfig.json

@@ -0,0 +1,22 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "target/types",
+    "types": [
+      "jest",
+      "node",
+      "react"
+    ]
+  },
+  "include": [
+    "**/*.ts",
+    "**/*.tsx",
+  ],
+  "exclude": [
+    "target/**/*"
+  ],
+  "kbn_references": [
+    "@kbn/data-views-plugin",
+    "@kbn/crypto-browser",
+  ]
+}

src/plugins/discover/public/application/main/utils/get_data_view_by_text_based_query_lang.ts

@@ -10,6 +10,7 @@ import {
   getIndexPatternFromSQLQuery,
   getIndexPatternFromESQLQuery,
 } from '@kbn/es-query';
+import { getESQLAdHocDataview } from '@kbn/esql-utils';
 import { DataView } from '@kbn/data-views-plugin/common';
 import { DiscoverServices } from '../../../build_services';
 
@@ -32,9 +33,7 @@ export async function getDataViewByTextBasedQueryLang(
     currentDataView?.isPersisted() ||
     indexPatternFromQuery !== currentDataView?.getIndexPattern()
   ) {
-    const dataViewObj = await services.dataViews.create({
-      title: indexPatternFromQuery,
-    });
+    const dataViewObj = await getESQLAdHocDataview(indexPatternFromQuery, services.dataViews);
 
     if (dataViewObj.fields.getByName('@timestamp')?.type === 'date') {
       dataViewObj.timeFieldName = '@timestamp';

src/plugins/discover/tsconfig.json

@@ -81,6 +81,7 @@
     "@kbn/shared-ux-button-toolbar",
     "@kbn/serverless",
     "@kbn/deeplinks-observability",
+    "@kbn/esql-utils",
     "@kbn/managed-content-badge"
   ],
   "exclude": ["target/**/*"]

tsconfig.base.json

@@ -760,6 +760,8 @@
       "@kbn/eso-model-version-example/*": ["examples/eso_model_version_example/*"],
       "@kbn/eso-plugin": ["x-pack/test/encrypted_saved_objects_api_integration/plugins/api_consumer_plugin"],
       "@kbn/eso-plugin/*": ["x-pack/test/encrypted_saved_objects_api_integration/plugins/api_consumer_plugin/*"],
+      "@kbn/esql-utils": ["packages/kbn-esql-utils"],
+      "@kbn/esql-utils/*": ["packages/kbn-esql-utils/*"],
       "@kbn/event-annotation-common": ["packages/kbn-event-annotation-common"],
       "@kbn/event-annotation-common/*": ["packages/kbn-event-annotation-common/*"],
       "@kbn/event-annotation-components": ["packages/kbn-event-annotation-components"],

x-pack/plugins/lens/public/app_plugin/shared/edit_on_the_fly/helpers.ts

@@ -7,6 +7,7 @@
 import { i18n } from '@kbn/i18n';
 import { getIndexPatternFromSQLQuery, getIndexPatternFromESQLQuery } from '@kbn/es-query';
 import type { AggregateQuery, Query, Filter } from '@kbn/es-query';
+import { getESQLAdHocDataview } from '@kbn/esql-utils';
 import { fetchFieldsFromESQL } from '@kbn/text-based-editor';
 import type { DataView, DataViewSpec } from '@kbn/data-views-plugin/public';
 import type { Suggestion } from '../../../types';
@@ -48,11 +49,10 @@ export const getSuggestions = async (
       return adHoc.name === indexPattern;
     });
 
-    const dataView = await deps.dataViews.create(
-      dataViewSpec ?? {
-        title: indexPattern,
-      }
-    );
+    const dataView = dataViewSpec
+      ? await deps.dataViews.create(dataViewSpec)
+      : await getESQLAdHocDataview(indexPattern, deps.dataViews);
+
     if (dataView.fields.getByName('@timestamp')?.type === 'date' && !dataViewSpec) {
       dataView.timeFieldName = '@timestamp';
     }

x-pack/plugins/lens/public/datasources/text_based/utils.ts

@@ -7,7 +7,7 @@
 import type { DataViewsPublicPluginStart } from '@kbn/data-views-plugin/public';
 import type { DataPublicPluginStart } from '@kbn/data-plugin/public';
 import type { ExpressionsStart } from '@kbn/expressions-plugin/public';
-
+import { getESQLAdHocDataview } from '@kbn/esql-utils';
 import {
   type AggregateQuery,
   getIndexPatternFromSQLQuery,
@@ -16,7 +16,6 @@ import {
 import type { DatatableColumn } from '@kbn/expressions-plugin/public';
 import { generateId } from '../../id_generator';
 import { fetchDataFromAggregateQuery } from './fetch_data_from_aggregate_query';
-
 import type { IndexPatternRef, TextBasedPrivateState, TextBasedLayerColumn } from './types';
 import type { DataViewsState } from '../../state_management';
 import { addColumnsToCache } from './fieldlist_cache';
@@ -89,9 +88,8 @@ export async function getStateFromAggregateQuery(
   let columnsFromQuery: DatatableColumn[] = [];
   let timeFieldName;
   try {
-    const dataView = await dataViews.create({
-      title: indexPattern,
-    });
+    const dataView = await getESQLAdHocDataview(indexPattern, dataViews);
+
     if (dataView && dataView.id) {
       if (dataView?.fields?.getByName('@timestamp')?.type === 'date') {
         dataView.timeFieldName = '@timestamp';

x-pack/plugins/lens/tsconfig.json

@@ -105,6 +105,7 @@
     "@kbn/visualization-utils",
     "@kbn/test-eui-helpers",
     "@kbn/shared-ux-utility",
+    "@kbn/esql-utils",
     "@kbn/text-based-editor",
     "@kbn/managed-content-badge",
     "@kbn/sort-predicates",

yarn.lock

@@ -4584,6 +4584,10 @@
   version "0.0.0"
   uid ""
 
+"@kbn/esql-utils@link:packages/kbn-esql-utils":
+  version "0.0.0"
+  uid ""
+
 "@kbn/event-annotation-common@link:packages/kbn-event-annotation-common":
   version "0.0.0"
   uid ""