[Lens] Move field existence from Lens to UnifiedFieldList plugin (#139453)

* [Lens] move field existence from to unified field list plugin * [Lens] update readme, move integration tests * [Lens] update wording paths * [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' * [Discover] fix loader tests, clean up code * [Discover] update datapanel tests, clean up code * [Discover] remove comments * [Discover] fix problem with filters * [Lens] apply suggestions * [Discover] remove spread * [Discover] fix type checks Co-authored-by: Joe Reuter <johannes.reuter@elastic.co>
2025-04-23 09:19:04 -04:00 · 2022-09-08 16:59:33 +03:00 · 2022-09-08 16:59:33 +03:00 · e11bea9178
commit e11bea9178
parent 5c9a11a25f
36 changed files with 778 additions and 614 deletions
--- a/src/plugins/data_views/public/data_views/data_views_api_client.ts
+++ b/src/plugins/data_views/public/data_views/data_views_api_client.ts
@ -27,18 +27,17 @@ export class DataViewsApiClient implements IDataViewsApiClient {
    this.http = http;
  }

-  private _request<T = unknown>(url: string, query?: {}): Promise<T | undefined> {
-    return this.http
-      .fetch<T>(url, {
-        query,
-      })
-      .catch((resp) => {
-        if (resp.body.statusCode === 404 && resp.body.attributes?.code === 'no_matching_indices') {
-          throw new DataViewMissingIndices(resp.body.message);
-        }
+  private _request<T = unknown>(url: string, query?: {}, body?: string): Promise<T | undefined> {
+    const request = body
+      ? this.http.post<T>(url, { query, body })
+      : this.http.fetch<T>(url, { query });
+    return request.catch((resp) => {
+      if (resp.body.statusCode === 404 && resp.body.attributes?.code === 'no_matching_indices') {
+        throw new DataViewMissingIndices(resp.body.message);
+      }

-        throw new Error(resp.body.message || resp.body.error || `${resp.body.statusCode} Response`);
-      });
+      throw new Error(resp.body.message || resp.body.error || `${resp.body.statusCode} Response`);
+    });
  }

  private _getUrl(path: string[]) {
@ -51,14 +50,17 @@ export class DataViewsApiClient implements IDataViewsApiClient {
   */
  getFieldsForWildcard(options: GetFieldsOptions) {
    const { pattern, metaFields, type, rollupIndex, allowNoIndex, filter } = options;
-    return this._request<FieldsForWildcardResponse>(this._getUrl(['_fields_for_wildcard']), {
-      pattern,
-      meta_fields: metaFields,
-      type,
-      rollup_index: rollupIndex,
-      allow_no_index: allowNoIndex,
-      filter,
-    }).then((response) => {
+    return this._request<FieldsForWildcardResponse>(
+      this._getUrl(['_fields_for_wildcard']),
+      {
+        pattern,
+        meta_fields: metaFields,
+        type,
+        rollup_index: rollupIndex,
+        allow_no_index: allowNoIndex,
+      },
+      filter ? JSON.stringify({ index_filter: filter }) : undefined
+    ).then((response) => {
      return response || { fields: [], indices: [] };
    });
  }
--- a/src/plugins/data_views/public/mocks.ts
+++ b/src/plugins/data_views/public/mocks.ts
@ -35,6 +35,7 @@ const createStartContract = (): Start => {
    clearCache: jest.fn(),
    getCanSaveSync: jest.fn(),
    getIdsWithTitle: jest.fn(),
+    getFieldsForIndexPattern: jest.fn(),
  } as unknown as jest.Mocked<DataViewsContract>;
 };

--- a/src/plugins/data_views/server/routes/fields_for.ts
+++ b/src/plugins/data_views/server/routes/fields_for.ts
@ -118,5 +118,6 @@ export const registerFieldForWildcard = (
  >
 ) => {
  router.put({ path, validate }, handler);
+  router.post({ path, validate }, handler);
  router.get({ path, validate }, handler);
 };
--- a/src/plugins/unified_field_list/README.md
+++ b/src/plugins/unified_field_list/README.md
@ -12,10 +12,14 @@ This Kibana plugin contains components and services for field list UI (as in fie

 * `loadStats(...)` - returns the loaded field stats (can also work with Ad-hoc data views)

+* `loadFieldExisting(...)` - returns the loaded existing fields (can also work with Ad-hoc data views)
+
 ## Server APIs

 * `/api/unified_field_list/field_stats` - returns the loaded field stats (except for Ad-hoc data views)

+* `/api/unified_field_list/existing_fields/{dataViewId}` - returns the loaded existing fields (except for Ad-hoc data views)
+
 ## Development

 See the [kibana contributing guide](https://github.com/elastic/kibana/blob/main/CONTRIBUTING.md) for instructions setting up your development environment.
--- a/src/plugins/unified_field_list/common/constants.ts
+++ b/src/plugins/unified_field_list/common/constants.ts
@ -8,3 +8,4 @@

 export const BASE_API_PATH = '/api/unified_field_list';
 export const FIELD_STATS_API_PATH = `${BASE_API_PATH}/field_stats`;
+export const FIELD_EXISTING_API_PATH = `${BASE_API_PATH}/existing_fields/{dataViewId}`;
--- a/src/plugins/unified_field_list/common/index.ts
+++ b/src/plugins/unified_field_list/common/index.ts
@ -7,3 +7,4 @@
 */

 export const PLUGIN_ID = 'unifiedFieldList';
+export const FIELD_EXISTENCE_SETTING = 'lens:useFieldExistenceSampling';
--- a/src/plugins/unified_field_list/common/utils/field_existing_utils.test.ts
+++ b/src/plugins/unified_field_list/common/utils/field_existing_utils.test.ts
@ -1,12 +1,18 @@
 /*
 * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
 * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
 */

 import { DataView } from '@kbn/data-views-plugin/common';
-import { legacyExistingFields, existingFields, Field, buildFieldList } from './existing_fields';
+import {
+  legacyExistingFields,
+  existingFields,
+  Field,
+  buildFieldList,
+} from './field_existing_utils';

 describe('existingFields', () => {
  it('should remove missing fields by matching names', () => {
--- a/src/plugins/unified_field_list/common/utils/field_existing_utils.ts
+++ b/src/plugins/unified_field_list/common/utils/field_existing_utils.ts
@ -0,0 +1,266 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import Boom from '@hapi/boom';
+import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import { RuntimeField } from '@kbn/data-views-plugin/common';
+import type { DataViewsContract, DataView, FieldSpec } from '@kbn/data-views-plugin/common';
+import { IKibanaSearchRequest } from '@kbn/data-plugin/common';
+
+export type SearchHandler = (
+  params: IKibanaSearchRequest['params']
+) => Promise<estypes.SearchResponse<Array<estypes.SearchHit<unknown>>>>;
+
+/**
+ * The number of docs to sample to determine field empty status.
+ */
+const SAMPLE_SIZE = 500;
+
+export interface Field {
+  name: string;
+  isScript: boolean;
+  isMeta: boolean;
+  lang?: estypes.ScriptLanguage;
+  script?: string;
+  runtimeField?: RuntimeField;
+}
+
+export async function fetchFieldExistence({
+  search,
+  dataViewsService,
+  dataView,
+  dslQuery = { match_all: {} },
+  fromDate,
+  toDate,
+  timeFieldName,
+  includeFrozen,
+  metaFields,
+  useSampling,
+}: {
+  search: SearchHandler;
+  dataView: DataView;
+  dslQuery: object;
+  fromDate?: string;
+  toDate?: string;
+  timeFieldName?: string;
+  includeFrozen: boolean;
+  useSampling: boolean;
+  metaFields: string[];
+  dataViewsService: DataViewsContract;
+}) {
+  if (useSampling) {
+    return legacyFetchFieldExistenceSampling({
+      search,
+      metaFields,
+      dataView,
+      dataViewsService,
+      dslQuery,
+      fromDate,
+      toDate,
+      timeFieldName,
+      includeFrozen,
+    });
+  }
+
+  const allFields = buildFieldList(dataView, metaFields);
+  const existingFieldList = await dataViewsService.getFieldsForIndexPattern(dataView, {
+    // filled in by data views service
+    pattern: '',
+    filter: toQuery(timeFieldName, fromDate, toDate, dslQuery),
+  });
+  return {
+    indexPatternTitle: dataView.title,
+    existingFieldNames: existingFields(existingFieldList, allFields),
+  };
+}
+
+async function legacyFetchFieldExistenceSampling({
+  search,
+  metaFields,
+  dataView,
+  dslQuery,
+  fromDate,
+  toDate,
+  timeFieldName,
+  includeFrozen,
+}: {
+  search: SearchHandler;
+  metaFields: string[];
+  dataView: DataView;
+  dataViewsService: DataViewsContract;
+  dslQuery: object;
+  fromDate?: string;
+  toDate?: string;
+  timeFieldName?: string;
+  includeFrozen: boolean;
+}) {
+  const fields = buildFieldList(dataView, metaFields);
+  const runtimeMappings = dataView.getRuntimeMappings();
+
+  const docs = await fetchDataViewStats({
+    search,
+    fromDate,
+    toDate,
+    dslQuery,
+    index: dataView.title,
+    timeFieldName: timeFieldName || dataView.timeFieldName,
+    fields,
+    runtimeMappings,
+    includeFrozen,
+  });
+
+  return {
+    indexPatternTitle: dataView.title,
+    existingFieldNames: legacyExistingFields(docs, fields),
+  };
+}
+
+/**
+ * Exported only for unit tests.
+ */
+export function buildFieldList(indexPattern: DataView, metaFields: string[]): Field[] {
+  return indexPattern.fields.map((field) => {
+    return {
+      name: field.name,
+      isScript: !!field.scripted,
+      lang: field.lang,
+      script: field.script,
+      // id is a special case - it doesn't show up in the meta field list,
+      // but as it's not part of source, it has to be handled separately.
+      isMeta: metaFields.includes(field.name) || field.name === '_id',
+      runtimeField: !field.isMapped ? field.runtimeField : undefined,
+    };
+  });
+}
+
+async function fetchDataViewStats({
+  search,
+  index,
+  dslQuery,
+  timeFieldName,
+  fromDate,
+  toDate,
+  fields,
+  runtimeMappings,
+  includeFrozen,
+}: {
+  search: SearchHandler;
+  index: string;
+  dslQuery: object;
+  timeFieldName?: string;
+  fromDate?: string;
+  toDate?: string;
+  fields: Field[];
+  runtimeMappings: estypes.MappingRuntimeFields;
+  includeFrozen: boolean;
+}) {
+  const query = toQuery(timeFieldName, fromDate, toDate, dslQuery);
+
+  const scriptedFields = fields.filter((f) => f.isScript);
+  const response = await search({
+    index,
+    ...(includeFrozen ? { ignore_throttled: false } : {}),
+    body: {
+      size: SAMPLE_SIZE,
+      query,
+      // Sorted queries are usually able to skip entire shards that don't match
+      sort: timeFieldName && fromDate && toDate ? [{ [timeFieldName]: 'desc' }] : [],
+      fields: ['*'],
+      _source: false,
+      runtime_mappings: runtimeMappings,
+      script_fields: scriptedFields.reduce((acc, field) => {
+        acc[field.name] = {
+          script: {
+            lang: field.lang!,
+            source: field.script!,
+          },
+        };
+        return acc;
+      }, {} as Record<string, estypes.ScriptField>),
+      // Small improvement because there is overhead in counting
+      track_total_hits: false,
+      // Per-shard timeout, must be lower than overall. Shards return partial results on timeout
+      timeout: '4500ms',
+    },
+  });
+
+  return response?.hits.hits;
+}
+
+function toQuery(
+  timeFieldName: string | undefined,
+  fromDate: string | undefined,
+  toDate: string | undefined,
+  dslQuery: object
+) {
+  const filter =
+    timeFieldName && fromDate && toDate
+      ? [
+          {
+            range: {
+              [timeFieldName]: {
+                format: 'strict_date_optional_time',
+                gte: fromDate,
+                lte: toDate,
+              },
+            },
+          },
+          dslQuery,
+        ]
+      : [dslQuery];
+
+  const query = {
+    bool: {
+      filter,
+    },
+  };
+  return query;
+}
+
+/**
+ * Exported only for unit tests.
+ */
+export function existingFields(filteredFields: FieldSpec[], allFields: Field[]): string[] {
+  const filteredFieldsSet = new Set(filteredFields.map((f) => f.name));
+
+  return allFields
+    .filter((field) => field.isScript || field.runtimeField || filteredFieldsSet.has(field.name))
+    .map((f) => f.name);
+}
+
+/**
+ * Exported only for unit tests.
+ */
+export function legacyExistingFields(docs: estypes.SearchHit[], fields: Field[]): string[] {
+  const missingFields = new Set(fields);
+
+  for (const doc of docs) {
+    if (missingFields.size === 0) {
+      break;
+    }
+
+    missingFields.forEach((field) => {
+      let fieldStore = doc.fields!;
+      if (field.isMeta) {
+        fieldStore = doc;
+      }
+      const value = fieldStore[field.name];
+      if (Array.isArray(value) && value.length) {
+        missingFields.delete(field);
+      } else if (!Array.isArray(value) && value) {
+        missingFields.delete(field);
+      }
+    });
+  }
+
+  return fields.filter((field) => !missingFields.has(field)).map((f) => f.name);
+}
+
+export function isBoomError(error: { isBoom?: boolean }): error is Boom.Boom {
+  return error.isBoom === true;
+}
--- a/src/plugins/unified_field_list/public/index.ts
+++ b/src/plugins/unified_field_list/public/index.ts
@ -17,6 +17,7 @@ export type {
 export type { FieldStatsProps, FieldStatsServices } from './components/field_stats';
 export { FieldStats } from './components/field_stats';
 export { loadFieldStats } from './services/field_stats';
+export { loadFieldExisting } from './services/field_existing';

 // This exports static code and TypeScript types,
 // as well as, Kibana Platform `plugin()` initializer.
--- a/src/plugins/unified_field_list/public/services/field_existing/index.ts
+++ b/src/plugins/unified_field_list/public/services/field_existing/index.ts
@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export { loadFieldExisting } from './load_field_existing';
--- a/src/plugins/unified_field_list/public/services/field_existing/load_field_existing.ts
+++ b/src/plugins/unified_field_list/public/services/field_existing/load_field_existing.ts
@ -0,0 +1,56 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { IUiSettingsClient } from '@kbn/core/public';
+import { DataPublicPluginStart, UI_SETTINGS } from '@kbn/data-plugin/public';
+import type { DataView, DataViewsContract } from '@kbn/data-views-plugin/common';
+import { lastValueFrom } from 'rxjs';
+import { FIELD_EXISTENCE_SETTING } from '../../../common';
+import { fetchFieldExistence } from '../../../common/utils/field_existing_utils';
+
+interface FetchFieldExistenceParams {
+  data: DataPublicPluginStart;
+  dataView: DataView;
+  fromDate: string;
+  toDate: string;
+  dslQuery: object;
+  timeFieldName?: string;
+  dataViewsService: DataViewsContract;
+  uiSettingsClient: IUiSettingsClient;
+}
+
+export async function loadFieldExisting({
+  data,
+  dslQuery,
+  fromDate,
+  toDate,
+  timeFieldName,
+  dataViewsService,
+  uiSettingsClient,
+  dataView,
+}: FetchFieldExistenceParams) {
+  const includeFrozen = uiSettingsClient.get(UI_SETTINGS.SEARCH_INCLUDE_FROZEN);
+  const useSampling = uiSettingsClient.get(FIELD_EXISTENCE_SETTING);
+  const metaFields = uiSettingsClient.get(UI_SETTINGS.META_FIELDS);
+
+  return await fetchFieldExistence({
+    dslQuery,
+    fromDate,
+    toDate,
+    timeFieldName,
+    dataViewsService,
+    includeFrozen,
+    useSampling,
+    metaFields,
+    dataView,
+    search: async (params) => {
+      const response = await lastValueFrom(data.search.search({ params }));
+      return response.rawResponse;
+    },
+  });
+}
--- a/src/plugins/unified_field_list/server/plugin.ts
+++ b/src/plugins/unified_field_list/server/plugin.ts
@ -14,6 +14,7 @@ import {
  PluginSetup,
 } from './types';
 import { defineRoutes } from './routes';
+import { getUiSettings } from './ui_settings';

 export class UnifiedFieldListPlugin
  implements Plugin<UnifiedFieldListServerPluginSetup, UnifiedFieldListServerPluginStart>
@ -26,8 +27,9 @@ export class UnifiedFieldListPlugin

  public setup(core: CoreSetup<PluginStart>, plugins: PluginSetup) {
    this.logger.debug('unifiedFieldList: Setup');
+    core.uiSettings.register(getUiSettings());

-    defineRoutes(core);
+    defineRoutes(core, this.logger);

    return {};
  }
--- a/src/plugins/unified_field_list/server/routes/existing_fields.ts
+++ b/src/plugins/unified_field_list/server/routes/existing_fields.ts
@ -0,0 +1,103 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { errors } from '@elastic/elasticsearch';
+import { schema } from '@kbn/config-schema';
+import { CoreSetup, Logger } from '@kbn/core/server';
+import { UI_SETTINGS } from '@kbn/data-plugin/server';
+import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+
+import { fetchFieldExistence, isBoomError } from '../../common/utils/field_existing_utils';
+import { FIELD_EXISTING_API_PATH } from '../../common/constants';
+import { FIELD_EXISTENCE_SETTING } from '../../common';
+import { PluginStart } from '../types';
+
+export async function existingFieldsRoute(setup: CoreSetup<PluginStart>, logger: Logger) {
+  const router = setup.http.createRouter();
+
+  router.post(
+    {
+      path: FIELD_EXISTING_API_PATH,
+      validate: {
+        params: schema.object({
+          dataViewId: schema.string(),
+        }),
+        body: schema.object({
+          dslQuery: schema.object({}, { unknowns: 'allow' }),
+          fromDate: schema.maybe(schema.string()),
+          toDate: schema.maybe(schema.string()),
+          timeFieldName: schema.maybe(schema.string()),
+        }),
+      },
+    },
+    async (context, req, res) => {
+      const [{ savedObjects, elasticsearch, uiSettings }, { dataViews }] =
+        await setup.getStartServices();
+      const savedObjectsClient = savedObjects.getScopedClient(req);
+      const uiSettingsClient = uiSettings.asScopedToClient(savedObjectsClient);
+      const [includeFrozen, useSampling, metaFields] = await Promise.all([
+        uiSettingsClient.get(UI_SETTINGS.SEARCH_INCLUDE_FROZEN),
+        uiSettingsClient.get(FIELD_EXISTENCE_SETTING),
+        uiSettingsClient.get(UI_SETTINGS.META_FIELDS),
+      ]);
+      const esClient = elasticsearch.client.asScoped(req).asCurrentUser;
+      try {
+        const dataViewsService = await dataViews.dataViewsServiceFactory(
+          savedObjectsClient,
+          esClient
+        );
+        return res.ok({
+          body: await fetchFieldExistence({
+            ...req.body,
+            dataViewsService,
+            includeFrozen,
+            useSampling,
+            metaFields,
+            dataView: await dataViewsService.get(req.params.dataViewId),
+            search: async (params) => {
+              const contextCore = await context.core;
+              return await contextCore.elasticsearch.client.asCurrentUser.search<
+                estypes.SearchHit[]
+              >(
+                { ...params },
+                {
+                  // Global request timeout. Will cancel the request if exceeded. Overrides the elasticsearch.requestTimeout
+                  requestTimeout: '5000ms',
+                  // Fails fast instead of retrying- default is to retry
+                  maxRetries: 0,
+                }
+              );
+            },
+          }),
+        });
+      } catch (e) {
+        if (e instanceof errors.TimeoutError) {
+          logger.info(`Field existence check timed out on ${req.params.dataViewId}`);
+          // 408 is Request Timeout
+          return res.customError({ statusCode: 408, body: e.message });
+        }
+        logger.info(
+          `Field existence check failed on ${req.params.dataViewId}: ${
+            isBoomError(e) ? e.output.payload.message : e.message
+          }`
+        );
+        if (e instanceof errors.ResponseError && e.statusCode === 404) {
+          return res.notFound({ body: e.message });
+        }
+        if (isBoomError(e)) {
+          if (e.output.statusCode === 404) {
+            return res.notFound({ body: e.output.payload.message });
+          }
+          throw new Error(e.output.payload.message);
+        } else {
+          throw e;
+        }
+      }
+    }
+  );
+}
--- a/src/plugins/unified_field_list/server/routes/index.ts
+++ b/src/plugins/unified_field_list/server/routes/index.ts
@ -6,10 +6,12 @@
 * Side Public License, v 1.
 */

-import { CoreSetup } from '@kbn/core/server';
+import { CoreSetup, Logger } from '@kbn/core/server';
 import { PluginStart } from '../types';
+import { existingFieldsRoute } from './existing_fields';
 import { initFieldStatsRoute } from './field_stats';

-export function defineRoutes(setup: CoreSetup<PluginStart>) {
+export function defineRoutes(setup: CoreSetup<PluginStart>, logger: Logger) {
  initFieldStatsRoute(setup);
+  existingFieldsRoute(setup, logger);
 }
--- a/src/plugins/unified_field_list/server/ui_settings.ts
+++ b/src/plugins/unified_field_list/server/ui_settings.ts
@ -1,34 +1,37 @@
 /*
 * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
 * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
 */

 import { i18n } from '@kbn/i18n';
 import { schema } from '@kbn/config-schema';

 import { UiSettingsParams } from '@kbn/core/server';
-
-export const FIELD_EXISTENCE_SETTING = 'lens:useFieldExistenceSampling';
+import { FIELD_EXISTENCE_SETTING } from '../common';

 export const getUiSettings: () => Record<string, UiSettingsParams> = () => ({
  [FIELD_EXISTENCE_SETTING]: {
-    name: i18n.translate('xpack.lens.advancedSettings.useFieldExistenceSampling.title', {
+    name: i18n.translate('unifiedFieldList.advancedSettings.useFieldExistenceSampling.title', {
      defaultMessage: 'Use field existence sampling',
    }),
    value: false,
    description: i18n.translate(
-      'xpack.lens.advancedSettings.useFieldExistenceSampling.description',
+      'unifiedFieldList.advancedSettings.useFieldExistenceSampling.description',
      {
        defaultMessage:
          'If enabled, document sampling is used to determine field existence (available or empty) for the Lens field list instead of relying on index mappings.',
      }
    ),
    deprecation: {
-      message: i18n.translate('xpack.lens.advancedSettings.useFieldExistenceSampling.deprecation', {
-        defaultMessage: 'This setting is deprecated and will not be supported as of 8.6.',
-      }),
+      message: i18n.translate(
+        'unifiedFieldList.advancedSettings.useFieldExistenceSampling.deprecation',
+        {
+          defaultMessage: 'This setting is deprecated and will not be supported as of 8.6.',
+        }
+      ),
      docLinksKey: 'visualizationSettings',
    },
    category: ['visualization'],
--- a/test/api_integration/apis/unified_field_list/existing_fields.ts
+++ b/test/api_integration/apis/unified_field_list/existing_fields.ts
@ -1,8 +1,9 @@
 /*
 * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
 * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
 */

 import expect from '@kbn/expect';
@ -29,26 +30,32 @@ export default ({ getService }: FtrProviderContext) => {
  const esArchiver = getService('esArchiver');
  const supertest = getService('supertest');
  const kibanaServer = getService('kibanaServer');
+  const dataViewId = 'existence_index';
+  const API_PATH = `/api/unified_field_list/existing_fields/${dataViewId}`;

  describe('existing_fields apis', () => {
    before(async () => {
-      await esArchiver.load('x-pack/test/api_integration/es_archives/lens/constant_keyword');
+      await esArchiver.load(
+        'test/api_integration/fixtures/es_archiver/index_patterns/constant_keyword'
+      );
      await kibanaServer.importExport.load(
-        'x-pack/test/api_integration/fixtures/kbn_archiver/lens/constant_keyword.json'
+        'test/api_integration/fixtures/kbn_archiver/index_patterns/constant_keyword.json'
      );
    });

    after(async () => {
-      await esArchiver.unload('x-pack/test/api_integration/es_archives/lens/constant_keyword');
+      await esArchiver.unload(
+        'test/api_integration/fixtures/es_archiver/index_patterns/constant_keyword'
+      );
      await kibanaServer.importExport.unload(
-        'x-pack/test/api_integration/fixtures/kbn_archiver/lens/constant_keyword.json'
+        'test/api_integration/fixtures/kbn_archiver/index_patterns/constant_keyword.json'
      );
    });

    describe('existence', () => {
      it('should find which fields exist in the sample documents', async () => {
        const { body } = await supertest
-          .post(`/api/lens/existing_fields/existence_index`)
+          .post(API_PATH)
          .set(COMMON_HEADERS)
          .send({
            dslQuery: {
@ -75,7 +82,7 @@ export default ({ getService }: FtrProviderContext) => {
        ];

        const { body } = await supertest
-          .post(`/api/lens/existing_fields/existence_index`)
+          .post(API_PATH)
          .set(COMMON_HEADERS)
          .send({
            dslQuery: {
@ -102,7 +109,7 @@ export default ({ getService }: FtrProviderContext) => {
        ];

        const { body } = await supertest
-          .post(`/api/lens/existing_fields/existence_index`)
+          .post(API_PATH)
          .set(COMMON_HEADERS)
          .send({
            dslQuery: {
@ -129,7 +136,7 @@ export default ({ getService }: FtrProviderContext) => {
        ];

        const { body } = await supertest
-          .post(`/api/lens/existing_fields/existence_index`)
+          .post(API_PATH)
          .set(COMMON_HEADERS)
          .send({
            dslQuery: {
--- a/test/api_integration/apis/unified_field_list/index.ts
+++ b/test/api_integration/apis/unified_field_list/index.ts
@ -10,6 +10,8 @@ import { FtrProviderContext } from '../../ftr_provider_context';

 export default function lensApiIntegrationTests({ loadTestFile }: FtrProviderContext) {
  describe('UnifiedFieldList', () => {
+    loadTestFile(require.resolve('./existing_fields'));
+    loadTestFile(require.resolve('./legacy_existing_fields'));
    loadTestFile(require.resolve('./field_stats'));
  });
 }
--- a/test/api_integration/apis/unified_field_list/legacy_existing_fields.ts
+++ b/test/api_integration/apis/unified_field_list/legacy_existing_fields.ts
@ -1,8 +1,9 @@
 /*
 * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
 * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
 */

 import expect from '@kbn/expect';
@ -185,7 +186,7 @@ export default ({ getService }: FtrProviderContext) => {
    describe('existence', () => {
      it('should find which fields exist in the sample documents', async () => {
        const { body } = await supertest
-          .post(`/api/lens/existing_fields/${encodeURIComponent('logstash-*')}`)
+          .post(`/api/unified_field_list/existing_fields/${encodeURIComponent('logstash-*')}`)
          .set(COMMON_HEADERS)
          .send({
            dslQuery: {
@ -204,7 +205,7 @@ export default ({ getService }: FtrProviderContext) => {

      it('should succeed for thousands of fields', async () => {
        const { body } = await supertest
-          .post(`/api/lens/existing_fields/${encodeURIComponent('metricbeat-*')}`)
+          .post(`/api/unified_field_list/existing_fields/${encodeURIComponent('metricbeat-*')}`)
          .set(COMMON_HEADERS)
          .send({
            dslQuery: { match_all: {} },
@ -255,7 +256,7 @@ export default ({ getService }: FtrProviderContext) => {
        ];

        const { body } = await supertest
-          .post(`/api/lens/existing_fields/${encodeURIComponent('logstash-*')}`)
+          .post(`/api/unified_field_list/existing_fields/${encodeURIComponent('logstash-*')}`)
          .set(COMMON_HEADERS)
          .send({
            dslQuery: {
--- a/test/api_integration/fixtures/es_archiver/index_patterns/constant_keyword/data.json
+++ b/test/api_integration/fixtures/es_archiver/index_patterns/constant_keyword/data.json
--- a/test/api_integration/fixtures/es_archiver/index_patterns/constant_keyword/mappings.json
+++ b/test/api_integration/fixtures/es_archiver/index_patterns/constant_keyword/mappings.json
--- a/test/api_integration/fixtures/kbn_archiver/index_patterns/constant_keyword.json
+++ b/test/api_integration/fixtures/kbn_archiver/index_patterns/constant_keyword.json
@ -0,0 +1,16 @@
+{
+  "attributes": {
+      "timeFieldName": "ts",
+      "title": "existence_index_*",
+      "runtimeFieldMap":"{\"data_view_runtime_field\":{\"type\":\"keyword\",\"script\":{\"source\":\"emit('a')\"}}}"
+  },
+  "coreMigrationVersion": "8.0.0",
+  "id": "existence_index",
+  "migrationVersion": {
+      "index-pattern": "7.11.0"
+  },
+  "references": [],
+  "type": "index-pattern",
+  "updated_at": "2018-12-21T00:43:07.096Z",
+  "version": "WzEzLDJd"
+}
--- a/x-pack/plugins/lens/public/app_plugin/app.tsx
+++ b/x-pack/plugins/lens/public/app_plugin/app.tsx
@ -62,6 +62,9 @@ export function App({

  const {
    data,
+    dataViews,
+    uiActions,
+    uiSettings,
    chrome,
    inspector: lensInspector,
    application,
@ -367,10 +370,10 @@ export function App({
  const indexPatternService = useMemo(
    () =>
      createIndexPatternService({
-        dataViews: lensAppServices.dataViews,
-        uiSettings: lensAppServices.uiSettings,
-        uiActions: lensAppServices.uiActions,
-        core: { http, notifications },
+        dataViews,
+        uiActions,
+        core: { http, notifications, uiSettings },
+        data,
        updateIndexPatterns: (newIndexPatternsState, options) => {
          dispatch(updateIndexPatterns(newIndexPatternsState));
          if (options?.applyImmediately) {
@ -384,7 +387,7 @@ export function App({
          }
        },
      }),
-    [dispatch, http, notifications, lensAppServices]
+    [dataViews, uiActions, http, notifications, uiSettings, data, dispatch]
  );

  return (
--- a/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.test.tsx
+++ b/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.test.tsx
@ -9,7 +9,10 @@ import React from 'react';
 import ReactDOM from 'react-dom';
 import { createMockedDragDropContext } from './mocks';
 import { dataPluginMock } from '@kbn/data-plugin/public/mocks';
-import { dataViewPluginMocks } from '@kbn/data-views-plugin/public/mocks';
+import {
+  dataViewPluginMocks,
+  Start as DataViewPublicStart,
+} from '@kbn/data-views-plugin/public/mocks';
 import { InnerIndexPatternDataPanel, IndexPatternDataPanel, Props } from './datapanel';
 import { FieldList } from './field_list';
 import { FieldItem } from './field_item';
@ -32,6 +35,8 @@ import { createMockFramePublicAPI } from '../mocks';
 import { DataViewsState } from '../state_management';
 import { ExistingFieldsMap, FramePublicAPI, IndexPattern } from '../types';
 import { IndexPatternServiceProps } from '../indexpattern_service/service';
+import { FieldSpec, DataView } from '@kbn/data-views-plugin/public';
+import { UI_SETTINGS } from '@kbn/data-plugin/public';

 const fieldsOne = [
  {
@ -275,13 +280,33 @@ const dslQuery = { bool: { must: [], filter: [], should: [], must_not: [] } };
 ReactDOM.createPortal = jest.fn((element) => element);

 describe('IndexPattern Data Panel', () => {
+  const indexPatterns = {
+    a: {
+      id: 'a',
+      title: 'aaa',
+      timeFieldName: 'atime',
+      fields: [{ name: 'aaa_field_1' }, { name: 'aaa_field_2' }],
+      getFieldByName: getFieldByNameFactory([]),
+      hasRestrictions: false,
+    },
+    b: {
+      id: 'b',
+      title: 'bbb',
+      timeFieldName: 'btime',
+      fields: [{ name: 'bbb_field_1' }, { name: 'bbb_field_2' }],
+      getFieldByName: getFieldByNameFactory([]),
+      hasRestrictions: false,
+    },
+  };
  let defaultProps: Parameters<typeof InnerIndexPatternDataPanel>[0] & {
    showNoDataPopover: () => void;
  };
  let core: ReturnType<typeof coreMock['createStart']>;
+  let dataViews: DataViewPublicStart;

  beforeEach(() => {
    core = coreMock.createStart();
+    dataViews = dataViewPluginMocks.createStartContract();
    defaultProps = {
      data: dataPluginMock.createStartContract(),
      dataViews: dataViewPluginMocks.createStartContract(),
@ -302,7 +327,7 @@ describe('IndexPattern Data Panel', () => {
      dropOntoWorkspace: jest.fn(),
      hasSuggestionForField: jest.fn(() => false),
      uiActions: uiActionsPluginMock.createStartContract(),
-      indexPatternService: createIndexPatternServiceMock(),
+      indexPatternService: createIndexPatternServiceMock({ core, dataViews }),
      frame: getFrameAPIMock(),
    };
  });
@ -328,19 +353,29 @@ describe('IndexPattern Data Panel', () => {

  describe('loading existence data', () => {
    function testProps(updateIndexPatterns: IndexPatternServiceProps['updateIndexPatterns']) {
-      core.http.post.mockImplementation(async (path) => {
-        const parts = (path as unknown as string).split('/');
-        const indexPatternTitle = parts[parts.length - 1];
-        return {
-          indexPatternTitle: `${indexPatternTitle}_testtitle`,
-          existingFieldNames: ['field_1', 'field_2'].map(
-            (fieldName) => `${indexPatternTitle}_${fieldName}`
-          ),
-        };
+      core.uiSettings.get.mockImplementation((key: string) => {
+        if (key === UI_SETTINGS.META_FIELDS) {
+          return [];
+        }
+      });
+      dataViews.getFieldsForIndexPattern.mockImplementation((dataView) => {
+        return Promise.resolve([
+          { name: `${dataView.title}_field_1` },
+          { name: `${dataView.title}_field_2` },
+        ]) as Promise<FieldSpec[]>;
+      });
+      dataViews.get.mockImplementation(async (id: string) => {
+        return [indexPatterns.a, indexPatterns.b].find(
+          (indexPattern) => indexPattern.id === id
+        ) as unknown as DataView;
      });
      return {
        ...defaultProps,
-        indexPatternService: createIndexPatternServiceMock({ updateIndexPatterns, core }),
+        indexPatternService: createIndexPatternServiceMock({
+          updateIndexPatterns,
+          core,
+          dataViews,
+        }),
        setState: jest.fn(),
        dragDropContext: {
          ...createMockedDragDropContext(),
@ -352,24 +387,7 @@ describe('IndexPattern Data Panel', () => {
            indexPatternRefs: [],
            existingFields: {},
            isFirstExistenceFetch: false,
-            indexPatterns: {
-              a: {
-                id: 'a',
-                title: 'aaa',
-                timeFieldName: 'atime',
-                fields: [],
-                getFieldByName: getFieldByNameFactory([]),
-                hasRestrictions: false,
-              },
-              b: {
-                id: 'b',
-                title: 'bbb',
-                timeFieldName: 'btime',
-                fields: [],
-                getFieldByName: getFieldByNameFactory([]),
-                hasRestrictions: false,
-              },
-            },
+            indexPatterns,
          },
        } as unknown as FramePublicAPI,
        state: {
@ -418,9 +436,9 @@ describe('IndexPattern Data Panel', () => {
      expect(updateIndexPatterns).toHaveBeenCalledWith(
        {
          existingFields: {
-            a_testtitle: {
-              a_field_1: true,
-              a_field_2: true,
+            aaa: {
+              aaa_field_1: true,
+              aaa_field_2: true,
            },
          },
          isFirstExistenceFetch: false,
@ -438,13 +456,13 @@ describe('IndexPattern Data Panel', () => {
      expect(updateIndexPatterns).toHaveBeenCalledWith(
        {
          existingFields: {
-            a_testtitle: {
-              a_field_1: true,
-              a_field_2: true,
+            aaa: {
+              aaa_field_1: true,
+              aaa_field_2: true,
            },
-            b_testtitle: {
-              b_field_1: true,
-              b_field_2: true,
+            bbb: {
+              bbb_field_1: true,
+              bbb_field_2: true,
            },
          },
          isFirstExistenceFetch: false,
@ -473,32 +491,41 @@ describe('IndexPattern Data Panel', () => {
      });

      expect(updateIndexPatterns).toHaveBeenCalledTimes(2);
-      expect(core.http.post).toHaveBeenCalledTimes(2);
+      expect(dataViews.getFieldsForIndexPattern).toHaveBeenCalledTimes(2);
+      expect(dataViews.get).toHaveBeenCalledTimes(2);

-      expect(core.http.post).toHaveBeenCalledWith('/api/lens/existing_fields/a', {
-        body: JSON.stringify({
-          dslQuery,
-          fromDate: '2019-01-01',
-          toDate: '2020-01-01',
-          timeFieldName: 'atime',
-        }),
+      const firstCall = dataViews.getFieldsForIndexPattern.mock.calls[0];
+      expect(firstCall[0]).toEqual(indexPatterns.a);
+      expect(firstCall[1]?.filter?.bool?.filter).toContainEqual(dslQuery);
+      expect(firstCall[1]?.filter?.bool?.filter).toContainEqual({
+        range: {
+          atime: {
+            format: 'strict_date_optional_time',
+            gte: '2019-01-01',
+            lte: '2020-01-01',
+          },
+        },
      });

-      expect(core.http.post).toHaveBeenCalledWith('/api/lens/existing_fields/a', {
-        body: JSON.stringify({
-          dslQuery,
-          fromDate: '2019-01-01',
-          toDate: '2020-01-02',
-          timeFieldName: 'atime',
-        }),
+      const secondCall = dataViews.getFieldsForIndexPattern.mock.calls[1];
+      expect(secondCall[0]).toEqual(indexPatterns.a);
+      expect(secondCall[1]?.filter?.bool?.filter).toContainEqual(dslQuery);
+      expect(secondCall[1]?.filter?.bool?.filter).toContainEqual({
+        range: {
+          atime: {
+            format: 'strict_date_optional_time',
+            gte: '2019-01-01',
+            lte: '2020-01-02',
+          },
+        },
      });

      expect(updateIndexPatterns).toHaveBeenCalledWith(
        {
          existingFields: {
-            a_testtitle: {
-              a_field_1: true,
-              a_field_2: true,
+            aaa: {
+              aaa_field_1: true,
+              aaa_field_2: true,
            },
          },
          isFirstExistenceFetch: false,
@ -521,34 +548,42 @@ describe('IndexPattern Data Panel', () => {

      expect(updateIndexPatterns).toHaveBeenCalledTimes(2);

-      expect(core.http.post).toHaveBeenCalledWith('/api/lens/existing_fields/a', {
-        body: JSON.stringify({
-          dslQuery,
-          fromDate: '2019-01-01',
-          toDate: '2020-01-01',
-          timeFieldName: 'atime',
-        }),
+      const secondCall = dataViews.getFieldsForIndexPattern.mock.calls[1];
+      expect(secondCall[0]).toEqual(indexPatterns.a);
+      expect(secondCall[1]?.filter?.bool?.filter).toContainEqual(dslQuery);
+      expect(secondCall[1]?.filter?.bool?.filter).toContainEqual({
+        range: {
+          atime: {
+            format: 'strict_date_optional_time',
+            gte: '2019-01-01',
+            lte: '2020-01-01',
+          },
+        },
      });

-      expect(core.http.post).toHaveBeenCalledWith('/api/lens/existing_fields/b', {
-        body: JSON.stringify({
-          dslQuery,
-          fromDate: '2019-01-01',
-          toDate: '2020-01-01',
-          timeFieldName: 'btime',
-        }),
+      const thirdCall = dataViews.getFieldsForIndexPattern.mock.calls[2];
+      expect(thirdCall[0]).toEqual(indexPatterns.b);
+      expect(thirdCall[1]?.filter?.bool?.filter).toContainEqual(dslQuery);
+      expect(thirdCall[1]?.filter?.bool?.filter).toContainEqual({
+        range: {
+          btime: {
+            format: 'strict_date_optional_time',
+            gte: '2019-01-01',
+            lte: '2020-01-01',
+          },
+        },
      });

      expect(updateIndexPatterns).toHaveBeenCalledWith(
        {
          existingFields: {
-            a_testtitle: {
-              a_field_1: true,
-              a_field_2: true,
+            aaa: {
+              aaa_field_1: true,
+              aaa_field_2: true,
            },
-            b_testtitle: {
-              b_field_1: true,
-              b_field_2: true,
+            bbb: {
+              bbb_field_1: true,
+              bbb_field_2: true,
            },
          },
          isFirstExistenceFetch: false,
@ -573,20 +608,15 @@ describe('IndexPattern Data Panel', () => {
      let overlapCount = 0;
      const props = testProps(updateIndexPatterns);

-      core.http.post.mockImplementation((path) => {
+      dataViews.getFieldsForIndexPattern.mockImplementation((dataView) => {
        if (queryCount) {
          ++overlapCount;
        }
        ++queryCount;
-
-        const parts = (path as unknown as string).split('/');
-        const indexPatternTitle = parts[parts.length - 1];
-        const result = Promise.resolve({
-          indexPatternTitle,
-          existingFieldNames: ['field_1', 'field_2'].map(
-            (fieldName) => `${indexPatternTitle}_${fieldName}`
-          ),
-        });
+        const result = Promise.resolve([
+          { name: `${dataView.title}_field_1` },
+          { name: `${dataView.title}_field_2` },
+        ]) as Promise<FieldSpec[]>;

        result.then(() => --queryCount);

@ -613,7 +643,7 @@ describe('IndexPattern Data Panel', () => {
        inst.update();
      });

-      expect(core.http.post).toHaveBeenCalledTimes(2);
+      expect(dataViews.getFieldsForIndexPattern).toHaveBeenCalledTimes(2);
      expect(overlapCount).toEqual(0);
    });

@ -628,13 +658,14 @@ describe('IndexPattern Data Panel', () => {
      };
      await testExistenceLoading(props, undefined, undefined);

-      expect((props.core.http.post as jest.Mock).mock.calls[0][1].body).toContain(
-        JSON.stringify({
+      const firstCall = dataViews.getFieldsForIndexPattern.mock.calls[0];
+      expect(firstCall[1]?.filter?.bool?.filter).toContainEqual({
+        bool: {
          must_not: {
            match_all: {},
          },
-        })
-      );
+        },
+      });
    });
  });

--- a/x-pack/plugins/lens/public/indexpattern_service/loader.test.ts
+++ b/x-pack/plugins/lens/public/indexpattern_service/loader.test.ts
@ -5,10 +5,7 @@
 * 2.0.
 */

-import { HttpHandler } from '@kbn/core/public';
-import { last } from 'lodash';
-import { DataViewsContract } from '@kbn/data-views-plugin/public';
-import { createHttpFetchError } from '@kbn/core-http-browser-mocks';
+import { DataViewsContract, DataViewSpec, FieldSpec } from '@kbn/data-views-plugin/public';
 import { IndexPattern, IndexPatternField } from '../types';
 import {
  ensureIndexPattern,
@ -18,6 +15,12 @@ import {
 } from './loader';
 import { sampleIndexPatterns, mockDataViewsService } from './mocks';
 import { documentField } from '../indexpattern_datasource/document_field';
+import { coreMock } from '@kbn/core/public/mocks';
+import { dataViewPluginMocks } from '@kbn/data-views-plugin/public/mocks';
+import type { DataView } from '@kbn/data-views-plugin/public';
+import { dataPluginMock } from '@kbn/data-plugin/public/mocks';
+import { UI_SETTINGS } from '@kbn/data-plugin/public';
+import { createHttpFetchError } from '@kbn/core-http-browser-mocks';

 describe('loader', () => {
  describe('loadIndexPatternRefs', () => {
@ -262,6 +265,10 @@ describe('loader', () => {
  });

  describe('syncExistingFields', () => {
+    const core = coreMock.createStart();
+    const dataViews = dataViewPluginMocks.createStartContract();
+    const data = dataPluginMock.createStartContract();
+
    const dslQuery = {
      bool: {
        must: [],
@ -273,27 +280,51 @@ describe('loader', () => {

    function getIndexPatternList() {
      return [
-        { id: '1', title: '1', fields: [], hasRestrictions: false },
-        { id: '2', title: '1', fields: [], hasRestrictions: false },
-        { id: '3', title: '1', fields: [], hasRestrictions: false },
+        {
+          id: '1',
+          title: '1',
+          fields: [{ name: 'ip1_field_1' }, { name: 'ip1_field_2' }],
+          hasRestrictions: false,
+        },
+        {
+          id: '2',
+          title: '2',
+          fields: [{ name: 'ip2_field_1' }, { name: 'ip2_field_2' }],
+          hasRestrictions: false,
+        },
+        {
+          id: '3',
+          title: '3',
+          fields: [{ name: 'ip3_field_1' }, { name: 'ip3_field_2' }],
+          hasRestrictions: false,
+        },
      ] as unknown as IndexPattern[];
    }

+    beforeEach(() => {
+      core.uiSettings.get.mockImplementation((key: string) => {
+        if (key === UI_SETTINGS.META_FIELDS) {
+          return [];
+        }
+      });
+      dataViews.get.mockImplementation((id: string) =>
+        Promise.resolve(
+          getIndexPatternList().find(
+            (indexPattern) => indexPattern.id === id
+          ) as unknown as DataView
+        )
+      );
+    });
+
    it('should call once for each index pattern', async () => {
      const updateIndexPatterns = jest.fn();
-      const fetchJson = jest.fn((path: string) => {
-        const indexPatternTitle = last(path.split('/'));
-        return {
-          indexPatternTitle,
-          existingFieldNames: ['field_1', 'field_2'].map(
-            (fieldName) => `ip${indexPatternTitle}_${fieldName}`
-          ),
-        };
-      }) as unknown as HttpHandler;
+      dataViews.getFieldsForIndexPattern.mockImplementation(
+        (dataView: DataViewSpec | DataView) =>
+          Promise.resolve(dataView.fields) as Promise<FieldSpec[]>
+      );

      await syncExistingFields({
        dateRange: { fromDate: '1900-01-01', toDate: '2000-01-01' },
-        fetchJson,
        indexPatternList: getIndexPatternList(),
        updateIndexPatterns,
        dslQuery,
@ -301,9 +332,13 @@ describe('loader', () => {
        currentIndexPatternTitle: 'abc',
        isFirstExistenceFetch: false,
        existingFields: {},
+        core,
+        data,
+        dataViews,
      });

-      expect(fetchJson).toHaveBeenCalledTimes(3);
+      expect(dataViews.get).toHaveBeenCalledTimes(3);
+      expect(dataViews.getFieldsForIndexPattern).toHaveBeenCalledTimes(3);
      expect(updateIndexPatterns).toHaveBeenCalledTimes(1);

      const [newState, options] = updateIndexPatterns.mock.calls[0];
@ -322,20 +357,16 @@ describe('loader', () => {
    it('should call onNoData callback if current index pattern returns no fields', async () => {
      const updateIndexPatterns = jest.fn();
      const onNoData = jest.fn();
-      const fetchJson = jest.fn((path: string) => {
-        const indexPatternTitle = last(path.split('/'));
-        return {
-          indexPatternTitle,
-          existingFieldNames:
-            indexPatternTitle === '1'
-              ? ['field_1', 'field_2'].map((fieldName) => `${indexPatternTitle}_${fieldName}`)
-              : [],
-        };
-      }) as unknown as HttpHandler;
+      dataViews.getFieldsForIndexPattern.mockImplementation(
+        async (dataView: DataViewSpec | DataView) => {
+          return (dataView.title === '1'
+            ? [{ name: `${dataView.title}_field_1` }, { name: `${dataView.title}_field_2` }]
+            : []) as unknown as Promise<FieldSpec[]>;
+        }
+      );

      const args = {
        dateRange: { fromDate: '1900-01-01', toDate: '2000-01-01' },
-        fetchJson,
        indexPatternList: getIndexPatternList(),
        updateIndexPatterns,
        dslQuery,
@ -343,6 +374,9 @@ describe('loader', () => {
        currentIndexPatternTitle: 'abc',
        isFirstExistenceFetch: false,
        existingFields: {},
+        core,
+        data,
+        dataViews,
      };

      await syncExistingFields(args);
@ -355,15 +389,14 @@ describe('loader', () => {

    it('should set all fields to available and existence error flag if the request fails', async () => {
      const updateIndexPatterns = jest.fn();
-      const fetchJson = jest.fn((path: string) => {
-        return new Promise((resolve, reject) => {
+      dataViews.getFieldsForIndexPattern.mockImplementation(() => {
+        return new Promise((_, reject) => {
          reject(new Error());
        });
-      }) as unknown as HttpHandler;
+      });

      const args = {
        dateRange: { fromDate: '1900-01-01', toDate: '2000-01-01' },
-        fetchJson,
        indexPatternList: [
          {
            id: '1',
@ -378,6 +411,9 @@ describe('loader', () => {
        currentIndexPatternTitle: 'abc',
        isFirstExistenceFetch: false,
        existingFields: {},
+        core,
+        data,
+        dataViews,
      };

      await syncExistingFields(args);
@ -395,8 +431,8 @@ describe('loader', () => {

    it('should set all fields to available and existence error flag if the request times out', async () => {
      const updateIndexPatterns = jest.fn();
-      const fetchJson = jest.fn((path: string) => {
-        return new Promise((resolve, reject) => {
+      dataViews.getFieldsForIndexPattern.mockImplementation(() => {
+        return new Promise((_, reject) => {
          const error = createHttpFetchError(
            'timeout',
            'error',
@ -405,11 +441,10 @@ describe('loader', () => {
          );
          reject(error);
        });
-      }) as unknown as HttpHandler;
+      });

      const args = {
        dateRange: { fromDate: '1900-01-01', toDate: '2000-01-01' },
-        fetchJson,
        indexPatternList: [
          {
            id: '1',
@ -424,6 +459,9 @@ describe('loader', () => {
        currentIndexPatternTitle: 'abc',
        isFirstExistenceFetch: false,
        existingFields: {},
+        core,
+        data,
+        dataViews,
      };

      await syncExistingFields(args);
--- a/x-pack/plugins/lens/public/indexpattern_service/loader.ts
+++ b/x-pack/plugins/lens/public/indexpattern_service/loader.ts
@ -8,10 +8,12 @@
 import { isNestedField } from '@kbn/data-views-plugin/common';
 import type { DataViewsContract, DataView, DataViewSpec } from '@kbn/data-views-plugin/public';
 import { keyBy } from 'lodash';
-import { HttpSetup } from '@kbn/core/public';
+import { CoreStart } from '@kbn/core/public';
+import { DataPublicPluginStart } from '@kbn/data-plugin/public';
+import { loadFieldExisting } from '@kbn/unified-field-list-plugin/public';
 import { IndexPattern, IndexPatternField, IndexPatternMap, IndexPatternRef } from '../types';
 import { documentField } from '../indexpattern_datasource/document_field';
-import { BASE_API_URL, DateRange, ExistingFields } from '../../common';
+import { DateRange } from '../../common';
 import { DataViewsState } from '../state_management';

 type ErrorHandler = (err: Error) => void;
@ -231,41 +233,40 @@ export async function ensureIndexPattern({

 async function refreshExistingFields({
  dateRange,
-  fetchJson,
  indexPatternList,
  dslQuery,
+  core,
+  data,
+  dataViews,
 }: {
  dateRange: DateRange;
  indexPatternList: IndexPattern[];
-  fetchJson: HttpSetup['post'];
  dslQuery: object;
+  core: Pick<CoreStart, 'http' | 'notifications' | 'uiSettings'>;
+  data: DataPublicPluginStart;
+  dataViews: DataViewsContract;
 }) {
  try {
    const emptinessInfo = await Promise.all(
-      indexPatternList.map((pattern) => {
+      indexPatternList.map(async (pattern) => {
        if (pattern.hasRestrictions) {
          return {
            indexPatternTitle: pattern.title,
            existingFieldNames: pattern.fields.map((field) => field.name),
          };
        }
-        const body: Record<string, string | object> = {
+
+        const dataView = await dataViews.get(pattern.id);
+        return await loadFieldExisting({
          dslQuery,
          fromDate: dateRange.fromDate,
          toDate: dateRange.toDate,
-        };
-
-        if (pattern.timeFieldName) {
-          body.timeFieldName = pattern.timeFieldName;
-        }
-
-        if (pattern.spec) {
-          body.spec = pattern.spec;
-        }
-
-        return fetchJson(`${BASE_API_URL}/existing_fields/${pattern.id}`, {
-          body: JSON.stringify(body),
-        }) as Promise<ExistingFields>;
+          timeFieldName: pattern.timeFieldName,
+          data,
+          uiSettingsClient: core.uiSettings,
+          dataViewsService: dataViews,
+          dataView,
+        });
      })
    );
    return { result: emptinessInfo, status: 200 };
@ -289,7 +290,6 @@ export async function syncExistingFields({
  dateRange: DateRange;
  indexPatternList: IndexPattern[];
  existingFields: Record<string, Record<string, boolean>>;
-  fetchJson: HttpSetup['post'];
  updateIndexPatterns: (
    newFieldState: FieldsPropsFromDataViewsState,
    options: { applyImmediately: boolean }
@ -298,6 +298,9 @@ export async function syncExistingFields({
  currentIndexPatternTitle: string;
  dslQuery: object;
  onNoData?: () => void;
+  core: Pick<CoreStart, 'http' | 'notifications' | 'uiSettings'>;
+  data: DataPublicPluginStart;
+  dataViews: DataViewsContract;
 }) {
  const { indexPatternList } = requestOptions;
  const newExistingFields = { ...existingFields };
--- a/x-pack/plugins/lens/public/indexpattern_service/service.ts
+++ b/x-pack/plugins/lens/public/indexpattern_service/service.ts
@ -6,8 +6,9 @@
 */

 import type { DataViewsContract, DataView } from '@kbn/data-views-plugin/public';
-import type { CoreStart, IUiSettingsClient } from '@kbn/core/public';
+import type { CoreStart } from '@kbn/core/public';
 import { i18n } from '@kbn/i18n';
+import { DataPublicPluginStart } from '@kbn/data-plugin/public';
 import { ActionExecutionContext, UiActionsStart } from '@kbn/ui-actions-plugin/public';
 import {
  UPDATE_FILTER_REFERENCES_ACTION,
@ -25,9 +26,9 @@ import type { DataViewsState } from '../state_management';
 import { generateId } from '../id_generator';

 export interface IndexPatternServiceProps {
-  core: Pick<CoreStart, 'http' | 'notifications'>;
+  core: Pick<CoreStart, 'http' | 'notifications' | 'uiSettings'>;
+  data: DataPublicPluginStart;
  dataViews: DataViewsContract;
-  uiSettings: IUiSettingsClient;
  uiActions: UiActionsStart;
  updateIndexPatterns: (
    newState: Partial<DataViewsState>,
@ -100,7 +101,7 @@ export interface IndexPatternServiceAPI {
 export function createIndexPatternService({
  core,
  dataViews,
-  uiSettings,
+  data,
  updateIndexPatterns,
  replaceIndexPattern,
  uiActions,
@ -145,11 +146,13 @@ export function createIndexPatternService({
    refreshExistingFields: (args) =>
      syncExistingFields({
        updateIndexPatterns,
-        fetchJson: core.http.post,
        ...args,
+        data,
+        dataViews,
+        core,
      }),
    loadIndexPatternRefs: async ({ isFullEditor }) =>
      isFullEditor ? loadIndexPatternRefs(dataViews) : [],
-    getDefaultIndex: () => uiSettings.get('defaultIndex'),
+    getDefaultIndex: () => core.uiSettings.get('defaultIndex'),
  };
 }
--- a/x-pack/plugins/lens/public/mocks/data_views_service_mock.ts
+++ b/x-pack/plugins/lens/public/mocks/data_views_service_mock.ts
@ -5,8 +5,8 @@
 * 2.0.
 */

-import { uiSettingsServiceMock } from '@kbn/core-ui-settings-browser-mocks';
 import { coreMock } from '@kbn/core/public/mocks';
+import { dataPluginMock } from '@kbn/data-plugin/public/mocks';
 import { dataViewPluginMocks } from '@kbn/data-views-plugin/public/mocks';
 import { uiActionsPluginMock } from '@kbn/ui-actions-plugin/public/mocks';
 import {
@ -17,18 +17,18 @@ import {

 export function createIndexPatternServiceMock({
  core = coreMock.createStart(),
-  uiSettings = uiSettingsServiceMock.createStartContract(),
  dataViews = dataViewPluginMocks.createStartContract(),
  uiActions = uiActionsPluginMock.createStartContract(),
+  data = dataPluginMock.createStartContract(),
  updateIndexPatterns = jest.fn(),
  replaceIndexPattern = jest.fn(),
 }: Partial<IndexPatternServiceProps> = {}): IndexPatternServiceAPI {
  return createIndexPatternService({
    core,
-    uiSettings,
+    dataViews,
+    data,
    updateIndexPatterns,
    replaceIndexPattern,
-    dataViews,
    uiActions,
  });
 }
--- a/x-pack/plugins/lens/server/index.ts
+++ b/x-pack/plugins/lens/server/index.ts
@ -8,12 +8,10 @@
 // TODO: https://github.com/elastic/kibana/issues/110891
 /* eslint-disable @kbn/eslint/no_export_all */

-import { PluginInitializerContext } from '@kbn/core/server';
 import { LensServerPlugin } from './plugin';

 export type { LensServerPluginSetup } from './plugin';
 export * from './plugin';
 export * from './migrations/types';

-export const plugin = (initializerContext: PluginInitializerContext) =>
-  new LensServerPlugin(initializerContext);
+export const plugin = () => new LensServerPlugin();
--- a/x-pack/plugins/lens/server/plugin.tsx
+++ b/x-pack/plugins/lens/server/plugin.tsx
@ -5,7 +5,7 @@
 * 2.0.
 */

-import { Plugin, CoreSetup, CoreStart, PluginInitializerContext } from '@kbn/core/server';
+import { Plugin, CoreSetup, CoreStart } from '@kbn/core/server';
 import { PluginStart as DataViewsServerPluginStart } from '@kbn/data-views-plugin/server';
 import {
  PluginStart as DataPluginStart,
@ -21,8 +21,6 @@ import {
 } from '@kbn/task-manager-plugin/server';
 import { EmbeddableSetup } from '@kbn/embeddable-plugin/server';
 import { DataViewPersistableStateService } from '@kbn/data-views-plugin/common';
-import { setupRoutes } from './routes';
-import { getUiSettings } from './ui_settings';
 import { setupSavedObjects } from './saved_objects';
 import { setupExpressions } from './expressions';
 import { makeLensEmbeddableFactory } from './embeddable/make_lens_embeddable_factory';
@ -59,16 +57,14 @@ export interface LensServerPluginSetup {
 export class LensServerPlugin implements Plugin<LensServerPluginSetup, {}, {}, {}> {
  private customVisualizationMigrations: CustomVisualizationMigrations = {};

-  constructor(private initializerContext: PluginInitializerContext) {}
+  constructor() {}

  setup(core: CoreSetup<PluginStartContract>, plugins: PluginSetupContract) {
    const getFilterMigrations = plugins.data.query.filterManager.getAllMigrations.bind(
      plugins.data.query.filterManager
    );
    setupSavedObjects(core, getFilterMigrations, this.customVisualizationMigrations);
-    setupRoutes(core, this.initializerContext.logger.get());
    setupExpressions(core, plugins.expressions);
-    core.uiSettings.register(getUiSettings());

    const lensEmbeddableFactory = makeLensEmbeddableFactory(
      getFilterMigrations,
--- a/x-pack/plugins/lens/server/routes/existing_fields.ts
+++ b/x-pack/plugins/lens/server/routes/existing_fields.ts
@ -1,356 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import Boom from '@hapi/boom';
-import { errors } from '@elastic/elasticsearch';
-import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
-import { schema } from '@kbn/config-schema';
-import { RequestHandlerContext, ElasticsearchClient } from '@kbn/core/server';
-import { CoreSetup, Logger } from '@kbn/core/server';
-import { RuntimeField } from '@kbn/data-views-plugin/common';
-import { DataViewsService, DataView, FieldSpec, DataViewSpec } from '@kbn/data-views-plugin/common';
-import { UI_SETTINGS } from '@kbn/data-plugin/server';
-import { BASE_API_URL } from '../../common';
-import { FIELD_EXISTENCE_SETTING } from '../ui_settings';
-import { PluginStartContract } from '../plugin';
-
-export function isBoomError(error: { isBoom?: boolean }): error is Boom.Boom {
-  return error.isBoom === true;
-}
-
-/**
- * The number of docs to sample to determine field empty status.
- */
-const SAMPLE_SIZE = 500;
-
-export interface Field {
-  name: string;
-  isScript: boolean;
-  isMeta: boolean;
-  lang?: estypes.ScriptLanguage;
-  script?: string;
-  runtimeField?: RuntimeField;
-}
-
-export async function existingFieldsRoute(setup: CoreSetup<PluginStartContract>, logger: Logger) {
-  const router = setup.http.createRouter();
-
-  router.post(
-    {
-      path: `${BASE_API_URL}/existing_fields/{indexPatternId}`,
-      validate: {
-        params: schema.object({
-          indexPatternId: schema.string(),
-        }),
-        body: schema.object({
-          dslQuery: schema.object({}, { unknowns: 'allow' }),
-          fromDate: schema.maybe(schema.string()),
-          toDate: schema.maybe(schema.string()),
-          timeFieldName: schema.maybe(schema.string()),
-          spec: schema.object({}, { unknowns: 'allow' }),
-        }),
-      },
-    },
-    async (context, req, res) => {
-      const [{ savedObjects, elasticsearch, uiSettings }, { dataViews }] =
-        await setup.getStartServices();
-      const savedObjectsClient = savedObjects.getScopedClient(req);
-      const uiSettingsClient = uiSettings.asScopedToClient(savedObjectsClient);
-      const [includeFrozen, useSampling]: boolean[] = await Promise.all([
-        uiSettingsClient.get(UI_SETTINGS.SEARCH_INCLUDE_FROZEN),
-        uiSettingsClient.get(FIELD_EXISTENCE_SETTING),
-      ]);
-      const esClient = elasticsearch.client.asScoped(req).asCurrentUser;
-      try {
-        return res.ok({
-          body: await fetchFieldExistence({
-            ...req.params,
-            ...req.body,
-            dataViewsService: await dataViews.dataViewsServiceFactory(savedObjectsClient, esClient),
-            context,
-            includeFrozen,
-            useSampling,
-          }),
-        });
-      } catch (e) {
-        if (e instanceof errors.TimeoutError) {
-          logger.info(`Field existence check timed out on ${req.params.indexPatternId}`);
-          // 408 is Request Timeout
-          return res.customError({ statusCode: 408, body: e.message });
-        }
-        logger.info(
-          `Field existence check failed on ${req.params.indexPatternId}: ${
-            isBoomError(e) ? e.output.payload.message : e.message
-          }`
-        );
-        if (e instanceof errors.ResponseError && e.statusCode === 404) {
-          return res.notFound({ body: e.message });
-        }
-        if (isBoomError(e)) {
-          if (e.output.statusCode === 404) {
-            return res.notFound({ body: e.output.payload.message });
-          }
-          throw new Error(e.output.payload.message);
-        } else {
-          throw e;
-        }
-      }
-    }
-  );
-}
-
-async function fetchFieldExistence({
-  context,
-  indexPatternId,
-  dataViewsService,
-  dslQuery = { match_all: {} },
-  fromDate,
-  toDate,
-  timeFieldName,
-  spec,
-  includeFrozen,
-  useSampling,
-}: {
-  indexPatternId: string;
-  context: RequestHandlerContext;
-  dataViewsService: DataViewsService;
-  dslQuery: object;
-  fromDate?: string;
-  toDate?: string;
-  timeFieldName?: string;
-  spec?: DataViewSpec;
-  includeFrozen: boolean;
-  useSampling: boolean;
-}) {
-  if (useSampling) {
-    return legacyFetchFieldExistenceSampling({
-      context,
-      indexPatternId,
-      dataViewsService,
-      dslQuery,
-      fromDate,
-      toDate,
-      timeFieldName,
-      spec,
-      includeFrozen,
-    });
-  }
-
-  const uiSettingsClient = (await context.core).uiSettings.client;
-  const metaFields: string[] = await uiSettingsClient.get(UI_SETTINGS.META_FIELDS);
-  const dataView =
-    spec && Object.keys(spec).length !== 0
-      ? await dataViewsService.create(spec)
-      : await dataViewsService.get(indexPatternId);
-  const allFields = buildFieldList(dataView, metaFields);
-  const existingFieldList = await dataViewsService.getFieldsForIndexPattern(dataView, {
-    // filled in by data views service
-    pattern: '',
-    filter: toQuery(timeFieldName, fromDate, toDate, dslQuery),
-  });
-  return {
-    indexPatternTitle: dataView.title,
-    existingFieldNames: existingFields(existingFieldList, allFields),
-  };
-}
-
-async function legacyFetchFieldExistenceSampling({
-  context,
-  indexPatternId,
-  dataViewsService,
-  dslQuery,
-  fromDate,
-  toDate,
-  timeFieldName,
-  spec,
-  includeFrozen,
-}: {
-  indexPatternId: string;
-  context: RequestHandlerContext;
-  dataViewsService: DataViewsService;
-  dslQuery: object;
-  fromDate?: string;
-  toDate?: string;
-  timeFieldName?: string;
-  spec?: DataViewSpec;
-  includeFrozen: boolean;
-}) {
-  const coreContext = await context.core;
-  const metaFields: string[] = await coreContext.uiSettings.client.get(UI_SETTINGS.META_FIELDS);
-  const indexPattern =
-    spec && Object.keys(spec).length !== 0
-      ? await dataViewsService.create(spec)
-      : await dataViewsService.get(indexPatternId);
-
-  const fields = buildFieldList(indexPattern, metaFields);
-  const runtimeMappings = indexPattern.getRuntimeMappings();
-
-  const docs = await fetchIndexPatternStats({
-    fromDate,
-    toDate,
-    dslQuery,
-    client: coreContext.elasticsearch.client.asCurrentUser,
-    index: indexPattern.title,
-    timeFieldName: timeFieldName || indexPattern.timeFieldName,
-    fields,
-    runtimeMappings,
-    includeFrozen,
-  });
-
-  return {
-    indexPatternTitle: indexPattern.title,
-    existingFieldNames: legacyExistingFields(docs, fields),
-  };
-}
-
-/**
- * Exported only for unit tests.
- */
-export function buildFieldList(indexPattern: DataView, metaFields: string[]): Field[] {
-  return indexPattern.fields.map((field) => {
-    return {
-      name: field.name,
-      isScript: !!field.scripted,
-      lang: field.lang,
-      script: field.script,
-      // id is a special case - it doesn't show up in the meta field list,
-      // but as it's not part of source, it has to be handled separately.
-      isMeta: metaFields.includes(field.name) || field.name === '_id',
-      runtimeField: !field.isMapped ? field.runtimeField : undefined,
-    };
-  });
-}
-
-async function fetchIndexPatternStats({
-  client,
-  index,
-  dslQuery,
-  timeFieldName,
-  fromDate,
-  toDate,
-  fields,
-  runtimeMappings,
-  includeFrozen,
-}: {
-  client: ElasticsearchClient;
-  index: string;
-  dslQuery: object;
-  timeFieldName?: string;
-  fromDate?: string;
-  toDate?: string;
-  fields: Field[];
-  runtimeMappings: estypes.MappingRuntimeFields;
-  includeFrozen: boolean;
-}) {
-  const query = toQuery(timeFieldName, fromDate, toDate, dslQuery);
-
-  const scriptedFields = fields.filter((f) => f.isScript);
-  const result = await client.search(
-    {
-      index,
-      ...(includeFrozen ? { ignore_throttled: false } : {}),
-      body: {
-        size: SAMPLE_SIZE,
-        query,
-        // Sorted queries are usually able to skip entire shards that don't match
-        sort: timeFieldName && fromDate && toDate ? [{ [timeFieldName]: 'desc' }] : [],
-        fields: ['*'],
-        _source: false,
-        runtime_mappings: runtimeMappings,
-        script_fields: scriptedFields.reduce((acc, field) => {
-          acc[field.name] = {
-            script: {
-              lang: field.lang!,
-              source: field.script!,
-            },
-          };
-          return acc;
-        }, {} as Record<string, estypes.ScriptField>),
-        // Small improvement because there is overhead in counting
-        track_total_hits: false,
-        // Per-shard timeout, must be lower than overall. Shards return partial results on timeout
-        timeout: '4500ms',
-      },
-    },
-    {
-      // Global request timeout. Will cancel the request if exceeded. Overrides the elasticsearch.requestTimeout
-      requestTimeout: '5000ms',
-      // Fails fast instead of retrying- default is to retry
-      maxRetries: 0,
-    }
-  );
-  return result.hits.hits;
-}
-
-function toQuery(
-  timeFieldName: string | undefined,
-  fromDate: string | undefined,
-  toDate: string | undefined,
-  dslQuery: object
-) {
-  const filter =
-    timeFieldName && fromDate && toDate
-      ? [
-          {
-            range: {
-              [timeFieldName]: {
-                format: 'strict_date_optional_time',
-                gte: fromDate,
-                lte: toDate,
-              },
-            },
-          },
-          dslQuery,
-        ]
-      : [dslQuery];
-
-  const query = {
-    bool: {
-      filter,
-    },
-  };
-  return query;
-}
-
-/**
- * Exported only for unit tests.
- */
-export function existingFields(filteredFields: FieldSpec[], allFields: Field[]): string[] {
-  const filteredFieldsSet = new Set(filteredFields.map((f) => f.name));
-
-  return allFields
-    .filter((field) => field.isScript || field.runtimeField || filteredFieldsSet.has(field.name))
-    .map((f) => f.name);
-}
-
-/**
- * Exported only for unit tests.
- */
-export function legacyExistingFields(docs: estypes.SearchHit[], fields: Field[]): string[] {
-  const missingFields = new Set(fields);
-
-  for (const doc of docs) {
-    if (missingFields.size === 0) {
-      break;
-    }
-
-    missingFields.forEach((field) => {
-      let fieldStore = doc.fields!;
-      if (field.isMeta) {
-        fieldStore = doc;
-      }
-      const value = fieldStore[field.name];
-      if (Array.isArray(value) && value.length) {
-        missingFields.delete(field);
-      } else if (!Array.isArray(value) && value) {
-        missingFields.delete(field);
-      }
-    });
-  }
-
-  return fields.filter((field) => !missingFields.has(field)).map((f) => f.name);
-}
--- a/x-pack/plugins/lens/server/routes/index.ts
+++ b/x-pack/plugins/lens/server/routes/index.ts
@ -1,14 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import { CoreSetup, Logger } from '@kbn/core/server';
-import { PluginStartContract } from '../plugin';
-import { existingFieldsRoute } from './existing_fields';
-
-export function setupRoutes(setup: CoreSetup<PluginStartContract>, logger: Logger) {
-  existingFieldsRoute(setup, logger);
-}
--- a/x-pack/plugins/translations/translations/fr-FR.json
+++ b/x-pack/plugins/translations/translations/fr-FR.json
@ -17142,8 +17142,6 @@
    "xpack.lens.indexPattern.percentileRanks.documentation.markdown": "\nRetourne le pourcentage de valeurs qui sont en dessous d'une certaine valeur. Par exemple, si une valeur est supérieure à 95 % des valeurs observées, elle est placée au 95e rang centile.\n\nExemple : Obtenir le pourcentage de valeurs qui sont en dessous de 100 :\n`percentile_rank(bytes, value=100)`\n      ",
    "xpack.lens.indexPattern.standardDeviation.documentation.markdown": "\nRetourne la taille de la variation ou de la dispersion du champ. Cette fonction ne s’applique qu’aux champs numériques.\n\n#### Exemples\n\nPour obtenir l'écart type d'un prix, utilisez standard_deviation(price).\n\nPour obtenir la variance du prix des commandes passées au Royaume-Uni, utilisez `square(standard_deviation(price, kql='location:UK'))`.\n      ",
    "xpack.lens.indexPattern.time_scale.documentation.markdown": "\n\nCette fonction avancée est utile pour normaliser les comptes et les sommes sur un intervalle de temps spécifique. Elle permet l'intégration avec les indicateurs qui sont stockés déjà normalisés sur un intervalle de temps spécifique.\n\nVous pouvez faire appel à cette fonction uniquement si une fonction d'histogramme des dates est utilisée dans le graphique actuel.\n\nExemple : Un rapport comparant un indicateur déjà normalisé à un autre indicateur devant être normalisé.\n\"normalize_by_unit(counter_rate(max(system.diskio.write.bytes)), unit='s') / last_value(apache.status.bytes_per_second)\"\n      ",
-    "xpack.lens.advancedSettings.useFieldExistenceSampling.description": "Lorsque cette option est activée, l’échantillonnage de document est utilisé pour déterminer l’existence des champs (disponibles ou vides) pour la liste de champs Lens au lieu de se fonder sur les mappings d’index.",
-    "xpack.lens.advancedSettings.useFieldExistenceSampling.title": "Utiliser l’échantillonnage d’existence des champs",
    "xpack.lens.app.addToLibrary": "Enregistrer dans la bibliothèque",
    "xpack.lens.app.cancel": "Annuler",
    "xpack.lens.app.cancelButtonAriaLabel": "Retour à la dernière application sans enregistrer les modifications",
--- a/x-pack/plugins/translations/translations/ja-JP.json
+++ b/x-pack/plugins/translations/translations/ja-JP.json
@ -17127,8 +17127,6 @@
    "xpack.lens.indexPattern.percentileRanks.documentation.markdown": "\n特定の値未満の値の割合が返されます。たとえば、値が観察された値の95%以上の場合、95パーセンタイルランクであるとされます。\n\n例：100未満の値のパーセンタイルを取得します。\n`percentile_rank(bytes, value=100)`\n      ",
    "xpack.lens.indexPattern.standardDeviation.documentation.markdown": "\nフィールドの分散または散布度が返されます。この関数は数値フィールドでのみ動作します。\n\n#### 例\n\n価格の標準偏差を取得するには、standard_deviation(price)を使用します。\n\n英国からの注文書の価格の分散を取得するには、square(standard_deviation(price, kql='location:UK'))を使用します。\n      ",
    "xpack.lens.indexPattern.time_scale.documentation.markdown": "\n\nこの高度な機能は、特定の期間に対してカウントと合計を正規化する際に役立ちます。すでに特定の期間に対して正規化され、保存されたメトリックとの統合が可能です。\n\nこの機能は、現在のグラフで日付ヒストグラム関数が使用されている場合にのみ使用できます。\n\n例：すでに正規化されているメトリックを、正規化が必要な別のメトリックと比較した比率。\n`normalize_by_unit(counter_rate(max(system.diskio.write.bytes)), unit='s') / last_value(apache.status.bytes_per_second)`\n      ",
-    "xpack.lens.advancedSettings.useFieldExistenceSampling.description": "有効な場合、インデックスマッピングを使用するのではなく、ドキュメントサンプリングを使用して、Lensフィールドリストのフィールドの存在（使用可能または空）を決定します。",
-    "xpack.lens.advancedSettings.useFieldExistenceSampling.title": "フィールド存在サンプリングを使用",
    "xpack.lens.app.addToLibrary": "ライブラリに保存",
    "xpack.lens.app.cancel": "キャンセル",
    "xpack.lens.app.cancelButtonAriaLabel": "変更を保存せずに最後に使用していたアプリに戻る",
--- a/x-pack/plugins/translations/translations/zh-CN.json
+++ b/x-pack/plugins/translations/translations/zh-CN.json
@ -17149,8 +17149,6 @@
    "xpack.lens.indexPattern.percentileRanks.documentation.markdown": "\n返回小于某个值的值的百分比。例如，如果某个值大于或等于 95% 的观察值，则称它处于第 95 个百分位等级\n\n例如：获取小于 100 的值的百分比：\n`percentile_rank(bytes, value=100)`\n      ",
    "xpack.lens.indexPattern.standardDeviation.documentation.markdown": "\n返回字段的变量或差量数量。此函数仅适用于数字字段。\n\n#### 示例\n\n要获取价格的标准偏差，请使用 `standard_deviation(price)`。\n\n要获取来自英国的订单的价格方差，请使用 `square(standard_deviation(price, kql='location:UK'))`。\n      ",
    "xpack.lens.indexPattern.time_scale.documentation.markdown": "\n\n此高级函数用于将计数和总和标准化为特定时间间隔。它允许集成所存储的已标准化为特定时间间隔的指标。\n\n此函数只能在当前图表中使用了日期直方图函数时使用。\n\n例如：将已标准化指标与其他需要标准化的指标进行比较的比率。\n`normalize_by_unit(counter_rate(max(system.diskio.write.bytes)), unit='s') / last_value(apache.status.bytes_per_second)`\n      ",
-    "xpack.lens.advancedSettings.useFieldExistenceSampling.description": "如果启用，文档采样将用于确定 Lens 字段列表中的字段是否存在（可用或为空），而不依赖索引映射。",
-    "xpack.lens.advancedSettings.useFieldExistenceSampling.title": "使用字段存在采样",
    "xpack.lens.app.addToLibrary": "保存到库",
    "xpack.lens.app.cancel": "取消",
    "xpack.lens.app.cancelButtonAriaLabel": "返回到上一个应用而不保存更改",
--- a/x-pack/test/api_integration/apis/index.ts
+++ b/x-pack/test/api_integration/apis/index.ts
@ -24,7 +24,6 @@ export default function ({ loadTestFile }: FtrProviderContext) {
    loadTestFile(require.resolve('./uptime'));
    loadTestFile(require.resolve('./maps'));
    loadTestFile(require.resolve('./security_solution'));
-    loadTestFile(require.resolve('./lens'));
    loadTestFile(require.resolve('./transform'));
    loadTestFile(require.resolve('./lists'));
    loadTestFile(require.resolve('./upgrade_assistant'));
--- a/x-pack/test/api_integration/apis/lens/index.ts
+++ b/x-pack/test/api_integration/apis/lens/index.ts
@ -1,15 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import { FtrProviderContext } from '../../ftr_provider_context';
-
-export default function lensApiIntegrationTests({ loadTestFile }: FtrProviderContext) {
-  describe('Lens', () => {
-    loadTestFile(require.resolve('./existing_fields'));
-    loadTestFile(require.resolve('./legacy_existing_fields'));
-  });
-}