[APM] Use excluded data tiers setting (#192373)

closes [#190559](https://github.com/elastic/kibana/issues/190559)

## Summary

This PR updates the ES clients in APM to respect the excluded tier
configuration. When this config is set, the ES clients will
automatically add a filter to exclude the specified tiers from queries.

<img width="600" alt="image"
src="https://github.com/user-attachments/assets/9b0de76d-242c-4343-bc30-d5c787316f59">

All queries in APM should have the `_tier` filter (via
`get_apm_events_client`)
<img width="600" alt="image"
src="https://github.com/user-attachments/assets/c525602f-f239-4be8-99c4-65d617962656">

This change also affects alerting (via `alerting_es_client`)
<img width="600" alt="image"
src="https://github.com/user-attachments/assets/750df4d7-5b49-4de5-9294-7afedf11d7e5">

And it impacts the alerts column (via `get_apm_alert_client`)
<img width="600" alt="image"
src="https://github.com/user-attachments/assets/44bd9129-1e72-4a3a-af32-d42a9cd9164d">

### What won't automatically add a filter for `_tier`

- Embeddables
- ML queries

### How to test
- Set the config in Advanced Settings to exclude `data_frozen` and
`data_cold` (optional)
- Navigate to APM and check the query `Inspect` to see if the filter is
present.
- Click through APM to confirm things still work.
- Create one of each type of APM alerts
- Without the config set, queries should not include the `_tier` filter`

---------

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

x-pack/packages/observability/observability_utils/es/queries/exclude_frozen_query.ts

@@ -5,19 +5,8 @@
  * 2.0.
  */
 import type { estypes } from '@elastic/elasticsearch';
+import { excludeTiersQuery } from './exclude_tiers_query';
 
 export function excludeFrozenQuery(): estypes.QueryDslQueryContainer[] {
-  return [
-    {
-      bool: {
-        must_not: [
-          {
-            term: {
-              _tier: 'data_frozen',
-            },
-          },
-        ],
-      },
-    },
-  ];
+  return excludeTiersQuery(['data_frozen']);
 }

x-pack/packages/observability/observability_utils/es/queries/exclude_tiers_query.ts

@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { estypes } from '@elastic/elasticsearch';
+
+export function excludeTiersQuery(
+  excludedDataTiers: Array<'data_frozen' | 'data_cold' | 'data_warm' | 'data_hot'>
+): estypes.QueryDslQueryContainer[] {
+  return [
+    {
+      bool: {
+        must_not: [
+          {
+            terms: {
+              _tier: excludedDataTiers,
+            },
+          },
+        ],
+      },
+    },
+  ];
+}

x-pack/plugins/observability_solution/apm/common/storage_explorer_types.ts

@@ -5,23 +5,13 @@
  * 2.0.
  */
 
+import {
+  IndexLifecyclePhaseSelectOption,
+  indexLifeCyclePhaseToDataTier,
+} from '@kbn/observability-shared-plugin/common';
 import * as t from 'io-ts';
 
-export enum IndexLifecyclePhaseSelectOption {
-  All = 'all',
-  Hot = 'hot',
-  Warm = 'warm',
-  Cold = 'cold',
-  Frozen = 'frozen',
-}
-
-export const indexLifeCyclePhaseToDataTier = {
-  [IndexLifecyclePhaseSelectOption.Hot]: 'data_hot',
-  [IndexLifecyclePhaseSelectOption.Warm]: 'data_warm',
-  [IndexLifecyclePhaseSelectOption.Cold]: 'data_cold',
-  [IndexLifecyclePhaseSelectOption.Frozen]: 'data_frozen',
-};
-
+export { IndexLifecyclePhaseSelectOption, indexLifeCyclePhaseToDataTier };
 export const indexLifecyclePhaseRt = t.type({
   indexLifecyclePhase: t.union([
     t.literal(IndexLifecyclePhaseSelectOption.All),

x-pack/plugins/observability_solution/apm/server/lib/helpers/get_apm_alerts_client.test.ts

@@ -0,0 +1,95 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { type ApmAlertsRequiredParams, getApmAlertsClient } from './get_apm_alerts_client';
+import type {
+  IScopedClusterClient,
+  IUiSettingsClient,
+  KibanaRequest,
+  SavedObjectsClientContract,
+} from '@kbn/core/server';
+import { AlertsClient, RuleRegistryPluginStartContract } from '@kbn/rule-registry-plugin/server';
+
+describe('get_apm_alerts_client', () => {
+  let ruleRegistryMock: jest.Mocked<RuleRegistryPluginStartContract>;
+  let alertClient: jest.Mocked<AlertsClient>;
+  let uiSettingsClientMock: jest.Mocked<IUiSettingsClient>;
+
+  const params: ApmAlertsRequiredParams = {
+    size: 10,
+    track_total_hits: true,
+    query: {
+      match: { field: 'value' },
+    },
+  };
+
+  beforeEach(async () => {
+    uiSettingsClientMock = {
+      get: jest.fn().mockResolvedValue(undefined),
+    } as unknown as jest.Mocked<IUiSettingsClient>;
+
+    alertClient = {
+      find: jest.fn().mockResolvedValue({}),
+      getAuthorizedAlertsIndices: jest.fn().mockResolvedValue(['apm']),
+    } as unknown as jest.Mocked<AlertsClient>;
+
+    ruleRegistryMock = {
+      getRacClientWithRequest: jest.fn().mockResolvedValue(alertClient),
+      alerting: jest.fn(),
+    } as unknown as jest.Mocked<RuleRegistryPluginStartContract>;
+  });
+
+  afterEach(() => {
+    jest.resetAllMocks();
+  });
+
+  // Helper function to create the APM alerts client
+  const createApmAlertsClient = async () => {
+    return await getApmAlertsClient({
+      context: {
+        core: Promise.resolve({
+          uiSettings: { client: uiSettingsClientMock },
+          elasticsearch: { client: {} as IScopedClusterClient },
+          savedObjects: { client: {} as SavedObjectsClientContract },
+        }),
+      } as any,
+      plugins: {
+        ruleRegistry: {
+          start: jest.fn().mockResolvedValue(ruleRegistryMock),
+          setup: {} as any,
+        },
+      } as any,
+      request: {} as KibanaRequest,
+    });
+  };
+
+  it('should call search', async () => {
+    const apmAlertsClient = await createApmAlertsClient();
+
+    await apmAlertsClient.search(params);
+
+    const searchParams = alertClient.find.mock.calls[0][0] as ApmAlertsRequiredParams;
+    expect(searchParams.query).toEqual({ match: { field: 'value' } });
+  });
+
+  it('should call search with filters containing excluded data tiers', async () => {
+    const excludedDataTiers = ['data_warm', 'data_cold'];
+    uiSettingsClientMock.get.mockResolvedValue(excludedDataTiers);
+
+    const apmAlertsClient = await createApmAlertsClient();
+
+    await apmAlertsClient.search(params);
+
+    const searchParams = alertClient.find.mock.calls[0][0] as ApmAlertsRequiredParams;
+    expect(searchParams.query?.bool).toEqual({
+      must: [
+        { match: { field: 'value' } },
+        { bool: { must_not: [{ terms: { _tier: ['data_warm', 'data_cold'] } }] } },
+      ],
+    });
+  });
+});

x-pack/plugins/observability_solution/apm/server/lib/helpers/get_apm_alerts_client.ts

@@ -8,14 +8,27 @@
 import { isEmpty } from 'lodash';
 import { ESSearchRequest, InferSearchResponseOf } from '@kbn/es-types';
 import { ParsedTechnicalFields } from '@kbn/rule-registry-plugin/common';
+import { DataTier } from '@kbn/observability-shared-plugin/common';
+import { searchExcludedDataTiers } from '@kbn/observability-plugin/common/ui_settings_keys';
+import { estypes } from '@elastic/elasticsearch';
+import { getDataTierFilterCombined } from '@kbn/apm-data-access-plugin/server/utils';
 import type { MinimalAPMRouteHandlerResources } from '../../routes/apm_routes/register_apm_server_routes';
 
 export type ApmAlertsClient = Awaited<ReturnType<typeof getApmAlertsClient>>;
 
+export type ApmAlertsRequiredParams = ESSearchRequest & {
+  size: number;
+  track_total_hits: boolean | number;
+  query?: estypes.QueryDslQueryContainer;
+};
+
 export async function getApmAlertsClient({
+  context,
   plugins,
   request,
-}: Pick<MinimalAPMRouteHandlerResources, 'plugins' | 'request'>) {
+}: Pick<MinimalAPMRouteHandlerResources, 'context' | 'plugins' | 'request'>) {
+  const coreContext = await context.core;
+
   const ruleRegistryPluginStart = await plugins.ruleRegistry.start();
   const alertsClient = await ruleRegistryPluginStart.getRacClientWithRequest(request);
   const apmAlertsIndices = await alertsClient.getAuthorizedAlertsIndices(['apm']);
@@ -24,17 +37,20 @@ export async function getApmAlertsClient({
     throw Error('No alert indices exist for "apm"');
   }
 
-  type RequiredParams = ESSearchRequest & {
-    size: number;
-    track_total_hits: boolean | number;
-  };
+  const excludedDataTiers = await coreContext.uiSettings.client.get<DataTier[]>(
+    searchExcludedDataTiers
+  );
 
   return {
-    search<TParams extends RequiredParams>(
+    search<TParams extends ApmAlertsRequiredParams>(
       searchParams: TParams
     ): Promise<InferSearchResponseOf<ParsedTechnicalFields, TParams>> {
       return alertsClient.find({
         ...searchParams,
+        query: getDataTierFilterCombined({
+          filter: searchParams.query,
+          excludedDataTiers,
+        }),
         index: apmAlertsIndices.join(','),
       }) as Promise<any>;
     },

x-pack/plugins/observability_solution/apm/server/lib/helpers/get_apm_event_client.ts

@@ -6,6 +6,8 @@
  */
 
 import { UI_SETTINGS } from '@kbn/data-plugin/common';
+import { DataTier } from '@kbn/observability-shared-plugin/common';
+import { searchExcludedDataTiers } from '@kbn/observability-plugin/common/ui_settings_keys';
 import { APMEventClient } from './create_es_client/create_apm_event_client';
 import { withApmSpan } from '../../utils/with_apm_span';
 import { MinimalAPMRouteHandlerResources } from '../../routes/apm_routes/register_apm_server_routes';
@@ -22,11 +24,18 @@ export async function getApmEventClient({
 >): Promise<APMEventClient> {
   return withApmSpan('get_apm_event_client', async () => {
     const coreContext = await context.core;
-    const [indices, includeFrozen] = await Promise.all([
+    const [indices, uiSettings] = await Promise.all([
       getApmIndices(),
-      withApmSpan('get_ui_settings', () =>
-        coreContext.uiSettings.client.get<boolean>(UI_SETTINGS.SEARCH_INCLUDE_FROZEN)
-      ),
+      withApmSpan('get_ui_settings', async () => {
+        const includeFrozen = await coreContext.uiSettings.client.get<boolean>(
+          UI_SETTINGS.SEARCH_INCLUDE_FROZEN
+        );
+        const excludedDataTiers = await coreContext.uiSettings.client.get<DataTier[]>(
+          searchExcludedDataTiers
+        );
+
+        return { includeFrozen, excludedDataTiers };
+      }),
     ]);
 
     return new APMEventClient({
@@ -35,7 +44,8 @@ export async function getApmEventClient({
       request,
       indices,
       options: {
-        includeFrozen,
+        includeFrozen: uiSettings.includeFrozen,
+        excludedDataTiers: uiSettings.excludedDataTiers,
         inspectableEsQueriesMap,
       },
     });

x-pack/plugins/observability_solution/apm/server/routes/alerts/alerting_es_client.test.ts

@@ -0,0 +1,93 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { type APMEventESSearchRequestParams, alertingEsClient } from './alerting_es_client';
+import type { RuleExecutorServices } from '@kbn/alerting-plugin/server';
+import type { ElasticsearchClient, IUiSettingsClient } from '@kbn/core/server';
+import type { ESSearchResponse } from '@kbn/es-types';
+
+describe('alertingEsClient', () => {
+  let scopedClusterClientMock: jest.Mocked<{
+    asCurrentUser: jest.Mocked<ElasticsearchClient>;
+  }>;
+
+  let uiSettingsClientMock: jest.Mocked<IUiSettingsClient>;
+
+  const params = {
+    body: {
+      size: 10,
+      track_total_hits: true,
+      query: {
+        match: { field: 'value' },
+      },
+    },
+  };
+
+  const mockSearchResponse = {
+    hits: {
+      total: { value: 1, relation: 'eq' },
+      hits: [{ _source: {}, _index: '' }],
+      max_score: 1,
+    },
+    took: 1,
+    _shards: { total: 1, successful: 1, skipped: 0, failed: 0 },
+    timed_out: false,
+  } as unknown as ESSearchResponse<unknown, typeof params>;
+
+  beforeEach(() => {
+    scopedClusterClientMock = {
+      asCurrentUser: {
+        search: jest.fn().mockResolvedValue(mockSearchResponse),
+      } as unknown as jest.Mocked<ElasticsearchClient>,
+    };
+
+    uiSettingsClientMock = {
+      get: jest.fn().mockResolvedValue(undefined),
+    } as unknown as jest.Mocked<IUiSettingsClient>;
+  });
+
+  afterEach(() => {
+    jest.resetAllMocks();
+  });
+
+  // Helper function to perform the search
+  const performSearch = async (searchParams: APMEventESSearchRequestParams) => {
+    return await alertingEsClient({
+      scopedClusterClient: scopedClusterClientMock as unknown as RuleExecutorServices<
+        never,
+        never,
+        never
+      >['scopedClusterClient'],
+      uiSettingsClient: uiSettingsClientMock,
+      params: searchParams,
+    });
+  };
+
+  it('should call search with default params', async () => {
+    await performSearch(params);
+
+    const searchParams = scopedClusterClientMock.asCurrentUser.search.mock
+      .calls[0][0] as APMEventESSearchRequestParams;
+    expect(searchParams.body?.query).toEqual({ match: { field: 'value' } });
+  });
+
+  it('should call search with filters containing excluded data tiers', async () => {
+    const excludedDataTiers = ['data_warm', 'data_cold'];
+    uiSettingsClientMock.get.mockResolvedValue(excludedDataTiers);
+
+    await performSearch(params);
+
+    const searchParams = scopedClusterClientMock.asCurrentUser.search.mock
+      .calls[0][0] as APMEventESSearchRequestParams;
+    expect(searchParams.body?.query?.bool).toEqual({
+      must: [
+        { match: { field: 'value' } },
+        { bool: { must_not: [{ terms: { _tier: ['data_warm', 'data_cold'] } }] } },
+      ],
+    });
+  });
+});

x-pack/plugins/observability_solution/apm/server/routes/alerts/alerting_es_client.ts

@@ -7,6 +7,10 @@
 
 import type { ESSearchRequest, ESSearchResponse } from '@kbn/es-types';
 import { RuleExecutorServices } from '@kbn/alerting-plugin/server';
+import { IUiSettingsClient } from '@kbn/core/server';
+import type { DataTier } from '@kbn/observability-shared-plugin/common';
+import { getDataTierFilterCombined } from '@kbn/apm-data-access-plugin/server/utils';
+import { searchExcludedDataTiers } from '@kbn/observability-plugin/common/ui_settings_keys';
 
 export type APMEventESSearchRequestParams = ESSearchRequest & {
   body: { size: number; track_total_hits: boolean | number };
@@ -14,13 +18,24 @@ export type APMEventESSearchRequestParams = ESSearchRequest & {
 
 export async function alertingEsClient<TParams extends APMEventESSearchRequestParams>({
   scopedClusterClient,
+  uiSettingsClient,
   params,
 }: {
   scopedClusterClient: RuleExecutorServices<never, never, never>['scopedClusterClient'];
+  uiSettingsClient: IUiSettingsClient;
   params: TParams;
 }): Promise<ESSearchResponse<unknown, TParams>> {
+  const excludedDataTiers = await uiSettingsClient.get<DataTier[]>(searchExcludedDataTiers);
+
   const response = await scopedClusterClient.asCurrentUser.search({
     ...params,
+    body: {
+      ...params.body,
+      query: getDataTierFilterCombined({
+        filter: params.body.query,
+        excludedDataTiers,
+      }),
+    },
     ignore_unavailable: true,
   });
 

x-pack/plugins/observability_solution/apm/server/routes/alerts/rule_types/anomaly/get_service_group_fields_for_anomaly.ts

@@ -5,7 +5,11 @@
  * 2.0.
  */
 
-import { IScopedClusterClient, SavedObjectsClientContract } from '@kbn/core/server';
+import type {
+  IScopedClusterClient,
+  IUiSettingsClient,
+  SavedObjectsClientContract,
+} from '@kbn/core/server';
 import type { APMIndices } from '@kbn/apm-data-access-plugin/server';
 import {
   SERVICE_ENVIRONMENT,
@@ -23,6 +27,7 @@ export async function getServiceGroupFieldsForAnomaly({
   apmIndices,
   scopedClusterClient,
   serviceName,
+  uiSettingsClient,
   environment,
   transactionType,
   timestamp,
@@ -31,6 +36,7 @@ export async function getServiceGroupFieldsForAnomaly({
   apmIndices: APMIndices;
   scopedClusterClient: IScopedClusterClient;
   savedObjectsClient: SavedObjectsClientContract;
+  uiSettingsClient: IUiSettingsClient;
   serviceName: string;
   environment: string;
   transactionType: string;
@@ -70,6 +76,7 @@ export async function getServiceGroupFieldsForAnomaly({
 
   const response = await alertingEsClient({
     scopedClusterClient,
+    uiSettingsClient,
     params,
   });
 

x-pack/plugins/observability_solution/apm/server/routes/alerts/rule_types/anomaly/register_anomaly_rule_type.ts

@@ -129,7 +129,7 @@ export function registerAnomalyRuleType({
       }
 
       const { params, services, spaceId, startedAt, getTimeRange } = options;
-      const { alertsClient, savedObjectsClient, scopedClusterClient } = services;
+      const { alertsClient, savedObjectsClient, scopedClusterClient, uiSettingsClient } = services;
       if (!alertsClient) {
         throw new AlertsClientError();
       }
@@ -283,6 +283,7 @@ export function registerAnomalyRuleType({
           apmIndices,
           scopedClusterClient,
           savedObjectsClient,
+          uiSettingsClient,
           serviceName,
           environment,
           transactionType,

x-pack/plugins/observability_solution/apm/server/routes/alerts/rule_types/error_count/register_error_count_rule_type.ts

@@ -128,7 +128,7 @@ export function registerErrorCountRuleType({
       >
     ) => {
       const { params: ruleParams, services, spaceId, startedAt, getTimeRange } = options;
-      const { alertsClient, savedObjectsClient, scopedClusterClient } = services;
+      const { alertsClient, savedObjectsClient, scopedClusterClient, uiSettingsClient } = services;
       if (!alertsClient) {
         throw new AlertsClientError();
       }
@@ -187,6 +187,7 @@ export function registerErrorCountRuleType({
 
       const response = await alertingEsClient({
         scopedClusterClient,
+        uiSettingsClient,
         params: searchParams,
       });
 

x-pack/plugins/observability_solution/apm/server/routes/alerts/rule_types/transaction_duration/register_transaction_duration_rule_type.ts

@@ -140,7 +140,7 @@ export function registerTransactionDurationRuleType({
       >
     ) => {
       const { params: ruleParams, services, spaceId, getTimeRange } = options;
-      const { alertsClient, savedObjectsClient, scopedClusterClient } = services;
+      const { alertsClient, savedObjectsClient, scopedClusterClient, uiSettingsClient } = services;
       if (!alertsClient) {
         throw new AlertsClientError();
       }
@@ -221,6 +221,7 @@ export function registerTransactionDurationRuleType({
 
       const response = await alertingEsClient({
         scopedClusterClient,
+        uiSettingsClient,
         params: searchParams,
       });
 

x-pack/plugins/observability_solution/apm/server/routes/alerts/rule_types/transaction_error_rate/register_transaction_error_rate_rule_type.ts

@@ -138,7 +138,7 @@ export function registerTransactionErrorRateRuleType({
       >
     ) => {
       const { services, spaceId, params: ruleParams, startedAt, getTimeRange } = options;
-      const { alertsClient, savedObjectsClient, scopedClusterClient } = services;
+      const { alertsClient, savedObjectsClient, scopedClusterClient, uiSettingsClient } = services;
       if (!alertsClient) {
         throw new AlertsClientError();
       }
@@ -223,6 +223,7 @@ export function registerTransactionErrorRateRuleType({
 
       const response = await alertingEsClient({
         scopedClusterClient,
+        uiSettingsClient,
         params: searchParams,
       });
 

x-pack/plugins/observability_solution/apm/server/routes/alerts/test_utils/index.ts

@@ -40,6 +40,9 @@ export const createRuleTypeMocks = () => {
     savedObjectsClient: {
       get: () => ({ attributes: { consumer: APM_SERVER_FEATURE_ID } }),
     },
+    uiSettingsClient: {
+      get: jest.fn(),
+    },
     alertFactory: {
       create: jest.fn(() => ({ scheduleActions, getUuid })),
       done: {},

x-pack/plugins/observability_solution/apm/server/routes/historical_data/has_historical_agent_data.ts

@@ -6,6 +6,7 @@
  */
 
 import { ProcessorEvent } from '@kbn/observability-plugin/common';
+import type { DataTier } from '@kbn/observability-shared-plugin/common';
 import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client';
 
 export async function hasHistoricalAgentData(apmEventClient: APMEventClient) {
@@ -23,8 +24,9 @@ export async function hasHistoricalAgentData(apmEventClient: APMEventClient) {
   return hasDataUnbounded;
 }
 
-type DataTier = 'data_hot' | 'data_warm' | 'data_cold' | 'data_frozen';
 async function hasDataRequest(apmEventClient: APMEventClient, dataTiers?: DataTier[]) {
+  // the `observability:searchExcludedDataTiers` setting will also be considered
+  // in the `search` function to exclude data tiers from the search
   const query = dataTiers ? { terms: { _tier: dataTiers } } : undefined;
 
   const params = {

x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/create_es_client/create_apm_event_client/index.test.ts

@@ -7,80 +7,245 @@
 import { setTimeout as setTimeoutPromise } from 'timers/promises';
 import { contextServiceMock, executionContextServiceMock } from '@kbn/core/server/mocks';
 import { createHttpService } from '@kbn/core-http-server-mocks';
+import type { ElasticsearchClient, KibanaRequest } from '@kbn/core/server';
+import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import type {
+  TermsEnumRequest,
+  MsearchMultisearchBody,
+} from '@elastic/elasticsearch/lib/api/types';
 import supertest from 'supertest';
-import { APMEventClient } from '.';
+import { APMEventClient, type APMEventESSearchRequest, type APMEventFieldCapsRequest } from '.';
+import { APMIndices } from '../../../..';
+
+import * as cancelEsRequestOnAbortModule from '../cancel_es_request_on_abort';
+import * as observabilityPluginModule from '@kbn/observability-plugin/server';
+
+jest.mock('@kbn/observability-plugin/server', () => ({
+  __esModule: true,
+  ...jest.requireActual('@kbn/observability-plugin/server'),
+}));
 
 describe('APMEventClient', () => {
-  let server: ReturnType<typeof createHttpService>;
-
-  beforeEach(() => {
-    server = createHttpService();
-  });
-
-  afterEach(async () => {
-    await server.stop();
-  });
-  it('cancels a search when a request is aborted', async () => {
-    await server.preboot({
-      context: contextServiceMock.createPrebootContract(),
+  describe('Abort controller', () => {
+    let server: ReturnType<typeof createHttpService>;
+    beforeEach(() => {
+      server = createHttpService();
     });
-    const { server: innerServer, createRouter } = await server.setup({
-      context: contextServiceMock.createSetupContract(),
-      executionContext: executionContextServiceMock.createInternalSetupContract(),
-    });
-    const router = createRouter('/');
 
-    let abortSignal: AbortSignal | undefined;
-    router.get({ path: '/', validate: false }, async (context, request, res) => {
-      const eventClient = new APMEventClient({
-        esClient: {
-          search: async (params: any, { signal }: { signal: AbortSignal }) => {
-            abortSignal = signal;
-            await setTimeoutPromise(3_000, undefined, {
-              signal: abortSignal,
-            });
-            return {};
+    afterEach(async () => {
+      await server.stop();
+    });
+
+    it('cancels a search when a request is aborted', async () => {
+      await server.preboot({
+        context: contextServiceMock.createPrebootContract(),
+      });
+      const { server: innerServer, createRouter } = await server.setup({
+        context: contextServiceMock.createSetupContract(),
+        executionContext: executionContextServiceMock.createInternalSetupContract(),
+      });
+      const router = createRouter('/');
+
+      let abortSignal: AbortSignal | undefined;
+      router.get({ path: '/', validate: false }, async (context, request, res) => {
+        const eventClient = new APMEventClient({
+          esClient: {
+            search: async (params: any, { signal }: { signal: AbortSignal }) => {
+              abortSignal = signal;
+              await setTimeoutPromise(3_000, undefined, {
+                signal: abortSignal,
+              });
+              return {};
+            },
+          } as any,
+          debug: false,
+          request,
+          indices: {} as APMIndices,
+          options: {
+            includeFrozen: false,
           },
-        } as any,
-        debug: false,
-        request,
-        indices: {} as any,
-        options: {
-          includeFrozen: false,
-        },
-      });
-
-      await eventClient.search('foo', {
-        apm: {
-          events: [],
-        },
-        body: { size: 0, track_total_hits: false },
-      });
-
-      return res.ok({ body: 'ok' });
-    });
-
-    await server.start();
-
-    expect(abortSignal?.aborted).toBeFalsy();
-
-    const incomingRequest = supertest(innerServer.listener)
-      .get('/')
-      // end required to send request
-      .end();
-
-    await new Promise((resolve) => {
-      setTimeout(() => {
-        void incomingRequest.on('abort', () => {
-          setTimeout(() => {
-            resolve(undefined);
-          }, 100);
         });
 
-        void incomingRequest.abort();
-      }, 200);
+        await eventClient.search('foo', {
+          apm: {
+            events: [],
+          },
+          body: { size: 0, track_total_hits: false },
+        });
+
+        return res.ok({ body: 'ok' });
+      });
+
+      await server.start();
+
+      expect(abortSignal?.aborted).toBeFalsy();
+
+      const incomingRequest = supertest(innerServer.listener)
+        .get('/')
+        // end required to send request
+        .end();
+
+      await new Promise((resolve) => {
+        setTimeout(() => {
+          void incomingRequest.on('abort', () => {
+            setTimeout(() => {
+              resolve(undefined);
+            }, 100);
+          });
+
+          void incomingRequest.abort();
+        }, 200);
+      });
+
+      expect(abortSignal?.aborted).toBe(true);
+    });
+  });
+
+  describe('excludedDataTiers filter', () => {
+    let esClientMock: jest.Mocked<ElasticsearchClient>;
+    let apmEventClient: APMEventClient;
+    let cancelEsRequestOnAbortSpy: jest.SpyInstance;
+    let unwrapEsResponseSpy: jest.SpyInstance;
+
+    const esResponse: estypes.SearchResponse = {
+      hits: {
+        total: { value: 1, relation: 'eq' },
+        hits: [{ _source: {}, _index: '' }],
+        max_score: 1,
+      },
+      took: 1,
+      _shards: { total: 1, successful: 1, skipped: 0, failed: 0 },
+      timed_out: false,
+    };
+
+    beforeAll(() => {
+      jest.resetModules();
     });
 
-    expect(abortSignal?.aborted).toBe(true);
+    beforeEach(() => {
+      cancelEsRequestOnAbortSpy = jest
+        .spyOn(cancelEsRequestOnAbortModule, 'cancelEsRequestOnAbort')
+        .mockImplementation(jest.fn());
+
+      unwrapEsResponseSpy = jest
+        .spyOn(observabilityPluginModule, 'unwrapEsResponse')
+        .mockImplementation(jest.fn());
+
+      esClientMock = {
+        search: jest.fn(),
+        msearch: jest.fn(),
+        eql: { search: jest.fn() },
+        fieldCaps: jest.fn(),
+        termsEnum: jest.fn(),
+      } as unknown as jest.Mocked<ElasticsearchClient>;
+
+      apmEventClient = new APMEventClient({
+        esClient: esClientMock,
+        debug: false,
+        request: {} as KibanaRequest,
+        indices: {} as APMIndices,
+        options: {
+          includeFrozen: false,
+          excludedDataTiers: ['data_warm', 'data_cold'],
+        },
+      });
+    });
+
+    afterAll(() => {
+      cancelEsRequestOnAbortSpy.mockReset();
+      unwrapEsResponseSpy.mockReset();
+    });
+
+    it('includes excludedDataTiers filter in search params', async () => {
+      esClientMock.search.mockResolvedValue(esResponse);
+
+      await apmEventClient.search('testOperation', {
+        apm: { events: [] },
+        body: {
+          size: 0,
+          track_total_hits: false,
+          query: { bool: { filter: [{ match_all: {} }] } },
+        },
+      });
+
+      const searchParams = esClientMock.search.mock.calls[0][0] as APMEventESSearchRequest;
+
+      expect(searchParams.body.query?.bool).toEqual({
+        filter: [
+          { terms: { 'processor.event': [] } },
+          { bool: { must_not: [{ terms: { _tier: ['data_warm', 'data_cold'] } }] } },
+        ],
+        must: [{ bool: { filter: [{ match_all: {} }] } }],
+      });
+    });
+
+    it('includes excludedDataTiers filter in msearch params', async () => {
+      esClientMock.msearch.mockResolvedValue({ responses: [esResponse], took: 1 });
+
+      await apmEventClient.msearch('testOperation', {
+        apm: { events: [] },
+        body: {
+          size: 0,
+          track_total_hits: false,
+          query: { bool: { filter: [{ match_all: {} }] } },
+        },
+      });
+
+      const msearchParams = esClientMock.msearch.mock.calls[0][0] as {
+        searches: MsearchMultisearchBody[];
+      };
+
+      expect(msearchParams.searches[1].query?.bool).toEqual({
+        filter: [
+          { bool: { filter: [{ match_all: {} }] } },
+          { terms: { 'processor.event': [] } },
+          { bool: { must_not: [{ terms: { _tier: ['data_warm', 'data_cold'] } }] } },
+        ],
+      });
+    });
+
+    it('includes excludedDataTiers filter in fieldCaps params', async () => {
+      esClientMock.fieldCaps.mockResolvedValue({
+        fields: {},
+        indices: '',
+      });
+
+      await apmEventClient.fieldCaps('testOperation', {
+        apm: { events: [] },
+        fields: ['field1'],
+        index_filter: { bool: { filter: [{ match_all: {} }] } },
+      });
+
+      const fieldCapsParams = esClientMock.fieldCaps.mock.calls[0][0] as APMEventFieldCapsRequest;
+      expect(fieldCapsParams?.index_filter?.bool).toEqual({
+        must: [
+          { bool: { filter: [{ match_all: {} }] } },
+          { bool: { must_not: [{ terms: { _tier: ['data_warm', 'data_cold'] } }] } },
+        ],
+      });
+    });
+
+    it('includes excludedDataTiers filter in termsEnum params', async () => {
+      esClientMock.termsEnum.mockResolvedValue({
+        terms: [''],
+        _shards: { total: 1, successful: 1, failed: 0 },
+        complete: true,
+      });
+
+      await apmEventClient.termsEnum('testOperation', {
+        apm: { events: [] },
+        field: 'field1',
+        index_filter: { bool: { filter: [{ match_all: {} }] } },
+      });
+
+      const termsEnumParams = esClientMock.termsEnum.mock.calls[0][0] as TermsEnumRequest;
+
+      expect(termsEnumParams.index_filter?.bool).toEqual({
+        must: [
+          { bool: { filter: [{ match_all: {} }] } },
+          { bool: { must_not: [{ terms: { _tier: ['data_warm', 'data_cold'] } }] } },
+        ],
+      });
+    });
   });
 });

x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/create_es_client/create_apm_event_client/index.ts

@@ -22,6 +22,8 @@ import { compact, omit } from 'lodash';
 import { ValuesType } from 'utility-types';
 import type { APMError, Metric, Span, Transaction, Event } from '@kbn/apm-types/es_schemas_ui';
 import type { InspectResponse } from '@kbn/observability-plugin/typings/common';
+import type { DataTier } from '@kbn/observability-shared-plugin/common';
+import { excludeTiersQuery } from '@kbn/observability-utils/es/queries/exclude_tiers_query';
 import { withApmSpan } from '../../../../utils';
 import type { ApmDataSource } from '../../../../../common/data_source';
 import { cancelEsRequestOnAbort } from '../cancel_es_request_on_abort';
@@ -29,6 +31,7 @@ import { callAsyncWithDebug, getDebugBody, getDebugTitle } from '../call_async_w
 import type { ProcessorEventOfDocumentType } from '../document_type';
 import type { APMIndices } from '../../../..';
 import { getRequestBase, processorEventsToIndex } from './get_request_base';
+import { getDataTierFilterCombined } from '../../tier_filter';
 
 export type APMEventESSearchRequest = Omit<ESSearchRequest, 'index'> & {
   apm: {
@@ -51,9 +54,9 @@ type APMEventWrapper<T> = Omit<T, 'index'> & {
   apm: { events: ProcessorEvent[] };
 };
 
-type APMEventTermsEnumRequest = APMEventWrapper<TermsEnumRequest>;
+export type APMEventTermsEnumRequest = APMEventWrapper<TermsEnumRequest>;
 type APMEventEqlSearchRequest = APMEventWrapper<EqlSearchRequest>;
-type APMEventFieldCapsRequest = APMEventWrapper<FieldCapsRequest>;
+export type APMEventFieldCapsRequest = APMEventWrapper<FieldCapsRequest>;
 
 type TypeOfProcessorEvent<T extends ProcessorEvent> = {
   [ProcessorEvent.error]: APMError;
@@ -88,6 +91,7 @@ export interface APMEventClientConfig {
   options: {
     includeFrozen: boolean;
     inspectableEsQueriesMap?: WeakMap<KibanaRequest, InspectResponse>;
+    excludedDataTiers?: DataTier[];
   };
 }
 
@@ -96,7 +100,10 @@ export class APMEventClient {
   private readonly debug: boolean;
   private readonly request: KibanaRequest;
   public readonly indices: APMIndices;
+  /** @deprecated Use {@link excludedDataTiers} instead.
+   * See https://www.elastic.co/guide/en/kibana/current/advanced-options.html **/
   private readonly includeFrozen: boolean;
+  private readonly excludedDataTiers?: DataTier[];
   private readonly inspectableEsQueriesMap?: WeakMap<KibanaRequest, InspectResponse>;
 
   constructor(config: APMEventClientConfig) {
@@ -105,6 +112,7 @@ export class APMEventClient {
     this.request = config.request;
     this.indices = config.indices;
     this.includeFrozen = config.options.includeFrozen;
+    this.excludedDataTiers = config.options.excludedDataTiers;
     this.inspectableEsQueriesMap = config.options.inspectableEsQueriesMap;
   }
 
@@ -159,6 +167,10 @@ export class APMEventClient {
       indices: this.indices,
     });
 
+    if (this.excludedDataTiers) {
+      filters.push(...excludeTiersQuery(this.excludedDataTiers));
+    }
+
     const searchParams = {
       ...omit(params, 'apm', 'body'),
       index,
@@ -195,6 +207,8 @@ export class APMEventClient {
     // Reusing indices configured for errors since both events and errors are stored as logs.
     const index = processorEventsToIndex([ProcessorEvent.error], this.indices);
 
+    const filter = this.excludedDataTiers ? excludeTiersQuery(this.excludedDataTiers) : undefined;
+
     const searchParams = {
       ...omit(params, 'body'),
       index,
@@ -202,6 +216,7 @@ export class APMEventClient {
         ...params.body,
         query: {
           bool: {
+            filter,
             must: compact([params.body.query]),
           },
         },
@@ -234,6 +249,10 @@ export class APMEventClient {
           indices: this.indices,
         });
 
+        if (this.excludedDataTiers) {
+          filters.push(...excludeTiersQuery(this.excludedDataTiers));
+        }
+
         const searchParams: [MsearchMultisearchHeader, MsearchMultisearchBody] = [
           {
             index,
@@ -295,9 +314,13 @@ export class APMEventClient {
   ): Promise<FieldCapsResponse> {
     const index = processorEventsToIndex(params.apm.events, this.indices);
 
-    const requestParams = {
+    const requestParams: Omit<APMEventFieldCapsRequest, 'apm'> & { index: string[] } = {
       ...omit(params, 'apm'),
       index,
+      index_filter: getDataTierFilterCombined({
+        filter: params.index_filter,
+        excludedDataTiers: this.excludedDataTiers,
+      }),
     };
 
     return this.callAsyncWithDebug({
@@ -314,9 +337,13 @@ export class APMEventClient {
   ): Promise<TermsEnumResponse> {
     const index = processorEventsToIndex(params.apm.events, this.indices);
 
-    const requestParams = {
+    const requestParams: Omit<APMEventTermsEnumRequest, 'apm'> & { index: string } = {
       ...omit(params, 'apm'),
       index: index.join(','),
+      index_filter: getDataTierFilterCombined({
+        filter: params.index_filter,
+        excludedDataTiers: this.excludedDataTiers,
+      }),
     };
 
     return this.callAsyncWithDebug({

x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/index.ts

@@ -21,3 +21,5 @@ export {
 } from './create_es_client/call_async_with_debug';
 
 export { cancelEsRequestOnAbort } from './create_es_client/cancel_es_request_on_abort';
+
+export { getDataTierFilterCombined } from './tier_filter';

x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/tier_filter.ts

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import type { DataTier } from '@kbn/observability-shared-plugin/common';
+import { excludeTiersQuery } from '@kbn/observability-utils/es/queries/exclude_tiers_query';
+
+export function getDataTierFilterCombined({
+  filter,
+  excludedDataTiers,
+}: {
+  filter?: QueryDslQueryContainer;
+  excludedDataTiers?: DataTier[];
+}): QueryDslQueryContainer | undefined {
+  if (!filter) {
+    return excludedDataTiers ? excludeTiersQuery(excludedDataTiers)[0] : undefined;
+  }
+
+  return !excludedDataTiers
+    ? filter
+    : {
+        bool: {
+          must: [filter, ...excludeTiersQuery(excludedDataTiers)],
+        },
+      };
+}

x-pack/plugins/observability_solution/apm_data_access/server/utils.ts

@@ -11,6 +11,7 @@ export {
   cancelEsRequestOnAbort,
   getDebugBody,
   getDebugTitle,
+  getDataTierFilterCombined,
 } from './lib/helpers';
 
 export { withApmSpan } from './utils/with_apm_span';

x-pack/plugins/observability_solution/apm_data_access/tsconfig.json

@@ -19,6 +19,7 @@
     "@kbn/core-http-server-mocks",
     "@kbn/apm-utils",
     "@kbn/core-http-server",
-    "@kbn/security-plugin-types-server"
+    "@kbn/security-plugin-types-server",
+    "@kbn/observability-utils"
   ]
 }

x-pack/plugins/observability_solution/observability/server/ui_settings.ts

@@ -649,8 +649,9 @@ export const uiSettings: Record<string, UiSettings> = {
     description: i18n.translate(
       'xpack.observability.advancedSettings.searchExcludedDataTiersDesc',
       {
-        defaultMessage: `Specify the data tiers to exclude from search, such as data_cold and/or data_frozen. 
+        defaultMessage: `{technicalPreviewLabel} Specify the data tiers to exclude from search, such as data_cold and/or data_frozen. 
         When configured, indices allocated in the selected tiers will be ignored from search requests. Affected apps: APM`,
+        values: { technicalPreviewLabel: `<em>[${technicalPreviewLabel}]</em>` },
       }
     ),
     value: [],

x-pack/plugins/observability_solution/observability_shared/common/ilm_types.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export enum IndexLifecyclePhaseSelectOption {
+  All = 'all',
+  Hot = 'hot',
+  Warm = 'warm',
+  Cold = 'cold',
+  Frozen = 'frozen',
+}
+
+export const indexLifeCyclePhaseToDataTier = {
+  [IndexLifecyclePhaseSelectOption.Hot]: 'data_hot',
+  [IndexLifecyclePhaseSelectOption.Warm]: 'data_warm',
+  [IndexLifecyclePhaseSelectOption.Cold]: 'data_cold',
+  [IndexLifecyclePhaseSelectOption.Frozen]: 'data_frozen',
+} as const;
+
+export type DataTier =
+  (typeof indexLifeCyclePhaseToDataTier)[keyof typeof indexLifeCyclePhaseToDataTier];

x-pack/plugins/observability_solution/observability_shared/common/index.ts

@@ -144,6 +144,11 @@ export {
 export { type Color, colorTransformer } from './color_palette';
 export { ObservabilityTriggerId } from './trigger_ids';
 export { getInspectResponse } from './utils/get_inspect_response';
+export {
+  type DataTier,
+  indexLifeCyclePhaseToDataTier,
+  IndexLifecyclePhaseSelectOption,
+} from './ilm_types';
 
 export const LOGS_ONBOARDING_FEEDBACK_LINK = 'https://ela.st/logs-onboarding-feedback';
 export const LOGS_EXPLORER_FEEDBACK_LINK = 'https://ela.st/explorer-feedback';

x-pack/plugins/observability_solution/profiling/common/storage_explorer.ts

@@ -4,16 +4,13 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
+import {
+  IndexLifecyclePhaseSelectOption,
+  indexLifeCyclePhaseToDataTier,
+} from '@kbn/observability-shared-plugin/common';
 import * as t from 'io-ts';
 
-export enum IndexLifecyclePhaseSelectOption {
-  All = 'all',
-  Hot = 'hot',
-  Warm = 'warm',
-  Cold = 'cold',
-  Frozen = 'frozen',
-}
-
+export { IndexLifecyclePhaseSelectOption, indexLifeCyclePhaseToDataTier };
 export const indexLifecyclePhaseRt = t.type({
   indexLifecyclePhase: t.union([
     t.literal(IndexLifecyclePhaseSelectOption.All),
@@ -24,13 +21,6 @@ export const indexLifecyclePhaseRt = t.type({
   ]),
 });
 
-export const indexLifeCyclePhaseToDataTier = {
-  [IndexLifecyclePhaseSelectOption.Hot]: 'data_hot',
-  [IndexLifecyclePhaseSelectOption.Warm]: 'data_warm',
-  [IndexLifecyclePhaseSelectOption.Cold]: 'data_cold',
-  [IndexLifecyclePhaseSelectOption.Frozen]: 'data_frozen',
-};
-
 export interface StorageExplorerSummaryAPIResponse {
   totalProfilingSizeBytes: number;
   totalSymbolsSizeBytes: number;