[8.x] [Infra] Exclude frozen/cold data tiers from source queries (#201804) (#202789)

# Backport This will backport the following commits from `main` to `8.x`: - [[Infra] Exclude frozen/cold data tiers from source queries (#201804)](https://github.com/elastic/kibana/pull/201804)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport)  Co-authored-by: Gonçalo Rica Pais da Silva <goncalo.rica@elastic.co>
2025-04-24 09:48:58 -04:00 · 2024-12-13 02:23:36 +11:00 · 2024-12-13 02:23:36 +11:00 · c59e9e2ad3
commit c59e9e2ad3
parent 8dcbb64d73
7 changed files with 198 additions and 9 deletions
--- a/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/tier_filter.ts
+++ b/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/tier_filter.ts
@ -13,10 +13,10 @@ export function getDataTierFilterCombined({
  excludedDataTiers,
 }: {
  filter?: QueryDslQueryContainer;
-  excludedDataTiers: DataTier[];
+  excludedDataTiers?: DataTier[];
 }): QueryDslQueryContainer | undefined {
  if (!filter) {
-    return excludedDataTiers.length > 0 ? excludeTiersQuery(excludedDataTiers)[0] : undefined;
+    return excludedDataTiers?.length ? excludeTiersQuery(excludedDataTiers)[0] : undefined;
  }

  return !excludedDataTiers
--- a/x-pack/plugins/observability_solution/infra/server/lib/adapters/source_status/elasticsearch_source_status_adapter.ts
+++ b/x-pack/plugins/observability_solution/infra/server/lib/adapters/source_status/elasticsearch_source_status_adapter.ts
@ -5,11 +5,14 @@
 * 2.0.
 */

+import type { DataTier } from '@kbn/observability-shared-plugin/common';
+import { searchExcludedDataTiers } from '@kbn/observability-plugin/common/ui_settings_keys';
+import { excludeTiersQuery } from '@kbn/observability-utils-common/es/queries/exclude_tiers_query';
 import type { InfraPluginRequestHandlerContext } from '../../../types';
 import { isNoSuchRemoteClusterMessage, NoSuchRemoteClusterError } from '../../sources/errors';
-import { InfraSourceStatusAdapter, SourceIndexStatus } from '../../source_status';
-import { InfraDatabaseGetIndicesResponse } from '../framework';
-import { KibanaFramework } from '../framework/kibana_framework_adapter';
+import type { InfraSourceStatusAdapter, SourceIndexStatus } from '../../source_status';
+import type { InfraDatabaseGetIndicesResponse } from '../framework';
+import type { KibanaFramework } from '../framework/kibana_framework_adapter';

 export class InfraElasticsearchSourceStatusAdapter implements InfraSourceStatusAdapter {
  constructor(private readonly framework: KibanaFramework) {}
@ -46,6 +49,12 @@ export class InfraElasticsearchSourceStatusAdapter implements InfraSourceStatusA
    requestContext: InfraPluginRequestHandlerContext,
    indexNames: string
  ): Promise<SourceIndexStatus> {
+    const { uiSettings } = await requestContext.core;
+
+    const excludedDataTiers = await uiSettings.client.get<DataTier[]>(searchExcludedDataTiers);
+
+    const filter = excludedDataTiers.length ? excludeTiersQuery(excludedDataTiers) : [];
+
    return await this.framework
      .callWithRequest(requestContext, 'search', {
        ignore_unavailable: true,
@ -54,6 +63,7 @@ export class InfraElasticsearchSourceStatusAdapter implements InfraSourceStatusA
        size: 0,
        terminate_after: 1,
        track_total_hits: 1,
+        query: { bool: { filter } },
      })
      .then(
        (response) => {
--- a/x-pack/plugins/observability_solution/infra/server/lib/helpers/get_infra_metrics_client.test.ts
+++ b/x-pack/plugins/observability_solution/infra/server/lib/helpers/get_infra_metrics_client.test.ts
@ -0,0 +1,151 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { KibanaRequest } from '@kbn/core-http-server';
+import { DataTier } from '@kbn/observability-shared-plugin/common';
+import { InfraBackendLibs } from '../infra_types';
+import { getInfraMetricsClient } from './get_infra_metrics_client';
+import { InfraPluginRequestHandlerContext } from '../../types';
+import { QueryDslQueryContainer } from '@kbn/data-views-plugin/common/types';
+
+const withExcludedDataTiers = (tiers: DataTier[]) => ({
+  uiSettings: {
+    client: {
+      get: () => Promise.resolve(tiers),
+    },
+  },
+});
+
+const mockedInfra = { getMetricsIndices: () => Promise.resolve(['*.indices']) };
+
+const infraMetricsTestHarness =
+  (tiers: DataTier[], input: QueryDslQueryContainer | undefined, output: QueryDslQueryContainer) =>
+  async () => {
+    const callWithRequest = jest.fn();
+
+    const mockedCore = withExcludedDataTiers(tiers);
+
+    const context = {
+      infra: Promise.resolve(mockedInfra),
+      core: Promise.resolve(mockedCore),
+    } as unknown as InfraPluginRequestHandlerContext;
+
+    const client = await getInfraMetricsClient({
+      libs: { framework: { callWithRequest } } as unknown as InfraBackendLibs,
+      context,
+      request: {} as unknown as KibanaRequest,
+    });
+
+    await client.search({
+      body: {
+        query: input,
+        size: 1,
+        track_total_hits: false,
+      },
+    });
+
+    expect(callWithRequest).toBeCalledWith(
+      context,
+      'search',
+      {
+        body: {
+          query: output,
+          size: 1,
+          track_total_hits: false,
+        },
+        ignore_unavailable: true,
+        index: ['*.indices'],
+      },
+      {}
+    );
+  };
+
+describe('getInfraMetricsClient', () => {
+  it(
+    'defines an empty must_not query if given no data tiers to filter by',
+    infraMetricsTestHarness([], undefined, { bool: { must_not: [] } })
+  );
+
+  it(
+    'includes excluded data tiers in the request filter by default',
+    infraMetricsTestHarness(['data_frozen'], undefined, {
+      bool: {
+        must_not: [
+          {
+            terms: {
+              _tier: ['data_frozen'],
+            },
+          },
+        ],
+      },
+    })
+  );
+
+  it(
+    'merges provided filters with the excluded data tier filter',
+    infraMetricsTestHarness(
+      ['data_frozen'],
+      {
+        bool: {
+          must_not: {
+            exists: {
+              field: 'a-field',
+            },
+          },
+        },
+      },
+      {
+        bool: {
+          must_not: [
+            {
+              exists: {
+                field: 'a-field',
+              },
+            },
+            {
+              terms: {
+                _tier: ['data_frozen'],
+              },
+            },
+          ],
+        },
+      }
+    )
+  );
+
+  it(
+    'merges other query params with the excluded data tiers filter',
+    infraMetricsTestHarness(
+      ['data_frozen'],
+      {
+        bool: {
+          must: {
+            exists: {
+              field: 'a-field',
+            },
+          },
+        },
+      },
+      {
+        bool: {
+          must: {
+            exists: {
+              field: 'a-field',
+            },
+          },
+          must_not: [
+            {
+              terms: {
+                _tier: ['data_frozen'],
+              },
+            },
+          ],
+        },
+      }
+    )
+  );
+});
--- a/x-pack/plugins/observability_solution/infra/server/lib/helpers/get_infra_metrics_client.ts
+++ b/x-pack/plugins/observability_solution/infra/server/lib/helpers/get_infra_metrics_client.ts
@ -6,6 +6,10 @@
 */
 import type { ESSearchRequest, InferSearchResponseOf } from '@kbn/es-types';
 import type { KibanaRequest } from '@kbn/core/server';
+import { searchExcludedDataTiers } from '@kbn/observability-plugin/common/ui_settings_keys';
+import type { DataTier } from '@kbn/observability-shared-plugin/common';
+import { excludeTiersQuery } from '@kbn/observability-utils-common/es/queries/exclude_tiers_query';
+import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types';
 import type { InfraPluginRequestHandlerContext } from '../../types';
 import type { InfraBackendLibs } from '../infra_types';

@ -29,12 +33,25 @@ export async function getInfraMetricsClient({
 }) {
  const { framework } = libs;
  const infraContext = await context.infra;
+  const { uiSettings } = await context.core;
+
+  const excludedDataTiers = await uiSettings.client.get<DataTier[]>(searchExcludedDataTiers);
  const metricsIndices = await infraContext.getMetricsIndices();

+  const excludedQuery = excludedDataTiers.length
+    ? excludeTiersQuery(excludedDataTiers)[0].bool!.must_not!
+    : [];
+
  return {
    search<TDocument, TParams extends RequiredParams>(
      searchParams: TParams
    ): Promise<InferSearchResponseOf<TDocument, TParams>> {
+      const searchFilter = searchParams.body.query?.bool?.must_not ?? [];
+
+      // This flattens arrays by one level, and non-array values can be added as well, so it all
+      // results in a nice [QueryDsl, QueryDsl, ...] array.
+      const mustNot = ([] as QueryDslQueryContainer[]).concat(searchFilter, excludedQuery);
+
      return framework.callWithRequest(
        context,
        'search',
@ -42,6 +59,16 @@ export async function getInfraMetricsClient({
          ...searchParams,
          ignore_unavailable: true,
          index: metricsIndices,
+          body: {
+            ...searchParams.body,
+            query: {
+              ...searchParams.body.query,
+              bool: {
+                ...searchParams.body.query?.bool,
+                must_not: mustNot,
+              },
+            },
+          },
        },
        request
      ) as Promise<any>;
--- a/x-pack/plugins/observability_solution/infra/server/routes/metrics_sources/index.ts
+++ b/x-pack/plugins/observability_solution/infra/server/routes/metrics_sources/index.ts
@ -208,7 +208,7 @@ export const initMetricsSourceConfigurationRoutes = (libs: InfraBackendLibs) =>
    },
    async (context, request, response) => {
      try {
-        const modules = castArray(request.query.modules);
+        const modules = request.query.modules ? castArray(request.query.modules) : [];

        if (modules.length > MAX_MODULES) {
          throw Boom.badRequest(
--- a/x-pack/plugins/observability_solution/infra/tsconfig.json
+++ b/x-pack/plugins/observability_solution/infra/tsconfig.json
@ -117,6 +117,7 @@
    "@kbn/entities-schema",
    "@kbn/zod",
    "@kbn/observability-utils-server",
+    "@kbn/observability-utils-common",
    "@kbn/core-plugins-server",
    "@kbn/config",
  ],
--- a/x-pack/plugins/observability_solution/observability/server/ui_settings.ts
+++ b/x-pack/plugins/observability_solution/observability/server/ui_settings.ts
@ -531,7 +531,7 @@ export const uiSettings: Record<string, UiSettings> = {
    value: 1.7,
    description: i18n.translate('xpack.observability.profilingDatacenterPUEUiSettingDescription', {
      defaultMessage: `Data center power usage effectiveness (PUE) measures how efficiently a data center uses energy. Defaults to 1.7, the average on-premise data center PUE according to the <a>Uptime Institute</a> survey
-      
+
      You can also use the PUE that corresponds with your cloud provider:
      '<ul style="list-style-type: none;margin-left: 4px;">
        <li><strong>AWS:</strong> 1.135</li>
@ -649,8 +649,8 @@ export const uiSettings: Record<string, UiSettings> = {
    description: i18n.translate(
      'xpack.observability.advancedSettings.searchExcludedDataTiersDesc',
      {
-        defaultMessage: `{technicalPreviewLabel} Specify the data tiers to exclude from search, such as data_cold and/or data_frozen. 
-        When configured, indices allocated in the selected tiers will be ignored from search requests. Affected apps: APM`,
+        defaultMessage: `{technicalPreviewLabel} Specify the data tiers to exclude from search, such as data_cold and/or data_frozen.
+        When configured, indices allocated in the selected tiers will be ignored from search requests. Affected apps: APM, Infrastructure`,
        values: { technicalPreviewLabel: `<em>[${technicalPreviewLabel}]</em>` },
      }
    ),