[8.17] [Profiling x APM Integration] Use both host.hostname or host.name and container.id to correlate profiling data on APM (#201403) (#201635)

# Backport This will backport the following commits from `main` to `8.17`: - [[Profiling x APM Integration] Use both host.hostname or host.name and container.id to correlate profiling data on APM (#201403)](https://github.com/elastic/kibana/pull/201403)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport)  Co-authored-by: Cauê Marcondes <55978943+cauemarcondes@users.noreply.github.com>
2025-04-24 09:48:58 -04:00 · 2024-11-26 04:49:46 +11:00 · 2024-11-26 04:49:46 +11:00 · 86288e107d
commit 86288e107d
parent 247ea32b2e
9 changed files with 145 additions and 74 deletions
--- a/x-pack/plugins/observability_solution/apm/public/components/app/profiling_overview/filter_warning.tsx
+++ b/x-pack/plugins/observability_solution/apm/public/components/app/profiling_overview/filter_warning.tsx
@ -0,0 +1,66 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { EuiFlexGroup, EuiFlexItem, EuiIcon, EuiText, EuiToolTip } from '@elastic/eui';
+import React from 'react';
+import { i18n } from '@kbn/i18n';
+
+interface Props {
+  hostNames?: string[];
+  containerIds?: string[];
+}
+
+export function FilterWarning({ containerIds = [], hostNames = [] }: Props) {
+  const hasContainerIds = containerIds.length > 0;
+
+  return hasContainerIds ? (
+    <FilterWarningToolTip
+      values={containerIds}
+      label={i18n.translate('xpack.apm.profiling.topFunctions.filteredLabel.containerId', {
+        defaultMessage: "Displaying profiling insights from the service's container id(s)",
+      })}
+    />
+  ) : (
+    <FilterWarningToolTip
+      values={hostNames}
+      label={i18n.translate('xpack.apm.profiling.topFunctions.filteredLabel.hostName', {
+        defaultMessage: "Displaying profiling insights from the service's host(s)",
+      })}
+    />
+  );
+}
+
+interface FilterWarningToolTipProps {
+  values: string[];
+  label: string;
+}
+function FilterWarningToolTip({ values = [], label }: FilterWarningToolTipProps) {
+  function renderTooltipOptions() {
+    return (
+      <ul>
+        {values.map((value) => (
+          <li key={value}>{`- ${value}`}</li>
+        ))}
+      </ul>
+    );
+  }
+
+  return (
+    <EuiFlexGroup gutterSize="none">
+      <EuiFlexItem grow={false}>
+        <EuiText size="xs" color="subdued">
+          {label}
+        </EuiText>
+      </EuiFlexItem>
+      <EuiFlexItem grow={false}>
+        <EuiToolTip content={renderTooltipOptions()}>
+          <EuiIcon type="questionInCircle" />
+        </EuiToolTip>
+      </EuiFlexItem>
+    </EuiFlexGroup>
+  );
+}
--- a/x-pack/plugins/observability_solution/apm/public/components/app/profiling_overview/host_names_filter_warning.tsx
+++ b/x-pack/plugins/observability_solution/apm/public/components/app/profiling_overview/host_names_filter_warning.tsx
@ -1,42 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import { EuiFlexGroup, EuiFlexItem, EuiIcon, EuiText, EuiToolTip } from '@elastic/eui';
-import { i18n } from '@kbn/i18n';
-import React from 'react';
-
-interface Props {
-  hostNames?: string[];
-}
-export function HostnamesFilterWarning({ hostNames = [] }: Props) {
-  function renderTooltipOptions() {
-    return (
-      <ul>
-        {hostNames.map((hostName) => (
-          <li key={hostName}>{`- ${hostName}`}</li>
-        ))}
-      </ul>
-    );
-  }
-
-  return (
-    <EuiFlexGroup gutterSize="none">
-      <EuiFlexItem grow={false}>
-        <EuiText size="xs" color="subdued">
-          {i18n.translate('xpack.apm.profiling.flamegraph.filteredLabel', {
-            defaultMessage: "Displaying profiling insights from the service's host(s)",
-          })}
-        </EuiText>
-      </EuiFlexItem>
-      <EuiFlexItem grow={false}>
-        <EuiToolTip content={renderTooltipOptions()}>
-          <EuiIcon type="questionInCircle" />
-        </EuiToolTip>
-      </EuiFlexItem>
-    </EuiFlexGroup>
-  );
-}
--- a/x-pack/plugins/observability_solution/apm/public/components/app/profiling_overview/profiling_hosts_flamegraph.tsx
+++ b/x-pack/plugins/observability_solution/apm/public/components/app/profiling_overview/profiling_hosts_flamegraph.tsx
@ -9,12 +9,12 @@ import { EuiFlexGroup, EuiFlexItem, EuiSpacer } from '@elastic/eui';
 import React from 'react';
 import { ApmDataSourceWithSummary } from '../../../../common/data_source';
 import { ApmDocumentType } from '../../../../common/document_type';
-import { HOST_NAME } from '../../../../common/es_fields/apm';
+import { CONTAINER_ID, HOST_NAME } from '../../../../common/es_fields/apm';
 import { mergeKueries, toKueryFilterFormat } from '../../../../common/utils/kuery_utils';
 import { useFetcher } from '../../../hooks/use_fetcher';
 import { FlamegraphChart } from '../../shared/charts/flamegraph';
 import { ProfilingFlamegraphLink } from '../../shared/profiling/flamegraph/flamegraph_link';
-import { HostnamesFilterWarning } from './host_names_filter_warning';
+import { FilterWarning } from './filter_warning';

 interface Props {
  serviceName: string;
@ -60,17 +60,20 @@ export function ProfilingHostsFlamegraph({
    [dataSource, serviceName, start, end, environment, kuery]
  );

-  const hostNamesKueryFormat = toKueryFilterFormat(HOST_NAME, data?.hostNames || []);
+  const profilingKueryFilter =
+    data?.containerIds && data.containerIds.length > 0
+      ? toKueryFilterFormat(CONTAINER_ID, data?.containerIds || [])
+      : toKueryFilterFormat(HOST_NAME, data?.hostNames || []);

  return (
    <>
      <EuiFlexGroup>
        <EuiFlexItem grow={false}>
-          <HostnamesFilterWarning hostNames={data?.hostNames} />
+          <FilterWarning containerIds={data?.containerIds} hostNames={data?.hostNames} />
        </EuiFlexItem>
        <EuiFlexItem>
          <ProfilingFlamegraphLink
-            kuery={mergeKueries([`(${hostNamesKueryFormat})`, kuery])}
+            kuery={mergeKueries([`(${profilingKueryFilter})`, kuery])}
            rangeFrom={rangeFrom}
            rangeTo={rangeTo}
            justifyContent="flexEnd"
--- a/x-pack/plugins/observability_solution/apm/public/components/app/profiling_overview/profiling_hosts_top_functions.tsx
+++ b/x-pack/plugins/observability_solution/apm/public/components/app/profiling_overview/profiling_hosts_top_functions.tsx
@ -10,11 +10,11 @@ import { EmbeddableFunctions } from '@kbn/observability-shared-plugin/public';
 import React from 'react';
 import { ApmDataSourceWithSummary } from '../../../../common/data_source';
 import { ApmDocumentType } from '../../../../common/document_type';
-import { HOST_NAME } from '../../../../common/es_fields/apm';
+import { CONTAINER_ID, HOST_NAME } from '../../../../common/es_fields/apm';
 import { mergeKueries, toKueryFilterFormat } from '../../../../common/utils/kuery_utils';
 import { isPending, useFetcher } from '../../../hooks/use_fetcher';
 import { ProfilingTopNFunctionsLink } from '../../shared/profiling/top_functions/top_functions_link';
-import { HostnamesFilterWarning } from './host_names_filter_warning';
+import { FilterWarning } from './filter_warning';

 interface Props {
  serviceName: string;
@ -66,17 +66,20 @@ export function ProfilingHostsTopNFunctions({
    [dataSource, serviceName, start, end, environment, startIndex, endIndex, kuery]
  );

-  const hostNamesKueryFormat = toKueryFilterFormat(HOST_NAME, data?.hostNames || []);
+  const profilingKueryFilter =
+    data?.containerIds && data.containerIds.length > 0
+      ? toKueryFilterFormat(CONTAINER_ID, data?.containerIds || [])
+      : toKueryFilterFormat(HOST_NAME, data?.hostNames || []);

  return (
    <>
      <EuiFlexGroup>
        <EuiFlexItem grow={false}>
-          <HostnamesFilterWarning hostNames={data?.hostNames} />
+          <FilterWarning containerIds={data?.containerIds} hostNames={data?.hostNames} />
        </EuiFlexItem>
        <EuiFlexItem>
          <ProfilingTopNFunctionsLink
-            kuery={mergeKueries([`(${hostNamesKueryFormat})`, kuery])}
+            kuery={mergeKueries([`(${profilingKueryFilter})`, kuery])}
            rangeFrom={rangeFrom}
            rangeTo={rangeTo}
            justifyContent="flexEnd"
--- a/x-pack/plugins/observability_solution/apm/server/routes/profiling/get_service_correlation_fields.ts
+++ b/x-pack/plugins/observability_solution/apm/server/routes/profiling/get_service_correlation_fields.ts
@ -5,13 +5,26 @@
 * 2.0.
 */
 import { rangeQuery } from '@kbn/observability-plugin/server';
-import { ApmServiceTransactionDocumentType } from '../../../common/document_type';
-import { HOST_HOSTNAME, SERVICE_NAME } from '../../../common/es_fields/apm';
-import { RollupInterval } from '../../../common/rollup';
+import type { ApmServiceTransactionDocumentType } from '../../../common/document_type';
+import {
+  CONTAINER_ID,
+  HOST_HOSTNAME,
+  HOST_NAME,
+  SERVICE_NAME,
+} from '../../../common/es_fields/apm';
+import type { RollupInterval } from '../../../common/rollup';
 import { environmentQuery } from '../../../common/utils/environment_query';
-import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client';
+import type { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client';

-export async function getServiceHostNames({
+const getBucketKeysAsString = (
+  buckets?: Array<{
+    doc_count: number;
+    key: string | number;
+    key_as_string?: string | undefined;
+  }>
+) => buckets?.map((bucket) => bucket.key as string) || [];
+
+export async function getServiceCorrelationFields({
  apmEventClient,
  serviceName,
  start,
@ -45,15 +58,36 @@ export async function getServiceHostNames({
        },
      },
      aggs: {
-        hostNames: {
+        hostHostNames: {
          terms: {
            field: HOST_HOSTNAME,
            size: 500,
          },
        },
+        hostNames: {
+          terms: {
+            field: HOST_NAME,
+            size: 500,
+          },
+        },
+        containerIds: {
+          terms: {
+            field: CONTAINER_ID,
+            size: 500,
+          },
+        },
      },
    },
  });

-  return response.aggregations?.hostNames.buckets.map((bucket) => bucket.key as string) || [];
+  const allHostNames = [
+    ...getBucketKeysAsString(response.aggregations?.hostHostNames.buckets),
+    ...getBucketKeysAsString(response.aggregations?.hostNames.buckets),
+  ];
+  const hostNames = new Set<string>(allHostNames);
+
+  return {
+    hostNames: Array.from(hostNames),
+    containerIds: getBucketKeysAsString(response.aggregations?.containerIds.buckets),
+  };
 }
--- a/x-pack/plugins/observability_solution/apm/server/routes/profiling/hosts/route.ts
+++ b/x-pack/plugins/observability_solution/apm/server/routes/profiling/hosts/route.ts
@ -8,7 +8,7 @@
 import { toNumberRt } from '@kbn/io-ts-utils';
 import type { BaseFlameGraph, TopNFunctions } from '@kbn/profiling-utils';
 import * as t from 'io-ts';
-import { HOST_NAME } from '../../../../common/es_fields/apm';
+import { CONTAINER_ID, HOST_NAME } from '../../../../common/es_fields/apm';
 import { mergeKueries, toKueryFilterFormat } from '../../../../common/utils/kuery_utils';
 import { getApmEventClient } from '../../../lib/helpers/get_apm_event_client';
 import { createApmServerRoute } from '../../apm_routes/create_apm_server_route';
@ -20,7 +20,7 @@ import {
 } from '../../default_api_types';
 import { fetchFlamegraph } from '../fetch_flamegraph';
 import { fetchFunctions } from '../fetch_functions';
-import { getServiceHostNames } from '../get_service_host_names';
+import { getServiceCorrelationFields } from '../get_service_correlation_fields';

 const profilingHostsFlamegraphRoute = createApmServerRoute({
  endpoint: 'GET /internal/apm/services/{serviceName}/profiling/hosts/flamegraph',
@ -31,7 +31,9 @@ const profilingHostsFlamegraphRoute = createApmServerRoute({
  options: { tags: ['access:apm'] },
  handler: async (
    resources
-  ): Promise<{ flamegraph: BaseFlameGraph; hostNames: string[] } | undefined> => {
+  ): Promise<
+    { flamegraph: BaseFlameGraph; hostNames: string[]; containerIds: string[] } | undefined
+  > => {
    const { context, plugins, params } = resources;
    const core = await context.core;
    const [esClient, apmEventClient, profilingDataAccessStart] = await Promise.all([
@ -43,7 +45,7 @@ const profilingHostsFlamegraphRoute = createApmServerRoute({
      const { start, end, environment, documentType, rollupInterval, kuery } = params.query;
      const { serviceName } = params.path;

-      const serviceHostNames = await getServiceHostNames({
+      const { hostNames, containerIds } = await getServiceCorrelationFields({
        apmEventClient,
        start,
        end,
@ -53,7 +55,7 @@ const profilingHostsFlamegraphRoute = createApmServerRoute({
        rollupInterval,
      });

-      if (!serviceHostNames.length) {
+      if (!hostNames.length && !containerIds.length) {
        return undefined;
      }
      const startSecs = start / 1000;
@ -65,10 +67,13 @@ const profilingHostsFlamegraphRoute = createApmServerRoute({
        esClient: esClient.asCurrentUser,
        start: startSecs,
        end: endSecs,
-        kuery: mergeKueries([`(${toKueryFilterFormat(HOST_NAME, serviceHostNames)})`, kuery]),
+        kuery:
+          containerIds.length > 0
+            ? mergeKueries([`(${toKueryFilterFormat(CONTAINER_ID, containerIds)})`, kuery])
+            : mergeKueries([`(${toKueryFilterFormat(HOST_NAME, hostNames)})`, kuery]),
      });

-      return { flamegraph, hostNames: serviceHostNames };
+      return { flamegraph, hostNames, containerIds };
    }

    return undefined;
@ -90,7 +95,9 @@ const profilingHostsFunctionsRoute = createApmServerRoute({
  options: { tags: ['access:apm'] },
  handler: async (
    resources
-  ): Promise<{ functions: TopNFunctions; hostNames: string[] } | undefined> => {
+  ): Promise<
+    { functions: TopNFunctions; hostNames: string[]; containerIds: string[] } | undefined
+  > => {
    const { context, plugins, params } = resources;
    const core = await context.core;
    const [esClient, apmEventClient, profilingDataAccessStart] = await Promise.all([
@ -103,7 +110,7 @@ const profilingHostsFunctionsRoute = createApmServerRoute({
        params.query;
      const { serviceName } = params.path;

-      const serviceHostNames = await getServiceHostNames({
+      const { hostNames, containerIds } = await getServiceCorrelationFields({
        apmEventClient,
        start,
        end,
@ -113,7 +120,7 @@ const profilingHostsFunctionsRoute = createApmServerRoute({
        rollupInterval,
      });

-      if (!serviceHostNames.length) {
+      if (!hostNames.length && !containerIds.length) {
        return undefined;
      }

@ -128,10 +135,13 @@ const profilingHostsFunctionsRoute = createApmServerRoute({
        endIndex,
        start: startSecs,
        end: endSecs,
-        kuery: mergeKueries([`(${toKueryFilterFormat(HOST_NAME, serviceHostNames)})`, kuery]),
+        kuery:
+          containerIds.length > 0
+            ? mergeKueries([`(${toKueryFilterFormat(CONTAINER_ID, containerIds)})`, kuery])
+            : mergeKueries([`(${toKueryFilterFormat(HOST_NAME, hostNames)})`, kuery]),
      });

-      return { functions, hostNames: serviceHostNames };
+      return { functions, hostNames, containerIds };
    }

    return undefined;
--- a/x-pack/plugins/translations/translations/fr-FR.json
+++ b/x-pack/plugins/translations/translations/fr-FR.json
@ -11542,7 +11542,6 @@
    "xpack.apm.profiling.callout.dismiss": "Rejeter",
    "xpack.apm.profiling.callout.learnMore": "En savoir plus",
    "xpack.apm.profiling.callout.title": "Affichage des informations de profilage de l'hôte ou des hôtes exécutant des services {serviceName}",
-    "xpack.apm.profiling.flamegraph.filteredLabel": "Affichage des informations de profilage du ou des hôtes des services",
    "xpack.apm.profiling.flamegraph.link": "Accéder au flame-graph d'Universal Profiling",
    "xpack.apm.profiling.flamegraph.noDataFound": "Aucune donnée trouvée",
    "xpack.apm.profiling.tabs.flamegraph": "Flame-graph",
--- a/x-pack/plugins/translations/translations/ja-JP.json
+++ b/x-pack/plugins/translations/translations/ja-JP.json
@ -11525,7 +11525,6 @@
    "xpack.apm.profiling.callout.dismiss": "閉じる",
    "xpack.apm.profiling.callout.learnMore": "詳細",
    "xpack.apm.profiling.callout.title": "{serviceName}サービスを実行しているホストからプロファイリングインサイトを表示しています",
-    "xpack.apm.profiling.flamegraph.filteredLabel": "サービスのホストからプロファイリングインサイトを表示しています",
    "xpack.apm.profiling.flamegraph.link": "ユニバーサルプロファイリングFlamegraphに移動",
    "xpack.apm.profiling.flamegraph.noDataFound": "データが見つかりません",
    "xpack.apm.profiling.tabs.flamegraph": "Flamegraph",
--- a/x-pack/plugins/translations/translations/zh-CN.json
+++ b/x-pack/plugins/translations/translations/zh-CN.json
@ -11551,7 +11551,6 @@
    "xpack.apm.profiling.callout.dismiss": "关闭",
    "xpack.apm.profiling.callout.learnMore": "了解详情",
    "xpack.apm.profiling.callout.title": "正在显示运行 {serviceName} 服务的主机中的分析洞见",
-    "xpack.apm.profiling.flamegraph.filteredLabel": "正在显示服务主机中的分析洞见",
    "xpack.apm.profiling.flamegraph.link": "前往 Universal Profiling 火焰图",
    "xpack.apm.profiling.flamegraph.noDataFound": "未找到任何数据",
    "xpack.apm.profiling.tabs.flamegraph": "火焰图",