[8.8] [APM] Check if metric fields exist (#145348) (#156006)

# Backport This will backport the following commits from `main` to `8.8`: - [[APM] Check if metric fields exist (#145348)](https://github.com/elastic/kibana/pull/145348)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport)  Co-authored-by: Katerina Patticha <aikaterini.patticha@elastic.co>
2025-04-24 01:38:56 -04:00 · 2023-05-03 13:17:11 -04:00 · 2023-05-03 13:17:11 -04:00 · b64f2f2d0b
commit b64f2f2d0b
parent e2047079d7
8 changed files with 263 additions and 35 deletions
--- a/packages/kbn-apm-synthtrace-client/src/lib/apm/apm_fields.ts
+++ b/packages/kbn-apm-synthtrace-client/src/lib/apm/apm_fields.ts
@ -12,6 +12,8 @@ export type ApmApplicationMetricFields = Partial<{
  'system.process.memory.size': number;
  'system.memory.actual.free': number;
  'system.memory.total': number;
+  'system.process.cgroup.memory.mem.limit.bytes': number;
+  'system.process.cgroup.memory.mem.usage.bytes': number;
  'system.cpu.total.norm.pct': number;
  'system.process.memory.rss.bytes': number;
  'system.process.cpu.total.norm.pct': number;
--- a/x-pack/plugins/apm/server/routes/metrics/snapshots/queries.test.ts.snap
+++ b/x-pack/plugins/apm/server/routes/metrics/snapshots/queries.test.ts.snap
@ -332,8 +332,27 @@ Object {
            },
          },
          Object {
-            "exists": Object {
-              "field": "system.process.cgroup.memory.mem.usage.bytes",
+            "bool": Object {
+              "filter": Array [
+                Object {
+                  "exists": Object {
+                    "field": "system.process.cgroup.memory.mem.usage.bytes",
+                  },
+                },
+              ],
+              "minimum_should_match": 1,
+              "should": Array [
+                Object {
+                  "exists": Object {
+                    "field": "system.process.cgroup.memory.mem.limit.bytes",
+                  },
+                },
+                Object {
+                  "exists": Object {
+                    "field": "system.memory.total",
+                  },
+                },
+              ],
            },
          },
        ],
@ -869,8 +888,27 @@ Object {
            },
          },
          Object {
-            "exists": Object {
-              "field": "system.process.cgroup.memory.mem.usage.bytes",
+            "bool": Object {
+              "filter": Array [
+                Object {
+                  "exists": Object {
+                    "field": "system.process.cgroup.memory.mem.usage.bytes",
+                  },
+                },
+              ],
+              "minimum_should_match": 1,
+              "should": Array [
+                Object {
+                  "exists": Object {
+                    "field": "system.process.cgroup.memory.mem.limit.bytes",
+                  },
+                },
+                Object {
+                  "exists": Object {
+                    "field": "system.memory.total",
+                  },
+                },
+              ],
            },
          },
        ],
@ -1385,8 +1423,27 @@ Object {
            },
          },
          Object {
-            "exists": Object {
-              "field": "system.process.cgroup.memory.mem.usage.bytes",
+            "bool": Object {
+              "filter": Array [
+                Object {
+                  "exists": Object {
+                    "field": "system.process.cgroup.memory.mem.usage.bytes",
+                  },
+                },
+              ],
+              "minimum_should_match": 1,
+              "should": Array [
+                Object {
+                  "exists": Object {
+                    "field": "system.process.cgroup.memory.mem.limit.bytes",
+                  },
+                },
+                Object {
+                  "exists": Object {
+                    "field": "system.memory.total",
+                  },
+                },
+              ],
            },
          },
        ],
--- a/x-pack/plugins/apm/server/routes/metrics/by_agent/shared/memory/index.ts
+++ b/x-pack/plugins/apm/server/routes/metrics/by_agent/shared/memory/index.ts
@ -47,6 +47,15 @@ const chartBase: ChartBase = {
  series,
 };

+export const systemMemoryFilter = {
+  bool: {
+    filter: [
+      { exists: { field: METRIC_SYSTEM_FREE_MEMORY } },
+      { exists: { field: METRIC_SYSTEM_TOTAL_MEMORY } },
+    ],
+  },
+};
+
 export const percentSystemMemoryUsedScript = {
  lang: 'painless',
  source: `
@ -60,6 +69,17 @@ export const percentSystemMemoryUsedScript = {
  `,
 } as const;

+export const cgroupMemoryFilter = {
+  bool: {
+    filter: [{ exists: { field: METRIC_CGROUP_MEMORY_USAGE_BYTES } }],
+    should: [
+      { exists: { field: METRIC_CGROUP_MEMORY_LIMIT_BYTES } },
+      { exists: { field: METRIC_SYSTEM_TOTAL_MEMORY } },
+    ],
+    minimum_should_match: 1,
+  },
+};
+
 export const percentCgroupMemoryUsedScript = {
  lang: 'painless',
  source: `
@ -147,7 +167,7 @@ export async function getMemoryChartData({
          memoryUsedMax: { max: { script: percentCgroupMemoryUsedScript } },
        },
        additionalFilters: [
-          { exists: { field: METRIC_CGROUP_MEMORY_USAGE_BYTES } },
+          cgroupMemoryFilter,
          ...termQuery(FAAS_ID, serverlessId),
        ],
        operationName: 'get_cgroup_memory_metrics_charts',
@ -169,8 +189,7 @@ export async function getMemoryChartData({
            memoryUsedMax: { max: { script: percentSystemMemoryUsedScript } },
          },
          additionalFilters: [
-            { exists: { field: METRIC_SYSTEM_FREE_MEMORY } },
-            { exists: { field: METRIC_SYSTEM_TOTAL_MEMORY } },
+            systemMemoryFilter,
            ...termQuery(FAAS_ID, serverlessId),
          ],
          operationName: 'get_system_memory_metrics_charts',
--- a/x-pack/plugins/apm/server/routes/service_map/get_service_map_service_node_info.ts
+++ b/x-pack/plugins/apm/server/routes/service_map/get_service_map_service_node_info.ts
@ -9,10 +9,7 @@ import type { ESFilter } from '@kbn/es-types';
 import { rangeQuery } from '@kbn/observability-plugin/server';
 import { ProcessorEvent } from '@kbn/observability-plugin/common';
 import {
-  METRIC_CGROUP_MEMORY_USAGE_BYTES,
  METRIC_SYSTEM_CPU_PERCENT,
-  METRIC_SYSTEM_FREE_MEMORY,
-  METRIC_SYSTEM_TOTAL_MEMORY,
  SERVICE_NAME,
  TRANSACTION_TYPE,
 } from '../../../common/es_fields/apm';
@ -31,6 +28,8 @@ import { withApmSpan } from '../../utils/with_apm_span';
 import {
  percentCgroupMemoryUsedScript,
  percentSystemMemoryUsedScript,
+  systemMemoryFilter,
+  cgroupMemoryFilter,
 } from '../metrics/by_agent/shared/memory';
 import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client';
 import { ApmDocumentType } from '../../../common/document_type';
@ -358,19 +357,14 @@ function getMemoryStats({
    };

    let memoryUsage = await getMemoryUsage({
-      additionalFilters: [
-        { exists: { field: METRIC_CGROUP_MEMORY_USAGE_BYTES } },
-      ],
      script: percentCgroupMemoryUsedScript,
+      additionalFilters: [cgroupMemoryFilter],
    });

    if (!memoryUsage) {
      memoryUsage = await getMemoryUsage({
-        additionalFilters: [
-          { exists: { field: METRIC_SYSTEM_FREE_MEMORY } },
-          { exists: { field: METRIC_SYSTEM_TOTAL_MEMORY } },
-        ],
        script: percentSystemMemoryUsedScript,
+        additionalFilters: [systemMemoryFilter],
      });
    }

--- a/x-pack/plugins/apm/server/routes/services/get_service_instances/get_service_instances_system_metric_statistics.ts
+++ b/x-pack/plugins/apm/server/routes/services/get_service_instances/get_service_instances_system_metric_statistics.ts
@ -9,10 +9,7 @@ import type { AggregationOptionsByType } from '@kbn/es-types';
 import { kqlQuery, rangeQuery } from '@kbn/observability-plugin/server';
 import { ProcessorEvent } from '@kbn/observability-plugin/common';
 import {
-  METRIC_CGROUP_MEMORY_USAGE_BYTES,
  METRIC_PROCESS_CPU_PERCENT,
-  METRIC_SYSTEM_FREE_MEMORY,
-  METRIC_SYSTEM_TOTAL_MEMORY,
  SERVICE_NAME,
  SERVICE_NODE_NAME,
 } from '../../../../common/es_fields/apm';
@ -24,6 +21,8 @@ import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm
 import {
  percentCgroupMemoryUsedScript,
  percentSystemMemoryUsedScript,
+  systemMemoryFilter,
+  cgroupMemoryFilter,
 } from '../../metrics/by_agent/shared/memory';
 import { getOffsetInMs } from '../../../../common/utils/get_offset_in_ms';

@ -82,19 +81,6 @@ export async function getServiceInstancesSystemMetricStatistics<
    numBuckets,
  });

-  const systemMemoryFilter = {
-    bool: {
-      filter: [
-        { exists: { field: METRIC_SYSTEM_FREE_MEMORY } },
-        { exists: { field: METRIC_SYSTEM_TOTAL_MEMORY } },
-      ],
-    },
-  };
-
-  const cgroupMemoryFilter = {
-    exists: { field: METRIC_CGROUP_MEMORY_USAGE_BYTES },
-  };
-
  const cpuUsageFilter = { exists: { field: METRIC_PROCESS_CPU_PERCENT } };

  function withTimeseries<TParams extends AggregationOptionsByType['avg']>(
--- a/x-pack/test/apm_api_integration/tests/metrics/memory/generate_data.ts
+++ b/x-pack/test/apm_api_integration/tests/metrics/memory/generate_data.ts
@ -0,0 +1,77 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { apm, timerange } from '@kbn/apm-synthtrace-client';
+import type { ApmSynthtraceEsClient } from '@kbn/apm-synthtrace';
+
+const gigabytesToBytes = (value: number) => value * Math.pow(1024, 3);
+
+export const config = {
+  memoryTotal: gigabytesToBytes(64),
+  memoryFree: gigabytesToBytes(5.5),
+  cGroupMemoryLimit: gigabytesToBytes(8),
+  cGroupMemoryUsage: gigabytesToBytes(1.5),
+};
+
+export const expectedValues = {
+  expectedMemoryUsedRate: (config.memoryTotal - config.memoryFree) / config.memoryTotal,
+  expectedMemoryUsed: config.memoryTotal - config.memoryFree,
+};
+
+export async function generateData({
+  synthtraceEsClient,
+  start,
+  end,
+}: {
+  synthtraceEsClient: ApmSynthtraceEsClient;
+  start: number;
+  end: number;
+}) {
+  const { memoryTotal, memoryFree, cGroupMemoryLimit, cGroupMemoryUsage } = config;
+
+  const systemMetricOnlyInstance = apm
+    .service({ name: 'system-metric-only-service', environment: 'production', agentName: 'go' })
+    .instance('system-metric-only-production');
+
+  const cGroupMemoryOnlyInstance = apm
+    .service({ name: 'cgroup-memory-only-service', environment: 'production', agentName: 'go' })
+    .instance('cgroup-memory-only-production');
+
+  const cGroupMemoryWithLimitInstance = apm
+    .service({
+      name: 'cgroup-memory-with-limit-production',
+      environment: 'production',
+      agentName: 'go',
+    })
+    .instance('cgroup-memory-with-limit-production');
+
+  const transactionsEvents = timerange(start, end)
+    .ratePerMinute(1)
+    .generator((timestamp) => [
+      systemMetricOnlyInstance
+        .appMetrics({
+          'system.memory.actual.free': memoryFree,
+          'system.memory.total': memoryTotal,
+        })
+        .timestamp(timestamp),
+      cGroupMemoryOnlyInstance
+        .appMetrics({
+          'system.process.cgroup.memory.mem.usage.bytes': cGroupMemoryUsage,
+        })
+        .timestamp(timestamp),
+
+      cGroupMemoryWithLimitInstance
+        .appMetrics({
+          'system.process.cgroup.memory.mem.usage.bytes': cGroupMemoryUsage,
+          'system.process.cgroup.memory.mem.limit.bytes': cGroupMemoryLimit,
+          'system.memory.total': memoryTotal,
+          'system.memory.actual.free': memoryFree,
+        })
+        .timestamp(timestamp),
+    ]);
+
+  await synthtraceEsClient.index(transactionsEvents);
+}
--- a/x-pack/test/apm_api_integration/tests/metrics/memory/memory_metrics.spec.ts
+++ b/x-pack/test/apm_api_integration/tests/metrics/memory/memory_metrics.spec.ts
@ -0,0 +1,83 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import { FtrProviderContext } from '../../../common/ftr_provider_context';
+import { config, generateData } from './generate_data';
+
+export default function ApiTest({ getService }: FtrProviderContext) {
+  const registry = getService('registry');
+  const apmApiClient = getService('apmApiClient');
+  const synthtraceEsClient = getService('synthtraceEsClient');
+
+  const start = new Date('2023-01-01T00:00:00.000Z').getTime();
+  const end = new Date('2023-01-01T00:15:00.000Z').getTime() - 1;
+
+  async function callMetricChartsAPI(serviceName: string) {
+    return await apmApiClient.readUser({
+      endpoint: `GET /internal/apm/services/{serviceName}/metrics/charts`,
+      params: {
+        path: { serviceName },
+        query: {
+          environment: 'production',
+          start: new Date(start).toISOString(),
+          end: new Date(end).toISOString(),
+          kuery: '',
+          agentName: 'go',
+        },
+      },
+    });
+  }
+
+  registry.when('Memory', { config: 'trial', archives: [] }, () => {
+    before(async () => {
+      await generateData({ start, end, synthtraceEsClient });
+    });
+
+    after(() => synthtraceEsClient.clean());
+
+    it('returns system memory stats', async () => {
+      const expectedFreeMemory = 1 - config.memoryFree / config.memoryTotal;
+
+      const { status, body } = await callMetricChartsAPI('system-metric-only-service');
+      const memoryChart = body.charts.find(({ key }) => key === 'memory_usage_chart');
+
+      expect(status).to.be(200);
+      [
+        { title: 'Max', expectedValue: expectedFreeMemory },
+        { title: 'Average', expectedValue: expectedFreeMemory },
+      ].map(({ title, expectedValue }) => {
+        const series = memoryChart?.series.find((item) => item.title === title);
+        expect(series?.overallValue).to.eql(expectedValue);
+      });
+    });
+
+    it('returns cgroup memory with system.process.cgroup.memory.mem.limit.bytes', async () => {
+      const expectedFreeMemory = config.cGroupMemoryUsage / config.cGroupMemoryLimit;
+
+      const { status, body } = await callMetricChartsAPI('cgroup-memory-with-limit-production');
+      const memoryChart = body.charts.find(({ key }) => key === 'memory_usage_chart');
+
+      expect(status).to.be(200);
+      [
+        { title: 'Max', expectedValue: expectedFreeMemory },
+        { title: 'Average', expectedValue: expectedFreeMemory },
+      ].map(({ title, expectedValue }) => {
+        const series = memoryChart?.series.find((item) => item.title === title);
+        expect(series?.overallValue).to.eql(expectedValue);
+      });
+    });
+
+    it('handles cgroup memory stats when system.process.cgroup.memory.mem.limit.bytes and system.memory.total are not present', async () => {
+      const { status, body } = await callMetricChartsAPI('cgroup-memory-only-production');
+      expect(status).to.be(200);
+
+      const memoryChart = body.charts.find(({ key }) => key === 'memory_usage_chart');
+      expect(memoryChart?.series).to.eql([]);
+    });
+  });
+}
--- a/x-pack/test/apm_api_integration/tests/service_overview/instances_main_statistics.spec.ts
+++ b/x-pack/test/apm_api_integration/tests/service_overview/instances_main_statistics.spec.ts
@ -418,6 +418,16 @@ export default function ApiTest({ getService }: FtrProviderContext) {
            expect(javaStats).to.be(undefined);
          });

+          it('does not return metrics', () => {
+            const goAStats = body.currentPeriod.find(
+              (stat) => stat.serviceNodeName === 'go-instance-a'
+            );
+
+            expect(goAStats).to.not.be(undefined);
+            expect(goAStats?.memoryUsage).to.be(undefined);
+            expect(goAStats?.cpuUsage).to.be(undefined);
+          });
+
          it('does not return data for missing service node name', () => {
            const missingNameStats = body.currentPeriod.find(
              (stat) => stat.serviceNodeName === SERVICE_NODE_NAME_MISSING