From 1ee13cae705f2bf564a18c492bca4f9809245f7e Mon Sep 17 00:00:00 2001 From: Marco Gavazzoni <138492709+marcogavaz@users.noreply.github.com> Date: Fri, 14 Feb 2025 15:11:29 +0100 Subject: [PATCH] APM telemetry collection Otel agent distro diferentiation - issue 489 (#210775) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary This PRs follows the [closed PR ](https://github.com/elastic/kibana/pull/208770) and Closes https://github.com/elastic/observability-bi/issues/489 As requested in [this comment ](https://github.com/elastic/kibana/issues/186281#issue-2357143823) and tracked by the issue https://github.com/elastic/kibana/issues/186281 this PR handles main changes introduced to address the request for capturing open-ended OTel distro agent names with patter `opentelemetry//`. These changes ensure that new agent names won’t be dropped in telemetry. ### Checklist Check the PR satisfies following conditions. Reviewers should verify this PR satisfies this list as well. - [ ] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/src/platform/packages/shared/kbn-i18n/README.md) - [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials - [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [ ] If a plugin configuration key changed, check if it needs to be allowlisted in the cloud and added to the [docker list](https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker) - [ ] This was checked for breaking HTTP API changes, and any breaking changes have been approved by the breaking-change committee. The `release_note:breaking` label should be applied in these situations. - [ ] [Flaky Test Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was used on any tests changed - [ ] The PR description includes the appropriate Release Notes section, and the correct `release_note:*` label is applied per the [guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Elastic Machine --- .../__snapshots__/tasks.test.ts.snap | 576 ------------------ .../collect_data_telemetry/tasks.ts | 139 ++--- 2 files changed, 56 insertions(+), 659 deletions(-) diff --git a/x-pack/solutions/observability/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/__snapshots__/tasks.test.ts.snap b/x-pack/solutions/observability/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/__snapshots__/tasks.test.ts.snap index e3bb9a252578..2561325ca776 100644 --- a/x-pack/solutions/observability/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/__snapshots__/tasks.test.ts.snap +++ b/x-pack/solutions/observability/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/__snapshots__/tasks.test.ts.snap @@ -254,121 +254,6 @@ Object { }, }, }, - "opentelemetry/android": Object { - "agent": Object { - "activation_method": Array [], - "version": Array [], - }, - "service": Object { - "framework": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - "language": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - "runtime": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - }, - }, - "opentelemetry/cpp": Object { - "agent": Object { - "activation_method": Array [], - "version": Array [], - }, - "service": Object { - "framework": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - "language": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - "runtime": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - }, - }, - "opentelemetry/dotnet": Object { - "agent": Object { - "activation_method": Array [], - "version": Array [], - }, - "service": Object { - "framework": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - "language": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - "runtime": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - }, - }, - "opentelemetry/erlang": Object { - "agent": Object { - "activation_method": Array [], - "version": Array [], - }, - "service": Object { - "framework": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - "language": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - "runtime": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - }, - }, - "opentelemetry/go": Object { - "agent": Object { - "activation_method": Array [], - "version": Array [], - }, - "service": Object { - "framework": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - "language": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - "runtime": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - }, - }, "opentelemetry/java": Object { "agent": Object { "activation_method": Array [ @@ -459,167 +344,6 @@ Object { }, }, }, - "opentelemetry/nodejs": Object { - "agent": Object { - "activation_method": Array [], - "version": Array [], - }, - "service": Object { - "framework": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - "language": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - "runtime": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - }, - }, - "opentelemetry/php": Object { - "agent": Object { - "activation_method": Array [], - "version": Array [], - }, - "service": Object { - "framework": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - "language": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - "runtime": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - }, - }, - "opentelemetry/python": Object { - "agent": Object { - "activation_method": Array [], - "version": Array [], - }, - "service": Object { - "framework": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - "language": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - "runtime": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - }, - }, - "opentelemetry/ruby": Object { - "agent": Object { - "activation_method": Array [], - "version": Array [], - }, - "service": Object { - "framework": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - "language": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - "runtime": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - }, - }, - "opentelemetry/rust": Object { - "agent": Object { - "activation_method": Array [], - "version": Array [], - }, - "service": Object { - "framework": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - "language": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - "runtime": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - }, - }, - "opentelemetry/swift": Object { - "agent": Object { - "activation_method": Array [], - "version": Array [], - }, - "service": Object { - "framework": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - "language": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - "runtime": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - }, - }, - "opentelemetry/webjs": Object { - "agent": Object { - "activation_method": Array [], - "version": Array [], - }, - "service": Object { - "framework": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - "language": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - "runtime": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - }, - }, "otlp": Object { "agent": Object { "activation_method": Array [ @@ -665,121 +389,6 @@ Object { }, }, }, - "otlp/android": Object { - "agent": Object { - "activation_method": Array [], - "version": Array [], - }, - "service": Object { - "framework": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - "language": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - "runtime": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - }, - }, - "otlp/cpp": Object { - "agent": Object { - "activation_method": Array [], - "version": Array [], - }, - "service": Object { - "framework": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - "language": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - "runtime": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - }, - }, - "otlp/dotnet": Object { - "agent": Object { - "activation_method": Array [], - "version": Array [], - }, - "service": Object { - "framework": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - "language": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - "runtime": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - }, - }, - "otlp/erlang": Object { - "agent": Object { - "activation_method": Array [], - "version": Array [], - }, - "service": Object { - "framework": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - "language": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - "runtime": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - }, - }, - "otlp/go": Object { - "agent": Object { - "activation_method": Array [], - "version": Array [], - }, - "service": Object { - "framework": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - "language": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - "runtime": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - }, - }, "otlp/java": Object { "agent": Object { "activation_method": Array [ @@ -870,167 +479,6 @@ Object { }, }, }, - "otlp/nodejs": Object { - "agent": Object { - "activation_method": Array [], - "version": Array [], - }, - "service": Object { - "framework": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - "language": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - "runtime": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - }, - }, - "otlp/php": Object { - "agent": Object { - "activation_method": Array [], - "version": Array [], - }, - "service": Object { - "framework": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - "language": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - "runtime": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - }, - }, - "otlp/python": Object { - "agent": Object { - "activation_method": Array [], - "version": Array [], - }, - "service": Object { - "framework": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - "language": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - "runtime": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - }, - }, - "otlp/ruby": Object { - "agent": Object { - "activation_method": Array [], - "version": Array [], - }, - "service": Object { - "framework": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - "language": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - "runtime": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - }, - }, - "otlp/rust": Object { - "agent": Object { - "activation_method": Array [], - "version": Array [], - }, - "service": Object { - "framework": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - "language": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - "runtime": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - }, - }, - "otlp/swift": Object { - "agent": Object { - "activation_method": Array [], - "version": Array [], - }, - "service": Object { - "framework": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - "language": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - "runtime": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - }, - }, - "otlp/webjs": Object { - "agent": Object { - "activation_method": Array [], - "version": Array [], - }, - "service": Object { - "framework": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - "language": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - "runtime": Object { - "composite": Array [], - "name": Array [], - "version": Array [], - }, - }, - }, "php": Object { "agent": Object { "activation_method": Array [], @@ -1757,35 +1205,11 @@ Object { "js-base": 0, "nodejs": 0, "opentelemetry": 4, - "opentelemetry/android": 0, - "opentelemetry/cpp": 0, - "opentelemetry/dotnet": 0, - "opentelemetry/erlang": 0, - "opentelemetry/go": 0, "opentelemetry/java": 5, "opentelemetry/java/elastic": 6, - "opentelemetry/nodejs": 0, - "opentelemetry/php": 0, - "opentelemetry/python": 0, - "opentelemetry/ruby": 0, - "opentelemetry/rust": 0, - "opentelemetry/swift": 0, - "opentelemetry/webjs": 0, "otlp": 1, - "otlp/android": 0, - "otlp/cpp": 0, - "otlp/dotnet": 0, - "otlp/erlang": 0, - "otlp/go": 0, "otlp/java": 2, "otlp/java/elastic": 3, - "otlp/nodejs": 0, - "otlp/php": 0, - "otlp/python": 0, - "otlp/ruby": 0, - "otlp/rust": 0, - "otlp/swift": 0, - "otlp/webjs": 0, "php": 0, "python": 0, "ruby": 0, diff --git a/x-pack/solutions/observability/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts b/x-pack/solutions/observability/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts index 3a8747a65936..1c6fb8c4d623 100644 --- a/x-pack/solutions/observability/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts +++ b/x-pack/solutions/observability/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts @@ -16,7 +16,6 @@ import { OPEN_TELEMETRY_AGENT_NAMES, OPEN_TELEMETRY_BASE_AGENT_NAMES, RUM_AGENT_NAMES, - type OpenTelemetryAgentName, } from '@kbn/elastic-agent-utils/src/agent_names'; import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils'; import { @@ -651,14 +650,9 @@ export const tasks: TelemetryTask[] = [ Promise.resolve({} as Record) ); - const initOtelAgents = OPEN_TELEMETRY_AGENT_NAMES.reduce((acc, agent) => { - acc[agent] = 0; - return acc; - }, {} as Record); - const servicesPerOtelAgents = await OPEN_TELEMETRY_BASE_AGENT_NAMES.reduce( - (prevJob, agentName) => { - return prevJob.then(async (data) => { + (prevJob, baseAgentName) => { + return prevJob.then(async (accData) => { const response = await telemetryClient.search({ index: [indices.error, indices.span, indices.metric, indices.transaction], body: { @@ -667,7 +661,7 @@ export const tasks: TelemetryTask[] = [ timeout, query: { bool: { - filter: [{ prefix: { [AGENT_NAME]: agentName } }, range1d], + filter: [{ prefix: { [AGENT_NAME]: baseAgentName } }, range1d], }, }, aggs: { @@ -687,22 +681,18 @@ export const tasks: TelemetryTask[] = [ }, }, }); - - const aggregatedServicesPerAgents = response.aggregations?.agent_name.buckets.reduce( - (acc, agent) => { - acc[agent.key as OpenTelemetryAgentName] = agent.services.value || 0; - return acc; - }, - initOtelAgents - ); - + const aggregatedServices: Record = {}; + for (const bucket of response.aggregations?.agent_name.buckets ?? []) { + const fullAgentName = bucket.key as string; + aggregatedServices[fullAgentName] = bucket.services.value || 0; + } return { - ...data, - ...aggregatedServicesPerAgents, + ...accData, + ...aggregatedServices, }; }); }, - Promise.resolve(initOtelAgents) + Promise.resolve({} as Record) ); const services = await telemetryClient.search({ @@ -967,6 +957,7 @@ export const tasks: TelemetryTask[] = [ const size = 3; const toComposite = (outerKey: string | number, innerKey: string | number) => `${outerKey}/${innerKey}`; + const agentNameAggs = { [AGENT_ACTIVATION_METHOD]: { terms: { @@ -1146,13 +1137,14 @@ export const tasks: TelemetryTask[] = [ }, }, }; + return data; }, Promise.resolve({} as NonNullable) ); const agentDataWithOtel = await OPEN_TELEMETRY_BASE_AGENT_NAMES.reduce( - async (prevJob, agentName) => { + async (prevJob, baseAgentName) => { const data = await prevJob; const response = await telemetryClient.search({ @@ -1163,7 +1155,7 @@ export const tasks: TelemetryTask[] = [ timeout, query: { bool: { - filter: [{ prefix: { [AGENT_NAME]: agentName } }, range1d], + filter: [{ prefix: { [AGENT_NAME]: baseAgentName } }, range1d], }, }, sort: { @@ -1181,64 +1173,39 @@ export const tasks: TelemetryTask[] = [ }, }); - const { aggregations } = response; - - if (!aggregations) { + if (!response.aggregations) { return data; } - const initAgentData = OPEN_TELEMETRY_AGENT_NAMES.reduce((acc, agent) => { - acc[agent] = { - agent: { - activation_method: [], - version: [], - }, - service: { - framework: { - name: [], - version: [], - composite: [], - }, - language: { - name: [], - version: [], - composite: [], - }, - runtime: { - name: [], - version: [], - composite: [], - }, - }, - }; - return acc; - }, {} as NonNullable); + const dynamicAgentData: NonNullable = {}; - const agentData = aggregations?.agent_name.buckets.reduce((acc, agentNamesAggs) => { - acc[agentNamesAggs.key as OpenTelemetryAgentName] = { + for (const agentBucket of response.aggregations.agent_name.buckets) { + const agentKey = agentBucket.key as string; + + dynamicAgentData[agentKey as AgentName] = { agent: { - activation_method: agentNamesAggs[AGENT_ACTIVATION_METHOD].buckets + activation_method: agentBucket[AGENT_ACTIVATION_METHOD].buckets .map((bucket) => bucket.key as string) .slice(0, size), - version: agentNamesAggs[AGENT_VERSION].buckets.map( - (bucket) => bucket.key as string - ), + version: agentBucket[AGENT_VERSION].buckets.map((bucket) => bucket.key as string), }, service: { framework: { - name: agentNamesAggs[SERVICE_FRAMEWORK_NAME].buckets + name: agentBucket[SERVICE_FRAMEWORK_NAME].buckets .map((bucket) => bucket.key as string) .slice(0, size), - version: agentNamesAggs[SERVICE_FRAMEWORK_VERSION].buckets + version: agentBucket[SERVICE_FRAMEWORK_VERSION].buckets .map((bucket) => bucket.key as string) .slice(0, size), composite: sortBy( flatten( - agentNamesAggs[SERVICE_FRAMEWORK_NAME].buckets.map((bucket) => - bucket[SERVICE_FRAMEWORK_VERSION].buckets.map((versionBucket) => ({ - doc_count: versionBucket.doc_count, - name: toComposite(bucket.key, versionBucket.key), - })) + agentBucket[SERVICE_FRAMEWORK_NAME].buckets.map((fwBucket: any) => + fwBucket[SERVICE_FRAMEWORK_VERSION].buckets.map( + (versionBucket: { key: string; doc_count: number }) => ({ + doc_count: versionBucket.doc_count, + name: `${fwBucket.key}/${versionBucket.key}`, + }) + ) ) ), 'doc_count' @@ -1248,19 +1215,21 @@ export const tasks: TelemetryTask[] = [ .map((composite) => composite.name), }, language: { - name: agentNamesAggs[SERVICE_LANGUAGE_NAME].buckets + name: agentBucket[SERVICE_LANGUAGE_NAME].buckets .map((bucket) => bucket.key as string) .slice(0, size), - version: agentNamesAggs[SERVICE_LANGUAGE_VERSION].buckets + version: agentBucket[SERVICE_LANGUAGE_VERSION].buckets .map((bucket) => bucket.key as string) .slice(0, size), composite: sortBy( flatten( - agentNamesAggs[SERVICE_LANGUAGE_NAME].buckets.map((bucket) => - bucket[SERVICE_LANGUAGE_VERSION].buckets.map((versionBucket) => ({ - doc_count: versionBucket.doc_count, - name: toComposite(bucket.key, versionBucket.key), - })) + agentBucket[SERVICE_LANGUAGE_NAME].buckets.map((langBucket: any) => + langBucket[SERVICE_LANGUAGE_VERSION].buckets.map( + (versionBucket: { key: string; doc_count: number }) => ({ + doc_count: versionBucket.doc_count, + name: `${langBucket.key}/${versionBucket.key}`, + }) + ) ) ), 'doc_count' @@ -1270,19 +1239,21 @@ export const tasks: TelemetryTask[] = [ .map((composite) => composite.name), }, runtime: { - name: agentNamesAggs[SERVICE_RUNTIME_NAME].buckets + name: agentBucket[SERVICE_RUNTIME_NAME].buckets .map((bucket) => bucket.key as string) .slice(0, size), - version: agentNamesAggs[SERVICE_RUNTIME_VERSION].buckets + version: agentBucket[SERVICE_RUNTIME_VERSION].buckets .map((bucket) => bucket.key as string) .slice(0, size), composite: sortBy( flatten( - agentNamesAggs[SERVICE_RUNTIME_NAME].buckets.map((bucket) => - bucket[SERVICE_RUNTIME_VERSION].buckets.map((versionBucket) => ({ - doc_count: versionBucket.doc_count, - name: toComposite(bucket.key, versionBucket.key), - })) + agentBucket[SERVICE_RUNTIME_NAME].buckets.map((runtimeBucket: any) => + runtimeBucket[SERVICE_RUNTIME_VERSION].buckets.map( + (versionBucket: { key: string; doc_count: number }) => ({ + doc_count: versionBucket.doc_count, + name: `${runtimeBucket.key}/${versionBucket.key}`, + }) + ) ) ), 'doc_count' @@ -1293,19 +1264,21 @@ export const tasks: TelemetryTask[] = [ }, }, }; - return acc; - }, initAgentData); + } return { ...data, - ...agentData, + ...dynamicAgentData, }; }, Promise.resolve({} as APMTelemetry['agents']) ); return { - agents: { ...agentDataWithoutOtel, ...agentDataWithOtel }, + agents: { + ...agentDataWithoutOtel, + ...agentDataWithOtel, + }, }; }, },