APM telemetry collection Otel agent distro diferentiation - issue 489 (#210775)

## Summary

This PRs follows the [closed PR
](https://github.com/elastic/kibana/pull/208770) and Closes
https://github.com/elastic/observability-bi/issues/489

As requested in [this comment
](https://github.com/elastic/kibana/issues/186281#issue-2357143823) and
tracked by the issue https://github.com/elastic/kibana/issues/186281
this PR handles main changes introduced to address the request for
capturing open-ended OTel distro agent names with patter
`opentelemetry/<LANGUAGE>/<DISTRO_NAME>`. These changes ensure that new
agent names won’t be dropped in telemetry.



### Checklist

Check the PR satisfies following conditions. 

Reviewers should verify this PR satisfies this list as well.

- [ ] Any text added follows [EUI's writing
guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses
sentence case text and includes [i18n
support](https://github.com/elastic/kibana/blob/main/src/platform/packages/shared/kbn-i18n/README.md)
- [ ]
[Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html)
was added for features that require explanation or tutorials
- [ ] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [ ] If a plugin configuration key changed, check if it needs to be
allowlisted in the cloud and added to the [docker
list](https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker)
- [ ] This was checked for breaking HTTP API changes, and any breaking
changes have been approved by the breaking-change committee. The
`release_note:breaking` label should be applied in these situations.
- [ ] [Flaky Test
Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was
used on any tests changed
- [ ] The PR description includes the appropriate Release Notes section,
and the correct `release_note:*` label is applied per the
[guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
This commit is contained in:
Marco Gavazzoni 2025-02-14 15:11:29 +01:00 committed by GitHub
parent 6ecb66df7f
commit 1ee13cae70
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 56 additions and 659 deletions

View file

@ -254,121 +254,6 @@ Object {
},
},
},
"opentelemetry/android": Object {
"agent": Object {
"activation_method": Array [],
"version": Array [],
},
"service": Object {
"framework": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
"language": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
"runtime": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
},
},
"opentelemetry/cpp": Object {
"agent": Object {
"activation_method": Array [],
"version": Array [],
},
"service": Object {
"framework": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
"language": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
"runtime": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
},
},
"opentelemetry/dotnet": Object {
"agent": Object {
"activation_method": Array [],
"version": Array [],
},
"service": Object {
"framework": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
"language": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
"runtime": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
},
},
"opentelemetry/erlang": Object {
"agent": Object {
"activation_method": Array [],
"version": Array [],
},
"service": Object {
"framework": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
"language": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
"runtime": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
},
},
"opentelemetry/go": Object {
"agent": Object {
"activation_method": Array [],
"version": Array [],
},
"service": Object {
"framework": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
"language": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
"runtime": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
},
},
"opentelemetry/java": Object {
"agent": Object {
"activation_method": Array [
@ -459,167 +344,6 @@ Object {
},
},
},
"opentelemetry/nodejs": Object {
"agent": Object {
"activation_method": Array [],
"version": Array [],
},
"service": Object {
"framework": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
"language": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
"runtime": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
},
},
"opentelemetry/php": Object {
"agent": Object {
"activation_method": Array [],
"version": Array [],
},
"service": Object {
"framework": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
"language": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
"runtime": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
},
},
"opentelemetry/python": Object {
"agent": Object {
"activation_method": Array [],
"version": Array [],
},
"service": Object {
"framework": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
"language": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
"runtime": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
},
},
"opentelemetry/ruby": Object {
"agent": Object {
"activation_method": Array [],
"version": Array [],
},
"service": Object {
"framework": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
"language": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
"runtime": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
},
},
"opentelemetry/rust": Object {
"agent": Object {
"activation_method": Array [],
"version": Array [],
},
"service": Object {
"framework": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
"language": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
"runtime": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
},
},
"opentelemetry/swift": Object {
"agent": Object {
"activation_method": Array [],
"version": Array [],
},
"service": Object {
"framework": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
"language": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
"runtime": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
},
},
"opentelemetry/webjs": Object {
"agent": Object {
"activation_method": Array [],
"version": Array [],
},
"service": Object {
"framework": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
"language": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
"runtime": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
},
},
"otlp": Object {
"agent": Object {
"activation_method": Array [
@ -665,121 +389,6 @@ Object {
},
},
},
"otlp/android": Object {
"agent": Object {
"activation_method": Array [],
"version": Array [],
},
"service": Object {
"framework": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
"language": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
"runtime": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
},
},
"otlp/cpp": Object {
"agent": Object {
"activation_method": Array [],
"version": Array [],
},
"service": Object {
"framework": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
"language": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
"runtime": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
},
},
"otlp/dotnet": Object {
"agent": Object {
"activation_method": Array [],
"version": Array [],
},
"service": Object {
"framework": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
"language": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
"runtime": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
},
},
"otlp/erlang": Object {
"agent": Object {
"activation_method": Array [],
"version": Array [],
},
"service": Object {
"framework": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
"language": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
"runtime": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
},
},
"otlp/go": Object {
"agent": Object {
"activation_method": Array [],
"version": Array [],
},
"service": Object {
"framework": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
"language": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
"runtime": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
},
},
"otlp/java": Object {
"agent": Object {
"activation_method": Array [
@ -870,167 +479,6 @@ Object {
},
},
},
"otlp/nodejs": Object {
"agent": Object {
"activation_method": Array [],
"version": Array [],
},
"service": Object {
"framework": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
"language": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
"runtime": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
},
},
"otlp/php": Object {
"agent": Object {
"activation_method": Array [],
"version": Array [],
},
"service": Object {
"framework": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
"language": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
"runtime": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
},
},
"otlp/python": Object {
"agent": Object {
"activation_method": Array [],
"version": Array [],
},
"service": Object {
"framework": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
"language": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
"runtime": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
},
},
"otlp/ruby": Object {
"agent": Object {
"activation_method": Array [],
"version": Array [],
},
"service": Object {
"framework": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
"language": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
"runtime": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
},
},
"otlp/rust": Object {
"agent": Object {
"activation_method": Array [],
"version": Array [],
},
"service": Object {
"framework": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
"language": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
"runtime": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
},
},
"otlp/swift": Object {
"agent": Object {
"activation_method": Array [],
"version": Array [],
},
"service": Object {
"framework": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
"language": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
"runtime": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
},
},
"otlp/webjs": Object {
"agent": Object {
"activation_method": Array [],
"version": Array [],
},
"service": Object {
"framework": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
"language": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
"runtime": Object {
"composite": Array [],
"name": Array [],
"version": Array [],
},
},
},
"php": Object {
"agent": Object {
"activation_method": Array [],
@ -1757,35 +1205,11 @@ Object {
"js-base": 0,
"nodejs": 0,
"opentelemetry": 4,
"opentelemetry/android": 0,
"opentelemetry/cpp": 0,
"opentelemetry/dotnet": 0,
"opentelemetry/erlang": 0,
"opentelemetry/go": 0,
"opentelemetry/java": 5,
"opentelemetry/java/elastic": 6,
"opentelemetry/nodejs": 0,
"opentelemetry/php": 0,
"opentelemetry/python": 0,
"opentelemetry/ruby": 0,
"opentelemetry/rust": 0,
"opentelemetry/swift": 0,
"opentelemetry/webjs": 0,
"otlp": 1,
"otlp/android": 0,
"otlp/cpp": 0,
"otlp/dotnet": 0,
"otlp/erlang": 0,
"otlp/go": 0,
"otlp/java": 2,
"otlp/java/elastic": 3,
"otlp/nodejs": 0,
"otlp/php": 0,
"otlp/python": 0,
"otlp/ruby": 0,
"otlp/rust": 0,
"otlp/swift": 0,
"otlp/webjs": 0,
"php": 0,
"python": 0,
"ruby": 0,

View file

@ -16,7 +16,6 @@ import {
OPEN_TELEMETRY_AGENT_NAMES,
OPEN_TELEMETRY_BASE_AGENT_NAMES,
RUM_AGENT_NAMES,
type OpenTelemetryAgentName,
} from '@kbn/elastic-agent-utils/src/agent_names';
import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
import {
@ -651,14 +650,9 @@ export const tasks: TelemetryTask[] = [
Promise.resolve({} as Record<AgentName, number>)
);
const initOtelAgents = OPEN_TELEMETRY_AGENT_NAMES.reduce((acc, agent) => {
acc[agent] = 0;
return acc;
}, {} as Record<OpenTelemetryAgentName, number>);
const servicesPerOtelAgents = await OPEN_TELEMETRY_BASE_AGENT_NAMES.reduce(
(prevJob, agentName) => {
return prevJob.then(async (data) => {
(prevJob, baseAgentName) => {
return prevJob.then(async (accData) => {
const response = await telemetryClient.search({
index: [indices.error, indices.span, indices.metric, indices.transaction],
body: {
@ -667,7 +661,7 @@ export const tasks: TelemetryTask[] = [
timeout,
query: {
bool: {
filter: [{ prefix: { [AGENT_NAME]: agentName } }, range1d],
filter: [{ prefix: { [AGENT_NAME]: baseAgentName } }, range1d],
},
},
aggs: {
@ -687,22 +681,18 @@ export const tasks: TelemetryTask[] = [
},
},
});
const aggregatedServicesPerAgents = response.aggregations?.agent_name.buckets.reduce(
(acc, agent) => {
acc[agent.key as OpenTelemetryAgentName] = agent.services.value || 0;
return acc;
},
initOtelAgents
);
const aggregatedServices: Record<string, number> = {};
for (const bucket of response.aggregations?.agent_name.buckets ?? []) {
const fullAgentName = bucket.key as string;
aggregatedServices[fullAgentName] = bucket.services.value || 0;
}
return {
...data,
...aggregatedServicesPerAgents,
...accData,
...aggregatedServices,
};
});
},
Promise.resolve(initOtelAgents)
Promise.resolve({} as Record<string, number>)
);
const services = await telemetryClient.search({
@ -967,6 +957,7 @@ export const tasks: TelemetryTask[] = [
const size = 3;
const toComposite = (outerKey: string | number, innerKey: string | number) =>
`${outerKey}/${innerKey}`;
const agentNameAggs = {
[AGENT_ACTIVATION_METHOD]: {
terms: {
@ -1146,13 +1137,14 @@ export const tasks: TelemetryTask[] = [
},
},
};
return data;
},
Promise.resolve({} as NonNullable<APMTelemetry['agents']>)
);
const agentDataWithOtel = await OPEN_TELEMETRY_BASE_AGENT_NAMES.reduce(
async (prevJob, agentName) => {
async (prevJob, baseAgentName) => {
const data = await prevJob;
const response = await telemetryClient.search({
@ -1163,7 +1155,7 @@ export const tasks: TelemetryTask[] = [
timeout,
query: {
bool: {
filter: [{ prefix: { [AGENT_NAME]: agentName } }, range1d],
filter: [{ prefix: { [AGENT_NAME]: baseAgentName } }, range1d],
},
},
sort: {
@ -1181,64 +1173,39 @@ export const tasks: TelemetryTask[] = [
},
});
const { aggregations } = response;
if (!aggregations) {
if (!response.aggregations) {
return data;
}
const initAgentData = OPEN_TELEMETRY_AGENT_NAMES.reduce((acc, agent) => {
acc[agent] = {
agent: {
activation_method: [],
version: [],
},
service: {
framework: {
name: [],
version: [],
composite: [],
},
language: {
name: [],
version: [],
composite: [],
},
runtime: {
name: [],
version: [],
composite: [],
},
},
};
return acc;
}, {} as NonNullable<APMTelemetry['agents']>);
const dynamicAgentData: NonNullable<APMTelemetry['agents']> = {};
const agentData = aggregations?.agent_name.buckets.reduce((acc, agentNamesAggs) => {
acc[agentNamesAggs.key as OpenTelemetryAgentName] = {
for (const agentBucket of response.aggregations.agent_name.buckets) {
const agentKey = agentBucket.key as string;
dynamicAgentData[agentKey as AgentName] = {
agent: {
activation_method: agentNamesAggs[AGENT_ACTIVATION_METHOD].buckets
activation_method: agentBucket[AGENT_ACTIVATION_METHOD].buckets
.map((bucket) => bucket.key as string)
.slice(0, size),
version: agentNamesAggs[AGENT_VERSION].buckets.map(
(bucket) => bucket.key as string
),
version: agentBucket[AGENT_VERSION].buckets.map((bucket) => bucket.key as string),
},
service: {
framework: {
name: agentNamesAggs[SERVICE_FRAMEWORK_NAME].buckets
name: agentBucket[SERVICE_FRAMEWORK_NAME].buckets
.map((bucket) => bucket.key as string)
.slice(0, size),
version: agentNamesAggs[SERVICE_FRAMEWORK_VERSION].buckets
version: agentBucket[SERVICE_FRAMEWORK_VERSION].buckets
.map((bucket) => bucket.key as string)
.slice(0, size),
composite: sortBy(
flatten(
agentNamesAggs[SERVICE_FRAMEWORK_NAME].buckets.map((bucket) =>
bucket[SERVICE_FRAMEWORK_VERSION].buckets.map((versionBucket) => ({
doc_count: versionBucket.doc_count,
name: toComposite(bucket.key, versionBucket.key),
}))
agentBucket[SERVICE_FRAMEWORK_NAME].buckets.map((fwBucket: any) =>
fwBucket[SERVICE_FRAMEWORK_VERSION].buckets.map(
(versionBucket: { key: string; doc_count: number }) => ({
doc_count: versionBucket.doc_count,
name: `${fwBucket.key}/${versionBucket.key}`,
})
)
)
),
'doc_count'
@ -1248,19 +1215,21 @@ export const tasks: TelemetryTask[] = [
.map((composite) => composite.name),
},
language: {
name: agentNamesAggs[SERVICE_LANGUAGE_NAME].buckets
name: agentBucket[SERVICE_LANGUAGE_NAME].buckets
.map((bucket) => bucket.key as string)
.slice(0, size),
version: agentNamesAggs[SERVICE_LANGUAGE_VERSION].buckets
version: agentBucket[SERVICE_LANGUAGE_VERSION].buckets
.map((bucket) => bucket.key as string)
.slice(0, size),
composite: sortBy(
flatten(
agentNamesAggs[SERVICE_LANGUAGE_NAME].buckets.map((bucket) =>
bucket[SERVICE_LANGUAGE_VERSION].buckets.map((versionBucket) => ({
doc_count: versionBucket.doc_count,
name: toComposite(bucket.key, versionBucket.key),
}))
agentBucket[SERVICE_LANGUAGE_NAME].buckets.map((langBucket: any) =>
langBucket[SERVICE_LANGUAGE_VERSION].buckets.map(
(versionBucket: { key: string; doc_count: number }) => ({
doc_count: versionBucket.doc_count,
name: `${langBucket.key}/${versionBucket.key}`,
})
)
)
),
'doc_count'
@ -1270,19 +1239,21 @@ export const tasks: TelemetryTask[] = [
.map((composite) => composite.name),
},
runtime: {
name: agentNamesAggs[SERVICE_RUNTIME_NAME].buckets
name: agentBucket[SERVICE_RUNTIME_NAME].buckets
.map((bucket) => bucket.key as string)
.slice(0, size),
version: agentNamesAggs[SERVICE_RUNTIME_VERSION].buckets
version: agentBucket[SERVICE_RUNTIME_VERSION].buckets
.map((bucket) => bucket.key as string)
.slice(0, size),
composite: sortBy(
flatten(
agentNamesAggs[SERVICE_RUNTIME_NAME].buckets.map((bucket) =>
bucket[SERVICE_RUNTIME_VERSION].buckets.map((versionBucket) => ({
doc_count: versionBucket.doc_count,
name: toComposite(bucket.key, versionBucket.key),
}))
agentBucket[SERVICE_RUNTIME_NAME].buckets.map((runtimeBucket: any) =>
runtimeBucket[SERVICE_RUNTIME_VERSION].buckets.map(
(versionBucket: { key: string; doc_count: number }) => ({
doc_count: versionBucket.doc_count,
name: `${runtimeBucket.key}/${versionBucket.key}`,
})
)
)
),
'doc_count'
@ -1293,19 +1264,21 @@ export const tasks: TelemetryTask[] = [
},
},
};
return acc;
}, initAgentData);
}
return {
...data,
...agentData,
...dynamicAgentData,
};
},
Promise.resolve({} as APMTelemetry['agents'])
);
return {
agents: { ...agentDataWithoutOtel, ...agentDataWithOtel },
agents: {
...agentDataWithoutOtel,
...agentDataWithOtel,
},
};
},
},