github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 17:59:23 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Inventory] Add k8s fields to Service entity type (#195407)

Browse source 
closes https://github.com/elastic/kibana/issues/195244

- Removed metrics definition from service, host and container
- Removed `metrics-apm` index patterns from the service definition
because k8s fields are not available on that scope.
- Added `traces-apm*` index pattern on the service definition


https://github.com/user-attachments/assets/6c6b4fd6-817a-494e-8649-e2d76a8e98e3
This commit is contained in:
Cauê Marcondes 2024-10-09 14:44:12 +01:00 • committed by GitHub
parent 02cc5a83b8
commit cc7fdba142
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
16 changed files with 12 additions and 1161 deletions
Download patch file Download diff file Expand all files Collapse all files

92
x-pack/plugins/entity_manager/server/lib/entities/built_in/containers_from_ecs_data.ts
View file

@ -12,7 +12,7 @@ export const builtInContainersFromEcsEntityDefinition: EntityDefinition =
entityDefinitionSchema.parse({
id: `${BUILT_IN_ID_PREFIX}containers_from_ecs_data`,
managed: true,
version: '1.0.0',
version: '0.1.0',
name: 'Containers from ECS data',
description:
'This definition extracts container entities from common data streams by looking for the ECS field container.id',
@ -65,94 +65,4 @@ export const builtInContainersFromEcsEntityDefinition: EntityDefinition =
'agent.type',
'agent.ephemeral_id',
],
metrics: [
{
name: 'log_rate',
equation: 'A',
metrics: [
{
name: 'A',
aggregation: 'doc_count',
filter: 'log.level: * OR error.log.level: *',
},
],
},
{
name: 'error_log_rate',
equation: 'A',
metrics: [
{
name: 'A',
aggregation: 'doc_count',
filter: '(log.level: "error" OR "ERROR") OR (error.log.level: "error" OR "ERROR")',
},
],
},
{
name: 'cpu_usage_avg',
equation: 'A',
metrics: [
{
name: 'A',
aggregation: 'avg',
field: 'docker.cpu.total.pct',
},
],
},
{
name: 'memory_usage_avg',
equation: 'A',
metrics: [
{
name: 'A',
aggregation: 'avg',
field: 'docker.memory.usage.pct',
},
],
},
{
name: 'network_in_avg',
equation: 'A',
metrics: [
{
name: 'A',
aggregation: 'avg',
field: 'docker.network.in.bytes',
},
],
},
{
name: 'network_out_avg',
equation: 'A',
metrics: [
{
name: 'A',
aggregation: 'avg',
field: 'docker.network.out.bytes',
},
],
},
{
name: 'disk_read_avg',
equation: 'A',
metrics: [
{
name: 'A',
aggregation: 'avg',
field: 'docker.diskio.read.ops',
},
],
},
{
name: 'disk_write_avg',
equation: 'A',
metrics: [
{
name: 'A',
aggregation: 'avg',
field: 'docker.diskio.write.ops',
},
],
},
],
});

113
x-pack/plugins/entity_manager/server/lib/entities/built_in/hosts_from_ecs_data.ts
View file

@ -11,7 +11,7 @@ import { BUILT_IN_ID_PREFIX } from './constants';
export const builtInHostsFromEcsEntityDefinition: EntityDefinition = entityDefinitionSchema.parse({
id: `${BUILT_IN_ID_PREFIX}hosts_from_ecs_data`,
managed: true,
version: '1.0.0',
version: '0.1.0',
name: 'Hosts from ECS data',
description:
'This definition extracts host entities from common data streams by looking for the ECS field host.name',
@ -65,115 +65,4 @@ export const builtInHostsFromEcsEntityDefinition: EntityDefinition = entityDefin
'agent.type',
'agent.version',
],
metrics: [
{
name: 'log_rate',
equation: 'A',
metrics: [
{
name: 'A',
aggregation: 'doc_count',
filter: 'log.level: * OR error.log.level: *',
},
],
},
{
name: 'error_log_rate',
equation: 'A',
metrics: [
{
name: 'A',
aggregation: 'doc_count',
filter: '(log.level: "error" OR "ERROR") OR (error.log.level: "error" OR "ERROR")',
},
],
},
{
name: 'cpu_usage_avg',
equation: 'A',
metrics: [
{
name: 'A',
aggregation: 'avg',
field: 'system.cpu.total.norm.pct',
},
],
},
{
name: 'normalized_load_avg',
equation: 'A / B',
metrics: [
{
name: 'A',
aggregation: 'avg',
field: 'system.load.1',
},
{
name: 'B',
aggregation: 'max',
field: 'system.load.cores',
},
],
},
{
name: 'memory_usage_avg',
equation: 'A',
metrics: [
{
name: 'A',
aggregation: 'avg',
field: 'system.memory.actual.used.pct',
},
],
},
{
name: 'memory_free_avg',
equation: 'A - B',
metrics: [
{
name: 'A',
aggregation: 'max',
field: 'system.memory.total',
},
{
name: 'B',
aggregation: 'avg',
field: 'system.memory.actual.used.bytes',
},
],
},
{
name: 'disk_usage_max',
equation: 'A',
metrics: [
{
name: 'A',
aggregation: 'max',
field: 'system.filesystem.used.pct',
},
],
},
{
name: 'rx_avg',
equation: 'A * 8',
metrics: [
{
name: 'A',
aggregation: 'sum',
field: 'host.network.ingress.bytes',
},
],
},
{
name: 'tx_avg',
equation: 'A * 8',
metrics: [
{
name: 'A',
aggregation: 'sum',
field: 'host.network.egress.bytes',
},
],
},
],
});

90
x-pack/plugins/entity_manager/server/lib/entities/built_in/services_from_ecs_data.ts
View file

@ -8,31 +8,16 @@
import { EntityDefinition, entityDefinitionSchema } from '@kbn/entities-schema';
import { BUILT_IN_ID_PREFIX } from './constants';
const serviceTransactionFilter = (additionalFilters: string[] = []) => {
const baseFilters = [
'processor.event: "metric"',
'metricset.name: "service_transaction"',
'metricset.interval: "1m"',
];
return [...baseFilters, ...additionalFilters].join(' AND ');
};
export const builtInServicesFromEcsEntityDefinition: EntityDefinition =
entityDefinitionSchema.parse({
version: '0.3.0',
version: '0.4.0',
id: `${BUILT_IN_ID_PREFIX}services_from_ecs_data`,
name: 'Services from ECS data',
description:
'This definition extracts service entities from common data streams by looking for the ECS field service.name',
type: 'service',
managed: true,
indexPatterns: [
'logs-*',
'filebeat*',
'metrics-apm.service_transaction.1m*',
'metrics-apm.service_summary.1m*',
],
indexPatterns: ['logs-*', 'filebeat*', 'traces-apm*'],
history: {
timestampField: '@timestamp',
interval: '1m',
@ -65,72 +50,9 @@ export const builtInServicesFromEcsEntityDefinition: EntityDefinition =
'cloud.provider',
'cloud.availability_zone',
'cloud.machine.type',
],
metrics: [
{
name: 'latency',
equation: 'A',
metrics: [
{
name: 'A',
aggregation: 'avg',
filter: serviceTransactionFilter(),
field: 'transaction.duration.histogram',
},
],
},
{
name: 'throughput',
equation: 'A',
metrics: [
{
name: 'A',
aggregation: 'value_count',
filter: serviceTransactionFilter(),
field: 'transaction.duration.summary',
},
],
},
{
name: 'failedTransactionRate',
equation: '1 - (A / B)',
metrics: [
{
name: 'A',
aggregation: 'sum',
filter: serviceTransactionFilter(),
field: 'event.success_count',
},
{
name: 'B',
aggregation: 'value_count',
filter: serviceTransactionFilter(),
field: 'event.success_count',
},
],
},
{
name: 'logErrorRate',
equation: 'A',
metrics: [
{
name: 'A',
aggregation: 'doc_count',
filter:
'log.level: "error" OR log.level: "ERROR" OR error.log.level: "error" OR error.log.level: "ERROR"',
},
],
},
{
name: 'logRate',
equation: 'A',
metrics: [
{
name: 'A',
aggregation: 'doc_count',
filter: 'data_stream.type: logs',
},
],
},
'kubernetes.namespace',
'orchestrator.cluster.name',
'k8s.namespace.name',
'k8s.cluster.name',
],
});

13
x-pack/plugins/observability_solution/apm/common/entities/types.ts
View file

@ -10,16 +10,3 @@ export enum EntityDataStreamType {
TRACES = 'traces',
LOGS = 'logs',
}
interface TraceMetrics {
latency?: number | null;
throughput?: number | null;
failedTransactionRate?: number | null;
}
interface LogsMetrics {
logRate?: number | null;
logErrorRate?: number | null;
}
export type EntityMetrics = TraceMetrics & LogsMetrics;

28
x-pack/plugins/observability_solution/apm/server/routes/entities/get_entities.ts
View file

@ -12,7 +12,6 @@ import {
import type { EntitiesESClient } from '../../lib/helpers/create_es_client/create_entities_es_client/create_entities_es_client';
import { getEntityLatestServices } from './get_entity_latest_services';
import type { EntityLatestServiceRaw } from './types';
import { getEntityHistoryServicesMetrics } from './get_entity_history_services_metrics';
export function entitiesRangeQuery(start?: number, end?: number): QueryDslQueryContainer[] {
if (!start || !end) {
@ -64,30 +63,5 @@ export async function getEntities({
serviceName,
});
const serviceEntitiesHistoryMetricsMap = entityLatestServices.length
? await getEntityHistoryServicesMetrics({
start,
end,
entitiesESClient,
entityIds: entityLatestServices.map((latestEntity) => latestEntity.entity.id),
size,
})
: undefined;
return entityLatestServices.map((latestEntity) => {
const historyEntityMetrics = serviceEntitiesHistoryMetricsMap?.[latestEntity.entity.id];
return {
...latestEntity,
entity: {
...latestEntity.entity,
metrics: historyEntityMetrics || {
latency: undefined,
logErrorRate: undefined,
failedTransactionRate: undefined,
logRate: undefined,
throughput: undefined,
},
},
};
});
return entityLatestServices;
}

84
x-pack/plugins/observability_solution/apm/server/routes/entities/get_entity_history_services_metrics.ts
View file

@ -1,84 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { rangeQuery, termsQuery } from '@kbn/observability-plugin/server';
import {
ENTITY_ID,
ENTITY_LAST_SEEN,
} from '@kbn/observability-shared-plugin/common/field_names/elasticsearch';
import { EntityMetrics } from '../../../common/entities/types';
import {
ENTITY_METRICS_FAILED_TRANSACTION_RATE,
ENTITY_METRICS_LATENCY,
ENTITY_METRICS_LOG_ERROR_RATE,
ENTITY_METRICS_LOG_RATE,
ENTITY_METRICS_THROUGHPUT,
} from '../../../common/es_fields/entities';
import { EntitiesESClient } from '../../lib/helpers/create_es_client/create_entities_es_client/create_entities_es_client';
interface Params {
entitiesESClient: EntitiesESClient;
start: number;
end: number;
entityIds: string[];
size: number;
}
export async function getEntityHistoryServicesMetrics({
end,
entityIds,
start,
entitiesESClient,
size,
}: Params) {
const response = await entitiesESClient.searchHistory('get_entities_history', {
body: {
size: 0,
track_total_hits: false,
query: {
bool: {
filter: [
...rangeQuery(start, end, ENTITY_LAST_SEEN),
...termsQuery(ENTITY_ID, ...entityIds),
],
},
},
aggs: {
entityIds: {
terms: { field: ENTITY_ID, size },
aggs: {
latency: { avg: { field: ENTITY_METRICS_LATENCY } },
logErrorRate: { avg: { field: ENTITY_METRICS_LOG_ERROR_RATE } },
logRate: { avg: { field: ENTITY_METRICS_LOG_RATE } },
throughput: { avg: { field: ENTITY_METRICS_THROUGHPUT } },
failedTransactionRate: { avg: { field: ENTITY_METRICS_FAILED_TRANSACTION_RATE } },
},
},
},
},
});
if (!response.aggregations) {
return {};
}
return response.aggregations.entityIds.buckets.reduce<Record<string, EntityMetrics>>(
(acc, currBucket) => {
return {
...acc,
[currBucket.key]: {
latency: currBucket.latency.value,
logErrorRate: currBucket.logErrorRate.value,
logRate: currBucket.logRate.value,
throughput: currBucket.throughput.value,
failedTransactionRate: currBucket.failedTransactionRate.value,
},
};
},
{}
);
}

116
x-pack/plugins/observability_solution/apm/server/routes/entities/get_entity_history_services_timeseries.ts
View file

@ -1,116 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { getBucketSize } from '@kbn/apm-data-access-plugin/common';
import { rangeQuery, termsQuery } from '@kbn/observability-plugin/server';
import { ENTITY_LAST_SEEN } from '@kbn/observability-shared-plugin/common/field_names/elasticsearch';
import { keyBy } from 'lodash';
import { SERVICE_NAME } from '../../../common/es_fields/apm';
import {
ENTITY_METRICS_FAILED_TRANSACTION_RATE,
ENTITY_METRICS_LATENCY,
ENTITY_METRICS_LOG_ERROR_RATE,
ENTITY_METRICS_LOG_RATE,
ENTITY_METRICS_THROUGHPUT,
} from '../../../common/es_fields/entities';
import { environmentQuery } from '../../../common/utils/environment_query';
import { EntitiesESClient } from '../../lib/helpers/create_es_client/create_entities_es_client/create_entities_es_client';
interface Params {
entitiesESClient: EntitiesESClient;
start: number;
end: number;
serviceNames: string[];
environment: string;
}
export async function getEntityHistoryServicesTimeseries({
start,
end,
serviceNames,
entitiesESClient,
environment,
}: Params) {
const { intervalString } = getBucketSize({
start,
end,
minBucketSize: 60,
});
const response = await entitiesESClient.searchHistory('get_entities_history_timeseries', {
body: {
size: 0,
track_total_hits: false,
query: {
bool: {
filter: [
...rangeQuery(start, end, ENTITY_LAST_SEEN),
...termsQuery(SERVICE_NAME, ...serviceNames),
...environmentQuery(environment),
],
},
},
aggs: {
serviceNames: {
terms: { field: SERVICE_NAME, size: serviceNames.length },
aggs: {
timeseries: {
date_histogram: {
field: '@timestamp',
fixed_interval: intervalString,
min_doc_count: 0,
extended_bounds: { min: start, max: end },
},
aggs: {
latency: { avg: { field: ENTITY_METRICS_LATENCY } },
logErrorRate: { avg: { field: ENTITY_METRICS_LOG_ERROR_RATE } },
logRate: { avg: { field: ENTITY_METRICS_LOG_RATE } },
throughput: { avg: { field: ENTITY_METRICS_THROUGHPUT } },
failedTransactionRate: { avg: { field: ENTITY_METRICS_FAILED_TRANSACTION_RATE } },
},
},
},
},
},
},
});
if (!response.aggregations) {
return {};
}
return keyBy(
response.aggregations.serviceNames.buckets.map((serviceBucket) => {
const serviceName = serviceBucket.key as string;
return {
serviceName,
latency: serviceBucket.timeseries.buckets.map((bucket) => ({
x: bucket.key,
y: bucket.latency.value ?? null,
})),
logErrorRate: serviceBucket.timeseries.buckets.map((bucket) => ({
x: bucket.key,
y: bucket.logErrorRate.value ?? null,
})),
logRate: serviceBucket.timeseries.buckets.map((bucket) => ({
x: bucket.key,
y: bucket.logRate.value ?? null,
})),
throughput: serviceBucket.timeseries.buckets.map((bucket) => ({
x: bucket.key,
y: bucket.throughput.value ?? null,
})),
failedTransactionRate: serviceBucket.timeseries.buckets.map((bucket) => ({
x: bucket.key,
y: bucket.failedTransactionRate.value ?? null,
})),
};
}),
'serviceName'
);
}

3
x-pack/plugins/observability_solution/apm/server/routes/entities/services/get_service_entities.ts
View file

@ -10,7 +10,6 @@ import { WrappedElasticsearchClientError } from '@kbn/observability-plugin/serve
import { EntitiesESClient } from '../../../lib/helpers/create_es_client/create_entities_es_client/create_entities_es_client';
import { withApmSpan } from '../../../utils/with_apm_span';
import { getEntities } from '../get_entities';
import { calculateAvgMetrics } from '../utils/calculate_avg_metrics';
import { mergeEntities } from '../utils/merge_entities';
export const MAX_NUMBER_OF_SERVICES = 1_000;
@ -41,7 +40,7 @@ export async function getServiceEntities({
size: MAX_NUMBER_OF_SERVICES,
});
return calculateAvgMetrics(mergeEntities({ entities }));
return mergeEntities({ entities });
} catch (error) {
// If the index does not exist, handle it gracefully
if (

3
x-pack/plugins/observability_solution/apm/server/routes/entities/services/get_service_entity_summary.ts
View file

@ -8,7 +8,6 @@
import type { EntitiesESClient } from '../../../lib/helpers/create_es_client/create_entities_es_client/create_entities_es_client';
import { withApmSpan } from '../../../utils/with_apm_span';
import { getEntityLatestServices } from '../get_entity_latest_services';
import { calculateAvgMetrics } from '../utils/calculate_avg_metrics';
import { mergeEntities } from '../utils/merge_entities';
import { MAX_NUMBER_OF_SERVICES } from './get_service_entities';
@ -27,7 +26,7 @@ export function getServiceEntitySummary({ entitiesESClient, environment, service
serviceName,
});
const serviceEntity = calculateAvgMetrics(mergeEntities({ entities: entityLatestServices }));
const serviceEntity = mergeEntities({ entities: entityLatestServices });
return serviceEntity[0];
});
}

44
x-pack/plugins/observability_solution/apm/server/routes/entities/services/routes.ts
View file

@ -4,8 +4,6 @@
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import Boom from '@hapi/boom';
import { jsonRt } from '@kbn/io-ts-utils';
import * as t from 'io-ts';
import { environmentQuery } from '../../../../common/utils/environment_query';
import { createEntitiesESClient } from '../../../lib/helpers/create_es_client/create_entities_es_client/create_entities_es_client';
@ -13,7 +11,6 @@ import { createApmServerRoute } from '../../apm_routes/create_apm_server_route';
import { environmentRt, kueryRt, rangeRt } from '../../default_api_types';
import { getServiceEntities } from './get_service_entities';
import { getServiceEntitySummary } from './get_service_entity_summary';
import { getEntityHistoryServicesTimeseries } from '../get_entity_history_services_timeseries';
const serviceEntitiesSummaryRoute = createApmServerRoute({
endpoint: 'GET /internal/apm/entities/services/{serviceName}/summary',
@ -72,46 +69,6 @@ const servicesEntitiesRoute = createApmServerRoute({
},
});
const servicesEntitiesDetailedStatisticsRoute = createApmServerRoute({
endpoint: 'POST /internal/apm/entities/services/detailed_statistics',
params: t.type({
query: t.intersection([environmentRt, kueryRt, rangeRt]),
body: t.type({ serviceNames: jsonRt.pipe(t.array(t.string)) }),
}),
options: { tags: ['access:apm'] },
handler: async (resources) => {
const { context, params, request } = resources;
const coreContext = await context.core;
const entitiesESClient = await createEntitiesESClient({
request,
esClient: coreContext.elasticsearch.client.asCurrentUser,
});
const { environment, start, end } = params.query;
const { serviceNames } = params.body;
if (!serviceNames.length) {
throw Boom.badRequest(`serviceNames cannot be empty`);
}
const serviceEntitiesTimeseries = await getEntityHistoryServicesTimeseries({
start,
end,
serviceNames,
environment,
entitiesESClient,
});
return {
currentPeriod: {
...serviceEntitiesTimeseries,
},
};
},
});
const serviceLogRateTimeseriesRoute = createApmServerRoute({
endpoint: 'GET /internal/apm/entities/services/{serviceName}/logs_rate_timeseries',
params: t.type({
@ -183,6 +140,5 @@ export const servicesEntitiesRoutesRepository = {
...servicesEntitiesRoute,
...serviceLogRateTimeseriesRoute,
...serviceLogErrorRateTimeseriesRoute,
...servicesEntitiesDetailedStatisticsRoute,
...serviceEntitiesSummaryRoute,
};

2
x-pack/plugins/observability_solution/apm/server/routes/entities/types.ts
View file

@ -5,7 +5,6 @@
* 2.0.
*/
import { AgentName } from '../../../typings/es_schemas/ui/fields/agent';
import { EntityMetrics } from '../../../common/entities/types';
export enum EntityType {
SERVICE = 'service',
@ -30,5 +29,4 @@ interface Entity {
lastSeenTimestamp: string;
firstSeenTimestamp: string;
identityFields: string[];
metrics: EntityMetrics;
}

236
x-pack/plugins/observability_solution/apm/server/routes/entities/utils/calculate_avg_metrics.test.ts
View file

@ -1,236 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { EntityMetrics, EntityDataStreamType } from '../../../../common/entities/types';
import { AgentName } from '../../../../typings/es_schemas/ui/fields/agent';
import { calculateAvgMetrics, mergeMetrics } from './calculate_avg_metrics';
describe('calculateAverageMetrics', () => {
it('calculates average metrics', () => {
const entities = [
{
agentName: 'nodejs' as AgentName,
dataStreamTypes: [EntityDataStreamType.METRICS, EntityDataStreamType.LOGS],
environments: [],
latestTimestamp: '2024-03-05T10:34:40.810Z',
metrics: [
{
failedTransactionRate: 5,
latency: 5,
logErrorRate: 5,
logRate: 5,
throughput: 5,
},
{
failedTransactionRate: 10,
latency: 10,
logErrorRate: 10,
logRate: 10,
throughput: 10,
},
],
serviceName: 'service-1',
hasLogMetrics: true,
},
{
agentName: 'java' as AgentName,
dataStreamTypes: [EntityDataStreamType.METRICS],
environments: [],
latestTimestamp: '2024-06-05T10:34:40.810Z',
metrics: [
{
failedTransactionRate: 15,
latency: 15,
logErrorRate: 15,
logRate: 15,
throughput: 15,
},
{
failedTransactionRate: 5,
latency: 5,
logErrorRate: 5,
logRate: 5,
throughput: 5,
},
],
serviceName: 'service-2',
hasLogMetrics: true,
},
];
const result = calculateAvgMetrics(entities);
expect(result).toEqual([
{
agentName: 'nodejs',
dataStreamTypes: [EntityDataStreamType.METRICS, EntityDataStreamType.LOGS],
environments: [],
latestTimestamp: '2024-03-05T10:34:40.810Z',
metrics: {
failedTransactionRate: 7.5,
latency: 7.5,
logErrorRate: 7.5,
logRate: 7.5,
throughput: 7.5,
},
serviceName: 'service-1',
hasLogMetrics: true,
},
{
agentName: 'java' as AgentName,
dataStreamTypes: [EntityDataStreamType.METRICS],
environments: [],
latestTimestamp: '2024-06-05T10:34:40.810Z',
metrics: {
failedTransactionRate: 10,
latency: 10,
logErrorRate: 10,
logRate: 10,
throughput: 10,
},
serviceName: 'service-2',
hasLogMetrics: true,
},
]);
});
it('calculates average metrics with null', () => {
const entities = [
{
agentName: 'nodejs' as AgentName,
dataStreamTypes: [EntityDataStreamType.METRICS],
environments: ['env-service-1', 'env-service-2'],
latestTimestamp: '2024-03-05T10:34:40.810Z',
metrics: [
{
failedTransactionRate: 5,
latency: null,
logErrorRate: 5,
logRate: 5,
throughput: 5,
},
{
failedTransactionRate: 10,
latency: null,
logErrorRate: 10,
logRate: 10,
throughput: 10,
},
],
serviceName: 'service-1',
hasLogMetrics: true,
},
];
const result = calculateAvgMetrics(entities);
expect(result).toEqual([
{
agentName: 'nodejs',
dataStreamTypes: [EntityDataStreamType.METRICS],
environments: ['env-service-1', 'env-service-2'],
latestTimestamp: '2024-03-05T10:34:40.810Z',
metrics: {
failedTransactionRate: 7.5,
logErrorRate: 7.5,
logRate: 7.5,
throughput: 7.5,
},
serviceName: 'service-1',
hasLogMetrics: true,
},
]);
});
});
describe('mergeMetrics', () => {
it('merges metrics correctly', () => {
const metrics = [
{
failedTransactionRate: 5,
latency: 5,
logErrorRate: 5,
logRate: 5,
throughput: 5,
},
{
failedTransactionRate: 10,
latency: 10,
logErrorRate: 10,
logRate: 10,
throughput: 10,
},
];
const result = mergeMetrics(metrics);
expect(result).toEqual({
failedTransactionRate: [5, 10],
latency: [5, 10],
logErrorRate: [5, 10],
logRate: [5, 10],
throughput: [5, 10],
});
});
it('handles empty metrics array', () => {
const metrics: EntityMetrics[] = [];
const result = mergeMetrics(metrics);
expect(result).toEqual({});
});
it('returns metrics with zero value', () => {
const metrics = [
{
failedTransactionRate: 0,
latency: 4,
logErrorRate: 5,
logRate: 5,
throughput: 5,
},
];
const result = mergeMetrics(metrics);
expect(result).toEqual({
failedTransactionRate: [0],
latency: [4],
logErrorRate: [5],
logRate: [5],
throughput: [5],
});
});
it('does not return metrics with null', () => {
const metrics = [
{
failedTransactionRate: null,
latency: null,
logErrorRate: 5,
logRate: 5,
throughput: 5,
},
{
failedTransactionRate: 5,
latency: null,
logErrorRate: 5,
logRate: 5,
throughput: 5,
},
];
const result = mergeMetrics(metrics);
expect(result).toEqual({
failedTransactionRate: [5],
logErrorRate: [5, 5],
logRate: [5, 5],
throughput: [5, 5],
});
});
});

45
x-pack/plugins/observability_solution/apm/server/routes/entities/utils/calculate_avg_metrics.ts
View file

@ -1,45 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { mapValues, isNumber } from 'lodash';
import { EntityMetrics } from '../../../../common/entities/types';
import type { MergedServiceEntity } from './merge_entities';
export function calculateAvgMetrics(entities: MergedServiceEntity[]) {
return entities.map((entity) => {
const transformedMetrics = mergeMetrics(entity.metrics);
const averages = mapValues(transformedMetrics, (values: number[]) => {
const sum = values.reduce((acc: number, val: number) => acc + (val !== null ? val : 0), 0);
return sum / values.length;
});
return {
...entity,
metrics: averages,
};
});
}
type MetricsKey = keyof EntityMetrics;
export function mergeMetrics(metrics: EntityMetrics[]) {
return metrics.reduce((acc, metric) => {
for (const key in metric) {
if (Object.hasOwn(metric, key)) {
const metricsKey = key as MetricsKey;
const value = metric[metricsKey];
if (isNumber(value)) {
if (!acc[metricsKey]) {
acc[metricsKey] = [];
}
acc[metricsKey].push(value);
}
}
}
return acc;
}, {} as { [key in MetricsKey]: number[] });
}

233
x-pack/plugins/observability_solution/apm/server/routes/entities/utils/merge_entities.test.ts
View file

@ -22,13 +22,6 @@ describe('mergeEntities', () => {
entity: {
firstSeenTimestamp: '2024-06-05T10:34:40.810Z',
lastSeenTimestamp: '2024-06-05T10:34:40.810Z',
metrics: {
logRate: 1,
logErrorRate: null,
throughput: 0,
failedTransactionRate: 0.3333333333333333,
latency: 10,
},
identityFields: ['service.name', 'service.environment'],
id: 'service-1:test',
},
@ -41,16 +34,6 @@ describe('mergeEntities', () => {
dataStreamTypes: ['metrics', 'logs'],
environments: ['test'],
lastSeenTimestamp: '2024-06-05T10:34:40.810Z',
hasLogMetrics: true,
metrics: [
{
failedTransactionRate: 0.3333333333333333,
latency: 10,
logErrorRate: null,
logRate: 1,
throughput: 0,
},
],
serviceName: 'service-1',
},
]);
@ -68,13 +51,6 @@ describe('mergeEntities', () => {
entity: {
firstSeenTimestamp: '2024-03-05T10:34:40.810Z',
lastSeenTimestamp: '2024-03-05T10:34:40.810Z',
metrics: {
logRate: 1,
logErrorRate: null,
throughput: 0,
failedTransactionRate: 0.3333333333333333,
latency: 10,
},
identityFields: ['service.name', 'service.environment'],
id: 'service-1:env-service-1',
},
@ -89,13 +65,6 @@ describe('mergeEntities', () => {
entity: {
firstSeenTimestamp: '2024-03-05T10:34:40.810Z',
lastSeenTimestamp: '2024-03-05T10:34:40.810Z',
metrics: {
logRate: 10,
logErrorRate: 10,
throughput: 10,
failedTransactionRate: 10,
latency: 10,
},
identityFields: ['service.name', 'service.environment'],
id: 'apm-only-1:synthtrace-env-2',
},
@ -110,13 +79,6 @@ describe('mergeEntities', () => {
entity: {
firstSeenTimestamp: '2024-06-05T10:34:40.810Z',
lastSeenTimestamp: '2024-06-05T10:34:40.810Z',
metrics: {
logRate: 15,
logErrorRate: 15,
throughput: 15,
failedTransactionRate: 15,
latency: 15,
},
identityFields: ['service.name', 'service.environment'],
id: 'service-2:env-service-3',
},
@ -131,13 +93,6 @@ describe('mergeEntities', () => {
entity: {
firstSeenTimestamp: '2024-06-05T10:34:40.810Z',
lastSeenTimestamp: '2024-06-05T10:34:40.810Z',
metrics: {
logRate: 5,
logErrorRate: 5,
throughput: 5,
failedTransactionRate: 5,
latency: 5,
},
identityFields: ['service.name', 'service.environment'],
id: 'service-2:env-service-3',
},
@ -151,23 +106,6 @@ describe('mergeEntities', () => {
dataStreamTypes: ['foo', 'bar'],
environments: ['env-service-1', 'env-service-2'],
lastSeenTimestamp: '2024-03-05T10:34:40.810Z',
hasLogMetrics: true,
metrics: [
{
failedTransactionRate: 0.3333333333333333,
latency: 10,
logErrorRate: null,
logRate: 1,
throughput: 0,
},
{
failedTransactionRate: 10,
latency: 10,
logErrorRate: 10,
logRate: 10,
throughput: 10,
},
],
serviceName: 'service-1',
},
{
@ -175,23 +113,6 @@ describe('mergeEntities', () => {
dataStreamTypes: ['baz'],
environments: ['env-service-3', 'env-service-4'],
lastSeenTimestamp: '2024-06-05T10:34:40.810Z',
hasLogMetrics: true,
metrics: [
{
failedTransactionRate: 15,
latency: 15,
logErrorRate: 15,
logRate: 15,
throughput: 15,
},
{
failedTransactionRate: 5,
latency: 5,
logErrorRate: 5,
logRate: 5,
throughput: 5,
},
],
serviceName: 'service-2',
},
]);
@ -208,13 +129,6 @@ describe('mergeEntities', () => {
entity: {
firstSeenTimestamp: '2024-06-05T10:34:40.810Z',
lastSeenTimestamp: '2024-06-05T10:34:40.810Z',
metrics: {
logRate: 5,
logErrorRate: 5,
throughput: 5,
failedTransactionRate: 5,
latency: 5,
},
identityFields: ['service.name', 'service.environment'],
id: 'service-1:test',
},
@ -229,13 +143,6 @@ describe('mergeEntities', () => {
entity: {
firstSeenTimestamp: '2024-06-05T10:34:40.810Z',
lastSeenTimestamp: '2024-06-05T10:34:40.810Z',
metrics: {
logRate: 10,
logErrorRate: 10,
throughput: 10,
failedTransactionRate: 0.3333333333333333,
latency: 10,
},
identityFields: ['service.name', 'service.environment'],
id: 'service-1:test',
},
@ -250,13 +157,6 @@ describe('mergeEntities', () => {
entity: {
firstSeenTimestamp: '2024-23-05T10:34:40.810Z',
lastSeenTimestamp: '2024-23-05T10:34:40.810Z',
metrics: {
logRate: 0.333,
logErrorRate: 0.333,
throughput: 0.333,
failedTransactionRate: 0.333,
latency: 0.333,
},
identityFields: ['service.name', 'service.environment'],
id: 'service-1:prod',
},
@ -269,30 +169,6 @@ describe('mergeEntities', () => {
dataStreamTypes: ['metrics', 'logs', 'foo'],
environments: ['test', 'prod'],
lastSeenTimestamp: '2024-23-05T10:34:40.810Z',
hasLogMetrics: true,
metrics: [
{
failedTransactionRate: 5,
latency: 5,
logErrorRate: 5,
logRate: 5,
throughput: 5,
},
{
failedTransactionRate: 0.3333333333333333,
latency: 10,
logErrorRate: 10,
logRate: 10,
throughput: 10,
},
{
failedTransactionRate: 0.333,
latency: 0.333,
logErrorRate: 0.333,
logRate: 0.333,
throughput: 0.333,
},
],
serviceName: 'service-1',
},
]);
@ -309,13 +185,6 @@ describe('mergeEntities', () => {
entity: {
firstSeenTimestamp: '2024-06-05T10:34:40.810Z',
lastSeenTimestamp: '2024-06-05T10:34:40.810Z',
metrics: {
logRate: 1,
logErrorRate: null,
throughput: 0,
failedTransactionRate: 0.3333333333333333,
latency: 10,
},
identityFields: ['service.name'],
id: 'service-1:test',
},
@ -328,16 +197,6 @@ describe('mergeEntities', () => {
dataStreamTypes: [],
environments: [],
lastSeenTimestamp: '2024-06-05T10:34:40.810Z',
hasLogMetrics: true,
metrics: [
{
failedTransactionRate: 0.3333333333333333,
latency: 10,
logErrorRate: null,
logRate: 1,
throughput: 0,
},
],
serviceName: 'service-1',
},
]);
@ -352,13 +211,6 @@ describe('mergeEntities', () => {
entity: {
firstSeenTimestamp: '2024-06-05T10:34:40.810Z',
lastSeenTimestamp: '2024-06-05T10:34:40.810Z',
metrics: {
logRate: 1,
logErrorRate: null,
throughput: 0,
failedTransactionRate: 0.3333333333333333,
latency: 10,
},
identityFields: ['service.name'],
id: 'service-1:test',
},
@ -372,13 +224,6 @@ describe('mergeEntities', () => {
entity: {
firstSeenTimestamp: '2024-06-05T10:34:40.810Z',
lastSeenTimestamp: '2024-06-05T10:34:40.810Z',
metrics: {
logRate: 1,
logErrorRate: null,
throughput: 0,
failedTransactionRate: 0.3333333333333333,
latency: 10,
},
identityFields: ['service.name'],
id: 'service-1:test',
},
@ -391,23 +236,6 @@ describe('mergeEntities', () => {
dataStreamTypes: [],
environments: [],
lastSeenTimestamp: '2024-06-05T10:34:40.810Z',
hasLogMetrics: true,
metrics: [
{
failedTransactionRate: 0.3333333333333333,
latency: 10,
logErrorRate: null,
logRate: 1,
throughput: 0,
},
{
logRate: 1,
logErrorRate: null,
throughput: 0,
failedTransactionRate: 0.3333333333333333,
latency: 10,
},
],
serviceName: 'service-1',
},
]);
@ -424,13 +252,6 @@ describe('mergeEntities', () => {
entity: {
firstSeenTimestamp: '2024-06-05T10:34:40.810Z',
lastSeenTimestamp: '2024-06-05T10:34:40.810Z',
metrics: {
logRate: 1,
logErrorRate: null,
throughput: 0,
failedTransactionRate: 0.3333333333333333,
latency: 10,
},
identityFields: ['service.name'],
id: 'service-1:test',
},
@ -443,16 +264,6 @@ describe('mergeEntities', () => {
dataStreamTypes: [],
environments: [],
lastSeenTimestamp: '2024-06-05T10:34:40.810Z',
hasLogMetrics: true,
metrics: [
{
failedTransactionRate: 0.3333333333333333,
latency: 10,
logErrorRate: null,
logRate: 1,
throughput: 0,
},
],
serviceName: 'service-1',
},
]);
@ -467,13 +278,6 @@ describe('mergeEntities', () => {
entity: {
firstSeenTimestamp: '2024-06-05T10:34:40.810Z',
lastSeenTimestamp: '2024-06-05T10:34:40.810Z',
metrics: {
logRate: 1,
logErrorRate: null,
throughput: 0,
failedTransactionRate: 0.3333333333333333,
latency: 10,
},
identityFields: ['service.name'],
id: 'service-1:test',
},
@ -487,13 +291,6 @@ describe('mergeEntities', () => {
entity: {
firstSeenTimestamp: '2024-06-05T10:34:40.810Z',
lastSeenTimestamp: '2024-06-05T10:34:40.810Z',
metrics: {
logRate: 1,
logErrorRate: null,
throughput: 0,
failedTransactionRate: 0.3333333333333333,
latency: 10,
},
identityFields: ['service.name'],
id: 'service-1:test',
},
@ -506,23 +303,6 @@ describe('mergeEntities', () => {
dataStreamTypes: [],
environments: [],
lastSeenTimestamp: '2024-06-05T10:34:40.810Z',
hasLogMetrics: true,
metrics: [
{
failedTransactionRate: 0.3333333333333333,
latency: 10,
logErrorRate: null,
logRate: 1,
throughput: 0,
},
{
logRate: 1,
logErrorRate: null,
throughput: 0,
failedTransactionRate: 0.3333333333333333,
latency: 10,
},
],
serviceName: 'service-1',
},
]);
@ -540,11 +320,6 @@ describe('mergeEntities', () => {
entity: {
firstSeenTimestamp: '2024-06-05T10:34:40.810Z',
lastSeenTimestamp: '2024-06-05T10:34:40.810Z',
metrics: {
throughput: 0,
failedTransactionRate: 0.3333333333333333,
latency: 10,
},
identityFields: ['service.name', 'service.environment'],
id: 'service-1:test',
},
@ -557,14 +332,6 @@ describe('mergeEntities', () => {
dataStreamTypes: ['metrics'],
environments: ['test'],
lastSeenTimestamp: '2024-06-05T10:34:40.810Z',
hasLogMetrics: false,
metrics: [
{
failedTransactionRate: 0.3333333333333333,
latency: 10,
throughput: 0,
},
],
serviceName: 'service-1',
},
]);

13
x-pack/plugins/observability_solution/apm/server/routes/entities/utils/merge_entities.ts
View file

@ -7,17 +7,14 @@
import { compact, uniq } from 'lodash';
import type { EntityLatestServiceRaw } from '../types';
import { isFiniteNumber } from '../../../../common/utils/is_finite_number';
import type { AgentName } from '../../../../typings/es_schemas/ui/fields/agent';
import type { EntityDataStreamType, EntityMetrics } from '../../../../common/entities/types';
import type { EntityDataStreamType } from '../../../../common/entities/types';
export interface MergedServiceEntity {
serviceName: string;
agentName: AgentName;
dataStreamTypes: EntityDataStreamType[];
environments: string[];
metrics: EntityMetrics[];
hasLogMetrics: boolean;
}
export function mergeEntities({
@ -40,10 +37,6 @@ export function mergeEntities({
}
function mergeFunc(entity: EntityLatestServiceRaw, existingEntity?: MergedServiceEntity) {
const hasLogMetrics = isFiniteNumber(entity.entity.metrics.logRate)
? entity.entity.metrics.logRate > 0
: false;
const commonEntityFields = {
serviceName: entity.service.name,
agentName: entity.agent.name[0],
@ -55,8 +48,6 @@ function mergeFunc(entity: EntityLatestServiceRaw, existingEntity?: MergedServic
...commonEntityFields,
dataStreamTypes: entity.source_data_stream.type,
environments: compact([entity?.service.environment]),
metrics: [entity.entity.metrics],
hasLogMetrics,
};
}
return {
@ -65,7 +56,5 @@ function mergeFunc(entity: EntityLatestServiceRaw, existingEntity?: MergedServic
compact([...(existingEntity?.dataStreamTypes ?? []), ...entity.source_data_stream.type])
),
environments: uniq(compact([...existingEntity?.environments, entity?.service.environment])),
metrics: [...existingEntity?.metrics, entity.entity.metrics],
hasLogMetrics: hasLogMetrics || existingEntity.hasLogMetrics,
};
}

58
x-pack/test/apm_api_integration/tests/entities/services/services_entities_detailed_statistics.spec.ts
View file

@ -1,58 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import expect from '@kbn/expect';
import { APIClientRequestParamsOf } from '@kbn/apm-plugin/public/services/rest/create_call_apm_api';
import { FtrProviderContext } from '../../../common/ftr_provider_context';
export default function ApiTest({ getService }: FtrProviderContext) {
const registry = getService('registry');
const apmApiClient = getService('apmApiClient');
const start = '2024-01-01T00:00:00.000Z';
const end = '2024-01-01T00:59:59.999Z';
const serviceNames = ['my-service', 'synth-go'];
async function getServiceEntitiesDetailedStats(
overrides?: Partial<
APIClientRequestParamsOf<'POST /internal/apm/entities/services/detailed_statistics'>['params']['query']
>
) {
const response = await apmApiClient.readUser({
endpoint: `POST /internal/apm/entities/services/detailed_statistics`,
params: {
query: {
start,
end,
environment: 'ENVIRONMENT_ALL',
kuery: '',
...overrides,
},
body: {
serviceNames: JSON.stringify(serviceNames),
},
},
});
return response;
}
registry.when(
'Services entities detailed statistics when no data is generated',
{ config: 'basic', archives: [] },
() => {
describe('Service entities detailed', () => {
it('handles the empty state', async () => {
const response = await getServiceEntitiesDetailedStats();
expect(response.status).to.be(200);
expect(response.body.currentPeriod).to.empty();
});
});
}
);
}