github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 01:38:56 -04:00
Code Issues Projects Releases Packages Wiki Activity

[APM][ECO] Fetching metrics from entities history (#189164)

Browse source 
closes https://github.com/elastic/kibana/issues/189082

<img width="1215" alt="Screenshot 2024-07-25 at 10 34 13"
src="https://github.com/user-attachments/assets/fe693e9c-cdb8-4136-b942-38069ae56f92">
This commit is contained in:
Cauê Marcondes 2024-07-29 12:27:46 +01:00 • committed by GitHub
parent c07595fe00
commit 587be2709e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 193 additions and 39 deletions
Download patch file Download diff file Expand all files Collapse all files

4
x-pack/plugins/observability_solution/apm/common/entities/types.ts
View file

@ -16,8 +16,8 @@ export enum SignalTypes {
export interface EntityMetrics {
latency: number | null;
throughput: number | null;
failedTransactionRate: number;
logRate: number;
failedTransactionRate: number | null;
logRate: number | null;
logErrorRate: number | null;
}

10
x-pack/plugins/observability_solution/apm/common/es_fields/entities.ts
View file

@ -5,7 +5,13 @@
* 2.0.
*/
export const ENTITY = 'entity';
export const LAST_SEEN = 'entity.lastSeenTimestamp';
export const FIRST_SEEN = 'entity.firstSeenTimestamp';
export const ENTITY = 'entity';
export const ENTITY_ID = 'entity.id';
export const ENTITY_METRICS_LATENCY = 'entity.metrics.latency';
export const ENTITY_METRICS_LOG_ERROR_RATE = 'entity.metrics.logErrorRate';
export const ENTITY_METRICS_LOG_RATE = 'entity.metrics.logRate';
export const ENTITY_METRICS_THROUGHPUT = 'entity.metrics.throughput';
export const ENTITY_METRICS_FAILED_TRANSACTION_RATE = 'entity.metrics.failedTransactionRate';
export const ENTITY_TYPE = 'entity.type';

6
x-pack/plugins/observability_solution/apm/common/utils/formatters/formatters.test.ts
View file

@ -71,6 +71,12 @@ describe('formatters', () => {
expect(asDecimalOrInteger(0.25435632645, 1)).toEqual('0.3');
});
});
it('returns fallback when valueNaN', () => {
expect(asDecimalOrInteger(NaN)).toEqual('N/A');
expect(asDecimalOrInteger(null)).toEqual('N/A');
expect(asDecimalOrInteger(undefined)).toEqual('N/A');
});
});
});

6
x-pack/plugins/observability_solution/apm/common/utils/formatters/formatters.ts
View file

@ -56,7 +56,11 @@ export function asPercent(
return numeral(decimal).format('0.0%');
}
export function asDecimalOrInteger(value: number, threshold = 10) {
export function asDecimalOrInteger(value: Maybe<number>, threshold = 10) {
if (!isFiniteNumber(value)) {
return NOT_AVAILABLE_LABEL;
}
// exact 0 or above threshold should not have decimal
if (value === 0 || value >= threshold) {
return asInteger(value);

4
x-pack/plugins/observability_solution/apm/ftr_e2e/cypress/e2e/settings/agent_configurations.cy.ts
View file

@ -121,8 +121,8 @@ describe('Agent configuration', () => {
cy.get('mark').contains('All').click({ force: true });
cy.contains('Next step').click();
cy.contains('Service name All');
cy.contains('Environment All');
cy.get('[data-test-subj="settingsPage_serviceName"]').contains('All');
cy.get('[data-test-subj="settingsPage_environmentName"]').contains('All');
cy.contains('Edit').click();
cy.wait('@serviceEnvironmentApi');
cy.getByTestSubj('serviceEnviromentComboBox')

20
x-pack/plugins/observability_solution/apm/public/components/app/settings/agent_configurations/agent_configuration_create_edit/settings_page/settings_page.tsx
View file

@ -132,7 +132,15 @@ export function SettingsPage({
<EuiFlexItem>
<EuiStat
titleSize="xs"
title={isLoading ? '-' : getOptionLabel(newConfig.service.name)}
title={
isLoading ? (
'-'
) : (
<span data-test-subj="settingsPage_serviceName">
{getOptionLabel(newConfig.service.name)}
</span>
)
}
description={i18n.translate(
'xpack.apm.agentConfig.chooseService.service.name.label',
{ defaultMessage: 'Service name' }
@ -142,7 +150,15 @@ export function SettingsPage({
<EuiFlexItem>
<EuiStat
titleSize="xs"
title={isLoading ? '-' : getOptionLabel(newConfig.service.environment)}
title={
isLoading ? (
'-'
) : (
<span data-test-subj="settingsPage_environmentName">
{getOptionLabel(newConfig.service.environment)}
</span>
)
}
description={i18n.translate(
'xpack.apm.agentConfig.chooseService.service.environment.label',
{ defaultMessage: 'Environment' }

64
x-pack/plugins/observability_solution/apm/server/lib/helpers/create_es_client/create_assets_es_client/create_assets_es_clients.ts
View file

@ -14,7 +14,8 @@ import {
} from '@elastic/elasticsearch/lib/api/types';
import { withApmSpan } from '../../../../utils/with_apm_span';
const ENTITIES_INDEX_NAME = '.entities.v1.latest.builtin_services*';
const ENTITIES_LATEST_INDEX_NAME = '.entities.v1.latest.builtin_services*';
const ENTITIES_HISTORY_INDEX_NAME = '.entities.v1.history.builtin_services*';
export function cancelEsRequestOnAbort<T extends Promise<any>>(
promise: T,
@ -29,7 +30,11 @@ export function cancelEsRequestOnAbort<T extends Promise<any>>(
}
export interface EntitiesESClient {
search<TDocument = unknown, TSearchRequest extends ESSearchRequest = ESSearchRequest>(
searchLatest<TDocument = unknown, TSearchRequest extends ESSearchRequest = ESSearchRequest>(
operationName: string,
searchRequest: TSearchRequest
): Promise<InferSearchResponseOf<TDocument, TSearchRequest>>;
searchHistory<TDocument = unknown, TSearchRequest extends ESSearchRequest = ESSearchRequest>(
operationName: string,
searchRequest: TSearchRequest
): Promise<InferSearchResponseOf<TDocument, TSearchRequest>>;
@ -45,30 +50,45 @@ export async function createEntitiesESClient({
request: KibanaRequest;
esClient: ElasticsearchClient;
}) {
function search<TDocument = unknown, TSearchRequest extends ESSearchRequest = ESSearchRequest>(
indexName: string,
operationName: string,
searchRequest: TSearchRequest
): Promise<InferSearchResponseOf<TDocument, TSearchRequest>> {
const controller = new AbortController();
const promise = withApmSpan(operationName, () => {
return cancelEsRequestOnAbort(
esClient.search(
{ ...searchRequest, index: [indexName] },
{
signal: controller.signal,
meta: true,
}
) as unknown as Promise<{
body: InferSearchResponseOf<TDocument, TSearchRequest>;
}>,
request,
controller
);
});
return unwrapEsResponse(promise);
}
return {
search<TDocument = unknown, TSearchRequest extends ESSearchRequest = ESSearchRequest>(
searchLatest<TDocument = unknown, TSearchRequest extends ESSearchRequest = ESSearchRequest>(
operationName: string,
searchRequest: TSearchRequest
): Promise<InferSearchResponseOf<TDocument, TSearchRequest>> {
const controller = new AbortController();
return search(ENTITIES_LATEST_INDEX_NAME, operationName, searchRequest);
},
const promise = withApmSpan(operationName, () => {
return cancelEsRequestOnAbort(
esClient.search(
{ ...searchRequest, index: [ENTITIES_INDEX_NAME] },
{
signal: controller.signal,
meta: true,
}
) as unknown as Promise<{
body: InferSearchResponseOf<TDocument, TSearchRequest>;
}>,
request,
controller
);
});
return unwrapEsResponse(promise);
searchHistory<TDocument = unknown, TSearchRequest extends ESSearchRequest = ESSearchRequest>(
operationName: string,
searchRequest: TSearchRequest
): Promise<InferSearchResponseOf<TDocument, TSearchRequest>> {
return search(ENTITIES_HISTORY_INDEX_NAME, operationName, searchRequest);
},
async msearch<TDocument = unknown, TSearchRequest extends ESSearchRequest = ESSearchRequest>(
@ -78,7 +98,7 @@ export async function createEntitiesESClient({
.map((params) => {
const searchParams: [MsearchMultisearchHeader, MsearchMultisearchBody] = [
{
index: [ENTITIES_INDEX_NAME],
index: [ENTITIES_LATEST_INDEX_NAME],
},
{
...params.body,

35
x-pack/plugins/observability_solution/apm/server/routes/entities/get_entities.ts
View file

@ -5,30 +5,31 @@
* 2.0.
*/
import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import { kqlQuery } from '@kbn/observability-plugin/server';
import { kqlQuery, termQuery } from '@kbn/observability-plugin/server';
import {
AGENT_NAME,
DATA_STEAM_TYPE,
SERVICE_ENVIRONMENT,
SERVICE_NAME,
} from '../../../common/es_fields/apm';
import { FIRST_SEEN, LAST_SEEN, ENTITY } from '../../../common/es_fields/entities';
import { FIRST_SEEN, LAST_SEEN, ENTITY, ENTITY_TYPE } from '../../../common/es_fields/entities';
import { environmentQuery } from '../../../common/utils/environment_query';
import { EntitiesESClient } from '../../lib/helpers/create_es_client/create_assets_es_client/create_assets_es_clients';
import { EntitiesRaw, ServiceEntities } from './types';
import { getServiceEntitiesHistoryMetrics } from './get_service_entities_history_metrics';
import { EntitiesRaw, EntityType, ServiceEntities } from './types';
export function entitiesRangeQuery(start: number, end: number): QueryDslQueryContainer[] {
return [
{
range: {
[FIRST_SEEN]: {
[LAST_SEEN]: {
gte: start,
},
},
},
{
range: {
[LAST_SEEN]: {
[FIRST_SEEN]: {
lte: end,
},
},
@ -52,7 +53,7 @@ export async function getEntities({
size: number;
}) {
const entities = (
await entitiesESClient.search(`get_entities`, {
await entitiesESClient.searchLatest(`get_entities`, {
body: {
size,
track_total_hits: false,
@ -63,6 +64,7 @@ export async function getEntities({
...kqlQuery(kuery),
...environmentQuery(environment, SERVICE_ENVIRONMENT),
...entitiesRangeQuery(start, end),
...termQuery(ENTITY_TYPE, EntityType.SERVICE),
],
},
},
@ -70,6 +72,15 @@ export async function getEntities({
})
).hits.hits.map((hit) => hit._source as EntitiesRaw);
const serviceEntitiesHistoryMetricsMap = entities.length
? await getServiceEntitiesHistoryMetrics({
start,
end,
entitiesESClient,
entityIds: entities.map((entity) => entity.entity.id),
})
: undefined;
return entities.map((entity): ServiceEntities => {
return {
serviceName: entity.service.name,
@ -78,7 +89,17 @@ export async function getEntities({
: entity.service.environment,
agentName: entity.agent.name[0],
signalTypes: entity.data_stream.type,
entity: entity.entity,
entity: {
...entity.entity,
// History metrics undefined means that for the selected time range there was no ingestion happening.
metrics: serviceEntitiesHistoryMetricsMap?.[entity.entity.id] || {
latency: null,
logErrorRate: null,
failedTransactionRate: null,
logRate: null,
throughput: null,
},
},
};
});
}

77
x-pack/plugins/observability_solution/apm/server/routes/entities/get_service_entities_history_metrics.ts Normal file
View file

@ -0,0 +1,77 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { termsQuery, rangeQuery } from '@kbn/observability-plugin/server';
import { EntityMetrics } from '../../../common/entities/types';
import {
ENTITY_ID,
ENTITY_METRICS_FAILED_TRANSACTION_RATE,
ENTITY_METRICS_LATENCY,
ENTITY_METRICS_LOG_ERROR_RATE,
ENTITY_METRICS_LOG_RATE,
ENTITY_METRICS_THROUGHPUT,
LAST_SEEN,
} from '../../../common/es_fields/entities';
import { EntitiesESClient } from '../../lib/helpers/create_es_client/create_assets_es_client/create_assets_es_clients';
interface Params {
entitiesESClient: EntitiesESClient;
start: number;
end: number;
entityIds: string[];
}
export async function getServiceEntitiesHistoryMetrics({
end,
entityIds,
start,
entitiesESClient,
}: Params) {
const response = await entitiesESClient.searchHistory('get_entities_history', {
body: {
size: 0,
track_total_hits: false,
query: {
bool: {
filter: [...rangeQuery(start, end, LAST_SEEN), ...termsQuery(ENTITY_ID, ...entityIds)],
},
},
aggs: {
entityIds: {
terms: { field: ENTITY_ID },
aggs: {
latency: { avg: { field: ENTITY_METRICS_LATENCY } },
logErrorRate: { avg: { field: ENTITY_METRICS_LOG_ERROR_RATE } },
logRate: { avg: { field: ENTITY_METRICS_LOG_RATE } },
throughput: { avg: { field: ENTITY_METRICS_THROUGHPUT } },
failedTransactionRate: { avg: { field: ENTITY_METRICS_FAILED_TRANSACTION_RATE } },
},
},
},
},
});
if (!response.aggregations) {
return {};
}
return response.aggregations.entityIds.buckets.reduce<Record<string, EntityMetrics>>(
(acc, currBucket) => {
return {
...acc,
[currBucket.key]: {
latency: currBucket.latency.value ?? null,
logErrorRate: currBucket.logErrorRate.value ?? null,
logRate: currBucket.logRate.value ?? null,
throughput: currBucket.throughput.value ?? null,
failedTransactionRate: currBucket.failedTransactionRate.value ?? null,
},
};
},
{}
);
}

4
x-pack/plugins/observability_solution/apm/server/routes/entities/types.ts
View file

@ -7,6 +7,10 @@
import { AgentName } from '../../../typings/es_schemas/ui/fields/agent';
import { SignalTypes, EntityMetrics } from '../../../common/entities/types';
export enum EntityType {
SERVICE = 'service',
}
export interface Entity {
id: string;
latestTimestamp: string;

2
x-pack/plugins/observability_solution/apm/server/routes/historical_data/has_historical_entities_data.ts
View file

@ -19,7 +19,7 @@ export async function hasEntitiesData(entitiesESClient: EntitiesESClient, logger
};
try {
const resp = await entitiesESClient.search('has_historical_entities_data', params);
const resp = await entitiesESClient.searchLatest('has_historical_entities_data', params);
return resp.hits.total.value > 0;
} catch (error) {
if (