github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-06-27 18:51:07 -04:00
Code Issues Projects Releases Packages Wiki Activity

[APM] Use (rolled up) service metrics for service inventory (#149938)

Browse source 
Closes https://github.com/elastic/kibana/issues/146682
This commit is contained in:
Dario Gieselaar 2023-02-04 12:00:25 +01:00 • committed by GitHub
parent 949cb5829b
commit 23e97feccb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
75 changed files with 1251 additions and 1474 deletions
Download patch file Download diff file Expand all files Collapse all files

1
packages/kbn-es-types/index.ts
View file

@ -15,6 +15,7 @@ export type {
ESSourceOptions,
InferSearchResponseOf,
AggregationResultOf,
AggregationResultOfMap,
ESFilter,
MaybeReadonlyArray,
} from './src';

9
packages/kbn-es-types/src/index.ts
View file

@ -6,7 +6,12 @@
* Side Public License, v 1.
*/
import * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import { InferSearchResponseOf, AggregateOf as AggregationResultOf, SearchHit } from './search';
import {
InferSearchResponseOf,
AggregateOf as AggregationResultOf,
AggregateOfMap as AggregationResultOfMap,
SearchHit,
} from './search';
export type ESFilter = estypes.QueryDslQueryContainer;
export type ESSearchRequest = estypes.SearchRequest;
@ -29,4 +34,4 @@ export type ESSearchResponse<
TOptions extends { restTotalHitsAsInt: boolean } = { restTotalHitsAsInt: false }
> = InferSearchResponseOf<TDocument, TSearchRequest, TOptions>;
export type { InferSearchResponseOf, AggregationResultOf, SearchHit };
export type { InferSearchResponseOf, AggregationResultOf, AggregationResultOfMap, SearchHit };

2
packages/kbn-es-types/src/search.ts
View file

@ -577,7 +577,7 @@ export type AggregateOf<
>
>;
type AggregateOfMap<TAggregationMap extends AggregationMap | undefined, TDocument> = {
export type AggregateOfMap<TAggregationMap extends AggregationMap | undefined, TDocument> = {
[TAggregationName in keyof TAggregationMap]: Required<TAggregationMap>[TAggregationName] extends AggregationsAggregationContainer
? AggregateOf<TAggregationMap[TAggregationName], TDocument>
: never; // using never means we effectively ignore optional keys, using {} creates a union type of { ... } | {}

22
x-pack/plugins/apm/common/data_source.ts Normal file
View file

@ -0,0 +1,22 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { ApmDocumentType } from './document_type';
import { RollupInterval } from './rollup';
type AnyApmDocumentType =
| ApmDocumentType.ServiceTransactionMetric
| ApmDocumentType.TransactionMetric
| ApmDocumentType.TransactionEvent
| ApmDocumentType.ServiceDestinationMetric;
export interface ApmDataSource<
TDocumentType extends AnyApmDocumentType = AnyApmDocumentType
> {
rollupInterval: RollupInterval;
documentType: TDocumentType;
}

13
x-pack/plugins/apm/common/document_type.ts Normal file
View file

@ -0,0 +1,13 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export enum ApmDocumentType {
TransactionMetric = 'transactionMetric',
ServiceTransactionMetric = 'serviceTransactionMetric',
TransactionEvent = 'transactionEvent',
ServiceDestinationMetric = 'serviceDestinationMetric',
}

6
x-pack/plugins/apm/common/es_fields/__snapshots__/es_fields.test.ts.snap generated
View file

@ -84,6 +84,8 @@ exports[`Error EVENT_NAME 1`] = `undefined`;
exports[`Error EVENT_OUTCOME 1`] = `undefined`;
exports[`Error EVENT_SUCCESS_COUNT 1`] = `undefined`;
exports[`Error FAAS_BILLED_DURATION 1`] = `undefined`;
exports[`Error FAAS_COLDSTART 1`] = `undefined`;
@ -371,6 +373,8 @@ exports[`Span EVENT_NAME 1`] = `undefined`;
exports[`Span EVENT_OUTCOME 1`] = `"unknown"`;
exports[`Span EVENT_SUCCESS_COUNT 1`] = `undefined`;
exports[`Span FAAS_BILLED_DURATION 1`] = `undefined`;
exports[`Span FAAS_COLDSTART 1`] = `undefined`;
@ -654,6 +658,8 @@ exports[`Transaction EVENT_NAME 1`] = `undefined`;
exports[`Transaction EVENT_OUTCOME 1`] = `"unknown"`;
exports[`Transaction EVENT_SUCCESS_COUNT 1`] = `undefined`;
exports[`Transaction FAAS_BILLED_DURATION 1`] = `undefined`;
exports[`Transaction FAAS_COLDSTART 1`] = `undefined`;

2
x-pack/plugins/apm/common/es_fields/apm.ts
View file

@ -21,6 +21,8 @@ export const CLOUD_INSTANCE_ID = 'cloud.instance.id';
export const CLOUD_INSTANCE_NAME = 'cloud.instance.name';
export const CLOUD_SERVICE_NAME = 'cloud.service.name';
export const EVENT_SUCCESS_COUNT = 'event.success_count';
export const SERVICE = 'service';
export const SERVICE_NAME = 'service.name';
export const SERVICE_ENVIRONMENT = 'service.environment';

13
x-pack/plugins/apm/common/rollup.ts Normal file
View file

@ -0,0 +1,13 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export enum RollupInterval {
OneMinute = '1m',
TenMinutes = '10m',
SixtyMinutes = '60m',
None = 'none',
}

3
x-pack/plugins/apm/common/time_range_metadata.ts
View file

@ -5,6 +5,9 @@
* 2.0.
*/
import { ApmDataSource } from './data_source';
export interface TimeRangeMetadata {
isUsingServiceDestinationMetrics: boolean;
sources: Array<ApmDataSource & { hasDocs: boolean }>;
}

0
x-pack/plugins/apm/server/lib/helpers/get_bucket_size/calculate_auto.js → x-pack/plugins/apm/common/utils/get_bucket_size/calculate_auto.js
View file

0
x-pack/plugins/apm/server/lib/helpers/get_bucket_size/index.ts → x-pack/plugins/apm/common/utils/get_bucket_size/index.ts
View file

0
x-pack/plugins/apm/server/lib/helpers/get_bucket_size/unit_to_seconds.js → x-pack/plugins/apm/common/utils/get_bucket_size/unit_to_seconds.js
View file

87
x-pack/plugins/apm/common/utils/get_preferred_bucket_size_and_data_source.test.ts Normal file
View file

@ -0,0 +1,87 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { ApmDataSource } from '../data_source';
import { ApmDocumentType } from '../document_type';
import { RollupInterval } from '../rollup';
import {
getPreferredBucketSizeAndDataSource,
intervalToSeconds,
} from './get_preferred_bucket_size_and_data_source';
const serviceTransactionMetricSources: ApmDataSource[] = [
{
documentType: ApmDocumentType.ServiceTransactionMetric,
rollupInterval: RollupInterval.OneMinute,
},
{
documentType: ApmDocumentType.ServiceTransactionMetric,
rollupInterval: RollupInterval.TenMinutes,
},
{
documentType: ApmDocumentType.ServiceTransactionMetric,
rollupInterval: RollupInterval.SixtyMinutes,
},
];
const txMetricSources: ApmDataSource[] = [
{
documentType: ApmDocumentType.TransactionMetric,
rollupInterval: RollupInterval.OneMinute,
},
{
documentType: ApmDocumentType.TransactionMetric,
rollupInterval: RollupInterval.TenMinutes,
},
{
documentType: ApmDocumentType.TransactionMetric,
rollupInterval: RollupInterval.SixtyMinutes,
},
];
const txEventSources: ApmDataSource[] = [
{
documentType: ApmDocumentType.TransactionEvent,
rollupInterval: RollupInterval.None,
},
];
describe('getPreferredBucketSizeAndDataSource', () => {
const tests: Array<{ in: string; out: string }> = [
{ in: '30s', out: '1m' },
{ in: '60s', out: '60s' },
{ in: '10m', out: '10m' },
{ in: '30m', out: '10m' },
{ in: '60m', out: '60m' },
{ in: '120m', out: '60m' },
];
tests.forEach((test) => {
it(`${test.in} => ${test.out}`, () => {
const { source, bucketSizeInSeconds } =
getPreferredBucketSizeAndDataSource({
sources: [
...serviceTransactionMetricSources,
...txMetricSources,
...txEventSources,
],
bucketSizeInSeconds: intervalToSeconds(test.in),
});
expect(source.documentType).toBe(
ApmDocumentType.ServiceTransactionMetric
);
expect(intervalToSeconds(source.rollupInterval)).toBe(
intervalToSeconds(test.out)
);
expect(bucketSizeInSeconds).toBeGreaterThanOrEqual(
intervalToSeconds(test.out)
);
});
});
});

81
x-pack/plugins/apm/common/utils/get_preferred_bucket_size_and_data_source.ts Normal file
View file

@ -0,0 +1,81 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { parseInterval } from '@kbn/data-plugin/common';
import { orderBy, last } from 'lodash';
import { ApmDataSource } from '../data_source';
import { ApmDocumentType } from '../document_type';
import { RollupInterval } from '../rollup';
const EVENT_PREFERENCE = [
ApmDocumentType.ServiceTransactionMetric,
ApmDocumentType.TransactionMetric,
ApmDocumentType.TransactionEvent,
];
export function intervalToSeconds(rollupInterval: string) {
if (rollupInterval === RollupInterval.None) {
return 0;
}
return parseInterval(rollupInterval)!.asSeconds();
}
export function getPreferredBucketSizeAndDataSource({
sources,
bucketSizeInSeconds,
}: {
sources: ApmDataSource[];
bucketSizeInSeconds: number;
}): {
source: ApmDataSource;
bucketSizeInSeconds: number;
} {
let preferred: ApmDataSource | undefined;
const sourcesInPreferredOrder = orderBy(
sources,
[
(source) => EVENT_PREFERENCE.indexOf(source.documentType),
(source) => intervalToSeconds(source.rollupInterval),
],
['asc', 'desc']
);
if (sourcesInPreferredOrder.length > 0) {
const preferredDocumentType = sourcesInPreferredOrder[0].documentType;
const sourcesFromPreferredDocumentType = sourcesInPreferredOrder.filter(
(source) => source.documentType === preferredDocumentType
);
preferred =
sourcesFromPreferredDocumentType.find((source) => {
const rollupIntervalInSeconds = intervalToSeconds(
source.rollupInterval
);
return rollupIntervalInSeconds <= bucketSizeInSeconds;
}) ||
// pick 1m from available docs if we can't find a matching bucket size
last(sourcesFromPreferredDocumentType);
}
if (!preferred) {
preferred = {
documentType: ApmDocumentType.TransactionEvent,
rollupInterval: RollupInterval.None,
};
}
return {
source: preferred,
bucketSizeInSeconds: Math.max(
bucketSizeInSeconds,
intervalToSeconds(preferred.rollupInterval)
),
};
}

102
x-pack/plugins/apm/public/components/app/service_inventory/index.tsx
View file

@ -14,16 +14,15 @@ import {
} from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import { FormattedMessage } from '@kbn/i18n-react';
import { useKibana } from '@kbn/kibana-react-plugin/public';
import { apmServiceInventoryOptimizedSorting } from '@kbn/observability-plugin/common';
import React from 'react';
import { v4 as uuidv4 } from 'uuid';
import { ApmDocumentType } from '../../../../common/document_type';
import { ServiceInventoryFieldName } from '../../../../common/service_inventory';
import { joinByKey } from '../../../../common/utils/join_by_key';
import { useAnomalyDetectionJobsContext } from '../../../context/anomaly_detection_jobs/use_anomaly_detection_jobs_context';
import { useApmParams } from '../../../hooks/use_apm_params';
import { FETCH_STATUS, useFetcher } from '../../../hooks/use_fetcher';
import { FETCH_STATUS, isPending } from '../../../hooks/use_fetcher';
import { useLocalStorage } from '../../../hooks/use_local_storage';
import { usePreferredDataSourceAndBucketSize } from '../../../hooks/use_preferred_data_source_and_bucket_size';
import { useProgressiveFetcher } from '../../../hooks/use_progressive_fetcher';
import { useTimeRange } from '../../../hooks/use_time_range';
import { MLCallout, shouldDisplayMlCallout } from '../../shared/ml_callout';
@ -58,26 +57,17 @@ function useServicesMainStatisticsFetcher() {
const { start, end } = useTimeRange({ rangeFrom, rangeTo });
const sortedAndFilteredServicesFetch = useFetcher(
(callApmApi) => {
return callApmApi('GET /internal/apm/sorted_and_filtered_services', {
params: {
query: {
start,
end,
environment,
kuery,
serviceGroup,
},
},
});
},
[start, end, environment, kuery, serviceGroup]
);
const dataSourceOptions = usePreferredDataSourceAndBucketSize({
rangeFrom,
rangeTo,
kuery,
type: ApmDocumentType.ServiceTransactionMetric,
numBuckets: 20,
});
const mainStatisticsFetch = useProgressiveFetcher(
(callApmApi) => {
if (start && end) {
if (start && end && dataSourceOptions) {
return callApmApi('GET /internal/apm/services', {
params: {
query: {
@ -86,6 +76,8 @@ function useServicesMainStatisticsFetcher() {
start,
end,
serviceGroup,
documentType: dataSourceOptions.source.documentType,
rollupInterval: dataSourceOptions.source.rollupInterval,
},
},
}).then((mainStatisticsData) => {
@ -103,6 +95,8 @@ function useServicesMainStatisticsFetcher() {
start,
end,
serviceGroup,
dataSourceOptions?.source.documentType,
dataSourceOptions?.source.rollupInterval,
// not used, but needed to update the requestId to call the details statistics API when table is options are updated
page,
pageSize,
@ -112,7 +106,6 @@ function useServicesMainStatisticsFetcher() {
);
return {
sortedAndFilteredServicesFetch,
mainStatisticsFetch,
};
}
@ -147,6 +140,14 @@ function useServicesDetailedStatisticsFetcher({
const { start, end } = useTimeRange({ rangeFrom, rangeTo });
const dataSourceOptions = usePreferredDataSourceAndBucketSize({
rangeFrom,
rangeTo,
kuery,
type: ApmDocumentType.ServiceTransactionMetric,
numBuckets: 20,
});
const { data: mainStatisticsData = initialData } = mainStatisticsFetch;
const currentPageItems = orderServiceItems({
@ -162,7 +163,8 @@ function useServicesDetailedStatisticsFetcher({
start &&
end &&
currentPageItems.length &&
mainStatisticsFetch.status === FETCH_STATUS.SUCCESS
mainStatisticsFetch.status === FETCH_STATUS.SUCCESS &&
dataSourceOptions
) {
return callApmApi('POST /internal/apm/services/detailed_statistics', {
params: {
@ -175,6 +177,9 @@ function useServicesDetailedStatisticsFetcher({
comparisonEnabled && isTimeComparison(offset)
? offset
: undefined,
documentType: dataSourceOptions.source.documentType,
rollupInterval: dataSourceOptions.source.rollupInterval,
bucketSizeInSeconds: dataSourceOptions.bucketSizeInSeconds,
},
body: {
serviceNames: JSON.stringify(
@ -198,16 +203,13 @@ function useServicesDetailedStatisticsFetcher({
}
export function ServiceInventory() {
const { sortedAndFilteredServicesFetch, mainStatisticsFetch } =
useServicesMainStatisticsFetcher();
const { mainStatisticsFetch } = useServicesMainStatisticsFetcher();
const mainStatisticsItems = mainStatisticsFetch.data?.items ?? [];
const preloadedServices = sortedAndFilteredServicesFetch.data?.services || [];
const displayHealthStatus = [
...mainStatisticsItems,
...preloadedServices,
].some((item) => 'healthStatus' in item);
const displayHealthStatus = mainStatisticsItems.some(
(item) => 'healthStatus' in item
);
const hasKibanaUiLimitRestrictedData =
mainStatisticsFetch.data?.maxServiceCountExceeded;
@ -215,25 +217,17 @@ export function ServiceInventory() {
const serviceOverflowCount =
mainStatisticsFetch.data?.serviceOverflowCount ?? 0;
const displayAlerts = [...mainStatisticsItems, ...preloadedServices].some(
const displayAlerts = mainStatisticsItems.some(
(item) => ServiceInventoryFieldName.AlertsCount in item
);
const useOptimizedSorting =
useKibana().services.uiSettings?.get<boolean>(
apmServiceInventoryOptimizedSorting
) || false;
const tiebreakerField = useOptimizedSorting
? ServiceInventoryFieldName.ServiceName
: ServiceInventoryFieldName.Throughput;
const tiebreakerField = ServiceInventoryFieldName.Throughput;
const initialSortField = displayHealthStatus
? ServiceInventoryFieldName.HealthStatus
: tiebreakerField;
const initialSortDirection =
initialSortField === ServiceInventoryFieldName.ServiceName ? 'asc' : 'desc';
const initialSortDirection = 'desc';
const { comparisonFetch } = useServicesDetailedStatisticsFetcher({
mainStatisticsFetch,
@ -253,18 +247,7 @@ export function ServiceInventory() {
!userHasDismissedCallout &&
shouldDisplayMlCallout(anomalyDetectionSetupState);
let isLoading: boolean;
if (useOptimizedSorting) {
isLoading =
// ensures table is usable when sorted and filtered services have loaded
sortedAndFilteredServicesFetch.status === FETCH_STATUS.LOADING ||
(sortedAndFilteredServicesFetch.status === FETCH_STATUS.SUCCESS &&
sortedAndFilteredServicesFetch.data?.services.length === 0 &&
mainStatisticsFetch.status === FETCH_STATUS.LOADING);
} else {
isLoading = mainStatisticsFetch.status === FETCH_STATUS.LOADING;
}
const isLoading = isPending(mainStatisticsFetch.status);
const isFailure = mainStatisticsFetch.status === FETCH_STATUS.FAILURE;
const noItemsMessage = (
@ -280,18 +263,7 @@ export function ServiceInventory() {
/>
);
const items = joinByKey(
[
// only use preloaded services if tiebreaker field is service.name,
// otherwise ignore them to prevent re-sorting of the table
// once the tiebreaking metric comes in
...(tiebreakerField === ServiceInventoryFieldName.ServiceName
? preloadedServices
: []),
...mainStatisticsItems,
],
'serviceName'
);
const items = mainStatisticsItems;
const mlCallout = (
<EuiFlexItem>

4
x-pack/plugins/apm/public/context/time_range_metadata/time_range_metadata_context.tsx
View file

@ -38,8 +38,8 @@ export function TimeRangeMetadataContextProvider({
const routePath = useApmRoutePath();
const isOperationView =
routePath === '/dependencies/operation' ||
routePath === '/dependencies/operations';
routePath.startsWith('/dependencies/operation') ||
routePath.startsWith('/dependencies/operations');
const fetcherResult = useFetcher(
(callApmApi) => {

86
x-pack/plugins/apm/public/hooks/use_preferred_data_source_and_bucket_size.ts Normal file
View file

@ -0,0 +1,86 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { ApmDataSource } from '../../common/data_source';
import { ApmDocumentType } from '../../common/document_type';
import { getBucketSize } from '../../common/utils/get_bucket_size';
import { getPreferredBucketSizeAndDataSource } from '../../common/utils/get_preferred_bucket_size_and_data_source';
import { useTimeRangeMetadata } from '../context/time_range_metadata/use_time_range_metadata_context';
import { useTimeRange } from './use_time_range';
export function usePreferredDataSourceAndBucketSize<
TDocumentType extends
| ApmDocumentType.ServiceTransactionMetric
| ApmDocumentType.TransactionMetric
>({
rangeFrom,
rangeTo,
kuery,
numBuckets,
type,
}: {
rangeFrom: string;
rangeTo: string;
kuery: string;
numBuckets: 20 | 100;
type: TDocumentType;
}): {
bucketSizeInSeconds: number;
source: ApmDataSource<
TDocumentType extends ApmDocumentType.ServiceTransactionMetric
?
| ApmDocumentType.ServiceTransactionMetric
| ApmDocumentType.TransactionMetric
| ApmDocumentType.TransactionMetric
: ApmDocumentType.TransactionMetric | ApmDocumentType.TransactionEvent
>;
} | null {
const timeRangeMetadataFetch = useTimeRangeMetadata({
rangeFrom,
rangeTo,
kuery,
});
const { start, end } = useTimeRange({ rangeFrom, rangeTo });
const sources = timeRangeMetadataFetch.data?.sources;
if (!sources) {
return null;
}
let suitableTypes: ApmDocumentType[];
if (type === ApmDocumentType.ServiceTransactionMetric) {
suitableTypes = [
ApmDocumentType.ServiceTransactionMetric,
ApmDocumentType.TransactionMetric,
ApmDocumentType.TransactionEvent,
];
} else if (type === ApmDocumentType.TransactionMetric) {
suitableTypes = [
ApmDocumentType.TransactionMetric,
ApmDocumentType.TransactionEvent,
];
}
const { bucketSizeInSeconds, source } = getPreferredBucketSizeAndDataSource({
bucketSizeInSeconds: getBucketSize({
numBuckets,
start: new Date(start).getTime(),
end: new Date(end).getTime(),
}).bucketSize,
sources: sources.filter(
(s) => s.hasDocs && suitableTypes.includes(s.documentType)
),
});
return {
bucketSizeInSeconds,
source: source as ApmDataSource<any>,
};
}

2
x-pack/plugins/apm/server/lib/anomaly_detection/get_anomaly_result_bucket_size.ts
View file

@ -5,7 +5,7 @@
* 2.0.
*/
import { getBucketSize } from '../helpers/get_bucket_size';
import { getBucketSize } from '../../../common/utils/get_bucket_size';
export function getAnomalyResultBucketSize({
start,

2
x-pack/plugins/apm/server/lib/connections/get_connection_stats/get_stats.ts
View file

@ -25,7 +25,7 @@ import {
SPAN_SUBTYPE,
SPAN_TYPE,
} from '../../../../common/es_fields/apm';
import { getBucketSize } from '../../helpers/get_bucket_size';
import { getBucketSize } from '../../../../common/utils/get_bucket_size';
import { EventOutcome } from '../../../../common/event_outcome';
import { NodeType } from '../../../../common/connections';
import { excludeRumExitSpansQuery } from '../exclude_rum_exit_spans_query';

18
x-pack/plugins/apm/server/lib/helpers/create_es_client/create_apm_event_client/unpack_processor_events.test.ts → x-pack/plugins/apm/server/lib/helpers/create_es_client/create_apm_event_client/get_request_base.test.ts
View file

@ -7,17 +7,16 @@
import { APMEventESSearchRequest } from '.';
import { ApmIndicesConfig } from '../../../../routes/settings/apm_indices/get_apm_indices';
import { unpackProcessorEvents } from './unpack_processor_events';
import { getRequestBase } from './get_request_base';
describe('unpackProcessorEvents', () => {
let res: ReturnType<typeof unpackProcessorEvents>;
describe('getRequestBase', () => {
let res: ReturnType<typeof getRequestBase>;
beforeEach(() => {
const request = {
apm: { events: ['transaction', 'error'] },
body: {
track_total_hits: false,
size: 0,
query: { bool: { filter: [{ terms: { foo: 'bar' } }] } },
},
} as APMEventESSearchRequest;
@ -29,22 +28,15 @@ describe('unpackProcessorEvents', () => {
onboarding: 'my-apm-*-onboarding-*',
} as ApmIndicesConfig;
res = unpackProcessorEvents(request, indices);
res = getRequestBase({ ...request, indices });
});
it('adds terms filter for apm events', () => {
expect(res.body.query.bool.filter).toContainEqual({
expect(res.filters).toContainEqual({
terms: { 'processor.event': ['transaction', 'error'] },
});
});
it('merges queries', () => {
expect(res.body.query.bool.filter).toEqual([
{ terms: { foo: 'bar' } },
{ terms: { 'processor.event': ['transaction', 'error'] } },
]);
});
it('searches the specified indices', () => {
expect(res.index).toEqual(['my-apm-*-transaction-*', 'my-apm-*-error-*']);
});

69
x-pack/plugins/apm/server/lib/helpers/create_es_client/create_apm_event_client/get_request_base.ts Normal file
View file

@ -0,0 +1,69 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { ESFilter } from '@kbn/es-types';
import { ProcessorEvent } from '@kbn/observability-plugin/common';
import { uniq } from 'lodash';
import { ApmDataSource } from '../../../../../common/data_source';
import {} from '../../../../../common/document_type';
import { PROCESSOR_EVENT } from '../../../../../common/es_fields/apm';
import { ApmIndicesConfig } from '../../../../routes/settings/apm_indices/get_apm_indices';
import {
getConfigForDocumentType,
getProcessorEventForDocumentType,
} from '../document_type';
const processorEventIndexMap = {
[ProcessorEvent.transaction]: 'transaction',
[ProcessorEvent.span]: 'span',
[ProcessorEvent.metric]: 'metric',
[ProcessorEvent.error]: 'error',
} as const;
export function processorEventsToIndex(
events: ProcessorEvent[],
indices: ApmIndicesConfig
) {
return uniq(events.map((event) => indices[processorEventIndexMap[event]]));
}
export function getRequestBase(options: {
apm: { events: ProcessorEvent[] } | { sources: ApmDataSource[] };
indices: ApmIndicesConfig;
}) {
const events =
'events' in options.apm
? options.apm.events
: options.apm.sources.map((source) =>
getProcessorEventForDocumentType(source.documentType)
);
const index = processorEventsToIndex(events, options.indices);
const filters: ESFilter[] = [
{
terms: {
[PROCESSOR_EVENT]: events,
},
},
];
if ('sources' in options.apm) {
options.apm.sources.forEach((source) => {
const { getQuery } = getConfigForDocumentType(source.documentType);
if (getQuery) {
filters.push(getQuery(source.rollupInterval));
}
});
}
return {
index,
events,
filters,
};
}

50
x-pack/plugins/apm/server/lib/helpers/create_es_client/create_apm_event_client/index.ts
View file

@ -15,31 +15,29 @@ import type {
import { ValuesType } from 'utility-types';
import { ElasticsearchClient, KibanaRequest } from '@kbn/core/server';
import type { ESSearchRequest, InferSearchResponseOf } from '@kbn/es-types';
import { unwrapEsResponse } from '@kbn/observability-plugin/server';
import { omit } from 'lodash';
import { ProcessorEvent } from '@kbn/observability-plugin/common';
import { withApmSpan } from '../../../../utils/with_apm_span';
import { unwrapEsResponse } from '@kbn/observability-plugin/server';
import { compact, omit } from 'lodash';
import { ApmDataSource } from '../../../../../common/data_source';
import { APMError } from '../../../../../typings/es_schemas/ui/apm_error';
import { Metric } from '../../../../../typings/es_schemas/ui/metric';
import { Span } from '../../../../../typings/es_schemas/ui/span';
import { Transaction } from '../../../../../typings/es_schemas/ui/transaction';
import { ApmIndicesConfig } from '../../../../routes/settings/apm_indices/get_apm_indices';
import { withApmSpan } from '../../../../utils/with_apm_span';
import {
callAsyncWithDebug,
getDebugBody,
getDebugTitle,
} from '../call_async_with_debug';
import { cancelEsRequestOnAbort } from '../cancel_es_request_on_abort';
import {
unpackProcessorEvents,
processorEventsToIndex,
} from './unpack_processor_events';
import { processorEventsToIndex, getRequestBase } from './get_request_base';
import { ProcessorEventOfDocumentType } from '../document_type';
export type APMEventESSearchRequest = Omit<ESSearchRequest, 'index'> & {
apm: {
events: ProcessorEvent[];
includeLegacyData?: boolean;
};
} & ({ events: ProcessorEvent[] } | { sources: ApmDataSource[] });
body: {
size: number;
track_total_hits: boolean | number;
@ -69,7 +67,17 @@ type TypeOfProcessorEvent<T extends ProcessorEvent> = {
type TypedSearchResponse<TParams extends APMEventESSearchRequest> =
InferSearchResponseOf<
TypeOfProcessorEvent<ValuesType<TParams['apm']['events']>>,
TypeOfProcessorEvent<
ValuesType<
TParams['apm'] extends { events: ProcessorEvent[] }
? TParams['apm']['events']
: TParams['apm'] extends { sources: ApmDataSource[] }
? ProcessorEventOfDocumentType<
ValuesType<TParams['apm']['sources']>['documentType']
>
: never
>
>,
TParams
>;
@ -146,17 +154,25 @@ export class APMEventClient {
operationName: string,
params: TParams
): Promise<TypedSearchResponse<TParams>> {
const withProcessorEventFilter = unpackProcessorEvents(
params,
this.indices
);
const { events, index, filters } = getRequestBase({
apm: params.apm,
indices: this.indices,
});
const forceSyntheticSourceForThisRequest =
this.forceSyntheticSource &&
params.apm.events.includes(ProcessorEvent.metric);
this.forceSyntheticSource && events.includes(ProcessorEvent.metric);
const searchParams = {
...withProcessorEventFilter,
...omit(params, 'apm'),
index,
body: {
...params.body,
query: {
bool: {
filter: compact([params.body.query, ...filters]),
},
},
},
...(this.includeFrozen ? { ignore_throttled: false } : {}),
ignore_unavailable: true,
preference: 'any',

62
x-pack/plugins/apm/server/lib/helpers/create_es_client/create_apm_event_client/unpack_processor_events.ts
View file

@ -1,62 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { uniq, defaultsDeep, cloneDeep } from 'lodash';
import type { ESSearchRequest, ESFilter } from '@kbn/es-types';
import { ProcessorEvent } from '@kbn/observability-plugin/common';
import { PROCESSOR_EVENT } from '../../../../../common/es_fields/apm';
import { ApmIndicesConfig } from '../../../../routes/settings/apm_indices/get_apm_indices';
const processorEventIndexMap = {
[ProcessorEvent.transaction]: 'transaction',
[ProcessorEvent.span]: 'span',
[ProcessorEvent.metric]: 'metric',
[ProcessorEvent.error]: 'error',
} as const;
export function processorEventsToIndex(
events: ProcessorEvent[],
indices: ApmIndicesConfig
) {
return uniq(events.map((event) => indices[processorEventIndexMap[event]]));
}
export function unpackProcessorEvents(
request: {
apm: {
events: ProcessorEvent[];
};
},
indices: ApmIndicesConfig
) {
const { apm, ...params } = request;
const events = uniq(apm.events);
const index = processorEventsToIndex(events, indices);
const withFilterForProcessorEvent: ESSearchRequest & {
body: { query: { bool: { filter: ESFilter[] } } };
} = defaultsDeep(cloneDeep(params), {
body: {
query: {
bool: {
filter: [],
},
},
},
});
withFilterForProcessorEvent.body.query.bool.filter.push({
terms: {
[PROCESSOR_EVENT]: events,
},
});
return {
index,
...withFilterForProcessorEvent,
};
}

103
x-pack/plugins/apm/server/lib/helpers/create_es_client/document_type.ts Normal file
View file

@ -0,0 +1,103 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types';
import { ProcessorEvent } from '@kbn/observability-plugin/common';
import { ApmDocumentType } from '../../../../common/document_type';
import {
METRICSET_INTERVAL,
METRICSET_NAME,
} from '../../../../common/es_fields/apm';
import { RollupInterval } from '../../../../common/rollup';
import { termQuery } from '../../../../common/utils/term_query';
import { getDocumentTypeFilterForServiceDestinationStatistics } from '../spans/get_is_using_service_destination_metrics';
import { getDocumentTypeFilterForTransactions } from '../transactions';
const defaultRollupIntervals = [
RollupInterval.OneMinute,
RollupInterval.TenMinutes,
RollupInterval.SixtyMinutes,
];
function getDefaultFilter(
metricsetName: string,
rollupInterval: RollupInterval
) {
return [
...termQuery(METRICSET_NAME, metricsetName),
...termQuery(METRICSET_INTERVAL, rollupInterval),
];
}
const documentTypeConfigMap: Record<
ApmDocumentType,
{
processorEvent: ProcessorEvent;
getQuery?: (rollupInterval: RollupInterval) => QueryDslQueryContainer;
rollupIntervals: RollupInterval[];
}
> = {
[ApmDocumentType.ServiceTransactionMetric]: {
processorEvent: ProcessorEvent.metric,
getQuery: (rollupInterval) => ({
bool: {
filter: getDefaultFilter('service_transaction', rollupInterval),
},
}),
rollupIntervals: defaultRollupIntervals,
},
[ApmDocumentType.TransactionMetric]: {
processorEvent: ProcessorEvent.metric,
getQuery: (rollupInterval) => ({
bool: {
filter:
rollupInterval === RollupInterval.OneMinute
? getDocumentTypeFilterForTransactions(true)
: getDefaultFilter('transaction', rollupInterval),
},
}),
rollupIntervals: defaultRollupIntervals,
},
[ApmDocumentType.TransactionEvent]: {
processorEvent: ProcessorEvent.transaction,
rollupIntervals: [RollupInterval.None],
},
[ApmDocumentType.ServiceDestinationMetric]: {
processorEvent: ProcessorEvent.metric,
rollupIntervals: defaultRollupIntervals,
getQuery: (rollupInterval) => ({
bool: {
filter:
rollupInterval === RollupInterval.OneMinute
? getDocumentTypeFilterForServiceDestinationStatistics(true)
: getDefaultFilter('service_destination', rollupInterval),
},
}),
},
};
type DocumentTypeConfigOf<TApmDocumentType extends ApmDocumentType> =
typeof documentTypeConfigMap[TApmDocumentType];
export function getConfigForDocumentType<
TApmDocumentType extends ApmDocumentType
>(docType: TApmDocumentType): DocumentTypeConfigOf<TApmDocumentType> {
return documentTypeConfigMap[docType];
}
export type ProcessorEventOfDocumentType<
TApmDocumentType extends ApmDocumentType
> = DocumentTypeConfigOf<TApmDocumentType>['processorEvent'];
export function getProcessorEventForDocumentType<
TApmDocumentType extends ApmDocumentType
>(
documentType: TApmDocumentType
): ProcessorEventOfDocumentType<TApmDocumentType> {
return getConfigForDocumentType(documentType).processorEvent;
}

2
x-pack/plugins/apm/server/lib/helpers/get_bucket_size_for_aggregated_transactions/index.ts
View file

@ -5,7 +5,7 @@
* 2.0.
*/
import { getBucketSize } from '../get_bucket_size';
import { getBucketSize } from '../../../../common/utils/get_bucket_size';
export function getBucketSizeForAggregatedTransactions({
start,

79
x-pack/plugins/apm/server/lib/helpers/get_document_sources.ts Normal file
View file

@ -0,0 +1,79 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { kqlQuery, rangeQuery } from '@kbn/observability-plugin/server';
import { flatten } from 'lodash';
import { ApmDataSource } from '../../../common/data_source';
import { ApmDocumentType } from '../../../common/document_type';
import { RollupInterval } from '../../../common/rollup';
import { APMEventClient } from './create_es_client/create_apm_event_client';
import { getConfigForDocumentType } from './create_es_client/document_type';
export async function getDocumentSources({
apmEventClient,
start,
end,
kuery,
}: {
apmEventClient: APMEventClient;
start: number;
end: number;
kuery: string;
}) {
const sources: Array<ApmDataSource & { hasDocs: boolean }> = flatten(
await Promise.all(
[
ApmDocumentType.ServiceTransactionMetric as const,
ApmDocumentType.TransactionMetric as const,
].map(async (documentType) => {
const docTypeConfig = getConfigForDocumentType(documentType);
const allHasDocs = await Promise.all(
docTypeConfig.rollupIntervals.map(async (rollupInterval) => {
const response = await apmEventClient.search(
'check_document_type_availability',
{
apm: {
sources: [
{
documentType,
rollupInterval,
},
],
},
body: {
track_total_hits: 1,
size: 0,
terminate_after: 1,
query: {
bool: {
filter: [...kqlQuery(kuery), ...rangeQuery(start, end)],
},
},
},
}
);
return {
documentType,
rollupInterval,
hasDocs: response.hits.total.value > 0,
};
})
);
return allHasDocs;
})
)
);
sources.push({
documentType: ApmDocumentType.TransactionEvent,
rollupInterval: RollupInterval.None,
hasDocs: true,
});
return sources;
}

49
x-pack/plugins/apm/server/lib/helpers/get_service_inventory_search_source.ts
View file

@ -1,49 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { APMEventClient } from './create_es_client/create_apm_event_client';
import { getSearchTransactionsEvents } from './transactions';
import { getSearchServiceMetrics } from './service_metrics';
import { APMConfig } from '../..';
export async function getServiceInventorySearchSource({
config,
serviceMetricsEnabled,
apmEventClient,
start,
end,
kuery,
}: {
serviceMetricsEnabled: boolean;
config: APMConfig;
apmEventClient: APMEventClient;
start: number;
end: number;
kuery: string;
}): Promise<{
searchAggregatedTransactions: boolean;
searchAggregatedServiceMetrics: boolean;
}> {
const commonProps = {
apmEventClient,
kuery,
start,
end,
};
const [searchAggregatedTransactions, searchAggregatedServiceMetrics] =
await Promise.all([
getSearchTransactionsEvents({ ...commonProps, config }),
getSearchServiceMetrics({
...commonProps,
serviceMetricsEnabled,
}),
]);
return {
searchAggregatedTransactions,
searchAggregatedServiceMetrics,
};
}

2
x-pack/plugins/apm/server/lib/helpers/metrics.ts
View file

@ -5,7 +5,7 @@
* 2.0.
*/
import { getBucketSize } from './get_bucket_size';
import { getBucketSize } from '../../../common/utils/get_bucket_size';
export function getMetricsDateHistogramParams({
start,

83
x-pack/plugins/apm/server/lib/helpers/service_metrics/index.ts
View file

@ -1,83 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { kqlQuery, rangeQuery } from '@kbn/observability-plugin/server';
import { ProcessorEvent } from '@kbn/observability-plugin/common';
import { METRICSET_NAME } from '../../../../common/es_fields/apm';
import { APMEventClient } from '../create_es_client/create_apm_event_client';
export async function getSearchServiceMetrics({
serviceMetricsEnabled,
start,
end,
apmEventClient,
kuery,
}: {
serviceMetricsEnabled: boolean;
start?: number;
end?: number;
apmEventClient: APMEventClient;
kuery: string;
}): Promise<boolean> {
if (serviceMetricsEnabled) {
return getHasServicesMetrics({
start,
end,
apmEventClient,
kuery,
});
}
return false;
}
export async function getHasServicesMetrics({
start,
end,
apmEventClient,
kuery,
}: {
start?: number;
end?: number;
apmEventClient: APMEventClient;
kuery: string;
}) {
const response = await apmEventClient.search(
'get_has_aggregated_service_metrics',
{
apm: {
events: [ProcessorEvent.metric],
},
body: {
track_total_hits: 1,
size: 0,
query: {
bool: {
filter: [
...getDocumentTypeFilterForServiceMetrics(),
...(start && end ? rangeQuery(start, end) : []),
...kqlQuery(kuery),
],
},
},
},
terminate_after: 1,
}
);
return response.hits.total.value > 0;
}
export function getDocumentTypeFilterForServiceMetrics() {
return [
{
term: {
[METRICSET_NAME]: 'service_transaction',
},
},
];
}

54
x-pack/plugins/apm/server/lib/helpers/transaction_error_rate.test.ts
View file

@ -1,54 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { calculateFailedTransactionRateFromServiceMetrics } from './transaction_error_rate';
describe('calculateFailedTransactionRateFromServiceMetrics', () => {
it('should return 0 when all params are null', () => {
expect(
calculateFailedTransactionRateFromServiceMetrics({
failedTransactions: null,
successfulTransactions: null,
})
).toBe(0);
});
it('should return 0 when failedTransactions:null', () => {
expect(
calculateFailedTransactionRateFromServiceMetrics({
failedTransactions: null,
successfulTransactions: 2,
})
).toBe(0);
});
it('should return 0 when failedTransactions:0', () => {
expect(
calculateFailedTransactionRateFromServiceMetrics({
failedTransactions: 0,
successfulTransactions: null,
})
).toBe(0);
});
it('should return 1 when failedTransactions:10 and successfulTransactions:0', () => {
expect(
calculateFailedTransactionRateFromServiceMetrics({
failedTransactions: 10,
successfulTransactions: 0,
})
).toBe(1);
});
it('should return 0,5 when failedTransactions:10 and successfulTransactions:10', () => {
expect(
calculateFailedTransactionRateFromServiceMetrics({
failedTransactions: 10,
successfulTransactions: 10,
})
).toBe(0.5);
});
});

118
x-pack/plugins/apm/server/lib/helpers/transaction_error_rate.ts
View file

@ -5,72 +5,104 @@
* 2.0.
*/
import {
AggregationsSumAggregation,
AggregationsValueCountAggregation,
QueryDslQueryContainer,
} from '@elastic/elasticsearch/lib/api/types';
import type {
AggregationOptionsByType,
AggregationResultOf,
AggregationResultOfMap,
} from '@kbn/es-types';
import { isNull } from 'lodash';
import { EVENT_OUTCOME } from '../../../common/es_fields/apm';
import { ApmDocumentType } from '../../../common/document_type';
import {
EVENT_OUTCOME,
EVENT_SUCCESS_COUNT,
} from '../../../common/es_fields/apm';
import { EventOutcome } from '../../../common/event_outcome';
export const getOutcomeAggregation = () => {
export const getOutcomeAggregation = (
documentType: ApmDocumentType
): {
successful_or_failed:
| { value_count: AggregationsValueCountAggregation }
| { filter: QueryDslQueryContainer };
successful:
| { sum: AggregationsSumAggregation }
| { filter: QueryDslQueryContainer };
} => {
if (documentType === ApmDocumentType.ServiceTransactionMetric) {
return {
successful_or_failed: {
value_count: {
field: EVENT_SUCCESS_COUNT,
},
},
successful: {
sum: {
field: EVENT_SUCCESS_COUNT,
},
},
};
}
return {
terms: {
field: EVENT_OUTCOME,
include: [EventOutcome.failure, EventOutcome.success],
successful_or_failed: {
filter: {
bool: {
filter: [
{
terms: {
[EVENT_OUTCOME]: [EventOutcome.failure, EventOutcome.success],
},
},
],
},
},
},
successful: {
filter: {
bool: {
filter: [
{
terms: {
[EVENT_OUTCOME]: [EventOutcome.success],
},
},
],
},
},
},
};
};
type OutcomeAggregation = ReturnType<typeof getOutcomeAggregation>;
export const getTimeseriesAggregation = (
start: number,
end: number,
intervalString: string
) => ({
date_histogram: {
field: '@timestamp',
fixed_interval: intervalString,
min_doc_count: 0,
extended_bounds: { min: start, max: end },
},
aggs: { outcomes: getOutcomeAggregation() },
});
export function calculateFailedTransactionRate(
outcomeResponse: AggregationResultOf<OutcomeAggregation, {}>
outcomeResponse: AggregationResultOfMap<OutcomeAggregation, {}>
) {
const outcomes = Object.fromEntries(
outcomeResponse.buckets.map(({ key, doc_count: count }) => [key, count])
);
const successfulTransactions =
'value' in outcomeResponse.successful
? outcomeResponse.successful.value ?? 0
: outcomeResponse.successful.doc_count;
const failedTransactions = outcomes[EventOutcome.failure] ?? 0;
const successfulTransactions = outcomes[EventOutcome.success] ?? 0;
const successfulOrFailedTransactions =
'value' in outcomeResponse.successful_or_failed
? outcomeResponse.successful_or_failed.value
: outcomeResponse.successful_or_failed.doc_count;
return failedTransactions / (successfulTransactions + failedTransactions);
}
const failedTransactions =
successfulOrFailedTransactions - successfulTransactions;
export function calculateFailedTransactionRateFromServiceMetrics({
failedTransactions,
successfulTransactions,
}: {
failedTransactions: number | null;
successfulTransactions: number | null;
}) {
if (isNull(failedTransactions) || failedTransactions === 0) {
return 0;
}
successfulTransactions = successfulTransactions ?? 0;
return failedTransactions / (successfulTransactions + failedTransactions);
return failedTransactions / successfulOrFailedTransactions;
}
export function getFailedTransactionRateTimeSeries(
buckets: AggregationResultOf<
{
date_histogram: AggregationOptionsByType['date_histogram'];
aggs: { outcomes: OutcomeAggregation };
aggs: OutcomeAggregation;
},
{}
>['buckets']
@ -78,7 +110,7 @@ export function getFailedTransactionRateTimeSeries(
return buckets.map((dateBucket) => {
return {
x: dateBucket.key,
y: calculateFailedTransactionRate(dateBucket.outcomes),
y: calculateFailedTransactionRate(dateBucket),
};
});
}

25
x-pack/plugins/apm/server/lib/helpers/transactions/index.ts
View file

@ -15,9 +15,11 @@ import {
PARENT_ID,
METRICSET_INTERVAL,
METRICSET_NAME,
TRANSACTION_DURATION_SUMMARY,
} from '../../../../common/es_fields/apm';
import { APMConfig } from '../../..';
import { APMEventClient } from '../create_es_client/create_apm_event_client';
import { ApmDocumentType } from '../../../../common/document_type';
export async function getHasTransactionsEvents({
start,
@ -89,11 +91,26 @@ export async function getSearchTransactionsEvents({
}
export function getDurationFieldForTransactions(
searchAggregatedTransactions: boolean
typeOrSearchAgggregatedTransactions: ApmDocumentType | boolean
) {
return searchAggregatedTransactions
? TRANSACTION_DURATION_HISTOGRAM
: TRANSACTION_DURATION;
let type: ApmDocumentType;
if (typeOrSearchAgggregatedTransactions === true) {
type = ApmDocumentType.TransactionMetric;
} else if (typeOrSearchAgggregatedTransactions === false) {
type = ApmDocumentType.TransactionEvent;
} else {
type = typeOrSearchAgggregatedTransactions;
}
if (type === ApmDocumentType.ServiceTransactionMetric) {
return TRANSACTION_DURATION_SUMMARY;
}
if (type === ApmDocumentType.TransactionMetric) {
return TRANSACTION_DURATION_HISTOGRAM;
}
return TRANSACTION_DURATION;
}
export function getDocumentTypeFilterForTransactions(

13
x-pack/plugins/apm/server/lib/transaction_groups/get_failed_transaction_rate.ts
View file

@ -31,6 +31,7 @@ import {
} from '../helpers/transaction_error_rate';
import { getOffsetInMs } from '../../../common/utils/get_offset_in_ms';
import { APMEventClient } from '../helpers/create_es_client/create_apm_event_client';
import { ApmDocumentType } from '../../../common/document_type';
export async function getFailedTransactionRate({
environment,
@ -81,7 +82,11 @@ export async function getFailedTransactionRate({
...kqlQuery(kuery),
];
const outcomes = getOutcomeAggregation();
const outcomes = getOutcomeAggregation(
searchAggregatedTransactions
? ApmDocumentType.TransactionMetric
: ApmDocumentType.TransactionEvent
);
const params = {
apm: {
@ -92,7 +97,7 @@ export async function getFailedTransactionRate({
size: 0,
query: { bool: { filter } },
aggs: {
outcomes,
...outcomes,
timeseries: {
date_histogram: {
field: '@timestamp',
@ -106,7 +111,7 @@ export async function getFailedTransactionRate({
extended_bounds: { min: startWithOffset, max: endWithOffset },
},
aggs: {
outcomes,
...outcomes,
},
},
},
@ -124,7 +129,7 @@ export async function getFailedTransactionRate({
const timeseries = getFailedTransactionRateTimeSeries(
resp.aggregations.timeseries.buckets
);
const average = calculateFailedTransactionRate(resp.aggregations.outcomes);
const average = calculateFailedTransactionRate(resp.aggregations);
return { timeseries, average };
}

12
x-pack/plugins/apm/server/routes/alerts/rule_types/transaction_error_rate/get_transaction_error_rate_chart_preview.ts
View file

@ -23,6 +23,7 @@ import {
} from '../../../../lib/helpers/transaction_error_rate';
import { APMConfig } from '../../../..';
import { APMEventClient } from '../../../../lib/helpers/create_es_client/create_apm_event_client';
import { ApmDocumentType } from '../../../../../common/document_type';
export async function getTransactionErrorRateChartPreview({
config,
@ -44,8 +45,6 @@ export async function getTransactionErrorRateChartPreview({
end,
});
const outcomes = getOutcomeAggregation();
const params = {
apm: {
events: [getProcessorEventForTransactions(searchAggregatedTransactions)],
@ -67,7 +66,6 @@ export async function getTransactionErrorRateChartPreview({
},
},
aggs: {
outcomes,
timeseries: {
date_histogram: {
field: '@timestamp',
@ -77,7 +75,11 @@ export async function getTransactionErrorRateChartPreview({
max: end,
},
},
aggs: { outcomes },
aggs: getOutcomeAggregation(
searchAggregatedTransactions
? ApmDocumentType.TransactionMetric
: ApmDocumentType.TransactionEvent
),
},
},
},
@ -95,7 +97,7 @@ export async function getTransactionErrorRateChartPreview({
return resp.aggregations.timeseries.buckets.map((bucket) => {
return {
x: bucket.key,
y: calculateFailedTransactionRate(bucket.outcomes),
y: calculateFailedTransactionRate(bucket),
};
});
}

16
x-pack/plugins/apm/server/routes/default_api_types.ts
View file

@ -7,6 +7,8 @@
import * as t from 'io-ts';
import { isoToEpochRt, toNumberRt } from '@kbn/io-ts-utils';
import { ApmDocumentType } from '../../common/document_type';
import { RollupInterval } from '../../common/rollup';
export { environmentRt } from '../../common/environment_rt';
@ -19,3 +21,17 @@ export const probabilityRt = t.type({
probability: toNumberRt,
});
export const kueryRt = t.type({ kuery: t.string });
export const dataSourceRt = t.type({
documentType: t.union([
t.literal(ApmDocumentType.ServiceTransactionMetric),
t.literal(ApmDocumentType.TransactionMetric),
t.literal(ApmDocumentType.TransactionEvent),
]),
rollupInterval: t.union([
t.literal(RollupInterval.OneMinute),
t.literal(RollupInterval.TenMinutes),
t.literal(RollupInterval.SixtyMinutes),
t.literal(RollupInterval.None),
]),
});

2
x-pack/plugins/apm/server/routes/dependencies/get_throughput_charts_for_dependency.ts
View file

@ -16,7 +16,7 @@ import {
} from '../../../common/es_fields/apm';
import { environmentQuery } from '../../../common/utils/environment_query';
import { getOffsetInMs } from '../../../common/utils/get_offset_in_ms';
import { getBucketSize } from '../../lib/helpers/get_bucket_size';
import { getBucketSize } from '../../../common/utils/get_bucket_size';
import {
getDocCountFieldForServiceDestinationStatistics,
getDocumentTypeFilterForServiceDestinationStatistics,

2
x-pack/plugins/apm/server/routes/errors/erroneous_transactions/get_top_erroneous_transactions.ts
View file

@ -26,7 +26,7 @@ import {
TRANSACTION_TYPE,
} from '../../../../common/es_fields/apm';
import { environmentQuery } from '../../../../common/utils/environment_query';
import { getBucketSize } from '../../../lib/helpers/get_bucket_size';
import { getBucketSize } from '../../../../common/utils/get_bucket_size';
import { getOffsetInMs } from '../../../../common/utils/get_offset_in_ms';
import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client';

2
x-pack/plugins/apm/server/routes/errors/get_error_groups/get_error_group_detailed_statistics.ts
View file

@ -16,7 +16,7 @@ import { offsetPreviousPeriodCoordinates } from '../../../../common/utils/offset
import { Coordinate } from '../../../../typings/timeseries';
import { ERROR_GROUP_ID, SERVICE_NAME } from '../../../../common/es_fields/apm';
import { environmentQuery } from '../../../../common/utils/environment_query';
import { getBucketSize } from '../../../lib/helpers/get_bucket_size';
import { getBucketSize } from '../../../../common/utils/get_bucket_size';
import { getOffsetInMs } from '../../../../common/utils/get_offset_in_ms';
import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client';

2
x-pack/plugins/apm/server/routes/metrics/by_agent/java/gc/fetch_and_transform_gc_metrics.ts
View file

@ -20,7 +20,7 @@ import {
METRIC_JAVA_GC_TIME,
SERVICE_NAME,
} from '../../../../../../common/es_fields/apm';
import { getBucketSize } from '../../../../../lib/helpers/get_bucket_size';
import { getBucketSize } from '../../../../../../common/utils/get_bucket_size';
import { getVizColorForIndex } from '../../../../../../common/viz_colors';
import { JAVA_AGENT_NAMES } from '../../../../../../common/agent_name';
import {

2
x-pack/plugins/apm/server/routes/metrics/serverless/get_active_instances_overview.ts
View file

@ -23,7 +23,7 @@ import {
import { getServerlessFunctionNameFromId } from '../../../../common/serverless';
import { environmentQuery } from '../../../../common/utils/environment_query';
import { Coordinate } from '../../../../typings/timeseries';
import { getBucketSize } from '../../../lib/helpers/get_bucket_size';
import { getBucketSize } from '../../../../common/utils/get_bucket_size';
import { calcMemoryUsed } from './helper';
import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client';

12
x-pack/plugins/apm/server/routes/mobile/get_mobile_http_requests.ts
View file

@ -13,17 +13,17 @@ import {
} from '@kbn/observability-plugin/server';
import {
SERVICE_NAME,
TRANSACTION_NAME,
SERVICE_TARGET_TYPE,
METRICSET_NAME,
TRANSACTION_NAME,
} from '../../../common/es_fields/apm';
import { environmentQuery } from '../../../common/utils/environment_query';
import { getBucketSize } from '../../../common/utils/get_bucket_size';
import { getOffsetInMs } from '../../../common/utils/get_offset_in_ms';
import { offsetPreviousPeriodCoordinates } from '../../../common/utils/offset_previous_period_coordinate';
import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client';
import { getBucketSize } from '../../lib/helpers/get_bucket_size';
import { Coordinate } from '../../../typings/timeseries';
import { Maybe } from '../../../typings/common';
import { Coordinate } from '../../../typings/timeseries';
import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client';
import { getDocumentTypeFilterForServiceDestinationStatistics } from '../../lib/helpers/spans/get_is_using_service_destination_metrics';
export interface HttpRequestsTimeseries {
currentPeriod: { timeseries: Coordinate[]; value: Maybe<number> };
@ -77,7 +77,7 @@ async function getHttpRequestsTimeseries({
bool: {
filter: [
{ exists: { field: SERVICE_TARGET_TYPE } },
...termQuery(METRICSET_NAME, 'service_destination'),
...getDocumentTypeFilterForServiceDestinationStatistics(true),
...termQuery(SERVICE_NAME, serviceName),
...termQuery(TRANSACTION_NAME, transactionName),
...rangeQuery(startWithOffset, endWithOffset),

2
x-pack/plugins/apm/server/routes/mobile/get_mobile_sessions.ts
View file

@ -20,7 +20,7 @@ import {
import { environmentQuery } from '../../../common/utils/environment_query';
import { getOffsetInMs } from '../../../common/utils/get_offset_in_ms';
import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client';
import { getBucketSize } from '../../lib/helpers/get_bucket_size';
import { getBucketSize } from '../../../common/utils/get_bucket_size';
import { Coordinate } from '../../../typings/timeseries';
import { Maybe } from '../../../typings/common';

31
x-pack/plugins/apm/server/routes/service_map/get_service_map_dependency_node_info.ts
View file

@ -5,24 +5,27 @@
* 2.0.
*/
import { rangeQuery } from '@kbn/observability-plugin/server';
import { ProcessorEvent } from '@kbn/observability-plugin/common';
import { rangeQuery } from '@kbn/observability-plugin/server';
import { ApmDocumentType } from '../../../common/document_type';
import {
EVENT_OUTCOME,
SPAN_DESTINATION_SERVICE_RESOURCE,
SPAN_DESTINATION_SERVICE_RESPONSE_TIME_COUNT,
SPAN_DESTINATION_SERVICE_RESPONSE_TIME_SUM,
} from '../../../common/es_fields/apm';
import { EventOutcome } from '../../../common/event_outcome';
import { environmentQuery } from '../../../common/utils/environment_query';
import { withApmSpan } from '../../utils/with_apm_span';
import { calculateThroughputWithRange } from '../../lib/helpers/calculate_throughput';
import { getBucketSize } from '../../lib/helpers/get_bucket_size';
import { getFailedTransactionRateTimeSeries } from '../../lib/helpers/transaction_error_rate';
import { NodeStats } from '../../../common/service_map';
import { environmentQuery } from '../../../common/utils/environment_query';
import { getBucketSize } from '../../../common/utils/get_bucket_size';
import { getOffsetInMs } from '../../../common/utils/get_offset_in_ms';
import { calculateThroughputWithRange } from '../../lib/helpers/calculate_throughput';
import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client';
import { getDocumentTypeFilterForServiceDestinationStatistics } from '../../lib/helpers/spans/get_is_using_service_destination_metrics';
import {
calculateFailedTransactionRate,
getFailedTransactionRateTimeSeries,
getOutcomeAggregation,
} from '../../lib/helpers/transaction_error_rate';
import { withApmSpan } from '../../utils/with_apm_span';
interface Options {
apmEventClient: APMEventClient;
@ -61,9 +64,7 @@ export function getServiceMapDependencyNodeInfo({
count: {
sum: { field: SPAN_DESTINATION_SERVICE_RESPONSE_TIME_COUNT },
},
outcomes: {
terms: { field: EVENT_OUTCOME, include: [EventOutcome.failure] },
},
...getOutcomeAggregation(ApmDocumentType.ServiceDestinationMetric),
};
const response = await apmEventClient.search(
@ -104,11 +105,13 @@ export function getServiceMapDependencyNodeInfo({
);
const count = response.aggregations?.count.value ?? 0;
const failedTransactionsRateCount =
response.aggregations?.outcomes.buckets[0]?.doc_count ?? 0;
const latencySum = response.aggregations?.latency_sum.value ?? 0;
const avgFailedTransactionsRate = failedTransactionsRateCount / count;
const avgFailedTransactionsRate = response.aggregations
? calculateFailedTransactionRate(response.aggregations)
: null;
const latency = latencySum / count;
const throughput = calculateThroughputWithRange({
start: startWithOffset,

49
x-pack/plugins/apm/server/routes/services/__snapshots__/queries.test.ts.snap generated
View file

@ -91,15 +91,18 @@ exports[`services queries fetches the service items 1`] = `
Array [
Object {
"apm": Object {
"events": Array [
"transaction",
"sources": Array [
Object {
"documentType": "transactionEvent",
"rollupInterval": "none",
},
],
},
"body": Object {
"aggs": Object {
"sample": Object {
"aggs": Object {
"service_overflow_count": Object {
"overflowCount": Object {
"sum": Object {
"field": "service_transaction.aggregation.overflow_count",
},
@ -118,15 +121,6 @@ Array [
"field": "service.environment",
},
},
"outcomes": Object {
"terms": Object {
"field": "event.outcome",
"include": Array [
"failure",
"success",
],
},
},
"sample": Object {
"top_metrics": Object {
"metrics": Array [
@ -139,6 +133,37 @@ Array [
},
},
},
"successful": Object {
"filter": Object {
"bool": Object {
"filter": Array [
Object {
"terms": Object {
"event.outcome": Array [
"success",
],
},
},
],
},
},
},
"successful_or_failed": Object {
"filter": Object {
"bool": Object {
"filter": Array [
Object {
"terms": Object {
"event.outcome": Array [
"failure",
"success",
],
},
},
],
},
},
},
},
"terms": Object {
"field": "transaction.type",

2
x-pack/plugins/apm/server/routes/services/get_service_instances/get_service_instances_system_metric_statistics.ts
View file

@ -19,7 +19,7 @@ import {
import { SERVICE_NODE_NAME_MISSING } from '../../../../common/service_nodes';
import { Coordinate } from '../../../../typings/timeseries';
import { environmentQuery } from '../../../../common/utils/environment_query';
import { getBucketSize } from '../../../lib/helpers/get_bucket_size';
import { getBucketSize } from '../../../../common/utils/get_bucket_size';
import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client';
import {
percentCgroupMemoryUsedScript,

39
x-pack/plugins/apm/server/routes/services/get_service_transaction_group_detailed_statistics.ts
View file

@ -5,32 +5,34 @@
* 2.0.
*/
import { keyBy } from 'lodash';
import { kqlQuery, rangeQuery } from '@kbn/observability-plugin/server';
import { keyBy } from 'lodash';
import { ApmDocumentType } from '../../../common/document_type';
import {
EVENT_OUTCOME,
SERVICE_NAME,
TRANSACTION_NAME,
TRANSACTION_TYPE,
} from '../../../common/es_fields/apm';
import { EventOutcome } from '../../../common/event_outcome';
import { LatencyAggregationType } from '../../../common/latency_aggregation_types';
import { offsetPreviousPeriodCoordinates } from '../../../common/utils/offset_previous_period_coordinate';
import { environmentQuery } from '../../../common/utils/environment_query';
import { getOffsetInMs } from '../../../common/utils/get_offset_in_ms';
import { offsetPreviousPeriodCoordinates } from '../../../common/utils/offset_previous_period_coordinate';
import { Coordinate } from '../../../typings/timeseries';
import {
getDocumentTypeFilterForTransactions,
getDurationFieldForTransactions,
getProcessorEventForTransactions,
} from '../../lib/helpers/transactions';
import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client';
import { getBucketSizeForAggregatedTransactions } from '../../lib/helpers/get_bucket_size_for_aggregated_transactions';
import {
getLatencyAggregation,
getLatencyValue,
} from '../../lib/helpers/latency_aggregation_type';
import { calculateFailedTransactionRate } from '../../lib/helpers/transaction_error_rate';
import { getOffsetInMs } from '../../../common/utils/get_offset_in_ms';
import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client';
import {
getDocumentTypeFilterForTransactions,
getDurationFieldForTransactions,
getProcessorEventForTransactions,
} from '../../lib/helpers/transactions';
import {
calculateFailedTransactionRate,
getOutcomeAggregation,
} from '../../lib/helpers/transaction_error_rate';
export async function getServiceTransactionGroupDetailedStatistics({
environment,
@ -131,12 +133,11 @@ export async function getServiceTransactionGroupDetailedStatistics({
},
aggs: {
...getLatencyAggregation(latencyAggregationType, field),
[EVENT_OUTCOME]: {
terms: {
field: EVENT_OUTCOME,
include: [EventOutcome.failure, EventOutcome.success],
},
},
...getOutcomeAggregation(
searchAggregatedTransactions
? ApmDocumentType.TransactionMetric
: ApmDocumentType.TransactionEvent
),
},
},
},
@ -164,7 +165,7 @@ export async function getServiceTransactionGroupDetailedStatistics({
}));
const errorRate = bucket.timeseries.buckets.map((timeseriesBucket) => ({
x: timeseriesBucket.key,
y: calculateFailedTransactionRate(timeseriesBucket[EVENT_OUTCOME]),
y: calculateFailedTransactionRate(timeseriesBucket),
}));
const transactionGroupTotalDuration =
bucket.transaction_group_total_duration.value || 0;

35
x-pack/plugins/apm/server/routes/services/get_service_transaction_groups.ts
View file

@ -5,30 +5,32 @@
* 2.0.
*/
import { rangeQuery, kqlQuery } from '@kbn/observability-plugin/server';
import { kqlQuery, rangeQuery } from '@kbn/observability-plugin/server';
import { APMConfig } from '../..';
import { ApmDocumentType } from '../../../common/document_type';
import {
EVENT_OUTCOME,
SERVICE_NAME,
TRANSACTION_NAME,
TRANSACTION_OVERFLOW_COUNT,
TRANSACTION_TYPE,
} from '../../../common/es_fields/apm';
import { EventOutcome } from '../../../common/event_outcome';
import { LatencyAggregationType } from '../../../common/latency_aggregation_types';
import { environmentQuery } from '../../../common/utils/environment_query';
import { calculateThroughputWithRange } from '../../lib/helpers/calculate_throughput';
import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client';
import {
getLatencyAggregation,
getLatencyValue,
} from '../../lib/helpers/latency_aggregation_type';
import {
getDocumentTypeFilterForTransactions,
getDurationFieldForTransactions,
getProcessorEventForTransactions,
} from '../../lib/helpers/transactions';
import { calculateThroughputWithRange } from '../../lib/helpers/calculate_throughput';
import {
getLatencyAggregation,
getLatencyValue,
} from '../../lib/helpers/latency_aggregation_type';
import { calculateFailedTransactionRate } from '../../lib/helpers/transaction_error_rate';
import { APMConfig } from '../..';
import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client';
calculateFailedTransactionRate,
getOutcomeAggregation,
} from '../../lib/helpers/transaction_error_rate';
const txGroupsDroppedBucketName = '_other';
@ -114,12 +116,11 @@ export async function getServiceTransactionGroups({
sum: { field },
},
...getLatencyAggregation(latencyAggregationType, field),
[EVENT_OUTCOME]: {
terms: {
field: EVENT_OUTCOME,
include: [EventOutcome.failure, EventOutcome.success],
},
},
...getOutcomeAggregation(
searchAggregatedTransactions
? ApmDocumentType.TransactionMetric
: ApmDocumentType.TransactionEvent
),
},
},
},
@ -131,7 +132,7 @@ export async function getServiceTransactionGroups({
const transactionGroups =
response.aggregations?.transaction_groups.buckets.map((bucket) => {
const errorRate = calculateFailedTransactionRate(bucket[EVENT_OUTCOME]);
const errorRate = calculateFailedTransactionRate(bucket);
const transactionGroupTotalDuration =
bucket.transaction_group_total_duration.value || 0;

43
x-pack/plugins/apm/server/routes/services/get_services/get_service_names_from_terms_enum.ts Normal file
View file

@ -0,0 +1,43 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { ProcessorEvent } from '@kbn/observability-plugin/common';
import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client';
import { SERVICE_NAME } from '../../../../common/es_fields/apm';
import { ENVIRONMENT_ALL } from '../../../../common/environment_filter_values';
import { Environment } from '../../../../common/environment_rt';
export async function getServiceNamesFromTermsEnum({
apmEventClient,
environment,
maxNumberOfServices,
}: {
apmEventClient: APMEventClient;
environment: Environment;
maxNumberOfServices: number;
}) {
if (environment !== ENVIRONMENT_ALL.value) {
return [];
}
const response = await apmEventClient.termsEnum(
'get_services_from_terms_enum',
{
apm: {
events: [
ProcessorEvent.transaction,
ProcessorEvent.span,
ProcessorEvent.metric,
ProcessorEvent.error,
],
},
size: maxNumberOfServices,
field: SERVICE_NAME,
}
);
return response.terms;
}

167
x-pack/plugins/apm/server/routes/services/get_services/get_service_stats.ts
View file

@ -1,167 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { kqlQuery, rangeQuery } from '@kbn/observability-plugin/server';
import {
AGENT_NAME,
SERVICE_ENVIRONMENT,
SERVICE_NAME,
SERVICE_OVERFLOW_COUNT,
TRANSACTION_TYPE,
} from '../../../../common/es_fields/apm';
import {
TRANSACTION_PAGE_LOAD,
TRANSACTION_REQUEST,
} from '../../../../common/transaction_types';
import { environmentQuery } from '../../../../common/utils/environment_query';
import { AgentName } from '../../../../typings/es_schemas/ui/fields/agent';
import {
getDocumentTypeFilterForTransactions,
getDurationFieldForTransactions,
getProcessorEventForTransactions,
} from '../../../lib/helpers/transactions';
import { calculateThroughputWithRange } from '../../../lib/helpers/calculate_throughput';
import {
calculateFailedTransactionRate,
getOutcomeAggregation,
} from '../../../lib/helpers/transaction_error_rate';
import { serviceGroupQuery } from '../../../lib/service_group_query';
import { ServiceGroup } from '../../../../common/service_groups';
import { RandomSampler } from '../../../lib/helpers/get_random_sampler';
import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client';
interface AggregationParams {
environment: string;
kuery: string;
apmEventClient: APMEventClient;
searchAggregatedTransactions: boolean;
maxNumServices: number;
start: number;
end: number;
serviceGroup: ServiceGroup | null;
randomSampler: RandomSampler;
}
export async function getServiceStats({
environment,
kuery,
apmEventClient,
searchAggregatedTransactions,
maxNumServices,
start,
end,
serviceGroup,
randomSampler,
}: AggregationParams) {
const outcomes = getOutcomeAggregation();
const metrics = {
avg_duration: {
avg: {
field: getDurationFieldForTransactions(searchAggregatedTransactions),
},
},
outcomes,
};
const response = await apmEventClient.search('get_service_stats', {
apm: {
events: [getProcessorEventForTransactions(searchAggregatedTransactions)],
},
body: {
track_total_hits: false,
size: 0,
query: {
bool: {
filter: [
...getDocumentTypeFilterForTransactions(
searchAggregatedTransactions
),
...rangeQuery(start, end),
...environmentQuery(environment),
...kqlQuery(kuery),
...serviceGroupQuery(serviceGroup),
],
},
},
aggs: {
sample: {
random_sampler: randomSampler,
aggs: {
service_overflow_count: {
sum: {
field: SERVICE_OVERFLOW_COUNT,
},
},
services: {
terms: {
field: SERVICE_NAME,
size: maxNumServices,
},
aggs: {
transactionType: {
terms: {
field: TRANSACTION_TYPE,
},
aggs: {
...metrics,
environments: {
terms: {
field: SERVICE_ENVIRONMENT,
},
},
sample: {
top_metrics: {
metrics: [{ field: AGENT_NAME } as const],
sort: {
'@timestamp': 'desc' as const,
},
},
},
},
},
},
},
},
},
},
},
});
return {
serviceStats:
response.aggregations?.sample.services.buckets.map((bucket) => {
const topTransactionTypeBucket =
bucket.transactionType.buckets.find(
({ key }) =>
key === TRANSACTION_REQUEST || key === TRANSACTION_PAGE_LOAD
) ?? bucket.transactionType.buckets[0];
return {
serviceName: bucket.key as string,
transactionType: topTransactionTypeBucket.key as string,
environments: topTransactionTypeBucket.environments.buckets.map(
(environmentBucket) => environmentBucket.key as string
),
agentName: topTransactionTypeBucket.sample.top[0].metrics[
AGENT_NAME
] as AgentName,
latency: topTransactionTypeBucket.avg_duration.value,
transactionErrorRate: calculateFailedTransactionRate(
topTransactionTypeBucket.outcomes
),
throughput: calculateThroughputWithRange({
start,
end,
value: topTransactionTypeBucket.doc_count,
}),
};
}) ?? [],
serviceOverflowCount:
response.aggregations?.sample?.service_overflow_count.value || 0,
};
}

74
x-pack/plugins/apm/server/routes/services/get_services/get_service_stats_for_service_metric.ts → x-pack/plugins/apm/server/routes/services/get_services/get_service_transaction_stats.ts
View file

@ -6,17 +6,16 @@
*/
import { kqlQuery, rangeQuery } from '@kbn/observability-plugin/server';
import { ProcessorEvent } from '@kbn/observability-plugin/common';
import { ApmDocumentType } from '../../../../common/document_type';
import {
AGENT_NAME,
SERVICE_ENVIRONMENT,
SERVICE_NAME,
SERVICE_OVERFLOW_COUNT,
TRANSACTION_TYPE,
TRANSACTION_DURATION_SUMMARY,
TRANSACTION_FAILURE_COUNT,
TRANSACTION_SUCCESS_COUNT,
SERVICE_OVERFLOW_COUNT,
} from '../../../../common/es_fields/apm';
import { RollupInterval } from '../../../../common/rollup';
import { ServiceGroup } from '../../../../common/service_groups';
import {
TRANSACTION_PAGE_LOAD,
TRANSACTION_REQUEST,
@ -24,12 +23,15 @@ import {
import { environmentQuery } from '../../../../common/utils/environment_query';
import { AgentName } from '../../../../typings/es_schemas/ui/fields/agent';
import { calculateThroughputWithRange } from '../../../lib/helpers/calculate_throughput';
import { calculateFailedTransactionRateFromServiceMetrics } from '../../../lib/helpers/transaction_error_rate';
import { serviceGroupQuery } from '../../../lib/service_group_query';
import { ServiceGroup } from '../../../../common/service_groups';
import { RandomSampler } from '../../../lib/helpers/get_random_sampler';
import { getDocumentTypeFilterForServiceMetrics } from '../../../lib/helpers/service_metrics';
import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client';
import { RandomSampler } from '../../../lib/helpers/get_random_sampler';
import { getDurationFieldForTransactions } from '../../../lib/helpers/transactions';
import {
calculateFailedTransactionRate,
getOutcomeAggregation,
} from '../../../lib/helpers/transaction_error_rate';
import { serviceGroupQuery } from '../../../lib/service_group_query';
interface AggregationParams {
environment: string;
kuery: string;
@ -39,9 +41,11 @@ interface AggregationParams {
end: number;
serviceGroup: ServiceGroup | null;
randomSampler: RandomSampler;
documentType: ApmDocumentType;
rollupInterval: RollupInterval;
}
export async function getServiceStatsForServiceMetrics({
export async function getServiceTransactionStats({
environment,
kuery,
apmEventClient,
@ -50,12 +54,30 @@ export async function getServiceStatsForServiceMetrics({
end,
serviceGroup,
randomSampler,
documentType,
rollupInterval,
}: AggregationParams) {
const outcomes = getOutcomeAggregation(documentType);
const metrics = {
avg_duration: {
avg: {
field: getDurationFieldForTransactions(documentType),
},
},
...outcomes,
};
const response = await apmEventClient.search(
'get_service_stats_for_service_metric',
'get_service_transaction_stats',
{
apm: {
events: [ProcessorEvent.metric],
sources: [
{
documentType,
rollupInterval,
},
],
},
body: {
track_total_hits: false,
@ -63,7 +85,6 @@ export async function getServiceStatsForServiceMetrics({
query: {
bool: {
filter: [
...getDocumentTypeFilterForServiceMetrics(),
...rangeQuery(start, end),
...environmentQuery(environment),
...kqlQuery(kuery),
@ -91,21 +112,7 @@ export async function getServiceStatsForServiceMetrics({
field: TRANSACTION_TYPE,
},
aggs: {
avg_duration: {
avg: {
field: TRANSACTION_DURATION_SUMMARY,
},
},
failure_count: {
sum: {
field: TRANSACTION_FAILURE_COUNT,
},
},
success_count: {
sum: {
field: TRANSACTION_SUCCESS_COUNT,
},
},
...metrics,
environments: {
terms: {
field: SERVICE_ENVIRONMENT,
@ -149,12 +156,9 @@ export async function getServiceStatsForServiceMetrics({
AGENT_NAME
] as AgentName,
latency: topTransactionTypeBucket.avg_duration.value,
transactionErrorRate:
calculateFailedTransactionRateFromServiceMetrics({
failedTransactions: topTransactionTypeBucket.failure_count.value,
successfulTransactions:
topTransactionTypeBucket.success_count.value,
}),
transactionErrorRate: calculateFailedTransactionRate(
topTransactionTypeBucket
),
throughput: calculateThroughputWithRange({
start,
end,

42
x-pack/plugins/apm/server/routes/services/get_services/get_services_items.ts
View file

@ -6,18 +6,19 @@
*/
import { Logger } from '@kbn/logging';
import { withApmSpan } from '../../../utils/with_apm_span';
import { ApmDocumentType } from '../../../../common/document_type';
import { RollupInterval } from '../../../../common/rollup';
import { ServiceGroup } from '../../../../common/service_groups';
import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client';
import { ApmAlertsClient } from '../../../lib/helpers/get_apm_alerts_client';
import { MlClient } from '../../../lib/helpers/get_ml_client';
import { RandomSampler } from '../../../lib/helpers/get_random_sampler';
import { withApmSpan } from '../../../utils/with_apm_span';
import { getHealthStatuses } from './get_health_statuses';
import { getServicesFromErrorAndMetricDocuments } from './get_services_from_error_and_metric_documents';
import { getServiceStats } from './get_service_stats';
import { getServiceStatsForServiceMetrics } from './get_service_stats_for_service_metric';
import { mergeServiceStats } from './merge_service_stats';
import { ServiceGroup } from '../../../../common/service_groups';
import { RandomSampler } from '../../../lib/helpers/get_random_sampler';
import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client';
import { getServicesAlerts } from './get_service_alerts';
import { ApmAlertsClient } from '../../../lib/helpers/get_apm_alerts_client';
import { getServiceTransactionStats } from './get_service_transaction_stats';
import { mergeServiceStats } from './merge_service_stats';
export const MAX_NUMBER_OF_SERVICES = 1_000;
@ -27,38 +28,38 @@ export async function getServicesItems({
mlClient,
apmEventClient,
apmAlertsClient,
searchAggregatedTransactions,
searchAggregatedServiceMetrics,
logger,
start,
end,
serviceGroup,
randomSampler,
documentType,
rollupInterval,
}: {
environment: string;
kuery: string;
mlClient?: MlClient;
apmEventClient: APMEventClient;
apmAlertsClient: ApmAlertsClient;
searchAggregatedTransactions: boolean;
searchAggregatedServiceMetrics: boolean;
logger: Logger;
start: number;
end: number;
serviceGroup: ServiceGroup | null;
randomSampler: RandomSampler;
documentType: ApmDocumentType;
rollupInterval: RollupInterval;
}) {
return withApmSpan('get_services_items', async () => {
const commonParams = {
environment,
kuery,
searchAggregatedTransactions,
searchAggregatedServiceMetrics,
maxNumServices: MAX_NUMBER_OF_SERVICES,
start,
end,
serviceGroup,
randomSampler,
documentType,
rollupInterval,
};
const [
@ -67,15 +68,10 @@ export async function getServicesItems({
healthStatuses,
alertCounts,
] = await Promise.all([
searchAggregatedServiceMetrics
? getServiceStatsForServiceMetrics({
...commonParams,
apmEventClient,
})
: getServiceStats({
...commonParams,
apmEventClient,
}),
getServiceTransactionStats({
...commonParams,
apmEventClient,
}),
getServicesFromErrorAndMetricDocuments({
...commonParams,
apmEventClient,

127
x-pack/plugins/apm/server/routes/services/get_services/get_sorted_and_filtered_services.ts
View file

@ -1,127 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { Logger } from '@kbn/logging';
import { ProcessorEvent } from '@kbn/observability-plugin/common';
import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client';
import { SERVICE_NAME } from '../../../../common/es_fields/apm';
import { ENVIRONMENT_ALL } from '../../../../common/environment_filter_values';
import { Environment } from '../../../../common/environment_rt';
import { joinByKey } from '../../../../common/utils/join_by_key';
import { ServiceGroup } from '../../../../common/service_groups';
import { MlClient } from '../../../lib/helpers/get_ml_client';
import { getHealthStatuses } from './get_health_statuses';
import { lookupServices } from '../../service_groups/lookup_services';
export async function getServiceNamesFromTermsEnum({
apmEventClient,
environment,
maxNumberOfServices,
}: {
apmEventClient: APMEventClient;
environment: Environment;
maxNumberOfServices: number;
}) {
if (environment !== ENVIRONMENT_ALL.value) {
return [];
}
const response = await apmEventClient.termsEnum(
'get_services_from_terms_enum',
{
apm: {
events: [
ProcessorEvent.transaction,
ProcessorEvent.span,
ProcessorEvent.metric,
ProcessorEvent.error,
],
},
size: maxNumberOfServices,
field: SERVICE_NAME,
}
);
return response.terms;
}
export async function getSortedAndFilteredServices({
mlClient,
apmEventClient,
start,
end,
environment,
logger,
serviceGroup,
maxNumberOfServices,
}: {
mlClient?: MlClient;
apmEventClient: APMEventClient;
start: number;
end: number;
environment: Environment;
logger: Logger;
serviceGroup: ServiceGroup | null;
maxNumberOfServices: number;
}) {
const [servicesWithHealthStatuses, selectedServices] = await Promise.all([
getHealthStatuses({
mlClient,
start,
end,
environment,
}).catch((error) => {
logger.error(error);
return [];
}),
serviceGroup
? getServiceNamesFromServiceGroup({
apmEventClient,
start,
end,
maxNumberOfServices,
serviceGroup,
})
: getServiceNamesFromTermsEnum({
apmEventClient,
environment,
maxNumberOfServices,
}),
]);
const services = joinByKey(
[
...servicesWithHealthStatuses,
...selectedServices.map((serviceName) => ({ serviceName })),
],
'serviceName'
);
return services;
}
async function getServiceNamesFromServiceGroup({
apmEventClient,
start,
end,
maxNumberOfServices,
serviceGroup: { kuery },
}: {
apmEventClient: APMEventClient;
start: number;
end: number;
maxNumberOfServices: number;
serviceGroup: ServiceGroup;
}) {
const services = await lookupServices({
apmEventClient,
kuery,
start,
end,
maxNumberOfServices,
});
return services.map(({ serviceName }) => serviceName);
}

67
x-pack/plugins/apm/server/routes/services/get_services/index.ts
View file

@ -1,67 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { Logger } from '@kbn/logging';
import { withApmSpan } from '../../../utils/with_apm_span';
import { MlClient } from '../../../lib/helpers/get_ml_client';
import { getServicesItems } from './get_services_items';
import { ServiceGroup } from '../../../../common/service_groups';
import { RandomSampler } from '../../../lib/helpers/get_random_sampler';
import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client';
import { ApmAlertsClient } from '../../../lib/helpers/get_apm_alerts_client';
export async function getServices({
environment,
kuery,
mlClient,
apmEventClient,
apmAlertsClient,
searchAggregatedTransactions,
searchAggregatedServiceMetrics,
logger,
start,
end,
serviceGroup,
randomSampler,
}: {
environment: string;
kuery: string;
mlClient?: MlClient;
apmEventClient: APMEventClient;
apmAlertsClient: ApmAlertsClient;
searchAggregatedTransactions: boolean;
searchAggregatedServiceMetrics: boolean;
logger: Logger;
start: number;
end: number;
serviceGroup: ServiceGroup | null;
randomSampler: RandomSampler;
}) {
return withApmSpan('get_services', async () => {
const { items, maxServiceCountExceeded, serviceOverflowCount } =
await getServicesItems({
environment,
kuery,
mlClient,
apmEventClient,
apmAlertsClient,
searchAggregatedTransactions,
searchAggregatedServiceMetrics,
logger,
start,
end,
serviceGroup,
randomSampler,
});
return {
items,
maxServiceCountExceeded,
serviceOverflowCount,
};
});
}

4
x-pack/plugins/apm/server/routes/services/get_services/merge_service_stats.test.ts
View file

@ -5,11 +5,11 @@
* 2.0.
*/
import { ServiceHealthStatus } from '../../../../common/service_health_status';
import { getServiceStats } from './get_service_stats';
import { getServiceTransactionStats } from './get_service_transaction_stats';
import { mergeServiceStats } from './merge_service_stats';
type ServiceTransactionStat = Awaited<
ReturnType<typeof getServiceStats>
ReturnType<typeof getServiceTransactionStats>
>['serviceStats'][number];
function stat(values: Partial<ServiceTransactionStat>): ServiceTransactionStat {

6
x-pack/plugins/apm/server/routes/services/get_services/merge_service_stats.ts
View file

@ -10,7 +10,7 @@ import { joinByKey } from '../../../../common/utils/join_by_key';
import { getServicesAlerts } from './get_service_alerts';
import { getHealthStatuses } from './get_health_statuses';
import { getServicesFromErrorAndMetricDocuments } from './get_services_from_error_and_metric_documents';
import { getServiceStats } from './get_service_stats';
import { getServiceTransactionStats } from './get_service_transaction_stats';
export function mergeServiceStats({
serviceStats,
@ -18,7 +18,9 @@ export function mergeServiceStats({
healthStatuses,
alertCounts,
}: {
serviceStats: Awaited<ReturnType<typeof getServiceStats>>['serviceStats'];
serviceStats: Awaited<
ReturnType<typeof getServiceTransactionStats>
>['serviceStats'];
servicesFromErrorAndMetricDocuments: Awaited<
ReturnType<typeof getServicesFromErrorAndMetricDocuments>
>['services'];

228
x-pack/plugins/apm/server/routes/services/get_services_detailed_statistics/get_service_aggregated_transaction_detailed_statistics.ts
View file

@ -1,228 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { keyBy } from 'lodash';
import { ProcessorEvent } from '@kbn/observability-plugin/common';
import { kqlQuery, rangeQuery } from '@kbn/observability-plugin/server';
import {
SERVICE_NAME,
TRANSACTION_TYPE,
TRANSACTION_DURATION_SUMMARY,
TRANSACTION_FAILURE_COUNT,
TRANSACTION_SUCCESS_COUNT,
} from '../../../../common/es_fields/apm';
import { withApmSpan } from '../../../utils/with_apm_span';
import {
TRANSACTION_PAGE_LOAD,
TRANSACTION_REQUEST,
} from '../../../../common/transaction_types';
import { environmentQuery } from '../../../../common/utils/environment_query';
import { getOffsetInMs } from '../../../../common/utils/get_offset_in_ms';
import { calculateThroughputWithRange } from '../../../lib/helpers/calculate_throughput';
import { getBucketSizeForAggregatedTransactions } from '../../../lib/helpers/get_bucket_size_for_aggregated_transactions';
import { calculateFailedTransactionRateFromServiceMetrics } from '../../../lib/helpers/transaction_error_rate';
import { RandomSampler } from '../../../lib/helpers/get_random_sampler';
import { getDocumentTypeFilterForServiceMetrics } from '../../../lib/helpers/service_metrics';
import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client';
export async function getServiceAggregatedTransactionDetailedStats({
serviceNames,
environment,
kuery,
apmEventClient,
searchAggregatedServiceMetrics,
offset,
start,
end,
randomSampler,
}: {
serviceNames: string[];
environment: string;
kuery: string;
apmEventClient: APMEventClient;
searchAggregatedServiceMetrics: boolean;
offset?: string;
start: number;
end: number;
randomSampler: RandomSampler;
}) {
const { offsetInMs, startWithOffset, endWithOffset } = getOffsetInMs({
start,
end,
offset,
});
const metrics = {
avg_duration: {
avg: {
field: TRANSACTION_DURATION_SUMMARY,
},
},
failure_count: {
sum: {
field: TRANSACTION_FAILURE_COUNT,
},
},
success_count: {
sum: {
field: TRANSACTION_SUCCESS_COUNT,
},
},
};
const response = await apmEventClient.search(
'get_service_aggregated_transaction_detail_stats',
{
apm: {
events: [ProcessorEvent.metric],
},
body: {
track_total_hits: false,
size: 0,
query: {
bool: {
filter: [
{ terms: { [SERVICE_NAME]: serviceNames } },
...getDocumentTypeFilterForServiceMetrics(),
...rangeQuery(startWithOffset, endWithOffset),
...environmentQuery(environment),
...kqlQuery(kuery),
],
},
},
aggs: {
sample: {
random_sampler: randomSampler,
aggs: {
services: {
terms: {
field: SERVICE_NAME,
size: serviceNames.length,
},
aggs: {
transactionType: {
terms: {
field: TRANSACTION_TYPE,
},
aggs: {
...metrics,
timeseries: {
date_histogram: {
field: '@timestamp',
fixed_interval:
getBucketSizeForAggregatedTransactions({
start: startWithOffset,
end: endWithOffset,
numBuckets: 20,
searchAggregatedServiceMetrics,
}).intervalString,
min_doc_count: 0,
extended_bounds: {
min: startWithOffset,
max: endWithOffset,
},
},
aggs: metrics,
},
},
},
},
},
},
},
},
},
}
);
return keyBy(
response.aggregations?.sample.services.buckets.map((bucket) => {
const topTransactionTypeBucket =
bucket.transactionType.buckets.find(
({ key }) =>
key === TRANSACTION_REQUEST || key === TRANSACTION_PAGE_LOAD
) ?? bucket.transactionType.buckets[0];
return {
serviceName: bucket.key as string,
latency: topTransactionTypeBucket.timeseries.buckets.map(
(dateBucket) => ({
x: dateBucket.key + offsetInMs,
y: dateBucket.avg_duration.value,
})
),
transactionErrorRate: topTransactionTypeBucket.timeseries.buckets.map(
(dateBucket) => ({
x: dateBucket.key + offsetInMs,
y: calculateFailedTransactionRateFromServiceMetrics({
failedTransactions: dateBucket.failure_count.value,
successfulTransactions: dateBucket.success_count.value,
}),
})
),
throughput: topTransactionTypeBucket.timeseries.buckets.map(
(dateBucket) => ({
x: dateBucket.key + offsetInMs,
y: calculateThroughputWithRange({
start,
end,
value: dateBucket.doc_count,
}),
})
),
};
}) ?? [],
'serviceName'
);
}
export async function getServiceAggregatedDetailedStatsPeriods({
serviceNames,
environment,
kuery,
apmEventClient,
searchAggregatedServiceMetrics,
offset,
start,
end,
randomSampler,
}: {
serviceNames: string[];
environment: string;
kuery: string;
apmEventClient: APMEventClient;
searchAggregatedServiceMetrics: boolean;
offset?: string;
start: number;
end: number;
randomSampler: RandomSampler;
}) {
return withApmSpan('get_service_aggregated_detailed_stats', async () => {
const commonProps = {
serviceNames,
environment,
kuery,
apmEventClient,
searchAggregatedServiceMetrics,
start,
end,
randomSampler,
};
const [currentPeriod, previousPeriod] = await Promise.all([
getServiceAggregatedTransactionDetailedStats(commonProps),
offset
? getServiceAggregatedTransactionDetailedStats({
...commonProps,
offset,
})
: Promise.resolve({}),
]);
return { currentPeriod, previousPeriod };
});
}

63
x-pack/plugins/apm/server/routes/services/get_services_detailed_statistics/get_service_transaction_detailed_statistics.ts
View file

@ -5,39 +5,38 @@
* 2.0.
*/
import { keyBy } from 'lodash';
import { kqlQuery, rangeQuery } from '@kbn/observability-plugin/server';
import { keyBy } from 'lodash';
import { ApmDocumentType } from '../../../../common/document_type';
import {
SERVICE_NAME,
TRANSACTION_TYPE,
} from '../../../../common/es_fields/apm';
import { withApmSpan } from '../../../utils/with_apm_span';
import { RollupInterval } from '../../../../common/rollup';
import {
TRANSACTION_PAGE_LOAD,
TRANSACTION_REQUEST,
} from '../../../../common/transaction_types';
import { environmentQuery } from '../../../../common/utils/environment_query';
import { getOffsetInMs } from '../../../../common/utils/get_offset_in_ms';
import {
getDocumentTypeFilterForTransactions,
getDurationFieldForTransactions,
getProcessorEventForTransactions,
} from '../../../lib/helpers/transactions';
import { calculateThroughputWithRange } from '../../../lib/helpers/calculate_throughput';
import { getBucketSizeForAggregatedTransactions } from '../../../lib/helpers/get_bucket_size_for_aggregated_transactions';
import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client';
import { RandomSampler } from '../../../lib/helpers/get_random_sampler';
import { getDurationFieldForTransactions } from '../../../lib/helpers/transactions';
import {
calculateFailedTransactionRate,
getOutcomeAggregation,
} from '../../../lib/helpers/transaction_error_rate';
import { RandomSampler } from '../../../lib/helpers/get_random_sampler';
import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client';
import { withApmSpan } from '../../../utils/with_apm_span';
export async function getServiceTransactionDetailedStats({
serviceNames,
environment,
kuery,
apmEventClient,
searchAggregatedTransactions,
documentType,
rollupInterval,
bucketSizeInSeconds,
offset,
start,
end,
@ -47,7 +46,9 @@ export async function getServiceTransactionDetailedStats({
environment: string;
kuery: string;
apmEventClient: APMEventClient;
searchAggregatedTransactions: boolean;
documentType: ApmDocumentType;
rollupInterval: RollupInterval;
bucketSizeInSeconds: number;
offset?: string;
start: number;
end: number;
@ -59,23 +60,26 @@ export async function getServiceTransactionDetailedStats({
offset,
});
const outcomes = getOutcomeAggregation();
const outcomes = getOutcomeAggregation(documentType);
const metrics = {
avg_duration: {
avg: {
field: getDurationFieldForTransactions(searchAggregatedTransactions),
field: getDurationFieldForTransactions(documentType),
},
},
outcomes,
...outcomes,
};
const response = await apmEventClient.search(
'get_service_transaction_detail_stats',
{
apm: {
events: [
getProcessorEventForTransactions(searchAggregatedTransactions),
sources: [
{
documentType,
rollupInterval,
},
],
},
body: {
@ -85,9 +89,6 @@ export async function getServiceTransactionDetailedStats({
bool: {
filter: [
{ terms: { [SERVICE_NAME]: serviceNames } },
...getDocumentTypeFilterForTransactions(
searchAggregatedTransactions
),
...rangeQuery(startWithOffset, endWithOffset),
...environmentQuery(environment),
...kqlQuery(kuery),
@ -113,13 +114,7 @@ export async function getServiceTransactionDetailedStats({
timeseries: {
date_histogram: {
field: '@timestamp',
fixed_interval:
getBucketSizeForAggregatedTransactions({
start: startWithOffset,
end: endWithOffset,
numBuckets: 20,
searchAggregatedTransactions,
}).intervalString,
fixed_interval: `${bucketSizeInSeconds}s`,
min_doc_count: 0,
extended_bounds: {
min: startWithOffset,
@ -158,7 +153,7 @@ export async function getServiceTransactionDetailedStats({
transactionErrorRate: topTransactionTypeBucket.timeseries.buckets.map(
(dateBucket) => ({
x: dateBucket.key + offsetInMs,
y: calculateFailedTransactionRate(dateBucket.outcomes),
y: calculateFailedTransactionRate(dateBucket),
})
),
throughput: topTransactionTypeBucket.timeseries.buckets.map(
@ -182,7 +177,9 @@ export async function getServiceDetailedStatsPeriods({
environment,
kuery,
apmEventClient,
searchAggregatedTransactions,
documentType,
rollupInterval,
bucketSizeInSeconds,
offset,
start,
end,
@ -192,7 +189,9 @@ export async function getServiceDetailedStatsPeriods({
environment: string;
kuery: string;
apmEventClient: APMEventClient;
searchAggregatedTransactions: boolean;
documentType: ApmDocumentType;
rollupInterval: RollupInterval;
bucketSizeInSeconds: number;
offset?: string;
start: number;
end: number;
@ -204,7 +203,9 @@ export async function getServiceDetailedStatsPeriods({
environment,
kuery,
apmEventClient,
searchAggregatedTransactions,
documentType,
rollupInterval,
bucketSizeInSeconds,
start,
end,
randomSampler,

33
x-pack/plugins/apm/server/routes/services/get_services_detailed_statistics/index.ts
View file

@ -5,18 +5,20 @@
* 2.0.
*/
import { getServiceDetailedStatsPeriods } from './get_service_transaction_detailed_statistics';
import { getServiceAggregatedDetailedStatsPeriods } from './get_service_aggregated_transaction_detailed_statistics';
import { RandomSampler } from '../../../lib/helpers/get_random_sampler';
import { ApmDocumentType } from '../../../../common/document_type';
import { RollupInterval } from '../../../../common/rollup';
import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client';
import { RandomSampler } from '../../../lib/helpers/get_random_sampler';
import { getServiceDetailedStatsPeriods } from './get_service_transaction_detailed_statistics';
export async function getServicesDetailedStatistics({
serviceNames,
environment,
kuery,
apmEventClient,
searchAggregatedTransactions,
searchAggregatedServiceMetrics,
documentType,
rollupInterval,
bucketSizeInSeconds,
offset,
start,
end,
@ -26,14 +28,15 @@ export async function getServicesDetailedStatistics({
environment: string;
kuery: string;
apmEventClient: APMEventClient;
searchAggregatedTransactions: boolean;
searchAggregatedServiceMetrics: boolean;
documentType: ApmDocumentType;
rollupInterval: RollupInterval;
bucketSizeInSeconds: number;
offset?: string;
start: number;
end: number;
randomSampler: RandomSampler;
}) {
const commonProps = {
return getServiceDetailedStatsPeriods({
serviceNames,
environment,
kuery,
@ -42,14 +45,8 @@ export async function getServicesDetailedStatistics({
end,
randomSampler,
offset,
};
return searchAggregatedServiceMetrics
? getServiceAggregatedDetailedStatsPeriods({
...commonProps,
searchAggregatedServiceMetrics,
})
: getServiceDetailedStatsPeriods({
...commonProps,
searchAggregatedTransactions,
});
documentType,
rollupInterval,
bucketSizeInSeconds,
});
}

6
x-pack/plugins/apm/server/routes/services/queries.test.ts
View file

@ -5,7 +5,9 @@
* 2.0.
*/
import { ApmDocumentType } from '../../../common/document_type';
import { ENVIRONMENT_ALL } from '../../../common/environment_filter_values';
import { RollupInterval } from '../../../common/rollup';
import {
inspectSearchParams,
SearchParamsMock,
@ -55,8 +57,8 @@ describe('services queries', () => {
getServicesItems({
mlClient: undefined,
apmEventClient: mockApmEventClient,
searchAggregatedTransactions: false,
searchAggregatedServiceMetrics: false,
documentType: ApmDocumentType.TransactionEvent,
rollupInterval: RollupInterval.None,
logger: {} as any,
environment: ENVIRONMENT_ALL.value,
kuery: '',

195
x-pack/plugins/apm/server/routes/services/route.ts
View file

@ -7,68 +7,68 @@
import Boom from '@hapi/boom';
import { isoToEpochRt, jsonRt, toNumberRt } from '@kbn/io-ts-utils';
import * as t from 'io-ts';
import { uniq, mergeWith } from 'lodash';
import {
UnknownMLCapabilitiesError,
InsufficientMLCapabilities,
MLPrivilegesUninitialized,
UnknownMLCapabilitiesError,
} from '@kbn/ml-plugin/server';
import { ScopedAnnotationsClient } from '@kbn/observability-plugin/server';
import { Annotation } from '@kbn/observability-plugin/common/annotations';
import { apmServiceGroupMaxNumberOfServices } from '@kbn/observability-plugin/common';
import { ScopedAnnotationsClient } from '@kbn/observability-plugin/server';
import * as t from 'io-ts';
import { mergeWith, uniq } from 'lodash';
import { ML_ERRORS } from '../../../common/anomaly_detection';
import { offsetRt } from '../../../common/comparison_rt';
import { ConnectionStatsItemWithImpact } from '../../../common/connections';
import { latencyAggregationTypeRt } from '../../../common/latency_aggregation_types';
import { getSearchTransactionsEvents } from '../../lib/helpers/transactions';
import { getServiceInventorySearchSource } from '../../lib/helpers/get_service_inventory_search_source';
import { ServerlessType } from '../../../common/serverless';
import { offsetPreviousPeriodCoordinates } from '../../../common/utils/offset_previous_period_coordinate';
import { getAnomalyTimeseries } from '../../lib/anomaly_detection/get_anomaly_timeseries';
import { createInfraMetricsClient } from '../../lib/helpers/create_es_client/create_infra_metrics_client/create_infra_metrics_client';
import { getApmAlertsClient } from '../../lib/helpers/get_apm_alerts_client';
import { getApmEventClient } from '../../lib/helpers/get_apm_event_client';
import { getMlClient } from '../../lib/helpers/get_ml_client';
import { getServiceAnnotations } from './annotations';
import { getServices } from './get_services';
import { getServiceAgent } from './get_service_agent';
import { getServiceDependencies } from './get_service_dependencies';
import { getServiceInstanceMetadataDetails } from './get_service_instance_metadata_details';
import { getServiceInstancesMainStatistics } from './get_service_instances/main_statistics';
import { getServiceMetadataDetails } from './get_service_metadata_details';
import { getServiceMetadataIcons } from './get_service_metadata_icons';
import { getServiceNodeMetadata } from './get_service_node_metadata';
import { getServiceTransactionTypes } from './get_service_transaction_types';
import { getThroughput } from './get_throughput';
import { getRandomSampler } from '../../lib/helpers/get_random_sampler';
import { getSearchTransactionsEvents } from '../../lib/helpers/transactions';
import { withApmSpan } from '../../utils/with_apm_span';
import { createApmServerRoute } from '../apm_routes/create_apm_server_route';
import {
dataSourceRt,
environmentRt,
kueryRt,
rangeRt,
probabilityRt,
rangeRt,
} from '../default_api_types';
import { offsetPreviousPeriodCoordinates } from '../../../common/utils/offset_previous_period_coordinate';
import { getServiceOverviewContainerMetadata } from './get_service_overview_container_metadata';
import { getServiceInstanceContainerMetadata } from './get_service_instance_container_metadata';
import { getServicesDetailedStatistics } from './get_services_detailed_statistics';
import { getServiceDependenciesBreakdown } from './get_service_dependencies_breakdown';
import { getAnomalyTimeseries } from '../../lib/anomaly_detection/get_anomaly_timeseries';
import { getServiceInstancesDetailedStatisticsPeriods } from './get_service_instances/detailed_statistics';
import { ML_ERRORS } from '../../../common/anomaly_detection';
import { ConnectionStatsItemWithImpact } from '../../../common/connections';
import { getSortedAndFilteredServices } from './get_services/get_sorted_and_filtered_services';
import { ServiceHealthStatus } from '../../../common/service_health_status';
import { getServiceGroup } from '../service_groups/get_service_group';
import { offsetRt } from '../../../common/comparison_rt';
import { getRandomSampler } from '../../lib/helpers/get_random_sampler';
import { createInfraMetricsClient } from '../../lib/helpers/create_es_client/create_infra_metrics_client/create_infra_metrics_client';
import { getApmEventClient } from '../../lib/helpers/get_apm_event_client';
import { getApmAlertsClient } from '../../lib/helpers/get_apm_alerts_client';
import { getServiceAnnotations } from './annotations';
import { getServicesItems } from './get_services/get_services_items';
import { getServicesAlerts } from './get_services/get_service_alerts';
import { ServerlessType } from '../../../common/serverless';
import { getServicesDetailedStatistics } from './get_services_detailed_statistics';
import { getServiceAgent } from './get_service_agent';
import { getServiceDependencies } from './get_service_dependencies';
import { getServiceDependenciesBreakdown } from './get_service_dependencies_breakdown';
import { getServiceInstancesDetailedStatisticsPeriods } from './get_service_instances/detailed_statistics';
import { getServiceInstancesMainStatistics } from './get_service_instances/main_statistics';
import { getServiceInstanceContainerMetadata } from './get_service_instance_container_metadata';
import { getServiceInstanceMetadataDetails } from './get_service_instance_metadata_details';
import { getServiceMetadataDetails } from './get_service_metadata_details';
import { getServiceMetadataIcons } from './get_service_metadata_icons';
import { getServiceNodeMetadata } from './get_service_node_metadata';
import { getServiceOverviewContainerMetadata } from './get_service_overview_container_metadata';
import { getServiceTransactionTypes } from './get_service_transaction_types';
import { getThroughput } from './get_throughput';
const servicesRoute = createApmServerRoute({
endpoint: 'GET /internal/apm/services',
params: t.type({
query: t.intersection([
environmentRt,
kueryRt,
rangeRt,
t.partial({ serviceGroup: t.string }),
probabilityRt,
t.intersection([
probabilityRt,
dataSourceRt,
environmentRt,
kueryRt,
rangeRt,
]),
]),
}),
options: { tags: ['access:apm'] },
@ -120,7 +120,6 @@ const servicesRoute = createApmServerRoute({
serviceOverflowCount: number;
}> {
const {
config,
context,
params,
logger,
@ -135,6 +134,8 @@ const servicesRoute = createApmServerRoute({
end,
serviceGroup: serviceGroupId,
probability,
documentType,
rollupInterval,
} = params.query;
const savedObjectsClient = (await context.core).savedObjects.client;
@ -154,29 +155,19 @@ const servicesRoute = createApmServerRoute({
getRandomSampler({ security, request, probability }),
]);
const { searchAggregatedTransactions, searchAggregatedServiceMetrics } =
await getServiceInventorySearchSource({
serviceMetricsEnabled: false, // Disable serviceMetrics for 8.5 & 8.6
config,
apmEventClient,
kuery,
start,
end,
});
return getServices({
return getServicesItems({
environment,
kuery,
mlClient,
apmEventClient,
apmAlertsClient,
searchAggregatedTransactions,
searchAggregatedServiceMetrics,
logger,
start,
end,
serviceGroup,
randomSampler,
documentType,
rollupInterval,
});
},
});
@ -188,8 +179,10 @@ const servicesDetailedStatisticsRoute = createApmServerRoute({
environmentRt,
kueryRt,
rangeRt,
offsetRt,
probabilityRt,
t.intersection([offsetRt, probabilityRt, dataSourceRt]),
t.type({
bucketSizeInSeconds: toNumberRt,
}),
]),
body: t.type({ serviceNames: jsonRt.pipe(t.array(t.string)) }),
}),
@ -229,14 +222,22 @@ const servicesDetailedStatisticsRoute = createApmServerRoute({
}>;
}> => {
const {
config,
params,
request,
plugins: { security },
} = resources;
const { environment, kuery, offset, start, end, probability } =
params.query;
const {
environment,
kuery,
offset,
start,
end,
probability,
documentType,
rollupInterval,
bucketSizeInSeconds,
} = params.query;
const { serviceNames } = params.body;
@ -245,16 +246,6 @@ const servicesDetailedStatisticsRoute = createApmServerRoute({
getRandomSampler({ security, request, probability }),
]);
const { searchAggregatedTransactions, searchAggregatedServiceMetrics } =
await getServiceInventorySearchSource({
serviceMetricsEnabled: false, // Disable serviceMetrics for 8.5 & 8.6
config,
apmEventClient,
kuery,
start,
end,
});
if (!serviceNames.length) {
throw Boom.badRequest(`serviceNames cannot be empty`);
}
@ -263,8 +254,9 @@ const servicesDetailedStatisticsRoute = createApmServerRoute({
environment,
kuery,
apmEventClient,
searchAggregatedTransactions,
searchAggregatedServiceMetrics,
documentType,
rollupInterval,
bucketSizeInSeconds,
offset,
serviceNames,
start,
@ -1158,66 +1150,6 @@ const serviceAnomalyChartsRoute = createApmServerRoute({
},
});
const sortedAndFilteredServicesRoute = createApmServerRoute({
endpoint: 'GET /internal/apm/sorted_and_filtered_services',
options: {
tags: ['access:apm'],
},
params: t.type({
query: t.intersection([
rangeRt,
environmentRt,
kueryRt,
t.partial({ serviceGroup: t.string }),
]),
}),
handler: async (
resources
): Promise<{
services: Array<{
serviceName: string;
healthStatus?: ServiceHealthStatus;
}>;
}> => {
const {
query: { start, end, environment, kuery, serviceGroup: serviceGroupId },
} = resources.params;
if (kuery) {
return {
services: [],
};
}
const {
savedObjects: { client: savedObjectsClient },
uiSettings: { client: uiSettingsClient },
} = await resources.context.core;
const [mlClient, apmEventClient, serviceGroup, maxNumberOfServices] =
await Promise.all([
getMlClient(resources),
getApmEventClient(resources),
serviceGroupId
? getServiceGroup({ savedObjectsClient, serviceGroupId })
: Promise.resolve(null),
uiSettingsClient.get<number>(apmServiceGroupMaxNumberOfServices),
]);
return {
services: await getSortedAndFilteredServices({
mlClient,
apmEventClient,
start,
end,
environment,
logger: resources.logger,
serviceGroup,
maxNumberOfServices,
}),
};
},
});
const serviceAlertsRoute = createApmServerRoute({
endpoint: 'GET /internal/apm/services/{serviceName}/alerts_count',
params: t.type({
@ -1271,6 +1203,5 @@ export const serviceRouteRepository = {
...serviceDependenciesRoute,
...serviceDependenciesBreakdownRoute,
...serviceAnomalyChartsRoute,
...sortedAndFilteredServicesRoute,
...serviceAlertsRoute,
};

2
x-pack/plugins/apm/server/routes/span_links/get_span_links_details.ts
View file

@ -20,6 +20,7 @@ import {
SPAN_SUBTYPE,
SPAN_TYPE,
AGENT_NAME,
SERVICE_ENVIRONMENT,
} from '../../../common/es_fields/apm';
import { Environment } from '../../../common/environment_rt';
import { SpanLinkDetails } from '../../../common/span_links';
@ -64,6 +65,7 @@ async function fetchSpanLinksDetails({
SPAN_SUBTYPE,
SPAN_TYPE,
AGENT_NAME,
SERVICE_ENVIRONMENT,
],
body: {
track_total_hits: false,

2
x-pack/plugins/apm/server/routes/storage_explorer/route.ts
View file

@ -37,7 +37,7 @@ import {
} from './get_summary_statistics';
import { getApmEventClient } from '../../lib/helpers/get_apm_event_client';
import { isCrossClusterSearch } from './is_cross_cluster_search';
import { getServiceNamesFromTermsEnum } from '../services/get_services/get_sorted_and_filtered_services';
import { getServiceNamesFromTermsEnum } from '../services/get_services/get_service_names_from_terms_enum';
const storageExplorerRoute = createApmServerRoute({
endpoint: 'GET /internal/apm/storage_explorer',

10
x-pack/plugins/apm/server/routes/time_range_metadata/route.ts
View file

@ -8,6 +8,7 @@ import { toBooleanRt } from '@kbn/io-ts-utils';
import * as t from 'io-ts';
import { TimeRangeMetadata } from '../../../common/time_range_metadata';
import { getApmEventClient } from '../../lib/helpers/get_apm_event_client';
import { getDocumentSources } from '../../lib/helpers/get_document_sources';
import { getIsUsingServiceDestinationMetrics } from '../../lib/helpers/spans/get_is_using_service_destination_metrics';
import { createApmServerRoute } from '../apm_routes/create_apm_server_route';
import { kueryRt, rangeRt } from '../default_api_types';
@ -31,7 +32,7 @@ export const timeRangeMetadataRoute = createApmServerRoute({
query: { useSpanName, start, end, kuery },
} = resources.params;
const [isUsingServiceDestinationMetrics] = await Promise.all([
const [isUsingServiceDestinationMetrics, sources] = await Promise.all([
getIsUsingServiceDestinationMetrics({
apmEventClient,
useSpanName,
@ -39,10 +40,17 @@ export const timeRangeMetadataRoute = createApmServerRoute({
end,
kuery,
}),
getDocumentSources({
apmEventClient,
start,
end,
kuery,
}),
]);
return {
isUsingServiceDestinationMetrics,
sources,
};
},
});

12
x-pack/test/apm_api_integration/tests/error_rate/service_apis.spec.ts
View file

@ -9,6 +9,9 @@ import expect from '@kbn/expect';
import { mean, meanBy, sumBy } from 'lodash';
import { LatencyAggregationType } from '@kbn/apm-plugin/common/latency_aggregation_types';
import { isFiniteNumber } from '@kbn/apm-plugin/common/utils/is_finite_number';
import { ApmDocumentType } from '@kbn/apm-plugin/common/document_type';
import { RollupInterval } from '@kbn/apm-plugin/common/rollup';
import { ProcessorEvent } from '@kbn/observability-plugin/common';
import { FtrProviderContext } from '../../common/ftr_provider_context';
export default function ApiTest({ getService }: FtrProviderContext) {
@ -43,6 +46,15 @@ export default function ApiTest({ getService }: FtrProviderContext) {
...commonQuery,
probability: 1,
kuery: `service.name : "${serviceName}" and processor.event : "${processorEvent}"`,
...(processorEvent === ProcessorEvent.metric
? {
documentType: ApmDocumentType.TransactionMetric,
rollupInterval: RollupInterval.OneMinute,
}
: {
documentType: ApmDocumentType.TransactionEvent,
rollupInterval: RollupInterval.None,
}),
},
},
}),

12
x-pack/test/apm_api_integration/tests/error_rate/service_maps.spec.ts
View file

@ -7,6 +7,9 @@
import { apm, timerange } from '@kbn/apm-synthtrace-client';
import expect from '@kbn/expect';
import { meanBy } from 'lodash';
import { ApmDocumentType } from '@kbn/apm-plugin/common/document_type';
import { RollupInterval } from '@kbn/apm-plugin/common/rollup';
import { ProcessorEvent } from '@kbn/observability-plugin/common';
import { FtrProviderContext } from '../../common/ftr_provider_context';
export default function ApiTest({ getService }: FtrProviderContext) {
@ -32,6 +35,15 @@ export default function ApiTest({ getService }: FtrProviderContext) {
...commonQuery,
kuery: `service.name : "${serviceName}" and processor.event : "${processorEvent}"`,
probability: 1,
...(processorEvent === ProcessorEvent.metric
? {
documentType: ApmDocumentType.TransactionMetric,
rollupInterval: RollupInterval.OneMinute,
}
: {
documentType: ApmDocumentType.TransactionEvent,
rollupInterval: RollupInterval.None,
}),
},
},
}),

2
x-pack/test/apm_api_integration/tests/feature_controls.spec.ts
View file

@ -103,7 +103,7 @@ export default function featureControlsTests({ getService }: FtrProviderContext)
},
{
req: {
url: `/internal/apm/services?start=${start}&end=${end}&environment=ENVIRONMENT_ALL&kuery=&probability=1`,
url: `/internal/apm/services?start=${start}&end=${end}&environment=ENVIRONMENT_ALL&kuery=&probability=1&documentType=transactionMetric&rollupInterval=1m`,
},
expectForbidden: expect403,
expectResponse: expect200,

12
x-pack/test/apm_api_integration/tests/latency/service_apis.spec.ts
View file

@ -9,6 +9,9 @@ import expect from '@kbn/expect';
import { meanBy, sumBy } from 'lodash';
import { LatencyAggregationType } from '@kbn/apm-plugin/common/latency_aggregation_types';
import { isFiniteNumber } from '@kbn/apm-plugin/common/utils/is_finite_number';
import { ApmDocumentType } from '@kbn/apm-plugin/common/document_type';
import { RollupInterval } from '@kbn/apm-plugin/common/rollup';
import { ProcessorEvent } from '@kbn/observability-plugin/common';
import { FtrProviderContext } from '../../common/ftr_provider_context';
export default function ApiTest({ getService }: FtrProviderContext) {
@ -45,6 +48,15 @@ export default function ApiTest({ getService }: FtrProviderContext) {
...commonQuery,
kuery: `service.name : "${serviceName}" and processor.event : "${processorEvent}"`,
probability: 1,
...(processorEvent === ProcessorEvent.metric
? {
documentType: ApmDocumentType.TransactionMetric,
rollupInterval: RollupInterval.OneMinute,
}
: {
documentType: ApmDocumentType.TransactionEvent,
rollupInterval: RollupInterval.None,
}),
},
},
}),

12
x-pack/test/apm_api_integration/tests/latency/service_maps.spec.ts
View file

@ -7,6 +7,9 @@
import { apm, timerange } from '@kbn/apm-synthtrace-client';
import expect from '@kbn/expect';
import { meanBy } from 'lodash';
import { ApmDocumentType } from '@kbn/apm-plugin/common/document_type';
import { RollupInterval } from '@kbn/apm-plugin/common/rollup';
import { ProcessorEvent } from '@kbn/observability-plugin/common';
import { FtrProviderContext } from '../../common/ftr_provider_context';
export default function ApiTest({ getService }: FtrProviderContext) {
@ -32,6 +35,15 @@ export default function ApiTest({ getService }: FtrProviderContext) {
...commonQuery,
kuery: `service.name : "${serviceName}" and processor.event : "${processorEvent}"`,
probability: 1,
...(processorEvent === ProcessorEvent.metric
? {
documentType: ApmDocumentType.TransactionMetric,
rollupInterval: RollupInterval.OneMinute,
}
: {
documentType: ApmDocumentType.TransactionEvent,
rollupInterval: RollupInterval.None,
}),
},
},
}),

4
x-pack/test/apm_api_integration/tests/observability_overview/observability_overview.spec.ts
View file

@ -7,6 +7,8 @@
import { apm, timerange } from '@kbn/apm-synthtrace-client';
import expect from '@kbn/expect';
import { meanBy, sumBy } from 'lodash';
import { ApmDocumentType } from '@kbn/apm-plugin/common/document_type';
import { RollupInterval } from '@kbn/apm-plugin/common/rollup';
import { FtrProviderContext } from '../../common/ftr_provider_context';
import { roundNumber } from '../../utils';
@ -32,6 +34,8 @@ export default function ApiTest({ getService }: FtrProviderContext) {
probability: 1,
environment: 'ENVIRONMENT_ALL',
kuery: '',
documentType: ApmDocumentType.TransactionMetric,
rollupInterval: RollupInterval.OneMinute,
},
},
}),

25
x-pack/test/apm_api_integration/tests/services/services_detailed_statistics.spec.ts
View file

@ -8,6 +8,8 @@ import expect from '@kbn/expect';
import moment from 'moment';
import { APIReturnType } from '@kbn/apm-plugin/public/services/rest/create_call_apm_api';
import { isFiniteNumber } from '@kbn/apm-plugin/common/utils/is_finite_number';
import { ApmDocumentType } from '@kbn/apm-plugin/common/document_type';
import { RollupInterval } from '@kbn/apm-plugin/common/rollup';
import archives_metadata from '../../common/fixtures/es_archiver/archives_metadata';
import { FtrProviderContext } from '../../common/ftr_provider_context';
import { ApmApiError } from '../../common/apm_api_supertest';
@ -40,6 +42,9 @@ export default function ApiTest({ getService }: FtrProviderContext) {
kuery: '',
offset: '1d',
probability: 1,
documentType: ApmDocumentType.TransactionMetric,
rollupInterval: RollupInterval.OneMinute,
bucketSizeInSeconds: 60,
},
body: {
serviceNames: JSON.stringify(serviceNames),
@ -69,21 +74,29 @@ export default function ApiTest({ getService }: FtrProviderContext) {
environment: 'ENVIRONMENT_ALL',
kuery: '',
probability: 1,
documentType: ApmDocumentType.TransactionMetric,
rollupInterval: RollupInterval.OneMinute,
bucketSizeInSeconds: 60,
_inspect: true,
},
body: {
serviceNames: JSON.stringify(serviceNames),
},
},
});
expect(response.status).to.be(200);
servicesDetailedStatistics = response.body;
});
it('returns current period data', async () => {
expect(servicesDetailedStatistics.currentPeriod).not.to.be.empty();
});
it("doesn't returns previous period data", async () => {
expect(servicesDetailedStatistics.previousPeriod).to.be.empty();
});
it('returns current data for requested service names', () => {
serviceNames.forEach((serviceName) => {
expect(servicesDetailedStatistics.currentPeriod[serviceName]).not.to.be.empty();
@ -124,6 +137,9 @@ export default function ApiTest({ getService }: FtrProviderContext) {
environment: 'ENVIRONMENT_ALL',
kuery: '',
probability: 1,
documentType: ApmDocumentType.TransactionMetric,
rollupInterval: RollupInterval.OneMinute,
bucketSizeInSeconds: 60,
},
body: {
serviceNames: JSON.stringify([]),
@ -149,6 +165,9 @@ export default function ApiTest({ getService }: FtrProviderContext) {
environment: 'production',
kuery: '',
probability: 1,
documentType: ApmDocumentType.TransactionMetric,
rollupInterval: RollupInterval.OneMinute,
bucketSizeInSeconds: 60,
},
body: {
serviceNames: JSON.stringify(serviceNames),
@ -169,6 +188,9 @@ export default function ApiTest({ getService }: FtrProviderContext) {
environment: 'ENVIRONMENT_ALL',
kuery: 'transaction.type : "invalid_transaction_type"',
probability: 1,
documentType: ApmDocumentType.TransactionMetric,
rollupInterval: RollupInterval.OneMinute,
bucketSizeInSeconds: 60,
},
body: {
serviceNames: JSON.stringify(serviceNames),
@ -197,6 +219,9 @@ export default function ApiTest({ getService }: FtrProviderContext) {
environment: 'ENVIRONMENT_ALL',
kuery: '',
probability: 1,
documentType: ApmDocumentType.TransactionMetric,
rollupInterval: RollupInterval.OneMinute,
bucketSizeInSeconds: 60,
},
body: {
serviceNames: JSON.stringify(serviceNames),

154
x-pack/test/apm_api_integration/tests/services/sorted_and_filtered_services.spec.ts
View file

@ -1,154 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { apm, timerange } from '@kbn/apm-synthtrace-client';
import expect from '@kbn/expect';
import { ValuesType } from 'utility-types';
import { FtrProviderContext } from '../../common/ftr_provider_context';
import { createAndRunApmMlJobs } from '../../common/utils/create_and_run_apm_ml_jobs';
export default function ApiTest({ getService }: FtrProviderContext) {
const registry = getService('registry');
const synthtraceClient = getService('synthtraceEsClient');
const apmApiClient = getService('apmApiClient');
const ml = getService('ml');
const es = getService('es');
const start = '2021-01-01T12:00:00.000Z';
const end = '2021-08-01T12:00:00.000Z';
// the terms enum API will return names for deleted services,
// so we add a prefix to make sure we don't get data from other
// tests
const SERVICE_NAME_PREFIX = 'sorted_and_filtered_';
async function getSortedAndFilteredServices({
environment = 'ENVIRONMENT_ALL',
kuery = '',
}: { environment?: string; kuery?: string } = {}) {
const response = await apmApiClient.readUser({
endpoint: 'GET /internal/apm/sorted_and_filtered_services',
params: {
query: {
start,
end,
environment,
kuery,
},
},
});
return response.body.services
.filter((service) => service.serviceName.startsWith(SERVICE_NAME_PREFIX))
.map((service) => ({
...service,
serviceName: service.serviceName.replace(SERVICE_NAME_PREFIX, ''),
}));
}
type ServiceListItem = ValuesType<Awaited<ReturnType<typeof getSortedAndFilteredServices>>>;
registry.when('Sorted and filtered services', { config: 'trial', archives: [] }, () => {
before(async () => {
const serviceA = apm
.service({ name: SERVICE_NAME_PREFIX + 'a', environment: 'production', agentName: 'java' })
.instance('a');
const serviceB = apm
.service({ name: SERVICE_NAME_PREFIX + 'b', environment: 'development', agentName: 'go' })
.instance('b');
const serviceC = apm
.service({ name: SERVICE_NAME_PREFIX + 'c', environment: 'development', agentName: 'go' })
.instance('c');
const spikeStart = new Date('2021-01-07T12:00:00.000Z').getTime();
const spikeEnd = new Date('2021-01-07T14:00:00.000Z').getTime();
const eventsWithinTimerange = timerange(new Date(start).getTime(), new Date(end).getTime())
.interval('15m')
.rate(1)
.generator((timestamp) => {
const isInSpike = spikeStart <= timestamp && spikeEnd >= timestamp;
return [
serviceA
.transaction({ transactionName: 'GET /api' })
.duration(isInSpike ? 1000 : 1100)
.timestamp(timestamp),
serviceB
.transaction({ transactionName: 'GET /api' })
.duration(isInSpike ? 1000 : 4000)
.timestamp(timestamp),
];
});
const eventsOutsideOfTimerange = timerange(
new Date('2021-01-01T00:00:00.000Z').getTime(),
new Date(start).getTime() - 1
)
.interval('15m')
.rate(1)
.generator((timestamp) => {
return serviceC
.transaction({ transactionName: 'GET /api', transactionType: 'custom' })
.duration(1000)
.timestamp(timestamp);
});
await synthtraceClient.index([eventsWithinTimerange, eventsOutsideOfTimerange]);
await createAndRunApmMlJobs({ es, ml, environments: ['production', 'development'] });
});
after(() => {
return Promise.all([synthtraceClient.clean(), ml.cleanMlIndices()]);
});
describe('with no kuery or environment are set', () => {
let items: ServiceListItem[];
before(async () => {
items = await getSortedAndFilteredServices();
});
it('returns services based on the terms enum API and ML data', () => {
const serviceNames = items.map((item) => item.serviceName);
expect(serviceNames.sort()).to.eql(['a', 'b', 'c']);
});
});
describe('with kuery set', () => {
let items: ServiceListItem[];
before(async () => {
items = await getSortedAndFilteredServices({
kuery: 'service.name:*',
});
});
it('does not return any services', () => {
expect(items.length).to.be(0);
});
});
describe('with environment set to production', () => {
let items: ServiceListItem[];
before(async () => {
items = await getSortedAndFilteredServices({
environment: 'production',
});
});
it('returns services for production only', () => {
const serviceNames = items.map((item) => item.serviceName);
expect(serviceNames.sort()).to.eql(['a']);
});
});
});
}

18
x-pack/test/apm_api_integration/tests/services/top_services.spec.ts
View file

@ -10,6 +10,8 @@ import { sortBy } from 'lodash';
import { apm, timerange } from '@kbn/apm-synthtrace-client';
import { APIReturnType } from '@kbn/apm-plugin/public/services/rest/create_call_apm_api';
import { ENVIRONMENT_ALL } from '@kbn/apm-plugin/common/environment_filter_values';
import { ApmDocumentType } from '@kbn/apm-plugin/common/document_type';
import { RollupInterval } from '@kbn/apm-plugin/common/rollup';
import { FtrProviderContext } from '../../common/ftr_provider_context';
import archives_metadata from '../../common/fixtures/es_archiver/archives_metadata';
import { SupertestReturnType } from '../../common/apm_api_supertest';
@ -45,6 +47,8 @@ export default function ApiTest({ getService }: FtrProviderContext) {
environment: ENVIRONMENT_ALL.value,
kuery: '',
probability: 1,
documentType: ApmDocumentType.TransactionMetric,
rollupInterval: RollupInterval.OneMinute,
},
},
});
@ -163,6 +167,8 @@ export default function ApiTest({ getService }: FtrProviderContext) {
environment: ENVIRONMENT_ALL.value,
kuery: '',
probability: 1,
documentType: ApmDocumentType.TransactionMetric,
rollupInterval: RollupInterval.OneMinute,
},
},
});
@ -215,6 +221,8 @@ export default function ApiTest({ getService }: FtrProviderContext) {
environment: 'production',
kuery: '',
probability: 1,
documentType: ApmDocumentType.TransactionMetric,
rollupInterval: RollupInterval.OneMinute,
},
},
});
@ -250,6 +258,8 @@ export default function ApiTest({ getService }: FtrProviderContext) {
environment: ENVIRONMENT_ALL.value,
kuery: 'service.node.name:"multiple-env-service-development"',
probability: 1,
documentType: ApmDocumentType.TransactionMetric,
rollupInterval: RollupInterval.OneMinute,
},
},
});
@ -285,6 +295,8 @@ export default function ApiTest({ getService }: FtrProviderContext) {
environment: ENVIRONMENT_ALL.value,
kuery: 'not (transaction.type:request)',
probability: 1,
documentType: ApmDocumentType.TransactionMetric,
rollupInterval: RollupInterval.OneMinute,
},
},
});
@ -322,6 +334,8 @@ export default function ApiTest({ getService }: FtrProviderContext) {
environment: ENVIRONMENT_ALL.value,
kuery: '',
probability: 1,
documentType: ApmDocumentType.TransactionMetric,
rollupInterval: RollupInterval.OneMinute,
},
},
});
@ -377,6 +391,8 @@ export default function ApiTest({ getService }: FtrProviderContext) {
environment: ENVIRONMENT_ALL.value,
kuery: '',
probability: 1,
documentType: ApmDocumentType.TransactionMetric,
rollupInterval: RollupInterval.OneMinute,
},
},
});
@ -411,6 +427,8 @@ export default function ApiTest({ getService }: FtrProviderContext) {
environment: ENVIRONMENT_ALL.value,
kuery: 'service.name:opbeans-java',
probability: 1,
documentType: ApmDocumentType.TransactionMetric,
rollupInterval: RollupInterval.OneMinute,
},
},
});

11
x-pack/test/apm_api_integration/tests/span_links/span_links.spec.ts
View file

@ -228,6 +228,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
transactionId: ids.producerConsumer.transactionCId,
spanName: 'Transaction C',
duration: 1000000,
environment: 'production',
});
const serviceDDetails = spanALinksDetails.childrenLinks.spanLinksDetails.find(
@ -244,6 +245,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
transactionId: ids.producerMultiple.transactionDId,
spanName: 'Transaction D',
duration: 1000000,
environment: 'production',
});
});
});
@ -299,6 +301,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
duration: 100000,
spanSubtype: 'http',
spanType: 'external',
environment: 'production',
},
},
]);
@ -341,6 +344,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
duration: 100000,
spanSubtype: 'http',
spanType: 'external',
environment: 'production',
},
},
{
@ -352,6 +356,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
transactionId: ids.producerExternalOnly.transactionBId,
duration: 1000000,
spanName: 'Transaction B',
environment: 'production',
},
},
{
@ -375,6 +380,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
duration: 100000,
spanSubtype: 'http',
spanType: 'external',
environment: 'production',
},
},
]);
@ -396,6 +402,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
transactionId: ids.producerMultiple.transactionDId,
spanName: 'Transaction D',
duration: 1000000,
environment: 'production',
},
},
]);
@ -438,6 +445,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
duration: 100000,
spanSubtype: 'http',
spanType: 'external',
environment: 'production',
},
},
{
@ -451,6 +459,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
duration: 100000,
spanSubtype: 'http',
spanType: 'external',
environment: 'production',
},
},
]);
@ -475,6 +484,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
duration: 100000,
spanSubtype: 'http',
spanType: 'external',
environment: 'production',
},
},
{
@ -486,6 +496,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
transactionId: ids.producerConsumer.transactionCId,
spanName: 'Transaction C',
duration: 1000000,
environment: 'production',
},
},
]);

12
x-pack/test/apm_api_integration/tests/throughput/service_apis.spec.ts
View file

@ -8,6 +8,9 @@ import { apm, timerange } from '@kbn/apm-synthtrace-client';
import expect from '@kbn/expect';
import { meanBy, sumBy } from 'lodash';
import { LatencyAggregationType } from '@kbn/apm-plugin/common/latency_aggregation_types';
import { ApmDocumentType } from '@kbn/apm-plugin/common/document_type';
import { RollupInterval } from '@kbn/apm-plugin/common/rollup';
import { ProcessorEvent } from '@kbn/observability-plugin/common';
import { FtrProviderContext } from '../../common/ftr_provider_context';
import { roundNumber } from '../../utils';
@ -39,6 +42,15 @@ export default function ApiTest({ getService }: FtrProviderContext) {
...commonQuery,
probability: 1,
kuery: `service.name : "${serviceName}" and processor.event : "${processorEvent}"`,
...(processorEvent === ProcessorEvent.metric
? {
documentType: ApmDocumentType.TransactionMetric,
rollupInterval: RollupInterval.OneMinute,
}
: {
documentType: ApmDocumentType.TransactionEvent,
rollupInterval: RollupInterval.None,
}),
},
},
}),

12
x-pack/test/apm_api_integration/tests/throughput/service_maps.spec.ts
View file

@ -6,6 +6,9 @@
*/
import { apm, timerange } from '@kbn/apm-synthtrace-client';
import expect from '@kbn/expect';
import { ApmDocumentType } from '@kbn/apm-plugin/common/document_type';
import { RollupInterval } from '@kbn/apm-plugin/common/rollup';
import { ProcessorEvent } from '@kbn/observability-plugin/common';
import { FtrProviderContext } from '../../common/ftr_provider_context';
import { roundNumber } from '../../utils';
@ -43,6 +46,15 @@ export default function ApiTest({ getService }: FtrProviderContext) {
...commonQuery,
kuery: `service.name : "${serviceName}" and processor.event : "${processorEvent}"`,
probability: 1,
...(processorEvent === ProcessorEvent.metric
? {
documentType: ApmDocumentType.TransactionMetric,
rollupInterval: RollupInterval.OneMinute,
}
: {
documentType: ApmDocumentType.TransactionEvent,
rollupInterval: RollupInterval.None,
}),
},
},
}),