github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 01:38:56 -04:00
Code Issues Projects Releases Packages Wiki Activity

[APM] Progressive fetching (experimental) (#127598)

Browse source 
Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Dario Gieselaar 2022-04-21 12:11:46 +02:00 • committed by GitHub
parent 6e8d198c06
commit 7af6915581
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
33 changed files with 663 additions and 238 deletions
Download patch file Download diff file Expand all files Collapse all files

5
.eslintrc.js
View file

@ -840,7 +840,10 @@ module.exports = {
},
],
'react-hooks/rules-of-hooks': 'error', // Checks rules of Hooks
'react-hooks/exhaustive-deps': ['error', { additionalHooks: '^useFetcher$' }],
'react-hooks/exhaustive-deps': [
'error',
{ additionalHooks: '^(useFetcher|useProgressiveFetcher)$' },
],
},
},
{

5
src/core/types/elasticsearch/search.ts
View file

@ -455,6 +455,11 @@ export type AggregateOf<
reverse_nested: {
doc_count: number;
} & SubAggregateOf<TAggregationContainer, TDocument>;
random_sampler: {
seed: number;
probability: number;
doc_count: number;
} & SubAggregateOf<TAggregationContainer, TDocument>;
sampler: {
doc_count: number;
} & SubAggregateOf<TAggregationContainer, TDocument>;

20
x-pack/plugins/apm/public/components/app/service_inventory/index.tsx
View file

@ -19,6 +19,7 @@ import { useTimeRange } from '../../../hooks/use_time_range';
import { SearchBar } from '../../shared/search_bar';
import { ServiceList } from './service_list';
import { MLCallout, shouldDisplayMlCallout } from '../../shared/ml_callout';
import { useProgressiveFetcher } from '../../../hooks/use_progressive_fetcher';
import { joinByKey } from '../../../../common/utils/join_by_key';
import { ServiceInventoryFieldName } from '../../../../common/service_inventory';
import { orderServiceItems } from './service_list/order_service_items';
@ -62,7 +63,7 @@ function useServicesFetcher() {
[start, end, environment, kuery, serviceGroup]
);
const mainStatisticsFetch = useFetcher(
const mainStatisticsFetch = useProgressiveFetcher(
(callApmApi) => {
if (start && end) {
return callApmApi('GET /internal/apm/services', {
@ -88,9 +89,14 @@ function useServicesFetcher() {
const { data: mainStatisticsData = initialData } = mainStatisticsFetch;
const comparisonFetch = useFetcher(
const comparisonFetch = useProgressiveFetcher(
(callApmApi) => {
if (start && end && mainStatisticsData.items.length) {
if (
start &&
end &&
mainStatisticsData.items.length &&
mainStatisticsFetch.status === FETCH_STATUS.SUCCESS
) {
return callApmApi('GET /internal/apm/services/detailed_statistics', {
params: {
query: {
@ -141,14 +147,16 @@ export function ServiceInventory() {
!userHasDismissedCallout &&
shouldDisplayMlCallout(anomalyDetectionSetupState);
const useOptimizedSorting = useKibana().services.uiSettings?.get<boolean>(
apmServiceInventoryOptimizedSorting
);
const useOptimizedSorting =
useKibana().services.uiSettings?.get<boolean>(
apmServiceInventoryOptimizedSorting
) || false;
let isLoading: boolean;
if (useOptimizedSorting) {
isLoading =
// ensures table is usable when sorted and filtered services have loaded
sortedAndFilteredServicesFetch.status === FETCH_STATUS.LOADING ||
(sortedAndFilteredServicesFetch.status === FETCH_STATUS.SUCCESS &&
sortedAndFilteredServicesFetch.data?.services.length === 0 &&

5
x-pack/plugins/apm/public/components/app/trace_overview/index.tsx
View file

@ -8,13 +8,14 @@
import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui';
import React from 'react';
import { useApmParams } from '../../../hooks/use_apm_params';
import { FETCH_STATUS, useFetcher } from '../../../hooks/use_fetcher';
import { FETCH_STATUS } from '../../../hooks/use_fetcher';
import { APIReturnType } from '../../../services/rest/create_call_apm_api';
import { SearchBar } from '../../shared/search_bar';
import { TraceList } from './trace_list';
import { useFallbackToTransactionsFetcher } from '../../../hooks/use_fallback_to_transactions_fetcher';
import { AggregatedTransactionsBadge } from '../../shared/aggregated_transactions_badge';
import { useTimeRange } from '../../../hooks/use_time_range';
import { useProgressiveFetcher } from '../../../hooks/use_progressive_fetcher';
type TracesAPIResponse = APIReturnType<'GET /internal/apm/traces'>;
const DEFAULT_RESPONSE: TracesAPIResponse = {
@ -31,7 +32,7 @@ export function TraceOverview() {
const { start, end } = useTimeRange({ rangeFrom, rangeTo });
const { status, data = DEFAULT_RESPONSE } = useFetcher(
const { status, data = DEFAULT_RESPONSE } = useProgressiveFetcher(
(callApmApi) => {
if (start && end) {
return callApmApi('GET /internal/apm/traces', {

155
x-pack/plugins/apm/public/hooks/use_progressive_fetcher.tsx Normal file
View file

@ -0,0 +1,155 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { OmitByValue, Assign } from 'utility-types';
import type {
ClientRequestParamsOf,
EndpointOf,
ReturnOf,
} from '@kbn/server-route-repository';
import { useKibana } from '@kbn/kibana-react-plugin/public';
import {
apmProgressiveLoading,
getProbabilityFromProgressiveLoadingQuality,
ProgressiveLoadingQuality,
} from '@kbn/observability-plugin/common';
// eslint-disable-next-line @kbn/eslint/no-restricted-paths
import type { APMServerRouteRepository } from '../../server';
import type {
APMClient,
APMClientOptions,
} from '../services/rest/create_call_apm_api';
import { FetcherResult, FETCH_STATUS, useFetcher } from './use_fetcher';
type APMProgressivelyLoadingServerRouteRepository = OmitByValue<
{
[key in keyof APMServerRouteRepository]: ClientRequestParamsOf<
APMServerRouteRepository,
key
> extends {
params: { query: { probability: any } };
}
? APMServerRouteRepository[key]
: undefined;
},
undefined
>;
type WithoutProbabilityParameter<T extends Record<string, any>> = {
params: { query: {} };
} & Assign<
T,
{
params: Omit<T['params'], 'query'> & {
query: Omit<T['params']['query'], 'probability'>;
};
}
>;
type APMProgressiveAPIClient = <
TEndpoint extends EndpointOf<APMProgressivelyLoadingServerRouteRepository>
>(
endpoint: TEndpoint,
options: Omit<APMClientOptions, 'signal'> &
WithoutProbabilityParameter<
ClientRequestParamsOf<
APMProgressivelyLoadingServerRouteRepository,
TEndpoint
>
>
) => Promise<ReturnOf<APMProgressivelyLoadingServerRouteRepository, TEndpoint>>;
function clientWithProbability(
regularCallApmApi: APMClient,
probability: number
) {
return <
TEndpoint extends EndpointOf<APMProgressivelyLoadingServerRouteRepository>
>(
endpoint: TEndpoint,
options: Omit<APMClientOptions, 'signal'> &
WithoutProbabilityParameter<
ClientRequestParamsOf<
APMProgressivelyLoadingServerRouteRepository,
TEndpoint
>
>
) => {
return regularCallApmApi(endpoint, {
...options,
params: {
...options.params,
query: {
...options.params.query,
probability,
},
},
} as any);
};
}
export function useProgressiveFetcher<TReturn>(
callback: (
callApmApi: APMProgressiveAPIClient
) => Promise<TReturn> | undefined,
dependencies: any[],
options?: Parameters<typeof useFetcher>[2]
): FetcherResult<TReturn> {
const {
services: { uiSettings },
} = useKibana();
const progressiveLoadingQuality =
uiSettings?.get<ProgressiveLoadingQuality>(apmProgressiveLoading) ??
ProgressiveLoadingQuality.off;
const sampledProbability = getProbabilityFromProgressiveLoadingQuality(
progressiveLoadingQuality
);
const sampledFetch = useFetcher(
(regularCallApmApi) => {
if (progressiveLoadingQuality === ProgressiveLoadingQuality.off) {
return;
}
return callback(
clientWithProbability(regularCallApmApi, sampledProbability)
);
},
// eslint-disable-next-line react-hooks/exhaustive-deps
dependencies,
options
);
const unsampledFetch = useFetcher(
(regularCallApmApi) => {
return callback(clientWithProbability(regularCallApmApi, 1));
},
// eslint-disable-next-line react-hooks/exhaustive-deps
dependencies
);
const fetches = [unsampledFetch, sampledFetch];
const isError = unsampledFetch.status === FETCH_STATUS.FAILURE;
const usedFetch =
(!isError &&
fetches.find((fetch) => fetch.status === FETCH_STATUS.SUCCESS)) ||
unsampledFetch;
const status =
unsampledFetch.status === FETCH_STATUS.LOADING &&
usedFetch.status === FETCH_STATUS.SUCCESS
? FETCH_STATUS.LOADING
: usedFetch.status;
return {
...usedFetch,
status,
};
}

4
x-pack/plugins/apm/server/routes/apm_routes/get_global_apm_server_route_repository.ts
View file

@ -38,7 +38,7 @@ import { eventMetadataRouteRepository } from '../event_metadata/route';
import { suggestionsRouteRepository } from '../suggestions/route';
import { agentKeysRouteRepository } from '../agent_keys/route';
const getTypedGlobalApmServerRouteRepository = () => {
function getTypedGlobalApmServerRouteRepository() {
const repository = {
...dataViewRouteRepository,
...environmentsRouteRepository,
@ -70,7 +70,7 @@ const getTypedGlobalApmServerRouteRepository = () => {
};
return repository;
};
}
const getGlobalApmServerRouteRepository = (): ServerRouteRepository => {
return getTypedGlobalApmServerRouteRepository();

5
x-pack/plugins/apm/server/routes/default_api_types.ts
View file

@ -6,7 +6,7 @@
*/
import * as t from 'io-ts';
import { isoToEpochRt } from '@kbn/io-ts-utils';
import { isoToEpochRt, toNumberRt } from '@kbn/io-ts-utils';
export { environmentRt } from '../../common/environment_rt';
@ -15,4 +15,7 @@ export const rangeRt = t.type({
end: isoToEpochRt,
});
export const probabilityRt = t.type({
probability: toNumberRt,
});
export const kueryRt = t.type({ kuery: t.string });

114
x-pack/plugins/apm/server/routes/services/__snapshots__/queries.test.ts.snap generated
View file

@ -81,50 +81,57 @@ Array [
},
"body": Object {
"aggs": Object {
"services": Object {
"sample": Object {
"aggs": Object {
"transactionType": Object {
"services": Object {
"aggs": Object {
"avg_duration": Object {
"avg": Object {
"field": "transaction.duration.us",
},
},
"environments": Object {
"terms": Object {
"field": "service.environment",
},
},
"outcomes": Object {
"terms": Object {
"field": "event.outcome",
"include": Array [
"failure",
"success",
],
},
},
"sample": Object {
"top_metrics": Object {
"metrics": Array [
Object {
"field": "agent.name",
"transactionType": Object {
"aggs": Object {
"avg_duration": Object {
"avg": Object {
"field": "transaction.duration.us",
},
],
"sort": Object {
"@timestamp": "desc",
},
"environments": Object {
"terms": Object {
"field": "service.environment",
},
},
"outcomes": Object {
"terms": Object {
"field": "event.outcome",
"include": Array [
"failure",
"success",
],
},
},
"sample": Object {
"top_metrics": Object {
"metrics": Array [
Object {
"field": "agent.name",
},
],
"sort": Object {
"@timestamp": "desc",
},
},
},
},
"terms": Object {
"field": "transaction.type",
},
},
},
"terms": Object {
"field": "transaction.type",
"field": "service.name",
"size": 50,
},
},
},
"terms": Object {
"field": "service.name",
"size": 50,
"random_sampler": Object {
"probability": 1,
},
},
},
@ -155,29 +162,36 @@ Array [
},
"body": Object {
"aggs": Object {
"services": Object {
"sample": Object {
"aggs": Object {
"environments": Object {
"terms": Object {
"field": "service.environment",
},
},
"latest": Object {
"top_metrics": Object {
"metrics": Array [
Object {
"field": "agent.name",
"services": Object {
"aggs": Object {
"environments": Object {
"terms": Object {
"field": "service.environment",
},
],
"sort": Object {
"@timestamp": "desc",
},
"latest": Object {
"top_metrics": Object {
"metrics": Array [
Object {
"field": "agent.name",
},
],
"sort": Object {
"@timestamp": "desc",
},
},
},
},
"terms": Object {
"field": "service.name",
"size": 50,
},
},
},
"terms": Object {
"field": "service.name",
"size": 50,
"random_sampler": Object {
"probability": 1,
},
},
},

41
x-pack/plugins/apm/server/routes/services/get_services/get_service_transaction_stats.ts
View file

@ -35,6 +35,7 @@ import { ServiceGroup } from '../../../../common/service_groups';
interface AggregationParams {
environment: string;
kuery: string;
probability: number;
setup: ServicesItemsSetup;
searchAggregatedTransactions: boolean;
maxNumServices: number;
@ -46,6 +47,7 @@ interface AggregationParams {
export async function getServiceTransactionStats({
environment,
kuery,
probability,
setup,
searchAggregatedTransactions,
maxNumServices,
@ -90,28 +92,35 @@ export async function getServiceTransactionStats({
},
},
aggs: {
services: {
terms: {
field: SERVICE_NAME,
size: maxNumServices,
sample: {
random_sampler: {
probability,
},
aggs: {
transactionType: {
services: {
terms: {
field: TRANSACTION_TYPE,
field: SERVICE_NAME,
size: maxNumServices,
},
aggs: {
...metrics,
environments: {
transactionType: {
terms: {
field: SERVICE_ENVIRONMENT,
field: TRANSACTION_TYPE,
},
},
sample: {
top_metrics: {
metrics: [{ field: AGENT_NAME } as const],
sort: {
'@timestamp': 'desc' as const,
aggs: {
...metrics,
environments: {
terms: {
field: SERVICE_ENVIRONMENT,
},
},
sample: {
top_metrics: {
metrics: [{ field: AGENT_NAME } as const],
sort: {
'@timestamp': 'desc' as const,
},
},
},
},
},
@ -125,7 +134,7 @@ export async function getServiceTransactionStats({
);
return (
response.aggregations?.services.buckets.map((bucket) => {
response.aggregations?.sample.services.buckets.map((bucket) => {
const topTransactionTypeBucket =
bucket.transactionType.buckets.find(
({ key }) =>

33
x-pack/plugins/apm/server/routes/services/get_services/get_services_from_error_and_metric_documents.ts
View file

@ -21,6 +21,7 @@ import { ServiceGroup } from '../../../../common/service_groups';
export async function getServicesFromErrorAndMetricDocuments({
environment,
setup,
probability,
maxNumServices,
kuery,
start,
@ -29,6 +30,7 @@ export async function getServicesFromErrorAndMetricDocuments({
}: {
setup: Setup;
environment: string;
probability: number;
maxNumServices: number;
kuery: string;
start: number;
@ -56,21 +58,28 @@ export async function getServicesFromErrorAndMetricDocuments({
},
},
aggs: {
services: {
terms: {
field: SERVICE_NAME,
size: maxNumServices,
sample: {
random_sampler: {
probability,
},
aggs: {
environments: {
services: {
terms: {
field: SERVICE_ENVIRONMENT,
field: SERVICE_NAME,
size: maxNumServices,
},
},
latest: {
top_metrics: {
metrics: [{ field: AGENT_NAME } as const],
sort: { '@timestamp': 'desc' },
aggs: {
environments: {
terms: {
field: SERVICE_ENVIRONMENT,
},
},
latest: {
top_metrics: {
metrics: [{ field: AGENT_NAME } as const],
sort: { '@timestamp': 'desc' },
},
},
},
},
},
@ -81,7 +90,7 @@ export async function getServicesFromErrorAndMetricDocuments({
);
return (
response.aggregations?.services.buckets.map((bucket) => {
response.aggregations?.sample.services.buckets.map((bucket) => {
return {
serviceName: bucket.key as string,
environments: bucket.environments.buckets.map(

3
x-pack/plugins/apm/server/routes/services/get_services/get_services_items.ts
View file

@ -21,6 +21,7 @@ const MAX_NUMBER_OF_SERVICES = 50;
export async function getServicesItems({
environment,
kuery,
probability,
setup,
searchAggregatedTransactions,
logger,
@ -30,6 +31,7 @@ export async function getServicesItems({
}: {
environment: string;
kuery: string;
probability: number;
setup: ServicesItemsSetup;
searchAggregatedTransactions: boolean;
logger: Logger;
@ -41,6 +43,7 @@ export async function getServicesItems({
const params = {
environment,
kuery,
probability,
setup,
searchAggregatedTransactions,
maxNumServices: MAX_NUMBER_OF_SERVICES,

3
x-pack/plugins/apm/server/routes/services/get_services/index.ts
View file

@ -14,6 +14,7 @@ import { ServiceGroup } from '../../../../common/service_groups';
export async function getServices({
environment,
kuery,
probability,
setup,
searchAggregatedTransactions,
logger,
@ -23,6 +24,7 @@ export async function getServices({
}: {
environment: string;
kuery: string;
probability: number;
setup: Setup;
searchAggregatedTransactions: boolean;
logger: Logger;
@ -34,6 +36,7 @@ export async function getServices({
const items = await getServicesItems({
environment,
kuery,
probability,
setup,
searchAggregatedTransactions,
logger,

53
x-pack/plugins/apm/server/routes/services/get_services_detailed_statistics/get_service_transaction_detailed_statistics.ts
View file

@ -39,6 +39,7 @@ export async function getServiceTransactionDetailedStatistics({
offset,
start,
end,
probability,
}: {
serviceNames: string[];
environment: string;
@ -48,6 +49,7 @@ export async function getServiceTransactionDetailedStatistics({
offset?: string;
start: number;
end: number;
probability: number;
}) {
const { apmEventClient } = setup;
const { offsetInMs, startWithOffset, endWithOffset } = getOffsetInMs({
@ -91,33 +93,42 @@ export async function getServiceTransactionDetailedStatistics({
},
},
aggs: {
services: {
terms: {
field: SERVICE_NAME,
sample: {
random_sampler: {
probability,
},
aggs: {
transactionType: {
services: {
terms: {
field: TRANSACTION_TYPE,
field: SERVICE_NAME,
size: serviceNames.length,
},
aggs: {
...metrics,
timeseries: {
date_histogram: {
field: '@timestamp',
fixed_interval: getBucketSizeForAggregatedTransactions({
start: startWithOffset,
end: endWithOffset,
numBuckets: 20,
searchAggregatedTransactions,
}).intervalString,
min_doc_count: 0,
extended_bounds: {
min: startWithOffset,
max: endWithOffset,
transactionType: {
terms: {
field: TRANSACTION_TYPE,
},
aggs: {
...metrics,
timeseries: {
date_histogram: {
field: '@timestamp',
fixed_interval:
getBucketSizeForAggregatedTransactions({
start: startWithOffset,
end: endWithOffset,
numBuckets: 20,
searchAggregatedTransactions,
}).intervalString,
min_doc_count: 0,
extended_bounds: {
min: startWithOffset,
max: endWithOffset,
},
},
aggs: metrics,
},
},
aggs: metrics,
},
},
},
@ -129,7 +140,7 @@ export async function getServiceTransactionDetailedStatistics({
);
return keyBy(
response.aggregations?.services.buckets.map((bucket) => {
response.aggregations?.sample.services.buckets.map((bucket) => {
const topTransactionTypeBucket =
bucket.transactionType.buckets.find(
({ key }) =>

3
x-pack/plugins/apm/server/routes/services/get_services_detailed_statistics/index.ts
View file

@ -18,6 +18,7 @@ export async function getServicesDetailedStatistics({
offset,
start,
end,
probability,
}: {
serviceNames: string[];
environment: string;
@ -27,6 +28,7 @@ export async function getServicesDetailedStatistics({
offset?: string;
start: number;
end: number;
probability: number;
}) {
return withApmSpan('get_service_detailed_statistics', async () => {
const commonProps = {
@ -37,6 +39,7 @@ export async function getServicesDetailedStatistics({
searchAggregatedTransactions,
start,
end,
probability,
};
const [currentPeriod, previousPeriod] = await Promise.all([

1
x-pack/plugins/apm/server/routes/services/queries.test.ts
View file

@ -60,6 +60,7 @@ describe('services queries', () => {
start: 0,
end: 50000,
serviceGroup: null,
probability: 1,
})
);

34
x-pack/plugins/apm/server/routes/services/route.ts
View file

@ -36,7 +36,12 @@ import { getServiceProfilingTimeline } from './profiling/get_service_profiling_t
import { getServiceInfrastructure } from './get_service_infrastructure';
import { withApmSpan } from '../../utils/with_apm_span';
import { createApmServerRoute } from '../apm_routes/create_apm_server_route';
import { environmentRt, kueryRt, rangeRt } from '../default_api_types';
import {
environmentRt,
kueryRt,
rangeRt,
probabilityRt,
} from '../default_api_types';
import { offsetPreviousPeriodCoordinates } from '../../../common/utils/offset_previous_period_coordinate';
import { getServicesDetailedStatistics } from './get_services_detailed_statistics';
import { getServiceDependenciesBreakdown } from './get_service_dependencies_breakdown';
@ -57,6 +62,7 @@ const servicesRoute = createApmServerRoute({
kueryRt,
rangeRt,
t.partial({ serviceGroup: t.string }),
probabilityRt,
]),
}),
options: { tags: ['access:apm'] },
@ -105,6 +111,7 @@ const servicesRoute = createApmServerRoute({
start,
end,
serviceGroup: serviceGroupId,
probability,
} = params.query;
const savedObjectsClient = context.core.savedObjects.client;
@ -123,6 +130,7 @@ const servicesRoute = createApmServerRoute({
return getServices({
environment,
kuery,
probability,
setup,
searchAggregatedTransactions,
logger,
@ -137,10 +145,14 @@ const servicesDetailedStatisticsRoute = createApmServerRoute({
endpoint: 'GET /internal/apm/services/detailed_statistics',
params: t.type({
query: t.intersection([
environmentRt,
kueryRt,
rangeRt,
offsetRt,
// t.intersection seemingly only supports 5 arguments so let's wrap them in another intersection
t.intersection([
environmentRt,
kueryRt,
rangeRt,
offsetRt,
probabilityRt,
]),
t.type({ serviceNames: jsonRt.pipe(t.array(t.string)) }),
]),
}),
@ -181,8 +193,15 @@ const servicesDetailedStatisticsRoute = createApmServerRoute({
}> => {
const setup = await setupRequest(resources);
const { params } = resources;
const { environment, kuery, offset, serviceNames, start, end } =
params.query;
const {
environment,
kuery,
offset,
serviceNames,
start,
end,
probability,
} = params.query;
const searchAggregatedTransactions = await getSearchAggregatedTransactions({
...setup,
start,
@ -203,6 +222,7 @@ const servicesDetailedStatisticsRoute = createApmServerRoute({
serviceNames,
start,
end,
probability,
});
},
});

89
x-pack/plugins/apm/server/routes/traces/get_top_traces_primary_stats.ts
View file

@ -41,6 +41,7 @@ export type BucketKey = Record<
interface TopTracesParams {
environment: string;
kuery: string;
probability: number;
transactionName?: string;
searchAggregatedTransactions: boolean;
start: number;
@ -50,6 +51,7 @@ interface TopTracesParams {
export function getTopTracesPrimaryStats({
environment,
kuery,
probability,
transactionName,
searchAggregatedTransactions,
start,
@ -101,47 +103,52 @@ export function getTopTracesPrimaryStats({
},
},
aggs: {
transaction_groups: {
composite: {
sources: asMutableArray([
{ [SERVICE_NAME]: { terms: { field: SERVICE_NAME } } },
{
[TRANSACTION_NAME]: {
terms: { field: TRANSACTION_NAME },
},
},
] as const),
// traces overview is hardcoded to 10000
size: 10000,
},
sample: {
random_sampler: { probability },
aggs: {
transaction_type: {
top_metrics: {
sort: {
'@timestamp': 'desc' as const,
transaction_groups: {
composite: {
sources: asMutableArray([
{ [SERVICE_NAME]: { terms: { field: SERVICE_NAME } } },
{
[TRANSACTION_NAME]: {
terms: { field: TRANSACTION_NAME },
},
},
] as const),
// traces overview is hardcoded to 10000
size: 10000,
},
aggs: {
transaction_type: {
top_metrics: {
sort: {
'@timestamp': 'desc' as const,
},
metrics: [
{
field: TRANSACTION_TYPE,
} as const,
{
field: AGENT_NAME,
} as const,
],
},
},
avg: {
avg: {
field: getDurationFieldForTransactions(
searchAggregatedTransactions
),
},
},
sum: {
sum: {
field: getDurationFieldForTransactions(
searchAggregatedTransactions
),
},
},
metrics: [
{
field: TRANSACTION_TYPE,
} as const,
{
field: AGENT_NAME,
} as const,
],
},
},
avg: {
avg: {
field: getDurationFieldForTransactions(
searchAggregatedTransactions
),
},
},
sum: {
sum: {
field: getDurationFieldForTransactions(
searchAggregatedTransactions
),
},
},
},
@ -152,12 +159,12 @@ export function getTopTracesPrimaryStats({
);
const calculateImpact = calculateImpactBuilder(
response.aggregations?.transaction_groups.buckets.map(
response.aggregations?.sample.transaction_groups.buckets.map(
({ sum }) => sum.value
)
);
const items = response.aggregations?.transaction_groups.buckets.map(
const items = response.aggregations?.sample.transaction_groups.buckets.map(
(bucket) => {
return {
key: bucket.key as BucketKey,

12
x-pack/plugins/apm/server/routes/traces/route.ts
View file

@ -10,7 +10,12 @@ import { setupRequest } from '../../lib/helpers/setup_request';
import { getTraceItems } from './get_trace_items';
import { getTopTracesPrimaryStats } from './get_top_traces_primary_stats';
import { createApmServerRoute } from '../apm_routes/create_apm_server_route';
import { environmentRt, kueryRt, rangeRt } from '../default_api_types';
import {
environmentRt,
kueryRt,
probabilityRt,
rangeRt,
} from '../default_api_types';
import { getSearchAggregatedTransactions } from '../../lib/helpers/transactions';
import { getRootTransactionByTraceId } from '../transactions/get_transaction_by_trace';
import { getTransaction } from '../transactions/get_transaction';
@ -18,7 +23,7 @@ import { getTransaction } from '../transactions/get_transaction';
const tracesRoute = createApmServerRoute({
endpoint: 'GET /internal/apm/traces',
params: t.type({
query: t.intersection([environmentRt, kueryRt, rangeRt]),
query: t.intersection([environmentRt, kueryRt, rangeRt, probabilityRt]),
}),
options: { tags: ['access:apm'] },
handler: async (
@ -37,7 +42,7 @@ const tracesRoute = createApmServerRoute({
}> => {
const setup = await setupRequest(resources);
const { params } = resources;
const { environment, kuery, start, end } = params.query;
const { environment, kuery, start, end, probability } = params.query;
const searchAggregatedTransactions = await getSearchAggregatedTransactions({
...setup,
kuery,
@ -48,6 +53,7 @@ const tracesRoute = createApmServerRoute({
return await getTopTracesPrimaryStats({
environment,
kuery,
probability,
setup,
searchAggregatedTransactions,
start,

6
x-pack/plugins/observability/common/index.ts
View file

@ -16,8 +16,14 @@ export {
enableInfrastructureView,
defaultApmServiceEnvironment,
apmServiceInventoryOptimizedSorting,
apmProgressiveLoading,
} from './ui_settings_keys';
export {
ProgressiveLoadingQuality,
getProbabilityFromProgressiveLoadingQuality,
} from './progressive_loading';
export const casesFeatureId = 'observabilityCases';
// The ID of the observability app. Should more appropriately be called

31
x-pack/plugins/observability/common/progressive_loading.ts Normal file
View file

@ -0,0 +1,31 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export const enum ProgressiveLoadingQuality {
low = 'low',
medium = 'medium',
high = 'high',
off = 'off',
}
export function getProbabilityFromProgressiveLoadingQuality(
quality: ProgressiveLoadingQuality
): number {
switch (quality) {
case ProgressiveLoadingQuality.high:
return 0.1;
case ProgressiveLoadingQuality.medium:
return 0.01;
case ProgressiveLoadingQuality.low:
return 0.001;
case ProgressiveLoadingQuality.off:
return 1;
}
}

1
x-pack/plugins/observability/common/ui_settings_keys.ts
View file

@ -10,6 +10,7 @@ export const maxSuggestions = 'observability:maxSuggestions';
export const enableComparisonByDefault = 'observability:enableComparisonByDefault';
export const enableInfrastructureView = 'observability:enableInfrastructureView';
export const defaultApmServiceEnvironment = 'observability:apmDefaultServiceEnvironment';
export const apmProgressiveLoading = 'observability:apmProgressiveLoading';
export const enableServiceGroups = 'observability:enableServiceGroups';
export const apmServiceInventoryOptimizedSorting =
'observability:apmServiceInventoryOptimizedSorting';

55
x-pack/plugins/observability/server/ui_settings.ts
View file

@ -8,13 +8,14 @@
import { schema } from '@kbn/config-schema';
import { i18n } from '@kbn/i18n';
import { UiSettingsParams } from '@kbn/core/types';
import { observabilityFeatureId } from '../common';
import { observabilityFeatureId, ProgressiveLoadingQuality } from '../common';
import {
enableComparisonByDefault,
enableInspectEsQueries,
maxSuggestions,
enableInfrastructureView,
defaultApmServiceEnvironment,
apmProgressiveLoading,
enableServiceGroups,
apmServiceInventoryOptimizedSorting,
} from '../common/ui_settings_keys';
@ -86,6 +87,58 @@ export const uiSettings: Record<string, UiSettingsParams<boolean | number | stri
value: '',
schema: schema.string(),
},
[apmProgressiveLoading]: {
category: [observabilityFeatureId],
name: i18n.translate('xpack.observability.apmProgressiveLoading', {
defaultMessage: 'Use progressive loading of selected APM views',
}),
description: i18n.translate('xpack.observability.apmProgressiveLoadingDescription', {
defaultMessage:
'{technicalPreviewLabel} Whether to load data progressively for APM views. Data may be requested with a lower sampling rate first, with lower accuracy but faster response times, while the unsampled data loads in the background',
values: { technicalPreviewLabel: `<em>[${technicalPreviewLabel}]</em>` },
}),
value: ProgressiveLoadingQuality.off,
schema: schema.oneOf([
schema.literal(ProgressiveLoadingQuality.off),
schema.literal(ProgressiveLoadingQuality.low),
schema.literal(ProgressiveLoadingQuality.medium),
schema.literal(ProgressiveLoadingQuality.high),
]),
requiresPageReload: false,
type: 'select',
options: [
ProgressiveLoadingQuality.off,
ProgressiveLoadingQuality.low,
ProgressiveLoadingQuality.medium,
ProgressiveLoadingQuality.high,
],
optionLabels: {
[ProgressiveLoadingQuality.off]: i18n.translate(
'xpack.observability.apmProgressiveLoadingQualityOff',
{
defaultMessage: 'Off',
}
),
[ProgressiveLoadingQuality.low]: i18n.translate(
'xpack.observability.apmProgressiveLoadingQualityLow',
{
defaultMessage: 'Low sampling rate (fastest, least accurate)',
}
),
[ProgressiveLoadingQuality.medium]: i18n.translate(
'xpack.observability.apmProgressiveLoadingQualityMedium',
{
defaultMessage: 'Medium sampling rate',
}
),
[ProgressiveLoadingQuality.high]: i18n.translate(
'xpack.observability.apmProgressiveLoadingQualityHigh',
{
defaultMessage: 'High sampling rate (slower, most accurate)',
}
),
},
},
[enableServiceGroups]: {
category: [observabilityFeatureId],
name: i18n.translate('xpack.observability.enableServiceGroups', {

1
x-pack/test/apm_api_integration/tests/error_rate/service_apis.spec.ts
View file

@ -41,6 +41,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
params: {
query: {
...commonQuery,
probability: 1,
kuery: `service.name : "${serviceName}" and processor.event : "${processorEvent}"`,
},
},

1
x-pack/test/apm_api_integration/tests/error_rate/service_maps.spec.ts
View file

@ -31,6 +31,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
query: {
...commonQuery,
kuery: `service.name : "${serviceName}" and processor.event : "${processorEvent}"`,
probability: 1,
},
},
}),

4
x-pack/test/apm_api_integration/tests/feature_controls.spec.ts
View file

@ -79,7 +79,7 @@ export default function featureControlsTests({ getService }: FtrProviderContext)
},
{
req: {
url: `/internal/apm/services?start=${start}&end=${end}&environment=ENVIRONMENT_ALL&kuery=`,
url: `/internal/apm/services?start=${start}&end=${end}&environment=ENVIRONMENT_ALL&kuery=&probability=1`,
},
expectForbidden: expect403,
expectResponse: expect200,
@ -98,7 +98,7 @@ export default function featureControlsTests({ getService }: FtrProviderContext)
},
{
req: {
url: `/internal/apm/traces?start=${start}&end=${end}&environment=ENVIRONMENT_ALL&kuery=`,
url: `/internal/apm/traces?start=${start}&end=${end}&environment=ENVIRONMENT_ALL&kuery=&probability=1`,
},
expectForbidden: expect403,
expectResponse: expect200,

1
x-pack/test/apm_api_integration/tests/latency/service_apis.spec.ts
View file

@ -44,6 +44,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
query: {
...commonQuery,
kuery: `service.name : "${serviceName}" and processor.event : "${processorEvent}"`,
probability: 1,
},
},
}),

1
x-pack/test/apm_api_integration/tests/latency/service_maps.spec.ts
View file

@ -31,6 +31,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
query: {
...commonQuery,
kuery: `service.name : "${serviceName}" and processor.event : "${processorEvent}"`,
probability: 1,
},
},
}),

1
x-pack/test/apm_api_integration/tests/observability_overview/observability_overview.spec.ts
View file

@ -29,6 +29,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
params: {
query: {
...commonQuery,
probability: 1,
environment: 'ENVIRONMENT_ALL',
kuery: '',
},

95
x-pack/test/apm_api_integration/tests/services/services_detailed_statistics.spec.ts
View file

@ -5,19 +5,20 @@
* 2.0.
*/
import expect from '@kbn/expect';
import url from 'url';
import moment from 'moment';
import { APIReturnType } from '@kbn/apm-plugin/public/services/rest/create_call_apm_api';
import { isFiniteNumber } from '@kbn/apm-plugin/common/utils/is_finite_number';
import archives_metadata from '../../common/fixtures/es_archiver/archives_metadata';
import { FtrProviderContext } from '../../common/ftr_provider_context';
import { ApmApiError } from '../../common/apm_api_supertest';
type ServicesDetailedStatisticsReturn =
APIReturnType<'GET /internal/apm/services/detailed_statistics'>;
export default function ApiTest({ getService }: FtrProviderContext) {
const registry = getService('registry');
const supertest = getService('legacySupertestAsApmReadUser');
const apmApiClient = getService('apmApiClient');
const archiveName = 'apm_8.0.0';
const metadata = archives_metadata[archiveName];
@ -29,9 +30,9 @@ export default function ApiTest({ getService }: FtrProviderContext) {
{ config: 'basic', archives: [] },
() => {
it('handles the empty state', async () => {
const response = await supertest.get(
url.format({
pathname: `/internal/apm/services/detailed_statistics`,
const response = await apmApiClient.readUser({
endpoint: `GET /internal/apm/services/detailed_statistics`,
params: {
query: {
start,
end,
@ -39,9 +40,11 @@ export default function ApiTest({ getService }: FtrProviderContext) {
environment: 'ENVIRONMENT_ALL',
kuery: '',
offset: '1d',
probability: 1,
},
})
);
},
});
expect(response.status).to.be(200);
expect(response.body.currentPeriod).to.be.empty();
expect(response.body.previousPeriod).to.be.empty();
@ -55,18 +58,19 @@ export default function ApiTest({ getService }: FtrProviderContext) {
() => {
let servicesDetailedStatistics: ServicesDetailedStatisticsReturn;
before(async () => {
const response = await supertest.get(
url.format({
pathname: `/internal/apm/services/detailed_statistics`,
const response = await apmApiClient.readUser({
endpoint: `GET /internal/apm/services/detailed_statistics`,
params: {
query: {
start,
end,
serviceNames: JSON.stringify(serviceNames),
environment: 'ENVIRONMENT_ALL',
kuery: '',
probability: 1,
},
})
);
},
});
expect(response.status).to.be(200);
servicesDetailedStatistics = response.body;
});
@ -106,52 +110,61 @@ export default function ApiTest({ getService }: FtrProviderContext) {
});
it('returns empty when empty service names is passed', async () => {
const response = await supertest.get(
url.format({
pathname: `/internal/apm/services/detailed_statistics`,
query: {
start,
end,
serviceNames: JSON.stringify([]),
environment: 'ENVIRONMENT_ALL',
kuery: '',
try {
await apmApiClient.readUser({
endpoint: `GET /internal/apm/services/detailed_statistics`,
params: {
query: {
start,
end,
serviceNames: JSON.stringify([]),
environment: 'ENVIRONMENT_ALL',
kuery: '',
probability: 1,
},
},
})
);
expect(response.status).to.be(400);
expect(response.body.message).to.equal('serviceNames cannot be empty');
});
expect().fail('Expected API call to throw an error');
} catch (error: unknown) {
const apiError = error as ApmApiError;
expect(apiError.res.status).eql(400);
expect(apiError.res.body.message).eql('serviceNames cannot be empty');
}
});
it('filters by environment', async () => {
const response = await supertest.get(
url.format({
pathname: `/internal/apm/services/detailed_statistics`,
const response = await apmApiClient.readUser({
endpoint: `GET /internal/apm/services/detailed_statistics`,
params: {
query: {
start,
end,
serviceNames: JSON.stringify(serviceNames),
environment: 'production',
kuery: '',
probability: 1,
},
})
);
},
});
expect(response.status).to.be(200);
expect(Object.keys(response.body.currentPeriod).length).to.be(1);
expect(response.body.currentPeriod['opbeans-java']).not.to.be.empty();
});
it('filters by kuery', async () => {
const response = await supertest.get(
url.format({
pathname: `/internal/apm/services/detailed_statistics`,
const response = await apmApiClient.readUser({
endpoint: `GET /internal/apm/services/detailed_statistics`,
params: {
query: {
start,
end,
serviceNames: JSON.stringify(serviceNames),
environment: 'ENVIRONMENT_ALL',
kuery: 'transaction.type : "invalid_transaction_type"',
probability: 1,
},
})
);
},
});
expect(response.status).to.be(200);
expect(Object.keys(response.body.currentPeriod)).to.be.empty();
});
@ -164,9 +177,9 @@ export default function ApiTest({ getService }: FtrProviderContext) {
() => {
let servicesDetailedStatistics: ServicesDetailedStatisticsReturn;
before(async () => {
const response = await supertest.get(
url.format({
pathname: `/internal/apm/services/detailed_statistics`,
const response = await apmApiClient.readUser({
endpoint: `GET /internal/apm/services/detailed_statistics`,
params: {
query: {
start: moment(end).subtract(15, 'minutes').toISOString(),
end,
@ -174,9 +187,11 @@ export default function ApiTest({ getService }: FtrProviderContext) {
offset: '15m',
environment: 'ENVIRONMENT_ALL',
kuery: '',
probability: 1,
},
})
);
},
});
expect(response.status).to.be(200);
servicesDetailedStatistics = response.body;
});

82
x-pack/test/apm_api_integration/tests/services/top_services.spec.ts
View file

@ -12,25 +12,21 @@ import { APIReturnType } from '@kbn/apm-plugin/public/services/rest/create_call_
import { ENVIRONMENT_ALL } from '@kbn/apm-plugin/common/environment_filter_values';
import { FtrProviderContext } from '../../common/ftr_provider_context';
import archives_metadata from '../../common/fixtures/es_archiver/archives_metadata';
import { SupertestReturnType } from '../../common/apm_api_supertest';
export default function ApiTest({ getService }: FtrProviderContext) {
const registry = getService('registry');
const supertest = getService('legacySupertestAsApmReadUser');
const apmApiClient = getService('apmApiClient');
const synthtrace = getService('synthtraceEsClient');
const supertestAsApmReadUserWithoutMlAccess = getService(
'legacySupertestAsApmReadUserWithoutMlAccess'
);
const archiveName = 'apm_8.0.0';
const archiveRange = archives_metadata[archiveName];
// url parameters
const archiveStart = encodeURIComponent(archiveRange.start);
const archiveEnd = encodeURIComponent(archiveRange.end);
const archiveStart = archiveRange.start;
const archiveEnd = archiveRange.end;
const start = '2021-10-01T00:00:00.000Z';
const end = '2021-10-01T00:05:00.000Z';
@ -40,9 +36,18 @@ export default function ApiTest({ getService }: FtrProviderContext) {
{ config: 'basic', archives: ['apm_mappings_only_8.0.0'] },
() => {
it('handles the empty state', async () => {
const response = await supertest.get(
`/internal/apm/services?start=${start}&end=${end}&environment=ENVIRONMENT_ALL&kuery=`
);
const response = await apmApiClient.readUser({
endpoint: `GET /internal/apm/services`,
params: {
query: {
start,
end,
environment: ENVIRONMENT_ALL.value,
kuery: '',
probability: 1,
},
},
});
expect(response.status).to.be(200);
expect(response.body.items.length).to.be(0);
@ -153,6 +158,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
end,
environment: ENVIRONMENT_ALL.value,
kuery: '',
probability: 1,
},
},
});
@ -204,6 +210,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
end,
environment: 'production',
kuery: '',
probability: 1,
},
},
});
@ -238,6 +245,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
end,
environment: ENVIRONMENT_ALL.value,
kuery: 'service.node.name:"multiple-env-service-development"',
probability: 1,
},
},
});
@ -272,6 +280,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
end,
environment: ENVIRONMENT_ALL.value,
kuery: 'not (transaction.type:request)',
probability: 1,
},
},
});
@ -300,9 +309,18 @@ export default function ApiTest({ getService }: FtrProviderContext) {
};
before(async () => {
response = await supertest.get(
`/internal/apm/services?start=${archiveStart}&end=${archiveEnd}&environment=ENVIRONMENT_ALL&kuery=`
);
response = await apmApiClient.readUser({
endpoint: `GET /internal/apm/services`,
params: {
query: {
start: archiveStart,
end: archiveEnd,
environment: ENVIRONMENT_ALL.value,
kuery: '',
probability: 1,
},
},
});
});
it('the response is successful', () => {
@ -344,11 +362,20 @@ export default function ApiTest({ getService }: FtrProviderContext) {
});
describe('with a user that does not have access to ML', () => {
let response: Awaited<ReturnType<typeof supertest.get>>;
let response: SupertestReturnType<'GET /internal/apm/services'>;
before(async () => {
response = await supertestAsApmReadUserWithoutMlAccess.get(
`/internal/apm/services?start=${archiveStart}&end=${archiveEnd}&environment=ENVIRONMENT_ALL&kuery=`
);
response = await apmApiClient.noMlAccessUser({
endpoint: 'GET /internal/apm/services',
params: {
query: {
start: archiveStart,
end: archiveEnd,
environment: ENVIRONMENT_ALL.value,
kuery: '',
probability: 1,
},
},
});
});
it('the response is successful', () => {
@ -361,7 +388,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
it('contains no health statuses', () => {
const definedHealthStatuses = response.body.items
.map((item: any) => item.healthStatus)
.map((item) => item.healthStatus)
.filter(Boolean);
expect(definedHealthStatuses.length).to.be(0);
@ -369,13 +396,20 @@ export default function ApiTest({ getService }: FtrProviderContext) {
});
describe('and fetching a list of services with a filter', () => {
let response: Awaited<ReturnType<typeof supertest.get>>;
let response: SupertestReturnType<'GET /internal/apm/services'>;
before(async () => {
response = await supertest.get(
`/internal/apm/services?environment=ENVIRONMENT_ALL&start=${archiveStart}&end=${archiveEnd}&kuery=${encodeURIComponent(
'service.name:opbeans-java'
)}`
);
response = await apmApiClient.noMlAccessUser({
endpoint: 'GET /internal/apm/services',
params: {
query: {
start: archiveStart,
end: archiveEnd,
environment: ENVIRONMENT_ALL.value,
kuery: 'service.name:opbeans-java',
probability: 1,
},
},
});
});
it('does not return health statuses for services that are not found in APM data', () => {

1
x-pack/test/apm_api_integration/tests/throughput/service_apis.spec.ts
View file

@ -37,6 +37,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
params: {
query: {
...commonQuery,
probability: 1,
kuery: `service.name : "${serviceName}" and processor.event : "${processorEvent}"`,
},
},

1
x-pack/test/apm_api_integration/tests/throughput/service_maps.spec.ts
View file

@ -42,6 +42,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
query: {
...commonQuery,
kuery: `service.name : "${serviceName}" and processor.event : "${processorEvent}"`,
probability: 1,
},
},
}),

35
x-pack/test/apm_api_integration/tests/traces/top_traces.spec.ts
View file

@ -12,20 +12,28 @@ import { FtrProviderContext } from '../../common/ftr_provider_context';
export default function ApiTest({ getService }: FtrProviderContext) {
const registry = getService('registry');
const supertest = getService('legacySupertestAsApmReadUser');
const apmApiClient = getService('apmApiClient');
const archiveName = 'apm_8.0.0';
const metadata = archives_metadata[archiveName];
// url parameters
const start = encodeURIComponent(metadata.start);
const end = encodeURIComponent(metadata.end);
const { start, end } = metadata;
registry.when('Top traces when data is not loaded', { config: 'basic', archives: [] }, () => {
it('handles empty state', async () => {
const response = await supertest.get(
`/internal/apm/traces?start=${start}&end=${end}&environment=ENVIRONMENT_ALL&kuery=`
);
const response = await apmApiClient.readUser({
endpoint: `GET /internal/apm/traces`,
params: {
query: {
start,
end,
kuery: '',
environment: 'ENVIRONMENT_ALL',
probability: 1,
},
},
});
expect(response.status).to.be(200);
expect(response.body.items.length).to.be(0);
@ -38,9 +46,18 @@ export default function ApiTest({ getService }: FtrProviderContext) {
() => {
let response: any;
before(async () => {
response = await supertest.get(
`/internal/apm/traces?start=${start}&end=${end}&environment=ENVIRONMENT_ALL&kuery=`
);
response = await apmApiClient.readUser({
endpoint: 'GET /internal/apm/traces',
params: {
query: {
start,
end,
kuery: '',
environment: 'ENVIRONMENT_ALL',
probability: 1,
},
},
});
});
it('returns the correct status code', async () => {