[APM] Service groups - dynamic query refresh (#140406)

* Service groups - dynamic query refresh

* fixes bug with group save API which omits the query params

* don't render 0 services if no services count is given

* adds support for dynamic refresh in getSortedAndFilteredServices

* only include a whitelisted set of metric doc fields in kuery bar suggestions

* clean up unused i18n translation

* address PR feedback

* fixes bug with a bad translation variable reference

Co-authored-by: Oliver Gupte <oliver.gupte@elastic.co>
This commit is contained in:
Giorgos Bamparopoulos 2022-09-20 11:29:09 +03:00 committed by GitHub
parent 260ea9a07b
commit 256f2e7353
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
20 changed files with 290 additions and 73 deletions

View file

@ -13,7 +13,6 @@ export interface ServiceGroup {
groupName: string;
kuery: string;
description?: string;
serviceNames: string[];
color?: string;
}

View file

@ -4,7 +4,6 @@
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import datemath from '@kbn/datemath';
import { EuiModal } from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import { useHistory } from 'react-router-dom';
@ -60,14 +59,9 @@ export function SaveGroupModal({ onClose, savedServiceGroup }: Props) {
async function (newServiceGroup: StagedServiceGroup) {
setIsLoading(true);
try {
const start = datemath.parse('now-24h')?.toISOString();
const end = datemath.parse('now', { roundUp: true })?.toISOString();
if (!start || !end) {
throw new Error('Unable to determine start/end time range.');
}
await callApmApi('POST /internal/apm/service-group', {
params: {
query: { start, end, serviceGroupId: savedServiceGroup?.id },
query: { serviceGroupId: savedServiceGroup?.id },
body: {
groupName: newServiceGroup.groupName,
kuery: newServiceGroup.kuery,

View file

@ -39,6 +39,13 @@ const MAX_CONTAINER_HEIGHT = 600;
const MODAL_HEADER_HEIGHT = 122;
const MODAL_FOOTER_HEIGHT = 80;
const suggestedFieldsWhitelist = [
'agent.name',
'service.name',
'service.language.name',
'service.environment',
];
const Container = styled.div`
width: 600px;
height: ${MAX_CONTAINER_HEIGHT}px;
@ -144,6 +151,21 @@ export function SelectServices({
setStagedKuery(value);
}}
value={kuery}
suggestionFilter={(querySuggestion) => {
if ('field' in querySuggestion) {
const {
field: {
spec: { name: fieldName },
},
} = querySuggestion;
return (
fieldName.startsWith('label') ||
suggestedFieldsWhitelist.includes(fieldName)
);
}
return true;
}}
/>
</EuiFlexItem>
<EuiFlexItem grow={false}>

View file

@ -15,11 +15,12 @@ import {
} from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import { isEmpty, sortBy } from 'lodash';
import React, { useState, useCallback } from 'react';
import React, { useState, useCallback, useMemo } from 'react';
import { FETCH_STATUS, useFetcher } from '../../../../hooks/use_fetcher';
import { ServiceGroupsListItems } from './service_groups_list';
import { Sort } from './sort';
import { RefreshServiceGroupsSubscriber } from '../refresh_service_groups_subscriber';
import { getDateRange } from '../../../../context/url_params_context/helpers';
export type ServiceGroupsSortType = 'recently_added' | 'alphabetical';
@ -38,6 +39,31 @@ export function ServiceGroupsList() {
[]
);
const { start, end } = useMemo(
() =>
getDateRange({
rangeFrom: 'now-24h',
rangeTo: 'now',
}),
[]
);
const { data: servicesCountData = { servicesCounts: {} } } = useFetcher(
(callApmApi) => {
if (start && end) {
return callApmApi('GET /internal/apm/service_groups/services_count', {
params: {
query: {
start,
end,
},
},
});
}
},
[start, end]
);
const { serviceGroups } = data;
const isLoading =
@ -133,7 +159,11 @@ export function ServiceGroupsList() {
</EuiFlexItem>
<EuiFlexItem>
{items.length ? (
<ServiceGroupsListItems items={items} isLoading={isLoading} />
<ServiceGroupsListItems
items={items}
servicesCounts={servicesCountData.servicesCounts}
isLoading={isLoading}
/>
) : (
<EuiEmptyPrompt
iconType="layers"

View file

@ -27,6 +27,7 @@ interface Props {
onClick?: () => void;
href?: string;
withTour?: boolean;
servicesCount?: number;
}
export function ServiceGroupsCard({
@ -35,6 +36,7 @@ export function ServiceGroupsCard({
onClick,
href,
withTour,
servicesCount,
}: Props) {
const { tourEnabled, dismissTour } = useServiceGroupsTour('serviceGroupCard');
@ -62,13 +64,17 @@ export function ServiceGroupsCard({
{!hideServiceCount && (
<EuiFlexItem>
<EuiText size="s">
{i18n.translate(
'xpack.apm.serviceGroups.cardsList.serviceCount',
{
defaultMessage:
'{servicesCount} {servicesCount, plural, one {service} other {services}}',
values: { servicesCount: serviceGroup.serviceNames.length },
}
{servicesCount === undefined ? (
<>&nbsp;</>
) : (
i18n.translate(
'xpack.apm.serviceGroups.cardsList.serviceCount',
{
defaultMessage:
'{servicesCount} {servicesCount, plural, one {service} other {services}}',
values: { servicesCount },
}
)
)}
</EuiText>
</EuiFlexItem>

View file

@ -16,10 +16,11 @@ import { useDefaultEnvironment } from '../../../../hooks/use_default_environment
interface Props {
items: SavedServiceGroup[];
servicesCounts: Record<string, number>;
isLoading: boolean;
}
export function ServiceGroupsListItems({ items }: Props) {
export function ServiceGroupsListItems({ items, servicesCounts }: Props) {
const router = useApmRouter();
const { query } = useApmParams('/service-groups');
@ -30,6 +31,7 @@ export function ServiceGroupsListItems({ items }: Props) {
{items.map((item) => (
<ServiceGroupsCard
serviceGroup={item}
servicesCount={servicesCounts[item.id]}
href={router.link('/services', {
query: {
...query,
@ -52,7 +54,6 @@ export function ServiceGroupsListItems({ items }: Props) {
'xpack.apm.serviceGroups.list.allServices.description',
{ defaultMessage: 'View all services' }
),
serviceNames: [],
color: SERVICE_GROUP_COLOR_DEFAULT,
}}
hideServiceCount

View file

@ -40,6 +40,7 @@ export function KueryBar(props: {
onSubmit?: (value: string) => void;
onChange?: (value: string) => void;
value?: string;
suggestionFilter?: (querySuggestion: QuerySuggestion) => boolean;
}) {
const { path, query } = useApmParams('/*');
@ -102,7 +103,7 @@ export function KueryBar(props: {
currentRequestCheck = currentRequest;
try {
const suggestions = (
const suggestions =
(await unifiedSearch.autocomplete.getQuerySuggestions({
language: 'kuery',
indexPatterns: [dataView],
@ -120,14 +121,21 @@ export function KueryBar(props: {
selectionEnd: selectionStart,
useTimeRange: true,
method: 'terms_agg',
})) || []
).slice(0, 15);
})) || [];
const filteredSuggestions = props.suggestionFilter
? suggestions.filter(props.suggestionFilter)
: suggestions;
if (currentRequest !== currentRequestCheck) {
return;
}
setState({ ...state, suggestions, isLoadingSuggestions: false });
setState({
...state,
suggestions: filteredSuggestions.slice(0, 15),
isLoadingSuggestions: false,
});
} catch (e) {
console.error('Error while fetching suggestions', e);
}

View file

@ -6,17 +6,11 @@
*/
import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import { SERVICE_NAME } from '../elasticsearch_fieldnames';
import { ServiceGroup } from '../service_groups';
import { kqlQuery } from '@kbn/observability-plugin/server';
import { ServiceGroup } from '../../common/service_groups';
export function serviceGroupQuery(
serviceGroup?: ServiceGroup | null
): QueryDslQueryContainer[] {
if (!serviceGroup) {
return [];
}
return serviceGroup?.serviceNames
? [{ terms: { [SERVICE_NAME]: serviceGroup.serviceNames } }]
: [];
return serviceGroup ? kqlQuery(serviceGroup?.kuery) : [];
}

View file

@ -0,0 +1,55 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { ProcessorEvent } from '@kbn/observability-plugin/common';
import { rangeQuery, kqlQuery } from '@kbn/observability-plugin/server';
import { Setup } from '../../lib/helpers/setup_request';
import { SERVICE_NAME } from '../../../common/elasticsearch_fieldnames';
export async function getServicesCounts({
setup,
kuery,
maxNumberOfServices,
start,
end,
}: {
setup: Setup;
kuery: string;
maxNumberOfServices: number;
start: number;
end: number;
}) {
const { apmEventClient } = setup;
const response = await apmEventClient.search('get_services_count', {
apm: {
events: [
ProcessorEvent.metric,
ProcessorEvent.transaction,
ProcessorEvent.span,
ProcessorEvent.error,
],
},
body: {
size: 0,
query: {
bool: {
filter: [...rangeQuery(start, end), ...kqlQuery(kuery)],
},
},
aggs: {
services_count: {
cardinality: {
field: SERVICE_NAME,
},
},
},
},
});
return response?.aggregations?.services_count.value ?? 0;
}

View file

@ -7,6 +7,7 @@
import * as t from 'io-ts';
import { apmServiceGroupMaxNumberOfServices } from '@kbn/observability-plugin/common';
import { keyBy, mapValues } from 'lodash';
import { setupRequest } from '../../lib/helpers/setup_request';
import { createApmServerRoute } from '../apm_routes/create_apm_server_route';
import { kueryRt, rangeRt } from '../default_api_types';
@ -15,10 +16,8 @@ import { getServiceGroup } from './get_service_group';
import { saveServiceGroup } from './save_service_group';
import { deleteServiceGroup } from './delete_service_group';
import { lookupServices } from './lookup_services';
import {
ServiceGroup,
SavedServiceGroup,
} from '../../../common/service_groups';
import { SavedServiceGroup } from '../../../common/service_groups';
import { getServicesCounts } from './get_services_counts';
const serviceGroupsRoute = createApmServerRoute({
endpoint: 'GET /internal/apm/service-groups',
@ -39,6 +38,67 @@ const serviceGroupsRoute = createApmServerRoute({
},
});
const serviceGroupsWithServiceCountRoute = createApmServerRoute({
endpoint: 'GET /internal/apm/service_groups/services_count',
params: t.type({
query: rangeRt,
}),
options: {
tags: ['access:apm'],
},
handler: async (
resources
): Promise<{ servicesCounts: Record<string, number> }> => {
const { context, params } = resources;
const {
savedObjects: { client: savedObjectsClient },
uiSettings: { client: uiSettingsClient },
} = await context.core;
const {
query: { start, end },
} = params;
const [setup, maxNumberOfServices] = await Promise.all([
setupRequest(resources),
uiSettingsClient.get<number>(apmServiceGroupMaxNumberOfServices),
]);
const serviceGroups = await getServiceGroups({
savedObjectsClient,
});
const serviceGroupsWithServiceCount = await Promise.all(
serviceGroups.map(
async ({
id,
kuery,
}): Promise<{ id: string; servicesCount: number }> => {
const servicesCount = await getServicesCounts({
setup,
kuery,
maxNumberOfServices,
start,
end,
});
return {
id,
servicesCount,
};
}
)
);
const servicesCounts = mapValues(
keyBy(serviceGroupsWithServiceCount, 'id'),
'servicesCount'
);
return { servicesCounts };
},
});
const serviceGroupRoute = createApmServerRoute({
endpoint: 'GET /internal/apm/service-group',
params: t.type({
@ -65,11 +125,11 @@ const serviceGroupRoute = createApmServerRoute({
const serviceGroupSaveRoute = createApmServerRoute({
endpoint: 'POST /internal/apm/service-group',
params: t.type({
query: t.intersection([
rangeRt,
query: t.union([
t.partial({
serviceGroupId: t.string,
}),
t.undefined,
]),
body: t.type({
groupName: t.string,
@ -81,32 +141,15 @@ const serviceGroupSaveRoute = createApmServerRoute({
options: { tags: ['access:apm', 'access:apm_write'] },
handler: async (resources): Promise<void> => {
const { context, params } = resources;
const { start, end, serviceGroupId } = params.query;
const { serviceGroupId } = params.query;
const {
savedObjects: { client: savedObjectsClient },
uiSettings: { client: uiSettingsClient },
} = await context.core;
const [setup, maxNumberOfServices] = await Promise.all([
setupRequest(resources),
uiSettingsClient.get<number>(apmServiceGroupMaxNumberOfServices),
]);
const items = await lookupServices({
setup,
kuery: params.body.kuery,
start,
end,
maxNumberOfServices,
});
const serviceNames = items.map(({ serviceName }): string => serviceName);
const serviceGroup: ServiceGroup = {
...params.body,
serviceNames,
};
await saveServiceGroup({
savedObjectsClient,
serviceGroupId,
serviceGroup,
serviceGroup: params.body,
});
},
});
@ -167,4 +210,5 @@ export const serviceGroupRouteRepository = {
...serviceGroupSaveRoute,
...serviceGroupDeleteRoute,
...serviceGroupServicesRoute,
...serviceGroupsWithServiceCountRoute,
};

View file

@ -26,6 +26,8 @@ import { getTraceSampleIds } from './get_trace_sample_ids';
import { transformServiceMapResponses } from './transform_service_map_responses';
import { ENVIRONMENT_ALL } from '../../../common/environment_filter_values';
import { getProcessorEventForTransactions } from '../../lib/helpers/transactions';
import { ServiceGroup } from '../../../common/service_groups';
import { serviceGroupQuery } from '../../lib/service_group_query';
export interface IEnvOptions {
setup: Setup;
@ -35,6 +37,7 @@ export interface IEnvOptions {
logger: Logger;
start: number;
end: number;
serviceGroup: ServiceGroup | null;
}
async function getConnectionData({
@ -43,6 +46,7 @@ async function getConnectionData({
environment,
start,
end,
serviceGroup,
}: IEnvOptions) {
return withApmSpan('get_service_map_connections', async () => {
const { traceIds } = await getTraceSampleIds({
@ -51,6 +55,7 @@ async function getConnectionData({
environment,
start,
end,
serviceGroup,
});
const chunks = chunk(traceIds, setup.config.serviceMapMaxTracesPerRequest);
@ -100,6 +105,7 @@ async function getServicesData(
start,
end,
maxNumberOfServices,
serviceGroup,
} = options;
const params = {
apm: {
@ -117,6 +123,7 @@ async function getServicesData(
...rangeQuery(start, end),
...environmentQuery(environment),
...termsQuery(SERVICE_NAME, ...(options.serviceNames ?? [])),
...serviceGroupQuery(serviceGroup),
],
},
},

View file

@ -19,6 +19,8 @@ import {
import { SERVICE_MAP_TIMEOUT_ERROR } from '../../../common/service_map';
import { environmentQuery } from '../../../common/utils/environment_query';
import { Setup } from '../../lib/helpers/setup_request';
import { serviceGroupQuery } from '../../lib/service_group_query';
import { ServiceGroup } from '../../../common/service_groups';
const MAX_TRACES_TO_INSPECT = 1000;
@ -28,18 +30,20 @@ export async function getTraceSampleIds({
setup,
start,
end,
serviceGroup,
}: {
serviceNames?: string[];
environment: string;
setup: Setup;
start: number;
end: number;
serviceGroup: ServiceGroup | null;
}) {
const { apmEventClient, config } = setup;
const query = {
bool: {
filter: [...rangeQuery(start, end)],
filter: [...rangeQuery(start, end), ...serviceGroupQuery(serviceGroup)],
},
};

View file

@ -7,6 +7,7 @@
import Boom from '@hapi/boom';
import * as t from 'io-ts';
import { compact } from 'lodash';
import { apmServiceGroupMaxNumberOfServices } from '@kbn/observability-plugin/common';
import { isActivePlatinumLicense } from '../../../common/license_check';
import { invalidLicenseMessage } from '../../../common/service_map';
@ -125,10 +126,7 @@ const serviceMapRoute = createApmServerRoute({
uiSettingsClient.get<number>(apmServiceGroupMaxNumberOfServices),
]);
const serviceNames = [
...(serviceName ? [serviceName] : []),
...(serviceGroup?.serviceNames ?? []),
];
const serviceNames = compact([serviceName]);
const searchAggregatedTransactions = await getSearchAggregatedTransactions({
apmEventClient: setup.apmEventClient,
@ -146,6 +144,7 @@ const serviceMapRoute = createApmServerRoute({
start,
end,
maxNumberOfServices,
serviceGroup,
});
},
});

View file

@ -29,7 +29,7 @@ import {
getOutcomeAggregation,
} from '../../../lib/helpers/transaction_error_rate';
import { ServicesItemsSetup } from './get_services_items';
import { serviceGroupQuery } from '../../../../common/utils/service_group_query';
import { serviceGroupQuery } from '../../../lib/service_group_query';
import { ServiceGroup } from '../../../../common/service_groups';
import { RandomSampler } from '../../../lib/helpers/get_random_sampler';

View file

@ -15,7 +15,7 @@ import {
} from '../../../../common/elasticsearch_fieldnames';
import { environmentQuery } from '../../../../common/utils/environment_query';
import { Setup } from '../../../lib/helpers/setup_request';
import { serviceGroupQuery } from '../../../../common/utils/service_group_query';
import { serviceGroupQuery } from '../../../lib/service_group_query';
import { ServiceGroup } from '../../../../common/service_groups';
import { RandomSampler } from '../../../lib/helpers/get_random_sampler';

View file

@ -14,6 +14,7 @@ import { joinByKey } from '../../../../common/utils/join_by_key';
import { ServiceGroup } from '../../../../common/service_groups';
import { Setup } from '../../../lib/helpers/setup_request';
import { getHealthStatuses } from './get_health_statuses';
import { lookupServices } from '../../service_groups/lookup_services';
export async function getSortedAndFilteredServices({
setup,
@ -70,7 +71,13 @@ export async function getSortedAndFilteredServices({
return [];
}),
serviceGroup
? getServiceNamesFromServiceGroup(serviceGroup)
? getServiceNamesFromServiceGroup({
setup,
start,
end,
maxNumberOfServices,
serviceGroup,
})
: getServiceNamesFromTermsEnum(),
]);
@ -85,6 +92,25 @@ export async function getSortedAndFilteredServices({
return services;
}
async function getServiceNamesFromServiceGroup(serviceGroup: ServiceGroup) {
return serviceGroup.serviceNames;
async function getServiceNamesFromServiceGroup({
setup,
start,
end,
maxNumberOfServices,
serviceGroup: { kuery },
}: {
setup: Setup;
start: number;
end: number;
maxNumberOfServices: number;
serviceGroup: ServiceGroup;
}) {
const services = await lookupServices({
setup,
kuery,
start,
end,
maxNumberOfServices,
});
return services.map(({ serviceName }) => serviceName);
}

View file

@ -7,8 +7,37 @@
import { SavedObjectsType } from '@kbn/core/server';
import { i18n } from '@kbn/i18n';
import type { SavedObjectMigrationFn } from '@kbn/core/server';
import { APM_SERVICE_GROUP_SAVED_OBJECT_TYPE } from '../../common/service_groups';
interface ApmServiceGroupsPre850 {
groupName: string;
kuery: string;
description: string;
serviceNames: string[];
color: string;
}
interface ApmServiceGroups {
groupName: string;
kuery: string;
description: string;
color: string;
}
const migrateApmServiceGroups850: SavedObjectMigrationFn<
ApmServiceGroupsPre850,
ApmServiceGroups
> = (doc) => {
const { serviceNames, ...rest } = doc.attributes;
return {
...doc,
attributes: {
...rest,
},
};
};
export const apmServiceGroups: SavedObjectsType = {
name: APM_SERVICE_GROUP_SAVED_OBJECT_TYPE,
hidden: false,
@ -18,7 +47,6 @@ export const apmServiceGroups: SavedObjectsType = {
groupName: { type: 'keyword' },
kuery: { type: 'text' },
description: { type: 'text' },
serviceNames: { type: 'keyword' },
color: { type: 'text' },
},
},
@ -26,8 +54,11 @@ export const apmServiceGroups: SavedObjectsType = {
importableAndExportable: false,
icon: 'apmApp',
getTitle: () =>
i18n.translate('xpack.apm.apmServiceGroups.index', {
defaultMessage: 'APM Service Groups - Index',
i18n.translate('xpack.apm.apmServiceGroups.title', {
defaultMessage: 'APM Service Groups',
}),
},
migrations: {
'8.5.0': migrateApmServiceGroups850,
},
};

View file

@ -6723,7 +6723,6 @@
"xpack.apm.api.fleet.fleetSecurityRequired": "Les plug-ins Fleet et Security sont requis",
"xpack.apm.apmDescription": "Collecte automatiquement les indicateurs et les erreurs de performances détaillés depuis vos applications.",
"xpack.apm.apmSchema.index": "Schéma du serveur APM - Index",
"xpack.apm.apmServiceGroups.index": "Groupes de services APM - Index",
"xpack.apm.apmSettings.index": "Paramètres APM - Index",
"xpack.apm.betaBadgeDescription": "Cette fonctionnalité est actuellement en version bêta. Si vous rencontrez des bugs ou si vous souhaitez apporter des commentaires, ouvrez un ticket de problème ou visitez notre forum de discussion.",
"xpack.apm.betaBadgeLabel": "Bêta",

View file

@ -6711,7 +6711,6 @@
"xpack.apm.api.fleet.fleetSecurityRequired": "FleetおよびSecurityプラグインが必要です",
"xpack.apm.apmDescription": "アプリケーション内から自動的に詳細なパフォーマンスメトリックやエラーを集めます。",
"xpack.apm.apmSchema.index": "APMサーバースキーマ - インデックス",
"xpack.apm.apmServiceGroups.index": "APMサービスグループ - インデックス",
"xpack.apm.apmSettings.index": "APM 設定 - インデックス",
"xpack.apm.betaBadgeDescription": "現在、この機能はベータです。不具合を見つけた場合やご意見がある場合、サポートに問い合わせるか、またはディスカッションフォーラムにご報告ください。",
"xpack.apm.betaBadgeLabel": "ベータ",

View file

@ -6724,7 +6724,6 @@
"xpack.apm.api.fleet.fleetSecurityRequired": "需要 Fleet 和 Security 插件",
"xpack.apm.apmDescription": "自动从您的应用程序内收集深层的性能指标和错误。",
"xpack.apm.apmSchema.index": "APM Server 架构 - 索引",
"xpack.apm.apmServiceGroups.index": "APM 服务组 - 索引",
"xpack.apm.apmSettings.index": "APM 设置 - 索引",
"xpack.apm.betaBadgeDescription": "此功能当前为公测版。如果遇到任何错误或有任何反馈,请报告问题或访问我们的论坛。",
"xpack.apm.betaBadgeLabel": "公测版",