Disable navigation to _other bucket and show warning tooltip (#148641)

Closes https://github.com/elastic/kibana/issues/146650

### Summary

This PR disables the link for the `_other` bucket on Services List page
and instead shows a Warning Tooltip with the message.

### Changes

 - `service_link.tsx` prevents the navigation to the bucket
- `other_service_group_bucket.ts` - Syntrace for the `_other` bucket
generation
 
### Demo



https://user-images.githubusercontent.com/7416358/211568522-e10bcf01-d07f-4259-996b-b3b612c7807d.mov

- When kibana limit has been reached
<img width="1561" alt="image"
src="https://user-images.githubusercontent.com/1313018/214614001-0bd8a5d9-c2fe-48c7-a231-0b0805708bce.png">


- When having only `_other`
<img width="1569" alt="image"
src="https://user-images.githubusercontent.com/1313018/214614142-0d47d5b1-40d6-40f5-9a0d-5f5e596f9b98.png">

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Yngrid Coello <yngrid.coello@elastic.co>

packages/kbn-apm-synthtrace/src/scenarios/other_service_group_bucket.ts

@@ -0,0 +1,72 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+import { range as lodashRange } from 'lodash';
+import { ApmFields, apm } from '@kbn/apm-synthtrace-client';
+import { Scenario } from '../cli/scenario';
+
+const scenario: Scenario<ApmFields> = async (runOptions) => {
+  const { logger } = runOptions;
+  const numServices = 10;
+
+  return {
+    generate: ({ range }) => {
+      const TRANSACTION_TYPES = ['request'];
+      const ENVIRONMENTS = ['production', 'development'];
+
+      const MIN_DURATION = 10;
+      const MAX_DURATION = 1000;
+
+      const MAX_BUCKETS = 50;
+
+      const BUCKET_SIZE = (MAX_DURATION - MIN_DURATION) / MAX_BUCKETS;
+
+      const serviceRange = [
+        ...lodashRange(0, numServices).map((groupId) => `service-${groupId}`),
+        '_other',
+      ];
+
+      const instances = serviceRange.flatMap((serviceName) => {
+        const services = ENVIRONMENTS.map((env) => apm.service(serviceName, env, 'go'));
+
+        return lodashRange(0, 2).flatMap((serviceNodeId) =>
+          services.map((service) => service.instance(`${serviceName}-${serviceNodeId}`))
+        );
+      });
+
+      const transactionGroupRange = [
+        ...lodashRange(0, 10).map((groupId) => `transaction-${groupId}`),
+        '_other',
+      ];
+
+      return range.ratePerMinute(60).generator((timestamp, timestampIndex) => {
+        return logger.perf(
+          'generate_events_for_timestamp ' + new Date(timestamp).toISOString(),
+          () => {
+            const events = instances.flatMap((instance) =>
+              transactionGroupRange.flatMap((groupId, groupIndex) => {
+                const duration = Math.round(
+                  (timestampIndex % MAX_BUCKETS) * BUCKET_SIZE + MIN_DURATION
+                );
+
+                return instance
+                  .transaction(groupId, TRANSACTION_TYPES[groupIndex % TRANSACTION_TYPES.length])
+                  .timestamp(timestamp)
+                  .duration(duration)
+                  .outcome('success' as const);
+              })
+            );
+
+            return events;
+          }
+        );
+      });
+    },
+  };
+};
+
+export default scenario;

x-pack/plugins/apm/common/es_fields/__snapshots__/es_fields.test.ts.snap

@@ -210,6 +210,8 @@ exports[`Error SERVICE_NAME 1`] = `"service name"`;
 
 exports[`Error SERVICE_NODE_NAME 1`] = `undefined`;
 
+exports[`Error SERVICE_OVERFLOW_COUNT 1`] = `undefined`;
+
 exports[`Error SERVICE_RUNTIME_NAME 1`] = `undefined`;
 
 exports[`Error SERVICE_RUNTIME_VERSION 1`] = `undefined`;
@@ -487,6 +489,8 @@ exports[`Span SERVICE_NAME 1`] = `"service name"`;
 
 exports[`Span SERVICE_NODE_NAME 1`] = `undefined`;
 
+exports[`Span SERVICE_OVERFLOW_COUNT 1`] = `undefined`;
+
 exports[`Span SERVICE_RUNTIME_NAME 1`] = `undefined`;
 
 exports[`Span SERVICE_RUNTIME_VERSION 1`] = `undefined`;
@@ -782,6 +786,8 @@ exports[`Transaction SERVICE_NAME 1`] = `"service name"`;
 
 exports[`Transaction SERVICE_NODE_NAME 1`] = `undefined`;
 
+exports[`Transaction SERVICE_OVERFLOW_COUNT 1`] = `undefined`;
+
 exports[`Transaction SERVICE_RUNTIME_NAME 1`] = `undefined`;
 
 exports[`Transaction SERVICE_RUNTIME_VERSION 1`] = `undefined`;

x-pack/plugins/apm/common/es_fields/apm.ts

@@ -33,6 +33,8 @@ export const SERVICE_RUNTIME_VERSION = 'service.runtime.version';
 export const SERVICE_NODE_NAME = 'service.node.name';
 export const SERVICE_VERSION = 'service.version';
 export const SERVICE_TARGET_TYPE = 'service.target.type';
+export const SERVICE_OVERFLOW_COUNT =
+  'service_transaction.aggregation.overflow_count';
 
 export const URL_FULL = 'url.full';
 export const HTTP_REQUEST_METHOD = 'http.request.method';

x-pack/plugins/apm/common/service_inventory.ts

@@ -18,6 +18,7 @@ export interface ServiceListItem {
   transactionErrorRate?: number | null;
   environments?: string[];
   alertsCount?: number;
+  overflowCount?: number | null;
 }
 
 export enum ServiceInventoryFieldName {

x-pack/plugins/apm/public/components/app/dependency_detail_overview/dependencies_detail_table.tsx

@@ -12,7 +12,7 @@ import { getNodeName, NodeType } from '../../../../common/connections';
 import { useApmParams } from '../../../hooks/use_apm_params';
 import { useFetcher } from '../../../hooks/use_fetcher';
 import { DependenciesTable } from '../../shared/dependencies_table';
-import { ServiceLink } from '../../shared/service_link';
+import { ServiceLink } from '../../shared/links/apm/service_link';
 import { useTimeRange } from '../../../hooks/use_time_range';
 import { getComparisonEnabled } from '../../shared/time_comparison/get_comparison_enabled';
 import { useApmPluginContext } from '../../../context/apm_plugin/use_apm_plugin_context';

x-pack/plugins/apm/public/components/app/service_inventory/index.tsx

@@ -5,24 +5,31 @@
  * 2.0.
  */
 
-import { EuiFlexGroup, EuiFlexItem, EuiEmptyPrompt } from '@elastic/eui';
+import {
+  EuiCallOut,
+  EuiEmptyPrompt,
+  EuiFlexGroup,
+  EuiFlexItem,
+  EuiText,
+} from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
-import React from 'react';
+import { FormattedMessage } from '@kbn/i18n-react';
-import { v4 as uuidv4 } from 'uuid';
 import { useKibana } from '@kbn/kibana-react-plugin/public';
 import { apmServiceInventoryOptimizedSorting } from '@kbn/observability-plugin/common';
-import { isTimeComparison } from '../../shared/time_comparison/get_comparison_options';
+import React from 'react';
+import { v4 as uuidv4 } from 'uuid';
+import { ServiceInventoryFieldName } from '../../../../common/service_inventory';
+import { joinByKey } from '../../../../common/utils/join_by_key';
 import { useAnomalyDetectionJobsContext } from '../../../context/anomaly_detection_jobs/use_anomaly_detection_jobs_context';
-import { useLocalStorage } from '../../../hooks/use_local_storage';
 import { useApmParams } from '../../../hooks/use_apm_params';
 import { FETCH_STATUS, useFetcher } from '../../../hooks/use_fetcher';
-import { useTimeRange } from '../../../hooks/use_time_range';
+import { useLocalStorage } from '../../../hooks/use_local_storage';
-import { SearchBar } from '../../shared/search_bar';
-import { ServiceList } from './service_list';
-import { MLCallout, shouldDisplayMlCallout } from '../../shared/ml_callout';
 import { useProgressiveFetcher } from '../../../hooks/use_progressive_fetcher';
-import { joinByKey } from '../../../../common/utils/join_by_key';
+import { useTimeRange } from '../../../hooks/use_time_range';
-import { ServiceInventoryFieldName } from '../../../../common/service_inventory';
+import { MLCallout, shouldDisplayMlCallout } from '../../shared/ml_callout';
+import { SearchBar } from '../../shared/search_bar';
+import { isTimeComparison } from '../../shared/time_comparison/get_comparison_options';
+import { ServiceList } from './service_list';
 import { orderServiceItems } from './service_list/order_service_items';
 
 const initialData = {
@@ -202,6 +209,12 @@ export function ServiceInventory() {
     ...preloadedServices,
   ].some((item) => 'healthStatus' in item);
 
+  const hasKibanaUiLimitRestrictedData =
+    mainStatisticsFetch.data?.maxServiceCountExceeded;
+
+  const serviceOverflowCount =
+    mainStatisticsFetch.data?.serviceOverflowCount ?? 0;
+
   const displayAlerts = [...mainStatisticsItems, ...preloadedServices].some(
     (item) => ServiceInventoryFieldName.AlertsCount in item
   );
@@ -280,19 +293,45 @@ export function ServiceInventory() {
     'serviceName'
   );
 
+  const mlCallout = (
+    <EuiFlexItem>
+      <MLCallout
+        isOnSettingsPage={false}
+        anomalyDetectionSetupState={anomalyDetectionSetupState}
+        onDismiss={() => setUserHasDismissedCallout(true)}
+      />
+    </EuiFlexItem>
+  );
+
+  const kibanaUiServiceLimitCallout = (
+    <EuiFlexItem>
+      <EuiCallOut
+        title={i18n.translate(
+          'xpack.apm.serviceList.ui.limit.warning.calloutTitle',
+          {
+            defaultMessage:
+              'Number of services exceed the allowed maximum that are displayed (1,000)',
+          }
+        )}
+        color="warning"
+        iconType="alert"
+      >
+        <EuiText size="s">
+          <FormattedMessage
+            defaultMessage="Max. number of services that can be viewed in Kibana has been reached. Try narrowing down results by using the query bar or consider using service groups."
+            id="xpack.apm.serviceList.ui.limit.warning.calloutDescription"
+          />
+        </EuiText>
+      </EuiCallOut>
+    </EuiFlexItem>
+  );
+
   return (
     <>
       <SearchBar showTimeComparison />
       <EuiFlexGroup direction="column" gutterSize="m">
-        {displayMlCallout && (
+        {displayMlCallout && mlCallout}
-          <EuiFlexItem>
+        {hasKibanaUiLimitRestrictedData && kibanaUiServiceLimitCallout}
-            <MLCallout
-              isOnSettingsPage={false}
-              anomalyDetectionSetupState={anomalyDetectionSetupState}
-              onDismiss={() => setUserHasDismissedCallout(true)}
-            />
-          </EuiFlexItem>
-        )}
         <EuiFlexItem>
           <ServiceList
             isLoading={isLoading}
@@ -316,6 +355,7 @@ export function ServiceInventory() {
             comparisonData={comparisonFetch?.data}
             noItemsMessage={noItemsMessage}
             initialPageSize={INITIAL_PAGE_SIZE}
+            serviceOverflowCount={serviceOverflowCount}
           />
         </EuiFlexItem>
       </EuiFlexGroup>

x-pack/plugins/apm/public/components/app/service_inventory/service_list/__fixtures__/service_api_mock_data.ts

@@ -29,3 +29,33 @@ export const items: ServiceListAPIResponse['items'] = [
     environments: [],
   },
 ];
+
+export const overflowItems: ServiceListAPIResponse['items'] = [
+  {
+    serviceName: '_other',
+    transactionType: 'page-load',
+    agentName: 'python',
+    throughput: 86.93333333333334,
+    transactionErrorRate: 12.6,
+    latency: 91535.42944785276,
+    environments: [],
+  },
+  {
+    serviceName: 'opbeans-node',
+    transactionType: 'request',
+    agentName: 'nodejs',
+    throughput: 0,
+    transactionErrorRate: 46.06666666666667,
+    latency: null,
+    environments: ['test'],
+  },
+  {
+    serviceName: 'opbeans-python',
+    transactionType: 'page-load',
+    agentName: 'python',
+    throughput: 86.93333333333334,
+    transactionErrorRate: 12.6,
+    latency: 91535.42944785276,
+    environments: [],
+  },
+];

x-pack/plugins/apm/public/components/app/service_inventory/service_list/index.tsx

@@ -17,7 +17,6 @@ import {
 import { i18n } from '@kbn/i18n';
 import { TypeOf } from '@kbn/typed-react-router-config';
 import React, { useMemo } from 'react';
-import { NOT_AVAILABLE_LABEL } from '../../../../../common/i18n';
 import { ServiceHealthStatus } from '../../../../../common/service_health_status';
 import {
   ServiceInventoryFieldName,
@@ -47,17 +46,12 @@ import {
 import { EnvironmentBadge } from '../../../shared/environment_badge';
 import { ListMetric } from '../../../shared/list_metric';
 import { ITableColumn, ManagedTable } from '../../../shared/managed_table';
-import { ServiceLink } from '../../../shared/service_link';
+import { ServiceLink } from '../../../shared/links/apm/service_link';
-import { TruncateWithTooltip } from '../../../shared/truncate_with_tooltip';
 import { HealthBadge } from './health_badge';
 
 type ServicesDetailedStatisticsAPIResponse =
   APIReturnType<'POST /internal/apm/services/detailed_statistics'>;
 
-function formatString(value?: string | null) {
-  return value || NOT_AVAILABLE_LABEL;
-}
-
 export function getServiceColumns({
   query,
   showTransactionTypeColumn,
@@ -67,6 +61,7 @@ export function getServiceColumns({
   showHealthStatusColumn,
   showAlertsColumn,
   link,
+  serviceOverflowCount,
 }: {
   query: TypeOf<ApmRoutes, '/services'>['query'];
   showTransactionTypeColumn: boolean;
@@ -76,6 +71,7 @@ export function getServiceColumns({
   breakpoints: Breakpoints;
   comparisonData?: ServicesDetailedStatisticsAPIResponse;
   link: any;
+  serviceOverflowCount: number;
 }): Array<ITableColumn<ServiceListItem>> {
   const { isSmall, isLarge, isXl } = breakpoints;
   const showWhenSmallOrGreaterThanLarge = isSmall || !isLarge;
@@ -136,16 +132,11 @@ export function getServiceColumns({
       }),
       sortable: true,
       render: (_, { serviceName, agentName, transactionType }) => (
-        <TruncateWithTooltip
+        <ServiceLink
-          data-test-subj="apmServiceListAppLink"
+          agentName={agentName}
-          text={formatString(serviceName)}
+          query={{ ...query, transactionType }}
-          content={
+          serviceName={serviceName}
-            <ServiceLink
+          serviceOverflowCount={serviceOverflowCount}
-              agentName={agentName}
-              query={{ ...query, transactionType }}
-              serviceName={serviceName}
-            />
-          }
         />
       ),
     },
@@ -285,8 +276,9 @@ interface Props {
     sortField: ServiceInventoryFieldName,
     sortDirection: 'asc' | 'desc'
   ) => ServiceListItem[];
-}
 
+  serviceOverflowCount: number;
+}
 export function ServiceList({
   items,
   noItemsMessage,
@@ -300,6 +292,7 @@ export function ServiceList({
   initialSortDirection,
   initialPageSize,
   sortFn,
+  serviceOverflowCount,
 }: Props) {
   const breakpoints = useBreakpoints();
   const { link } = useApmRouter();
@@ -337,6 +330,7 @@ export function ServiceList({
         showHealthStatusColumn: displayHealthStatus,
         showAlertsColumn: displayAlerts,
         link,
+        serviceOverflowCount,
       }),
     [
       query,
@@ -347,6 +341,7 @@ export function ServiceList({
       displayHealthStatus,
       displayAlerts,
       link,
+      serviceOverflowCount,
     ]
   );
 

x-pack/plugins/apm/public/components/app/service_inventory/service_list/order_service_items.test.ts

@@ -191,40 +191,130 @@ describe('orderServiceItems', () => {
   });
 
   describe('when sorting by alphabetical fields', () => {
-    const sortedItems = orderServiceItems({
+    it('sorts correctly', () => {
-      primarySortField: ServiceInventoryFieldName.ServiceName,
+      const sortedItems = orderServiceItems({
-      sortDirection: 'asc',
+        primarySortField: ServiceInventoryFieldName.ServiceName,
-      tiebreakerField: ServiceInventoryFieldName.ServiceName,
+        sortDirection: 'asc',
-      items: [
+        tiebreakerField: ServiceInventoryFieldName.ServiceName,
-        {
+        items: [
-          serviceName: 'd-service',
+          {
-          healthStatus: ServiceHealthStatus.unknown,
+            serviceName: 'd-service',
-        },
+            healthStatus: ServiceHealthStatus.unknown,
-        {
+          },
-          serviceName: 'a-service',
+          {
-          healthStatus: ServiceHealthStatus.unknown,
+            serviceName: 'a-service',
-        },
+            healthStatus: ServiceHealthStatus.unknown,
-        {
+          },
-          serviceName: 'b-service',
+          {
-          healthStatus: ServiceHealthStatus.unknown,
+            serviceName: 'b-service',
-        },
+            healthStatus: ServiceHealthStatus.unknown,
-        {
+          },
-          serviceName: 'c-service',
+          {
-          healthStatus: ServiceHealthStatus.unknown,
+            serviceName: 'c-service',
-        },
+            healthStatus: ServiceHealthStatus.unknown,
-        {
+          },
-          serviceName: '0-service',
+          {
-          healthStatus: ServiceHealthStatus.unknown,
+            serviceName: '0-service',
-        },
+            healthStatus: ServiceHealthStatus.unknown,
-      ],
+          },
+        ],
+      });
+
+      expect(sortedItems.map((item) => item.serviceName)).toEqual([
+        '0-service',
+        'a-service',
+        'b-service',
+        'c-service',
+        'd-service',
+      ]);
+    });
+  });
+
+  describe('when sorting by alphabetical fields, service overflow bucket always appears 1st', () => {
+    it('asc', () => {
+      const sortedItems = orderServiceItems({
+        primarySortField: ServiceInventoryFieldName.ServiceName,
+        sortDirection: 'asc',
+        tiebreakerField: ServiceInventoryFieldName.ServiceName,
+        items: [
+          {
+            serviceName: 'd-service',
+            healthStatus: ServiceHealthStatus.unknown,
+          },
+          {
+            serviceName: 'a-service',
+            healthStatus: ServiceHealthStatus.unknown,
+          },
+          {
+            serviceName: 'b-service',
+            healthStatus: ServiceHealthStatus.unknown,
+          },
+          {
+            serviceName: 'c-service',
+            healthStatus: ServiceHealthStatus.unknown,
+          },
+          {
+            serviceName: '0-service',
+            healthStatus: ServiceHealthStatus.unknown,
+          },
+          {
+            serviceName: '_other',
+            healthStatus: ServiceHealthStatus.unknown,
+          },
+        ],
+      });
+
+      expect(sortedItems.map((item) => item.serviceName)).toEqual([
+        '_other',
+        '0-service',
+        'a-service',
+        'b-service',
+        'c-service',
+        'd-service',
+      ]);
     });
 
-    expect(sortedItems.map((item) => item.serviceName)).toEqual([
+    it('desc', () => {
-      '0-service',
+      const sortedItems = orderServiceItems({
-      'a-service',
+        primarySortField: ServiceInventoryFieldName.ServiceName,
-      'b-service',
+        sortDirection: 'desc',
-      'c-service',
+        tiebreakerField: ServiceInventoryFieldName.ServiceName,
-      'd-service',
+        items: [
-    ]);
+          {
+            serviceName: 'd-service',
+            healthStatus: ServiceHealthStatus.unknown,
+          },
+          {
+            serviceName: 'a-service',
+            healthStatus: ServiceHealthStatus.unknown,
+          },
+          {
+            serviceName: 'b-service',
+            healthStatus: ServiceHealthStatus.unknown,
+          },
+          {
+            serviceName: 'c-service',
+            healthStatus: ServiceHealthStatus.unknown,
+          },
+          {
+            serviceName: '0-service',
+            healthStatus: ServiceHealthStatus.unknown,
+          },
+          {
+            serviceName: '_other',
+            healthStatus: ServiceHealthStatus.unknown,
+          },
+        ],
+      });
+
+      expect(sortedItems.map((item) => item.serviceName)).toEqual([
+        '_other',
+        'd-service',
+        'c-service',
+        'b-service',
+        'a-service',
+        '0-service',
+      ]);
+    });
   });
 });

x-pack/plugins/apm/public/components/app/service_inventory/service_list/order_service_items.ts

@@ -10,6 +10,7 @@ import {
   ServiceListItem,
   ServiceInventoryFieldName,
 } from '../../../../../common/service_inventory';
+import { OTHER_SERVICE_NAME } from '../../../shared/links/apm/service_link/service_max_groups_message';
 
 type SortValueGetter = (item: ServiceListItem) => string | number;
 
@@ -56,6 +57,11 @@ export function orderServiceItems({
   // For healthStatus, sort items by healthStatus first, then by tie-breaker
 
   const sortFn = sorts[primarySortField as ServiceInventoryFieldName];
+  const sortOtherBucketFirst = (item: ServiceListItem) => {
+    return item.serviceName === OTHER_SERVICE_NAME ? -1 : 0;
+  };
+
+  const sortOtherBucketFirstDirection = 'asc';
 
   if (primarySortField === ServiceInventoryFieldName.HealthStatus) {
     const tiebreakerSortDirection =
@@ -67,10 +73,13 @@ export function orderServiceItems({
 
     return orderBy(
       items,
-      [sortFn, tiebreakerSortFn],
+      [sortOtherBucketFirst, sortFn, tiebreakerSortFn],
-      [sortDirection, tiebreakerSortDirection]
+      [sortOtherBucketFirstDirection, sortDirection, tiebreakerSortDirection]
     );
   }
-
+  return orderBy(
-  return orderBy(items, sortFn, sortDirection);
+    items,
+    [sortOtherBucketFirst, sortFn],
+    [sortOtherBucketFirstDirection, sortDirection]
+  );
 }

x-pack/plugins/apm/public/components/app/service_inventory/service_list/service_list.stories.tsx

@@ -13,7 +13,7 @@ import { ServiceHealthStatus } from '../../../../../common/service_health_status
 import { ServiceInventoryFieldName } from '../../../../../common/service_inventory';
 import type { ApmPluginContextValue } from '../../../../context/apm_plugin/apm_plugin_context';
 import { MockApmPluginStorybook } from '../../../../context/apm_plugin/mock_apm_plugin_storybook';
-import { items } from './__fixtures__/service_api_mock_data';
+import { items, overflowItems } from './__fixtures__/service_api_mock_data';
 
 type Args = ComponentProps<typeof ServiceList>;
 
@@ -81,3 +81,17 @@ WithHealthWarnings.args = {
   })),
   sortFn: (sortItems) => sortItems,
 };
+
+export const WithOverflowBucket: Story<Args> = (args) => {
+  return <ServiceList {...args} />;
+};
+
+WithOverflowBucket.args = {
+  isLoading: false,
+  items: overflowItems,
+  displayHealthStatus: false,
+  initialSortField: ServiceInventoryFieldName.HealthStatus,
+  initialSortDirection: 'desc',
+  initialPageSize: 25,
+  sortFn: (sortItems) => sortItems,
+};

x-pack/plugins/apm/public/components/app/service_inventory/service_list/service_list.test.tsx

@@ -82,6 +82,7 @@ describe('ServiceList', () => {
           } as Breakpoints,
           showAlertsColumn: true,
           link: apmRouter.link,
+          serviceOverflowCount: 0,
         }).map((c) =>
           c.render ? c.render!(service[c.field!], service) : service[c.field!]
         );
@@ -122,6 +123,7 @@ describe('ServiceList', () => {
           } as Breakpoints,
           showAlertsColumn: true,
           link: apmRouter.link,
+          serviceOverflowCount: 0,
         }).map((c) =>
           c.render ? c.render!(service[c.field!], service) : service[c.field!]
         );
@@ -151,6 +153,7 @@ describe('ServiceList', () => {
             } as Breakpoints,
             showAlertsColumn: true,
             link: apmRouter.link,
+            serviceOverflowCount: 0,
           }).map((c) =>
             c.render ? c.render!(service[c.field!], service) : service[c.field!]
           );
@@ -190,6 +193,7 @@ describe('ServiceList', () => {
             } as Breakpoints,
             showAlertsColumn: true,
             link: apmRouter.link,
+            serviceOverflowCount: 0,
           }).map((c) =>
             c.render ? c.render!(service[c.field!], service) : service[c.field!]
           );
@@ -232,6 +236,7 @@ describe('ServiceList', () => {
         } as Breakpoints,
         showAlertsColumn: true,
         link: apmRouter.link,
+        serviceOverflowCount: 0,
       }).map((c) => c.field);
       expect(renderedColumns.includes('healthStatus')).toBeFalsy();
     });
@@ -251,6 +256,7 @@ describe('ServiceList', () => {
         } as Breakpoints,
         showAlertsColumn: true,
         link: apmRouter.link,
+        serviceOverflowCount: 0,
       }).map((c) => c.field);
       expect(renderedColumns.includes('healthStatus')).toBeTruthy();
     });
@@ -270,6 +276,7 @@ describe('ServiceList', () => {
         } as Breakpoints,
         showAlertsColumn: false,
         link: apmRouter.link,
+        serviceOverflowCount: 0,
       }).map((c) => c.field);
       expect(renderedColumns.includes('alertsCount')).toBeFalsy();
     });
@@ -289,6 +296,7 @@ describe('ServiceList', () => {
         } as Breakpoints,
         showAlertsColumn: true,
         link: apmRouter.link,
+        serviceOverflowCount: 0,
       }).map((c) => c.field);
       expect(renderedColumns.includes('alertsCount')).toBeTruthy();
     });

x-pack/plugins/apm/public/components/app/service_overview/service_overview_dependencies_table/index.tsx

@@ -18,7 +18,7 @@ import { useFetcher } from '../../../../hooks/use_fetcher';
 import { useTimeRange } from '../../../../hooks/use_time_range';
 import { DependencyLink } from '../../../shared/dependency_link';
 import { DependenciesTable } from '../../../shared/dependencies_table';
-import { ServiceLink } from '../../../shared/service_link';
+import { ServiceLink } from '../../../shared/links/apm/service_link';
 
 interface ServiceOverviewDependenciesTableProps {
   fixedHeight?: boolean;

x-pack/plugins/apm/public/components/app/settings/agent_explorer/agent_instances/agent_contextual_information/index.tsx

@@ -13,7 +13,7 @@ import { AgentName } from '../../../../../../../typings/es_schemas/ui/fields/age
 import { useApmPluginContext } from '../../../../../../context/apm_plugin/use_apm_plugin_context';
 import { useDefaultTimeRange } from '../../../../../../hooks/use_default_time_range';
 import { ApmRoutes } from '../../../../../routing/apm_route_config';
-import { ServiceLink } from '../../../../../shared/service_link';
+import { ServiceLink } from '../../../../../shared/links/apm/service_link';
 import { StickyProperties } from '../../../../../shared/sticky_properties';
 import { getComparisonEnabled } from '../../../../../shared/time_comparison/get_comparison_enabled';
 import { TruncateWithTooltip } from '../../../../../shared/truncate_with_tooltip';

x-pack/plugins/apm/public/components/app/storage_explorer/services_table/index.tsx

@@ -23,7 +23,7 @@ import { apmServiceInventoryOptimizedSorting } from '@kbn/observability-plugin/c
 import { AgentName } from '../../../../../typings/es_schemas/ui/fields/agent';
 import { EnvironmentBadge } from '../../../shared/environment_badge';
 import { asPercent } from '../../../../../common/utils/formatters';
-import { ServiceLink } from '../../../shared/service_link';
+import { ServiceLink } from '../../../shared/links/apm/service_link';
 import { TruncateWithTooltip } from '../../../shared/truncate_with_tooltip';
 import { StorageDetailsPerService } from './storage_details_per_service';
 import { getComparisonEnabled } from '../../../shared/time_comparison/get_comparison_enabled';

x-pack/plugins/apm/public/components/app/top_traces_overview/trace_list.tsx

@@ -23,7 +23,7 @@ import { EmptyMessage } from '../../shared/empty_message';
 import { ImpactBar } from '../../shared/impact_bar';
 import { TransactionDetailLink } from '../../shared/links/apm/transaction_detail_link';
 import { ITableColumn, ManagedTable } from '../../shared/managed_table';
-import { ServiceLink } from '../../shared/service_link';
+import { ServiceLink } from '../../shared/links/apm/service_link';
 import { TruncateWithTooltip } from '../../shared/truncate_with_tooltip';
 import { NOT_AVAILABLE_LABEL } from '../../../../common/i18n';
 

x-pack/plugins/apm/public/components/app/transaction_details/waterfall_with_summary/waterfall_container/waterfall/flyout_top_level_properties.tsx

@@ -16,7 +16,7 @@ import { LatencyAggregationType } from '../../../../../../../common/latency_aggr
 import { Transaction } from '../../../../../../../typings/es_schemas/ui/transaction';
 import { useAnyOfApmParams } from '../../../../../../hooks/use_apm_params';
 import { TransactionDetailLink } from '../../../../../shared/links/apm/transaction_detail_link';
-import { ServiceLink } from '../../../../../shared/service_link';
+import { ServiceLink } from '../../../../../shared/links/apm/service_link';
 import { StickyProperties } from '../../../../../shared/sticky_properties';
 
 interface Props {

x-pack/plugins/apm/public/components/app/transaction_details/waterfall_with_summary/waterfall_container/waterfall/span_flyout/sticky_span_properties.tsx

@@ -21,7 +21,7 @@ import { Transaction } from '../../../../../../../../typings/es_schemas/ui/trans
 import { useAnyOfApmParams } from '../../../../../../../hooks/use_apm_params';
 import { DependencyLink } from '../../../../../../shared/dependency_link';
 import { TransactionDetailLink } from '../../../../../../shared/links/apm/transaction_detail_link';
-import { ServiceLink } from '../../../../../../shared/service_link';
+import { ServiceLink } from '../../../../../../shared/links/apm/service_link';
 import { StickyProperties } from '../../../../../../shared/sticky_properties';
 import { LatencyAggregationType } from '../../../../../../../../common/latency_aggregation_types';
 

x-pack/plugins/apm/public/components/shared/links/apm/service_link/index.tsx

@@ -0,0 +1,107 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { EuiFlexGroup, EuiFlexItem, EuiLink, EuiText } from '@elastic/eui';
+import { i18n } from '@kbn/i18n';
+import { euiStyled } from '@kbn/kibana-react-plugin/common';
+import { TypeOf } from '@kbn/typed-react-router-config';
+import React from 'react';
+import { isMobileAgentName } from '../../../../../../common/agent_name';
+import { NOT_AVAILABLE_LABEL } from '../../../../../../common/i18n';
+import { AgentName } from '../../../../../../typings/es_schemas/ui/fields/agent';
+import { useApmRouter } from '../../../../../hooks/use_apm_router';
+import { truncate, unit } from '../../../../../utils/style';
+import { ApmRoutes } from '../../../../routing/apm_route_config';
+import { AgentIcon } from '../../../agent_icon';
+import { PopoverTooltip } from '../../../popover_tooltip';
+import { TruncateWithTooltip } from '../../../truncate_with_tooltip';
+import {
+  OTHER_SERVICE_NAME,
+  ServiceMaxGroupsMessage,
+} from './service_max_groups_message';
+
+const StyledLink = euiStyled(EuiLink)`${truncate('100%')};`;
+
+function formatString(value?: string | null) {
+  return value || NOT_AVAILABLE_LABEL;
+}
+
+interface ServiceLinkProps {
+  agentName?: AgentName;
+  query: TypeOf<ApmRoutes, '/services/{serviceName}/overview'>['query'];
+  serviceName: string;
+  serviceOverflowCount?: number;
+}
+export function ServiceLink({
+  agentName,
+  query,
+  serviceName,
+  serviceOverflowCount,
+}: ServiceLinkProps) {
+  const { link } = useApmRouter();
+
+  const serviceLink = isMobileAgentName(agentName)
+    ? '/mobile-services/{serviceName}/overview'
+    : '/services/{serviceName}/overview';
+
+  if (serviceName === OTHER_SERVICE_NAME) {
+    return (
+      <EuiFlexGroup alignItems="center" gutterSize="xs">
+        <EuiFlexItem grow={false}>
+          <EuiText
+            grow={false}
+            style={{ fontStyle: 'italic', fontSize: '1rem' }}
+          >
+            {i18n.translate('xpack.apm.serviceLink.otherBucketName', {
+              defaultMessage: 'Remaining Services',
+            })}
+          </EuiText>
+        </EuiFlexItem>
+        <EuiFlexItem>
+          <PopoverTooltip
+            ariaLabel={i18n.translate('xpack.apm.serviceLink.tooltip', {
+              defaultMessage:
+                'Number of services instrumented has reached the current capacity of the APM server',
+            })}
+            iconType="alert"
+          >
+            <EuiText style={{ width: `${unit * 28}px` }} size="s">
+              <ServiceMaxGroupsMessage
+                serviceOverflowCount={serviceOverflowCount}
+              />
+            </EuiText>
+          </PopoverTooltip>
+        </EuiFlexItem>
+      </EuiFlexGroup>
+    );
+  }
+
+  return (
+    <TruncateWithTooltip
+      data-test-subj="apmServiceListAppLink"
+      text={formatString(serviceName)}
+      content={
+        <StyledLink
+          data-test-subj={`serviceLink_${agentName}`}
+          href={link(serviceLink, {
+            path: { serviceName },
+            query,
+          })}
+        >
+          <EuiFlexGroup alignItems="center" gutterSize="s" responsive={false}>
+            <EuiFlexItem grow={false}>
+              <AgentIcon agentName={agentName} />
+            </EuiFlexItem>
+            <EuiFlexItem className="eui-textTruncate">
+              <span className="eui-textTruncate">{serviceName}</span>
+            </EuiFlexItem>
+          </EuiFlexGroup>
+        </StyledLink>
+      }
+    />
+  );
+}

x-pack/plugins/apm/public/components/shared/links/apm/service_link/service_link.stories.tsx

@@ -7,8 +7,8 @@
 
 import { Story } from '@storybook/react';
 import React, { ComponentProps, ComponentType } from 'react';
-import { MockApmPluginStorybook } from '../../context/apm_plugin/mock_apm_plugin_storybook';
+import { ServiceLink } from '.';
-import { ServiceLink } from './service_link';
+import { MockApmPluginStorybook } from '../../../../../context/apm_plugin/mock_apm_plugin_storybook';
 
 type Args = ComponentProps<typeof ServiceLink>;
 

x-pack/plugins/apm/public/components/shared/links/apm/service_link/service_max_groups_message.tsx

@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FormattedMessage } from '@kbn/i18n-react';
+import React from 'react';
+
+export const OTHER_SERVICE_NAME = '_other';
+
+export function ServiceMaxGroupsMessage({
+  serviceOverflowCount = 0,
+}: {
+  serviceOverflowCount?: number;
+}) {
+  return (
+    <FormattedMessage
+      defaultMessage="Number of services that have been instrumented has reached the current max. capacity that can be handled by the APM server. There are at least {serviceOverflowCount, plural, one {1 service} other {# services}} missing in this list. Please increase the memory allocated to APM server."
+      id="xpack.apm.serviceDetail.maxGroups.message"
+      values={{
+        serviceOverflowCount,
+      }}
+    />
+  );
+}

x-pack/plugins/apm/public/components/shared/popover_tooltip/index.tsx

@@ -10,13 +10,13 @@ import React, { useState } from 'react';
 
 interface PopoverTooltipProps {
   ariaLabel?: string;
-  children: React.ReactNode;
   iconType?: string;
+  children: React.ReactNode;
 }
 
 export function PopoverTooltip({
   ariaLabel,
-  iconType,
+  iconType = 'questionInCircle',
   children,
 }: PopoverTooltipProps) {
   const [isPopoverOpen, setIsPopoverOpen] = useState(false);
@@ -35,7 +35,7 @@ export function PopoverTooltip({
           }}
           size="xs"
           color="primary"
-          iconType={iconType ?? 'questionInCircle'}
+          iconType={iconType}
           style={{ height: 'auto' }}
         />
       }

x-pack/plugins/apm/public/components/shared/service_link.tsx

@@ -1,56 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import { EuiFlexGroup, EuiFlexItem, EuiLink } from '@elastic/eui';
-import React from 'react';
-import { TypeOf } from '@kbn/typed-react-router-config';
-import { euiStyled } from '@kbn/kibana-react-plugin/common';
-import { truncate } from '../../utils/style';
-import { useApmRouter } from '../../hooks/use_apm_router';
-import { AgentIcon } from './agent_icon';
-import { AgentName } from '../../../typings/es_schemas/ui/fields/agent';
-import { ApmRoutes } from '../routing/apm_route_config';
-import { isMobileAgentName } from '../../../common/agent_name';
-
-const StyledLink = euiStyled(EuiLink)`${truncate('100%')};`;
-
-interface ServiceLinkProps {
-  agentName?: AgentName;
-  query: TypeOf<ApmRoutes, '/services/{serviceName}/overview'>['query'];
-  serviceName: string;
-}
-
-export function ServiceLink({
-  agentName,
-  query,
-  serviceName,
-}: ServiceLinkProps) {
-  const { link } = useApmRouter();
-
-  const serviceLink = isMobileAgentName(agentName)
-    ? '/mobile-services/{serviceName}/overview'
-    : '/services/{serviceName}/overview';
-
-  return (
-    <StyledLink
-      data-test-subj={`serviceLink_${agentName}`}
-      href={link(serviceLink, {
-        path: { serviceName },
-        query,
-      })}
-    >
-      <EuiFlexGroup alignItems="center" gutterSize="s" responsive={false}>
-        <EuiFlexItem grow={false}>
-          <AgentIcon agentName={agentName} />
-        </EuiFlexItem>
-        <EuiFlexItem className="eui-textTruncate">
-          <span className="eui-textTruncate">{serviceName}</span>
-        </EuiFlexItem>
-      </EuiFlexGroup>
-    </StyledLink>
-  );
-}

x-pack/plugins/apm/public/components/shared/span_links/span_links_table.tsx

@@ -23,7 +23,7 @@ import { SpanLinkDetails } from '../../../../common/span_links';
 import { asDuration } from '../../../../common/utils/formatters';
 import { useAnyOfApmParams } from '../../../hooks/use_apm_params';
 import { useApmRouter } from '../../../hooks/use_apm_router';
-import { ServiceLink } from '../service_link';
+import { ServiceLink } from '../links/apm/service_link';
 import { getSpanIcon } from '../span_icon/get_span_icon';
 
 interface Props {

x-pack/plugins/apm/server/routes/services/__snapshots__/queries.test.ts.snap

@@ -99,6 +99,11 @@ Array [
       "aggs": Object {
         "sample": Object {
           "aggs": Object {
+            "service_overflow_count": Object {
+              "sum": Object {
+                "field": "service_transaction.aggregation.overflow_count",
+              },
+            },
             "services": Object {
               "aggs": Object {
                 "transactionType": Object {
@@ -142,7 +147,7 @@ Array [
               },
               "terms": Object {
                 "field": "service.name",
-                "size": 500,
+                "size": 1000,
               },
             },
           },
@@ -204,7 +209,7 @@ Array [
               },
               "terms": Object {
                 "field": "service.name",
-                "size": 500,
+                "size": 1000,
               },
             },
           },

x-pack/plugins/apm/server/routes/services/get_services/get_service_stats.ts

@@ -0,0 +1,167 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { kqlQuery, rangeQuery } from '@kbn/observability-plugin/server';
+import {
+  AGENT_NAME,
+  SERVICE_ENVIRONMENT,
+  SERVICE_NAME,
+  SERVICE_OVERFLOW_COUNT,
+  TRANSACTION_TYPE,
+} from '../../../../common/es_fields/apm';
+import {
+  TRANSACTION_PAGE_LOAD,
+  TRANSACTION_REQUEST,
+} from '../../../../common/transaction_types';
+import { environmentQuery } from '../../../../common/utils/environment_query';
+import { AgentName } from '../../../../typings/es_schemas/ui/fields/agent';
+import {
+  getDocumentTypeFilterForTransactions,
+  getDurationFieldForTransactions,
+  getProcessorEventForTransactions,
+} from '../../../lib/helpers/transactions';
+import { calculateThroughputWithRange } from '../../../lib/helpers/calculate_throughput';
+import {
+  calculateFailedTransactionRate,
+  getOutcomeAggregation,
+} from '../../../lib/helpers/transaction_error_rate';
+import { serviceGroupQuery } from '../../../lib/service_group_query';
+import { ServiceGroup } from '../../../../common/service_groups';
+import { RandomSampler } from '../../../lib/helpers/get_random_sampler';
+import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client';
+
+interface AggregationParams {
+  environment: string;
+  kuery: string;
+  apmEventClient: APMEventClient;
+  searchAggregatedTransactions: boolean;
+  maxNumServices: number;
+  start: number;
+  end: number;
+  serviceGroup: ServiceGroup | null;
+  randomSampler: RandomSampler;
+}
+
+export async function getServiceStats({
+  environment,
+  kuery,
+  apmEventClient,
+  searchAggregatedTransactions,
+  maxNumServices,
+  start,
+  end,
+  serviceGroup,
+  randomSampler,
+}: AggregationParams) {
+  const outcomes = getOutcomeAggregation();
+
+  const metrics = {
+    avg_duration: {
+      avg: {
+        field: getDurationFieldForTransactions(searchAggregatedTransactions),
+      },
+    },
+    outcomes,
+  };
+
+  const response = await apmEventClient.search('get_service_stats', {
+    apm: {
+      events: [getProcessorEventForTransactions(searchAggregatedTransactions)],
+    },
+    body: {
+      track_total_hits: false,
+      size: 0,
+      query: {
+        bool: {
+          filter: [
+            ...getDocumentTypeFilterForTransactions(
+              searchAggregatedTransactions
+            ),
+            ...rangeQuery(start, end),
+            ...environmentQuery(environment),
+            ...kqlQuery(kuery),
+            ...serviceGroupQuery(serviceGroup),
+          ],
+        },
+      },
+      aggs: {
+        sample: {
+          random_sampler: randomSampler,
+          aggs: {
+            service_overflow_count: {
+              sum: {
+                field: SERVICE_OVERFLOW_COUNT,
+              },
+            },
+            services: {
+              terms: {
+                field: SERVICE_NAME,
+                size: maxNumServices,
+              },
+              aggs: {
+                transactionType: {
+                  terms: {
+                    field: TRANSACTION_TYPE,
+                  },
+                  aggs: {
+                    ...metrics,
+                    environments: {
+                      terms: {
+                        field: SERVICE_ENVIRONMENT,
+                      },
+                    },
+                    sample: {
+                      top_metrics: {
+                        metrics: [{ field: AGENT_NAME } as const],
+                        sort: {
+                          '@timestamp': 'desc' as const,
+                        },
+                      },
+                    },
+                  },
+                },
+              },
+            },
+          },
+        },
+      },
+    },
+  });
+
+  return {
+    serviceStats:
+      response.aggregations?.sample.services.buckets.map((bucket) => {
+        const topTransactionTypeBucket =
+          bucket.transactionType.buckets.find(
+            ({ key }) =>
+              key === TRANSACTION_REQUEST || key === TRANSACTION_PAGE_LOAD
+          ) ?? bucket.transactionType.buckets[0];
+
+        return {
+          serviceName: bucket.key as string,
+          transactionType: topTransactionTypeBucket.key as string,
+          environments: topTransactionTypeBucket.environments.buckets.map(
+            (environmentBucket) => environmentBucket.key as string
+          ),
+          agentName: topTransactionTypeBucket.sample.top[0].metrics[
+            AGENT_NAME
+          ] as AgentName,
+          latency: topTransactionTypeBucket.avg_duration.value,
+          transactionErrorRate: calculateFailedTransactionRate(
+            topTransactionTypeBucket.outcomes
+          ),
+          throughput: calculateThroughputWithRange({
+            start,
+            end,
+            value: topTransactionTypeBucket.doc_count,
+          }),
+        };
+      }) ?? [],
+    serviceOverflowCount:
+      response.aggregations?.sample?.service_overflow_count.value || 0,
+  };
+}

x-pack/plugins/apm/server/routes/services/get_services/get_service_stats_for_service_metric.ts

@@ -11,6 +11,7 @@ import {
   AGENT_NAME,
   SERVICE_ENVIRONMENT,
   SERVICE_NAME,
+  SERVICE_OVERFLOW_COUNT,
   TRANSACTION_TYPE,
   TRANSACTION_DURATION_SUMMARY,
   TRANSACTION_FAILURE_COUNT,
@@ -40,7 +41,7 @@ interface AggregationParams {
   randomSampler: RandomSampler;
 }
 
-export async function getServiceAggregatedTransactionStats({
+export async function getServiceStatsForServiceMetrics({
   environment,
   kuery,
   apmEventClient,
@@ -51,7 +52,7 @@ export async function getServiceAggregatedTransactionStats({
   randomSampler,
 }: AggregationParams) {
   const response = await apmEventClient.search(
-    'get_service_aggregated_transaction_stats',
+    'get_service_stats_for_service_metric',
     {
       apm: {
         events: [ProcessorEvent.metric],
@@ -74,6 +75,11 @@ export async function getServiceAggregatedTransactionStats({
           sample: {
             random_sampler: randomSampler,
             aggs: {
+              overflowCount: {
+                sum: {
+                  field: SERVICE_OVERFLOW_COUNT,
+                },
+              },
               services: {
                 terms: {
                   field: SERVICE_NAME,
@@ -124,34 +130,39 @@ export async function getServiceAggregatedTransactionStats({
     }
   );
 
-  return (
+  return {
-    response.aggregations?.sample.services.buckets.map((bucket) => {
+    serviceStats:
-      const topTransactionTypeBucket =
+      response.aggregations?.sample.services.buckets.map((bucket) => {
-        bucket.transactionType.buckets.find(
+        const topTransactionTypeBucket =
-          ({ key }) =>
+          bucket.transactionType.buckets.find(
-            key === TRANSACTION_REQUEST || key === TRANSACTION_PAGE_LOAD
+            ({ key }) =>
-        ) ?? bucket.transactionType.buckets[0];
+              key === TRANSACTION_REQUEST || key === TRANSACTION_PAGE_LOAD
+          ) ?? bucket.transactionType.buckets[0];
 
-      return {
+        return {
-        serviceName: bucket.key as string,
+          serviceName: bucket.key as string,
-        transactionType: topTransactionTypeBucket.key as string,
+          transactionType: topTransactionTypeBucket.key as string,
-        environments: topTransactionTypeBucket.environments.buckets.map(
+          environments: topTransactionTypeBucket.environments.buckets.map(
-          (environmentBucket) => environmentBucket.key as string
+            (environmentBucket) => environmentBucket.key as string
-        ),
+          ),
-        agentName: topTransactionTypeBucket.sample.top[0].metrics[
+          agentName: topTransactionTypeBucket.sample.top[0].metrics[
-          AGENT_NAME
+            AGENT_NAME
-        ] as AgentName,
+          ] as AgentName,
-        latency: topTransactionTypeBucket.avg_duration.value,
+          latency: topTransactionTypeBucket.avg_duration.value,
-        transactionErrorRate: calculateFailedTransactionRateFromServiceMetrics({
+          transactionErrorRate:
-          failedTransactions: topTransactionTypeBucket.failure_count.value,
+            calculateFailedTransactionRateFromServiceMetrics({
-          successfulTransactions: topTransactionTypeBucket.success_count.value,
+              failedTransactions: topTransactionTypeBucket.failure_count.value,
-        }),
+              successfulTransactions:
-        throughput: calculateThroughputWithRange({
+                topTransactionTypeBucket.success_count.value,
-          start,
+            }),
-          end,
+          throughput: calculateThroughputWithRange({
-          value: topTransactionTypeBucket.doc_count,
+            start,
-        }),
+            end,
-      };
+            value: topTransactionTypeBucket.doc_count,
-    }) ?? []
+          }),
-  );
+        };
+      }) ?? [],
+    serviceOverflowCount:
+      response.aggregations?.sample?.overflowCount.value || 0,
+  };
 }

x-pack/plugins/apm/server/routes/services/get_services/get_service_transaction_stats.ts

@@ -1,163 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import { kqlQuery, rangeQuery } from '@kbn/observability-plugin/server';
-import {
-  AGENT_NAME,
-  SERVICE_ENVIRONMENT,
-  SERVICE_NAME,
-  TRANSACTION_TYPE,
-} from '../../../../common/es_fields/apm';
-import {
-  TRANSACTION_PAGE_LOAD,
-  TRANSACTION_REQUEST,
-} from '../../../../common/transaction_types';
-import { environmentQuery } from '../../../../common/utils/environment_query';
-import { AgentName } from '../../../../typings/es_schemas/ui/fields/agent';
-import {
-  getDocumentTypeFilterForTransactions,
-  getDurationFieldForTransactions,
-  getProcessorEventForTransactions,
-} from '../../../lib/helpers/transactions';
-import { calculateThroughputWithRange } from '../../../lib/helpers/calculate_throughput';
-import {
-  calculateFailedTransactionRate,
-  getOutcomeAggregation,
-} from '../../../lib/helpers/transaction_error_rate';
-import { serviceGroupQuery } from '../../../lib/service_group_query';
-import { ServiceGroup } from '../../../../common/service_groups';
-import { RandomSampler } from '../../../lib/helpers/get_random_sampler';
-import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client';
-
-interface AggregationParams {
-  environment: string;
-  kuery: string;
-  apmEventClient: APMEventClient;
-  searchAggregatedTransactions: boolean;
-  maxNumServices: number;
-  start: number;
-  end: number;
-  serviceGroup: ServiceGroup | null;
-  randomSampler: RandomSampler;
-}
-
-export async function getServiceTransactionStats({
-  environment,
-  kuery,
-  apmEventClient,
-  searchAggregatedTransactions,
-  maxNumServices,
-  start,
-  end,
-  serviceGroup,
-  randomSampler,
-}: AggregationParams) {
-  const outcomes = getOutcomeAggregation();
-
-  const metrics = {
-    avg_duration: {
-      avg: {
-        field: getDurationFieldForTransactions(searchAggregatedTransactions),
-      },
-    },
-    outcomes,
-  };
-
-  const response = await apmEventClient.search(
-    'get_service_transaction_stats',
-    {
-      apm: {
-        events: [
-          getProcessorEventForTransactions(searchAggregatedTransactions),
-        ],
-      },
-      body: {
-        track_total_hits: false,
-        size: 0,
-        query: {
-          bool: {
-            filter: [
-              ...getDocumentTypeFilterForTransactions(
-                searchAggregatedTransactions
-              ),
-              ...rangeQuery(start, end),
-              ...environmentQuery(environment),
-              ...kqlQuery(kuery),
-              ...serviceGroupQuery(serviceGroup),
-            ],
-          },
-        },
-        aggs: {
-          sample: {
-            random_sampler: randomSampler,
-            aggs: {
-              services: {
-                terms: {
-                  field: SERVICE_NAME,
-                  size: maxNumServices,
-                },
-                aggs: {
-                  transactionType: {
-                    terms: {
-                      field: TRANSACTION_TYPE,
-                    },
-                    aggs: {
-                      ...metrics,
-                      environments: {
-                        terms: {
-                          field: SERVICE_ENVIRONMENT,
-                        },
-                      },
-                      sample: {
-                        top_metrics: {
-                          metrics: [{ field: AGENT_NAME } as const],
-                          sort: {
-                            '@timestamp': 'desc' as const,
-                          },
-                        },
-                      },
-                    },
-                  },
-                },
-              },
-            },
-          },
-        },
-      },
-    }
-  );
-
-  return (
-    response.aggregations?.sample.services.buckets.map((bucket) => {
-      const topTransactionTypeBucket =
-        bucket.transactionType.buckets.find(
-          ({ key }) =>
-            key === TRANSACTION_REQUEST || key === TRANSACTION_PAGE_LOAD
-        ) ?? bucket.transactionType.buckets[0];
-
-      return {
-        serviceName: bucket.key as string,
-        transactionType: topTransactionTypeBucket.key as string,
-        environments: topTransactionTypeBucket.environments.buckets.map(
-          (environmentBucket) => environmentBucket.key as string
-        ),
-        agentName: topTransactionTypeBucket.sample.top[0].metrics[
-          AGENT_NAME
-        ] as AgentName,
-        latency: topTransactionTypeBucket.avg_duration.value,
-        transactionErrorRate: calculateFailedTransactionRate(
-          topTransactionTypeBucket.outcomes
-        ),
-        throughput: calculateThroughputWithRange({
-          start,
-          end,
-          value: topTransactionTypeBucket.doc_count,
-        }),
-      };
-    }) ?? []
-  );
-}

x-pack/plugins/apm/server/routes/services/get_services/get_services_from_error_and_metric_documents.ts

@@ -87,15 +87,18 @@ export async function getServicesFromErrorAndMetricDocuments({
     }
   );
 
-  return (
+  return {
-    response.aggregations?.sample.services.buckets.map((bucket) => {
+    services:
-      return {
+      response.aggregations?.sample.services.buckets.map((bucket) => {
-        serviceName: bucket.key as string,
+        return {
-        environments: bucket.environments.buckets.map(
+          serviceName: bucket.key as string,
-          (envBucket) => envBucket.key as string
+          environments: bucket.environments.buckets.map(
-        ),
+            (envBucket) => envBucket.key as string
-        agentName: bucket.latest.top[0].metrics[AGENT_NAME] as AgentName,
+          ),
-      };
+          agentName: bucket.latest.top[0].metrics[AGENT_NAME] as AgentName,
-    }) ?? []
+        };
-  );
+      }) ?? [],
+    maxServiceCountExceeded:
+      (response.aggregations?.sample.services.sum_other_doc_count ?? 0) > 0,
+  };
 }

x-pack/plugins/apm/server/routes/services/get_services/get_services_items.ts

@@ -10,8 +10,8 @@ import { withApmSpan } from '../../../utils/with_apm_span';
 import { MlClient } from '../../../lib/helpers/get_ml_client';
 import { getHealthStatuses } from './get_health_statuses';
 import { getServicesFromErrorAndMetricDocuments } from './get_services_from_error_and_metric_documents';
-import { getServiceTransactionStats } from './get_service_transaction_stats';
+import { getServiceStats } from './get_service_stats';
-import { getServiceAggregatedTransactionStats } from './get_service_aggregated_transaction_stats';
+import { getServiceStatsForServiceMetrics } from './get_service_stats_for_service_metric';
 import { mergeServiceStats } from './merge_service_stats';
 import { ServiceGroup } from '../../../../common/service_groups';
 import { RandomSampler } from '../../../lib/helpers/get_random_sampler';
@@ -19,7 +19,7 @@ import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm
 import { getServicesAlerts } from './get_service_alerts';
 import { ApmAlertsClient } from '../../../lib/helpers/get_apm_alerts_client';
 
-export const MAX_NUMBER_OF_SERVICES = 500;
+export const MAX_NUMBER_OF_SERVICES = 1_000;
 
 export async function getServicesItems({
   environment,
@@ -62,17 +62,17 @@ export async function getServicesItems({
     };
 
     const [
-      transactionStats,
+      { serviceStats, serviceOverflowCount },
-      servicesFromErrorAndMetricDocuments,
+      { services, maxServiceCountExceeded },
       healthStatuses,
       alertCounts,
     ] = await Promise.all([
       searchAggregatedServiceMetrics
-        ? getServiceAggregatedTransactionStats({
+        ? getServiceStatsForServiceMetrics({
             ...commonParams,
             apmEventClient,
           })
-        : getServiceTransactionStats({
+        : getServiceStats({
             ...commonParams,
             apmEventClient,
           }),
@@ -90,11 +90,16 @@ export async function getServicesItems({
       }),
     ]);
 
-    return mergeServiceStats({
+    return {
-      transactionStats,
+      items:
-      servicesFromErrorAndMetricDocuments,
+        mergeServiceStats({
-      healthStatuses,
+          serviceStats,
-      alertCounts,
+          servicesFromErrorAndMetricDocuments: services,
-    });
+          healthStatuses,
+          alertCounts,
+        }) ?? [],
+      maxServiceCountExceeded,
+      serviceOverflowCount,
+    };
   });
 }

x-pack/plugins/apm/server/routes/services/get_services/index.ts

@@ -42,23 +42,26 @@ export async function getServices({
   randomSampler: RandomSampler;
 }) {
   return withApmSpan('get_services', async () => {
-    const items = await getServicesItems({
+    const { items, maxServiceCountExceeded, serviceOverflowCount } =
-      environment,
+      await getServicesItems({
-      kuery,
+        environment,
-      mlClient,
+        kuery,
-      apmEventClient,
+        mlClient,
-      apmAlertsClient,
+        apmEventClient,
-      searchAggregatedTransactions,
+        apmAlertsClient,
-      searchAggregatedServiceMetrics,
+        searchAggregatedTransactions,
-      logger,
+        searchAggregatedServiceMetrics,
-      start,
+        logger,
-      end,
+        start,
-      serviceGroup,
+        end,
-      randomSampler,
+        serviceGroup,
-    });
+        randomSampler,
+      });
 
     return {
       items,
+      maxServiceCountExceeded,
+      serviceOverflowCount,
     };
   });
 }

x-pack/plugins/apm/server/routes/services/get_services/merge_service_stats.test.ts

@@ -5,12 +5,12 @@
  * 2.0.
  */
 import { ServiceHealthStatus } from '../../../../common/service_health_status';
-import { getServiceTransactionStats } from './get_service_transaction_stats';
+import { getServiceStats } from './get_service_stats';
 import { mergeServiceStats } from './merge_service_stats';
 
 type ServiceTransactionStat = Awaited<
-  ReturnType<typeof getServiceTransactionStats>
+  ReturnType<typeof getServiceStats>
->[number];
+>['serviceStats'][number];
 
 function stat(values: Partial<ServiceTransactionStat>): ServiceTransactionStat {
   return {
@@ -29,7 +29,7 @@ describe('mergeServiceStats', () => {
   it('joins stats by service name', () => {
     expect(
       mergeServiceStats({
-        transactionStats: [
+        serviceStats: [
           stat({
             serviceName: 'opbeans-java',
             environments: ['production'],
@@ -87,7 +87,7 @@ describe('mergeServiceStats', () => {
   it('shows services that only have metric documents', () => {
     expect(
       mergeServiceStats({
-        transactionStats: [
+        serviceStats: [
           stat({
             serviceName: 'opbeans-java-2',
             environments: ['staging'],
@@ -136,7 +136,7 @@ describe('mergeServiceStats', () => {
   it('does not show services that only have ML data', () => {
     expect(
       mergeServiceStats({
-        transactionStats: [
+        serviceStats: [
           stat({
             serviceName: 'opbeans-java-2',
             environments: ['staging'],
@@ -173,7 +173,7 @@ describe('mergeServiceStats', () => {
   it('concatenates environments from metric/transaction data', () => {
     expect(
       mergeServiceStats({
-        transactionStats: [
+        serviceStats: [
           stat({
             serviceName: 'opbeans-java',
             environments: ['staging'],

x-pack/plugins/apm/server/routes/services/get_services/merge_service_stats.ts

@@ -10,24 +10,22 @@ import { joinByKey } from '../../../../common/utils/join_by_key';
 import { getServicesAlerts } from './get_service_alerts';
 import { getHealthStatuses } from './get_health_statuses';
 import { getServicesFromErrorAndMetricDocuments } from './get_services_from_error_and_metric_documents';
-import { getServiceTransactionStats } from './get_service_transaction_stats';
+import { getServiceStats } from './get_service_stats';
 
 export function mergeServiceStats({
-  transactionStats,
+  serviceStats,
   servicesFromErrorAndMetricDocuments,
   healthStatuses,
   alertCounts,
 }: {
-  transactionStats: Awaited<ReturnType<typeof getServiceTransactionStats>>;
+  serviceStats: Awaited<ReturnType<typeof getServiceStats>>['serviceStats'];
   servicesFromErrorAndMetricDocuments: Awaited<
     ReturnType<typeof getServicesFromErrorAndMetricDocuments>
-  >;
+  >['services'];
   healthStatuses: Awaited<ReturnType<typeof getHealthStatuses>>;
   alertCounts: Awaited<ReturnType<typeof getServicesAlerts>>;
 }) {
-  const foundServiceNames = transactionStats.map(
+  const foundServiceNames = serviceStats.map(({ serviceName }) => serviceName);
-    ({ serviceName }) => serviceName
-  );
 
   const servicesWithOnlyMetricDocuments =
     servicesFromErrorAndMetricDocuments.filter(
@@ -46,7 +44,7 @@ export function mergeServiceStats({
 
   return joinByKey(
     asMutableArray([
-      ...transactionStats,
+      ...serviceStats,
       ...servicesFromErrorAndMetricDocuments,
       ...matchedHealthStatuses,
       ...alertCounts,

x-pack/plugins/apm/server/routes/services/route.ts

@@ -116,6 +116,8 @@ const servicesRoute = createApmServerRoute({
         alertsCount: number;
       }
     >;
+    maxServiceCountExceeded: boolean;
+    serviceOverflowCount: number;
   }> {
     const {
       config,

x-pack/test/apm_api_integration/tests/services/top_services.spec.ts

@@ -51,6 +51,8 @@ export default function ApiTest({ getService }: FtrProviderContext) {
 
         expect(response.status).to.be(200);
         expect(response.body.items.length).to.be(0);
+        expect(response.body.maxServiceCountExceeded).to.be(false);
+        expect(response.body.serviceOverflowCount).to.be(0);
       });
     }
   );