[APM] Diagnostics Tool: Communicate that CCS is not supported (#160730)

It turns out that most of the APIs used by the diagnostics tool is not
supported by CCS. For now I'm just handling the errors and communicating
to the user that cross cluster search is not supported.

<img width="1452" alt="image"
src="df55cdbf-88d6-41a0-a760-0d21e57660c2">

x-pack/plugins/apm/public/components/app/diagnostics/apm_documents_tab.tsx

@@ -11,7 +11,7 @@ import {
   EuiBasicTableColumn,
   EuiSpacer,
 } from '@elastic/eui';
-import React, { useState } from 'react';
+import React, { useState, useMemo } from 'react';
 import { useKibana } from '@kbn/kibana-react-plugin/public';
 import { orderBy } from 'lodash';
 import { useApmParams } from '../../../hooks/use_apm_params';
@@ -22,6 +22,14 @@ import { useDiagnosticsContext } from './context/use_diagnostics';
 import { ApmPluginStartDeps } from '../../../plugin';
 import { SearchBar } from '../../shared/search_bar/search_bar';
 
+function formatDocCount(count?: number) {
+  if (count === undefined) {
+    return '-';
+  }
+
+  return asInteger(count);
+}
+
 export function DiagnosticsApmDocuments() {
   const { diagnosticsBundle, isImported } = useDiagnosticsContext();
   const { discover } = useKibana<ApmPluginStartDeps>().services;
@@ -31,7 +39,20 @@ export function DiagnosticsApmDocuments() {
     query: { rangeFrom, rangeTo },
   } = useApmParams('/diagnostics/documents');
 
-  const items = diagnosticsBundle?.apmEvents ?? [];
+  const items = useMemo<ApmEvent[]>(() => {
+    return (
+      diagnosticsBundle?.apmEvents.filter(({ legacy, docCount, intervals }) => {
+        const isLegacyAndUnused =
+          legacy === true &&
+          docCount === 0 &&
+          intervals &&
+          Object.values(intervals).every((interval) => interval === 0);
+
+        return !isLegacyAndUnused;
+      }) ?? []
+    );
+  }, [diagnosticsBundle?.apmEvents]);
+
   const columns: Array<EuiBasicTableColumn<ApmEvent>> = [
     {
       name: 'Name',
@@ -48,24 +69,24 @@ export function DiagnosticsApmDocuments() {
       name: '1m',
       field: 'intervals.1m',
       render: (_, { intervals }) => {
-        const interval = intervals?.['1m'];
-        return interval ? asInteger(interval) : '-';
+        const docCount = intervals?.['1m'];
+        return formatDocCount(docCount);
       },
     },
     {
       name: '10m',
       field: 'intervals.10m',
       render: (_, { intervals }) => {
-        const interval = intervals?.['10m'];
-        return interval ? asInteger(interval) : '-';
+        const docCount = intervals?.['10m'];
+        return formatDocCount(docCount);
       },
     },
     {
       name: '60m',
       field: 'intervals.60m',
       render: (_, { intervals }) => {
-        const interval = intervals?.['60m'];
-        return interval ? asInteger(interval) : '-';
+        const docCount = intervals?.['60m'];
+        return formatDocCount(docCount);
       },
     },
     {

x-pack/plugins/apm/public/components/app/diagnostics/context/diagnostics_context.tsx

@@ -39,6 +39,7 @@ export function DiagnosticsContextProvider({
   const { data, status, refetch } = useFetcher(
     (callApmApi) => {
       return callApmApi(`GET /internal/apm/diagnostics`, {
+        isCachable: false,
         params: {
           query: {
             start,

x-pack/plugins/apm/public/components/app/diagnostics/index_pattern_settings_tab.tsx

@@ -36,7 +36,7 @@ export function DiagnosticsIndexPatternSettings() {
     !indexTemplatesByIndexPattern ||
     indexTemplatesByIndexPattern?.length === 0
   ) {
-    return null;
+    return <EuiText>No settings to display</EuiText>;
   }
 
   const elms = indexTemplatesByIndexPattern.map(

x-pack/plugins/apm/public/components/app/diagnostics/summary_tab/index.tsx

@@ -6,13 +6,30 @@
  */
 
 import React from 'react';
-import { EuiFlexGroup } from '@elastic/eui';
+import { EuiFlexGroup, EuiCallOut } from '@elastic/eui';
 import { ApmIntegrationPackageStatus } from './apm_integration_package_status';
 import { IndexTemplatesStatus } from './index_templates_status';
 import { FieldMappingStatus } from './indicies_status';
 import { DataStreamsStatus } from './data_streams_status';
+import { useDiagnosticsContext } from '../context/use_diagnostics';
 
 export function DiagnosticsSummary() {
+  const { diagnosticsBundle } = useDiagnosticsContext();
+
+  const isCrossCluster = Object.values(
+    diagnosticsBundle?.apmIndices ?? {}
+  ).some((indicies) => indicies.includes(':'));
+
+  if (isCrossCluster) {
+    return (
+      <EuiCallOut title="Cross cluster search not supported" color="warning">
+        The APM index settings is targetting remote clusters. Please note: this
+        is not currently supported by the Diagnostics Tool and functionality
+        will therefore be limited.
+      </EuiCallOut>
+    );
+  }
+
   return (
     <EuiFlexGroup direction="column">
       <ApmIntegrationPackageStatus />

x-pack/plugins/apm/server/routes/diagnostics/bundle/get_apm_events.ts

@@ -7,6 +7,7 @@
 
 import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
 import { kqlQuery, rangeQuery } from '@kbn/observability-plugin/server';
+import { merge } from 'lodash';
 import {
   PROCESSOR_EVENT,
   METRICSET_NAME,
@@ -18,6 +19,7 @@ import { getTypedSearch, TypedSearch } from '../create_typed_es_client';
 import { getApmIndexPatterns } from './get_indices';
 
 export interface ApmEvent {
+  legacy?: boolean;
   name: string;
   kuery: string;
   index: string[];
@@ -53,7 +55,7 @@ export async function getApmEvents({
     }),
     getEventWithMetricsetInterval({
       ...commonProps,
-      name: 'Metric: Service transaction (with summary field)',
+      name: 'Metric: Service transaction (8.7+)',
       index: getApmIndexPatterns([apmIndices.metric]),
       kuery: mergeKueries(
         `${PROCESSOR_EVENT}: "metric" AND ${METRICSET_NAME}: "service_transaction" AND ${TRANSACTION_DURATION_SUMMARY} :* `,
@@ -62,7 +64,7 @@ export async function getApmEvents({
     }),
     getEventWithMetricsetInterval({
       ...commonProps,
-      name: 'Metric: Transaction (with summary field)',
+      name: 'Metric: Transaction (8.7+)',
       index: getApmIndexPatterns([apmIndices.metric]),
       kuery: mergeKueries(
         `${PROCESSOR_EVENT}: "metric" AND ${METRICSET_NAME}: "transaction" AND ${TRANSACTION_DURATION_SUMMARY} :* `,
@@ -71,7 +73,8 @@ export async function getApmEvents({
     }),
     getEventWithMetricsetInterval({
       ...commonProps,
-      name: 'Metric: Service transaction (without summary field)',
+      legacy: true,
+      name: 'Metric: Service transaction (pre-8.7)',
       index: getApmIndexPatterns([apmIndices.metric]),
       kuery: mergeKueries(
         `${PROCESSOR_EVENT}: "metric" AND ${METRICSET_NAME}: "service_transaction" AND not ${TRANSACTION_DURATION_SUMMARY} :* `,
@@ -80,7 +83,8 @@ export async function getApmEvents({
     }),
     getEventWithMetricsetInterval({
       ...commonProps,
-      name: 'Metric: Transaction (without summary field)',
+      legacy: true,
+      name: 'Metric: Transaction (pre-8.7)',
       index: getApmIndexPatterns([apmIndices.metric]),
       kuery: mergeKueries(
         `${PROCESSOR_EVENT}: "metric" AND ${METRICSET_NAME}: "transaction" AND not ${TRANSACTION_DURATION_SUMMARY} :* `,
@@ -129,6 +133,7 @@ export async function getApmEvents({
 }
 
 async function getEventWithMetricsetInterval({
+  legacy,
   name,
   index,
   start,
@@ -136,6 +141,7 @@ async function getEventWithMetricsetInterval({
   kuery,
   typedSearch,
 }: {
+  legacy?: boolean;
   name: string;
   index: string[];
   start: number;
@@ -163,17 +169,23 @@ async function getEventWithMetricsetInterval({
     },
   });
 
+  const defaultIntervals = { '1m': 0, '10m': 0, '60m': 0 };
+  const foundIntervals = res.aggregations?.metricset_intervals.buckets.reduce<
+    Record<string, number>
+  >((acc, item) => {
+    acc[item.key] = item.doc_count;
+    return acc;
+  }, {});
+
+  const intervals = merge(defaultIntervals, foundIntervals);
+
   return {
+    legacy,
     name,
     kuery,
     index,
     docCount: res.hits.total.value,
-    intervals: res.aggregations?.metricset_intervals.buckets.reduce<
-      Record<string, number>
-    >((acc, item) => {
-      acc[item.key] = item.doc_count;
-      return acc;
-    }, {}),
+    intervals,
   };
 }
 

x-pack/plugins/apm/server/routes/diagnostics/bundle/get_existing_index_templates.ts

@@ -6,6 +6,7 @@
  */
 
 import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
+import { getApmIndexTemplateNames } from '../helpers/get_apm_index_template_names';
 import { getIndexTemplate } from './get_index_template';
 
 export type ApmIndexTemplateStates = Record<
@@ -16,11 +17,10 @@ export type ApmIndexTemplateStates = Record<
 // Check whether the default APM index templates exist
 export async function getExistingApmIndexTemplates({
   esClient,
-  apmIndexTemplateNames,
 }: {
   esClient: ElasticsearchClient;
-  apmIndexTemplateNames: string[];
 }) {
+  const apmIndexTemplateNames = getApmIndexTemplateNames();
   const values = await Promise.all(
     apmIndexTemplateNames.map(async (indexTemplateName) => {
       const res = await getIndexTemplate(esClient, { name: indexTemplateName });

x-pack/plugins/apm/server/routes/diagnostics/bundle/get_index_templates_by_index_pattern.ts

@@ -11,7 +11,7 @@ import { orderBy } from 'lodash';
 import { ApmIndicesConfig } from '../../settings/apm_indices/get_apm_indices';
 import { getApmIndexPatterns } from './get_indices';
 import { getIndexTemplate } from './get_index_template';
-import { getApmIndexTemplateNames } from '../get_apm_index_template_names';
+import { getApmIndexTemplateNames } from '../helpers/get_apm_index_template_names';
 
 export async function getIndexTemplatesByIndexPattern({
   esClient,
@@ -27,11 +27,16 @@ export async function getIndexTemplatesByIndexPattern({
     apmIndices.transaction,
   ]);
 
-  return Promise.all(
-    indexPatterns.map(async (indexPattern) =>
-      getSimulatedIndexTemplateForIndexPattern({ indexPattern, esClient })
-    )
-  );
+  try {
+    return await Promise.all(
+      indexPatterns.map(async (indexPattern) =>
+        getSimulatedIndexTemplateForIndexPattern({ indexPattern, esClient })
+      )
+    );
+  } catch (e) {
+    console.error(e);
+    return [];
+  }
 }
 
 async function getSimulatedIndexTemplateForIndexPattern({

x-pack/plugins/apm/server/routes/diagnostics/bundle/get_indices_states.ts

@@ -11,7 +11,7 @@ import {
   IngestGetPipelineResponse,
 } from '@elastic/elasticsearch/lib/api/types';
 import { SERVICE_NAME } from '../../../../common/es_fields/apm';
-import { getApmIndexTemplateNames } from '../get_apm_index_template_names';
+import { getApmIndexTemplateNames } from '../helpers/get_apm_index_template_names';
 
 export function getIndicesStates({
   indices,

x-pack/plugins/apm/server/routes/diagnostics/get_apm_index_template_names.ts

@@ -1,28 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-export function getApmIndexTemplateNames() {
-  const indexTemplateNames = [
-    'logs-apm.app',
-    'logs-apm.error',
-    'metrics-apm.app',
-    'metrics-apm.internal',
-    'traces-apm.rum',
-    'traces-apm.sampled',
-    'traces-apm',
-  ];
-
-  const rollupIndexTemplateNames = ['1m', '10m', '60m'].flatMap((interval) => {
-    return [
-      'metrics-apm.service_destination',
-      'metrics-apm.service_summary',
-      'metrics-apm.service_transaction',
-      'metrics-apm.transaction',
-    ].map((ds) => `${ds}.${interval}`);
-  });
-
-  return [...indexTemplateNames, ...rollupIndexTemplateNames];
-}

x-pack/plugins/apm/server/routes/diagnostics/get_diagnostics_bundle.ts

@@ -5,12 +5,10 @@
  * 2.0.
  */
 
-import { IndicesGetIndexTemplateIndexTemplateItem } from '@elastic/elasticsearch/lib/api/types';
 import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
 import { ApmIndicesConfig } from '../settings/apm_indices/get_apm_indices';
 import { getDataStreams } from './bundle/get_data_streams';
 import { getNonDataStreamIndices } from './bundle/get_non_data_stream_indices';
-import { getApmIndexTemplateNames } from './get_apm_index_template_names';
 import { getElasticsearchVersion } from './get_elasticsearch_version';
 import { getIndexTemplatesByIndexPattern } from './bundle/get_index_templates_by_index_pattern';
 import { getExistingApmIndexTemplates } from './bundle/get_existing_index_templates';
@@ -18,6 +16,7 @@ import { getFieldCaps } from './bundle/get_field_caps';
 import { getIndicesAndIngestPipelines } from './bundle/get_indices';
 import { getIndicesStates } from './bundle/get_indices_states';
 import { getApmEvents } from './bundle/get_apm_events';
+import { getApmIndexTemplates } from './helpers/get_apm_index_template_names';
 
 const DEFEAULT_START = Date.now() - 60 * 5 * 1000; // 5 minutes
 const DEFAULT_END = Date.now();
@@ -35,8 +34,6 @@ export async function getDiagnosticsBundle({
   end: number | undefined;
   kuery: string | undefined;
 }) {
-  const apmIndexTemplateNames = getApmIndexTemplateNames();
-
   const { indices, ingestPipelines } = await getIndicesAndIngestPipelines({
     esClient,
     apmIndices,
@@ -49,7 +46,6 @@ export async function getDiagnosticsBundle({
 
   const existingIndexTemplates = await getExistingApmIndexTemplates({
     esClient,
-    apmIndexTemplateNames,
   });
 
   const fieldCaps = await getFieldCaps({ esClient, apmIndices });
@@ -75,6 +71,7 @@ export async function getDiagnosticsBundle({
 
   return {
     created_at: new Date().toISOString(),
+    apmIndices,
     elasticsearchVersion: await getElasticsearchVersion(esClient),
     esResponses: {
       fieldCaps,
@@ -82,10 +79,7 @@ export async function getDiagnosticsBundle({
       ingestPipelines,
       existingIndexTemplates,
     },
-    apmIndexTemplates: getApmIndexTemplates(
-      apmIndexTemplateNames,
-      existingIndexTemplates
-    ),
+    apmIndexTemplates: getApmIndexTemplates(existingIndexTemplates),
     invalidIndices,
     validIndices,
     indexTemplatesByIndexPattern,
@@ -95,35 +89,3 @@ export async function getDiagnosticsBundle({
     params: { start, end, kuery },
   };
 }
-
-function getApmIndexTemplates(
-  apmIndexTemplateNames: string[],
-  existingIndexTemplates: IndicesGetIndexTemplateIndexTemplateItem[]
-) {
-  const standardIndexTemplates = apmIndexTemplateNames.map((templateName) => {
-    const matchingTemplate = existingIndexTemplates.find(
-      ({ name }) => name === templateName
-    );
-
-    return {
-      name: templateName,
-      exists: Boolean(matchingTemplate),
-      isNonStandard: false,
-    };
-  });
-
-  const nonStandardIndexTemplates = existingIndexTemplates
-    .filter(
-      (indexTemplate) =>
-        standardIndexTemplates.some(
-          ({ name }) => name === indexTemplate.name
-        ) === false
-    )
-    .map((indexTemplate) => ({
-      name: indexTemplate.name,
-      isNonStandard: true,
-      exists: true,
-    }));
-
-  return [...standardIndexTemplates, ...nonStandardIndexTemplates];
-}

x-pack/plugins/apm/server/routes/diagnostics/helpers/get_apm_index_template_names.ts

@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { IndicesGetIndexTemplateIndexTemplateItem } from '@elastic/elasticsearch/lib/api/types';
+
+export function getApmIndexTemplateNames() {
+  const indexTemplateNames = [
+    'logs-apm.app',
+    'logs-apm.error',
+    'metrics-apm.app',
+    'metrics-apm.internal',
+    'traces-apm.rum',
+    'traces-apm.sampled',
+    'traces-apm',
+  ];
+
+  const rollupIndexTemplateNames = ['1m', '10m', '60m'].flatMap((interval) => {
+    return [
+      'metrics-apm.service_destination',
+      'metrics-apm.service_summary',
+      'metrics-apm.service_transaction',
+      'metrics-apm.transaction',
+    ].map((ds) => `${ds}.${interval}`);
+  });
+
+  return [...indexTemplateNames, ...rollupIndexTemplateNames];
+}
+
+export function getApmIndexTemplates(
+  existingIndexTemplates: IndicesGetIndexTemplateIndexTemplateItem[]
+) {
+  const apmIndexTemplateNames = getApmIndexTemplateNames();
+  const standardIndexTemplates = apmIndexTemplateNames.map((templateName) => {
+    const matchingTemplate = existingIndexTemplates.find(
+      ({ name }) => name === templateName
+    );
+
+    return {
+      name: templateName,
+      exists: Boolean(matchingTemplate),
+      isNonStandard: false,
+    };
+  });
+
+  const nonStandardIndexTemplates = existingIndexTemplates
+    .filter(
+      (indexTemplate) =>
+        standardIndexTemplates.some(
+          ({ name }) => name === indexTemplate.name
+        ) === false
+    )
+    .map((indexTemplate) => ({
+      name: indexTemplate.name,
+      isNonStandard: true,
+      exists: true,
+    }));
+
+  return [...standardIndexTemplates, ...nonStandardIndexTemplates];
+}

x-pack/plugins/apm/server/routes/diagnostics/route.ts

@@ -15,7 +15,10 @@ import {
 import * as t from 'io-ts';
 import { isoToEpochRt } from '@kbn/io-ts-utils';
 import { createApmServerRoute } from '../apm_routes/create_apm_server_route';
-import { getApmIndices } from '../settings/apm_indices/get_apm_indices';
+import {
+  ApmIndicesConfig,
+  getApmIndices,
+} from '../settings/apm_indices/get_apm_indices';
 import { ApmEvent } from './bundle/get_apm_events';
 import { getDiagnosticsBundle } from './get_diagnostics_bundle';
 import { getFleetPackageInfo } from './get_fleet_package_info';
@@ -53,6 +56,7 @@ const getDiagnosticsRoute = createApmServerRoute({
       indices: IndicesGetResponse;
       ingestPipelines: IngestGetPipelineResponse;
     };
+    apmIndices: ApmIndicesConfig;
     apmIndexTemplates: Array<{
       name: string;
       isNonStandard: boolean;

x-pack/test/apm_api_integration/tests/diagnostics/index_pattern_settings.ts

@@ -7,7 +7,7 @@
 
 import expect from '@kbn/expect';
 import { apm, timerange } from '@kbn/apm-synthtrace-client';
-import { getApmIndexTemplateNames } from '@kbn/apm-plugin/server/routes/diagnostics/get_apm_index_template_names';
+import { getApmIndexTemplateNames } from '@kbn/apm-plugin/server/routes/diagnostics/helpers/get_apm_index_template_names';
 import { FtrProviderContext } from '../../common/ftr_provider_context';
 
 export default function ApiTest({ getService }: FtrProviderContext) {

x-pack/test/apm_api_integration/tests/diagnostics/index_templates.spec.ts

@@ -7,7 +7,7 @@
 
 import expect from '@kbn/expect';
 import { apm, timerange } from '@kbn/apm-synthtrace-client';
-import { getApmIndexTemplateNames } from '@kbn/apm-plugin/server/routes/diagnostics/get_apm_index_template_names';
+import { getApmIndexTemplateNames } from '@kbn/apm-plugin/server/routes/diagnostics/helpers/get_apm_index_template_names';
 import { FtrProviderContext } from '../../common/ftr_provider_context';
 
 export default function ApiTest({ getService }: FtrProviderContext) {