[APM] Add data section to Diagnostics tool (#159884)

This adds an overview of the number of documents per document type. It's
possible to filter the data via search bar and time range.

<img width="1478" alt="image"
src="32c1c79c-c76a-4b84-ab16-c1b1faccba32">

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

.gitignore

@@ -92,7 +92,7 @@ npm-debug.log*
 
 ## @cypress/snapshot from apm plugin
 /snapshots.js
-x-pack/plugins/apm/scripts/apm-diagnostics*.json
+/apm-diagnostics*.json
 
 # transpiled cypress config
 x-pack/plugins/fleet/cypress.config.d.ts
@@ -132,3 +132,4 @@ fleet-server.yml
 **/.journeys/
 x-pack/test/security_api_integration/plugins/audit_log/audit.log
 
+

src/dev/precommit_hook/casing_check_config.js

@@ -30,6 +30,7 @@ export const IGNORE_FILE_GLOBS = [
   'x-pack/plugins/cases/docs/**/*',
   'x-pack/plugins/monitoring/public/lib/jquery_flot/**/*',
   'x-pack/plugins/fleet/cypress/packages/*.zip',
+  'x-pack/plugins/apm/ftr_e2e/cypress/e2e/power_user/diagnostics/apm-diagnostics-*.json',
   '**/.*',
   '**/__mocks__/**/*',
   'x-pack/docs/**/*',

x-pack/plugins/apm/ftr_e2e/cypress/e2e/power_user/diagnostics/diagnostics.cy.ts

@@ -45,7 +45,7 @@ describe('Diagnostics', () => {
       cy.loginAs({ username: 'elastic', password: 'changeme' });
       cy.visitKibana('/app/apm/diagnostics/import-export');
       cy.get('#file-picker').selectFile(
-        './cypress/e2e/power_user/diagnostics/apm_diagnostics_8.9.0_1685708312530.json'
+        './cypress/e2e/power_user/diagnostics/apm-diagnostics-8.8.0-1687436214804.json'
       );
     });
 
@@ -56,7 +56,7 @@ describe('Diagnostics', () => {
     });
 
     it('can display summary tab', () => {
-      cy.get('[href="/app/apm/diagnostics"]').click();
+      cy.get('[data-test-subj="summary-tab"]').click();
 
       // integration package
       cy.get('[data-test-subj="integrationPackageStatus_Badge"]').should(
@@ -66,7 +66,7 @@ describe('Diagnostics', () => {
 
       cy.get('[data-test-subj="integrationPackageStatus_Content"]').should(
         'have.text',
-        'APM integration (8.9.0-preview-1685091758)'
+        'APM integration (8.8.0)'
       );
 
       // data stream
@@ -89,26 +89,35 @@ describe('Diagnostics', () => {
     });
 
     it('can display index template tab', () => {
-      cy.get('[href="/app/apm/diagnostics/index-templates"]').click();
+      cy.get('[data-test-subj="index-templates-tab"]').click();
       cy.get('.euiTableRow').should('have.length', 19);
     });
 
     it('can display data streams tab', () => {
-      cy.get('[href="/app/apm/diagnostics/data-streams"]').click();
-      cy.get('.euiTableRow').should('have.length', 17);
+      cy.get('[data-test-subj="data-streams-tab"]').click();
+      cy.get('.euiTableRow').should('have.length', 8);
     });
 
     it('can display indices tab', () => {
-      cy.get('[href="/app/apm/diagnostics/indices"]').click();
+      cy.get('[data-test-subj="indices-tab"]').click();
 
       cy.get('[data-test-subj="indicedWithProblems"] .euiTableRow').should(
         'have.length',
-        18
+        138
       );
 
       cy.get('[data-test-subj="indicedWithoutProblems"] .euiTableRow').should(
         'have.length',
-        17
+        27
+      );
+    });
+
+    it('can display documents tab', () => {
+      cy.get('[data-test-subj="documents-tab"]').click();
+
+      cy.get('[data-test-subj="documents-table"] .euiTableRow').should(
+        'have.length',
+        10
       );
     });
   });

x-pack/plugins/apm/kibana.jsonc

@@ -35,6 +35,7 @@
       "cases",
       "charts",
       "cloud",
+      "discover",
       "fleet",
       "fieldFormats",
       "home",

x-pack/plugins/apm/public/components/app/diagnostics/apm_documents_tab.tsx

@@ -0,0 +1,140 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+  EuiBadge,
+  EuiBasicTable,
+  EuiBasicTableColumn,
+  EuiSpacer,
+} from '@elastic/eui';
+import React, { useState } from 'react';
+import { useKibana } from '@kbn/kibana-react-plugin/public';
+import { orderBy } from 'lodash';
+import { useApmParams } from '../../../hooks/use_apm_params';
+import { asInteger } from '../../../../common/utils/formatters';
+import { APM_STATIC_DATA_VIEW_ID } from '../../../../common/data_view_constants';
+import type { ApmEvent } from '../../../../server/routes/diagnostics/bundle/get_apm_events';
+import { useDiagnosticsContext } from './context/use_diagnostics';
+import { ApmPluginStartDeps } from '../../../plugin';
+import { SearchBar } from '../../shared/search_bar/search_bar';
+
+export function DiagnosticsApmDocuments() {
+  const { diagnosticsBundle, isImported } = useDiagnosticsContext();
+  const { discover } = useKibana<ApmPluginStartDeps>().services;
+  const [sortField, setSortField] = useState<keyof ApmEvent>('name');
+  const [sortDirection, setSortDirection] = useState<'asc' | 'desc'>('asc');
+  const {
+    query: { rangeFrom, rangeTo },
+  } = useApmParams('/diagnostics/documents');
+
+  const items = diagnosticsBundle?.apmEvents ?? [];
+  const columns: Array<EuiBasicTableColumn<ApmEvent>> = [
+    {
+      name: 'Name',
+      field: 'name',
+      width: '40%',
+    },
+    {
+      name: 'Doc count',
+      field: 'docCount',
+      render: (_, { docCount }) => asInteger(docCount),
+      sortable: true,
+    },
+    {
+      name: '1m',
+      field: 'intervals.1m',
+      render: (_, { intervals }) => {
+        const interval = intervals?.['1m'];
+        return interval ? asInteger(interval) : '-';
+      },
+    },
+    {
+      name: '10m',
+      field: 'intervals.10m',
+      render: (_, { intervals }) => {
+        const interval = intervals?.['10m'];
+        return interval ? asInteger(interval) : '-';
+      },
+    },
+    {
+      name: '60m',
+      field: 'intervals.60m',
+      render: (_, { intervals }) => {
+        const interval = intervals?.['60m'];
+        return interval ? asInteger(interval) : '-';
+      },
+    },
+    {
+      name: 'Actions',
+      actions: [
+        {
+          name: 'View',
+          description: 'View in Discover',
+          type: 'icon',
+          icon: 'discoverApp',
+          onClick: async (item) => {
+            await discover?.locator?.navigate({
+              query: {
+                language: 'kuery',
+                query: item.kuery,
+              },
+              dataViewId: APM_STATIC_DATA_VIEW_ID,
+              timeRange:
+                rangeTo && rangeFrom
+                  ? {
+                      to: rangeTo,
+                      from: rangeFrom,
+                    }
+                  : undefined,
+            });
+          },
+        },
+      ],
+    },
+  ];
+
+  return (
+    <>
+      {isImported && diagnosticsBundle ? (
+        <>
+          <EuiBadge>
+            From: {new Date(diagnosticsBundle.params.start).toISOString()}
+          </EuiBadge>
+          <EuiBadge>
+            To: {new Date(diagnosticsBundle.params.end).toISOString()}
+          </EuiBadge>
+          <EuiBadge>
+            Filter: {diagnosticsBundle?.params.kuery ?? <em>Empty</em>}
+          </EuiBadge>
+          <EuiSpacer />
+        </>
+      ) : (
+        <SearchBar />
+      )}
+
+      <EuiBasicTable
+        data-test-subj="documents-table"
+        items={orderBy(items, sortField, sortDirection)}
+        sorting={{
+          enableAllColumns: true,
+          sort: {
+            direction: sortDirection,
+            field: sortField,
+          },
+        }}
+        rowHeader="firstName"
+        columns={columns}
+        onChange={({ sort }) => {
+          if (sort) {
+            setSortField(sort.field);
+            setSortDirection(sort.direction);
+          }
+        }}
+      />
+    </>
+  );
+}

x-pack/plugins/apm/public/components/app/diagnostics/context/diagnostics_context.tsx

@@ -6,6 +6,8 @@
  */
 
 import React, { useMemo, useState } from 'react';
+import { useApmParams } from '../../../../hooks/use_apm_params';
+import { useTimeRange } from '../../../../hooks/use_time_range';
 import { APIReturnType } from '../../../../services/rest/create_call_apm_api';
 import { FETCH_STATUS, useFetcher } from '../../../../hooks/use_fetcher';
 
@@ -29,9 +31,25 @@ export function DiagnosticsContextProvider({
 }: {
   children: React.ReactChild;
 }) {
-  const { data, status, refetch } = useFetcher((callApmApi) => {
-    return callApmApi(`GET /internal/apm/diagnostics`);
-  }, []);
+  const {
+    query: { kuery, rangeFrom, rangeTo },
+  } = useApmParams('/diagnostics/*');
+
+  const { start, end } = useTimeRange({ rangeFrom, rangeTo, optional: true });
+  const { data, status, refetch } = useFetcher(
+    (callApmApi) => {
+      return callApmApi(`GET /internal/apm/diagnostics`, {
+        params: {
+          query: {
+            start,
+            end,
+            kuery,
+          },
+        },
+      });
+    },
+    [start, end, kuery]
+  );
 
   const [importedDiagnosticsBundle, setImportedDiagnosticsBundle] = useState<
     DiagnosticsBundle | undefined

x-pack/plugins/apm/public/components/app/diagnostics/data_stream_tab.tsx

@@ -63,7 +63,6 @@ function DataStreamsTable({ data }: { data?: DiagnosticsBundle }) {
 
   return (
     <EuiBasicTable
-      tableCaption="Demo of EuiBasicTable"
       items={data?.dataStreams ?? []}
       rowHeader="firstName"
       columns={columns}

x-pack/plugins/apm/public/components/app/diagnostics/import_export_tab.tsx

@@ -87,7 +87,11 @@ function ImportCard() {
     <EuiCard
       icon={<EuiIcon size="xxl" type="exportAction" />}
       title="Import diagnostics report"
-      description="Import a diagnostics report in order to view the results in the UI"
+      description={
+        isImported
+          ? 'Diagnostics report was imported'
+          : `Import a diagnostics report in order to view the results in the UI`
+      }
       footer={
         <div>
           {isImported ? (
@@ -131,6 +135,7 @@ function ImportCard() {
                           setImportError(true);
                         }
                       } catch (e) {
+                        setImportError(true);
                         console.error(
                           `Could not parse file ${file.name}. ${e.message}`
                         );

x-pack/plugins/apm/public/components/app/diagnostics/index.tsx

@@ -8,7 +8,9 @@
 import { i18n } from '@kbn/i18n';
 import { Outlet } from '@kbn/typed-react-router-config';
 import React from 'react';
+import * as t from 'io-ts';
 import { EuiButton, EuiCallOut, EuiIcon } from '@elastic/eui';
+import { useApmParams } from '../../../hooks/use_apm_params';
 import { useApmRouter } from '../../../hooks/use_apm_router';
 import { useApmRoutePath } from '../../../hooks/use_apm_route_path';
 import { DiagnosticsSummary } from './summary_tab';
@@ -26,6 +28,21 @@ import { useDiagnosticsContext } from './context/use_diagnostics';
 import { getIndexTemplateStatus } from './summary_tab/index_templates_status';
 import { getDataStreamTabStatus } from './summary_tab/data_streams_status';
 import { getIndicesTabStatus } from './summary_tab/indicies_status';
+import { DiagnosticsApmDocuments } from './apm_documents_tab';
+
+const params = t.type({
+  query: t.intersection([
+    t.type({
+      rangeFrom: t.string,
+      rangeTo: t.string,
+    }),
+    t.partial({
+      refreshPaused: t.union([t.literal('true'), t.literal('false')]),
+      refreshInterval: t.string,
+      kuery: t.string,
+    }),
+  ]),
+});
 
 export const diagnosticsRoute = {
   '/diagnostics': {
@@ -36,24 +53,35 @@ export const diagnosticsRoute = {
         </DiagnosticsTemplate>
       </DiagnosticsContextProvider>
     ),
+    params,
     children: {
       '/diagnostics': {
         element: <DiagnosticsSummary />,
+        params,
       },
       '/diagnostics/index-pattern-settings': {
         element: <DiagnosticsIndexPatternSettings />,
+        params,
       },
       '/diagnostics/index-templates': {
         element: <DiagnosticsIndexTemplates />,
+        params,
       },
       '/diagnostics/data-streams': {
         element: <DiagnosticsDataStreams />,
+        params,
       },
       '/diagnostics/indices': {
         element: <DiagnosticsIndices />,
+        params,
+      },
+      '/diagnostics/documents': {
+        element: <DiagnosticsApmDocuments />,
+        params,
       },
       '/diagnostics/import-export': {
         element: <DiagnosticsImportExport />,
+        params,
       },
     },
   },
@@ -63,6 +91,7 @@ function DiagnosticsTemplate({ children }: { children: React.ReactChild }) {
   const routePath = useApmRoutePath();
   const router = useApmRouter();
   const { diagnosticsBundle } = useDiagnosticsContext();
+  const { query } = useApmParams('/diagnostics/*');
 
   return (
     <ApmMainTemplate
@@ -76,17 +105,19 @@ function DiagnosticsTemplate({ children }: { children: React.ReactChild }) {
         description: <TemplateDescription />,
         tabs: [
           {
-            href: router.link('/diagnostics'),
+            'data-test-subj': 'summary-tab',
+            href: router.link('/diagnostics', { query }),
             label: i18n.translate('xpack.apm.diagnostics.tab.summary', {
               defaultMessage: 'Summary',
             }),
             isSelected: routePath === '/diagnostics',
           },
           {
+            'data-test-subj': 'index-pattern-tab',
             prepend: !getIndexPatternTabStatus(diagnosticsBundle) && (
               <EuiIcon type="warning" color="red" />
             ),
-            href: router.link('/diagnostics/index-pattern-settings'),
+            href: router.link('/diagnostics/index-pattern-settings', { query }),
             label: i18n.translate(
               'xpack.apm.diagnostics.tab.index_pattern_settings',
               {
@@ -96,37 +127,49 @@ function DiagnosticsTemplate({ children }: { children: React.ReactChild }) {
             isSelected: routePath === '/diagnostics/index-pattern-settings',
           },
           {
+            'data-test-subj': 'index-templates-tab',
             prepend: !getIndexTemplateStatus(diagnosticsBundle) && (
               <EuiIcon type="warning" color="red" />
             ),
-            href: router.link('/diagnostics/index-templates'),
+            href: router.link('/diagnostics/index-templates', { query }),
             label: i18n.translate('xpack.apm.diagnostics.tab.index_templates', {
               defaultMessage: 'Index templates',
             }),
             isSelected: routePath === '/diagnostics/index-templates',
           },
           {
+            'data-test-subj': 'data-streams-tab',
             prepend: !getDataStreamTabStatus(diagnosticsBundle) && (
               <EuiIcon type="warning" color="red" />
             ),
-            href: router.link('/diagnostics/data-streams'),
+            href: router.link('/diagnostics/data-streams', { query }),
             label: i18n.translate('xpack.apm.diagnostics.tab.datastreams', {
               defaultMessage: 'Data streams',
             }),
             isSelected: routePath === '/diagnostics/data-streams',
           },
           {
+            'data-test-subj': 'indices-tab',
             prepend: !getIndicesTabStatus(diagnosticsBundle) && (
               <EuiIcon type="warning" color="red" />
             ),
-            href: router.link('/diagnostics/indices'),
+            href: router.link('/diagnostics/indices', { query }),
             label: i18n.translate('xpack.apm.diagnostics.tab.indices', {
               defaultMessage: 'Indices',
             }),
             isSelected: routePath === '/diagnostics/indices',
           },
           {
-            href: router.link('/diagnostics/import-export'),
+            'data-test-subj': 'documents-tab',
+            href: router.link('/diagnostics/documents', { query }),
+            label: i18n.translate('xpack.apm.diagnostics.tab.apmEvents', {
+              defaultMessage: 'Documents',
+            }),
+            isSelected: routePath === '/diagnostics/documents',
+          },
+          {
+            'data-test-subj': 'import-export-tab',
+            href: router.link('/diagnostics/import-export', { query }),
             label: i18n.translate('xpack.apm.diagnostics.tab.import_export', {
               defaultMessage: 'Import/Export',
             }),

x-pack/plugins/apm/public/components/app/diagnostics/summary_tab/data_streams_status.tsx

@@ -7,6 +7,7 @@
 
 import React from 'react';
 import { EuiLink } from '@elastic/eui';
+import { useApmParams } from '../../../../hooks/use_apm_params';
 import { APIReturnType } from '../../../../services/rest/create_call_apm_api';
 import { useApmRouter } from '../../../../hooks/use_apm_router';
 import { FETCH_STATUS } from '../../../../hooks/use_fetcher';
@@ -19,6 +20,7 @@ type DiagnosticsBundle = APIReturnType<'GET /internal/apm/diagnostics'>;
 export function DataStreamsStatus() {
   const { diagnosticsBundle, status } = useDiagnosticsContext();
   const router = useApmRouter();
+  const { query } = useApmParams('/diagnostics/*');
   const isLoading = status === FETCH_STATUS.LOADING;
   const tabStatus = getDataStreamTabStatus(diagnosticsBundle);
 
@@ -31,7 +33,7 @@ export function DataStreamsStatus() {
       Data streams
       <EuiLink
         data-test-subj="apmDataStreamsStatusSeeDetailsLink"
-        href={router.link('/diagnostics/data-streams')}
+        href={router.link('/diagnostics/data-streams', { query })}
       >
         See details
       </EuiLink>

x-pack/plugins/apm/public/components/app/diagnostics/summary_tab/index_templates_status.tsx

@@ -7,6 +7,7 @@
 import React from 'react';
 import { EuiLink } from '@elastic/eui';
 import { FETCH_STATUS } from '@kbn/observability-shared-plugin/public';
+import { useApmParams } from '../../../../hooks/use_apm_params';
 import { APIReturnType } from '../../../../services/rest/create_call_apm_api';
 import { useApmRouter } from '../../../../hooks/use_apm_router';
 import { useDiagnosticsContext } from '../context/use_diagnostics';
@@ -16,6 +17,7 @@ type DiagnosticsBundle = APIReturnType<'GET /internal/apm/diagnostics'>;
 
 export function IndexTemplatesStatus() {
   const router = useApmRouter();
+  const { query } = useApmParams('/diagnostics/*');
   const { diagnosticsBundle, status } = useDiagnosticsContext();
   const isLoading = status === FETCH_STATUS.LOADING;
   const tabStatus = getIndexTemplateStatus(diagnosticsBundle);
@@ -29,7 +31,7 @@ export function IndexTemplatesStatus() {
       Index templates
       <EuiLink
         data-test-subj="apmIndexTemplatesStatusSeeDetailsLink"
-        href={router.link('/diagnostics/index-templates')}
+        href={router.link('/diagnostics/index-templates', { query })}
       >
         See details
       </EuiLink>

x-pack/plugins/apm/public/components/app/diagnostics/summary_tab/indicies_status.tsx

@@ -7,6 +7,7 @@
 
 import React from 'react';
 import { EuiLink } from '@elastic/eui';
+import { useApmParams } from '../../../../hooks/use_apm_params';
 import { FETCH_STATUS } from '../../../../hooks/use_fetcher';
 import { APIReturnType } from '../../../../services/rest/create_call_apm_api';
 import { useApmRouter } from '../../../../hooks/use_apm_router';
@@ -17,6 +18,7 @@ type DiagnosticsBundle = APIReturnType<'GET /internal/apm/diagnostics'>;
 
 export function FieldMappingStatus() {
   const router = useApmRouter();
+  const { query } = useApmParams('/diagnostics/*');
   const { diagnosticsBundle, status } = useDiagnosticsContext();
   const isLoading = status === FETCH_STATUS.LOADING;
   const isOk = getIndicesTabStatus(diagnosticsBundle);
@@ -30,7 +32,7 @@ export function FieldMappingStatus() {
       Indices
       <EuiLink
         data-test-subj="apmFieldMappingStatusSeeDetailsLink"
-        href={router.link('/diagnostics/indices')}
+        href={router.link('/diagnostics/indices', { query })}
       >
         See details
       </EuiLink>

x-pack/plugins/apm/public/components/app/service_map/popover/service_contents.tsx

@@ -16,8 +16,8 @@ import {
 import { i18n } from '@kbn/i18n';
 import React from 'react';
 import { NodeDataDefinition } from 'cytoscape';
+import { useAnyOfApmParams } from '../../../../hooks/use_apm_params';
 import { isTimeComparison } from '../../../shared/time_comparison/get_comparison_options';
-import { useApmParams } from '../../../../hooks/use_apm_params';
 import type { ContentsProps } from '.';
 import { useApmRouter } from '../../../../hooks/use_apm_router';
 import { FETCH_STATUS, useFetcher } from '../../../../hooks/use_fetcher';
@@ -43,7 +43,11 @@ export function ServiceContents({
   const nodeData = elementData as NodeDataDefinition;
   const apmRouter = useApmRouter();
 
-  const { query } = useApmParams('/*');
+  const { query } = useAnyOfApmParams(
+    '/service-map',
+    '/services/{serviceName}/service-map',
+    '/mobile-services/{serviceName}/service-map'
+  );
 
   if (
     !('rangeFrom' in query && 'rangeTo' in query) ||

x-pack/plugins/apm/public/components/shared/is_route_with_time_range.ts

@@ -17,6 +17,7 @@ export function isRouteWithTimeRange({
   const matchingRoutes = apmRouter.getRoutesToMatch(location.pathname);
   const matchesRoute = matchingRoutes.some((route) => {
     return (
+      route.path === '/diagnostics' ||
       route.path === '/services' ||
       route.path === '/traces' ||
       route.path === '/service-map' ||

x-pack/plugins/apm/public/context/apm_plugin/mock_apm_plugin_storybook.tsx

@@ -15,6 +15,7 @@ import { RouterProvider } from '@kbn/typed-react-router-config';
 import { createMemoryHistory } from 'history';
 import { merge } from 'lodash';
 import React, { ReactNode } from 'react';
+import { __IntlProvider as IntlProvider } from '@kbn/i18n-react';
 import { Observable, of } from 'rxjs';
 import { apmRouter } from '../../components/routing/apm_route_config';
 import { createCallApmApi } from '../../services/rest/create_call_apm_api';
@@ -144,20 +145,22 @@ export function MockApmPluginStorybook({
   });
 
   return (
-    <EuiThemeProvider darkMode={false}>
-      <KibanaReactContext.Provider>
-        <ApmPluginContext.Provider value={contextMock}>
-          <APMServiceContext.Provider value={serviceContextValue}>
-            <RouterProvider router={apmRouter as any} history={history2}>
-              <MockTimeRangeContextProvider>
-                <ApmTimeRangeMetadataContextProvider>
-                  {children}
-                </ApmTimeRangeMetadataContextProvider>
-              </MockTimeRangeContextProvider>
-            </RouterProvider>
-          </APMServiceContext.Provider>
-        </ApmPluginContext.Provider>
-      </KibanaReactContext.Provider>
-    </EuiThemeProvider>
+    <IntlProvider locale="en">
+      <EuiThemeProvider darkMode={false}>
+        <KibanaReactContext.Provider>
+          <ApmPluginContext.Provider value={contextMock}>
+            <APMServiceContext.Provider value={serviceContextValue}>
+              <RouterProvider router={apmRouter as any} history={history2}>
+                <MockTimeRangeContextProvider>
+                  <ApmTimeRangeMetadataContextProvider>
+                    {children}
+                  </ApmTimeRangeMetadataContextProvider>
+                </MockTimeRangeContextProvider>
+              </RouterProvider>
+            </APMServiceContext.Provider>
+          </ApmPluginContext.Provider>
+        </KibanaReactContext.Provider>
+      </EuiThemeProvider>
+    </IntlProvider>
   );
 }

x-pack/plugins/apm/public/context/time_range_metadata/time_range_metadata_context.tsx

@@ -32,7 +32,7 @@ export function ApmTimeRangeMetadataContextProvider({
 
   const { query } = useApmParams('/*');
 
-  const kuery = 'kuery' in query ? query.kuery : '';
+  const kuery = 'kuery' in query && query.kuery ? query.kuery : '';
 
   const range =
     'rangeFrom' in query && 'rangeTo' in query

x-pack/plugins/apm/public/plugin.ts

@@ -61,6 +61,10 @@ import { FieldFormatsStart } from '@kbn/field-formats-plugin/public';
 import { UiActionsStart, UiActionsSetup } from '@kbn/ui-actions-plugin/public';
 import { ObservabilityTriggerId } from '@kbn/observability-shared-plugin/common';
 import { LicenseManagementUIPluginSetup } from '@kbn/license-management-plugin/public';
+import {
+  DiscoverStart,
+  DiscoverSetup,
+} from '@kbn/discover-plugin/public/plugin';
 import { registerApmRuleTypes } from './components/alerting/rule_types/register_apm_rule_types';
 import {
   getApmEnrollmentFlyoutData,
@@ -80,6 +84,7 @@ export type ApmPluginStart = void;
 export interface ApmPluginSetupDeps {
   alerting?: AlertingPluginPublicSetup;
   data: DataPublicPluginSetup;
+  discover?: DiscoverSetup;
   exploratoryView: ExploratoryViewPublicSetup;
   unifiedSearch: UnifiedSearchPublicPluginStart;
   features: FeaturesPluginSetup;
@@ -98,6 +103,7 @@ export interface ApmPluginStartDeps {
   alerting?: AlertingPluginPublicStart;
   charts?: ChartsPluginStart;
   data: DataPublicPluginStart;
+  discover?: DiscoverStart;
   embeddable: EmbeddableStart;
   home: void;
   inspector: InspectorPluginStart;

x-pack/plugins/apm/scripts/create_diagnostics_bundle.js

@@ -6,4 +6,4 @@
  */
 
 require('@kbn/babel-register').install();
-require('./diagnostics_bundle/main');
+require('./diagnostics_bundle/cli');

x-pack/plugins/apm/scripts/diagnostics_bundle/cli.ts

@@ -0,0 +1,86 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/* eslint-disable no-console */
+
+import datemath from '@elastic/datemath';
+import yargs from 'yargs';
+import { initDiagnosticsBundle } from './diagnostics_bundle';
+
+const { argv } = yargs(process.argv.slice(2))
+  .option('esHost', {
+    demandOption: true,
+    type: 'string',
+    description: 'Elasticsearch host name',
+  })
+  .option('kbHost', {
+    demandOption: true,
+    type: 'string',
+    description: 'Kibana host name',
+  })
+  .option('username', {
+    demandOption: true,
+    type: 'string',
+    description: 'Kibana host name',
+  })
+  .option('password', {
+    demandOption: true,
+    type: 'string',
+    description: 'Kibana host name',
+  })
+  .option('rangeFrom', {
+    type: 'string',
+    description: 'Time-range start',
+    coerce: convertDate,
+  })
+  .option('rangeTo', {
+    type: 'string',
+    description: 'Time range end',
+    coerce: convertDate,
+  })
+  .option('kuery', {
+    type: 'string',
+    description: 'KQL query to filter documents by',
+  })
+  .help();
+
+const { esHost, kbHost, password, username, kuery } = argv;
+const rangeFrom = argv.rangeFrom as unknown as number;
+const rangeTo = argv.rangeTo as unknown as number;
+
+if (rangeFrom) {
+  console.log(`rangeFrom = ${new Date(rangeFrom).toISOString()}`);
+}
+
+if (rangeTo) {
+  console.log(`rangeTo = ${new Date(rangeTo).toISOString()}`);
+}
+
+initDiagnosticsBundle({
+  esHost,
+  kbHost,
+  password,
+  username,
+  start: rangeFrom,
+  end: rangeTo,
+  kuery,
+})
+  .then((res) => {
+    console.log(res);
+  })
+  .catch((err) => {
+    console.log(err);
+  });
+
+function convertDate(dateString: string): number {
+  const parsed = datemath.parse(dateString);
+  if (parsed && parsed.isValid()) {
+    return parsed.valueOf();
+  }
+
+  throw new Error(`Incorrect argument: ${dateString}`);
+}

x-pack/plugins/apm/scripts/diagnostics_bundle/diagnostics_bundle.ts

@@ -21,11 +21,17 @@ export async function initDiagnosticsBundle({
   kbHost,
   username,
   password,
+  start,
+  end,
+  kuery,
 }: {
   esHost: string;
   kbHost: string;
   username: string;
   password: string;
+  start: number | undefined;
+  end: number | undefined;
+  kuery: string | undefined;
 }) {
   const esClient = new Client({ node: esHost, auth: { username, password } });
 
@@ -34,7 +40,13 @@ export async function initDiagnosticsBundle({
     auth: { username, password },
   });
   const apmIndices = await getApmIndices(kibanaClient);
-  const bundle = await getDiagnosticsBundle(esClient, apmIndices);
+  const bundle = await getDiagnosticsBundle({
+    esClient,
+    apmIndices,
+    start,
+    end,
+    kuery,
+  });
   const fleetPackageInfo = await getFleetPackageInfo(kibanaClient);
   const kibanaVersion = await getKibanaVersion(kibanaClient);
 

x-pack/plugins/apm/scripts/diagnostics_bundle/main.ts

@@ -1,44 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-/* eslint-disable no-console */
-
-import yargs from 'yargs';
-import { initDiagnosticsBundle } from './diagnostics_bundle';
-
-const { argv } = yargs(process.argv.slice(2))
-  .option('esHost', {
-    demandOption: true,
-    type: 'string',
-    description: 'Elasticsearch host name',
-  })
-  .option('kbHost', {
-    demandOption: true,
-    type: 'string',
-    description: 'Kibana host name',
-  })
-  .option('username', {
-    demandOption: true,
-    type: 'string',
-    description: 'Kibana host name',
-  })
-  .option('password', {
-    demandOption: true,
-    type: 'string',
-    description: 'Kibana host name',
-  })
-  .help();
-
-const { esHost, kbHost, password, username } = argv;
-
-initDiagnosticsBundle({ esHost, kbHost, password, username })
-  .then((res) => {
-    console.log(res);
-  })
-  .catch((err) => {
-    console.log(err);
-  });

x-pack/plugins/apm/server/routes/diagnostics/bundle/get_apm_events.ts

@@ -0,0 +1,220 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
+import { kqlQuery, rangeQuery } from '@kbn/observability-plugin/server';
+import {
+  PROCESSOR_EVENT,
+  METRICSET_NAME,
+  METRICSET_INTERVAL,
+  TRANSACTION_DURATION_SUMMARY,
+} from '../../../../common/es_fields/apm';
+import { ApmIndicesConfig } from '../../settings/apm_indices/get_apm_indices';
+import { getTypedSearch, TypedSearch } from '../create_typed_es_client';
+import { getApmIndexPatterns } from './get_indices';
+
+export interface ApmEvent {
+  name: string;
+  kuery: string;
+  index: string[];
+  docCount: number;
+  intervals?: Record<string, number>;
+}
+
+export async function getApmEvents({
+  esClient,
+  apmIndices,
+  start,
+  end,
+  kuery,
+}: {
+  esClient: ElasticsearchClient;
+  apmIndices: ApmIndicesConfig;
+  start: number;
+  end: number;
+  kuery?: string;
+}): Promise<ApmEvent[]> {
+  const typedSearch = getTypedSearch(esClient);
+
+  const commonProps = { start, end, typedSearch };
+  const items = await Promise.all([
+    getEventWithMetricsetInterval({
+      ...commonProps,
+      name: 'Metric: Service destination',
+      index: getApmIndexPatterns([apmIndices.metric]),
+      kuery: mergeKueries(
+        `${PROCESSOR_EVENT}: "metric" AND ${METRICSET_NAME}: "service_destination"`,
+        kuery
+      ),
+    }),
+    getEventWithMetricsetInterval({
+      ...commonProps,
+      name: 'Metric: Service transaction (with summary field)',
+      index: getApmIndexPatterns([apmIndices.metric]),
+      kuery: mergeKueries(
+        `${PROCESSOR_EVENT}: "metric" AND ${METRICSET_NAME}: "service_transaction" AND ${TRANSACTION_DURATION_SUMMARY} :* `,
+        kuery
+      ),
+    }),
+    getEventWithMetricsetInterval({
+      ...commonProps,
+      name: 'Metric: Transaction (with summary field)',
+      index: getApmIndexPatterns([apmIndices.metric]),
+      kuery: mergeKueries(
+        `${PROCESSOR_EVENT}: "metric" AND ${METRICSET_NAME}: "transaction" AND ${TRANSACTION_DURATION_SUMMARY} :* `,
+        kuery
+      ),
+    }),
+    getEventWithMetricsetInterval({
+      ...commonProps,
+      name: 'Metric: Service transaction (without summary field)',
+      index: getApmIndexPatterns([apmIndices.metric]),
+      kuery: mergeKueries(
+        `${PROCESSOR_EVENT}: "metric" AND ${METRICSET_NAME}: "service_transaction" AND not ${TRANSACTION_DURATION_SUMMARY} :* `,
+        kuery
+      ),
+    }),
+    getEventWithMetricsetInterval({
+      ...commonProps,
+      name: 'Metric: Transaction (without summary field)',
+      index: getApmIndexPatterns([apmIndices.metric]),
+      kuery: mergeKueries(
+        `${PROCESSOR_EVENT}: "metric" AND ${METRICSET_NAME}: "transaction" AND not ${TRANSACTION_DURATION_SUMMARY} :* `,
+        kuery
+      ),
+    }),
+    getEventWithMetricsetInterval({
+      ...commonProps,
+      name: 'Metric: Span breakdown',
+      index: getApmIndexPatterns([apmIndices.metric]),
+      kuery: mergeKueries(
+        `${PROCESSOR_EVENT}: "metric" AND ${METRICSET_NAME}: "span_breakdown"`,
+        kuery
+      ),
+    }),
+    getEventWithMetricsetInterval({
+      ...commonProps,
+      name: 'Metric: Service summary',
+      index: getApmIndexPatterns([apmIndices.metric]),
+      kuery: mergeKueries(
+        `${PROCESSOR_EVENT}: "metric" AND ${METRICSET_NAME}: "service_summary"`,
+        kuery
+      ),
+    }),
+    getEvent({
+      ...commonProps,
+      name: 'Event: Transaction',
+      index: getApmIndexPatterns([apmIndices.transaction]),
+      kuery: mergeKueries(`${PROCESSOR_EVENT}: "transaction"`, kuery),
+    }),
+    getEvent({
+      ...commonProps,
+      name: 'Event: Span',
+      index: getApmIndexPatterns([apmIndices.span]),
+      kuery: mergeKueries(`${PROCESSOR_EVENT}: "span"`, kuery),
+    }),
+    getEvent({
+      ...commonProps,
+      name: 'Event: Error',
+      index: getApmIndexPatterns([apmIndices.error]),
+      kuery: mergeKueries(`${PROCESSOR_EVENT}: "error"`, kuery),
+    }),
+  ]);
+
+  return items;
+}
+
+async function getEventWithMetricsetInterval({
+  name,
+  index,
+  start,
+  end,
+  kuery,
+  typedSearch,
+}: {
+  name: string;
+  index: string[];
+  start: number;
+  end: number;
+  kuery: string;
+  typedSearch: TypedSearch;
+}) {
+  const res = await typedSearch({
+    expand_wildcards: 'all',
+    track_total_hits: true,
+    index,
+    size: 0,
+    query: {
+      bool: {
+        filter: [...kqlQuery(kuery), ...rangeQuery(start, end)],
+      },
+    },
+    aggs: {
+      metricset_intervals: {
+        terms: {
+          size: 1000,
+          field: METRICSET_INTERVAL,
+        },
+      },
+    },
+  });
+
+  return {
+    name,
+    kuery,
+    index,
+    docCount: res.hits.total.value,
+    intervals: res.aggregations?.metricset_intervals.buckets.reduce<
+      Record<string, number>
+    >((acc, item) => {
+      acc[item.key] = item.doc_count;
+      return acc;
+    }, {}),
+  };
+}
+
+async function getEvent({
+  name,
+  index,
+  start,
+  end,
+  kuery,
+  typedSearch,
+}: {
+  name: string;
+  index: string[];
+  start: number;
+  end: number;
+  kuery: string;
+  typedSearch: TypedSearch;
+}) {
+  const res = await typedSearch({
+    track_total_hits: true,
+    index,
+    size: 0,
+    query: {
+      bool: {
+        filter: [...kqlQuery(kuery), ...rangeQuery(start, end)],
+      },
+    },
+  });
+
+  return {
+    name,
+    kuery,
+    index,
+    docCount: res.hits.total.value,
+  };
+}
+
+function mergeKueries(fixedKuery: string, kuery?: string) {
+  if (!kuery) {
+    return fixedKuery;
+  }
+
+  return `(${fixedKuery}) AND (${kuery})`;
+}

x-pack/plugins/apm/server/routes/diagnostics/create_typed_es_client.ts

@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
+import { ESSearchRequest, InferSearchResponseOf } from '@kbn/es-types';
+
+type RequiredParams = ESSearchRequest & {
+  size: number;
+  track_total_hits: boolean | number;
+};
+
+export type TypedSearch = ReturnType<typeof getTypedSearch>;
+export function getTypedSearch(esClient: ElasticsearchClient) {
+  async function search<TDocument, TParams extends RequiredParams>(
+    opts: TParams
+  ): Promise<InferSearchResponseOf<TDocument, TParams>> {
+    return esClient.search<TDocument>(opts) as Promise<any>;
+  }
+
+  return search;
+}

x-pack/plugins/apm/server/routes/diagnostics/get_diagnostics_bundle.ts

@@ -17,11 +17,24 @@ import { getExistingApmIndexTemplates } from './bundle/get_existing_index_templa
 import { getFieldCaps } from './bundle/get_field_caps';
 import { getIndicesAndIngestPipelines } from './bundle/get_indices';
 import { getIndicesStates } from './bundle/get_indices_states';
+import { getApmEvents } from './bundle/get_apm_events';
 
-export async function getDiagnosticsBundle(
-  esClient: ElasticsearchClient,
-  apmIndices: ApmIndicesConfig
-) {
+const DEFEAULT_START = Date.now() - 60 * 5 * 1000; // 5 minutes
+const DEFAULT_END = Date.now();
+
+export async function getDiagnosticsBundle({
+  esClient,
+  apmIndices,
+  start = DEFEAULT_START,
+  end = DEFAULT_END,
+  kuery,
+}: {
+  esClient: ElasticsearchClient;
+  apmIndices: ApmIndicesConfig;
+  start: number | undefined;
+  end: number | undefined;
+  kuery: string | undefined;
+}) {
   const apmIndexTemplateNames = getApmIndexTemplateNames();
 
   const { indices, ingestPipelines } = await getIndicesAndIngestPipelines({
@@ -52,6 +65,14 @@ export async function getDiagnosticsBundle(
     ingestPipelines,
   });
 
+  const apmEvents = await getApmEvents({
+    esClient,
+    apmIndices,
+    start,
+    end,
+    kuery,
+  });
+
   return {
     created_at: new Date().toISOString(),
     elasticsearchVersion: await getElasticsearchVersion(esClient),
@@ -70,6 +91,8 @@ export async function getDiagnosticsBundle(
     indexTemplatesByIndexPattern,
     dataStreams,
     nonDataStreamIndices,
+    apmEvents,
+    params: { start, end, kuery },
   };
 }
 

x-pack/plugins/apm/server/routes/diagnostics/route.ts

@@ -12,8 +12,11 @@ import {
   IndicesGetResponse,
   IngestGetPipelineResponse,
 } from '@elastic/elasticsearch/lib/api/types';
+import * as t from 'io-ts';
+import { isoToEpochRt } from '@kbn/io-ts-utils';
 import { createApmServerRoute } from '../apm_routes/create_apm_server_route';
 import { getApmIndices } from '../settings/apm_indices/get_apm_indices';
+import { ApmEvent } from './bundle/get_apm_events';
 import { getDiagnosticsBundle } from './get_diagnostics_bundle';
 import { getFleetPackageInfo } from './get_fleet_package_info';
 
@@ -33,8 +36,14 @@ export interface IndiciesItem {
 
 const getDiagnosticsRoute = createApmServerRoute({
   endpoint: 'GET /internal/apm/diagnostics',
-
   options: { tags: ['access:apm'] },
+  params: t.partial({
+    query: t.partial({
+      kuery: t.string,
+      start: isoToEpochRt,
+      end: isoToEpochRt,
+    }),
+  }),
   handler: async (
     resources
   ): Promise<{
@@ -55,6 +64,7 @@ const getDiagnosticsRoute = createApmServerRoute({
     };
     kibanaVersion: string;
     elasticsearchVersion: string;
+    apmEvents: ApmEvent[];
     invalidIndices: IndiciesItem[];
     validIndices: IndiciesItem[];
     dataStreams: IndicesDataStream[];
@@ -68,7 +78,9 @@ const getDiagnosticsRoute = createApmServerRoute({
         templateName: string;
       }>;
     }>;
+    params: { start: number; end: number; kuery?: string };
   }> => {
+    const { start, end, kuery } = resources.params.query;
     const coreContext = await resources.context.core;
     const { asCurrentUser: esClient } = coreContext.elasticsearch.client;
     const apmIndices = await getApmIndices({
@@ -76,7 +88,14 @@ const getDiagnosticsRoute = createApmServerRoute({
       config: resources.config,
     });
 
-    const bundle = await getDiagnosticsBundle(esClient, apmIndices);
+    const bundle = await getDiagnosticsBundle({
+      esClient,
+      apmIndices,
+      start,
+      end,
+      kuery,
+    });
+
     const fleetPackageInfo = await getFleetPackageInfo(resources);
     const kibanaVersion = resources.kibanaVersion;
 

x-pack/plugins/apm/tsconfig.json

@@ -1,7 +1,7 @@
 {
   "extends": "../../../tsconfig.base.json",
   "compilerOptions": {
-    "outDir": "target/types",
+    "outDir": "target/types"
   },
   "include": [
     "../../../typings/**/*",
@@ -13,7 +13,7 @@
     "jest.config.js",
     // have to declare *.json explicitly due to https://github.com/microsoft/TypeScript/issues/25636
     "public/**/*.json",
-    "server/**/*.json",
+    "server/**/*.json"
   ],
   "kbn_references": [
     "@kbn/core",
@@ -94,8 +94,7 @@
     "@kbn/core-http-server",
     "@kbn/unified-field-list",
     "@kbn/slo-schema",
+    "@kbn/discover-plugin"
   ],
-  "exclude": [
-    "target/**/*",
-  ]
+  "exclude": ["target/**/*"]
 }

x-pack/test/apm_api_integration/tests/diagnostics/apm_events.spec.ts

@@ -0,0 +1,131 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import { apm, timerange } from '@kbn/apm-synthtrace-client';
+import { FtrProviderContext } from '../../common/ftr_provider_context';
+
+export default function ApiTest({ getService }: FtrProviderContext) {
+  const registry = getService('registry');
+  const apmApiClient = getService('apmApiClient');
+  const es = getService('es');
+  const synthtraceEsClient = getService('synthtraceEsClient');
+
+  const start = new Date('2021-01-01T00:00:00.000Z').getTime();
+  const end = new Date('2021-01-01T00:15:00.000Z').getTime() - 1;
+
+  registry.when('Diagnostics: APM Events', { config: 'basic', archives: [] }, () => {
+    describe('When there is no data', () => {
+      before(async () => {
+        // delete APM data streams
+        await es.indices.deleteDataStream({ name: '*apm*' });
+      });
+
+      it('returns zero data streams`', async () => {
+        const { status, body } = await apmApiClient.adminUser({
+          endpoint: 'GET /internal/apm/diagnostics',
+        });
+        expect(status).to.be(200);
+        expect(body.apmEvents.every(({ docCount }) => docCount === 0)).to.be(true);
+      });
+    });
+
+    describe('When data is ingested', () => {
+      before(async () => {
+        const instance = apm
+          .service({ name: 'synth-go', environment: 'production', agentName: 'go' })
+          .instance('instance-a');
+
+        await synthtraceEsClient.index(
+          timerange(start, end)
+            .interval('1m')
+            .rate(30)
+            .generator((timestamp) =>
+              instance
+                .transaction({ transactionName: 'GET /users' })
+                .timestamp(timestamp)
+                .duration(100)
+                .success()
+            )
+        );
+      });
+
+      after(() => synthtraceEsClient.clean());
+
+      it('returns zero doc_counts when no time range is specified', async () => {
+        const { body } = await apmApiClient.adminUser({
+          endpoint: 'GET /internal/apm/diagnostics',
+        });
+
+        expect(body.apmEvents.every(({ docCount }) => docCount === 0)).to.be(true);
+      });
+
+      it('returns non-zero doc_counts when time range is specified', async () => {
+        const { body } = await apmApiClient.adminUser({
+          endpoint: 'GET /internal/apm/diagnostics',
+          params: {
+            query: { start: new Date(start).toISOString(), end: new Date(end).toISOString() },
+          },
+        });
+
+        expect(body.apmEvents.every(({ docCount }) => docCount === 0)).to.be(false);
+        expect(
+          body.apmEvents
+            .filter(({ docCount }) => docCount > 0)
+            .map(({ kuery, docCount }) => ({ kuery, docCount }))
+        ).to.eql([
+          {
+            kuery:
+              'processor.event: "metric" AND metricset.name: "service_transaction" AND transaction.duration.summary :* ',
+            docCount: 21,
+          },
+          {
+            kuery:
+              'processor.event: "metric" AND metricset.name: "transaction" AND transaction.duration.summary :* ',
+            docCount: 21,
+          },
+          { kuery: 'processor.event: "metric" AND metricset.name: "span_breakdown"', docCount: 15 },
+          {
+            kuery: 'processor.event: "metric" AND metricset.name: "service_summary"',
+            docCount: 21,
+          },
+          { kuery: 'processor.event: "transaction"', docCount: 450 },
+        ]);
+      });
+
+      it('returns zero doc_counts when filtering by a non-existing service', async () => {
+        const { body } = await apmApiClient.adminUser({
+          endpoint: 'GET /internal/apm/diagnostics',
+          params: {
+            query: {
+              start: new Date(start).toISOString(),
+              end: new Date(end).toISOString(),
+              kuery: 'service.name: "foo"',
+            },
+          },
+        });
+
+        expect(body.apmEvents.every(({ docCount }) => docCount === 0)).to.be(true);
+      });
+
+      it('returns non-zero doc_counts when filtering by an existing service', async () => {
+        const { body } = await apmApiClient.adminUser({
+          endpoint: 'GET /internal/apm/diagnostics',
+          params: {
+            query: {
+              start: new Date(start).toISOString(),
+              end: new Date(end).toISOString(),
+              kuery: 'service.name: "synth-go"',
+            },
+          },
+        });
+
+        expect(body.apmEvents.every(({ docCount }) => docCount === 0)).to.be(false);
+      });
+    });
+  });
+}