[APM] Initial diagnostics tool (#157500)

Pitch: https://github.com/elastic/apm-dev/issues/872 (_internal_)

Introduces the Diagnostics tool under a hidden page:
`/app/apm/diagnostics`. The Diagnostics tool can help find problems with
missing index templates, data streams with incorrect index templates and
indices with incorrect mappings.

It will be released in 8.9 but customers can take advantage of it as
soon as this PR is merged. They can export a diagnostics bundle from
their system by running this script:

```
node x-pack/plugins/apm/scripts/diagnostics_bundle \ 
--kbHost https://foo.kb.europe-west2.gcp.elastic-cloud.com:9243 \
--esHost https://foo.es.europe-west2.gcp.elastic-cloud.com:9243 \
--username elastic
--password changeme
```


![image](a783d25e-652a-441f-abcf-acc3444876ee)



![image](fe60e83e-d559-459a-9fda-e2adb7d583dc)



![image](557aebdf-005d-4cfb-bd16-895c9b1b90cb)

---------

Co-authored-by: Yngrid Coello <yngrdyn@gmail.com>

.gitignore

@@ -92,6 +92,7 @@ npm-debug.log*
 
 ## @cypress/snapshot from apm plugin
 /snapshots.js
+x-pack/plugins/apm/scripts/apm-diagnostics*.json
 
 # transpiled cypress config
 x-pack/plugins/fleet/cypress.config.d.ts
@@ -130,3 +131,4 @@ fleet-server.yml
 **/.synthetics/
 **/.journeys/
 x-pack/test/security_api_integration/plugins/audit_log/audit.log
+

packages/kbn-apm-synthtrace/src/lib/apm/client/apm_synthtrace_es_client/get_routing_transform.ts

@@ -15,31 +15,35 @@ export function getRoutingTransform() {
     transform(document: ESDocumentWithOperation<ApmFields>, encoding, callback) {
       let index: string | undefined;
 
+      const namespace = 'default';
+
       switch (document['processor.event']) {
         case 'transaction':
         case 'span':
           index =
-            document['agent.name'] === 'rum-js' ? 'traces-apm.rum-default' : 'traces-apm-default';
+            document['agent.name'] === 'rum-js'
+              ? `traces-apm.rum-${namespace}`
+              : `traces-apm-${namespace}`;
           break;
 
         case 'error':
-          index = 'logs-apm.error-default';
+          index = `logs-apm.error-${namespace}`;
           break;
 
         case 'metric':
           const metricsetName = document['metricset.name'];
 
           if (metricsetName === 'app') {
-            index = `metrics-apm.app.${document['service.name']}-default`;
+            index = `metrics-apm.app.${document['service.name']}-${namespace}`;
           } else if (
             metricsetName === 'transaction' ||
             metricsetName === 'service_transaction' ||
             metricsetName === 'service_destination' ||
             metricsetName === 'service_summary'
           ) {
-            index = `metrics-apm.${metricsetName}.${document['metricset.interval']!}-default`;
+            index = `metrics-apm.${metricsetName}.${document['metricset.interval']!}-${namespace}`;
           } else {
-            index = `metrics-apm.internal-default`;
+            index = `metrics-apm.internal-${namespace}`;
           }
           break;
       }

x-pack/plugins/apm/ftr_e2e/cypress/e2e/power_user/diagnostics/diagnostics.cy.ts

@@ -0,0 +1,115 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+describe('Diagnostics', () => {
+  describe('when no data is loaded', () => {
+    beforeEach(() => {
+      cy.loginAs({ username: 'elastic', password: 'changeme' });
+    });
+
+    it('can display summary tab', () => {
+      cy.visitKibana('/app/apm/diagnostics');
+
+      // integration package
+      cy.get('[data-test-subj="integrationPackageStatus_Badge"]').should(
+        'have.text',
+        'OK'
+      );
+
+      // data stream
+      cy.get('[data-test-subj="dataStreamsStatus_Badge"]').should(
+        'have.text',
+        'OK'
+      );
+
+      // Index template
+      cy.get('[data-test-subj="indexTemplatesStatus_Badge"]').should(
+        'have.text',
+        'OK'
+      );
+
+      // Index template
+      cy.get('[data-test-subj="fieldMappingStatus_Badge"]').should(
+        'have.text',
+        'OK'
+      );
+    });
+  });
+
+  describe('when importing a file', () => {
+    beforeEach(() => {
+      cy.loginAs({ username: 'elastic', password: 'changeme' });
+      cy.visitKibana('/app/apm/diagnostics/import-export');
+      cy.get('#file-picker').selectFile(
+        './cypress/e2e/power_user/diagnostics/apm_diagnostics_8.9.0_1685708312530.json'
+      );
+    });
+
+    it('shows the remove button', () => {
+      cy.get('[data-test-subj="apmImportCardRemoveReportButton"]').should(
+        'exist'
+      );
+    });
+
+    it('can display summary tab', () => {
+      cy.get('[href="/app/apm/diagnostics"]').click();
+
+      // integration package
+      cy.get('[data-test-subj="integrationPackageStatus_Badge"]').should(
+        'have.text',
+        'OK'
+      );
+
+      cy.get('[data-test-subj="integrationPackageStatus_Content"]').should(
+        'have.text',
+        'APM integration (8.9.0-preview-1685091758)'
+      );
+
+      // data stream
+      cy.get('[data-test-subj="dataStreamsStatus_Badge"]').should(
+        'have.text',
+        'OK'
+      );
+
+      // Index template
+      cy.get('[data-test-subj="indexTemplatesStatus_Badge"]').should(
+        'have.text',
+        'OK'
+      );
+
+      // Index template
+      cy.get('[data-test-subj="fieldMappingStatus_Badge"]').should(
+        'have.text',
+        'Warning'
+      );
+    });
+
+    it('can display index template tab', () => {
+      cy.get('[href="/app/apm/diagnostics/index-templates"]').click();
+      cy.get('.euiTableRow').should('have.length', 19);
+    });
+
+    it('can display data streams tab', () => {
+      cy.get('[href="/app/apm/diagnostics/data-streams"]').click();
+      cy.get('.euiTableRow').should('have.length', 17);
+    });
+
+    it('can display indices tab', () => {
+      cy.get('[href="/app/apm/diagnostics/indices"]').click();
+
+      cy.get('[data-test-subj="indicedWithProblems"] .euiTableRow').should(
+        'have.length',
+        18
+      );
+
+      cy.get('[data-test-subj="indicedWithoutProblems"] .euiTableRow').should(
+        'have.length',
+        17
+      );
+    });
+  });
+});

x-pack/plugins/apm/public/components/app/diagnostics/context/diagnostics_context.tsx

@@ -0,0 +1,67 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React, { useMemo, useState } from 'react';
+import { APIReturnType } from '../../../../services/rest/create_call_apm_api';
+import { FETCH_STATUS, useFetcher } from '../../../../hooks/use_fetcher';
+
+type DiagnosticsBundle = APIReturnType<'GET /internal/apm/diagnostics'>;
+
+export const DiagnosticsContext = React.createContext<{
+  diagnosticsBundle?: DiagnosticsBundle;
+  setImportedDiagnosticsBundle: (bundle: DiagnosticsBundle | undefined) => void;
+  status: FETCH_STATUS;
+  isImported?: boolean;
+  refetch: () => void;
+}>({
+  diagnosticsBundle: undefined,
+  setImportedDiagnosticsBundle: () => undefined,
+  status: FETCH_STATUS.NOT_INITIATED,
+  refetch: () => undefined,
+});
+
+export function DiagnosticsContextProvider({
+  children,
+}: {
+  children: React.ReactChild;
+}) {
+  const { data, status, refetch } = useFetcher((callApmApi) => {
+    return callApmApi(`GET /internal/apm/diagnostics`);
+  }, []);
+
+  const [importedDiagnosticsBundle, setImportedDiagnosticsBundle] = useState<
+    DiagnosticsBundle | undefined
+  >(undefined);
+
+  const value = useMemo(() => {
+    if (importedDiagnosticsBundle) {
+      return {
+        refetch,
+        diagnosticsBundle: importedDiagnosticsBundle,
+        setImportedDiagnosticsBundle,
+        status: FETCH_STATUS.SUCCESS,
+        isImported: true,
+      };
+    }
+
+    return {
+      refetch,
+      diagnosticsBundle: data,
+      setImportedDiagnosticsBundle,
+      status,
+      isImported: false,
+    };
+  }, [
+    importedDiagnosticsBundle,
+    setImportedDiagnosticsBundle,
+    status,
+    data,
+    refetch,
+  ]);
+
+  return <DiagnosticsContext.Provider value={value} children={children} />;
+}

x-pack/plugins/apm/public/components/app/diagnostics/context/use_diagnostics.tsx

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { useContext } from 'react';
+import { DiagnosticsContext } from './diagnostics_context';
+
+export function useDiagnosticsContext() {
+  return useContext(DiagnosticsContext);
+}

x-pack/plugins/apm/public/components/app/diagnostics/data_stream_tab.tsx

@@ -0,0 +1,81 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { IndicesDataStream } from '@elastic/elasticsearch/lib/api/types';
+import {
+  EuiBadge,
+  EuiBasicTable,
+  EuiBasicTableColumn,
+  EuiSpacer,
+  EuiText,
+} from '@elastic/eui';
+import React from 'react';
+import { APIReturnType } from '../../../services/rest/create_call_apm_api';
+import { useDiagnosticsContext } from './context/use_diagnostics';
+
+type DiagnosticsBundle = APIReturnType<'GET /internal/apm/diagnostics'>;
+
+export function DiagnosticsDataStreams() {
+  const { diagnosticsBundle } = useDiagnosticsContext();
+
+  return (
+    <>
+      <EuiText>
+        This section shows the APM data streams and their underlying index
+        template.
+      </EuiText>
+      <EuiSpacer />
+      <DataStreamsTable data={diagnosticsBundle} />
+    </>
+  );
+}
+
+function DataStreamsTable({ data }: { data?: DiagnosticsBundle }) {
+  const columns: Array<EuiBasicTableColumn<IndicesDataStream>> = [
+    {
+      field: 'name',
+      name: 'Data stream name',
+    },
+    {
+      field: 'template',
+      name: 'Index template name',
+      render: (templateName: string) => {
+        const indexTemplate = data && getIndexTemplateState(data, templateName);
+
+        return indexTemplate?.exists && !indexTemplate?.isNonStandard ? (
+          <>
+            {templateName}&nbsp;
+            <EuiBadge color="green">OK</EuiBadge>
+          </>
+        ) : (
+          <>
+            {templateName}&nbsp;
+            <EuiBadge color="warning">Non-standard</EuiBadge>
+          </>
+        );
+      },
+    },
+  ];
+
+  return (
+    <EuiBasicTable
+      tableCaption="Demo of EuiBasicTable"
+      items={data?.dataStreams ?? []}
+      rowHeader="firstName"
+      columns={columns}
+    />
+  );
+}
+
+export function getIndexTemplateState(
+  diagnosticsBundle: DiagnosticsBundle,
+  templateName: string
+) {
+  return diagnosticsBundle.apmIndexTemplates.find(
+    ({ name }) => templateName === name
+  );
+}

x-pack/plugins/apm/public/components/app/diagnostics/import_export_tab.tsx

@@ -0,0 +1,161 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React, { useState } from 'react';
+import {
+  EuiButton,
+  EuiCard,
+  EuiIcon,
+  EuiFlexGroup,
+  EuiFlexItem,
+  EuiFilePicker,
+  EuiCallOut,
+  EuiSpacer,
+} from '@elastic/eui';
+import { APIReturnType } from '../../../services/rest/create_call_apm_api';
+import { useDiagnosticsContext } from './context/use_diagnostics';
+import { getIndexTemplateStatus } from './summary_tab/index_templates_status';
+import { getIndicesTabStatus } from './summary_tab/indicies_status';
+import { getDataStreamTabStatus } from './summary_tab/data_streams_status';
+
+type DiagnosticsBundle = APIReturnType<'GET /internal/apm/diagnostics'>;
+
+export function DiagnosticsImportExport() {
+  return (
+    <EuiFlexGroup gutterSize="l">
+      <EuiFlexItem>
+        <ExportCard />
+      </EuiFlexItem>
+      <EuiFlexItem>
+        <ImportCard />
+      </EuiFlexItem>
+    </EuiFlexGroup>
+  );
+}
+
+function ExportCard() {
+  const { diagnosticsBundle, isImported } = useDiagnosticsContext();
+
+  return (
+    <EuiCard
+      isDisabled={isImported}
+      icon={<EuiIcon size="xxl" type="importAction" />}
+      title="Export"
+      description="Export the diagnostics report in order to provide it to Elastic Support"
+      footer={
+        <div>
+          <EuiButton
+            isDisabled={isImported}
+            data-test-subj="apmDiagnosticsImportExportGoForItButton"
+            aria-label="Export diagnostics report"
+            onClick={() => {
+              if (!diagnosticsBundle) {
+                return;
+              }
+
+              const blob = new Blob(
+                [JSON.stringify(diagnosticsBundle, null, 2)],
+                {
+                  type: 'text/plain',
+                }
+              );
+              const fileURL = URL.createObjectURL(blob);
+
+              const { kibanaVersion } = diagnosticsBundle;
+              const link = document.createElement('a');
+              link.href = fileURL;
+              link.download = `apm-diagnostics-${kibanaVersion}-${Date.now()}.json`;
+              link.click();
+            }}
+          >
+            Export
+          </EuiButton>
+        </div>
+      }
+    />
+  );
+}
+
+function ImportCard() {
+  const { setImportedDiagnosticsBundle, isImported } = useDiagnosticsContext();
+  const [importError, setImportError] = useState(false);
+  return (
+    <EuiCard
+      icon={<EuiIcon size="xxl" type="exportAction" />}
+      title="Import diagnostics report"
+      description="Import a diagnostics report in order to view the results in the UI"
+      footer={
+        <div>
+          {isImported ? (
+            <EuiButton
+              data-test-subj="apmImportCardRemoveReportButton"
+              onClick={() => setImportedDiagnosticsBundle(undefined)}
+              color="danger"
+            >
+              Remove report
+            </EuiButton>
+          ) : (
+            <>
+              {importError && (
+                <>
+                  <EuiCallOut color="danger" iconType="warning">
+                    The uploaded file could not be parsed
+                  </EuiCallOut>
+                  <EuiSpacer />
+                </>
+              )}
+              <EuiFilePicker
+                fullWidth
+                id="file-picker"
+                multiple
+                onChange={(_files) => {
+                  setImportError(false);
+
+                  if (_files && _files.length > 0) {
+                    const file = Array.from(_files)[0];
+                    const reader = new FileReader();
+                    reader.onload = (evt: ProgressEvent<FileReader>) => {
+                      try {
+                        const diagnosticsBundle = JSON.parse(
+                          // @ts-expect-error
+                          evt?.target?.result
+                        ) as DiagnosticsBundle;
+
+                        if (isBundleValid(diagnosticsBundle)) {
+                          setImportedDiagnosticsBundle(diagnosticsBundle);
+                        } else {
+                          setImportError(true);
+                        }
+                      } catch (e) {
+                        console.error(
+                          `Could not parse file ${file.name}. ${e.message}`
+                        );
+                      }
+                    };
+
+                    reader.readAsText(file);
+                  }
+                }}
+              />
+            </>
+          )}
+        </div>
+      }
+    />
+  );
+}
+
+function isBundleValid(diagnosticsBundle: DiagnosticsBundle) {
+  try {
+    getIndexTemplateStatus(diagnosticsBundle);
+    getIndicesTabStatus(diagnosticsBundle);
+    getDataStreamTabStatus(diagnosticsBundle);
+    return true;
+  } catch (e) {
+    return false;
+  }
+}

x-pack/plugins/apm/public/components/app/diagnostics/index.tsx

@@ -0,0 +1,169 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { i18n } from '@kbn/i18n';
+import { Outlet } from '@kbn/typed-react-router-config';
+import React from 'react';
+import { EuiButton, EuiCallOut, EuiIcon } from '@elastic/eui';
+import { useApmRouter } from '../../../hooks/use_apm_router';
+import { useApmRoutePath } from '../../../hooks/use_apm_route_path';
+import { DiagnosticsSummary } from './summary_tab';
+import { ApmMainTemplate } from '../../routing/templates/apm_main_template';
+import { DiagnosticsIndexTemplates } from './index_templates_tab';
+import { DiagnosticsIndices } from './indices_tab';
+import { DiagnosticsDataStreams } from './data_stream_tab';
+import {
+  DiagnosticsIndexPatternSettings,
+  getIndexPatternTabStatus,
+} from './index_pattern_settings_tab';
+import { DiagnosticsImportExport } from './import_export_tab';
+import { DiagnosticsContextProvider } from './context/diagnostics_context';
+import { useDiagnosticsContext } from './context/use_diagnostics';
+import { getIndexTemplateStatus } from './summary_tab/index_templates_status';
+import { getDataStreamTabStatus } from './summary_tab/data_streams_status';
+import { getIndicesTabStatus } from './summary_tab/indicies_status';
+
+export const diagnosticsRoute = {
+  '/diagnostics': {
+    element: (
+      <DiagnosticsContextProvider>
+        <DiagnosticsTemplate>
+          <Outlet />
+        </DiagnosticsTemplate>
+      </DiagnosticsContextProvider>
+    ),
+    children: {
+      '/diagnostics': {
+        element: <DiagnosticsSummary />,
+      },
+      '/diagnostics/index-pattern-settings': {
+        element: <DiagnosticsIndexPatternSettings />,
+      },
+      '/diagnostics/index-templates': {
+        element: <DiagnosticsIndexTemplates />,
+      },
+      '/diagnostics/data-streams': {
+        element: <DiagnosticsDataStreams />,
+      },
+      '/diagnostics/indices': {
+        element: <DiagnosticsIndices />,
+      },
+      '/diagnostics/import-export': {
+        element: <DiagnosticsImportExport />,
+      },
+    },
+  },
+};
+
+function DiagnosticsTemplate({ children }: { children: React.ReactChild }) {
+  const routePath = useApmRoutePath();
+  const router = useApmRouter();
+  const { diagnosticsBundle } = useDiagnosticsContext();
+
+  return (
+    <ApmMainTemplate
+      pageTitle="Diagnostics"
+      environmentFilter={false}
+      showServiceGroupSaveButton={false}
+      selectedNavButton="serviceGroups"
+      pageHeader={{
+        iconType: 'magnifyWithExclamation',
+        rightSideItems: [<RefreshButton />],
+        description: <TemplateDescription />,
+        tabs: [
+          {
+            href: router.link('/diagnostics'),
+            label: i18n.translate('xpack.apm.diagnostics.tab.summary', {
+              defaultMessage: 'Summary',
+            }),
+            isSelected: routePath === '/diagnostics',
+          },
+          {
+            prepend: !getIndexPatternTabStatus(diagnosticsBundle) && (
+              <EuiIcon type="warning" color="red" />
+            ),
+            href: router.link('/diagnostics/index-pattern-settings'),
+            label: i18n.translate(
+              'xpack.apm.diagnostics.tab.index_pattern_settings',
+              {
+                defaultMessage: 'Index pattern settings',
+              }
+            ),
+            isSelected: routePath === '/diagnostics/index-pattern-settings',
+          },
+          {
+            prepend: !getIndexTemplateStatus(diagnosticsBundle) && (
+              <EuiIcon type="warning" color="red" />
+            ),
+            href: router.link('/diagnostics/index-templates'),
+            label: i18n.translate('xpack.apm.diagnostics.tab.index_templates', {
+              defaultMessage: 'Index templates',
+            }),
+            isSelected: routePath === '/diagnostics/index-templates',
+          },
+          {
+            prepend: !getDataStreamTabStatus(diagnosticsBundle) && (
+              <EuiIcon type="warning" color="red" />
+            ),
+            href: router.link('/diagnostics/data-streams'),
+            label: i18n.translate('xpack.apm.diagnostics.tab.datastreams', {
+              defaultMessage: 'Data streams',
+            }),
+            isSelected: routePath === '/diagnostics/data-streams',
+          },
+          {
+            prepend: !getIndicesTabStatus(diagnosticsBundle) && (
+              <EuiIcon type="warning" color="red" />
+            ),
+            href: router.link('/diagnostics/indices'),
+            label: i18n.translate('xpack.apm.diagnostics.tab.indices', {
+              defaultMessage: 'Indices',
+            }),
+            isSelected: routePath === '/diagnostics/indices',
+          },
+          {
+            href: router.link('/diagnostics/import-export'),
+            label: i18n.translate('xpack.apm.diagnostics.tab.import_export', {
+              defaultMessage: 'Import/Export',
+            }),
+            isSelected: routePath === '/diagnostics/import-export',
+          },
+        ],
+      }}
+    >
+      {children}
+    </ApmMainTemplate>
+  );
+}
+
+function TemplateDescription() {
+  const { isImported } = useDiagnosticsContext();
+  if (isImported) {
+    return (
+      <EuiCallOut
+        title="Displaying results from the uploaded diagnostics report"
+        iconType="exportAction"
+      />
+    );
+  }
+
+  return null;
+}
+
+function RefreshButton() {
+  const { isImported, refetch } = useDiagnosticsContext();
+  return (
+    <EuiButton
+      isDisabled={isImported}
+      data-test-subj="apmDiagnosticsTemplateRefreshButton"
+      fill
+      onClick={refetch}
+    >
+      Refresh
+    </EuiButton>
+  );
+}

x-pack/plugins/apm/public/components/app/diagnostics/index_pattern_settings_tab.tsx

@@ -0,0 +1,118 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { EuiLink, EuiLoadingElastic } from '@elastic/eui';
+import {
+  EuiBadge,
+  EuiSpacer,
+  EuiText,
+  EuiTitle,
+  EuiToolTip,
+} from '@elastic/eui';
+import React from 'react';
+import { APIReturnType } from '../../../services/rest/create_call_apm_api';
+import { useApmRouter } from '../../../hooks/use_apm_router';
+import { FETCH_STATUS } from '../../../hooks/use_fetcher';
+import { useDiagnosticsContext } from './context/use_diagnostics';
+
+type DiagnosticsBundle = APIReturnType<'GET /internal/apm/diagnostics'>;
+
+export function DiagnosticsIndexPatternSettings() {
+  const router = useApmRouter();
+  const { diagnosticsBundle, status } = useDiagnosticsContext();
+
+  if (status === FETCH_STATUS.LOADING) {
+    return <EuiLoadingElastic size="m" />;
+  }
+
+  const indexTemplatesByIndexPattern =
+    diagnosticsBundle?.indexTemplatesByIndexPattern;
+
+  if (
+    !indexTemplatesByIndexPattern ||
+    indexTemplatesByIndexPattern?.length === 0
+  ) {
+    return null;
+  }
+
+  const elms = indexTemplatesByIndexPattern.map(
+    ({ indexPattern, indexTemplates }) => {
+      return (
+        <div key={indexPattern}>
+          <EuiTitle size="xs">
+            <h4>{indexPattern}</h4>
+          </EuiTitle>
+
+          {!indexTemplates?.length && <em>No matching index templates</em>}
+
+          {indexTemplates?.map(
+            ({
+              templateName,
+              templateIndexPatterns,
+              priority,
+              isNonStandard,
+            }) => {
+              const text = priority
+                ? `(Priority: ${priority})`
+                : isNonStandard
+                ? `(legacy template)`
+                : '';
+              return (
+                <EuiToolTip
+                  key={templateName}
+                  content={`${templateIndexPatterns.join(', ')} ${text}`}
+                >
+                  <EuiBadge
+                    color={isNonStandard ? 'warning' : 'hollow'}
+                    css={{ marginRight: '5px', marginTop: '5px' }}
+                  >
+                    {templateName}
+                  </EuiBadge>
+                </EuiToolTip>
+              );
+            }
+          )}
+
+          <EuiSpacer />
+        </div>
+      );
+    }
+  );
+
+  return (
+    <>
+      <EuiText>
+        This section lists the index patterns specified in{' '}
+        <EuiLink
+          data-test-subj="apmMatchingIndexTemplatesSeeDetailsLink"
+          href={router.link('/settings/apm-indices')}
+        >
+          APM Index Settings
+        </EuiLink>{' '}
+        and which index templates they match. The priority and index pattern of
+        each index template can be seen by hovering over the item.
+      </EuiText>
+      <EuiSpacer />
+      {elms}
+    </>
+  );
+}
+
+export function getIndexPatternTabStatus(
+  diagnosticsBundle?: DiagnosticsBundle
+) {
+  if (!diagnosticsBundle) {
+    return true;
+  }
+
+  const hasError = diagnosticsBundle.indexTemplatesByIndexPattern.some(
+    ({ indexTemplates }) =>
+      indexTemplates.some(({ isNonStandard }) => isNonStandard)
+  );
+
+  return !hasError;
+}

x-pack/plugins/apm/public/components/app/diagnostics/index_templates_tab.tsx

@@ -0,0 +1,106 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { EuiCallOut, EuiLoadingElastic } from '@elastic/eui';
+import {
+  EuiBadge,
+  EuiBasicTable,
+  EuiBasicTableColumn,
+  EuiSpacer,
+  EuiText,
+} from '@elastic/eui';
+import React from 'react';
+import { APIReturnType } from '../../../services/rest/create_call_apm_api';
+import { FETCH_STATUS } from '../../../hooks/use_fetcher';
+import { useDiagnosticsContext } from './context/use_diagnostics';
+
+type DiagnosticsBundle = APIReturnType<'GET /internal/apm/diagnostics'>;
+
+export function DiagnosticsIndexTemplates() {
+  const { diagnosticsBundle, status } = useDiagnosticsContext();
+
+  if (status === FETCH_STATUS.LOADING) {
+    return <EuiLoadingElastic size="m" />;
+  }
+
+  const items = diagnosticsBundle?.apmIndexTemplates ?? [];
+  const columns: Array<EuiBasicTableColumn<typeof items[0]>> = [
+    {
+      name: 'Index template name',
+      field: 'name',
+      render: (_, { name }) => name,
+      truncateText: true,
+    },
+    {
+      name: 'Status',
+      field: 'status',
+      render: (_, { exists, isNonStandard }) => {
+        if (isNonStandard) {
+          return <EuiBadge color="warning">Non standard</EuiBadge>;
+        }
+
+        if (!exists) {
+          return <EuiBadge color="danger">Not found</EuiBadge>;
+        }
+
+        return <EuiBadge color="green">OK</EuiBadge>;
+      },
+      truncateText: true,
+    },
+  ];
+
+  return (
+    <>
+      <NonStandardIndexTemplateCalout diagnosticsBundle={diagnosticsBundle} />
+      <EuiText>
+        This section lists the names of the default APM Index Templates and
+        whether it exists or not
+      </EuiText>
+
+      <EuiSpacer />
+
+      <EuiBasicTable
+        tableCaption="Expected Index Templates"
+        items={items}
+        rowHeader="firstName"
+        columns={columns}
+      />
+    </>
+  );
+}
+
+function NonStandardIndexTemplateCalout({
+  diagnosticsBundle,
+}: {
+  diagnosticsBundle?: DiagnosticsBundle;
+}) {
+  const nonStandardIndexTemplates =
+    diagnosticsBundle?.apmIndexTemplates?.filter(
+      ({ isNonStandard }) => isNonStandard
+    );
+
+  if (!nonStandardIndexTemplates?.length) {
+    return null;
+  }
+
+  return (
+    <>
+      <EuiCallOut
+        title="Non-standard index templates"
+        color="warning"
+        iconType="warning"
+      >
+        The following index templates do not follow the recommended naming
+        scheme:{' '}
+        {nonStandardIndexTemplates.map(({ name }) => (
+          <EuiBadge>{name}</EuiBadge>
+        ))}
+      </EuiCallOut>
+      <EuiSpacer />
+    </>
+  );
+}

x-pack/plugins/apm/public/components/app/diagnostics/indices_tab.tsx

@@ -0,0 +1,138 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React from 'react';
+import {
+  EuiBasicTable,
+  EuiBasicTableColumn,
+  EuiIcon,
+  EuiLoadingElastic,
+  EuiSpacer,
+  EuiText,
+  EuiTitle,
+  EuiToolTip,
+} from '@elastic/eui';
+import { type IndiciesItem } from '../../../../server/routes/diagnostics/route';
+import { FETCH_STATUS } from '../../../hooks/use_fetcher';
+import { useDiagnosticsContext } from './context/use_diagnostics';
+
+export function DiagnosticsIndices() {
+  const { diagnosticsBundle, status } = useDiagnosticsContext();
+
+  if (!diagnosticsBundle || status === FETCH_STATUS.LOADING) {
+    return <EuiLoadingElastic size="m" />;
+  }
+
+  const { invalidIndices, validIndices } = diagnosticsBundle;
+  const columns: Array<EuiBasicTableColumn<IndiciesItem>> = [
+    {
+      field: 'index',
+      name: 'Index name',
+      truncateText: true,
+    },
+    {
+      field: 'dataStream',
+      name: 'Data stream',
+      truncateText: true,
+      render: (_, { dataStream }) => {
+        if (!dataStream) {
+          return (
+            <EuiToolTip
+              content={`This index does not belong to a data stream. This will most likely cause mapping issues. Consider deleting the index and re-install the APM integration to ensure you have index templates and data streams correctly installed`}
+            >
+              <EuiIcon type="warning" />
+            </EuiToolTip>
+          );
+        }
+
+        return dataStream;
+      },
+    },
+    {
+      field: 'ingestPipeline',
+      name: 'Ingest pipelines',
+      truncateText: true,
+      render: (_, { ingestPipeline }) => {
+        if (ingestPipeline.id === undefined) {
+          return (
+            <EuiToolTip content={`Pipeline is missing`}>
+              <EuiIcon type="warning" />
+            </EuiToolTip>
+          );
+        }
+
+        return (
+          <>
+            {ingestPipeline.isValid ? (
+              ingestPipeline.id
+            ) : (
+              <EuiToolTip
+                content={`The expected processor for "observer.version" was not found in "${ingestPipeline.id}"`}
+              >
+                <EuiIcon type="warning" />
+              </EuiToolTip>
+            )}
+          </>
+        );
+      },
+    },
+    {
+      field: 'fieldMappings',
+      name: 'Mappings',
+      width: '75px',
+      align: 'center',
+      render: (_, { fieldMappings }) => {
+        return (
+          <>
+            {fieldMappings.isValid ? (
+              <EuiIcon type="check" />
+            ) : (
+              <EuiToolTip
+                content={`The field "service.name" should be mapped as keyword but is mapped as "${fieldMappings.invalidType}"`}
+              >
+                <EuiIcon type="warning" />
+              </EuiToolTip>
+            )}
+          </>
+        );
+      },
+    },
+  ];
+
+  return (
+    <>
+      <EuiText>
+        This section shows the concrete indices backing the data streams, and
+        highlights mapping issues and missing ingest pipelines.
+      </EuiText>
+
+      <EuiSpacer />
+
+      <EuiTitle size="s">
+        <h3>Indices with problems</h3>
+      </EuiTitle>
+      <EuiBasicTable
+        data-test-subj="indicedWithProblems"
+        items={invalidIndices}
+        rowHeader="index"
+        columns={columns}
+      />
+
+      <EuiSpacer />
+
+      <EuiTitle size="s">
+        <h3>Indices without problems</h3>
+      </EuiTitle>
+      <EuiBasicTable
+        data-test-subj="indicedWithoutProblems"
+        items={validIndices}
+        rowHeader="index"
+        columns={columns}
+      />
+    </>
+  );
+}

x-pack/plugins/apm/public/components/app/diagnostics/summary_tab/apm_integration_package_status.tsx

@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React from 'react';
+
+import { EuiLink } from '@elastic/eui';
+import { useApmPluginContext } from '../../../../context/apm_plugin/use_apm_plugin_context';
+import { FETCH_STATUS } from '../../../../hooks/use_fetcher';
+import { useDiagnosticsContext } from '../context/use_diagnostics';
+import { TabStatus } from './tab_status';
+
+export function ApmIntegrationPackageStatus() {
+  const { diagnosticsBundle, status, isImported } = useDiagnosticsContext();
+  const { core } = useApmPluginContext();
+  const { basePath } = core.http;
+
+  const isLoading = status === FETCH_STATUS.LOADING;
+  const isInstalled = diagnosticsBundle?.fleetPackageInfo.isInstalled;
+  const packageVersion = diagnosticsBundle?.fleetPackageInfo.version;
+
+  return (
+    <TabStatus
+      isLoading={isLoading}
+      isOk={isInstalled}
+      data-test-subj="integrationPackageStatus"
+    >
+      {isLoading
+        ? '...'
+        : isInstalled
+        ? `APM integration (${packageVersion})`
+        : 'APM integration: not installed'}
+
+      {!isImported ? (
+        <EuiLink
+          data-test-subj="apmApmIntegrationPackageStatusGoToApmIntegrationLink"
+          href={basePath.prepend('/app/integrations/detail/apm/overview')}
+        >
+          Go to APM Integration
+        </EuiLink>
+      ) : null}
+    </TabStatus>
+  );
+}

x-pack/plugins/apm/public/components/app/diagnostics/summary_tab/data_streams_status.tsx

@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React from 'react';
+import { EuiLink } from '@elastic/eui';
+import { APIReturnType } from '../../../../services/rest/create_call_apm_api';
+import { useApmRouter } from '../../../../hooks/use_apm_router';
+import { FETCH_STATUS } from '../../../../hooks/use_fetcher';
+import { useDiagnosticsContext } from '../context/use_diagnostics';
+import { getIndexTemplateState } from '../data_stream_tab';
+import { TabStatus } from './tab_status';
+
+type DiagnosticsBundle = APIReturnType<'GET /internal/apm/diagnostics'>;
+
+export function DataStreamsStatus() {
+  const { diagnosticsBundle, status } = useDiagnosticsContext();
+  const router = useApmRouter();
+  const isLoading = status === FETCH_STATUS.LOADING;
+  const tabStatus = getDataStreamTabStatus(diagnosticsBundle);
+
+  return (
+    <TabStatus
+      isLoading={isLoading}
+      isOk={tabStatus}
+      data-test-subj="dataStreamsStatus"
+    >
+      Data streams
+      <EuiLink
+        data-test-subj="apmDataStreamsStatusSeeDetailsLink"
+        href={router.link('/diagnostics/data-streams')}
+      >
+        See details
+      </EuiLink>
+    </TabStatus>
+  );
+}
+
+export function getDataStreamTabStatus(diagnosticsBundle?: DiagnosticsBundle) {
+  if (!diagnosticsBundle) {
+    return false;
+  }
+
+  return diagnosticsBundle.dataStreams.every((ds) => {
+    const match = getIndexTemplateState(diagnosticsBundle, ds.template);
+    return match?.exists && !match.isNonStandard;
+  });
+}

x-pack/plugins/apm/public/components/app/diagnostics/summary_tab/index.tsx

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React from 'react';
+import { EuiFlexGroup } from '@elastic/eui';
+import { ApmIntegrationPackageStatus } from './apm_integration_package_status';
+import { IndexTemplatesStatus } from './index_templates_status';
+import { FieldMappingStatus } from './indicies_status';
+import { DataStreamsStatus } from './data_streams_status';
+
+export function DiagnosticsSummary() {
+  return (
+    <EuiFlexGroup direction="column">
+      <ApmIntegrationPackageStatus />
+      <IndexTemplatesStatus />
+      <DataStreamsStatus />
+      <FieldMappingStatus />
+    </EuiFlexGroup>
+  );
+}

x-pack/plugins/apm/public/components/app/diagnostics/summary_tab/index_templates_status.tsx

@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import React from 'react';
+import { EuiLink } from '@elastic/eui';
+import { FETCH_STATUS } from '@kbn/observability-shared-plugin/public';
+import { APIReturnType } from '../../../../services/rest/create_call_apm_api';
+import { useApmRouter } from '../../../../hooks/use_apm_router';
+import { useDiagnosticsContext } from '../context/use_diagnostics';
+import { TabStatus } from './tab_status';
+
+type DiagnosticsBundle = APIReturnType<'GET /internal/apm/diagnostics'>;
+
+export function IndexTemplatesStatus() {
+  const router = useApmRouter();
+  const { diagnosticsBundle, status } = useDiagnosticsContext();
+  const isLoading = status === FETCH_STATUS.LOADING;
+  const tabStatus = getIndexTemplateStatus(diagnosticsBundle);
+
+  return (
+    <TabStatus
+      isLoading={isLoading}
+      isOk={tabStatus}
+      data-test-subj="indexTemplatesStatus"
+    >
+      Index templates
+      <EuiLink
+        data-test-subj="apmIndexTemplatesStatusSeeDetailsLink"
+        href={router.link('/diagnostics/index-templates')}
+      >
+        See details
+      </EuiLink>
+    </TabStatus>
+  );
+}
+
+export function getIndexTemplateStatus(diagnosticsBundle?: DiagnosticsBundle) {
+  const hasNonStandardIndexTemplates =
+    diagnosticsBundle?.apmIndexTemplates?.some(
+      ({ isNonStandard }) => isNonStandard
+    );
+
+  const isEveryExpectedApmIndexTemplateInstalled =
+    diagnosticsBundle?.apmIndexTemplates.every(
+      ({ exists, isNonStandard }) => isNonStandard || exists
+    );
+
+  return (
+    isEveryExpectedApmIndexTemplateInstalled && !hasNonStandardIndexTemplates
+  );
+}

x-pack/plugins/apm/public/components/app/diagnostics/summary_tab/indicies_status.tsx

@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React from 'react';
+import { EuiLink } from '@elastic/eui';
+import { FETCH_STATUS } from '../../../../hooks/use_fetcher';
+import { APIReturnType } from '../../../../services/rest/create_call_apm_api';
+import { useApmRouter } from '../../../../hooks/use_apm_router';
+import { useDiagnosticsContext } from '../context/use_diagnostics';
+import { TabStatus } from './tab_status';
+
+type DiagnosticsBundle = APIReturnType<'GET /internal/apm/diagnostics'>;
+
+export function FieldMappingStatus() {
+  const router = useApmRouter();
+  const { diagnosticsBundle, status } = useDiagnosticsContext();
+  const isLoading = status === FETCH_STATUS.LOADING;
+  const isOk = getIndicesTabStatus(diagnosticsBundle);
+
+  return (
+    <TabStatus
+      isLoading={isLoading}
+      isOk={isOk}
+      data-test-subj="fieldMappingStatus"
+    >
+      Indices
+      <EuiLink
+        data-test-subj="apmFieldMappingStatusSeeDetailsLink"
+        href={router.link('/diagnostics/indices')}
+      >
+        See details
+      </EuiLink>
+    </TabStatus>
+  );
+}
+
+export function getIndicesTabStatus(diagnosticsBundle?: DiagnosticsBundle) {
+  return diagnosticsBundle?.invalidIndices.length === 0;
+}

x-pack/plugins/apm/public/components/app/diagnostics/summary_tab/tab_status.tsx

@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React from 'react';
+import { EuiBadge, EuiFlexGroup, EuiFlexItem } from '@elastic/eui';
+
+export function TabStatus({
+  isLoading,
+  isOk,
+  children,
+  ...props
+}: {
+  isLoading: boolean;
+  isOk?: boolean;
+  children: React.ReactNode;
+} & React.ComponentProps<typeof EuiFlexItem>) {
+  return (
+    <EuiFlexItem {...props}>
+      <EuiFlexGroup>
+        <EuiFlexItem grow={1}>
+          <EuiFlexGroup justifyContent="flexEnd">
+            <EuiFlexItem
+              grow={false}
+              data-test-subj={`${props['data-test-subj']}_Badge`}
+            >
+              {isLoading ? (
+                <EuiBadge color="default">-</EuiBadge>
+              ) : isOk ? (
+                <EuiBadge color="green">OK</EuiBadge>
+              ) : (
+                <EuiBadge color="warning">Warning</EuiBadge>
+              )}
+            </EuiFlexItem>
+          </EuiFlexGroup>
+        </EuiFlexItem>
+        <EuiFlexItem
+          grow={10}
+          data-test-subj={`${props['data-test-subj']}_Content`}
+        >
+          {children}
+        </EuiFlexItem>
+      </EuiFlexGroup>
+    </EuiFlexItem>
+  );
+}

x-pack/plugins/apm/public/components/routing/apm_route_config.tsx

@@ -13,13 +13,14 @@ import { toBooleanRt } from '@kbn/io-ts-utils';
 import { Breadcrumb } from '../app/breadcrumb';
 import { TraceLink } from '../app/trace_link';
 import { TransactionLink } from '../app/transaction_link';
-import { home } from './home';
-import { serviceDetail } from './service_detail';
-import { mobileServiceDetail } from './mobile_service_detail';
-import { settings } from './settings';
+import { homeRoute } from './home';
+import { serviceDetailRoute } from './service_detail';
+import { mobileServiceDetailRoute } from './mobile_service_detail';
+import { settingsRoute } from './settings';
 import { ApmMainTemplate } from './templates/apm_main_template';
 import { ServiceGroupsList } from '../app/service_groups';
 import { offsetRt } from '../../../common/comparison_rt';
+import { diagnosticsRoute } from '../app/diagnostics';
 
 const ServiceGroupsTitle = i18n.translate(
   'xpack.apm.views.serviceGroups.title',
@@ -104,10 +105,11 @@ const apmRoutes = {
           ]),
         }),
       },
-      ...settings,
-      ...serviceDetail,
-      ...mobileServiceDetail,
-      ...home,
+      ...diagnosticsRoute,
+      ...settingsRoute,
+      ...serviceDetailRoute,
+      ...mobileServiceDetailRoute,
+      ...homeRoute,
     },
   },
 };

x-pack/plugins/apm/public/components/routing/home/index.tsx

@@ -96,7 +96,7 @@ export const DependenciesOperationsTitle = i18n.translate(
   }
 );
 
-export const home = {
+export const homeRoute = {
   '/': {
     element: (
       <ApmTimeRangeMetadataContextProvider>
@@ -214,6 +214,7 @@ export const home = {
           },
         },
       },
+
       ...dependencies,
       ...legacyBackends,
       ...storageExplorer,

x-pack/plugins/apm/public/components/routing/mobile_service_detail/index.tsx

@@ -55,7 +55,7 @@ export function page({
   };
 }
 
-export const mobileServiceDetail = {
+export const mobileServiceDetailRoute = {
   '/mobile-services/{serviceName}': {
     element: (
       <ApmTimeRangeMetadataContextProvider>

x-pack/plugins/apm/public/components/routing/service_detail/index.tsx

@@ -94,7 +94,7 @@ function RedirectNodeMetricsToMetricsDetails() {
   );
 }
 
-export const serviceDetail = {
+export const serviceDetailRoute = {
   '/services/{serviceName}': {
     element: (
       <ApmTimeRangeMetadataContextProvider>

x-pack/plugins/apm/public/components/routing/settings/index.tsx

@@ -44,7 +44,7 @@ function page({
   };
 }
 
-export const settings = {
+export const settingsRoute = {
   '/settings': {
     element: (
       <Breadcrumb

x-pack/plugins/apm/public/components/routing/templates/apm_main_template.tsx

@@ -20,7 +20,7 @@ import { ApmEnvironmentFilter } from '../../shared/environment_filter';
 import { getNoDataConfig } from './no_data_config';
 
 // Paths that must skip the no data screen
-const bypassNoDataScreenPaths = ['/settings'];
+const bypassNoDataScreenPaths = ['/settings', '/diagnostics'];
 
 /*
  * This template contains:

x-pack/plugins/apm/scripts/diagnostics_bundle.js

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+require('@kbn/babel-register').install();
+require('./diagnostics_bundle/main');

x-pack/plugins/apm/scripts/diagnostics_bundle/diagnostics_bundle.ts

@@ -0,0 +1,89 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/* eslint-disable no-console */
+
+import { Client } from '@elastic/elasticsearch';
+import fs from 'fs/promises';
+import axios, { AxiosInstance } from 'axios';
+import { APIReturnType } from '../../public/services/rest/create_call_apm_api';
+import { getDiagnosticsBundle } from '../../server/routes/diagnostics/get_diagnostics_bundle';
+import { ApmIndicesConfig } from '../../server/routes/settings/apm_indices/get_apm_indices';
+
+type DiagnosticsBundle = APIReturnType<'GET /internal/apm/diagnostics'>;
+
+export async function initDiagnosticsBundle({
+  esHost,
+  kbHost,
+  username,
+  password,
+}: {
+  esHost: string;
+  kbHost: string;
+  username: string;
+  password: string;
+}) {
+  const esClient = new Client({ node: esHost, auth: { username, password } });
+
+  const kibanaClient = axios.create({
+    baseURL: kbHost,
+    auth: { username, password },
+  });
+  const apmIndices = await getApmIndices(kibanaClient);
+  const bundle = await getDiagnosticsBundle(esClient, apmIndices);
+  const fleetPackageInfo = await getFleetPackageInfo(kibanaClient);
+  const kibanaVersion = await getKibanaVersion(kibanaClient);
+
+  await saveReportToFile({ ...bundle, fleetPackageInfo, kibanaVersion });
+}
+
+async function saveReportToFile(combinedReport: DiagnosticsBundle) {
+  const filename = `apm-diagnostics-${
+    combinedReport.kibanaVersion
+  }-${Date.now()}.json`;
+  await fs.writeFile(filename, JSON.stringify(combinedReport, null, 2), {
+    encoding: 'utf8',
+    flag: 'w',
+  });
+  console.log(`Diagnostics report written to "${filename}"`);
+}
+
+async function getApmIndices(kibanaClient: AxiosInstance) {
+  interface Response {
+    apmIndexSettings: Array<{
+      configurationName: string;
+      defaultValue: string;
+      savedValue?: string;
+    }>;
+  }
+
+  const res = await kibanaClient.get<Response>(
+    '/internal/apm/settings/apm-index-settings'
+  );
+
+  return Object.fromEntries(
+    res.data.apmIndexSettings.map(
+      ({ configurationName, defaultValue, savedValue }) => [
+        configurationName,
+        savedValue ?? defaultValue,
+      ]
+    )
+  ) as ApmIndicesConfig;
+}
+
+async function getFleetPackageInfo(kibanaClient: AxiosInstance) {
+  const res = await kibanaClient.get('/api/fleet/epm/packages/apm');
+  return {
+    version: res.data.response.version,
+    isInstalled: res.data.response.status,
+  };
+}
+
+async function getKibanaVersion(kibanaClient: AxiosInstance) {
+  const res = await kibanaClient.get('/api/status');
+  return res.data.version.number;
+}

x-pack/plugins/apm/scripts/diagnostics_bundle/main.ts

@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/* eslint-disable no-console */
+
+import yargs from 'yargs';
+import { initDiagnosticsBundle } from './diagnostics_bundle';
+
+const { argv } = yargs(process.argv.slice(2))
+  .option('esHost', {
+    demandOption: true,
+    type: 'string',
+    description: 'Elasticsearch host name',
+  })
+  .option('kbHost', {
+    demandOption: true,
+    type: 'string',
+    description: 'Kibana host name',
+  })
+  .option('username', {
+    demandOption: true,
+    type: 'string',
+    description: 'Kibana host name',
+  })
+  .option('password', {
+    demandOption: true,
+    type: 'string',
+    description: 'Kibana host name',
+  })
+  .help();
+
+const { esHost, kbHost, password, username } = argv;
+
+initDiagnosticsBundle({ esHost, kbHost, password, username })
+  .then((res) => {
+    console.log(res);
+  })
+  .catch((err) => {
+    console.log(err);
+  });

x-pack/plugins/apm/scripts/test/api.js

@@ -39,7 +39,8 @@ const { argv } = yargs(process.argv.slice(2))
   })
   .option('grep-files', {
     alias: 'files',
-    type: 'string',
+    type: 'array',
+    string: true,
     description: 'Specify the files to run',
   })
   .option('inspect', {
@@ -107,7 +108,7 @@ function runTests() {
   childProcess.execSync(cmd, {
     cwd: path.join(__dirname),
     stdio: 'inherit',
-    env: { ...process.env, APM_TEST_GREP_FILES: grepFiles },
+    env: { ...process.env, APM_TEST_GREP_FILES: JSON.stringify(grepFiles) },
   });
 }
 

x-pack/plugins/apm/server/lib/helpers/create_es_client/create_apm_event_client/index.ts

@@ -11,8 +11,8 @@ import type {
   FieldCapsResponse,
   MsearchMultisearchBody,
   MsearchMultisearchHeader,
-  TermsEnumRequest,
   TermsEnumResponse,
+  TermsEnumRequest,
 } from '@elastic/elasticsearch/lib/api/types';
 import { ElasticsearchClient, KibanaRequest } from '@kbn/core/server';
 import type { ESSearchRequest, InferSearchResponseOf } from '@kbn/es-types';
@@ -46,17 +46,13 @@ export type APMEventESSearchRequest = Omit<ESSearchRequest, 'index'> & {
   };
 };
 
-export type APMEventESTermsEnumRequest = Omit<TermsEnumRequest, 'index'> & {
+type APMEventWrapper<T> = Omit<T, 'index'> & {
   apm: { events: ProcessorEvent[] };
 };
 
-export type APMEventEqlSearchRequest = Omit<EqlSearchRequest, 'index'> & {
-  apm: { events: ProcessorEvent[] };
-};
-
-export type APMEventFieldCapsRequest = Omit<FieldCapsRequest, 'index'> & {
-  apm: { events: ProcessorEvent[] };
-};
+type APMEventTermsEnumRequest = APMEventWrapper<TermsEnumRequest>;
+type APMEventEqlSearchRequest = APMEventWrapper<EqlSearchRequest>;
+type APMEventFieldCapsRequest = APMEventWrapper<FieldCapsRequest>;
 
 // These keys shoul all be `ProcessorEvent.x`, but until TypeScript 4.2 we're inlining them here.
 // See https://github.com/microsoft/TypeScript/issues/37888
@@ -280,7 +276,7 @@ export class APMEventClient {
 
     return this.callAsyncWithDebug({
       operationName,
-      requestType: 'field_caps',
+      requestType: '_field_caps',
       params: requestParams,
       cb: (opts) => this.esClient.fieldCaps(requestParams, opts),
     });
@@ -288,7 +284,7 @@ export class APMEventClient {
 
   async termsEnum(
     operationName: string,
-    params: APMEventESTermsEnumRequest
+    params: APMEventTermsEnumRequest
   ): Promise<TermsEnumResponse> {
     const index = processorEventsToIndex(params.apm.events, this.indices);
 
@@ -299,7 +295,7 @@ export class APMEventClient {
 
     return this.callAsyncWithDebug({
       operationName,
-      requestType: 'terms_enum',
+      requestType: '_terms_enum',
       params: requestParams,
       cb: (opts) => this.esClient.termsEnum(requestParams, opts),
     });

x-pack/plugins/apm/server/routes/apm_routes/get_global_apm_server_route_repository.ts

@@ -35,6 +35,7 @@ import { agentConfigurationRouteRepository } from '../settings/agent_configurati
 import { anomalyDetectionRouteRepository } from '../settings/anomaly_detection/route';
 import { apmIndicesRouteRepository } from '../settings/apm_indices/route';
 import { customLinkRouteRepository } from '../settings/custom_link/route';
+import { diagnosticsRepository } from '../diagnostics/route';
 import { labsRouteRepository } from '../settings/labs/route';
 import { sourceMapsRouteRepository } from '../source_maps/route';
 import { spanLinksRouteRepository } from '../span_links/route';
@@ -79,6 +80,7 @@ function getTypedGlobalApmServerRouteRepository() {
     ...labsRouteRepository,
     ...agentExplorerRouteRepository,
     ...mobileRouteRepository,
+    ...diagnosticsRepository,
   };
 
   return repository;

x-pack/plugins/apm/server/routes/diagnostics/bundle/get_data_streams.ts

@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
+import { ApmIndicesConfig } from '../../settings/apm_indices/get_apm_indices';
+import { getApmIndexPatterns } from './get_indices';
+
+export async function getDataStreams({
+  esClient,
+  apmIndices,
+}: {
+  esClient: ElasticsearchClient;
+  apmIndices: ApmIndicesConfig;
+}) {
+  const apmIndexPatterns = getApmIndexPatterns([
+    apmIndices.error,
+    apmIndices.metric,
+    apmIndices.span,
+    apmIndices.transaction,
+  ]);
+
+  // fetch APM data streams
+  const { data_streams: dataStreams } = await esClient.indices.getDataStream({
+    name: apmIndexPatterns,
+    filter_path: ['data_streams.name', 'data_streams.template'],
+    // @ts-expect-error
+    ignore_unavailable: true,
+  });
+
+  return dataStreams ?? [];
+}

x-pack/plugins/apm/server/routes/diagnostics/bundle/get_existing_index_templates.ts

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
+import { getIndexTemplate } from './get_index_template';
+
+export type ApmIndexTemplateStates = Record<
+  string,
+  { exists: boolean; name?: string | undefined }
+>;
+
+// Check whether the default APM index templates exist
+export async function getExistingApmIndexTemplates({
+  esClient,
+  apmIndexTemplateNames,
+}: {
+  esClient: ElasticsearchClient;
+  apmIndexTemplateNames: string[];
+}) {
+  const values = await Promise.all(
+    apmIndexTemplateNames.map(async (indexTemplateName) => {
+      const res = await getIndexTemplate(esClient, { name: indexTemplateName });
+      return res.index_templates[0];
+    })
+  );
+
+  return values.filter((v) => v !== undefined);
+}

x-pack/plugins/apm/server/routes/diagnostics/bundle/get_field_caps.ts

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
+import { SERVICE_NAME } from '../../../../common/es_fields/apm';
+import { ApmIndicesConfig } from '../../settings/apm_indices/get_apm_indices';
+import { getApmIndexPatterns } from './get_indices';
+
+export function getFieldCaps({
+  esClient,
+  apmIndices,
+}: {
+  esClient: ElasticsearchClient;
+  apmIndices: ApmIndicesConfig;
+}) {
+  return esClient.fieldCaps({
+    index: getApmIndexPatterns([apmIndices.metric, apmIndices.transaction]),
+    fields: [SERVICE_NAME],
+    filter_path: ['fields'],
+    filters: '-parent',
+    include_unmapped: true,
+    ignore_unavailable: true,
+  });
+}

x-pack/plugins/apm/server/routes/diagnostics/bundle/get_index_template.ts

@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+  IndicesGetIndexTemplateRequest,
+  IndicesGetIndexTemplateResponse,
+} from '@elastic/elasticsearch/lib/api/types';
+import { errors } from '@elastic/elasticsearch';
+import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
+
+export async function getIndexTemplate(
+  esClient: ElasticsearchClient,
+  params: IndicesGetIndexTemplateRequest
+): Promise<IndicesGetIndexTemplateResponse> {
+  try {
+    return await esClient.indices.getIndexTemplate(params, {
+      signal: new AbortController().signal,
+    });
+  } catch (e) {
+    if (e instanceof errors.ResponseError && e.statusCode === 404) {
+      return { index_templates: [] };
+    }
+
+    throw e;
+  }
+}

x-pack/plugins/apm/server/routes/diagnostics/bundle/get_index_templates_by_index_pattern.ts

@@ -0,0 +1,92 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
+import { IndicesSimulateTemplateResponse } from '@elastic/elasticsearch/lib/api/types';
+import { orderBy } from 'lodash';
+import { ApmIndicesConfig } from '../../settings/apm_indices/get_apm_indices';
+import { getApmIndexPatterns } from './get_indices';
+import { getIndexTemplate } from './get_index_template';
+import { getApmIndexTemplateNames } from '../get_apm_index_template_names';
+
+export async function getIndexTemplatesByIndexPattern({
+  esClient,
+  apmIndices,
+}: {
+  esClient: ElasticsearchClient;
+  apmIndices: ApmIndicesConfig;
+}) {
+  const indexPatterns = getApmIndexPatterns([
+    apmIndices.error,
+    apmIndices.metric,
+    apmIndices.span,
+    apmIndices.transaction,
+  ]);
+
+  return Promise.all(
+    indexPatterns.map(async (indexPattern) =>
+      getSimulatedIndexTemplateForIndexPattern({ indexPattern, esClient })
+    )
+  );
+}
+
+async function getSimulatedIndexTemplateForIndexPattern({
+  esClient,
+  indexPattern,
+}: {
+  esClient: ElasticsearchClient;
+  indexPattern: string;
+}) {
+  const simulatedIndexTemplate =
+    await esClient.transport.request<IndicesSimulateTemplateResponse>({
+      method: 'POST',
+      path: '/_index_template/_simulate',
+      body: { index_patterns: [indexPattern] },
+    });
+
+  const indexTemplates = await Promise.all(
+    (simulatedIndexTemplate.overlapping ?? []).map(
+      async ({ index_patterns: templateIndexPatterns, name: templateName }) => {
+        const priority = await getTemplatePriority(esClient, templateName);
+        const isNonStandard = getIsNonStandardIndexTemplate(templateName);
+        return {
+          isNonStandard,
+          priority,
+          templateIndexPatterns,
+          templateName,
+        };
+      }
+    )
+  );
+
+  return {
+    indexPattern,
+    indexTemplates: orderBy(indexTemplates, ({ priority }) => priority, 'desc'),
+  };
+}
+
+async function getTemplatePriority(
+  esClient: ElasticsearchClient,
+  name: string
+) {
+  const res = await getIndexTemplate(esClient, { name });
+  return res.index_templates[0]?.index_template?.priority;
+}
+
+function getIsNonStandardIndexTemplate(templateName: string) {
+  const apmIndexTemplateNames = getApmIndexTemplateNames();
+  const stackIndexTemplateNames = ['logs', 'metrics'];
+  const isNonStandard = [
+    ...apmIndexTemplateNames,
+    ...stackIndexTemplateNames,
+  ].every((apmIndexTemplateName) => {
+    const notMatch = templateName !== apmIndexTemplateName;
+    return notMatch;
+  });
+
+  return isNonStandard;
+}

x-pack/plugins/apm/server/routes/diagnostics/bundle/get_indices.ts

@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { compact, uniq } from 'lodash';
+import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
+import { ApmIndicesConfig } from '../../settings/apm_indices/get_apm_indices';
+
+export function getApmIndexPatterns(indices: string[]) {
+  return uniq(indices.flatMap((index): string[] => index.split(',')));
+}
+
+export async function getIndicesAndIngestPipelines({
+  esClient,
+  apmIndices,
+}: {
+  esClient: ElasticsearchClient;
+  apmIndices: ApmIndicesConfig;
+}) {
+  const indices = await esClient.indices.get({
+    index: getApmIndexPatterns([
+      apmIndices.error,
+      apmIndices.metric,
+      apmIndices.span,
+      apmIndices.transaction,
+    ]),
+    filter_path: [
+      '*.settings.index.default_pipeline',
+      '*.data_stream',
+      '*.settings.index.provided_name',
+    ],
+    ignore_unavailable: true,
+  });
+
+  const pipelineIds = compact(
+    uniq(
+      Object.values(indices).map(
+        (index) => index.settings?.index?.default_pipeline
+      )
+    )
+  ).join(',');
+
+  const ingestPipelines = await esClient.ingest.getPipeline({
+    id: pipelineIds,
+    filter_path: ['*.processors.grok.field', '*.processors.grok.patterns'],
+  });
+
+  return { indices, ingestPipelines };
+}

x-pack/plugins/apm/server/routes/diagnostics/bundle/get_indices_states.ts

@@ -0,0 +1,92 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+  FieldCapsResponse,
+  IndicesGetResponse,
+  IngestGetPipelineResponse,
+} from '@elastic/elasticsearch/lib/api/types';
+import { SERVICE_NAME } from '../../../../common/es_fields/apm';
+import { getApmIndexTemplateNames } from '../get_apm_index_template_names';
+
+export function getIndicesStates({
+  indices,
+  fieldCaps,
+  ingestPipelines,
+}: {
+  indices: IndicesGetResponse;
+  fieldCaps: FieldCapsResponse;
+  ingestPipelines: IngestGetPipelineResponse;
+}) {
+  const indicesWithPipelineId = Object.entries(indices).map(([key, value]) => ({
+    index: key,
+    dataStream: value.data_stream,
+    pipelineId: value.settings?.index?.default_pipeline,
+  }));
+
+  const invalidFieldMappings = Object.values(
+    fieldCaps.fields[SERVICE_NAME] ?? {}
+  ).filter(({ type }): boolean => type !== 'keyword');
+
+  const items = indicesWithPipelineId.map(
+    ({ index, dataStream, pipelineId }) => {
+      const hasObserverVersionProcessor = pipelineId
+        ? ingestPipelines[pipelineId]?.processors?.some((processor) => {
+            return (
+              processor?.grok?.field === 'observer.version' &&
+              processor?.grok?.patterns[0] ===
+                '%{DIGITS:observer.version_major:int}.%{DIGITS:observer.version_minor:int}.%{DIGITS:observer.version_patch:int}(?:[-+].*)?'
+            );
+          })
+        : false;
+
+      const invalidFieldMapping = invalidFieldMappings.find((fieldMappings) =>
+        fieldMappings.indices?.includes(index)
+      );
+
+      const isValidFieldMappings = invalidFieldMapping === undefined;
+      const isValidIngestPipeline =
+        hasObserverVersionProcessor === true &&
+        validateIngestPipelineName(dataStream, pipelineId);
+
+      return {
+        isValid: isValidFieldMappings && isValidIngestPipeline,
+        fieldMappings: {
+          isValid: isValidFieldMappings,
+          invalidType: invalidFieldMapping?.type,
+        },
+        ingestPipeline: {
+          isValid: isValidIngestPipeline,
+          id: pipelineId,
+        },
+        index,
+        dataStream,
+      };
+    }
+  );
+
+  const invalidIndices = items.filter((item) => !item.isValid);
+  const validIndices = items.filter((item) => item.isValid);
+
+  return { invalidIndices, validIndices };
+}
+
+export function validateIngestPipelineName(
+  dataStream: string | undefined,
+  ingestPipelineId: string | undefined
+) {
+  if (!dataStream || !ingestPipelineId) {
+    return false;
+  }
+
+  const indexTemplateNames = getApmIndexTemplateNames();
+  return indexTemplateNames.some(
+    (indexTemplateName) =>
+      dataStream.startsWith(indexTemplateName) &&
+      ingestPipelineId.startsWith(indexTemplateName)
+  );
+}

x-pack/plugins/apm/server/routes/diagnostics/bundle/get_non_data_stream_indices.ts

@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
+import { ApmIndicesConfig } from '@kbn/observability-plugin/common/typings';
+import { getApmIndexPatterns } from './get_indices';
+
+export async function getNonDataStreamIndices({
+  esClient,
+  apmIndices,
+}: {
+  esClient: ElasticsearchClient;
+  apmIndices: ApmIndicesConfig;
+}) {
+  const apmIndexPatterns = getApmIndexPatterns([
+    apmIndices.error,
+    apmIndices.metric,
+    apmIndices.span,
+    apmIndices.transaction,
+  ]);
+
+  // TODO: indices already retrieved by `getIndicesAndIngestPipelines`
+  const nonDataStreamIndicesResponse = await esClient.indices.get({
+    index: apmIndexPatterns,
+    filter_path: ['*.data_stream', '*.settings.index.uuid'],
+    ignore_unavailable: true,
+  });
+
+  const nonDataStreamIndices = Object.entries(nonDataStreamIndicesResponse)
+    .filter(([indexName, { data_stream: dataStream }]): boolean => !dataStream)
+    .map(([indexName]): string => indexName);
+
+  return nonDataStreamIndices;
+}

x-pack/plugins/apm/server/routes/diagnostics/get_apm_index_template_names.ts

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+export function getApmIndexTemplateNames() {
+  const indexTemplateNames = [
+    'logs-apm.app',
+    'logs-apm.error',
+    'metrics-apm.app',
+    'metrics-apm.internal',
+    'traces-apm.rum',
+    'traces-apm.sampled',
+    'traces-apm',
+  ];
+
+  const rollupIndexTemplateNames = ['1m', '10m', '60m'].flatMap((interval) => {
+    return [
+      'metrics-apm.service_destination',
+      'metrics-apm.service_summary',
+      'metrics-apm.service_transaction',
+      'metrics-apm.transaction',
+    ].map((ds) => `${ds}.${interval}`);
+  });
+
+  return [...indexTemplateNames, ...rollupIndexTemplateNames];
+}

x-pack/plugins/apm/server/routes/diagnostics/get_diagnostics_bundle.ts

@@ -0,0 +1,106 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { IndicesGetIndexTemplateIndexTemplateItem } from '@elastic/elasticsearch/lib/api/types';
+import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
+import { ApmIndicesConfig } from '../settings/apm_indices/get_apm_indices';
+import { getDataStreams } from './bundle/get_data_streams';
+import { getNonDataStreamIndices } from './bundle/get_non_data_stream_indices';
+import { getApmIndexTemplateNames } from './get_apm_index_template_names';
+import { getElasticsearchVersion } from './get_elasticsearch_version';
+import { getIndexTemplatesByIndexPattern } from './bundle/get_index_templates_by_index_pattern';
+import { getExistingApmIndexTemplates } from './bundle/get_existing_index_templates';
+import { getFieldCaps } from './bundle/get_field_caps';
+import { getIndicesAndIngestPipelines } from './bundle/get_indices';
+import { getIndicesStates } from './bundle/get_indices_states';
+
+export async function getDiagnosticsBundle(
+  esClient: ElasticsearchClient,
+  apmIndices: ApmIndicesConfig
+) {
+  const apmIndexTemplateNames = getApmIndexTemplateNames();
+
+  const { indices, ingestPipelines } = await getIndicesAndIngestPipelines({
+    esClient,
+    apmIndices,
+  });
+
+  const indexTemplatesByIndexPattern = await getIndexTemplatesByIndexPattern({
+    esClient,
+    apmIndices,
+  });
+
+  const existingIndexTemplates = await getExistingApmIndexTemplates({
+    esClient,
+    apmIndexTemplateNames,
+  });
+
+  const fieldCaps = await getFieldCaps({ esClient, apmIndices });
+  const dataStreams = await getDataStreams({ esClient, apmIndices });
+  const nonDataStreamIndices = await getNonDataStreamIndices({
+    esClient,
+    apmIndices,
+  });
+
+  const { invalidIndices, validIndices } = getIndicesStates({
+    fieldCaps,
+    indices,
+    ingestPipelines,
+  });
+
+  return {
+    created_at: new Date().toISOString(),
+    elasticsearchVersion: await getElasticsearchVersion(esClient),
+    esResponses: {
+      fieldCaps,
+      indices,
+      ingestPipelines,
+      existingIndexTemplates,
+    },
+    apmIndexTemplates: getApmIndexTemplates(
+      apmIndexTemplateNames,
+      existingIndexTemplates
+    ),
+    invalidIndices,
+    validIndices,
+    indexTemplatesByIndexPattern,
+    dataStreams,
+    nonDataStreamIndices,
+  };
+}
+
+function getApmIndexTemplates(
+  apmIndexTemplateNames: string[],
+  existingIndexTemplates: IndicesGetIndexTemplateIndexTemplateItem[]
+) {
+  const standardIndexTemplates = apmIndexTemplateNames.map((templateName) => {
+    const matchingTemplate = existingIndexTemplates.find(
+      ({ name }) => name === templateName
+    );
+
+    return {
+      name: templateName,
+      exists: Boolean(matchingTemplate),
+      isNonStandard: false,
+    };
+  });
+
+  const nonStandardIndexTemplates = existingIndexTemplates
+    .filter(
+      (indexTemplate) =>
+        standardIndexTemplates.some(
+          ({ name }) => name === indexTemplate.name
+        ) === false
+    )
+    .map((indexTemplate) => ({
+      name: indexTemplate.name,
+      isNonStandard: true,
+      exists: true,
+    }));
+
+  return [...standardIndexTemplates, ...nonStandardIndexTemplates];
+}

x-pack/plugins/apm/server/routes/diagnostics/get_elasticsearch_version.ts

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
+
+export async function getElasticsearchVersion(esClient: ElasticsearchClient) {
+  const { version } = await esClient.info();
+  return version.number;
+}

x-pack/plugins/apm/server/routes/diagnostics/get_fleet_package_info.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { APMRouteHandlerResources } from '../typings';
+
+export async function getFleetPackageInfo(resources: APMRouteHandlerResources) {
+  const fleetPluginStart = await resources.plugins.fleet?.start();
+  const packageInfo = await fleetPluginStart?.packageService
+    .asScoped(resources.request)
+    .getInstallation('apm');
+
+  return {
+    isInstalled: packageInfo?.install_status === 'installed',
+    version: packageInfo?.version,
+  };
+}

x-pack/plugins/apm/server/routes/diagnostics/route.ts

@@ -0,0 +1,89 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+  FieldCapsResponse,
+  IndicesDataStream,
+  IndicesGetIndexTemplateIndexTemplateItem,
+  IndicesGetResponse,
+  IngestGetPipelineResponse,
+} from '@elastic/elasticsearch/lib/api/types';
+import { createApmServerRoute } from '../apm_routes/create_apm_server_route';
+import { getApmIndices } from '../settings/apm_indices/get_apm_indices';
+import { getDiagnosticsBundle } from './get_diagnostics_bundle';
+import { getFleetPackageInfo } from './get_fleet_package_info';
+
+export interface IndiciesItem {
+  index: string;
+  fieldMappings: {
+    isValid: boolean;
+    invalidType?: string;
+  };
+  ingestPipeline: {
+    isValid?: boolean;
+    id?: string;
+  };
+  dataStream?: string;
+  isValid: boolean;
+}
+
+const getDiagnosticsRoute = createApmServerRoute({
+  endpoint: 'GET /internal/apm/diagnostics',
+
+  options: { tags: ['access:apm'] },
+  handler: async (
+    resources
+  ): Promise<{
+    esResponses: {
+      existingIndexTemplates: IndicesGetIndexTemplateIndexTemplateItem[];
+      fieldCaps: FieldCapsResponse;
+      indices: IndicesGetResponse;
+      ingestPipelines: IngestGetPipelineResponse;
+    };
+    apmIndexTemplates: Array<{
+      name: string;
+      isNonStandard: boolean;
+      exists: boolean;
+    }>;
+    fleetPackageInfo: {
+      isInstalled: boolean;
+      version?: string;
+    };
+    kibanaVersion: string;
+    elasticsearchVersion: string;
+    invalidIndices: IndiciesItem[];
+    validIndices: IndiciesItem[];
+    dataStreams: IndicesDataStream[];
+    nonDataStreamIndices: string[];
+    indexTemplatesByIndexPattern: Array<{
+      indexPattern: string;
+      indexTemplates: Array<{
+        priority: number | undefined;
+        isNonStandard: boolean;
+        templateIndexPatterns: string[];
+        templateName: string;
+      }>;
+    }>;
+  }> => {
+    const coreContext = await resources.context.core;
+    const { asCurrentUser: esClient } = coreContext.elasticsearch.client;
+    const apmIndices = await getApmIndices({
+      savedObjectsClient: coreContext.savedObjects.client,
+      config: resources.config,
+    });
+
+    const bundle = await getDiagnosticsBundle(esClient, apmIndices);
+    const fleetPackageInfo = await getFleetPackageInfo(resources);
+    const kibanaVersion = resources.kibanaVersion;
+
+    return { ...bundle, fleetPackageInfo, kibanaVersion };
+  },
+});
+
+export const diagnosticsRepository = {
+  ...getDiagnosticsRoute,
+};

x-pack/test/apm_api_integration/common/bootstrap_apm_synthtrace.ts

@@ -16,10 +16,24 @@ import { InheritedFtrProviderContext } from './ftr_provider_context';
 
 export async function bootstrapApmSynthtrace(
   context: InheritedFtrProviderContext,
-  kibanaServerUrl: string
+  kibanaClient: ApmSynthtraceKibanaClient
 ) {
   const es = context.getService('es');
 
+  const kibanaVersion = await kibanaClient.fetchLatestApmPackageVersion();
+  await kibanaClient.installApmPackage(kibanaVersion);
+
+  const esClient = new ApmSynthtraceEsClient({
+    client: es,
+    logger: createLogger(LogLevel.info),
+    version: kibanaVersion,
+    refreshAfterIndex: true,
+  });
+
+  return esClient;
+}
+
+export function getApmSynthtraceKibanaClient(kibanaServerUrl: string) {
   const kibanaServerUrlWithAuth = url
     .format({
       ...url.parse(kibanaServerUrl),
@@ -32,16 +46,5 @@ export async function bootstrapApmSynthtrace(
     logger: createLogger(LogLevel.debug),
   });
 
-  const kibanaVersion = await kibanaClient.fetchLatestApmPackageVersion();
-
-  await kibanaClient.installApmPackage(kibanaVersion);
-
-  const esClient = new ApmSynthtraceEsClient({
-    client: es,
-    logger: createLogger(LogLevel.info),
-    version: kibanaVersion,
-    refreshAfterIndex: true,
-  });
-
-  return esClient;
+  return kibanaClient;
 }

x-pack/test/apm_api_integration/common/config.ts

@@ -10,14 +10,14 @@ import {
   APM_TEST_PASSWORD,
 } from '@kbn/apm-plugin/server/test_helpers/create_apm_users/authentication';
 import { createApmUsers } from '@kbn/apm-plugin/server/test_helpers/create_apm_users/create_apm_users';
-import { ApmSynthtraceEsClient } from '@kbn/apm-synthtrace';
+import { ApmSynthtraceEsClient, ApmSynthtraceKibanaClient } from '@kbn/apm-synthtrace';
 import { FtrConfigProviderContext } from '@kbn/test';
 import supertest from 'supertest';
 import { format, UrlObject } from 'url';
 import { MachineLearningAPIProvider } from '../../functional/services/ml/api';
 import { APMFtrConfigName } from '../configs';
 import { createApmApiClient } from './apm_api_supertest';
-import { bootstrapApmSynthtrace } from './bootstrap_apm_synthtrace';
+import { bootstrapApmSynthtrace, getApmSynthtraceKibanaClient } from './bootstrap_apm_synthtrace';
 import {
   FtrProviderContext,
   InheritedFtrProviderContext,
@@ -36,7 +36,7 @@ async function getApmApiClient({
   username,
 }: {
   kibanaServer: UrlObject;
-  username: ApmUsername;
+  username: ApmUsername | 'elastic';
 }) {
   const url = format({
     ...kibanaServer,
@@ -51,6 +51,7 @@ export type CreateTestConfig = ReturnType<typeof createTestConfig>;
 type ApmApiClientKey =
   | 'noAccessUser'
   | 'readUser'
+  | 'adminUser'
   | 'writeUser'
   | 'annotationWriterUser'
   | 'noMlAccessUser'
@@ -69,6 +70,9 @@ export interface CreateTest {
     apmFtrConfig: () => ApmFtrConfig;
     registry: ({ getService }: FtrProviderContext) => ReturnType<typeof RegistryProvider>;
     synthtraceEsClient: (context: InheritedFtrProviderContext) => Promise<ApmSynthtraceEsClient>;
+    synthtraceKibanaClient: (
+      context: InheritedFtrProviderContext
+    ) => Promise<ApmSynthtraceKibanaClient>;
     apmApiClient: (context: InheritedFtrProviderContext) => ApmApiClient;
     ml: ({ getService }: FtrProviderContext) => ReturnType<typeof MachineLearningAPIProvider>;
   };
@@ -92,6 +96,7 @@ export function createTestConfig(
     const kibanaServer = servers.kibana as UrlObject;
     const kibanaServerUrl = format(kibanaServer);
     const esServer = servers.elasticsearch as UrlObject;
+    const synthtraceKibanaClient = getApmSynthtraceKibanaClient(kibanaServerUrl);
 
     return {
       testFiles: [require.resolve('../tests')],
@@ -102,8 +107,9 @@ export function createTestConfig(
         apmFtrConfig: () => config,
         registry: RegistryProvider,
         synthtraceEsClient: (context: InheritedFtrProviderContext) => {
-          return bootstrapApmSynthtrace(context, kibanaServerUrl);
+          return bootstrapApmSynthtrace(context, synthtraceKibanaClient);
         },
+        synthtraceKibanaClient: () => synthtraceKibanaClient,
         apmApiClient: async (context: InheritedFtrProviderContext) => {
           const { username, password } = servers.kibana;
           const esUrl = format(esServer);
@@ -123,6 +129,10 @@ export function createTestConfig(
               kibanaServer,
               username: ApmUsername.viewerUser,
             }),
+            adminUser: await getApmApiClient({
+              kibanaServer,
+              username: 'elastic',
+            }),
             writeUser: await getApmApiClient({
               kibanaServer,
               username: ApmUsername.editorUser,

x-pack/test/apm_api_integration/common/registry.ts

@@ -178,7 +178,7 @@ export function RegistryProvider({ getService }: FtrProviderContext) {
               }
             };
 
-            describe(condition.archives.join(',') || 'no data', () => {
+            describe(condition.archives.join(',') || 'no archive', () => {
               before(runBefore);
 
               runs.forEach((run) => {

x-pack/test/apm_api_integration/tests/diagnostics/data_streams.spec.ts

@@ -0,0 +1,100 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import { apm, timerange } from '@kbn/apm-synthtrace-client';
+import { FtrProviderContext } from '../../common/ftr_provider_context';
+
+export default function ApiTest({ getService }: FtrProviderContext) {
+  const registry = getService('registry');
+  const apmApiClient = getService('apmApiClient');
+  const es = getService('es');
+  const synthtraceEsClient = getService('synthtraceEsClient');
+  const synthtraceKibanaClient = getService('synthtraceKibanaClient');
+
+  const start = new Date('2021-01-01T00:00:00.000Z').getTime();
+  const end = new Date('2021-01-01T00:15:00.000Z').getTime() - 1;
+
+  registry.when('Diagnostics: Data streams', { config: 'basic', archives: [] }, () => {
+    describe('When there is no data', () => {
+      before(async () => {
+        // delete APM data streams
+        await es.indices.deleteDataStream({ name: '*apm*' });
+      });
+
+      it('returns zero data streams`', async () => {
+        const { status, body } = await apmApiClient.adminUser({
+          endpoint: 'GET /internal/apm/diagnostics',
+        });
+        expect(status).to.be(200);
+        expect(body.dataStreams).to.eql([]);
+      });
+
+      it('returns zero non-data stream indices`', async () => {
+        const { status, body } = await apmApiClient.adminUser({
+          endpoint: 'GET /internal/apm/diagnostics',
+        });
+        expect(status).to.be(200);
+        expect(body.nonDataStreamIndices).to.eql([]);
+      });
+    });
+
+    describe('When data is ingested', () => {
+      before(async () => {
+        const latestVersion = await synthtraceKibanaClient.fetchLatestApmPackageVersion();
+        await synthtraceKibanaClient.installApmPackage(latestVersion);
+
+        const instance = apm
+          .service({ name: 'synth-go', environment: 'production', agentName: 'go' })
+          .instance('instance-a');
+
+        await synthtraceEsClient.index(
+          timerange(start, end)
+            .interval('1m')
+            .rate(30)
+            .generator((timestamp) =>
+              instance
+                .transaction({ transactionName: 'GET /users' })
+                .timestamp(timestamp)
+                .duration(100)
+                .success()
+            )
+        );
+      });
+
+      after(() => synthtraceEsClient.clean());
+
+      it('returns 5 data streams', async () => {
+        const { status, body } = await apmApiClient.adminUser({
+          endpoint: 'GET /internal/apm/diagnostics',
+        });
+        expect(status).to.be(200);
+        expect(body.dataStreams).to.eql([
+          { name: 'metrics-apm.internal-default', template: 'metrics-apm.internal' },
+          {
+            name: 'metrics-apm.service_summary.1m-default',
+            template: 'metrics-apm.service_summary.1m',
+          },
+          {
+            name: 'metrics-apm.service_transaction.1m-default',
+            template: 'metrics-apm.service_transaction.1m',
+          },
+          { name: 'metrics-apm.transaction.1m-default', template: 'metrics-apm.transaction.1m' },
+          { name: 'traces-apm-default', template: 'traces-apm' },
+        ]);
+      });
+
+      it('returns zero non-data stream indices', async () => {
+        const { status, body } = await apmApiClient.adminUser({
+          endpoint: 'GET /internal/apm/diagnostics',
+        });
+        expect(status).to.be(200);
+        expect(body.nonDataStreamIndices).to.eql([]);
+      });
+    });
+  });
+}

x-pack/test/apm_api_integration/tests/diagnostics/index_pattern_settings.ts

@@ -0,0 +1,104 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import { apm, timerange } from '@kbn/apm-synthtrace-client';
+import { getApmIndexTemplateNames } from '@kbn/apm-plugin/server/routes/diagnostics/get_apm_index_template_names';
+import { FtrProviderContext } from '../../common/ftr_provider_context';
+
+export default function ApiTest({ getService }: FtrProviderContext) {
+  const registry = getService('registry');
+  const apmApiClient = getService('apmApiClient');
+  const es = getService('es');
+  const synthtraceEsClient = getService('synthtraceEsClient');
+  const synthtraceKibanaClient = getService('synthtraceKibanaClient');
+
+  const start = new Date('2021-01-01T00:00:00.000Z').getTime();
+  const end = new Date('2021-01-01T00:15:00.000Z').getTime() - 1;
+
+  registry.when('Diagnostics: Index pattern settings', { config: 'basic', archives: [] }, () => {
+    describe('When there is no data', () => {
+      before(async () => {
+        // delete APM index templates
+        await es.indices.deleteIndexTemplate({ name: getApmIndexTemplateNames() });
+      });
+
+      it('returns the built-in (non-APM) index templates`', async () => {
+        const { status, body } = await apmApiClient.adminUser({
+          endpoint: 'GET /internal/apm/diagnostics',
+        });
+        expect(status).to.be(200);
+
+        const templateNames = body.indexTemplatesByIndexPattern.flatMap(({ indexTemplates }) => {
+          return indexTemplates?.map(({ templateName }) => templateName);
+        });
+
+        expect(templateNames).to.eql(['logs', 'metrics']);
+      });
+    });
+
+    describe('When data is ingested', () => {
+      before(async () => {
+        const latestVersion = await synthtraceKibanaClient.fetchLatestApmPackageVersion();
+        await synthtraceKibanaClient.installApmPackage(latestVersion);
+
+        const instance = apm
+          .service({ name: 'synth-go', environment: 'production', agentName: 'go' })
+          .instance('instance-a');
+        await synthtraceEsClient.index(
+          timerange(start, end)
+            .interval('1m')
+            .rate(30)
+            .generator((timestamp) =>
+              instance
+                .transaction({ transactionName: 'GET /users' })
+                .timestamp(timestamp)
+                .duration(100)
+                .success()
+            )
+        );
+      });
+
+      after(() => synthtraceEsClient.clean());
+
+      it('returns APM index templates', async () => {
+        const { status, body } = await apmApiClient.adminUser({
+          endpoint: 'GET /internal/apm/diagnostics',
+        });
+        expect(status).to.be(200);
+
+        const templateNames = body.indexTemplatesByIndexPattern.flatMap(({ indexTemplates }) => {
+          return indexTemplates?.map(({ templateName }) => templateName);
+        });
+
+        expect(templateNames).to.eql([
+          'logs-apm.error',
+          'logs-apm.app',
+          'logs',
+          'metrics-apm.service_transaction.60m',
+          'metrics-apm.service_destination.10m',
+          'metrics-apm.transaction.1m',
+          'metrics-apm.service_destination.1m',
+          'metrics-apm.service_transaction.10m',
+          'metrics-apm.service_transaction.1m',
+          'metrics-apm.transaction.60m',
+          'metrics-apm.service_destination.60m',
+          'metrics-apm.service_summary.1m',
+          'metrics-apm.transaction.10m',
+          'metrics-apm.internal',
+          'metrics-apm.service_summary.10m',
+          'metrics-apm.service_summary.60m',
+          'metrics-apm.app',
+          'metrics',
+          'traces-apm',
+          'traces-apm.rum',
+          'traces-apm.sampled',
+        ]);
+      });
+    });
+  });
+}

x-pack/test/apm_api_integration/tests/diagnostics/index_templates.spec.ts

@@ -0,0 +1,80 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import { apm, timerange } from '@kbn/apm-synthtrace-client';
+import { getApmIndexTemplateNames } from '@kbn/apm-plugin/server/routes/diagnostics/get_apm_index_template_names';
+import { FtrProviderContext } from '../../common/ftr_provider_context';
+
+export default function ApiTest({ getService }: FtrProviderContext) {
+  const registry = getService('registry');
+  const apmApiClient = getService('apmApiClient');
+  const es = getService('es');
+  const synthtraceEsClient = getService('synthtraceEsClient');
+  const synthtraceKibanaClient = getService('synthtraceKibanaClient');
+
+  const start = new Date('2021-01-01T00:00:00.000Z').getTime();
+  const end = new Date('2021-01-01T00:15:00.000Z').getTime() - 1;
+
+  registry.when('Diagnostics: Index Templates', { config: 'basic', archives: [] }, () => {
+    describe('When there is no data', () => {
+      before(async () => {
+        // delete APM index templates
+        await es.indices.deleteIndexTemplate({ name: getApmIndexTemplateNames() });
+      });
+
+      it('verifies that none of the default APM index templates exists`', async () => {
+        const { status, body } = await apmApiClient.adminUser({
+          endpoint: 'GET /internal/apm/diagnostics',
+        });
+        expect(status).to.be(200);
+        const noApmIndexTemplateExists = body.apmIndexTemplates.every(
+          ({ exists }) => exists === false
+        );
+        expect(noApmIndexTemplateExists).to.eql(true);
+      });
+    });
+
+    describe('When data is ingested', () => {
+      before(async () => {
+        const latestVersion = await synthtraceKibanaClient.fetchLatestApmPackageVersion();
+        await synthtraceKibanaClient.installApmPackage(latestVersion);
+
+        const instance = apm
+          .service({ name: 'synth-go', environment: 'production', agentName: 'go' })
+          .instance('instance-a');
+        await synthtraceEsClient.index(
+          timerange(start, end)
+            .interval('1m')
+            .rate(30)
+            .generator((timestamp) =>
+              instance
+                .transaction({ transactionName: 'GET /users' })
+                .timestamp(timestamp)
+                .duration(100)
+                .success()
+            )
+        );
+      });
+
+      after(() => synthtraceEsClient.clean());
+
+      it('verifies that all the default APM index templates exist', async () => {
+        const { status, body } = await apmApiClient.adminUser({
+          endpoint: 'GET /internal/apm/diagnostics',
+        });
+        expect(status).to.be(200);
+
+        const everyApmIndexTemplateExists = body.apmIndexTemplates.every(
+          ({ exists }) => exists === true
+        );
+
+        expect(everyApmIndexTemplateExists).to.eql(true);
+      });
+    });
+  });
+}

x-pack/test/apm_api_integration/tests/diagnostics/indices.spec.ts

@@ -0,0 +1,202 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import { apm, timerange } from '@kbn/apm-synthtrace-client';
+import { omit } from 'lodash';
+import { FtrProviderContext } from '../../common/ftr_provider_context';
+
+export default function ApiTest({ getService }: FtrProviderContext) {
+  const registry = getService('registry');
+  const apmApiClient = getService('apmApiClient');
+  const synthtraceEsClient = getService('synthtraceEsClient');
+  const es = getService('es');
+  const synthtraceKibanaClient = getService('synthtraceKibanaClient');
+
+  const start = new Date('2021-01-01T00:00:00.000Z').getTime();
+  const end = new Date('2021-01-01T00:15:00.000Z').getTime() - 1;
+
+  registry.when('Diagnostics: Indices', { config: 'basic', archives: [] }, () => {
+    describe('When there is no data', () => {
+      it('returns empty response`', async () => {
+        const { status, body } = await apmApiClient.adminUser({
+          endpoint: 'GET /internal/apm/diagnostics',
+        });
+
+        expect(status).to.be(200);
+        expect(body.validIndices).to.eql([]);
+        expect(body.invalidIndices).to.eql([]);
+      });
+    });
+
+    describe('When data is ingested', () => {
+      before(async () => {
+        const instance = apm
+          .service({ name: 'synth-go', environment: 'production', agentName: 'go' })
+          .instance('instance-a');
+
+        await synthtraceEsClient.index(
+          timerange(start, end)
+            .interval('1m')
+            .rate(30)
+            .generator((timestamp) =>
+              instance
+                .transaction({ transactionName: 'GET /users' })
+                .timestamp(timestamp)
+                .duration(100)
+                .success()
+            )
+        );
+      });
+
+      after(() => synthtraceEsClient.clean());
+
+      it('returns empty response', async () => {
+        const { status, body } = await apmApiClient.adminUser({
+          endpoint: 'GET /internal/apm/diagnostics',
+        });
+
+        expect(status).to.be(200);
+        expect(body.validIndices.length).to.be.greaterThan(0);
+        expect(body.invalidIndices).to.eql([]);
+      });
+    });
+
+    describe('When data is ingested without the necessary index templates', () => {
+      before(async () => {
+        await es.indices.deleteDataStream({ name: 'traces-apm-*' });
+        await es.indices.deleteIndexTemplate({ name: ['traces-apm'] });
+
+        const instance = apm
+          .service({ name: 'synth-go', environment: 'production', agentName: 'go' })
+          .instance('instance-a');
+
+        await synthtraceEsClient.index(
+          timerange(start, end)
+            .interval('1m')
+            .rate(30)
+            .generator((timestamp) =>
+              instance
+                .transaction({ transactionName: 'GET /users' })
+                .timestamp(timestamp)
+                .duration(100)
+                .success()
+            )
+        );
+      });
+
+      after(async () => {
+        await es.indices.delete({ index: 'traces-apm-default' });
+        const latestVersion = await synthtraceKibanaClient.fetchLatestApmPackageVersion();
+        await synthtraceKibanaClient.installApmPackage(latestVersion);
+        await synthtraceEsClient.clean();
+      });
+
+      it('returns a list of items with mapping issues', async () => {
+        const { status, body } = await apmApiClient.adminUser({
+          endpoint: 'GET /internal/apm/diagnostics',
+        });
+
+        expect(status).to.be(200);
+        expect(body.validIndices.length).to.be.greaterThan(0);
+        expect(body.invalidIndices).to.eql([
+          {
+            isValid: false,
+            fieldMappings: { isValid: false, invalidType: 'text' },
+            ingestPipeline: { isValid: false },
+            index: 'traces-apm-default',
+          },
+        ]);
+      });
+    });
+
+    describe('ingest pipelines', () => {
+      before(async () => {
+        const instance = apm
+          .service({ name: 'synth-go', environment: 'production', agentName: 'go' })
+          .instance('instance-a');
+
+        await synthtraceEsClient.index(
+          timerange(start, end)
+            .interval('1m')
+            .rate(30)
+            .generator((timestamp) =>
+              instance
+                .transaction({ transactionName: 'GET /users' })
+                .timestamp(timestamp)
+                .duration(100)
+                .success()
+            )
+        );
+      });
+
+      after(async () => {
+        const latestVersion = await synthtraceKibanaClient.fetchLatestApmPackageVersion();
+        await synthtraceKibanaClient.installApmPackage(latestVersion);
+        await synthtraceEsClient.clean();
+      });
+
+      describe('an ingest pipeline is removed', () => {
+        before(async () => {
+          const datastreamToUpdate = await es.indices.getDataStream({
+            name: 'metrics-apm.internal-default',
+          });
+          await es.indices.putSettings({
+            index: datastreamToUpdate.data_streams[0].indices[0].index_name,
+            // @ts-expect-error: Allow null values in https://github.com/elastic/elasticsearch-specification/pull/2126
+            body: { index: { default_pipeline: null } },
+          });
+        });
+
+        it('returns the item without an ingest pipeline', async () => {
+          const { status, body } = await apmApiClient.adminUser({
+            endpoint: 'GET /internal/apm/diagnostics',
+          });
+
+          expect(status).to.be(200);
+          expect(body.validIndices.length).to.be.greaterThan(0);
+          expect(body.invalidIndices.length).to.be(1);
+
+          expect(omit(body.invalidIndices[0], 'index')).to.eql({
+            isValid: false,
+            fieldMappings: { isValid: true },
+            ingestPipeline: { isValid: false },
+            dataStream: 'metrics-apm.internal-default',
+          });
+        });
+      });
+
+      describe('an ingest pipeline is changed', () => {
+        before(async () => {
+          const datastreamToUpdate = await es.indices.getDataStream({
+            name: 'metrics-apm.internal-default',
+          });
+          await es.indices.putSettings({
+            index: datastreamToUpdate.data_streams[0].indices[0].index_name,
+            body: { index: { default_pipeline: 'logs-default-pipeline' } },
+          });
+        });
+
+        it('returns the item without an ingest pipeline', async () => {
+          const { status, body } = await apmApiClient.adminUser({
+            endpoint: 'GET /internal/apm/diagnostics',
+          });
+
+          expect(status).to.be(200);
+          expect(body.validIndices.length).to.be.greaterThan(0);
+          expect(body.invalidIndices.length).to.be(1);
+          expect(omit(body.invalidIndices[0], 'index')).to.eql({
+            isValid: false,
+            fieldMappings: { isValid: true },
+            ingestPipeline: { isValid: false, id: 'logs-default-pipeline' },
+            dataStream: 'metrics-apm.internal-default',
+          });
+        });
+      });
+    });
+  });
+}

x-pack/test/apm_api_integration/tests/index.ts

@@ -9,37 +9,39 @@ import path from 'path';
 import { FtrProviderContext } from '../common/ftr_provider_context';
 
 const cwd = path.join(__dirname);
+const envGrepFiles = process.env.APM_TEST_GREP_FILES as string;
 
-const envGrepFiles = process.env.APM_TEST_GREP_FILES;
 function getGlobPattern() {
-  if (!envGrepFiles) {
-    return '**/*.spec.ts';
+  try {
+    const envGrepFilesParsed = JSON.parse(envGrepFiles as string) as string[];
+    return envGrepFilesParsed.map((pattern) => `**/${pattern}**`);
+  } catch (e) {
+    // ignore
   }
-
-  return envGrepFiles.includes('**') ? envGrepFiles : `**/*${envGrepFiles}*`;
+  return '**/*.spec.ts';
 }
 
 export default function apmApiIntegrationTests({ getService, loadTestFile }: FtrProviderContext) {
   const registry = getService('registry');
 
   describe('APM API tests', function () {
-    const tests = globby.sync(getGlobPattern(), { cwd });
+    const filePattern = getGlobPattern();
+    const tests = globby.sync(filePattern, { cwd });
 
     if (envGrepFiles) {
       // eslint-disable-next-line no-console
       console.log(
-        `\nCommand "--grep-files=${envGrepFiles}" matched ${tests.length} file(s):\n${tests
+        `\nCommand "--grep-files=${filePattern}" matched ${tests.length} file(s):\n${tests
           .map((name) => ` - ${name}`)
           .join('\n')}\n`
       );
     }
 
-    tests.forEach((test) => {
-      describe(test, function () {
-        loadTestFile(require.resolve(`./${test}`));
+    tests.forEach((testName) => {
+      describe(testName, () => {
+        loadTestFile(require.resolve(`./${testName}`));
+        registry.run();
       });
     });
-
-    registry.run();
   });
 }

x-pack/test/apm_api_integration/tests/service_overview/__snapshots__/instances_detailed_statistics.spec.snap

@@ -1,6 +1,6 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
-exports[`APM API tests basic apm_8.0.0 Service overview instances detailed statistics when data is loaded fetching data with comparison returns the right data for current and previous periods 5`] = `
+exports[`APM API tests service_overview/instances_detailed_statistics.spec.ts basic apm_8.0.0 Service overview instances detailed statistics when data is loaded fetching data with comparison returns the right data for current and previous periods 5`] = `
 Object {
   "currentPeriod": Object {
     "31651f3c624b81c55dd4633df0b5b9f9ab06b151121b0404ae796632cd1f87ad": Object {
@@ -675,7 +675,7 @@ Object {
 }
 `;
 
-exports[`APM API tests basic apm_8.0.0 Service overview instances detailed statistics when data is loaded fetching data without comparison returns the right data 3`] = `
+exports[`APM API tests service_overview/instances_detailed_statistics.spec.ts basic apm_8.0.0 Service overview instances detailed statistics when data is loaded fetching data without comparison returns the right data 3`] = `
 Object {
   "currentPeriod": Object {
     "31651f3c624b81c55dd4633df0b5b9f9ab06b151121b0404ae796632cd1f87ad": Object {

x-pack/test/apm_api_integration/tests/traces/__snapshots__/top_traces.spec.snap

@@ -1,6 +1,6 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
-exports[`APM API tests basic apm_8.0.0 Top traces when data is loaded returns the correct buckets 1`] = `
+exports[`APM API tests traces/top_traces.spec.ts basic apm_8.0.0 Top traces when data is loaded returns the correct buckets 1`] = `
 Array [
   Object {
     "agentName": "java",

x-pack/test/apm_api_integration/tests/transactions/__snapshots__/breakdown.spec.snap

@@ -1,6 +1,6 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
-exports[`APM API tests basic apm_8.0.0 Breakdown when data is loaded returns the transaction breakdown for a service 1`] = `
+exports[`APM API tests transactions/breakdown.spec.ts basic apm_8.0.0 Breakdown when data is loaded returns the transaction breakdown for a service 1`] = `
 Object {
   "timeseries": Array [
     Object {
@@ -1019,7 +1019,7 @@ Object {
 }
 `;
 
-exports[`APM API tests basic apm_8.0.0 Breakdown when data is loaded returns the transaction breakdown for a transaction group 9`] = `
+exports[`APM API tests transactions/breakdown.spec.ts basic apm_8.0.0 Breakdown when data is loaded returns the transaction breakdown for a transaction group 9`] = `
 Array [
   Object {
     "x": 1627973400000,