[APM] Remove index_pattern.json and add custom field formatters (#119915) (#120012)

* [APM] Remove index_pattern.json and add custom field formatters

* Fix tests

* Fix tests

* Fix tutorial

Co-authored-by: Søren Louv-Jansen <soren.louv@elastic.co>

x-pack/plugins/apm/ftr_e2e/cypress/integration/read_only_user/errors/error_details.spec.ts

@@ -40,7 +40,7 @@ describe('Error details', () => {
     });
 
     describe('when error has no occurrences', () => {
-      it('shows empty an message', () => {
+      it('shows an empty message', () => {
         cy.visit(
           url.format({
             pathname:

x-pack/plugins/apm/ftr_e2e/cypress_start.ts

@@ -20,8 +20,6 @@ export async function cypressStart(
 ) {
   const config = getService('config');
 
-  const archiveName = 'apm_mappings_only_8.0.0';
-
   const kibanaUrl = Url.format({
     protocol: config.get('servers.kibana.protocol'),
     hostname: config.get('servers.kibana.hostname'),
@@ -50,8 +48,9 @@ export async function cypressStart(
   });
 
   const esRequestTimeout = config.get('timeouts.esRequestTimeout');
+  const archiveName = 'apm_mappings_only_8.0.0';
 
-  console.log(`Loading ES archive "${archiveName}"`);
+  console.log(`Creating APM mappings`);
   await esArchiverLoad(archiveName);
 
   const spec = argv.grep as string | undefined;
@@ -66,7 +65,7 @@ export async function cypressStart(
     },
   });
 
-  console.log('Unloading ES archives...');
+  console.log('Removing APM mappings');
   await esArchiverUnload(archiveName);
 
   return res;

x-pack/plugins/apm/scripts/test/e2e.js

@@ -7,6 +7,7 @@
 
 /* eslint-disable no-console */
 
+const { times } = require('lodash');
 const path = require('path');
 const yargs = require('yargs');
 const childProcess = require('child_process');
@@ -45,6 +46,11 @@ const { argv } = yargs(process.argv.slice(2))
     type: 'boolean',
     description: 'stop tests after the first failure',
   })
+  .option('times', {
+    default: 1,
+    type: 'number',
+    description: 'Repeat the test n number of times',
+  })
   .help();
 
 const { server, runner, open, grep, bail, kibanaInstallDir } = argv;
@@ -63,5 +69,23 @@ const grepArg = grep ? `--grep "${grep}"` : '';
 const bailArg = bail ? `--bail` : '';
 const cmd = `node ../../../../scripts/${ftrScript} --config ${config} ${grepArg} ${bailArg} --kibana-install-dir '${kibanaInstallDir}'`;
 
-console.log(`Running ${cmd}`);
-childProcess.execSync(cmd, { cwd: e2eDir, stdio: 'inherit' });
+console.log(`Running "${cmd}"`);
+
+if (argv.times > 1) {
+  console.log(`The command will be executed ${argv.times} times`);
+}
+
+const runCounter = { succeeded: 0, failed: 0, remaining: argv.times };
+times(argv.times, () => {
+  try {
+    childProcess.execSync(cmd, { cwd: e2eDir, stdio: 'inherit' });
+    runCounter.succeeded++;
+  } catch (e) {
+    runCounter.failed++;
+  }
+  runCounter.remaining--;
+
+  if (argv.times > 1) {
+    console.log(runCounter);
+  }
+});

x-pack/plugins/apm/server/routes/data_view/create_static_data_view.ts

@@ -7,30 +7,28 @@
 
 import { SavedObjectsErrorHelpers } from '../../../../../../src/core/server';
 import { APM_STATIC_INDEX_PATTERN_ID } from '../../../common/index_pattern_constants';
-import apmDataView from '../../tutorial/index_pattern.json';
 import { hasHistoricalAgentData } from '../../routes/historical_data/has_historical_agent_data';
 import { Setup } from '../../lib/helpers/setup_request';
 import { APMRouteHandlerResources } from '../../routes/typings';
 import { InternalSavedObjectsClient } from '../../lib/helpers/get_internal_saved_objects_client.js';
 import { withApmSpan } from '../../utils/with_apm_span';
 import { getApmDataViewTitle } from './get_apm_data_view_title';
+import { getApmDataViewAttributes } from './get_apm_data_view_attributes';
 
-type ApmDataViewAttributes = typeof apmDataView.attributes & {
+interface ApmDataViewAttributes {
   title: string;
-};
+}
 
 export async function createStaticDataView({
   setup,
   config,
   savedObjectsClient,
   spaceId,
-  overwrite = false,
 }: {
   setup: Setup;
   config: APMRouteHandlerResources['config'];
   savedObjectsClient: InternalSavedObjectsClient;
   spaceId?: string;
-  overwrite?: boolean;
 }): Promise<boolean> {
   return withApmSpan('create_static_data_view', async () => {
     // don't autocreate APM data view if it's been disabled via the config
@@ -48,7 +46,6 @@ export async function createStaticDataView({
     const apmDataViewTitle = getApmDataViewTitle(setup.indices);
     const forceOverwrite = await getForceOverwrite({
       apmDataViewTitle,
-      overwrite,
       savedObjectsClient,
     });
 
@@ -56,17 +53,15 @@ export async function createStaticDataView({
       await withApmSpan('create_index_pattern_saved_object', () =>
         savedObjectsClient.create(
           'index-pattern',
-          {
-            ...apmDataView.attributes,
-            title: apmDataViewTitle,
-          },
+          getApmDataViewAttributes(apmDataViewTitle),
           {
             id: APM_STATIC_INDEX_PATTERN_ID,
-            overwrite: forceOverwrite ? true : overwrite,
+            overwrite: forceOverwrite,
             namespace: spaceId,
           }
         )
       );
+
       return true;
     } catch (e) {
       // if the data view (saved object) already exists a conflict error (code: 409) will be thrown
@@ -82,30 +77,26 @@ export async function createStaticDataView({
 // force an overwrite of the data view if the data view has been changed
 async function getForceOverwrite({
   savedObjectsClient,
-  overwrite,
   apmDataViewTitle,
 }: {
   savedObjectsClient: InternalSavedObjectsClient;
-  overwrite: boolean;
   apmDataViewTitle: string;
 }) {
-  if (!overwrite) {
-    try {
-      const existingDataView =
-        await savedObjectsClient.get<ApmDataViewAttributes>(
-          'index-pattern',
-          APM_STATIC_INDEX_PATTERN_ID
-        );
+  try {
+    const existingDataView =
+      await savedObjectsClient.get<ApmDataViewAttributes>(
+        'index-pattern',
+        APM_STATIC_INDEX_PATTERN_ID
+      );
 
-      // if the existing data view does not matches the new one, force an update
-      return existingDataView.attributes.title !== apmDataViewTitle;
-    } catch (e) {
-      // ignore exception if the data view (saved object) is not found
-      if (SavedObjectsErrorHelpers.isNotFoundError(e)) {
-        return false;
-      }
-
-      throw e;
+    // if the existing data view does not matches the new one, force an update
+    return existingDataView.attributes.title !== apmDataViewTitle;
+  } catch (e) {
+    // ignore exception if the data view (saved object) is not found
+    if (SavedObjectsErrorHelpers.isNotFoundError(e)) {
+      return false;
     }
+
+    throw e;
   }
 }

x-pack/plugins/apm/server/routes/data_view/get_apm_data_view_attributes.ts

@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+  TRACE_ID,
+  TRANSACTION_ID,
+} from '../../../common/elasticsearch_fieldnames';
+
+export function getApmDataViewAttributes(title: string) {
+  return {
+    // required fields (even if empty)
+    title,
+    fieldAttrs: '{}',
+    fields: '[]',
+    runtimeFieldMap: '{}',
+    timeFieldName: '@timestamp',
+    typeMeta: '{}',
+
+    // link to APM from Discover
+    fieldFormatMap: JSON.stringify({
+      [TRACE_ID]: {
+        id: 'url',
+        params: {
+          urlTemplate: 'apm/link-to/trace/{{value}}',
+          labelTemplate: '{{value}}',
+        },
+      },
+      [TRANSACTION_ID]: {
+        id: 'url',
+        params: {
+          urlTemplate: 'apm/link-to/transaction/{{value}}',
+          labelTemplate: '{{value}}',
+        },
+      },
+    }),
+  };
+}

x-pack/plugins/apm/server/tutorial/index.ts

@@ -14,11 +14,11 @@ import {
 } from '../../../../../src/plugins/home/server';
 import { CloudSetup } from '../../../cloud/server';
 import { APM_STATIC_INDEX_PATTERN_ID } from '../../common/index_pattern_constants';
+import { getApmDataViewAttributes } from '../routes/data_view/get_apm_data_view_attributes';
 import { getApmDataViewTitle } from '../routes/data_view/get_apm_data_view_title';
 import { ApmIndicesConfig } from '../routes/settings/apm_indices/get_apm_indices';
 import { createElasticCloudInstructions } from './envs/elastic_cloud';
 import { onPremInstructions } from './envs/on_prem';
-import apmDataView from './index_pattern.json';
 
 const apmIntro = i18n.translate('xpack.apm.tutorial.introduction', {
   defaultMessage:
@@ -39,16 +39,12 @@ export const tutorialProvider =
     isFleetPluginEnabled: boolean;
   }) =>
   () => {
-    const indexPatternTitle = getApmDataViewTitle(apmIndices);
-
+    const dataViewTitle = getApmDataViewTitle(apmIndices);
     const savedObjects = [
       {
-        ...apmDataView,
         id: APM_STATIC_INDEX_PATTERN_ID,
-        attributes: {
-          ...apmDataView.attributes,
-          title: indexPatternTitle,
-        },
+        attributes: getApmDataViewAttributes(dataViewTitle),
+        type: 'index-pattern',
       },
     ];
 

x-pack/test/apm_api_integration/tests/data_view/static.spec.ts

@@ -0,0 +1,149 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { apm, ApmSynthtraceEsClient, timerange } from '@elastic/apm-synthtrace';
+import expect from '@kbn/expect';
+import { APM_STATIC_INDEX_PATTERN_ID } from '../../../../plugins/apm/common/index_pattern_constants';
+import { FtrProviderContext } from '../../common/ftr_provider_context';
+import { SupertestReturnType } from '../../common/apm_api_supertest';
+
+export default function ApiTest({ getService }: FtrProviderContext) {
+  const registry = getService('registry');
+  const apmApiClient = getService('apmApiClient');
+  const supertest = getService('supertest');
+  const synthtrace = getService('synthtraceEsClient');
+
+  const dataViewPattern = 'traces-apm*,apm-*,logs-apm*,apm-*,metrics-apm*,apm-*';
+
+  function createDataViewViaApmApi() {
+    return apmApiClient.readUser({ endpoint: 'POST /internal/apm/data_view/static' });
+  }
+
+  function deleteDataView() {
+    // return supertest.delete('/api/saved_objects/<type>/<id>').set('kbn-xsrf', 'foo').expect(200)
+    return supertest
+      .delete(`/api/saved_objects/index-pattern/${APM_STATIC_INDEX_PATTERN_ID}`)
+      .set('kbn-xsrf', 'foo')
+      .expect(200);
+  }
+
+  function getDataView() {
+    return supertest.get(`/api/saved_objects/index-pattern/${APM_STATIC_INDEX_PATTERN_ID}`);
+  }
+
+  function getDataViewSuggestions(field: string) {
+    return supertest
+      .post(`/api/kibana/suggestions/values/${dataViewPattern}`)
+      .set('kbn-xsrf', 'foo')
+      .send({ query: '', field, method: 'terms_agg' });
+  }
+
+  registry.when('no mappings exist', { config: 'basic', archives: [] }, () => {
+    let response: SupertestReturnType<'POST /internal/apm/data_view/static'>;
+    describe('when no data is generated', () => {
+      before(async () => {
+        response = await createDataViewViaApmApi();
+      });
+
+      it('does not create data view', async () => {
+        expect(response.status).to.be(200);
+        expect(response.body.created).to.be(false);
+      });
+
+      it('cannot fetch data view', async () => {
+        await getDataView().expect(404);
+      });
+    });
+  });
+
+  registry.when(
+    'mappings exists',
+    { config: 'basic', archives: ['apm_mappings_only_8.0.0'] },
+    () => {
+      describe('when data is generated', () => {
+        let response: SupertestReturnType<'POST /internal/apm/data_view/static'>;
+
+        before(async () => {
+          await generateApmData(synthtrace);
+          response = await createDataViewViaApmApi();
+        });
+
+        after(async () => {
+          await deleteDataView();
+          await synthtrace.clean();
+        });
+
+        it('successfully creates the apm data view', async () => {
+          expect(response.status).to.be(200);
+          expect(response.body.created).to.be(true);
+        });
+
+        describe('when fetching the data view', async () => {
+          let resBody: any;
+
+          before(async () => {
+            const res = await getDataView().expect(200);
+            resBody = res.body;
+          });
+
+          it('has correct id', () => {
+            expect(resBody.id).to.be('apm_static_index_pattern_id');
+          });
+
+          it('has correct title', () => {
+            expect(resBody.attributes.title).to.be(dataViewPattern);
+          });
+
+          it('has correct attributes', () => {
+            expect(resBody.attributes.fieldFormatMap).to.be(
+              JSON.stringify({
+                'trace.id': {
+                  id: 'url',
+                  params: {
+                    urlTemplate: 'apm/link-to/trace/{{value}}',
+                    labelTemplate: '{{value}}',
+                  },
+                },
+                'transaction.id': {
+                  id: 'url',
+                  params: {
+                    urlTemplate: 'apm/link-to/transaction/{{value}}',
+                    labelTemplate: '{{value}}',
+                  },
+                },
+              })
+            );
+          });
+
+          // this test ensures that the default APM Data View doesn't interfere with suggestions returned in the kuery bar (this has been a problem in the past)
+          it('can get suggestions for `trace.id`', async () => {
+            const suggestions = await getDataViewSuggestions('trace.id');
+            expect(suggestions.body.length).to.be(10);
+          });
+        });
+      });
+    }
+  );
+}
+
+function generateApmData(synthtrace: ApmSynthtraceEsClient) {
+  const range = timerange(
+    new Date('2021-10-01T00:00:00.000Z').getTime(),
+    new Date('2021-10-01T00:01:00.000Z').getTime()
+  );
+
+  const instance = apm.service('multiple-env-service', 'production', 'go').instance('my-instance');
+
+  return synthtrace.index([
+    ...range
+      .interval('1s')
+      .rate(1)
+      .flatMap((timestamp) => [
+        ...instance.transaction('GET /api').timestamp(timestamp).duration(30).success().serialize(),
+      ]),
+  ]);
+}

x-pack/test/apm_api_integration/tests/historical_data/has_data.spec.ts

@@ -10,7 +10,7 @@ import { FtrProviderContext } from '../../common/ftr_provider_context';
 
 export default function ApiTest({ getService }: FtrProviderContext) {
   const registry = getService('registry');
-  const supertest = getService('supertest');
+  const apmApiClient = getService('apmApiClient');
   const archiveName = 'apm_8.0.0';
 
   registry.when(
@@ -18,8 +18,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
     { config: 'basic', archives: [] },
     () => {
       it('handles the empty state', async () => {
-        const response = await supertest.get(`/internal/apm/has_data`);
-
+        const response = await apmApiClient.readUser({ endpoint: `GET /internal/apm/has_data` });
         expect(response.status).to.be(200);
         expect(response.body.hasData).to.be(false);
       });
@@ -31,8 +30,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
     { config: 'basic', archives: [archiveName] },
     () => {
       it('returns hasData: true', async () => {
-        const response = await supertest.get(`/internal/apm/has_data`);
-
+        const response = await apmApiClient.readUser({ endpoint: `GET /internal/apm/has_data` });
         expect(response.status).to.be(200);
         expect(response.body.hasData).to.be(true);
       });

x-pack/test/apm_api_integration/tests/traces/trace_by_id.spec.ts

@@ -8,12 +8,11 @@
 import expect from '@kbn/expect';
 import archives_metadata from '../../common/fixtures/es_archiver/archives_metadata';
 import { FtrProviderContext } from '../../common/ftr_provider_context';
-import { createApmApiClient, SupertestReturnType } from '../../common/apm_api_supertest';
+import { SupertestReturnType } from '../../common/apm_api_supertest';
 
 export default function ApiTest({ getService }: FtrProviderContext) {
   const registry = getService('registry');
-  const supertest = getService('supertest');
-  const apmApiSupertest = createApmApiClient(supertest);
+  const apmApiClient = getService('apmApiClient');
 
   const archiveName = 'apm_8.0.0';
   const metadata = archives_metadata[archiveName];
@@ -21,7 +20,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
 
   registry.when('Trace does not exist', { config: 'basic', archives: [] }, () => {
     it('handles empty state', async () => {
-      const response = await apmApiSupertest({
+      const response = await apmApiClient.readUser({
         endpoint: `GET /internal/apm/traces/{traceId}`,
         params: {
           path: { traceId: 'foo' },
@@ -37,7 +36,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
   registry.when('Trace exists', { config: 'basic', archives: [archiveName] }, () => {
     let response: SupertestReturnType<`GET /internal/apm/traces/{traceId}`>;
     before(async () => {
-      response = await apmApiSupertest({
+      response = await apmApiClient.readUser({
         endpoint: `GET /internal/apm/traces/{traceId}`,
         params: {
           path: { traceId: '64d0014f7530df24e549dd17cc0a8895' },