[APM] Add apm_core plugin and query errors, spans, transactions from separate indices (#21022)

* [APM] Change index pattern

* WIP: change how indicies are queried in APM

* Create apm_core plugin

* Update config keys from `xpack.apm` to `apm_core`

* Revert tutorial formatting changes

* Rename to apm_oss

src/core_plugins/apm_oss/index.js

@@ -0,0 +1,40 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export default function apmOss(kibana) {
+  return new kibana.Plugin({
+    id: 'apm_oss',
+
+    config(Joi) {
+      return Joi.object({
+        // enable plugin
+        enabled: Joi.boolean().default(true),
+
+        // Kibana Index pattern
+        indexPattern: Joi.string().default('apm-*'),
+
+        // ES Indices
+        errorIndices: Joi.string().default('apm-*-error-*'),
+        onboardingIndices: Joi.string().default('apm-*-onboarding-*'),
+        spanIndices: Joi.string().default('apm-*-span-*'),
+        transactionIndices: Joi.string().default('apm-*-transaction-*'),
+      }).default();
+    },
+  });
+}

src/core_plugins/apm_oss/package.json

@@ -0,0 +1,4 @@
+{
+  "name": "apm_oss",
+  "version": "kibana"
+}

src/core_plugins/kibana/server/tutorials/apm/index.js

@@ -24,8 +24,8 @@ import { getSavedObjects } from './saved_objects/get_saved_objects';
 
 const apmIntro = 'Collect in-depth performance metrics and errors from inside your applications.';
 
-const ENABLED_KEY = 'xpack.apm.ui.enabled';
 function isEnabled(config) {
+  const ENABLED_KEY = 'xpack.apm.ui.enabled';
   if (config.has(ENABLED_KEY)) {
     return config.get(ENABLED_KEY);
   }
@@ -33,19 +33,9 @@ function isEnabled(config) {
   return false;
 }
 
-const TITLE_KEY = 'xpack.apm.indexPattern';
-const DEFAULT_TITLE = 'apm*';
-function getIndexPatternTitle(config) {
-  if (config.has(TITLE_KEY)) {
-    return config.get(TITLE_KEY);
-  }
-
-  return DEFAULT_TITLE;
-}
-
 export function apmSpecProvider(server) {
   const config = server.config();
-  const apmIndexPattern = getIndexPatternTitle(config);
+  const apmIndexPattern = config.get('apm_oss.indexPattern');
 
   const artifacts = {
     dashboards: [

src/core_plugins/kibana/server/tutorials/apm/saved_objects/get_saved_objects.test.js

@@ -22,7 +22,7 @@ import { getSavedObjects } from './get_saved_objects';
 
 const indexPatternTitle = 'dynamic index pattern title';
 
-test('should dynamically set index title to "xpack.apm.indexPattern" yaml config value', () => {
+test('should dynamically set index title to "apm_oss.indexPattern" yaml config value', () => {
   const savedObjects = getSavedObjects(indexPatternTitle);
   const indexPattern = savedObjects[0];
   expect(indexPattern.type).to.be('index-pattern');

x-pack/plugins/apm/index.js

@@ -12,7 +12,7 @@ import { initStatusApi } from './server/routes/status_check';
 
 export function apm(kibana) {
   return new kibana.Plugin({
-    require: ['kibana', 'elasticsearch', 'xpack_main'],
+    require: ['kibana', 'elasticsearch', 'xpack_main', 'apm_oss'],
     id: 'apm',
     configPrefix: 'xpack.apm',
     publicDir: resolve(__dirname, 'public'),
@@ -30,7 +30,7 @@ export function apm(kibana) {
         return {
           mlEnabled: config.get('xpack.ml.enabled'),
           apmUiEnabled: config.get('xpack.apm.ui.enabled'),
-          apmIndexPattern: config.get('xpack.apm.indexPattern')
+          apmIndexPattern: config.get('apm_oss.indexPattern')
         };
       },
       hacks: ['plugins/apm/hacks/toggle_app_link_in_nav']
@@ -38,11 +38,15 @@ export function apm(kibana) {
 
     config(Joi) {
       return Joi.object({
+        // display menu item
         ui: Joi.object({
           enabled: Joi.boolean().default(true)
         }).default(),
+
+        // enable plugin
         enabled: Joi.boolean().default(true),
-        indexPattern: Joi.string().default('apm*'),
+
+        // buckets
         minimumBucketSize: Joi.number().default(15),
         bucketTargetCount: Joi.number().default(27)
       }).default();

x-pack/plugins/apm/server/lib/errors/distribution/get_buckets.js

@@ -10,7 +10,7 @@ export async function getBuckets({ serviceName, groupId, bucketSize, setup }) {
   const { start, end, esFilterQuery, client, config } = setup;
 
   const params = {
-    index: config.get('xpack.apm.indexPattern'),
+    index: config.get('apm_oss.errorIndices'),
     body: {
       size: 0,
       query: {

x-pack/plugins/apm/server/lib/errors/get_error_group.js

@@ -11,7 +11,7 @@ export async function getErrorGroup({ serviceName, groupId, setup }) {
   const { start, end, esFilterQuery, client, config } = setup;
 
   const params = {
-    index: config.get('xpack.apm.indexPattern'),
+    index: config.get('apm_oss.errorIndices'),
     body: {
       size: 1,
       query: {

x-pack/plugins/apm/server/lib/errors/get_error_groups.js

@@ -24,7 +24,7 @@ export async function getErrorGroups({
   const { start, end, esFilterQuery, client, config } = setup;
 
   const params = {
-    index: config.get('xpack.apm.indexPattern'),
+    index: config.get('apm_oss.errorIndices'),
     body: {
       size: 0,
       query: {

x-pack/plugins/apm/server/lib/services/get_service.js

@@ -15,7 +15,10 @@ export async function getService({ serviceName, setup }) {
   const { start, end, esFilterQuery, client, config } = setup;
 
   const params = {
-    index: config.get('xpack.apm.indexPattern'),
+    index: [
+      config.get('apm_oss.errorIndices'),
+      config.get('apm_oss.transactionIndices')
+    ],
     body: {
       size: 0,
       query: {

x-pack/plugins/apm/server/lib/services/get_services.js

@@ -16,7 +16,10 @@ export async function getServices({ setup }) {
   const { start, end, esFilterQuery, client, config } = setup;
 
   const params = {
-    index: config.get('xpack.apm.indexPattern'),
+    index: [
+      config.get('apm_oss.errorIndices'),
+      config.get('apm_oss.transactionIndices')
+    ],
     body: {
       size: 0,
       query: {

x-pack/plugins/apm/server/lib/status_check/agent_check.js

@@ -10,7 +10,10 @@ export async function getAgentStatus({ setup }) {
   const { client, config } = setup;
 
   const params = {
-    index: config.get('xpack.apm.indexPattern'),
+    index: [
+      config.get('apm_oss.errorIndices'),
+      config.get('apm_oss.transactionIndices')
+    ],
     body: {
       size: 0,
       query: {

x-pack/plugins/apm/server/lib/status_check/server_check.js

@@ -8,7 +8,7 @@ export async function getServerStatus({ setup }) {
   const { client, config } = setup;
 
   const params = {
-    index: config.get('xpack.apm.indexPattern'),
+    index: config.get('apm_oss.onboardingIndices'),
     body: {
       size: 0,
       query: {

x-pack/plugins/apm/server/lib/transactions/charts/get_timeseries_data.js

@@ -26,7 +26,7 @@ export async function getTimeseriesData({
   const { intervalString, bucketSize } = getBucketSize(start, end, 'auto');
 
   const params = {
-    index: config.get('xpack.apm.indexPattern'),
+    index: config.get('apm_oss.transactionIndices'),
     body: {
       size: 0,
       query: {

x-pack/plugins/apm/server/lib/transactions/distribution/calculate_bucket_size.js

@@ -18,7 +18,7 @@ export async function calculateBucketSize({
   const { start, end, esFilterQuery, client, config } = setup;
 
   const params = {
-    index: config.get('xpack.apm.indexPattern'),
+    index: config.get('apm_oss.transactionIndices'),
     body: {
       size: 0,
       query: {

x-pack/plugins/apm/server/lib/transactions/distribution/get_buckets.js

@@ -24,7 +24,7 @@ export async function getBuckets({
   const bucketTargetCount = config.get('xpack.apm.bucketTargetCount');
 
   const params = {
-    index: config.get('xpack.apm.indexPattern'),
+    index: config.get('apm_oss.transactionIndices'),
     body: {
       size: 0,
       query: {

x-pack/plugins/apm/server/lib/transactions/get_top_transactions.js

@@ -25,7 +25,7 @@ export async function getTopTransactions({
   const minutes = duration.asMinutes();
 
   const params = {
-    index: config.get('xpack.apm.indexPattern'),
+    index: config.get('apm_oss.transactionIndices'),
     body: {
       size: 0,
       query: {

x-pack/plugins/apm/server/lib/transactions/get_transaction.js

@@ -11,7 +11,7 @@ async function getTransaction({ transactionId, setup }) {
   const { start, end, esFilterQuery, client, config } = setup;
 
   const params = {
-    index: config.get('xpack.apm.indexPattern'),
+    index: config.get('apm_oss.transactionIndices'),
     body: {
       size: 1,
       query: {

x-pack/plugins/apm/server/lib/transactions/get_transaction_duration.js

@@ -15,7 +15,7 @@ export async function getTransactionDuration({ transactionId, setup }) {
   const { start, end, esFilterQuery, client, config } = setup;
 
   const params = {
-    index: config.get('xpack.apm.indexPattern'),
+    index: config.get('apm_oss.transactionIndices'),
     body: {
       size: 1,
       _source: TRANSACTION_DURATION,

x-pack/plugins/apm/server/lib/transactions/spans/get_spans.js

@@ -15,7 +15,7 @@ async function getSpans({ transactionId, setup }) {
   const { start, end, client, config } = setup;
 
   const params = {
-    index: config.get('xpack.apm.indexPattern'),
+    index: config.get('apm_oss.spanIndices'),
     body: {
       size: 500,
       query: {