Add service groups telemetry (#132789)

* [APM] Add telemetry to service groups quries

* Add service groups in telemetry schema

* Add an internal route to test apm telemetry

* Update endpoint to run telemetry jobs and display data

* Move service_groups task work to another PR

* Clean up

* Add telemetry for service groups kueries

* Flatten map

* Update schema

* Update snapshots

x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap

@@ -1014,6 +1014,13 @@ exports[`APM telemetry helpers getApmTelemetry generates a JSON object with the
                         }
                       }
                     },
+                    "service_groups": {
+                      "properties": {
+                        "kuery_fields": {
+                          "type": "keyword"
+                        }
+                      }
+                    },
                     "tasks": {
                       "properties": {
                         "aggregated_transactions": {
@@ -1158,6 +1165,17 @@ exports[`APM telemetry helpers getApmTelemetry generates a JSON object with the
                               }
                             }
                           }
+                        },
+                        "service_groups": {
+                          "properties": {
+                            "took": {
+                              "properties": {
+                                "ms": {
+                                  "type": "long"
+                                }
+                              }
+                            }
+                          }
                         }
                       }
                     }

x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts

@@ -4,12 +4,18 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
+import { fromKueryExpression } from '@kbn/es-query';
 import { flatten, merge, sortBy, sum, pickBy } from 'lodash';
 import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { asMutableArray } from '../../../../common/utils/as_mutable_array';
 import { ProcessorEvent } from '../../../../common/processor_event';
 import { TelemetryTask } from '.';
 import { AGENT_NAMES, RUM_AGENT_NAMES } from '../../../../common/agent_name';
+import {
+  SavedServiceGroup,
+  APM_SERVICE_GROUP_SAVED_OBJECT_TYPE,
+} from '../../../../common/service_groups';
+import { getKueryFields } from '../../helpers/get_kuery_fields';
 import {
   AGENT_NAME,
   AGENT_VERSION,
@@ -1120,4 +1126,28 @@ export const tasks: TelemetryTask[] = [
       };
     },
   },
+  {
+    name: 'service_groups',
+    executor: async ({ savedObjectsClient }) => {
+      const response = await savedObjectsClient.find<SavedServiceGroup>({
+        type: APM_SERVICE_GROUP_SAVED_OBJECT_TYPE,
+        page: 1,
+        perPage: 50,
+        sortField: 'updated_at',
+        sortOrder: 'desc',
+      });
+
+      const kueryNodes = response.saved_objects.map(
+        ({ attributes: { kuery } }) => fromKueryExpression(kuery)
+      );
+
+      const kueryFields = getKueryFields(kueryNodes);
+
+      return {
+        service_groups: {
+          kuery_fields: kueryFields,
+        },
+      };
+    },
+  },
 ];

x-pack/plugins/apm/server/lib/apm_telemetry/schema.ts

@@ -187,6 +187,9 @@ export const apmSchema: MakeSchemaFrom<APMUsage> = {
       },
     },
   },
+  service_groups: {
+    kuery_fields: { type: 'array', items: { type: 'keyword' } },
+  },
   tasks: {
     aggregated_transactions: { took: { ms: long } },
     cloud: { took: { ms: long } },
@@ -201,5 +204,6 @@ export const apmSchema: MakeSchemaFrom<APMUsage> = {
     indices_stats: { took: { ms: long } },
     cardinality: { took: { ms: long } },
     environments: { took: { ms: long } },
+    service_groups: { took: { ms: long } },
   },
 };

x-pack/plugins/apm/server/lib/apm_telemetry/types.ts

@@ -130,6 +130,9 @@ export interface APMUsage {
       };
     };
   };
+  service_groups: {
+    kuery_fields: string[];
+  };
   tasks: Record<
     | 'aggregated_transactions'
     | 'cloud'
@@ -143,7 +146,8 @@ export interface APMUsage {
     | 'agents'
     | 'indices_stats'
     | 'cardinality'
-    | 'environments',
+    | 'environments'
+    | 'service_groups',
     { took: { ms: number } }
   >;
 }

x-pack/plugins/apm/server/lib/helpers/get_kuery_fields.test.ts

@@ -0,0 +1,89 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { getKueryFields } from './get_kuery_fields';
+import { fromKueryExpression } from '@kbn/es-query';
+
+describe('get kuery fields', () => {
+  it('returns single kuery field', () => {
+    const kuery = 'service.name: my-service';
+    const kueryNode = fromKueryExpression(kuery);
+    expect(getKueryFields([kueryNode])).toEqual(['service.name']);
+  });
+
+  it('returns kuery fields with wildcard', () => {
+    const kuery = 'service.name: *';
+    const kueryNode = fromKueryExpression(kuery);
+    expect(getKueryFields([kueryNode])).toEqual(['service.name']);
+  });
+
+  it('returns multiple fields used AND operator', () => {
+    const kuery =
+      'service.name: my-service AND service.environment: production';
+    const kueryNode = fromKueryExpression(kuery);
+    expect(getKueryFields([kueryNode])).toEqual([
+      'service.name',
+      'service.environment',
+    ]);
+  });
+
+  it('returns multiple kuery fields with OR operator', () => {
+    const kuery = 'network.carrier.mcc: test or child.id: 33';
+    const kueryNode = fromKueryExpression(kuery);
+    expect(getKueryFields([kueryNode])).toEqual([
+      'network.carrier.mcc',
+      'child.id',
+    ]);
+  });
+
+  it('returns multiple kuery fields with wildcard', () => {
+    const kuery = 'network.carrier.mcc:* or child.id: *';
+    const kueryNode = fromKueryExpression(kuery);
+    expect(getKueryFields([kueryNode])).toEqual([
+      'network.carrier.mcc',
+      'child.id',
+    ]);
+  });
+
+  it('returns single kuery fields with gt operator', () => {
+    const kuery = 'transaction.duration.aggregate > 10';
+    const kueryNode = fromKueryExpression(kuery);
+    expect(getKueryFields([kueryNode])).toEqual([
+      'transaction.duration.aggregate',
+    ]);
+  });
+
+  it('returns dublicate fields', () => {
+    const kueries = [
+      'service.name: my-service',
+      'service.name: my-service and trace.id: trace',
+    ];
+
+    const kueryNodes = kueries.map((kuery) => fromKueryExpression(kuery));
+    expect(getKueryFields(kueryNodes)).toEqual([
+      'service.name',
+      'service.name',
+      'trace.id',
+    ]);
+  });
+
+  it('returns multiple fields with multiple logical operators', () => {
+    const kuery =
+      '(service.name:opbeans-* OR service.name:kibana) and (service.environment:production)';
+    const kueryNode = fromKueryExpression(kuery);
+    expect(getKueryFields([kueryNode])).toEqual([
+      'service.name',
+      'service.name',
+      'service.environment',
+    ]);
+  });
+
+  it('do not return if kuery field is null', () => {
+    const kuery = 'opbean';
+    const kueryNode = fromKueryExpression(kuery);
+    expect(getKueryFields([kueryNode])).toEqual([]);
+  });
+});

x-pack/plugins/apm/server/lib/helpers/get_kuery_fields.ts

@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { KueryNode } from '@kbn/es-query';
+import { compact } from 'lodash';
+
+export function getKueryFields(nodes: KueryNode[]): string[] {
+  const allFields = nodes
+    .map((node) => {
+      const {
+        arguments: [fieldNameArg],
+      } = node;
+
+      if (fieldNameArg.type === 'function') {
+        return getKueryFields(node.arguments);
+      }
+
+      return fieldNameArg.value;
+    })
+    .flat();
+
+  return compact(allFields);
+}

x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json

@@ -3918,6 +3918,16 @@
             }
           }
         },
+        "service_groups": {
+          "properties": {
+            "kuery_fields": {
+              "type": "array",
+              "items": {
+                "type": "keyword"
+              }
+            }
+          }
+        },
         "tasks": {
           "properties": {
             "aggregated_transactions": {
@@ -4062,6 +4072,17 @@
                   }
                 }
               }
+            },
+            "service_groups": {
+              "properties": {
+                "took": {
+                  "properties": {
+                    "ms": {
+                      "type": "long"
+                    }
+                  }
+                }
+              }
             }
           }
         }