[Synthetics] Migrate service list query out of APM (#132548)

* Migrate service list query out of APM.

* Delete obsolete e2e test.

* Rename non-snakecase files.

* Add tests for new filter generator.

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

x-pack/plugins/apm/server/routes/rum_client/__snapshots__/queries.test.ts.snap

@@ -679,50 +679,3 @@ Object {
   },
 }
 `;
-
-exports[`rum client dashboard queries fetches rum services 1`] = `
-Object {
-  "apm": Object {
-    "events": Array [
-      "transaction",
-    ],
-  },
-  "body": Object {
-    "aggs": Object {
-      "services": Object {
-        "terms": Object {
-          "field": "service.name",
-          "size": 1000,
-        },
-      },
-    },
-    "query": Object {
-      "bool": Object {
-        "filter": Array [
-          Object {
-            "range": Object {
-              "@timestamp": Object {
-                "format": "epoch_millis",
-                "gte": 0,
-                "lte": 50000,
-              },
-            },
-          },
-          Object {
-            "term": Object {
-              "transaction.type": "page-load",
-            },
-          },
-          Object {
-            "exists": Object {
-              "field": "transaction.marks.navigationTiming.fetchStart",
-            },
-          },
-        ],
-        "must_not": Array [],
-      },
-    },
-    "size": 0,
-  },
-}
-`;

x-pack/plugins/apm/server/routes/rum_client/queries.test.ts

@@ -12,7 +12,6 @@ import {
 import { getClientMetrics } from './get_client_metrics';
 import { getPageViewTrends } from './get_page_view_trends';
 import { getPageLoadDistribution } from './get_page_load_distribution';
-import { getRumServices } from './get_rum_services';
 import { getLongTaskMetrics } from './get_long_task_metrics';
 import { getWebCoreVitals } from './get_web_core_vitals';
 import { getJSErrors } from './get_js_errors';
@@ -68,17 +67,6 @@ describe('rum client dashboard queries', () => {
     expect(mock.params).toMatchSnapshot();
   });
 
-  it('fetches rum services', async () => {
-    mock = await inspectSearchParams((setup) =>
-      getRumServices({
-        setup,
-        start: 0,
-        end: 50000,
-      })
-    );
-    expect(mock.params).toMatchSnapshot();
-  });
-
   it('fetches rum core vitals', async () => {
     mock = await inspectSearchParams(
       (setup) =>

x-pack/plugins/apm/server/routes/rum_client/route.ts

@@ -14,7 +14,6 @@ import { getLongTaskMetrics } from './get_long_task_metrics';
 import { getPageLoadDistribution } from './get_page_load_distribution';
 import { getPageViewTrends } from './get_page_view_trends';
 import { getPageLoadDistBreakdown } from './get_pl_dist_breakdown';
-import { getRumServices } from './get_rum_services';
 import { getUrlSearch } from './get_url_search';
 import { getVisitorBreakdown } from './get_visitor_breakdown';
 import { getWebCoreVitals } from './get_web_core_vitals';
@@ -190,22 +189,6 @@ const rumPageViewsTrendRoute = createApmServerRoute({
   },
 });
 
-const rumServicesRoute = createApmServerRoute({
-  endpoint: 'GET /internal/apm/ux/services',
-  params: t.type({
-    query: t.intersection([uiFiltersRt, rangeRt]),
-  }),
-  options: { tags: ['access:apm'] },
-  handler: async (resources): Promise<{ rumServices: string[] }> => {
-    const setup = await setupUXRequest(resources);
-    const {
-      query: { start, end },
-    } = resources.params;
-    const rumServices = await getRumServices({ setup, start, end });
-    return { rumServices };
-  },
-});
-
 const rumVisitorsBreakdownRoute = createApmServerRoute({
   endpoint: 'GET /internal/apm/ux/visitor-breakdown',
   params: t.type({
@@ -426,7 +409,6 @@ export const rumRouteRepository = {
   ...rumPageLoadDistributionRoute,
   ...rumPageLoadDistBreakdownRoute,
   ...rumPageViewsTrendRoute,
-  ...rumServicesRoute,
   ...rumVisitorsBreakdownRoute,
   ...rumWebCoreVitals,
   ...rumLongTaskMetrics,

x-pack/plugins/ux/common/processor_event.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export enum ProcessorEvent {
+  transaction = 'transaction',
+  error = 'error',
+  metric = 'metric',
+  span = 'span',
+  profile = 'profile',
+}

x-pack/plugins/ux/common/utils/merge_projection.ts

@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { DeepPartial } from 'utility-types';
+import { cloneDeep, isPlainObject, mergeWith } from 'lodash';
+
+type PlainObject = Record<string | number | symbol, any>;
+
+type SourceProjection = DeepPartial<any>;
+
+type DeepMerge<T, U> = U extends PlainObject
+  ? T extends PlainObject
+    ? Omit<T, keyof U> & {
+        [key in keyof U]: T extends { [k in key]: any }
+          ? DeepMerge<T[key], U[key]>
+          : U[key];
+      }
+    : U
+  : U;
+
+export function mergeProjection<T extends any, U extends SourceProjection>(
+  target: T,
+  source: U
+): DeepMerge<T, U> {
+  return mergeWith({}, cloneDeep(target), source, (a, b) => {
+    if (isPlainObject(a) && isPlainObject(b)) {
+      return undefined;
+    }
+    return b;
+  }) as DeepMerge<T, U>;
+}

x-pack/plugins/ux/public/components/app/rum_dashboard/panels/web_application_select.tsx

@@ -6,42 +6,42 @@
  */
 
 import React from 'react';
+import datemath from '@kbn/datemath';
+import { useEsSearch } from '@kbn/observability-plugin/public';
+import { serviceNameQuery } from '../../../../services/data/service_name_query';
 import { ServiceNameFilter } from '../url_filter/service_name_filter';
-import { useFetcher } from '../../../../hooks/use_fetcher';
 import { useLegacyUrlParams } from '../../../../context/url_params_context/use_url_params';
-import { RUM_AGENT_NAMES } from '../../../../../common/agent_name';
+import { useDataView } from '../local_uifilters/use_data_view';
+
+function callDateMath(value: unknown): number {
+  const DEFAULT_RETURN_VALUE = 0;
+  if (typeof value === 'string') {
+    return datemath.parse(value)?.valueOf() ?? DEFAULT_RETURN_VALUE;
+  }
+  return DEFAULT_RETURN_VALUE;
+}
 
 export function WebApplicationSelect() {
   const {
     rangeId,
     urlParams: { start, end },
   } = useLegacyUrlParams();
+  const { dataViewTitle } = useDataView();
 
-  const { data, status } = useFetcher(
-    (callApmApi) => {
-      if (start && end) {
-        return callApmApi('GET /internal/apm/ux/services', {
-          params: {
-            query: {
-              start,
-              end,
-              uiFilters: JSON.stringify({ agentName: RUM_AGENT_NAMES }),
-            },
-          },
-        });
-      }
+  const { data, loading } = useEsSearch(
+    {
+      index: dataViewTitle,
+      ...serviceNameQuery(callDateMath(start), callDateMath(end)),
     },
     // `rangeId` works as a cache buster for ranges that never change, like `Today`
-    // eslint-disable-next-line react-hooks/exhaustive-deps
-    [start, end, rangeId]
+    [start, end, rangeId, dataViewTitle],
+    { name: 'UxApplicationServices' }
   );
 
-  const rumServiceNames = data?.rumServices ?? [];
+  const rumServiceNames =
+    data?.aggregations?.services?.buckets.map(({ key }) => key as string) ?? [];
 
   return (
-    <ServiceNameFilter
-      loading={status !== 'success'}
-      serviceNames={rumServiceNames}
-    />
+    <ServiceNameFilter loading={!!loading} serviceNames={rumServiceNames} />
   );
 }

x-pack/plugins/ux/public/services/data/__snapshots__/service_name_query.test.ts.snap

@@ -0,0 +1,43 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`serviceNameQuery fetches rum services 1`] = `
+Object {
+  "body": Object {
+    "aggs": Object {
+      "services": Object {
+        "terms": Object {
+          "field": "service.name",
+          "size": 1000,
+        },
+      },
+    },
+    "query": Object {
+      "bool": Object {
+        "filter": Array [
+          Object {
+            "range": Object {
+              "@timestamp": Object {
+                "format": "epoch_millis",
+                "gte": 0,
+                "lte": 50000,
+              },
+            },
+          },
+          Object {
+            "term": Object {
+              "transaction.type": "page-load",
+            },
+          },
+          Object {
+            "exists": Object {
+              "field": "transaction.marks.navigationTiming.fetchStart",
+            },
+          },
+        ],
+        "must_not": Array [],
+      },
+    },
+    "size": 0,
+  },
+}
+`;

x-pack/plugins/ux/public/services/data/get_es_filter.test.ts

@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { getEsFilter } from './get_es_filter';
+
+describe('getEsFilters', function () {
+  it('should return environment in include filters', function () {
+    const result = getEsFilter({
+      browser: ['Chrome'],
+      environment: 'production',
+    });
+
+    expect(result).toEqual([
+      { terms: { 'user_agent.name': ['Chrome'] } },
+      { term: { 'service.environment': 'production' } },
+    ]);
+  });
+
+  it('should not return environment in exclude filters', function () {
+    const result = getEsFilter(
+      { browserExcluded: ['Chrome'], environment: 'production' },
+      true
+    );
+
+    expect(result).toEqual([{ terms: { 'user_agent.name': ['Chrome'] } }]);
+  });
+});

x-pack/plugins/ux/public/services/data/get_es_filter.ts

@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ESFilter } from '@kbn/core/types/elasticsearch';
+import { ENVIRONMENT_ALL } from '../../../common/environment_filter_values';
+import {
+  uxLocalUIFilterNames,
+  uxLocalUIFilters,
+} from '../../../common/ux_ui_filter';
+import { UxUIFilters } from '../../../typings/ui_filters';
+import { environmentQuery } from '../../components/app/rum_dashboard/local_uifilters/queries';
+
+export function getEsFilter(uiFilters: UxUIFilters, exclude?: boolean) {
+  const localFilterValues = uiFilters;
+  const mappedFilters = uxLocalUIFilterNames
+    .filter((name) => {
+      const validFilter = name in localFilterValues;
+      if (typeof name !== 'string') return false;
+      if (exclude) {
+        return name.includes('Excluded') && validFilter;
+      }
+      return !name.includes('Excluded') && validFilter;
+    })
+    .map((filterName) => {
+      const field = uxLocalUIFilters[filterName];
+      const value = localFilterValues[filterName];
+
+      return {
+        terms: {
+          [field.fieldName]: value,
+        },
+      };
+    }) as ESFilter[];
+
+  return [
+    ...mappedFilters,
+    ...(exclude
+      ? []
+      : environmentQuery(uiFilters.environment || ENVIRONMENT_ALL.value)),
+  ];
+}

x-pack/plugins/ux/public/services/data/projections.ts

@@ -0,0 +1,68 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { TRANSACTION_TYPE } from '../../../common/elasticsearch_fieldnames';
+import { ProcessorEvent } from '../../../common/processor_event';
+import { TRANSACTION_PAGE_LOAD } from '../../../common/transaction_types';
+import { SetupUX } from '../../../typings/ui_filters';
+import { getEsFilter } from './get_es_filter';
+import { rangeQuery } from './range_query';
+
+export function getRumPageLoadTransactionsProjection({
+  setup,
+  urlQuery,
+  checkFetchStartFieldExists = true,
+  start,
+  end,
+}: {
+  setup: SetupUX;
+  urlQuery?: string;
+  checkFetchStartFieldExists?: boolean;
+  start: number;
+  end: number;
+}) {
+  const { uiFilters } = setup;
+
+  const bool = {
+    filter: [
+      ...rangeQuery(start, end),
+      { term: { [TRANSACTION_TYPE]: TRANSACTION_PAGE_LOAD } },
+      ...(checkFetchStartFieldExists
+        ? [
+            {
+              // Adding this filter to cater for some inconsistent rum data
+              // not available on aggregated transactions
+              exists: {
+                field: 'transaction.marks.navigationTiming.fetchStart',
+              },
+            },
+          ]
+        : []),
+      ...(urlQuery
+        ? [
+            {
+              wildcard: {
+                'url.full': `*${urlQuery}*`,
+              },
+            },
+          ]
+        : []),
+      ...getEsFilter(uiFilters),
+    ],
+    must_not: [...getEsFilter(uiFilters, true)],
+  };
+
+  return {
+    apm: {
+      events: [ProcessorEvent.transaction],
+    },
+    body: {
+      query: {
+        bool,
+      },
+    },
+  };
+}

x-pack/plugins/ux/public/services/data/range_query.ts

@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+
+export function rangeQuery(
+  start?: number,
+  end?: number,
+  field = '@timestamp'
+): estypes.QueryDslQueryContainer[] {
+  return [
+    {
+      range: {
+        [field]: {
+          gte: start,
+          lte: end,
+          format: 'epoch_millis',
+        },
+      },
+    },
+  ];
+}

x-pack/plugins/ux/public/services/data/service_name_query.test.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { serviceNameQuery } from './service_name_query';
+
+describe('serviceNameQuery', () => {
+  it('fetches rum services', () => {
+    expect(serviceNameQuery(0, 50000, {})).toMatchSnapshot();
+  });
+});

x-pack/plugins/ux/public/services/data/service_name_query.ts

@@ -4,26 +4,23 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-import { SERVICE_NAME } from '../../../common/elasticsearch_fieldnames';
-import { SetupUX } from './route';
-import { getRumPageLoadTransactionsProjection } from '../../projections/rum_page_load_transactions';
-import { mergeProjection } from '../../projections/util/merge_projection';
 
-export async function getRumServices({
-  setup,
-  start,
-  end,
-}: {
-  setup: SetupUX;
-  start: number;
-  end: number;
-}) {
+import { SERVICE_NAME } from '../../../common/elasticsearch_fieldnames';
+import { mergeProjection } from '../../../common/utils/merge_projection';
+import { SetupUX, UxUIFilters } from '../../../typings/ui_filters';
+import { getRumPageLoadTransactionsProjection } from './projections';
+
+export function serviceNameQuery(
+  start: number,
+  end: number,
+  uiFilters?: UxUIFilters
+) {
+  const setup: SetupUX = { uiFilters: uiFilters ? uiFilters : {} };
   const projection = getRumPageLoadTransactionsProjection({
     setup,
     start,
     end,
   });
-
   const params = mergeProjection(projection, {
     body: {
       size: 0,
@@ -40,12 +37,6 @@ export async function getRumServices({
       },
     },
   });
-
-  const { apmEventClient } = setup;
-
-  const response = await apmEventClient.search('get_rum_services', params);
-
-  const result = response.aggregations?.services.buckets ?? [];
-
-  return result.map(({ key }) => key as string);
+  const { apm: _apm, ...rest } = params;
+  return rest;
 }

x-pack/plugins/ux/typings/ui_filters.ts

@@ -13,6 +13,10 @@ export type UxUIFilters = {
   [key in UxLocalUIFilterName]?: string[];
 };
 
+export interface SetupUX {
+  uiFilters: UxUIFilters;
+}
+
 export interface BreakdownItem {
   name: string;
   type: string;

x-pack/test/apm_api_integration/tests/csm/csm_services.spec.ts

@@ -1,45 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import expect from '@kbn/expect';
-import { FtrProviderContext } from '../../common/ftr_provider_context';
-
-export default function rumServicesApiTests({ getService }: FtrProviderContext) {
-  const registry = getService('registry');
-  const supertest = getService('legacySupertestAsApmReadUser');
-
-  registry.when('CSM Services without data', { config: 'trial', archives: [] }, () => {
-    it('returns empty list', async () => {
-      const response = await supertest.get('/internal/apm/ux/services').query({
-        start: '2020-06-28T10:24:46.055Z',
-        end: '2020-07-29T10:24:46.055Z',
-        uiFilters: '{"agentName":["js-base","rum-js"]}',
-      });
-
-      expect(response.status).to.be(200);
-      expect(response.body.rumServices).to.eql([]);
-    });
-  });
-
-  registry.when(
-    'CSM services with data',
-    { config: 'trial', archives: ['8.0.0', 'rum_8.0.0'] },
-    () => {
-      it('returns rum services list', async () => {
-        const response = await supertest.get('/internal/apm/ux/services').query({
-          start: '2020-06-28T10:24:46.055Z',
-          end: '2020-07-29T10:24:46.055Z',
-          uiFilters: '{"agentName":["js-base","rum-js"]}',
-        });
-
-        expect(response.status).to.be(200);
-
-        expectSnapshot(response.body.rumServices).toMatchInline(`Array []`);
-      });
-    }
-  );
-}