Migrate client metrics from APM to UX (#133322)

* Migrate client metrics from APM to UX

* Add e2e test

x-pack/plugins/apm/server/routes/rum_client/__snapshots__/queries.test.ts.snap

@@ -1,78 +1,5 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
-exports[`rum client dashboard queries fetches client metrics 1`] = `
-Object {
-  "apm": Object {
-    "events": Array [
-      "transaction",
-    ],
-  },
-  "body": Object {
-    "aggs": Object {
-      "hasFetchStartField": Object {
-        "aggs": Object {
-          "backEnd": Object {
-            "percentiles": Object {
-              "field": "transaction.marks.agent.timeToFirstByte",
-              "hdr": Object {
-                "number_of_significant_value_digits": 3,
-              },
-              "percents": Array [
-                50,
-              ],
-            },
-          },
-          "totalPageLoadDuration": Object {
-            "percentiles": Object {
-              "field": "transaction.duration.us",
-              "hdr": Object {
-                "number_of_significant_value_digits": 3,
-              },
-              "percents": Array [
-                50,
-              ],
-            },
-          },
-        },
-        "filter": Object {
-          "exists": Object {
-            "field": "transaction.marks.navigationTiming.fetchStart",
-          },
-        },
-      },
-    },
-    "query": Object {
-      "bool": Object {
-        "filter": Array [
-          Object {
-            "range": Object {
-              "@timestamp": Object {
-                "format": "epoch_millis",
-                "gte": 0,
-                "lte": 50000,
-              },
-            },
-          },
-          Object {
-            "term": Object {
-              "transaction.type": "page-load",
-            },
-          },
-          Object {
-            "term": Object {
-              "service.environment": "staging",
-            },
-          },
-        ],
-        "must_not": Array [],
-      },
-    },
-    "size": 0,
-    "track_total_hits": true,
-  },
-}
-`;
-
 exports[`rum client dashboard queries fetches long task metrics 1`] = `
 Object {
   "apm": Object {
@@ -464,4 +391,4 @@ Object {
     "size": 0,
   },
 }
-`;
+`;

x-pack/plugins/apm/server/routes/rum_client/queries.test.ts

@@ -9,7 +9,6 @@ import {
   SearchParamsMock,
   inspectSearchParams,
 } from '../../utils/test_helpers';
-import { getClientMetrics } from './get_client_metrics';
 import { getPageViewTrends } from './get_page_view_trends';
 import { getPageLoadDistribution } from './get_page_load_distribution';
 import { getLongTaskMetrics } from './get_long_task_metrics';
@@ -21,20 +20,6 @@ describe('rum client dashboard queries', () => {
     mock.teardown();
   });
 
-  it('fetches client metrics', async () => {
-    mock = await inspectSearchParams(
-      (setup) =>
-        getClientMetrics({
-          setup,
-          start: 0,
-          end: 50000,
-        }),
-      { uiFilters: { environment: 'staging' } }
-    );
-
-    expect(mock.params).toMatchSnapshot();
-  });
-
   it('fetches page view trends', async () => {
     mock = await inspectSearchParams(
       (setup) =>

x-pack/plugins/apm/server/routes/rum_client/route.ts

@@ -7,7 +7,6 @@
 import * as t from 'io-ts';
 import { Logger } from '@kbn/core/server';
 import { setupRequest, Setup } from '../../lib/helpers/setup_request';
-import { getClientMetrics } from './get_client_metrics';
 import { getLongTaskMetrics } from './get_long_task_metrics';
 import { getPageLoadDistribution } from './get_page_load_distribution';
 import { getPageViewTrends } from './get_page_view_trends';
@@ -57,36 +56,6 @@ const uxQueryRt = t.intersection([
   t.partial({ urlQuery: t.string, percentile: t.string }),
 ]);
 
-const rumClientMetricsRoute = createApmServerRoute({
-  endpoint: 'GET /internal/apm/ux/client-metrics',
-  params: t.type({
-    query: uxQueryRt,
-  }),
-  options: { tags: ['access:apm'] },
-  handler: async (
-    resources
-  ): Promise<{
-    pageViews: { value: number };
-    totalPageLoadDuration: { value: number };
-    backEnd: { value: number };
-    frontEnd: { value: number };
-  }> => {
-    const setup = await setupUXRequest(resources);
-
-    const {
-      query: { urlQuery, percentile, start, end },
-    } = resources.params;
-
-    return getClientMetrics({
-      setup,
-      urlQuery,
-      percentile: percentile ? Number(percentile) : undefined,
-      start,
-      end,
-    });
-  },
-});
-
 const rumPageLoadDistributionRoute = createApmServerRoute({
   endpoint: 'GET /internal/apm/ux/page-load-distribution',
   params: t.type({
@@ -270,7 +239,6 @@ async function setupUXRequest<TParams extends SetupUXRequestParams>(
 }
 
 export const rumRouteRepository = {
-  ...rumClientMetricsRoute,
   ...rumPageLoadDistributionRoute,
   ...rumPageLoadDistBreakdownRoute,
   ...rumPageViewsTrendRoute,

x-pack/plugins/ux/e2e/journeys/index.ts

@@ -8,3 +8,4 @@
 export * from './core_web_vitals';
 export * from './url_ux_query.journey';
 export * from './ux_js_errors.journey';
+export * from './ux_client_metrics.journey';

x-pack/plugins/ux/e2e/journeys/ux_client_metrics.journey.ts

@@ -0,0 +1,78 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { journey, step, expect, before } from '@elastic/synthetics';
+import { UXDashboardDatePicker } from '../page_objects/date_picker';
+import { byTestId, loginToKibana, waitForLoadingToFinish } from './utils';
+
+const totalPageLoadValue = '4.24 s';
+const totalPageLoadLabel = `Total
+
+${totalPageLoadValue}`;
+const backendLabel = `Backend
+
+359 ms`;
+const frontendLabel = `Frontend
+
+3.88 s`;
+const pageViewsLabel = `Total page views
+
+524`;
+
+journey('UX ClientMetrics', async ({ page, params }) => {
+  before(async () => {
+    await waitForLoadingToFinish({ page });
+  });
+
+  const queryParams = {
+    percentile: '50',
+    rangeFrom: '2020-05-18T11:51:00.000Z',
+    rangeTo: '2021-10-30T06:37:15.536Z',
+  };
+  const queryString = new URLSearchParams(queryParams).toString();
+
+  const baseUrl = `${params.kibanaUrl}/app/ux`;
+
+  step('Go to UX Dashboard', async () => {
+    await page.goto(`${baseUrl}?${queryString}`, {
+      waitUntil: 'networkidle',
+    });
+    await loginToKibana({
+      page,
+      user: { username: 'viewer_user', password: 'changeme' },
+    });
+  });
+
+  step('Set date range', async () => {
+    const datePickerPage = new UXDashboardDatePicker(page);
+    await datePickerPage.setDefaultE2eRange();
+  });
+
+  step('Confirm metrics values', async () => {
+    // Wait until chart data is loaded
+    page.waitForLoadState('networkidle');
+    await page.waitForSelector(`text=${totalPageLoadValue}`);
+
+    const totalPageLoad = await (
+      await page.waitForSelector(byTestId('uxClientMetrics-totalPageLoad'))
+    ).innerText();
+    const backend = await (
+      await page.waitForSelector(byTestId('uxClientMetrics-backend'))
+    ).innerText();
+    const frontend = await (
+      await page.waitForSelector(byTestId('uxClientMetrics-frontend'))
+    ).innerText();
+    const pageViews = await (
+      await page.waitForSelector(byTestId('uxClientMetrics-pageViews'))
+    ).innerText();
+
+    expect(totalPageLoad).toBe(totalPageLoadLabel);
+    expect(backend).toBe(backendLabel);
+    expect(frontend).toBe(frontendLabel);
+    expect(pageViews).toBe(pageViewsLabel);
+  });
+});

x-pack/plugins/ux/public/components/app/rum_dashboard/client_metrics/metrics.tsx

@@ -16,9 +16,8 @@ import {
   EuiToolTip,
   EuiIconTip,
 } from '@elastic/eui';
-import { useFetcher } from '../../../../hooks/use_fetcher';
+import { useClientMetricsQuery } from '../../../../hooks/use_client_metrics_query';
 import { I18LABELS } from '../translations';
-import { useUxQuery } from '../hooks/use_ux_query';
 import { formatToSec } from '../ux_metrics/key_ux_metrics';
 import { CsmSharedContext } from '../csm_shared_context';
 
@@ -49,23 +48,7 @@ function PageViewsTotalTitle({ pageViews }: { pageViews?: number }) {
 }
 
 export function Metrics() {
-  const uxQuery = useUxQuery();
-
-  const { data, status } = useFetcher(
-    (callApmApi) => {
-      if (uxQuery) {
-        return callApmApi('GET /internal/apm/ux/client-metrics', {
-          params: {
-            query: {
-              ...uxQuery,
-            },
-          },
-        });
-      }
-      return Promise.resolve(null);
-    },
-    [uxQuery]
-  );
+  const { data, loading } = useClientMetricsQuery();
 
   const { setSharedData } = useContext(CsmSharedContext);
 
@@ -79,6 +62,7 @@ export function Metrics() {
     <ClFlexGroup wrap responsive={false}>
       <EuiFlexItem style={STAT_STYLE}>
         <EuiStat
+          data-test-subj={'uxClientMetrics-totalPageLoad'}
           titleSize="l"
           title={formatTitle('ms', data?.totalPageLoadDuration?.value)}
           description={
@@ -90,11 +74,12 @@ export function Metrics() {
               />
             </>
           }
-          isLoading={status !== 'success'}
+          isLoading={!!loading}
         />
       </EuiFlexItem>
       <EuiFlexItem style={STAT_STYLE}>
         <EuiStat
+          data-test-subj={'uxClientMetrics-backend'}
           titleSize="l"
           title={formatTitle('ms', data?.backEnd?.value)}
           description={
@@ -106,11 +91,12 @@ export function Metrics() {
               />
             </>
           }
-          isLoading={status !== 'success'}
+          isLoading={!!loading}
         />
       </EuiFlexItem>
       <EuiFlexItem style={STAT_STYLE}>
         <EuiStat
+          data-test-subj={'uxClientMetrics-frontend'}
           titleSize="l"
           title={formatTitle('ms', data?.frontEnd?.value)}
           description={
@@ -122,15 +108,16 @@ export function Metrics() {
               />
             </>
           }
-          isLoading={status !== 'success'}
+          isLoading={!!loading}
         />
       </EuiFlexItem>
       <EuiFlexItem style={STAT_STYLE}>
         <EuiStat
+          data-test-subj={'uxClientMetrics-pageViews'}
           titleSize="l"
           title={<PageViewsTotalTitle pageViews={data?.pageViews?.value} />}
           description={I18LABELS.pageViews}
-          isLoading={status !== 'success'}
+          isLoading={!!loading}
         />
       </EuiFlexItem>
     </ClFlexGroup>

x-pack/plugins/ux/public/hooks/use_client_metrics_query.ts

@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { useEsSearch } from '@kbn/observability-plugin/public';
+import { useMemo } from 'react';
+import { useDataView } from '../components/app/rum_dashboard/local_uifilters/use_data_view';
+import { useLegacyUrlParams } from '../context/url_params_context/use_url_params';
+import { callDateMath } from '../services/data/call_date_math';
+import { clientMetricsQuery } from '../services/data/client_metrics_query';
+
+export function useClientMetricsQuery() {
+  const {
+    rangeId,
+    urlParams: { start, end, percentile = 50, searchTerm },
+    uxUiFilters,
+  } = useLegacyUrlParams();
+  const { dataViewTitle } = useDataView();
+  const { data: esQueryResponse, loading } = useEsSearch(
+    {
+      index: dataViewTitle,
+      ...clientMetricsQuery(
+        callDateMath(start),
+        callDateMath(end),
+        percentile,
+        searchTerm,
+        uxUiFilters
+      ),
+    },
+    [start, end, percentile, searchTerm, uxUiFilters, dataViewTitle, rangeId],
+    { name: 'UxClientMetrics' }
+  );
+
+  const data = useMemo(() => {
+    if (!esQueryResponse) return {};
+
+    const {
+      hasFetchStartField: { backEnd, totalPageLoadDuration },
+    } = esQueryResponse.aggregations!;
+
+    const pkey = percentile.toFixed(1);
+
+    const totalPageLoadDurationValue = totalPageLoadDuration.values[pkey] ?? 0;
+    const totalPageLoadDurationValueMs = totalPageLoadDurationValue / 1000; // Microseconds to milliseconds
+    const backendValue = backEnd.values[pkey] ?? 0;
+
+    return {
+      pageViews: { value: (esQueryResponse.hits.total as any as number) ?? 0 },
+      totalPageLoadDuration: { value: totalPageLoadDurationValueMs },
+      backEnd: { value: backendValue },
+      frontEnd: { value: totalPageLoadDurationValueMs - backendValue },
+    };
+  }, [esQueryResponse, percentile]);
+
+  return { data, loading };
+}

x-pack/plugins/ux/public/services/data/__snapshots__/client_metrics_query.test.ts.snap

@@ -0,0 +1,76 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`clientMetricsQuery fetches client metrics 1`] = `
+Object {
+  "body": Object {
+    "aggs": Object {
+      "hasFetchStartField": Object {
+        "aggs": Object {
+          "backEnd": Object {
+            "percentiles": Object {
+              "field": "transaction.marks.agent.timeToFirstByte",
+              "hdr": Object {
+                "number_of_significant_value_digits": 3,
+              },
+              "percents": Array [
+                50,
+              ],
+            },
+          },
+          "totalPageLoadDuration": Object {
+            "percentiles": Object {
+              "field": "transaction.duration.us",
+              "hdr": Object {
+                "number_of_significant_value_digits": 3,
+              },
+              "percents": Array [
+                50,
+              ],
+            },
+          },
+        },
+        "filter": Object {
+          "exists": Object {
+            "field": "transaction.marks.navigationTiming.fetchStart",
+          },
+        },
+      },
+    },
+    "query": Object {
+      "bool": Object {
+        "filter": Array [
+          Object {
+            "range": Object {
+              "@timestamp": Object {
+                "format": "epoch_millis",
+                "gte": 0,
+                "lte": 50000,
+              },
+            },
+          },
+          Object {
+            "term": Object {
+              "transaction.type": "page-load",
+            },
+          },
+          Object {
+            "terms": Object {
+              "processor.event": Array [
+                "transaction",
+              ],
+            },
+          },
+          Object {
+            "term": Object {
+              "service.environment": "staging",
+            },
+          },
+        ],
+        "must_not": Array [],
+      },
+    },
+    "size": 0,
+    "track_total_hits": true,
+  },
+}
+`;

x-pack/plugins/ux/public/services/data/__snapshots__/core_web_vitals_query.test.ts.snap

@@ -99,6 +99,13 @@ Object {
               "transaction.type": "page-load",
             },
           },
+          Object {
+            "terms": Object {
+              "processor.event": Array [
+                "transaction",
+              ],
+            },
+          },
           Object {
             "exists": Object {
               "field": "transaction.marks.navigationTiming.fetchStart",

x-pack/plugins/ux/public/services/data/__snapshots__/service_name_query.test.ts.snap

@@ -28,6 +28,13 @@ Object {
               "transaction.type": "page-load",
             },
           },
+          Object {
+            "terms": Object {
+              "processor.event": Array [
+                "transaction",
+              ],
+            },
+          },
           Object {
             "exists": Object {
               "field": "transaction.marks.navigationTiming.fetchStart",

x-pack/plugins/ux/public/services/data/client_metrics_query.test.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { clientMetricsQuery } from './client_metrics_query';
+
+describe('clientMetricsQuery', () => {
+  it('fetches client metrics', () => {
+    const query = clientMetricsQuery(0, 50000, 50, '', {
+      environment: 'staging',
+    });
+    expect(query).toMatchSnapshot();
+  });
+});

x-pack/plugins/ux/public/services/data/client_metrics_query.ts

@@ -5,27 +5,22 @@
  * 2.0.
  */
 
-import { getRumPageLoadTransactionsProjection } from '../../projections/rum_page_load_transactions';
-import { mergeProjection } from '../../projections/util/merge_projection';
-import { SetupUX } from './route';
+import { mergeProjection } from '../../../common/utils/merge_projection';
+import { SetupUX, UxUIFilters } from '../../../typings/ui_filters';
 import {
   TRANSACTION_TIME_TO_FIRST_BYTE,
   TRANSACTION_DURATION,
 } from '../../../common/elasticsearch_fieldnames';
+import { getRumPageLoadTransactionsProjection } from './projections';
 
-export async function getClientMetrics({
-  setup,
-  urlQuery,
-  percentile = 50,
-  start,
-  end,
-}: {
-  setup: SetupUX;
-  urlQuery?: string;
-  percentile?: number;
-  start: number;
-  end: number;
-}) {
+export function clientMetricsQuery(
+  start: number,
+  end: number,
+  percentile: number = 50,
+  urlQuery?: string,
+  uiFilters?: UxUIFilters
+) {
+  const setup: SetupUX = { uiFilters: uiFilters ? uiFilters : {} };
   const projection = getRumPageLoadTransactionsProjection({
     setup,
     urlQuery,
@@ -68,22 +63,5 @@ export async function getClientMetrics({
     },
   });
 
-  const { apmEventClient } = setup;
-  const response = await apmEventClient.search('get_client_metrics', params);
-  const {
-    hasFetchStartField: { backEnd, totalPageLoadDuration },
-  } = response.aggregations!;
-
-  const pkey = percentile.toFixed(1);
-
-  const totalPageLoadDurationValue = totalPageLoadDuration.values[pkey] ?? 0;
-  const totalPageLoadDurationValueMs = totalPageLoadDurationValue / 1000; // Microseconds to milliseconds
-  const backendValue = backEnd.values[pkey] ?? 0;
-
-  return {
-    pageViews: { value: response.hits.total.value ?? 0 },
-    totalPageLoadDuration: { value: totalPageLoadDurationValueMs },
-    backEnd: { value: backendValue },
-    frontEnd: { value: totalPageLoadDurationValueMs - backendValue },
-  };
+  return params;
 }

x-pack/plugins/ux/public/services/data/core_web_vitals_query.ts

@@ -158,6 +158,5 @@ export function coreWebVitalsQuery(
       },
     },
   });
-  const { apm, ...rest } = params;
-  return rest;
+  return params;
 }

x-pack/plugins/ux/public/services/data/projections.ts

@@ -36,6 +36,7 @@ export function getRumPageLoadTransactionsProjection({
     filter: [
       ...rangeQuery(start, end),
       { term: { [TRANSACTION_TYPE]: TRANSACTION_PAGE_LOAD } },
+      { terms: { [PROCESSOR_EVENT]: [ProcessorEvent.transaction] } },
       ...(checkFetchStartFieldExists
         ? [
             {
@@ -62,9 +63,6 @@ export function getRumPageLoadTransactionsProjection({
   };
 
   return {
-    apm: {
-      events: [ProcessorEvent.transaction],
-    },
     body: {
       query: {
         bool,

x-pack/plugins/ux/public/services/data/service_name_query.ts

@@ -37,6 +37,6 @@ export function serviceNameQuery(
       },
     },
   });
-  const { apm, ...rest } = params;
-  return rest;
+
+  return params;
 }

x-pack/plugins/ux/public/services/data/url_search_query.ts

@@ -53,6 +53,5 @@ export function urlSearchQuery(
       },
     },
   });
-  const { apm: _apm, ...rest } = params;
-  return rest;
+  return params;
 }