[infra] hosts table metrics (#173708)

## Summary

Closes https://github.com/elastic/kibana/issues/172475

- record performance event of the host table data loading
- infra synthtrace now installs `system` package and can produce all
datasets queried by host view
- [lens
viz](f88d3180-9f10-11ee-b4ef-7533a045abfc?_g=h@97e8101&_a=h@a8f21d5)
(internal)

### Testing
- run journey with `PERFORMANCE_ENABLE_TELEMETRY=1 node
scripts/run_performance.js -v --journey-path
x-pack/performance/journeys/infra_hosts_view.ts`
- check data is ingested in lens

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Dzmitry Lemechko <dzmitry.lemechko@elastic.co>

.buildkite/ftr_configs.yml

@@ -459,6 +459,7 @@ enabled:
   - x-pack/performance/journeys/tags_listing_page.ts
   - x-pack/performance/journeys/cloud_security_dashboard.ts
   - x-pack/performance/journeys/apm_service_inventory.ts
+  - x-pack/performance/journeys/infra_hosts_view.ts
   - x-pack/test/custom_branding/config.ts
   - x-pack/test/profiling_api_integration/cloud/config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/configs/serverless.config.ts

packages/kbn-apm-synthtrace-client/src/lib/infra/container.ts

@@ -15,6 +15,7 @@ interface ContainerDocument extends Fields {
   'container.id': string;
   'kubernetes.pod.uid': string;
   'kubernetes.node.name': string;
+  'metricset.name'?: string;
 }
 
 export class Container extends Entity<ContainerDocument> {

packages/kbn-apm-synthtrace-client/src/lib/infra/host.ts

@@ -13,14 +13,79 @@ import { Serializable } from '../serializable';
 import { pod } from './pod';
 
 interface HostDocument extends Fields {
+  'agent.id': string;
   'host.hostname': string;
+  'host.name': string;
+  'metricset.name'?: string;
 }
 
 class Host extends Entity<HostDocument> {
-  metrics() {
+  cpu() {
     return new HostMetrics({
       ...this.fields,
-      'system.cpu.total.norm.pct': 46,
+      'system.cpu.total.norm.pct': 0.094,
+      'system.cpu.user.pct': 0.805,
+      'system.cpu.system.pct': 0.704,
+      'system.cpu.cores': 16,
+      'metricset.period': 10000,
+      'metricset.name': 'cpu',
+    });
+  }
+
+  memory() {
+    return new HostMetrics({
+      ...this.fields,
+      'system.memory.actual.free': 44704067584,
+      'system.memory.actual.used.bytes': 24015409152,
+      'system.memory.actual.used.pct': 0.35,
+      'system.memory.total': 68719476736,
+      'system.memory.used.bytes': 39964708864,
+      'system.memory.used.pct': 0.582,
+      'metricset.period': 10000,
+      'metricset.name': 'memory',
+    });
+  }
+
+  network() {
+    return new HostMetrics({
+      ...this.fields,
+      'host.network.ingress.bytes': 2871285,
+      'host.network.egress.bytes': 2904987,
+      'metricset.period': 10000,
+      'metricset.name': 'network',
+    });
+  }
+
+  load() {
+    return new HostMetrics({
+      ...this.fields,
+      'system.load': {
+        1: 3,
+        cores: 16,
+      },
+      'metricset.period': 10000,
+      'metricset.name': 'load',
+    });
+  }
+
+  filesystem() {
+    return new HostMetrics({
+      ...this.fields,
+      'system.filesystem.used.pct': 12.23,
+      'metricset.period': 10000,
+      'metricset.name': 'filesystem',
+    });
+  }
+
+  diskio() {
+    return new HostMetrics({
+      ...this.fields,
+      'system.diskio.read.count': 3538413,
+      'system.diskio.write.count': 4694333,
+      'system.diskio.read.bytes': 33147297792,
+      'system.diskio.write.bytes': 48595652608,
+      'metricset.period': 10000,
+      'metricset.name': 'diskio',
     });
   }
 
@@ -39,13 +104,35 @@ class Host extends Entity<HostDocument> {
 }
 
 export interface HostMetricsDocument extends HostDocument {
-  'system.cpu.total.norm.pct': number;
+  'agent.id': string;
+  'metricset.period'?: number;
+  'metricset.name'?: string;
+  'system.cpu.total.norm.pct'?: number;
+  'system.cpu.user.pct'?: number;
+  'system.cpu.system.pct'?: number;
+  'system.cpu.cores'?: number;
+  'system.diskio.read.count'?: number;
+  'system.diskio.write.count'?: number;
+  'system.diskio.read.bytes'?: number;
+  'system.diskio.write.bytes'?: number;
+  'system.filesystem.used.pct'?: number;
+  'system.memory.actual.used.pct'?: number;
+  'system.memory.total'?: number;
+  'system.memory.actual.used.bytes'?: number;
+  'system.memory.actual.free'?: number;
+  'system.memory.used.bytes'?: number;
+  'system.memory.used.pct'?: number;
+  'system.load'?: { 1: number; cores: number };
+  'host.network.ingress.bytes'?: number;
+  'host.network.egress.bytes'?: number;
 }
 
 class HostMetrics extends Serializable<HostMetricsDocument> {}
 
 export function host(name: string): Host {
   return new Host({
+    'agent.id': 'synthtrace',
     'host.hostname': name,
+    'host.name': name,
   });
 }

packages/kbn-apm-synthtrace-client/src/lib/infra/pod.ts

@@ -15,6 +15,7 @@ import { container } from './container';
 interface PodDocument extends Fields {
   'kubernetes.pod.uid': string;
   'kubernetes.node.name': string;
+  'metricset.name'?: string;
 }
 
 export class Pod extends Entity<PodDocument> {

packages/kbn-apm-synthtrace/index.ts

@@ -12,6 +12,7 @@ export { ApmSynthtraceEsClient } from './src/lib/apm/client/apm_synthtrace_es_cl
 export { ApmSynthtraceKibanaClient } from './src/lib/apm/client/apm_synthtrace_kibana_client';
 
 export { InfraSynthtraceEsClient } from './src/lib/infra/infra_synthtrace_es_client';
+export { InfraSynthtraceKibanaClient } from './src/lib/infra/infra_synthtrace_kibana_client';
 
 export { AssetsSynthtraceEsClient } from './src/lib/assets/assets_synthtrace_es_client';
 

packages/kbn-apm-synthtrace/src/lib/apm/client/apm_synthtrace_kibana_client.ts

@@ -9,6 +9,7 @@
 import fetch from 'node-fetch';
 import pRetry from 'p-retry';
 import { Logger } from '../../utils/create_logger';
+import { kibanaHeaders } from '../../shared/client_headers';
 
 export class ApmSynthtraceKibanaClient {
   private readonly logger: Logger;
@@ -63,12 +64,3 @@ export class ApmSynthtraceKibanaClient {
     this.logger.info(`Installed APM package ${packageVersion}`);
   }
 }
-
-function kibanaHeaders() {
-  return {
-    Accept: 'application/json',
-    'Content-Type': 'application/json',
-    'kbn-xsrf': 'kibana',
-    'elastic-api-version': '2023-10-31',
-  };
-}

packages/kbn-apm-synthtrace/src/lib/infra/infra_synthtrace_es_client.ts

@@ -46,8 +46,20 @@ function getRoutingTransform() {
   return new Transform({
     objectMode: true,
     transform(document: ESDocumentWithOperation<InfraDocument>, encoding, callback) {
-      if ('host.hostname' in document) {
+      const metricset = document['metricset.name'];
+
+      if (metricset === 'cpu') {
         document._index = 'metrics-system.cpu-default';
+      } else if (metricset === 'memory') {
+        document._index = 'metrics-system.memory-default';
+      } else if (metricset === 'network') {
+        document._index = 'metrics-system.network-default';
+      } else if (metricset === 'load') {
+        document._index = 'metrics-system.load-default';
+      } else if (metricset === 'filesystem') {
+        document._index = 'metrics-system.filesystem-default';
+      } else if (metricset === 'diskio') {
+        document._index = 'metrics-system.diskio-default';
       } else if ('container.id' in document) {
         document._index = 'metrics-kubernetes.container-default';
       } else if ('kubernetes.pod.uid' in document) {

packages/kbn-apm-synthtrace/src/lib/infra/infra_synthtrace_kibana_client.ts

@@ -0,0 +1,70 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { join } from 'path';
+import fetch from 'node-fetch';
+import pRetry from 'p-retry';
+import { Logger } from '../utils/create_logger';
+import { kibanaHeaders } from '../shared/client_headers';
+
+export class InfraSynthtraceKibanaClient {
+  private readonly logger: Logger;
+  private target: string;
+
+  constructor(options: { logger: Logger; target: string; username: string; password: string }) {
+    this.logger = options.logger;
+    const url = new URL(options.target);
+    url.username = options.username;
+    url.password = options.password;
+    this.target = url.toString();
+  }
+
+  async fetchLatestSystemPackageVersion() {
+    const fleetPackageApiUrl = join(this.target, '/api/fleet/epm/packages/system?prerelease=true');
+    this.logger.debug(`Fetching latest System package version from ${fleetPackageApiUrl}`);
+    const response = await fetch(fleetPackageApiUrl, {
+      method: 'GET',
+      headers: kibanaHeaders(),
+    });
+
+    const responseJson = await response.json();
+
+    if (response.status !== 200) {
+      throw new Error(
+        `Failed to fetch latest System package version, received HTTP ${response.status} and message: ${responseJson.message}`
+      );
+    }
+
+    const { latestVersion } = responseJson.item;
+
+    return latestVersion as string;
+  }
+
+  async installSystemPackage(packageVersion: string) {
+    this.logger.debug(`Installing System package ${packageVersion}`);
+
+    const url = join(this.target, `/api/fleet/epm/packages/system/${packageVersion}`);
+    const response = await pRetry(() => {
+      return fetch(url, {
+        method: 'POST',
+        headers: kibanaHeaders(),
+        body: '{"force":true}',
+      });
+    });
+
+    const responseJson = await response.json();
+
+    if (!responseJson.items) {
+      throw new Error(
+        `Failed to install System package version ${packageVersion}, received HTTP ${response.status} and message: ${responseJson.message} for url ${url}`
+      );
+    }
+
+    this.logger.info(`Installed System package ${packageVersion}`);
+  }
+}

packages/kbn-apm-synthtrace/src/lib/shared/client_headers.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export function kibanaHeaders() {
+  return {
+    Accept: 'application/json',
+    'Content-Type': 'application/json',
+    'kbn-xsrf': 'kibana',
+    'elastic-api-version': '2023-10-31',
+  };
+}

x-pack/performance/journeys/infra_hosts_view.ts

@@ -0,0 +1,86 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { Journey } from '@kbn/journeys';
+import {
+  createLogger,
+  InfraSynthtraceEsClient,
+  LogLevel,
+  InfraSynthtraceKibanaClient,
+} from '@kbn/apm-synthtrace';
+import { infra, timerange } from '@kbn/apm-synthtrace-client';
+import { subj } from '@kbn/test-subj-selector';
+
+export const journey = new Journey({
+  beforeSteps: async ({ kbnUrl, auth, es }) => {
+    const logger = createLogger(LogLevel.debug);
+    const synthKibanaClient = new InfraSynthtraceKibanaClient({
+      logger,
+      target: kbnUrl.get(),
+      username: auth.getUsername(),
+      password: auth.getPassword(),
+    });
+
+    const pkgVersion = await synthKibanaClient.fetchLatestSystemPackageVersion();
+    await synthKibanaClient.installSystemPackage(pkgVersion);
+
+    const synthEsClient = new InfraSynthtraceEsClient({
+      logger,
+      client: es,
+      refreshAfterIndex: true,
+    });
+
+    const start = Date.now() - 1000 * 60 * 10;
+    await synthEsClient.index(
+      generateHostsData({
+        from: new Date(start).toISOString(),
+        to: new Date().toISOString(),
+        count: 1000,
+      })
+    );
+  },
+}).step('Navigate to Hosts view and load 500 hosts', async ({ page, kbnUrl, kibanaPage }) => {
+  await page.goto(
+    kbnUrl.get(
+      `app/metrics/hosts?_a=(dateRange:(from:now-15m,to:now),filters:!(),limit:500,panelFilters:!(),query:(language:kuery,query:''))`
+    )
+  );
+  // wait for table to be loaded
+  await page.waitForSelector(subj('hostsView-table-loaded'));
+  // wait for metric charts to be loaded
+  await kibanaPage.waitForCharts({ count: 5, timeout: 60000 });
+});
+
+export function generateHostsData({
+  from,
+  to,
+  count = 1,
+}: {
+  from: string;
+  to: string;
+  count: number;
+}) {
+  const range = timerange(from, to);
+
+  const hosts = Array(count)
+    .fill(0)
+    .map((_, idx) => infra.host(`my-host-${idx}`));
+
+  return range
+    .interval('30s')
+    .rate(1)
+    .generator((timestamp, index) =>
+      hosts.flatMap((host) => [
+        host.cpu().timestamp(timestamp),
+        host.memory().timestamp(timestamp),
+        host.network().timestamp(timestamp),
+        host.load().timestamp(timestamp),
+        host.filesystem().timestamp(timestamp),
+        host.diskio().timestamp(timestamp),
+      ])
+    );
+}

x-pack/plugins/infra/public/containers/metrics_source/source.tsx

@@ -8,8 +8,8 @@
 import createContainer from 'constate';
 import React, { useEffect, useState } from 'react';
 
-import { useKibana } from '@kbn/kibana-react-plugin/public';
 import { IHttpFetchError } from '@kbn/core-http-browser';
+import { useKibanaContextForPlugin } from '../../hooks/use_kibana';
 import type {
   MetricsSourceConfigurationResponse,
   MetricsSourceConfiguration,
@@ -34,11 +34,13 @@ export const pickIndexPattern = (
 };
 
 export const useSource = ({ sourceId }: { sourceId: string }) => {
-  const { services } = useKibana();
+  const {
+    services: { http, telemetry },
+  } = useKibanaContextForPlugin();
 
   const notify = useSourceNotifier();
 
-  const fetchService = services.http;
+  const fetchService = http;
   const API_URL = `/api/metrics/source/${sourceId}`;
 
   const [source, setSource] = useState<MetricsSourceConfiguration | undefined>(undefined);
@@ -46,12 +48,22 @@ export const useSource = ({ sourceId }: { sourceId: string }) => {
   const [loadSourceRequest, loadSource] = useTrackedPromise(
     {
       cancelPreviousOn: 'resolution',
-      createPromise: () => {
+      createPromise: async () => {
         if (!fetchService) {
           throw new MissingHttpClientException();
         }
 
-        return fetchService.fetch<MetricsSourceConfigurationResponse>(API_URL, { method: 'GET' });
+        const start = performance.now();
+        const response = await fetchService.fetch<MetricsSourceConfigurationResponse>(API_URL, {
+          method: 'GET',
+        });
+        telemetry.reportPerformanceMetricEvent(
+          'infra_source_load',
+          performance.now() - start,
+          {},
+          {}
+        );
+        return response;
       },
       onResolve: (response) => {
         if (response) {

x-pack/plugins/infra/public/pages/metrics/hosts/components/hosts_table.tsx

@@ -44,7 +44,7 @@ export const HostsTable = () => {
       />
       <EuiBasicTable
         ref={refs.tableRef}
-        data-test-subj="hostsView-table"
+        data-test-subj={`hostsView-table-${loading ? 'loading' : 'loaded'}`}
         itemId="id"
         isSelectable
         selection={selection}

x-pack/plugins/infra/public/pages/metrics/hosts/hooks/use_hosts_view.ts

@@ -41,7 +41,7 @@ const BASE_INFRA_METRICS_PATH = '/api/metrics/infra';
 export const useHostsView = () => {
   const { sourceId } = useSourceContext();
   const {
-    services: { http, data },
+    services: { http, data, telemetry },
   } = useKibanaContextForPlugin();
   const { buildQuery, parsedDateRange, searchCriteria } = useUnifiedSearchContext();
   const abortCtrlRef = useRef(new AbortController());
@@ -59,14 +59,26 @@ export const useHostsView = () => {
   );
 
   const [state, refetch] = useAsyncFn(
-    () => {
+    async () => {
       abortCtrlRef.current.abort();
       abortCtrlRef.current = new AbortController();
 
-      return http.post<GetInfraMetricsResponsePayload>(`${BASE_INFRA_METRICS_PATH}`, {
-        signal: abortCtrlRef.current.signal,
-        body: JSON.stringify(baseRequest),
-      });
+      const start = performance.now();
+      const metricsResponse = await http.post<GetInfraMetricsResponsePayload>(
+        `${BASE_INFRA_METRICS_PATH}`,
+        {
+          signal: abortCtrlRef.current.signal,
+          body: JSON.stringify(baseRequest),
+        }
+      );
+      const duration = performance.now() - start;
+      telemetry.reportPerformanceMetricEvent(
+        'infra_hosts_table_load',
+        duration,
+        { key1: 'data_load', value1: duration },
+        { limit: searchCriteria.limit }
+      );
+      return metricsResponse;
     },
     [baseRequest, http],
     { loading: true }

x-pack/plugins/infra/public/services/telemetry/telemetry_client.mock.ts

@@ -15,4 +15,5 @@ export const createTelemetryClientMock = (): jest.Mocked<ITelemetryClient> => ({
   reportHostsViewTotalHostCountRetrieved: jest.fn(),
   reportAssetDetailsFlyoutViewed: jest.fn(),
   reportAssetDetailsPageViewed: jest.fn(),
+  reportPerformanceMetricEvent: jest.fn(),
 });

x-pack/plugins/infra/public/services/telemetry/telemetry_client.ts

@@ -6,6 +6,7 @@
  */
 
 import type { AnalyticsServiceSetup } from '@kbn/core-analytics-server';
+import { reportPerformanceMetricEvent } from '@kbn/ebt-tools';
 import {
   AssetDetailsFlyoutViewedParams,
   AssetDetailsPageViewedParams,
@@ -15,6 +16,7 @@ import {
   HostsViewQuerySubmittedParams,
   InfraTelemetryEventTypes,
   ITelemetryClient,
+  PerformanceMetricInnerEvents,
 } from './types';
 
 /**
@@ -70,4 +72,18 @@ export class TelemetryClient implements ITelemetryClient {
   public reportAssetDetailsPageViewed = (params: AssetDetailsPageViewedParams) => {
     this.analytics.reportEvent(InfraTelemetryEventTypes.ASSET_DETAILS_PAGE_VIEWED, params);
   };
+
+  public reportPerformanceMetricEvent = (
+    eventName: string,
+    duration: number,
+    innerEvents: PerformanceMetricInnerEvents = {},
+    meta: Record<string, unknown> = {}
+  ) => {
+    reportPerformanceMetricEvent(this.analytics, {
+      eventName,
+      duration,
+      meta,
+      ...innerEvents,
+    });
+  };
 }

x-pack/plugins/infra/public/services/telemetry/types.ts

@@ -61,6 +61,11 @@ export type InfraTelemetryEventParams =
   | HostsViewQueryHostsCountRetrievedParams
   | AssetDetailsFlyoutViewedParams;
 
+export interface PerformanceMetricInnerEvents {
+  key1?: string;
+  value1?: number;
+}
+
 export interface ITelemetryClient {
   reportHostEntryClicked(params: HostEntryClickedParams): void;
   reportHostFlyoutFilterRemoved(params: HostFlyoutFilterActionParams): void;
@@ -69,6 +74,12 @@ export interface ITelemetryClient {
   reportHostsViewQuerySubmitted(params: HostsViewQuerySubmittedParams): void;
   reportAssetDetailsFlyoutViewed(params: AssetDetailsFlyoutViewedParams): void;
   reportAssetDetailsPageViewed(params: AssetDetailsPageViewedParams): void;
+  reportPerformanceMetricEvent(
+    eventName: string,
+    duration: number,
+    innerEvents: PerformanceMetricInnerEvents,
+    meta: Record<string, unknown>
+  ): void;
 }
 
 export type InfraTelemetryEvent =

x-pack/plugins/infra/tsconfig.json

@@ -79,7 +79,8 @@
     "@kbn/profiling-utils",
     "@kbn/profiling-data-access-plugin",
     "@kbn/core-http-request-handler-context-server",
-    "@kbn/observability-get-padded-alert-time-range-util"
+    "@kbn/observability-get-padded-alert-time-range-util",
+    "@kbn/ebt-tools"
   ],
   "exclude": ["target/**/*"]
 }

x-pack/test/api_integration/apis/asset_manager/tests/helpers.ts

@@ -100,5 +100,5 @@ export function generateHostsData({
   return range
     .interval('1m')
     .rate(1)
-    .generator((timestamp, index) => hosts.map((host) => host.metrics().timestamp(timestamp)));
+    .generator((timestamp, index) => hosts.map((host) => host.cpu().timestamp(timestamp)));
 }

x-pack/test/functional/apps/infra/hosts_view.ts

@@ -122,7 +122,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
     await retry.waitFor(
       'wait for table and KPI charts to load',
       async () =>
-        (await pageObjects.infraHostsView.isHostTableLoading()) &&
+        (await pageObjects.infraHostsView.isHostTableLoaded()) &&
         (await pageObjects.infraHostsView.isKPIChartsLoaded())
     );
 

x-pack/test/functional/page_objects/infra_hosts_view.ts

@@ -49,11 +49,11 @@ export function InfraHostsViewProvider({ getService }: FtrProviderContext) {
     // Table
 
     async getHostsTable() {
-      return testSubjects.find('hostsView-table');
+      return testSubjects.find('hostsView-table-loaded');
     },
 
-    async isHostTableLoading() {
-      return !(await testSubjects.exists('tbody[class*=euiBasicTableBodyLoading]'));
+    async isHostTableLoaded() {
+      return !(await testSubjects.exists('hostsView-table-loading'));
     },
 
     async getHostsTableData() {

x-pack/test_serverless/functional/test_suites/observability/infra/hosts_page.ts

@@ -36,7 +36,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
     await retry.waitFor(
       'wait for table and KPI charts to load',
       async () =>
-        (await pageObjects.infraHostsView.isHostTableLoading()) &&
+        (await pageObjects.infraHostsView.isHostTableLoaded()) &&
         (await pageObjects.infraHostsView.isKPIChartsLoaded())
     );