[Infra UI] Show only hosts with metrics collected by system module in the hosts view (#177239)

Closes #176403

## Summary

This PR adds a filter for the `event.module` to be `system` because the
Hosts View is only compatible with the metrics-system indices - I added
a
[comment](https://github.com/elastic/kibana/issues/176403#issuecomment-1954232722)
to explain the change in the query. It adds infra client as part of the
synthtrace and a scenario to test the change

## Testing 
- Use the new synthtrace scenario: `node scripts/synthtrace --clean
infra_hosts_with_apm_hosts.ts`
- By default there should be `10` host visible on the host view and 3
separate services in APM (the APM hosts should not be visible)
- The scenario can be used with different numbers of services/hosts for
example:
`node scripts/synthtrace --clean --scenarioOpts.numServices=5
--scenarioOpts.numHosts=5 infra_hosts_with_apm_hosts.ts`
- 5 hosts shown on Infrastructure > Hosts (the APM hosts should not be
visible)

![image](d8763a24-95c2-43cd-991b-23edd102f47a)
       - 5 services shown on APM > Services

![image](0638cfdb-3d6f-4f72-915d-d67764bc9349)
- Use remote cluster (with APM)
   - The hosts with `0` metrics coming from APM should not be visible:
<img width="1920" alt="image"
src="af69efc0-bbd9-47ae-8431-2a56fa0626c4">

packages/kbn-apm-synthtrace-client/src/lib/infra/host.ts

@@ -17,6 +17,7 @@ interface HostDocument extends Fields {
   'host.hostname': string;
   'host.name': string;
   'metricset.name'?: string;
+  'event.module'?: string;
 }
 
 class Host extends Entity<HostDocument> {
@@ -131,6 +132,7 @@ class HostMetrics extends Serializable<HostMetricsDocument> {}
 
 export function host(name: string): Host {
   return new Host({
+    'event.module': 'system',
     'agent.id': 'synthtrace',
     'host.hostname': name,
     'host.name': name,

packages/kbn-apm-synthtrace/src/cli/scenario.ts

@@ -9,7 +9,7 @@
 import { Timerange } from '@kbn/apm-synthtrace-client';
 import { Logger } from '../lib/utils/create_logger';
 import { RunOptions } from './utils/parse_run_cli_flags';
-import { ApmSynthtraceEsClient, LogsSynthtraceEsClient } from '../..';
+import { ApmSynthtraceEsClient, InfraSynthtraceEsClient, LogsSynthtraceEsClient } from '../..';
 import { ScenarioReturnType } from '../lib/utils/with_client';
 
 type Generate<TFields> = (options: {
@@ -17,6 +17,7 @@ type Generate<TFields> = (options: {
   clients: {
     apmEsClient: ApmSynthtraceEsClient;
     logsEsClient: LogsSynthtraceEsClient;
+    infraEsClient: InfraSynthtraceEsClient;
   };
 }) => ScenarioReturnType<TFields> | Array<ScenarioReturnType<TFields>>;
 
@@ -24,6 +25,7 @@ export type Scenario<TFields> = (options: RunOptions & { logger: Logger }) => Pr
   bootstrap?: (options: {
     apmEsClient: ApmSynthtraceEsClient;
     logsEsClient: LogsSynthtraceEsClient;
+    infraEsClient: InfraSynthtraceEsClient;
   }) => Promise<void>;
   generate: Generate<TFields>;
 }>;

packages/kbn-apm-synthtrace/src/cli/utils/bootstrap.ts

@@ -9,6 +9,7 @@
 import { createLogger } from '../../lib/utils/create_logger';
 import { getApmEsClient } from './get_apm_es_client';
 import { getLogsEsClient } from './get_logs_es_client';
+import { getInfraEsClient } from './get_infra_es_client';
 import { getKibanaClient } from './get_kibana_client';
 import { getServiceUrls } from './get_service_urls';
 import { RunOptions } from './parse_run_cli_flags';
@@ -47,15 +48,23 @@ export async function bootstrap(runOptions: RunOptions) {
     concurrency: runOptions.concurrency,
   });
 
+  const infraEsClient = getInfraEsClient({
+    target: esUrl,
+    logger,
+    concurrency: runOptions.concurrency,
+  });
+
   if (runOptions.clean) {
     await apmEsClient.clean();
     await logsEsClient.clean();
+    await infraEsClient.clean();
   }
 
   return {
     logger,
     apmEsClient,
     logsEsClient,
+    infraEsClient,
     version,
     kibanaUrl,
     esUrl,

packages/kbn-apm-synthtrace/src/cli/utils/get_infra_es_client.ts

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { Client } from '@elastic/elasticsearch';
+import { InfraSynthtraceEsClient } from '../../lib/infra/infra_synthtrace_es_client';
+import { Logger } from '../../lib/utils/create_logger';
+import { RunOptions } from './parse_run_cli_flags';
+
+export function getInfraEsClient({
+  target,
+  logger,
+  concurrency,
+}: Pick<RunOptions, 'concurrency'> & {
+  target: string;
+  logger: Logger;
+}) {
+  const client = new Client({
+    node: target,
+  });
+
+  return new InfraSynthtraceEsClient({
+    client,
+    logger,
+    concurrency,
+    refreshAfterIndex: true,
+  });
+}

packages/kbn-apm-synthtrace/src/cli/utils/start_live_data_upload.ts

@@ -25,7 +25,7 @@ export async function startLiveDataUpload({
 }) {
   const file = runOptions.file;
 
-  const { logger, apmEsClient, logsEsClient } = await bootstrap(runOptions);
+  const { logger, apmEsClient, logsEsClient, infraEsClient } = await bootstrap(runOptions);
 
   const scenario = await getScenario({ file, logger });
   const { generate } = await scenario({ ...runOptions, logger });
@@ -62,7 +62,7 @@ export async function startLiveDataUpload({
 
       const generatorsAndClients = generate({
         range: timerange(bucketFrom.getTime(), bucketTo.getTime()),
-        clients: { logsEsClient, apmEsClient },
+        clients: { logsEsClient, apmEsClient, infraEsClient },
       });
 
       const generatorsAndClientsArray = castArray(generatorsAndClients);

packages/kbn-apm-synthtrace/src/cli/utils/synthtrace_worker.ts

@@ -15,6 +15,7 @@ import { getScenario } from './get_scenario';
 import { loggerProxy } from './logger_proxy';
 import { RunOptions } from './parse_run_cli_flags';
 import { getLogsEsClient } from './get_logs_es_client';
+import { getInfraEsClient } from './get_infra_es_client';
 
 export interface WorkerData {
   bucketFrom: Date;
@@ -42,6 +43,12 @@ async function start() {
     logger,
   });
 
+  const infraEsClient = getInfraEsClient({
+    concurrency: runOptions.concurrency,
+    target: esUrl,
+    logger,
+  });
+
   const file = runOptions.file;
 
   const scenario = await logger.perf('get_scenario', () => getScenario({ file, logger }));
@@ -51,13 +58,20 @@ async function start() {
   const { generate, bootstrap } = await scenario({ ...runOptions, logger });
 
   if (bootstrap) {
-    await bootstrap({ apmEsClient, logsEsClient });
+    await bootstrap({
+      apmEsClient,
+      logsEsClient,
+      infraEsClient,
+    });
   }
 
   logger.debug('Generating scenario');
 
   const generatorsAndClients = logger.perf('generate_scenario', () =>
-    generate({ range: timerange(bucketFrom, bucketTo), clients: { logsEsClient, apmEsClient } })
+    generate({
+      range: timerange(bucketFrom, bucketTo),
+      clients: { logsEsClient, apmEsClient, infraEsClient },
+    })
   );
 
   const generatorsAndClientsArray = castArray(generatorsAndClients);

packages/kbn-apm-synthtrace/src/scenarios/infra_hosts_with_apm_hosts.ts

@@ -0,0 +1,82 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+import { InfraDocument, apm, Instance, infra, ApmFields } from '@kbn/apm-synthtrace-client';
+import { Scenario } from '../cli/scenario';
+import { withClient } from '../lib/utils/with_client';
+import { getSynthtraceEnvironment } from '../lib/utils/get_synthtrace_environment';
+
+const ENVIRONMENT = getSynthtraceEnvironment(__filename);
+
+const scenario: Scenario<InfraDocument | ApmFields> = async (runOptions) => {
+  return {
+    generate: ({ range, clients: { infraEsClient, apmEsClient } }) => {
+      const { numServices = 3, numHosts = 10 } = runOptions.scenarioOpts || {};
+      const { logger } = runOptions;
+
+      // Infra hosts Data logic
+
+      const HOSTS = Array(numHosts)
+        .fill(0)
+        .map((_, idx) => infra.host(`my-host-${idx}`));
+
+      const hosts = range
+        .interval('30s')
+        .rate(1)
+        .generator((timestamp) =>
+          HOSTS.flatMap((host) => [
+            host.cpu().timestamp(timestamp),
+            host.memory().timestamp(timestamp),
+            host.network().timestamp(timestamp),
+            host.load().timestamp(timestamp),
+            host.filesystem().timestamp(timestamp),
+            host.diskio().timestamp(timestamp),
+          ])
+        );
+
+      // APM Simple Trace
+
+      const instances = [...Array(numServices).keys()].map((index) =>
+        apm
+          .service({ name: `synth-node-${index}`, environment: ENVIRONMENT, agentName: 'nodejs' })
+          .instance('instance')
+      );
+      const instanceSpans = (instance: Instance) => {
+        const metricsets = range
+          .interval('30s')
+          .rate(1)
+          .generator((timestamp) =>
+            instance
+              .appMetrics({
+                'system.memory.actual.free': 800,
+                'system.memory.total': 1000,
+                'system.cpu.total.norm.pct': 0.6,
+                'system.process.cpu.total.norm.pct': 0.7,
+              })
+              .timestamp(timestamp)
+          );
+
+        return [metricsets];
+      };
+
+      return [
+        withClient(
+          infraEsClient,
+          logger.perf('generating_infra_hosts', () => hosts)
+        ),
+        withClient(
+          apmEsClient,
+          logger.perf('generating_apm_events', () =>
+            instances.flatMap((instance) => instanceSpans(instance))
+          )
+        ),
+      ];
+    },
+  };
+};
+
+export default scenario;

x-pack/plugins/infra/public/pages/metrics/hosts/hooks/use_host_count.ts

@@ -32,6 +32,24 @@ export const useHostCount = () => {
       const filters: QueryDslQueryContainer = {
         bool: {
           ...query.bool,
+          must: [
+            {
+              bool: {
+                should: [
+                  {
+                    term: {
+                      'event.module': 'system',
+                    },
+                  },
+                  {
+                    term: {
+                      'metricset.module': 'system', // Needed for hosts where metricbeat version < 8
+                    },
+                  },
+                ],
+              },
+            },
+          ],
           filter: [
             ...query.bool.filter,
             {

x-pack/plugins/infra/server/routes/infra/lib/helpers/get_infra_alerts_client.ts

@@ -29,7 +29,7 @@ export async function getInfraAlertsClient({
   const infraAlertsIndices = await alertsClient.getAuthorizedAlertsIndices(['infrastructure']);
 
   if (!infraAlertsIndices || isEmpty(infraAlertsIndices)) {
-    throw Error('No alert indices exist for "infrastrucuture"');
+    throw Error('No alert indices exist for "infrastructure"');
   }
 
   return {

x-pack/plugins/infra/server/routes/infra/lib/helpers/query.ts

@@ -31,7 +31,6 @@ export const createFilters = ({
       ? extrafilterClause
       : [extrafilterClause]
     : [];
-
   const hostNamesFilter =
     hostNamesShortList.length > 0
       ? [
@@ -86,6 +85,27 @@ export const runQuery = <T>(
     );
 };
 
+export const systemMetricsFilter = {
+  must: [
+    {
+      bool: {
+        should: [
+          {
+            term: {
+              'event.module': 'system',
+            },
+          },
+          {
+            term: {
+              'metricset.module': 'system', // Needed for hosts where metricbeat version < 8
+            },
+          },
+        ],
+      },
+    },
+  ],
+};
+
 export const getInventoryModelAggregations = (
   metrics: InfraAssetMetricType[]
 ): Record<string, estypes.AggregationsAggregationContainer> => {

x-pack/plugins/infra/server/routes/infra/lib/host/get_all_hosts.ts

@@ -17,7 +17,12 @@ import {
   HostsMetricsSearchAggregationResponse,
   HostsMetricsSearchAggregationResponseRT,
 } from '../types';
-import { createFilters, getInventoryModelAggregations, runQuery } from '../helpers/query';
+import {
+  createFilters,
+  systemMetricsFilter,
+  getInventoryModelAggregations,
+  runQuery,
+} from '../helpers/query';
 
 export const getAllHosts = async (
   { searchClient, sourceConfig, params }: GetHostsArgs,
@@ -44,6 +49,7 @@ const createQuery = (
       size: 0,
       query: {
         bool: {
+          ...systemMetricsFilter,
           filter: createFilters({
             params,
             hostNamesShortList,

x-pack/plugins/infra/server/routes/infra/lib/host/get_filtered_hosts.ts

@@ -18,7 +18,7 @@ import {
 } from '../types';
 import { BUCKET_KEY, MAX_SIZE } from '../constants';
 import { assertQueryStructure } from '../utils';
-import { createFilters, runQuery } from '../helpers/query';
+import { createFilters, runQuery, systemMetricsFilter } from '../helpers/query';
 
 export const getFilteredHosts = async ({
   searchClient,
@@ -46,6 +46,7 @@ const createQuery = (
       query: {
         bool: {
           ...params.query.bool,
+          ...systemMetricsFilter,
           filter: createFilters({ params, extraFilter: params.query }),
         },
       },