[Infra] Change host count KPI query (#188950)

Closes #188757 

## Summary
 
This PR adds an endpoint to get the hosts (monitored by the system
integration) count. Currently, it supports only hosts but it can be
extended to other asset types (if/when needed). So the endpoint ( **POST
/api/infra/{assetType}/count** ) supports only 'host' as `{assetType}`.

⚠️ This PR adds only the endpoint - it is not used yet! To avoid having
different host counts and results shown on the UI this PR is not
updating the hook responsible for the request because currently the
hosts shown in the table are not filtered by the system integration
(showing the filtered result of this endpoint can lead to
inconsistencies between the count and the results shown in the table)
Once [#188756](https://github.com/elastic/kibana/issues/188756) and
[#188752](https://github.com/elastic/kibana/issues/188752) are done we
can use this endpoint.

## Testing

It can't be tested in the UI so we can test it:

<details>

<summary>Using curl:</summary> 

```bash

curl --location -u elastic:changeme 'http://0.0.0.0:5601/ftw/api/infra/host/count' \
--header 'kbn-xsrf: xxxx' \
--header 'Content-Type: application/json' \
--data '{
   "query": {
      "bool": {
         "must": [],
         "filter": [],
         "should": [],
         "must_not": []
      }
   },
   "from": "2024-07-23T11:34:11.640Z",
   "to": "2024-07-23T11:49:11.640Z",
   "sourceId": "default"
}'

```
</details>

In case of testing with oblt replace the `elastic:changeme` with your
user and password

x-pack/plugins/observability_solution/infra/common/constants.ts

@@ -24,6 +24,8 @@ export const HOST_NAME_FIELD = 'host.name';
 export const CONTAINER_ID_FIELD = 'container.id';
 export const KUBERNETES_POD_UID_FIELD = 'kubernetes.pod.uid';
 export const SYSTEM_PROCESS_CMDLINE_FIELD = 'system.process.cmdline';
+export const EVENT_MODULE = 'event.module';
+export const METRICSET_MODULE = 'metricset.module';
 
 // logs
 export const MESSAGE_FIELD = 'message';

x-pack/plugins/observability_solution/infra/common/http_api/asset_count_api.ts

@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { dateRt } from '@kbn/io-ts-utils';
+import * as rt from 'io-ts';
+
+const AssetTypeRT = rt.type({
+  assetType: rt.literal('host'),
+});
+
+export const GetInfraAssetCountRequestBodyPayloadRT = rt.intersection([
+  rt.partial({
+    query: rt.UnknownRecord,
+  }),
+  rt.type({
+    sourceId: rt.string,
+    from: dateRt,
+    to: dateRt,
+  }),
+]);
+
+export const GetInfraAssetCountRequestParamsPayloadRT = AssetTypeRT;
+
+export const GetInfraAssetCountResponsePayloadRT = rt.intersection([
+  AssetTypeRT,
+  rt.type({
+    count: rt.number,
+  }),
+]);
+
+export type GetInfraAssetCountRequestParamsPayload = rt.TypeOf<
+  typeof GetInfraAssetCountRequestParamsPayloadRT
+>;
+export type GetInfraAssetCountRequestBodyPayload = Omit<
+  rt.TypeOf<typeof GetInfraAssetCountRequestBodyPayloadRT>,
+  'from' | 'to'
+> & {
+  from: string;
+  to: string;
+};
+
+export type GetInfraAssetCountResponsePayload = rt.TypeOf<
+  typeof GetInfraAssetCountResponsePayloadRT
+>;

x-pack/plugins/observability_solution/infra/common/http_api/index.ts

@@ -11,6 +11,7 @@ export * from './metrics_api';
 export * from './snapshot_api';
 export * from './host_details';
 export * from './infra';
+export * from './asset_count_api';
 
 /**
  * Exporting versioned APIs types

x-pack/plugins/observability_solution/infra/server/infra_server.ts

@@ -31,7 +31,7 @@ import { initNodeDetailsRoute } from './routes/node_details';
 import { initOverviewRoute } from './routes/overview';
 import { initProcessListRoute } from './routes/process_list';
 import { initSnapshotRoute } from './routes/snapshot';
-import { initInfraMetricsRoute } from './routes/infra';
+import { initInfraAssetRoutes } from './routes/infra';
 import { initMetricsExplorerViewRoutes } from './routes/metrics_explorer_views';
 import { initProfilingRoutes } from './routes/profiling';
 import { initServicesRoute } from './routes/services';
@@ -67,7 +67,7 @@ export const initInfraServer = (
   initGetLogAlertsChartPreviewDataRoute(libs);
   initProcessListRoute(libs);
   initOverviewRoute(libs);
-  initInfraMetricsRoute(libs);
+  initInfraAssetRoutes(libs);
   initProfilingRoutes(libs);
   initServicesRoute(libs);
   initCustomDashboardsRoutes(libs.framework);

x-pack/plugins/observability_solution/infra/server/routes/infra/README.md

@@ -1,9 +1,10 @@
-# Infra Hosts API
+# Infra Assets API
 
-This API returns a list of hosts and their metrics.
+## **POST /api/metrics/infra**
 
-**POST /api/metrics/infra**
-parameters:
+This endpoint returns a list of hosts and their metrics.
+
+### Parameters:
 
 - type: asset type. 'host' is the only one supported now
 - metrics: list of metrics to be calculated and returned for each host
@@ -18,7 +19,7 @@ The response includes:
 - metrics: object containing name of the metric and value
 - metadata: object containing name of the metadata and value
 
-## Examples
+### Examples:
 
 Request
 
@@ -113,3 +114,49 @@ Response
    ]
 }
 ```
+
+## **POST /api/infra/{assetType}/count**
+
+This endpoint returns the count of the hosts monitored with the system integration.
+
+### Parameters:
+
+- type: asset type. 'host' is the only one supported now
+- sourceId: sourceId to retrieve configuration such as index-pattern used to query the results
+- from: Start date
+- to: End date
+- (optional) query: filter
+
+The response includes:
+
+- count: number - the count of the hosts monitored with the system integration 
+- type: string - the type of the asset **(currently only 'host' is supported)**
+
+### Examples:
+
+Request
+
+```bash
+curl --location -u elastic:changeme 'http://0.0.0.0:5601/ftw/api/infra/host/count' \
+--header 'kbn-xsrf: xxxx' \
+--header 'Content-Type: application/json' \
+--data '{
+   "query": {
+      "bool": {
+         "must": [],
+         "filter": [],
+         "should": [],
+         "must_not": []
+      }
+   },
+   "from": "2024-07-23T11:34:11.640Z",
+   "to": "2024-07-23T11:49:11.640Z",
+   "sourceId": "default"
+}'
+```
+
+Response
+
+```json
+{"type":"host","count":22}
+```

x-pack/plugins/observability_solution/infra/server/routes/infra/index.ts

@@ -8,18 +8,27 @@
 import Boom from '@hapi/boom';
 import { createRouteValidationFunction } from '@kbn/io-ts-utils';
 
+import type { BoolQuery } from '@kbn/es-query';
 import {
+  type GetInfraMetricsRequestBodyPayload,
   GetInfraMetricsRequestBodyPayloadRT,
-  GetInfraMetricsRequestBodyPayload,
   GetInfraMetricsResponsePayloadRT,
 } from '../../../common/http_api/infra';
-import { InfraBackendLibs } from '../../lib/infra_types';
+import {
+  type GetInfraAssetCountRequestBodyPayload,
+  type GetInfraAssetCountRequestParamsPayload,
+  GetInfraAssetCountRequestBodyPayloadRT,
+  GetInfraAssetCountResponsePayloadRT,
+  GetInfraAssetCountRequestParamsPayloadRT,
+} from '../../../common/http_api/asset_count_api';
+import type { InfraBackendLibs } from '../../lib/infra_types';
 import { getInfraAlertsClient } from '../../lib/helpers/get_infra_alerts_client';
 import { getHosts } from './lib/host/get_hosts';
+import { getHostsCount } from './lib/host/get_hosts_count';
 import { getInfraMetricsClient } from '../../lib/helpers/get_infra_metrics_client';
 
-export const initInfraMetricsRoute = (libs: InfraBackendLibs) => {
-  const validateBody = createRouteValidationFunction(GetInfraMetricsRequestBodyPayloadRT);
+export const initInfraAssetRoutes = (libs: InfraBackendLibs) => {
+  const validateMetricsBody = createRouteValidationFunction(GetInfraMetricsRequestBodyPayloadRT);
 
   const { framework } = libs;
 
@@ -28,7 +37,7 @@ export const initInfraMetricsRoute = (libs: InfraBackendLibs) => {
       method: 'post',
       path: '/api/metrics/infra',
       validate: {
-        body: validateBody,
+        body: validateMetricsBody,
       },
     },
     async (requestContext, request, response) => {
@@ -74,4 +83,64 @@ export const initInfraMetricsRoute = (libs: InfraBackendLibs) => {
       }
     }
   );
+
+  const validateCountBody = createRouteValidationFunction(GetInfraAssetCountRequestBodyPayloadRT);
+  const validateCountParams = createRouteValidationFunction(
+    GetInfraAssetCountRequestParamsPayloadRT
+  );
+
+  framework.registerRoute(
+    {
+      method: 'post',
+      path: '/api/infra/{assetType}/count',
+      validate: {
+        body: validateCountBody,
+        params: validateCountParams,
+      },
+    },
+    async (requestContext, request, response) => {
+      const body: GetInfraAssetCountRequestBodyPayload = request.body;
+      const params: GetInfraAssetCountRequestParamsPayload = request.params;
+      const { assetType } = params;
+      const { query, from, to, sourceId } = body;
+
+      try {
+        const infraMetricsClient = await getInfraMetricsClient({
+          framework,
+          request,
+          infraSources: libs.sources,
+          requestContext,
+          sourceId,
+        });
+
+        const assetCount = await getHostsCount({
+          infraMetricsClient,
+          query: (query?.bool as BoolQuery) ?? undefined,
+          from,
+          to,
+        });
+
+        return response.ok({
+          body: GetInfraAssetCountResponsePayloadRT.encode({
+            assetType,
+            count: assetCount,
+          }),
+        });
+      } catch (err) {
+        if (Boom.isBoom(err)) {
+          return response.customError({
+            statusCode: err.output.statusCode,
+            body: { message: err.output.payload.message },
+          });
+        }
+
+        return response.customError({
+          statusCode: err.statusCode ?? 500,
+          body: {
+            message: err.message ?? 'An unexpected error occurred',
+          },
+        });
+      }
+    }
+  );
 };

x-pack/plugins/observability_solution/infra/server/routes/infra/lib/host/get_hosts_count.ts

@@ -0,0 +1,64 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { rangeQuery, termQuery } from '@kbn/observability-plugin/server';
+import { BoolQuery } from '@kbn/es-query';
+import { InfraMetricsClient } from '../../../../lib/helpers/get_infra_metrics_client';
+import { HOST_NAME_FIELD, EVENT_MODULE, METRICSET_MODULE } from '../../../../../common/constants';
+
+export async function getHostsCount({
+  infraMetricsClient,
+  query,
+  from,
+  to,
+}: {
+  infraMetricsClient: InfraMetricsClient;
+  query?: BoolQuery;
+  from: string;
+  to: string;
+}) {
+  const queryFilter = query?.filter ?? [];
+  const queryBool = query ?? {};
+
+  const params = {
+    allow_no_indices: true,
+    ignore_unavailable: true,
+    body: {
+      size: 0,
+      track_total_hits: false,
+      query: {
+        bool: {
+          ...queryBool,
+          filter: [
+            ...queryFilter,
+            ...rangeQuery(new Date(from).getTime(), new Date(to).getTime()),
+            {
+              bool: {
+                should: [
+                  ...termQuery(EVENT_MODULE, 'system'),
+                  ...termQuery(METRICSET_MODULE, 'system'),
+                ],
+                minimum_should_match: 1,
+              },
+            },
+          ],
+        },
+      },
+      aggs: {
+        count: {
+          cardinality: {
+            field: HOST_NAME_FIELD,
+          },
+        },
+      },
+    },
+  };
+
+  const result = await infraMetricsClient.search(params);
+
+  return result.aggregations?.count.value ?? 0;
+}

x-pack/test_serverless/api_integration/test_suites/observability/infra/asset_count.ts

@@ -0,0 +1,92 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import type {
+  GetInfraAssetCountRequestBodyPayload,
+  GetInfraAssetCountResponsePayload,
+  GetInfraAssetCountRequestParamsPayload,
+} from '@kbn/infra-plugin/common/http_api';
+import type { RoleCredentials } from '../../../../shared/services';
+import type { FtrProviderContext } from '../../../ftr_provider_context';
+
+import { DATES, ARCHIVE_NAME } from './constants';
+
+const timeRange = {
+  from: DATES.serverlessTestingHostDateString.min,
+  to: DATES.serverlessTestingHostDateString.max,
+};
+
+export default function ({ getService }: FtrProviderContext) {
+  const esArchiver = getService('esArchiver');
+  const supertestWithoutAuth = getService('supertestWithoutAuth');
+  const svlUserManager = getService('svlUserManager');
+  const svlCommonApi = getService('svlCommonApi');
+
+  const fetchHostsCount = async ({
+    params,
+    body,
+    roleAuthc,
+  }: {
+    params: GetInfraAssetCountRequestParamsPayload;
+    body: GetInfraAssetCountRequestBodyPayload;
+    roleAuthc: RoleCredentials;
+  }): Promise<GetInfraAssetCountResponsePayload | undefined> => {
+    const { assetType } = params;
+    const response = await supertestWithoutAuth
+      .post(`/api/infra/${assetType}/count`)
+      .set(svlCommonApi.getInternalRequestHeader())
+      .set(roleAuthc.apiKeyHeader)
+      .send(body)
+      .expect(200);
+    return response.body;
+  };
+
+  describe('API /api/infra/{assetType}/count', () => {
+    let roleAuthc: RoleCredentials;
+    describe('works', () => {
+      describe('with host', () => {
+        before(async () => {
+          roleAuthc = await svlUserManager.createM2mApiKeyWithRoleScope('admin');
+          return esArchiver.load(ARCHIVE_NAME);
+        });
+        after(async () => {
+          await svlUserManager.invalidateM2mApiKeyWithRoleScope(roleAuthc);
+          return esArchiver.unload(ARCHIVE_NAME);
+        });
+
+        it('received data', async () => {
+          const infraHosts = await fetchHostsCount({
+            params: { assetType: 'host' },
+            body: {
+              query: {
+                bool: {
+                  must: [],
+                  filter: [],
+                  should: [],
+                  must_not: [],
+                },
+              },
+              from: timeRange.from,
+              to: timeRange.to,
+              sourceId: 'default',
+            },
+            roleAuthc,
+          });
+
+          if (infraHosts) {
+            const { count, assetType } = infraHosts;
+            expect(count).to.equal(1);
+            expect(assetType).to.be('host');
+          } else {
+            throw new Error('Hosts count response should not be empty');
+          }
+        });
+      });
+    });
+  });
+}

x-pack/test_serverless/api_integration/test_suites/observability/infra/index.ts

@@ -17,5 +17,6 @@ export default function ({ loadTestFile }: FtrProviderContext) {
     loadTestFile(require.resolve('./snapshot'));
     loadTestFile(require.resolve('./processes'));
     loadTestFile(require.resolve('./infra'));
+    loadTestFile(require.resolve('./asset_count'));
   });
 }