[Fleet] Add agent incoming data endpoint and presentational component (#127177)

* [Fleet] Create endpoint to check if agent has incoming data

* Document new endpoint

* Improvements to component

* Update endpoint schema

* Remove button for now

* Address review comments

* Add dynamic button functionality

* Add option to hide button and improve query

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

x-pack/plugins/fleet/common/constants/routes.ts

@@ -105,6 +105,7 @@ export const AGENT_API_ROUTES = {
   REASSIGN_PATTERN: `${API_ROOT}/agents/{agentId}/reassign`,
   BULK_REASSIGN_PATTERN: `${API_ROOT}/agents/bulk_reassign`,
   STATUS_PATTERN: `${API_ROOT}/agent_status`,
+  DATA_PATTERN: `${API_ROOT}/agent_status/data`,
   // deprecated since 8.0
   STATUS_PATTERN_DEPRECATED: `${API_ROOT}/agent-status`,
   UPGRADE_PATTERN: `${API_ROOT}/agents/{agentId}/upgrade`,

x-pack/plugins/fleet/common/openapi/bundled.json

@@ -1119,6 +1119,51 @@
         ]
       }
     },
+    "/agent_status/data": {
+      "get": {
+        "summary": "Agents - Get incoming data",
+        "tags": [],
+        "responses": {
+          "200": {
+            "description": "OK",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "type": "object",
+                  "properties": {
+                    "items": {
+                      "type": "array",
+                      "items": {
+                        "type": "object",
+                        "additionalProperties": {
+                          "type": "object",
+                          "properties": {
+                            "data": {
+                              "type": "boolean"
+                            }
+                          }
+                        }
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          }
+        },
+        "operationId": "get-agent-data",
+        "parameters": [
+          {
+            "schema": {
+              "type": "array"
+            },
+            "name": "agentsIds",
+            "in": "query",
+            "required": true
+          }
+        ]
+      }
+    },
     "/agents": {
       "get": {
         "summary": "Agents - List",

x-pack/plugins/fleet/common/openapi/bundled.yaml

@@ -686,6 +686,34 @@ paths:
           name: kuery
           in: query
           required: false
+  /agent_status/data:
+    get:
+      summary: Agents - Get incoming data
+      tags: []
+      responses:
+        '200':
+          description: OK
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  items:
+                    type: array
+                    items:
+                      type: object
+                      additionalProperties:
+                        type: object
+                        properties:
+                          data:
+                            type: boolean
+      operationId: get-agent-data
+      parameters:
+        - schema:
+            type: array
+          name: agentsId
+          in: query
+          required: true
   /agents:
     get:
       summary: Agents - List

x-pack/plugins/fleet/common/openapi/entrypoint.yaml

@@ -49,6 +49,8 @@ paths:
     $ref: paths/agent_status_deprecated.yaml
   /agent_status:
     $ref: paths/agent_status.yaml
+  /agent_status/data:
+    $ref: paths/agent_status@data.yaml
   /agents:
     $ref: paths/agents.yaml
   /agents/bulk_upgrade:

x-pack/plugins/fleet/common/openapi/paths/agent_status@data.yaml

@@ -0,0 +1,27 @@
+get:
+  summary: Agents - Get incoming data
+  tags: []
+  responses:
+    '200':
+      description: OK
+      content:
+        application/json:
+          schema:
+            type: object
+            properties:
+              items:
+                type: array
+                items:
+                  type: object
+                  additionalProperties:
+                    type: object
+                    properties:
+                      data:
+                        type: boolean
+  operationId: get-agent-data
+  parameters:
+    - schema:
+        type: array
+      name: agentsId
+      in: query
+      required: true

x-pack/plugins/fleet/common/services/routes.ts

@@ -164,6 +164,7 @@ export const agentRouteService = {
   getBulkUpgradePath: () => AGENT_API_ROUTES.BULK_UPGRADE_PATTERN,
   getListPath: () => AGENT_API_ROUTES.LIST_PATTERN,
   getStatusPath: () => AGENT_API_ROUTES.STATUS_PATTERN,
+  getIncomingDataPath: () => AGENT_API_ROUTES.DATA_PATTERN,
   getCreateActionPath: (agentId: string) =>
     AGENT_API_ROUTES.ACTIONS_PATTERN.replace('{agentId}', agentId),
 };

x-pack/plugins/fleet/common/types/rest_spec/agent.ts

@@ -161,3 +161,16 @@ export interface GetAgentStatusResponse {
     updating: number;
   };
 }
+
+export interface GetAgentIncomingDataRequest {
+  query: {
+    agentsIds: string[];
+  };
+}
+
+export interface IncomingDataList {
+  [key: string]: { data: boolean };
+}
+export interface GetAgentIncomingDataResponse {
+  items: IncomingDataList[];
+}

x-pack/plugins/fleet/public/components/agent_enrollment_flyout/confirm_incoming_data.tsx

@@ -0,0 +1,75 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React from 'react';
+import { EuiCallOut, EuiText, EuiSpacer, EuiButton } from '@elastic/eui';
+import { i18n } from '@kbn/i18n';
+
+import type { InstalledIntegrationPolicy } from '../../hooks';
+import { useGetAgentIncomingData } from '../../hooks';
+interface Props {
+  agentsIds: string[];
+  installedPolicy?: InstalledIntegrationPolicy;
+}
+
+export const ConfirmIncomingData: React.FunctionComponent<Props> = ({
+  agentsIds,
+  installedPolicy,
+}) => {
+  const { enrolledAgents, numAgentsWithData, isLoading, linkButton } = useGetAgentIncomingData(
+    agentsIds,
+    installedPolicy
+  );
+
+  return (
+    <>
+      {isLoading ? (
+        <EuiText size="s">
+          {i18n.translate('xpack.fleet.confirmIncomingData.loading', {
+            defaultMessage:
+              'It may take a few minutes for data to arrive in Elasticsearch. If the system is not generating data, it may help to generate some to ensure data is being collected correctly. If you’re having trouble, see our troubleshooting guide. You may close this dialog and check later by viewing our integration assets.',
+          })}
+        </EuiText>
+      ) : (
+        <>
+          <EuiCallOut
+            data-test-subj="IncomingDataConfirmedCallOut"
+            title={i18n.translate('xpack.fleet.confirmIncomingData.title', {
+              defaultMessage:
+                'Incoming data received from {numAgentsWithData} of {enrolledAgents} recently enrolled { enrolledAgents, plural, one {agent} other {agents}}.',
+              values: {
+                numAgentsWithData,
+                enrolledAgents,
+              },
+            })}
+            color="success"
+            iconType="check"
+          />
+          <EuiSpacer size="m" />
+          <EuiText size="s">
+            {i18n.translate('xpack.fleet.confirmIncomingData.subtitle', {
+              defaultMessage: 'Your agent is enrolled successfully and your data is received.',
+            })}
+          </EuiText>
+        </>
+      )}
+
+      <EuiSpacer size="m" />
+      {installedPolicy && (
+        <EuiButton
+          href={linkButton.href}
+          isDisabled={isLoading}
+          color="primary"
+          fill
+          data-test-subj="IncomingDataConfirmedButton"
+        >
+          {linkButton.text}
+        </EuiButton>
+      )}
+    </>
+  );
+};

x-pack/plugins/fleet/public/hooks/index.ts

@@ -27,3 +27,4 @@ export * from './use_platform';
 export * from './use_agent_policy_refresh';
 export * from './use_package_installations';
 export * from './use_agent_enrollment_flyout_data';
+export * from './use_get_agent_incoming_data';

x-pack/plugins/fleet/public/hooks/use_get_agent_incoming_data.tsx

@@ -0,0 +1,78 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { useEffect, useState, useMemo } from 'react';
+import { i18n } from '@kbn/i18n';
+
+import type { IncomingDataList } from '../../common/types/rest_spec/agent';
+
+import { sendGetAgentIncomingData, useLink } from './index';
+
+export interface InstalledIntegrationPolicy {
+  name: string;
+  version: string;
+}
+
+export const useGetAgentIncomingData = (
+  agentsIds: string[],
+  installedPolicy?: InstalledIntegrationPolicy
+) => {
+  const [isLoading, setIsLoading] = useState<boolean>(true);
+  const [incomingData, setIncomingData] = useState<IncomingDataList[]>([]);
+
+  useEffect(() => {
+    const getIncomingData = async () => {
+      const { data } = await sendGetAgentIncomingData({ agentsIds });
+      if (data?.items) {
+        setIncomingData(data?.items);
+        setIsLoading(false);
+      }
+    };
+    if (agentsIds) {
+      getIncomingData();
+    }
+  }, [agentsIds]);
+
+  const enrolledAgents = useMemo(() => incomingData.length, [incomingData.length]);
+  const numAgentsWithData = useMemo(
+    () =>
+      incomingData.reduce((acc, curr) => {
+        const agentData = Object.values(curr)[0];
+        return !!agentData.data ? acc + 1 : acc;
+      }, 0),
+    [incomingData]
+  );
+  const { getAbsolutePath, getHref } = useLink();
+
+  let href;
+  let text;
+  if (!installedPolicy) {
+    href = '';
+    text = '';
+  }
+
+  if (installedPolicy?.name === 'apm') {
+    href = getAbsolutePath('/app/home#/tutorial/apm');
+    text = i18n.translate('xpack.fleet.confirmIncomingData.installApmAgentButtonText', {
+      defaultMessage: 'Install APM Agent',
+    });
+  } else {
+    href = getHref('integration_details_assets', {
+      pkgkey: `${installedPolicy?.name}-${installedPolicy?.version}`,
+    });
+    text = i18n.translate('xpack.fleet.confirmIncomingData.viewDataAssetsButtonText', {
+      defaultMessage: 'View assets',
+    });
+  }
+  const linkButton = { href, text };
+
+  return {
+    enrolledAgents,
+    numAgentsWithData,
+    isLoading,
+    linkButton,
+  };
+};

x-pack/plugins/fleet/public/hooks/use_request/agents.ts

@@ -21,6 +21,8 @@ import type {
   GetAgentsResponse,
   GetAgentStatusRequest,
   GetAgentStatusResponse,
+  GetAgentIncomingDataRequest,
+  GetAgentIncomingDataResponse,
   PostAgentUpgradeRequest,
   PostBulkAgentUpgradeRequest,
   PostAgentUpgradeResponse,
@@ -68,6 +70,13 @@ export function useGetAgentStatus(query: GetAgentStatusRequest['query'], options
     ...options,
   });
 }
+export function sendGetAgentIncomingData(query: GetAgentIncomingDataRequest['query']) {
+  return sendRequest<GetAgentIncomingDataResponse>({
+    method: 'get',
+    path: agentRouteService.getIncomingDataPath(),
+    query,
+  });
+}
 
 export function sendGetAgentStatus(
   query: GetAgentStatusRequest['query'],

x-pack/plugins/fleet/public/types/index.ts

@@ -61,6 +61,9 @@ export type {
   PostBulkAgentUpgradeResponse,
   GetAgentStatusRequest,
   GetAgentStatusResponse,
+  GetAgentIncomingDataRequest,
+  IncomingDataList,
+  GetAgentIncomingDataResponse,
   PutAgentReassignRequest,
   PutAgentReassignResponse,
   PostBulkAgentReassignRequest,

x-pack/plugins/fleet/server/routes/agent/handlers.ts

@@ -21,6 +21,7 @@ import type {
   UpdateAgentRequestSchema,
   DeleteAgentRequestSchema,
   GetAgentStatusRequestSchema,
+  GetAgentDataRequestSchema,
   PutAgentReassignRequestSchema,
   PostBulkAgentReassignRequestSchema,
 } from '../../types';
@@ -218,3 +219,29 @@ export const getAgentStatusForAgentPolicyHandler: RequestHandler<
     return defaultIngestErrorHandler({ error, response });
   }
 };
+
+export const getAgentDataHandler: RequestHandler<
+  undefined,
+  TypeOf<typeof GetAgentDataRequestSchema.query>
+> = async (context, request, response) => {
+  const esClient = context.core.elasticsearch.client.asCurrentUser;
+  try {
+    let items;
+
+    if (isStringArray(request.query.agentsIds)) {
+      items = await AgentService.getIncomingDataByAgentsId(esClient, request.query.agentsIds);
+    } else {
+      items = await AgentService.getIncomingDataByAgentsId(esClient, [request.query.agentsIds]);
+    }
+
+    const body = { items };
+
+    return response.ok({ body });
+  } catch (error) {
+    return defaultIngestErrorHandler({ error, response });
+  }
+};
+
+function isStringArray(arr: unknown | string[]): arr is string[] {
+  return Array.isArray(arr) && arr.every((p) => typeof p === 'string');
+}

x-pack/plugins/fleet/server/routes/agent/index.ts

@@ -14,6 +14,7 @@ import {
   PostAgentUnenrollRequestSchema,
   PostBulkAgentUnenrollRequestSchema,
   GetAgentStatusRequestSchema,
+  GetAgentDataRequestSchema,
   PostNewAgentActionRequestSchema,
   PutAgentReassignRequestSchema,
   PostBulkAgentReassignRequestSchema,
@@ -32,6 +33,7 @@ import {
   getAgentStatusForAgentPolicyHandler,
   putAgentsReassignHandler,
   postBulkAgentsReassignHandler,
+  getAgentDataHandler,
 } from './handlers';
 import { postNewAgentActionHandlerBuilder } from './actions_handlers';
 import { postAgentUnenrollHandler, postBulkAgentsUnenrollHandler } from './unenroll_handler';
@@ -141,6 +143,17 @@ export const registerAPIRoutes = (router: FleetAuthzRouter, config: FleetConfigT
     },
     getAgentStatusForAgentPolicyHandler
   );
+  // Agent data
+  router.get(
+    {
+      path: AGENT_API_ROUTES.DATA_PATTERN,
+      validate: GetAgentDataRequestSchema,
+      fleetAuthz: {
+        fleet: { all: true },
+      },
+    },
+    getAgentDataHandler
+  );
 
   // upgrade agent
   router.post(

x-pack/plugins/fleet/server/services/agents/status.ts

@@ -14,9 +14,12 @@ import { fromKueryExpression } from '@kbn/es-query';
 import { AGENTS_PREFIX } from '../../constants';
 import type { AgentStatus } from '../../types';
 import { AgentStatusKueryHelper } from '../../../common/services';
+import { FleetUnauthorizedError } from '../../errors';
 
 import { getAgentById, getAgentsByKuery, removeSOAttributes } from './crud';
 
+const DATA_STREAM_INDEX_PATTERN = 'logs-*-*,metrics-*-*,traces-*-*,synthetics-*-*';
+
 export async function getAgentStatusById(
   esClient: ElasticsearchClient,
   agentId: string
@@ -92,3 +95,77 @@ export async function getAgentStatusForAgentPolicy(
     events: 0,
   };
 }
+export async function getIncomingDataByAgentsId(
+  esClient: ElasticsearchClient,
+  agentsIds: string[]
+) {
+  try {
+    const { has_all_requested: hasAllPrivileges } = await esClient.security.hasPrivileges({
+      body: {
+        index: [
+          {
+            names: [DATA_STREAM_INDEX_PATTERN],
+            privileges: ['read'],
+          },
+        ],
+      },
+    });
+    if (!hasAllPrivileges) {
+      throw new FleetUnauthorizedError('Missing permissions to read data streams indices');
+    }
+
+    const searchResult = await esClient.search({
+      index: DATA_STREAM_INDEX_PATTERN,
+      allow_partial_search_results: true,
+      _source: false,
+      timeout: '5s',
+      size: 0,
+      body: {
+        query: {
+          bool: {
+            filter: [
+              {
+                terms: {
+                  'agent.id': agentsIds,
+                },
+              },
+              {
+                range: {
+                  '@timestamp': {
+                    gte: 'now-5m',
+                    lte: 'now',
+                  },
+                },
+              },
+            ],
+          },
+        },
+        aggs: {
+          agent_ids: {
+            terms: {
+              field: 'agent.id',
+              size: 10,
+            },
+          },
+        },
+      },
+    });
+
+    if (!searchResult.aggregations?.agent_ids) {
+      return agentsIds.map((id) => {
+        return { [id]: { data: false } };
+      });
+    }
+
+    // @ts-expect-error aggregation type is not specified
+    const agentIdsWithData: string[] = searchResult.aggregations.agent_ids.buckets.map(
+      (bucket: any) => bucket.key as string
+    );
+
+    return agentsIds.map((id) =>
+      agentIdsWithData.includes(id) ? { [id]: { data: true } } : { [id]: { data: false } }
+    );
+  } catch (e) {
+    throw new Error(e);
+  }
+}

x-pack/plugins/fleet/server/types/rest_spec/agent.ts

@@ -111,3 +111,9 @@ export const GetAgentStatusRequestSchema = {
     kuery: schema.maybe(schema.string()),
   }),
 };
+
+export const GetAgentDataRequestSchema = {
+  query: schema.object({
+    agentsIds: schema.oneOf([schema.arrayOf(schema.string()), schema.string()]),
+  }),
+};