[Fleet] Show remote es output error state on UI (#172181)

## Summary

Relates https://github.com/elastic/fleet-server/issues/3116

Relates https://github.com/elastic/kibana/issues/104986

Reading latest output health state from
`logs-fleet_server.output_health-default` data stream by output id, and
displaying error state on UI - Edit Output flyout.

Steps to verify:
- enable feature flag `remoteESOutput`
- add `remote_elasticsearch` output, can be a non-existent host for this
test
- add the output as monitoring output of an agent policy
- run fleet-server with the changes
[here](https://github.com/elastic/fleet-server/issues/3116)
- enroll an agent
- wait until fleet-server starts reporting degraded state in the output
health data stream
- open edit output flyout on UI and verify that the error state is
visible
- when the connection is back again (update host to a valid one, or
remote es was temporarily down), the error state goes away

<img width="568" alt="image"
src="46d0cf95-6aa4-4f7c-8608-4362ada4eb6c">

The UI was suggested in the design doc:
https://docs.google.com/document/d/19D0bX7oURf0yms4qemfqDyisw_IYB-OVw4oU-t4lf18/edit#bookmark=id.595r8l91kaq8

### Notes/suggestions:

- We might want to add the output state to the output list as well
(maybe as badges like agent health?) as it's not too visible in the
flyout (have to scroll down).
- Also the error state will be reported earliest when an agent is
enrolled and fleet-server can't create api key, so not immediately when
the output is added. It would be good to show the time of the last state
(e.g. how we display on agents last checkin x minutes ago)
- I think it would be beneficial to display the healthy state too.

Added badges to output list:
<img width="1233" alt="image"
src="07ff06ec-b778-4420-975b-b46a0a18c7cc">

Added healthy state UI to Edit output:
<img width="627" alt="image"
src="4222d849-c957-41d7-9606-b58493264115">


### Checklist

Delete any items that are not applicable to this PR.

- [x] Any text added follows [EUI's writing
guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses
sentence case text and includes [i18n
support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md)
- [ ] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

x-pack/plugins/fleet/common/constants/output.ts

@@ -119,3 +119,5 @@ export const kafkaSupportedVersions = [
   '2.5.1',
   '2.6.0',
 ];
+
+export const OUTPUT_HEALTH_DATA_STREAM = 'logs-fleet_server.output_health-default';

x-pack/plugins/fleet/common/constants/routes.ts

@@ -94,6 +94,7 @@ export const OUTPUT_API_ROUTES = {
   UPDATE_PATTERN: `${API_ROOT}/outputs/{outputId}`,
   DELETE_PATTERN: `${API_ROOT}/outputs/{outputId}`,
   CREATE_PATTERN: `${API_ROOT}/outputs`,
+  GET_OUTPUT_HEALTH_PATTERN: `${API_ROOT}/outputs/{outputId}/health`,
   LOGSTASH_API_KEY_PATTERN: `${API_ROOT}/logstash_api_keys`,
 };
 

x-pack/plugins/fleet/common/openapi/bundled.json

@@ -4680,6 +4680,54 @@
         ]
       }
     },
+    "/outputs/{outputId}/health": {
+      "get": {
+        "summary": "Get latest output health",
+        "tags": [
+          "Outputs"
+        ],
+        "responses": {
+          "200": {
+            "description": "OK",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "type": "object",
+                  "properties": {
+                    "state": {
+                      "type": "string",
+                      "description": "state of output, HEALTHY or DEGRADED"
+                    },
+                    "message": {
+                      "type": "string",
+                      "description": "long message if unhealthy"
+                    },
+                    "timestamp": {
+                      "type": "string",
+                      "description": "timestamp of reported state"
+                    }
+                  }
+                }
+              }
+            }
+          },
+          "400": {
+            "$ref": "#/components/responses/error"
+          }
+        },
+        "operationId": "get-output-health"
+      },
+      "parameters": [
+        {
+          "schema": {
+            "type": "string"
+          },
+          "name": "outputId",
+          "in": "path",
+          "required": true
+        }
+      ]
+    },
     "/logstash_api_keys": {
       "post": {
         "summary": "Generate Logstash API key",

x-pack/plugins/fleet/common/openapi/bundled.yaml

@@ -2916,6 +2916,37 @@ paths:
           $ref: '#/components/responses/error'
       parameters:
         - $ref: '#/components/parameters/kbn_xsrf'
+  /outputs/{outputId}/health:
+    get:
+      summary: Get latest output health
+      tags:
+        - Outputs
+      responses:
+        '200':
+          description: OK
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  state:
+                    type: string
+                    description: state of output, HEALTHY or DEGRADED
+                  message:
+                    type: string
+                    description: long message if unhealthy
+                  timestamp:
+                    type: string
+                    description: timestamp of reported state
+        '400':
+          $ref: '#/components/responses/error'
+      operationId: get-output-health
+    parameters:
+      - schema:
+          type: string
+        name: outputId
+        in: path
+        required: true
   /logstash_api_keys:
     post:
       summary: Generate Logstash API key

x-pack/plugins/fleet/common/openapi/entrypoint.yaml

@@ -142,6 +142,8 @@ paths:
     $ref: paths/outputs.yaml
   /outputs/{outputId}:
     $ref: paths/outputs@{output_id}.yaml
+  /outputs/{outputId}/health:
+    $ref: paths/output_health@{output_id}.yaml
   /logstash_api_keys:
     $ref: paths/logstash_api_keys.yaml
 

x-pack/plugins/fleet/common/openapi/paths/output_health@{output_id}.yaml

@@ -0,0 +1,31 @@
+get:
+  summary: Get latest output health
+  tags:
+    - Outputs
+  responses:
+    '200':
+      description: OK
+      content:
+        application/json:
+          schema:
+            type: object
+            properties:
+              state:
+                type: string
+                description: state of output, HEALTHY or DEGRADED
+              message:
+                type: string
+                description: long message if unhealthy
+              timestamp:
+                type: string
+                description: timestamp of reported state
+    '400':
+      $ref: ../components/responses/error.yaml
+  operationId: get-output-health
+parameters:
+  - schema:
+      type: string
+    name: outputId
+    in: path
+    required: true
+

x-pack/plugins/fleet/common/services/routes.ts

@@ -248,6 +248,8 @@ export const outputRoutesService = {
     OUTPUT_API_ROUTES.DELETE_PATTERN.replace('{outputId}', outputId),
   getCreatePath: () => OUTPUT_API_ROUTES.CREATE_PATTERN,
   getCreateLogstashApiKeyPath: () => OUTPUT_API_ROUTES.LOGSTASH_API_KEY_PATTERN,
+  getOutputHealthPath: (outputId: string) =>
+    OUTPUT_API_ROUTES.GET_OUTPUT_HEALTH_PATTERN.replace('{outputId}', outputId),
 };
 
 export const fleetProxiesRoutesService = {

x-pack/plugins/fleet/common/types/rest_spec/output.ts

@@ -43,3 +43,9 @@ export type GetOutputsResponse = ListResult<Output>;
 export interface PostLogstashApiKeyResponse {
   api_key: string;
 }
+
+export interface GetOutputHealthResponse {
+  state: string;
+  message: string;
+  timestamp: string;
+}

x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/index.tsx

@@ -55,6 +55,7 @@ import { useOutputForm } from './use_output_form';
 import { EncryptionKeyRequiredCallout } from './encryption_key_required_callout';
 import { AdvancedOptionsSection } from './advanced_options_section';
 import { OutputFormRemoteEsSection } from './output_form_remote_es';
+import { OutputHealth } from './output_health';
 
 export interface EditOutputFlyoutProps {
   output?: Output;
@@ -576,6 +577,9 @@ export const EditOutputFlyout: React.FunctionComponent<EditOutputFlyoutProps> =
           <EuiSpacer size="l" />
           <AdvancedOptionsSection enabled={form.isShipperEnabled} inputs={inputs} />
         </EuiForm>
+        {output?.id && output.type === 'remote_elasticsearch' ? (
+          <OutputHealth output={output} />
+        ) : null}
       </EuiFlyoutBody>
       <EuiFlyoutFooter>
         <EuiFlexGroup justifyContent="spaceBetween">

x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/output_health.test.tsx

@@ -0,0 +1,143 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React from 'react';
+import { waitFor } from '@testing-library/react';
+
+import type { Output } from '../../../../types';
+import { createFleetTestRendererMock } from '../../../../../../mock';
+
+import { sendGetOutputHealth, useStartServices } from '../../../../hooks';
+
+import { OutputHealth } from './output_health';
+
+jest.mock('../../../../hooks', () => {
+  return {
+    ...jest.requireActual('../../../../hooks'),
+    useStartServices: jest.fn(),
+    sendGetOutputHealth: jest.fn(),
+  };
+});
+
+jest.mock('@elastic/eui', () => {
+  return {
+    ...jest.requireActual('@elastic/eui'),
+    EuiToolTip: (props: any) => (
+      <div data-test-subj="outputHealthBadgeTooltip" data-tooltip-content={props.content}>
+        {props.children}
+      </div>
+    ),
+  };
+});
+
+const mockUseStartServices = useStartServices as jest.Mock;
+
+const mockSendGetOutputHealth = sendGetOutputHealth as jest.Mock;
+
+describe('OutputHealth', () => {
+  function render(output: Output, showBadge?: boolean) {
+    const renderer = createFleetTestRendererMock();
+
+    const utils = renderer.render(<OutputHealth output={output} showBadge={showBadge} />);
+
+    return { utils };
+  }
+
+  const mockStartServices = () => {
+    mockUseStartServices.mockReturnValue({
+      notifications: { toasts: {} },
+    });
+  };
+
+  beforeEach(() => {
+    mockStartServices();
+  });
+
+  it('should render output health component when degraded', async () => {
+    mockSendGetOutputHealth.mockResolvedValue({
+      data: { state: 'DEGRADED', message: 'connection error', timestamp: '2023-11-30T14:25:31Z' },
+    });
+    const { utils } = render({
+      type: 'remote_elasticsearch',
+      id: 'remote',
+      name: 'Remote ES',
+      hosts: ['https://remote-es:443'],
+    } as Output);
+
+    expect(mockSendGetOutputHealth).toHaveBeenCalled();
+
+    await waitFor(async () => {
+      expect(utils.getByTestId('outputHealthDegradedCallout').textContent).toContain(
+        'Unable to connect to "Remote ES" at https://remote-es:443. Please check the details are correct.'
+      );
+    });
+  });
+
+  it('should render output health component when healthy', async () => {
+    mockSendGetOutputHealth.mockResolvedValue({
+      data: { state: 'HEALTHY', message: '', timestamp: '2023-11-30T14:25:31Z' },
+    });
+    const { utils } = render({
+      type: 'remote_elasticsearch',
+      id: 'remote',
+      name: 'Remote ES',
+      hosts: ['https://remote-es:443'],
+    } as Output);
+
+    expect(mockSendGetOutputHealth).toHaveBeenCalled();
+
+    await waitFor(async () => {
+      expect(utils.getByTestId('outputHealthHealthyCallout').textContent).toContain(
+        'Connection with remote output established.'
+      );
+    });
+  });
+
+  it('should render output health badge when degraded', async () => {
+    mockSendGetOutputHealth.mockResolvedValue({
+      data: { state: 'DEGRADED', message: 'connection error', timestamp: '2023-11-30T14:25:31Z' },
+    });
+    const { utils } = render(
+      {
+        type: 'remote_elasticsearch',
+        id: 'remote',
+        name: 'Remote ES',
+        hosts: ['https://remote-es:443'],
+      } as Output,
+      true
+    );
+
+    expect(mockSendGetOutputHealth).toHaveBeenCalled();
+
+    await waitFor(async () => {
+      expect(utils.getByTestId('outputHealthDegradedBadge')).not.toBeNull();
+      expect(utils.getByTestId('outputHealthBadgeTooltip')).not.toBeNull();
+    });
+  });
+
+  it('should render output health badge when healthy', async () => {
+    mockSendGetOutputHealth.mockResolvedValue({
+      data: { state: 'HEALTHY', message: '', timestamp: '2023-11-30T14:25:31Z' },
+    });
+    const { utils } = render(
+      {
+        type: 'remote_elasticsearch',
+        id: 'remote',
+        name: 'Remote ES',
+        hosts: ['https://remote-es:443'],
+      } as Output,
+      true
+    );
+
+    expect(mockSendGetOutputHealth).toHaveBeenCalled();
+
+    await waitFor(async () => {
+      expect(utils.getByTestId('outputHealthHealthyBadge')).not.toBeNull();
+      expect(utils.getByTestId('outputHealthBadgeTooltip')).not.toBeNull();
+    });
+  });
+});

x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/output_health.tsx

@@ -0,0 +1,135 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { EuiBadge, EuiCallOut, EuiToolTip } from '@elastic/eui';
+
+import { i18n } from '@kbn/i18n';
+import { FormattedMessage, FormattedRelative } from '@kbn/i18n-react';
+import React, { useEffect, useState } from 'react';
+import { useQuery } from '@tanstack/react-query';
+
+import type { GetOutputHealthResponse } from '../../../../../../../common/types';
+
+import { sendGetOutputHealth, useStartServices } from '../../../../hooks';
+import type { Output } from '../../../../types';
+
+interface Props {
+  output: Output;
+  showBadge?: boolean;
+}
+const REFRESH_INTERVAL_MS = 10000;
+
+export const OutputHealth: React.FunctionComponent<Props> = ({ output, showBadge }) => {
+  const { notifications } = useStartServices();
+  const [outputHealth, setOutputHealth] = useState<GetOutputHealthResponse | null>();
+
+  const { data: outputHealthResponse } = useQuery(
+    ['outputHealth', output.id],
+    () => sendGetOutputHealth(output.id),
+    { refetchInterval: REFRESH_INTERVAL_MS }
+  );
+  useEffect(() => {
+    if (outputHealthResponse?.error) {
+      notifications.toasts.addError(outputHealthResponse?.error, {
+        title: i18n.translate('xpack.fleet.output.errorFetchingOutputHealth', {
+          defaultMessage: 'Error fetching output state',
+        }),
+      });
+    }
+    setOutputHealth(outputHealthResponse?.data);
+  }, [outputHealthResponse, notifications.toasts]);
+
+  const EditOutputStatus: { [status: string]: JSX.Element | null } = {
+    DEGRADED: (
+      <EuiCallOut
+        title="Error"
+        color="danger"
+        iconType="error"
+        data-test-subj="outputHealthDegradedCallout"
+      >
+        <p>
+          {i18n.translate('xpack.fleet.output.calloutText', {
+            defaultMessage: 'Unable to connect to "{name}" at {host}.',
+            values: {
+              name: output.name,
+              host: output.hosts?.join(',') ?? '',
+            },
+          })}
+        </p>{' '}
+        <p>
+          {i18n.translate('xpack.fleet.output.calloutPromptText', {
+            defaultMessage: 'Please check the details are correct.',
+          })}
+        </p>
+      </EuiCallOut>
+    ),
+    HEALTHY: (
+      <EuiCallOut
+        title="Healthy"
+        color="success"
+        iconType="check"
+        data-test-subj="outputHealthHealthyCallout"
+      >
+        <p>
+          {i18n.translate('xpack.fleet.output.successCalloutText', {
+            defaultMessage: 'Connection with remote output established.',
+          })}
+        </p>
+      </EuiCallOut>
+    ),
+  };
+
+  const OutputStatusBadge: { [status: string]: JSX.Element | null } = {
+    DEGRADED: (
+      <EuiBadge color="danger" data-test-subj="outputHealthDegradedBadge">
+        <FormattedMessage
+          id="xpack.fleet.outputHealth.degradedStatusText"
+          defaultMessage="Unhealthy"
+        />
+      </EuiBadge>
+    ),
+    HEALTHY: (
+      <EuiBadge color="success" data-test-subj="outputHealthHealthyBadge">
+        <FormattedMessage
+          id="xpack.fleet.outputHealth.healthyStatusText"
+          defaultMessage="Healthy"
+        />
+      </EuiBadge>
+    ),
+  };
+
+  const msLastTimestamp = new Date(outputHealth?.timestamp || 0).getTime();
+  const lastTimestampText = msLastTimestamp ? (
+    <>
+      <FormattedMessage
+        id="xpack.fleet.outputHealth.timestampTooltipText"
+        defaultMessage="Last reported {timestamp}"
+        values={{
+          timestamp: <FormattedRelative value={msLastTimestamp} />,
+        }}
+      />
+    </>
+  ) : null;
+
+  const outputBadge = (outputHealth?.state && OutputStatusBadge[outputHealth?.state]) || null;
+
+  return showBadge ? (
+    lastTimestampText && outputHealth?.state ? (
+      <EuiToolTip
+        position="top"
+        content={lastTimestampText}
+        data-test-subj="outputHealthBadgeTooltip"
+      >
+        <>{outputBadge} </>
+      </EuiToolTip>
+    ) : (
+      outputBadge
+    )
+  ) : (
+    (outputHealth?.state && EditOutputStatus[outputHealth.state]) || null
+  );
+};

x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/outputs_table/index.tsx

@@ -14,6 +14,8 @@ import { i18n } from '@kbn/i18n';
 import { useLink } from '../../../../hooks';
 import type { Output } from '../../../../types';
 
+import { OutputHealth } from '../edit_output_flyout/output_health';
+
 import { DefaultBadges } from './badges';
 
 export interface OutputsTableProps {
@@ -76,14 +78,14 @@ export const OutputsTable: React.FunctionComponent<OutputsTableProps> = ({
           </EuiFlexGroup>
         ),
         width: '288px',
-        name: i18n.translate('xpack.fleet.settings.outputsTable.nameColomnTitle', {
+        name: i18n.translate('xpack.fleet.settings.outputsTable.nameColumnTitle', {
           defaultMessage: 'Name',
         }),
       },
       {
         width: '172px',
         render: (output: Output) => displayOutputType(output.type),
-        name: i18n.translate('xpack.fleet.settings.outputsTable.typeColomnTitle', {
+        name: i18n.translate('xpack.fleet.settings.outputsTable.typeColumnTitle', {
           defaultMessage: 'Type',
         }),
       },
@@ -100,14 +102,24 @@ export const OutputsTable: React.FunctionComponent<OutputsTableProps> = ({
             ))}
           </FlexGroupWithMinWidth>
         ),
-        name: i18n.translate('xpack.fleet.settings.outputsTable.hostColomnTitle', {
+        name: i18n.translate('xpack.fleet.settings.outputsTable.hostColumnTitle', {
           defaultMessage: 'Hosts',
         }),
       },
+      {
+        render: (output: Output) => {
+          return output?.id && output.type === 'remote_elasticsearch' ? (
+            <OutputHealth output={output} showBadge={true} />
+          ) : null;
+        },
+        name: i18n.translate('xpack.fleet.settings.outputsTable.statusColumnTitle', {
+          defaultMessage: 'Status',
+        }),
+      },
       {
         render: (output: Output) => <DefaultBadges output={output} />,
         width: '200px',
-        name: i18n.translate('xpack.fleet.settings.outputSection.defaultColomnTitle', {
+        name: i18n.translate('xpack.fleet.settings.outputSection.defaultColumnTitle', {
           defaultMessage: 'Default',
         }),
       },
@@ -145,7 +157,7 @@ export const OutputsTable: React.FunctionComponent<OutputsTableProps> = ({
             </EuiFlexGroup>
           );
         },
-        name: i18n.translate('xpack.fleet.settings.outputSection.actionsColomnTitle', {
+        name: i18n.translate('xpack.fleet.settings.outputSection.actionsColumnTitle', {
           defaultMessage: 'Actions',
         }),
       },

x-pack/plugins/fleet/public/hooks/use_request/outputs.ts

@@ -5,6 +5,8 @@
  * 2.0.
  */
 
+import type { GetOutputHealthResponse } from '../../../common/types';
+
 import { outputRoutesService } from '../../services';
 import type {
   PutOutputRequest,
@@ -65,3 +67,11 @@ export function sendDeleteOutput(outputId: string) {
     version: API_VERSIONS.public.v1,
   });
 }
+
+export function sendGetOutputHealth(outputId: string) {
+  return sendRequest<GetOutputHealthResponse>({
+    method: 'get',
+    path: outputRoutesService.getOutputHealthPath(outputId),
+    version: API_VERSIONS.public.v1,
+  });
+}

x-pack/plugins/fleet/server/constants/index.ts

@@ -81,6 +81,8 @@ export {
   // secrets
   SECRETS_ENDPOINT_PATH,
   SECRETS_MINIMUM_FLEET_SERVER_VERSION,
+  // outputs
+  OUTPUT_HEALTH_DATA_STREAM,
   type PrivilegeMapObject,
 } from '../../common/constants';
 

x-pack/plugins/fleet/server/routes/output/handler.ts

@@ -16,6 +16,7 @@ import { outputType } from '../../../common/constants';
 
 import type {
   DeleteOutputRequestSchema,
+  GetLatestOutputHealthRequestSchema,
   GetOneOutputRequestSchema,
   PostOutputRequestSchema,
   PutOutputRequestSchema,
@@ -207,3 +208,18 @@ export const postLogstashApiKeyHandler: RequestHandler = async (context, request
     return defaultFleetErrorHandler({ error, response });
   }
 };
+
+export const getLatestOutputHealth: RequestHandler<
+  TypeOf<typeof GetLatestOutputHealthRequestSchema.params>
+> = async (context, request, response) => {
+  const esClient = (await context.core).elasticsearch.client.asCurrentUser;
+  try {
+    const outputHealth = await outputService.getLatestOutputHealth(
+      esClient,
+      request.params.outputId
+    );
+    return response.ok({ body: outputHealth });
+  } catch (error) {
+    return defaultFleetErrorHandler({ error, response });
+  }
+};

x-pack/plugins/fleet/server/routes/output/index.ts

@@ -12,6 +12,7 @@ import { API_VERSIONS } from '../../../common/constants';
 import { OUTPUT_API_ROUTES } from '../../constants';
 import {
   DeleteOutputRequestSchema,
+  GetLatestOutputHealthRequestSchema,
   GetOneOutputRequestSchema,
   GetOutputsRequestSchema,
   PostOutputRequestSchema,
@@ -25,6 +26,7 @@ import {
   postOutputHandler,
   putOutputHandler,
   postLogstashApiKeyHandler,
+  getLatestOutputHealth,
 } from './handler';
 
 export const registerRoutes = (router: FleetAuthzRouter) => {
@@ -115,4 +117,19 @@ export const registerRoutes = (router: FleetAuthzRouter) => {
       },
       postLogstashApiKeyHandler
     );
+
+  router.versioned
+    .get({
+      path: OUTPUT_API_ROUTES.GET_OUTPUT_HEALTH_PATTERN,
+      fleetAuthz: {
+        fleet: { all: true },
+      },
+    })
+    .addVersion(
+      {
+        version: API_VERSIONS.public.v1,
+        validate: { request: GetLatestOutputHealthRequestSchema },
+      },
+      getLatestOutputHealth
+    );
 };

x-pack/plugins/fleet/server/services/output.test.ts

@@ -1679,4 +1679,46 @@ describe('Output Service', () => {
       expect(hosts).toEqual(['http://localhost:9200']);
     });
   });
+
+  describe('getLatestOutputHealth', () => {
+    it('should return unkown state if no hits', async () => {
+      esClientMock.search.mockResolvedValue({
+        hits: {
+          hits: [],
+        },
+      } as any);
+
+      const response = await outputService.getLatestOutputHealth(esClientMock, 'id');
+
+      expect(response).toEqual({
+        state: 'UNKOWN',
+        message: '',
+        timestamp: '',
+      });
+    });
+
+    it('should return state from hits', async () => {
+      esClientMock.search.mockResolvedValue({
+        hits: {
+          hits: [
+            {
+              _source: {
+                state: 'DEGRADED',
+                message: 'connection error',
+                '@timestamp': '2023-11-30T14:25:31Z',
+              },
+            },
+          ],
+        },
+      } as any);
+
+      const response = await outputService.getLatestOutputHealth(esClientMock, 'id');
+
+      expect(response).toEqual({
+        state: 'DEGRADED',
+        message: 'connection error',
+        timestamp: '2023-11-30T14:25:31Z',
+      });
+    });
+  });
 });

x-pack/plugins/fleet/server/services/output.ts

@@ -33,6 +33,7 @@ import {
   DEFAULT_OUTPUT,
   DEFAULT_OUTPUT_ID,
   OUTPUT_SAVED_OBJECT_TYPE,
+  OUTPUT_HEALTH_DATA_STREAM,
 } from '../constants';
 import {
   SO_SEARCH_LIMIT,
@@ -962,6 +963,34 @@ class OutputService {
       }
     }
   }
+
+  async getLatestOutputHealth(esClient: ElasticsearchClient, id: string): Promise<OutputHealth> {
+    const response = await esClient.search({
+      index: OUTPUT_HEALTH_DATA_STREAM,
+      query: { bool: { filter: { term: { output: id } } } },
+      sort: { '@timestamp': 'desc' },
+      size: 1,
+    });
+    if (response.hits.hits.length === 0) {
+      return {
+        state: 'UNKOWN',
+        message: '',
+        timestamp: '',
+      };
+    }
+    const latestHit = response.hits.hits[0]._source as any;
+    return {
+      state: latestHit.state,
+      message: latestHit.message ?? '',
+      timestamp: latestHit['@timestamp'],
+    };
+  }
+}
+
+interface OutputHealth {
+  state: string;
+  message: string;
+  timestamp: string;
 }
 
 export const outputService = new OutputService();

x-pack/plugins/fleet/server/types/rest_spec/output.ts

@@ -33,3 +33,9 @@ export const PutOutputRequestSchema = {
   }),
   body: UpdateOutputSchema,
 };
+
+export const GetLatestOutputHealthRequestSchema = {
+  params: schema.object({
+    outputId: schema.string(),
+  }),
+};

x-pack/plugins/translations/translations/fr-FR.json

@@ -18206,17 +18206,17 @@
     "xpack.fleet.settings.outputForm.sslKeyRequiredErrorMessage": "Clé SSL requise",
     "xpack.fleet.settings.outputs.defaultMonitoringOutputBadgeTitle": "Monitoring des agents",
     "xpack.fleet.settings.outputs.defaultOutputBadgeTitle": "Intégrations d’agent",
-    "xpack.fleet.settings.outputSection.actionsColomnTitle": "Actions",
-    "xpack.fleet.settings.outputSection.defaultColomnTitle": "Par défaut",
+    "xpack.fleet.settings.outputSection.actionsColumnTitle": "Actions",
+    "xpack.fleet.settings.outputSection.defaultColumnTitle": "Par défaut",
     "xpack.fleet.settings.outputSection.deleteButtonTitle": "Supprimer",
     "xpack.fleet.settings.outputSection.editButtonTitle": "Modifier",
     "xpack.fleet.settings.outputSectionSubtitle": "Indiquez l’emplacement vers lequel les agents doivent envoyer les données.",
     "xpack.fleet.settings.outputSectionTitle": "Sorties",
     "xpack.fleet.settings.outputsTable.elasticsearchTypeLabel": "Elasticsearch",
-    "xpack.fleet.settings.outputsTable.hostColomnTitle": "Hôtes",
+    "xpack.fleet.settings.outputsTable.hostColumnTitle": "Hôtes",
     "xpack.fleet.settings.outputsTable.managedTooltip": "Cette sortie est gérée en dehors de Fleet.",
-    "xpack.fleet.settings.outputsTable.nameColomnTitle": "Nom",
-    "xpack.fleet.settings.outputsTable.typeColomnTitle": "Type",
+    "xpack.fleet.settings.outputsTable.nameColumnTitle": "Nom",
+    "xpack.fleet.settings.outputsTable.typeColumnTitle": "Type",
     "xpack.fleet.settings.sortHandle": "Trier par identification d'hôte",
     "xpack.fleet.settings.updateDownloadSourceModal.confirmModalTitle": "Enregistrer et déployer les modifications ?",
     "xpack.fleet.settings.updateOutput.confirmModalTitle": "Enregistrer et déployer les modifications ?",

x-pack/plugins/translations/translations/ja-JP.json

@@ -18219,17 +18219,17 @@
     "xpack.fleet.settings.outputForm.sslKeyRequiredErrorMessage": "SSL鍵が必要です",
     "xpack.fleet.settings.outputs.defaultMonitoringOutputBadgeTitle": "アラート監視",
     "xpack.fleet.settings.outputs.defaultOutputBadgeTitle": "エージェント統合",
-    "xpack.fleet.settings.outputSection.actionsColomnTitle": "アクション",
-    "xpack.fleet.settings.outputSection.defaultColomnTitle": "デフォルト",
+    "xpack.fleet.settings.outputSection.actionsColumnTitle": "アクション",
+    "xpack.fleet.settings.outputSection.defaultColumnTitle": "デフォルト",
     "xpack.fleet.settings.outputSection.deleteButtonTitle": "削除",
     "xpack.fleet.settings.outputSection.editButtonTitle": "編集",
     "xpack.fleet.settings.outputSectionSubtitle": "エージェントがデータを送信する場合を指定します。",
     "xpack.fleet.settings.outputSectionTitle": "アウトプット",
     "xpack.fleet.settings.outputsTable.elasticsearchTypeLabel": "Elasticsearch",
-    "xpack.fleet.settings.outputsTable.hostColomnTitle": "ホスト",
+    "xpack.fleet.settings.outputsTable.hostColumnTitle": "ホスト",
     "xpack.fleet.settings.outputsTable.managedTooltip": "この出力はFleet外で管理されます。",
-    "xpack.fleet.settings.outputsTable.nameColomnTitle": "名前",
-    "xpack.fleet.settings.outputsTable.typeColomnTitle": "型",
+    "xpack.fleet.settings.outputsTable.nameColumnTitle": "名前",
+    "xpack.fleet.settings.outputsTable.typeColumnTitle": "型",
     "xpack.fleet.settings.sortHandle": "ホストハンドルの並べ替え",
     "xpack.fleet.settings.updateDownloadSourceModal.confirmModalTitle": "変更を保存してデプロイしますか？",
     "xpack.fleet.settings.updateOutput.confirmModalTitle": "変更を保存してデプロイしますか？",

x-pack/plugins/translations/translations/zh-CN.json

@@ -18219,17 +18219,17 @@
     "xpack.fleet.settings.outputForm.sslKeyRequiredErrorMessage": "SSL 密钥必填",
     "xpack.fleet.settings.outputs.defaultMonitoringOutputBadgeTitle": "代理监测",
     "xpack.fleet.settings.outputs.defaultOutputBadgeTitle": "代理集成",
-    "xpack.fleet.settings.outputSection.actionsColomnTitle": "操作",
-    "xpack.fleet.settings.outputSection.defaultColomnTitle": "默认",
+    "xpack.fleet.settings.outputSection.actionsColumnTitle": "操作",
+    "xpack.fleet.settings.outputSection.defaultColumnTitle": "默认",
     "xpack.fleet.settings.outputSection.deleteButtonTitle": "删除",
     "xpack.fleet.settings.outputSection.editButtonTitle": "编辑",
     "xpack.fleet.settings.outputSectionSubtitle": "指定代理将发送数据的位置。",
     "xpack.fleet.settings.outputSectionTitle": "输出",
     "xpack.fleet.settings.outputsTable.elasticsearchTypeLabel": "Elasticsearch",
-    "xpack.fleet.settings.outputsTable.hostColomnTitle": "主机",
+    "xpack.fleet.settings.outputsTable.hostColumnTitle": "主机",
     "xpack.fleet.settings.outputsTable.managedTooltip": "此输出在 Fleet 外部进行管理。",
-    "xpack.fleet.settings.outputsTable.nameColomnTitle": "名称",
-    "xpack.fleet.settings.outputsTable.typeColomnTitle": "类型",
+    "xpack.fleet.settings.outputsTable.nameColumnTitle": "名称",
+    "xpack.fleet.settings.outputsTable.typeColumnTitle": "类型",
     "xpack.fleet.settings.sortHandle": "排序主机手柄",
     "xpack.fleet.settings.updateDownloadSourceModal.confirmModalTitle": "保存并部署更改？",
     "xpack.fleet.settings.updateOutput.confirmModalTitle": "保存并部署更改？",

x-pack/test/fleet_api_integration/apis/outputs/crud.ts

@@ -6,7 +6,10 @@
  */
 
 import expect from '@kbn/expect';
-import { GLOBAL_SETTINGS_SAVED_OBJECT_TYPE } from '@kbn/fleet-plugin/common/constants';
+import {
+  GLOBAL_SETTINGS_SAVED_OBJECT_TYPE,
+  OUTPUT_HEALTH_DATA_STREAM,
+} from '@kbn/fleet-plugin/common/constants';
 import { FtrProviderContext } from '../../../api_integration/ftr_provider_context';
 import { skipIfNoDockerRegistry } from '../../helpers';
 import { setupFleetAndAgents } from '../agents/services';
@@ -189,6 +192,52 @@ export default function (providerContext: FtrProviderContext) {
       });
     });
 
+    describe('GET /outputs/{outputId}/health', () => {
+      before(async () => {
+        await es.index({
+          refresh: 'wait_for',
+          index: OUTPUT_HEALTH_DATA_STREAM,
+          document: {
+            state: 'HEALTHY',
+            message: '',
+            '@timestamp': '' + Date.parse('2023-11-29T14:25:31Z'),
+            output: defaultOutputId,
+          },
+        });
+
+        await es.index({
+          refresh: 'wait_for',
+          index: OUTPUT_HEALTH_DATA_STREAM,
+          document: {
+            state: 'DEGRADED',
+            message: 'connection error',
+            '@timestamp': '' + Date.parse('2023-11-30T14:25:31Z'),
+            output: defaultOutputId,
+          },
+        });
+
+        await es.index({
+          refresh: 'wait_for',
+          index: OUTPUT_HEALTH_DATA_STREAM,
+          document: {
+            state: 'HEALTHY',
+            message: '',
+            '@timestamp': '' + Date.parse('2023-11-31T14:25:31Z'),
+            output: 'remote2',
+          },
+        });
+      });
+      it('should allow return the latest output health', async () => {
+        const { body: outputHealth } = await supertest
+          .get(`/api/fleet/outputs/${defaultOutputId}/health`)
+          .expect(200);
+
+        expect(outputHealth.state).to.equal('DEGRADED');
+        expect(outputHealth.message).to.equal('connection error');
+        expect(outputHealth.timestamp).not.to.be.empty();
+      });
+    });
+
     describe('PUT /outputs/{outputId}', () => {
       it('should explicitly set port on ES hosts', async function () {
         await supertest