[Stack Monitoring] Add stale status reporting for Kibana (#132613)

* [Stack Monitoring] Add stale status reporting for Kibana (#126386)

* Fix stale message grammar and update stale indicator to use EuiBadge

* Fix i18n ids

* Remove unused i18n key

* Fix Jest tests

* Update exposeToBrowser test

* Update API integration tests

* Fix functional tests

* Fix API integration tests

* Update snapshots

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

docs/settings/monitoring-settings.asciidoc

@@ -137,6 +137,10 @@ represent. Defaults to 10. If you modify the
 `monitoring.ui.collection.interval` in `elasticsearch.yml`, use the same
 value in this setting.
 
+`monitoring.ui.kibana.reporting.stale_status_threshold_seconds`::
+Specifies how many seconds can pass before the Kibana status reports are considered stale.
+Defaults to `120`.
+
 [float]
 [[monitoring-ui-cgroup-settings]]
 ===== Monitoring UI container settings

test/plugin_functional/test_suites/core_plugins/rendering.ts

@@ -130,6 +130,7 @@ export default function ({ getService }: PluginFunctionalProviderContext) {
         'monitoring.kibana.collection.enabled (boolean)',
         'monitoring.kibana.collection.interval (number)',
         'monitoring.ui.ccs.enabled (boolean)',
+        'monitoring.ui.kibana.reporting.stale_status_threshold_seconds (number)',
         'monitoring.ui.container.apm.enabled (boolean)',
         'monitoring.ui.container.elasticsearch.enabled (boolean)',
         'monitoring.ui.container.logstash.enabled (boolean)',

x-pack/plugins/monitoring/public/application/contexts/external_config_context.tsx

@@ -12,6 +12,7 @@ export interface ExternalConfig {
   showCgroupMetricsElasticsearch: boolean;
   showCgroupMetricsLogstash: boolean;
   renderReactApp: boolean;
+  staleStatusThresholdSeconds: number;
 }
 
 export const ExternalConfigContext = createContext({} as ExternalConfig);

x-pack/plugins/monitoring/public/components/cluster/overview/helpers.js

@@ -70,7 +70,7 @@ export function HealthStatusIndicator(props) {
   return (
     <EuiFlexGroup alignItems="center" gutterSize="s">
       <EuiFlexItem grow={false}>
-        <EuiHealth color={statusColor} data-test-subj="statusIcon">
+        <EuiHealth color={statusColor} data-test-subj="status">
           <HealthLabel {...props} />
         </EuiHealth>
       </EuiFlexItem>

x-pack/plugins/monitoring/public/components/cluster/overview/kibana_panel.js

@@ -5,37 +5,40 @@
  * 2.0.
  */
 
-import React from 'react';
-import { formatNumber } from '../../../lib/format_number';
-import {
-  ClusterItemContainer,
-  HealthStatusIndicator,
-  BytesPercentageUsage,
-  DisabledIfNoDataAndInSetupModeLink,
-} from './helpers';
-import { get } from 'lodash';
 import {
+  EuiBadge,
+  EuiDescriptionList,
+  EuiDescriptionListDescription,
+  EuiDescriptionListTitle,
   EuiFlexGrid,
   EuiFlexGroup,
   EuiFlexItem,
-  EuiLink,
-  EuiTitle,
-  EuiPanel,
-  EuiDescriptionList,
-  EuiDescriptionListTitle,
-  EuiDescriptionListDescription,
   EuiHorizontalRule,
+  EuiLink,
+  EuiPanel,
+  EuiTitle,
+  EuiToolTip,
 } from '@elastic/eui';
-import { FormattedMessage } from '@kbn/i18n-react';
 import { i18n } from '@kbn/i18n';
-import { SetupModeTooltip } from '../../setup_mode/tooltip';
+import { FormattedMessage } from '@kbn/i18n-react';
+import { get } from 'lodash';
+import React from 'react';
 import { KIBANA_SYSTEM_ID, RULE_KIBANA_VERSION_MISMATCH } from '../../../../common/constants';
-import { getSafeForExternalLink } from '../../../lib/get_safe_for_external_link';
+import { SetupModeFeature } from '../../../../common/enums';
 import { AlertsBadge } from '../../../alerts/badge';
 import { shouldShowAlertBadge } from '../../../alerts/lib/should_show_alert_badge';
+import { ExternalConfigContext } from '../../../application/contexts/external_config_context';
+import { formatNumber } from '../../../lib/format_number';
+import { getSafeForExternalLink } from '../../../lib/get_safe_for_external_link';
 import { isSetupModeFeatureEnabled } from '../../../lib/setup_mode';
-import { SetupModeFeature } from '../../../../common/enums';
 import { SetupModeContext } from '../../setup_mode/setup_mode_context';
+import { SetupModeTooltip } from '../../setup_mode/tooltip';
+import {
+  BytesPercentageUsage,
+  ClusterItemContainer,
+  DisabledIfNoDataAndInSetupModeLink,
+  HealthStatusIndicator,
+} from './helpers';
 
 const INSTANCES_PANEL_ALERTS = [RULE_KIBANA_VERSION_MISMATCH];
 
@@ -43,14 +46,13 @@ export function KibanaPanel(props) {
   const setupMode = props.setupMode;
   const alerts = props.alerts;
   const setupModeContext = React.useContext(SetupModeContext);
+  const { staleStatusThresholdSeconds } = React.useContext(ExternalConfigContext);
   const showDetectedKibanas =
     setupMode.enabled && get(setupMode.data, 'kibana.detected.doesExist', false);
   if (!props.count && !showDetectedKibanas) {
     return null;
   }
 
-  const statusIndicator = <HealthStatusIndicator status={props.status} product={'kb'} />;
-
   const goToKibana = () => getSafeForExternalLink('#/kibana');
   const goToInstances = () => getSafeForExternalLink('#/kibana/instances');
 
@@ -78,7 +80,12 @@ export function KibanaPanel(props) {
   return (
     <ClusterItemContainer
       {...props}
-      statusIndicator={statusIndicator}
+      statusIndicator={statusIndicator(
+        props.status,
+        props.some_status_is_stale,
+        goToInstances(),
+        staleStatusThresholdSeconds
+      )}
       url="kibana"
       title={i18n.translate('xpack.monitoring.cluster.overview.kibanaPanel.kibanaTitle', {
         defaultMessage: 'Kibana',
@@ -203,3 +210,42 @@ export function KibanaPanel(props) {
     </ClusterItemContainer>
   );
 }
+
+function statusIndicator(status, someStatusIsStale, instancesHref, staleStatusThresholdSeconds) {
+  if (!someStatusIsStale) {
+    return <HealthStatusIndicator status={status} product={'kb'} />;
+  }
+
+  const staleMessage = i18n.translate(
+    'xpack.monitoring.cluster.overview.kibanaPanel.staleStatusTooltip',
+    {
+      defaultMessage:
+        "It's been more than {staleStatusThresholdSeconds} seconds since we have heard from some instances.",
+      values: {
+        staleStatusThresholdSeconds,
+      },
+    }
+  );
+
+  return (
+    <>
+      <div style={{ marginBottom: '8px' }}>
+        <EuiToolTip position="top" content={staleMessage}>
+          <EuiBadge iconType="alert" color="warning" data-test-subj="status">
+            {i18n.translate('xpack.monitoring.cluster.overview.kibanaPanel.staleStatusLabel', {
+              defaultMessage: 'Stale',
+            })}
+          </EuiBadge>
+        </EuiToolTip>
+      </div>
+      <EuiLink href={instancesHref}>
+        {i18n.translate(
+          'xpack.monitoring.cluster.overview.kibanaPanel.staleStatusLinkToInstancesLabel',
+          {
+            defaultMessage: 'View all instances',
+          }
+        )}
+      </EuiLink>
+    </>
+  );
+}

x-pack/plugins/monitoring/public/components/kibana/cluster_status/index.js

@@ -5,11 +5,15 @@
  * 2.0.
  */
 
-import React from 'react';
+import { EuiBadge, EuiLink, EuiStat, EuiToolTip } from '@elastic/eui';
-import { SummaryStatus } from '../../summary_status';
-import { KibanaStatusIcon } from '../status_icon';
-import { formatMetric } from '../../../lib/format_number';
 import { i18n } from '@kbn/i18n';
+import React from 'react';
+import { useLocation } from 'react-router-dom';
+import { ExternalConfigContext } from '../../../application/contexts/external_config_context';
+import { formatMetric } from '../../../lib/format_number';
+import { getSafeForExternalLink } from '../../../lib/get_safe_for_external_link';
+import { DefaultStatusIndicator, SummaryStatus } from '../../summary_status';
+import { KibanaStatusIcon } from '../status_icon';
 
 export function ClusterStatus({ stats, alerts }) {
   const {
@@ -20,8 +24,12 @@ export function ClusterStatus({ stats, alerts }) {
     requests_total: requests,
     response_time_max: maxResponseTime,
     status,
+    some_status_is_stale: someStatusIsStale,
   } = stats;
 
+  const { staleStatusThresholdSeconds } = React.useContext(ExternalConfigContext);
+  const location = useLocation();
+
   const metrics = [
     {
       label: i18n.translate('xpack.monitoring.kibana.clusterStatus.instancesLabel', {
@@ -60,15 +68,107 @@ export function ClusterStatus({ stats, alerts }) {
     },
   ];
 
-  const IconComponent = ({ status }) => <KibanaStatusIcon status={status} />;
+  const StatusIndicator = () => {
+    if (!someStatusIsStale) {
+      return (
+        <DefaultStatusIndicator status={status} isOnline={true} IconComponent={KibanaStatusIcon} />
+      );
+    }
+
+    const staleMessage = i18n.translate(
+      'xpack.monitoring.kibana.clusterStatus.staleStatusTooltip',
+      {
+        defaultMessage:
+          "It's been more than {staleStatusThresholdSeconds} seconds since we have heard from some instances.",
+        values: {
+          staleStatusThresholdSeconds,
+        },
+      }
+    );
+
+    if (location.pathname === '/kibana') {
+      return <OverviewPageStatusIndicator staleMessage={staleMessage} />;
+    }
+
+    return <InstancesPageStatusIndicator staleMessage={staleMessage} />;
+  };
 
   return (
     <SummaryStatus
-      metrics={metrics}
+      StatusIndicator={StatusIndicator}
-      status={status}
       alerts={alerts}
-      IconComponent={IconComponent}
+      metrics={metrics}
       data-test-subj="kibanaClusterStatus"
     />
   );
 }
+
+function OverviewPageStatusIndicator({ staleMessage }) {
+  const instancesHref = getSafeForExternalLink('#/kibana/instances');
+
+  const title = (
+    <>
+      <div style={{ marginBottom: '8px' }}>
+        <EuiToolTip position="top" content={staleMessage}>
+          <EuiBadge iconType="alert" color="warning">
+            {i18n.translate(
+              'xpack.monitoring.kibana.clusterStatus.overview.staleStatusInstancesLabel',
+              {
+                defaultMessage: 'Stale',
+              }
+            )}
+          </EuiBadge>
+        </EuiToolTip>
+      </div>
+      <EuiLink href={instancesHref}>
+        {i18n.translate(
+          'xpack.monitoring.kibana.clusterStatus.overview.staleStatusLinkToInstancesLabel',
+          {
+            defaultMessage: 'View all instances',
+          }
+        )}
+      </EuiLink>
+    </>
+  );
+
+  return (
+    <EuiStat
+      data-test-subj="status"
+      description={i18n.translate('xpack.monitoring.kibana.clusterStatus.overview.statusLabel', {
+        defaultMessage: 'Status',
+      })}
+      title={title}
+      titleSize="xxxs"
+      textAlign="left"
+      className="monSummaryStatusNoWrap__stat"
+    />
+  );
+}
+
+function InstancesPageStatusIndicator({ staleMessage }) {
+  const title = (
+    <EuiToolTip position="top" content={staleMessage}>
+      <EuiBadge iconType="alert" color="warning">
+        {i18n.translate(
+          'xpack.monitoring.kibana.clusterStatus.instances.staleStatusInstancesLabel',
+          {
+            defaultMessage: 'Stale',
+          }
+        )}
+      </EuiBadge>
+    </EuiToolTip>
+  );
+
+  return (
+    <EuiStat
+      data-test-subj="status"
+      description={i18n.translate('xpack.monitoring.kibana.clusterStatus.instances.statusLabel', {
+        defaultMessage: 'Status',
+      })}
+      title={title}
+      titleSize="xxxs"
+      textAlign="left"
+      className="monSummaryStatusNoWrap__stat"
+    />
+  );
+}

x-pack/plugins/monitoring/public/components/kibana/detail_status/index.js

@@ -5,11 +5,16 @@
  * 2.0.
  */
 
-import React from 'react';
+import { EuiBadge, EuiStat, EuiToolTip } from '@elastic/eui';
-import { SummaryStatus } from '../../summary_status';
-import { KibanaStatusIcon } from '../status_icon';
-import { formatMetric } from '../../../lib/format_number';
 import { i18n } from '@kbn/i18n';
+import { useUiSetting } from '@kbn/kibana-react-plugin/public';
+import { capitalize } from 'lodash';
+import React from 'react';
+import { ExternalConfigContext } from '../../../application/contexts/external_config_context';
+import { formatMetric } from '../../../lib/format_number';
+import { DefaultStatusIndicator, SummaryStatus } from '../../summary_status';
+import { formatLastSeenTimestamp } from '../format_last_seen_timestamp';
+import { KibanaStatusIcon } from '../status_icon';
 
 export function DetailStatus({ stats }) {
   const {
@@ -18,8 +23,13 @@ export function DetailStatus({ stats }) {
     version,
     uptime,
     status,
+    statusIsStale,
+    lastSeenTimestamp,
   } = stats;
 
+  const { staleStatusThresholdSeconds } = React.useContext(ExternalConfigContext);
+  const dateFormat = useUiSetting('dateFormat');
+
   const metrics = [
     {
       label: i18n.translate('xpack.monitoring.kibana.detailStatus.transportAddressLabel', {
@@ -51,14 +61,82 @@ export function DetailStatus({ stats }) {
     },
   ];
 
-  const IconComponent = ({ status }) => <KibanaStatusIcon status={status} />;
+  const StatusIndicator = () => {
+    if (!statusIsStale) {
+      return (
+        <DefaultStatusIndicator status={status} isOnline={true} IconComponent={KibanaStatusIcon} />
+      );
+    }
+
+    const { description, title } = prepareStaleMessage(
+      status,
+      lastSeenTimestamp,
+      staleStatusThresholdSeconds,
+      dateFormat
+    );
+
+    return (
+      <EuiStat
+        data-test-subj="status"
+        description={description}
+        title={title}
+        titleSize="xxxs"
+        textAlign="left"
+        className="monSummaryStatusNoWrap__stat"
+      />
+    );
+  };
 
   return (
     <SummaryStatus
+      StatusIndicator={StatusIndicator}
       metrics={metrics}
-      status={status}
-      IconComponent={IconComponent}
       data-test-subj="kibanaDetailStatus"
     />
   );
 }
+
+function prepareStaleMessage(status, lastSeenTimestamp, staleStatusThresholdSeconds, dateFormat) {
+  const { shouldShowRelativeTime, relativeTime, formattedTimestamp } = formatLastSeenTimestamp(
+    lastSeenTimestamp,
+    dateFormat
+  );
+
+  const staleMessage = i18n.translate('xpack.monitoring.kibana.detailStatus.staleStatusTooltip', {
+    defaultMessage:
+      "It's been more than {staleStatusThresholdSeconds} seconds since we have heard from this instance. Last seen: {lastSeenTimestamp}",
+    values: {
+      staleStatusThresholdSeconds,
+      lastSeenTimestamp: shouldShowRelativeTime ? relativeTime : formattedTimestamp,
+    },
+  });
+
+  const description = i18n.translate(
+    'xpack.monitoring.kibana.detailStatus.staleStatusMetricDescription',
+    {
+      defaultMessage: 'Last Reported Status',
+    }
+  );
+
+  const title = (
+    <>
+      <KibanaStatusIcon status={status} />
+      &nbsp;
+      {capitalize(status)}
+      <span style={{ marginLeft: '8px' }}>
+        <EuiToolTip position="top" content={staleMessage}>
+          <EuiBadge iconType="alert" color="warning">
+            {i18n.translate('xpack.monitoring.kibana.detailStatus.staleStatusLabel', {
+              defaultMessage: 'Stale',
+            })}
+          </EuiBadge>
+        </EuiToolTip>
+      </span>
+    </>
+  );
+
+  return {
+    description,
+    title,
+  };
+}

x-pack/plugins/monitoring/public/components/kibana/format_last_seen_timestamp.ts

@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { capitalize } from 'lodash';
+import moment from 'moment';
+
+export function formatLastSeenTimestamp(lastSeenTimestampRaw: string, dateFormat: string) {
+  const lastSeenTimestamp = moment(lastSeenTimestampRaw);
+
+  const formattedTimestamp = lastSeenTimestamp.format(dateFormat);
+  const relativeTime = capitalize(lastSeenTimestamp.fromNow());
+
+  const sixHoursAgo = moment().subtract(6, 'hours');
+  const shouldShowRelativeTime = !sixHoursAgo.isAfter(lastSeenTimestamp);
+
+  return {
+    shouldShowRelativeTime,
+    formattedTimestamp,
+    relativeTime,
+  };
+}

x-pack/plugins/monitoring/public/components/kibana/instances/instances.tsx

@@ -5,42 +5,51 @@
  * 2.0.
  */
 
-import React, { Fragment } from 'react';
 import {
+  EuiCallOut,
+  EuiHealth,
+  EuiIconTip,
+  EuiLink,
   EuiPage,
   EuiPageBody,
   EuiPageContent,
   EuiPanel,
-  EuiSpacer,
-  EuiLink,
-  EuiCallOut,
   EuiScreenReaderOnly,
+  EuiSpacer,
   EuiToolTip,
-  EuiHealth,
 } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
 import { FormattedMessage } from '@kbn/i18n-react';
+import { useUiSetting } from '@kbn/kibana-react-plugin/public';
 import { capitalize, get } from 'lodash';
-// @ts-ignore
+import React, { Fragment } from 'react';
-import { ClusterStatus } from '../cluster_status';
+import { KIBANA_SYSTEM_ID } from '../../../../common/constants';
-// @ts-ignore
+import { SetupModeFeature } from '../../../../common/enums';
-import { EuiMonitoringTable } from '../../table';
+import { CommonAlertStatus } from '../../../../common/types/alerts';
-import { STATUS_ICON_TYPES } from '../../status_icon';
+import { ElasticsearchSourceKibanaStats } from '../../../../common/types/es';
+import { AlertsStatus } from '../../../alerts/status';
+import { ExternalConfigContext } from '../../../application/contexts/external_config_context';
 // @ts-ignore
 import { formatMetric, formatNumber } from '../../../lib/format_number';
 import { getSafeForExternalLink } from '../../../lib/get_safe_for_external_link';
+import { isSetupModeFeatureEnabled } from '../../../lib/setup_mode';
 // @ts-ignore
 import { SetupModeBadge } from '../../setup_mode/badge';
-import { KIBANA_SYSTEM_ID } from '../../../../common/constants';
-import { CommonAlertStatus } from '../../../../common/types/alerts';
-import { ElasticsearchSourceKibanaStats } from '../../../../common/types/es';
 // @ts-ignore
 import { ListingCallOut } from '../../setup_mode/listing_callout';
-import { AlertsStatus } from '../../../alerts/status';
+import { STATUS_ICON_TYPES } from '../../status_icon';
-import { isSetupModeFeatureEnabled } from '../../../lib/setup_mode';
+// @ts-ignore
-import { SetupModeFeature } from '../../../../common/enums';
+import { EuiMonitoringTable } from '../../table';
+// @ts-ignore
+import { ClusterStatus } from '../cluster_status';
+import { formatLastSeenTimestamp } from '../format_last_seen_timestamp';
 
-const getColumns = (setupMode: any, alerts: { [alertTypeId: string]: CommonAlertStatus[] }) => {
+const getColumns = (
+  setupMode: any,
+  alerts: { [alertTypeId: string]: CommonAlertStatus[] },
+  dateFormat: string,
+  staleStatusThresholdSeconds: number
+) => {
   const columns = [
     {
       name: i18n.translate('xpack.monitoring.kibana.listing.nameColumnTitle', {
@@ -95,29 +104,57 @@ const getColumns = (setupMode: any, alerts: { [alertTypeId: string]: CommonAlert
       name: i18n.translate('xpack.monitoring.kibana.listing.alertsColumnTitle', {
         defaultMessage: 'Alerts',
       }),
-      field: 'isOnline',
+      field: 'alerts_column',
       width: '175px',
       sortable: true,
       render: () => <AlertsStatus showBadge={true} alerts={alerts} />,
     },
     {
-      name: i18n.translate('xpack.monitoring.kibana.listing.statusColumnTitle', {
+      name: i18n.translate('xpack.monitoring.kibana.listing.lastReportedStatusColumnTitle', {
-        defaultMessage: 'Status',
+        defaultMessage: 'Last Reported Status',
       }),
       field: 'status',
-      render: (
+      render: (status: string) => {
-        status: string,
-        kibana: Pick<ElasticsearchSourceKibanaStats, 'kibana'> & { availability: boolean }
-      ) => {
         return (
-          <EuiToolTip content={status} position="bottom">
+          <EuiHealth color={statusIconColor(status)} data-test-subj="status">
-            <EuiHealth
+            {capitalize(status)}
-              color={kibana.availability ? 'success' : 'subdued'}
+          </EuiHealth>
-              data-test-subj="statusIcon"
+        );
-            >
+      },
-              {capitalize(status)}
+    },
-            </EuiHealth>
+    {
-          </EuiToolTip>
+      name: i18n.translate('xpack.monitoring.kibana.listing.lastSeenColumnTitle', {
+        defaultMessage: 'Last Seen',
+      }),
+      field: 'lastSeenTimestamp',
+      render: (
+        lastSeenTimestampRaw: string,
+        kibana: Pick<ElasticsearchSourceKibanaStats, 'kibana'> & { statusIsStale: boolean }
+      ) => {
+        const lastSeenTimestamp = prepareLastSeenTimestamp(lastSeenTimestampRaw, dateFormat);
+        const staleMessage = i18n.translate('xpack.monitoring.kibana.listing.staleStatusTooltip', {
+          defaultMessage:
+            "It's been more than {staleStatusThresholdSeconds} seconds since we have heard from this instance.",
+          values: {
+            staleStatusThresholdSeconds,
+          },
+        });
+        return (
+          <span data-test-subj="lastSeen">
+            {lastSeenTimestamp}
+            {kibana.statusIsStale && (
+              <>
+                &nbsp;
+                <EuiIconTip
+                  aria-label={staleMessage}
+                  content={staleMessage}
+                  size="l"
+                  type="alert"
+                  color="warning"
+                />
+              </>
+            )}
+          </span>
         );
       },
     },
@@ -183,6 +220,9 @@ interface Props {
 export const KibanaInstances: React.FC<Props> = (props: Props) => {
   const { clusterStatus, alerts, setupMode, sorting, pagination, onTableChange } = props;
 
+  const { staleStatusThresholdSeconds } = React.useContext(ExternalConfigContext);
+  const dateFormat = useUiSetting<string>('dateFormat');
+
   let setupModeCallOut = null;
   // Merge the instances data with the setup data if enabled
   const instances = props.instances || [];
@@ -286,7 +326,7 @@ export const KibanaInstances: React.FC<Props> = (props: Props) => {
           <EuiMonitoringTable
             className="kibanaInstancesTable"
             rows={dataFlattened}
-            columns={getColumns(setupMode, alerts)}
+            columns={getColumns(setupMode, alerts, dateFormat, staleStatusThresholdSeconds)}
             sorting={sorting}
             pagination={pagination}
             setupMode={setupMode}
@@ -312,3 +352,33 @@ export const KibanaInstances: React.FC<Props> = (props: Props) => {
     </EuiPage>
   );
 };
+
+function statusIconColor(status: string) {
+  switch (status) {
+    case 'red':
+      return 'danger';
+    case 'yellow':
+      return 'warning';
+    case 'green':
+      return 'success';
+    default:
+      return 'subdued';
+  }
+}
+
+function prepareLastSeenTimestamp(lastSeenTimestampRaw: string, dateFormat: string) {
+  const { shouldShowRelativeTime, formattedTimestamp, relativeTime } = formatLastSeenTimestamp(
+    lastSeenTimestampRaw,
+    dateFormat
+  );
+
+  if (shouldShowRelativeTime) {
+    return (
+      <EuiToolTip position="top" content={formattedTimestamp}>
+        <span>{relativeTime}</span>
+      </EuiToolTip>
+    );
+  }
+
+  return formattedTimestamp;
+}

x-pack/plugins/monitoring/public/components/summary_status/__snapshots__/summary_status.test.js.snap

@@ -14,6 +14,7 @@ exports[`Summary Status Component should allow label to be optional 1`] = `
     >
       <div
         class="euiStat euiStat--leftAligned monSummaryStatusNoWrap__stat"
+        data-test-subj="status"
       >
         <div
           class="euiText euiText--small euiStat__description"
@@ -109,6 +110,10 @@ exports[`Summary Status Component should allow status to be optional 1`] = `
   <div
     class="euiFlexGroup euiFlexGroup--gutterMedium euiFlexGroup--alignItemsCenter euiFlexGroup--justifyContentSpaceBetween euiFlexGroup--directionRow euiFlexGroup--responsive"
   >
+    <div
+      class="euiFlexItem euiFlexItem--flexGrowZero eui-textTruncate"
+      style="max-width:200px"
+    />
     <div
       class="euiFlexItem euiFlexItem--flexGrowZero"
       data-test-subj="freeDiskSpace"
@@ -187,6 +192,7 @@ exports[`Summary Status Component should render metrics in a summary bar 1`] = `
     >
       <div
         class="euiStat euiStat--leftAligned monSummaryStatusNoWrap__stat"
+        data-test-subj="status"
       >
         <div
           class="euiText euiText--small euiStat__description"

x-pack/plugins/monitoring/public/components/summary_status/index.js

@@ -5,4 +5,4 @@
  * 2.0.
  */
 
-export { SummaryStatus } from './summary_status';
+export { SummaryStatus, DefaultStatusIndicator } from './summary_status';

x-pack/plugins/monitoring/public/components/summary_status/summary_status.js

@@ -54,49 +54,51 @@ const DefaultIconComponent = ({ status }) => (
   </Fragment>
 );
 
-const StatusIndicator = ({ status, isOnline, IconComponent }) => {
+export const DefaultStatusIndicator = ({ status, isOnline, IconComponent }) => {
   if (isEmpty(status)) {
     return null;
   }
 
   return (
-    <EuiFlexItem
+    <EuiStat
-      className="eui-textTruncate"
+      data-test-subj="status"
-      style={{ maxWidth: 200 }}
+      title={
-      key={`summary-status-item-status`}
+        <>
-      grow={false}
+          <IconComponent status={status} isOnline={isOnline} />
-    >
+          &nbsp;
-      <EuiStat
+          {capitalize(status)}
-        title={
+        </>
-          <Fragment>
+      }
-            <IconComponent status={status} isOnline={isOnline} />
+      titleSize="xxxs"
-            &nbsp;
+      textAlign="left"
-            {capitalize(status)}
+      className="monSummaryStatusNoWrap__stat"
-          </Fragment>
+      description={i18n.translate('xpack.monitoring.summaryStatus.statusDescription', {
-        }
+        defaultMessage: 'Status',
-        titleSize="xxxs"
+      })}
-        textAlign="left"
+    />
-        className="monSummaryStatusNoWrap__stat"
-        description={i18n.translate('xpack.monitoring.summaryStatus.statusDescription', {
-          defaultMessage: 'Status',
-        })}
-      />
-    </EuiFlexItem>
   );
 };
 
 export function SummaryStatus({
-  metrics,
+  StatusIndicator = DefaultStatusIndicator,
   status,
-  alerts,
   isOnline,
   IconComponent = DefaultIconComponent,
+  alerts,
+  metrics,
   ...props
 }) {
   return (
     <div {...props} className="monSummaryStatusNoWrap">
       <EuiFlexGroup gutterSize="m" alignItems="center" justifyContent="spaceBetween">
-        <StatusIndicator status={status} IconComponent={IconComponent} isOnline={isOnline} />
+        <EuiFlexItem
+          className="eui-textTruncate"
+          style={{ maxWidth: 200 }}
+          key={`summary-status-item-status`}
+          grow={false}
+        >
+          <StatusIndicator status={status} isOnline={isOnline} IconComponent={IconComponent} />
+        </EuiFlexItem>
         {alerts ? (
           <EuiFlexItem grow={false}>
             <EuiStat

x-pack/plugins/monitoring/public/plugin.ts

@@ -145,6 +145,10 @@ export class MonitoringPlugin
       ['showLicenseExpiration', monitoring.ui.show_license_expiration],
       ['showCgroupMetricsElasticsearch', monitoring.ui.container.elasticsearch.enabled],
       ['showCgroupMetricsLogstash', monitoring.ui.container.logstash.enabled],
+      [
+        'staleStatusThresholdSeconds',
+        monitoring.ui.kibana.reporting.stale_status_threshold_seconds,
+      ],
     ];
   }
 

x-pack/plugins/monitoring/server/config.test.ts

@@ -8,23 +8,6 @@
 import fs from 'fs';
 import { configSchema, createConfig } from './config';
 
-const MOCKED_PATHS = [
-  '/proc/self/cgroup',
-  'packages/kbn-dev-utils/certs/ca.crt',
-  'packages/kbn-dev-utils/certs/elasticsearch.crt',
-  'packages/kbn-dev-utils/certs/elasticsearch.key',
-];
-
-beforeEach(() => {
-  jest.spyOn(fs, 'readFileSync').mockImplementation((path, enc) => {
-    if (typeof path === 'string' && MOCKED_PATHS.includes(path) && enc === 'utf8') {
-      return `contents-of-${path}`;
-    }
-
-    throw new Error(`unpexpected arguments to fs.readFileSync: ${path}, ${enc}`);
-  });
-});
-
 describe('config schema', () => {
   it('generates proper defaults', () => {
     expect(configSchema.validate({})).toMatchInlineSnapshot(`
@@ -99,6 +82,11 @@ describe('config schema', () => {
             },
           },
           "enabled": true,
+          "kibana": Object {
+            "reporting": Object {
+              "stale_status_threshold_seconds": 120,
+            },
+          },
           "logs": Object {
             "index": "filebeat-*",
           },
@@ -115,6 +103,22 @@ describe('config schema', () => {
 });
 
 describe('createConfig()', () => {
+  const MOCKED_PATHS = [
+    'packages/kbn-dev-utils/certs/ca.crt',
+    'packages/kbn-dev-utils/certs/elasticsearch.crt',
+    'packages/kbn-dev-utils/certs/elasticsearch.key',
+  ];
+
+  beforeEach(() => {
+    jest.spyOn(fs, 'readFileSync').mockImplementation((path, enc) => {
+      if (typeof path === 'string' && MOCKED_PATHS.includes(path) && enc === 'utf8') {
+        return `contents-of-${path}`;
+      }
+
+      throw new Error(`unpexpected arguments to fs.readFileSync: ${path}, ${enc}`);
+    });
+  });
+
   it('should wrap in Elasticsearch config', async () => {
     const config = createConfig(
       configSchema.validate({

x-pack/plugins/monitoring/server/config.ts

@@ -35,6 +35,11 @@ export const configSchema = schema.object({
     }),
     max_bucket_size: schema.number({ defaultValue: 10000 }),
     elasticsearch: monitoringElasticsearchConfigSchema,
+    kibana: schema.object({
+      reporting: schema.object({
+        stale_status_threshold_seconds: schema.number({ defaultValue: 120 }),
+      }),
+    }),
     container: schema.object({
       elasticsearch: schema.object({
         enabled: schema.boolean({ defaultValue: false }),

x-pack/plugins/monitoring/server/index.ts

@@ -28,6 +28,11 @@ export const config: PluginConfigDescriptor<TypeOf<typeof configSchema>> = {
       ccs: {
         enabled: true,
       },
+      kibana: {
+        reporting: {
+          stale_status_threshold_seconds: true,
+        },
+      },
     },
     kibana: true,
   },

x-pack/plugins/monitoring/server/lib/kibana/get_kibana_info.test.js

@@ -8,6 +8,22 @@
 import moment from 'moment';
 import { handleResponse } from './get_kibana_info';
 
+jest.mock('../../static_globals', () => ({
+  Globals: {
+    app: {
+      config: {
+        ui: {
+          kibana: {
+            reporting: {
+              stale_status_threshold_seconds: 120,
+            },
+          },
+        },
+      },
+    },
+  },
+}));
+
 describe('get_kibana_info', () => {
   // TODO: test was not running before and is not up to date
   it.skip('return undefined for empty response', () => {
@@ -16,13 +32,14 @@ describe('get_kibana_info', () => {
   });
 
   it('return mapped data for result with hits, availability = true', () => {
+    const timestamp = moment().format();
     const result = handleResponse({
       hits: {
         hits: [
           {
             _source: {
               kibana_stats: {
-                timestamp: moment().format(),
+                timestamp,
                 kibana: {
                   data: 123,
                 },
@@ -31,6 +48,9 @@ describe('get_kibana_info', () => {
                     free_in_bytes: 123000,
                   },
                 },
+                process: {
+                  uptime_in_millis: 3000,
+                },
               },
             },
           },
@@ -38,20 +58,23 @@ describe('get_kibana_info', () => {
       },
     });
     expect(result).toEqual({
-      availability: true,
+      lastSeenTimestamp: timestamp,
+      statusIsStale: false,
       data: 123,
       os_memory_free: 123000,
+      uptime: 3000,
     });
   });
 
   it('return mapped data for result with hits, availability = false', () => {
+    const timestamp = moment().subtract(11, 'minutes').format();
     const result = handleResponse({
       hits: {
         hits: [
           {
             _source: {
               kibana_stats: {
-                timestamp: moment().subtract(11, 'minutes').format(),
+                timestamp,
                 kibana: {
                   data: 123,
                 },
@@ -60,6 +83,9 @@ describe('get_kibana_info', () => {
                     free_in_bytes: 123000,
                   },
                 },
+                process: {
+                  uptime_in_millis: 3000,
+                },
               },
             },
           },
@@ -67,9 +93,11 @@ describe('get_kibana_info', () => {
       },
     });
     expect(result).toEqual({
-      availability: false,
+      lastSeenTimestamp: timestamp,
+      statusIsStale: true,
       data: 123,
       os_memory_free: 123000,
+      uptime: 3000,
     });
   });
 });

x-pack/plugins/monitoring/server/lib/kibana/get_kibana_info.ts

@@ -6,27 +6,26 @@
  */
 
 import { merge } from 'lodash';
-// @ts-ignore
-import { MissingRequiredError } from '../error_missing_required';
-// @ts-ignore
-import { calculateAvailability } from '../calculate_availability';
-import { LegacyRequest } from '../../types';
 import { ElasticsearchResponse } from '../../../common/types/es';
-import { getNewIndexPatterns } from '../cluster/get_index_patterns';
 import { Globals } from '../../static_globals';
+import { LegacyRequest } from '../../types';
+import { getNewIndexPatterns } from '../cluster/get_index_patterns';
+import { MissingRequiredError } from '../error_missing_required';
 import { buildKibanaInfo } from './build_kibana_info';
+import { isKibanaStatusStale } from './is_kibana_status_stale';
 
 export function handleResponse(resp: ElasticsearchResponse) {
   const hit = resp.hits?.hits[0];
   const legacySource = hit?._source.kibana_stats;
   const mbSource = hit?._source.kibana?.stats;
-  const availabilityTimestamp = hit?._source['@timestamp'] ?? legacySource?.timestamp;
+  const lastSeenTimestamp = hit?._source['@timestamp'] ?? legacySource?.timestamp;
-  if (!availabilityTimestamp) {
+  if (!lastSeenTimestamp) {
     throw new MissingRequiredError('timestamp');
   }
 
   return merge(buildKibanaInfo(hit!), {
-    availability: calculateAvailability(availabilityTimestamp),
+    statusIsStale: isKibanaStatusStale(lastSeenTimestamp),
+    lastSeenTimestamp,
     os_memory_free: mbSource?.os?.memory?.free_in_bytes ?? legacySource?.os?.memory?.free_in_bytes,
     uptime: mbSource?.process?.uptime?.ms ?? legacySource?.process?.uptime_in_millis,
   });

x-pack/plugins/monitoring/server/lib/kibana/get_kibanas.ts

@@ -6,17 +6,14 @@
  */
 
 import moment from 'moment';
-// @ts-ignore
+import { ElasticsearchResponse, ElasticsearchResponseHit } from '../../../common/types/es';
-import { createQuery } from '../create_query';
+import { Globals } from '../../static_globals';
-// @ts-ignore
-import { calculateAvailability } from '../calculate_availability';
-// @ts-ignore
-import { KibanaMetric } from '../metrics';
 import { LegacyRequest } from '../../types';
 import { getNewIndexPatterns } from '../cluster/get_index_patterns';
-import { Globals } from '../../static_globals';
+import { createQuery } from '../create_query';
-import { ElasticsearchResponse, ElasticsearchResponseHit } from '../../../common/types/es';
+import { KibanaMetric } from '../metrics';
-import { KibanaInfo, buildKibanaInfo } from './build_kibana_info';
+import { buildKibanaInfo, KibanaInfo } from './build_kibana_info';
+import { isKibanaStatusStale } from './is_kibana_status_stale';
 
 interface Kibana {
   process?: {
@@ -38,7 +35,8 @@ interface Kibana {
   };
   concurrent_connections?: number;
   kibana?: KibanaInfo;
-  availability: boolean;
+  statusIsStale: boolean;
+  lastSeenTimestamp: string;
 }
 
 /*
@@ -120,6 +118,8 @@ export async function getKibanas(req: LegacyRequest, { clusterUuid }: { clusterU
     const legacyStats = hit._source.kibana_stats;
     const mbStats = hit._source.kibana?.stats;
 
+    const lastSeenTimestamp = hit._source['@timestamp'] ?? hit._source.timestamp;
+
     const kibana: Kibana = {
       kibana: buildKibanaInfo(hit),
       concurrent_connections:
@@ -143,7 +143,8 @@ export async function getKibanas(req: LegacyRequest, { clusterUuid }: { clusterU
       requests: {
         total: mbStats?.request?.total ?? legacyStats?.requests?.total,
       },
-      availability: calculateAvailability(hit._source['@timestamp'] ?? hit._source.timestamp),
+      statusIsStale: isKibanaStatusStale(lastSeenTimestamp),
+      lastSeenTimestamp,
     };
     return kibana;
   });

x-pack/plugins/monitoring/server/lib/kibana/get_kibanas_for_clusters.ts

@@ -6,11 +6,12 @@
  */
 
 import { chain, find } from 'lodash';
-import { LegacyRequest, Cluster, Bucket } from '../../types';
+import { Globals } from '../../static_globals';
+import { Bucket, Cluster, LegacyRequest } from '../../types';
+import { getNewIndexPatterns } from '../cluster/get_index_patterns';
 import { createQuery } from '../create_query';
 import { KibanaClusterMetric } from '../metrics';
-import { getNewIndexPatterns } from '../cluster/get_index_patterns';
+import { isKibanaStatusStale } from './is_kibana_status_stale';
-import { Globals } from '../../static_globals';
 
 /*
  * Get high-level info for Kibanas in a set of clusters
@@ -74,6 +75,11 @@ export function getKibanasForClusters(req: LegacyRequest, clusters: Cluster[], c
                     },
                   },
                   aggs: {
+                    last_seen: {
+                      max: {
+                        field: 'kibana_stats.timestamp',
+                      },
+                    },
                     response_time_max: {
                       max: {
                         field: 'kibana_stats.response_times.max',
@@ -185,6 +191,7 @@ export function getKibanasForClusters(req: LegacyRequest, clusters: Cluster[], c
         let responseTime = 0;
         let memorySize = 0;
         let memoryLimit = 0;
+        let someStatusIsStale = true;
 
         // if the cluster has kibana instances at all
         if (kibanaUuids.length) {
@@ -204,6 +211,8 @@ export function getKibanasForClusters(req: LegacyRequest, clusters: Cluster[], c
           responseTime = aggregations.response_time_max?.value;
           memorySize = aggregations.memory_rss?.value;
           memoryLimit = aggregations.memory_heap_size_limit?.value;
+
+          someStatusIsStale = kibanaUuids.some(hasStaleStatus);
         }
 
         return {
@@ -211,6 +220,7 @@ export function getKibanasForClusters(req: LegacyRequest, clusters: Cluster[], c
           stats: {
             uuids: kibanaUuids.map(({ key }: Bucket) => key),
             status,
+            some_status_is_stale: someStatusIsStale,
             requests_total: requestsTotal,
             concurrent_connections: connections,
             response_time_max: responseTime,
@@ -223,3 +233,19 @@ export function getKibanasForClusters(req: LegacyRequest, clusters: Cluster[], c
     })
   );
 }
+
+function hasStaleStatus(kibana: any) {
+  const buckets: any[] = kibana?.latest_report?.buckets ?? [];
+
+  if (buckets.length === 0) {
+    return true;
+  }
+
+  const lastSeenTimestamp: string | null = buckets[0]?.last_seen?.value_as_string ?? null;
+
+  if (lastSeenTimestamp === null) {
+    return true;
+  }
+
+  return isKibanaStatusStale(lastSeenTimestamp);
+}

x-pack/plugins/monitoring/server/lib/kibana/is_kibana_status_stale.ts

@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import moment from 'moment';
+import { Globals } from '../../static_globals';
+
+export function isKibanaStatusStale(lastSeenTimestamp: string) {
+  const lastSeen = moment(lastSeenTimestamp);
+  const staleThreshold = moment().subtract(
+    Globals.app.config.ui.kibana.reporting.stale_status_threshold_seconds,
+    'seconds'
+  );
+  return staleThreshold.isAfter(lastSeen);
+}

x-pack/plugins/translations/translations/fr-FR.json

@@ -20508,7 +20508,6 @@
     "xpack.monitoring.kibana.listing.nameColumnTitle": "Nom",
     "xpack.monitoring.kibana.listing.requestsColumnTitle": "Demandes",
     "xpack.monitoring.kibana.listing.responseTimeColumnTitle": "Temps de réponse",
-    "xpack.monitoring.kibana.listing.statusColumnTitle": "Statut",
     "xpack.monitoring.kibana.overview.pageTitle": "Aperçu Kibana",
     "xpack.monitoring.kibana.overview.title": "Kibana",
     "xpack.monitoring.kibana.shardActivity.bytesTitle": "Octets",

x-pack/plugins/translations/translations/ja-JP.json

@@ -20648,7 +20648,6 @@
     "xpack.monitoring.kibana.listing.nameColumnTitle": "名前",
     "xpack.monitoring.kibana.listing.requestsColumnTitle": "リクエスト",
     "xpack.monitoring.kibana.listing.responseTimeColumnTitle": "応答時間",
-    "xpack.monitoring.kibana.listing.statusColumnTitle": "ステータス",
     "xpack.monitoring.kibana.overview.pageTitle": "Kibanaの概要",
     "xpack.monitoring.kibana.overview.title": "Kibana",
     "xpack.monitoring.kibana.shardActivity.bytesTitle": "バイト",

x-pack/plugins/translations/translations/zh-CN.json

@@ -20679,7 +20679,6 @@
     "xpack.monitoring.kibana.listing.nameColumnTitle": "名称",
     "xpack.monitoring.kibana.listing.requestsColumnTitle": "请求",
     "xpack.monitoring.kibana.listing.responseTimeColumnTitle": "响应时间",
-    "xpack.monitoring.kibana.listing.statusColumnTitle": "状态",
     "xpack.monitoring.kibana.overview.pageTitle": "Kibana 概览",
     "xpack.monitoring.kibana.overview.title": "Kibana",
     "xpack.monitoring.kibana.shardActivity.bytesTitle": "字节",

x-pack/test/api_integration/apis/monitoring/cluster/fixtures/multicluster.json

@@ -59,6 +59,7 @@
     },
     "kibana": {
       "status": null,
+      "some_status_is_stale": true,
       "requests_total": 0,
       "concurrent_connections": 0,
       "response_time_max": 0,
@@ -170,6 +171,7 @@
     },
     "kibana": {
       "status": null,
+      "some_status_is_stale": true,
       "requests_total": 0,
       "concurrent_connections": 0,
       "response_time_max": 0,
@@ -283,6 +285,7 @@
     },
     "kibana": {
       "status": "green",
+      "some_status_is_stale": true,
       "requests_total": 571,
       "concurrent_connections": 307,
       "response_time_max": 1930,

x-pack/test/api_integration/apis/monitoring/cluster/fixtures/overview.json

@@ -75,6 +75,7 @@
     },
     "kibana": {
       "status": "green",
+      "some_status_is_stale": true,
       "requests_total": 914,
       "concurrent_connections": 646,
       "response_time_max": 2873,

x-pack/test/api_integration/apis/monitoring/kibana/fixtures/instance.json

@@ -645,7 +645,8 @@
     "transport_address": "tsullivan.local:5601",
     "uuid": "de3b8f2a-7bb9-4931-9bf3-997ba7824cf9",
     "version": "7.0.0-alpha1",
-    "availability": false,
+    "lastSeenTimestamp": "2017-08-29T17:25:43.192Z",
+    "statusIsStale": true,
     "os_memory_free": 1645989888,
     "uptime": 200102
   }

x-pack/test/api_integration/apis/monitoring/kibana/fixtures/listing.json

@@ -4,6 +4,7 @@
       "de3b8f2a-7bb9-4931-9bf3-997ba7824cf9"
     ],
     "status": "green",
+    "some_status_is_stale": true,
     "requests_total": 174,
     "concurrent_connections": 174,
     "response_time_max": 2203,
@@ -38,7 +39,7 @@
         "uuid": "de3b8f2a-7bb9-4931-9bf3-997ba7824cf9",
         "status": "green"
       },
-      "availability": false
+      "statusIsStale": true
     }
   ]
 }

x-pack/test/api_integration/apis/monitoring/kibana/fixtures/overview.json

@@ -7,6 +7,7 @@
     "requests_total": 174,
     "response_time_max": 2203,
     "status": "green",
+    "some_status_is_stale": true,
     "uuids": [
       "de3b8f2a-7bb9-4931-9bf3-997ba7824cf9"
     ]

x-pack/test/api_integration/apis/monitoring/kibana/listing.js

@@ -34,7 +34,13 @@ export default function ({ getService }) {
         .send({ timeRange })
         .expect(200);
 
+      // Fixture is shared between internal and Metricbeat collection tests
+      // But timestamps of documents differ by a few miliseconds
+      const lastSeenTimestamp = body.kibanas[0].lastSeenTimestamp;
+      delete body.kibanas[0].lastSeenTimestamp;
+
       expect(body).to.eql(listingFixture);
+      expect(lastSeenTimestamp).to.eql('2017-08-29T17:25:47.825Z');
     });
   });
 }

x-pack/test/api_integration/apis/monitoring/kibana/listing_mb.js

@@ -36,7 +36,13 @@ export default function ({ getService }) {
         .send({ timeRange })
         .expect(200);
 
+      // Fixture is shared between internal and Metricbeat collection tests
+      // But timestamps of documents differ by a few miliseconds
+      const lastSeenTimestamp = body.kibanas[0].lastSeenTimestamp;
+      delete body.kibanas[0].lastSeenTimestamp;
+
       expect(body).to.eql(listingFixture);
+      expect(lastSeenTimestamp).to.eql('2017-08-29T17:25:43.192Z');
     });
   });
 }

x-pack/test/api_integration/apis/monitoring/standalone_cluster/fixtures/clusters.json

@@ -59,6 +59,7 @@
     },
     "kibana": {
       "status": "green",
+      "some_status_is_stale": true,
       "requests_total": 42,
       "concurrent_connections": 0,
       "response_time_max": 864,
@@ -148,6 +149,7 @@
     },
     "kibana": {
       "status": null,
+      "some_status_is_stale": true,
       "requests_total": 0,
       "concurrent_connections": 0,
       "response_time_max": 0,

x-pack/test/functional/apps/monitoring/cluster/overview.js

@@ -105,7 +105,7 @@ export default function ({ getService, getPageObjects }) {
       });
 
       it('shows kibana panel', async () => {
-        expect(await overview.getKbnStatus()).to.be('Healthy');
+        expect(await overview.getKbnStatus()).to.be('Stale');
         expect(await overview.getKbnRequests()).to.be('174');
         expect(await overview.getKbnMaxResponseTime()).to.be('2203 ms');
         expect(await overview.getKbnInstances()).to.be('Instances: 1');

x-pack/test/functional/apps/monitoring/kibana/instance.js

@@ -43,7 +43,7 @@ export default function ({ getService, getPageObjects }) {
         osFreeMemory: 'OS Free Memory\n1.5 GB',
         version: 'Version\n7.0.0-alpha1',
         uptime: 'Uptime\n3 minutes',
-        health: 'Health: green',
+        health: 'Last Reported Status\n Green\nStale',
       });
     });
   });

x-pack/test/functional/apps/monitoring/kibana/instance_mb.js

@@ -47,7 +47,7 @@ export default function ({ getService, getPageObjects }) {
         osFreeMemory: 'OS Free Memory\n1.5 GB',
         version: 'Version\n7.0.0-alpha1',
         uptime: 'Uptime\n3 minutes',
-        health: 'Health: green',
+        health: 'Last Reported Status\n Green\nStale',
       });
     });
   });

x-pack/test/functional/apps/monitoring/kibana/instances.js

@@ -40,7 +40,7 @@ export default function ({ getService, getPageObjects }) {
         requests: 'Requests\n174',
         connections: 'Connections\n174',
         maxResponseTime: 'Max. Response Time\n2203 ms',
-        health: 'Health: green',
+        health: 'Status\nStale',
       });
     });
   });

x-pack/test/functional/apps/monitoring/kibana/instances_mb.js

@@ -45,7 +45,7 @@ export default function ({ getService, getPageObjects }) {
         requests: 'Requests\n174',
         connections: 'Connections\n174',
         maxResponseTime: 'Max. Response Time\n2203 ms',
-        health: 'Health: green',
+        health: 'Status\nStale',
       });
     });
   });

x-pack/test/functional/apps/monitoring/kibana/overview.js

@@ -40,7 +40,7 @@ export default function ({ getService, getPageObjects }) {
         requests: 'Requests\n174',
         connections: 'Connections\n174',
         maxResponseTime: 'Max. Response Time\n2203 ms',
-        health: 'Health: green',
+        health: 'Status\nStale\nView all instances',
       });
     });
   });

x-pack/test/functional/apps/monitoring/kibana/overview_mb.js

@@ -44,7 +44,7 @@ export default function ({ getService, getPageObjects }) {
         requests: 'Requests\n174',
         connections: 'Connections\n174',
         maxResponseTime: 'Max. Response Time\n2203 ms',
-        health: 'Health: green',
+        health: 'Status\nStale\nView all instances',
       });
     });
   });

x-pack/test/functional/services/monitoring/cluster_overview.js

@@ -33,7 +33,7 @@ export function MonitoringClusterOverviewProvider({ getService }) {
   const SUBJ_ES_ML_JOBS = `${SUBJ_ES_PANEL} > esMlJobs`;
 
   const SUBJ_KBN_PANEL = `${SUBJ_CLUSTER_ITEM_CONTAINER_PREFIX}Kibana`;
-  const SUBJ_KBN_STATUS = `${SUBJ_KBN_PANEL} > statusIcon`;
+  const SUBJ_KBN_STATUS = `${SUBJ_KBN_PANEL} > status`;
   const SUBJ_KBN_REQUESTS = `${SUBJ_KBN_PANEL} > kbnRequests`;
   const SUBJ_KBN_MAX_RESPONSE_TIME = `${SUBJ_KBN_PANEL} > kbnMaxResponseTime`;
   const SUBJ_KBN_CONNECTIONS = `${SUBJ_KBN_PANEL} > kbnConnections`;

x-pack/test/functional/services/monitoring/kibana_instance.js

@@ -16,7 +16,7 @@ export function MonitoringKibanaInstanceProvider({ getService }) {
   const SUBJ_SUMMARY_OS_FREE_MEMORY = `${SUBJ_SUMMARY} > osFreeMemory`;
   const SUBJ_SUMMARY_VERSION = `${SUBJ_SUMMARY} > version`;
   const SUBJ_SUMMARY_UPTIME = `${SUBJ_SUMMARY} > uptime`;
-  const SUBJ_SUMMARY_HEALTH = `${SUBJ_SUMMARY} > statusIcon`;
+  const SUBJ_SUMMARY_HEALTH = `${SUBJ_SUMMARY} > status`;
 
   return new (class KibanaInstance {
     async isOnInstance() {
@@ -30,7 +30,7 @@ export function MonitoringKibanaInstanceProvider({ getService }) {
         osFreeMemory: await testSubjects.getVisibleText(SUBJ_SUMMARY_OS_FREE_MEMORY),
         version: await testSubjects.getVisibleText(SUBJ_SUMMARY_VERSION),
         uptime: await testSubjects.getVisibleText(SUBJ_SUMMARY_UPTIME),
-        health: await testSubjects.getAttribute(SUBJ_SUMMARY_HEALTH, 'alt'),
+        health: await testSubjects.getVisibleText(SUBJ_SUMMARY_HEALTH),
       };
     }
   })();

x-pack/test/functional/services/monitoring/kibana_summary_status.js

@@ -14,7 +14,7 @@ export function MonitoringKibanaSummaryStatusProvider({ getService }) {
   const SUBJ_SUMMARY_REQUESTS = `${SUBJ_SUMMARY} > requests`;
   const SUBJ_SUMMARY_CONNECTIONS = `${SUBJ_SUMMARY} > connections`;
   const SUBJ_SUMMARY_MAX_RESPONSE_TIME = `${SUBJ_SUMMARY} > maxResponseTime`;
-  const SUBJ_SUMMARY_HEALTH = `${SUBJ_SUMMARY} > statusIcon`;
+  const SUBJ_SUMMARY_HEALTH = `${SUBJ_SUMMARY} > status`;
 
   return new (class KibanaSummaryStatus {
     async getContent() {
@@ -24,7 +24,7 @@ export function MonitoringKibanaSummaryStatusProvider({ getService }) {
         requests: await testSubjects.getVisibleText(SUBJ_SUMMARY_REQUESTS),
         connections: await testSubjects.getVisibleText(SUBJ_SUMMARY_CONNECTIONS),
         maxResponseTime: await testSubjects.getVisibleText(SUBJ_SUMMARY_MAX_RESPONSE_TIME),
-        health: await testSubjects.getAttribute(SUBJ_SUMMARY_HEALTH, 'alt'),
+        health: await testSubjects.getVisibleText(SUBJ_SUMMARY_HEALTH),
       };
     }
   })();