[8.13] [Fleet] Fix status summary when showUpgradeable is selected (#177618) (#177741)

# Backport

This will backport the following commits from `main` to `8.13`:
- [[Fleet] Fix status summary when showUpgradeable is selected
(#177618)](https://github.com/elastic/kibana/pull/177618)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Cristina
Amico","email":"criamico@users.noreply.github.com"},"sourceCommit":{"committedDate":"2024-02-23T16:56:01Z","message":"[Fleet]
Fix status summary when showUpgradeable is selected (#177618)\n\nFixes
https://github.com/elastic/kibana/issues/159446\r\n\r\n##
Summary\r\n\r\nFix status summary in agents list. The problem is not
actually in the UI\r\nbut in the `GET api/fleet/agents` endpoint,
specifically when querying\r\nwith `getStatusSummary` and
`showUpgradeable` (query used in the UI to\r\nshow the status bar above
the
table:\r\n```\r\n/api/fleet/agents?getStatusSummary=true&perPage=5&showUpgradeable=true\r\n```\r\nThis
codepath was not taking into account the upgradeable agents so
the\r\nresults were wrong, both in the API and in the table
header:\r\n![Screenshot 2024-02-21 at 10
38\r\n19](d66a0e55-4e14-4874-8264-757eba9017eb)\r\n\r\n###
Testing\r\nEnroll some agents and make sure that at least one of them
is\r\nupgradeable (version < fleet server). Then select the
`upgrade\r\navailable` button on top and check that the values shown in
the summary\r\nupdate correctly.\r\n\r\n### Checklist\r\n\r\n- [ ] [Unit
or
functional\r\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\r\nwere
updated or added to match the most common
scenarios\r\n\r\n---------\r\n\r\nCo-authored-by: Kibana Machine
<42973632+kibanamachine@users.noreply.github.com>","sha":"a60617e9f1f6a68a2ee00b71be45108480fea1fb","branchLabelMapping":{"^v8.14.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:fix","Team:Fleet","backport:prev-minor","v8.14.0"],"title":"[Fleet]
Fix status summary when showUpgradeable is
selected","number":177618,"url":"https://github.com/elastic/kibana/pull/177618","mergeCommit":{"message":"[Fleet]
Fix status summary when showUpgradeable is selected (#177618)\n\nFixes
https://github.com/elastic/kibana/issues/159446\r\n\r\n##
Summary\r\n\r\nFix status summary in agents list. The problem is not
actually in the UI\r\nbut in the `GET api/fleet/agents` endpoint,
specifically when querying\r\nwith `getStatusSummary` and
`showUpgradeable` (query used in the UI to\r\nshow the status bar above
the
table:\r\n```\r\n/api/fleet/agents?getStatusSummary=true&perPage=5&showUpgradeable=true\r\n```\r\nThis
codepath was not taking into account the upgradeable agents so
the\r\nresults were wrong, both in the API and in the table
header:\r\n![Screenshot 2024-02-21 at 10
38\r\n19](d66a0e55-4e14-4874-8264-757eba9017eb)\r\n\r\n###
Testing\r\nEnroll some agents and make sure that at least one of them
is\r\nupgradeable (version < fleet server). Then select the
`upgrade\r\navailable` button on top and check that the values shown in
the summary\r\nupdate correctly.\r\n\r\n### Checklist\r\n\r\n- [ ] [Unit
or
functional\r\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\r\nwere
updated or added to match the most common
scenarios\r\n\r\n---------\r\n\r\nCo-authored-by: Kibana Machine
<42973632+kibanamachine@users.noreply.github.com>","sha":"a60617e9f1f6a68a2ee00b71be45108480fea1fb"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"main","label":"v8.14.0","branchLabelMappingKey":"^v8.14.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/177618","number":177618,"mergeCommit":{"message":"[Fleet]
Fix status summary when showUpgradeable is selected (#177618)\n\nFixes
https://github.com/elastic/kibana/issues/159446\r\n\r\n##
Summary\r\n\r\nFix status summary in agents list. The problem is not
actually in the UI\r\nbut in the `GET api/fleet/agents` endpoint,
specifically when querying\r\nwith `getStatusSummary` and
`showUpgradeable` (query used in the UI to\r\nshow the status bar above
the
table:\r\n```\r\n/api/fleet/agents?getStatusSummary=true&perPage=5&showUpgradeable=true\r\n```\r\nThis
codepath was not taking into account the upgradeable agents so
the\r\nresults were wrong, both in the API and in the table
header:\r\n![Screenshot 2024-02-21 at 10
38\r\n19](d66a0e55-4e14-4874-8264-757eba9017eb)\r\n\r\n###
Testing\r\nEnroll some agents and make sure that at least one of them
is\r\nupgradeable (version < fleet server). Then select the
`upgrade\r\navailable` button on top and check that the values shown in
the summary\r\nupdate correctly.\r\n\r\n### Checklist\r\n\r\n- [ ] [Unit
or
functional\r\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\r\nwere
updated or added to match the most common
scenarios\r\n\r\n---------\r\n\r\nCo-authored-by: Kibana Machine
<42973632+kibanamachine@users.noreply.github.com>","sha":"a60617e9f1f6a68a2ee00b71be45108480fea1fb"}}]}]
BACKPORT-->

Co-authored-by: Cristina Amico <criamico@users.noreply.github.com>

x-pack/plugins/fleet/common/constants/agent.ts

@@ -35,3 +35,15 @@ export const FleetServerAgentComponentStatuses = [
   'STOPPING',
   'STOPPED',
 ] as const;
+
+export const AgentStatuses = [
+  'offline',
+  'error',
+  'online',
+  'inactive',
+  'enrolling',
+  'unenrolling',
+  'unenrolled',
+  'updating',
+  'degraded',
+] as const;

x-pack/plugins/fleet/common/types/models/agent.ts

@@ -10,6 +10,7 @@ import type {
   AGENT_TYPE_PERMANENT,
   AGENT_TYPE_TEMPORARY,
   FleetServerAgentComponentStatuses,
+  AgentStatuses,
 } from '../../constants';
 
 export type AgentType =
@@ -17,16 +18,8 @@ export type AgentType =
   | typeof AGENT_TYPE_PERMANENT
   | typeof AGENT_TYPE_TEMPORARY;
 
-export type AgentStatus =
-  | 'offline'
-  | 'error'
-  | 'online'
-  | 'inactive'
-  | 'enrolling'
-  | 'unenrolling'
-  | 'unenrolled'
-  | 'updating'
-  | 'degraded';
+type AgentStatusTuple = typeof AgentStatuses;
+export type AgentStatus = AgentStatusTuple[number];
 
 export type SimplifiedAgentStatus =
   | 'healthy'

x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_list_page/hooks/use_fetch_agents_data.tsx

@@ -33,7 +33,7 @@ export function useFetchAgentsData() {
   const { displayAgentMetrics } = ExperimentalFeaturesService.get();
 
   const { notifications } = useStartServices();
-  // useBreadcrumbs('agent_list');
+
   const history = useHistory();
   const { urlParams, toUrlParams } = useUrlParams();
   const defaultKuery: string = (urlParams.kuery as string) || '';

x-pack/plugins/fleet/server/services/agents/crud.test.ts

@@ -12,6 +12,7 @@ import { AGENTS_INDEX } from '../../constants';
 import { createAppContextStartContractMock } from '../../mocks';
 import type { Agent } from '../../types';
 import { appContextService } from '../app_context';
+import type { AgentStatus } from '../../../common/types';
 
 import { auditLoggingService } from '../audit_logging';
 
@@ -57,7 +58,7 @@ describe('Agents CRUD test', () => {
     appContextService.start(mockContract);
   });
 
-  function getEsResponse(ids: string[], total: number) {
+  function getEsResponse(ids: string[], total: number, status: AgentStatus) {
     return {
       hits: {
         total,
@@ -65,7 +66,7 @@ describe('Agents CRUD test', () => {
           _id: id,
           _source: {},
           fields: {
-            status: ['inactive'],
+            status: [status],
           },
         })),
       },
@@ -162,9 +163,11 @@ describe('Agents CRUD test', () => {
   describe('getAgentsByKuery', () => {
     it('should return upgradeable on first page', async () => {
       searchMock
-        .mockImplementationOnce(() => Promise.resolve(getEsResponse(['1', '2', '3', '4', '5'], 7)))
         .mockImplementationOnce(() =>
-          Promise.resolve(getEsResponse(['1', '2', '3', '4', '5', 'up', '7'], 7))
+          Promise.resolve(getEsResponse(['1', '2', '3', '4', '5'], 7, 'inactive'))
+        )
+        .mockImplementationOnce(() =>
+          Promise.resolve(getEsResponse(['1', '2', '3', '4', '5', 'up', '7'], 7, 'inactive'))
         );
       const result = await getAgentsByKuery(esClientMock, soClientMock, {
         showUpgradeable: true,
@@ -191,9 +194,11 @@ describe('Agents CRUD test', () => {
 
     it('should return upgradeable from all pages', async () => {
       searchMock
-        .mockImplementationOnce(() => Promise.resolve(getEsResponse(['1', '2', '3', 'up', '5'], 7)))
         .mockImplementationOnce(() =>
-          Promise.resolve(getEsResponse(['1', '2', '3', 'up', '5', 'up2', '7'], 7))
+          Promise.resolve(getEsResponse(['1', '2', '3', 'up', '5'], 7, 'inactive'))
+        )
+        .mockImplementationOnce(() =>
+          Promise.resolve(getEsResponse(['1', '2', '3', 'up', '5', 'up2', '7'], 7, 'inactive'))
         );
       const result = await getAgentsByKuery(esClientMock, soClientMock, {
         showUpgradeable: true,
@@ -227,9 +232,11 @@ describe('Agents CRUD test', () => {
 
     it('should return upgradeable on second page', async () => {
       searchMock
-        .mockImplementationOnce(() => Promise.resolve(getEsResponse(['up6', '7'], 7)))
+        .mockImplementationOnce(() => Promise.resolve(getEsResponse(['up6', '7'], 7, 'inactive')))
         .mockImplementationOnce(() =>
-          Promise.resolve(getEsResponse(['up1', 'up2', 'up3', 'up4', 'up5', 'up6', '7'], 7))
+          Promise.resolve(
+            getEsResponse(['up1', 'up2', 'up3', 'up4', 'up5', 'up6', '7'], 7, 'inactive')
+          )
         );
       const result = await getAgentsByKuery(esClientMock, soClientMock, {
         showUpgradeable: true,
@@ -256,7 +263,7 @@ describe('Agents CRUD test', () => {
 
     it('should return upgradeable from one page when total is more than limit', async () => {
       searchMock.mockImplementationOnce(() =>
-        Promise.resolve(getEsResponse(['1', '2', '3', 'up', '5'], 10001))
+        Promise.resolve(getEsResponse(['1', '2', '3', 'up', '5'], 10001, 'inactive'))
       );
       const result = await getAgentsByKuery(esClientMock, soClientMock, {
         showUpgradeable: true,
@@ -281,8 +288,79 @@ describe('Agents CRUD test', () => {
       });
     });
 
+    it('should return correct status summary when showUpgradeable is selected and total is less than limit', async () => {
+      searchMock.mockImplementationOnce(() =>
+        Promise.resolve(getEsResponse(['1', '2', '3', 'up', '5'], 100, 'updating'))
+      );
+      searchMock.mockImplementationOnce(() =>
+        Promise.resolve(getEsResponse(['1', '2', '3', 'up', '5'], 100, 'updating'))
+      );
+      const result = await getAgentsByKuery(esClientMock, soClientMock, {
+        showUpgradeable: true,
+        showInactive: false,
+        getStatusSummary: true,
+        page: 1,
+        perPage: 5,
+      });
+
+      expect(result).toEqual(
+        expect.objectContaining({
+          page: 1,
+          perPage: 5,
+          statusSummary: {
+            degraded: 0,
+            enrolling: 0,
+            error: 0,
+            inactive: 0,
+            offline: 0,
+            online: 0,
+            unenrolled: 0,
+            unenrolling: 0,
+            updating: 1,
+          },
+          total: 1,
+        })
+      );
+    });
+
+    it('should return correct status summary when showUpgradeable is selected and total is more than limit', async () => {
+      searchMock.mockImplementationOnce(() =>
+        Promise.resolve(getEsResponse(['1', '2', '3', 'up', '5'], 10001, 'updating'))
+      );
+      searchMock.mockImplementationOnce(() =>
+        Promise.resolve(getEsResponse(['1', '2', '3', 'up', '5'], 10001, 'updating'))
+      );
+      const result = await getAgentsByKuery(esClientMock, soClientMock, {
+        showUpgradeable: true,
+        showInactive: false,
+        getStatusSummary: true,
+        page: 1,
+        perPage: 5,
+      });
+      expect(result).toEqual(
+        expect.objectContaining({
+          page: 1,
+          perPage: 5,
+          statusSummary: {
+            degraded: 0,
+            enrolling: 0,
+            error: 0,
+            inactive: 0,
+            offline: 0,
+            online: 0,
+            unenrolled: 0,
+            unenrolling: 0,
+            updating: 1,
+          },
+          total: 10001,
+        })
+      );
+    });
+
     it('should return second page', async () => {
-      searchMock.mockImplementationOnce(() => Promise.resolve(getEsResponse(['6', '7'], 7)));
+      searchMock.mockImplementationOnce(() =>
+        Promise.resolve(getEsResponse(['6', '7'], 7, 'inactive'))
+      );
       const result = await getAgentsByKuery(esClientMock, soClientMock, {
         showUpgradeable: false,
         showInactive: false,
@@ -314,7 +392,9 @@ describe('Agents CRUD test', () => {
     });
 
     it('should pass secondary sort for default sort', async () => {
-      searchMock.mockImplementationOnce(() => Promise.resolve(getEsResponse(['1', '2'], 2)));
+      searchMock.mockImplementationOnce(() =>
+        Promise.resolve(getEsResponse(['1', '2'], 2, 'inactive'))
+      );
       await getAgentsByKuery(esClientMock, soClientMock, {
         showInactive: false,
       });
@@ -326,7 +406,9 @@ describe('Agents CRUD test', () => {
     });
 
     it('should not pass secondary sort for non-default sort', async () => {
-      searchMock.mockImplementationOnce(() => Promise.resolve(getEsResponse(['1', '2'], 2)));
+      searchMock.mockImplementationOnce(() =>
+        Promise.resolve(getEsResponse(['1', '2'], 2, 'inactive'))
+      );
       await getAgentsByKuery(esClientMock, soClientMock, {
         showInactive: false,
         sortField: 'policy_id',

x-pack/plugins/fleet/server/services/agents/crud.ts

@@ -350,9 +350,19 @@ export async function getAgentsByKuery(
   }
 
   if (getStatusSummary) {
-    res.aggregations?.status.buckets.forEach((bucket) => {
-      statusSummary[bucket.key] = bucket.doc_count;
-    });
+    if (showUpgradeable) {
+      // when showUpgradeable is selected, calculate the summary status manually from the upgradeable agents above
+      // the bucket count doesn't take in account the upgradeable agents
+      agents.forEach((agent) => {
+        if (!agent?.status) return;
+        if (!statusSummary[agent.status]) statusSummary[agent.status] = 0;
+        statusSummary[agent.status]++;
+      });
+    } else {
+      res.aggregations?.status.buckets.forEach((bucket) => {
+        statusSummary[bucket.key] = bucket.doc_count;
+      });
+    }
   }
 
   return {

x-pack/test/fleet_api_integration/apis/agents/list.ts

@@ -6,7 +6,7 @@
  */
 
 import expect from '@kbn/expect';
-import { type Agent, FLEET_ELASTIC_AGENT_PACKAGE } from '@kbn/fleet-plugin/common';
+import { type Agent, FLEET_ELASTIC_AGENT_PACKAGE, AGENTS_INDEX } from '@kbn/fleet-plugin/common';
 
 import { FtrProviderContext } from '../../../api_integration/ftr_provider_context';
 import { testUsers } from '../test_users';
@@ -236,5 +236,53 @@ export default function ({ getService }: FtrProviderContext) {
         updating: 0,
       });
     });
+
+    it('should return correct status summary if showUpgradeable is provided', async () => {
+      await es.update({
+        id: 'agent1',
+        refresh: 'wait_for',
+        index: AGENTS_INDEX,
+        body: {
+          doc: {
+            policy_revision_idx: 1,
+            last_checkin: new Date().toISOString(),
+            status: 'online',
+            local_metadata: { elastic: { agent: { upgradeable: true, version: '0.0.0' } } },
+          },
+        },
+      });
+      // 1 agent inactive
+      await es.update({
+        id: 'agent4',
+        refresh: 'wait_for',
+        index: AGENTS_INDEX,
+        body: {
+          doc: {
+            policy_id: 'policy-inactivity-timeout',
+            policy_revision_idx: 1,
+            last_checkin: new Date(Date.now() - 1000 * 60).toISOString(), // policy timeout 1 min
+            status: 'online',
+            local_metadata: { elastic: { agent: { upgradeable: true, version: '0.0.0' } } },
+          },
+        },
+      });
+
+      const { body: apiResponse } = await supertest
+        .get('/api/fleet/agents?getStatusSummary=true&perPage=5&showUpgradeable=true')
+        .set('kbn-xsrf', 'xxxx')
+        .expect(200);
+
+      expect(apiResponse.statusSummary).to.eql({
+        degraded: 0,
+        enrolling: 0,
+        error: 0,
+        inactive: 0,
+        offline: 0,
+        online: 2,
+        unenrolled: 0,
+        unenrolling: 0,
+        updating: 0,
+      });
+    });
   });
 }