[Fleet] Skip 10m wait after agent upgrade if agent cleared watching state (#179917)

## Summary

Closes https://github.com/elastic/kibana/issues/176910
Closes https://github.com/elastic/kibana/issues/179741

Updated agent upgradeable check to consider the new `UPG_WATCHING` state
in upgrade details. For agents version 8.12+, upgrade is immediately
allowed after watching state is cleared. For agents with lower version,
the 10m wait is preserved after upgrade.

To verify:
- enroll an agent with version 8.11.0
- upgrade to 8.11.4
- verify that the 10m wait is still there after upgrade before allowing
another upgrade
- upgrade to 8.12.2
- verify that the agent can't be upgraded while the watching state is
active
- verify that the agent can be upgraded again as soon as the watching
state clears (no more 10m wait for agents 8.12+)

<img width="1284" alt="image"
src="https://github.com/elastic/kibana/assets/90178898/243ee69d-fbef-4012-96b3-2c7ad75be5d2">
<img width="1307" alt="image"
src="https://github.com/elastic/kibana/assets/90178898/5708e356-2999-4493-bbed-b6bd7b6758c2">
<img width="1251" alt="image"
src="https://github.com/elastic/kibana/assets/90178898/fc52ac27-6d4b-4b16-b221-47f196950d0d">

<img width="1298" alt="image"
src="https://github.com/elastic/kibana/assets/90178898/91eec4af-7beb-4de7-bc8a-e80210adedea">



### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

x-pack/plugins/fleet/common/services/is_agent_upgradeable.test.ts

@@ -208,6 +208,23 @@ describe('Fleet - isAgentUpgradeable', () => {
       isAgentUpgradeable(getAgent({ version: '7.9.0', upgradeable: true, minutesSinceUpgrade: 11 }))
     ).toBe(true);
   });
+  it('returns false if the agent reports upgradeable but is in watching state', () => {
+    expect(
+      isAgentUpgradeable(
+        getAgent({
+          version: '8.12.0',
+          upgradeable: true,
+          minutesSinceUpgrade: 11,
+          upgradeDetails: { state: 'UPG_WATCHING' } as any,
+        })
+      )
+    ).toBe(false);
+  });
+  it('returns true if agent watching state cleared and it was upgraded less than 10 minutes ago', () => {
+    expect(
+      isAgentUpgradeable(getAgent({ version: '8.12.1', upgradeable: true, minutesSinceUpgrade: 1 }))
+    ).toBe(true);
+  });
 });
 describe('Fleet - isAgentUpgradeableToVersion', () => {
   it('returns true if agent reports upgradeable, with upgrade to agent snapshot version newer than latest agent version', () => {
@@ -398,6 +415,13 @@ describe('hasAgentBeenUpgradedRecently', () => {
       getRecentUpgradeInfoForAgent(getAgent({ version: '7.9.0' })).hasBeenUpgradedRecently
     ).toBe(false);
   });
+
+  it('returns false if the agent was upgraded more less 10 minutes ago, but supports upgrade details', () => {
+    expect(
+      getRecentUpgradeInfoForAgent(getAgent({ version: '8.12.0', minutesSinceUpgrade: 1 }))
+        .hasBeenUpgradedRecently
+    ).toBe(false);
+  });
 });
 
 describe('isAgentUpgrading', () => {

x-pack/plugins/fleet/common/services/is_agent_upgradeable.ts

@@ -8,12 +8,14 @@
 import semverCoerce from 'semver/functions/coerce';
 import semverLt from 'semver/functions/lt';
 import semverGt from 'semver/functions/gt';
+import semverGte from 'semver/functions/gte';
 import semverEq from 'semver/functions/eq';
 import moment from 'moment';
 
 import type { Agent } from '../types';
 
 export const AGENT_UPGRADE_COOLDOWN_IN_MIN = 10;
+export const AGENT_UPGARDE_DETAILS_SUPPORTED_VERSION = '8.12.0';
 
 // Error messages for agent not upgradeable
 export const VERSION_MISSING_ERROR = `agent version is missing.`;
@@ -48,6 +50,9 @@ export function isAgentUpgradeable(agent: Agent): boolean {
   if (isAgentUpgrading(agent)) {
     return false;
   }
+  if (isAgentInWatchingState(agent)) {
+    return false;
+  }
   if (getRecentUpgradeInfoForAgent(agent).hasBeenUpgradedRecently) {
     return false;
   }
@@ -150,7 +155,12 @@ export function getRecentUpgradeInfoForAgent(agent: Agent): {
   elapsedMinsSinceUpgrade: number;
   timeToWaitMins: number;
 } {
-  if (!agent.upgraded_at) {
+  // no need for 10m wait if agent has upgrade details watching state
+  const agentHasUpgradeDetailsSupport = semverGte(
+    agent.local_metadata.elastic.agent.version,
+    AGENT_UPGARDE_DETAILS_SUPPORTED_VERSION
+  );
+  if (!agent.upgraded_at || agentHasUpgradeDetailsSupport) {
     return {
       hasBeenUpgradedRecently: false,
       timeToWaitMs: 0,
@@ -170,6 +180,10 @@ export function getRecentUpgradeInfoForAgent(agent: Agent): {
   return { hasBeenUpgradedRecently, timeToWaitMs, elapsedMinsSinceUpgrade, timeToWaitMins };
 }
 
+export function isAgentInWatchingState(agent: Agent) {
+  return agent.upgrade_details?.state === 'UPG_WATCHING';
+}
+
 export function isAgentUpgrading(agent: Agent) {
   if (agent.upgrade_details) {
     return agent.upgrade_details.state !== 'UPG_FAILED';