[Fleet] Don't upgrade agent(s) in a managed policy (#91303)

## Summary

 - Make sure any agents requesting to be upgraded, are not enrolled in a managed policy.
 - `force: true` will only bypass agent / kibana version checks. It will not bypass managed policy check. To workaround, the enrolled policy should be changed to unmanaged (`is_managed: false`) as we do with enroll, reassign, etc.
 - Took more efficient approach to bulk actions. One `bulkGet` for N agents/policies vs N `get`s  approach used for bulk reassignment of agents. See discussion in https://github.com/elastic/kibana/pull/88688/files#r568941761
 - [x] API
 - [ ] UI
 - [x] tests

### Checklist
- [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios


### Manual tests
#### upgrade one
```
curl --location --request POST '8d9748e0-6d52-11eb-8cbd-47e38cd1c8de/upgrade' --header 'kbn-xsrf: <string>' --header 'Content-Type: application/json' --header 'Authorization: Basic ZWxhc3RpYzpjaGFuZ2VtZQ==' --data-raw '{
    "version": "8.0.0"
}'
{"statusCode":400,"error":"Bad Request","message":"Cannot upgrade agent 8d9748e0-6d52-11eb-8cbd-47e38cd1c8de in managed policy bf319100-6d50-11eb-8859-15a87f509a99"}
```

```
curl --location --request POST '8d9748e0-6d52-11eb-8cbd-47e38cd1c8de/upgrade' --header 'kbn-xsrf: <string>' --header 'Content-Type: application/json' --header 'Authorization: Basic ZWxhc3RpYzpjaGFuZ2VtZQ==' --data-raw '{
    "version": "8.0.0", "force": true
}'
{"statusCode":400,"error":"Bad Request","message":"Cannot upgrade agent 8d9748e0-6d52-11eb-8cbd-47e38cd1c8de in managed policy bf319100-6d50-11eb-8859-15a87f509a99"}
```

#### bulk upgrade
```
curl --location --request POST 'http://localhost:5601/api/fleet/agents/bulk_upgrade' --header 'kbn-xsrf: <string>' --header 'Content-Type: application/json' --header 'Authorization: Basic ZWxhc3RpYzpjaGFuZ2VtZQ==' --data-raw '{
    "version": "8.0.0",
    "agents": [
        "8d9748e0-6d52-11eb-8cbd-47e38cd1c8de"
    ]
}'
{}
```

```
curl --location --request POST 'http://localhost:5601/api/fleet/agents/bulk_upgrade' --header 'kbn-xsrf: <string>' --header 'Content-Type: application/json' --header 'Authorization: Basic ZWxhc3RpYzpjaGFuZ2VtZQ==' --data-raw '{
    "version": "8.0.0",
    "agents": [
        "8d9748e0-6d52-11eb-8cbd-47e38cd1c8de"
    ], "force": true
}'
{"statusCode":400,"error":"Bad Request","message":"Cannot update agent in managed policy bf319100-6d50-11eb-8859-15a87f509a99"}```
```

x-pack/plugins/fleet/server/services/agent_policy.ts

@@ -209,6 +209,21 @@ class AgentPolicyService {
     return agentPolicy;
   }
 
+  public async getByIDs(
+    soClient: SavedObjectsClientContract,
+    ids: string[],
+    options: { fields?: string[] } = {}
+  ): Promise<AgentPolicy[]> {
+    const objects = ids.map((id) => ({ ...options, id, type: SAVED_OBJECT_TYPE }));
+    const agentPolicySO = await soClient.bulkGet<AgentPolicySOAttributes>(objects);
+
+    return agentPolicySO.saved_objects.map((so) => ({
+      id: so.id,
+      version: so.version,
+      ...so.attributes,
+    }));
+  }
+
   public async list(
     soClient: SavedObjectsClientContract,
     options: ListWithKuery & {

x-pack/plugins/fleet/server/services/agents/upgrade.ts

@@ -8,8 +8,16 @@
 import { ElasticsearchClient, SavedObjectsClientContract } from 'src/core/server';
 import { AgentAction, AgentActionSOAttributes } from '../../types';
 import { AGENT_ACTION_SAVED_OBJECT_TYPE } from '../../constants';
+import { agentPolicyService } from '../../services';
+import { IngestManagerError } from '../../errors';
 import { bulkCreateAgentActions, createAgentAction } from './actions';
-import { getAgents, listAllAgents, updateAgent, bulkUpdateAgents } from './crud';
+import {
+  getAgents,
+  listAllAgents,
+  updateAgent,
+  bulkUpdateAgents,
+  getAgentPolicyForAgent,
+} from './crud';
 import { isAgentUpgradeable } from '../../../common/services';
 import { appContextService } from '../app_context';
 
@@ -31,6 +39,14 @@ export async function sendUpgradeAgentAction({
     version,
     source_uri: sourceUri,
   };
+
+  const agentPolicy = await getAgentPolicyForAgent(soClient, esClient, agentId);
+  if (agentPolicy?.is_managed) {
+    throw new IngestManagerError(
+      `Cannot upgrade agent ${agentId} in managed policy ${agentPolicy.id}`
+    );
+  }
+
   await createAgentAction(soClient, esClient, {
     agent_id: agentId,
     created_at: now,
@@ -89,19 +105,40 @@ export async function sendUpgradeAgentsActions(
             showInactive: false,
           })
         ).agents;
-  const agentsToUpdate = options.force
+
+  // upgradeable if they pass the version check
+  const upgradeableAgents = options.force
     ? agents
     : agents.filter((agent) => isAgentUpgradeable(agent, kibanaVersion));
+
+  // get any policy ids from upgradable agents
+  const policyIdsToGet = new Set(
+    upgradeableAgents.filter((agent) => agent.policy_id).map((agent) => agent.policy_id!)
+  );
+
+  // get the agent policies for those ids
+  const agentPolicies = await agentPolicyService.getByIDs(soClient, Array.from(policyIdsToGet), {
+    fields: ['is_managed'],
+  });
+
+  // throw if any of those agent policies are managed
+  for (const policy of agentPolicies) {
+    if (policy.is_managed) {
+      throw new IngestManagerError(`Cannot upgrade agent in managed policy ${policy.id}`);
+    }
+  }
+
+  // Create upgrade action for each agent
   const now = new Date().toISOString();
   const data = {
     version: options.version,
     source_uri: options.sourceUri,
   };
-  // Create upgrade action for each agent
+
   await bulkCreateAgentActions(
     soClient,
     esClient,
-    agentsToUpdate.map((agent) => ({
+    upgradeableAgents.map((agent) => ({
       agent_id: agent.id,
       created_at: now,
       data,
@@ -113,7 +150,7 @@ export async function sendUpgradeAgentsActions(
   return await bulkUpdateAgents(
     soClient,
     esClient,
-    agentsToUpdate.map((agent) => ({
+    upgradeableAgents.map((agent) => ({
       agentId: agent.id,
       data: {
         upgraded_at: null,

x-pack/test/fleet_api_integration/apis/agents/upgrade.ts

@@ -427,5 +427,84 @@ export default function (providerContext: FtrProviderContext) {
         })
         .expect(400);
     });
+
+    describe('fleet upgrade agent(s) in a managed policy', function () {
+      it('should respond 400 to bulk upgrade and not update the agent SOs', async () => {
+        // update enrolled policy to managed
+        await supertest.put(`/api/fleet/agent_policies/policy1`).set('kbn-xsrf', 'xxxx').send({
+          name: 'Test policy',
+          namespace: 'default',
+          is_managed: true,
+        });
+
+        const kibanaVersion = await kibanaServer.version.get();
+        await kibanaServer.savedObjects.update({
+          id: 'agent1',
+          type: AGENT_SAVED_OBJECT_TYPE,
+          attributes: {
+            local_metadata: { elastic: { agent: { upgradeable: true, version: '0.0.0' } } },
+          },
+        });
+        await kibanaServer.savedObjects.update({
+          id: 'agent2',
+          type: AGENT_SAVED_OBJECT_TYPE,
+          attributes: {
+            local_metadata: {
+              elastic: {
+                agent: { upgradeable: true, version: semver.inc(kibanaVersion, 'patch') },
+              },
+            },
+          },
+        });
+
+        // attempt to upgrade agent in managed policy
+        const { body } = await supertest
+          .post(`/api/fleet/agents/bulk_upgrade`)
+          .set('kbn-xsrf', 'xxx')
+          .send({
+            version: kibanaVersion,
+            agents: ['agent1', 'agent2'],
+          })
+          .expect(400);
+        expect(body.message).to.contain('Cannot upgrade agent in managed policy policy1');
+
+        const [agent1data, agent2data] = await Promise.all([
+          supertest.get(`/api/fleet/agents/agent1`),
+          supertest.get(`/api/fleet/agents/agent2`),
+        ]);
+
+        expect(typeof agent1data.body.item.upgrade_started_at).to.be('undefined');
+        expect(typeof agent2data.body.item.upgrade_started_at).to.be('undefined');
+      });
+
+      it('should respond 400 to upgrade and not update the agent SOs', async () => {
+        // update enrolled policy to managed
+        await supertest.put(`/api/fleet/agent_policies/policy1`).set('kbn-xsrf', 'xxxx').send({
+          name: 'Test policy',
+          namespace: 'default',
+          is_managed: true,
+        });
+
+        const kibanaVersion = await kibanaServer.version.get();
+        await kibanaServer.savedObjects.update({
+          id: 'agent1',
+          type: AGENT_SAVED_OBJECT_TYPE,
+          attributes: {
+            local_metadata: { elastic: { agent: { upgradeable: true, version: '0.0.0' } } },
+          },
+        });
+
+        // attempt to upgrade agent in managed policy
+        const { body } = await supertest
+          .post(`/api/fleet/agents/agent1/upgrade`)
+          .set('kbn-xsrf', 'xxx')
+          .send({ version: kibanaVersion })
+          .expect(400);
+        expect(body.message).to.contain('Cannot upgrade agent agent1 in managed policy policy1');
+
+        const agent1data = await supertest.get(`/api/fleet/agents/agent1`);
+        expect(typeof agent1data.body.item.upgrade_started_at).to.be('undefined');
+      });
+    });
   });
 }