[Fleet] Add automatic agent upgrades functional tests (#220829)

## Summary

Closes https://github.com/elastic/ingest-dev/issues/5303

This PR adds functional tests for automatic upgrades of Fleet agents.

The test cases aim to cover the logic of whether agents should be
upgraded or not in a single run.

I noticed some rare flakiness when running these tests locally, hence
the few Flaky Test Runner runs. Since these all passed, I am leaving the
task interval and sleep parameters as is for now (with a comment in case
flakiness does happen).

Running:
```
yarn test:ftr:server --config x-pack/test/fleet_tasks/config.ts
```
```
yarn test:ftr:runner --config x-pack/test/fleet_tasks/config.ts
```

### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [x] [Flaky Test
Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was
used on any tests changed
- 🟢 25 runs:
https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/8264
- 🟢 100 runs:
https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/8265
- 🟢 100 runs:
https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/8267
- 🟢 100 runs:
https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/8268
- [x] The PR description includes the appropriate Release Notes section,
and the correct `release_note:*` label is applied per the
[guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)

### Identify risks

No impact on functionality. Risk of functional tests flakiness.

---------

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

.buildkite/ftr_platform_stateful_configs.yml

@@ -171,6 +171,7 @@ enabled:
   - x-pack/test/fleet_api_integration/config.package_policy.ts
   - x-pack/test/fleet_api_integration/config.space_awareness.ts
   - x-pack/test/fleet_functional/config.ts
+  - x-pack/test/fleet_tasks/config.ts
   - x-pack/test/ftr_apis/security_and_spaces/config.ts
   - x-pack/test/functional_basic/apps/ml/permissions/config.ts
   - x-pack/test/functional_basic/apps/ml/data_visualizer/group1/config.ts

.github/CODEOWNERS

@@ -1445,6 +1445,7 @@ src/platform/plugins/shared/discover/public/context_awareness/profile_providers/
 /x-pack/test/api_integration/services/fleet_and_agents.ts @elastic/fleet
 /x-pack/test/fleet_api_integration @elastic/fleet
 /x-pack/test/fleet_packages @elastic/fleet
+/x-pack/test/fleet_tasks @elastic/fleet
 /src/platform/test/api_integration/apis/custom_integration/*.ts @elastic/fleet
 /x-pack/test/fleet_cypress @elastic/fleet
 /x-pack/test/fleet_functional @elastic/fleet

x-pack/test/fleet_tasks/config.ts

@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrConfigProviderContext } from '@kbn/test';
+
+export default async function ({ readConfigFile }: FtrConfigProviderContext) {
+  const xPackAPITestsConfig = await readConfigFile(require.resolve('../api_integration/config.ts'));
+
+  return {
+    testFiles: [require.resolve('./tests')],
+    servers: xPackAPITestsConfig.get('servers'),
+    services: xPackAPITestsConfig.get('services'),
+    junit: {
+      reportName: 'X-Pack Fleet tasks tests',
+    },
+    esTestCluster: xPackAPITestsConfig.get('esTestCluster'),
+    kbnTestServer: {
+      ...xPackAPITestsConfig.get('kbnTestServer'),
+      serverArgs: [
+        ...xPackAPITestsConfig.get('kbnTestServer.serverArgs'),
+        '--xpack.cloudSecurityPosture.enabled=true',
+        // Enable debug fleet logs by default
+        `--logging.loggers[0].name=plugins.fleet`,
+        `--logging.loggers[0].level=debug`,
+        `--logging.loggers[0].appenders=${JSON.stringify(['default'])}`,
+        `--xpack.fleet.enableExperimental=${JSON.stringify(['enableAutomaticAgentUpgrades'])}`,
+        `--xpack.fleet.autoUpgrades.taskInterval=30s`,
+        `--xpack.fleet.autoUpgrades.retryDelays=${JSON.stringify(['1m'])}`,
+      ],
+    },
+  };
+}

x-pack/test/fleet_tasks/ftr_provider_context.d.ts

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { GenericFtrProviderContext } from '@kbn/test';
+
+import { services } from '../api_integration/services';
+
+export type FtrProviderContextWithServices = GenericFtrProviderContext<typeof services, {}>;
+export type FtrProviderContext = GenericFtrProviderContext<{}, {}>;

x-pack/test/fleet_tasks/helpers.ts

@@ -0,0 +1,70 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { AGENTS_INDEX } from '@kbn/fleet-plugin/common';
+import { FtrProviderContext } from '../api_integration/ftr_provider_context';
+
+export async function createAgentDoc(
+  providerContext: FtrProviderContext,
+  id: string,
+  policyId: string,
+  version: string,
+  active: boolean = true,
+  additionalData: any = {}
+) {
+  const { getService } = providerContext;
+  const es = getService('es');
+  const lastCheckin = active
+    ? new Date().toISOString()
+    : new Date(new Date().getTime() - 21 * 24 * 60 * 60 * 1000).toISOString(); // 3 weeks ago
+
+  await es.index({
+    index: AGENTS_INDEX,
+    id,
+    document: {
+      id,
+      type: 'PERMANENT',
+      active: true,
+      enrolled_at: new Date().toISOString(),
+      last_checkin: lastCheckin,
+      policy_id: policyId,
+      policy_revision: 1,
+      policy_revision_idx: 1,
+      agent: {
+        id,
+        version,
+      },
+      local_metadata: {
+        elastic: {
+          agent: {
+            version,
+            upgradeable: true,
+          },
+        },
+      },
+      ...additionalData,
+    },
+    refresh: 'wait_for',
+  });
+}
+
+export async function cleanupAgentDocs(providerContext: FtrProviderContext) {
+  const { getService } = providerContext;
+  const es = getService('es');
+
+  try {
+    await es.deleteByQuery({
+      index: AGENTS_INDEX,
+      refresh: true,
+      query: {
+        match_all: {},
+      },
+    });
+  } catch (err) {
+    // index doesn't exist
+  }
+}

x-pack/test/fleet_tasks/tests/automatic_upgrades.ts

@@ -0,0 +1,260 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+
+import { FtrProviderContextWithServices } from '../ftr_provider_context';
+import { cleanupAgentDocs, createAgentDoc } from '../helpers';
+
+export default function (providerContext: FtrProviderContextWithServices) {
+  const { getService } = providerContext;
+  const supertest = getService('supertest');
+  const TASK_INTERVAL = 30000; // as set in the config
+  const RETRY_DELAY = 60000; // as set in the config
+  let policyId: string;
+
+  async function waitForTask() {
+    // Sleep for the duration of the task interval.
+    // In case of test flakiness, the sleep duration can be increased.
+    await new Promise((resolve) => setTimeout(resolve, TASK_INTERVAL));
+  }
+
+  describe('Automatic agent upgrades', () => {
+    before(async () => {
+      const { body: agentPolicyResponse } = await supertest
+        .post('/api/fleet/agent_policies')
+        .set('kbn-xsrf', 'xxxx')
+        .send({
+          name: 'Test policy',
+          namespace: 'default',
+          force: true,
+        })
+        .expect(200);
+      policyId = agentPolicyResponse.item.id;
+    });
+
+    after(async () => {
+      await supertest
+        .post('/api/fleet/agent_policies/delete')
+        .send({ agentPolicyId: policyId })
+        .set('kbn-xsrf', 'xxxx')
+        .expect(200);
+    });
+
+    afterEach(async () => {
+      await cleanupAgentDocs(providerContext);
+    });
+
+    it('should only upgrade active agents', async () => {
+      // Create an active agent and an inactive agent on 8.17.0.
+      await createAgentDoc(providerContext, 'agent1', policyId, '8.17.0');
+      await createAgentDoc(providerContext, 'agent2', policyId, '8.17.0', false);
+      // Update the policy to require version 8.17.1 for all active agents.
+      await supertest
+        .put(`/api/fleet/agent_policies/${policyId}`)
+        .set('kbn-xsrf', 'xxxx')
+        .send({
+          name: 'Test policy',
+          namespace: 'default',
+          required_versions: [
+            {
+              version: '8.17.1',
+              percentage: 100,
+            },
+          ],
+        })
+        .expect(200);
+      await waitForTask();
+      // Check that only the active agent was upgraded.
+      let res = await supertest.get('/api/fleet/agents/agent1').set('kbn-xsrf', 'xxx').expect(200);
+      expect(typeof res.body.item.upgrade_started_at).to.be('string');
+      expect(res.body.item.upgrade_attempts.length).to.be(1);
+      res = await supertest.get('/api/fleet/agents/agent2').set('kbn-xsrf', 'xxx').expect(200);
+      expect(res.body.item.upgrade_started_at).to.be(undefined);
+      expect(res.body.item.upgrade_attempts).to.be(undefined);
+    });
+
+    it('should take agents on target version into account', async () => {
+      // Create 3 active agents on 8.17.0 and 1 active agent on 8.17.1.
+      await createAgentDoc(providerContext, 'agent1', policyId, '8.17.0');
+      await createAgentDoc(providerContext, 'agent2', policyId, '8.17.0');
+      await createAgentDoc(providerContext, 'agent3', policyId, '8.17.0');
+      await createAgentDoc(providerContext, 'agent4', policyId, '8.17.1');
+      // Update the policy to require version 8.17.1 for 50% of active agents.
+      await supertest
+        .put(`/api/fleet/agent_policies/${policyId}`)
+        .set('kbn-xsrf', 'xxxx')
+        .send({
+          name: 'Test policy',
+          namespace: 'default',
+          required_versions: [
+            {
+              version: '8.17.1',
+              percentage: 50,
+            },
+          ],
+        })
+        .expect(200);
+      await waitForTask();
+      // Check that only one agent on 8.17.0 was upgraded.
+      const res = await supertest.get('/api/fleet/agents').set('kbn-xsrf', 'xxx').expect(200);
+      expect(res.body.items.length).to.be(4);
+      expect(res.body.items.filter((item: any) => item.status === 'updating').length).to.be(1);
+      expect(
+        res.body.items.filter((item: any) => item.upgrade_started_at !== undefined).length
+      ).to.be(1);
+      expect(
+        res.body.items.filter((item: any) => item.upgrade_attempts !== undefined).length
+      ).to.be(1);
+    });
+
+    it('should not take inactive agents on target version into account', async () => {
+      // Create 2 active agents on 8.17.0, 1 active agent on 8.17.1 and 1 inactive agent on 8.17.1.
+      await createAgentDoc(providerContext, 'agent1', policyId, '8.17.0');
+      await createAgentDoc(providerContext, 'agent2', policyId, '8.17.0');
+      await createAgentDoc(providerContext, 'agent3', policyId, '8.17.1');
+      await createAgentDoc(providerContext, 'agent4', policyId, '8.17.1', false);
+      // Update the policy to require version 8.17.1 for 50% of active agents.
+      await supertest
+        .put(`/api/fleet/agent_policies/${policyId}`)
+        .set('kbn-xsrf', 'xxxx')
+        .send({
+          name: 'Test policy',
+          namespace: 'default',
+          required_versions: [
+            {
+              version: '8.17.1',
+              percentage: 50,
+            },
+          ],
+        })
+        .expect(200);
+      await waitForTask();
+      // Check that two agents on 8.17.0 were upgraded.
+      const res = await supertest
+        .get('/api/fleet/agents?showInactive=true')
+        .set('kbn-xsrf', 'xxx')
+        .expect(200);
+      expect(res.body.items.length).to.be(4);
+      expect(res.body.items.filter((item: any) => item.status === 'updating').length).to.be(1);
+      expect(
+        res.body.items.filter((item: any) => item.upgrade_started_at !== undefined).length
+      ).to.be(1);
+      expect(
+        res.body.items.filter((item: any) => item.upgrade_attempts !== undefined).length
+      ).to.be(1);
+    });
+
+    it('should take already upgrading agents into account', async () => {
+      // Create an active agent on 8.17.0 and an active agent on 8.17.0 upgrading to 8.17.1.
+      await createAgentDoc(providerContext, 'agent1', policyId, '8.17.0');
+      await createAgentDoc(providerContext, 'agent2', policyId, '8.17.0', true, {
+        upgrade_details: {
+          target_version: '8.17.1',
+          state: 'UPG_DOWNLOADING',
+          action_id: '123',
+        },
+      });
+      // Update the policy to require version 8.17.1 or 50% of active agents.
+      await supertest
+        .put(`/api/fleet/agent_policies/${policyId}`)
+        .set('kbn-xsrf', 'xxxx')
+        .send({
+          name: 'Test policy',
+          namespace: 'default',
+          required_versions: [
+            {
+              version: '8.17.1',
+              percentage: 50,
+            },
+          ],
+        })
+        .expect(200);
+      await waitForTask();
+      // Check that no agent was upgraded.
+      let res = await supertest.get('/api/fleet/agents/agent1').set('kbn-xsrf', 'xxx').expect(200);
+      expect(res.body.item.upgrade_started_at).to.be(undefined);
+      res = await supertest.get('/api/fleet/agents/agent2').set('kbn-xsrf', 'xxx').expect(200);
+      expect(res.body.item.upgrade_started_at).to.be(undefined);
+    });
+
+    it('should take agents marked but not ready for retry into account but not upgrade them', async () => {
+      // Create an active agent on 8.17.0 and an active agent on 8.17.0 marked but not ready for retry.
+      await createAgentDoc(providerContext, 'agent1', policyId, '8.17.0');
+      await createAgentDoc(providerContext, 'agent2', policyId, '8.17.0', true, {
+        upgrade_details: {
+          target_version: '8.17.1',
+          state: 'UPG_FAILED',
+          action_id: '123',
+        },
+        upgrade_attempts: [new Date().toISOString()],
+      });
+      // Update the policy to require version 8.17.1 for 50% of active agents.
+      await supertest
+        .put(`/api/fleet/agent_policies/${policyId}`)
+        .set('kbn-xsrf', 'xxxx')
+        .send({
+          name: 'Test policy',
+          namespace: 'default',
+          required_versions: [
+            {
+              version: '8.17.1',
+              percentage: 50,
+            },
+          ],
+        })
+        .expect(200);
+      await waitForTask();
+      // Check that no agent was upgraded.
+      let res = await supertest.get('/api/fleet/agents/agent1').set('kbn-xsrf', 'xxx').expect(200);
+      expect(res.body.item.upgrade_started_at).to.be(undefined);
+      res = await supertest.get('/api/fleet/agents/agent2').set('kbn-xsrf', 'xxx').expect(200);
+      expect(res.body.item.upgrade_started_at).to.be(undefined);
+    });
+
+    it('should take agents marked and ready for retry into account and upgrade them', async () => {
+      // Create an active agent on 8.17.0 and an active agent on 8.17.0 marked and ready for retry.
+      await createAgentDoc(providerContext, 'agent1', policyId, '8.17.0');
+      await createAgentDoc(providerContext, 'agent2', policyId, '8.17.0', true, {
+        upgrade_details: {
+          target_version: '8.17.1',
+          state: 'UPG_FAILED',
+          action_id: '123',
+        },
+        upgrade_attempts: [new Date(new Date().getTime() - RETRY_DELAY).toISOString()],
+      });
+      // Update the policy to require version 8.17.1 for 50% of active agents.
+      await supertest
+        .put(`/api/fleet/agent_policies/${policyId}`)
+        .set('kbn-xsrf', 'xxxx')
+        .send({
+          name: 'Test policy',
+          namespace: 'default',
+          required_versions: [
+            {
+              version: '8.17.1',
+              percentage: 50,
+            },
+          ],
+        })
+        .expect(200);
+      await waitForTask();
+      // Check that agent1 was upgraded.
+      let res = await supertest.get('/api/fleet/agents/agent1').set('kbn-xsrf', 'xxx').expect(200);
+      expect(res.body.item.upgrade_started_at).to.be(undefined);
+      res = await supertest.get('/api/fleet/agents/agent2').set('kbn-xsrf', 'xxx').expect(200);
+      expect(typeof res.body.item.upgrade_started_at).to.be('string');
+    });
+  });
+}

x-pack/test/fleet_tasks/tests/index.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrProviderContext } from '../ftr_provider_context';
+
+export default function ({ loadTestFile }: FtrProviderContext) {
+  describe('Fleet packages test', function () {
+    loadTestFile(require.resolve('./automatic_upgrades'));
+  });
+}