From 2dba27e8ba8fda8005b0947e7da8468a38b7721e Mon Sep 17 00:00:00 2001 From: Jill Guyonnet Date: Tue, 20 May 2025 09:33:23 +0200 Subject: [PATCH] [Fleet] Add automatic agent upgrades functional tests (#220829) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Closes https://github.com/elastic/ingest-dev/issues/5303 This PR adds functional tests for automatic upgrades of Fleet agents. The test cases aim to cover the logic of whether agents should be upgraded or not in a single run. I noticed some rare flakiness when running these tests locally, hence the few Flaky Test Runner runs. Since these all passed, I am leaving the task interval and sleep parameters as is for now (with a comment in case flakiness does happen). Running: ``` yarn test:ftr:server --config x-pack/test/fleet_tasks/config.ts ``` ``` yarn test:ftr:runner --config x-pack/test/fleet_tasks/config.ts ``` ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [x] [Flaky Test Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was used on any tests changed - 🟢 25 runs: https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/8264 - 🟢 100 runs: https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/8265 - 🟢 100 runs: https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/8267 - 🟢 100 runs: https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/8268 - [x] The PR description includes the appropriate Release Notes section, and the correct `release_note:*` label is applied per the [guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) ### Identify risks No impact on functionality. Risk of functional tests flakiness. --------- Co-authored-by: Elastic Machine --- .buildkite/ftr_platform_stateful_configs.yml | 1 + .github/CODEOWNERS | 1 + x-pack/test/fleet_tasks/config.ts | 36 +++ .../fleet_tasks/ftr_provider_context.d.ts | 13 + x-pack/test/fleet_tasks/helpers.ts | 70 +++++ .../fleet_tasks/tests/automatic_upgrades.ts | 260 ++++++++++++++++++ x-pack/test/fleet_tasks/tests/index.ts | 14 + 7 files changed, 395 insertions(+) create mode 100644 x-pack/test/fleet_tasks/config.ts create mode 100644 x-pack/test/fleet_tasks/ftr_provider_context.d.ts create mode 100644 x-pack/test/fleet_tasks/helpers.ts create mode 100644 x-pack/test/fleet_tasks/tests/automatic_upgrades.ts create mode 100644 x-pack/test/fleet_tasks/tests/index.ts diff --git a/.buildkite/ftr_platform_stateful_configs.yml b/.buildkite/ftr_platform_stateful_configs.yml index ec31da46e58b..07709eff9400 100644 --- a/.buildkite/ftr_platform_stateful_configs.yml +++ b/.buildkite/ftr_platform_stateful_configs.yml @@ -171,6 +171,7 @@ enabled: - x-pack/test/fleet_api_integration/config.package_policy.ts - x-pack/test/fleet_api_integration/config.space_awareness.ts - x-pack/test/fleet_functional/config.ts + - x-pack/test/fleet_tasks/config.ts - x-pack/test/ftr_apis/security_and_spaces/config.ts - x-pack/test/functional_basic/apps/ml/permissions/config.ts - x-pack/test/functional_basic/apps/ml/data_visualizer/group1/config.ts diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index bfc41e288572..13859767f89f 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1445,6 +1445,7 @@ src/platform/plugins/shared/discover/public/context_awareness/profile_providers/ /x-pack/test/api_integration/services/fleet_and_agents.ts @elastic/fleet /x-pack/test/fleet_api_integration @elastic/fleet /x-pack/test/fleet_packages @elastic/fleet +/x-pack/test/fleet_tasks @elastic/fleet /src/platform/test/api_integration/apis/custom_integration/*.ts @elastic/fleet /x-pack/test/fleet_cypress @elastic/fleet /x-pack/test/fleet_functional @elastic/fleet diff --git a/x-pack/test/fleet_tasks/config.ts b/x-pack/test/fleet_tasks/config.ts new file mode 100644 index 000000000000..78fb30552ec0 --- /dev/null +++ b/x-pack/test/fleet_tasks/config.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrConfigProviderContext } from '@kbn/test'; + +export default async function ({ readConfigFile }: FtrConfigProviderContext) { + const xPackAPITestsConfig = await readConfigFile(require.resolve('../api_integration/config.ts')); + + return { + testFiles: [require.resolve('./tests')], + servers: xPackAPITestsConfig.get('servers'), + services: xPackAPITestsConfig.get('services'), + junit: { + reportName: 'X-Pack Fleet tasks tests', + }, + esTestCluster: xPackAPITestsConfig.get('esTestCluster'), + kbnTestServer: { + ...xPackAPITestsConfig.get('kbnTestServer'), + serverArgs: [ + ...xPackAPITestsConfig.get('kbnTestServer.serverArgs'), + '--xpack.cloudSecurityPosture.enabled=true', + // Enable debug fleet logs by default + `--logging.loggers[0].name=plugins.fleet`, + `--logging.loggers[0].level=debug`, + `--logging.loggers[0].appenders=${JSON.stringify(['default'])}`, + `--xpack.fleet.enableExperimental=${JSON.stringify(['enableAutomaticAgentUpgrades'])}`, + `--xpack.fleet.autoUpgrades.taskInterval=30s`, + `--xpack.fleet.autoUpgrades.retryDelays=${JSON.stringify(['1m'])}`, + ], + }, + }; +} diff --git a/x-pack/test/fleet_tasks/ftr_provider_context.d.ts b/x-pack/test/fleet_tasks/ftr_provider_context.d.ts new file mode 100644 index 000000000000..ca3325e0682b --- /dev/null +++ b/x-pack/test/fleet_tasks/ftr_provider_context.d.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { GenericFtrProviderContext } from '@kbn/test'; + +import { services } from '../api_integration/services'; + +export type FtrProviderContextWithServices = GenericFtrProviderContext; +export type FtrProviderContext = GenericFtrProviderContext<{}, {}>; diff --git a/x-pack/test/fleet_tasks/helpers.ts b/x-pack/test/fleet_tasks/helpers.ts new file mode 100644 index 000000000000..5eb8cde4763f --- /dev/null +++ b/x-pack/test/fleet_tasks/helpers.ts @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { AGENTS_INDEX } from '@kbn/fleet-plugin/common'; +import { FtrProviderContext } from '../api_integration/ftr_provider_context'; + +export async function createAgentDoc( + providerContext: FtrProviderContext, + id: string, + policyId: string, + version: string, + active: boolean = true, + additionalData: any = {} +) { + const { getService } = providerContext; + const es = getService('es'); + const lastCheckin = active + ? new Date().toISOString() + : new Date(new Date().getTime() - 21 * 24 * 60 * 60 * 1000).toISOString(); // 3 weeks ago + + await es.index({ + index: AGENTS_INDEX, + id, + document: { + id, + type: 'PERMANENT', + active: true, + enrolled_at: new Date().toISOString(), + last_checkin: lastCheckin, + policy_id: policyId, + policy_revision: 1, + policy_revision_idx: 1, + agent: { + id, + version, + }, + local_metadata: { + elastic: { + agent: { + version, + upgradeable: true, + }, + }, + }, + ...additionalData, + }, + refresh: 'wait_for', + }); +} + +export async function cleanupAgentDocs(providerContext: FtrProviderContext) { + const { getService } = providerContext; + const es = getService('es'); + + try { + await es.deleteByQuery({ + index: AGENTS_INDEX, + refresh: true, + query: { + match_all: {}, + }, + }); + } catch (err) { + // index doesn't exist + } +} diff --git a/x-pack/test/fleet_tasks/tests/automatic_upgrades.ts b/x-pack/test/fleet_tasks/tests/automatic_upgrades.ts new file mode 100644 index 000000000000..0b9fbd12c10d --- /dev/null +++ b/x-pack/test/fleet_tasks/tests/automatic_upgrades.ts @@ -0,0 +1,260 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; + +import { FtrProviderContextWithServices } from '../ftr_provider_context'; +import { cleanupAgentDocs, createAgentDoc } from '../helpers'; + +export default function (providerContext: FtrProviderContextWithServices) { + const { getService } = providerContext; + const supertest = getService('supertest'); + const TASK_INTERVAL = 30000; // as set in the config + const RETRY_DELAY = 60000; // as set in the config + let policyId: string; + + async function waitForTask() { + // Sleep for the duration of the task interval. + // In case of test flakiness, the sleep duration can be increased. + await new Promise((resolve) => setTimeout(resolve, TASK_INTERVAL)); + } + + describe('Automatic agent upgrades', () => { + before(async () => { + const { body: agentPolicyResponse } = await supertest + .post('/api/fleet/agent_policies') + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'Test policy', + namespace: 'default', + force: true, + }) + .expect(200); + policyId = agentPolicyResponse.item.id; + }); + + after(async () => { + await supertest + .post('/api/fleet/agent_policies/delete') + .send({ agentPolicyId: policyId }) + .set('kbn-xsrf', 'xxxx') + .expect(200); + }); + + afterEach(async () => { + await cleanupAgentDocs(providerContext); + }); + + it('should only upgrade active agents', async () => { + // Create an active agent and an inactive agent on 8.17.0. + await createAgentDoc(providerContext, 'agent1', policyId, '8.17.0'); + await createAgentDoc(providerContext, 'agent2', policyId, '8.17.0', false); + // Update the policy to require version 8.17.1 for all active agents. + await supertest + .put(`/api/fleet/agent_policies/${policyId}`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'Test policy', + namespace: 'default', + required_versions: [ + { + version: '8.17.1', + percentage: 100, + }, + ], + }) + .expect(200); + await waitForTask(); + // Check that only the active agent was upgraded. + let res = await supertest.get('/api/fleet/agents/agent1').set('kbn-xsrf', 'xxx').expect(200); + expect(typeof res.body.item.upgrade_started_at).to.be('string'); + expect(res.body.item.upgrade_attempts.length).to.be(1); + res = await supertest.get('/api/fleet/agents/agent2').set('kbn-xsrf', 'xxx').expect(200); + expect(res.body.item.upgrade_started_at).to.be(undefined); + expect(res.body.item.upgrade_attempts).to.be(undefined); + }); + + it('should take agents on target version into account', async () => { + // Create 3 active agents on 8.17.0 and 1 active agent on 8.17.1. + await createAgentDoc(providerContext, 'agent1', policyId, '8.17.0'); + await createAgentDoc(providerContext, 'agent2', policyId, '8.17.0'); + await createAgentDoc(providerContext, 'agent3', policyId, '8.17.0'); + await createAgentDoc(providerContext, 'agent4', policyId, '8.17.1'); + // Update the policy to require version 8.17.1 for 50% of active agents. + await supertest + .put(`/api/fleet/agent_policies/${policyId}`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'Test policy', + namespace: 'default', + required_versions: [ + { + version: '8.17.1', + percentage: 50, + }, + ], + }) + .expect(200); + await waitForTask(); + // Check that only one agent on 8.17.0 was upgraded. + const res = await supertest.get('/api/fleet/agents').set('kbn-xsrf', 'xxx').expect(200); + expect(res.body.items.length).to.be(4); + expect(res.body.items.filter((item: any) => item.status === 'updating').length).to.be(1); + expect( + res.body.items.filter((item: any) => item.upgrade_started_at !== undefined).length + ).to.be(1); + expect( + res.body.items.filter((item: any) => item.upgrade_attempts !== undefined).length + ).to.be(1); + }); + + it('should not take inactive agents on target version into account', async () => { + // Create 2 active agents on 8.17.0, 1 active agent on 8.17.1 and 1 inactive agent on 8.17.1. + await createAgentDoc(providerContext, 'agent1', policyId, '8.17.0'); + await createAgentDoc(providerContext, 'agent2', policyId, '8.17.0'); + await createAgentDoc(providerContext, 'agent3', policyId, '8.17.1'); + await createAgentDoc(providerContext, 'agent4', policyId, '8.17.1', false); + // Update the policy to require version 8.17.1 for 50% of active agents. + await supertest + .put(`/api/fleet/agent_policies/${policyId}`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'Test policy', + namespace: 'default', + required_versions: [ + { + version: '8.17.1', + percentage: 50, + }, + ], + }) + .expect(200); + await waitForTask(); + // Check that two agents on 8.17.0 were upgraded. + const res = await supertest + .get('/api/fleet/agents?showInactive=true') + .set('kbn-xsrf', 'xxx') + .expect(200); + expect(res.body.items.length).to.be(4); + expect(res.body.items.filter((item: any) => item.status === 'updating').length).to.be(1); + expect( + res.body.items.filter((item: any) => item.upgrade_started_at !== undefined).length + ).to.be(1); + expect( + res.body.items.filter((item: any) => item.upgrade_attempts !== undefined).length + ).to.be(1); + }); + + it('should take already upgrading agents into account', async () => { + // Create an active agent on 8.17.0 and an active agent on 8.17.0 upgrading to 8.17.1. + await createAgentDoc(providerContext, 'agent1', policyId, '8.17.0'); + await createAgentDoc(providerContext, 'agent2', policyId, '8.17.0', true, { + upgrade_details: { + target_version: '8.17.1', + state: 'UPG_DOWNLOADING', + action_id: '123', + }, + }); + // Update the policy to require version 8.17.1 or 50% of active agents. + await supertest + .put(`/api/fleet/agent_policies/${policyId}`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'Test policy', + namespace: 'default', + required_versions: [ + { + version: '8.17.1', + percentage: 50, + }, + ], + }) + .expect(200); + await waitForTask(); + // Check that no agent was upgraded. + let res = await supertest.get('/api/fleet/agents/agent1').set('kbn-xsrf', 'xxx').expect(200); + expect(res.body.item.upgrade_started_at).to.be(undefined); + res = await supertest.get('/api/fleet/agents/agent2').set('kbn-xsrf', 'xxx').expect(200); + expect(res.body.item.upgrade_started_at).to.be(undefined); + }); + + it('should take agents marked but not ready for retry into account but not upgrade them', async () => { + // Create an active agent on 8.17.0 and an active agent on 8.17.0 marked but not ready for retry. + await createAgentDoc(providerContext, 'agent1', policyId, '8.17.0'); + await createAgentDoc(providerContext, 'agent2', policyId, '8.17.0', true, { + upgrade_details: { + target_version: '8.17.1', + state: 'UPG_FAILED', + action_id: '123', + }, + upgrade_attempts: [new Date().toISOString()], + }); + // Update the policy to require version 8.17.1 for 50% of active agents. + await supertest + .put(`/api/fleet/agent_policies/${policyId}`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'Test policy', + namespace: 'default', + required_versions: [ + { + version: '8.17.1', + percentage: 50, + }, + ], + }) + .expect(200); + await waitForTask(); + // Check that no agent was upgraded. + let res = await supertest.get('/api/fleet/agents/agent1').set('kbn-xsrf', 'xxx').expect(200); + expect(res.body.item.upgrade_started_at).to.be(undefined); + res = await supertest.get('/api/fleet/agents/agent2').set('kbn-xsrf', 'xxx').expect(200); + expect(res.body.item.upgrade_started_at).to.be(undefined); + }); + + it('should take agents marked and ready for retry into account and upgrade them', async () => { + // Create an active agent on 8.17.0 and an active agent on 8.17.0 marked and ready for retry. + await createAgentDoc(providerContext, 'agent1', policyId, '8.17.0'); + await createAgentDoc(providerContext, 'agent2', policyId, '8.17.0', true, { + upgrade_details: { + target_version: '8.17.1', + state: 'UPG_FAILED', + action_id: '123', + }, + upgrade_attempts: [new Date(new Date().getTime() - RETRY_DELAY).toISOString()], + }); + // Update the policy to require version 8.17.1 for 50% of active agents. + await supertest + .put(`/api/fleet/agent_policies/${policyId}`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'Test policy', + namespace: 'default', + required_versions: [ + { + version: '8.17.1', + percentage: 50, + }, + ], + }) + .expect(200); + await waitForTask(); + // Check that agent1 was upgraded. + let res = await supertest.get('/api/fleet/agents/agent1').set('kbn-xsrf', 'xxx').expect(200); + expect(res.body.item.upgrade_started_at).to.be(undefined); + res = await supertest.get('/api/fleet/agents/agent2').set('kbn-xsrf', 'xxx').expect(200); + expect(typeof res.body.item.upgrade_started_at).to.be('string'); + }); + }); +} diff --git a/x-pack/test/fleet_tasks/tests/index.ts b/x-pack/test/fleet_tasks/tests/index.ts new file mode 100644 index 000000000000..28eb53363c15 --- /dev/null +++ b/x-pack/test/fleet_tasks/tests/index.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrProviderContext } from '../ftr_provider_context'; + +export default function ({ loadTestFile }: FtrProviderContext) { + describe('Fleet packages test', function () { + loadTestFile(require.resolve('./automatic_upgrades')); + }); +}