github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-06-27 18:51:07 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Fleet] Added FTR test to verify sync integrations to a remote cluster (#222573)

Browse source 
## Summary

Closes https://github.com/elastic/kibana/issues/208017

Added test that starts 2 clusters, sets up remote ES output with sync
feature, sets up CCR between the clusters, and verifies the sync
installs the packages.

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Dzmitry Lemechko <dzmitry.lemechko@elastic.co>
This commit is contained in:
Julia Bardi 2025-06-17 12:56:58 +02:00 • committed by GitHub
parent 46e5f3bacf
commit cc4b6609d7
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
18 changed files with 487 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

1
.buildkite/ftr_platform_stateful_configs.yml
View file

@ -400,6 +400,7 @@ enabled:
- x-pack/platform/test/automatic_import_api_integration/apis/config_basic.ts
- x-pack/platform/test/automatic_import_api_integration/apis/config_graphs.ts
- x-pack/platform/test/encrypted_saved_objects_api_integration/config.ts
- x-pack/platform/test/fleet_multi_cluster/config.ts
- x-pack/platform/test/monitoring_api_integration/config.ts
- x-pack/platform/test/plugin_api_integration/config.ts
- x-pack/platform/test/saved_object_api_integration/security_and_spaces/config_basic.ts

1
.github/CODEOWNERS vendored
View file

@ -1477,6 +1477,7 @@ src/platform/plugins/shared/discover/public/context_awareness/profile_providers/
/src/platform/test/api_integration/apis/custom_integration/*.ts @elastic/fleet
/x-pack/test/fleet_cypress @elastic/fleet
/x-pack/test/fleet_functional @elastic/fleet
/x-pack/platform/test/fleet_multi_cluster @elastic/fleet
/src/dev/build/tasks/bundle_fleet_packages.ts @elastic/fleet @elastic/kibana-operations
/x-pack/platform/plugins/shared/fleet/server/services/elastic_agent_manifest.ts @elastic/fleet @elastic/obs-cloudnative-monitoring
/x-pack/test_serverless/**/test_suites/**/fleet/ @elastic/fleet

1
src/platform/packages/shared/kbn-test/src/functional_test_runner/lib/config/schema.ts
View file

@ -259,6 +259,7 @@ export const schema = Joi.object()
.default(),
env: Joi.object().unknown().default(),
delayShutdown: Joi.number(),
startRemoteKibana: Joi.boolean().default(false),
})
.default(),

8
src/platform/packages/shared/kbn-test/src/functional_tests/lib/run_kibana_server.ts
View file

@ -26,6 +26,7 @@ export async function runKibanaServer(options: {
logsDir?: string;
onEarlyExit?: (msg: string) => void;
inspect?: boolean;
remote?: boolean;
}) {
const { config, procs } = options;
const runOptions = options.config.get('kbnTestServer.runOptions');
@ -74,7 +75,7 @@ export async function runKibanaServer(options: {
kbnFlags = remapPluginPaths(kbnFlags, installDir);
}
const mainName = useTaskRunner ? 'kbn-ui' : 'kibana';
const mainName = (useTaskRunner ? 'kbn-ui' : 'kibana') + (options.remote ? '-remote' : '');
const promises = [
// main process
procs.run(mainName, {
@ -99,13 +100,14 @@ export async function runKibanaServer(options: {
if (useTaskRunner) {
const mainUuid = getArgValue(kbnFlags, 'server.uuid');
const tasksProcName = 'kbn-tasks' + (options.remote ? '-remote' : '');
// dedicated task runner
promises.push(
procs.run('kbn-tasks', {
procs.run(tasksProcName, {
...procRunnerOpts,
writeLogsToPath: options.logsDir
? Path.resolve(options.logsDir, 'kbn-tasks.log')
? Path.resolve(options.logsDir, `${tasksProcName}.log`)
: undefined,
args: [
...prefixArgs,

33
src/platform/packages/shared/kbn-test/src/functional_tests/run_tests/run_tests.ts
View file

@ -127,6 +127,39 @@ export async function runTests(log: ToolingLog, options: RunTestsOptions) {
],
});
const startRemoteKibana = config.get('kbnTestServer.startRemoteKibana');
if (startRemoteKibana) {
await runKibanaServer({
procs,
config: new Config({
settings: {
...config.getAll(),
kbnTestServer: {
sourceArgs: ['--no-base-path'],
serverArgs: [
...config.get('kbnTestServer.serverArgs'),
`--xpack.fleet.syncIntegrations.taskInterval=5s`,
`--elasticsearch.hosts=http://localhost:9221`,
`--server.port=5621`,
],
},
},
path: config.path,
module: config.module,
}),
logsDir: options.logsDir,
installDir: options.installDir,
onEarlyExit,
extraKbnOpts: [
config.get('serverless')
? '--server.versioned.versionResolution=newest'
: '--server.versioned.versionResolution=oldest',
],
remote: true,
});
}
if (abortCtrl.signal.aborted) {
return;
}

36
src/platform/packages/shared/kbn-test/src/functional_tests/start_servers/start_servers.ts
View file

@ -59,6 +59,42 @@ export async function startServers(log: ToolingLog, options: StartServerOptions)
],
});
const startRemoteKibana = config.get('kbnTestServer.startRemoteKibana');
if (startRemoteKibana) {
await runKibanaServer({
procs,
config: new Config({
settings: {
...config.getAll(),
kbnTestServer: {
sourceArgs: ['--no-base-path'],
serverArgs: [
...config.get('kbnTestServer.serverArgs'),
`--xpack.fleet.syncIntegrations.taskInterval=5s`,
`--elasticsearch.hosts=http://localhost:9221`,
`--server.port=5621`,
],
},
},
path: config.path,
module: config.module,
}),
installDir: options.installDir,
extraKbnOpts: options.installDir
? []
: [
'--dev',
'--no-dev-config',
'--no-dev-credentials',
config.get('serverless')
? '--server.versioned.versionResolution=newest'
: '--server.versioned.versionResolution=oldest',
],
remote: true,
});
}
reportTime(runStartTime, 'ready', {
success: true,
...options,

3
x-pack/platform/plugins/shared/fleet/common/types/index.ts
View file

@ -92,6 +92,9 @@ export interface FleetConfigType {
taskInterval?: string;
retryDelays?: string[];
};
syncIntegrations?: {
taskInterval?: string;
};
integrationsHomeOverride?: string;
prereleaseEnabledByDefault?: boolean;
}

5
x-pack/platform/plugins/shared/fleet/server/config.ts
View file

@ -309,6 +309,11 @@ export const config: PluginConfigDescriptor = {
retryDelays: schema.maybe(schema.arrayOf(schema.string())),
})
),
syncIntegrations: schema.maybe(
schema.object({
taskInterval: schema.maybe(schema.string()),
})
),
integrationsHomeOverride: schema.maybe(schema.string()),
prereleaseEnabledByDefault: schema.boolean({ defaultValue: false }),
},

3
x-pack/platform/plugins/shared/fleet/server/plugin.ts
View file

@ -667,6 +667,9 @@ export class FleetPlugin
core,
taskManager: deps.taskManager,
logFactory: this.initializerContext.logger,
config: {
taskInterval: config.syncIntegrations?.taskInterval,
},
});
this.automaticAgentUpgradeTask = new AutomaticAgentUpgradeTask({
core,

3
x-pack/platform/plugins/shared/fleet/server/tasks/sync_integrations/sync_integrations_task.test.ts
View file

@ -111,6 +111,9 @@ describe('SyncIntegrationsTask', () => {
core: mockCore,
taskManager: mockTaskManagerSetup,
logFactory: loggingSystemMock.create(),
config: {
taskInterval: '1m',
},
});
});

15
x-pack/platform/plugins/shared/fleet/server/tasks/sync_integrations/sync_integrations_task.ts
View file

@ -35,13 +35,18 @@ export const TYPE = 'fleet:sync-integrations-task';
export const VERSION = '1.0.5';
const TITLE = 'Fleet Sync Integrations Task';
const SCOPE = ['fleet'];
const INTERVAL = '5m';
const DEFAULT_INTERVAL = '5m';
const TIMEOUT = '5m';
interface SyncIntegrationsTaskConfig {
taskInterval?: string;
}
interface SyncIntegrationsTaskSetupContract {
core: CoreSetup;
taskManager: TaskManagerSetupContract;
logFactory: LoggerFactory;
config: SyncIntegrationsTaskConfig;
}
interface SyncIntegrationsTaskStartContract {
@ -52,10 +57,12 @@ export class SyncIntegrationsTask {
private logger: Logger;
private wasStarted: boolean = false;
private abortController = new AbortController();
private taskInterval: string;
constructor(setupContract: SyncIntegrationsTaskSetupContract) {
const { core, taskManager, logFactory } = setupContract;
const { core, taskManager, logFactory, config } = setupContract;
this.logger = logFactory.get(this.taskId);
this.taskInterval = config.taskInterval ?? DEFAULT_INTERVAL;
taskManager.registerTaskDefinitions({
[TYPE]: {
@ -82,7 +89,7 @@ export class SyncIntegrationsTask {
}
this.wasStarted = true;
this.logger.info(`[SyncIntegrationsTask] Started with interval of [${INTERVAL}]`);
this.logger.info(`[SyncIntegrationsTask] Started with interval of [${this.taskInterval}]`);
try {
await taskManager.ensureScheduled({
@ -90,7 +97,7 @@ export class SyncIntegrationsTask {
taskType: TYPE,
scope: SCOPE,
schedule: {
interval: INTERVAL,
interval: this.taskInterval,
},
state: {},
params: { version: VERSION },

231
x-pack/platform/test/fleet_multi_cluster/apps/fleet/sync_integrations_flow.ts Normal file
View file

@ -0,0 +1,231 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import axios from 'axios';
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../ftr_provider_context';
interface IntegrationPackage {
name: string;
version: string;
}
export default ({ getService }: FtrProviderContext) => {
const security = getService('security');
const retry = getService('retry');
const remoteEs = getService('remoteEs' as 'es');
const localEs = getService('es');
const supertest = getService('supertest');
describe('Fleet Multi Cluster Sync Integrations E2E', function () {
before(async () => {
await security.testUser.setRoles(['superuser']);
});
const installPackage = ({ name, version }: IntegrationPackage) => {
return supertest
.post(`/api/fleet/epm/packages/${name}/${version}`)
.set('kbn-xsrf', 'xxxx')
.send({ force: true })
.expect(200);
};
const uninstallPackage = ({ name, version }: IntegrationPackage) => {
return supertest
.delete(`/api/fleet/epm/packages/${name}/${version}`)
.set('kbn-xsrf', 'xxxx')
.send({ force: true })
.expect(200);
};
async function createRemoteServiceToken(): Promise<string> {
const { token } = await remoteEs.security.createServiceToken({
namespace: 'elastic',
service: 'fleet-server-remote',
});
return token.value;
}
async function createRemoteAPIKey(): Promise<string> {
const apiKeyResp = await remoteEs.security.createApiKey({
name: 'integration_sync_api_key',
role_descriptors: {
integration_writer: {
cluster: [],
indices: [],
applications: [
{
application: 'kibana-.kibana',
privileges: ['feature_fleet.read', 'feature_fleetv2.read'],
resources: ['*'],
},
],
},
},
});
return apiKeyResp.encoded;
}
async function createRemoteOutput() {
await supertest
.post('/api/fleet/outputs')
.set('kbn-xsrf', 'xxxx')
.send({
id: 'remote-elasticsearch1',
name: 'Remote ES Output',
type: 'remote_elasticsearch',
hosts: ['http://localhost:9221'],
kibana_api_key: await createRemoteAPIKey(),
kibana_url: 'http://localhost:5621',
sync_integrations: true,
sync_uninstalled_integrations: true,
secrets: {
service_token: await createRemoteServiceToken(),
},
})
.expect(200);
}
async function createLocalOutputOnRemote() {
const response = await axios.post(
'http://localhost:5621/api/fleet/outputs',
{
id: 'es',
type: 'elasticsearch',
name: 'Local ES Output',
hosts: ['http://localhost:9221'],
},
{
auth: { username: 'elastic', password: 'changeme' },
headers: { 'kbn-xsrf': 'true', 'x-elastic-internal-origin': 'fleet-e2e' },
}
);
expect(response.status).to.be(200);
}
async function addRemoteCluster() {
const resp = await remoteEs.cluster.putSettings({
persistent: {
cluster: {
remote: {
local: {
seeds: ['localhost:9300'],
},
},
},
},
});
expect(resp.acknowledged).to.be(true);
}
async function createFollowerIndex() {
const resp = await remoteEs.ccr.follow({
index: 'fleet-synced-integrations-ccr-local',
leader_index: 'fleet-synced-integrations',
remote_cluster: 'local',
wait_for_active_shards: 'all',
});
expect(resp.follow_index_created).to.be(true);
}
async function queryFollowerIndexDoc() {
const resp = await remoteEs.get({
id: 'fleet-synced-integrations',
index: 'fleet-synced-integrations-ccr-local',
});
expect(resp.found).to.be(true);
}
async function verifySyncIntegrationsStatus(isUninstalled = false) {
const resp = await supertest
.get('/api/fleet/remote_synced_integrations/remote-elasticsearch1/remote_status')
.set('kbn-xsrf', 'xxxx')
.expect(200);
const respJson = JSON.parse(resp.text);
const nginxIntegration = respJson.integrations.find(
(int: any) => int.package_name === 'nginx'
);
expect(nginxIntegration?.sync_status).to.be('completed');
if (isUninstalled) {
expect(nginxIntegration?.install_status.remote).to.be('not_installed');
}
}
async function verifyPackageInstalledOnRemote() {
const resp = await remoteEs.get({
id: 'epm-packages:nginx',
index: '.kibana_ingest',
});
expect(resp.found).to.be(true);
expect((resp._source as any)?.['epm-packages'].install_status).to.be('installed');
}
async function verifyPackageUninstalledOnRemote() {
const resp = await remoteEs.get(
{
id: 'epm-packages:nginx',
index: '.kibana_ingest',
},
{ ignore: [404] }
);
expect(resp.found).to.be(false);
}
it('should sync integrations to remote cluster when enabled on remote ES output', async () => {
await installPackage({ name: 'nginx', version: '2.0.0' });
await createRemoteOutput();
await createLocalOutputOnRemote();
await addRemoteCluster();
await createFollowerIndex();
await retry.tryForTime(10000, async () => {
await queryFollowerIndexDoc();
});
// check nginx package is installed on remote
await retry.tryForTime(20000, async () => {
await verifySyncIntegrationsStatus();
await verifyPackageInstalledOnRemote();
});
// verify uninstalled packages are synced
await uninstallPackage({ name: 'nginx', version: '2.0.0' });
await retry.tryForTime(20000, async () => {
await verifySyncIntegrationsStatus(true);
await verifyPackageUninstalledOnRemote();
});
});
after(async () => {
// Clean up the remote output
await supertest
.delete('/api/fleet/outputs/remote-elasticsearch1')
.set('kbn-xsrf', 'xxxx')
.expect(200);
// Clean up the local output on remote
const response = await axios.delete('http://localhost:5621/api/fleet/outputs/es', {
auth: { username: 'elastic', password: 'changeme' },
headers: { 'kbn-xsrf': 'true', 'x-elastic-internal-origin': 'fleet-e2e' },
});
expect(response.status).to.be(200);
await localEs.indices.delete({
index: 'fleet-synced-integrations',
});
await remoteEs.indices.delete({
index: 'fleet-synced-integrations-ccr-local',
});
await security.testUser.restoreDefaults();
});
});
};

83
x-pack/platform/test/fleet_multi_cluster/config.ts Normal file
View file

@ -0,0 +1,83 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { resolve } from 'path';
import { FtrConfigProviderContext, getKibanaCliLoggers } from '@kbn/test';
import { RemoteEsArchiverProvider } from './services/remote_es/remote_es_archiver';
import { RemoteEsProvider } from './services/remote_es/remote_es';
export default async function ({ readConfigFile }: FtrConfigProviderContext) {
const xpackFunctionalConfig = await readConfigFile(
require.resolve('@kbn/test-suites-xpack/functional/config.base')
);
return {
...xpackFunctionalConfig.getAll(),
testFiles: [resolve(__dirname, './apps/fleet/sync_integrations_flow')],
junit: {
reportName: 'X-Pack Fleet Multi Cluster Tests',
},
services: {
...xpackFunctionalConfig.get('services'),
remoteEs: RemoteEsProvider,
remoteEsArchiver: RemoteEsArchiverProvider,
},
apps: {
...xpackFunctionalConfig.get('apps'),
['fleet']: {
pathname: '/app/fleet',
},
},
kbnTestServer: {
...xpackFunctionalConfig.get('kbnTestServer'),
serverArgs: [
...xpackFunctionalConfig.get('kbnTestServer.serverArgs'),
`--xpack.fleet.syncIntegrations.taskInterval=5s`,
`--logging.loggers=${JSON.stringify([
...getKibanaCliLoggers(xpackFunctionalConfig.get('kbnTestServer.serverArgs')),
// Enable debug fleet logs by default
{
name: 'plugins.fleet',
level: 'debug',
appenders: ['default'],
},
])}`,
],
startRemoteKibana: true,
},
security: {
...xpackFunctionalConfig.get('security'),
remoteEsRoles: {
ccs_remote_search: {
cluster: ['manage', 'manage_ccr'],
indices: [
{
names: ['*'],
privileges: ['read', 'view_index_metadata', 'read_cross_cluster', 'monitor'],
},
],
},
},
defaultRoles: [
...(xpackFunctionalConfig.get('security.defaultRoles') ?? []),
'ccs_remote_search',
],
},
esTestCluster: {
...xpackFunctionalConfig.get('esTestCluster'),
ccs: {
remoteClusterUrl:
process.env.REMOTE_CLUSTER_URL ??
'http://elastic:changeme@localhost:' +
`${xpackFunctionalConfig.get('servers.elasticsearch.port') + 1}`,
},
serverArgs: ['xpack.ml.enabled=false'],
license: 'trial',
},
};
}

10
x-pack/platform/test/fleet_multi_cluster/ftr_provider_context.d.ts vendored Normal file
View file

@ -0,0 +1,10 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { GenericFtrProviderContext } from '@kbn/test';
import { services } from './services';
export type FtrProviderContext = GenericFtrProviderContext<typeof services, {}>;

16
x-pack/platform/test/fleet_multi_cluster/services/index.ts Normal file
View file

@ -0,0 +1,16 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { services as kibanaFunctionalServices } from '@kbn/test-suites-src/functional/services';
import { RemoteEsProvider } from '@kbn/test-suites-src/functional/services/remote_es/remote_es';
import { RemoteEsArchiverProvider } from '@kbn/test-suites-src/functional/services/remote_es/remote_es_archiver';
export const services = {
...kibanaFunctionalServices,
remoteEs: RemoteEsProvider,
remoteEsArchiver: RemoteEsArchiverProvider,
};

23
x-pack/platform/test/fleet_multi_cluster/services/remote_es/remote_es.ts Normal file
View file

@ -0,0 +1,23 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { Client } from '@elastic/elasticsearch';
import { systemIndicesSuperuser, createRemoteEsClientForFtrConfig } from '@kbn/test';
import { FtrProviderContext } from '../../ftr_provider_context';
/**
* Kibana-specific @elastic/elasticsearch client instance.
*/
export function RemoteEsProvider({ getService }: FtrProviderContext): Client {
const config = getService('config');
return createRemoteEsClientForFtrConfig(config, {
// Use system indices user so tests can write to system indices
authOverride: systemIndicesSuperuser,
});
}

21
x-pack/platform/test/fleet_multi_cluster/services/remote_es/remote_es_archiver.ts Normal file
View file

@ -0,0 +1,21 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { EsArchiver } from '@kbn/es-archiver';
import { FtrProviderContext } from '../../ftr_provider_context';
export function RemoteEsArchiverProvider({ getService }: FtrProviderContext): EsArchiver {
const remoteEs = getService('remoteEs' as 'es');
const log = getService('log');
const kibanaServer = getService('kibanaServer');
return new EsArchiver({
client: remoteEs,
log,
kbnClient: kibanaServer,
});
}

1
x-pack/platform/test/tsconfig.json
View file

@ -90,6 +90,7 @@
"@kbn/rule-data-utils",
"@kbn/actions-plugin",
"@kbn/core-saved-objects-api-server",
"@kbn/es-archiver",
"@kbn/es-query",
"@kbn/stack-alerts-plugin",
"@kbn/triggers-actions-ui-plugin",