[Fleet] Add reset preconfigured policies API (#122467)

This commit is contained in:
Nicolas Chaulet 2022-01-13 09:24:19 -05:00 committed by GitHub
parent c9dcd9332a
commit d0850b0dd1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 584 additions and 57 deletions

View file

@ -7,6 +7,8 @@
// Base API paths
export const INTERNAL_ROOT = `/internal/fleet`;
export const API_ROOT = `/api/fleet`;
export const EPM_API_ROOT = `${API_ROOT}/epm`;
export const DATA_STREAM_API_ROOT = `${API_ROOT}/data_streams`;
@ -133,4 +135,6 @@ export const INSTALL_SCRIPT_API_ROUTES = `${API_ROOT}/install/{osType}`;
// Policy preconfig API routes
export const PRECONFIGURATION_API_ROUTES = {
UPDATE_PATTERN: `${API_ROOT}/setup/preconfiguration`,
RESET_PATTERN: `${INTERNAL_ROOT}/reset_preconfigured_agent_policies`,
RESET_ONE_PATTERN: `${INTERNAL_ROOT}/reset_preconfigured_agent_policies/{agentPolicyId}`,
};

View file

@ -0,0 +1,236 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import Path from 'path';
import * as kbnTestServer from 'src/core/test_helpers/kbn_server';
import type { AgentPolicySOAttributes } from '../types';
const logFilePath = Path.join(__dirname, 'logs.log');
type Root = ReturnType<typeof kbnTestServer.createRoot>;
const waitForFleetSetup = async (root: Root) => {
const isFleetSetupRunning = async () => {
const statusApi = kbnTestServer.getSupertest(root, 'get', '/api/status');
const resp = await statusApi.send();
const fleetStatus = resp.body?.status?.plugins?.fleet;
if (fleetStatus?.meta?.error) {
throw new Error(`Setup failed: ${JSON.stringify(fleetStatus)}`);
}
return !fleetStatus || fleetStatus?.summary === 'Fleet is setting up';
};
while (await isFleetSetupRunning()) {
await new Promise((resolve) => setTimeout(resolve, 2000));
}
};
describe('Fleet preconfiguration rest', () => {
let esServer: kbnTestServer.TestElasticsearchUtils;
let kbnServer: kbnTestServer.TestKibanaUtils;
const startServers = async () => {
const { startES } = kbnTestServer.createTestServers({
adjustTimeout: (t) => jest.setTimeout(t),
settings: {
es: {
license: 'trial',
},
kbn: {},
},
});
esServer = await startES();
const startKibana = async () => {
const root = kbnTestServer.createRootWithCorePlugins(
{
xpack: {
fleet: {
agentPolicies: [
{
name: 'Elastic Cloud agent policy 0001',
description: 'Default agent policy for agents hosted on Elastic Cloud',
is_default: false,
is_managed: true,
id: 'test-12345',
namespace: 'default',
monitoring_enabled: [],
package_policies: [
{
name: 'fleet_server123456789',
package: {
name: 'fleet_server',
},
inputs: [
{
type: 'fleet-server',
keep_enabled: true,
vars: [
{
name: 'host',
value: '127.0.0.1',
frozen: true,
},
],
},
],
},
],
},
],
},
},
logging: {
appenders: {
file: {
type: 'file',
fileName: logFilePath,
layout: {
type: 'json',
},
},
},
loggers: [
{
name: 'root',
appenders: ['file'],
},
{
name: 'plugins.fleet',
level: 'all',
},
],
},
},
{ oss: false }
);
await root.preboot();
const coreSetup = await root.setup();
const coreStart = await root.start();
return {
root,
coreSetup,
coreStart,
stop: async () => await root.shutdown(),
};
};
kbnServer = await startKibana();
await waitForFleetSetup(kbnServer.root);
};
const stopServers = async () => {
if (kbnServer) {
await kbnServer.stop();
}
if (esServer) {
await esServer.stop();
}
await new Promise((res) => setTimeout(res, 10000));
};
beforeEach(async () => {
await startServers();
});
afterEach(async () => {
await stopServers();
});
describe('Reset all policy', () => {
it('Works and reset all preconfigured policies', async () => {
const resetAPI = kbnTestServer.getSupertest(
kbnServer.root,
'post',
'/internal/fleet/reset_preconfigured_agent_policies'
);
await resetAPI.set('kbn-sxrf', 'xx').send();
const agentPolicies = await kbnServer.coreStart.savedObjects
.createInternalRepository()
.find<AgentPolicySOAttributes>({
type: 'ingest-agent-policies',
perPage: 10000,
});
expect(agentPolicies.saved_objects).toHaveLength(1);
expect(agentPolicies.saved_objects.map((ap) => ap.attributes)).toEqual(
expect.arrayContaining([
expect.objectContaining({
name: 'Elastic Cloud agent policy 0001',
}),
])
);
});
});
describe('Reset one preconfigured policy', () => {
const POLICY_ID = 'test-12345';
it('Works and reset one preconfigured policies if the policy is already deleted (with a ghost package policy)', async () => {
const soClient = kbnServer.coreStart.savedObjects.createInternalRepository();
await soClient.delete('ingest-agent-policies', POLICY_ID);
const resetAPI = kbnTestServer.getSupertest(
kbnServer.root,
'post',
'/internal/fleet/reset_preconfigured_agent_policies/test-12345'
);
await resetAPI.set('kbn-sxrf', 'xx').send();
const agentPolicies = await kbnServer.coreStart.savedObjects
.createInternalRepository()
.find<AgentPolicySOAttributes>({
type: 'ingest-agent-policies',
perPage: 10000,
});
expect(agentPolicies.saved_objects).toHaveLength(1);
expect(agentPolicies.saved_objects.map((ap) => ap.attributes)).toEqual(
expect.arrayContaining([
expect.objectContaining({
name: 'Elastic Cloud agent policy 0001',
}),
])
);
});
it('Works if the preconfigured policies already exists with a missing package policy', async () => {
const soClient = kbnServer.coreStart.savedObjects.createInternalRepository();
await soClient.update('ingest-agent-policies', POLICY_ID, {
package_policies: [],
});
const resetAPI = kbnTestServer.getSupertest(
kbnServer.root,
'post',
'/internal/fleet/reset_preconfigured_agent_policies/test-12345'
);
await resetAPI.set('kbn-sxrf', 'xx').send();
const agentPolicies = await soClient.find<AgentPolicySOAttributes>({
type: 'ingest-agent-policies',
perPage: 10000,
});
expect(agentPolicies.saved_objects).toHaveLength(1);
expect(agentPolicies.saved_objects.map((ap) => ap.attributes)).toEqual(
expect.arrayContaining([
expect.objectContaining({
name: 'Elastic Cloud agent policy 0001',
package_policies: expect.arrayContaining([expect.stringMatching(/.*/)]),
}),
])
);
});
});
});

View file

@ -0,0 +1,77 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { TypeOf } from '@kbn/config-schema';
import type { PreconfiguredAgentPolicy } from '../../../common';
import type { FleetRequestHandler } from '../../types';
import type {
PutPreconfigurationSchema,
PostResetOnePreconfiguredAgentPolicies,
} from '../../types';
import { defaultIngestErrorHandler } from '../../errors';
import { ensurePreconfiguredPackagesAndPolicies, outputService } from '../../services';
import { resetPreconfiguredAgentPolicies } from '../../services/preconfiguration/index';
export const updatePreconfigurationHandler: FleetRequestHandler<
undefined,
undefined,
TypeOf<typeof PutPreconfigurationSchema.body>
> = async (context, request, response) => {
const soClient = context.core.savedObjects.client;
const esClient = context.core.elasticsearch.client.asInternalUser;
const defaultOutput = await outputService.ensureDefaultOutput(soClient);
const spaceId = context.fleet.spaceId;
const { agentPolicies, packages } = request.body;
try {
const body = await ensurePreconfiguredPackagesAndPolicies(
soClient,
esClient,
(agentPolicies as PreconfiguredAgentPolicy[]) ?? [],
packages ?? [],
defaultOutput,
spaceId
);
return response.ok({ body });
} catch (error) {
return defaultIngestErrorHandler({ error, response });
}
};
export const resetPreconfigurationHandler: FleetRequestHandler<
TypeOf<typeof PostResetOnePreconfiguredAgentPolicies.params>,
undefined,
undefined
> = async (context, request, response) => {
const soClient = context.core.savedObjects.client;
const esClient = context.core.elasticsearch.client.asInternalUser;
try {
await resetPreconfiguredAgentPolicies(soClient, esClient, request.params.agentPolicyid);
return response.ok({});
} catch (error) {
return defaultIngestErrorHandler({ error, response });
}
};
export const resetOnePreconfigurationHandler: FleetRequestHandler<
undefined,
undefined,
undefined
> = async (context, request, response) => {
const soClient = context.core.savedObjects.client;
const esClient = context.core.elasticsearch.client.asInternalUser;
try {
await resetPreconfiguredAgentPolicies(soClient, esClient);
return response.ok({});
} catch (error) {
return defaultIngestErrorHandler({ error, response });
}
};

View file

@ -5,45 +5,38 @@
* 2.0.
*/
import type { RequestHandler } from 'src/core/server';
import type { TypeOf } from '@kbn/config-schema';
import type { PreconfiguredAgentPolicy } from '../../../common';
import { PRECONFIGURATION_API_ROUTES } from '../../constants';
import type { FleetRequestHandler } from '../../types';
import { PutPreconfigurationSchema } from '../../types';
import { defaultIngestErrorHandler } from '../../errors';
import { ensurePreconfiguredPackagesAndPolicies, outputService } from '../../services';
import type { FleetAuthzRouter } from '../security';
export const updatePreconfigurationHandler: FleetRequestHandler<
undefined,
undefined,
TypeOf<typeof PutPreconfigurationSchema.body>
> = async (context, request, response) => {
const soClient = context.core.savedObjects.client;
const esClient = context.core.elasticsearch.client.asInternalUser;
const defaultOutput = await outputService.ensureDefaultOutput(soClient);
const spaceId = context.fleet.spaceId;
const { agentPolicies, packages } = request.body;
try {
const body = await ensurePreconfiguredPackagesAndPolicies(
soClient,
esClient,
(agentPolicies as PreconfiguredAgentPolicy[]) ?? [],
packages ?? [],
defaultOutput,
spaceId
);
return response.ok({ body });
} catch (error) {
return defaultIngestErrorHandler({ error, response });
}
};
import {
updatePreconfigurationHandler,
resetPreconfigurationHandler,
resetOnePreconfigurationHandler,
} from './handler';
export const registerRoutes = (router: FleetAuthzRouter) => {
router.post(
{
path: PRECONFIGURATION_API_ROUTES.RESET_PATTERN,
validate: false,
fleetAuthz: {
fleet: { all: true },
},
},
resetPreconfigurationHandler
);
router.post(
{
path: PRECONFIGURATION_API_ROUTES.RESET_ONE_PATTERN,
validate: false,
fleetAuthz: {
fleet: { all: true },
},
},
resetOnePreconfigurationHandler
);
router.put(
{
path: PRECONFIGURATION_API_ROUTES.UPDATE_PATTERN,
@ -52,6 +45,6 @@ export const registerRoutes = (router: FleetAuthzRouter) => {
fleet: { all: true },
},
},
updatePreconfigurationHandler as RequestHandler
updatePreconfigurationHandler
);
};

View file

@ -618,22 +618,23 @@ class AgentPolicyService {
public async delete(
soClient: SavedObjectsClientContract,
esClient: ElasticsearchClient,
id: string
id: string,
options?: { force?: boolean; removeFleetServerDocuments?: boolean }
): Promise<DeleteAgentPolicyResponse> {
const agentPolicy = await this.get(soClient, id, false);
if (!agentPolicy) {
throw new Error('Agent policy not found');
}
if (agentPolicy.is_managed) {
if (agentPolicy.is_managed && !options?.force) {
throw new HostedAgentPolicyRestrictionRelatedError(`Cannot delete hosted agent policy ${id}`);
}
if (agentPolicy.is_default) {
if (agentPolicy.is_default && !options?.force) {
throw new Error('The default agent policy cannot be deleted');
}
if (agentPolicy.is_default_fleet_server) {
if (agentPolicy.is_default_fleet_server && !options?.force) {
throw new Error('The default fleet server agent policy cannot be deleted');
}
@ -655,6 +656,7 @@ class AgentPolicyService {
esClient,
agentPolicy.package_policies as string[],
{
force: options?.force,
skipUnassignFromAgentPolicies: true,
}
);
@ -667,7 +669,7 @@ class AgentPolicyService {
}
}
if (agentPolicy.is_preconfigured) {
if (agentPolicy.is_preconfigured && !options?.force) {
await soClient.create(PRECONFIGURATION_DELETION_RECORD_SAVED_OBJECT_TYPE, {
id: String(id),
});
@ -675,6 +677,11 @@ class AgentPolicyService {
await soClient.delete(SAVED_OBJECT_TYPE, id);
await this.triggerAgentPolicyUpdatedEvent(soClient, esClient, 'deleted', id);
if (options?.removeFleetServerDocuments) {
this.deleteFleetServerPoliciesForPolicyId(esClient, id);
}
return {
id,
name: agentPolicy.name,
@ -720,6 +727,23 @@ class AgentPolicyService {
});
}
public async deleteFleetServerPoliciesForPolicyId(
esClient: ElasticsearchClient,
agentPolicyId: string
) {
await esClient.deleteByQuery({
index: AGENT_POLICY_INDEX,
ignore_unavailable: true,
body: {
query: {
term: {
policy_id: agentPolicyId,
},
},
},
});
}
public async getLatestFleetPolicy(esClient: ElasticsearchClient, agentPolicyId: string) {
const res = await esClient.search({
index: AGENT_POLICY_INDEX,

View file

@ -98,21 +98,33 @@ export async function getEnrollmentAPIKey(
* Invalidate an api key and mark it as inactive
* @param id
*/
export async function deleteEnrollmentApiKey(esClient: ElasticsearchClient, id: string) {
export async function deleteEnrollmentApiKey(
esClient: ElasticsearchClient,
id: string,
forceDelete = false
) {
const enrollmentApiKey = await getEnrollmentAPIKey(esClient, id);
await invalidateAPIKeys([enrollmentApiKey.api_key_id]);
await esClient.update({
index: ENROLLMENT_API_KEYS_INDEX,
id,
body: {
doc: {
active: false,
if (forceDelete) {
await esClient.delete({
index: ENROLLMENT_API_KEYS_INDEX,
id,
refresh: 'wait_for',
});
} else {
await esClient.update({
index: ENROLLMENT_API_KEYS_INDEX,
id,
body: {
doc: {
active: false,
},
},
},
refresh: 'wait_for',
});
refresh: 'wait_for',
});
}
}
export async function deleteEnrollmentApiKeyForAgentPolicyId(

View file

@ -0,0 +1,8 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export { resetPreconfiguredAgentPolicies } from './reset_agent_policies';

View file

@ -0,0 +1,149 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import pMap from 'p-map';
import type { ElasticsearchClient, SavedObjectsClientContract, Logger } from 'src/core/server';
import { appContextService } from '../app_context';
import { setupFleet } from '../setup';
import {
AGENT_POLICY_SAVED_OBJECT_TYPE,
SO_SEARCH_LIMIT,
PACKAGE_POLICY_SAVED_OBJECT_TYPE,
} from '../../constants';
import { agentPolicyService } from '../agent_policy';
import { packagePolicyService } from '../package_policy';
import { getAgentsByKuery, forceUnenrollAgent } from '../agents';
import { listEnrollmentApiKeys, deleteEnrollmentApiKey } from '../api_keys';
import type { AgentPolicy } from '../../types';
export async function resetPreconfiguredAgentPolicies(
soClient: SavedObjectsClientContract,
esClient: ElasticsearchClient,
agentPolicyId?: string
) {
const logger = appContextService.getLogger();
logger.warn('Reseting Fleet preconfigured agent policies');
await _deleteExistingData(soClient, esClient, logger, agentPolicyId);
await _deleteGhostPackagePolicies(soClient, esClient, logger);
await setupFleet(soClient, esClient);
}
/**
* Delete all package policies that are not used in any agent policies
*/
async function _deleteGhostPackagePolicies(
soClient: SavedObjectsClientContract,
esClient: ElasticsearchClient,
logger: Logger
) {
const { items: packagePolicies } = await packagePolicyService.list(soClient, {
perPage: SO_SEARCH_LIMIT,
});
const policyIds = Array.from(
packagePolicies.reduce((acc, packagePolicy) => {
acc.add(packagePolicy.policy_id);
return acc;
}, new Set<string>())
);
const objects = policyIds.map((id) => ({ id, type: AGENT_POLICY_SAVED_OBJECT_TYPE }));
const agentPolicyExistsMap = (await soClient.bulkGet(objects)).saved_objects.reduce((acc, so) => {
if (so.error && so.error.statusCode === 404) {
acc.set(so.id, false);
} else {
acc.set(so.id, true);
}
return acc;
}, new Map<string, boolean>());
await pMap(
packagePolicies,
(packagePolicy) => {
if (agentPolicyExistsMap.get(packagePolicy.policy_id) === false) {
logger.info(`Deleting ghost package policy ${packagePolicy.name} (${packagePolicy.id})`);
return soClient.delete(PACKAGE_POLICY_SAVED_OBJECT_TYPE, packagePolicy.id);
}
},
{
concurrency: 20,
}
);
}
async function _deleteExistingData(
soClient: SavedObjectsClientContract,
esClient: ElasticsearchClient,
logger: Logger,
agentPolicyId?: string
) {
let existingPolicies: AgentPolicy[];
if (agentPolicyId) {
const policy = await agentPolicyService.get(soClient, agentPolicyId);
if (!policy || !policy.is_preconfigured) {
throw new Error('Invalid policy');
}
existingPolicies = [policy];
}
{
existingPolicies = (
await agentPolicyService.list(soClient, {
perPage: SO_SEARCH_LIMIT,
kuery: `${AGENT_POLICY_SAVED_OBJECT_TYPE}.is_preconfigured:true`,
})
).items;
}
// unenroll all the agents enroled in this policies
const { agents } = await getAgentsByKuery(esClient, {
showInactive: false,
perPage: SO_SEARCH_LIMIT,
kuery: existingPolicies.map((policy) => `policy_id:"${policy.id}"`).join(' or '),
});
// Delete
if (agents.length > 0) {
logger.info(`Force unenrolling ${agents.length} agents`);
await pMap(agents, (agent) => forceUnenrollAgent(soClient, esClient, agent.id), {
concurrency: 20,
});
}
const { items: enrollmentApiKeys } = await listEnrollmentApiKeys(esClient, {
perPage: SO_SEARCH_LIMIT,
showInactive: true,
});
if (enrollmentApiKeys.length > 0) {
logger.info(`Deleting ${enrollmentApiKeys.length} enrollment api keys`);
await pMap(
enrollmentApiKeys,
(enrollmentKey) => deleteEnrollmentApiKey(esClient, enrollmentKey.id, true),
{
concurrency: 20,
}
);
}
if (existingPolicies.length > 0) {
logger.info(`Deleting ${existingPolicies.length} agent policies`);
await pMap(
existingPolicies,
(policy) =>
agentPolicyService.delete(soClient, esClient, policy.id, {
force: true,
removeFleetServerDocuments: true,
}),
{
concurrency: 20,
}
);
}
}

View file

@ -15,3 +15,9 @@ export const PutPreconfigurationSchema = {
packages: schema.maybe(PreconfiguredPackagesSchema),
}),
};
export const PostResetOnePreconfiguredAgentPolicies = {
params: schema.object({
agentPolicyid: schema.string(),
}),
};

View file

@ -49,7 +49,31 @@ export class ReindexWorker {
private readonly log: Logger;
private readonly security: SecurityPluginStart;
constructor(
public static create(
client: SavedObjectsClientContract,
credentialStore: CredentialStore,
clusterClient: IClusterClient,
log: Logger,
licensing: LicensingPluginSetup,
security: SecurityPluginStart
): ReindexWorker {
if (ReindexWorker.workerSingleton) {
log.debug(`More than one ReindexWorker cannot be created, returning existing worker.`);
} else {
ReindexWorker.workerSingleton = new ReindexWorker(
client,
credentialStore,
clusterClient,
log,
licensing,
security
);
}
return ReindexWorker.workerSingleton;
}
private constructor(
private client: SavedObjectsClientContract,
private credentialStore: CredentialStore,
private clusterClient: IClusterClient,
@ -60,10 +84,6 @@ export class ReindexWorker {
this.log = log.get('reindex_worker');
this.security = security;
if (ReindexWorker.workerSingleton) {
throw new Error(`More than one ReindexWorker cannot be created.`);
}
const callAsInternalUser = this.clusterClient.asInternalUser;
this.reindexService = reindexServiceFactory(
@ -72,8 +92,6 @@ export class ReindexWorker {
log,
this.licensing
);
ReindexWorker.workerSingleton = this;
}
/**

View file

@ -34,5 +34,5 @@ export function createReindexWorker({
security,
}: CreateReindexWorker) {
const esClient = elasticsearchService.client;
return new ReindexWorker(savedObjects, credentialStore, esClient, logger, licensing, security);
return ReindexWorker.create(savedObjects, credentialStore, esClient, logger, licensing, security);
}