mirror of
https://github.com/elastic/kibana.git
synced 2025-04-23 09:19:04 -04:00
[Cloud Security] Add upgrade agentless deployment background task (#207143)
## Summary Summarize your PR. If it involves visual changes include a screenshot or gif. This PR add background task to upgrade Agentless Deployments after Kibana Stack has been upgrade in ESS. Once the Kibana stack upgrades, the task will do following: 1. Fetch agentless policies with package policies that have agents 2. Check if agentless agents version is upgradeable by use `semverLT` which see if current agent version less than latest available upgrade version and current kibana version 3. If agent version is upgradedable, then task will calls Agentless Upgrade Endpoint to upgrade agentless deployment. 4. Agent should be upgraded to latest available upgraded version  **How to test PR:** Prerequisite: Install [QAF Tool](https://docs.elastic.dev/appex-qa/qaf/getting-started) Create EC cloud api key [QAF Elastic Cloud](https://docs.elastic.dev/appex-qa/qaf/features/ec-deployments) 1. Go to Elastic Cloud and Create ESS Deployment in `8.17.0-SNAPSHOT` ```qaf elastic-cloud deployments create --environment production --region gcp-us-west2 --stack-version 8.17.0-SNAPSHOT --version-validation --deployment-name <DEPLOYMENT_NAME> ``` 2. Create an Agentless Integration 3. Upgrade stack to `8.18.0-SNAPSHOT` > `8.19.0-SNAPSHOT` 4. Run the following QAF command ```qaf elastic-cloud deployments upgrade <DEPLOYMENT_NAME> 9.1.0-SNAPSHOT --kb-docker-image docker.elastic.co/kibana-ci/kibana-cloud:9.1.0-SNAPSHOT-5e00106755e7084d1325e784eb27f91db9724c89``` --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Lola
GitHub
parent
b7976175e5
commit
f8e31e5fcb
10 changed files with 999 additions and 43 deletions
|
|
@ -15,6 +15,23 @@ export const AGENTLESS_GLOBAL_TAG_NAME_ORGANIZATION = 'organization';
|
|||
export const AGENTLESS_GLOBAL_TAG_NAME_DIVISION = 'division';
|
||||
export const AGENTLESS_GLOBAL_TAG_NAME_TEAM = 'team';
|
||||
|
||||
export const MAXIMUM_RETRIES = 3;
|
||||
|
||||
const HTTP_500_INTERNAL_SERVER_ERROR = 500;
|
||||
const HTTP_502_BAD_GATEWAY = 502;
|
||||
const HTTP_503_SERVICE_UNAVAILABLE = 503;
|
||||
const HTTP_504_GATEWAY_TIMEOUT = 504;
|
||||
|
||||
const ECONNREFUSED_CODE = 'ECONNREFUSED';
|
||||
|
||||
export const RETRYABLE_HTTP_STATUSES = [
|
||||
HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
HTTP_502_BAD_GATEWAY,
|
||||
HTTP_503_SERVICE_UNAVAILABLE,
|
||||
HTTP_504_GATEWAY_TIMEOUT,
|
||||
];
|
||||
|
||||
export const RETRYABLE_SERVER_CODES = [ECONNREFUSED_CODE];
|
||||
// Allowed output types for agentless integrations
|
||||
export const AGENTLESS_ALLOWED_OUTPUT_TYPES = [outputType.Elasticsearch];
|
||||
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ const _allowedExperimentalValues = {
|
|||
enableAutomaticAgentUpgrades: false,
|
||||
enableSyncIntegrationsOnRemote: false,
|
||||
enableSSLSecrets: false,
|
||||
enabledUpgradeAgentlessDeploymentsTask: false,
|
||||
};
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -62,6 +62,12 @@ export class AgentlessAgentDeleteError extends FleetError {
|
|||
super(`Error deleting agentless agent in Fleet, ${message}`);
|
||||
}
|
||||
}
|
||||
|
||||
export class AgentlessAgentUpgradeError extends FleetError {
|
||||
constructor(message: string) {
|
||||
super(`Error upgrading agentless agent in Fleet, ${message}`);
|
||||
}
|
||||
}
|
||||
export class AgentlessAgentConfigError extends FleetError {
|
||||
constructor(message: string) {
|
||||
super(`Error validating Agentless API configuration in Fleet, ${message}`);
|
||||
|
|
|
|||
|
|
@ -140,6 +140,7 @@ export const createAppContextStartContractMock = (
|
|||
: {}),
|
||||
unenrollInactiveAgentsTask: {} as any,
|
||||
deleteUnenrolledAgentsTask: {} as any,
|
||||
updateAgentlessDeploymentsTask: {} as any,
|
||||
syncIntegrationsTask: {} as any,
|
||||
automaticAgentUpgradeTask: {} as any,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -147,6 +147,7 @@ import { registerUpgradeManagedPackagePoliciesTask } from './services/setup/mana
|
|||
import { registerDeployAgentPoliciesTask } from './services/agent_policies/deploy_agent_policies_task';
|
||||
import { DeleteUnenrolledAgentsTask } from './tasks/delete_unenrolled_agents_task';
|
||||
import { registerBumpAgentPoliciesTask } from './services/agent_policies/bump_agent_policies_task';
|
||||
import { UpgradeAgentlessDeploymentsTask } from './tasks/upgrade_agentless_deployment';
|
||||
import { SyncIntegrationsTask } from './tasks/sync_integrations_task';
|
||||
import { AutomaticAgentUpgradeTask } from './tasks/automatic_agent_upgrade_task';
|
||||
|
||||
|
|
@ -200,6 +201,7 @@ export interface FleetAppContext {
|
|||
uninstallTokenService: UninstallTokenServiceInterface;
|
||||
unenrollInactiveAgentsTask: UnenrollInactiveAgentsTask;
|
||||
deleteUnenrolledAgentsTask: DeleteUnenrolledAgentsTask;
|
||||
updateAgentlessDeploymentsTask: UpgradeAgentlessDeploymentsTask;
|
||||
automaticAgentUpgradeTask: AutomaticAgentUpgradeTask;
|
||||
taskManagerStart?: TaskManagerStartContract;
|
||||
fetchUsage?: (abortController: AbortController) => Promise<FleetUsage | undefined>;
|
||||
|
|
@ -305,6 +307,7 @@ export class FleetPlugin
|
|||
private fleetMetricsTask?: FleetMetricsTask;
|
||||
private unenrollInactiveAgentsTask?: UnenrollInactiveAgentsTask;
|
||||
private deleteUnenrolledAgentsTask?: DeleteUnenrolledAgentsTask;
|
||||
private updateAgentlessDeploymentsTask?: UpgradeAgentlessDeploymentsTask;
|
||||
private syncIntegrationsTask?: SyncIntegrationsTask;
|
||||
private automaticAgentUpgradeTask?: AutomaticAgentUpgradeTask;
|
||||
|
||||
|
|
@ -653,6 +656,11 @@ export class FleetPlugin
|
|||
taskManager: deps.taskManager,
|
||||
logFactory: this.initializerContext.logger,
|
||||
});
|
||||
this.updateAgentlessDeploymentsTask = new UpgradeAgentlessDeploymentsTask({
|
||||
core,
|
||||
taskManager: deps.taskManager,
|
||||
logFactory: this.initializerContext.logger,
|
||||
});
|
||||
this.syncIntegrationsTask = new SyncIntegrationsTask({
|
||||
core,
|
||||
taskManager: deps.taskManager,
|
||||
|
|
@ -710,6 +718,7 @@ export class FleetPlugin
|
|||
uninstallTokenService,
|
||||
unenrollInactiveAgentsTask: this.unenrollInactiveAgentsTask!,
|
||||
deleteUnenrolledAgentsTask: this.deleteUnenrolledAgentsTask!,
|
||||
updateAgentlessDeploymentsTask: this.updateAgentlessDeploymentsTask!,
|
||||
automaticAgentUpgradeTask: this.automaticAgentUpgradeTask!,
|
||||
taskManagerStart: plugins.taskManager,
|
||||
fetchUsage: this.fetchUsage,
|
||||
|
|
@ -722,7 +731,11 @@ export class FleetPlugin
|
|||
this.checkDeletedFilesTask?.start({ taskManager: plugins.taskManager }).catch(() => {});
|
||||
this.unenrollInactiveAgentsTask?.start({ taskManager: plugins.taskManager }).catch(() => {});
|
||||
this.deleteUnenrolledAgentsTask?.start({ taskManager: plugins.taskManager }).catch(() => {});
|
||||
this.updateAgentlessDeploymentsTask
|
||||
?.start({ taskManager: plugins.taskManager })
|
||||
.catch(() => {});
|
||||
this.automaticAgentUpgradeTask?.start({ taskManager: plugins.taskManager }).catch(() => {});
|
||||
|
||||
startFleetUsageLogger(plugins.taskManager).catch(() => {});
|
||||
this.fleetMetricsTask
|
||||
?.start(plugins.taskManager, core.elasticsearch.client.asInternalUser)
|
||||
|
|
|
|||
|
|
@ -512,6 +512,47 @@ describe('Agentless Agent service', () => {
|
|||
);
|
||||
});
|
||||
|
||||
it('should upgraded agentless agent for ESS', async () => {
|
||||
const returnValue = {
|
||||
id: 'mocked',
|
||||
regional_id: 'mocked',
|
||||
};
|
||||
(axios as jest.MockedFunction<typeof axios>).mockResolvedValueOnce(returnValue);
|
||||
jest.spyOn(appContextService, 'getConfig').mockReturnValue({
|
||||
agentless: {
|
||||
enabled: true,
|
||||
api: {
|
||||
url: 'http://api.agentless.com',
|
||||
tls: {
|
||||
certificate: '/path/to/cert',
|
||||
key: '/path/to/key',
|
||||
ca: '/path/to/ca',
|
||||
},
|
||||
},
|
||||
},
|
||||
} as any);
|
||||
jest.spyOn(appContextService, 'getCloud').mockReturnValue({ isCloudEnabled: true } as any);
|
||||
|
||||
await agentlessAgentService.upgradeAgentlessDeployment(
|
||||
'mocked-agentless-agent-policy-id',
|
||||
'8.17.0'
|
||||
);
|
||||
|
||||
expect(axios).toHaveBeenCalledTimes(1);
|
||||
|
||||
expect(axios).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
headers: expect.anything(),
|
||||
httpsAgent: expect.anything(),
|
||||
method: 'PUT',
|
||||
data: {
|
||||
stack_version: '8.17.0',
|
||||
},
|
||||
url: 'http://api.agentless.com/api/v1/ess/deployments/mocked-agentless-agent-policy-id',
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it('should delete agentless agent for serverless', async () => {
|
||||
const returnValue = {
|
||||
id: 'mocked',
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ import {
|
|||
AgentlessAgentConfigError,
|
||||
AgentlessAgentCreateError,
|
||||
AgentlessAgentDeleteError,
|
||||
AgentlessAgentUpgradeError,
|
||||
} from '../../errors';
|
||||
import {
|
||||
AGENTLESS_GLOBAL_TAG_NAME_ORGANIZATION,
|
||||
|
|
@ -36,6 +37,11 @@ import { listEnrollmentApiKeys } from '../api_keys';
|
|||
import { listFleetServerHosts } from '../fleet_server_host';
|
||||
import type { AgentlessConfig } from '../utils/agentless';
|
||||
import { prependAgentlessApiBasePathToEndpoint, isAgentlessEnabled } from '../utils/agentless';
|
||||
import {
|
||||
MAXIMUM_RETRIES,
|
||||
RETRYABLE_HTTP_STATUSES,
|
||||
RETRYABLE_SERVER_CODES,
|
||||
} from '../../../common/constants/agentless';
|
||||
|
||||
interface AgentlessAgentErrorHandlingMessages {
|
||||
[key: string]: {
|
||||
|
|
@ -205,6 +211,72 @@ class AgentlessAgentService {
|
|||
return response;
|
||||
}
|
||||
|
||||
public async upgradeAgentlessDeployment(policyId: string, version: string) {
|
||||
const logger = appContextService.getLogger();
|
||||
const traceId = apm.currentTransaction?.traceparent;
|
||||
const agentlessConfig = appContextService.getConfig()?.agentless;
|
||||
const tlsConfig = this.createTlsConfig(agentlessConfig);
|
||||
const urlEndpoint = prependAgentlessApiBasePathToEndpoint(
|
||||
agentlessConfig,
|
||||
`/deployments/${policyId}`
|
||||
).split('/api')[1];
|
||||
logger.info(
|
||||
`[Agentless API] Call Agentless API endpoint ${urlEndpoint} to upgrade agentless deployment`
|
||||
);
|
||||
const requestConfig = {
|
||||
url: prependAgentlessApiBasePathToEndpoint(agentlessConfig, `/deployments/${policyId}`),
|
||||
method: 'PUT',
|
||||
data: {
|
||||
stack_version: version,
|
||||
},
|
||||
...this.getHeaders(tlsConfig, traceId),
|
||||
};
|
||||
|
||||
const errorMetadata: LogMeta = {
|
||||
trace: {
|
||||
id: traceId,
|
||||
},
|
||||
};
|
||||
|
||||
const requestConfigDebugStatus = this.createRequestConfigDebug(requestConfig);
|
||||
|
||||
logger.info(
|
||||
`[Agentless API] Start upgrading agentless deployment for agent policy ${requestConfigDebugStatus}`
|
||||
);
|
||||
|
||||
if (!isAgentlessEnabled) {
|
||||
logger.error(
|
||||
'[Agentless API] Agentless API is not supported. Upgrading agentless agent is not supported in non-cloud'
|
||||
);
|
||||
}
|
||||
|
||||
if (!agentlessConfig) {
|
||||
logger.error('[Agentless API] kibana.yml is currently missing Agentless API configuration');
|
||||
}
|
||||
|
||||
logger.info(`[Agentless API] Upgrading agentless agent with TLS config with certificate`);
|
||||
|
||||
logger.info(
|
||||
`[Agentless API] Upgrade agentless deployment with request config ${requestConfigDebugStatus}`
|
||||
);
|
||||
|
||||
const response = await axios(requestConfig).catch(async (error: AxiosError) => {
|
||||
await this.handleErrorsWithRetries(
|
||||
error,
|
||||
requestConfig,
|
||||
'upgrade',
|
||||
logger,
|
||||
MAXIMUM_RETRIES,
|
||||
policyId,
|
||||
requestConfigDebugStatus,
|
||||
errorMetadata,
|
||||
traceId
|
||||
);
|
||||
});
|
||||
|
||||
return response;
|
||||
}
|
||||
|
||||
private getHeaders(tlsConfig: SslConfig, traceId: string | undefined) {
|
||||
return {
|
||||
headers: {
|
||||
|
|
@ -301,7 +373,7 @@ class AgentlessAgentService {
|
|||
}
|
||||
|
||||
private catchAgentlessApiError(
|
||||
action: 'create' | 'delete',
|
||||
action: 'create' | 'delete' | 'upgrade',
|
||||
error: Error | AxiosError,
|
||||
logger: Logger,
|
||||
agentlessPolicyId: string,
|
||||
|
|
@ -324,12 +396,19 @@ class AgentlessAgentService {
|
|||
`${axiosError.code} ${this.convertCauseErrorsToString(axiosError)}`;
|
||||
|
||||
if (!axios.isAxiosError(error)) {
|
||||
let errorLogMessage;
|
||||
|
||||
if (action === 'create') {
|
||||
errorLogMessage = `[Agentless API] Creating agentless failed with an error that is not an AxiosError for agentless policy`;
|
||||
}
|
||||
if (action === 'delete') {
|
||||
errorLogMessage = `[Agentless API] Deleting agentless deployment failed with an error that is not an Axios error for agentless policy`;
|
||||
}
|
||||
if (action === 'upgrade') {
|
||||
errorLogMessage = `[Agentless API] Upgrading agentless deployment failed with an error that is not an Axios error for agentless policy`;
|
||||
}
|
||||
logger.error(
|
||||
`${
|
||||
action === 'create'
|
||||
? `[Agentless API] Creating agentless failed with an error that is not an AxiosError for agentless policy`
|
||||
: `[Agentless API] Deleting agentless deployment failed with an error that is not an Axios error for agentless policy`
|
||||
} ${error} ${requestConfigDebugStatus}`,
|
||||
`${errorLogMessage} ${error} ${requestConfigDebugStatus}`,
|
||||
errorMetadataWithRequestConfig
|
||||
);
|
||||
|
||||
|
|
@ -378,9 +457,9 @@ class AgentlessAgentService {
|
|||
} else {
|
||||
// Something happened in setting up the request that triggered an Error
|
||||
logger.error(
|
||||
`[Agentless API] ${
|
||||
action === 'create' ? 'Creating' : 'Deleting'
|
||||
} the agentless agent failed ${errorLogCodeCause(error)} ${requestConfigDebugStatus}`,
|
||||
`[Agentless API] ${action + 'ing'} the agentless agent failed ${errorLogCodeCause(
|
||||
error
|
||||
)} ${requestConfigDebugStatus}`,
|
||||
errorMetadataWithRequestConfig
|
||||
);
|
||||
|
||||
|
|
@ -393,7 +472,7 @@ class AgentlessAgentService {
|
|||
}
|
||||
|
||||
private handleResponseError(
|
||||
action: 'create' | 'delete',
|
||||
action: 'create' | 'delete' | 'upgrade',
|
||||
response: AxiosResponse,
|
||||
logger: Logger,
|
||||
errorMetadataWithRequestConfig: LogMeta,
|
||||
|
|
@ -429,9 +508,15 @@ class AgentlessAgentService {
|
|||
};
|
||||
|
||||
private getAgentlessAgentError(action: string, userMessage: string, traceId: string | undefined) {
|
||||
return action === 'create'
|
||||
? new AgentlessAgentCreateError(this.withRequestIdMessage(userMessage, traceId))
|
||||
: new AgentlessAgentDeleteError(this.withRequestIdMessage(userMessage, traceId));
|
||||
if (action === 'create') {
|
||||
return new AgentlessAgentCreateError(this.withRequestIdMessage(userMessage, traceId));
|
||||
}
|
||||
if (action === 'delete') {
|
||||
return new AgentlessAgentDeleteError(this.withRequestIdMessage(userMessage, traceId));
|
||||
}
|
||||
if (action === 'upgrade') {
|
||||
return new AgentlessAgentUpgradeError(this.withRequestIdMessage(userMessage, traceId));
|
||||
}
|
||||
}
|
||||
|
||||
private getErrorHandlingMessages(agentlessPolicyId: string): AgentlessAgentErrorHandlingMessages {
|
||||
|
|
@ -439,93 +524,214 @@ class AgentlessAgentService {
|
|||
400: {
|
||||
create: {
|
||||
log: '[Agentless API] Creating the agentless agent failed with a status 400, bad request for agentless policy.',
|
||||
message: `the Agentless API could not create the agentless agent. Please delete the agentless policy ${agentlessPolicyId} and try again or contact your administrator.`,
|
||||
message: `The Agentless API could not create the agentless agent. Please delete the agentless policy ${agentlessPolicyId} and try again or contact your administrator.`,
|
||||
},
|
||||
delete: {
|
||||
log: '[Agentless API] Deleting the agentless deployment failed with a status 400, bad request for agentless policy',
|
||||
message: `the Agentless API could not create the agentless agent. Please delete the agentless policy ${agentlessPolicyId} and try again or contact your administrator.`,
|
||||
log: '[Agentless API] Deleting the agentless deployment failed with a status 400, bad request for agentless policy.',
|
||||
message: `The Agentless API could not delete the agentless deployment. Please delete the agentless policy ${agentlessPolicyId} and try again or contact your administrator.`,
|
||||
},
|
||||
upgrade: {
|
||||
log: '[Agentless API] Upgrading the agentless agent failed with a status 400, bad request for agentless policy.',
|
||||
message: `The Agentless API could not upgrade the agentless agent. Please delete the agentless policy ${agentlessPolicyId} and try again or contact your administrator.`,
|
||||
},
|
||||
},
|
||||
401: {
|
||||
create: {
|
||||
log: '[Agentless API] Creating the agentless agent failed with a status 401 unauthorized for agentless policy.',
|
||||
message: `the Agentless API could not create the agentless agent because an unauthorized request was sent. Please delete the agentless policy ${agentlessPolicyId} and try again or contact your administrator.`,
|
||||
message: `The Agentless API could not create the agentless agent because an unauthorized request was sent. Please delete the agentless policy ${agentlessPolicyId} and try again or contact your administrator.`,
|
||||
},
|
||||
delete: {
|
||||
log: '[Agentless API] Deleting the agentless deployment failed with a status 401 unauthorized for agentless policy. Check the Kibana Agentless API tls configuration',
|
||||
message: `the Agentless API could not delete the agentless deployment because an unauthorized request was sent. Please delete the agentless policy ${agentlessPolicyId} and try again or contact your administrator.`,
|
||||
log: '[Agentless API] Deleting the agentless deployment failed with a status 401 unauthorized for agentless policy.',
|
||||
message: `The Agentless API could not delete the agentless deployment because an unauthorized request was sent. Please delete the agentless policy ${agentlessPolicyId} and try again or contact your administrator.`,
|
||||
},
|
||||
upgrade: {
|
||||
log: '[Agentless API] Upgrading the agentless agent failed with a status 401 unauthorized for agentless policy.',
|
||||
message: `The Agentless API could not upgrade the agentless agent because an unauthorized request was sent. Please delete the agentless policy ${agentlessPolicyId} and try again or contact your administrator.`,
|
||||
},
|
||||
},
|
||||
403: {
|
||||
create: {
|
||||
log: '[Agentless API] Creating the agentless agent failed with a status 403 forbidden for agentless policy. Check the Kibana Agentless API configuration and endpoints.',
|
||||
message: `the Agentless API could not create the agentless agent because a forbidden request was sent. Please delete the agentless policy ${agentlessPolicyId} and try again or contact your administrator.`,
|
||||
log: '[Agentless API] Creating the agentless agent failed with a status 403 forbidden for agentless policy.',
|
||||
message: `The Agentless API could not create the agentless agent because a forbidden request was sent. Please delete the agentless policy ${agentlessPolicyId} and try again or contact your administrator.`,
|
||||
},
|
||||
delete: {
|
||||
log: '[Agentless API] Deleting the agentless deployment failed with a status 403 forbidden for agentless policy. Check the Kibana Agentless API configuration and endpoints.',
|
||||
message: `the Agentless API could not delete the agentless deployment because a forbidden request was sent. Please delete the agentless policy ${agentlessPolicyId} and try again or contact your administrator.`,
|
||||
log: '[Agentless API] Deleting the agentless deployment failed with a status 403 forbidden for agentless policy.',
|
||||
message: `The Agentless API could not delete the agentless deployment because a forbidden request was sent. Please delete the agentless policy ${agentlessPolicyId} and try again or contact your administrator.`,
|
||||
},
|
||||
upgrade: {
|
||||
log: '[Agentless API] Upgrading the agentless agent failed with a status 403 forbidden for agentless policy.',
|
||||
message: `The Agentless API could not upgrade the agentless agent because a forbidden request was sent. Please delete the agentless policy ${agentlessPolicyId} and try again or contact your administrator.`,
|
||||
},
|
||||
},
|
||||
404: {
|
||||
// this is likely to happen when creating agentless agents, but covering it in case
|
||||
create: {
|
||||
log: '[Agentless API] Creating the agentless agent failed with a status 404 not found.',
|
||||
message: `the Agentless API could not create the agentless agent because it returned a 404 error not found. Please delete the agentless policy ${agentlessPolicyId} and try again or contact your administrator.`,
|
||||
message: `The Agentless API could not create the agentless agent because it returned a 404 error. Please delete the agentless policy ${agentlessPolicyId} and try again or contact your administrator.`,
|
||||
},
|
||||
delete: {
|
||||
log: '[Agentless API] Deleting the agentless deployment failed with a status 404 not found',
|
||||
message: `the Agentless API could not delete the agentless deployment ${agentlessPolicyId} because it could not be found.`,
|
||||
log: '[Agentless API] Deleting the agentless deployment failed with a status 404 not found.',
|
||||
message: `The Agentless API could not delete the agentless deployment because it could not be found. Please delete the agentless policy ${agentlessPolicyId} and try again or contact your administrator.`,
|
||||
},
|
||||
upgrade: {
|
||||
log: '[Agentless API] Upgrading the agentless agent failed with a status 404 not found.',
|
||||
message: `The Agentless API could not upgrade the agentless agent because it returned a 404 error. Please delete the agentless policy ${agentlessPolicyId} and try again or contact your administrator.`,
|
||||
},
|
||||
},
|
||||
408: {
|
||||
create: {
|
||||
log: '[Agentless API] Creating the agentless agent failed with a status 408, the request timed out',
|
||||
message: `the Agentless API request timed out waiting for the agentless agent status to respond, please wait a few minutes for the agent to enroll with fleet. If agent fails to enroll with Fleet please delete the agentless policy ${agentlessPolicyId} and try again or contact your administrator.`,
|
||||
log: '[Agentless API] Creating the agentless agent failed with a status 408, the request timed out.',
|
||||
message: `The Agentless API request timed out. Please wait a few minutes for the agent to enroll with Fleet. If the agent fails to enroll, delete the agentless policy ${agentlessPolicyId} and try again or contact your administrator.`,
|
||||
},
|
||||
delete: {
|
||||
log: '[Agentless API] Deleting the agentless deployment failed with a status 408, the request timed out',
|
||||
message: `the Agentless API could not delete the agentless deployment because the request timed out, please wait a few minutes for the agentless agent deployment to be removed. If it continues to persist please delete the agentless policy ${agentlessPolicyId} and try again or contact your administrator.`,
|
||||
log: '[Agentless API] Deleting the agentless deployment failed with a status 408, the request timed out.',
|
||||
message: `The Agentless API request timed out. Please wait a few minutes for the deployment to be removed. If it persists, delete the agentless policy ${agentlessPolicyId} and try again or contact your administrator.`,
|
||||
},
|
||||
upgrade: {
|
||||
log: '[Agentless API] Upgrading the agentless agent failed with a status 408, the request timed out.',
|
||||
message: `The Agentless API request timed out during the upgrade process. Please try again later or contact your administrator.`,
|
||||
},
|
||||
},
|
||||
429: {
|
||||
create: {
|
||||
log: '[Agentless API] Creating the agentless agent failed with a status 429 for agentless policy, agentless agent limit has been reached for this deployment or project.',
|
||||
log: '[Agentless API] Creating the agentless agent failed with a status 429, agentless agent limit reached.',
|
||||
message:
|
||||
'you have reached the limit for agentless provisioning. Please remove some or switch to agent-based integration.',
|
||||
'You have reached the limit for agentless provisioning. Please remove some or switch to agent-based integration.',
|
||||
},
|
||||
upgrade: {
|
||||
log: '[Agentless API] Upgrading the agentless agent failed with a status 429, agentless agent limit reached.',
|
||||
message:
|
||||
'You have reached the limit for agentless provisioning. Please remove some or switch to agent-based integration.',
|
||||
},
|
||||
},
|
||||
500: {
|
||||
create: {
|
||||
log: '[Agentless API] Creating the agentless agent failed with a status 500 internal service error.',
|
||||
message: `the Agentless API could not create the agentless agent because it returned a 500 internal error. Please delete the agentless policy ${agentlessPolicyId} and try again or contact your administrator.`,
|
||||
message: `The Agentless API could not create the agentless agent because it returned a 500 error. Please delete the agentless policy ${agentlessPolicyId} and try again or contact your administrator.`,
|
||||
},
|
||||
delete: {
|
||||
log: '[Agentless API] Deleting the agentless deployment failed with a status 500 internal service error.',
|
||||
message: `the Agentless API could not delete the agentless deployment because it returned a 500 internal error. Please delete the agentless policy ${agentlessPolicyId} and try again or contact your administrator.`,
|
||||
message: `The Agentless API could not delete the agentless deployment because it returned a 500 error. Please delete the agentless policy ${agentlessPolicyId} and try again or contact your administrator.`,
|
||||
},
|
||||
upgrade: {
|
||||
log: '[Agentless API] Upgrading the agentless agent failed with a status 500 internal service error.',
|
||||
message: `The Agentless API could not upgrade the agentless agent because it returned a 500 error. Please delete the agentless policy ${agentlessPolicyId} and try again or contact your administrator.`,
|
||||
},
|
||||
},
|
||||
unhandled_response: {
|
||||
create: {
|
||||
log: '[Agentless API] Creating agentless agent failed because the Agentless API responded with an unhandled status code that falls out of the range of 2xx:',
|
||||
message: `the Agentless API could not create the agentless agent due to an unexpected error. Please delete the agentless policy ${agentlessPolicyId} and try again or contact your administrator.`,
|
||||
log: '[Agentless API] Creating the agentless agent failed with an unhandled response.',
|
||||
message: `The Agentless API could not create the agentless agent due to an unexpected error. Please delete the agentless policy ${agentlessPolicyId} and try again or contact your administrator.`,
|
||||
},
|
||||
delete: {
|
||||
log: '[Agentless API] Deleting agentless deployment failed because the Agentless API responded with an unhandled status code that falls out of the range of 2xx:',
|
||||
message: `the Agentless API could not delete the agentless deployment. Please delete the agentless policy ${agentlessPolicyId} and try again or contact your administrator.`,
|
||||
log: '[Agentless API] Deleting the agentless deployment failed with an unhandled response.',
|
||||
message: `The Agentless API could not delete the agentless deployment due to an unexpected error. Please delete the agentless policy ${agentlessPolicyId} and try again or contact your administrator.`,
|
||||
},
|
||||
upgrade: {
|
||||
log: '[Agentless API] Upgrading the agentless agent failed with an unhandled response.',
|
||||
message: `The Agentless API could not upgrade the agentless agent due to an unexpected error. Please delete the agentless policy ${agentlessPolicyId} and try again or contact your administrator.`,
|
||||
},
|
||||
},
|
||||
request_error: {
|
||||
create: {
|
||||
log: '[Agentless API] Creating agentless agent failed with a request error:',
|
||||
message: `the Agentless API could not create the agentless agent due to a request error. Please delete the agentless policy ${agentlessPolicyId} and try again or contact your administrator.`,
|
||||
log: '[Agentless API] Creating the agentless agent failed with a request error.',
|
||||
message: `The Agentless API could not create the agentless agent due to a request error. Please delete the agentless policy ${agentlessPolicyId} and try again or contact your administrator.`,
|
||||
},
|
||||
delete: {
|
||||
log: '[Agentless API] Deleting agentless deployment failed with a request error:',
|
||||
message: `the Agentless API could not delete the agentless deployment due to a request error. Please delete the agentless policy ${agentlessPolicyId} and try again or contact your administrator.`,
|
||||
log: '[Agentless API] Deleting the agentless deployment failed with a request error.',
|
||||
message: `The Agentless API could not delete the agentless deployment due to a request error. Please delete the agentless policy ${agentlessPolicyId} and try again or contact your administrator.`,
|
||||
},
|
||||
upgrade: {
|
||||
log: '[Agentless API] Upgrading the agentless agent failed with a request error.',
|
||||
message: `The Agentless API could not upgrade the agentless agent due to a request error. Please delete the agentless policy ${agentlessPolicyId} and try again or contact your administrator.`,
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private handleErrorsWithRetries = async (
|
||||
error: AxiosError,
|
||||
requestConfig: AxiosRequestConfig,
|
||||
action: 'create' | 'delete' | 'upgrade',
|
||||
logger: Logger,
|
||||
retries: number,
|
||||
id: string,
|
||||
requestConfigDebugStatus: string,
|
||||
errorMetadata: any,
|
||||
traceId?: string
|
||||
) => {
|
||||
const hasRetryableStatusError = this.hasRetryableStatusError(error, RETRYABLE_HTTP_STATUSES);
|
||||
const hasRetryableCodeError = this.hasRetryableCodeError(error, RETRYABLE_SERVER_CODES);
|
||||
|
||||
if (hasRetryableStatusError || hasRetryableCodeError) {
|
||||
await this.retry(
|
||||
async () => await axios(requestConfig),
|
||||
action,
|
||||
requestConfigDebugStatus,
|
||||
logger,
|
||||
retries,
|
||||
() =>
|
||||
this.catchAgentlessApiError(
|
||||
action,
|
||||
error,
|
||||
logger,
|
||||
id,
|
||||
requestConfig,
|
||||
requestConfigDebugStatus,
|
||||
errorMetadata,
|
||||
traceId
|
||||
)
|
||||
);
|
||||
} else {
|
||||
this.catchAgentlessApiError(
|
||||
action,
|
||||
error,
|
||||
logger,
|
||||
id,
|
||||
requestConfig,
|
||||
requestConfigDebugStatus,
|
||||
errorMetadata,
|
||||
traceId
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
private retry = async <T>(
|
||||
fn: () => Promise<unknown>,
|
||||
action: 'create' | 'delete' | 'upgrade',
|
||||
requestConfigDebugStatus: string,
|
||||
logger: Logger,
|
||||
retries = MAXIMUM_RETRIES,
|
||||
throwAgentlessError: () => void
|
||||
) => {
|
||||
for (let i = 0; i < retries; i++) {
|
||||
try {
|
||||
await fn();
|
||||
} catch (e) {
|
||||
logger.info(
|
||||
`[Agentless API] Attempt ${i + 1} failed to ${action} agentless deployment, retrying...`
|
||||
);
|
||||
if (i === retries - 1) {
|
||||
logger.error(
|
||||
`[Agentless API] Reached maximum ${retries} attempts. Failed to ${action} agentless deployment with [REQUEST]: ${requestConfigDebugStatus}`
|
||||
);
|
||||
throwAgentlessError();
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
private hasRetryableStatusError = (
|
||||
error: AxiosError,
|
||||
retryableStatusErrors: number[]
|
||||
): boolean => {
|
||||
const status = error?.response?.status;
|
||||
return !!status && retryableStatusErrors.some((errorStatus) => errorStatus === status);
|
||||
};
|
||||
|
||||
private hasRetryableCodeError = (error: AxiosError, retryableCodeErrors: string[]): boolean => {
|
||||
const code = error?.code;
|
||||
return !!code && retryableCodeErrors.includes(code);
|
||||
};
|
||||
}
|
||||
|
||||
export const agentlessAgentService = new AgentlessAgentService();
|
||||
|
|
|
|||
|
|
@ -0,0 +1,376 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { TaskStatus } from '@kbn/task-manager-plugin/server';
|
||||
|
||||
import { coreMock, loggingSystemMock } from '@kbn/core/server/mocks';
|
||||
import { taskManagerMock } from '@kbn/task-manager-plugin/server/mocks';
|
||||
|
||||
import { agentPolicyService, appContextService } from '../services';
|
||||
import { createAppContextStartContractMock } from '../mocks';
|
||||
import { createAgentPolicyMock } from '../../common/mocks';
|
||||
|
||||
import type { AgentPolicy } from '../types';
|
||||
|
||||
import { agentlessAgentService } from '../services/agents/agentless_agent';
|
||||
|
||||
import { getAgentsByKuery, getLatestAvailableAgentVersion } from '../services/agents';
|
||||
|
||||
import {
|
||||
UPGRADE_AGENT_DEPLOYMENTS_TASK_VERSION,
|
||||
UPGRADE_AGENTLESS_DEPLOYMENTS_TASK_TYPE,
|
||||
UpgradeAgentlessDeploymentsTask,
|
||||
} from './upgrade_agentless_deployment';
|
||||
|
||||
const systemMock = {
|
||||
id: 'c6d16e42-c32d-4dce-8a88-113cfe276ad1',
|
||||
name: 'system-1',
|
||||
description: '',
|
||||
namespace: 'default',
|
||||
enabled: true,
|
||||
policy_id: '93c46720-c217-11ea-9906-b5b8a21b268e',
|
||||
policy_ids: ['93c46720-c217-11ea-9906-b5b8a21b268e'],
|
||||
revision: 1,
|
||||
package: {
|
||||
name: 'system',
|
||||
title: 'System',
|
||||
version: '0.9.0',
|
||||
},
|
||||
updated_at: '2020-06-25T16:03:38.159292',
|
||||
updated_by: 'kibana',
|
||||
created_at: '2020-06-25T16:03:38.159292',
|
||||
created_by: 'kibana',
|
||||
inputs: [
|
||||
{
|
||||
config: {},
|
||||
enabled: true,
|
||||
type: 'system',
|
||||
streams: [],
|
||||
},
|
||||
],
|
||||
};
|
||||
const MOCK_TASK_INSTANCE = {
|
||||
id: `${UPGRADE_AGENTLESS_DEPLOYMENTS_TASK_TYPE}:${UPGRADE_AGENT_DEPLOYMENTS_TASK_VERSION}`,
|
||||
runAt: new Date(),
|
||||
attempts: 1,
|
||||
ownerId: '',
|
||||
status: TaskStatus.Running,
|
||||
startedAt: new Date(),
|
||||
scheduledAt: new Date(),
|
||||
retryAt: new Date(),
|
||||
params: {},
|
||||
state: {},
|
||||
taskType: UPGRADE_AGENTLESS_DEPLOYMENTS_TASK_TYPE,
|
||||
};
|
||||
|
||||
const mockAgentPolicy: AgentPolicy = createAgentPolicyMock({
|
||||
agents: 1,
|
||||
id: '93c46720-c217-11ea-9906-b5b8a21b268e',
|
||||
package_policies: [systemMock],
|
||||
});
|
||||
|
||||
jest.mock('../services/agent_policy_update', () => ({
|
||||
agentPolicyUpdateEventHandler: jest.fn(),
|
||||
}));
|
||||
|
||||
jest.mock('../services/agents', () => ({
|
||||
getAgentsByKuery: jest.fn(),
|
||||
getLatestAvailableAgentVersion: jest.fn(),
|
||||
}));
|
||||
|
||||
describe('Upgrade Agentless Deployments', () => {
|
||||
const { createSetup: coreSetupMock } = coreMock;
|
||||
const { createSetup: tmSetupMock, createStart: tmStartMock } = taskManagerMock;
|
||||
let mockContract: ReturnType<typeof createAppContextStartContractMock>;
|
||||
let mockTask: UpgradeAgentlessDeploymentsTask;
|
||||
let mockCore: ReturnType<typeof coreSetupMock>;
|
||||
let mockTaskManagerSetup: ReturnType<typeof tmSetupMock>;
|
||||
|
||||
const getMockAgentPolicyFetchAllAgentPolicies = (items: AgentPolicy[]) =>
|
||||
jest.fn().mockResolvedValue(
|
||||
jest.fn(async function* () {
|
||||
yield items;
|
||||
})()
|
||||
);
|
||||
|
||||
beforeEach(() => {
|
||||
mockContract = createAppContextStartContractMock();
|
||||
appContextService.start(mockContract);
|
||||
mockCore = coreSetupMock();
|
||||
mockTaskManagerSetup = tmSetupMock();
|
||||
|
||||
mockTask = new UpgradeAgentlessDeploymentsTask({
|
||||
core: mockCore,
|
||||
taskManager: mockTaskManagerSetup,
|
||||
logFactory: loggingSystemMock.create(),
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
describe('Task lifecycle', () => {
|
||||
it('Should create task', async () => {
|
||||
expect(mockTask).toBeInstanceOf(UpgradeAgentlessDeploymentsTask);
|
||||
});
|
||||
|
||||
it('Should register task', async () => {
|
||||
expect(mockTaskManagerSetup.registerTaskDefinitions).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('Should schedule task', async () => {
|
||||
const mockTaskManagerStart = tmStartMock();
|
||||
await mockTask.start({ taskManager: mockTaskManagerStart });
|
||||
expect(mockTaskManagerStart.ensureScheduled).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('Task Logic', () => {
|
||||
const runTask = async (taskInstance = MOCK_TASK_INSTANCE) => {
|
||||
const mockTaskManagerStart = tmStartMock();
|
||||
await mockTask.start({ taskManager: mockTaskManagerStart });
|
||||
const createTaskRunner =
|
||||
mockTaskManagerSetup.registerTaskDefinitions.mock.calls[0][0][
|
||||
UPGRADE_AGENTLESS_DEPLOYMENTS_TASK_TYPE
|
||||
].createTaskRunner;
|
||||
const taskRunner = createTaskRunner({ taskInstance });
|
||||
return taskRunner.run();
|
||||
};
|
||||
|
||||
const mockAgentPolicyService = agentPolicyService as jest.Mocked<typeof agentPolicyService>;
|
||||
const agents = [
|
||||
{
|
||||
id: 'agent-1',
|
||||
policy_id: '93c46720-c217-11ea-9906-b5b8a21b268e',
|
||||
status: 'online',
|
||||
agent: {
|
||||
version: '8.16.0',
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 'agent-2',
|
||||
policy_id: 'agent-policy-2',
|
||||
status: 'inactive',
|
||||
agent: {
|
||||
version: '8.17.0',
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 'agent-3',
|
||||
policy_id: 'agent-policy-3',
|
||||
status: 'active',
|
||||
agent: {
|
||||
version: '8.17.0',
|
||||
},
|
||||
},
|
||||
];
|
||||
const mockedGetAgentsByKuery = getAgentsByKuery as jest.Mock;
|
||||
const mockedGetLatestAvailableAgentVersion = getLatestAvailableAgentVersion as jest.Mock;
|
||||
|
||||
beforeEach(() => {
|
||||
mockAgentPolicyService.fetchAllAgentPolicies = getMockAgentPolicyFetchAllAgentPolicies([
|
||||
mockAgentPolicy,
|
||||
]);
|
||||
jest
|
||||
.spyOn(appContextService, 'getExperimentalFeatures')
|
||||
.mockReturnValue({ enabledUpgradeAgentlessDeploymentsTask: true } as any);
|
||||
|
||||
mockedGetAgentsByKuery.mockResolvedValue({
|
||||
agents,
|
||||
});
|
||||
|
||||
mockedGetLatestAvailableAgentVersion.mockResolvedValue('8.17.0');
|
||||
|
||||
jest
|
||||
.spyOn(agentlessAgentService, 'upgradeAgentlessDeployment')
|
||||
.mockResolvedValueOnce(undefined);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
it('should upgrade agentless deployments', async () => {
|
||||
await runTask();
|
||||
|
||||
expect(mockAgentPolicyService.fetchAllAgentPolicies).toHaveBeenCalled();
|
||||
expect(agentlessAgentService.upgradeAgentlessDeployment).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should not upgrade agentless deployments when the latest version is up to date', async () => {
|
||||
mockedGetAgentsByKuery.mockResolvedValue({
|
||||
agents: [
|
||||
{
|
||||
id: 'agent-1',
|
||||
policy_id: '93c46720-c217-11ea-9906-b5b8a21b268e',
|
||||
status: 'online',
|
||||
agent: {
|
||||
version: '8.17.0',
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
await runTask();
|
||||
|
||||
expect(mockAgentPolicyService.fetchAllAgentPolicies).toHaveBeenCalled();
|
||||
expect(agentlessAgentService.upgradeAgentlessDeployment).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should not upgrade agentless deployments when agent status is updating', async () => {
|
||||
mockedGetLatestAvailableAgentVersion.mockResolvedValue('8.17.1');
|
||||
mockedGetAgentsByKuery.mockResolvedValue({
|
||||
agents: [
|
||||
{
|
||||
id: 'agent-1',
|
||||
policy_id: '93c46720-c217-11ea-9906-b5b8a21b268e',
|
||||
status: 'updating',
|
||||
agent: {
|
||||
version: '8.17.0',
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
await runTask();
|
||||
|
||||
expect(mockAgentPolicyService.fetchAllAgentPolicies).toHaveBeenCalled();
|
||||
expect(agentlessAgentService.upgradeAgentlessDeployment).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should not upgrade agentless deployments when agent status is unhealthy', async () => {
|
||||
mockedGetLatestAvailableAgentVersion.mockResolvedValue('8.17.1');
|
||||
mockedGetAgentsByKuery.mockResolvedValue({
|
||||
agents: [
|
||||
{
|
||||
id: 'agent-1',
|
||||
policy_id: '93c46720-c217-11ea-9906-b5b8a21b268e',
|
||||
status: 'updating',
|
||||
agent: {
|
||||
version: '8.17.0',
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
await runTask();
|
||||
|
||||
expect(mockAgentPolicyService.fetchAllAgentPolicies).toHaveBeenCalled();
|
||||
expect(agentlessAgentService.upgradeAgentlessDeployment).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should upgrade agentless deployments when agent status is online', async () => {
|
||||
mockedGetLatestAvailableAgentVersion.mockResolvedValue('8.17.1');
|
||||
mockedGetAgentsByKuery.mockResolvedValue({
|
||||
agents: [
|
||||
{
|
||||
id: 'agent-1',
|
||||
policy_id: '93c46720-c217-11ea-9906-b5b8a21b268e',
|
||||
status: 'online',
|
||||
agent: {
|
||||
version: '8.17.0',
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
await runTask();
|
||||
|
||||
expect(mockAgentPolicyService.fetchAllAgentPolicies).toHaveBeenCalled();
|
||||
expect(agentlessAgentService.upgradeAgentlessDeployment).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should not upgrade agentless deployments when agent status is unenroll', async () => {
|
||||
mockedGetLatestAvailableAgentVersion.mockResolvedValue('8.17.1');
|
||||
mockedGetAgentsByKuery.mockResolvedValue({
|
||||
agents: [
|
||||
{
|
||||
id: 'agent-1',
|
||||
policy_id: '93c46720-c217-11ea-9906-b5b8a21b268e',
|
||||
status: 'unenroll',
|
||||
agent: {
|
||||
version: '8.17.0',
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
await runTask();
|
||||
|
||||
expect(mockAgentPolicyService.fetchAllAgentPolicies).toHaveBeenCalled();
|
||||
expect(agentlessAgentService.upgradeAgentlessDeployment).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should upgrade agentless deployments when agent for target bg task release', async () => {
|
||||
mockedGetLatestAvailableAgentVersion.mockResolvedValue('8.18.1');
|
||||
mockedGetAgentsByKuery.mockResolvedValue({
|
||||
agents: [
|
||||
{
|
||||
id: 'agent-1',
|
||||
policy_id: '93c46720-c217-11ea-9906-b5b8a21b268e',
|
||||
status: 'online',
|
||||
agent: {
|
||||
version: '8.18.0',
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
await runTask();
|
||||
|
||||
expect(mockAgentPolicyService.fetchAllAgentPolicies).toHaveBeenCalled();
|
||||
expect(agentlessAgentService.upgradeAgentlessDeployment).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should upgrade agentless deployments when agent version is up to date', async () => {
|
||||
mockedGetLatestAvailableAgentVersion.mockResolvedValue('8.17.1');
|
||||
mockedGetAgentsByKuery.mockResolvedValue({
|
||||
agents: [
|
||||
{
|
||||
id: 'agent-1',
|
||||
policy_id: '93c46720-c217-11ea-9906-b5b8a21b268e',
|
||||
status: 'online',
|
||||
agent: {
|
||||
version: '8.17.0',
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
await runTask();
|
||||
|
||||
expect(mockAgentPolicyService.fetchAllAgentPolicies).toHaveBeenCalled();
|
||||
expect(agentlessAgentService.upgradeAgentlessDeployment).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should not call upgrade agentless api to upgrade when 0 agents', async () => {
|
||||
mockedGetAgentsByKuery.mockResolvedValue({
|
||||
agents: [],
|
||||
});
|
||||
await runTask();
|
||||
|
||||
expect(mockAgentPolicyService.fetchAllAgentPolicies).toHaveBeenCalled();
|
||||
expect(agentlessAgentService.upgradeAgentlessDeployment).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should throw an error if task is aborted', async () => {
|
||||
mockTask.abortController = new AbortController();
|
||||
mockTask.abortController.signal.throwIfAborted = jest.fn(() => {
|
||||
throw new Error('Task aborted!');
|
||||
});
|
||||
|
||||
mockTask.abortController.abort();
|
||||
await runTask();
|
||||
|
||||
expect(mockTask.abortController.signal.throwIfAborted).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should not called upgrade agentless api to upgrade when agent policy is not found', async () => {
|
||||
jest
|
||||
.spyOn(appContextService, 'getExperimentalFeatures')
|
||||
.mockReturnValue({ enabledUpgradeAgentlessDeploymentsTask: false } as any);
|
||||
|
||||
await runTask();
|
||||
|
||||
expect(agentlessAgentService.upgradeAgentlessDeployment).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,294 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { SavedObjectsClient, type CoreSetup, type Logger } from '@kbn/core/server';
|
||||
import type { ElasticsearchClient, LoggerFactory } from '@kbn/core/server';
|
||||
|
||||
import type { TaskManagerStartContract } from '@kbn/task-manager-plugin/server';
|
||||
import {
|
||||
type ConcreteTaskInstance,
|
||||
type TaskManagerSetupContract,
|
||||
} from '@kbn/task-manager-plugin/server';
|
||||
import { getDeleteTaskRunResult } from '@kbn/task-manager-plugin/server/task';
|
||||
|
||||
import { isAgentVersionLessThanLatest } from '../../common/services';
|
||||
|
||||
import { agentPolicyService, appContextService } from '../services';
|
||||
|
||||
import type { Agent, AgentPolicy } from '../types';
|
||||
|
||||
import { AGENTS_PREFIX } from '../constants';
|
||||
import { getAgentsByKuery, getLatestAvailableAgentVersion } from '../services/agents';
|
||||
import { agentlessAgentService } from '../services/agents/agentless_agent';
|
||||
|
||||
export const UPGRADE_AGENTLESS_DEPLOYMENTS_TASK_TYPE = 'fleet:upgrade-agentless-deployments-task';
|
||||
export const UPGRADE_AGENT_DEPLOYMENTS_TASK_VERSION = '1.0.0';
|
||||
const TITLE = 'Fleet upgrade agentless deployments Task';
|
||||
const TIMEOUT = '2m';
|
||||
const INTERVAL = '1d';
|
||||
const LOGGER_SUBJECT = '[UpgradeAgentlessDeploymentsTask]';
|
||||
const BATCH_SIZE = 10;
|
||||
const AGENTLESS_DEPLOYMENTS_SIZE = 40;
|
||||
interface UpgradeAgentlessDeploymentsTaskSetupContract {
|
||||
core: CoreSetup;
|
||||
taskManager: TaskManagerSetupContract;
|
||||
logFactory: LoggerFactory;
|
||||
}
|
||||
|
||||
interface UpgradeAgentlessDeploymentsTaskStartContract {
|
||||
taskManager: TaskManagerStartContract;
|
||||
}
|
||||
|
||||
export class UpgradeAgentlessDeploymentsTask {
|
||||
private logger: Logger;
|
||||
private startedTaskRunner: boolean = false;
|
||||
public abortController = new AbortController();
|
||||
|
||||
constructor(setupContract: UpgradeAgentlessDeploymentsTaskSetupContract) {
|
||||
const { core, taskManager, logFactory } = setupContract;
|
||||
this.logger = logFactory.get(this.taskId);
|
||||
|
||||
taskManager.registerTaskDefinitions({
|
||||
[UPGRADE_AGENTLESS_DEPLOYMENTS_TASK_TYPE]: {
|
||||
title: TITLE,
|
||||
timeout: TIMEOUT,
|
||||
createTaskRunner: ({ taskInstance }: { taskInstance: ConcreteTaskInstance }) => {
|
||||
return {
|
||||
run: async () => {
|
||||
return this.runTask(taskInstance, core);
|
||||
},
|
||||
cancel: async () => {
|
||||
this.abortController.abort(`${TITLE} timed out`);
|
||||
},
|
||||
};
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
public start = async ({ taskManager }: UpgradeAgentlessDeploymentsTaskStartContract) => {
|
||||
if (!taskManager) {
|
||||
this.logger.error(`${LOGGER_SUBJECT} Missing required service during start`);
|
||||
}
|
||||
|
||||
this.startedTaskRunner = true;
|
||||
this.logger.info(`${LOGGER_SUBJECT} Started with interval of [${INTERVAL}]`);
|
||||
|
||||
try {
|
||||
await taskManager.ensureScheduled({
|
||||
id: this.taskId,
|
||||
taskType: UPGRADE_AGENTLESS_DEPLOYMENTS_TASK_TYPE,
|
||||
schedule: {
|
||||
interval: INTERVAL,
|
||||
},
|
||||
state: {},
|
||||
params: { version: UPGRADE_AGENT_DEPLOYMENTS_TASK_VERSION },
|
||||
});
|
||||
} catch (e) {
|
||||
this.logger.error(`Error scheduling task ${LOGGER_SUBJECT}, error: ${e.message}`, e);
|
||||
}
|
||||
};
|
||||
|
||||
private get taskId(): string {
|
||||
return `${UPGRADE_AGENTLESS_DEPLOYMENTS_TASK_TYPE}:${UPGRADE_AGENT_DEPLOYMENTS_TASK_VERSION}`;
|
||||
}
|
||||
|
||||
private endRun(msg: string = '') {
|
||||
this.logger.info(`${LOGGER_SUBJECT} runTask ended${msg ? ': ' + msg : ''}`);
|
||||
}
|
||||
|
||||
private processInBatches = async (
|
||||
{
|
||||
agentlessPolicies,
|
||||
agents,
|
||||
}: {
|
||||
agentlessPolicies: AgentPolicy[];
|
||||
agents: Agent[];
|
||||
},
|
||||
batchSize: number,
|
||||
processFunction: (agentPolicy: AgentPolicy, agentlessAgent: Agent) => void
|
||||
) => {
|
||||
if (!agents.length) {
|
||||
this.endRun('No agents found');
|
||||
return;
|
||||
}
|
||||
|
||||
for (let i = 0; i < agentlessPolicies.length; i += batchSize) {
|
||||
const currentAgentPolicyBatch = agentlessPolicies.slice(i, i + batchSize);
|
||||
|
||||
await Promise.allSettled(
|
||||
await currentAgentPolicyBatch.map(async (agentPolicy) => {
|
||||
const agentlessAgent = agents.find((agent) => agent.policy_id === agentPolicy.id);
|
||||
|
||||
if (!agentlessAgent) {
|
||||
this.endRun('No active online agentless agent found');
|
||||
return;
|
||||
}
|
||||
|
||||
this.logger.info(
|
||||
`${LOGGER_SUBJECT} processing agentless agent ${JSON.stringify(agentlessAgent.agent)}`
|
||||
);
|
||||
|
||||
if (this.abortController.signal.aborted) {
|
||||
this.logger.info(`${LOGGER_SUBJECT} Task runner canceled!`);
|
||||
this.abortController.signal.throwIfAborted();
|
||||
}
|
||||
return processFunction(agentPolicy, agentlessAgent);
|
||||
})
|
||||
);
|
||||
|
||||
if (this.abortController.signal.aborted) {
|
||||
this.logger.info(`${LOGGER_SUBJECT} Task runner canceled!`);
|
||||
this.abortController.signal.throwIfAborted();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
private processUpgradeAgentlessDeployments = async (
|
||||
esClient: ElasticsearchClient,
|
||||
soClient: SavedObjectsClient
|
||||
) => {
|
||||
const SAVED_OBJECT_TYPE = 'fleet-agent-policies';
|
||||
|
||||
const policiesKuery = `${SAVED_OBJECT_TYPE}.supports_agentless: true`;
|
||||
|
||||
try {
|
||||
const agentPolicyFetcher = await agentPolicyService.fetchAllAgentPolicies(soClient, {
|
||||
kuery: policiesKuery,
|
||||
perPage: BATCH_SIZE,
|
||||
spaceId: '*',
|
||||
});
|
||||
this.logger.info(
|
||||
`[${LOGGER_SUBJECT}] running task to upgrade agentless deployments with kuery: ${policiesKuery}`
|
||||
);
|
||||
|
||||
for await (const agentlessPolicies of agentPolicyFetcher) {
|
||||
this.logger.info(
|
||||
`[${LOGGER_SUBJECT}] Found "${agentlessPolicies.length}" agentless policies`
|
||||
);
|
||||
|
||||
if (!agentlessPolicies.length) {
|
||||
this.endRun('Found no agentless policies to upgrade');
|
||||
return;
|
||||
}
|
||||
|
||||
// Upgrade agentless deployments
|
||||
try {
|
||||
const kuery = `(${AGENTS_PREFIX}.policy_id:${agentlessPolicies
|
||||
.map((policy) => `"${policy.id}"`)
|
||||
.join(' or ')}) and ${AGENTS_PREFIX}.status:online`;
|
||||
|
||||
const res = await getAgentsByKuery(esClient, soClient, {
|
||||
kuery,
|
||||
showInactive: false,
|
||||
page: 1,
|
||||
perPage: AGENTLESS_DEPLOYMENTS_SIZE,
|
||||
});
|
||||
this.logger.info(`${LOGGER_SUBJECT} Found "${res.agents.length}" agentless agents`);
|
||||
await this.processInBatches(
|
||||
{
|
||||
agentlessPolicies,
|
||||
agents: res.agents,
|
||||
},
|
||||
BATCH_SIZE,
|
||||
this.upgradeAgentlessDeployments
|
||||
);
|
||||
} catch (e) {
|
||||
this.logger.error(`${LOGGER_SUBJECT} Failed to get agentless agents error: ${e}`);
|
||||
}
|
||||
|
||||
if (this.abortController.signal.aborted) {
|
||||
this.logger.info(`${LOGGER_SUBJECT} Task runner canceled!`);
|
||||
this.abortController.signal.throwIfAborted();
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
this.logger.error(`${LOGGER_SUBJECT} Failed to get agentless policies error: ${e}`);
|
||||
}
|
||||
this.logger.info(`${LOGGER_SUBJECT} [runTask()] finished`);
|
||||
};
|
||||
|
||||
private upgradeAgentlessDeployments = async (agentPolicy: AgentPolicy, agent: Agent) => {
|
||||
this.logger.info(`${agentPolicy.id} agentless policy id`);
|
||||
|
||||
let latestAgentVersion;
|
||||
const currentAgentVersion = agent.agent?.version;
|
||||
// Get latest available agent version
|
||||
try {
|
||||
this.logger.info(`${LOGGER_SUBJECT} getting latest available agent version in ess`);
|
||||
latestAgentVersion = await getLatestAvailableAgentVersion();
|
||||
this.logger.info(
|
||||
`${LOGGER_SUBJECT} latest version ${latestAgentVersion} and current agent version ${currentAgentVersion}`
|
||||
);
|
||||
} catch (e) {
|
||||
this.logger.error(`${LOGGER_SUBJECT} Failed to get latest version error: ${e}`);
|
||||
throw e;
|
||||
}
|
||||
|
||||
// Compare the current agent version with the latest agent version And upgrade if necessary
|
||||
if (
|
||||
agent.status === 'online' &&
|
||||
latestAgentVersion &&
|
||||
currentAgentVersion &&
|
||||
isAgentVersionLessThanLatest(currentAgentVersion, latestAgentVersion)
|
||||
) {
|
||||
this.logger.info(
|
||||
`${LOGGER_SUBJECT} Upgrade Available to ${latestAgentVersion} for agentless policy ${agentPolicy.id} current version ${currentAgentVersion}`
|
||||
);
|
||||
try {
|
||||
this.logger.info(
|
||||
`${LOGGER_SUBJECT} upgrading agentless policy ${agentPolicy.id} current agent version ${currentAgentVersion} to version ${latestAgentVersion}`
|
||||
);
|
||||
await agentlessAgentService.upgradeAgentlessDeployment(agentPolicy.id, latestAgentVersion);
|
||||
|
||||
this.logger.info(
|
||||
`${LOGGER_SUBJECT} Successfully upgraded agentless deployment to ${latestAgentVersion} for ${agentPolicy.id}`
|
||||
);
|
||||
} catch (e) {
|
||||
this.logger.error(
|
||||
`${LOGGER_SUBJECT} Failed to upgrade agentless deployment to ${latestAgentVersion} for ${agentPolicy.id} error: ${e}`
|
||||
);
|
||||
throw e;
|
||||
}
|
||||
} else {
|
||||
this.logger.info(
|
||||
`${LOGGER_SUBJECT} No upgrade available for agentless policy ${agentPolicy.id} current agent version ${currentAgentVersion} and latest version ${latestAgentVersion}`
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
public runTask = async (taskInstance: ConcreteTaskInstance, core: CoreSetup) => {
|
||||
const cloudSetup = appContextService.getCloud();
|
||||
if (!this.startedTaskRunner) {
|
||||
this.logger.info(`${LOGGER_SUBJECT} runTask Aborted. Task not started yet`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (taskInstance.id !== this.taskId) {
|
||||
this.logger.info(
|
||||
`${LOGGER_SUBJECT} Outdated task version: Received [${taskInstance.id}] from task instance. Current version is [${this.taskId}]`
|
||||
);
|
||||
return getDeleteTaskRunResult();
|
||||
}
|
||||
|
||||
if (!appContextService.getExperimentalFeatures().enabledUpgradeAgentlessDeploymentsTask) {
|
||||
this.endRun('Upgrade Agentless Deployments Task is disabled');
|
||||
return;
|
||||
}
|
||||
|
||||
if (cloudSetup?.isServerlessEnabled) {
|
||||
this.endRun('Upgrading Agentless deployments is only supported in cloud');
|
||||
return;
|
||||
}
|
||||
|
||||
this.logger.info(`[runTask()] started`);
|
||||
const [coreStart] = await core.getStartServices();
|
||||
const esClient = coreStart.elasticsearch.client.asInternalUser;
|
||||
const soClient = new SavedObjectsClient(coreStart.savedObjects.createInternalRepository());
|
||||
await this.processUpgradeAgentlessDeployments(esClient, soClient);
|
||||
};
|
||||
}
|
||||
|
|
@ -156,6 +156,7 @@ export default function ({ getService }: FtrProviderContext) {
|
|||
'fleet:unenroll-inactive-agents-task',
|
||||
'fleet:unenroll_action:retry',
|
||||
'fleet:update_agent_tags:retry',
|
||||
'fleet:upgrade-agentless-deployments-task',
|
||||
'fleet:upgrade_action:retry',
|
||||
'logs-data-telemetry',
|
||||
'obs-ai-assistant:knowledge-base-migration',
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue