[Security Solution][Endpoint] Cypress test to validate that Endpoint can stream alerts to ES/Kibana (#155455)

## Summary

- Adds cypress test that stands up a real endpoint and validates that it
can trigger alerts and send those to ES/Kbn and that they show up on the
Alerts list
This commit is contained in:
Paul Tavares 2023-04-25 09:16:09 -04:00 committed by GitHub
parent 9dec953894
commit 96fcd5a497
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
20 changed files with 1067 additions and 11 deletions

View file

@ -344,7 +344,7 @@ describe('When showing Endpoint Agent Status', () => {
});
it('should keep agent status up to date when autoRefresh is true', async () => {
renderProps.autoFresh = true;
renderProps.autoRefresh = true;
apiMocks.responseProvider.metadataDetails.mockReturnValueOnce(endpointDetails);
const { getByTestId } = render();

View file

@ -138,7 +138,7 @@ export interface EndpointAgentStatusByIdProps {
* If set to `true` (Default), then the endpoint status and isolation/action counts will
* be kept up to date by querying the API periodically
*/
autoFresh?: boolean;
autoRefresh?: boolean;
'data-test-subj'?: string;
}
@ -150,9 +150,9 @@ export interface EndpointAgentStatusByIdProps {
* instead in order to avoid duplicate API calls.
*/
export const EndpointAgentStatusById = memo<EndpointAgentStatusByIdProps>(
({ endpointAgentId, autoFresh, 'data-test-subj': dataTestSubj }) => {
({ endpointAgentId, autoRefresh, 'data-test-subj': dataTestSubj }) => {
const { data } = useGetEndpointDetails(endpointAgentId, {
refetchInterval: autoFresh ? DEFAULT_POLL_INTERVAL : false,
refetchInterval: autoRefresh ? DEFAULT_POLL_INTERVAL : false,
});
const emptyValue = (
@ -169,7 +169,7 @@ export const EndpointAgentStatusById = memo<EndpointAgentStatusByIdProps>(
<EndpointAgentStatus
endpointHostInfo={data}
data-test-subj={dataTestSubj}
autoRefresh={autoFresh}
autoRefresh={autoRefresh}
/>
);
}

View file

@ -10,6 +10,7 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import type { CasePostRequest } from '@kbn/cases-plugin/common/api';
import type { DeleteAllEndpointDataResponse } from '../../../scripts/endpoint/common/delete_all_endpoint_data';
import type { IndexedEndpointPolicyResponse } from '../../../common/endpoint/data_loaders/index_endpoint_policy_response';
import type {
HostPolicyResponse,
@ -56,6 +57,20 @@ declare global {
...args: Parameters<Cypress.Chainable<E>['find']>
): Chainable<JQuery<E>>;
/**
* Continuously call provided callback function until it either return `true`
* or fail if `timeout` is reached.
* @param fn
* @param options
*/
waitUntil(
fn: (subject?: any) => boolean | Promise<boolean> | Chainable<boolean>,
options?: Partial<{
interval: number;
timeout: number;
}>
): Chainable<Subject>;
task(
name: 'indexFleetEndpointPolicy',
arg: {
@ -124,6 +139,12 @@ declare global {
arg: HostActionResponse,
options?: Partial<Loggable & Timeoutable>
): Chainable<LogsEndpointActionResponse>;
task(
name: 'deleteAllEndpointData',
arg: { endpointAgentIds: string[] },
options?: Partial<Loggable & Timeoutable>
): Chainable<DeleteAllEndpointDataResponse>;
}
}
}

View file

@ -0,0 +1,107 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { deleteAllLoadedEndpointData } from '../../tasks/delete_all_endpoint_data';
import { getAlertsTableRows, navigateToAlertsList } from '../../screens/alerts';
import { waitForEndpointAlerts } from '../../tasks/alerts';
import { request } from '../../tasks/common';
import { getEndpointIntegrationVersion } from '../../tasks/fleet';
import type { IndexedFleetEndpointPolicyResponse } from '../../../../../common/endpoint/data_loaders/index_fleet_endpoint_policy';
import { enableAllPolicyProtections } from '../../tasks/endpoint_policy';
import type { PolicyData, ResponseActionApiResponse } from '../../../../../common/endpoint/types';
import type { CreateAndEnrollEndpointHostResponse } from '../../../../../scripts/endpoint/common/endpoint_host_services';
import { login } from '../../tasks/login';
import { EXECUTE_ROUTE } from '../../../../../common/endpoint/constants';
import { waitForActionToComplete } from '../../tasks/response_actions';
describe('Endpoint generated alerts', () => {
let indexedPolicy: IndexedFleetEndpointPolicyResponse;
let policy: PolicyData;
let createdHost: CreateAndEnrollEndpointHostResponse;
before(() => {
getEndpointIntegrationVersion().then((version) => {
const policyName = `alerts test ${Math.random().toString(36).substring(2, 7)}`;
cy.task<IndexedFleetEndpointPolicyResponse>('indexFleetEndpointPolicy', {
policyName,
endpointPackageVersion: version,
agentPolicyName: policyName,
}).then((data) => {
indexedPolicy = data;
policy = indexedPolicy.integrationPolicies[0];
return enableAllPolicyProtections(policy.id).then(() => {
// Create and enroll a new Endpoint host
return cy
.task(
'createEndpointHost',
{
agentPolicyId: policy.policy_id,
},
{ timeout: 180000 }
)
.then((host) => {
createdHost = host as CreateAndEnrollEndpointHostResponse;
});
});
});
});
});
after(() => {
if (createdHost) {
cy.task('destroyEndpointHost', createdHost).then(() => {});
}
if (indexedPolicy) {
cy.task('deleteIndexedFleetEndpointPolicies', indexedPolicy);
}
if (createdHost) {
deleteAllLoadedEndpointData({ endpointAgentIds: [createdHost.agentId] });
}
});
beforeEach(() => {
login();
});
it('should create a Detection Engine alert from an endpoint alert', () => {
// Triggers a Malicious Behaviour alert on Linux system (`grep *` was added only to identify this specific alert)
const executeMaliciousCommand = `bash -c cat /dev/tcp/foo | grep ${Math.random()
.toString(16)
.substring(2)}`;
// Send `execute` command that triggers malicious behaviour using the `execute` response action
request<ResponseActionApiResponse>({
method: 'POST',
url: EXECUTE_ROUTE,
body: {
endpoint_ids: [createdHost.agentId],
parameters: {
command: executeMaliciousCommand,
},
},
})
.then((response) => waitForActionToComplete(response.body.data.id))
.then(() => {
return waitForEndpointAlerts(createdHost.agentId, [
{
term: { 'process.group_leader.args': executeMaliciousCommand },
},
]);
})
.then(() => {
return navigateToAlertsList(
`query=(language:kuery,query:'agent.id: "${createdHost.agentId}" ')`
);
});
getAlertsTableRows().should('have.length.greaterThan', 0);
});
});

View file

@ -0,0 +1,41 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { APP_ALERTS_PATH } from '../../../../common/constants';
export const navigateToAlertsList = (urlQueryParams: string = '') => {
cy.visit(`${APP_ALERTS_PATH}${urlQueryParams ? `?${urlQueryParams}` : ''}`);
};
export const clickAlertListRefreshButton = (): Cypress.Chainable => {
return cy.getByTestSubj('querySubmitButton').click().should('be.enabled');
};
/**
* Waits until the Alerts list has alerts data and return the number of rows that are currently displayed
* @param timeout
*/
export const getAlertsTableRows = (timeout?: number): Cypress.Chainable<JQuery<HTMLDivElement>> => {
let $rows: JQuery<HTMLDivElement> = Cypress.$();
return cy
.waitUntil(
() => {
clickAlertListRefreshButton();
return cy
.getByTestSubj('alertsTable')
.find<HTMLDivElement>('.euiDataGridRow')
.then(($rowsFound) => {
$rows = $rowsFound;
return Boolean($rows);
});
},
{ timeout }
)
.then(() => $rows);
};

View file

@ -6,7 +6,7 @@
*/
import { APP_PATH } from '../../../../common/constants';
import { getEndpointDetailsPath } from '../../common/routing';
import { getEndpointDetailsPath, getEndpointListPath } from '../../common/routing';
export const AGENT_HOSTNAME_CELL = 'hostnameCellLink';
export const AGENT_POLICY_CELL = 'policyNameCellLink';
@ -21,3 +21,7 @@ export const navigateToEndpointPolicyResponse = (
getEndpointDetailsPath({ name: 'endpointPolicyResponse', selected_endpoint: endpointAgentId })
);
};
export const navigateToEndpointList = (): Cypress.Chainable<Cypress.AUTWindow> => {
return cy.visit(APP_PATH + getEndpointListPath({ name: 'endpointList' }));
};

View file

@ -9,6 +9,13 @@
import type { CasePostRequest } from '@kbn/cases-plugin/common/api';
import { sendEndpointActionResponse } from '../../../../scripts/endpoint/agent_emulator/services/endpoint_response_actions';
import type { DeleteAllEndpointDataResponse } from '../../../../scripts/endpoint/common/delete_all_endpoint_data';
import { deleteAllEndpointData } from '../../../../scripts/endpoint/common/delete_all_endpoint_data';
import { waitForEndpointToStreamData } from '../../../../scripts/endpoint/common/endpoint_metadata_services';
import type {
CreateAndEnrollEndpointHostOptions,
CreateAndEnrollEndpointHostResponse,
} from '../../../../scripts/endpoint/common/endpoint_host_services';
import type { IndexedEndpointPolicyResponse } from '../../../../common/endpoint/data_loaders/index_endpoint_policy_response';
import {
deleteIndexedEndpointPolicyResponse,
@ -39,6 +46,10 @@ import {
deleteIndexedEndpointRuleAlerts,
indexEndpointRuleAlerts,
} from '../../../../common/endpoint/data_loaders/index_endpoint_rule_alerts';
import {
createAndEnrollEndpointHost,
destroyEndpointHost,
} from '../../../../scripts/endpoint/common/endpoint_host_services';
/**
* Cypress plugin for adding data loading related `task`s
@ -155,5 +166,47 @@ export const dataLoaders = (
const { esClient } = await stackServicesPromise;
return sendEndpointActionResponse(esClient, data.action, { state: data.state.state });
},
deleteAllEndpointData: async ({
endpointAgentIds,
}: {
endpointAgentIds: string[];
}): Promise<DeleteAllEndpointDataResponse> => {
const { esClient } = await stackServicesPromise;
return deleteAllEndpointData(esClient, endpointAgentIds);
},
});
};
export const dataLoadersForRealEndpoints = (
on: Cypress.PluginEvents,
config: Cypress.PluginConfigOptions
): void => {
const stackServicesPromise = createRuntimeServices({
kibanaUrl: config.env.KIBANA_URL,
elasticsearchUrl: config.env.ELASTICSEARCH_URL,
username: config.env.ELASTICSEARCH_USERNAME,
password: config.env.ELASTICSEARCH_PASSWORD,
asSuperuser: true,
});
on('task', {
createEndpointHost: async (
options: Omit<CreateAndEnrollEndpointHostOptions, 'log' | 'kbnClient'>
): Promise<CreateAndEnrollEndpointHostResponse> => {
const { kbnClient, log } = await stackServicesPromise;
return createAndEnrollEndpointHost({ ...options, log, kbnClient }).then((newHost) => {
return waitForEndpointToStreamData(kbnClient, newHost.agentId, 120000).then(() => {
return newHost;
});
});
},
destroyEndpointHost: async (
createdHost: CreateAndEnrollEndpointHostResponse
): Promise<null> => {
const { kbnClient } = await stackServicesPromise;
return destroyEndpointHost(kbnClient, createdHost).then(() => null);
},
});
};

View file

@ -50,4 +50,42 @@ Cypress.Commands.addQuery<'findByTestSubj'>(
}
);
Cypress.Commands.add(
'waitUntil',
{ prevSubject: 'optional' },
(subject, fn, { interval = 500, timeout = 30000 } = {}) => {
let attempts = Math.floor(timeout / interval);
const completeOrRetry = (result: boolean) => {
if (result) {
return result;
}
if (attempts < 1) {
throw new Error(`Timed out while retrying, last result was: {${result}}`);
}
cy.wait(interval, { log: false }).then(() => {
attempts--;
return evaluate();
});
};
const evaluate = () => {
const result = fn(subject);
if (typeof result === 'boolean') {
return completeOrRetry(result);
} else if ('then' in result) {
// @ts-expect-error
return result.then(completeOrRetry);
} else {
throw new Error(
`Unknown return type from callback: ${Object.prototype.toString.call(result)}`
);
}
};
return evaluate();
}
);
Cypress.on('uncaught:exception', () => false);

View file

@ -0,0 +1,170 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { estypes } from '@elastic/elasticsearch';
import type { Rule } from '../../../detection_engine/rule_management/logic';
import {
DETECTION_ENGINE_QUERY_SIGNALS_URL,
DETECTION_ENGINE_RULES_BULK_ACTION,
DETECTION_ENGINE_RULES_URL,
} from '../../../../common/constants';
import { ELASTIC_SECURITY_RULE_ID } from '../../../../common';
import { request } from './common';
import { ENDPOINT_ALERTS_INDEX } from '../../../../scripts/endpoint/common/constants';
const ES_URL = Cypress.env('ELASTICSEARCH_URL');
/**
* Continuously check for any alert to have been received by the given endpoint.
*
* NOTE: This is tno the same as the alerts that populate the Alerts list. To check for
* those types of alerts, use `waitForDetectionAlerts()`
*/
export const waitForEndpointAlerts = (
endpointAgentId: string,
additionalFilters?: object[],
timeout = 120000
): Cypress.Chainable => {
return cy
.waitUntil(
() => {
return request<estypes.SearchResponse>({
method: 'GET',
url: `${ES_URL}/${ENDPOINT_ALERTS_INDEX}/_search`,
body: {
query: {
match: {
'agent.id': endpointAgentId,
},
},
size: 1,
_source: false,
},
}).then(({ body: streamedAlerts }) => {
return (streamedAlerts.hits.total as estypes.SearchTotalHits).value > 0;
});
},
{ timeout }
)
.then(() => {
// Stop/start Endpoint rule so that it can pickup and create Detection alerts
cy.log(
`Received endpoint alerts for agent [${endpointAgentId}] in index [${ENDPOINT_ALERTS_INDEX}]`
);
return stopStartEndpointDetectionsRule();
})
.then(() => {
// wait until the Detection alert shows up in the API
return waitForDetectionAlerts(getEndpointDetectionAlertsQueryForAgentId(endpointAgentId));
});
};
export const fetchEndpointSecurityDetectionRule = (): Cypress.Chainable<Rule> => {
return request<Rule>({
method: 'GET',
url: DETECTION_ENGINE_RULES_URL,
qs: {
rule_id: ELASTIC_SECURITY_RULE_ID,
},
}).then(({ body }) => {
return body;
});
};
export const stopStartEndpointDetectionsRule = (): Cypress.Chainable<Rule> => {
return fetchEndpointSecurityDetectionRule()
.then((endpointRule) => {
// Disabled it
return request({
method: 'POST',
url: DETECTION_ENGINE_RULES_BULK_ACTION,
body: {
action: 'disable',
ids: [endpointRule.id],
},
}).then(() => {
return endpointRule;
});
})
.then((endpointRule) => {
cy.log(`Endpoint rule id [${endpointRule.id}] has been disabled`);
// Re-enable it
return request({
method: 'POST',
url: DETECTION_ENGINE_RULES_BULK_ACTION,
body: {
action: 'enable',
ids: [endpointRule.id],
},
}).then(() => endpointRule);
})
.then((endpointRule) => {
cy.log(`Endpoint rule id [${endpointRule.id}] has been re-enabled`);
return cy.wrap(endpointRule);
});
};
/**
* Waits for alerts to have been loaded by continuously calling the detections engine alerts
* api until data shows up
* @param query
* @param timeout
*/
export const waitForDetectionAlerts = (
/** The ES query. Defaults to `{ match_all: {} }` */
query: object = { match_all: {} },
timeout?: number
): Cypress.Chainable => {
return cy.waitUntil(
() => {
return request<estypes.SearchResponse>({
method: 'POST',
url: DETECTION_ENGINE_QUERY_SIGNALS_URL,
body: {
query,
size: 1,
},
}).then(({ body: alertsResponse }) => {
return Boolean((alertsResponse.hits.total as estypes.SearchTotalHits)?.value ?? 0);
});
},
{ timeout }
);
};
/**
* Builds and returns the ES `query` object for use in querying for Endpoint Detection Engine
* alerts. Can be used in ES searches or with the Detection Engine query signals (alerts) url.
* @param endpointAgentId
*/
export const getEndpointDetectionAlertsQueryForAgentId = (endpointAgentId: string) => {
return {
bool: {
filter: [
{
bool: {
should: [{ match_phrase: { 'agent.type': 'endpoint' } }],
minimum_should_match: 1,
},
},
{
bool: {
should: [{ match_phrase: { 'agent.id': endpointAgentId } }],
minimum_should_match: 1,
},
},
{
bool: {
should: [{ exists: { field: 'kibana.alert.rule.uuid' } }],
minimum_should_match: 1,
},
},
],
},
};
};

View file

@ -0,0 +1,14 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { DeleteAllEndpointDataResponse } from '../../../../scripts/endpoint/common/delete_all_endpoint_data';
export const deleteAllLoadedEndpointData = (options: {
endpointAgentIds: string[];
}): Cypress.Chainable<DeleteAllEndpointDataResponse> => {
return cy.task('deleteAllEndpointData', options);
};

View file

@ -0,0 +1,63 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type {
GetOnePackagePolicyResponse,
UpdatePackagePolicy,
UpdatePackagePolicyResponse,
} from '@kbn/fleet-plugin/common';
import { packagePolicyRouteService } from '@kbn/fleet-plugin/common';
import { request } from './common';
import { ProtectionModes } from '../../../../common/endpoint/types';
/**
* Updates the given Endpoint policy and enables all of the policy protections
* @param endpointPolicyId
*/
export const enableAllPolicyProtections = (
endpointPolicyId: string
): Cypress.Chainable<Cypress.Response<UpdatePackagePolicyResponse>> => {
return request<GetOnePackagePolicyResponse>({
method: 'GET',
url: packagePolicyRouteService.getInfoPath(endpointPolicyId),
}).then(({ body: { item: endpointPolicy } }) => {
const {
created_by: _createdBy,
created_at: _createdAt,
updated_at: _updatedAt,
updated_by: _updatedBy,
id,
version,
revision,
...restOfPolicy
} = endpointPolicy;
const updatedEndpointPolicy: UpdatePackagePolicy = restOfPolicy;
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
const policy = updatedEndpointPolicy!.inputs[0]!.config!.policy.value;
policy.mac.malware.mode = ProtectionModes.prevent;
policy.windows.malware.mode = ProtectionModes.prevent;
policy.linux.malware.mode = ProtectionModes.prevent;
policy.mac.memory_protection.mode = ProtectionModes.prevent;
policy.windows.memory_protection.mode = ProtectionModes.prevent;
policy.linux.memory_protection.mode = ProtectionModes.prevent;
policy.mac.behavior_protection.mode = ProtectionModes.prevent;
policy.windows.behavior_protection.mode = ProtectionModes.prevent;
policy.linux.behavior_protection.mode = ProtectionModes.prevent;
policy.windows.ransomware.mode = ProtectionModes.prevent;
return request<UpdatePackagePolicyResponse>({
method: 'PUT',
url: packagePolicyRouteService.getUpdatePath(endpointPolicyId),
body: updatedEndpointPolicy,
});
});
};

View file

@ -5,6 +5,10 @@
* 2.0.
*/
import { request } from './common';
import { resolvePathVariables } from '../../../common/utils/resolve_path_variables';
import { ACTION_DETAILS_ROUTE } from '../../../../common/endpoint/constants';
import type { ActionDetails, ActionDetailsApiResponse } from '../../../../common/endpoint/types';
import { ENABLED_AUTOMATED_RESPONSE_ACTION_COMMANDS } from '../../../../common/endpoint/service/response_actions/constants';
export const validateAvailableCommands = () => {
@ -59,3 +63,40 @@ export const tryAddingDisabledResponseAction = (itemNumber = 0) => {
});
cy.getByTestSubj(`response-actions-list-item-${itemNumber}`).should('not.exist');
};
/**
* Continuously checks an Response Action until it completes (or timeout is reached)
* @param actionId
* @param timeout
*/
export const waitForActionToComplete = (
actionId: string,
timeout = 60000
): Cypress.Chainable<ActionDetails> => {
let action: ActionDetails | undefined;
return cy
.waitUntil(
() => {
return request<ActionDetailsApiResponse>({
method: 'GET',
url: resolvePathVariables(ACTION_DETAILS_ROUTE, { action_id: actionId || 'undefined' }),
}).then((response) => {
if (response.body.data.isCompleted) {
action = response.body.data;
return true;
}
return false;
});
},
{ timeout }
)
.then(() => {
if (!action) {
throw new Error(`Failed to retrieve completed action`);
}
return action;
});
};

View file

@ -7,7 +7,7 @@
import { defineCypressConfig } from '@kbn/cypress-config';
// eslint-disable-next-line @kbn/imports/no_boundary_crossing
import { dataLoaders } from './cypress/support/data_loaders';
import { dataLoaders, dataLoadersForRealEndpoints } from './cypress/support/data_loaders';
// eslint-disable-next-line import/no-default-export
export default defineCypressConfig({
@ -40,7 +40,9 @@ export default defineCypressConfig({
specPattern: 'public/management/cypress/e2e/endpoint/*.cy.{js,jsx,ts,tsx}',
experimentalRunAllSpecs: true,
setupNodeEvents: (on: Cypress.PluginEvents, config: Cypress.PluginConfigOptions) => {
return dataLoaders(on, config);
dataLoaders(on, config);
// Data loaders specific to "real" Endpoint testing
dataLoadersForRealEndpoints(on, config);
},
},
});

View file

@ -0,0 +1,77 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { Client, estypes } from '@elastic/elasticsearch';
import assert from 'assert';
import { createEsClient } from './stack_services';
import { createSecuritySuperuser } from './security_user_services';
export interface DeleteAllEndpointDataResponse {
count: number;
query: string;
response: estypes.DeleteByQueryResponse;
}
/**
* Attempts to delete all data associated with the provided endpoint agent IDs.
*
* **NOTE:** This utility will create a new role and user that has elevated privileges and access to system indexes.
*
* @param esClient
* @param endpointAgentIds
*/
export const deleteAllEndpointData = async (
esClient: Client,
endpointAgentIds: string[]
): Promise<DeleteAllEndpointDataResponse> => {
assert(endpointAgentIds.length > 0, 'At least one endpoint agent id must be defined');
const unrestrictedUser = await createSecuritySuperuser(esClient, 'super_superuser');
const esUrl = getEsUrlFromClient(esClient);
const esClientUnrestricted = createEsClient({
url: esUrl,
username: unrestrictedUser.username,
password: unrestrictedUser.password,
});
const queryString = endpointAgentIds.map((id) => `(${id})`).join(' OR ');
const deleteResponse = await esClientUnrestricted.deleteByQuery({
index: '*,.*',
body: {
query: {
query_string: {
query: queryString,
},
},
},
ignore_unavailable: true,
conflicts: 'proceed',
});
return {
count: deleteResponse.deleted ?? 0,
query: queryString,
response: deleteResponse,
};
};
const getEsUrlFromClient = (esClient: Client) => {
const connection = esClient.connectionPool.connections.find((entry) => entry.status === 'alive');
if (!connection) {
throw new Error(
'Unable to get esClient connection information. No connection found with status `alive`'
);
}
const url = new URL(connection.url.href);
url.username = '';
url.password = '';
return url.href;
};

View file

@ -0,0 +1,205 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { kibanaPackageJson } from '@kbn/repo-info';
import type { KbnClient } from '@kbn/test';
import type { ToolingLog } from '@kbn/tooling-log';
import execa from 'execa';
import assert from 'assert';
import {
fetchAgentPolicyEnrollmentKey,
fetchFleetServerUrl,
getAgentDownloadUrl,
unEnrollFleetAgent,
waitForHostToEnroll,
} from './fleet_services';
export interface CreateAndEnrollEndpointHostOptions
extends Pick<CreateMultipassVmOptions, 'disk' | 'cpus' | 'memory'> {
kbnClient: KbnClient;
log: ToolingLog;
/** The fleet Agent Policy ID to use for enrolling the agent */
agentPolicyId: string;
/** version of the Agent to install. Defaults to stack version */
version?: string;
/** The name for the host. Will also be the name of the VM */
hostname?: string;
}
export interface CreateAndEnrollEndpointHostResponse {
hostname: string;
agentId: string;
}
/**
* Creates a new virtual machine (host) and enrolls that with Fleet
*/
export const createAndEnrollEndpointHost = async ({
kbnClient,
log,
agentPolicyId,
cpus,
disk,
memory,
hostname,
version = kibanaPackageJson.version,
}: CreateAndEnrollEndpointHostOptions): Promise<CreateAndEnrollEndpointHostResponse> => {
const [vm, agentDownloadUrl, fleetServerUrl, enrollmentToken] = await Promise.all([
createMultipassVm({
vmName: hostname ?? `test-host-${Math.random().toString().substring(2, 6)}`,
disk,
cpus,
memory,
}),
getAgentDownloadUrl(version, true, log),
fetchFleetServerUrl(kbnClient),
fetchAgentPolicyEnrollmentKey(kbnClient, agentPolicyId),
]);
// Some validations before we proceed
assert(agentDownloadUrl, 'Missing agent download URL');
assert(fleetServerUrl, 'Fleet server URL not set');
assert(enrollmentToken, `No enrollment token for agent policy id [${agentPolicyId}]`);
log.verbose(`Enrolling host [${vm.vmName}]
with fleet-server [${fleetServerUrl}]
using enrollment token [${enrollmentToken}]`);
const { agentId } = await enrollHostWithFleet({
kbnClient,
log,
fleetServerUrl,
agentDownloadUrl,
enrollmentToken,
vmName: vm.vmName,
});
return {
hostname: vm.vmName,
agentId,
};
};
/**
* Destroys the Endpoint Host VM and un-enrolls the Fleet agent
* @param kbnClient
* @param createdHost
*/
export const destroyEndpointHost = async (
kbnClient: KbnClient,
createdHost: CreateAndEnrollEndpointHostResponse
): Promise<void> => {
await Promise.all([
deleteMultipassVm(createdHost.hostname),
unEnrollFleetAgent(kbnClient, createdHost.agentId, true),
]);
};
interface CreateMultipassVmOptions {
vmName: string;
/** Number of CPUs */
cpus?: number;
/** Disk size */
disk?: string;
/** Amount of memory */
memory?: string;
}
interface CreateMultipassVmResponse {
vmName: string;
}
/**
* Creates a new VM using `multipass`
*/
const createMultipassVm = async ({
vmName,
disk = '8G',
cpus = 1,
memory = '1G',
}: CreateMultipassVmOptions): Promise<CreateMultipassVmResponse> => {
await execa.command(
`multipass launch --name ${vmName} --disk ${disk} --cpus ${cpus} --memory ${memory}`
);
return {
vmName,
};
};
const deleteMultipassVm = async (vmName: string): Promise<void> => {
await execa.command(`multipass delete -p ${vmName}`);
};
interface EnrollHostWithFleetOptions {
kbnClient: KbnClient;
log: ToolingLog;
vmName: string;
agentDownloadUrl: string;
fleetServerUrl: string;
enrollmentToken: string;
}
const enrollHostWithFleet = async ({
kbnClient,
log,
vmName,
fleetServerUrl,
agentDownloadUrl,
enrollmentToken,
}: EnrollHostWithFleetOptions): Promise<{ agentId: string }> => {
const agentDownloadedFile = agentDownloadUrl.substring(agentDownloadUrl.lastIndexOf('/') + 1);
const vmDirName = agentDownloadedFile.replace(/\.tar\.gz$/, '');
await execa.command(
`multipass exec ${vmName} -- curl -L ${agentDownloadUrl} -o ${agentDownloadedFile}`
);
await execa.command(`multipass exec ${vmName} -- tar -zxf ${agentDownloadedFile}`);
await execa.command(`multipass exec ${vmName} -- rm -f ${agentDownloadedFile}`);
const agentInstallArguments = [
'exec',
vmName,
'--working-directory',
`/home/ubuntu/${vmDirName}`,
'--',
'sudo',
'./elastic-agent',
'install',
'--insecure',
'--force',
'--url',
fleetServerUrl,
'--enrollment-token',
enrollmentToken,
];
log.info(`Enrolling elastic agent with Fleet`);
log.verbose(`Command: multipass ${agentInstallArguments.join(' ')}`);
await execa(`multipass`, agentInstallArguments);
log.info(`Waiting for Agent to check-in with Fleet`);
const agent = await waitForHostToEnroll(kbnClient, vmName, 120000);
return {
agentId: agent.id,
};
};

View file

@ -131,3 +131,46 @@ const fetchLastStreamedEndpointUpdate = async (
return queryResult.hits?.hits[0]?._source;
};
/**
* Waits for an endpoint to have streamed data to ES and for that data to have made it to the
* Endpoint Details API (transform destination index)
* @param kbnClient
* @param endpointAgentId
* @param timeoutMs
*/
export const waitForEndpointToStreamData = async (
kbnClient: KbnClient,
endpointAgentId: string,
timeoutMs: number = 60000
): Promise<HostInfo> => {
const started = new Date();
const hasTimedOut = (): boolean => {
const elapsedTime = Date.now() - started.getTime();
return elapsedTime > timeoutMs;
};
let found: HostInfo | undefined;
while (!found && !hasTimedOut()) {
found = await fetchEndpointMetadata(kbnClient, 'invalid-id-test').catch((error) => {
// Ignore `not found` (404) responses. Endpoint could be new and thus documents might not have
// been streamed yet.
if (error?.response?.status === 404) {
return undefined;
}
throw error;
});
if (!found) {
// sleep and check again
await new Promise((r) => setTimeout(r, 2000));
}
}
if (!found) {
throw new Error(`Timed out waiting for Endpoint id [${endpointAgentId}] to stream data to ES`);
}
return found;
};

View file

@ -14,7 +14,12 @@ import type {
GetAgentPoliciesResponse,
GetAgentsResponse,
} from '@kbn/fleet-plugin/common';
import { AGENT_API_ROUTES, agentPolicyRouteService, AGENTS_INDEX } from '@kbn/fleet-plugin/common';
import {
AGENT_API_ROUTES,
agentPolicyRouteService,
agentRouteService,
AGENTS_INDEX,
} from '@kbn/fleet-plugin/common';
import { ToolingLog } from '@kbn/tooling-log';
import type { KbnClient } from '@kbn/test';
import type { GetFleetServerHostsResponse } from '@kbn/fleet-plugin/common/types/rest_spec/fleet_server_hosts';
@ -26,7 +31,10 @@ import type {
EnrollmentAPIKey,
GetAgentsRequest,
GetEnrollmentAPIKeysResponse,
PostAgentUnenrollResponse,
} from '@kbn/fleet-plugin/common/types';
import nodeFetch from 'node-fetch';
import semver from 'semver';
import { FleetAgentGenerator } from '../../../common/endpoint/data_generators/fleet_agent_generator';
const fleetGenerator = new FleetAgentGenerator();
@ -236,3 +244,135 @@ export const getAgentVersionMatchingCurrentStack = async (
return version;
};
interface ElasticArtifactSearchResponse {
manifest: {
'last-update-time': string;
'seconds-since-last-update': number;
};
packages: {
[packageFileName: string]: {
architecture: string;
os: string[];
type: string;
asc_url: string;
sha_url: string;
url: string;
};
};
}
/**
* Retrieves the download URL to the Linux installation package for a given version of the Elastic Agent
* @param version
* @param closestMatch
* @param log
*/
export const getAgentDownloadUrl = async (
version: string,
/**
* When set to true a check will be done to determine the latest version of the agent that
* is less than or equal to the `version` provided
*/
closestMatch: boolean = false,
log?: ToolingLog
): Promise<string> => {
const agentVersion = closestMatch ? await getLatestAgentDownloadVersion(version, log) : version;
const downloadArch =
{ arm64: 'arm64', x64: 'x86_64' }[process.arch] ?? `UNSUPPORTED_ARCHITECTURE_${process.arch}`;
const agentFile = `elastic-agent-${agentVersion}-linux-${downloadArch}.tar.gz`;
const artifactSearchUrl = `https://artifacts-api.elastic.co/v1/search/${agentVersion}/${agentFile}`;
log?.verbose(`Retrieving elastic agent download URL from:\n ${artifactSearchUrl}`);
const searchResult: ElasticArtifactSearchResponse = await nodeFetch(artifactSearchUrl).then(
(response) => {
if (!response.ok) {
throw new Error(
`Failed to search elastic's artifact repository: ${response.statusText} (HTTP ${response.status}) {URL: ${artifactSearchUrl})`
);
}
return response.json();
}
);
log?.verbose(searchResult);
if (!searchResult.packages[agentFile]) {
throw new Error(`Unable to find an Agent download URL for version [${agentVersion}]`);
}
return searchResult.packages[agentFile].url;
};
/**
* Given a stack version number, function will return the closest Agent download version available
* for download. THis could be the actual version passed in or lower.
* @param version
*/
export const getLatestAgentDownloadVersion = async (
version: string,
log?: ToolingLog
): Promise<string> => {
const artifactsUrl = 'https://artifacts-api.elastic.co/v1/versions';
const semverMatch = `<=${version}`;
const artifactVersionsResponse: { versions: string[] } = await nodeFetch(artifactsUrl).then(
(response) => {
if (!response.ok) {
throw new Error(
`Failed to retrieve list of versions from elastic's artifact repository: ${response.statusText} (HTTP ${response.status}) {URL: ${artifactsUrl})`
);
}
return response.json();
}
);
const stackVersionToArtifactVersion: Record<string, string> =
artifactVersionsResponse.versions.reduce((acc, artifactVersion) => {
const stackVersion = artifactVersion.split('-SNAPSHOT')[0];
acc[stackVersion] = artifactVersion;
return acc;
}, {} as Record<string, string>);
log?.verbose(
`Versions found from [${artifactsUrl}]:\n${JSON.stringify(
stackVersionToArtifactVersion,
null,
2
)}`
);
const matchedVersion = semver.maxSatisfying(
Object.keys(stackVersionToArtifactVersion),
semverMatch
);
if (!matchedVersion) {
throw new Error(`Unable to find a semver version that meets ${semverMatch}`);
}
return stackVersionToArtifactVersion[matchedVersion];
};
/**
* Un-enrolls a Fleet agent
*
* @param kbnClient
* @param agentId
* @param force
*/
export const unEnrollFleetAgent = async (
kbnClient: KbnClient,
agentId: string,
force = false
): Promise<PostAgentUnenrollResponse> => {
const { data } = await kbnClient.request<PostAgentUnenrollResponse>({
method: 'POST',
path: agentRouteService.getUnenrollPath(agentId),
body: { revoke: force },
});
return data;
};

View file

@ -17,12 +17,41 @@ export const createSecuritySuperuser = async (
throw new Error(`username and password require values.`);
}
// Create a role which has full access to restricted indexes
await esClient.transport.request({
method: 'POST',
path: '_security/role/superuser_restricted_indices',
body: {
cluster: ['all'],
indices: [
{
names: ['*'],
privileges: ['all'],
allow_restricted_indices: true,
},
{
names: ['*'],
privileges: ['monitor', 'read', 'view_index_metadata', 'read_cross_cluster'],
allow_restricted_indices: true,
},
],
applications: [
{
application: '*',
privileges: ['*'],
resources: ['*'],
},
],
run_as: ['*'],
},
});
const addedUser = await esClient.transport.request<Promise<{ created: boolean }>>({
method: 'POST',
path: `_security/user/${username}`,
body: {
password,
roles: ['superuser', 'kibana_system'],
roles: ['superuser', 'kibana_system', 'superuser_restricted_indices'],
full_name: username,
},
});

View file

@ -99,7 +99,11 @@ export const createRuntimeServices = async ({
};
};
const buildUrlWithCredentials = (url: string, username: string, password: string): string => {
export const buildUrlWithCredentials = (
url: string,
username: string,
password: string
): string => {
const newUrl = new URL(url);
newUrl.username = username;

View file

@ -8,6 +8,7 @@
import { getLocalhostRealIp } from '@kbn/security-solution-plugin/scripts/endpoint/common/localhost_services';
import { FtrConfigProviderContext } from '@kbn/test';
import { ExperimentalFeatures } from '@kbn/security-solution-plugin/common/experimental_features';
import { DefendWorkflowsCypressEndpointTestRunner } from './runner';
export default async function ({ readConfigFile }: FtrConfigProviderContext) {
@ -15,6 +16,8 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
const config = defendWorkflowsCypressConfig.getAll();
const hostIp = getLocalhostRealIp();
const enabledFeatureFlags: Array<keyof ExperimentalFeatures> = ['responseActionExecuteEnabled'];
return {
...config,
kbnTestServer: {
@ -27,6 +30,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
)}`,
// set the packagerTaskInterval to 5s in order to speed up test executions when checking fleet artifacts
'--xpack.securitySolution.packagerTaskInterval=5s',
`--xpack.securitySolution.enableExperimental=${JSON.stringify(enabledFeatureFlags)}`,
],
},
testRunner: DefendWorkflowsCypressEndpointTestRunner,