github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 09:48:58 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Security Solution][Endpoint] Add action list API for action log table view (#133213)

Browse source 
* Add new API to for action log table view

refs elastic/security-team/issues/3395

* add tests

refs elastic/security-team/issues/3395

* Update schema

review changes

* review changes

* review changes

* fix paging

review changes

* revert change

review changes

* update action details by action id API

Only search on endpoint actions index and not `.fleet-actions` index

* fix type

size and from are numbers at this point

* fix types for details API and tests

refs c88dbb3c5a

* Better error handling

* [CI] Auto-commit changed files from 'node scripts/precommit_hook.js --ref HEAD~1..HEAD --fix'

* remove fleet actions dependency

* also return agentIds if in request

same as other query params

* add tests

refs elastic/security-team/issues/3871

* add more tests

* [CI] Auto-commit changed files from 'node scripts/precommit_hook.js --ref HEAD~1..HEAD --fix'

* authz tests for route handler

* Update action_list.test.ts

refs elastic/kibana/pull/133213/commits/77565f0445367347168bd2c6a3cc82593a837f21

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Ashokaditya 2022-06-14 20:52:17 +02:00 • committed by GitHub
parent e353f98b5e
commit 6a6b1eeafa
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
27 changed files with 1997 additions and 343 deletions
Download patch file Download diff file Expand all files Collapse all files

1
x-pack/plugins/security_solution/common/endpoint/constants.ts
View file

@ -67,6 +67,7 @@ export const SUSPEND_PROCESS_ROUTE = `${BASE_ENDPOINT_ACTION_ROUTE}/suspend_proc
export const ENDPOINT_ACTION_LOG_ROUTE = `${BASE_ENDPOINT_ROUTE}/action_log/{agent_id}`;
export const ACTION_STATUS_ROUTE = `${BASE_ENDPOINT_ROUTE}/action_status`;
export const ACTION_DETAILS_ROUTE = `${BASE_ENDPOINT_ACTION_ROUTE}/{action_id}`;
export const ENDPOINTS_ACTION_LIST_ROUTE = `${BASE_ENDPOINT_ROUTE}/action`;
export const failedFleetActionErrorCode = '424';

20
x-pack/plugins/security_solution/common/endpoint/data_generators/endpoint_action_generator.ts
View file

@ -8,11 +8,12 @@
import { DeepPartial } from 'utility-types';
import { merge } from 'lodash';
import * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import { ENDPOINT_ACTION_RESPONSES_DS, ENDPOINT_ACTIONS_INDEX } from '../constants';
import { ENDPOINT_ACTION_RESPONSES_DS, ENDPOINT_ACTIONS_DS } from '../constants';
import { BaseDataGenerator } from './base_data_generator';
import {
ActionDetails,
ActivityLogItemTypes,
EndpointActivityLogAction,
EndpointActivityLogActionResponse,
EndpointPendingActions,
ISOLATION_ACTIONS,
@ -58,7 +59,7 @@ export class EndpointActionGenerator extends BaseDataGenerator {
overrides: DeepPartial<LogsEndpointAction> = {}
): estypes.SearchHit<LogsEndpointAction> {
return Object.assign(this.toEsSearchHit(this.generate(overrides)), {
_index: `.ds-${ENDPOINT_ACTIONS_INDEX}-some_namespace`,
_index: `.ds-${ENDPOINT_ACTIONS_DS}-some_namespace`,
});
}
@ -181,6 +182,21 @@ export class EndpointActionGenerator extends BaseDataGenerator {
return merge(details, overrides);
}
generateActivityLogAction(
overrides: DeepPartial<EndpointActivityLogAction>
): EndpointActivityLogAction {
return merge(
{
type: ActivityLogItemTypes.ACTION,
item: {
id: this.seededUUIDv4(),
data: this.generate(),
},
},
overrides
);
}
generateActivityLogActionResponse(
overrides: DeepPartial<EndpointActivityLogActionResponse>
): EndpointActivityLogActionResponse {

18
x-pack/plugins/security_solution/common/endpoint/data_generators/fleet_action_generator.ts
View file

@ -11,6 +11,7 @@ import * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import { AGENT_ACTIONS_INDEX, AGENT_ACTIONS_RESULTS_INDEX } from '@kbn/fleet-plugin/common';
import { BaseDataGenerator } from './base_data_generator';
import {
ActivityLogAction,
ActivityLogActionResponse,
ActivityLogItemTypes,
EndpointAction,
@ -90,6 +91,23 @@ export class FleetActionGenerator extends BaseDataGenerator {
});
}
/**
* An Activity Log entry as returned by the Activity log API
* @param overrides
*/
generateActivityLogAction(overrides: DeepPartial<ActivityLogAction> = {}): ActivityLogAction {
return merge(
{
type: ActivityLogItemTypes.FLEET_ACTION,
item: {
id: this.seededUUIDv4(),
data: this.generate(),
},
},
overrides
);
}
/**
* An Activity Log entry as returned by the Activity log API
* @param overrides

141
x-pack/plugins/security_solution/common/endpoint/schema/actions.test.ts
View file

@ -5,9 +5,148 @@
* 2.0.
*/
import { HostIsolationRequestSchema, KillProcessRequestSchema } from './actions';
import uuid from 'uuid';
import {
EndpointActionListRequestSchema,
HostIsolationRequestSchema,
KillProcessRequestSchema,
} from './actions';
describe('actions schemas', () => {
describe('Endpoint action list API Schema', () => {
it('should work without any query keys ', () => {
expect(() => {
EndpointActionListRequestSchema.query.validate({}); // no agent_ids provided
}).not.toThrow();
});
it('should require at least 1 agent ID', () => {
expect(() => {
EndpointActionListRequestSchema.query.validate({ agentIds: [] }); // no agent_ids provided
}).toThrow();
});
it('should not accept an agent ID if not in an array', () => {
expect(() => {
EndpointActionListRequestSchema.query.validate({ agentIds: uuid.v4() });
}).toThrow();
});
it('should accept an agent ID in an array', () => {
expect(() => {
EndpointActionListRequestSchema.query.validate({ agentIds: [uuid.v4()] });
}).not.toThrow();
});
it('should accept multiple agent IDs in an array', () => {
expect(() => {
EndpointActionListRequestSchema.query.validate({
agentIds: [uuid.v4(), uuid.v4(), uuid.v4()],
});
}).not.toThrow();
});
it('should limit multiple agent IDs in an array to 50', () => {
expect(() => {
EndpointActionListRequestSchema.query.validate({
agentIds: Array(51)
.fill(1)
.map(() => uuid.v4()),
});
}).toThrow();
});
it('should work with all required query params', () => {
expect(() => {
EndpointActionListRequestSchema.query.validate({
page: 10,
pageSize: 100,
startDate: new Date(new Date().setDate(new Date().getDate() - 1)).toISOString(), // yesterday
endDate: new Date().toISOString(), // today
});
}).not.toThrow();
});
it('should not work without allowed page and pageSize params', () => {
expect(() => {
EndpointActionListRequestSchema.query.validate({ pageSize: 101 });
}).toThrow();
});
it('should not work without valid userIds', () => {
expect(() => {
EndpointActionListRequestSchema.query.validate({
page: 10,
pageSize: 100,
startDate: new Date(new Date().setDate(new Date().getDate() - 1)).toISOString(), // yesterday
endDate: new Date().toISOString(), // today
userIds: [],
});
}).toThrow();
});
it('should work with a single userIds query params', () => {
expect(() => {
EndpointActionListRequestSchema.query.validate({
page: 10,
pageSize: 100,
startDate: new Date(new Date().setDate(new Date().getDate() - 1)).toISOString(), // yesterday
endDate: new Date().toISOString(), // today
userIds: ['elastic'],
});
}).not.toThrow();
});
it('should work with multiple userIds query params', () => {
expect(() => {
EndpointActionListRequestSchema.query.validate({
page: 10,
pageSize: 100,
startDate: new Date(new Date().setDate(new Date().getDate() - 1)).toISOString(), // yesterday
endDate: new Date().toISOString(), // today
userIds: ['elastic', 'fleet'],
});
}).not.toThrow();
});
it('should work with commands query params with a single action type', () => {
expect(() => {
EndpointActionListRequestSchema.query.validate({
page: 10,
pageSize: 100,
startDate: new Date(new Date().setDate(new Date().getDate() - 1)).toISOString(), // yesterday
endDate: new Date().toISOString(), // today
commands: ['isolate'],
});
}).not.toThrow();
});
it('should not work with commands query params with empty array', () => {
expect(() => {
EndpointActionListRequestSchema.query.validate({
page: 10,
pageSize: 100,
startDate: new Date(new Date().setDate(new Date().getDate() - 1)).toISOString(), // yesterday
endDate: new Date().toISOString(), // today
commands: [],
});
}).toThrow();
});
it('should work with commands query params with multiple types', () => {
expect(() => {
EndpointActionListRequestSchema.query.validate({
page: 10,
pageSize: 100,
startDate: new Date(new Date().setDate(new Date().getDate() - 1)).toISOString(), // yesterday
endDate: new Date().toISOString(), // today
commands: ['isolate', 'unisolate'],
});
}).not.toThrow();
});
});
describe('HostIsolationRequestSchema', () => {
it('should require at least 1 Endpoint ID', () => {
expect(() => {

16
x-pack/plugins/security_solution/common/endpoint/schema/actions.ts
View file

@ -66,3 +66,19 @@ export const ActionDetailsRequestSchema = {
action_id: schema.string(),
}),
};
export const EndpointActionListRequestSchema = {
query: schema.object({
agentIds: schema.maybe(
schema.arrayOf(schema.string({ minLength: 1 }), { minSize: 1, maxSize: 50 })
),
commands: schema.maybe(schema.arrayOf(schema.string({ minLength: 1 }), { minSize: 1 })),
page: schema.maybe(schema.number({ defaultValue: 1, min: 1 })),
pageSize: schema.maybe(schema.number({ defaultValue: 10, min: 1, max: 100 })),
startDate: schema.maybe(schema.string()), // date ISO strings or moment date
endDate: schema.maybe(schema.string()), // date ISO strings or moment date
userIds: schema.maybe(schema.arrayOf(schema.string({ minLength: 1 }), { minSize: 1 })),
}),
};
export type EndpointActionListRequestQuery = TypeOf<typeof EndpointActionListRequestSchema.query>;

16
x-pack/plugins/security_solution/common/endpoint/types/actions.ts
View file

@ -101,12 +101,9 @@ export interface FleetActionResponseData {
/**
* And endpoint action created in Fleet's `.fleet-actions`
*/
export interface EndpointAction {
export interface EndpointAction extends ActionRequestFields {
action_id: string;
'@timestamp': string;
expiration: string;
type: 'INPUT_ACTION';
input_type: 'endpoint';
agents: string[];
user_id: string;
// the number of seconds Elastic Agent (on the host) should
@ -246,3 +243,14 @@ export interface ActionDetails {
export interface ActionDetailsApiResponse {
data: ActionDetails;
}
export interface ActionListApiResponse {
page: number | undefined;
pageSize: number | undefined;
startDate: string | undefined;
elasticAgentIds: string[] | undefined;
endDate: string | undefined;
userIds: string[] | undefined; // users that requested the actions
commands: string[] | undefined; // type of actions
data: ActionDetails[];
total: number;
}

4
x-pack/plugins/security_solution/server/endpoint/routes/actions/audit_log.ts
View file

@ -7,7 +7,7 @@
import { ENDPOINT_ACTION_LOG_ROUTE } from '../../../../common/endpoint/constants';
import { EndpointActionLogRequestSchema } from '../../../../common/endpoint/schema/actions';
import { actionsLogRequestHandler } from './audit_log_handler';
import { auditLogRequestHandler } from './audit_log_handler';
import { SecuritySolutionPluginRouter } from '../../../types';
import { EndpointAppContext } from '../../types';
@ -29,7 +29,7 @@ export function registerActionAuditLogRoutes(
withEndpointAuthz(
{ all: ['canIsolateHost'] },
endpointContext.logFactory.get('hostIsolationLogs'),
actionsLogRequestHandler(endpointContext)
auditLogRequestHandler(endpointContext)
)
);
}

2
x-pack/plugins/security_solution/server/endpoint/routes/actions/audit_log_handler.ts
View file

@ -14,7 +14,7 @@ import { getAuditLogResponse } from '../../services';
import { SecuritySolutionRequestHandlerContext } from '../../../types';
import { EndpointAppContext } from '../../types';
export const actionsLogRequestHandler = (
export const auditLogRequestHandler = (
endpointContext: EndpointAppContext
): RequestHandler<
EndpointActionLogRequestParams,

2
x-pack/plugins/security_solution/server/endpoint/routes/actions/index.ts
View file

@ -11,6 +11,7 @@ import { EndpointAppContext } from '../../types';
import { registerHostIsolationRoutes } from './isolation';
import { registerActionStatusRoutes } from './status';
import { registerActionAuditLogRoutes } from './audit_log';
import { registerActionListRoutes } from './list';
import { registerResponseActionRoutes } from './response_actions';
export * from './isolation';
@ -24,6 +25,7 @@ export function registerActionRoutes(
registerHostIsolationRoutes(router, endpointContext);
registerActionStatusRoutes(router, endpointContext);
registerActionAuditLogRoutes(router, endpointContext);
registerActionListRoutes(router, endpointContext);
registerActionDetailsRoutes(router, endpointContext);
registerResponseActionRoutes(router, endpointContext);
}

133
x-pack/plugins/security_solution/server/endpoint/routes/actions/list.test.ts Normal file
View file

@ -0,0 +1,133 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { SecuritySolutionRequestHandlerContextMock } from '../../../lib/detection_engine/routes/__mocks__/request_context';
import { AwaitedProperties } from '@kbn/utility-types';
import { EndpointActionListRequestQuery } from '../../../../common/endpoint/schema/actions';
import { EndpointAuthz } from '../../../../common/endpoint/types/authz';
import {
createMockEndpointAppContextServiceSetupContract,
createMockEndpointAppContextServiceStartContract,
createRouteHandlerContext,
} from '../../mocks';
import {
elasticsearchServiceMock,
httpServerMock,
httpServiceMock,
loggingSystemMock,
savedObjectsClientMock,
} from '@kbn/core/server/mocks';
import { KibanaResponseFactory, RequestHandler, RouteConfig } from '@kbn/core/server';
import { ENDPOINTS_ACTION_LIST_ROUTE } from '../../../../common/endpoint/constants';
import { EndpointAppContextService } from '../../endpoint_app_context_services';
import { createMockConfig } from '../../../lib/detection_engine/routes/__mocks__';
import { LicenseService, parseExperimentalConfigValue } from '@kbn/fleet-plugin/common';
import { Subject } from 'rxjs';
import { ILicense } from '@kbn/licensing-plugin/common/types';
import { licenseMock } from '@kbn/licensing-plugin/common/licensing.mock';
import { registerActionListRoutes } from './list';
interface CallApiRouteInterface {
query?: EndpointActionListRequestQuery;
authz?: Partial<EndpointAuthz>;
}
const Platinum = licenseMock.createLicense({ license: { type: 'platinum', mode: 'platinum' } });
describe('Action List Route', () => {
const superUser = {
username: 'superuser',
roles: ['superuser'],
};
let endpointAppContextService: EndpointAppContextService;
let mockResponse: jest.Mocked<KibanaResponseFactory>;
let licenseService: LicenseService;
let licenseEmitter: Subject<ILicense>;
let callApiRoute: (
routePrefix: string,
opts: CallApiRouteInterface
) => Promise<AwaitedProperties<SecuritySolutionRequestHandlerContextMock>>;
beforeEach(() => {
const mockScopedClient = elasticsearchServiceMock.createScopedClusterClient();
const mockSavedObjectClient = savedObjectsClientMock.create();
const startContract = createMockEndpointAppContextServiceStartContract();
const routerMock = httpServiceMock.createRouter();
mockResponse = httpServerMock.createResponseFactory();
endpointAppContextService = new EndpointAppContextService();
licenseEmitter = new Subject();
licenseService = new LicenseService();
licenseService.start(licenseEmitter);
endpointAppContextService.setup(createMockEndpointAppContextServiceSetupContract());
endpointAppContextService.start({
...startContract,
// @ts-expect-error
licenseService,
});
registerActionListRoutes(routerMock, {
logFactory: loggingSystemMock.create(),
service: endpointAppContextService,
config: () => Promise.resolve(createMockConfig()),
// @ts-expect-error
experimentalFeatures: parseExperimentalConfigValue(createMockConfig().enableExperimental),
});
callApiRoute = async (
routePrefix: string,
{ query, authz = {} }: CallApiRouteInterface
): Promise<AwaitedProperties<SecuritySolutionRequestHandlerContextMock>> => {
(startContract.security.authc.getCurrentUser as jest.Mock).mockImplementationOnce(
() => superUser
);
const ctx = createRouteHandlerContext(mockScopedClient, mockSavedObjectClient);
ctx.securitySolution.endpointAuthz = {
...ctx.securitySolution.endpointAuthz,
...authz,
};
licenseEmitter.next(Platinum);
const mockRequest = httpServerMock.createKibanaRequest({ query });
const [, routeHandler]: [
// eslint-disable-next-line @typescript-eslint/no-explicit-any
RouteConfig<any, any, any, any>,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
RequestHandler<any, any, any, any>
] = routerMock.get.mock.calls.find(([{ path }]) => path.startsWith(routePrefix))!;
await routeHandler(ctx, mockRequest, mockResponse);
return ctx;
};
});
afterEach(() => {
endpointAppContextService.stop();
licenseService.stop();
licenseEmitter.complete();
});
describe('User auth level', () => {
it('allows user with canAccessEndpointManagement access to allow requests to API', async () => {
await callApiRoute(ENDPOINTS_ACTION_LIST_ROUTE, {});
expect(mockResponse.ok).toBeCalled();
});
it('does not allow user with canAccessEndpointManagement access to allow requests to API', async () => {
await callApiRoute(ENDPOINTS_ACTION_LIST_ROUTE, {
authz: { canAccessEndpointManagement: false },
});
expect(mockResponse.forbidden).toBeCalled();
});
});
});

41
x-pack/plugins/security_solution/server/endpoint/routes/actions/list.ts Normal file
View file

@ -0,0 +1,41 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { ENDPOINTS_ACTION_LIST_ROUTE } from '../../../../common/endpoint/constants';
import { EndpointActionListRequestSchema } from '../../../../common/endpoint/schema/actions';
import { actionListHandler } from './list_handler';
import type { SecuritySolutionPluginRouter } from '../../../types';
import type { EndpointAppContext } from '../../types';
import { withEndpointAuthz } from '../with_endpoint_authz';
/**
* Registers the endpoint activity_log route
*/
export function registerActionListRoutes(
router: SecuritySolutionPluginRouter,
endpointContext: EndpointAppContext
) {
router.get(
{
path: ENDPOINTS_ACTION_LIST_ROUTE,
validate: EndpointActionListRequestSchema,
options: { authRequired: true, tags: ['access:securitySolution'] },
},
withEndpointAuthz(
{ all: ['canAccessEndpointManagement'] },
endpointContext.logFactory.get('endpointActionList'),
actionListHandler(endpointContext)
)
);
}

56
x-pack/plugins/security_solution/server/endpoint/routes/actions/list_handler.ts Normal file
View file

@ -0,0 +1,56 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { RequestHandler } from '@kbn/core/server';
import type { EndpointActionListRequestQuery } from '../../../../common/endpoint/schema/actions';
import { getActionList } from '../../services';
import type { SecuritySolutionRequestHandlerContext } from '../../../types';
import type { EndpointAppContext } from '../../types';
import { errorHandler } from '../error_handler';
export const actionListHandler = (
endpointContext: EndpointAppContext
): RequestHandler<
unknown,
EndpointActionListRequestQuery,
unknown,
SecuritySolutionRequestHandlerContext
> => {
const logger = endpointContext.logFactory.get('endpoint_action_list');
return async (context, req, res) => {
const {
query: { agentIds: elasticAgentIds, page, pageSize, startDate, endDate, userIds, commands },
} = req;
const esClient = (await context.core).elasticsearch.client.asInternalUser;
try {
const body = await getActionList({
commands,
esClient,
elasticAgentIds,
page,
pageSize,
startDate,
endDate,
userIds,
logger,
});
return res.ok({
body,
});
} catch (error) {
return errorHandler(logger, res, error);
}
};
};

59
x-pack/plugins/security_solution/server/endpoint/services/actions/action_details_by_id.test.ts
View file

@ -9,7 +9,6 @@ import type { ElasticsearchClientMock } from '@kbn/core/server/mocks';
import { elasticsearchServiceMock } from '@kbn/core/server/mocks';
import * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import {
EndpointAction,
EndpointActionResponse,
LogsEndpointAction,
LogsEndpointActionResponse,
@ -26,7 +25,7 @@ import {
describe('When using `getActionDetailsById()', () => {
let esClient: ElasticsearchClientMock;
let endpointActionGenerator: EndpointActionGenerator;
let actionRequests: estypes.SearchResponse<EndpointAction | LogsEndpointAction>;
let actionRequests: estypes.SearchResponse<LogsEndpointAction>;
let actionResponses: estypes.SearchResponse<EndpointActionResponse | LogsEndpointActionResponse>;
beforeEach(() => {
@ -45,7 +44,7 @@ describe('When using `getActionDetailsById()', () => {
command: 'isolate',
completedAt: '2022-04-30T16:08:47.449Z',
wasSuccessful: true,
error: undefined,
errors: undefined,
id: '123',
isCompleted: true,
isExpired: false,
@ -53,21 +52,24 @@ describe('When using `getActionDetailsById()', () => {
{
item: {
data: {
'@timestamp': '2022-04-27T16:08:47.449Z',
action_id: '123',
agents: ['agent-a'],
data: {
command: 'isolate',
comment: '5wb6pu6kh2xix5i',
'@timestamp': '2022-04-30T16:08:47.449Z',
EndpointActions: {
action_id: '123',
completed_at: '2022-04-30T16:08:47.449Z',
data: {
command: 'unisolate',
comment: '',
},
started_at: expect.any(String),
},
expiration: '2022-04-29T16:08:47.449Z',
input_type: 'endpoint',
type: 'INPUT_ACTION',
user_id: 'elastic',
agent: {
id: 'agent-a',
},
error: undefined,
},
id: '44d8b915-c69c-4c48-8c86-b57d0bd631d0',
id: expect.any(String),
},
type: 'fleetAction',
type: 'response',
},
{
item: {
@ -90,23 +92,26 @@ describe('When using `getActionDetailsById()', () => {
{
item: {
data: {
'@timestamp': '2022-04-30T16:08:47.449Z',
'@timestamp': '2022-04-27T16:08:47.449Z',
EndpointActions: {
action_id: '123',
completed_at: '2022-04-30T16:08:47.449Z',
data: {
command: 'unisolate',
comment: '',
command: 'isolate',
comment: '5wb6pu6kh2xix5i',
},
started_at: expect.any(String),
expiration: expect.any(String),
input_type: 'endpoint',
type: 'INPUT_ACTION',
},
agent: {
id: 'agent-a',
agent: { id: 'agent-a' },
user: {
id: expect.any(String),
},
error: undefined,
},
id: expect.any(String),
},
type: 'response',
type: 'action',
},
],
startedAt: '2022-04-27T16:08:47.449Z',
@ -144,7 +149,9 @@ describe('When using `getActionDetailsById()', () => {
});
it('should have `isExpired` of `true` if NOT complete and expiration is in the past', async () => {
(actionRequests.hits.hits[0]._source as EndpointAction).expiration = `2021-04-30T16:08:47.449Z`;
(
actionRequests.hits.hits[0]._source as LogsEndpointAction
).EndpointActions.expiration = `2021-04-30T16:08:47.449Z`;
actionResponses.hits.hits.pop(); // remove the endpoint response
await expect(getActionDetailsById(esClient, '123')).resolves.toEqual(
@ -156,7 +163,9 @@ describe('When using `getActionDetailsById()', () => {
});
it('should have `isExpired` of `false` if complete and expiration is in the past', async () => {
(actionRequests.hits.hits[0]._source as EndpointAction).expiration = `2021-04-30T16:08:47.449Z`;
(
actionRequests.hits.hits[0]._source as LogsEndpointAction
).EndpointActions.expiration = `2021-04-30T16:08:47.449Z`;
await expect(getActionDetailsById(esClient, '123')).resolves.toEqual(
expect.objectContaining({

41
x-pack/plugins/security_solution/server/endpoint/services/actions/action_details_by_id.ts
View file

@ -6,35 +6,33 @@
*/
import { ElasticsearchClient } from '@kbn/core/server';
import { getActionCompletionInfo, mapToNormalizedActionRequest } from './utils';
import { ENDPOINT_ACTIONS_INDEX } from '../../../../common/endpoint/constants';
import {
formatEndpointActionResults,
categorizeResponseResults,
getActionCompletionInfo,
mapToNormalizedActionRequest,
} from './utils';
import type {
ActionDetails,
ActivityLogAction,
ActivityLogActionResponse,
EndpointAction,
EndpointActionResponse,
EndpointActivityLogAction,
EndpointActivityLogActionResponse,
LogsEndpointAction,
LogsEndpointActionResponse,
} from '../../../../common/endpoint/types';
import {
ACTION_REQUEST_INDICES,
ACTION_RESPONSE_INDICES,
catchAndWrapError,
categorizeActionResults,
categorizeResponseResults,
getUniqueLogData,
} from '../../utils';
import { catchAndWrapError, getTimeSortedActionListLogEntries } from '../../utils';
import { EndpointError } from '../../../../common/endpoint/errors';
import { NotFoundError } from '../../errors';
import { ACTIONS_SEARCH_PAGE_SIZE } from './constants';
import { ACTION_RESPONSE_INDICES, ACTIONS_SEARCH_PAGE_SIZE } from './constants';
export const getActionDetailsById = async (
esClient: ElasticsearchClient,
actionId: string
): Promise<ActionDetails> => {
let actionRequestsLogEntries: Array<ActivityLogAction | EndpointActivityLogAction>;
let actionRequestsLogEntries: EndpointActivityLogAction[];
let normalizedActionRequest: ReturnType<typeof mapToNormalizedActionRequest> | undefined;
let actionResponses: Array<ActivityLogActionResponse | EndpointActivityLogActionResponse>;
@ -44,9 +42,9 @@ export const getActionDetailsById = async (
const [actionRequestEsSearchResults, actionResponsesEsSearchResults] = await Promise.all([
// Get the action request(s)
esClient
.search<EndpointAction | LogsEndpointAction>(
.search<LogsEndpointAction>(
{
index: ACTION_REQUEST_INDICES,
index: ENDPOINT_ACTIONS_INDEX,
body: {
query: {
bool: {
@ -84,11 +82,9 @@ export const getActionDetailsById = async (
.catch(catchAndWrapError),
]);
actionRequestsLogEntries = getUniqueLogData(
categorizeActionResults({
results: actionRequestEsSearchResults?.hits?.hits ?? [],
})
) as Array<ActivityLogAction | EndpointActivityLogAction>;
actionRequestsLogEntries = formatEndpointActionResults(
actionRequestEsSearchResults?.hits?.hits ?? []
);
// Multiple Action records could have been returned, but we only really
// need one since they both hold similar data
@ -120,7 +116,10 @@ export const getActionDetailsById = async (
agents: normalizedActionRequest.agents,
command: normalizedActionRequest.command,
startedAt: normalizedActionRequest.createdAt,
logEntries: [...actionRequestsLogEntries, ...actionResponses],
logEntries: getTimeSortedActionListLogEntries([
...actionRequestsLogEntries,
...actionResponses,
]),
isCompleted,
completedAt,
wasSuccessful,

284
x-pack/plugins/security_solution/server/endpoint/services/actions/action_list.test.ts Normal file
View file

@ -0,0 +1,284 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { ElasticsearchClientMock } from '@kbn/core/server/mocks';
import { elasticsearchServiceMock, loggingSystemMock } from '@kbn/core/server/mocks';
import * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import {
EndpointActionResponse,
LogsEndpointAction,
LogsEndpointActionResponse,
} from '../../../../common/endpoint/types';
import { EndpointActionGenerator } from '../../../../common/endpoint/data_generators/endpoint_action_generator';
import { getActionList } from './action_list';
import { CustomHttpRequestError } from '../../../utils/custom_http_request_error';
import {
applyActionListEsSearchMock,
createActionRequestsEsSearchResultsMock,
createActionResponsesEsSearchResultsMock,
} from './mocks';
import { MockedLogger } from '@kbn/logging-mocks';
describe('When using `getActionList()', () => {
let esClient: ElasticsearchClientMock;
let logger: MockedLogger;
let endpointActionGenerator: EndpointActionGenerator;
let actionRequests: estypes.SearchResponse<LogsEndpointAction>;
let actionResponses: estypes.SearchResponse<EndpointActionResponse | LogsEndpointActionResponse>;
beforeEach(() => {
esClient = elasticsearchServiceMock.createScopedClusterClient().asInternalUser;
logger = loggingSystemMock.createLogger();
endpointActionGenerator = new EndpointActionGenerator('seed');
actionRequests = createActionRequestsEsSearchResultsMock();
actionResponses = createActionResponsesEsSearchResultsMock();
applyActionListEsSearchMock(esClient, actionRequests, actionResponses);
});
it('should return expected output', async () => {
await expect(getActionList({ esClient, logger, page: 1, pageSize: 10 })).resolves.toEqual({
page: 0,
pageSize: 10,
commands: undefined,
userIds: undefined,
startDate: undefined,
elasticAgentIds: undefined,
endDate: undefined,
data: [
{
agents: ['agent-a'],
command: 'isolate',
completedAt: '2022-04-30T16:08:47.449Z',
wasSuccessful: true,
errors: undefined,
id: '123',
isCompleted: true,
isExpired: false,
logEntries: [
{
item: {
data: {
'@timestamp': '2022-04-30T16:08:47.449Z',
EndpointActions: {
action_id: '123',
completed_at: '2022-04-30T16:08:47.449Z',
data: {
command: 'unisolate',
comment: '',
},
started_at: expect.any(String),
},
agent: {
id: 'agent-a',
},
error: undefined,
},
id: expect.any(String),
},
type: 'response',
},
{
item: {
data: {
'@timestamp': '2022-04-30T16:08:47.449Z',
action_data: {
command: 'unisolate',
comment: '',
},
action_id: '123',
agent_id: 'agent-a',
completed_at: '2022-04-30T16:08:47.449Z',
error: '',
started_at: expect.any(String),
},
id: expect.any(String),
},
type: 'fleetResponse',
},
{
item: {
data: {
'@timestamp': '2022-04-27T16:08:47.449Z',
EndpointActions: {
action_id: '123',
data: {
command: 'isolate',
comment: '5wb6pu6kh2xix5i',
},
expiration: expect.any(String),
input_type: 'endpoint',
type: 'INPUT_ACTION',
},
agent: { id: 'agent-a' },
user: {
id: expect.any(String),
},
error: undefined,
},
id: expect.any(String),
},
type: 'action',
},
],
startedAt: '2022-04-27T16:08:47.449Z',
},
],
total: 1,
});
});
it('should call query with expected filters when querying for Action Request', async () => {
await getActionList({
esClient,
logger,
elasticAgentIds: ['123'],
pageSize: 20,
startDate: 'now-10d',
endDate: 'now',
commands: ['isolate', 'unisolate', 'get-file'],
userIds: ['elastic'],
});
expect(esClient.search).toHaveBeenNthCalledWith(
1,
expect.objectContaining({
body: {
query: {
bool: {
filter: [
{
term: {
input_type: 'endpoint',
},
},
{
term: {
type: 'INPUT_ACTION',
},
},
{
range: {
'@timestamp': {
gte: 'now-10d',
},
},
},
{
range: {
'@timestamp': {
lte: 'now',
},
},
},
{
terms: {
'data.command': ['isolate', 'unisolate', 'get-file'],
},
},
{
terms: {
user_id: ['elastic'],
},
},
{
terms: {
agents: ['123'],
},
},
],
},
},
sort: [
{
'@timestamp': {
order: 'desc',
},
},
],
},
from: 0,
index: '.logs-endpoint.actions-default',
size: 20,
}),
expect.objectContaining({
ignore: [404],
meta: true,
})
);
});
it('should return an empty array if no actions are found', async () => {
actionRequests.hits.hits = [];
(actionResponses.hits.total as estypes.SearchTotalHits).value = 0;
actionRequests = endpointActionGenerator.toEsSearchResponse([]);
await expect(getActionList({ esClient, logger })).resolves.toEqual(
expect.objectContaining({
commands: undefined,
data: [],
elasticAgentIds: undefined,
endDate: undefined,
page: 0,
pageSize: undefined,
startDate: undefined,
total: 0,
userIds: undefined,
})
);
});
it('should have `isExpired` as `true` if NOT complete and expiration is in the past', async () => {
(
actionRequests.hits.hits[0]._source as LogsEndpointAction
).EndpointActions.expiration = `2021-04-30T16:08:47.449Z`;
actionResponses.hits.hits.pop(); // remove the endpoint response
await expect(
await (
await getActionList({ esClient, logger, elasticAgentIds: ['123'] })
).data[0]
).toEqual(
expect.objectContaining({
isExpired: true,
isCompleted: false,
})
);
});
it('should have `isExpired` as `false` if complete and expiration is in the past', async () => {
(
actionRequests.hits.hits[0]._source as LogsEndpointAction
).EndpointActions.expiration = `2021-04-30T16:08:47.449Z`;
await expect(
await (
await getActionList({ esClient, logger, elasticAgentIds: ['123'] })
).data[0]
).toEqual(
expect.objectContaining({
isExpired: false,
isCompleted: true,
})
);
});
it('should throw custom errors', async () => {
const error = new Error('Some odd error!');
esClient.search.mockImplementation(async () => {
return Promise.reject(error);
});
const getActionListPromise = getActionList({ esClient, logger });
await expect(getActionListPromise).rejects.toThrowError(
'Unknown error while fetching action requests'
);
await expect(getActionListPromise).rejects.toBeInstanceOf(CustomHttpRequestError);
});
});

193
x-pack/plugins/security_solution/server/endpoint/services/actions/action_list.ts Normal file
View file

@ -0,0 +1,193 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { ElasticsearchClient, Logger } from '@kbn/core/server';
import { CustomHttpRequestError } from '../../../utils/custom_http_request_error';
import type { ActionDetails, ActionListApiResponse } from '../../../../common/endpoint/types';
import {
getActions,
getActionResponses,
getTimeSortedActionListLogEntries,
} from '../../utils/action_list_helpers';
import {
formatEndpointActionResults,
categorizeResponseResults,
getActionCompletionInfo,
mapToNormalizedActionRequest,
} from './utils';
interface OptionalFilterParams {
commands?: string[];
elasticAgentIds?: string[];
endDate?: string;
page?: number;
pageSize?: number;
startDate?: string;
userIds?: string[];
}
export const getActionList = async ({
commands,
elasticAgentIds,
esClient,
endDate,
logger,
page: _page,
pageSize,
startDate,
userIds,
}: OptionalFilterParams & {
esClient: ElasticsearchClient;
logger: Logger;
}): Promise<ActionListApiResponse> => {
const size = pageSize ?? 10;
const page = (_page ?? 1) - 1;
// # of hits to skip
const from = page * size;
const data = await getActionDetailsList({
commands,
elasticAgentIds,
esClient,
endDate,
from,
logger,
size,
startDate,
userIds,
});
return {
page,
pageSize,
startDate,
endDate,
elasticAgentIds,
userIds,
commands,
data,
total: data.length,
};
};
export type GetActionDetailsListParam = OptionalFilterParams & {
esClient: ElasticsearchClient;
from: number;
logger: Logger;
size: number;
};
const getActionDetailsList = async ({
commands,
elasticAgentIds,
esClient,
endDate,
from,
logger,
size,
startDate,
userIds,
}: GetActionDetailsListParam): Promise<ActionDetails[]> => {
let actionRequests;
let actionReqIds;
let actionResponses;
try {
// fetch actions with matching agent_ids if any
const { actionIds, actionRequests: _actionRequests } = await getActions({
commands,
esClient,
elasticAgentIds,
startDate,
endDate,
from,
size,
userIds,
});
actionRequests = _actionRequests;
actionReqIds = actionIds;
} catch (error) {
// all other errors
const err = new CustomHttpRequestError(
error.meta?.meta?.body?.error?.reason ?? 'Unknown error while fetching action requests',
error.meta?.meta?.statusCode ?? 500,
error
);
logger.error(err);
throw err;
}
// return empty details array
if (!actionRequests?.body?.hits?.hits) return [];
// format endpoint actions into { type, item } structure
const formattedActionRequests = formatEndpointActionResults(actionRequests?.body?.hits?.hits);
// normalized actions with a flat structure to access relevant values
const normalizedActionRequests: Array<ReturnType<typeof mapToNormalizedActionRequest>> =
formattedActionRequests.map((action) => mapToNormalizedActionRequest(action.item.data));
try {
// get all responses for given action Ids and agent Ids
actionResponses = await getActionResponses({
actionIds: actionReqIds,
elasticAgentIds,
esClient,
});
} catch (error) {
// all other errors
const err = new CustomHttpRequestError(
error.meta?.meta?.body?.error?.reason ?? 'Unknown error while fetching action responses',
error.meta?.meta?.statusCode ?? 500,
error
);
logger.error(err);
throw err;
}
// categorize responses as fleet and endpoint responses
const categorizedResponses = categorizeResponseResults({
results: actionResponses?.body?.hits?.hits,
});
// compute action details list for each action id
const actionDetails: ActionDetails[] = normalizedActionRequests.map((action) => {
// pick only those actions that match the current action id
const matchedActions = formattedActionRequests.filter(
(categorizedAction) => categorizedAction.item.data.EndpointActions.action_id === action.id
);
// pick only those responses that match the current action id
const matchedResponses = categorizedResponses.filter((categorizedResponse) =>
categorizedResponse.type === 'response'
? categorizedResponse.item.data.EndpointActions.action_id === action.id
: categorizedResponse.item.data.action_id === action.id
);
// find the specific response's details using that set of matching responses
const { isCompleted, completedAt, wasSuccessful, errors } = getActionCompletionInfo(
action.agents,
matchedResponses
);
return {
id: action.id,
agents: action.agents,
command: action.command,
startedAt: action.createdAt,
// sort the list by @timestamp in desc order, newest first
logEntries: getTimeSortedActionListLogEntries([...matchedActions, ...matchedResponses]),
isCompleted,
completedAt,
wasSuccessful,
errors,
isExpired: !isCompleted && action.expiration < new Date().toISOString(),
};
});
return actionDetails;
};

57
x-pack/plugins/security_solution/server/endpoint/services/actions/actions.ts
View file

@ -5,13 +5,13 @@
* 2.0.
*/
import { ElasticsearchClient, Logger } from '@kbn/core/server';
import type { ElasticsearchClient, Logger } from '@kbn/core/server';
import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import { TransportResult } from '@elastic/elasticsearch';
import type { TransportResult } from '@elastic/elasticsearch';
import { AGENT_ACTIONS_INDEX, AGENT_ACTIONS_RESULTS_INDEX } from '@kbn/fleet-plugin/common';
import { ENDPOINT_ACTION_RESPONSES_INDEX_PATTERN } from '../../../../common/endpoint/constants';
import { SecuritySolutionRequestHandlerContext } from '../../../types';
import {
import type { SecuritySolutionRequestHandlerContext } from '../../../types';
import type {
ActivityLog,
ActivityLogEntry,
EndpointAction,
@ -22,16 +22,22 @@ import {
} from '../../../../common/endpoint/types';
import {
catchAndWrapError,
categorizeActionResults,
categorizeResponseResults,
getActionRequestsResult,
getActionResponsesResult,
getTimeSortedData,
getUniqueLogData,
} from '../../utils';
import { EndpointMetadataService } from '../metadata';
import type { EndpointMetadataService } from '../metadata';
import { ACTIONS_SEARCH_PAGE_SIZE } from './constants';
import {
categorizeActionResults,
categorizeResponseResults,
hasAckInResponse,
getUniqueLogData,
hasNoEndpointResponse,
hasNoFleetResponse,
} from './utils';
const PENDING_ACTION_RESPONSE_MAX_LAPSED_TIME = 300000; // 300k ms === 5 minutes
export const getAuditLogResponse = async ({
@ -148,41 +154,6 @@ const getActivityLog = async ({
return sortedData;
};
const hasAckInResponse = (response: EndpointActionResponse): boolean => {
return response.action_response?.endpoint?.ack ?? false;
};
// return TRUE if for given action_id/agent_id
// there is no doc in .logs-endpoint.action.response-default
const hasNoEndpointResponse = ({
action,
agentId,
indexedActionIds,
}: {
action: EndpointAction;
agentId: string;
indexedActionIds: string[];
}): boolean => {
return action.agents.includes(agentId) && !indexedActionIds.includes(action.action_id);
};
// return TRUE if for given action_id/agent_id
// there is no doc in .fleet-actions-results
const hasNoFleetResponse = ({
action,
agentId,
agentResponses,
}: {
action: EndpointAction;
agentId: string;
agentResponses: EndpointActionResponse[];
}): boolean => {
return (
action.agents.includes(agentId) &&
!agentResponses.map((e) => e.action_id).includes(action.action_id)
);
};
export const getPendingActionCounts = async (
esClient: ElasticsearchClient,
metadataService: EndpointMetadataService,

12
x-pack/plugins/security_solution/server/endpoint/services/actions/constants.ts
View file

@ -5,7 +5,19 @@
* 2.0.
*/
import { AGENT_ACTIONS_INDEX, AGENT_ACTIONS_RESULTS_INDEX } from '@kbn/fleet-plugin/common';
import {
ENDPOINT_ACTIONS_INDEX,
ENDPOINT_ACTION_RESPONSES_INDEX_PATTERN,
} from '../../../../common/endpoint/constants';
/**
* The Page Size to be used when searching against the Actions indexes (both requests and responses)
*/
export const ACTIONS_SEARCH_PAGE_SIZE = 10000;
export const ACTION_REQUEST_INDICES = [AGENT_ACTIONS_INDEX, ENDPOINT_ACTIONS_INDEX];
// search all responses indices irrelevant of namespace
export const ACTION_RESPONSE_INDICES = [
AGENT_ACTIONS_RESULTS_INDEX,
ENDPOINT_ACTION_RESPONSES_INDEX_PATTERN,
];

1
x-pack/plugins/security_solution/server/endpoint/services/actions/index.ts
View file

@ -7,3 +7,4 @@
export * from './actions';
export { getActionDetailsById } from './action_details_by_id';
export { getActionList } from './action_list';

76
x-pack/plugins/security_solution/server/endpoint/services/actions/mocks.ts
View file

@ -7,11 +7,10 @@
import * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import { ElasticsearchClientMock } from '@kbn/core/server/mocks';
import { AGENT_ACTIONS_INDEX, AGENT_ACTIONS_RESULTS_INDEX } from '@kbn/fleet-plugin/common';
import { AGENT_ACTIONS_RESULTS_INDEX } from '@kbn/fleet-plugin/common';
import { EndpointActionGenerator } from '../../../../common/endpoint/data_generators/endpoint_action_generator';
import { FleetActionGenerator } from '../../../../common/endpoint/data_generators/fleet_action_generator';
import {
EndpointAction,
EndpointActionResponse,
LogsEndpointAction,
LogsEndpointActionResponse,
@ -21,25 +20,18 @@ import {
ENDPOINT_ACTIONS_INDEX,
} from '../../../../common/endpoint/constants';
export const createActionRequestsEsSearchResultsMock = (): estypes.SearchResponse<
EndpointAction | LogsEndpointAction
> => {
const endpointActionGenerator = new EndpointActionGenerator('seed');
const fleetActionGenerator = new FleetActionGenerator('seed');
export const createActionRequestsEsSearchResultsMock =
(): estypes.SearchResponse<LogsEndpointAction> => {
const endpointActionGenerator = new EndpointActionGenerator('seed');
return endpointActionGenerator.toEsSearchResponse<EndpointAction | LogsEndpointAction>([
fleetActionGenerator.generateActionEsHit({
action_id: '123',
agents: ['agent-a'],
'@timestamp': '2022-04-27T16:08:47.449Z',
}),
endpointActionGenerator.generateActionEsHit({
EndpointActions: { action_id: '123' },
agent: { id: 'agent-a' },
'@timestamp': '2022-04-27T16:08:47.449Z',
}),
]);
};
return endpointActionGenerator.toEsSearchResponse<LogsEndpointAction>([
endpointActionGenerator.generateActionEsHit({
EndpointActions: { action_id: '123' },
agent: { id: 'agent-a' },
'@timestamp': '2022-04-27T16:08:47.449Z',
}),
]);
};
export const createActionResponsesEsSearchResultsMock = (): estypes.SearchResponse<
LogsEndpointActionResponse | EndpointActionResponse
@ -73,9 +65,7 @@ export const createActionResponsesEsSearchResultsMock = (): estypes.SearchRespon
*/
export const applyActionsEsSearchMock = (
esClient: ElasticsearchClientMock,
actionRequests: estypes.SearchResponse<
EndpointAction | LogsEndpointAction
> = createActionRequestsEsSearchResultsMock(),
actionRequests: estypes.SearchResponse<LogsEndpointAction> = createActionRequestsEsSearchResultsMock(),
actionResponses: estypes.SearchResponse<
LogsEndpointActionResponse | EndpointActionResponse
> = createActionResponsesEsSearchResultsMock()
@ -86,7 +76,7 @@ export const applyActionsEsSearchMock = (
const params = args[0] ?? {};
const indexes = Array.isArray(params.index) ? params.index : [params.index];
if (indexes.includes(AGENT_ACTIONS_INDEX) || indexes.includes(ENDPOINT_ACTIONS_INDEX)) {
if (indexes.includes(ENDPOINT_ACTIONS_INDEX)) {
return actionRequests;
} else if (
indexes.includes(AGENT_ACTIONS_RESULTS_INDEX) ||
@ -102,3 +92,41 @@ export const applyActionsEsSearchMock = (
return new EndpointActionGenerator().toEsSearchResponse([]);
});
};
/**
* Applies a mock implementation to the `esClient.search()` method that will return action requests or responses
* depending on what indexes the `.search()` was called with.
* @param esClient
* @param actionRequests
* @param actionResponses
*/
export const applyActionListEsSearchMock = (
esClient: ElasticsearchClientMock,
actionRequests: estypes.SearchResponse<LogsEndpointAction> = createActionRequestsEsSearchResultsMock(),
actionResponses: estypes.SearchResponse<
LogsEndpointActionResponse | EndpointActionResponse
> = createActionResponsesEsSearchResultsMock()
) => {
const priorSearchMockImplementation = esClient.search.getMockImplementation();
// @ts-expect-error incorrect type
esClient.search.mockImplementation(async (...args) => {
const params = args[0] ?? {};
const indexes = Array.isArray(params.index) ? params.index : [params.index];
if (indexes.includes(ENDPOINT_ACTIONS_INDEX)) {
return { body: { ...actionRequests } };
} else if (
indexes.includes(AGENT_ACTIONS_RESULTS_INDEX) ||
indexes.includes(ENDPOINT_ACTION_RESPONSES_INDEX_PATTERN)
) {
return { body: { ...actionResponses } };
}
if (priorSearchMockImplementation) {
return priorSearchMockImplementation(...args);
}
return new EndpointActionGenerator().toEsSearchResponse([]);
});
};

303
x-pack/plugins/security_solution/server/endpoint/services/actions/utils.test.ts
View file

@ -5,18 +5,31 @@
* 2.0.
*/
import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import { EndpointActionGenerator } from '../../../../common/endpoint/data_generators/endpoint_action_generator';
import { FleetActionGenerator } from '../../../../common/endpoint/data_generators/fleet_action_generator';
import {
categorizeActionResults,
categorizeResponseResults,
formatEndpointActionResults,
getUniqueLogData,
getActionCompletionInfo,
isLogsEndpointAction,
isLogsEndpointActionResponse,
mapToNormalizedActionRequest,
} from './utils';
import type {
ActivityLogAction,
ActivityLogActionResponse,
EndpointAction,
EndpointActionResponse,
EndpointActivityLogAction,
EndpointActivityLogActionResponse,
LogsEndpointAction,
LogsEndpointActionResponse,
} from '../../../../common/endpoint/types';
import uuid from 'uuid';
import { mockAuditLogSearchResult, Results } from '../../routes/actions/mocks';
describe('When using Actions service utilities', () => {
let fleetActionGenerator: FleetActionGenerator;
@ -58,7 +71,7 @@ describe('When using Actions service utilities', () => {
).toEqual({
agents: ['6e6796b0-af39-4f12-b025-fcb06db499e5'],
command: 'isolate',
comment: 'isolate',
comment: expect.any(String),
createdAt: '2022-04-27T16:08:47.449Z',
createdBy: 'elastic',
expiration: '2022-04-29T16:08:47.449Z',
@ -77,7 +90,7 @@ describe('When using Actions service utilities', () => {
).toEqual({
agents: ['90d62689-f72d-4a05-b5e3-500cad0dc366'],
command: 'isolate',
comment: 'isolate',
comment: expect.any(String),
createdAt: '2022-04-27T16:08:47.449Z',
createdBy: 'Shanel',
expiration: '2022-05-10T16:08:47.449Z',
@ -118,22 +131,16 @@ describe('When using Actions service utilities', () => {
});
it('should show complete as `true` with completion date if Endpoint Response received', () => {
expect(
getActionCompletionInfo(
['123'],
[
endpointActionGenerator.generateActivityLogActionResponse({
item: {
data: {
'@timestamp': COMPLETED_AT,
agent: { id: '123' },
EndpointActions: { completed_at: COMPLETED_AT },
},
},
}),
]
)
).toEqual({
const endpointResponse = endpointActionGenerator.generateActivityLogActionResponse({
item: {
data: {
'@timestamp': COMPLETED_AT,
agent: { id: '123' },
EndpointActions: { completed_at: COMPLETED_AT },
},
},
});
expect(getActionCompletionInfo(['123'], [endpointResponse])).toEqual({
isCompleted: true,
completedAt: COMPLETED_AT,
errors: undefined,
@ -146,8 +153,11 @@ describe('When using Actions service utilities', () => {
let endpointResponseAtError: EndpointActivityLogActionResponse;
beforeEach(() => {
const actionId = uuid.v4();
fleetResponseAtError = fleetActionGenerator.generateActivityLogActionResponse({
item: { data: { agent_id: '123', error: 'agent failed to deliver' } },
item: {
data: { agent_id: '123', action_id: actionId, error: 'agent failed to deliver' },
},
});
endpointResponseAtError = endpointActionGenerator.generateActivityLogActionResponse({
@ -159,6 +169,7 @@ describe('When using Actions service utilities', () => {
message: 'endpoint failed to apply',
},
EndpointActions: {
action_id: actionId,
completed_at: '2022-05-18T13:03:54.756Z',
},
},
@ -201,22 +212,24 @@ describe('When using Actions service utilities', () => {
describe('with multiple agent ids', () => {
let agentIds: string[];
let actionId: string;
let action123Responses: Array<ActivityLogActionResponse | EndpointActivityLogActionResponse>;
let action456Responses: Array<ActivityLogActionResponse | EndpointActivityLogActionResponse>;
let action789Responses: Array<ActivityLogActionResponse | EndpointActivityLogActionResponse>;
beforeEach(() => {
agentIds = ['123', '456', '789'];
actionId = uuid.v4();
action123Responses = [
fleetActionGenerator.generateActivityLogActionResponse({
item: { data: { agent_id: '123', error: '' } },
item: { data: { agent_id: '123', error: '', action_id: actionId } },
}),
endpointActionGenerator.generateActivityLogActionResponse({
item: {
data: {
'@timestamp': '2022-01-05T19:27:23.816Z',
agent: { id: '123' },
EndpointActions: { completed_at: '2022-01-05T19:27:23.816Z' },
EndpointActions: { action_id: actionId, completed_at: '2022-01-05T19:27:23.816Z' },
},
},
}),
@ -224,14 +237,14 @@ describe('When using Actions service utilities', () => {
action456Responses = [
fleetActionGenerator.generateActivityLogActionResponse({
item: { data: { agent_id: '456', error: '' } },
item: { data: { action_id: actionId, agent_id: '456', error: '' } },
}),
endpointActionGenerator.generateActivityLogActionResponse({
item: {
data: {
'@timestamp': COMPLETED_AT,
agent: { id: '456' },
EndpointActions: { completed_at: COMPLETED_AT },
EndpointActions: { action_id: actionId, completed_at: COMPLETED_AT },
},
},
}),
@ -239,14 +252,14 @@ describe('When using Actions service utilities', () => {
action789Responses = [
fleetActionGenerator.generateActivityLogActionResponse({
item: { data: { agent_id: '789', error: '' } },
item: { data: { action_id: actionId, agent_id: '789', error: '' } },
}),
endpointActionGenerator.generateActivityLogActionResponse({
item: {
data: {
'@timestamp': '2022-03-05T19:27:23.816Z',
agent: { id: '789' },
EndpointActions: { completed_at: '2022-03-05T19:27:23.816Z' },
EndpointActions: { action_id: actionId, completed_at: '2022-03-05T19:27:23.816Z' },
},
},
}),
@ -257,7 +270,7 @@ describe('When using Actions service utilities', () => {
expect(getActionCompletionInfo(agentIds, [])).toEqual(NOT_COMPLETED_OUTPUT);
});
it('should complete as `false` if at least one agent id is has not received a response', () => {
it('should complete as `false` if at least one agent id has not received a response', () => {
expect(
getActionCompletionInfo(agentIds, [
...action123Responses,
@ -307,4 +320,242 @@ describe('When using Actions service utilities', () => {
});
});
});
describe('#getUniqueLogData()', () => {
let actionRequests123: Array<ActivityLogAction | EndpointActivityLogAction>;
let actionResponses123: Array<ActivityLogActionResponse | EndpointActivityLogActionResponse>;
let errorActionRequests: Array<ActivityLogAction | EndpointActivityLogAction>;
let errorResponses: Array<ActivityLogActionResponse | EndpointActivityLogActionResponse>;
beforeEach(() => {
const actionId0 = uuid.v4();
const actionId1 = uuid.v4();
actionRequests123 = [
fleetActionGenerator.generateActivityLogAction({
item: {
data: { agents: ['123'], action_id: actionId0 },
},
}),
endpointActionGenerator.generateActivityLogAction({
item: {
data: { agent: { id: '123' }, EndpointActions: { action_id: actionId0 } },
},
}),
fleetActionGenerator.generateActivityLogAction({
item: {
data: { agents: ['123'], action_id: actionId1 },
},
}),
endpointActionGenerator.generateActivityLogAction({
item: {
data: { agent: { id: '123' }, EndpointActions: { action_id: actionId1 } },
},
}),
];
actionResponses123 = [
fleetActionGenerator.generateActivityLogActionResponse({
item: {
data: { agent_id: '123', action_id: actionId0 },
},
}),
endpointActionGenerator.generateActivityLogActionResponse({
item: {
data: {
agent: { id: '123' },
EndpointActions: {
action_id: actionId0,
completed_at: new Date().toISOString(),
},
},
},
}),
];
errorActionRequests = [
endpointActionGenerator.generateActivityLogAction({
item: {
data: {
agent: { id: '456' },
EndpointActions: { action_id: actionId0 },
error: { message: 'Did not deliver to Fleet' },
},
},
}),
fleetActionGenerator.generateActivityLogAction({
item: {
data: { agents: ['456'], action_id: actionId1 },
},
}),
endpointActionGenerator.generateActivityLogAction({
item: {
data: {
agent: { id: '456' },
EndpointActions: { action_id: actionId1 },
},
},
}),
];
errorResponses = [
endpointActionGenerator.generateActivityLogActionResponse({
item: {
data: {
agent: { id: '456' },
EndpointActions: {
action_id: actionId0,
completed_at: new Date().toISOString(),
},
error: { code: '424', message: 'Did not deliver to Fleet' },
},
},
}),
fleetActionGenerator.generateActivityLogActionResponse({
item: {
data: { agent_id: '456', action_id: actionId1, error: 'some other error!' },
},
}),
endpointActionGenerator.generateActivityLogActionResponse({
item: {
data: {
agent: { id: '456' },
EndpointActions: {
action_id: actionId1,
completed_at: new Date().toISOString(),
},
error: { message: 'some other error!' },
},
},
}),
];
});
it('should exclude endpoint actions on successful actions', () => {
const uniqueData = getUniqueLogData([...actionRequests123, ...actionResponses123]);
expect(uniqueData.length).toEqual(4);
expect(uniqueData.find((u) => u.type === 'action')).toEqual(undefined);
});
it('should include endpoint action when no fleet response but endpoint response with error', () => {
const uniqueData = getUniqueLogData([...errorActionRequests, ...errorResponses]);
expect(uniqueData.length).toEqual(5);
});
});
describe('#categorizeActionResults(), #categorizeResponseResults() and #formatEndpointActionResults()', () => {
let fleetActions: Results[];
let endpointActions: Results[];
let fleetResponses: Results[];
let endpointResponses: Results[];
beforeEach(() => {
const agents = ['agent-id'];
const actionIds = [uuid.v4(), uuid.v4()];
fleetActions = actionIds.map((id) => {
return {
_index: '.fleet-actions-7',
_source: fleetActionGenerator.generate({
agents,
action_id: id,
}),
};
});
endpointActions = actionIds.map((id) => {
return {
_index: '.ds-.logs-endpoint.actions-default-2021.19.10-000001',
_source: endpointActionGenerator.generate({
agent: { id: agents[0] },
EndpointActions: {
action_id: id,
},
}),
};
});
fleetResponses = actionIds.map((id) => {
return {
_index: '.ds-.fleet-actions-results-2021.19.10-000001',
_source: fleetActionGenerator.generate({
agents,
action_id: id,
}),
};
});
endpointResponses = actionIds.map((id) => {
return {
_index: '.ds-.logs-endpoint.action.responses-default-2021.19.10-000001',
_source: endpointActionGenerator.generate({
agent: { id: agents[0] },
EndpointActions: {
action_id: id,
},
}),
};
});
});
it('should correctly categorize fleet actions and endpoint actions', () => {
const actionResults = mockAuditLogSearchResult([...fleetActions, ...endpointActions]);
const categorized = categorizeActionResults({
results: actionResults.body.hits.hits as Array<
estypes.SearchHit<EndpointAction | LogsEndpointAction>
>,
});
const categorizedActions = categorized.filter((e) => e.type === 'action');
const categorizedFleetActions = categorized.filter((e) => e.type === 'fleetAction');
expect(categorizedActions.length).toEqual(2);
expect(categorizedFleetActions.length).toEqual(2);
expect(
(categorizedActions as EndpointActivityLogAction[]).map(
(e) => 'EndpointActions' in e.item.data
)[0]
).toBeTruthy();
expect(
(categorizedFleetActions as ActivityLogAction[]).map((e) => 'data' in e.item.data)[0]
).toBeTruthy();
});
it('should correctly categorize fleet responses and endpoint responses', () => {
const actionResponses = mockAuditLogSearchResult([...fleetResponses, ...endpointResponses]);
const categorized = categorizeResponseResults({
results: actionResponses.body.hits.hits as Array<
estypes.SearchHit<EndpointActionResponse | LogsEndpointActionResponse>
>,
});
const categorizedResponses = categorized.filter((e) => e.type === 'response');
const categorizedFleetResponses = categorized.filter((e) => e.type === 'fleetResponse');
expect(categorizedResponses.length).toEqual(2);
expect(categorizedFleetResponses.length).toEqual(2);
expect(
(categorizedResponses as EndpointActivityLogActionResponse[]).map(
(e) => 'EndpointActions' in e.item.data
)[0]
).toBeTruthy();
expect(
(categorizedFleetResponses as ActivityLogActionResponse[]).map(
(e) => 'data' in e.item.data
)[0]
).toBeTruthy();
});
it('should correctly format endpoint actions', () => {
const actionResults = mockAuditLogSearchResult(endpointActions);
const formattedActions = formatEndpointActionResults(
actionResults.body.hits.hits as Array<estypes.SearchHit<LogsEndpointAction>>
);
const formattedActionRequests = formattedActions.filter((e) => e.type === 'action');
expect(formattedActionRequests.length).toEqual(2);
expect(
(formattedActionRequests as EndpointActivityLogAction[]).map(
(e) => 'EndpointActions' in e.item.data
)[0]
).toBeTruthy();
});
});
});

313
x-pack/plugins/security_solution/server/endpoint/services/actions/utils.ts
View file

@ -5,15 +5,24 @@
* 2.0.
*/
import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import {
ENDPOINT_ACTIONS_DS,
ENDPOINT_ACTION_RESPONSES_DS,
failedFleetActionErrorCode,
} from '../../../../common/endpoint/constants';
import type {
ActivityLogAction,
ActivityLogActionResponse,
ActivityLogEntry,
EndpointAction,
EndpointActionResponse,
EndpointActivityLogAction,
EndpointActivityLogActionResponse,
LogsEndpointAction,
LogsEndpointActionResponse,
} from '../../../../common/endpoint/types';
import { ActivityLogItemTypes } from '../../../../common/endpoint/types';
/**
* Type guard to check if a given Action is in the shape of the Endpoint Action.
* @param item
@ -54,18 +63,19 @@ interface NormalizedActionRequest {
export const mapToNormalizedActionRequest = (
actionRequest: EndpointAction | LogsEndpointAction
): NormalizedActionRequest => {
const type = 'ACTION_REQUEST';
if (isLogsEndpointAction(actionRequest)) {
return {
agents: Array.isArray(actionRequest.agent.id)
? actionRequest.agent.id
: [actionRequest.agent.id],
command: actionRequest.EndpointActions.data.command,
comment: actionRequest.EndpointActions.data.command,
type: 'ACTION_REQUEST',
id: actionRequest.EndpointActions.action_id,
expiration: actionRequest.EndpointActions.expiration,
comment: actionRequest.EndpointActions.data.comment,
createdBy: actionRequest.user.id,
createdAt: actionRequest['@timestamp'],
expiration: actionRequest.EndpointActions.expiration,
id: actionRequest.EndpointActions.action_id,
type,
};
}
@ -73,12 +83,12 @@ export const mapToNormalizedActionRequest = (
return {
agents: actionRequest.agents,
command: actionRequest.data.command,
comment: actionRequest.data.command,
type: 'ACTION_REQUEST',
id: actionRequest.action_id,
expiration: actionRequest.expiration,
comment: actionRequest.data.comment,
createdBy: actionRequest.user_id,
createdAt: actionRequest['@timestamp'],
expiration: actionRequest.expiration,
id: actionRequest.action_id,
type,
};
};
@ -96,13 +106,13 @@ export const getActionCompletionInfo = (
actionResponses: Array<ActivityLogActionResponse | EndpointActivityLogActionResponse>
): ActionCompletionInfo => {
const completedInfo: ActionCompletionInfo = {
isCompleted: Boolean(agentIds.length),
completedAt: undefined,
wasSuccessful: Boolean(agentIds.length),
errors: undefined,
isCompleted: Boolean(agentIds.length),
wasSuccessful: Boolean(agentIds.length),
};
const responsesByAgentId = mapActionResponsesByAgentId(actionResponses);
const responsesByAgentId: ActionResponseByAgentId = mapActionResponsesByAgentId(actionResponses);
for (const agentId of agentIds) {
if (!responsesByAgentId[agentId] || !responsesByAgentId[agentId].isCompleted) {
@ -153,7 +163,7 @@ type ActionResponseByAgentId = Record<string, NormalizedAgentActionResponse>;
/**
* Given a list of Action Responses, it will return a Map where keys are the Agent ID and
* value is a object having information about the action response's associated with that agent id
* value is a object having information about the action responses associated with that agent id
* @param actionResponses
*/
const mapActionResponsesByAgentId = (
@ -162,75 +172,74 @@ const mapActionResponsesByAgentId = (
const response: ActionResponseByAgentId = {};
for (const actionResponse of actionResponses) {
if (actionResponse.type === 'fleetResponse' || actionResponse.type === 'response') {
const agentId = getAgentIdFromActionResponse(actionResponse);
let thisAgentActionResponses = response[agentId];
const agentId = getAgentIdFromActionResponse(actionResponse);
let thisAgentActionResponses = response[agentId];
if (!thisAgentActionResponses) {
response[agentId] = {
isCompleted: false,
completedAt: undefined,
wasSuccessful: false,
errors: undefined,
fleetResponse: undefined,
endpointResponse: undefined,
};
if (!thisAgentActionResponses) {
response[agentId] = {
isCompleted: false,
completedAt: undefined,
wasSuccessful: false,
errors: undefined,
fleetResponse: undefined,
endpointResponse: undefined,
};
thisAgentActionResponses = response[agentId];
thisAgentActionResponses = response[agentId];
}
if (actionResponse.type === 'fleetResponse') {
thisAgentActionResponses.fleetResponse = actionResponse;
} else {
thisAgentActionResponses.endpointResponse = actionResponse;
}
thisAgentActionResponses.isCompleted =
// Action is complete if an Endpoint Action Response was received
Boolean(thisAgentActionResponses.endpointResponse) ||
// OR:
// If we did not have an endpoint response and the Fleet response has `error`, then
// action is complete. Elastic Agent was unable to deliver the action request to the
// endpoint, so we are unlikely to ever receive an Endpoint Response.
Boolean(thisAgentActionResponses.fleetResponse?.item.data.error);
// When completed, calculate additional properties about the action
if (thisAgentActionResponses.isCompleted) {
if (thisAgentActionResponses.endpointResponse) {
thisAgentActionResponses.completedAt =
thisAgentActionResponses.endpointResponse?.item.data['@timestamp'];
thisAgentActionResponses.wasSuccessful = true;
} else if (
// Check if perhaps the Fleet action response returned an error, in which case, the Fleet Agent
// failed to deliver the Action to the Endpoint. If that's the case, we are not going to get
// a Response from endpoint, thus mark the Action as completed and use the Fleet Message's
// timestamp for the complete data/time.
thisAgentActionResponses.fleetResponse &&
thisAgentActionResponses.fleetResponse.item.data.error
) {
thisAgentActionResponses.isCompleted = true;
thisAgentActionResponses.completedAt =
thisAgentActionResponses.fleetResponse.item.data['@timestamp'];
}
if (actionResponse.type === 'fleetResponse') {
thisAgentActionResponses.fleetResponse = actionResponse;
} else {
thisAgentActionResponses.endpointResponse = actionResponse;
const errors: NormalizedAgentActionResponse['errors'] = [];
// only one of the errors should be in there
if (thisAgentActionResponses.endpointResponse?.item.data.error?.message) {
errors.push(
`Endpoint action response error: ${thisAgentActionResponses.endpointResponse.item.data.error.message}`
);
}
thisAgentActionResponses.isCompleted =
// Action is complete if an Endpoint Action Response was received
Boolean(thisAgentActionResponses.endpointResponse) ||
// OR:
// If we did not have an endpoint response and the Fleet response has `error`, then
// action is complete. Elastic Agent was unable to deliver the action request to the
// endpoint, so we are unlikely to ever receive an Endpoint Response.
Boolean(thisAgentActionResponses.fleetResponse?.item.data.error);
if (thisAgentActionResponses.fleetResponse?.item.data.error) {
errors.push(
`Fleet action response error: ${thisAgentActionResponses.fleetResponse?.item.data.error}`
);
}
// When completed, calculate additional properties about the action
if (thisAgentActionResponses.isCompleted) {
if (thisAgentActionResponses.endpointResponse) {
thisAgentActionResponses.completedAt =
thisAgentActionResponses.endpointResponse?.item.data['@timestamp'];
thisAgentActionResponses.wasSuccessful = true;
} else if (
// Check if perhaps the Fleet action response returned an error, in which case, the Fleet Agent
// failed to deliver the Action to the Endpoint. If that's the case, we are not going to get
// a Response from endpoint, thus mark the Action as completed and use the Fleet Message's
// timestamp for the complete data/time.
thisAgentActionResponses.fleetResponse &&
thisAgentActionResponses.fleetResponse.item.data.error
) {
thisAgentActionResponses.isCompleted = true;
thisAgentActionResponses.completedAt =
thisAgentActionResponses.fleetResponse.item.data['@timestamp'];
}
const errors: NormalizedAgentActionResponse['errors'] = [];
if (thisAgentActionResponses.endpointResponse?.item.data.error?.message) {
errors.push(
`Endpoint action response error: ${thisAgentActionResponses.endpointResponse.item.data.error.message}`
);
}
if (thisAgentActionResponses.fleetResponse?.item.data.error) {
errors.push(
`Fleet action response error: ${thisAgentActionResponses.fleetResponse?.item.data.error}`
);
}
if (errors.length) {
thisAgentActionResponses.wasSuccessful = false;
thisAgentActionResponses.errors = errors;
}
if (errors.length) {
thisAgentActionResponses.wasSuccessful = false;
thisAgentActionResponses.errors = errors;
}
}
}
@ -253,3 +262,157 @@ const getAgentIdFromActionResponse = (
return responseData.agent_id;
};
// common helpers used by old and new log API
export const getDateFilters = ({
startDate,
endDate,
}: {
startDate?: string;
endDate?: string;
}) => {
const dateFilters = [];
if (startDate) {
dateFilters.push({ range: { '@timestamp': { gte: startDate } } });
}
if (endDate) {
dateFilters.push({ range: { '@timestamp': { lte: endDate } } });
}
return dateFilters;
};
export const getUniqueLogData = (activityLogEntries: ActivityLogEntry[]): ActivityLogEntry[] => {
// find the error responses for actions that didn't make it to fleet index
const onlyResponsesForFleetErrors: string[] = activityLogEntries.reduce<string[]>((acc, curr) => {
if (
curr.type === ActivityLogItemTypes.RESPONSE &&
curr.item.data.error?.code === failedFleetActionErrorCode
) {
acc.push(curr.item.data.EndpointActions.action_id);
}
return acc;
}, []);
// all actions and responses minus endpoint actions.
const nonEndpointActionsDocs = activityLogEntries.filter(
(e) => e.type !== ActivityLogItemTypes.ACTION
);
// only endpoint actions that match the error responses
const onlyEndpointActionsDocWithoutFleetActions: ActivityLogEntry[] = activityLogEntries.filter(
(e) =>
e.type === ActivityLogItemTypes.ACTION &&
onlyResponsesForFleetErrors.includes(
(e.item.data as LogsEndpointAction).EndpointActions.action_id
)
);
// join the error actions and the rest
return [...nonEndpointActionsDocs, ...onlyEndpointActionsDocWithoutFleetActions];
};
export const hasAckInResponse = (response: EndpointActionResponse): boolean => {
return response.action_response?.endpoint?.ack ?? false;
};
// return TRUE if for given action_id/agent_id
// there is no doc in .logs-endpoint.action.response-default
export const hasNoEndpointResponse = ({
action,
agentId,
indexedActionIds,
}: {
action: EndpointAction;
agentId: string;
indexedActionIds: string[];
}): boolean => {
return action.agents.includes(agentId) && !indexedActionIds.includes(action.action_id);
};
// return TRUE if for given action_id/agent_id
// there is no doc in .fleet-actions-results
export const hasNoFleetResponse = ({
action,
agentId,
agentResponses,
}: {
action: EndpointAction;
agentId: string;
agentResponses: EndpointActionResponse[];
}): boolean => {
return (
action.agents.includes(agentId) &&
!agentResponses.map((e) => e.action_id).includes(action.action_id)
);
};
const matchesDsNamePattern = ({
dataStreamName,
index,
}: {
dataStreamName: string;
index: string;
}): boolean => index.includes(dataStreamName);
export const categorizeResponseResults = ({
results,
}: {
results: Array<estypes.SearchHit<EndpointActionResponse | LogsEndpointActionResponse>>;
}): Array<ActivityLogActionResponse | EndpointActivityLogActionResponse> => {
return results?.length
? results?.map((e) => {
const isResponseDoc: boolean = matchesDsNamePattern({
dataStreamName: ENDPOINT_ACTION_RESPONSES_DS,
index: e._index,
});
return isResponseDoc
? {
type: ActivityLogItemTypes.RESPONSE,
item: { id: e._id, data: e._source as LogsEndpointActionResponse },
}
: {
type: ActivityLogItemTypes.FLEET_RESPONSE,
item: { id: e._id, data: e._source as EndpointActionResponse },
};
})
: [];
};
export const categorizeActionResults = ({
results,
}: {
results: Array<estypes.SearchHit<EndpointAction | LogsEndpointAction>>;
}): Array<ActivityLogAction | EndpointActivityLogAction> => {
return results?.length
? results?.map((e) => {
const isActionDoc: boolean = matchesDsNamePattern({
dataStreamName: ENDPOINT_ACTIONS_DS,
index: e._index,
});
return isActionDoc
? {
type: ActivityLogItemTypes.ACTION,
item: { id: e._id, data: e._source as LogsEndpointAction },
}
: {
type: ActivityLogItemTypes.FLEET_ACTION,
item: { id: e._id, data: e._source as EndpointAction },
};
})
: [];
};
// for 8.4+ we only search on endpoint actions index
// and thus there are only endpoint actions in the results
export const formatEndpointActionResults = (
results: Array<estypes.SearchHit<LogsEndpointAction>>
): EndpointActivityLogAction[] => {
return results?.length
? results?.map((e) => {
return {
type: ActivityLogItemTypes.ACTION,
item: { id: e._id, data: e._source as LogsEndpointAction },
};
})
: [];
};

258
x-pack/plugins/security_solution/server/endpoint/utils/action_list_helpers.test.ts Normal file
View file

@ -0,0 +1,258 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import uuid from 'uuid';
import { ScopedClusterClientMock, elasticsearchServiceMock } from '@kbn/core/server/mocks';
import {
applyActionListEsSearchMock,
createActionRequestsEsSearchResultsMock,
createActionResponsesEsSearchResultsMock,
} from '../services/actions/mocks';
import { getActions, getActionResponses } from './action_list_helpers';
describe('action helpers', () => {
let mockScopedEsClient: ScopedClusterClientMock;
beforeEach(() => {
mockScopedEsClient = elasticsearchServiceMock.createScopedClusterClient();
});
describe('#getActions', () => {
it('should call with base filter query correctly when no other filter options provided', async () => {
const esClient = mockScopedEsClient.asInternalUser;
applyActionListEsSearchMock(esClient);
await getActions({ esClient, size: 10, from: 0 });
expect(esClient.search).toHaveBeenCalledWith(
{
body: {
query: {
bool: {
filter: [
{
term: {
input_type: 'endpoint',
},
},
{
term: {
type: 'INPUT_ACTION',
},
},
],
},
},
sort: [
{
'@timestamp': {
order: 'desc',
},
},
],
},
from: 0,
index: '.logs-endpoint.actions-default',
size: 10,
},
{
ignore: [404],
meta: true,
}
);
});
it('should query with additional filter options provided', async () => {
const esClient = mockScopedEsClient.asInternalUser;
applyActionListEsSearchMock(esClient);
await getActions({
esClient,
size: 20,
from: 5,
startDate: 'now-10d',
elasticAgentIds: ['agent-123', 'agent-456'],
endDate: 'now',
commands: ['isolate', 'unisolate', 'get-file'],
userIds: ['elastic'],
});
expect(esClient.search).toHaveBeenCalledWith(
{
body: {
query: {
bool: {
filter: [
{
term: {
input_type: 'endpoint',
},
},
{
term: {
type: 'INPUT_ACTION',
},
},
{
range: {
'@timestamp': {
gte: 'now-10d',
},
},
},
{
range: {
'@timestamp': {
lte: 'now',
},
},
},
{
terms: {
'data.command': ['isolate', 'unisolate', 'get-file'],
},
},
{
terms: {
user_id: ['elastic'],
},
},
{
terms: {
agents: ['agent-123', 'agent-456'],
},
},
],
},
},
sort: [
{
'@timestamp': {
order: 'desc',
},
},
],
},
from: 5,
index: '.logs-endpoint.actions-default',
size: 20,
},
{
ignore: [404],
meta: true,
}
);
});
it('should return expected output', async () => {
const esClient = mockScopedEsClient.asInternalUser;
const actionRequests = createActionRequestsEsSearchResultsMock();
applyActionListEsSearchMock(esClient, actionRequests);
const actions = await getActions({
esClient,
size: 10,
from: 0,
elasticAgentIds: ['agent-a'],
});
expect(actions.actionIds).toEqual(['123']);
expect(actions.actionRequests?.body?.hits?.hits[0]._source?.agent.id).toEqual('agent-a');
});
});
describe('#getActionResponses', () => {
it('should use base filters correctly when no other filter options provided', async () => {
const esClient = mockScopedEsClient.asInternalUser;
applyActionListEsSearchMock(esClient);
await getActionResponses({ esClient, actionIds: [] });
expect(esClient.search).toHaveBeenCalledWith(
{
body: {
query: {
bool: {
filter: [],
},
},
},
from: 0,
index: ['.fleet-actions-results', '.logs-endpoint.action.responses-*'],
size: 10000,
},
{
headers: {
'X-elastic-product-origin': 'fleet',
},
ignore: [404],
meta: true,
}
);
});
it('should query with actionIds and elasticAgentIds when provided', async () => {
const actionIds = [uuid.v4(), uuid.v4()];
const elasticAgentIds = ['123', '456'];
const esClient = mockScopedEsClient.asInternalUser;
applyActionListEsSearchMock(esClient);
await getActionResponses({ esClient, actionIds, elasticAgentIds });
expect(esClient.search).toHaveBeenCalledWith(
{
body: {
query: {
bool: {
filter: [
{
terms: {
agent_id: elasticAgentIds,
},
},
{
terms: {
action_id: actionIds,
},
},
],
},
},
},
from: 0,
index: ['.fleet-actions-results', '.logs-endpoint.action.responses-*'],
size: 10000,
},
{
headers: {
'X-elastic-product-origin': 'fleet',
},
ignore: [404],
meta: true,
}
);
});
it('should return expected output', async () => {
const esClient = mockScopedEsClient.asInternalUser;
const actionRes = createActionResponsesEsSearchResultsMock();
applyActionListEsSearchMock(esClient, undefined, actionRes);
const responses = await getActionResponses({
esClient,
actionIds: ['123'],
elasticAgentIds: ['agent-a'],
});
const responseHits = responses.body.hits.hits;
expect(responseHits.length).toEqual(2);
expect(
responseHits.map((e) => e._index).filter((e) => e.includes('.fleet-actions-results')).length
).toEqual(1);
expect(
responseHits
.map((e) => e._index)
.filter((e) => e.includes('.logs-endpoint.action.responses')).length
).toEqual(1);
});
});
});

164
x-pack/plugins/security_solution/server/endpoint/utils/action_list_helpers.ts Normal file
View file

@ -0,0 +1,164 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { ElasticsearchClient } from '@kbn/core/server';
// eslint-disable-next-line @kbn/eslint/no-restricted-paths
import type { SearchRequest } from '@kbn/data-plugin/public';
import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import type { TransportResult } from '@elastic/elasticsearch';
import { ENDPOINT_ACTIONS_INDEX } from '../../../common/endpoint/constants';
import type {
LogsEndpointAction,
ActionListApiResponse,
EndpointActionResponse,
LogsEndpointActionResponse,
} from '../../../common/endpoint/types';
import { ACTIONS_SEARCH_PAGE_SIZE, ACTION_RESPONSE_INDICES } from '../services/actions/constants';
import { getDateFilters } from '../services/actions/utils';
import { catchAndWrapError } from './wrap_errors';
import { GetActionDetailsListParam } from '../services/actions/action_list';
const queryOptions = Object.freeze({
ignore: [404],
});
// This is same as the one for audit log
// but we want to deprecate audit log at some point
// thus creating this one for sorting action list log entries
export const getTimeSortedActionListLogEntries = (
data: ActionListApiResponse['data'][number]['logEntries']
): ActionListApiResponse['data'][number]['logEntries'] => {
return data.sort((a, b) =>
new Date(b.item.data['@timestamp']) > new Date(a.item.data['@timestamp']) ? 1 : -1
);
};
export const getActions = async ({
commands,
elasticAgentIds,
esClient,
endDate,
from,
size,
startDate,
userIds,
}: Omit<GetActionDetailsListParam, 'logger'>): Promise<{
actionIds: string[];
actionRequests: TransportResult<estypes.SearchResponse<LogsEndpointAction>, unknown>;
}> => {
const additionalFilters = [];
if (commands?.length) {
additionalFilters.push({
terms: {
'data.command': commands,
},
});
}
if (userIds?.length) {
additionalFilters.push({ terms: { user_id: userIds } });
}
if (elasticAgentIds?.length) {
additionalFilters.push({ terms: { agents: elasticAgentIds } });
}
const dateFilters = getDateFilters({ startDate, endDate });
const baseActionFilters = [
{ term: { input_type: 'endpoint' } },
{ term: { type: 'INPUT_ACTION' } },
];
const actionsFilters = [...baseActionFilters, ...dateFilters];
const actionsSearchQuery: SearchRequest = {
index: ENDPOINT_ACTIONS_INDEX,
size,
from,
body: {
query: {
bool: {
filter: additionalFilters.length
? [...actionsFilters, ...additionalFilters]
: actionsFilters,
},
},
sort: [
{
'@timestamp': {
order: 'desc',
},
},
],
},
};
const actionRequests: TransportResult<
estypes.SearchResponse<LogsEndpointAction>,
unknown
> = await esClient
.search<LogsEndpointAction>(actionsSearchQuery, {
...queryOptions,
meta: true,
})
.catch(catchAndWrapError);
// only one type of actions
const actionIds = actionRequests?.body?.hits?.hits.map((e) => {
return (e._source as LogsEndpointAction).EndpointActions.action_id;
});
return { actionIds, actionRequests };
};
export const getActionResponses = async ({
actionIds,
elasticAgentIds,
esClient,
}: {
actionIds: string[];
elasticAgentIds?: string[];
esClient: ElasticsearchClient;
}): Promise<
TransportResult<
estypes.SearchResponse<EndpointActionResponse | LogsEndpointActionResponse>,
unknown
>
> => {
const filter = [];
if (elasticAgentIds?.length) {
filter.push({ terms: { agent_id: elasticAgentIds } });
}
if (actionIds.length) {
filter.push({ terms: { action_id: actionIds } });
}
const responsesSearchQuery: SearchRequest = {
index: ACTION_RESPONSE_INDICES,
size: ACTIONS_SEARCH_PAGE_SIZE,
from: 0,
body: {
query: {
bool: {
filter: filter.length ? filter : [],
},
},
},
};
const actionResponses: TransportResult<
estypes.SearchResponse<EndpointActionResponse | LogsEndpointActionResponse>,
unknown
> = await esClient
.search<EndpointActionResponse | LogsEndpointActionResponse>(responsesSearchQuery, {
...queryOptions,
headers: {
'X-elastic-product-origin': 'fleet',
},
meta: true,
})
.catch(catchAndWrapError);
return actionResponses;
};

126
x-pack/plugins/security_solution/server/endpoint/utils/audit_log_helpers.ts
View file

@ -5,44 +5,27 @@
* 2.0.
*/
import { Logger } from '@kbn/core/server';
import type { Logger } from '@kbn/core/server';
// eslint-disable-next-line @kbn/eslint/no-restricted-paths
import type { SearchRequest } from '@kbn/data-plugin/public';
import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import { TransportResult } from '@elastic/elasticsearch';
import type { TransportResult } from '@elastic/elasticsearch';
import { AGENT_ACTIONS_INDEX, AGENT_ACTIONS_RESULTS_INDEX } from '@kbn/fleet-plugin/common';
import {
ENDPOINT_ACTIONS_DS,
ENDPOINT_ACTIONS_INDEX,
ENDPOINT_ACTION_RESPONSES_INDEX_PATTERN,
failedFleetActionErrorCode,
} from '../../../common/endpoint/constants';
import { SecuritySolutionRequestHandlerContext } from '../../types';
import {
import type { SecuritySolutionRequestHandlerContext } from '../../types';
import type {
ActivityLog,
ActivityLogAction,
EndpointActivityLogAction,
ActivityLogActionResponse,
EndpointActivityLogActionResponse,
ActivityLogItemTypes,
EndpointAction,
LogsEndpointAction,
EndpointActionResponse,
LogsEndpointActionResponse,
ActivityLogEntry,
} from '../../../common/endpoint/types';
import { doesLogsEndpointActionsIndexExist } from './yes_no_data_stream';
import { getDateFilters } from '../services/actions/utils';
import { ACTION_REQUEST_INDICES, ACTION_RESPONSE_INDICES } from '../services/actions/constants';
export const ACTION_REQUEST_INDICES = [AGENT_ACTIONS_INDEX, ENDPOINT_ACTIONS_INDEX];
// search all responses indices irrelevant of namespace
export const ACTION_RESPONSE_INDICES = [
AGENT_ACTIONS_RESULTS_INDEX,
ENDPOINT_ACTION_RESPONSES_INDEX_PATTERN,
];
export const logsEndpointActionsRegex = new RegExp(`(^\.ds-\.logs-endpoint\.actions-default-).+`);
// matches index names like .ds-.logs-endpoint.action.responses-name_space---suffix-2022.01.25-000001
export const logsEndpointResponsesRegex = new RegExp(
`(^\.ds-\.logs-endpoint\.action\.responses-\\w+-).+`
);
const queryOptions = {
headers: {
'X-elastic-product-origin': 'fleet',
@ -50,91 +33,6 @@ const queryOptions = {
ignore: [404],
};
const getDateFilters = ({ startDate, endDate }: { startDate: string; endDate: string }) => {
return [
{ range: { '@timestamp': { gte: startDate } } },
{ range: { '@timestamp': { lte: endDate } } },
];
};
export const getUniqueLogData = (activityLogEntries: ActivityLogEntry[]): ActivityLogEntry[] => {
// find the error responses for actions that didn't make it to fleet index
const onlyResponsesForFleetErrors = activityLogEntries
.filter(
(e) =>
e.type === ActivityLogItemTypes.RESPONSE &&
e.item.data.error?.code === failedFleetActionErrorCode
)
.map(
(e: ActivityLogEntry) => (e.item.data as LogsEndpointActionResponse).EndpointActions.action_id
);
// all actions and responses minus endpoint actions.
const nonEndpointActionsDocs = activityLogEntries.filter(
(e) => e.type !== ActivityLogItemTypes.ACTION
);
// only endpoint actions that match the error responses
const onlyEndpointActionsDocWithoutFleetActions = activityLogEntries
.filter((e) => e.type === ActivityLogItemTypes.ACTION)
.filter((e: ActivityLogEntry) =>
onlyResponsesForFleetErrors.includes(
(e.item.data as LogsEndpointAction).EndpointActions.action_id
)
);
// join the error actions and the rest
return [...nonEndpointActionsDocs, ...onlyEndpointActionsDocWithoutFleetActions];
};
export const categorizeResponseResults = ({
results,
}: {
results: Array<estypes.SearchHit<EndpointActionResponse | LogsEndpointActionResponse>>;
}): Array<ActivityLogActionResponse | EndpointActivityLogActionResponse> => {
return results?.length
? results?.map((e) => {
const isResponseDoc: boolean = matchesIndexPattern({
regexPattern: logsEndpointResponsesRegex,
index: e._index,
});
return isResponseDoc
? {
type: ActivityLogItemTypes.RESPONSE,
item: { id: e._id, data: e._source as LogsEndpointActionResponse },
}
: {
type: ActivityLogItemTypes.FLEET_RESPONSE,
item: { id: e._id, data: e._source as EndpointActionResponse },
};
})
: [];
};
export const categorizeActionResults = ({
results,
}: {
results: Array<estypes.SearchHit<EndpointAction | LogsEndpointAction>>;
}): Array<ActivityLogAction | EndpointActivityLogAction> => {
return results?.length
? results?.map((e) => {
const isActionDoc: boolean = matchesIndexPattern({
regexPattern: logsEndpointActionsRegex,
index: e._index,
});
return isActionDoc
? {
type: ActivityLogItemTypes.ACTION,
item: { id: e._id, data: e._source as LogsEndpointAction },
}
: {
type: ActivityLogItemTypes.FLEET_ACTION,
item: { id: e._id, data: e._source as EndpointAction },
};
})
: [];
};
export const getTimeSortedData = (data: ActivityLog['data']): ActivityLog['data'] => {
return data.sort((a, b) =>
new Date(b.item.data['@timestamp']) > new Date(a.item.data['@timestamp']) ? 1 : -1
@ -200,7 +98,7 @@ export const getActionRequestsResult = async ({
const esClient = (await context.core).elasticsearch.client.asInternalUser;
actionRequests = await esClient.search(actionsSearchQuery, { ...queryOptions, meta: true });
const actionIds = actionRequests?.body?.hits?.hits?.map((e) => {
return logsEndpointActionsRegex.test(e._index)
return e._index.includes(ENDPOINT_ACTIONS_DS)
? (e._source as LogsEndpointAction).EndpointActions.action_id
: (e._source as EndpointAction).action_id;
});
@ -264,11 +162,3 @@ export const getActionResponsesResult = async ({
}
return actionResponses;
};
const matchesIndexPattern = ({
regexPattern,
index,
}: {
regexPattern: RegExp;
index: string;
}): boolean => regexPattern.test(index);

1
x-pack/plugins/security_solution/server/endpoint/utils/index.ts
View file

@ -8,4 +8,5 @@
export * from './fleet_agent_status_to_endpoint_host_status';
export * from './wrap_errors';
export * from './audit_log_helpers';
export * from './action_list_helpers';
export * from './yes_no_data_stream';

2
x-pack/plugins/security_solution/server/utils/custom_http_request_error.ts
View file

@ -5,7 +5,7 @@
* 2.0.
*/
export class CustomHttpRequestError extends Error {
constructor(message: string, public readonly statusCode: number = 500) {
constructor(message: string, public readonly statusCode: number = 500, meta?: unknown) {
super(message);
// For debugging - capture name of subclasses
this.name = this.constructor.name;