github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-06-28 11:05:39 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Security Solution] update endpoint list api to support united index (#112758)

Browse source
This commit is contained in:
Joey F. Poon 2021-09-27 14:51:31 -05:00 • committed by GitHub
parent 1767bee636
commit 94e7844301
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
22 changed files with 1741 additions and 456 deletions
Download patch file Download diff file Expand all files Collapse all files

42
x-pack/plugins/fleet/common/services/agent_status.ts
View file

@ -8,7 +8,7 @@
import { AGENT_POLLING_THRESHOLD_MS } from '../constants';
import type { Agent, AgentStatus } from '../types';
export function getAgentStatus(agent: Agent, now: number = Date.now()): AgentStatus {
export function getAgentStatus(agent: Agent): AgentStatus {
const { last_checkin: lastCheckIn } = agent;
if (!agent.active) {
@ -41,36 +41,42 @@ export function getAgentStatus(agent: Agent, now: number = Date.now()): AgentSta
return 'online';
}
export function buildKueryForEnrollingAgents() {
return 'not (last_checkin:*)';
export function buildKueryForEnrollingAgents(path: string = '') {
return `not (${path}last_checkin:*)`;
}
export function buildKueryForUnenrollingAgents() {
return 'unenrollment_started_at:*';
export function buildKueryForUnenrollingAgents(path: string = '') {
return `${path}unenrollment_started_at:*`;
}
export function buildKueryForOnlineAgents() {
return `not (${buildKueryForOfflineAgents()}) AND not (${buildKueryForErrorAgents()}) AND not (${buildKueryForUpdatingAgents()})`;
export function buildKueryForOnlineAgents(path: string = '') {
return `not (${buildKueryForOfflineAgents(path)}) AND not (${buildKueryForErrorAgents(
path
)}) AND not (${buildKueryForUpdatingAgents(path)})`;
}
export function buildKueryForErrorAgents() {
return `(last_checkin_status:error or last_checkin_status:degraded) AND not (${buildKueryForUpdatingAgents()})`;
export function buildKueryForErrorAgents(path: string = '') {
return `(${path}last_checkin_status:error or ${path}last_checkin_status:degraded) AND not (${buildKueryForUpdatingAgents(
path
)})`;
}
export function buildKueryForOfflineAgents() {
return `last_checkin < now-${
export function buildKueryForOfflineAgents(path: string = '') {
return `${path}last_checkin < now-${
(4 * AGENT_POLLING_THRESHOLD_MS) / 1000
}s AND not (${buildKueryForErrorAgents()}) AND not ( ${buildKueryForUpdatingAgents()} )`;
}s AND not (${buildKueryForErrorAgents(path)}) AND not ( ${buildKueryForUpdatingAgents(path)} )`;
}
export function buildKueryForUpgradingAgents() {
return '(upgrade_started_at:*) and not (upgraded_at:*)';
export function buildKueryForUpgradingAgents(path: string = '') {
return `(${path}upgrade_started_at:*) and not (${path}upgraded_at:*)`;
}
export function buildKueryForUpdatingAgents() {
return `(${buildKueryForUpgradingAgents()}) or (${buildKueryForEnrollingAgents()}) or (${buildKueryForUnenrollingAgents()})`;
export function buildKueryForUpdatingAgents(path: string = '') {
return `(${buildKueryForUpgradingAgents(path)}) or (${buildKueryForEnrollingAgents(
path
)}) or (${buildKueryForUnenrollingAgents(path)})`;
}
export function buildKueryForInactiveAgents() {
return `active:false`;
export function buildKueryForInactiveAgents(path: string = '') {
return `${path}active:false`;
}

3
x-pack/plugins/security_solution/common/endpoint/constants.ts
View file

@ -20,6 +20,9 @@ export const metadataTransformPrefix = 'endpoint.metadata_current-default';
/** The metadata Transform Name prefix with NO namespace and NO (package) version) */
export const metadataTransformPattern = 'endpoint.metadata_current-*';
export const METADATA_UNITED_TRANSFORM = 'endpoint.metadata_united-default';
export const METADATA_UNITED_INDEX = '.metrics-endpoint.metadata_united_default';
export const policyIndexPattern = 'metrics-endpoint.policy-*';
export const telemetryIndexPattern = 'metrics-endpoint.telemetry-*';
export const LIMITED_CONCURRENCY_ENDPOINT_ROUTE_TAG = 'endpoint:limited-concurrency';

8
x-pack/plugins/security_solution/common/endpoint/data_loaders/index_endpoint_hosts.ts
View file

@ -181,6 +181,14 @@ export async function indexEndpointHostDocs({
await indexFleetActionsForHost(client, hostMetadata);
}
hostMetadata = {
...hostMetadata,
// since the united transform uses latest metadata transform as a source
// there is an extra delay and fleet-agents gets populated much sooner.
// we manually add a delay to the time sync field so that the united transform
// will pick up the latest metadata doc.
'@timestamp': hostMetadata['@timestamp'] + 60000,
};
await client
.index({
index: metadataIndex,

12
x-pack/plugins/security_solution/common/endpoint/types/index.ts
View file

@ -6,7 +6,7 @@
*/
import { ApplicationStart } from 'kibana/public';
import { PackagePolicy, UpdatePackagePolicy } from '../../../../fleet/common';
import { Agent, PackagePolicy, UpdatePackagePolicy } from '../../../../fleet/common';
import { ManifestSchema } from '../schema/manifest';
export * from './actions';
@ -546,6 +546,16 @@ export type HostMetadata = Immutable<{
data_stream: DataStream;
}>;
export type UnitedAgentMetadata = Immutable<{
agent: {
id: string;
};
united: {
endpoint: HostMetadata;
agent: Agent;
};
}>;
export interface LegacyEndpointEvent {
'@timestamp': number;
endgame: {

2
x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/middleware.test.ts
View file

@ -53,7 +53,7 @@ import {
jest.mock('../../policy/store/services/ingest', () => ({
sendGetAgentConfigList: () => Promise.resolve({ items: [] }),
sendGetAgentPolicyList: () => Promise.resolve({ items: [] }),
sendGetEndpointSecurityPackage: () => Promise.resolve({}),
sendGetEndpointSecurityPackage: () => Promise.resolve({ version: '1.1.1' }),
sendGetFleetAgentsWithEndpoint: () => Promise.resolve({ total: 0 }),
}));

27
x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/middleware.ts
View file

@ -6,6 +6,8 @@
*/
import { Dispatch } from 'redux';
import semverGte from 'semver/functions/gte';
import { CoreStart, HttpStart } from 'kibana/public';
import {
ActivityLog,
@ -40,6 +42,7 @@ import {
getMetadataTransformStats,
isMetadataTransformStatsLoading,
getActivityLogIsUninitializedOrHasSubsequentAPIError,
endpointPackageVersion,
} from './selectors';
import {
AgentIdsPendingActions,
@ -61,6 +64,7 @@ import {
HOST_METADATA_LIST_ROUTE,
BASE_POLICY_RESPONSE_ROUTE,
metadataCurrentIndexPattern,
METADATA_UNITED_INDEX,
} from '../../../../../common/endpoint/constants';
import { IIndexPattern, Query } from '../../../../../../../../src/plugins/data/public';
import {
@ -85,13 +89,26 @@ export const endpointMiddlewareFactory: ImmutableMiddlewareFactory<EndpointState
coreStart,
depsStart
) => {
async function fetchIndexPatterns(): Promise<IIndexPattern[]> {
// this needs to be called after endpointPackageVersion is loaded (getEndpointPackageInfo)
// or else wrong pattern might be loaded
async function fetchIndexPatterns(
state: ImmutableObject<EndpointState>
): Promise<IIndexPattern[]> {
const packageVersion = endpointPackageVersion(state) ?? '';
const parsedPackageVersion = packageVersion.includes('-')
? packageVersion.substring(0, packageVersion.indexOf('-'))
: packageVersion;
const minUnitedIndexVersion = '1.2.0';
const indexPatternToFetch = semverGte(parsedPackageVersion, minUnitedIndexVersion)
? METADATA_UNITED_INDEX
: metadataCurrentIndexPattern;
const { indexPatterns } = depsStart.data;
const fields = await indexPatterns.getFieldsForWildcard({
pattern: metadataCurrentIndexPattern,
pattern: indexPatternToFetch,
});
const indexPattern: IIndexPattern = {
title: metadataCurrentIndexPattern,
title: indexPatternToFetch,
fields,
};
return [indexPattern];
@ -379,7 +396,7 @@ async function endpointDetailsListMiddleware({
}: {
store: ImmutableMiddlewareAPI<EndpointState, AppAction>;
coreStart: CoreStart;
fetchIndexPatterns: () => Promise<IIndexPattern[]>;
fetchIndexPatterns: (state: ImmutableObject<EndpointState>) => Promise<IIndexPattern[]>;
}) {
const { getState, dispatch } = store;
@ -441,7 +458,7 @@ async function endpointDetailsListMiddleware({
// get index pattern and fields for search bar
if (patterns(getState()).length === 0) {
try {
const indexPatterns = await fetchIndexPatterns();
const indexPatterns = await fetchIndexPatterns(getState());
if (indexPatterns !== undefined) {
dispatch({
type: 'serverReturnedMetadataPatterns',

4
x-pack/plugins/security_solution/server/endpoint/routes/actions/isolation.test.ts
View file

@ -42,7 +42,7 @@ import {
HostMetadata,
} from '../../../../common/endpoint/types';
import { EndpointDocGenerator } from '../../../../common/endpoint/generate_data';
import { createV2SearchResponse } from '../metadata/support/test_support';
import { legacyMetadataSearchResponse } from '../metadata/support/test_support';
import { ElasticsearchAssetType } from '../../../../../fleet/common';
import { CasesClientMock } from '../../../../../cases/server/client/mocks';
@ -188,7 +188,7 @@ describe('Host Isolation', () => {
ctx.core.elasticsearch.client.asCurrentUser.search = jest
.fn()
.mockImplementation(() =>
Promise.resolve({ body: createV2SearchResponse(searchResponse) })
Promise.resolve({ body: legacyMetadataSearchResponse(searchResponse) })
);
const withLicense = license ? license : Platinum;
licenseEmitter.next(withLicense);

220
x-pack/plugins/security_solution/server/endpoint/routes/metadata/handlers.ts
View file

@ -6,11 +6,14 @@
*/
import Boom from '@hapi/boom';
import { ApiResponse } from '@elastic/elasticsearch';
import { SearchResponse, SearchTotalHits } from '@elastic/elasticsearch/api/types';
import { TypeOf } from '@kbn/config-schema';
import {
IKibanaResponse,
IScopedClusterClient,
KibanaRequest,
KibanaResponseFactory,
Logger,
RequestHandler,
@ -19,18 +22,24 @@ import {
import {
HostInfo,
HostMetadata,
UnitedAgentMetadata,
HostResultList,
HostStatus,
} from '../../../../common/endpoint/types';
import type { SecuritySolutionRequestHandlerContext } from '../../../types';
import { getESQueryHostMetadataByID, kibanaRequestToMetadataListESQuery } from './query_builders';
import { Agent, PackagePolicy } from '../../../../../fleet/common/types/models';
import {
getESQueryHostMetadataByID,
kibanaRequestToMetadataListESQuery,
buildUnitedIndexQuery,
} from './query_builders';
import { Agent, AgentPolicy, PackagePolicy } from '../../../../../fleet/common/types/models';
import { AgentNotFoundError } from '../../../../../fleet/server';
import { EndpointAppContext, HostListQueryResult } from '../../types';
import { GetMetadataListRequestSchema, GetMetadataRequestSchema } from './index';
import { findAllUnenrolledAgentIds } from './support/unenroll';
import { findAgentIDsByStatus } from './support/agent_status';
import { getAllEndpointPackagePolicies } from './support/endpoint_package_policies';
import { findAgentIdsByStatus } from './support/agent_status';
import { EndpointAppContextService } from '../../endpoint_app_context_services';
import { fleetAgentStatusToEndpointHostStatus } from '../../utils';
import {
@ -104,41 +113,32 @@ export const getMetadataListRequestHandler = function (
throw new Error('agentService not available');
}
const metadataRequestContext: MetadataRequestContext = {
esClient: context.core.elasticsearch.client,
endpointAppContextService: endpointAppContext.service,
logger,
requestHandlerContext: context,
savedObjectsClient: context.core.savedObjects.client,
};
const unenrolledAgentIds = await findAllUnenrolledAgentIds(
agentService,
const endpointPolicies = await getAllEndpointPackagePolicies(
endpointAppContext.service.getPackagePolicyService()!,
context.core.savedObjects.client,
context.core.elasticsearch.client.asCurrentUser
context.core.savedObjects.client
);
const statusIDs = request?.body?.filters?.host_status?.length
? await findAgentIDsByStatus(
agentService,
context.core.savedObjects.client,
context.core.elasticsearch.client.asCurrentUser,
request.body?.filters?.host_status
)
: undefined;
const queryParams = await kibanaRequestToMetadataListESQuery(request, endpointAppContext, {
unenrolledAgentIds: unenrolledAgentIds.concat(IGNORED_ELASTIC_AGENT_IDS),
statusAgentIDs: statusIDs,
});
const result = await context.core.elasticsearch.client.asCurrentUser.search<HostMetadata>(
queryParams
const { unitedIndexExists, unitedQueryResponse } = await queryUnitedIndex(
context,
request,
endpointAppContext,
logger,
endpointPolicies
);
const hostListQueryResult = queryResponseToHostListResult(result.body);
if (unitedIndexExists) {
return response.ok({
body: await mapToHostResultList(queryParams, hostListQueryResult, metadataRequestContext),
body: unitedQueryResponse,
});
}
return response.ok({
body: await legacyListMetadataQuery(
context,
request,
endpointAppContext,
logger,
endpointPolicies
),
});
};
};
@ -395,3 +395,157 @@ export async function enrichHostMetadata(
policy_info: policyInfo,
};
}
async function legacyListMetadataQuery(
context: SecuritySolutionRequestHandlerContext,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
request: KibanaRequest<any, any, any>,
endpointAppContext: EndpointAppContext,
logger: Logger,
endpointPolicies: PackagePolicy[]
): Promise<HostResultList> {
const agentService = endpointAppContext.service.getAgentService()!;
const metadataRequestContext: MetadataRequestContext = {
esClient: context.core.elasticsearch.client,
endpointAppContextService: endpointAppContext.service,
logger,
requestHandlerContext: context,
savedObjectsClient: context.core.savedObjects.client,
};
const endpointPolicyIds = endpointPolicies.map((policy) => policy.policy_id);
const unenrolledAgentIds = await findAllUnenrolledAgentIds(
agentService,
context.core.elasticsearch.client.asCurrentUser,
endpointPolicyIds
);
const statusesToFilter = request?.body?.filters?.host_status ?? [];
const statusIds = await findAgentIdsByStatus(
agentService,
context.core.elasticsearch.client.asCurrentUser,
statusesToFilter
);
const queryParams = await kibanaRequestToMetadataListESQuery(request, endpointAppContext, {
unenrolledAgentIds: unenrolledAgentIds.concat(IGNORED_ELASTIC_AGENT_IDS),
statusAgentIds: statusIds,
});
const result = await context.core.elasticsearch.client.asCurrentUser.search<HostMetadata>(
queryParams
);
const hostListQueryResult = queryResponseToHostListResult(result.body);
return mapToHostResultList(queryParams, hostListQueryResult, metadataRequestContext);
}
async function queryUnitedIndex(
context: SecuritySolutionRequestHandlerContext,
request: KibanaRequest,
endpointAppContext: EndpointAppContext,
logger: Logger,
endpointPolicies: PackagePolicy[]
): Promise<{
unitedIndexExists: boolean;
unitedQueryResponse: HostResultList;
}> {
const endpointPolicyIds = endpointPolicies.map((policy) => policy.policy_id);
const unitedIndexQuery = await buildUnitedIndexQuery(
request,
endpointAppContext,
IGNORED_ELASTIC_AGENT_IDS,
endpointPolicyIds
);
let unitedMetadataQueryResponse: ApiResponse<SearchResponse<UnitedAgentMetadata>>;
try {
unitedMetadataQueryResponse =
await context.core.elasticsearch.client.asCurrentUser.search<UnitedAgentMetadata>(
unitedIndexQuery
);
} catch (error) {
const errorType = error?.meta?.body?.error?.type ?? '';
// no united index means that the endpoint package hasn't been upgraded yet
// this is expected so we fall back to the legacy query
// errors other than index_not_found_exception are unexpected
if (errorType !== 'index_not_found_exception') {
logger.error(error);
throw error;
}
return {
unitedIndexExists: false,
unitedQueryResponse: {} as HostResultList,
};
}
const { hits: docs, total: docsCount } = unitedMetadataQueryResponse?.body?.hits || {};
const agentPolicyIds: string[] = docs.map((doc) => doc._source?.united?.agent?.policy_id ?? '');
const agentPolicies =
(await endpointAppContext.service
.getAgentPolicyService()
?.getByIds(context.core.savedObjects.client, agentPolicyIds)) ?? [];
const agentPoliciesMap: Record<string, AgentPolicy> = agentPolicies.reduce(
(acc, agentPolicy) => ({
...acc,
[agentPolicy.id]: {
...agentPolicy,
},
}),
{}
);
const endpointPoliciesMap: Record<string, PackagePolicy> = endpointPolicies.reduce(
(acc, packagePolicy) => ({
...acc,
[packagePolicy.policy_id]: packagePolicy,
}),
{}
);
const hosts = docs
.filter((doc) => {
const { endpoint: metadata, agent } = doc?._source?.united ?? {};
return metadata && agent;
})
.map((doc) => {
const { endpoint: metadata, agent } = doc!._source!.united!;
const agentPolicy = agentPoliciesMap[agent.policy_id!];
const endpointPolicy = endpointPoliciesMap[agent.policy_id!];
return {
metadata,
host_status: fleetAgentStatusToEndpointHostStatus(agent.last_checkin_status!),
policy_info: {
agent: {
applied: {
id: agent.policy_id || '',
revision: agent.policy_revision || 0,
},
configured: {
id: agentPolicy?.id || '',
revision: agentPolicy?.revision || 0,
},
},
endpoint: {
id: endpointPolicy?.id || '',
revision: endpointPolicy?.revision || 0,
},
},
} as HostInfo;
});
const unitedQueryResponse: HostResultList = {
request_page_size: unitedIndexQuery.size,
request_page_index: unitedIndexQuery.from,
total: (docsCount as SearchTotalHits).value,
hosts,
};
return {
unitedIndexExists: true,
unitedQueryResponse,
};
}

328
x-pack/plugins/security_solution/server/endpoint/routes/metadata/metadata.test.ts
View file

@ -33,11 +33,13 @@ import {
import { createMockConfig } from '../../../lib/detection_engine/routes/__mocks__';
import { EndpointDocGenerator } from '../../../../common/endpoint/generate_data';
import { Agent, ElasticsearchAssetType } from '../../../../../fleet/common/types/models';
import { createV2SearchResponse } from './support/test_support';
import { legacyMetadataSearchResponse, unitedMetadataSearchResponse } from './support/test_support';
import { PackageService } from '../../../../../fleet/server/services';
import {
HOST_METADATA_LIST_ROUTE,
metadataCurrentIndexPattern,
metadataTransformPrefix,
METADATA_UNITED_INDEX,
} from '../../../../common/endpoint/constants';
import type { SecuritySolutionPluginRouter } from '../../../types';
import { AgentNotFoundError, PackagePolicyServiceInterface } from '../../../../../fleet/server';
@ -49,6 +51,15 @@ import {
import { EndpointHostNotFoundError } from '../../services/metadata';
import { FleetAgentGenerator } from '../../../../common/endpoint/data_generators/fleet_agent_generator';
class IndexNotFoundException extends Error {
meta: { body: { error: { type: string } } };
constructor() {
super();
this.meta = { body: { error: { type: 'index_not_found_exception' } } };
}
}
describe('test endpoint route', () => {
let routerMock: jest.Mocked<SecuritySolutionPluginRouter>;
let mockResponse: jest.Mocked<KibanaResponseFactory>;
@ -95,7 +106,266 @@ describe('test endpoint route', () => {
});
});
describe('with new transform package', () => {
describe('with .metrics-endpoint.metadata_united_default index', () => {
beforeEach(() => {
endpointAppContextService = new EndpointAppContextService();
mockPackageService = createMockPackageService();
mockPackageService.getInstallation.mockReturnValue(
Promise.resolve({
installed_kibana: [],
package_assets: [],
es_index_patterns: {},
name: '',
version: '',
install_status: 'installed',
install_version: '',
install_started_at: '',
install_source: 'registry',
installed_es: [
{
id: 'logs-endpoint.events.security',
type: ElasticsearchAssetType.indexTemplate,
},
{
id: `${metadataTransformPrefix}-0.16.0-dev.0`,
type: ElasticsearchAssetType.transform,
},
],
})
);
endpointAppContextService.start({ ...startContract, packageService: mockPackageService });
mockAgentService = startContract.agentService!;
registerEndpointRoutes(routerMock, {
logFactory: loggingSystemMock.create(),
service: endpointAppContextService,
config: () => Promise.resolve(createMockConfig()),
experimentalFeatures: parseExperimentalConfigValue(createMockConfig().enableExperimental),
});
});
afterEach(() => endpointAppContextService.stop());
it('should fallback to legacy index if index not found', async () => {
const mockRequest = httpServerMock.createKibanaRequest({});
const response = legacyMetadataSearchResponse(
new EndpointDocGenerator().generateHostMetadata()
);
(mockScopedClient.asCurrentUser.search as jest.Mock)
.mockImplementationOnce(() => {
throw new IndexNotFoundException();
})
.mockImplementationOnce(() => Promise.resolve({ body: response }));
[routeConfig, routeHandler] = routerMock.post.mock.calls.find(([{ path }]) =>
path.startsWith(`${HOST_METADATA_LIST_ROUTE}`)
)!;
mockAgentService.getAgentStatusById = jest.fn().mockReturnValue('error');
mockAgentService.listAgents = jest.fn().mockReturnValue(noUnenrolledAgent);
await routeHandler(
createRouteHandlerContext(mockScopedClient, mockSavedObjectClient),
mockRequest,
mockResponse
);
const esSearchMock = mockScopedClient.asCurrentUser.search;
// should be called twice, united index first, then legacy index
expect(esSearchMock).toHaveBeenCalledTimes(2);
expect(esSearchMock.mock.calls[0][0]!.index).toEqual(METADATA_UNITED_INDEX);
expect(esSearchMock.mock.calls[1][0]!.index).toEqual(metadataCurrentIndexPattern);
expect(routeConfig.options).toEqual({
authRequired: true,
tags: ['access:securitySolution'],
});
expect(mockResponse.ok).toBeCalled();
const endpointResultList = mockResponse.ok.mock.calls[0][0]?.body as HostResultList;
expect(endpointResultList.hosts.length).toEqual(1);
expect(endpointResultList.total).toEqual(1);
expect(endpointResultList.request_page_index).toEqual(0);
expect(endpointResultList.request_page_size).toEqual(10);
});
it('should return expected metadata', async () => {
const mockRequest = httpServerMock.createKibanaRequest({
body: {
paging_properties: [
{
page_size: 10,
},
{
page_index: 1,
},
],
filters: {
kql: 'not host.ip:10.140.73.246',
host_status: ['updating'],
},
},
});
mockAgentService.getAgentStatusById = jest.fn().mockReturnValue('error');
mockAgentService.listAgents = jest.fn().mockReturnValue(noUnenrolledAgent);
const metadata = new EndpointDocGenerator().generateHostMetadata();
const esSearchMock = mockScopedClient.asCurrentUser.search as jest.Mock;
esSearchMock.mockImplementationOnce(() =>
Promise.resolve({
body: unitedMetadataSearchResponse(metadata),
})
);
[routeConfig, routeHandler] = routerMock.post.mock.calls.find(([{ path }]) =>
path.startsWith(`${HOST_METADATA_LIST_ROUTE}`)
)!;
await routeHandler(
createRouteHandlerContext(mockScopedClient, mockSavedObjectClient),
mockRequest,
mockResponse
);
expect(esSearchMock).toHaveBeenCalledTimes(1);
expect(esSearchMock.mock.calls[0][0]!.index).toEqual(METADATA_UNITED_INDEX);
expect(esSearchMock.mock.calls[0][0]?.body?.query).toEqual({
bool: {
must: [
{
bool: {
filter: [
{
terms: {
'united.agent.policy_id': [],
},
},
{
exists: {
field: 'united.endpoint.agent.id',
},
},
{
exists: {
field: 'united.agent.agent.id',
},
},
{
term: {
'united.agent.active': {
value: true,
},
},
},
],
must_not: {
terms: {
'agent.id': [
'00000000-0000-0000-0000-000000000000',
'11111111-1111-1111-1111-111111111111',
],
},
},
},
},
{
bool: {
should: [
{
bool: {
filter: [
{
bool: {
should: [
{
exists: {
field: 'united.agent.upgrade_started_at',
},
},
],
minimum_should_match: 1,
},
},
{
bool: {
must_not: {
bool: {
should: [
{
exists: {
field: 'united.agent.upgraded_at',
},
},
],
minimum_should_match: 1,
},
},
},
},
],
},
},
{
bool: {
must_not: {
bool: {
should: [
{
exists: {
field: 'united.agent.last_checkin',
},
},
],
minimum_should_match: 1,
},
},
},
},
{
bool: {
should: [
{
exists: {
field: 'united.agent.unenrollment_started_at',
},
},
],
minimum_should_match: 1,
},
},
],
minimum_should_match: 1,
},
},
{
bool: {
must_not: {
bool: {
should: [
{
match: {
'host.ip': '10.140.73.246',
},
},
],
minimum_should_match: 1,
},
},
},
},
],
},
});
expect(routeConfig.options).toEqual({
authRequired: true,
tags: ['access:securitySolution'],
});
expect(mockResponse.ok).toBeCalled();
const endpointResultList = mockResponse.ok.mock.calls[0][0]?.body as HostResultList;
expect(endpointResultList.hosts.length).toEqual(1);
expect(endpointResultList.hosts[0].metadata).toEqual(metadata);
expect(endpointResultList.total).toEqual(1);
expect(endpointResultList.request_page_index).toEqual(10);
expect(endpointResultList.request_page_size).toEqual(10);
});
});
describe('with metrics-endpoint.metadata_current_default index', () => {
beforeEach(() => {
endpointAppContextService = new EndpointAppContextService();
mockPackageService = createMockPackageService();
@ -137,10 +407,14 @@ describe('test endpoint route', () => {
it('test find the latest of all endpoints', async () => {
const mockRequest = httpServerMock.createKibanaRequest({});
const response = createV2SearchResponse(new EndpointDocGenerator().generateHostMetadata());
(mockScopedClient.asCurrentUser.search as jest.Mock).mockImplementationOnce(() =>
Promise.resolve({ body: response })
const response = legacyMetadataSearchResponse(
new EndpointDocGenerator().generateHostMetadata()
);
(mockScopedClient.asCurrentUser.search as jest.Mock)
.mockImplementationOnce(() => {
throw new IndexNotFoundException();
})
.mockImplementationOnce(() => Promise.resolve({ body: response }));
[routeConfig, routeHandler] = routerMock.post.mock.calls.find(([{ path }]) =>
path.startsWith(`${HOST_METADATA_LIST_ROUTE}`)
)!;
@ -152,7 +426,7 @@ describe('test endpoint route', () => {
mockResponse
);
expect(mockScopedClient.asCurrentUser.search).toHaveBeenCalledTimes(1);
expect(mockScopedClient.asCurrentUser.search).toHaveBeenCalledTimes(2);
expect(routeConfig.options).toEqual({
authRequired: true,
tags: ['access:securitySolution'],
@ -181,9 +455,13 @@ describe('test endpoint route', () => {
mockAgentService.getAgentStatusById = jest.fn().mockReturnValue('error');
mockAgentService.listAgents = jest.fn().mockReturnValue(noUnenrolledAgent);
(mockScopedClient.asCurrentUser.search as jest.Mock).mockImplementationOnce(() =>
(mockScopedClient.asCurrentUser.search as jest.Mock)
.mockImplementationOnce(() => {
throw new IndexNotFoundException();
})
.mockImplementationOnce(() =>
Promise.resolve({
body: createV2SearchResponse(new EndpointDocGenerator().generateHostMetadata()),
body: legacyMetadataSearchResponse(new EndpointDocGenerator().generateHostMetadata()),
})
);
[routeConfig, routeHandler] = routerMock.post.mock.calls.find(([{ path }]) =>
@ -195,9 +473,9 @@ describe('test endpoint route', () => {
mockRequest,
mockResponse
);
expect(mockScopedClient.asCurrentUser.search).toHaveBeenCalledTimes(1);
expect(mockScopedClient.asCurrentUser.search).toHaveBeenCalledTimes(2);
expect(
(mockScopedClient.asCurrentUser.search as jest.Mock).mock.calls[0][0]?.body?.query.bool
(mockScopedClient.asCurrentUser.search as jest.Mock).mock.calls[1][0]?.body?.query.bool
.must_not
).toContainEqual({
terms: {
@ -237,9 +515,13 @@ describe('test endpoint route', () => {
mockAgentService.getAgentStatusById = jest.fn().mockReturnValue('error');
mockAgentService.listAgents = jest.fn().mockReturnValue(noUnenrolledAgent);
(mockScopedClient.asCurrentUser.search as jest.Mock).mockImplementationOnce(() =>
(mockScopedClient.asCurrentUser.search as jest.Mock)
.mockImplementationOnce(() => {
throw new IndexNotFoundException();
})
.mockImplementationOnce(() =>
Promise.resolve({
body: createV2SearchResponse(new EndpointDocGenerator().generateHostMetadata()),
body: legacyMetadataSearchResponse(new EndpointDocGenerator().generateHostMetadata()),
})
);
[routeConfig, routeHandler] = routerMock.post.mock.calls.find(([{ path }]) =>
@ -255,7 +537,7 @@ describe('test endpoint route', () => {
expect(mockScopedClient.asCurrentUser.search).toBeCalled();
expect(
// KQL filter to be passed through
(mockScopedClient.asCurrentUser.search as jest.Mock).mock.calls[0][0]?.body?.query.bool.must
(mockScopedClient.asCurrentUser.search as jest.Mock).mock.calls[1][0]?.body?.query.bool.must
).toContainEqual({
bool: {
must_not: {
@ -273,7 +555,7 @@ describe('test endpoint route', () => {
},
});
expect(
(mockScopedClient.asCurrentUser.search as jest.Mock).mock.calls[0][0]?.body?.query.bool.must
(mockScopedClient.asCurrentUser.search as jest.Mock).mock.calls[1][0]?.body?.query.bool.must
).toContainEqual({
bool: {
must_not: [
@ -315,7 +597,7 @@ describe('test endpoint route', () => {
const mockRequest = httpServerMock.createKibanaRequest({ params: { id: 'BADID' } });
(mockScopedClient.asCurrentUser.search as jest.Mock).mockImplementationOnce(() =>
Promise.resolve({ body: createV2SearchResponse() })
Promise.resolve({ body: legacyMetadataSearchResponse() })
);
mockAgentService.getAgentStatusById = jest.fn().mockReturnValue('error');
@ -343,7 +625,9 @@ describe('test endpoint route', () => {
});
it('should return a single endpoint with status healthy', async () => {
const response = createV2SearchResponse(new EndpointDocGenerator().generateHostMetadata());
const response = legacyMetadataSearchResponse(
new EndpointDocGenerator().generateHostMetadata()
);
const mockRequest = httpServerMock.createKibanaRequest({
params: { id: response.hits.hits[0]._id },
});
@ -377,7 +661,9 @@ describe('test endpoint route', () => {
});
it('should return a single endpoint with status unhealthy when AgentService throw 404', async () => {
const response = createV2SearchResponse(new EndpointDocGenerator().generateHostMetadata());
const response = legacyMetadataSearchResponse(
new EndpointDocGenerator().generateHostMetadata()
);
const mockRequest = httpServerMock.createKibanaRequest({
params: { id: response.hits.hits[0]._id },
@ -412,7 +698,9 @@ describe('test endpoint route', () => {
});
it('should return a single endpoint with status unhealthy when status is not offline, online or enrolling', async () => {
const response = createV2SearchResponse(new EndpointDocGenerator().generateHostMetadata());
const response = legacyMetadataSearchResponse(
new EndpointDocGenerator().generateHostMetadata()
);
const mockRequest = httpServerMock.createKibanaRequest({
params: { id: response.hits.hits[0]._id },
@ -448,7 +736,9 @@ describe('test endpoint route', () => {
});
it('should throw error when endpoint agent is not active', async () => {
const response = createV2SearchResponse(new EndpointDocGenerator().generateHostMetadata());
const response = legacyMetadataSearchResponse(
new EndpointDocGenerator().generateHostMetadata()
);
const mockRequest = httpServerMock.createKibanaRequest({
params: { id: response.hits.hits[0]._id },

461
x-pack/plugins/security_solution/server/endpoint/routes/metadata/query_builders.fixtures.ts Normal file
View file

@ -0,0 +1,461 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export const expectedCompleteUnitedIndexQuery = {
bool: {
must: [
{
bool: {
must_not: {
terms: {
'agent.id': ['test-agent-id'],
},
},
filter: [
{
terms: {
'united.agent.policy_id': ['test-endpoint-policy-id'],
},
},
{
exists: {
field: 'united.endpoint.agent.id',
},
},
{
exists: {
field: 'united.agent.agent.id',
},
},
{
term: {
'united.agent.active': {
value: true,
},
},
},
],
},
},
{
bool: {
filter: [
{
bool: {
must_not: {
bool: {
filter: [
{
bool: {
should: [
{
range: {
'united.agent.last_checkin': {
lt: 'now-120s',
},
},
},
],
minimum_should_match: 1,
},
},
{
bool: {
must_not: {
bool: {
filter: [
{
bool: {
should: [
{
bool: {
should: [
{
match: {
'united.agent.last_checkin_status': 'error',
},
},
],
minimum_should_match: 1,
},
},
{
bool: {
should: [
{
match: {
'united.agent.last_checkin_status': 'degraded',
},
},
],
minimum_should_match: 1,
},
},
],
minimum_should_match: 1,
},
},
{
bool: {
must_not: {
bool: {
should: [
{
bool: {
filter: [
{
bool: {
should: [
{
exists: {
field: 'united.agent.upgrade_started_at',
},
},
],
minimum_should_match: 1,
},
},
{
bool: {
must_not: {
bool: {
should: [
{
exists: {
field: 'united.agent.upgraded_at',
},
},
],
minimum_should_match: 1,
},
},
},
},
],
},
},
{
bool: {
must_not: {
bool: {
should: [
{
exists: {
field: 'united.agent.last_checkin',
},
},
],
minimum_should_match: 1,
},
},
},
},
{
bool: {
should: [
{
exists: {
field: 'united.agent.unenrollment_started_at',
},
},
],
minimum_should_match: 1,
},
},
],
minimum_should_match: 1,
},
},
},
},
],
},
},
},
},
{
bool: {
must_not: {
bool: {
should: [
{
bool: {
filter: [
{
bool: {
should: [
{
exists: {
field: 'united.agent.upgrade_started_at',
},
},
],
minimum_should_match: 1,
},
},
{
bool: {
must_not: {
bool: {
should: [
{
exists: {
field: 'united.agent.upgraded_at',
},
},
],
minimum_should_match: 1,
},
},
},
},
],
},
},
{
bool: {
must_not: {
bool: {
should: [
{
exists: {
field: 'united.agent.last_checkin',
},
},
],
minimum_should_match: 1,
},
},
},
},
{
bool: {
should: [
{
exists: {
field: 'united.agent.unenrollment_started_at',
},
},
],
minimum_should_match: 1,
},
},
],
minimum_should_match: 1,
},
},
},
},
],
},
},
},
},
{
bool: {
must_not: {
bool: {
filter: [
{
bool: {
should: [
{
bool: {
should: [
{
match: {
'united.agent.last_checkin_status': 'error',
},
},
],
minimum_should_match: 1,
},
},
{
bool: {
should: [
{
match: {
'united.agent.last_checkin_status': 'degraded',
},
},
],
minimum_should_match: 1,
},
},
],
minimum_should_match: 1,
},
},
{
bool: {
must_not: {
bool: {
should: [
{
bool: {
filter: [
{
bool: {
should: [
{
exists: {
field: 'united.agent.upgrade_started_at',
},
},
],
minimum_should_match: 1,
},
},
{
bool: {
must_not: {
bool: {
should: [
{
exists: {
field: 'united.agent.upgraded_at',
},
},
],
minimum_should_match: 1,
},
},
},
},
],
},
},
{
bool: {
must_not: {
bool: {
should: [
{
exists: {
field: 'united.agent.last_checkin',
},
},
],
minimum_should_match: 1,
},
},
},
},
{
bool: {
should: [
{
exists: {
field: 'united.agent.unenrollment_started_at',
},
},
],
minimum_should_match: 1,
},
},
],
minimum_should_match: 1,
},
},
},
},
],
},
},
},
},
{
bool: {
must_not: {
bool: {
should: [
{
bool: {
filter: [
{
bool: {
should: [
{
exists: {
field: 'united.agent.upgrade_started_at',
},
},
],
minimum_should_match: 1,
},
},
{
bool: {
must_not: {
bool: {
should: [
{
exists: {
field: 'united.agent.upgraded_at',
},
},
],
minimum_should_match: 1,
},
},
},
},
],
},
},
{
bool: {
must_not: {
bool: {
should: [
{
exists: {
field: 'united.agent.last_checkin',
},
},
],
minimum_should_match: 1,
},
},
},
},
{
bool: {
should: [
{
exists: {
field: 'united.agent.unenrollment_started_at',
},
},
],
minimum_should_match: 1,
},
},
],
minimum_should_match: 1,
},
},
},
},
],
},
},
{
bool: {
should: [
{
exists: {
field: 'united.endpoint.host.os.name',
},
},
],
minimum_should_match: 1,
},
},
],
},
};

73
x-pack/plugins/security_solution/server/endpoint/routes/metadata/query_builders.test.ts
View file

@ -6,12 +6,19 @@
*/
import { httpServerMock, loggingSystemMock } from '../../../../../../../src/core/server/mocks';
import { kibanaRequestToMetadataListESQuery, getESQueryHostMetadataByID } from './query_builders';
import {
kibanaRequestToMetadataListESQuery,
getESQueryHostMetadataByID,
buildUnitedIndexQuery,
} from './query_builders';
import { EndpointAppContextService } from '../../endpoint_app_context_services';
import { createMockConfig } from '../../../lib/detection_engine/routes/__mocks__';
import { metadataCurrentIndexPattern } from '../../../../common/endpoint/constants';
import { parseExperimentalConfigValue } from '../../../../common/experimental_features';
import { get } from 'lodash';
import { KibanaRequest } from 'kibana/server';
import { EndpointAppContext } from '../../types';
import { expectedCompleteUnitedIndexQuery } from './query_builders.fixtures';
describe('query builder', () => {
describe('MetadataListESQuery', () => {
@ -200,4 +207,68 @@ describe('query builder', () => {
});
});
});
describe('buildUnitedIndexQuery', () => {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
let mockRequest: KibanaRequest<any, any, any>;
let mockEndpointAppContext: EndpointAppContext;
const filters = { kql: '', host_status: [] };
beforeEach(() => {
mockRequest = httpServerMock.createKibanaRequest({ body: { filters } });
mockEndpointAppContext = {
logFactory: loggingSystemMock.create(),
service: new EndpointAppContextService(),
config: () => Promise.resolve(createMockConfig()),
experimentalFeatures: parseExperimentalConfigValue(createMockConfig().enableExperimental),
};
});
it('correctly builds empty query', async () => {
const query = await buildUnitedIndexQuery(mockRequest, mockEndpointAppContext, [], []);
const expected = {
bool: {
filter: [
{
terms: {
'united.agent.policy_id': [],
},
},
{
exists: {
field: 'united.endpoint.agent.id',
},
},
{
exists: {
field: 'united.agent.agent.id',
},
},
{
term: {
'united.agent.active': {
value: true,
},
},
},
],
},
};
expect(query.body.query).toEqual(expected);
});
it('correctly builds query', async () => {
mockRequest.body.filters.kql = 'united.endpoint.host.os.name : *';
mockRequest.body.filters.host_status = ['healthy'];
const ignoredAgentIds: string[] = ['test-agent-id'];
const endpointPolicyIds: string[] = ['test-endpoint-policy-id'];
const query = await buildUnitedIndexQuery(
mockRequest,
mockEndpointAppContext,
ignoredAgentIds,
endpointPolicyIds
);
const expected = expectedCompleteUnitedIndexQuery;
expect(query.body.query).toEqual(expected);
});
});
});

101
x-pack/plugins/security_solution/server/endpoint/routes/metadata/query_builders.ts
View file

@ -7,13 +7,17 @@
import type { estypes } from '@elastic/elasticsearch';
import { fromKueryExpression, toElasticsearchQuery } from '@kbn/es-query';
import { metadataCurrentIndexPattern } from '../../../../common/endpoint/constants';
import {
metadataCurrentIndexPattern,
METADATA_UNITED_INDEX,
} from '../../../../common/endpoint/constants';
import { KibanaRequest } from '../../../../../../../src/core/server';
import { EndpointAppContext } from '../../types';
import { buildStatusesKuery } from './support/agent_status';
export interface QueryBuilderOptions {
unenrolledAgentIds?: string[];
statusAgentIDs?: string[];
statusAgentIds?: string[];
}
// sort using either event.created, or HostDetails.event.created,
@ -21,7 +25,7 @@ export interface QueryBuilderOptions {
// using unmapped_type avoids errors when the given field doesn't exist, and sets to the 0-value for that type
// effectively ignoring it
// https://www.elastic.co/guide/en/elasticsearch/reference/current/sort-search-results.html#_ignoring_unmapped_fields
const MetadataSortMethod: estypes.SearchSortContainer[] = [
export const MetadataSortMethod: estypes.SearchSortContainer[] = [
{
'event.created': {
order: 'desc',
@ -50,7 +54,7 @@ export async function kibanaRequestToMetadataListESQuery(
query: buildQueryBody(
request,
queryBuilderOptions?.unenrolledAgentIds!,
queryBuilderOptions?.statusAgentIDs!
queryBuilderOptions?.statusAgentIds!
),
track_total_hits: true,
sort: MetadataSortMethod,
@ -86,7 +90,7 @@ function buildQueryBody(
// eslint-disable-next-line @typescript-eslint/no-explicit-any
request: KibanaRequest<any, any, any>,
unerolledAgentIds: string[] | undefined,
statusAgentIDs: string[] | undefined
statusAgentIds: string[] | undefined
// eslint-disable-next-line @typescript-eslint/no-explicit-any
): Record<string, any> {
// the filtered properties may be preceded by 'HostDetails' under an older index mapping
@ -99,15 +103,16 @@ function buildQueryBody(
],
}
: null;
const filterStatusAgents = statusAgentIDs
const filterStatusAgents =
statusAgentIds && statusAgentIds.length
? {
filter: [
{
bool: {
// OR's the two together
should: [
{ terms: { 'elastic.agent.id': statusAgentIDs } },
{ terms: { 'HostDetails.elastic.agent.id': statusAgentIDs } },
{ terms: { 'elastic.agent.id': statusAgentIds } },
{ terms: { 'HostDetails.elastic.agent.id': statusAgentIds } },
],
},
},
@ -208,3 +213,83 @@ export function getESQueryHostMetadataByIDs(agentIDs: string[]) {
index: metadataCurrentIndexPattern,
};
}
export async function buildUnitedIndexQuery(
// eslint-disable-next-line @typescript-eslint/no-explicit-any
request: KibanaRequest<any, any, any>,
endpointAppContext: EndpointAppContext,
ignoredAgentIds: string[] | undefined,
endpointPolicyIds: string[] = []
// eslint-disable-next-line @typescript-eslint/no-explicit-any
): Promise<Record<string, any>> {
const pagingProperties = await getPagingProperties(request, endpointAppContext);
const statusesToFilter = request?.body?.filters?.host_status ?? [];
const statusesKuery = buildStatusesKuery(statusesToFilter);
const filterIgnoredAgents =
ignoredAgentIds && ignoredAgentIds.length > 0
? {
must_not: { terms: { 'agent.id': ignoredAgentIds } },
}
: null;
const filterEndpointPolicyAgents = {
filter: [
// must contain an endpoint policy id
{
terms: { 'united.agent.policy_id': endpointPolicyIds },
},
// doc contains both agent and metadata
{ exists: { field: 'united.endpoint.agent.id' } },
{ exists: { field: 'united.agent.agent.id' } },
// agent is enrolled
{
term: {
'united.agent.active': {
value: true,
},
},
},
],
};
const idFilter = {
bool: {
...filterIgnoredAgents,
...filterEndpointPolicyAgents,
},
};
// eslint-disable-next-line @typescript-eslint/no-explicit-any
let query: Record<string, any> =
filterIgnoredAgents || filterEndpointPolicyAgents
? idFilter
: {
match_all: {},
};
if (statusesKuery || request?.body?.filters?.kql) {
const kqlQuery = toElasticsearchQuery(fromKueryExpression(request.body.filters.kql));
const q = [];
if (filterIgnoredAgents || filterEndpointPolicyAgents) {
q.push(idFilter);
}
if (statusesKuery) {
q.push(toElasticsearchQuery(fromKueryExpression(statusesKuery)));
}
q.push({ ...kqlQuery });
query = {
bool: { must: q },
};
}
return {
body: {
query,
track_total_hits: true,
sort: MetadataSortMethod,
},
from: pagingProperties.pageIndex * pagingProperties.pageSize,
size: pagingProperties.pageSize,
index: METADATA_UNITED_INDEX,
};
}

88
x-pack/plugins/security_solution/server/endpoint/routes/metadata/support/agent_status.test.ts
View file

@ -5,23 +5,18 @@
* 2.0.
*/
import { ElasticsearchClient, SavedObjectsClientContract } from 'kibana/server';
import { findAgentIDsByStatus } from './agent_status';
import {
elasticsearchServiceMock,
savedObjectsClientMock,
} from '../../../../../../../../src/core/server/mocks';
import { ElasticsearchClient } from 'kibana/server';
import { buildStatusesKuery, findAgentIdsByStatus } from './agent_status';
import { elasticsearchServiceMock } from '../../../../../../../../src/core/server/mocks';
import { AgentService } from '../../../../../../fleet/server/services';
import { createMockAgentService } from '../../../../../../fleet/server/mocks';
import { Agent } from '../../../../../../fleet/common/types/models';
import { AgentStatusKueryHelper } from '../../../../../../fleet/common/services';
describe('test filtering endpoint hosts by agent status', () => {
let mockSavedObjectClient: jest.Mocked<SavedObjectsClientContract>;
let mockElasticsearchClient: jest.Mocked<ElasticsearchClient>;
let mockAgentService: jest.Mocked<AgentService>;
beforeEach(() => {
mockSavedObjectClient = savedObjectsClientMock.create();
mockElasticsearchClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
mockAgentService = createMockAgentService();
});
@ -36,12 +31,9 @@ describe('test filtering endpoint hosts by agent status', () => {
})
);
const result = await findAgentIDsByStatus(
mockAgentService,
mockSavedObjectClient,
mockElasticsearchClient,
['healthy']
);
const result = await findAgentIdsByStatus(mockAgentService, mockElasticsearchClient, [
'healthy',
]);
expect(result).toBeDefined();
});
@ -64,12 +56,9 @@ describe('test filtering endpoint hosts by agent status', () => {
})
);
const result = await findAgentIDsByStatus(
mockAgentService,
mockSavedObjectClient,
mockElasticsearchClient,
['offline']
);
const result = await findAgentIdsByStatus(mockAgentService, mockElasticsearchClient, [
'offline',
]);
const offlineKuery = AgentStatusKueryHelper.buildKueryForOfflineAgents();
expect(mockAgentService.listAgents.mock.calls[0][1].kuery).toEqual(
expect.stringContaining(offlineKuery)
@ -97,12 +86,10 @@ describe('test filtering endpoint hosts by agent status', () => {
})
);
const result = await findAgentIDsByStatus(
mockAgentService,
mockSavedObjectClient,
mockElasticsearchClient,
['updating', 'unhealthy']
);
const result = await findAgentIdsByStatus(mockAgentService, mockElasticsearchClient, [
'updating',
'unhealthy',
]);
const unenrollKuery = AgentStatusKueryHelper.buildKueryForUpdatingAgents();
const errorKuery = AgentStatusKueryHelper.buildKueryForErrorAgents();
expect(mockAgentService.listAgents.mock.calls[0][1].kuery).toEqual(
@ -111,4 +98,53 @@ describe('test filtering endpoint hosts by agent status', () => {
expect(result).toBeDefined();
expect(result).toEqual(['A', 'B']);
});
describe('buildStatusesKuery', () => {
it('correctly builds kuery for healthy status', () => {
const status = ['healthy'];
const kuery = buildStatusesKuery(status);
const expected =
'(not (united.agent.last_checkin < now-120s AND not ((united.agent.last_checkin_status:error or united.agent.last_checkin_status:degraded) AND not (((united.agent.upgrade_started_at:*) and not (united.agent.upgraded_at:*)) or (not (united.agent.last_checkin:*)) or (united.agent.unenrollment_started_at:*))) AND not ( ((united.agent.upgrade_started_at:*) and not (united.agent.upgraded_at:*)) or (not (united.agent.last_checkin:*)) or (united.agent.unenrollment_started_at:*) )) AND not ((united.agent.last_checkin_status:error or united.agent.last_checkin_status:degraded) AND not (((united.agent.upgrade_started_at:*) and not (united.agent.upgraded_at:*)) or (not (united.agent.last_checkin:*)) or (united.agent.unenrollment_started_at:*))) AND not (((united.agent.upgrade_started_at:*) and not (united.agent.upgraded_at:*)) or (not (united.agent.last_checkin:*)) or (united.agent.unenrollment_started_at:*)))';
expect(kuery).toEqual(expected);
});
it('correctly builds kuery for offline status', () => {
const status = ['offline'];
const kuery = buildStatusesKuery(status);
const expected =
'(united.agent.last_checkin < now-120s AND not ((united.agent.last_checkin_status:error or united.agent.last_checkin_status:degraded) AND not (((united.agent.upgrade_started_at:*) and not (united.agent.upgraded_at:*)) or (not (united.agent.last_checkin:*)) or (united.agent.unenrollment_started_at:*))) AND not ( ((united.agent.upgrade_started_at:*) and not (united.agent.upgraded_at:*)) or (not (united.agent.last_checkin:*)) or (united.agent.unenrollment_started_at:*) ))';
expect(kuery).toEqual(expected);
});
it('correctly builds kuery for unhealthy status', () => {
const status = ['unhealthy'];
const kuery = buildStatusesKuery(status);
const expected =
'((united.agent.last_checkin_status:error or united.agent.last_checkin_status:degraded) AND not (((united.agent.upgrade_started_at:*) and not (united.agent.upgraded_at:*)) or (not (united.agent.last_checkin:*)) or (united.agent.unenrollment_started_at:*)))';
expect(kuery).toEqual(expected);
});
it('correctly builds kuery for updating status', () => {
const status = ['updating'];
const kuery = buildStatusesKuery(status);
const expected =
'(((united.agent.upgrade_started_at:*) and not (united.agent.upgraded_at:*)) or (not (united.agent.last_checkin:*)) or (united.agent.unenrollment_started_at:*))';
expect(kuery).toEqual(expected);
});
it('correctly builds kuery for inactive status', () => {
const status = ['inactive'];
const kuery = buildStatusesKuery(status);
const expected = '(united.agent.active:false)';
expect(kuery).toEqual(expected);
});
it('correctly builds kuery for multiple statuses', () => {
const statuses = ['offline', 'unhealthy'];
const kuery = buildStatusesKuery(statuses);
const expected =
'(united.agent.last_checkin < now-120s AND not ((united.agent.last_checkin_status:error or united.agent.last_checkin_status:degraded) AND not (((united.agent.upgrade_started_at:*) and not (united.agent.upgraded_at:*)) or (not (united.agent.last_checkin:*)) or (united.agent.unenrollment_started_at:*))) AND not ( ((united.agent.upgrade_started_at:*) and not (united.agent.upgraded_at:*)) or (not (united.agent.last_checkin:*)) or (united.agent.unenrollment_started_at:*) ) OR (united.agent.last_checkin_status:error or united.agent.last_checkin_status:degraded) AND not (((united.agent.upgrade_started_at:*) and not (united.agent.upgraded_at:*)) or (not (united.agent.last_checkin:*)) or (united.agent.unenrollment_started_at:*)))';
expect(kuery).toEqual(expected);
});
});
});

41
x-pack/plugins/security_solution/server/endpoint/routes/metadata/support/agent_status.ts
View file

@ -5,28 +5,45 @@
* 2.0.
*/
import { ElasticsearchClient, SavedObjectsClientContract } from 'kibana/server';
import { ElasticsearchClient } from 'kibana/server';
import { AgentService } from '../../../../../../fleet/server';
import { AgentStatusKueryHelper } from '../../../../../../fleet/common/services';
import { Agent } from '../../../../../../fleet/common/types/models';
import { HostStatus } from '../../../../../common/endpoint/types';
const STATUS_QUERY_MAP = new Map([
[HostStatus.HEALTHY.toString(), AgentStatusKueryHelper.buildKueryForOnlineAgents()],
[HostStatus.OFFLINE.toString(), AgentStatusKueryHelper.buildKueryForOfflineAgents()],
[HostStatus.UNHEALTHY.toString(), AgentStatusKueryHelper.buildKueryForErrorAgents()],
[HostStatus.UPDATING.toString(), AgentStatusKueryHelper.buildKueryForUpdatingAgents()],
[HostStatus.INACTIVE.toString(), AgentStatusKueryHelper.buildKueryForInactiveAgents()],
]);
const getStatusQueryMap = (path: string = '') =>
new Map([
[HostStatus.HEALTHY.toString(), AgentStatusKueryHelper.buildKueryForOnlineAgents(path)],
[HostStatus.OFFLINE.toString(), AgentStatusKueryHelper.buildKueryForOfflineAgents(path)],
[HostStatus.UNHEALTHY.toString(), AgentStatusKueryHelper.buildKueryForErrorAgents(path)],
[HostStatus.UPDATING.toString(), AgentStatusKueryHelper.buildKueryForUpdatingAgents(path)],
[HostStatus.INACTIVE.toString(), AgentStatusKueryHelper.buildKueryForInactiveAgents(path)],
]);
export async function findAgentIDsByStatus(
export function buildStatusesKuery(statusesToFilter: string[]): string | undefined {
if (!statusesToFilter.length) {
return;
}
const STATUS_QUERY_MAP = getStatusQueryMap('united.agent.');
const statusQueries = statusesToFilter.map((status) => STATUS_QUERY_MAP.get(status));
if (!statusQueries.length) {
return;
}
return `(${statusQueries.join(' OR ')})`;
}
export async function findAgentIdsByStatus(
agentService: AgentService,
soClient: SavedObjectsClientContract,
esClient: ElasticsearchClient,
status: string[],
statuses: string[],
pageSize: number = 1000
): Promise<string[]> {
const helpers = status.map((s) => STATUS_QUERY_MAP.get(s));
if (!statuses.length) {
return [];
}
const STATUS_QUERY_MAP = getStatusQueryMap();
const helpers = statuses.map((s) => STATUS_QUERY_MAP.get(s));
const searchOptions = (pageNum: number) => {
return {
page: pageNum,

52
x-pack/plugins/security_solution/server/endpoint/routes/metadata/support/endpoint_package_policies.test.ts Normal file
View file

@ -0,0 +1,52 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { SavedObjectsClientContract } from 'kibana/server';
import { savedObjectsClientMock } from '../../../../../../../../src/core/server/mocks';
import { createPackagePolicyServiceMock } from '../../../../../../fleet/server/mocks';
import { PackagePolicy } from '../../../../../../fleet/common/types/models';
import { PackagePolicyServiceInterface } from '../../../../../../fleet/server';
import { getAllEndpointPackagePolicies } from './endpoint_package_policies';
describe('endpoint_package_policies', () => {
describe('getAllEndpointPackagePolicies', () => {
let mockSavedObjectClient: jest.Mocked<SavedObjectsClientContract>;
let mockPackagePolicyService: jest.Mocked<PackagePolicyServiceInterface>;
beforeEach(() => {
mockSavedObjectClient = savedObjectsClientMock.create();
mockPackagePolicyService = createPackagePolicyServiceMock();
});
it('gets all endpoint package policies', async () => {
const mockPolicy: PackagePolicy = {
id: '1',
policy_id: 'test-id-1',
} as PackagePolicy;
mockPackagePolicyService.list
.mockResolvedValueOnce({
items: [mockPolicy],
total: 1,
perPage: 10,
page: 1,
})
.mockResolvedValueOnce({
items: [],
total: 1,
perPage: 10,
page: 1,
});
const endpointPackagePolicies = await getAllEndpointPackagePolicies(
mockPackagePolicyService,
mockSavedObjectClient
);
const expected: PackagePolicy[] = [mockPolicy];
expect(endpointPackagePolicies).toEqual(expected);
});
});
});

35
x-pack/plugins/security_solution/server/endpoint/routes/metadata/support/endpoint_package_policies.ts Normal file
View file

@ -0,0 +1,35 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { SavedObjectsClientContract } from 'kibana/server';
import { PackagePolicyServiceInterface } from '../../../../../../fleet/server';
import { PackagePolicy } from '../../../../../../fleet/common/types/models';
export const getAllEndpointPackagePolicies = async (
packagePolicyService: PackagePolicyServiceInterface,
soClient: SavedObjectsClientContract
): Promise<PackagePolicy[]> => {
const result: PackagePolicy[] = [];
const perPage = 1000;
let page = 1;
let hasMore = true;
while (hasMore) {
const endpointPoliciesResponse = await packagePolicyService.list(soClient, {
perPage,
page: page++,
kuery: 'ingest-package-policies.package.name:endpoint',
});
if (endpointPoliciesResponse.items.length > 0) {
result.push(...endpointPoliciesResponse.items);
} else {
hasMore = false;
}
}
return result;
};

48
x-pack/plugins/security_solution/server/endpoint/routes/metadata/support/test_support.ts
View file

@ -6,9 +6,10 @@
*/
import type { estypes } from '@elastic/elasticsearch';
import { HostMetadata } from '../../../../../common/endpoint/types';
import { METADATA_UNITED_INDEX } from '../../../../../common/endpoint/constants';
import { HostMetadata, UnitedAgentMetadata } from '../../../../../common/endpoint/types';
export function createV2SearchResponse(
export function legacyMetadataSearchResponse(
hostMetadata?: HostMetadata
): estypes.SearchResponse<HostMetadata> {
return {
@ -42,3 +43,46 @@ export function createV2SearchResponse(
},
} as unknown as estypes.SearchResponse<HostMetadata>;
}
export function unitedMetadataSearchResponse(
hostMetadata?: HostMetadata
): estypes.SearchResponse<UnitedAgentMetadata> {
return {
took: 15,
timed_out: false,
_shards: {
total: 1,
successful: 1,
skipped: 0,
failed: 0,
},
hits: {
total: {
value: 1,
relation: 'eq',
},
max_score: null,
hits: hostMetadata
? [
{
_index: METADATA_UNITED_INDEX,
_id: '8FhM0HEBYyRTvb6lOQnw',
_score: null,
_source: {
agent: {
id: 'test-agent-id',
},
united: {
agent: {},
endpoint: {
...hostMetadata,
},
},
},
sort: [1588337587997],
},
]
: [],
},
} as unknown as estypes.SearchResponse<UnitedAgentMetadata>;
}

22
x-pack/plugins/security_solution/server/endpoint/routes/metadata/support/unenroll.test.ts
View file

@ -5,12 +5,9 @@
* 2.0.
*/
import { ElasticsearchClient, SavedObjectsClientContract } from 'kibana/server';
import { ElasticsearchClient } from 'kibana/server';
import { findAllUnenrolledAgentIds } from './unenroll';
import {
elasticsearchServiceMock,
savedObjectsClientMock,
} from '../../../../../../../../src/core/server/mocks';
import { elasticsearchServiceMock } from '../../../../../../../../src/core/server/mocks';
import { AgentService } from '../../../../../../fleet/server/services';
import {
createMockAgentService,
@ -20,13 +17,11 @@ import { Agent, PackagePolicy } from '../../../../../../fleet/common/types/model
import { PackagePolicyServiceInterface } from '../../../../../../fleet/server';
describe('test find all unenrolled Agent id', () => {
let mockSavedObjectClient: jest.Mocked<SavedObjectsClientContract>;
let mockElasticsearchClient: jest.Mocked<ElasticsearchClient>;
let mockAgentService: jest.Mocked<AgentService>;
let mockPackagePolicyService: jest.Mocked<PackagePolicyServiceInterface>;
beforeEach(() => {
mockSavedObjectClient = savedObjectsClientMock.create();
mockElasticsearchClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
mockAgentService = createMockAgentService();
mockPackagePolicyService = createPackagePolicyServiceMock();
@ -84,26 +79,21 @@ describe('test find all unenrolled Agent id', () => {
perPage: 1,
})
);
const endpointPolicyIds = ['test-endpoint-policy-id'];
const agentIds = await findAllUnenrolledAgentIds(
mockAgentService,
mockPackagePolicyService,
mockSavedObjectClient,
mockElasticsearchClient
mockElasticsearchClient,
endpointPolicyIds
);
expect(agentIds).toBeTruthy();
expect(agentIds).toEqual(['id1', 'id2']);
expect(mockPackagePolicyService.list).toHaveBeenNthCalledWith(1, mockSavedObjectClient, {
kuery: 'ingest-package-policies.package.name:endpoint',
page: 1,
perPage: 1000,
});
expect(mockAgentService.listAgents).toHaveBeenNthCalledWith(1, mockElasticsearchClient, {
page: 1,
perPage: 1000,
showInactive: true,
kuery: '(active : false) OR (active: true AND NOT policy_id:("abc123"))',
kuery: `(active : false) OR (active: true AND NOT policy_id:("${endpointPolicyIds[0]}"))`,
});
});
});

43
x-pack/plugins/security_solution/server/endpoint/routes/metadata/support/unenroll.ts
View file

@ -5,56 +5,23 @@
* 2.0.
*/
import { ElasticsearchClient, SavedObjectsClientContract } from 'kibana/server';
import { AgentService, PackagePolicyServiceInterface } from '../../../../../../fleet/server';
import { ElasticsearchClient } from 'kibana/server';
import { AgentService } from '../../../../../../fleet/server';
import { Agent } from '../../../../../../fleet/common/types/models';
const getAllAgentPolicyIdsWithEndpoint = async (
packagePolicyService: PackagePolicyServiceInterface,
soClient: SavedObjectsClientContract
): Promise<string[]> => {
const result: string[] = [];
const perPage = 1000;
let page = 1;
let hasMore = true;
while (hasMore) {
const endpointPoliciesResponse = await packagePolicyService.list(soClient, {
perPage,
page: page++,
kuery: 'ingest-package-policies.package.name:endpoint',
});
if (endpointPoliciesResponse.items.length > 0) {
result.push(
...endpointPoliciesResponse.items.map((endpointPolicy) => endpointPolicy.policy_id)
);
} else {
hasMore = false;
}
}
return result;
};
export async function findAllUnenrolledAgentIds(
agentService: AgentService,
packagePolicyService: PackagePolicyServiceInterface,
soClient: SavedObjectsClientContract,
esClient: ElasticsearchClient,
endpointPolicyIds: string[],
pageSize: number = 1000
): Promise<string[]> {
const agentPoliciesWithEndpoint = await getAllAgentPolicyIdsWithEndpoint(
packagePolicyService,
soClient
);
// We want:
// 1. if no endpoint policies exist, then get all Agents
// 2. if we have a list of agent policies, then Agents that are Active and that are
// NOT enrolled with an Agent Policy that has endpoint
const kuery =
agentPoliciesWithEndpoint.length > 0
? `(active : false) OR (active: true AND NOT policy_id:("${agentPoliciesWithEndpoint.join(
endpointPolicyIds.length > 0
? `(active : false) OR (active: true AND NOT policy_id:("${endpointPolicyIds.join(
'" OR "'
)}"))`
: undefined;

4
x-pack/plugins/security_solution/server/endpoint/services/metadata/endpoint_metadata_service.test.ts
View file

@ -12,7 +12,7 @@ import {
import { elasticsearchServiceMock } from '../../../../../../../src/core/server/mocks';
// eslint-disable-next-line @kbn/eslint/no-restricted-paths
import { ElasticsearchClientMock } from '../../../../../../../src/core/server/elasticsearch/client/mocks';
import { createV2SearchResponse } from '../../routes/metadata/support/test_support';
import { legacyMetadataSearchResponse } from '../../routes/metadata/support/test_support';
import { EndpointDocGenerator } from '../../../../common/endpoint/generate_data';
import { getESQueryHostMetadataByFleetAgentIds } from '../../routes/metadata/query_builders';
import { EndpointError } from '../../errors';
@ -38,7 +38,7 @@ describe('EndpointMetadataService', () => {
endpointMetadataDoc = new EndpointDocGenerator().generateHostMetadata();
esClient.search.mockReturnValue(
elasticsearchServiceMock.createSuccessTransportRequestPromise(
createV2SearchResponse(endpointMetadataDoc)
legacyMetadataSearchResponse(endpointMetadataDoc)
)
);
});

22
x-pack/test/security_solution_endpoint_api_int/apis/data_stream_helper.ts
View file

@ -15,9 +15,9 @@ import {
telemetryIndexPattern,
} from '../../../plugins/security_solution/common/endpoint/constants';
export async function deleteDataStream(getService: (serviceName: 'es') => Client, index: string) {
export function deleteDataStream(getService: (serviceName: 'es') => Client, index: string) {
const client = getService('es');
await client.transport.request(
return client.transport.request(
{
method: 'DELETE',
path: `_data_stream/${index}`,
@ -41,6 +41,8 @@ export async function deleteAllDocsFromIndex(
},
},
index: `${index}`,
wait_for_completion: true,
refresh: true,
},
{
ignore: [404],
@ -48,6 +50,11 @@ export async function deleteAllDocsFromIndex(
);
}
export async function deleteIndex(getService: (serviceName: 'es') => Client, index: string) {
const client = getService('es');
await client.indices.delete({ index, ignore_unavailable: true });
}
export async function deleteMetadataStream(getService: (serviceName: 'es') => Client) {
await deleteDataStream(getService, metadataIndexPattern);
}
@ -77,3 +84,14 @@ export async function deletePolicyStream(getService: (serviceName: 'es') => Clie
export async function deleteTelemetryStream(getService: (serviceName: 'es') => Client) {
await deleteDataStream(getService, telemetryIndexPattern);
}
export function stopTransform(getService: (serviceName: 'es') => Client, transformId: string) {
const client = getService('es');
const stopRequest = {
transform_id: transformId,
force: true,
wait_for_completion: true,
allow_no_match: true,
};
return client.transform.stopTransform(stopRequest);
}

37
x-pack/test/security_solution_endpoint_api_int/apis/metadata.ts
View file

@ -11,21 +11,34 @@ import {
deleteAllDocsFromMetadataCurrentIndex,
deleteAllDocsFromMetadataIndex,
deleteMetadataStream,
deleteIndex,
stopTransform,
} from './data_stream_helper';
import { HOST_METADATA_LIST_ROUTE } from '../../../plugins/security_solution/common/endpoint/constants';
/**
* The number of host documents in the es archive.
*/
const numberOfHostsInFixture = 3;
import {
HOST_METADATA_LIST_ROUTE,
METADATA_UNITED_INDEX,
METADATA_UNITED_TRANSFORM,
} from '../../../plugins/security_solution/common/endpoint/constants';
export default function ({ getService }: FtrProviderContext) {
const esArchiver = getService('esArchiver');
const supertest = getService('supertest');
describe('test metadata api', () => {
// TODO add this after endpoint package changes are merged and in snapshot
// describe('with .metrics-endpoint.metadata_united_default index', () => {
// });
describe('with metrics-endpoint.metadata_current_default index', () => {
/**
* The number of host documents in the es archive.
*/
const numberOfHostsInFixture = 3;
describe(`POST ${HOST_METADATA_LIST_ROUTE} when index is empty`, () => {
it('metadata api should return empty result when index is empty', async () => {
await stopTransform(getService, `${METADATA_UNITED_TRANSFORM}*`);
await deleteIndex(getService, METADATA_UNITED_INDEX);
await deleteMetadataStream(getService);
await deleteAllDocsFromMetadataIndex(getService);
await deleteAllDocsFromMetadataCurrentIndex(getService);
@ -43,9 +56,16 @@ export default function ({ getService }: FtrProviderContext) {
describe(`POST ${HOST_METADATA_LIST_ROUTE} when index is not empty`, () => {
before(async () => {
await esArchiver.load('x-pack/test/functional/es_archives/endpoint/metadata/api_feature', {
// stop the united transform and delete the index
// otherwise it won't hit metrics-endpoint.metadata_current_default index
await stopTransform(getService, `${METADATA_UNITED_TRANSFORM}*`);
await deleteIndex(getService, METADATA_UNITED_INDEX);
await esArchiver.load(
'x-pack/test/functional/es_archives/endpoint/metadata/api_feature',
{
useCreate: true,
});
}
);
// wait for transform
await new Promise((r) => setTimeout(r, 120000));
});
@ -287,4 +307,5 @@ export default function ({ getService }: FtrProviderContext) {
});
});
});
});
}