github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 09:19:04 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Security Solution][Artifacts] implemented policy specific trusted apps support in the manifest manager (#90991)

Browse source 
* Implemented policy specific trusted apps support in the manifest manager.
This commit is contained in:
Bohdan Tsymbala 2021-02-16 20:31:36 +01:00 • committed by GitHub
parent 312351c52c
commit e81b5c1e40
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 1093 additions and 644 deletions
Download patch file Download diff file Expand all files Collapse all files

7
x-pack/plugins/fleet/common/types/rest_spec/common.ts
View file

@ -14,3 +14,10 @@ export interface ListWithKuery extends HttpFetchQuery {
sortOrder?: 'desc' | 'asc';
kuery?: string;
}
export interface ListResult<T> {
items: T[];
total: number;
page: number;
perPage: number;
}

1
x-pack/plugins/fleet/server/mocks.ts
View file

@ -53,6 +53,7 @@ export const createPackagePolicyServiceMock = () => {
get: jest.fn(),
getByIDs: jest.fn(),
list: jest.fn(),
listIds: jest.fn(),
update: jest.fn(),
runExternalCallbacks: jest.fn(),
} as jest.Mocked<PackagePolicyServiceInterface>;

1
x-pack/plugins/fleet/server/routes/package_policy/handlers.test.ts
View file

@ -47,6 +47,7 @@ jest.mock('../../services/package_policy', (): {
get: jest.fn(),
getByIDs: jest.fn(),
list: jest.fn(),
listIds: jest.fn(),
update: jest.fn(),
runExternalCallbacks: jest.fn((callbackType, newPackagePolicy, context, request) =>
Promise.resolve(newPackagePolicy)

27
x-pack/plugins/fleet/server/services/package_policy.ts
View file

@ -20,6 +20,7 @@ import {
PackagePolicyInputStream,
PackageInfo,
ListWithKuery,
ListResult,
packageToPackagePolicy,
isPackageLimited,
doesAgentPolicyAlreadyIncludePackage,
@ -248,7 +249,7 @@ class PackagePolicyService {
public async list(
soClient: SavedObjectsClientContract,
options: ListWithKuery
): Promise<{ items: PackagePolicy[]; total: number; page: number; perPage: number }> {
): Promise<ListResult<PackagePolicy>> {
const { page = 1, perPage = 20, sortField = 'updated_at', sortOrder = 'desc', kuery } = options;
const packagePolicies = await soClient.find<PackagePolicySOAttributes>({
@ -272,6 +273,30 @@ class PackagePolicyService {
};
}
public async listIds(
soClient: SavedObjectsClientContract,
options: ListWithKuery
): Promise<ListResult<string>> {
const { page = 1, perPage = 20, sortField = 'updated_at', sortOrder = 'desc', kuery } = options;
const packagePolicies = await soClient.find<{}>({
type: SAVED_OBJECT_TYPE,
sortField,
sortOrder,
page,
perPage,
fields: [],
filter: kuery ? normalizeKuery(SAVED_OBJECT_TYPE, kuery) : undefined,
});
return {
items: packagePolicies.saved_objects.map((packagePolicySO) => packagePolicySO.id),
total: packagePolicies.total,
page,
perPage,
};
}
public async update(
soClient: SavedObjectsClientContract,
esClient: ElasticsearchClient,

2
x-pack/plugins/lists/common/shared_exports.ts
View file

@ -47,4 +47,4 @@ export {
OsTypeArray,
} from './schemas';
export { ENDPOINT_LIST_ID } from './constants';
export { ENDPOINT_LIST_ID, ENDPOINT_TRUSTED_APPS_LIST_ID } from './constants';

251
x-pack/plugins/lists/server/saved_objects/migrations.test.ts
View file

@ -6,61 +6,102 @@
*/
import { SavedObjectUnsanitizedDoc } from 'kibana/server';
import uuid from 'uuid';
import { ENDPOINT_LIST_ID } from '../../common/constants';
import { ENDPOINT_LIST_ID, ENDPOINT_TRUSTED_APPS_LIST_ID } from '../../common/constants';
import { ExceptionListSoSchema } from '../../common/schemas/saved_objects';
import { OldExceptionListSoSchema, migrations } from './migrations';
const DEFAULT_EXCEPTION_LIST_SO: ExceptionListSoSchema = {
comments: undefined,
created_at: '2020-06-09T20:18:20.349Z',
created_by: 'user',
description: 'description',
entries: undefined,
immutable: false,
item_id: undefined,
list_id: 'some_list',
list_type: 'list',
meta: undefined,
name: 'name',
os_types: [],
tags: [],
tie_breaker_id: uuid.v4(),
type: 'endpoint',
updated_by: 'user',
version: undefined,
};
const DEFAULT_OLD_EXCEPTION_LIST_SO: OldExceptionListSoSchema = {
...DEFAULT_EXCEPTION_LIST_SO,
_tags: [],
};
const createOldExceptionListSoSchemaSavedObject = (
attributes: Partial<OldExceptionListSoSchema>
): SavedObjectUnsanitizedDoc<OldExceptionListSoSchema> => ({
attributes: { ...DEFAULT_OLD_EXCEPTION_LIST_SO, ...attributes },
id: 'abcd',
migrationVersion: {},
references: [],
type: 'so-type',
updated_at: '2020-06-09T20:18:20.349Z',
});
const createExceptionListSoSchemaSavedObject = (
attributes: Partial<ExceptionListSoSchema>
): SavedObjectUnsanitizedDoc<ExceptionListSoSchema> => ({
attributes: { ...DEFAULT_EXCEPTION_LIST_SO, ...attributes },
id: 'abcd',
migrationVersion: {},
references: [],
type: 'so-type',
updated_at: '2020-06-09T20:18:20.349Z',
});
describe('7.10.0 lists migrations', () => {
const migration = migrations['7.10.0'];
test('properly converts .text fields to .caseless', () => {
const doc = {
attributes: {
entries: [
{
field: 'file.path.text',
operator: 'included',
type: 'match',
value: 'C:\\Windows\\explorer.exe',
},
{
field: 'host.os.name',
operator: 'included',
type: 'match',
value: 'my-host',
},
{
entries: [
{
field: 'process.command_line.text',
operator: 'included',
type: 'match',
value: '/usr/bin/bash',
},
{
field: 'process.parent.command_line.text',
operator: 'included',
type: 'match',
value: '/usr/bin/bash',
},
],
field: 'nested.field',
type: 'nested',
},
],
list_id: ENDPOINT_LIST_ID,
},
id: 'abcd',
migrationVersion: {},
references: [],
type: 'so-type',
updated_at: '2020-06-09T20:18:20.349Z',
};
expect(
migration((doc as unknown) as SavedObjectUnsanitizedDoc<OldExceptionListSoSchema>)
).toEqual({
attributes: {
const doc = createOldExceptionListSoSchemaSavedObject({
entries: [
{
field: 'file.path.text',
operator: 'included',
type: 'match',
value: 'C:\\Windows\\explorer.exe',
},
{
field: 'host.os.name',
operator: 'included',
type: 'match',
value: 'my-host',
},
{
entries: [
{
field: 'process.command_line.text',
operator: 'included',
type: 'match',
value: '/usr/bin/bash',
},
{
field: 'process.parent.command_line.text',
operator: 'included',
type: 'match',
value: '/usr/bin/bash',
},
],
field: 'nested.field',
type: 'nested',
},
],
list_id: ENDPOINT_LIST_ID,
});
expect(migration(doc)).toEqual(
createOldExceptionListSoSchemaSavedObject({
entries: [
{
field: 'file.path.caseless',
@ -94,40 +135,98 @@ describe('7.10.0 lists migrations', () => {
},
],
list_id: ENDPOINT_LIST_ID,
},
id: 'abcd',
migrationVersion: {},
references: [],
type: 'so-type',
updated_at: '2020-06-09T20:18:20.349Z',
});
})
);
});
test('properly copies os tags to os_types', () => {
const doc = {
attributes: {
_tags: ['1234', 'os:windows'],
comments: [],
},
id: 'abcd',
migrationVersion: {},
references: [],
type: 'so-type',
updated_at: '2020-06-09T20:18:20.349Z',
};
expect(
migration((doc as unknown) as SavedObjectUnsanitizedDoc<OldExceptionListSoSchema>)
).toEqual({
attributes: {
const doc = createOldExceptionListSoSchemaSavedObject({
_tags: ['1234', 'os:windows'],
comments: [],
});
expect(migration(doc)).toEqual(
createOldExceptionListSoSchemaSavedObject({
_tags: ['1234', 'os:windows'],
comments: [],
os_types: ['windows'],
},
id: 'abcd',
migrationVersion: {},
references: [],
type: 'so-type',
updated_at: '2020-06-09T20:18:20.349Z',
});
})
);
});
});
describe('7.12.0 lists migrations', () => {
const migration = migrations['7.12.0'];
test('should not convert non trusted apps lists', () => {
const doc = createExceptionListSoSchemaSavedObject({ list_id: ENDPOINT_LIST_ID, tags: [] });
expect(migration(doc)).toEqual(
createExceptionListSoSchemaSavedObject({
list_id: ENDPOINT_LIST_ID,
tags: [],
tie_breaker_id: expect.anything(),
})
);
});
test('converts empty tags to contain list containing "policy:all" tag', () => {
const doc = createExceptionListSoSchemaSavedObject({
list_id: ENDPOINT_TRUSTED_APPS_LIST_ID,
tags: [],
});
expect(migration(doc)).toEqual(
createExceptionListSoSchemaSavedObject({
list_id: ENDPOINT_TRUSTED_APPS_LIST_ID,
tags: ['policy:all'],
tie_breaker_id: expect.anything(),
})
);
});
test('preserves existing non policy related tags', () => {
const doc = createExceptionListSoSchemaSavedObject({
list_id: ENDPOINT_TRUSTED_APPS_LIST_ID,
tags: ['tag1', 'tag2'],
});
expect(migration(doc)).toEqual(
createExceptionListSoSchemaSavedObject({
list_id: ENDPOINT_TRUSTED_APPS_LIST_ID,
tags: ['tag1', 'tag2', 'policy:all'],
tie_breaker_id: expect.anything(),
})
);
});
test('preserves existing "policy:all" tag and does not add another one', () => {
const doc = createExceptionListSoSchemaSavedObject({
list_id: ENDPOINT_TRUSTED_APPS_LIST_ID,
tags: ['policy:all', 'tag1', 'tag2'],
});
expect(migration(doc)).toEqual(
createExceptionListSoSchemaSavedObject({
list_id: ENDPOINT_TRUSTED_APPS_LIST_ID,
tags: ['policy:all', 'tag1', 'tag2'],
tie_breaker_id: expect.anything(),
})
);
});
test('preserves existing policy reference tag and does not add "policy:all" tag', () => {
const doc = createExceptionListSoSchemaSavedObject({
list_id: ENDPOINT_TRUSTED_APPS_LIST_ID,
tags: ['policy:056d2d4645421fb92e5cd39f33d70856', 'tag1', 'tag2'],
});
expect(migration(doc)).toEqual(
createExceptionListSoSchemaSavedObject({
list_id: ENDPOINT_TRUSTED_APPS_LIST_ID,
tags: ['policy:056d2d4645421fb92e5cd39f33d70856', 'tag1', 'tag2'],
tie_breaker_id: expect.anything(),
})
);
});
});

24
x-pack/plugins/lists/server/saved_objects/migrations.ts
View file

@ -40,6 +40,9 @@ const reduceOsTypes = (acc: string[], tag: string): string[] => {
return [...acc];
};
const containsPolicyTags = (tags: string[]): boolean =>
tags.some((tag) => tag.startsWith('policy:'));
export type OldExceptionListSoSchema = ExceptionListSoSchema & {
_tags: string[];
};
@ -64,4 +67,25 @@ export const migrations = {
},
references: doc.references || [],
}),
'7.12.0': (
doc: SavedObjectUnsanitizedDoc<ExceptionListSoSchema>
): SavedObjectSanitizedDoc<ExceptionListSoSchema> => {
if (doc.attributes.list_id === ENDPOINT_TRUSTED_APPS_LIST_ID) {
return {
...doc,
...{
attributes: {
...doc.attributes,
tags: [
...(doc.attributes.tags || []),
...(containsPolicyTags(doc.attributes.tags) ? [] : ['policy:all']),
],
},
},
references: doc.references || [],
};
} else {
return { ...doc, references: doc.references || [] };
}
},
};

1
x-pack/plugins/security_solution/common/shared_imports.ts
View file

@ -43,6 +43,7 @@ export {
ExceptionListType,
Type,
ENDPOINT_LIST_ID,
ENDPOINT_TRUSTED_APPS_LIST_ID,
osTypeArray,
OsTypeArray,
} from '../../lists/common';

940
x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts
View file

File diff suppressed because it is too large Load diff

40
x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts
View file

@ -12,7 +12,7 @@ import { validate } from '../../../../common/validate';
import { Entry, EntryNested } from '../../../../../lists/common/schemas/types';
import { ExceptionListClient } from '../../../../../lists/server';
import { ENDPOINT_LIST_ID } from '../../../../common/shared_imports';
import { ENDPOINT_LIST_ID, ENDPOINT_TRUSTED_APPS_LIST_ID } from '../../../../common/shared_imports';
import {
InternalArtifactSchema,
TranslatedEntry,
@ -28,12 +28,11 @@ import {
internalArtifactCompleteSchema,
InternalArtifactCompleteSchema,
} from '../../schemas';
import { ENDPOINT_TRUSTED_APPS_LIST_ID } from '../../../../../lists/common/constants';
export async function buildArtifact(
exceptions: WrappedTranslatedExceptionList,
os: string,
schemaVersion: string,
os: string,
name: string
): Promise<InternalArtifactCompleteSchema> {
const exceptionsBuffer = Buffer.from(JSON.stringify(exceptions));
@ -74,10 +73,10 @@ export function isCompressed(artifact: InternalArtifactSchema) {
return artifact.compressionAlgorithm === 'zlib';
}
export async function getFullEndpointExceptionList(
export async function getFilteredEndpointExceptionList(
eClient: ExceptionListClient,
os: string,
schemaVersion: string,
filter: string,
listId: typeof ENDPOINT_LIST_ID | typeof ENDPOINT_TRUSTED_APPS_LIST_ID
): Promise<WrappedTranslatedExceptionList> {
const exceptions: WrappedTranslatedExceptionList = { entries: [] };
@ -88,7 +87,7 @@ export async function getFullEndpointExceptionList(
const response = await eClient.findExceptionListItem({
listId,
namespaceType: 'agnostic',
filter: `exception-list-agnostic.attributes.os_types:\"${os}\"`,
filter,
perPage: 100,
page,
sortField: 'created_at',
@ -114,6 +113,35 @@ export async function getFullEndpointExceptionList(
return validated as WrappedTranslatedExceptionList;
}
export async function getEndpointExceptionList(
eClient: ExceptionListClient,
schemaVersion: string,
os: string
): Promise<WrappedTranslatedExceptionList> {
const filter = `exception-list-agnostic.attributes.os_types:\"${os}\"`;
return getFilteredEndpointExceptionList(eClient, schemaVersion, filter, ENDPOINT_LIST_ID);
}
export async function getEndpointTrustedAppsList(
eClient: ExceptionListClient,
schemaVersion: string,
os: string,
policyId?: string
): Promise<WrappedTranslatedExceptionList> {
const osFilter = `exception-list-agnostic.attributes.os_types:\"${os}\"`;
const policyFilter = `(exception-list-agnostic.attributes.tags:\"policy:all\"${
policyId ? ` or exception-list-agnostic.attributes.tags:\"policy:${policyId}\"` : ''
})`;
return getFilteredEndpointExceptionList(
eClient,
schemaVersion,
`${osFilter} and ${policyFilter}`,
ENDPOINT_TRUSTED_APPS_LIST_ID
);
}
/**
* Translates Exception list items to Exceptions the endpoint can understand
* @param exceptions

4
x-pack/plugins/security_solution/server/endpoint/routes/trusted_apps/mapping.test.ts
View file

@ -35,7 +35,7 @@ const createExceptionListItemOptions = (
name: '',
namespaceType: 'agnostic',
osTypes: [],
tags: [],
tags: ['policy:all'],
type: 'simple',
...options,
});
@ -56,7 +56,7 @@ const exceptionListItemSchema = (
name: '',
namespace_type: 'agnostic',
os_types: [],
tags: [],
tags: ['policy:all'],
type: 'simple',
tie_breaker_id: '123',
updated_at: '11/11/2011T11:11:11.111',

4
x-pack/plugins/security_solution/server/endpoint/routes/trusted_apps/mapping.ts
View file

@ -15,7 +15,7 @@ import {
ExceptionListItemSchema,
NestedEntriesArray,
} from '../../../../../lists/common/shared_exports';
import { ENDPOINT_TRUSTED_APPS_LIST_ID } from '../../../../../lists/common/constants';
import { ENDPOINT_TRUSTED_APPS_LIST_ID } from '../../../../../lists/common';
import { CreateExceptionListItemOptions } from '../../../../../lists/server';
import {
ConditionEntry,
@ -184,7 +184,7 @@ export const newTrustedAppToCreateExceptionListItemOptions = ({
name,
namespaceType: 'agnostic',
osTypes: [OPERATING_SYSTEM_TO_OS_TYPE[os]],
tags: [],
tags: ['policy:all'],
type: 'simple',
};
};

2
x-pack/plugins/security_solution/server/endpoint/routes/trusted_apps/service.ts
View file

@ -6,7 +6,7 @@
*/
import { ExceptionListClient } from '../../../../../lists/server';
import { ENDPOINT_TRUSTED_APPS_LIST_ID } from '../../../../../lists/common/constants';
import { ENDPOINT_TRUSTED_APPS_LIST_ID } from '../../../../../lists/common';
import {
DeleteTrustedAppsRequestParams,

6
x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts
View file

@ -36,8 +36,8 @@ export const getInternalArtifactMock = async (
): Promise<InternalArtifactCompleteSchema> => {
const artifact = await buildArtifact(
getTranslatedExceptionListMock(),
os,
schemaVersion,
os,
artifactName
);
return opts?.compress ? compressArtifact(artifact) : artifact;
@ -49,7 +49,7 @@ export const getEmptyInternalArtifactMock = async (
opts?: { compress: boolean },
artifactName: string = ArtifactConstants.GLOBAL_ALLOWLIST_NAME
): Promise<InternalArtifactCompleteSchema> => {
const artifact = await buildArtifact({ entries: [] }, os, schemaVersion, artifactName);
const artifact = await buildArtifact({ entries: [] }, schemaVersion, os, artifactName);
return opts?.compress ? compressArtifact(artifact) : artifact;
};
@ -62,8 +62,8 @@ export const getInternalArtifactMockWithDiffs = async (
mock.entries.pop();
const artifact = await buildArtifact(
mock,
os,
schemaVersion,
os,
ArtifactConstants.GLOBAL_ALLOWLIST_NAME
);
return opts?.compress ? compressArtifact(artifact) : artifact;

19
x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts
View file

@ -33,17 +33,24 @@ export const createExceptionListResponse = (data: ExceptionListItemSchema[], tot
type FindExceptionListItemOptions = Parameters<ExceptionListClient['findExceptionListItem']>[0];
const FILTER_REGEXP = /^exception-list-agnostic\.attributes\.os_types:"(\w+)"$/;
const FILTER_PROPERTY_PREFIX = 'exception-list-agnostic\\.attributes';
const FILTER_REGEXP = new RegExp(
`^${FILTER_PROPERTY_PREFIX}\.os_types:"([^"]+)"( and \\(${FILTER_PROPERTY_PREFIX}\.tags:"policy:all"( or ${FILTER_PROPERTY_PREFIX}\.tags:"policy:([^"]+)")?\\))?$`
);
export const mockFindExceptionListItemResponses = (
responses: Record<string, Record<string, ExceptionListItemSchema[]>>
) => {
return jest.fn().mockImplementation((options: FindExceptionListItemOptions) => {
const os = FILTER_REGEXP.test(options.filter || '')
? options.filter!.match(FILTER_REGEXP)![1]
: '';
const matches = options.filter!.match(FILTER_REGEXP) || [];
return createExceptionListResponse(responses[options.listId]?.[os] || []);
if (matches[4] && responses[options.listId]?.[`${matches![1]}-${matches[4]}`]) {
return createExceptionListResponse(
responses[options.listId]?.[`${matches![1]}-${matches[4]}`] || []
);
} else {
return createExceptionListResponse(responses[options.listId]?.[matches![1] || ''] || []);
}
});
};
@ -118,7 +125,7 @@ export const getManifestManagerMock = (
context.exceptionListClient.findExceptionListItem = jest
.fn()
.mockRejectedValue(new Error('unexpected thing happened'));
return super.buildExceptionListArtifacts('v1');
return super.buildExceptionListArtifacts();
case ManifestManagerMockType.NormalFlow:
return getMockArtifactsWithDiff();
}

139
x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts
View file

@ -8,8 +8,7 @@
import { inflateSync } from 'zlib';
import { SavedObjectsErrorHelpers } from 'src/core/server';
import { savedObjectsClientMock } from 'src/core/server/mocks';
import { ENDPOINT_LIST_ID } from '../../../../../../lists/common';
import { ENDPOINT_TRUSTED_APPS_LIST_ID } from '../../../../../../lists/common/constants';
import { ENDPOINT_LIST_ID, ENDPOINT_TRUSTED_APPS_LIST_ID } from '../../../../../../lists/common';
import { getExceptionListItemSchemaMock } from '../../../../../../lists/common/schemas/response/exception_list_item_schema.mock';
import { PackagePolicy } from '../../../../../../fleet/common/types/models';
import { getEmptyInternalArtifactMock } from '../../../schemas/artifacts/saved_objects.mock';
@ -211,10 +210,19 @@ describe('ManifestManager', () => {
ARTIFACT_NAME_TRUSTED_APPS_LINUX,
];
const getArtifactIds = (artifacts: InternalArtifactSchema[]) =>
artifacts.map((artifact) => artifact.identifier);
const getArtifactIds = (artifacts: InternalArtifactSchema[]) => [
...new Set(artifacts.map((artifact) => artifact.identifier)).values(),
];
test('Fails when exception list list client fails', async () => {
const mockPolicyListIdsResponse = (items: string[]) =>
jest.fn().mockResolvedValue({
items,
page: 1,
per_page: 100,
total: items.length,
});
test('Fails when exception list client fails', async () => {
const context = buildManifestManagerContextMock({});
const manifestManager = new ManifestManager(context);
@ -228,6 +236,7 @@ describe('ManifestManager', () => {
const manifestManager = new ManifestManager(context);
context.exceptionListClient.findExceptionListItem = mockFindExceptionListItemResponses({});
context.packagePolicyService.listIds = mockPolicyListIdsResponse([TEST_POLICY_ID_1]);
const manifest = await manifestManager.buildNewManifest();
@ -237,11 +246,16 @@ describe('ManifestManager', () => {
const artifacts = manifest.getAllArtifacts();
expect(artifacts.length).toBe(5);
expect(getArtifactIds(artifacts)).toStrictEqual(SUPPORTED_ARTIFACT_NAMES);
expect(artifacts.every(isCompressed)).toBe(true);
for (const artifact of artifacts) {
expect(await uncompressArtifact(artifact)).toStrictEqual({ entries: [] });
expect(manifest.isDefaultArtifact(artifact)).toBe(true);
expect(manifest.getArtifactTargetPolicies(artifact)).toStrictEqual(
new Set([TEST_POLICY_ID_1])
);
}
});
@ -255,6 +269,7 @@ describe('ManifestManager', () => {
[ENDPOINT_LIST_ID]: { macos: [exceptionListItem] },
[ENDPOINT_TRUSTED_APPS_LIST_ID]: { linux: [trustedAppListItem] },
});
context.packagePolicyService.listIds = mockPolicyListIdsResponse([TEST_POLICY_ID_1]);
const manifest = await manifestManager.buildNewManifest();
@ -264,21 +279,25 @@ describe('ManifestManager', () => {
const artifacts = manifest.getAllArtifacts();
expect(artifacts.length).toBe(5);
expect(getArtifactIds(artifacts)).toStrictEqual(SUPPORTED_ARTIFACT_NAMES);
expect(artifacts.every(isCompressed)).toBe(true);
expect(await uncompressArtifact(artifacts[0])).toStrictEqual({
entries: translateToEndpointExceptions([exceptionListItem], 'v1'),
});
expect(await uncompressArtifact(artifacts[1])).toStrictEqual({ entries: [] });
expect(await uncompressArtifact(artifacts[2])).toStrictEqual({ entries: [] });
expect(await uncompressArtifact(artifacts[3])).toStrictEqual({ entries: [] });
expect(await uncompressArtifact(artifacts[4])).toStrictEqual({
entries: translateToEndpointExceptions([trustedAppListItem], 'v1'),
});
for (const artifact of artifacts) {
if (artifact.identifier === ARTIFACT_NAME_EXCEPTIONS_MACOS) {
expect(await uncompressArtifact(artifact)).toStrictEqual({
entries: translateToEndpointExceptions([exceptionListItem], 'v1'),
});
} else if (artifact.identifier === 'endpoint-trustlist-linux-v1') {
expect(await uncompressArtifact(artifact)).toStrictEqual({
entries: translateToEndpointExceptions([trustedAppListItem], 'v1'),
});
} else {
expect(await uncompressArtifact(artifact)).toStrictEqual({ entries: [] });
}
expect(manifest.isDefaultArtifact(artifact)).toBe(true);
expect(manifest.getArtifactTargetPolicies(artifact)).toStrictEqual(
new Set([TEST_POLICY_ID_1])
);
}
});
@ -291,6 +310,7 @@ describe('ManifestManager', () => {
context.exceptionListClient.findExceptionListItem = mockFindExceptionListItemResponses({
[ENDPOINT_LIST_ID]: { macos: [exceptionListItem] },
});
context.packagePolicyService.listIds = mockPolicyListIdsResponse([TEST_POLICY_ID_1]);
const oldManifest = await manifestManager.buildNewManifest();
@ -307,21 +327,90 @@ describe('ManifestManager', () => {
const artifacts = manifest.getAllArtifacts();
expect(artifacts.length).toBe(5);
expect(getArtifactIds(artifacts)).toStrictEqual(SUPPORTED_ARTIFACT_NAMES);
expect(artifacts.every(isCompressed)).toBe(true);
expect(artifacts[0]).toStrictEqual(oldManifest.getAllArtifacts()[0]);
expect(await uncompressArtifact(artifacts[1])).toStrictEqual({ entries: [] });
expect(await uncompressArtifact(artifacts[2])).toStrictEqual({ entries: [] });
expect(await uncompressArtifact(artifacts[3])).toStrictEqual({ entries: [] });
expect(await uncompressArtifact(artifacts[4])).toStrictEqual({
entries: translateToEndpointExceptions([trustedAppListItem], 'v1'),
});
for (const artifact of artifacts) {
if (artifact.identifier === ARTIFACT_NAME_EXCEPTIONS_MACOS) {
expect(artifact).toStrictEqual(oldManifest.getAllArtifacts()[0]);
} else if (artifact.identifier === 'endpoint-trustlist-linux-v1') {
expect(await uncompressArtifact(artifact)).toStrictEqual({
entries: translateToEndpointExceptions([trustedAppListItem], 'v1'),
});
} else {
expect(await uncompressArtifact(artifact)).toStrictEqual({ entries: [] });
}
expect(manifest.isDefaultArtifact(artifact)).toBe(true);
expect(manifest.getArtifactTargetPolicies(artifact)).toStrictEqual(
new Set([TEST_POLICY_ID_1])
);
}
});
test('Builds manifest with policy specific exception list items for trusted apps', async () => {
const exceptionListItem = getExceptionListItemSchemaMock({ os_types: ['macos'] });
const trustedAppListItem = getExceptionListItemSchemaMock({ os_types: ['linux'] });
const trustedAppListItemPolicy2 = getExceptionListItemSchemaMock({
os_types: ['linux'],
entries: [
{ field: 'other.field', operator: 'included', type: 'match', value: 'other value' },
],
});
const context = buildManifestManagerContextMock({});
const manifestManager = new ManifestManager(context);
context.exceptionListClient.findExceptionListItem = mockFindExceptionListItemResponses({
[ENDPOINT_LIST_ID]: { macos: [exceptionListItem] },
[ENDPOINT_TRUSTED_APPS_LIST_ID]: {
linux: [trustedAppListItem],
[`linux-${TEST_POLICY_ID_2}`]: [trustedAppListItem, trustedAppListItemPolicy2],
},
});
context.packagePolicyService.listIds = mockPolicyListIdsResponse([
TEST_POLICY_ID_1,
TEST_POLICY_ID_2,
]);
const manifest = await manifestManager.buildNewManifest();
expect(manifest?.getSchemaVersion()).toStrictEqual('v1');
expect(manifest?.getSemanticVersion()).toStrictEqual('1.0.0');
expect(manifest?.getSavedObjectVersion()).toBeUndefined();
const artifacts = manifest.getAllArtifacts();
expect(artifacts.length).toBe(6);
expect(getArtifactIds(artifacts)).toStrictEqual(SUPPORTED_ARTIFACT_NAMES);
expect(artifacts.every(isCompressed)).toBe(true);
expect(await uncompressArtifact(artifacts[0])).toStrictEqual({
entries: translateToEndpointExceptions([exceptionListItem], 'v1'),
});
expect(await uncompressArtifact(artifacts[1])).toStrictEqual({ entries: [] });
expect(await uncompressArtifact(artifacts[2])).toStrictEqual({ entries: [] });
expect(await uncompressArtifact(artifacts[3])).toStrictEqual({ entries: [] });
expect(await uncompressArtifact(artifacts[4])).toStrictEqual({
entries: translateToEndpointExceptions([trustedAppListItem], 'v1'),
});
expect(await uncompressArtifact(artifacts[5])).toStrictEqual({
entries: translateToEndpointExceptions(
[trustedAppListItem, trustedAppListItemPolicy2],
'v1'
),
});
for (const artifact of artifacts.slice(0, 4)) {
expect(manifest.isDefaultArtifact(artifact)).toBe(true);
expect(manifest.getArtifactTargetPolicies(artifact)).toStrictEqual(
new Set([TEST_POLICY_ID_1, TEST_POLICY_ID_2])
);
}
expect(manifest.isDefaultArtifact(artifacts[5])).toBe(false);
expect(manifest.getArtifactTargetPolicies(artifacts[5])).toStrictEqual(
new Set([TEST_POLICY_ID_2])
);
});
});
describe('deleteArtifacts', () => {

269
x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts
View file

@ -9,6 +9,7 @@ import semver from 'semver';
import LRU from 'lru-cache';
import { isEqual } from 'lodash';
import { Logger, SavedObjectsClientContract } from 'src/core/server';
import { ListResult } from '../../../../../../fleet/common';
import { PackagePolicyServiceInterface } from '../../../../../../fleet/server';
import { ExceptionListClient } from '../../../../../../lists/server';
import { ManifestSchemaVersion } from '../../../../../common/endpoint/schema/common';
@ -21,7 +22,8 @@ import {
ArtifactConstants,
buildArtifact,
getArtifactId,
getFullEndpointExceptionList,
getEndpointExceptionList,
getEndpointTrustedAppsList,
isCompressed,
Manifest,
maybeCompressArtifact,
@ -32,9 +34,45 @@ import {
} from '../../../schemas/artifacts';
import { ArtifactClient } from '../artifact_client';
import { ManifestClient } from '../manifest_client';
import { ENDPOINT_LIST_ID } from '../../../../../../lists/common';
import { ENDPOINT_TRUSTED_APPS_LIST_ID } from '../../../../../../lists/common/constants';
import { PackagePolicy } from '../../../../../../fleet/common/types/models';
interface ArtifactsBuildResult {
defaultArtifacts: InternalArtifactCompleteSchema[];
policySpecificArtifacts: Record<string, InternalArtifactCompleteSchema[]>;
}
const iterateArtifactsBuildResult = async (
result: ArtifactsBuildResult,
callback: (artifact: InternalArtifactCompleteSchema, policyId?: string) => Promise<void>
) => {
for (const artifact of result.defaultArtifacts) {
await callback(artifact);
}
for (const policyId of Object.keys(result.policySpecificArtifacts)) {
for (const artifact of result.policySpecificArtifacts[policyId]) {
await callback(artifact, policyId);
}
}
};
const iterateAllListItems = async <T>(
pageSupplier: (page: number) => Promise<ListResult<T>>,
itemCallback: (item: T) => void
) => {
let paging = true;
let page = 1;
while (paging) {
const { items, total } = await pageSupplier(page);
for (const item of items) {
await itemCallback(item);
}
paging = (page - 1) * 20 + items.length < total;
page++;
}
};
export interface ManifestManagerContext {
savedObjectsClient: SavedObjectsClientContract;
@ -81,6 +119,19 @@ export class ManifestManager {
return new ManifestClient(this.savedObjectsClient, this.schemaVersion);
}
/**
* Builds an artifact (one per supported OS) based on the current
* state of exception-list-agnostic SOs.
*/
protected async buildExceptionListArtifact(os: string): Promise<InternalArtifactCompleteSchema> {
return buildArtifact(
await getEndpointExceptionList(this.exceptionListClient, this.schemaVersion, os),
this.schemaVersion,
os,
ArtifactConstants.GLOBAL_ALLOWLIST_NAME
);
}
/**
* Builds an array of artifacts (one per supported OS) based on the current
* state of exception-list-agnostic SOs.
@ -88,54 +139,60 @@ export class ManifestManager {
* @returns {Promise<InternalArtifactCompleteSchema[]>} An array of uncompressed artifacts built from exception-list-agnostic SOs.
* @throws Throws/rejects if there are errors building the list.
*/
protected async buildExceptionListArtifacts(
artifactSchemaVersion?: string
): Promise<InternalArtifactCompleteSchema[]> {
const artifacts: InternalArtifactCompleteSchema[] = [];
protected async buildExceptionListArtifacts(): Promise<ArtifactsBuildResult> {
const defaultArtifacts: InternalArtifactCompleteSchema[] = [];
const policySpecificArtifacts: Record<string, InternalArtifactCompleteSchema[]> = {};
for (const os of ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS) {
const exceptionList = await getFullEndpointExceptionList(
this.exceptionListClient,
os,
artifactSchemaVersion ?? 'v1',
ENDPOINT_LIST_ID
);
const artifact = await buildArtifact(
exceptionList,
os,
artifactSchemaVersion ?? 'v1',
ArtifactConstants.GLOBAL_ALLOWLIST_NAME
);
artifacts.push(artifact);
defaultArtifacts.push(await this.buildExceptionListArtifact(os));
}
return artifacts;
await iterateAllListItems(
(page) => this.listEndpointPolicyIds(page),
async (policyId) => {
policySpecificArtifacts[policyId] = defaultArtifacts;
}
);
return { defaultArtifacts, policySpecificArtifacts };
}
/**
* Builds an artifact (one per supported OS) based on the current state of the
* Trusted Apps list (which uses the `exception-list-agnostic` SO type)
*/
protected async buildTrustedAppsArtifact(os: string, policyId?: string) {
return buildArtifact(
await getEndpointTrustedAppsList(this.exceptionListClient, this.schemaVersion, os, policyId),
this.schemaVersion,
os,
ArtifactConstants.GLOBAL_TRUSTED_APPS_NAME
);
}
/**
* Builds an array of artifacts (one per supported OS) based on the current state of the
* Trusted Apps list (which uses the `exception-list-agnostic` SO type)
* @param artifactSchemaVersion
*/
protected async buildTrustedAppsArtifacts(
artifactSchemaVersion?: string
): Promise<InternalArtifactCompleteSchema[]> {
const artifacts: InternalArtifactCompleteSchema[] = [];
protected async buildTrustedAppsArtifacts(): Promise<ArtifactsBuildResult> {
const defaultArtifacts: InternalArtifactCompleteSchema[] = [];
const policySpecificArtifacts: Record<string, InternalArtifactCompleteSchema[]> = {};
for (const os of ArtifactConstants.SUPPORTED_TRUSTED_APPS_OPERATING_SYSTEMS) {
const trustedApps = await getFullEndpointExceptionList(
this.exceptionListClient,
os,
artifactSchemaVersion ?? 'v1',
ENDPOINT_TRUSTED_APPS_LIST_ID
);
const artifact = await buildArtifact(
trustedApps,
os,
'v1',
ArtifactConstants.GLOBAL_TRUSTED_APPS_NAME
);
artifacts.push(artifact);
defaultArtifacts.push(await this.buildTrustedAppsArtifact(os));
}
return artifacts;
await iterateAllListItems(
(page) => this.listEndpointPolicyIds(page),
async (policyId) => {
for (const os of ArtifactConstants.SUPPORTED_TRUSTED_APPS_OPERATING_SYSTEMS) {
policySpecificArtifacts[policyId] = policySpecificArtifacts[policyId] || [];
policySpecificArtifacts[policyId].push(await this.buildTrustedAppsArtifact(os, policyId));
}
}
);
return { defaultArtifacts, policySpecificArtifacts };
}
/**
@ -251,32 +308,33 @@ export class ManifestManager {
public async buildNewManifest(
baselineManifest: Manifest = Manifest.getDefault(this.schemaVersion)
): Promise<Manifest> {
// Build new exception list artifacts
const artifacts = (
await Promise.all([this.buildExceptionListArtifacts(), this.buildTrustedAppsArtifacts()])
).flat();
const results = await Promise.all([
this.buildExceptionListArtifacts(),
this.buildTrustedAppsArtifacts(),
]);
// Build new manifest
const manifest = new Manifest({
schemaVersion: this.schemaVersion,
semanticVersion: baselineManifest.getSemanticVersion(),
soVersion: baselineManifest.getSavedObjectVersion(),
});
for (const artifact of artifacts) {
let artifactToAdd = baselineManifest.getArtifact(getArtifactId(artifact)) || artifact;
if (!isCompressed(artifactToAdd)) {
artifactToAdd = await maybeCompressArtifact(artifactToAdd);
for (const result of results) {
await iterateArtifactsBuildResult(result, async (artifact, policyId) => {
let artifactToAdd = baselineManifest.getArtifact(getArtifactId(artifact)) || artifact;
if (!isCompressed(artifactToAdd)) {
throw new Error(`Unable to compress artifact: ${getArtifactId(artifactToAdd)}`);
} else if (!internalArtifactCompleteSchema.is(artifactToAdd)) {
throw new Error(`Incomplete artifact detected: ${getArtifactId(artifactToAdd)}`);
}
}
artifactToAdd = await maybeCompressArtifact(artifactToAdd);
manifest.addEntry(artifactToAdd);
if (!isCompressed(artifactToAdd)) {
throw new Error(`Unable to compress artifact: ${getArtifactId(artifactToAdd)}`);
} else if (!internalArtifactCompleteSchema.is(artifactToAdd)) {
throw new Error(`Incomplete artifact detected: ${getArtifactId(artifactToAdd)}`);
}
}
manifest.addEntry(artifactToAdd, policyId);
});
}
return manifest;
@ -292,49 +350,52 @@ export class ManifestManager {
public async tryDispatch(manifest: Manifest): Promise<Error[]> {
const errors: Error[] = [];
await this.forEachPolicy(async (packagePolicy) => {
// eslint-disable-next-line @typescript-eslint/naming-convention
const { id, revision, updated_at, updated_by, ...newPackagePolicy } = packagePolicy;
if (newPackagePolicy.inputs.length > 0 && newPackagePolicy.inputs[0].config !== undefined) {
const oldManifest = newPackagePolicy.inputs[0].config.artifact_manifest ?? {
value: {},
};
await iterateAllListItems(
(page) => this.listEndpointPolicies(page),
async (packagePolicy) => {
// eslint-disable-next-line @typescript-eslint/naming-convention
const { id, revision, updated_at, updated_by, ...newPackagePolicy } = packagePolicy;
if (newPackagePolicy.inputs.length > 0 && newPackagePolicy.inputs[0].config !== undefined) {
const oldManifest = newPackagePolicy.inputs[0].config.artifact_manifest ?? {
value: {},
};
const newManifestVersion = manifest.getSemanticVersion();
if (semver.gt(newManifestVersion, oldManifest.value.manifest_version)) {
const serializedManifest = manifest.toPackagePolicyManifest(packagePolicy.id);
const newManifestVersion = manifest.getSemanticVersion();
if (semver.gt(newManifestVersion, oldManifest.value.manifest_version)) {
const serializedManifest = manifest.toPackagePolicyManifest(packagePolicy.id);
if (!manifestDispatchSchema.is(serializedManifest)) {
errors.push(new Error(`Invalid manifest for policy ${packagePolicy.id}`));
} else if (!manifestsEqual(serializedManifest, oldManifest.value)) {
newPackagePolicy.inputs[0].config.artifact_manifest = { value: serializedManifest };
if (!manifestDispatchSchema.is(serializedManifest)) {
errors.push(new Error(`Invalid manifest for policy ${packagePolicy.id}`));
} else if (!manifestsEqual(serializedManifest, oldManifest.value)) {
newPackagePolicy.inputs[0].config.artifact_manifest = { value: serializedManifest };
try {
await this.packagePolicyService.update(
this.savedObjectsClient,
// @ts-ignore
undefined,
id,
newPackagePolicy
);
try {
await this.packagePolicyService.update(
this.savedObjectsClient,
// @ts-ignore
undefined,
id,
newPackagePolicy
);
this.logger.debug(
`Updated package policy ${id} with manifest version ${manifest.getSemanticVersion()}`
);
} catch (err) {
errors.push(err);
}
} else {
this.logger.debug(
`Updated package policy ${id} with manifest version ${manifest.getSemanticVersion()}`
`No change in manifest content for package policy: ${id}. Staying on old version`
);
} catch (err) {
errors.push(err);
}
} else {
this.logger.debug(
`No change in manifest content for package policy: ${id}. Staying on old version`
);
this.logger.debug(`No change in manifest version for package policy: ${id}`);
}
} else {
this.logger.debug(`No change in manifest version for package policy: ${id}`);
errors.push(new Error(`Package Policy ${id} has no config.`));
}
} else {
errors.push(new Error(`Package Policy ${id} has no config.`));
}
});
);
return errors;
}
@ -363,23 +424,19 @@ export class ManifestManager {
this.logger.info(`Committed manifest ${manifest.getSemanticVersion()}`);
}
private async forEachPolicy(callback: (policy: PackagePolicy) => Promise<void>) {
let paging = true;
let page = 1;
private async listEndpointPolicies(page: number) {
return this.packagePolicyService.list(this.savedObjectsClient, {
page,
perPage: 20,
kuery: 'ingest-package-policies.package.name:endpoint',
});
}
while (paging) {
const { items, total } = await this.packagePolicyService.list(this.savedObjectsClient, {
page,
perPage: 20,
kuery: 'ingest-package-policies.package.name:endpoint',
});
for (const packagePolicy of items) {
await callback(packagePolicy);
}
paging = (page - 1) * 20 + items.length < total;
page++;
}
private async listEndpointPolicyIds(page: number) {
return this.packagePolicyService.listIds(this.savedObjectsClient, {
page,
perPage: 20,
kuery: 'ingest-package-policies.package.name:endpoint',
});
}
}