github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 17:28:26 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Security Solution][Platform] - Export exceptions with rule (#115144)

Browse source 
### Summary

Introduces exports of exception lists with rules. Import of exception lists not yet supported.
This commit is contained in:
Yara Tercero 2021-10-19 22:17:08 -07:00 • committed by GitHub
parent a01165ab30
commit 6a2b7fe3d3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
28 changed files with 523 additions and 83 deletions
Download patch file Download diff file Expand all files Collapse all files

1
x-pack/plugins/lists/server/index.ts
View file

@ -18,6 +18,7 @@ export {
} from './services/exception_lists/exception_list_client_types';
export { ExceptionListClient } from './services/exception_lists/exception_list_client';
export type { ListPluginSetup, ListsApiRequestHandlerContext } from './types';
export type { ExportExceptionListAndItemsReturn } from './services/exception_lists/export_exception_list_and_items';
export const config: PluginConfigDescriptor = {
schema: ConfigSchema,

63
x-pack/plugins/lists/server/routes/export_exception_list_route.ts
View file

@ -6,7 +6,6 @@
*/
import { transformError } from '@kbn/securitysolution-es-utils';
import { transformDataToNdjson } from '@kbn/securitysolution-utils';
import { exportExceptionListQuerySchema } from '@kbn/securitysolution-io-ts-list-types';
import { EXCEPTION_LIST_URL } from '@kbn/securitysolution-list-constants';
@ -30,43 +29,28 @@ export const exportExceptionListRoute = (router: ListsPluginRouter): void => {
try {
const { id, list_id: listId, namespace_type: namespaceType } = request.query;
const exceptionLists = getExceptionListClient(context);
const exceptionList = await exceptionLists.getExceptionList({
const exceptionListsClient = getExceptionListClient(context);
const exportContent = await exceptionListsClient.exportExceptionListAndItems({
id,
listId,
namespaceType,
});
if (exceptionList == null) {
if (exportContent == null) {
return siemResponse.error({
body: `exception list with list_id: ${listId} does not exist`,
body: `exception list with list_id: ${listId} or id: ${id} does not exist`,
statusCode: 400,
});
} else {
const listItems = await exceptionLists.findExceptionListItem({
filter: undefined,
listId,
namespaceType,
page: 1,
perPage: 10000,
sortField: 'exception-list.created_at',
sortOrder: 'desc',
});
const exceptionItems = listItems?.data ?? [];
const { exportData } = getExport([exceptionList, ...exceptionItems]);
const { exportDetails } = getExportDetails(exceptionItems);
// TODO: Allow the API to override the name of the file to export
const fileName = exceptionList.list_id;
return response.ok({
body: `${exportData}${exportDetails}`,
headers: {
'Content-Disposition': `attachment; filename="${fileName}"`,
'Content-Type': 'application/ndjson',
},
});
}
return response.ok({
body: `${exportContent.exportData}${JSON.stringify(exportContent.exportDetails)}\n`,
headers: {
'Content-Disposition': `attachment; filename="${listId}"`,
'Content-Type': 'application/ndjson',
},
});
} catch (err) {
const error = transformError(err);
return siemResponse.error({
@ -77,24 +61,3 @@ export const exportExceptionListRoute = (router: ListsPluginRouter): void => {
}
);
};
export const getExport = (
data: unknown[]
): {
exportData: string;
} => {
const ndjson = transformDataToNdjson(data);
return { exportData: ndjson };
};
export const getExportDetails = (
items: unknown[]
): {
exportDetails: string;
} => {
const exportDetails = JSON.stringify({
exported_list_items_count: items.length,
});
return { exportDetails: `${exportDetails}\n` };
};

11
x-pack/plugins/lists/server/services/exception_lists/exception_list_client.mock.ts
View file

@ -30,6 +30,17 @@ export class ExceptionListClientMock extends ExceptionListClient {
public findExceptionList = jest.fn().mockResolvedValue(getFoundExceptionListSchemaMock());
public createTrustedAppsList = jest.fn().mockResolvedValue(getTrustedAppsListSchemaMock());
public createEndpointList = jest.fn().mockResolvedValue(getExceptionListSchemaMock());
public exportExceptionListAndItems = jest.fn().mockResolvedValue({
exportData: 'exportString',
exportDetails: {
exported_exception_list_count: 0,
exported_exception_list_item_count: 0,
missing_exception_list_item_count: 0,
missing_exception_list_items: [],
missing_exception_lists: [],
missing_exception_lists_count: 0,
},
});
}
export const getExceptionListClientMock = (

20
x-pack/plugins/lists/server/services/exception_lists/exception_list_client.ts
View file

@ -24,6 +24,7 @@ import {
DeleteExceptionListItemByIdOptions,
DeleteExceptionListItemOptions,
DeleteExceptionListOptions,
ExportExceptionListAndItemsOptions,
FindEndpointListItemOptions,
FindExceptionListItemOptions,
FindExceptionListOptions,
@ -38,6 +39,10 @@ import {
UpdateExceptionListOptions,
} from './exception_list_client_types';
import { getExceptionList } from './get_exception_list';
import {
ExportExceptionListAndItemsReturn,
exportExceptionListAndItems,
} from './export_exception_list_and_items';
import { getExceptionListSummary } from './get_exception_list_summary';
import { createExceptionList } from './create_exception_list';
import { getExceptionListItem } from './get_exception_list_item';
@ -492,4 +497,19 @@ export class ExceptionListClient {
sortOrder,
});
};
public exportExceptionListAndItems = async ({
listId,
id,
namespaceType,
}: ExportExceptionListAndItemsOptions): Promise<ExportExceptionListAndItemsReturn | null> => {
const { savedObjectsClient } = this;
return exportExceptionListAndItems({
id,
listId,
namespaceType,
savedObjectsClient,
});
};
}

18
x-pack/plugins/lists/server/services/exception_lists/exception_list_client_types.ts
View file

@ -220,3 +220,21 @@ export interface FindExceptionListOptions {
sortField: SortFieldOrUndefined;
sortOrder: SortOrderOrUndefined;
}
export interface ExportExceptionListAndItemsOptions {
listId: ListIdOrUndefined;
id: IdOrUndefined;
namespaceType: NamespaceType;
}
export interface ExportExceptionListAndItemsReturn {
exportData: string;
exportDetails: {
exported_exception_list_count: number;
exported_exception_list_item_count: number;
missing_exception_list_item_count: number;
missing_exception_list_items: string[];
missing_exception_lists: string[];
missing_exception_lists_count: number;
};
}

62
x-pack/plugins/lists/server/services/exception_lists/export_exception_list_and_items.test.ts Normal file
View file

@ -0,0 +1,62 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { SavedObjectsClientContract } from 'kibana/server';
import { getExceptionListItemSchemaMock } from '../../../common/schemas/response/exception_list_item_schema.mock';
import { getExceptionListSchemaMock } from '../../../common/schemas/response/exception_list_schema.mock';
import { exportExceptionListAndItems } from './export_exception_list_and_items';
import { findExceptionListItem } from './find_exception_list_item';
import { getExceptionList } from './get_exception_list';
jest.mock('./get_exception_list');
jest.mock('./find_exception_list_item');
describe('export_exception_list_and_items', () => {
describe('exportExceptionListAndItems', () => {
test('it should return null if no matching exception list found', async () => {
(getExceptionList as jest.Mock).mockResolvedValue(null);
(findExceptionListItem as jest.Mock).mockResolvedValue({ data: [] });
const result = await exportExceptionListAndItems({
id: '123',
listId: 'non-existent',
namespaceType: 'single',
savedObjectsClient: {} as SavedObjectsClientContract,
});
expect(result).toBeNull();
});
test('it should return stringified list and items', async () => {
(getExceptionList as jest.Mock).mockResolvedValue(getExceptionListSchemaMock());
(findExceptionListItem as jest.Mock).mockResolvedValue({
data: [getExceptionListItemSchemaMock()],
});
const result = await exportExceptionListAndItems({
id: '123',
listId: 'non-existent',
namespaceType: 'single',
savedObjectsClient: {} as SavedObjectsClientContract,
});
expect(result?.exportData).toEqual(
`${JSON.stringify(getExceptionListSchemaMock())}\n${JSON.stringify(
getExceptionListItemSchemaMock()
)}\n`
);
expect(result?.exportDetails).toEqual({
exported_exception_list_count: 1,
exported_exception_list_item_count: 1,
missing_exception_list_item_count: 0,
missing_exception_list_items: [],
missing_exception_lists: [],
missing_exception_lists_count: 0,
});
});
});
});

93
x-pack/plugins/lists/server/services/exception_lists/export_exception_list_and_items.ts Normal file
View file

@ -0,0 +1,93 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type {
IdOrUndefined,
ListIdOrUndefined,
NamespaceType,
} from '@kbn/securitysolution-io-ts-list-types';
import { transformDataToNdjson } from '@kbn/securitysolution-utils';
import { SavedObjectsClientContract } from 'kibana/server';
import { findExceptionListItem } from './find_exception_list_item';
import { getExceptionList } from './get_exception_list';
interface ExportExceptionListAndItemsOptions {
id: IdOrUndefined;
listId: ListIdOrUndefined;
savedObjectsClient: SavedObjectsClientContract;
namespaceType: NamespaceType;
}
export interface ExportExceptionListAndItemsReturn {
exportData: string;
exportDetails: {
exported_exception_list_count: number;
exported_exception_list_item_count: number;
missing_exception_list_item_count: number;
missing_exception_list_items: string[];
missing_exception_lists: string[];
missing_exception_lists_count: number;
};
}
export const exportExceptionListAndItems = async ({
id,
listId,
namespaceType,
savedObjectsClient,
}: ExportExceptionListAndItemsOptions): Promise<ExportExceptionListAndItemsReturn | null> => {
const exceptionList = await getExceptionList({
id,
listId,
namespaceType,
savedObjectsClient,
});
if (exceptionList == null) {
return null;
} else {
// TODO: Will need to address this when we switch over to
// using PIT, don't want it to get lost
// https://github.com/elastic/kibana/issues/103944
const listItems = await findExceptionListItem({
filter: undefined,
listId: exceptionList.list_id,
namespaceType: exceptionList.namespace_type,
page: 1,
perPage: 10000,
savedObjectsClient,
sortField: 'exception-list.created_at',
sortOrder: 'desc',
});
const exceptionItems = listItems?.data ?? [];
const { exportData } = getExport([exceptionList, ...exceptionItems]);
// TODO: Add logic for missing lists and items on errors
return {
exportData: `${exportData}`,
exportDetails: {
exported_exception_list_count: 1,
exported_exception_list_item_count: exceptionItems.length,
missing_exception_list_item_count: 0,
missing_exception_list_items: [],
missing_exception_lists: [],
missing_exception_lists_count: 0,
},
};
}
};
export const getExport = (
data: unknown[]
): {
exportData: string;
} => {
const ndjson = transformDataToNdjson(data);
return { exportData: ndjson };
};

1
x-pack/plugins/lists/server/services/exception_lists/index.ts
View file

@ -10,6 +10,7 @@ export * from './create_exception_list_item';
export * from './delete_exception_list';
export * from './delete_exception_list_item';
export * from './delete_exception_list_items_by_list';
export * from './export_exception_list_and_items';
export * from './find_exception_list';
export * from './find_exception_list_item';
export * from './find_exception_list_items';

2
x-pack/plugins/security_solution/cypress/downloads/test_exception_list.ndjson
View file

@ -1,2 +0,0 @@
{"_version":"WzQyNjA0LDFd","created_at":"2021-10-14T01:30:22.034Z","created_by":"elastic","description":"Test exception list description","id":"4c65a230-2c8e-11ec-be1c-2bbdec602f88","immutable":false,"list_id":"test_exception_list","name":"Test exception list","namespace_type":"single","os_types":[],"tags":[],"tie_breaker_id":"b04983b4-1617-441c-bb6c-c729281fa2e9","type":"detection","updated_at":"2021-10-14T01:30:22.036Z","updated_by":"elastic","version":1}
{"exported_list_items_count":0}

2
x-pack/plugins/security_solution/cypress/objects/exception.ts
View file

@ -41,5 +41,5 @@ export const expectedExportedExceptionList = (
exceptionListResponse: Cypress.Response<ExceptionListItemSchema>
): string => {
const jsonrule = exceptionListResponse.body;
return `{"_version":"${jsonrule._version}","created_at":"${jsonrule.created_at}","created_by":"elastic","description":"${jsonrule.description}","id":"${jsonrule.id}","immutable":false,"list_id":"test_exception_list","name":"Test exception list","namespace_type":"single","os_types":[],"tags":[],"tie_breaker_id":"${jsonrule.tie_breaker_id}","type":"detection","updated_at":"${jsonrule.updated_at}","updated_by":"elastic","version":1}\n{"exported_list_items_count":0}\n`;
return `{"_version":"${jsonrule._version}","created_at":"${jsonrule.created_at}","created_by":"elastic","description":"${jsonrule.description}","id":"${jsonrule.id}","immutable":false,"list_id":"test_exception_list","name":"Test exception list","namespace_type":"single","os_types":[],"tags":[],"tie_breaker_id":"${jsonrule.tie_breaker_id}","type":"detection","updated_at":"${jsonrule.updated_at}","updated_by":"elastic","version":1}\n{"exported_exception_list_count":1,"exported_exception_list_item_count":0,"missing_exception_list_item_count":0,"missing_exception_list_items":[],"missing_exception_lists":[],"missing_exception_lists_count":0}\n`;
};

2
x-pack/plugins/security_solution/cypress/objects/rule.ts
View file

@ -421,5 +421,5 @@ export const getEditedRule = (): CustomRule => ({
export const expectedExportedRule = (ruleResponse: Cypress.Response<RulesSchema>): string => {
const jsonrule = ruleResponse.body;
return `{"id":"${jsonrule.id}","updated_at":"${jsonrule.updated_at}","updated_by":"elastic","created_at":"${jsonrule.created_at}","created_by":"elastic","name":"${jsonrule.name}","tags":[],"interval":"100m","enabled":false,"description":"${jsonrule.description}","risk_score":${jsonrule.risk_score},"severity":"${jsonrule.severity}","output_index":".siem-signals-default","author":[],"false_positives":[],"from":"now-50000h","rule_id":"rule_testing","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[],"immutable":false,"type":"query","language":"kuery","index":["exceptions-*"],"query":"${jsonrule.query}","throttle":"no_actions","actions":[]}\n{"exported_count":1,"missing_rules":[],"missing_rules_count":0}\n`;
return `{"id":"${jsonrule.id}","updated_at":"${jsonrule.updated_at}","updated_by":"elastic","created_at":"${jsonrule.created_at}","created_by":"elastic","name":"${jsonrule.name}","tags":[],"interval":"100m","enabled":false,"description":"${jsonrule.description}","risk_score":${jsonrule.risk_score},"severity":"${jsonrule.severity}","output_index":".siem-signals-default","author":[],"false_positives":[],"from":"now-50000h","rule_id":"rule_testing","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[],"immutable":false,"type":"query","language":"kuery","index":["exceptions-*"],"query":"${jsonrule.query}","throttle":"no_actions","actions":[]}\n{"exported_rules_count":1,"missing_rules":[],"missing_rules_count":0,"exported_exception_list_count":0,"exported_exception_list_item_count":0,"missing_exception_list_item_count":0,"missing_exception_list_items":[],"missing_exception_lists":[],"missing_exception_lists_count":0}\n`;
};

16
x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/export_rules_route.ts
View file

@ -46,6 +46,7 @@ export const exportRulesRoute = (
async (context, request, response) => {
const siemResponse = buildSiemResponse(response);
const rulesClient = context.alerting?.getRulesClient();
const exceptionsClient = context.lists?.getExceptionListClient();
const savedObjectsClient = context.core.savedObjects.client;
if (!rulesClient) {
@ -72,20 +73,27 @@ export const exportRulesRoute = (
}
}
const exported =
const exportedRulesAndExceptions =
request.body?.objects != null
? await getExportByObjectIds(
rulesClient,
exceptionsClient,
savedObjectsClient,
request.body.objects,
logger,
isRuleRegistryEnabled
)
: await getExportAll(rulesClient, savedObjectsClient, logger, isRuleRegistryEnabled);
: await getExportAll(
rulesClient,
exceptionsClient,
savedObjectsClient,
logger,
isRuleRegistryEnabled
);
const responseBody = request.query.exclude_export_details
? exported.rulesNdjson
: `${exported.rulesNdjson}${exported.exportDetails}`;
? exportedRulesAndExceptions.rulesNdjson
: `${exportedRulesAndExceptions.rulesNdjson}${exportedRulesAndExceptions.exceptionLists}${exportedRulesAndExceptions.exportDetails}`;
return response.ok({
headers: {

4
x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/perform_bulk_action_route.ts
View file

@ -47,6 +47,7 @@ export const performBulkActionRoute = (
try {
const rulesClient = context.alerting?.getRulesClient();
const exceptionsClient = context.lists?.getExceptionListClient();
const savedObjectsClient = context.core.savedObjects.client;
const ruleStatusClient = context.securitySolution.getExecutionLogClient();
@ -136,13 +137,14 @@ export const performBulkActionRoute = (
case BulkAction.export:
const exported = await getExportByObjectIds(
rulesClient,
exceptionsClient,
savedObjectsClient,
rules.data.map(({ params }) => ({ rule_id: params.ruleId })),
logger,
isRuleRegistryEnabled
);
const responseBody = `${exported.rulesNdjson}${exported.exportDetails}`;
const responseBody = `${exported.rulesNdjson}${exported.exceptionLists}${exported.exportDetails}`;
return response.ok({
headers: {

2
x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules_stream_from_ndjson.test.ts
View file

@ -207,7 +207,7 @@ describe('create_rules_stream_from_ndjson', () => {
read() {
this.push(getSampleAsNdjson(sample1));
this.push(getSampleAsNdjson(sample2));
this.push('{"exported_count":1,"missing_rules":[],"missing_rules_count":0}\n');
this.push('{"exported_rules_count":1,"missing_rules":[],"missing_rules_count":0}\n');
this.push(null);
},
});

6
x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules_stream_from_ndjson.ts
View file

@ -21,7 +21,8 @@ import {
} from '../../../../common/detection_engine/schemas/request/import_rules_schema';
import {
parseNdjsonStrings,
filterExportedCounts,
filterExportedRulesCounts,
filterExceptions,
createLimitStream,
} from '../../../utils/read_stream/create_stream_from_ndjson';
@ -59,7 +60,8 @@ export const createRulesStreamFromNdJson = (ruleLimit: number) => {
return [
createSplitStream('\n'),
parseNdjsonStrings(),
filterExportedCounts(),
filterExportedRulesCounts(),
filterExceptions(),
validateRules(),
createLimitStream(ruleLimit),
createConcatStream([]),

17
x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_export_all.test.ts
View file

@ -17,9 +17,12 @@ import { getListArrayMock } from '../../../../common/detection_engine/schemas/ty
import { getThreatMock } from '../../../../common/detection_engine/schemas/types/threat.mock';
import { getQueryRuleParams } from '../schemas/rule_schemas.mock';
import { getExceptionListClientMock } from '../../../../../lists/server/services/exception_lists/exception_list_client.mock';
import { loggingSystemMock } from 'src/core/server/mocks';
import { requestContextMock } from '../routes/__mocks__/request_context';
const exceptionsClient = getExceptionListClientMock();
describe.each([
['Legacy', false],
['RAC', true],
@ -49,6 +52,7 @@ describe.each([
const exports = await getExportAll(
rulesClient,
exceptionsClient,
clients.savedObjectsClient,
logger,
isRuleRegistryEnabled
@ -97,7 +101,13 @@ describe.each([
exceptions_list: getListArrayMock(),
});
expect(detailsJson).toEqual({
exported_count: 1,
exported_exception_list_count: 0,
exported_exception_list_item_count: 0,
exported_rules_count: 1,
missing_exception_list_item_count: 0,
missing_exception_list_items: [],
missing_exception_lists: [],
missing_exception_lists_count: 0,
missing_rules: [],
missing_rules_count: 0,
});
@ -116,13 +126,16 @@ describe.each([
const exports = await getExportAll(
rulesClient,
exceptionsClient,
clients.savedObjectsClient,
logger,
isRuleRegistryEnabled
);
expect(exports).toEqual({
rulesNdjson: '',
exportDetails: '{"exported_count":0,"missing_rules":[],"missing_rules_count":0}\n',
exportDetails:
'{"exported_rules_count":0,"missing_rules":[],"missing_rules_count":0,"exported_exception_list_count":0,"exported_exception_list_item_count":0,"missing_exception_list_item_count":0,"missing_exception_list_items":[],"missing_exception_lists":[],"missing_exception_lists_count":0}\n',
exceptionLists: '',
});
});
});

21
x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_export_all.ts
View file

@ -8,35 +8,44 @@
import { transformDataToNdjson } from '@kbn/securitysolution-utils';
import { Logger } from 'src/core/server';
import { ExceptionListClient } from '../../../../../lists/server';
import { RulesClient, AlertServices } from '../../../../../alerting/server';
import { getNonPackagedRules } from './get_existing_prepackaged_rules';
import { getExportDetailsNdjson } from './get_export_details_ndjson';
import { transformAlertsToRules } from '../routes/rules/utils';
import { getRuleExceptionsForExport } from './get_export_rule_exceptions';
// eslint-disable-next-line no-restricted-imports
import { legacyGetBulkRuleActionsSavedObject } from '../rule_actions/legacy_get_bulk_rule_actions_saved_object';
export const getExportAll = async (
rulesClient: RulesClient,
exceptionsClient: ExceptionListClient | undefined,
savedObjectsClient: AlertServices['savedObjectsClient'],
logger: Logger,
isRuleRegistryEnabled: boolean
): Promise<{
rulesNdjson: string;
exportDetails: string;
exceptionLists: string | null;
}> => {
const ruleAlertTypes = await getNonPackagedRules({ rulesClient, isRuleRegistryEnabled });
const alertIds = ruleAlertTypes.map((rule) => rule.id);
// Gather actions
const legacyActions = await legacyGetBulkRuleActionsSavedObject({
alertIds,
savedObjectsClient,
logger,
});
const rules = transformAlertsToRules(ruleAlertTypes, legacyActions);
// We do not support importing/exporting actions. When we do, delete this line of code
const rulesWithoutActions = rules.map((rule) => ({ ...rule, actions: [] }));
const rulesNdjson = transformDataToNdjson(rulesWithoutActions);
const exportDetails = getExportDetailsNdjson(rules);
return { rulesNdjson, exportDetails };
// Gather exceptions
const exceptions = rules.flatMap((rule) => rule.exceptions_list ?? []);
const { exportData: exceptionLists, exportDetails: exceptionDetails } =
await getRuleExceptionsForExport(exceptions, exceptionsClient);
const rulesNdjson = transformDataToNdjson(rules);
const exportDetails = getExportDetailsNdjson(rules, [], exceptionDetails);
return { rulesNdjson, exportDetails, exceptionLists };
};

16
x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_export_by_object_ids.test.ts
View file

@ -16,6 +16,9 @@ import { rulesClientMock } from '../../../../../alerting/server/mocks';
import { getListArrayMock } from '../../../../common/detection_engine/schemas/types/lists.mock';
import { getThreatMock } from '../../../../common/detection_engine/schemas/types/threat.mock';
import { getQueryRuleParams } from '../schemas/rule_schemas.mock';
import { getExceptionListClientMock } from '../../../../../lists/server/services/exception_lists/exception_list_client.mock';
const exceptionsClient = getExceptionListClientMock();
import { loggingSystemMock } from 'src/core/server/mocks';
import { requestContextMock } from '../routes/__mocks__/request_context';
@ -42,6 +45,7 @@ describe.each([
const objects = [{ rule_id: 'rule-1' }];
const exports = await getExportByObjectIds(
rulesClient,
exceptionsClient,
clients.savedObjectsClient,
objects,
logger,
@ -94,7 +98,13 @@ describe.each([
exceptions_list: getListArrayMock(),
},
exportDetails: {
exported_count: 1,
exported_exception_list_count: 0,
exported_exception_list_item_count: 0,
exported_rules_count: 1,
missing_exception_list_item_count: 0,
missing_exception_list_items: [],
missing_exception_lists: [],
missing_exception_lists_count: 0,
missing_rules: [],
missing_rules_count: 0,
},
@ -119,6 +129,7 @@ describe.each([
const objects = [{ rule_id: 'rule-1' }];
const exports = await getExportByObjectIds(
rulesClient,
exceptionsClient,
clients.savedObjectsClient,
objects,
logger,
@ -127,7 +138,8 @@ describe.each([
expect(exports).toEqual({
rulesNdjson: '',
exportDetails:
'{"exported_count":0,"missing_rules":[{"rule_id":"rule-1"}],"missing_rules_count":1}\n',
'{"exported_rules_count":0,"missing_rules":[{"rule_id":"rule-1"}],"missing_rules_count":1,"exported_exception_list_count":0,"exported_exception_list_item_count":0,"missing_exception_list_item_count":0,"missing_exception_list_items":[],"missing_exception_lists":[],"missing_exception_lists_count":0}\n',
exceptionLists: '',
});
});
});

18
x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_export_by_object_ids.ts
View file

@ -9,6 +9,7 @@ import { chunk } from 'lodash';
import { transformDataToNdjson } from '@kbn/securitysolution-utils';
import { Logger } from 'src/core/server';
import { ExceptionListClient } from '../../../../../lists/server';
import { RulesSchema } from '../../../../common/detection_engine/schemas/response/rules_schema';
import { RulesClient, AlertServices } from '../../../../../alerting/server';
@ -18,6 +19,7 @@ import { isAlertType } from '../rules/types';
import { transformAlertToRule } from '../routes/rules/utils';
import { INTERNAL_RULE_ID_KEY } from '../../../../common/constants';
import { findRules } from './find_rules';
import { getRuleExceptionsForExport } from './get_export_rule_exceptions';
// eslint-disable-next-line no-restricted-imports
import { legacyGetBulkRuleActionsSavedObject } from '../rule_actions/legacy_get_bulk_rule_actions_saved_object';
@ -40,6 +42,7 @@ export interface RulesErrors {
export const getExportByObjectIds = async (
rulesClient: RulesClient,
exceptionsClient: ExceptionListClient | undefined,
savedObjectsClient: AlertServices['savedObjectsClient'],
objects: Array<{ rule_id: string }>,
logger: Logger,
@ -47,6 +50,7 @@ export const getExportByObjectIds = async (
): Promise<{
rulesNdjson: string;
exportDetails: string;
exceptionLists: string | null;
}> => {
const rulesAndErrors = await getRulesFromObjects(
rulesClient,
@ -56,9 +60,19 @@ export const getExportByObjectIds = async (
isRuleRegistryEnabled
);
// Retrieve exceptions
const exceptions = rulesAndErrors.rules.flatMap((rule) => rule.exceptions_list ?? []);
const { exportData: exceptionLists, exportDetails: exceptionDetails } =
await getRuleExceptionsForExport(exceptions, exceptionsClient);
const rulesNdjson = transformDataToNdjson(rulesAndErrors.rules);
const exportDetails = getExportDetailsNdjson(rulesAndErrors.rules, rulesAndErrors.missingRules);
return { rulesNdjson, exportDetails };
const exportDetails = getExportDetailsNdjson(
rulesAndErrors.rules,
rulesAndErrors.missingRules,
exceptionDetails
);
return { rulesNdjson, exportDetails, exceptionLists };
};
export const getRulesFromObjects = async (

6
x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_export_details_ndjson.test.ts
View file

@ -20,7 +20,7 @@ describe('getExportDetailsNdjson', () => {
const details = getExportDetailsNdjson([rule]);
const reParsed = JSON.parse(details);
expect(reParsed).toEqual({
exported_count: 1,
exported_rules_count: 1,
missing_rules: [],
missing_rules_count: 0,
});
@ -31,7 +31,7 @@ describe('getExportDetailsNdjson', () => {
const details = getExportDetailsNdjson([], [missingRule]);
const reParsed = JSON.parse(details);
expect(reParsed).toEqual({
exported_count: 0,
exported_rules_count: 0,
missing_rules: [{ rule_id: 'rule-1' }],
missing_rules_count: 1,
});
@ -49,7 +49,7 @@ describe('getExportDetailsNdjson', () => {
const details = getExportDetailsNdjson([rule1, rule2], [missingRule1, missingRule2]);
const reParsed = JSON.parse(details);
expect(reParsed).toEqual({
exported_count: 2,
exported_rules_count: 2,
missing_rules: [missingRule1, missingRule2],
missing_rules_count: 2,
});

6
x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_export_details_ndjson.ts
View file

@ -9,12 +9,14 @@ import { RulesSchema } from '../../../../common/detection_engine/schemas/respons
export const getExportDetailsNdjson = (
rules: Array<Partial<RulesSchema>>,
missingRules: Array<{ rule_id: string }> = []
missingRules: Array<{ rule_id: string }> = [],
extraMeta: Record<string, number | string | string[]> = {}
): string => {
const stringified = JSON.stringify({
exported_count: rules.length,
exported_rules_count: rules.length,
missing_rules: missingRules,
missing_rules_count: missingRules.length,
...extraMeta,
});
return `${stringified}\n`;
};

79
x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_export_rule_exceptions.test.ts Normal file
View file

@ -0,0 +1,79 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { ENDPOINT_LIST_ID } from '@kbn/securitysolution-list-constants';
import { getExceptionListClientMock } from '../../../../../lists/server/services/exception_lists/exception_list_client.mock';
import {
getRuleExceptionsForExport,
getExportableExceptions,
getDefaultExportDetails,
} from './get_export_rule_exceptions';
import {
getListArrayMock,
getListMock,
} from '../../../../common/detection_engine/schemas/types/lists.mock';
describe('get_export_rule_exceptions', () => {
describe('getRuleExceptionsForExport', () => {
test('it returns empty exceptions array if no rules have exceptions associated', async () => {
const { exportData, exportDetails } = await getRuleExceptionsForExport(
[],
getExceptionListClientMock()
);
expect(exportData).toEqual('');
expect(exportDetails).toEqual(getDefaultExportDetails());
});
test('it returns stringified exceptions ready for export', async () => {
const { exportData } = await getRuleExceptionsForExport(
[getListMock()],
getExceptionListClientMock()
);
expect(exportData).toEqual('exportString');
});
test('it does not return a global endpoint list', async () => {
const { exportData } = await getRuleExceptionsForExport(
[
{
id: ENDPOINT_LIST_ID,
list_id: ENDPOINT_LIST_ID,
namespace_type: 'agnostic',
type: 'endpoint',
},
],
getExceptionListClientMock()
);
expect(exportData).toEqual('');
});
});
describe('getExportableExceptions', () => {
test('it returns stringified exception lists and items', async () => {
// This rule has 2 exception lists tied to it
const { exportData } = await getExportableExceptions(
getListArrayMock(),
getExceptionListClientMock()
);
expect(exportData).toEqual('exportStringexportString');
});
test('it throws error if error occurs in getting exceptions', async () => {
const exceptionsClient = getExceptionListClientMock();
exceptionsClient.exportExceptionListAndItems = jest.fn().mockRejectedValue(new Error('oops'));
// This rule has 2 exception lists tied to it
await expect(async () => {
await getExportableExceptions(getListArrayMock(), exceptionsClient);
}).rejects.toThrowErrorMatchingInlineSnapshot(`"oops"`);
});
});
});

102
x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_export_rule_exceptions.ts Normal file
View file

@ -0,0 +1,102 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { chunk } from 'lodash/fp';
import { ListArray } from '@kbn/securitysolution-io-ts-list-types';
import { ENDPOINT_LIST_ID } from '@kbn/securitysolution-list-constants';
import {
ExceptionListClient,
ExportExceptionListAndItemsReturn,
} from '../../../../../lists/server';
const NON_EXPORTABLE_LIST_IDS = [ENDPOINT_LIST_ID];
export const EXCEPTIONS_EXPORT_CHUNK_SIZE = 50;
export const getRuleExceptionsForExport = async (
exceptions: ListArray,
exceptionsListClient: ExceptionListClient | undefined
): Promise<ExportExceptionListAndItemsReturn> => {
if (exceptionsListClient != null) {
const exceptionsWithoutUnexportableLists = exceptions.filter(
({ list_id: listId }) => !NON_EXPORTABLE_LIST_IDS.includes(listId)
);
return getExportableExceptions(exceptionsWithoutUnexportableLists, exceptionsListClient);
} else {
return { exportData: '', exportDetails: getDefaultExportDetails() };
}
};
export const getExportableExceptions = async (
exceptions: ListArray,
exceptionsListClient: ExceptionListClient
): Promise<ExportExceptionListAndItemsReturn> => {
let exportString = '';
const exportDetails = getDefaultExportDetails();
const exceptionChunks = chunk(EXCEPTIONS_EXPORT_CHUNK_SIZE, exceptions);
for await (const exceptionChunk of exceptionChunks) {
const promises = createPromises(exceptionsListClient, exceptionChunk);
const responses = await Promise.all(promises);
for (const res of responses) {
if (res != null) {
const {
exportDetails: {
exported_exception_list_count: exportedExceptionListCount,
exported_exception_list_item_count: exportedExceptionListItemCount,
},
exportData,
} = res;
exportDetails.exported_exception_list_count =
exportDetails.exported_exception_list_count + exportedExceptionListCount;
exportDetails.exported_exception_list_item_count =
exportDetails.exported_exception_list_item_count + exportedExceptionListItemCount;
exportString = `${exportString}${exportData}`;
}
}
}
return {
exportDetails,
exportData: exportString,
};
};
/**
* Creates promises of the rules and returns them.
* @param exceptionsListClient Exception Lists client
* @param exceptions The rules to apply the update for
* @returns Promise of export ready exceptions.
*/
export const createPromises = (
exceptionsListClient: ExceptionListClient,
exceptions: ListArray
): Array<Promise<ExportExceptionListAndItemsReturn | null>> => {
return exceptions.map<Promise<ExportExceptionListAndItemsReturn | null>>(
async ({ id, list_id: listId, namespace_type: namespaceType }) => {
return exceptionsListClient.exportExceptionListAndItems({
id,
listId,
namespaceType,
});
}
);
};
export const getDefaultExportDetails = () => ({
exported_exception_list_count: 0,
exported_exception_list_item_count: 0,
missing_exception_list_item_count: 0,
missing_exception_list_items: [],
missing_exception_lists: [],
missing_exception_lists_count: 0,
});

12
x-pack/plugins/security_solution/server/utils/read_stream/create_stream_from_ndjson.ts
View file

@ -34,6 +34,18 @@ export const filterExportedCounts = (): Transform => {
);
};
export const filterExportedRulesCounts = (): Transform => {
return createFilterStream<ImportRulesSchemaDecoded | RulesObjectsExportResultDetails>(
(obj) => obj != null && !has('exported_rules_count', obj)
);
};
export const filterExceptions = (): Transform => {
return createFilterStream<ImportRulesSchemaDecoded | RulesObjectsExportResultDetails>(
(obj) => obj != null && !has('list_id', obj)
);
};
// Adaptation from: saved_objects/import/create_limit_stream.ts
export const createLimitStream = (limit: number): Transform => {
let counter = 0;

8
x-pack/test/detection_engine_api_integration/basic/tests/export_rules.ts
View file

@ -76,7 +76,13 @@ export default ({ getService }: FtrProviderContext): void => {
const bodySplitAndParsed = JSON.parse(body.toString().split(/\n/)[1]);
expect(bodySplitAndParsed).to.eql({
exported_count: 1,
exported_exception_list_count: 0,
exported_exception_list_item_count: 0,
exported_rules_count: 1,
missing_exception_list_item_count: 0,
missing_exception_list_items: [],
missing_exception_lists: [],
missing_exception_lists_count: 0,
missing_rules: [],
missing_rules_count: 0,
});

8
x-pack/test/detection_engine_api_integration/security_and_spaces/tests/export_rules.ts
View file

@ -76,7 +76,13 @@ export default ({ getService }: FtrProviderContext): void => {
const bodySplitAndParsed = JSON.parse(body.toString().split(/\n/)[1]);
expect(bodySplitAndParsed).to.eql({
exported_count: 1,
exported_exception_list_count: 0,
exported_exception_list_item_count: 0,
exported_rules_count: 1,
missing_exception_list_item_count: 0,
missing_exception_list_items: [],
missing_exception_lists: [],
missing_exception_lists_count: 0,
missing_rules: [],
missing_rules_count: 0,
});

8
x-pack/test/detection_engine_api_integration/security_and_spaces/tests/perform_bulk_action.ts
View file

@ -57,7 +57,13 @@ export default ({ getService }: FtrProviderContext): void => {
const exportDetails = JSON.parse(exportDetailsJson);
expect(exportDetails).to.eql({
exported_count: 1,
exported_exception_list_count: 0,
exported_exception_list_item_count: 0,
exported_rules_count: 1,
missing_exception_list_item_count: 0,
missing_exception_list_items: [],
missing_exception_lists: [],
missing_exception_lists_count: 0,
missing_rules: [],
missing_rules_count: 0,
});

2
x-pack/test/lists_api_integration/security_and_spaces/tests/export_exception_list.ts
View file

@ -77,7 +77,7 @@ export default ({ getService }: FtrProviderContext): void => {
.expect(400);
expect(exportBody).to.eql({
message: 'exception list with list_id: not_exist does not exist',
message: 'exception list with list_id: not_exist or id: not_exist does not exist',
status_code: 400,
});
});