github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 09:48:58 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Security Solution] Siem signals -> alerts as data field and index aliases (#106049)

Browse source 
* Add aliases mapping signal fields to alerts as data fields

* Add aliases mapping alerts as data fields to signal fields

* Replace siem signals templates per space and add AAD index aliases to siem signals indices

* Remove first version of new mapping json file

* Convert existing legacy siem-signals templates to new ES templates

* Catch 404 if siem signals templates were already updated

* Enhance error message when index exists but is not write index for alias

* Check if alias write index exists before creating new write index

* More robust write target creation logic

* Add RBAC required fields for AAD to siem signals indices

* Fix index name in index mapping update

* Throw errors if bulk retry fails or existing indices are not writeable

* Add new template to routes even without experimental rule registry flag enabled

* Check template version before updating template

* First pass at modifying routes to handle inserting field aliases

* Always insert field aliases when create_index_route is called

* Update snapshot test

* Remove template update logic from plugin setup

* Use aliases_version field to detect if aliases need update

* Fix bugs

* oops update snapshot

* Use internal user for PUT alias to fix perms issue

* Update comment

* Disable new resource creation if ruleRegistryEnabled

* Only attempt to add aliases if siem-signals index already exists

* Fix types, add aliases to aad indices, use package field names

* Undo adding aliases to AAD indices

* Remove unused import

* Update test and snapshot oops

* Filter out kibana.* fields from generated signals

* Update cypress test to account for new fields in table

* Properly handle space ids with dashes in them

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Marshall Main 2021-08-05 15:11:17 -07:00 • committed by GitHub
parent eed9723c85
commit 28084f858d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
19 changed files with 5513 additions and 4533 deletions
Download patch file Download diff file Expand all files Collapse all files

6
packages/kbn-rule-data-utils/src/technical_field_names.ts
View file

@ -34,6 +34,7 @@ const ALERT_EVALUATION_THRESHOLD = `${ALERT_NAMESPACE}.evaluation.threshold` as
const ALERT_EVALUATION_VALUE = `${ALERT_NAMESPACE}.evaluation.value` as const;
const ALERT_ID = `${ALERT_NAMESPACE}.id` as const;
const ALERT_OWNER = `${ALERT_NAMESPACE}.owner` as const;
const ALERT_CONSUMERS = `${ALERT_NAMESPACE}.consumers` as const;
const ALERT_PRODUCER = `${ALERT_NAMESPACE}.producer` as const;
const ALERT_REASON = `${ALERT_NAMESPACE}.reason` as const;
const ALERT_RISK_SCORE = `${ALERT_NAMESPACE}.risk_score` as const;
@ -70,6 +71,7 @@ const ALERT_RULE_SEVERITY_MAPPING = `${ALERT_RULE_NAMESPACE}.severity_mapping` a
const ALERT_RULE_TAGS = `${ALERT_RULE_NAMESPACE}.tags` as const;
const ALERT_RULE_TO = `${ALERT_RULE_NAMESPACE}.to` as const;
const ALERT_RULE_TYPE = `${ALERT_RULE_NAMESPACE}.type` as const;
const ALERT_RULE_TYPE_ID = `${ALERT_RULE_NAMESPACE}.rule_type_id` as const;
const ALERT_RULE_UPDATED_AT = `${ALERT_RULE_NAMESPACE}.updated_at` as const;
const ALERT_RULE_UPDATED_BY = `${ALERT_RULE_NAMESPACE}.updated_by` as const;
const ALERT_RULE_VERSION = `${ALERT_RULE_NAMESPACE}.version` as const;
@ -99,6 +101,7 @@ const fields = {
ALERT_EVALUATION_VALUE,
ALERT_ID,
ALERT_OWNER,
ALERT_CONSUMERS,
ALERT_PRODUCER,
ALERT_REASON,
ALERT_RISK_SCORE,
@ -124,6 +127,7 @@ const fields = {
ALERT_RULE_TAGS,
ALERT_RULE_TO,
ALERT_RULE_TYPE,
ALERT_RULE_TYPE_ID,
ALERT_RULE_UPDATED_AT,
ALERT_RULE_UPDATED_BY,
ALERT_RULE_VERSION,
@ -151,6 +155,7 @@ export {
ALERT_NAMESPACE,
ALERT_RULE_NAMESPACE,
ALERT_OWNER,
ALERT_CONSUMERS,
ALERT_PRODUCER,
ALERT_REASON,
ALERT_RISK_SCORE,
@ -179,6 +184,7 @@ export {
ALERT_RULE_TAGS,
ALERT_RULE_TO,
ALERT_RULE_TYPE,
ALERT_RULE_TYPE_ID,
ALERT_RULE_UPDATED_AT,
ALERT_RULE_UPDATED_BY,
ALERT_RULE_VERSION,

2
x-pack/plugins/rule_registry/server/index.ts
View file

@ -11,11 +11,11 @@ import { RuleRegistryPlugin } from './plugin';
export * from './config';
export type { RuleRegistryPluginSetupContract, RuleRegistryPluginStartContract } from './plugin';
export type { RacRequestHandlerContext, RacApiRequestHandlerContext } from './types';
export { RuleDataPluginService } from './rule_data_plugin_service';
export { RuleDataClient } from './rule_data_client';
export { IRuleDataClient } from './rule_data_client/types';
export { getRuleData, RuleExecutorData } from './utils/get_rule_executor_data';
export { createLifecycleRuleTypeFactory } from './utils/create_lifecycle_rule_type_factory';
export { RuleDataPluginService } from './rule_data_plugin_service';
export {
LifecycleRuleExecutor,
LifecycleAlertService,

51
x-pack/plugins/rule_registry/server/rule_data_client/index.ts
View file

@ -96,10 +96,20 @@ export class RuleDataClient implements IRuleDataClient {
if (response.body.errors) {
if (
response.body.items.length > 0 &&
response.body.items?.[0]?.index?.error?.type === 'index_not_found_exception'
(response.body.items.every(
(item) => item.index?.error?.type === 'index_not_found_exception'
) ||
response.body.items.every(
(item) => item.index?.error?.type === 'illegal_argument_exception'
))
) {
return this.createWriteTargetIfNeeded({ namespace }).then(() => {
return clusterClient.bulk(requestWithDefaultParameters);
return clusterClient.bulk(requestWithDefaultParameters).then((retryResponse) => {
if (retryResponse.body.errors) {
throw new ResponseError(retryResponse);
}
return retryResponse;
});
});
}
const error = new ResponseError(response);
@ -116,13 +126,14 @@ export class RuleDataClient implements IRuleDataClient {
const clusterClient = await this.getClusterClient();
const { body: aliasExists } = await clusterClient.indices.existsAlias({
name: alias,
const { body: indicesExist } = await clusterClient.indices.exists({
index: `${alias}-*`,
allow_no_indices: false,
});
const concreteIndexName = `${alias}-000001`;
if (!aliasExists) {
if (!indicesExist) {
try {
await clusterClient.indices.create({
index: concreteIndexName,
@ -135,11 +146,37 @@ export class RuleDataClient implements IRuleDataClient {
},
});
} catch (err) {
// something might have created the index already, that sounds OK
if (err?.meta?.body?.error?.type !== 'resource_already_exists_exception') {
// If the index already exists and it's the write index for the alias,
// something else created it so suppress the error. If it's not the write
// index, that's bad, throw an error.
if (err?.meta?.body?.error?.type === 'resource_already_exists_exception') {
const { body: existingIndices } = await clusterClient.indices.get({
index: concreteIndexName,
});
if (!existingIndices[concreteIndexName]?.aliases?.[alias]?.is_write_index) {
throw Error(
`Attempted to create index: ${concreteIndexName} as the write index for alias: ${alias}, but the index already exists and is not the write index for the alias`
);
}
} else {
throw err;
}
}
} else {
// If we find indices matching the pattern, then we expect one of them to be the write index for the alias.
// Throw an error if none of them are the write index.
const { body: aliasesResponse } = await clusterClient.indices.getAlias({
index: `${alias}-*`,
});
if (
!Object.entries(aliasesResponse).some(
([_, aliasesObject]) => aliasesObject.aliases[alias]?.is_write_index
)
) {
throw Error(
`Indices matching pattern ${alias}-* exist but none are set as the write index for alias ${alias}`
);
}
}
}
}

2
x-pack/plugins/security_solution/cypress/ccs_integration/detection_alerts/alerts_details.spec.ts
View file

@ -58,7 +58,7 @@ describe('Alert details with unmapped fields', () => {
it('Displays the unmapped field on the table', () => {
const expectedUnmmappedField = {
row: 55,
row: 88,
field: 'unmapped',
text: 'This is the unmapped field',
};

2
x-pack/plugins/security_solution/cypress/integration/detection_alerts/alerts_details.spec.ts
View file

@ -54,7 +54,7 @@ describe('Alert details with unmapped fields', () => {
it('Displays the unmapped field on the table', () => {
const expectedUnmmappedField = {
row: 55,
row: 88,
field: 'unmapped',
text: 'This is the unmapped field',
};

7
x-pack/plugins/security_solution/server/client/client.ts
View file

@ -9,12 +9,15 @@ import { ConfigType } from '../config';
export class AppClient {
private readonly signalsIndex: string;
private readonly spaceId: string;
constructor(private spaceId: string, private config: ConfigType) {
constructor(_spaceId: string, private config: ConfigType) {
const configuredSignalsIndex = this.config.signalsIndex;
this.signalsIndex = `${configuredSignalsIndex}-${this.spaceId}`;
this.signalsIndex = `${configuredSignalsIndex}-${_spaceId}`;
this.spaceId = _spaceId;
}
public getSignalsIndex = (): string => this.signalsIndex;
public getSpaceId = (): string => this.spaceId;
}

9272
x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/__snapshots__/get_signals_template.test.ts.snap generated
View file

File diff suppressed because it is too large Load diff

24
x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/check_template_version.ts
View file

@ -5,9 +5,14 @@
* 2.0.
*/
import { get } from 'lodash';
import { ElasticsearchClient } from 'src/core/server';
import { isOutdated } from '../../migrations/helpers';
import { SIGNALS_TEMPLATE_VERSION } from './get_signals_template';
import {
ALIAS_VERSION_FIELD,
SIGNALS_FIELD_ALIASES_VERSION,
SIGNALS_TEMPLATE_VERSION,
} from './get_signals_template';
export const getTemplateVersion = async ({
alias,
@ -17,10 +22,8 @@ export const getTemplateVersion = async ({
alias: string;
}): Promise<number> => {
try {
const response = await esClient.indices.getTemplate<{
[templateName: string]: { version: number };
}>({ name: alias });
return response.body[alias].version ?? 0;
const response = await esClient.indices.getIndexTemplate({ name: alias });
return response.body.index_templates[0].index_template.version ?? 0;
} catch (e) {
return 0;
}
@ -37,3 +40,14 @@ export const templateNeedsUpdate = async ({
return isOutdated({ current: templateVersion, target: SIGNALS_TEMPLATE_VERSION });
};
export const fieldAliasesOutdated = async (esClient: ElasticsearchClient, index: string) => {
const { body: indexMappings } = await esClient.indices.get({ index });
for (const [_, mapping] of Object.entries(indexMappings)) {
const aliasesVersion = get(mapping.mappings?._meta, ALIAS_VERSION_FIELD) ?? 0;
if (aliasesVersion < SIGNALS_FIELD_ALIASES_VERSION) {
return true;
}
}
return false;
};

124
x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/create_index_route.ts
View file

@ -5,12 +5,14 @@
* 2.0.
*/
import { get } from 'lodash';
import { estypes } from '@elastic/elasticsearch';
import { ElasticsearchClient } from 'src/core/server';
import {
transformError,
getIndexExists,
getPolicyExists,
setPolicy,
setTemplate,
createBootstrapIndex,
} from '@kbn/securitysolution-es-utils';
import type {
@ -20,14 +22,29 @@ import type {
} from '../../../../types';
import { DETECTION_ENGINE_INDEX_URL } from '../../../../../common/constants';
import { buildSiemResponse } from '../utils';
import { getSignalsTemplate, SIGNALS_TEMPLATE_VERSION } from './get_signals_template';
import {
createSignalsFieldAliases,
getSignalsTemplate,
getRbacRequiredFields,
SIGNALS_TEMPLATE_VERSION,
SIGNALS_FIELD_ALIASES_VERSION,
ALIAS_VERSION_FIELD,
} from './get_signals_template';
import { ensureMigrationCleanupPolicy } from '../../migrations/migration_cleanup';
import signalsPolicy from './signals_policy.json';
import { templateNeedsUpdate } from './check_template_version';
import { getIndexVersion } from './get_index_version';
import { isOutdated } from '../../migrations/helpers';
import { RuleDataPluginService } from '../../../../../../rule_registry/server';
import signalExtraFields from './signal_extra_fields.json';
import { ConfigType } from '../../../../config';
import { parseExperimentalConfigValue } from '../../../../../common/experimental_features';
export const createIndexRoute = (router: SecuritySolutionPluginRouter) => {
export const createIndexRoute = (
router: SecuritySolutionPluginRouter,
ruleDataService: RuleDataPluginService,
config: ConfigType
) => {
router.post(
{
path: DETECTION_ENGINE_INDEX_URL,
@ -38,13 +55,14 @@ export const createIndexRoute = (router: SecuritySolutionPluginRouter) => {
},
async (context, request, response) => {
const siemResponse = buildSiemResponse(response);
const { ruleRegistryEnabled } = parseExperimentalConfigValue(config.enableExperimental);
try {
const siemClient = context.securitySolution?.getAppClient();
if (!siemClient) {
return siemResponse.error({ statusCode: 404 });
}
await createDetectionIndex(context, siemClient!);
await createDetectionIndex(context, siemClient!, ruleDataService, ruleRegistryEnabled);
return response.ok({ body: { acknowledged: true } });
} catch (err) {
const error = transformError(err);
@ -67,25 +85,58 @@ class CreateIndexError extends Error {
export const createDetectionIndex = async (
context: SecuritySolutionRequestHandlerContext,
siemClient: AppClient
siemClient: AppClient,
ruleDataService: RuleDataPluginService,
ruleRegistryEnabled: boolean
): Promise<void> => {
const esClient = context.core.elasticsearch.client.asCurrentUser;
const spaceId = siemClient.getSpaceId();
if (!siemClient) {
throw new CreateIndexError('', 404);
}
const index = siemClient.getSignalsIndex();
const indexExists = await getIndexExists(esClient, index);
// If using the rule registry implementation, we don't want to create new .siem-signals indices -
// only create/update resources if there are existing indices
if (ruleRegistryEnabled && !indexExists) {
return;
}
await ensureMigrationCleanupPolicy({ alias: index, esClient });
const policyExists = await getPolicyExists(esClient, index);
if (!policyExists) {
await setPolicy(esClient, index, signalsPolicy);
}
const aadIndexAliasName = `${ruleDataService.getFullAssetName('security.alerts')}-${spaceId}`;
if (await templateNeedsUpdate({ alias: index, esClient })) {
await setTemplate(esClient, index, getSignalsTemplate(index));
await esClient.indices.putIndexTemplate({
name: index,
body: getSignalsTemplate(index, spaceId, aadIndexAliasName) as Record<string, unknown>,
});
}
const indexExists = await getIndexExists(esClient, index);
// Check if the old legacy siem signals template exists and remove it
try {
await esClient.indices.deleteTemplate({ name: index });
} catch (err) {
if (err.statusCode !== 404) {
throw err;
}
}
if (indexExists) {
await addFieldAliasesToIndices({ esClient, index, spaceId });
// The internal user is used here because Elasticsearch requires the PUT alias requestor to have 'manage' permissions
// for BOTH the index AND alias name. However, through 7.14 admins only needed permissions for .siem-signals (the index)
// and not .alerts-security.alerts (the alias). From the security solution perspective, all .siem-signals-<space id>-*
// indices should have an alias to .alerts-security.alerts-<space id> so it's safe to add those aliases as the internal user.
await addIndexAliases({
esClient: context.core.elasticsearch.client.asInternalUser,
index,
aadIndexAliasName,
});
const indexVersion = await getIndexVersion(esClient, index);
if (isOutdated({ current: indexVersion, target: SIGNALS_TEMPLATE_VERSION })) {
await esClient.indices.rollover({ alias: index });
@ -94,3 +145,62 @@ export const createDetectionIndex = async (
await createBootstrapIndex(esClient, index);
}
};
const addFieldAliasesToIndices = async ({
esClient,
index,
spaceId,
}: {
esClient: ElasticsearchClient;
index: string;
spaceId: string;
}) => {
const { body: indexMappings } = await esClient.indices.get({ index });
// Make sure that all signal fields we add aliases for are guaranteed to exist in the mapping for ALL historical
// signals indices (either by adding them to signalExtraFields or ensuring they exist in the original signals
// mapping) or else this call will fail and not update ANY signals indices
const fieldAliases = createSignalsFieldAliases();
for (const [indexName, mapping] of Object.entries(indexMappings)) {
const currentVersion: number | undefined = get(mapping.mappings?._meta, 'version');
const newMapping = {
properties: {
...signalExtraFields,
...fieldAliases,
...getRbacRequiredFields(spaceId),
},
_meta: {
version: currentVersion,
[ALIAS_VERSION_FIELD]: SIGNALS_FIELD_ALIASES_VERSION,
},
};
await esClient.indices.putMapping({
index: indexName,
body: newMapping,
allow_no_indices: true,
} as estypes.IndicesPutMappingRequest);
}
};
const addIndexAliases = async ({
esClient,
index,
aadIndexAliasName,
}: {
esClient: ElasticsearchClient;
index: string;
aadIndexAliasName: string;
}) => {
const { body: indices } = await esClient.indices.getAlias({ name: index });
const aliasActions = {
actions: Object.keys(indices).map((concreteIndexName) => {
return {
add: {
index: concreteIndexName,
alias: aadIndexAliasName,
is_write_index: false,
},
};
}),
};
await esClient.indices.updateAliases({ body: aliasActions });
};

11
x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/delete_index_route.ts
View file

@ -10,9 +10,7 @@ import {
getIndexExists,
getPolicyExists,
deletePolicy,
getTemplateExists,
deleteAllIndex,
deleteTemplate,
} from '@kbn/securitysolution-es-utils';
import type { SecuritySolutionPluginRouter } from '../../../../types';
import { DETECTION_ENGINE_INDEX_URL } from '../../../../../common/constants';
@ -22,6 +20,7 @@ import { buildSiemResponse } from '../utils';
* Deletes all of the indexes, template, ilm policies, and aliases. You can check
* this by looking at each of these settings from ES after a deletion:
* GET /_template/.siem-signals-default
* GET /_index_template/.siem-signals-default
* GET /.siem-signals-default-000001/
* GET /_ilm/policy/.signals-default
* GET /_alias/.siem-signals-default
@ -63,9 +62,13 @@ export const deleteIndexRoute = (router: SecuritySolutionPluginRouter) => {
if (policyExists) {
await deletePolicy(esClient, index);
}
const templateExists = await getTemplateExists(esClient, index);
const templateExists = await esClient.indices.existsIndexTemplate({ name: index });
if (templateExists) {
await deleteTemplate(esClient, index);
await esClient.indices.deleteIndexTemplate({ name: index });
}
const legacyTemplateExists = await esClient.indices.existsTemplate({ name: index });
if (legacyTemplateExists) {
await esClient.indices.deleteTemplate({ name: index });
}
return response.ok({ body: { acknowledged: true } });
}

57
x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.test.ts
View file

@ -9,8 +9,12 @@ import { getSignalsTemplate } from './get_signals_template';
describe('get_signals_template', () => {
test('it should set the lifecycle "name" and "rollover_alias" to be the name of the index passed in', () => {
const template = getSignalsTemplate('test-index');
expect(template.settings).toEqual({
const template = getSignalsTemplate(
'test-index',
'space-id',
'.alerts-security.alerts-space-id'
);
expect(template.template.settings).toEqual({
index: {
lifecycle: {
name: 'test-index',
@ -24,23 +28,39 @@ describe('get_signals_template', () => {
});
test('it should set have the index patterns with an ending glob in it', () => {
const template = getSignalsTemplate('test-index');
const template = getSignalsTemplate(
'test-index',
'space-id',
'.alerts-security.alerts-space-id'
);
expect(template.index_patterns).toEqual(['test-index-*']);
});
test('it should have a mappings section which is an object type', () => {
const template = getSignalsTemplate('test-index');
expect(typeof template.mappings).toEqual('object');
const template = getSignalsTemplate(
'test-index',
'space-id',
'.alerts-security.alerts-space-id'
);
expect(typeof template.template.mappings).toEqual('object');
});
test('it should have a signals section which is an object type', () => {
const template = getSignalsTemplate('test-index');
expect(typeof template.mappings.properties.signal).toEqual('object');
const template = getSignalsTemplate(
'test-index',
'space-id',
'.alerts-security.alerts-space-id'
);
expect(typeof template.template.mappings.properties.signal).toEqual('object');
});
test('it should have a "total_fields" section that is at least 10k in size', () => {
const template = getSignalsTemplate('test-index');
expect(template.settings.mapping.total_fields.limit).toBeGreaterThanOrEqual(10000);
const template = getSignalsTemplate(
'test-index',
'space-id',
'.alerts-security.alerts-space-id'
);
expect(template.template.settings.mapping.total_fields.limit).toBeGreaterThanOrEqual(10000);
});
// If you see this test fail, you should track down any and all "constant_keyword" in your ecs_mapping.json and replace
@ -62,7 +82,11 @@ describe('get_signals_template', () => {
// Instead you have to use "keyword". This test was first introduced when ECS 1.10 came out and data_stream.* values which had
// "constant_keyword" fields and we needed to change those to be "keyword" instead.
test('it should NOT have any "constant_keyword" and instead those should be replaced with regular "keyword" in the mapping', () => {
const template = getSignalsTemplate('test-index');
const template = getSignalsTemplate(
'test-index',
'space-id',
'.alerts-security.alerts-space-id'
);
// Small recursive function to find any values of "constant_keyword" and mark which fields it was found on and then error on those fields
// The matchers from jest such as jest.toMatchObject do not support recursion, so I have to write it here:
@ -83,11 +107,20 @@ describe('get_signals_template', () => {
}
}, []);
const constantKeywordsFound = recursiveConstantKeywordFound('', template);
expect(constantKeywordsFound).toEqual([]);
expect(constantKeywordsFound).toEqual([
'template.mappings.properties.kibana.space_ids',
'template.mappings.properties.kibana.alert.consumers',
'template.mappings.properties.kibana.alert.producer',
'template.mappings.properties.kibana.alert.rule.rule_type_id',
]);
});
test('it should match snapshot', () => {
const template = getSignalsTemplate('test-index');
const template = getSignalsTemplate(
'test-index',
'space-id',
'.alerts-security.alerts-space-id'
);
expect(template).toMatchSnapshot();
});
});

130
x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts
View file

@ -5,9 +5,16 @@
* 2.0.
*/
import {
SPACE_IDS,
ALERT_CONSUMERS,
ALERT_PRODUCER,
ALERT_RULE_TYPE_ID,
} from '@kbn/rule-data-utils';
import signalsMapping from './signals_mapping.json';
import ecsMapping from './ecs_mapping.json';
import otherMapping from './other_mappings.json';
import aadFieldConversion from './signal_aad_mapping.json';
/**
@constant
@ -22,50 +29,107 @@ import otherMapping from './other_mappings.json';
incremented by 10 in order to add "room" for the aforementioned patch
release
*/
export const SIGNALS_TEMPLATE_VERSION = 45;
export const SIGNALS_TEMPLATE_VERSION = 55;
/**
@constant
@type {number}
@description This value represents the version of the field aliases that map the new field names
used for alerts-as-data to the old signal.* field names. If any .siem-signals-<space id> indices
have an aliases_version less than this value, the detections UI will call create_index_route and
and go through the index update process. Increment this number if making changes to the field
aliases we use to make signals forwards-compatible.
*/
export const SIGNALS_FIELD_ALIASES_VERSION = 1;
export const MIN_EQL_RULE_INDEX_VERSION = 2;
export const ALIAS_VERSION_FIELD = 'aliases_version';
export const getSignalsTemplate = (index: string) => {
export const getSignalsTemplate = (index: string, spaceId: string, aadIndexAliasName: string) => {
const fieldAliases = createSignalsFieldAliases();
const template = {
settings: {
index: {
lifecycle: {
name: index,
rollover_alias: index,
},
},
mapping: {
total_fields: {
limit: 10000,
},
},
},
index_patterns: [`${index}-*`],
mappings: {
dynamic: false,
properties: {
...ecsMapping.mappings.properties,
...otherMapping.mappings.properties,
signal: signalsMapping.mappings.properties.signal,
threat: {
...ecsMapping.mappings.properties.threat,
properties: {
...ecsMapping.mappings.properties.threat.properties,
indicator: {
...otherMapping.mappings.properties.threat.properties.indicator,
properties: {
...otherMapping.mappings.properties.threat.properties.indicator.properties,
event: ecsMapping.mappings.properties.event,
template: {
aliases: {
[aadIndexAliasName]: {
is_write_index: false,
},
},
settings: {
index: {
lifecycle: {
name: index,
rollover_alias: index,
},
},
mapping: {
total_fields: {
limit: 10000,
},
},
},
mappings: {
dynamic: false,
properties: {
...ecsMapping.mappings.properties,
...otherMapping.mappings.properties,
...fieldAliases,
...getRbacRequiredFields(spaceId),
signal: signalsMapping.mappings.properties.signal,
threat: {
...ecsMapping.mappings.properties.threat,
properties: {
...ecsMapping.mappings.properties.threat.properties,
indicator: {
...otherMapping.mappings.properties.threat.properties.indicator,
properties: {
...otherMapping.mappings.properties.threat.properties.indicator.properties,
event: ecsMapping.mappings.properties.event,
},
},
},
},
},
},
_meta: {
version: SIGNALS_TEMPLATE_VERSION,
_meta: {
version: SIGNALS_TEMPLATE_VERSION,
[ALIAS_VERSION_FIELD]: SIGNALS_FIELD_ALIASES_VERSION,
},
},
},
version: SIGNALS_TEMPLATE_VERSION,
};
return template;
};
export const createSignalsFieldAliases = () => {
const fieldAliases: Record<string, unknown> = {};
Object.entries(aadFieldConversion).forEach(([key, value]) => {
fieldAliases[value] = {
type: 'alias',
path: key,
};
});
return fieldAliases;
};
export const getRbacRequiredFields = (spaceId: string) => {
return {
[SPACE_IDS]: {
type: 'constant_keyword',
value: spaceId,
},
[ALERT_CONSUMERS]: {
type: 'constant_keyword',
value: 'siem',
},
[ALERT_PRODUCER]: {
type: 'constant_keyword',
value: 'siem',
},
// TODO: discuss naming of this field and what the value will be for legacy signals.
// Can we leave it as 'siem.signals' or do we need a runtime field that will map signal.rule.type
// to the new ruleTypeId?
[ALERT_RULE_TYPE_ID]: {
type: 'constant_keyword',
value: 'siem.signals',
},
};
};

34
x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/read_index_route.ts
View file

@ -15,6 +15,7 @@ import { buildSiemResponse } from '../utils';
import { SIGNALS_TEMPLATE_VERSION } from './get_signals_template';
import { getIndexVersion } from './get_index_version';
import { isOutdated } from '../../migrations/helpers';
import { fieldAliasesOutdated } from './check_template_version';
export const readIndexRoute = (router: SecuritySolutionPluginRouter, config: ConfigType) => {
router.get(
@ -38,23 +39,20 @@ export const readIndexRoute = (router: SecuritySolutionPluginRouter, config: Con
// TODO: Once we are past experimental phase this code should be removed
const { ruleRegistryEnabled } = parseExperimentalConfigValue(config.enableExperimental);
if (ruleRegistryEnabled) {
return response.ok({
body: { name: DEFAULT_ALERTS_INDEX, index_mapping_outdated: false },
});
}
const index = siemClient.getSignalsIndex();
const indexExists = ruleRegistryEnabled ? true : await getIndexExists(esClient, index);
const indexExists = await getIndexExists(esClient, index);
if (indexExists) {
let mappingOutdated: boolean | null = null;
let aliasesOutdated: boolean | null = null;
try {
const indexVersion = await getIndexVersion(esClient, index);
mappingOutdated = isOutdated({
current: indexVersion,
target: SIGNALS_TEMPLATE_VERSION,
});
aliasesOutdated = await fieldAliasesOutdated(esClient, index);
} catch (err) {
const error = transformError(err);
// Some users may not have the view_index_metadata permission necessary to check the index mapping version
@ -66,12 +64,26 @@ export const readIndexRoute = (router: SecuritySolutionPluginRouter, config: Con
});
}
}
return response.ok({ body: { name: index, index_mapping_outdated: mappingOutdated } });
} else {
return siemResponse.error({
statusCode: 404,
body: 'index for this space does not exist',
return response.ok({
body: {
name: ruleRegistryEnabled ? DEFAULT_ALERTS_INDEX : index,
index_mapping_outdated: mappingOutdated || aliasesOutdated,
},
});
} else {
if (ruleRegistryEnabled) {
return response.ok({
body: {
name: DEFAULT_ALERTS_INDEX,
index_mapping_outdated: false,
},
});
} else {
return siemResponse.error({
statusCode: 404,
body: 'index for this space does not exist',
});
}
}
} catch (err) {
const error = transformError(err);

93
x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signal_aad_mapping.json Normal file
View file

@ -0,0 +1,93 @@
{
"signal.ancestors.depth": "kibana.alert.ancestors.depth",
"signal.ancestors.id": "kibana.alert.ancestors.id",
"signal.ancestors.index": "kibana.alert.ancestors.index",
"signal.ancestors.type": "kibana.alert.ancestors.type",
"signal.depth": "kibana.alert.depth",
"signal.original_event.action": "kibana.alert.original_event.action",
"signal.original_event.category": "kibana.alert.original_event.category",
"signal.original_event.code": "kibana.alert.original_event.code",
"signal.original_event.created": "kibana.alert.original_event.created",
"signal.original_event.dataset": "kibana.alert.original_event.dataset",
"signal.original_event.duration": "kibana.alert.original_event.duration",
"signal.original_event.end": "kibana.alert.original_event.end",
"signal.original_event.hash": "kibana.alert.original_event.hash",
"signal.original_event.id": "kibana.alert.original_event.id",
"signal.original_event.kind": "kibana.alert.original_event.kind",
"signal.original_event.module": "kibana.alert.original_event.module",
"signal.original_event.outcome": "kibana.alert.original_event.outcome",
"signal.original_event.provider": "kibana.alert.original_event.provider",
"signal.original_event.risk_score": "kibana.alert.original_event.risk_score",
"signal.original_event.risk_score_norm": "kibana.alert.original_event.risk_score_norm",
"signal.original_event.sequence": "kibana.alert.original_event.sequence",
"signal.original_event.severity": "kibana.alert.original_event.severity",
"signal.original_event.start": "kibana.alert.original_event.start",
"signal.original_event.timezone": "kibana.alert.original_event.timezone",
"signal.original_event.type": "kibana.alert.original_event.type",
"signal.original_time": "kibana.alert.original_time",
"signal.rule.author": "kibana.alert.rule.author",
"signal.rule.building_block_type": "kibana.alert.rule.building_block_type",
"signal.rule.created_at": "kibana.alert.rule.created_at",
"signal.rule.created_by": "kibana.alert.rule.created_by",
"signal.rule.description": "kibana.alert.rule.description",
"signal.rule.enabled": "kibana.alert.rule.enabled",
"signal.rule.false_positives": "kibana.alert.rule.false_positives",
"signal.rule.from": "kibana.alert.rule.from",
"signal.rule.id": "kibana.alert.rule.id",
"signal.rule.immutable": "kibana.alert.rule.immutable",
"signal.rule.index": "kibana.alert.rule.index",
"signal.rule.interval": "kibana.alert.rule.interval",
"signal.rule.language": "kibana.alert.rule.language",
"signal.rule.license": "kibana.alert.rule.license",
"signal.rule.max_signals": "kibana.alert.rule.max_signals",
"signal.rule.name": "kibana.alert.rule.name",
"signal.rule.note": "kibana.alert.rule.note",
"signal.rule.query": "kibana.alert.rule.query",
"signal.rule.references": "kibana.alert.rule.references",
"signal.rule.risk_score": "kibana.alert.risk_score",
"signal.rule.risk_score_mapping.field": "kibana.alert.rule.risk_score_mapping.field",
"signal.rule.risk_score_mapping.operator": "kibana.alert.rule.risk_score_mapping.operator",
"signal.rule.risk_score_mapping.value": "kibana.alert.rule.risk_score_mapping.value",
"signal.rule.rule_id": "kibana.alert.rule.rule_id",
"signal.rule.rule_name_override": "kibana.alert.rule.rule_name_override",
"signal.rule.saved_id": "kibana.alert.rule.saved_id",
"signal.rule.severity": "kibana.alert.severity",
"signal.rule.severity_mapping.field": "kibana.alert.rule.severity_mapping.field",
"signal.rule.severity_mapping.operator": "kibana.alert.rule.severity_mapping.operator",
"signal.rule.severity_mapping.value": "kibana.alert.rule.severity_mapping.value",
"signal.rule.severity_mapping.severity": "kibana.alert.rule.severity_mapping.severity",
"signal.rule.tags": "kibana.alert.rule.tags",
"signal.rule.threat.framework": "kibana.alert.rule.threat.framework",
"signal.rule.threat.tactic.id": "kibana.alert.rule.threat.tactic.id",
"signal.rule.threat.tactic.name": "kibana.alert.rule.threat.tactic.name",
"signal.rule.threat.tactic.reference": "kibana.alert.rule.threat.tactic.reference",
"signal.rule.threat.technique.id": "kibana.alert.rule.threat.technique.id",
"signal.rule.threat.technique.name": "kibana.alert.rule.threat.technique.name",
"signal.rule.threat.technique.reference": "kibana.alert.rule.threat.technique.reference",
"signal.rule.threat.technique.subtechnique.id": "kibana.alert.rule.threat.technique.subtechnique.id",
"signal.rule.threat.technique.subtechnique.name": "kibana.alert.rule.threat.technique.subtechnique.name",
"signal.rule.threat.technique.subtechnique.reference": "kibana.alert.rule.threat.technique.subtechnique.reference",
"signal.rule.threat_index": "kibana.alert.rule.threat_index",
"signal.rule.threat_indicator_path": "kibana.alert.rule.threat_indicator_path",
"signal.rule.threat_language": "kibana.alert.rule.threat_language",
"signal.rule.threat_mapping.entries.field": "kibana.alert.rule.threat_mapping.entries.field",
"signal.rule.threat_mapping.entries.value": "kibana.alert.rule.threat_mapping.entries.value",
"signal.rule.threat_mapping.entries.type": "kibana.alert.rule.threat_mapping.entries.type",
"signal.rule.threat_query": "kibana.alert.rule.threat_query",
"signal.rule.threshold.field": "kibana.alert.rule.threshold.field",
"signal.rule.threshold.value": "kibana.alert.rule.threshold.value",
"signal.rule.timeline_id": "kibana.alert.rule.timeline_id",
"signal.rule.timeline_title": "kibana.alert.rule.timeline_title",
"signal.rule.to": "kibana.alert.rule.to",
"signal.rule.type": "kibana.alert.rule.type",
"signal.rule.updated_at": "kibana.alert.rule.updated_at",
"signal.rule.updated_by": "kibana.alert.rule.updated_by",
"signal.rule.version": "kibana.alert.rule.version",
"signal.status": "kibana.alert.workflow_status",
"signal.threshold_result.from": "kibana.alert.threshold_result.from",
"signal.threshold_result.terms.field": "kibana.alert.threshold_result.terms.field",
"signal.threshold_result.terms.value": "kibana.alert.threshold_result.terms.value",
"signal.threshold_result.cardinality.field": "kibana.alert.threshold_result.cardinality.field",
"signal.threshold_result.cardinality.value": "kibana.alert.threshold_result.cardinality.value",
"signal.threshold_result.count": "kibana.alert.threshold_result.count"
}

195
x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signal_extra_fields.json Normal file
View file

@ -0,0 +1,195 @@
{
"signal": {
"type": "object",
"properties": {
"_meta": {
"type": "object",
"properties": {
"version": {
"type": "long"
}
}
},
"ancestors": {
"properties": {
"rule": {
"type": "keyword"
},
"index": {
"type": "keyword"
},
"id": {
"type": "keyword"
},
"type": {
"type": "keyword"
},
"depth": {
"type": "long"
}
}
},
"depth": {
"type": "integer"
},
"group": {
"type": "object",
"properties": {
"id": {
"type": "keyword"
},
"index": {
"type": "integer"
}
}
},
"rule": {
"type": "object",
"properties": {
"author": {
"type": "keyword"
},
"building_block_type": {
"type": "keyword"
},
"license": {
"type": "keyword"
},
"note": {
"type": "text"
},
"risk_score_mapping": {
"type": "object",
"properties": {
"field": {
"type": "keyword"
},
"operator": {
"type": "keyword"
},
"value": {
"type": "keyword"
}
}
},
"rule_name_override": {
"type": "keyword"
},
"severity_mapping": {
"type": "object",
"properties": {
"field": {
"type": "keyword"
},
"operator": {
"type": "keyword"
},
"value": {
"type": "keyword"
},
"severity": {
"type": "keyword"
}
}
},
"threat": {
"type": "object",
"properties": {
"technique": {
"type": "object",
"properties": {
"subtechnique": {
"type": "object",
"properties": {
"id": {
"type": "keyword"
},
"name": {
"type": "keyword"
},
"reference": {
"type": "keyword"
}
}
}
}
}
}
},
"threat_index": {
"type": "keyword"
},
"threat_indicator_path": {
"type": "keyword"
},
"threat_language": {
"type": "keyword"
},
"threat_mapping": {
"type": "object",
"properties": {
"entries": {
"type": "object",
"properties": {
"field": {
"type": "keyword"
},
"value": {
"type": "keyword"
},
"type": {
"type": "keyword"
}
}
}
}
},
"threat_query": {
"type": "keyword"
},
"threshold": {
"type": "object",
"properties": {
"field": {
"type": "keyword"
},
"value": {
"type": "float"
}
}
}
}
},
"threshold_result": {
"properties": {
"from": {
"type": "date"
},
"terms": {
"properties": {
"field": {
"type": "keyword"
},
"value": {
"type": "keyword"
}
}
},
"cardinality": {
"properties": {
"field": {
"type": "keyword"
},
"value": {
"type": "long"
}
}
},
"count": {
"type": "long"
}
}
}
}
}
}

11
x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts
View file

@ -44,7 +44,10 @@ export const buildBulkBody = (
...additionalSignalFields(mergedDoc),
};
const event = buildEventTypeSignal(mergedDoc);
const { threshold_result: thresholdResult, ...filteredSource } = mergedDoc._source || {
// Filter out any kibana.* fields from the generated signal - kibana.* fields are aliases
// in siem-signals so we can't write to them, but for signals-on-signals they'll be returned
// in the fields API response and merged into the mergedDoc source
const { threshold_result: thresholdResult, kibana, ...filteredSource } = mergedDoc._source || {
threshold_result: null,
};
const signalHit: SignalHit = {
@ -145,9 +148,13 @@ export const buildSignalFromEvent = (
...additionalSignalFields(mergedEvent),
};
const eventFields = buildEventTypeSignal(mergedEvent);
// Filter out any kibana.* fields from the generated signal - kibana.* fields are aliases
// in siem-signals so we can't write to them, but for signals-on-signals they'll be returned
// in the fields API response and merged into the mergedDoc source
const { kibana, ...filteredSource } = mergedEvent._source || {};
// TODO: better naming for SignalHit - it's really a new signal to be inserted
const signalHit: SignalHit = {
...mergedEvent._source,
...filteredSource,
'@timestamp': new Date().toISOString(),
event: eventFields,
signal,

1
x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts
View file

@ -121,6 +121,7 @@ export interface SignalSource {
original_time?: string;
threshold_result?: ThresholdResult;
};
kibana?: SearchTypes;
}
export interface BulkItem {

19
x-pack/plugins/security_solution/server/plugin.ts
View file

@ -8,6 +8,7 @@
import { once } from 'lodash';
import { Observable } from 'rxjs';
import LRU from 'lru-cache';
import { estypes } from '@elastic/elasticsearch';
import {
CoreSetup,
@ -87,6 +88,7 @@ import { licenseService } from './lib/license';
import { PolicyWatcher } from './endpoint/lib/policy/license_watch';
import { parseExperimentalConfigValue } from '../common/experimental_features';
import { migrateArtifactsToFleet } from './endpoint/lib/artifacts/migrate_artifacts_to_fleet';
import aadFieldConversion from './lib/detection_engine/routes/index/signal_aad_mapping.json';
import { alertsFieldMap } from './lib/detection_engine/rule_types/field_maps/alerts';
import { rulesFieldMap } from './lib/detection_engine/rule_types/field_maps/rules';
import { RuleExecutionLogClient } from './lib/detection_engine/rule_execution_log/rule_execution_log_client';
@ -201,17 +203,27 @@ export class Plugin implements IPlugin<PluginSetup, PluginStart, SetupPlugins, S
const isRuleRegistryEnabled = experimentalFeatures.ruleRegistryEnabled;
let ruleDataClient: RuleDataClient | null = null;
const { ruleDataService } = plugins.ruleRegistry;
if (isRuleRegistryEnabled) {
const { ruleDataService } = plugins.ruleRegistry;
const alertsIndexPattern = ruleDataService.getFullAssetName('security.alerts*');
const initializeRuleDataTemplates = once(async () => {
if (!ruleDataService.isWriteEnabled()) {
return;
}
const componentTemplateName = ruleDataService.getFullAssetName('security.alerts-mappings');
// TODO: convert the aliases to FieldMaps. Requires enhancing FieldMap to support alias path.
// Split aliases by component template since we need to alias some fields in technical field mappings,
// some fields in security solution specific component template.
const aliases: Record<string, estypes.MappingProperty> = {};
Object.entries(aadFieldConversion).forEach(([key, value]) => {
aliases[key] = {
type: 'alias',
path: value,
};
});
const componentTemplateName = ruleDataService.getFullAssetName('security.alerts-mappings');
await ruleDataService.createOrUpdateComponentTemplate({
name: componentTemplateName,
body: {
@ -273,6 +285,7 @@ export class Plugin implements IPlugin<PluginSetup, PluginStart, SetupPlugins, S
plugins.encryptedSavedObjects?.canEncrypt === true,
plugins.security,
plugins.ml,
ruleDataService,
ruleDataClient
);
registerEndpointRoutes(router, endpointContext);

5
x-pack/plugins/security_solution/server/routes/index.ts
View file

@ -5,7 +5,7 @@
* 2.0.
*/
import { RuleDataClient } from '../../../rule_registry/server';
import { RuleDataClient, RuleDataPluginService } from '../../../rule_registry/server';
import { SecuritySolutionPluginRouter } from '../types';
@ -63,6 +63,7 @@ export const initRoutes = (
hasEncryptionKey: boolean,
security: SetupPlugins['security'],
ml: SetupPlugins['ml'],
ruleDataService: RuleDataPluginService,
ruleDataClient: RuleDataClient | null
) => {
// Detection Engine Rule routes that have the REST endpoints of /api/detection_engine/rules
@ -117,7 +118,7 @@ export const initRoutes = (
// Detection Engine index routes that have the REST endpoints of /api/detection_engine/index
// All REST index creation, policy management for spaces
createIndexRoute(router);
createIndexRoute(router, ruleDataService, config);
readIndexRoute(router, config);
deleteIndexRoute(router);