mirror of
https://github.com/elastic/kibana.git
synced 2025-04-24 01:38:56 -04:00
[Security Solution][Detection Engine] adds legacy siem signals telemetry (#202671)
## Summary - partly addresses https://github.com/elastic/kibana/issues/195523 - adds snapshot telemetry that shows number of legacy siem signals and number of spaces they are in - while working on PR, discovered and fixed few issues in APIs - get migration status API did not work correctly with new `.alerts-*` indices, listing them as outdated - finalize migration API did account for spaces, when adding alias to migrated index - remove migration API failed due to lack of permissions to removed migration task from `.tasks` index ### How to test #### How to create legacy siem index? run script that used for FTR tests ```bash node scripts/es_archiver --kibana-url=http://elastic:changeme@localhost:5601 --es-url=http://elastic:changeme@localhost:9200 load x-pack/test/functional/es_archives/signals/legacy_signals_index ``` These would create legacy siem indices. But be aware, it might break Kibana .alerts indices creation. But sufficient for testing #### How to test snapshot telemetry Snapshot For snapshot telemetry use [API](https://docs.elastic.dev/telemetry/collection/snapshot-telemetry#telemetry-usage-payload-api) call OR Check snapshots in Kibana adv settings -> Global Settings Tab -> Usage collection section -> Click on cluster data example link -> Check `legacy_siem_signals ` fields in flyout <details> <summary> Snapshot telemetry </summary> <img width="2549" alt="Screenshot 2024-12-03 at 13 08 03" src="https://github.com/user-attachments/assets/28ffe983-01c7-4435-a82a-9a968d32d5e0"> </details> --------- Co-authored-by: Ryland Herrick <ryalnd@gmail.com>
This commit is contained in:
parent
f1f3a4fddd
commit
8821e034e9
27 changed files with 714 additions and 14 deletions
|
@ -20,26 +20,30 @@ interface IndexAlias {
|
|||
*
|
||||
* @param esClient An {@link ElasticsearchClient}
|
||||
* @param alias alias name used to filter results
|
||||
* @param index index name used to filter results
|
||||
*
|
||||
* @returns an array of {@link IndexAlias} objects
|
||||
*/
|
||||
export const getIndexAliases = async ({
|
||||
esClient,
|
||||
alias,
|
||||
index,
|
||||
}: {
|
||||
esClient: ElasticsearchClient;
|
||||
alias: string;
|
||||
index?: string;
|
||||
}): Promise<IndexAlias[]> => {
|
||||
const response = await esClient.indices.getAlias(
|
||||
{
|
||||
name: alias,
|
||||
...(index ? { index } : {}),
|
||||
},
|
||||
{ meta: true }
|
||||
);
|
||||
|
||||
return Object.keys(response.body).map((index) => ({
|
||||
return Object.keys(response.body).map((indexName) => ({
|
||||
alias,
|
||||
index,
|
||||
isWriteIndex: response.body[index].aliases[alias]?.is_write_index === true,
|
||||
index: indexName,
|
||||
isWriteIndex: response.body[indexName].aliases[alias]?.is_write_index === true,
|
||||
}));
|
||||
};
|
||||
|
|
|
@ -14,7 +14,6 @@ import type { SignalsMigrationSO } from './saved_objects_schema';
|
|||
/**
|
||||
* Deletes a completed migration:
|
||||
* * deletes the migration SO
|
||||
* * deletes the underlying task document
|
||||
* * applies deletion policy to the relevant index
|
||||
*
|
||||
* @param esClient An {@link ElasticsearchClient}
|
||||
|
@ -40,7 +39,7 @@ export const deleteMigration = async ({
|
|||
return migration;
|
||||
}
|
||||
|
||||
const { destinationIndex, sourceIndex, taskId } = migration.attributes;
|
||||
const { destinationIndex, sourceIndex } = migration.attributes;
|
||||
|
||||
if (isMigrationFailed(migration)) {
|
||||
await applyMigrationCleanupPolicy({
|
||||
|
@ -57,7 +56,6 @@ export const deleteMigration = async ({
|
|||
});
|
||||
}
|
||||
|
||||
await esClient.delete({ index: '.tasks', id: taskId });
|
||||
await deleteMigrationSavedObject({ id: migration.id, soClient });
|
||||
|
||||
return migration;
|
||||
|
|
|
@ -40,6 +40,7 @@ describe('finalizeMigration', () => {
|
|||
signalsAlias: 'my-signals-alias',
|
||||
soClient,
|
||||
username: 'username',
|
||||
legacySiemSignalsAlias: '.siem-signals-default',
|
||||
});
|
||||
|
||||
expect(updateMigrationSavedObject).not.toHaveBeenCalled();
|
||||
|
@ -54,6 +55,7 @@ describe('finalizeMigration', () => {
|
|||
signalsAlias: 'my-signals-alias',
|
||||
soClient,
|
||||
username: 'username',
|
||||
legacySiemSignalsAlias: '.siem-signals-default',
|
||||
});
|
||||
|
||||
expect(updateMigrationSavedObject).not.toHaveBeenCalled();
|
||||
|
@ -72,6 +74,7 @@ describe('finalizeMigration', () => {
|
|||
signalsAlias: 'my-signals-alias',
|
||||
soClient,
|
||||
username: 'username',
|
||||
legacySiemSignalsAlias: '.siem-signals-default',
|
||||
});
|
||||
|
||||
expect(updateMigrationSavedObject).toHaveBeenCalledWith(
|
||||
|
|
|
@ -35,12 +35,14 @@ export const finalizeMigration = async ({
|
|||
signalsAlias,
|
||||
soClient,
|
||||
username,
|
||||
legacySiemSignalsAlias,
|
||||
}: {
|
||||
esClient: ElasticsearchClient;
|
||||
migration: SignalsMigrationSO;
|
||||
signalsAlias: string;
|
||||
soClient: SavedObjectsClientContract;
|
||||
username: string;
|
||||
legacySiemSignalsAlias: string;
|
||||
}): Promise<SignalsMigrationSO> => {
|
||||
if (!isMigrationPending(migration)) {
|
||||
return migration;
|
||||
|
@ -86,6 +88,7 @@ export const finalizeMigration = async ({
|
|||
esClient,
|
||||
newIndex: destinationIndex,
|
||||
oldIndex: sourceIndex,
|
||||
legacySiemSignalsAlias,
|
||||
});
|
||||
|
||||
const updatedMigration = await updateMigrationSavedObject({
|
||||
|
|
|
@ -0,0 +1,88 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { elasticsearchServiceMock } from '@kbn/core/server/mocks';
|
||||
import { getIndexAliasPerSpace } from './get_index_alias_per_space';
|
||||
|
||||
describe('getIndexAliasPerSpace', () => {
|
||||
let esClient: ReturnType<typeof elasticsearchServiceMock.createElasticsearchClient>;
|
||||
|
||||
beforeEach(() => {
|
||||
esClient = elasticsearchServiceMock.createElasticsearchClient();
|
||||
});
|
||||
|
||||
it('returns object with index alias and space', async () => {
|
||||
esClient.indices.getAlias.mockResponseOnce({
|
||||
'.siem-signals-default-old-one': {
|
||||
aliases: {
|
||||
'.siem-signals-default': {
|
||||
is_write_index: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
'.siem-signals-another-1-legacy': {
|
||||
aliases: {
|
||||
'.siem-signals-another-1': {
|
||||
is_write_index: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const result = await getIndexAliasPerSpace({
|
||||
esClient,
|
||||
signalsIndex: '.siem-signals',
|
||||
signalsAliasAllSpaces: '.siem-signals-*',
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
'.siem-signals-another-1-legacy': {
|
||||
alias: '.siem-signals-another-1',
|
||||
indexName: '.siem-signals-another-1-legacy',
|
||||
space: 'another-1',
|
||||
},
|
||||
'.siem-signals-default-old-one': {
|
||||
alias: '.siem-signals-default',
|
||||
indexName: '.siem-signals-default-old-one',
|
||||
space: 'default',
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('filters out .internal.alert indices', async () => {
|
||||
esClient.indices.getAlias.mockResponseOnce({
|
||||
'.siem-signals-default-old-one': {
|
||||
aliases: {
|
||||
'.siem-signals-default': {
|
||||
is_write_index: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
'.internal.alerts-security.alerts-another-2-000001': {
|
||||
aliases: {
|
||||
'.siem-signals-another-2': {
|
||||
is_write_index: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const result = await getIndexAliasPerSpace({
|
||||
esClient,
|
||||
signalsIndex: '.siem-signals',
|
||||
signalsAliasAllSpaces: '.siem-signals-*',
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
'.siem-signals-default-old-one': {
|
||||
alias: '.siem-signals-default',
|
||||
indexName: '.siem-signals-default-old-one',
|
||||
space: 'default',
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
|
@ -0,0 +1,52 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
import type { ElasticsearchClient } from '@kbn/core/server';
|
||||
|
||||
interface IndexAlias {
|
||||
alias: string;
|
||||
space: string;
|
||||
indexName: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieves index, its alias and Kibana space
|
||||
*/
|
||||
export const getIndexAliasPerSpace = async ({
|
||||
esClient,
|
||||
signalsIndex,
|
||||
signalsAliasAllSpaces,
|
||||
}: {
|
||||
esClient: ElasticsearchClient;
|
||||
signalsIndex: string;
|
||||
signalsAliasAllSpaces: string;
|
||||
}): Promise<Record<string, IndexAlias>> => {
|
||||
const response = await esClient.indices.getAlias(
|
||||
{
|
||||
name: signalsAliasAllSpaces,
|
||||
},
|
||||
{ meta: true }
|
||||
);
|
||||
|
||||
const indexAliasesMap = Object.keys(response.body).reduce<Record<string, IndexAlias>>(
|
||||
(acc, indexName) => {
|
||||
if (!indexName.startsWith('.internal.alerts-')) {
|
||||
const alias = Object.keys(response.body[indexName].aliases)[0];
|
||||
|
||||
acc[indexName] = {
|
||||
alias,
|
||||
space: alias.replace(`${signalsIndex}-`, ''),
|
||||
indexName,
|
||||
};
|
||||
}
|
||||
|
||||
return acc;
|
||||
},
|
||||
{}
|
||||
);
|
||||
|
||||
return indexAliasesMap;
|
||||
};
|
|
@ -0,0 +1,59 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
import type { IndicesGetIndexTemplateResponse } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
|
||||
import { elasticsearchServiceMock } from '@kbn/core/server/mocks';
|
||||
import { getLatestIndexTemplateVersion } from './get_latest_index_template_version';
|
||||
|
||||
describe('getIndexAliasPerSpace', () => {
|
||||
let esClient: ReturnType<typeof elasticsearchServiceMock.createElasticsearchClient>;
|
||||
|
||||
beforeEach(() => {
|
||||
esClient = elasticsearchServiceMock.createElasticsearchClient();
|
||||
});
|
||||
|
||||
it('returns latest index template version', async () => {
|
||||
esClient.indices.getIndexTemplate.mockResponseOnce({
|
||||
index_templates: [
|
||||
{ index_template: { version: 77 } },
|
||||
{ index_template: { version: 10 } },
|
||||
{ index_template: { version: 23 } },
|
||||
{ index_template: { version: 0 } },
|
||||
],
|
||||
} as IndicesGetIndexTemplateResponse);
|
||||
|
||||
const version = await getLatestIndexTemplateVersion({
|
||||
esClient,
|
||||
name: '.siem-signals-*',
|
||||
});
|
||||
|
||||
expect(version).toBe(77);
|
||||
});
|
||||
|
||||
it('returns 0 if templates empty', async () => {
|
||||
esClient.indices.getIndexTemplate.mockResponseOnce({
|
||||
index_templates: [],
|
||||
});
|
||||
|
||||
const version = await getLatestIndexTemplateVersion({
|
||||
esClient,
|
||||
name: '.siem-signals-*',
|
||||
});
|
||||
|
||||
expect(version).toBe(0);
|
||||
});
|
||||
|
||||
it('returns 0 if request fails', async () => {
|
||||
esClient.indices.getIndexTemplate.mockRejectedValueOnce('Failure');
|
||||
|
||||
const version = await getLatestIndexTemplateVersion({
|
||||
esClient,
|
||||
name: '.siem-signals-*',
|
||||
});
|
||||
|
||||
expect(version).toBe(0);
|
||||
});
|
||||
});
|
|
@ -0,0 +1,34 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
import type { ElasticsearchClient } from '@kbn/core/server';
|
||||
|
||||
/**
|
||||
* Retrieves the latest version of index template
|
||||
* There are can be multiple index templates across different Kibana spaces,
|
||||
* so we get them all and return the latest(greatest) number
|
||||
*/
|
||||
export const getLatestIndexTemplateVersion = async ({
|
||||
esClient,
|
||||
name,
|
||||
}: {
|
||||
esClient: ElasticsearchClient;
|
||||
name: string;
|
||||
}): Promise<number> => {
|
||||
let latestTemplateVersion: number;
|
||||
try {
|
||||
const response = await esClient.indices.getIndexTemplate({ name });
|
||||
const versions = response.index_templates.map(
|
||||
(template) => template.index_template.version ?? 0
|
||||
);
|
||||
|
||||
latestTemplateVersion = versions.length ? Math.max(...versions) : 0;
|
||||
} catch (e) {
|
||||
latestTemplateVersion = 0;
|
||||
}
|
||||
|
||||
return latestTemplateVersion;
|
||||
};
|
|
@ -0,0 +1,177 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
import { elasticsearchServiceMock } from '@kbn/core/server/mocks';
|
||||
import { loggerMock } from '@kbn/logging-mocks';
|
||||
|
||||
import { getNonMigratedSignalsInfo } from './get_non_migrated_signals_info';
|
||||
import { getIndexVersionsByIndex } from './get_index_versions_by_index';
|
||||
import { getSignalVersionsByIndex } from './get_signal_versions_by_index';
|
||||
import { getLatestIndexTemplateVersion } from './get_latest_index_template_version';
|
||||
import { getIndexAliasPerSpace } from './get_index_alias_per_space';
|
||||
|
||||
jest.mock('./get_index_versions_by_index', () => ({ getIndexVersionsByIndex: jest.fn() }));
|
||||
jest.mock('./get_signal_versions_by_index', () => ({ getSignalVersionsByIndex: jest.fn() }));
|
||||
jest.mock('./get_latest_index_template_version', () => ({
|
||||
getLatestIndexTemplateVersion: jest.fn(),
|
||||
}));
|
||||
jest.mock('./get_index_alias_per_space', () => ({ getIndexAliasPerSpace: jest.fn() }));
|
||||
|
||||
const getIndexVersionsByIndexMock = getIndexVersionsByIndex as jest.Mock;
|
||||
const getSignalVersionsByIndexMock = getSignalVersionsByIndex as jest.Mock;
|
||||
const getLatestIndexTemplateVersionMock = getLatestIndexTemplateVersion as jest.Mock;
|
||||
const getIndexAliasPerSpaceMock = getIndexAliasPerSpace as jest.Mock;
|
||||
|
||||
const TEMPLATE_VERSION = 77;
|
||||
|
||||
describe('getNonMigratedSignalsInfo', () => {
|
||||
let esClient: ReturnType<typeof elasticsearchServiceMock.createElasticsearchClient>;
|
||||
const logger = loggerMock.create();
|
||||
|
||||
beforeEach(() => {
|
||||
esClient = elasticsearchServiceMock.createElasticsearchClient();
|
||||
|
||||
getLatestIndexTemplateVersionMock.mockReturnValue(TEMPLATE_VERSION);
|
||||
getIndexVersionsByIndexMock.mockReturnValue({
|
||||
'.siem-signals-another-1-legacy': 10,
|
||||
'.siem-signals-default-old-one': 42,
|
||||
});
|
||||
getSignalVersionsByIndexMock.mockReturnValue({
|
||||
'.siem-signals-another-1-legacy': [{ count: 2, version: 10 }],
|
||||
});
|
||||
getIndexAliasPerSpaceMock.mockReturnValue({
|
||||
'.siem-signals-another-1-legacy': {
|
||||
alias: '.siem-signals-another-1',
|
||||
indexName: '.siem-signals-another-1-legacy',
|
||||
space: 'another-1',
|
||||
},
|
||||
'.siem-signals-default-old-one': {
|
||||
alias: '.siem-signals-default',
|
||||
indexName: '.siem-signals-default-old-one',
|
||||
space: 'default',
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('returns empty results if no siem indices found', async () => {
|
||||
getIndexAliasPerSpaceMock.mockReturnValue({});
|
||||
|
||||
const result = await getNonMigratedSignalsInfo({
|
||||
esClient,
|
||||
signalsIndex: 'siem-signals',
|
||||
logger,
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
isMigrationRequired: false,
|
||||
spaces: [],
|
||||
indices: [],
|
||||
});
|
||||
});
|
||||
|
||||
it('returns empty when error happens', async () => {
|
||||
getLatestIndexTemplateVersionMock.mockRejectedValueOnce(new Error('Test failure'));
|
||||
const debugSpy = jest.spyOn(logger, 'debug');
|
||||
|
||||
const result = await getNonMigratedSignalsInfo({
|
||||
esClient,
|
||||
signalsIndex: 'siem-signals',
|
||||
logger,
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
isMigrationRequired: false,
|
||||
spaces: [],
|
||||
indices: [],
|
||||
});
|
||||
expect(debugSpy).toHaveBeenCalledWith(expect.stringContaining('Test failure'));
|
||||
});
|
||||
|
||||
it('returns empty results if no siem indices or signals outdated', async () => {
|
||||
getIndexVersionsByIndexMock.mockReturnValue({
|
||||
'.siem-signals-another-1-legacy': TEMPLATE_VERSION,
|
||||
'.siem-signals-default-old-one': TEMPLATE_VERSION,
|
||||
});
|
||||
getSignalVersionsByIndexMock.mockReturnValue({
|
||||
'.siem-signals-another-1-legacy': [{ count: 2, version: TEMPLATE_VERSION }],
|
||||
});
|
||||
|
||||
const result = await getNonMigratedSignalsInfo({
|
||||
esClient,
|
||||
signalsIndex: 'siem-signals',
|
||||
logger,
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
isMigrationRequired: false,
|
||||
spaces: [],
|
||||
indices: [],
|
||||
});
|
||||
});
|
||||
it('returns results for outdated index', async () => {
|
||||
getIndexVersionsByIndexMock.mockReturnValue({
|
||||
'.siem-signals-another-1-legacy': TEMPLATE_VERSION,
|
||||
'.siem-signals-default-old-one': 16,
|
||||
});
|
||||
getSignalVersionsByIndexMock.mockReturnValue({
|
||||
'.siem-signals-another-1-legacy': [{ count: 2, version: TEMPLATE_VERSION }],
|
||||
});
|
||||
|
||||
const result = await getNonMigratedSignalsInfo({
|
||||
esClient,
|
||||
signalsIndex: 'siem-signals',
|
||||
logger,
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
indices: ['.siem-signals-default-old-one'],
|
||||
isMigrationRequired: true,
|
||||
spaces: ['default'],
|
||||
});
|
||||
});
|
||||
it('returns results for outdated signals in index', async () => {
|
||||
getIndexVersionsByIndexMock.mockReturnValue({
|
||||
'.siem-signals-another-1-legacy': TEMPLATE_VERSION,
|
||||
'.siem-signals-default-old-one': TEMPLATE_VERSION,
|
||||
});
|
||||
getSignalVersionsByIndexMock.mockReturnValue({
|
||||
'.siem-signals-another-1-legacy': [{ count: 2, version: 12 }],
|
||||
});
|
||||
|
||||
const result = await getNonMigratedSignalsInfo({
|
||||
esClient,
|
||||
signalsIndex: 'siem-signals',
|
||||
logger,
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
indices: ['.siem-signals-another-1-legacy'],
|
||||
isMigrationRequired: true,
|
||||
spaces: ['another-1'],
|
||||
});
|
||||
});
|
||||
it('returns indices in multiple spaces', async () => {
|
||||
getIndexVersionsByIndexMock.mockReturnValue({
|
||||
'.siem-signals-another-1-legacy': 11,
|
||||
'.siem-signals-default-old-one': 11,
|
||||
});
|
||||
getSignalVersionsByIndexMock.mockReturnValue({
|
||||
'.siem-signals-another-1-legacy': [{ count: 2, version: 11 }],
|
||||
});
|
||||
|
||||
const result = await getNonMigratedSignalsInfo({
|
||||
esClient,
|
||||
signalsIndex: 'siem-signals',
|
||||
logger,
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
indices: ['.siem-signals-another-1-legacy', '.siem-signals-default-old-one'],
|
||||
isMigrationRequired: true,
|
||||
spaces: ['another-1', 'default'],
|
||||
});
|
||||
});
|
||||
});
|
|
@ -0,0 +1,131 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { ElasticsearchClient, Logger } from '@kbn/core/server';
|
||||
|
||||
import type { IndexVersionsByIndex } from './get_index_versions_by_index';
|
||||
import { getIndexVersionsByIndex } from './get_index_versions_by_index';
|
||||
import {
|
||||
getSignalVersionsByIndex,
|
||||
type SignalVersionsByIndex,
|
||||
} from './get_signal_versions_by_index';
|
||||
import { isOutdated as getIsOutdated, signalsAreOutdated } from './helpers';
|
||||
import { getLatestIndexTemplateVersion } from './get_latest_index_template_version';
|
||||
import { getIndexAliasPerSpace } from './get_index_alias_per_space';
|
||||
|
||||
interface OutdatedSpaces {
|
||||
isMigrationRequired: boolean;
|
||||
spaces: string[];
|
||||
indices: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* gets lists of spaces and non-migrated signal indices
|
||||
*/
|
||||
export const getNonMigratedSignalsInfo = async ({
|
||||
esClient,
|
||||
signalsIndex,
|
||||
logger,
|
||||
}: {
|
||||
esClient: ElasticsearchClient;
|
||||
signalsIndex: string;
|
||||
logger: Logger;
|
||||
}): Promise<OutdatedSpaces> => {
|
||||
const signalsAliasAllSpaces = `${signalsIndex}-*`;
|
||||
|
||||
try {
|
||||
const latestTemplateVersion = await getLatestIndexTemplateVersion({
|
||||
esClient,
|
||||
name: signalsAliasAllSpaces,
|
||||
});
|
||||
const indexAliasesMap = await getIndexAliasPerSpace({
|
||||
esClient,
|
||||
signalsAliasAllSpaces,
|
||||
signalsIndex,
|
||||
});
|
||||
|
||||
const indices = Object.keys(indexAliasesMap);
|
||||
|
||||
if (indices.length === 0) {
|
||||
return {
|
||||
isMigrationRequired: false,
|
||||
spaces: [],
|
||||
indices: [],
|
||||
};
|
||||
}
|
||||
|
||||
let indexVersionsByIndex: IndexVersionsByIndex = {};
|
||||
try {
|
||||
indexVersionsByIndex = await getIndexVersionsByIndex({
|
||||
esClient,
|
||||
index: indices,
|
||||
});
|
||||
} catch (e) {
|
||||
logger.debug(
|
||||
`Getting information about legacy siem signals index version failed:"${e?.message}"`
|
||||
);
|
||||
}
|
||||
|
||||
let signalVersionsByIndex: SignalVersionsByIndex = {};
|
||||
try {
|
||||
signalVersionsByIndex = await getSignalVersionsByIndex({
|
||||
esClient,
|
||||
index: indices,
|
||||
});
|
||||
} catch (e) {
|
||||
logger.debug(`Getting information about legacy siem signals versions failed:"${e?.message}"`);
|
||||
}
|
||||
|
||||
const outdatedIndices = indices.reduce<Array<{ indexName: string; space: string }>>(
|
||||
(acc, indexName) => {
|
||||
const version = indexVersionsByIndex[indexName] ?? 0;
|
||||
const signalVersions = signalVersionsByIndex[indexName] ?? [];
|
||||
|
||||
const isOutdated =
|
||||
getIsOutdated({ current: version, target: latestTemplateVersion }) ||
|
||||
signalsAreOutdated({ signalVersions, target: latestTemplateVersion });
|
||||
|
||||
if (isOutdated) {
|
||||
acc.push({
|
||||
indexName,
|
||||
space: indexAliasesMap[indexName].space,
|
||||
});
|
||||
}
|
||||
|
||||
return acc;
|
||||
},
|
||||
[]
|
||||
);
|
||||
|
||||
const outdatedIndexNames = outdatedIndices.map((outdatedIndex) => outdatedIndex.indexName);
|
||||
|
||||
// remove duplicated spaces
|
||||
const spaces = [...new Set<string>(outdatedIndices.map((indexStatus) => indexStatus.space))];
|
||||
const isMigrationRequired = outdatedIndices.length > 0;
|
||||
|
||||
logger.debug(
|
||||
isMigrationRequired
|
||||
? `Legacy siem signals indices require migration: "${outdatedIndexNames.join(
|
||||
', '
|
||||
)}" in "${spaces.join(', ')}" spaces`
|
||||
: 'No legacy siem indices require migration'
|
||||
);
|
||||
|
||||
return {
|
||||
isMigrationRequired,
|
||||
spaces,
|
||||
indices: outdatedIndexNames,
|
||||
};
|
||||
} catch (e) {
|
||||
logger.debug(`Getting information about legacy siem signals failed:"${e?.message}"`);
|
||||
return {
|
||||
isMigrationRequired: false,
|
||||
spaces: [],
|
||||
indices: [],
|
||||
};
|
||||
}
|
||||
};
|
|
@ -22,6 +22,7 @@ export interface CreateParams {
|
|||
export interface FinalizeParams {
|
||||
signalsAlias: string;
|
||||
migration: SignalsMigrationSO;
|
||||
legacySiemSignalsAlias: string;
|
||||
}
|
||||
|
||||
export interface DeleteParams {
|
||||
|
@ -59,13 +60,14 @@ export const signalsMigrationService = ({
|
|||
username,
|
||||
});
|
||||
},
|
||||
finalize: ({ migration, signalsAlias }) =>
|
||||
finalize: ({ migration, signalsAlias, legacySiemSignalsAlias }) =>
|
||||
finalizeMigration({
|
||||
esClient,
|
||||
migration,
|
||||
signalsAlias,
|
||||
soClient,
|
||||
username,
|
||||
legacySiemSignalsAlias,
|
||||
}),
|
||||
delete: ({ migration, signalsAlias }) =>
|
||||
deleteMigration({
|
||||
|
|
|
@ -26,11 +26,13 @@ export const replaceSignalsIndexAlias = async ({
|
|||
esClient,
|
||||
newIndex,
|
||||
oldIndex,
|
||||
legacySiemSignalsAlias,
|
||||
}: {
|
||||
alias: string;
|
||||
esClient: ElasticsearchClient;
|
||||
newIndex: string;
|
||||
oldIndex: string;
|
||||
legacySiemSignalsAlias: string;
|
||||
}): Promise<void> => {
|
||||
await esClient.indices.updateAliases({
|
||||
body: {
|
||||
|
@ -40,12 +42,11 @@ export const replaceSignalsIndexAlias = async ({
|
|||
],
|
||||
},
|
||||
});
|
||||
// TODO: space-aware?
|
||||
await esClient.indices.updateAliases({
|
||||
body: {
|
||||
actions: [
|
||||
{ remove: { index: oldIndex, alias: '.siem-signals-default' } },
|
||||
{ add: { index: newIndex, alias: '.siem-signals-default', is_write_index: false } },
|
||||
{ remove: { index: oldIndex, alias: legacySiemSignalsAlias } },
|
||||
{ add: { index: newIndex, alias: legacySiemSignalsAlias, is_write_index: false } },
|
||||
],
|
||||
},
|
||||
});
|
||||
|
|
|
@ -74,6 +74,7 @@ export const finalizeSignalsMigrationRoute = (
|
|||
});
|
||||
|
||||
const spaceId = securitySolution.getSpaceId();
|
||||
const legacySiemSignalsAlias = appClient.getSignalsIndex();
|
||||
const signalsAlias = ruleDataService.getResourceName(`security.alerts-${spaceId}`);
|
||||
const finalizeResults = await Promise.all(
|
||||
migrations.map(async (migration) => {
|
||||
|
@ -81,6 +82,7 @@ export const finalizeSignalsMigrationRoute = (
|
|||
const finalizedMigration = await migrationService.finalize({
|
||||
migration,
|
||||
signalsAlias,
|
||||
legacySiemSignalsAlias,
|
||||
});
|
||||
|
||||
if (isMigrationFailed(finalizedMigration)) {
|
||||
|
|
|
@ -65,7 +65,11 @@ export const getSignalsMigrationStatusRoute = (
|
|||
|
||||
const signalsAlias = appClient.getSignalsIndex();
|
||||
const currentVersion = await getTemplateVersion({ alias: signalsAlias, esClient });
|
||||
const indexAliases = await getIndexAliases({ alias: signalsAlias, esClient });
|
||||
const indexAliases = await getIndexAliases({
|
||||
alias: signalsAlias,
|
||||
esClient,
|
||||
index: `${signalsAlias}-*`,
|
||||
});
|
||||
const signalsIndices = indexAliases.map((indexAlias) => indexAlias.index);
|
||||
const indicesInRange = await getSignalsIndicesInRange({
|
||||
esClient,
|
||||
|
|
|
@ -281,6 +281,7 @@ export class Plugin implements ISecuritySolutionPlugin {
|
|||
all: allRiskScoreIndexPattern,
|
||||
latest: latestRiskScoreIndexPattern,
|
||||
},
|
||||
legacySignalsIndex: config.signalsIndex,
|
||||
});
|
||||
|
||||
this.telemetryUsageCounter = plugins.usageCollection?.createUsageCounter(APP_ID);
|
||||
|
|
|
@ -32,6 +32,7 @@ export const registerCollector: RegisterCollector = ({
|
|||
usageCollection,
|
||||
logger,
|
||||
riskEngineIndexPatterns,
|
||||
legacySignalsIndex,
|
||||
}) => {
|
||||
if (!usageCollection) {
|
||||
logger.debug('Usage collection is undefined, therefore returning early without registering it');
|
||||
|
@ -3076,6 +3077,21 @@ export const registerCollector: RegisterCollector = ({
|
|||
},
|
||||
},
|
||||
},
|
||||
legacy_siem_signals: {
|
||||
non_migrated_indices_total: {
|
||||
type: 'long',
|
||||
_meta: {
|
||||
description: 'Total number of non migrated legacy siem signals indices',
|
||||
},
|
||||
},
|
||||
spaces_total: {
|
||||
type: 'long',
|
||||
_meta: {
|
||||
description:
|
||||
'Total number of Kibana spaces that have non migrated legacy siem signals indices',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
endpointMetrics: {
|
||||
unique_endpoint_count: {
|
||||
|
@ -3130,6 +3146,7 @@ export const registerCollector: RegisterCollector = ({
|
|||
savedObjectsClient,
|
||||
logger,
|
||||
mlClient: ml,
|
||||
legacySignalsIndex,
|
||||
}),
|
||||
getEndpointMetrics({ esClient, logger }),
|
||||
getDashboardMetrics({
|
||||
|
|
|
@ -9,6 +9,8 @@ import type { DetectionMetrics } from './types';
|
|||
|
||||
import { getInitialMlJobUsage } from './ml_jobs/get_initial_usage';
|
||||
import { getInitialEventLogUsage, getInitialRulesUsage } from './rules/get_initial_usage';
|
||||
// eslint-disable-next-line no-restricted-imports
|
||||
import { getInitialLegacySiemSignalsUsage } from './legacy_siem_signals/get_initial_usage';
|
||||
|
||||
/**
|
||||
* Initial detection metrics initialized.
|
||||
|
@ -23,4 +25,5 @@ export const getInitialDetectionMetrics = (): DetectionMetrics => ({
|
|||
detection_rule_usage: getInitialRulesUsage(),
|
||||
detection_rule_status: getInitialEventLogUsage(),
|
||||
},
|
||||
legacy_siem_signals: getInitialLegacySiemSignalsUsage(),
|
||||
});
|
||||
|
|
|
@ -57,6 +57,7 @@ describe('Detections Usage and Metrics', () => {
|
|||
savedObjectsClient,
|
||||
logger,
|
||||
mlClient,
|
||||
legacySignalsIndex: '',
|
||||
});
|
||||
expect(result).toEqual<DetectionMetrics>(getInitialDetectionMetrics());
|
||||
});
|
||||
|
@ -79,6 +80,7 @@ describe('Detections Usage and Metrics', () => {
|
|||
savedObjectsClient,
|
||||
logger,
|
||||
mlClient,
|
||||
legacySignalsIndex: '',
|
||||
});
|
||||
|
||||
expect(result).toEqual<DetectionMetrics>({
|
||||
|
@ -154,6 +156,7 @@ describe('Detections Usage and Metrics', () => {
|
|||
savedObjectsClient,
|
||||
logger,
|
||||
mlClient,
|
||||
legacySignalsIndex: '',
|
||||
});
|
||||
|
||||
expect(result).toEqual<DetectionMetrics>({
|
||||
|
@ -210,6 +213,7 @@ describe('Detections Usage and Metrics', () => {
|
|||
savedObjectsClient,
|
||||
logger,
|
||||
mlClient,
|
||||
legacySignalsIndex: '',
|
||||
});
|
||||
|
||||
expect(result).toEqual<DetectionMetrics>({
|
||||
|
@ -290,6 +294,7 @@ describe('Detections Usage and Metrics', () => {
|
|||
savedObjectsClient,
|
||||
logger,
|
||||
mlClient,
|
||||
legacySignalsIndex: '',
|
||||
});
|
||||
expect(result).toEqual<DetectionMetrics>(getInitialDetectionMetrics());
|
||||
});
|
||||
|
@ -329,6 +334,7 @@ describe('Detections Usage and Metrics', () => {
|
|||
savedObjectsClient,
|
||||
logger,
|
||||
mlClient,
|
||||
legacySignalsIndex: '',
|
||||
});
|
||||
|
||||
expect(result).toEqual(
|
||||
|
|
|
@ -13,6 +13,10 @@ import { getMlJobMetrics } from './ml_jobs/get_metrics';
|
|||
import { getRuleMetrics } from './rules/get_metrics';
|
||||
import { getInitialEventLogUsage, getInitialRulesUsage } from './rules/get_initial_usage';
|
||||
import { getInitialMlJobUsage } from './ml_jobs/get_initial_usage';
|
||||
// eslint-disable-next-line no-restricted-imports
|
||||
import { getInitialLegacySiemSignalsUsage } from './legacy_siem_signals/get_initial_usage';
|
||||
// eslint-disable-next-line no-restricted-imports
|
||||
import { getLegacySiemSignalsUsage } from './legacy_siem_signals/get_legacy_siem_signals_metrics';
|
||||
|
||||
export interface GetDetectionsMetricsOptions {
|
||||
signalsIndex: string;
|
||||
|
@ -21,6 +25,7 @@ export interface GetDetectionsMetricsOptions {
|
|||
logger: Logger;
|
||||
mlClient: MlPluginSetup | undefined;
|
||||
eventLogIndex: string;
|
||||
legacySignalsIndex: string;
|
||||
}
|
||||
|
||||
export const getDetectionsMetrics = async ({
|
||||
|
@ -30,10 +35,12 @@ export const getDetectionsMetrics = async ({
|
|||
savedObjectsClient,
|
||||
logger,
|
||||
mlClient,
|
||||
legacySignalsIndex,
|
||||
}: GetDetectionsMetricsOptions): Promise<DetectionMetrics> => {
|
||||
const [mlJobMetrics, detectionRuleMetrics] = await Promise.allSettled([
|
||||
const [mlJobMetrics, detectionRuleMetrics, legacySiemSignalsUsage] = await Promise.allSettled([
|
||||
getMlJobMetrics({ mlClient, savedObjectsClient, logger }),
|
||||
getRuleMetrics({ signalsIndex, eventLogIndex, esClient, savedObjectsClient, logger }),
|
||||
getLegacySiemSignalsUsage({ signalsIndex: legacySignalsIndex, esClient, logger }),
|
||||
]);
|
||||
|
||||
return {
|
||||
|
@ -49,5 +56,9 @@ export const getDetectionsMetrics = async ({
|
|||
detection_rule_usage: getInitialRulesUsage(),
|
||||
detection_rule_status: getInitialEventLogUsage(),
|
||||
},
|
||||
legacy_siem_signals:
|
||||
legacySiemSignalsUsage.status === 'fulfilled'
|
||||
? legacySiemSignalsUsage.value
|
||||
: getInitialLegacySiemSignalsUsage(),
|
||||
};
|
||||
};
|
||||
|
|
|
@ -0,0 +1,13 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { LegacySiemSignals } from './types';
|
||||
|
||||
export const getInitialLegacySiemSignalsUsage = (): LegacySiemSignals => ({
|
||||
non_migrated_indices_total: 0,
|
||||
spaces_total: 0,
|
||||
});
|
|
@ -0,0 +1,33 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { ElasticsearchClient, Logger } from '@kbn/core/server';
|
||||
import { getNonMigratedSignalsInfo } from '../../../lib/detection_engine/migrations/get_non_migrated_signals_info';
|
||||
import type { LegacySiemSignals } from './types';
|
||||
|
||||
export interface GetLegacySiemSignalsUsageOptions {
|
||||
signalsIndex: string;
|
||||
esClient: ElasticsearchClient;
|
||||
logger: Logger;
|
||||
}
|
||||
|
||||
export const getLegacySiemSignalsUsage = async ({
|
||||
signalsIndex,
|
||||
esClient,
|
||||
logger,
|
||||
}: GetLegacySiemSignalsUsageOptions): Promise<LegacySiemSignals> => {
|
||||
const { indices, spaces } = await getNonMigratedSignalsInfo({
|
||||
esClient,
|
||||
signalsIndex,
|
||||
logger,
|
||||
});
|
||||
|
||||
return {
|
||||
non_migrated_indices_total: indices.length,
|
||||
spaces_total: spaces.length,
|
||||
};
|
||||
};
|
|
@ -0,0 +1,10 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
export interface LegacySiemSignals {
|
||||
non_migrated_indices_total: number;
|
||||
spaces_total: number;
|
||||
}
|
|
@ -7,8 +7,11 @@
|
|||
|
||||
import type { MlJobUsageMetric } from './ml_jobs/types';
|
||||
import type { RuleAdoption } from './rules/types';
|
||||
// eslint-disable-next-line no-restricted-imports
|
||||
import type { LegacySiemSignals } from './legacy_siem_signals/types';
|
||||
|
||||
export interface DetectionMetrics {
|
||||
ml_jobs: MlJobUsageMetric;
|
||||
detection_rules: RuleAdoption;
|
||||
legacy_siem_signals: LegacySiemSignals;
|
||||
}
|
||||
|
|
|
@ -33,6 +33,7 @@ export type CollectorDependencies = {
|
|||
all: string;
|
||||
latest: string;
|
||||
};
|
||||
legacySignalsIndex: string;
|
||||
} & Pick<SetupPlugins, 'ml' | 'usageCollection'>;
|
||||
|
||||
export interface AlertBucket {
|
||||
|
|
|
@ -19470,6 +19470,22 @@
|
|||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"legacy_siem_signals": {
|
||||
"properties": {
|
||||
"non_migrated_indices_total": {
|
||||
"type": "long",
|
||||
"_meta": {
|
||||
"description": "Total number of non migrated legacy siem signals indices"
|
||||
}
|
||||
},
|
||||
"spaces_total": {
|
||||
"type": "long",
|
||||
"_meta": {
|
||||
"description": "Total number of Kibana spaces that have non migrated legacy siem signals indices"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
|
@ -13,7 +13,7 @@ import {
|
|||
DETECTION_ENGINE_SIGNALS_MIGRATION_URL,
|
||||
} from '@kbn/security-solution-plugin/common/constants';
|
||||
import { ROLES } from '@kbn/security-solution-plugin/common/test';
|
||||
import { deleteMigrations, getIndexNameFromLoad } from '../../../../../utils';
|
||||
import { deleteMigrationsIfExistent, getIndexNameFromLoad } from '../../../../../utils';
|
||||
import {
|
||||
createAlertsIndex,
|
||||
deleteAllAlerts,
|
||||
|
@ -84,10 +84,12 @@ export default ({ getService }: FtrProviderContext): void => {
|
|||
|
||||
afterEach(async () => {
|
||||
await esArchiver.unload('x-pack/test/functional/es_archives/signals/outdated_signals_index');
|
||||
await deleteMigrations({
|
||||
await deleteMigrationsIfExistent({
|
||||
kbnClient,
|
||||
ids: [createdMigration.migration_id],
|
||||
});
|
||||
// we need to delete migrated index, otherwise create migration call(in beforeEach hook) will fail
|
||||
await es.indices.delete({ index: createdMigration.migration_index });
|
||||
await deleteAllAlerts(supertest, log, es);
|
||||
});
|
||||
|
||||
|
@ -99,6 +101,7 @@ export default ({ getService }: FtrProviderContext): void => {
|
|||
.expect(200);
|
||||
|
||||
const deletedMigration = body.migrations[0];
|
||||
expect(deletedMigration.error).to.eql(undefined);
|
||||
expect(deletedMigration.id).to.eql(createdMigration.migration_id);
|
||||
expect(deletedMigration.sourceIndex).to.eql(outdatedAlertsIndexName);
|
||||
});
|
||||
|
|
|
@ -25,3 +25,28 @@ export const deleteMigrations = async ({
|
|||
)
|
||||
);
|
||||
};
|
||||
|
||||
export const deleteMigrationsIfExistent = async ({
|
||||
ids,
|
||||
kbnClient,
|
||||
}: {
|
||||
ids: string[];
|
||||
kbnClient: KbnClient;
|
||||
}): Promise<void> => {
|
||||
await Promise.all(
|
||||
ids.map(async (id) => {
|
||||
try {
|
||||
const res = await kbnClient.savedObjects.delete({
|
||||
id,
|
||||
type: signalsMigrationType,
|
||||
});
|
||||
return res;
|
||||
} catch (e) {
|
||||
// do not throw error when migration already deleted/not found
|
||||
if (e?.response?.status !== 404) {
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
})
|
||||
);
|
||||
};
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue