[Security Solution][Detection Engine] adds legacy siem signals telemetry (#202671)

## Summary

- partly addresses https://github.com/elastic/kibana/issues/195523
- adds snapshot telemetry that shows number of legacy siem signals and
number of spaces they are in
- while working on PR, discovered and fixed few issues in APIs
- get migration status API did not work correctly with new `.alerts-*`
indices, listing them as outdated
- finalize migration API did account for spaces, when adding alias to
migrated index
- remove migration API failed due to lack of permissions to removed
migration task from `.tasks` index

### How to test

#### How to create legacy siem index?

run script that used for FTR tests

```bash
node scripts/es_archiver --kibana-url=http://elastic:changeme@localhost:5601 --es-url=http://elastic:changeme@localhost:9200 load x-pack/test/functional/es_archives/signals/legacy_signals_index

```
These would create legacy siem indices. But be aware, it might break
Kibana .alerts indices creation. But sufficient for testing


#### How to test snapshot telemetry

Snapshot
For snapshot telemetry use
[API](https://docs.elastic.dev/telemetry/collection/snapshot-telemetry#telemetry-usage-payload-api)
call
OR
Check snapshots in Kibana adv settings -> Global Settings Tab -> Usage
collection section -> Click on cluster data example link -> Check
`legacy_siem_signals ` fields in flyout

<details>
<summary> Snapshot telemetry </summary>


<img width="2549" alt="Screenshot 2024-12-03 at 13 08 03"
src="https://github.com/user-attachments/assets/28ffe983-01c7-4435-a82a-9a968d32d5e0">


 </details>

---------

Co-authored-by: Ryland Herrick <ryalnd@gmail.com>
This commit is contained in:
Vitalii Dmyterko 2024-12-11 10:03:19 +00:00 committed by GitHub
parent f1f3a4fddd
commit 8821e034e9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
27 changed files with 714 additions and 14 deletions

View file

@ -20,26 +20,30 @@ interface IndexAlias {
*
* @param esClient An {@link ElasticsearchClient}
* @param alias alias name used to filter results
* @param index index name used to filter results
*
* @returns an array of {@link IndexAlias} objects
*/
export const getIndexAliases = async ({
esClient,
alias,
index,
}: {
esClient: ElasticsearchClient;
alias: string;
index?: string;
}): Promise<IndexAlias[]> => {
const response = await esClient.indices.getAlias(
{
name: alias,
...(index ? { index } : {}),
},
{ meta: true }
);
return Object.keys(response.body).map((index) => ({
return Object.keys(response.body).map((indexName) => ({
alias,
index,
isWriteIndex: response.body[index].aliases[alias]?.is_write_index === true,
index: indexName,
isWriteIndex: response.body[indexName].aliases[alias]?.is_write_index === true,
}));
};

View file

@ -14,7 +14,6 @@ import type { SignalsMigrationSO } from './saved_objects_schema';
/**
* Deletes a completed migration:
* * deletes the migration SO
* * deletes the underlying task document
* * applies deletion policy to the relevant index
*
* @param esClient An {@link ElasticsearchClient}
@ -40,7 +39,7 @@ export const deleteMigration = async ({
return migration;
}
const { destinationIndex, sourceIndex, taskId } = migration.attributes;
const { destinationIndex, sourceIndex } = migration.attributes;
if (isMigrationFailed(migration)) {
await applyMigrationCleanupPolicy({
@ -57,7 +56,6 @@ export const deleteMigration = async ({
});
}
await esClient.delete({ index: '.tasks', id: taskId });
await deleteMigrationSavedObject({ id: migration.id, soClient });
return migration;

View file

@ -40,6 +40,7 @@ describe('finalizeMigration', () => {
signalsAlias: 'my-signals-alias',
soClient,
username: 'username',
legacySiemSignalsAlias: '.siem-signals-default',
});
expect(updateMigrationSavedObject).not.toHaveBeenCalled();
@ -54,6 +55,7 @@ describe('finalizeMigration', () => {
signalsAlias: 'my-signals-alias',
soClient,
username: 'username',
legacySiemSignalsAlias: '.siem-signals-default',
});
expect(updateMigrationSavedObject).not.toHaveBeenCalled();
@ -72,6 +74,7 @@ describe('finalizeMigration', () => {
signalsAlias: 'my-signals-alias',
soClient,
username: 'username',
legacySiemSignalsAlias: '.siem-signals-default',
});
expect(updateMigrationSavedObject).toHaveBeenCalledWith(

View file

@ -35,12 +35,14 @@ export const finalizeMigration = async ({
signalsAlias,
soClient,
username,
legacySiemSignalsAlias,
}: {
esClient: ElasticsearchClient;
migration: SignalsMigrationSO;
signalsAlias: string;
soClient: SavedObjectsClientContract;
username: string;
legacySiemSignalsAlias: string;
}): Promise<SignalsMigrationSO> => {
if (!isMigrationPending(migration)) {
return migration;
@ -86,6 +88,7 @@ export const finalizeMigration = async ({
esClient,
newIndex: destinationIndex,
oldIndex: sourceIndex,
legacySiemSignalsAlias,
});
const updatedMigration = await updateMigrationSavedObject({

View file

@ -0,0 +1,88 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { elasticsearchServiceMock } from '@kbn/core/server/mocks';
import { getIndexAliasPerSpace } from './get_index_alias_per_space';
describe('getIndexAliasPerSpace', () => {
let esClient: ReturnType<typeof elasticsearchServiceMock.createElasticsearchClient>;
beforeEach(() => {
esClient = elasticsearchServiceMock.createElasticsearchClient();
});
it('returns object with index alias and space', async () => {
esClient.indices.getAlias.mockResponseOnce({
'.siem-signals-default-old-one': {
aliases: {
'.siem-signals-default': {
is_write_index: false,
},
},
},
'.siem-signals-another-1-legacy': {
aliases: {
'.siem-signals-another-1': {
is_write_index: false,
},
},
},
});
const result = await getIndexAliasPerSpace({
esClient,
signalsIndex: '.siem-signals',
signalsAliasAllSpaces: '.siem-signals-*',
});
expect(result).toEqual({
'.siem-signals-another-1-legacy': {
alias: '.siem-signals-another-1',
indexName: '.siem-signals-another-1-legacy',
space: 'another-1',
},
'.siem-signals-default-old-one': {
alias: '.siem-signals-default',
indexName: '.siem-signals-default-old-one',
space: 'default',
},
});
});
it('filters out .internal.alert indices', async () => {
esClient.indices.getAlias.mockResponseOnce({
'.siem-signals-default-old-one': {
aliases: {
'.siem-signals-default': {
is_write_index: false,
},
},
},
'.internal.alerts-security.alerts-another-2-000001': {
aliases: {
'.siem-signals-another-2': {
is_write_index: false,
},
},
},
});
const result = await getIndexAliasPerSpace({
esClient,
signalsIndex: '.siem-signals',
signalsAliasAllSpaces: '.siem-signals-*',
});
expect(result).toEqual({
'.siem-signals-default-old-one': {
alias: '.siem-signals-default',
indexName: '.siem-signals-default-old-one',
space: 'default',
},
});
});
});

View file

@ -0,0 +1,52 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { ElasticsearchClient } from '@kbn/core/server';
interface IndexAlias {
alias: string;
space: string;
indexName: string;
}
/**
* Retrieves index, its alias and Kibana space
*/
export const getIndexAliasPerSpace = async ({
esClient,
signalsIndex,
signalsAliasAllSpaces,
}: {
esClient: ElasticsearchClient;
signalsIndex: string;
signalsAliasAllSpaces: string;
}): Promise<Record<string, IndexAlias>> => {
const response = await esClient.indices.getAlias(
{
name: signalsAliasAllSpaces,
},
{ meta: true }
);
const indexAliasesMap = Object.keys(response.body).reduce<Record<string, IndexAlias>>(
(acc, indexName) => {
if (!indexName.startsWith('.internal.alerts-')) {
const alias = Object.keys(response.body[indexName].aliases)[0];
acc[indexName] = {
alias,
space: alias.replace(`${signalsIndex}-`, ''),
indexName,
};
}
return acc;
},
{}
);
return indexAliasesMap;
};

View file

@ -0,0 +1,59 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { IndicesGetIndexTemplateResponse } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import { elasticsearchServiceMock } from '@kbn/core/server/mocks';
import { getLatestIndexTemplateVersion } from './get_latest_index_template_version';
describe('getIndexAliasPerSpace', () => {
let esClient: ReturnType<typeof elasticsearchServiceMock.createElasticsearchClient>;
beforeEach(() => {
esClient = elasticsearchServiceMock.createElasticsearchClient();
});
it('returns latest index template version', async () => {
esClient.indices.getIndexTemplate.mockResponseOnce({
index_templates: [
{ index_template: { version: 77 } },
{ index_template: { version: 10 } },
{ index_template: { version: 23 } },
{ index_template: { version: 0 } },
],
} as IndicesGetIndexTemplateResponse);
const version = await getLatestIndexTemplateVersion({
esClient,
name: '.siem-signals-*',
});
expect(version).toBe(77);
});
it('returns 0 if templates empty', async () => {
esClient.indices.getIndexTemplate.mockResponseOnce({
index_templates: [],
});
const version = await getLatestIndexTemplateVersion({
esClient,
name: '.siem-signals-*',
});
expect(version).toBe(0);
});
it('returns 0 if request fails', async () => {
esClient.indices.getIndexTemplate.mockRejectedValueOnce('Failure');
const version = await getLatestIndexTemplateVersion({
esClient,
name: '.siem-signals-*',
});
expect(version).toBe(0);
});
});

View file

@ -0,0 +1,34 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { ElasticsearchClient } from '@kbn/core/server';
/**
* Retrieves the latest version of index template
* There are can be multiple index templates across different Kibana spaces,
* so we get them all and return the latest(greatest) number
*/
export const getLatestIndexTemplateVersion = async ({
esClient,
name,
}: {
esClient: ElasticsearchClient;
name: string;
}): Promise<number> => {
let latestTemplateVersion: number;
try {
const response = await esClient.indices.getIndexTemplate({ name });
const versions = response.index_templates.map(
(template) => template.index_template.version ?? 0
);
latestTemplateVersion = versions.length ? Math.max(...versions) : 0;
} catch (e) {
latestTemplateVersion = 0;
}
return latestTemplateVersion;
};

View file

@ -0,0 +1,177 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { elasticsearchServiceMock } from '@kbn/core/server/mocks';
import { loggerMock } from '@kbn/logging-mocks';
import { getNonMigratedSignalsInfo } from './get_non_migrated_signals_info';
import { getIndexVersionsByIndex } from './get_index_versions_by_index';
import { getSignalVersionsByIndex } from './get_signal_versions_by_index';
import { getLatestIndexTemplateVersion } from './get_latest_index_template_version';
import { getIndexAliasPerSpace } from './get_index_alias_per_space';
jest.mock('./get_index_versions_by_index', () => ({ getIndexVersionsByIndex: jest.fn() }));
jest.mock('./get_signal_versions_by_index', () => ({ getSignalVersionsByIndex: jest.fn() }));
jest.mock('./get_latest_index_template_version', () => ({
getLatestIndexTemplateVersion: jest.fn(),
}));
jest.mock('./get_index_alias_per_space', () => ({ getIndexAliasPerSpace: jest.fn() }));
const getIndexVersionsByIndexMock = getIndexVersionsByIndex as jest.Mock;
const getSignalVersionsByIndexMock = getSignalVersionsByIndex as jest.Mock;
const getLatestIndexTemplateVersionMock = getLatestIndexTemplateVersion as jest.Mock;
const getIndexAliasPerSpaceMock = getIndexAliasPerSpace as jest.Mock;
const TEMPLATE_VERSION = 77;
describe('getNonMigratedSignalsInfo', () => {
let esClient: ReturnType<typeof elasticsearchServiceMock.createElasticsearchClient>;
const logger = loggerMock.create();
beforeEach(() => {
esClient = elasticsearchServiceMock.createElasticsearchClient();
getLatestIndexTemplateVersionMock.mockReturnValue(TEMPLATE_VERSION);
getIndexVersionsByIndexMock.mockReturnValue({
'.siem-signals-another-1-legacy': 10,
'.siem-signals-default-old-one': 42,
});
getSignalVersionsByIndexMock.mockReturnValue({
'.siem-signals-another-1-legacy': [{ count: 2, version: 10 }],
});
getIndexAliasPerSpaceMock.mockReturnValue({
'.siem-signals-another-1-legacy': {
alias: '.siem-signals-another-1',
indexName: '.siem-signals-another-1-legacy',
space: 'another-1',
},
'.siem-signals-default-old-one': {
alias: '.siem-signals-default',
indexName: '.siem-signals-default-old-one',
space: 'default',
},
});
});
it('returns empty results if no siem indices found', async () => {
getIndexAliasPerSpaceMock.mockReturnValue({});
const result = await getNonMigratedSignalsInfo({
esClient,
signalsIndex: 'siem-signals',
logger,
});
expect(result).toEqual({
isMigrationRequired: false,
spaces: [],
indices: [],
});
});
it('returns empty when error happens', async () => {
getLatestIndexTemplateVersionMock.mockRejectedValueOnce(new Error('Test failure'));
const debugSpy = jest.spyOn(logger, 'debug');
const result = await getNonMigratedSignalsInfo({
esClient,
signalsIndex: 'siem-signals',
logger,
});
expect(result).toEqual({
isMigrationRequired: false,
spaces: [],
indices: [],
});
expect(debugSpy).toHaveBeenCalledWith(expect.stringContaining('Test failure'));
});
it('returns empty results if no siem indices or signals outdated', async () => {
getIndexVersionsByIndexMock.mockReturnValue({
'.siem-signals-another-1-legacy': TEMPLATE_VERSION,
'.siem-signals-default-old-one': TEMPLATE_VERSION,
});
getSignalVersionsByIndexMock.mockReturnValue({
'.siem-signals-another-1-legacy': [{ count: 2, version: TEMPLATE_VERSION }],
});
const result = await getNonMigratedSignalsInfo({
esClient,
signalsIndex: 'siem-signals',
logger,
});
expect(result).toEqual({
isMigrationRequired: false,
spaces: [],
indices: [],
});
});
it('returns results for outdated index', async () => {
getIndexVersionsByIndexMock.mockReturnValue({
'.siem-signals-another-1-legacy': TEMPLATE_VERSION,
'.siem-signals-default-old-one': 16,
});
getSignalVersionsByIndexMock.mockReturnValue({
'.siem-signals-another-1-legacy': [{ count: 2, version: TEMPLATE_VERSION }],
});
const result = await getNonMigratedSignalsInfo({
esClient,
signalsIndex: 'siem-signals',
logger,
});
expect(result).toEqual({
indices: ['.siem-signals-default-old-one'],
isMigrationRequired: true,
spaces: ['default'],
});
});
it('returns results for outdated signals in index', async () => {
getIndexVersionsByIndexMock.mockReturnValue({
'.siem-signals-another-1-legacy': TEMPLATE_VERSION,
'.siem-signals-default-old-one': TEMPLATE_VERSION,
});
getSignalVersionsByIndexMock.mockReturnValue({
'.siem-signals-another-1-legacy': [{ count: 2, version: 12 }],
});
const result = await getNonMigratedSignalsInfo({
esClient,
signalsIndex: 'siem-signals',
logger,
});
expect(result).toEqual({
indices: ['.siem-signals-another-1-legacy'],
isMigrationRequired: true,
spaces: ['another-1'],
});
});
it('returns indices in multiple spaces', async () => {
getIndexVersionsByIndexMock.mockReturnValue({
'.siem-signals-another-1-legacy': 11,
'.siem-signals-default-old-one': 11,
});
getSignalVersionsByIndexMock.mockReturnValue({
'.siem-signals-another-1-legacy': [{ count: 2, version: 11 }],
});
const result = await getNonMigratedSignalsInfo({
esClient,
signalsIndex: 'siem-signals',
logger,
});
expect(result).toEqual({
indices: ['.siem-signals-another-1-legacy', '.siem-signals-default-old-one'],
isMigrationRequired: true,
spaces: ['another-1', 'default'],
});
});
});

View file

@ -0,0 +1,131 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { ElasticsearchClient, Logger } from '@kbn/core/server';
import type { IndexVersionsByIndex } from './get_index_versions_by_index';
import { getIndexVersionsByIndex } from './get_index_versions_by_index';
import {
getSignalVersionsByIndex,
type SignalVersionsByIndex,
} from './get_signal_versions_by_index';
import { isOutdated as getIsOutdated, signalsAreOutdated } from './helpers';
import { getLatestIndexTemplateVersion } from './get_latest_index_template_version';
import { getIndexAliasPerSpace } from './get_index_alias_per_space';
interface OutdatedSpaces {
isMigrationRequired: boolean;
spaces: string[];
indices: string[];
}
/**
* gets lists of spaces and non-migrated signal indices
*/
export const getNonMigratedSignalsInfo = async ({
esClient,
signalsIndex,
logger,
}: {
esClient: ElasticsearchClient;
signalsIndex: string;
logger: Logger;
}): Promise<OutdatedSpaces> => {
const signalsAliasAllSpaces = `${signalsIndex}-*`;
try {
const latestTemplateVersion = await getLatestIndexTemplateVersion({
esClient,
name: signalsAliasAllSpaces,
});
const indexAliasesMap = await getIndexAliasPerSpace({
esClient,
signalsAliasAllSpaces,
signalsIndex,
});
const indices = Object.keys(indexAliasesMap);
if (indices.length === 0) {
return {
isMigrationRequired: false,
spaces: [],
indices: [],
};
}
let indexVersionsByIndex: IndexVersionsByIndex = {};
try {
indexVersionsByIndex = await getIndexVersionsByIndex({
esClient,
index: indices,
});
} catch (e) {
logger.debug(
`Getting information about legacy siem signals index version failed:"${e?.message}"`
);
}
let signalVersionsByIndex: SignalVersionsByIndex = {};
try {
signalVersionsByIndex = await getSignalVersionsByIndex({
esClient,
index: indices,
});
} catch (e) {
logger.debug(`Getting information about legacy siem signals versions failed:"${e?.message}"`);
}
const outdatedIndices = indices.reduce<Array<{ indexName: string; space: string }>>(
(acc, indexName) => {
const version = indexVersionsByIndex[indexName] ?? 0;
const signalVersions = signalVersionsByIndex[indexName] ?? [];
const isOutdated =
getIsOutdated({ current: version, target: latestTemplateVersion }) ||
signalsAreOutdated({ signalVersions, target: latestTemplateVersion });
if (isOutdated) {
acc.push({
indexName,
space: indexAliasesMap[indexName].space,
});
}
return acc;
},
[]
);
const outdatedIndexNames = outdatedIndices.map((outdatedIndex) => outdatedIndex.indexName);
// remove duplicated spaces
const spaces = [...new Set<string>(outdatedIndices.map((indexStatus) => indexStatus.space))];
const isMigrationRequired = outdatedIndices.length > 0;
logger.debug(
isMigrationRequired
? `Legacy siem signals indices require migration: "${outdatedIndexNames.join(
', '
)}" in "${spaces.join(', ')}" spaces`
: 'No legacy siem indices require migration'
);
return {
isMigrationRequired,
spaces,
indices: outdatedIndexNames,
};
} catch (e) {
logger.debug(`Getting information about legacy siem signals failed:"${e?.message}"`);
return {
isMigrationRequired: false,
spaces: [],
indices: [],
};
}
};

View file

@ -22,6 +22,7 @@ export interface CreateParams {
export interface FinalizeParams {
signalsAlias: string;
migration: SignalsMigrationSO;
legacySiemSignalsAlias: string;
}
export interface DeleteParams {
@ -59,13 +60,14 @@ export const signalsMigrationService = ({
username,
});
},
finalize: ({ migration, signalsAlias }) =>
finalize: ({ migration, signalsAlias, legacySiemSignalsAlias }) =>
finalizeMigration({
esClient,
migration,
signalsAlias,
soClient,
username,
legacySiemSignalsAlias,
}),
delete: ({ migration, signalsAlias }) =>
deleteMigration({

View file

@ -26,11 +26,13 @@ export const replaceSignalsIndexAlias = async ({
esClient,
newIndex,
oldIndex,
legacySiemSignalsAlias,
}: {
alias: string;
esClient: ElasticsearchClient;
newIndex: string;
oldIndex: string;
legacySiemSignalsAlias: string;
}): Promise<void> => {
await esClient.indices.updateAliases({
body: {
@ -40,12 +42,11 @@ export const replaceSignalsIndexAlias = async ({
],
},
});
// TODO: space-aware?
await esClient.indices.updateAliases({
body: {
actions: [
{ remove: { index: oldIndex, alias: '.siem-signals-default' } },
{ add: { index: newIndex, alias: '.siem-signals-default', is_write_index: false } },
{ remove: { index: oldIndex, alias: legacySiemSignalsAlias } },
{ add: { index: newIndex, alias: legacySiemSignalsAlias, is_write_index: false } },
],
},
});

View file

@ -74,6 +74,7 @@ export const finalizeSignalsMigrationRoute = (
});
const spaceId = securitySolution.getSpaceId();
const legacySiemSignalsAlias = appClient.getSignalsIndex();
const signalsAlias = ruleDataService.getResourceName(`security.alerts-${spaceId}`);
const finalizeResults = await Promise.all(
migrations.map(async (migration) => {
@ -81,6 +82,7 @@ export const finalizeSignalsMigrationRoute = (
const finalizedMigration = await migrationService.finalize({
migration,
signalsAlias,
legacySiemSignalsAlias,
});
if (isMigrationFailed(finalizedMigration)) {

View file

@ -65,7 +65,11 @@ export const getSignalsMigrationStatusRoute = (
const signalsAlias = appClient.getSignalsIndex();
const currentVersion = await getTemplateVersion({ alias: signalsAlias, esClient });
const indexAliases = await getIndexAliases({ alias: signalsAlias, esClient });
const indexAliases = await getIndexAliases({
alias: signalsAlias,
esClient,
index: `${signalsAlias}-*`,
});
const signalsIndices = indexAliases.map((indexAlias) => indexAlias.index);
const indicesInRange = await getSignalsIndicesInRange({
esClient,

View file

@ -281,6 +281,7 @@ export class Plugin implements ISecuritySolutionPlugin {
all: allRiskScoreIndexPattern,
latest: latestRiskScoreIndexPattern,
},
legacySignalsIndex: config.signalsIndex,
});
this.telemetryUsageCounter = plugins.usageCollection?.createUsageCounter(APP_ID);

View file

@ -32,6 +32,7 @@ export const registerCollector: RegisterCollector = ({
usageCollection,
logger,
riskEngineIndexPatterns,
legacySignalsIndex,
}) => {
if (!usageCollection) {
logger.debug('Usage collection is undefined, therefore returning early without registering it');
@ -3076,6 +3077,21 @@ export const registerCollector: RegisterCollector = ({
},
},
},
legacy_siem_signals: {
non_migrated_indices_total: {
type: 'long',
_meta: {
description: 'Total number of non migrated legacy siem signals indices',
},
},
spaces_total: {
type: 'long',
_meta: {
description:
'Total number of Kibana spaces that have non migrated legacy siem signals indices',
},
},
},
},
endpointMetrics: {
unique_endpoint_count: {
@ -3130,6 +3146,7 @@ export const registerCollector: RegisterCollector = ({
savedObjectsClient,
logger,
mlClient: ml,
legacySignalsIndex,
}),
getEndpointMetrics({ esClient, logger }),
getDashboardMetrics({

View file

@ -9,6 +9,8 @@ import type { DetectionMetrics } from './types';
import { getInitialMlJobUsage } from './ml_jobs/get_initial_usage';
import { getInitialEventLogUsage, getInitialRulesUsage } from './rules/get_initial_usage';
// eslint-disable-next-line no-restricted-imports
import { getInitialLegacySiemSignalsUsage } from './legacy_siem_signals/get_initial_usage';
/**
* Initial detection metrics initialized.
@ -23,4 +25,5 @@ export const getInitialDetectionMetrics = (): DetectionMetrics => ({
detection_rule_usage: getInitialRulesUsage(),
detection_rule_status: getInitialEventLogUsage(),
},
legacy_siem_signals: getInitialLegacySiemSignalsUsage(),
});

View file

@ -57,6 +57,7 @@ describe('Detections Usage and Metrics', () => {
savedObjectsClient,
logger,
mlClient,
legacySignalsIndex: '',
});
expect(result).toEqual<DetectionMetrics>(getInitialDetectionMetrics());
});
@ -79,6 +80,7 @@ describe('Detections Usage and Metrics', () => {
savedObjectsClient,
logger,
mlClient,
legacySignalsIndex: '',
});
expect(result).toEqual<DetectionMetrics>({
@ -154,6 +156,7 @@ describe('Detections Usage and Metrics', () => {
savedObjectsClient,
logger,
mlClient,
legacySignalsIndex: '',
});
expect(result).toEqual<DetectionMetrics>({
@ -210,6 +213,7 @@ describe('Detections Usage and Metrics', () => {
savedObjectsClient,
logger,
mlClient,
legacySignalsIndex: '',
});
expect(result).toEqual<DetectionMetrics>({
@ -290,6 +294,7 @@ describe('Detections Usage and Metrics', () => {
savedObjectsClient,
logger,
mlClient,
legacySignalsIndex: '',
});
expect(result).toEqual<DetectionMetrics>(getInitialDetectionMetrics());
});
@ -329,6 +334,7 @@ describe('Detections Usage and Metrics', () => {
savedObjectsClient,
logger,
mlClient,
legacySignalsIndex: '',
});
expect(result).toEqual(

View file

@ -13,6 +13,10 @@ import { getMlJobMetrics } from './ml_jobs/get_metrics';
import { getRuleMetrics } from './rules/get_metrics';
import { getInitialEventLogUsage, getInitialRulesUsage } from './rules/get_initial_usage';
import { getInitialMlJobUsage } from './ml_jobs/get_initial_usage';
// eslint-disable-next-line no-restricted-imports
import { getInitialLegacySiemSignalsUsage } from './legacy_siem_signals/get_initial_usage';
// eslint-disable-next-line no-restricted-imports
import { getLegacySiemSignalsUsage } from './legacy_siem_signals/get_legacy_siem_signals_metrics';
export interface GetDetectionsMetricsOptions {
signalsIndex: string;
@ -21,6 +25,7 @@ export interface GetDetectionsMetricsOptions {
logger: Logger;
mlClient: MlPluginSetup | undefined;
eventLogIndex: string;
legacySignalsIndex: string;
}
export const getDetectionsMetrics = async ({
@ -30,10 +35,12 @@ export const getDetectionsMetrics = async ({
savedObjectsClient,
logger,
mlClient,
legacySignalsIndex,
}: GetDetectionsMetricsOptions): Promise<DetectionMetrics> => {
const [mlJobMetrics, detectionRuleMetrics] = await Promise.allSettled([
const [mlJobMetrics, detectionRuleMetrics, legacySiemSignalsUsage] = await Promise.allSettled([
getMlJobMetrics({ mlClient, savedObjectsClient, logger }),
getRuleMetrics({ signalsIndex, eventLogIndex, esClient, savedObjectsClient, logger }),
getLegacySiemSignalsUsage({ signalsIndex: legacySignalsIndex, esClient, logger }),
]);
return {
@ -49,5 +56,9 @@ export const getDetectionsMetrics = async ({
detection_rule_usage: getInitialRulesUsage(),
detection_rule_status: getInitialEventLogUsage(),
},
legacy_siem_signals:
legacySiemSignalsUsage.status === 'fulfilled'
? legacySiemSignalsUsage.value
: getInitialLegacySiemSignalsUsage(),
};
};

View file

@ -0,0 +1,13 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { LegacySiemSignals } from './types';
export const getInitialLegacySiemSignalsUsage = (): LegacySiemSignals => ({
non_migrated_indices_total: 0,
spaces_total: 0,
});

View file

@ -0,0 +1,33 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { ElasticsearchClient, Logger } from '@kbn/core/server';
import { getNonMigratedSignalsInfo } from '../../../lib/detection_engine/migrations/get_non_migrated_signals_info';
import type { LegacySiemSignals } from './types';
export interface GetLegacySiemSignalsUsageOptions {
signalsIndex: string;
esClient: ElasticsearchClient;
logger: Logger;
}
export const getLegacySiemSignalsUsage = async ({
signalsIndex,
esClient,
logger,
}: GetLegacySiemSignalsUsageOptions): Promise<LegacySiemSignals> => {
const { indices, spaces } = await getNonMigratedSignalsInfo({
esClient,
signalsIndex,
logger,
});
return {
non_migrated_indices_total: indices.length,
spaces_total: spaces.length,
};
};

View file

@ -0,0 +1,10 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export interface LegacySiemSignals {
non_migrated_indices_total: number;
spaces_total: number;
}

View file

@ -7,8 +7,11 @@
import type { MlJobUsageMetric } from './ml_jobs/types';
import type { RuleAdoption } from './rules/types';
// eslint-disable-next-line no-restricted-imports
import type { LegacySiemSignals } from './legacy_siem_signals/types';
export interface DetectionMetrics {
ml_jobs: MlJobUsageMetric;
detection_rules: RuleAdoption;
legacy_siem_signals: LegacySiemSignals;
}

View file

@ -33,6 +33,7 @@ export type CollectorDependencies = {
all: string;
latest: string;
};
legacySignalsIndex: string;
} & Pick<SetupPlugins, 'ml' | 'usageCollection'>;
export interface AlertBucket {

View file

@ -19470,6 +19470,22 @@
}
}
}
},
"legacy_siem_signals": {
"properties": {
"non_migrated_indices_total": {
"type": "long",
"_meta": {
"description": "Total number of non migrated legacy siem signals indices"
}
},
"spaces_total": {
"type": "long",
"_meta": {
"description": "Total number of Kibana spaces that have non migrated legacy siem signals indices"
}
}
}
}
}
},

View file

@ -13,7 +13,7 @@ import {
DETECTION_ENGINE_SIGNALS_MIGRATION_URL,
} from '@kbn/security-solution-plugin/common/constants';
import { ROLES } from '@kbn/security-solution-plugin/common/test';
import { deleteMigrations, getIndexNameFromLoad } from '../../../../../utils';
import { deleteMigrationsIfExistent, getIndexNameFromLoad } from '../../../../../utils';
import {
createAlertsIndex,
deleteAllAlerts,
@ -84,10 +84,12 @@ export default ({ getService }: FtrProviderContext): void => {
afterEach(async () => {
await esArchiver.unload('x-pack/test/functional/es_archives/signals/outdated_signals_index');
await deleteMigrations({
await deleteMigrationsIfExistent({
kbnClient,
ids: [createdMigration.migration_id],
});
// we need to delete migrated index, otherwise create migration call(in beforeEach hook) will fail
await es.indices.delete({ index: createdMigration.migration_index });
await deleteAllAlerts(supertest, log, es);
});
@ -99,6 +101,7 @@ export default ({ getService }: FtrProviderContext): void => {
.expect(200);
const deletedMigration = body.migrations[0];
expect(deletedMigration.error).to.eql(undefined);
expect(deletedMigration.id).to.eql(createdMigration.migration_id);
expect(deletedMigration.sourceIndex).to.eql(outdatedAlertsIndexName);
});

View file

@ -25,3 +25,28 @@ export const deleteMigrations = async ({
)
);
};
export const deleteMigrationsIfExistent = async ({
ids,
kbnClient,
}: {
ids: string[];
kbnClient: KbnClient;
}): Promise<void> => {
await Promise.all(
ids.map(async (id) => {
try {
const res = await kbnClient.savedObjects.delete({
id,
type: signalsMigrationType,
});
return res;
} catch (e) {
// do not throw error when migration already deleted/not found
if (e?.response?.status !== 404) {
throw e;
}
}
})
);
};