github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 01:38:56 -04:00
Code Issues Projects Releases Packages Wiki Activity

[EventLog] change to use Data stream lifecycle instead of ILM (#163210)

Browse source 
resolves https://github.com/elastic/kibana/issues/162886

The default continues to be 90 days for data detetention of event log
documents, and the rollover is now controlled by DLM, as described in
[Data stream lifecycle][].

[Data stream lifecycle]:
https://www.elastic.co/guide/en/elasticsearch/reference/8.9/data-stream-lifecycle.html

## Release note

Fixes the event log data stream to use Data stream lifecycle instead of
Index Lifecycle Management. If you had previously customized the Kibana
event log ILM policy, you should now update the lifecycle of the event
log data stream itself.
This commit is contained in:
Patrick Mueller 2023-08-08 15:23:59 -04:00 • committed by GitHub
parent 596663c3ce
commit 7e234b1a78
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 10 additions and 213 deletions
Download patch file Download diff file Expand all files Collapse all files

10
docs/user/production-considerations/alerting-production-considerations.asciidoc
View file

@ -56,14 +56,10 @@ Predicting the buffer required to account for actions depends heavily on the rul
experimental[]
Alerts and actions log activity in a set of "event log" indices. These indices are configured with an index lifecycle management (ILM) policy, which you can customize. The default policy rolls over the index when it reaches 50GB, or after 30 days. Indices over 90 days old are deleted.
Alerts and actions log activity in a set of "event log" data streams, one per Kibana version, named `.kibana-event-log-{VERSION}`. These data streams are configured with a lifecycle data retention of 90 days. This can be updated to other values via the standard data stream lifecycle APIs. Note that the event log data contains the data shown in the alerting pages in {kib}, so reducing the data retention period will result in less data being available to view.
The name of the index policy is `kibana-event-log-policy`. {kib} creates the index policy on startup, if it doesn't already exist. The index policy can be customized for your environment, but {kib} never modifies the index policy after creating it.
Because {kib} uses the documents to display historic data, you should set the delete phase longer than you would like the historic data to be shown. For example, if you would like to see one month's worth of historic data, you should set the delete phase to at least one month.
For more information on index lifecycle management, see:
{ref}/index-lifecycle-management.html[Index Lifecycle Policies].
For more information on data stream lifecycle management, see:
{ref}/data-stream-lifecycle.html[Data stream lifecycle].
[float]
[[alerting-circuit-breakers]]

2
x-pack/plugins/event_log/server/es/cluster_client_adapter.mock.ts
View file

@ -11,8 +11,6 @@ const createClusterClientMock = () => {
const mock: jest.Mocked<IClusterClientAdapter> = {
indexDocument: jest.fn(),
indexDocuments: jest.fn(),
doesIlmPolicyExist: jest.fn(),
createIlmPolicy: jest.fn(),
doesIndexTemplateExist: jest.fn(),
createIndexTemplate: jest.fn(),
doesDataStreamExist: jest.fn(),

50
x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts
View file

@ -165,56 +165,6 @@ describe('buffering documents', () => {
});
});
describe('doesIlmPolicyExist', () => {
// ElasticsearchError can be a bit random in shape, we need an any here
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const notFoundError = new Error('Not found') as any;
notFoundError.statusCode = 404;
test('should call cluster with proper arguments', async () => {
await clusterClientAdapter.doesIlmPolicyExist('foo');
expect(clusterClient.transport.request).toHaveBeenCalledWith({
method: 'GET',
path: '/_ilm/policy/foo',
});
});
test('should return false when 404 error is returned by Elasticsearch', async () => {
clusterClient.transport.request.mockRejectedValue(notFoundError);
await expect(clusterClientAdapter.doesIlmPolicyExist('foo')).resolves.toEqual(false);
});
test('should throw error when error is not 404', async () => {
clusterClient.transport.request.mockRejectedValue(new Error('Fail'));
await expect(
clusterClientAdapter.doesIlmPolicyExist('foo')
).rejects.toThrowErrorMatchingInlineSnapshot(`"error checking existance of ilm policy: Fail"`);
});
test('should return true when no error is thrown', async () => {
await expect(clusterClientAdapter.doesIlmPolicyExist('foo')).resolves.toEqual(true);
});
});
describe('createIlmPolicy', () => {
test('should call cluster client with given policy', async () => {
clusterClient.transport.request.mockResolvedValue({ success: true });
await clusterClientAdapter.createIlmPolicy('foo', { args: true });
expect(clusterClient.transport.request).toHaveBeenCalledWith({
method: 'PUT',
path: '/_ilm/policy/foo',
body: { args: true },
});
});
test('should throw error when call cluster client throws', async () => {
clusterClient.transport.request.mockRejectedValue(new Error('Fail'));
await expect(
clusterClientAdapter.createIlmPolicy('foo', { args: true })
).rejects.toThrowErrorMatchingInlineSnapshot(`"error creating ilm policy: Fail"`);
});
});
describe('doesIndexTemplateExist', () => {
test('should call cluster with proper arguments', async () => {
await clusterClientAdapter.doesIndexTemplateExist('foo');

30
x-pack/plugins/event_log/server/es/cluster_client_adapter.ts
View file

@ -178,36 +178,6 @@ export class ClusterClientAdapter<TDoc extends { body: AliasAny; index: string }
}
}
public async doesIlmPolicyExist(policyName: string): Promise<boolean> {
const request = {
method: 'GET',
path: `/_ilm/policy/${policyName}`,
};
try {
const esClient = await this.elasticsearchClientPromise;
await esClient.transport.request(request);
} catch (err) {
if (err.statusCode === 404) return false;
throw new Error(`error checking existance of ilm policy: ${err.message}`);
}
return true;
}
public async createIlmPolicy(policyName: string, policy: Record<string, unknown>): Promise<void> {
this.logger.info(`Installing ILM policy ${policyName}`);
const request = {
method: 'PUT',
path: `/_ilm/policy/${policyName}`,
body: policy,
};
try {
const esClient = await this.elasticsearchClientPromise;
await esClient.transport.request(request);
} catch (err) {
throw new Error(`error creating ilm policy: ${err.message}`);
}
}
public async doesIndexTemplateExist(name: string): Promise<boolean> {
try {
const esClient = await this.elasticsearchClientPromise;

10
x-pack/plugins/event_log/server/es/context.test.ts
View file

@ -57,13 +57,12 @@ describe('createEsContext', () => {
expect(esNames).toStrictEqual({
base: 'test-index',
dataStream: 'test-index-event-log-1.2.3',
ilmPolicy: 'test-index-event-log-policy',
indexPattern: 'test-index-event-log-*',
indexTemplate: 'test-index-event-log-1.2.3-template',
});
});
test('should return exist false for esAdapter ilm policy, index template and data stream before initialize', async () => {
test('should return exist false for esAdapter index template and data stream before initialize', async () => {
const context = createEsContext({
logger,
indexNameRoot: 'test1',
@ -84,7 +83,7 @@ describe('createEsContext', () => {
expect(doesIndexTemplateExist).toBeFalsy();
});
test('should return exist true for esAdapter ilm policy, index template and data stream after initialize', async () => {
test('should return exist true for esAdapter index template and data stream after initialize', async () => {
const context = createEsContext({
logger,
indexNameRoot: 'test2',
@ -94,11 +93,6 @@ describe('createEsContext', () => {
elasticsearchClient.indices.existsTemplate.mockResponse(true);
context.initialize();
const doesIlmPolicyExist = await context.esAdapter.doesIlmPolicyExist(
context.esNames.ilmPolicy
);
expect(doesIlmPolicyExist).toBeTruthy();
elasticsearchClient.indices.getDataStream.mockResolvedValue(GetDataStreamsResponse);
const doesDataStreamExist = await context.esAdapter.doesDataStreamExist(
context.esNames.dataStream

13
x-pack/plugins/event_log/server/es/documents.test.ts
View file

@ -5,19 +5,9 @@
* 2.0.
*/
import { getIndexTemplate, getIlmPolicy } from './documents';
import { getIndexTemplate } from './documents';
import { getEsNames } from './names';
describe('getIlmPolicy()', () => {
test('returns the basic structure of an ilm policy', () => {
expect(getIlmPolicy()).toMatchObject({
policy: {
phases: {},
},
});
});
});
describe('getIndexTemplate()', () => {
const kibanaVersion = '1.2.3';
const esNames = getEsNames('XYZ', kibanaVersion);
@ -27,7 +17,6 @@ describe('getIndexTemplate()', () => {
expect(indexTemplate.index_patterns).toEqual([esNames.dataStream]);
expect(indexTemplate.template.settings.number_of_shards).toBeGreaterThanOrEqual(0);
expect(indexTemplate.template.settings.auto_expand_replicas).toBe('0-1');
expect(indexTemplate.template.settings['index.lifecycle.name']).toBe(esNames.ilmPolicy);
expect(indexTemplate.template.mappings).toMatchObject({});
});
});

34
x-pack/plugins/event_log/server/es/documents.ts
View file

@ -25,7 +25,9 @@ export function getIndexTemplate(esNames: EsNames) {
hidden: true,
number_of_shards: 1,
auto_expand_replicas: '0-1',
'index.lifecycle.name': esNames.ilmPolicy,
},
lifecycle: {
data_retention: '90d',
},
mappings,
},
@ -33,33 +35,3 @@ export function getIndexTemplate(esNames: EsNames) {
return indexTemplateBody;
}
// returns the body of an ilm policy used in an ES PUT _ilm/policy call
export function getIlmPolicy() {
return {
policy: {
_meta: {
description:
'ilm policy the Kibana event log, created initially by Kibana, but updated by the user, not Kibana',
managed: false,
},
phases: {
hot: {
actions: {
rollover: {
max_size: '50GB',
max_age: '30d',
// max_docs: 1, // you know, for testing
},
},
},
delete: {
min_age: '90d',
actions: {
delete: {},
},
},
},
},
};
}

44
x-pack/plugins/event_log/server/es/init.test.ts
View file

@ -83,7 +83,6 @@ describe('initializeEs', () => {
`error getting existing index templates - Fail`
);
expect(esContext.esAdapter.setLegacyIndexTemplateToHidden).not.toHaveBeenCalled();
expect(esContext.esAdapter.doesIlmPolicyExist).toHaveBeenCalled();
});
test(`should continue initialization if updating existing index templates throws an error`, async () => {
@ -124,7 +123,6 @@ describe('initializeEs', () => {
expect(esContext.logger.error).toHaveBeenCalledWith(
`error setting existing \"foo-bar-template\" index template to hidden - Fail`
);
expect(esContext.esAdapter.doesIlmPolicyExist).toHaveBeenCalled();
});
test(`should update existing index settings if any exist and are not hidden`, async () => {
@ -207,7 +205,6 @@ describe('initializeEs', () => {
expect(esContext.esAdapter.getExistingIndices).toHaveBeenCalled();
expect(esContext.logger.error).toHaveBeenCalledWith(`error getting existing indices - Fail`);
expect(esContext.esAdapter.setIndexToHidden).not.toHaveBeenCalled();
expect(esContext.esAdapter.doesIlmPolicyExist).toHaveBeenCalled();
});
test(`should continue initialization if updating existing index settings throws an error`, async () => {
@ -251,7 +248,6 @@ describe('initializeEs', () => {
expect(esContext.logger.error).toHaveBeenCalledWith(
`error setting existing \"foo-bar-000001\" index to hidden - Fail`
);
expect(esContext.esAdapter.doesIlmPolicyExist).toHaveBeenCalled();
});
test(`should update existing index aliases if any exist and are not hidden`, async () => {
@ -300,7 +296,6 @@ describe('initializeEs', () => {
`error getting existing index aliases - Fail`
);
expect(esContext.esAdapter.setIndexAliasToHidden).not.toHaveBeenCalled();
expect(esContext.esAdapter.doesIlmPolicyExist).toHaveBeenCalled();
});
test(`should continue initialization if updating existing index aliases throws an error`, async () => {
@ -336,23 +331,6 @@ describe('initializeEs', () => {
expect(esContext.logger.error).toHaveBeenCalledWith(
`error setting existing \"foo-bar\" index aliases - Fail`
);
expect(esContext.esAdapter.doesIlmPolicyExist).toHaveBeenCalled();
});
test(`should create ILM policy if it doesn't exist`, async () => {
esContext.esAdapter.doesIlmPolicyExist.mockResolvedValue(false);
await initializeEs(esContext);
expect(esContext.esAdapter.doesIlmPolicyExist).toHaveBeenCalled();
expect(esContext.esAdapter.createIlmPolicy).toHaveBeenCalled();
});
test(`shouldn't create ILM policy if it exists`, async () => {
esContext.esAdapter.doesIlmPolicyExist.mockResolvedValue(true);
await initializeEs(esContext);
expect(esContext.esAdapter.doesIlmPolicyExist).toHaveBeenCalled();
expect(esContext.esAdapter.createIlmPolicy).not.toHaveBeenCalled();
});
test(`should create index template if it doesn't exist`, async () => {
@ -463,30 +441,10 @@ describe('retries', () => {
esContext.esAdapter.getExistingLegacyIndexTemplates.mockResolvedValue({});
esContext.esAdapter.getExistingIndices.mockResolvedValue({});
esContext.esAdapter.getExistingIndexAliases.mockResolvedValue({});
esContext.esAdapter.doesIlmPolicyExist.mockResolvedValue(true);
esContext.esAdapter.doesIndexTemplateExist.mockResolvedValue(true);
esContext.esAdapter.doesDataStreamExist.mockResolvedValue(true);
});
test('createIlmPolicyIfNotExists with 1 retry', async () => {
esContext.esAdapter.doesIlmPolicyExist.mockRejectedValueOnce(new Error('retry 1'));
const timeStart = performance.now();
await initializeEs(esContext);
const timeElapsed = Math.ceil(performance.now() - timeStart);
expect(timeElapsed).toBeGreaterThanOrEqual(MOCK_RETRY_DELAY);
expect(esContext.esAdapter.getExistingLegacyIndexTemplates).toHaveBeenCalledTimes(1);
expect(esContext.esAdapter.doesIlmPolicyExist).toHaveBeenCalledTimes(2);
expect(esContext.esAdapter.doesIndexTemplateExist).toHaveBeenCalledTimes(1);
expect(esContext.esAdapter.doesDataStreamExist).toHaveBeenCalledTimes(1);
const prefix = `eventLog initialization operation failed and will be retried: createIlmPolicyIfNotExists`;
expect(esContext.logger.warn).toHaveBeenCalledTimes(1);
expect(esContext.logger.warn).toHaveBeenCalledWith(`${prefix}; 4 more times; error: retry 1`);
});
test('createIndexTemplateIfNotExists with 2 retries', async () => {
esContext.esAdapter.doesIndexTemplateExist.mockRejectedValueOnce(new Error('retry 2a'));
esContext.esAdapter.doesIndexTemplateExist.mockRejectedValueOnce(new Error('retry 2b'));
@ -498,7 +456,6 @@ describe('retries', () => {
expect(timeElapsed).toBeGreaterThanOrEqual(MOCK_RETRY_DELAY * (1 + 2));
expect(esContext.esAdapter.getExistingLegacyIndexTemplates).toHaveBeenCalledTimes(1);
expect(esContext.esAdapter.doesIlmPolicyExist).toHaveBeenCalledTimes(1);
expect(esContext.esAdapter.doesIndexTemplateExist).toHaveBeenCalledTimes(3);
expect(esContext.esAdapter.doesDataStreamExist).toHaveBeenCalledTimes(1);
@ -524,7 +481,6 @@ describe('retries', () => {
expect(timeElapsed).toBeGreaterThanOrEqual(MOCK_RETRY_DELAY * (1 + 2 + 4 + 8));
expect(esContext.esAdapter.getExistingLegacyIndexTemplates).toHaveBeenCalledTimes(1);
expect(esContext.esAdapter.doesIlmPolicyExist).toHaveBeenCalledTimes(1);
expect(esContext.esAdapter.doesIndexTemplateExist).toHaveBeenCalledTimes(1);
expect(esContext.esAdapter.doesDataStreamExist).toHaveBeenCalledTimes(5);
expect(esContext.esAdapter.createDataStream).toHaveBeenCalledTimes(0);

15
x-pack/plugins/event_log/server/es/init.ts
View file

@ -9,7 +9,7 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import { asyncForEach } from '@kbn/std';
import { groupBy } from 'lodash';
import pRetry, { FailedAttemptError } from 'p-retry';
import { getIlmPolicy, getIndexTemplate } from './documents';
import { getIndexTemplate } from './documents';
import { EsContext } from './context';
const MAX_RETRY_DELAY = 30000;
@ -33,7 +33,6 @@ async function initializeEsResources(esContext: EsContext) {
// today, setExistingAssetsToHidden() never throws, but just in case ...
await retry(steps.setExistingAssetsToHidden);
await retry(steps.createIlmPolicyIfNotExists);
await retry(steps.createIndexTemplateIfNotExists);
await retry(steps.createDataStreamIfNotExists);
@ -202,18 +201,6 @@ class EsInitializationSteps {
await this.setExistingIndexAliasesToHidden();
}
async createIlmPolicyIfNotExists(): Promise<void> {
const exists = await this.esContext.esAdapter.doesIlmPolicyExist(
this.esContext.esNames.ilmPolicy
);
if (!exists) {
await this.esContext.esAdapter.createIlmPolicy(
this.esContext.esNames.ilmPolicy,
getIlmPolicy()
);
}
}
async createIndexTemplateIfNotExists(): Promise<void> {
const exists = await this.esContext.esAdapter.doesIndexTemplateExist(
this.esContext.esNames.indexTemplate

1
x-pack/plugins/event_log/server/es/names.mock.ts
View file

@ -11,7 +11,6 @@ const createNamesMock = () => {
const mock: jest.Mocked<EsNames> = {
base: '.kibana',
dataStream: '.kibana-event-log-8.0.0',
ilmPolicy: 'kibana-event-log-policy',
indexPattern: '.kibana-event-log-*',
indexTemplate: '.kibana-event-log-8.0.0-template',
};

9
x-pack/plugins/event_log/server/es/names.test.ts
View file

@ -18,16 +18,7 @@ describe('getEsNames()', () => {
const esNames = getEsNames(base, kibanaVersion);
expect(esNames.base).toEqual(base);
expect(esNames.dataStream).toEqual(`${base}-event-log-${kibanaVersion}`);
expect(esNames.ilmPolicy).toEqual(`${base}-event-log-policy`);
expect(esNames.indexPattern).toEqual(`${base}-event-log-*`);
expect(esNames.indexTemplate).toEqual(`${base}-event-log-${kibanaVersion}-template`);
});
test('ilm policy name does not contain dot prefix', () => {
const base = '.XYZ';
const kibanaVersion = '1.2.3';
const esNames = getEsNames(base, kibanaVersion);
expect(esNames.ilmPolicy).toEqual('XYZ-event-log-policy');
});
});

5
x-pack/plugins/event_log/server/es/names.ts
View file

@ -10,7 +10,6 @@ const EVENT_LOG_NAME_SUFFIX = `-event-log`;
export interface EsNames {
base: string;
dataStream: string;
ilmPolicy: string;
indexPattern: string;
indexTemplate: string;
}
@ -19,13 +18,9 @@ export function getEsNames(baseName: string, kibanaVersion: string): EsNames {
const EVENT_LOG_VERSION_SUFFIX = `-${kibanaVersion.toLocaleLowerCase()}`;
const eventLogName = `${baseName}${EVENT_LOG_NAME_SUFFIX}`;
const eventLogNameWithVersion = `${eventLogName}${EVENT_LOG_VERSION_SUFFIX}`;
const eventLogPolicyName = `${
baseName.startsWith('.') ? baseName.substring(1) : baseName
}${EVENT_LOG_NAME_SUFFIX}-policy`;
return {
base: baseName,
dataStream: eventLogNameWithVersion,
ilmPolicy: `${eventLogPolicyName}`,
indexPattern: `${eventLogName}-*`,
indexTemplate: `${eventLogNameWithVersion}-template`,
};