[8.x] [Response Ops][Event Log] Updating event log mappings if data stream and index template already exist (#193205) (#193589)

# Backport

This will backport the following commits from `main` to `8.x`:
- [[Response Ops][Event Log] Updating event log mappings if data stream
and index template already exist
(#193205)](https://github.com/elastic/kibana/pull/193205)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Ying
Mao","email":"ying.mao@elastic.co"},"sourceCommit":{"committedDate":"2024-09-20T13:55:48Z","message":"[Response
Ops][Event Log] Updating event log mappings if data stream and index
template already exist (#193205)\n\nResolves
https://github.com/elastic/kibana/issues/192682\r\n\r\n##
Summary\r\n\r\nAs of 8.8, we started writing all event log documents to
the\r\n`.kibana-event-log-ds` index. Prior to this, we created a new
index\r\ntemplate and data stream for every version
(`.kibana-event-log-8.7` for\r\nexample) so any mapping updates that
were added for the version were\r\ncreated in the new index on
upgrade.\r\n\r\nWith the static index name and serverless, we need a way
to update\r\nmappings in existing indices. This PR uses the same
mechanism that we\r\nuse for the alerts index to update the index
template mappings and the\r\nmappings for the concrete backing indices
of a datastream.\r\n\r\n## To Verify\r\n\r\nRun ES and Kibana in `main`
to test the upgrade path for serverless \r\na. Check out `main`, run ES:
`yarn es snapshot --license trial --ssl
-E\r\npath.data=../test_el_upgrade` and Kibana `yarn start --ssl`\r\n b.
Create a rule and let it run to populate the event log index\r\nc.
Switch to this PR branch. Make a mapping update to the event
log\r\nindex:\r\n\r\n```\r\n---
a/x-pack/plugins/event_log/generated/mappings.json\r\n+++
b/x-pack/plugins/event_log/generated/mappings.json\r\n@@ -172,6 +172,9
@@\r\n },\r\n \"rule\": {\r\n \"properties\": {\r\n+ \"test\": {\r\n+
\"type\": \"keyword\"\r\n+ },\r\n \"author\": {\r\n \"ignore_above\":
1024,\r\n \"type\": \"keyword\",\r\n```\r\n d. Start ES and Kibana with
the same commands as above\r\ne. Verify that the `.kibana-event-log-ds`
index is created and has the\r\nupdated
mapping:\r\n-\r\nhttps://localhost:5601/app/management/data/index_management/templates/.kibana-event-log-template\r\n-\r\nhttps://localhost:5601/app/management/data/index_management/indices/index_details?indexName=.ds-.kibana-event-log-ds-2024.09.17-000001&filter=.kibana-&includeHiddenIndices=true&tab=mappings\r\n\r\nI
also verified the following:\r\n1. Run ES and Kibana in 8.7 to test the
upgrade path from 8.7 (when\r\nevent log indices were versioned) to
now\r\n2. Run ES and Kibana in 8.15 to test the upgrade path from the
previous\r\nrelease to now\r\n\r\nHowever, I had to create an 8.x branch
and cherry pick this commit\r\nbecause `main` is now on 9.0 and we can't
upgrade directly from older\r\n8.x version to
9.0!\r\n\r\n---------\r\n\r\nCo-authored-by: Elastic Machine
<elasticmachine@users.noreply.github.com>\r\nCo-authored-by:
kibanamachine
<42973632+kibanamachine@users.noreply.github.com>","sha":"e2798def07d50595806748dd64cccaa216c5e234","branchLabelMapping":{"^v9.0.0$":"main","^v8.16.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","Team:ResponseOps","v9.0.0","Feature:EventLog","backport:prev-minor","v8.16.0"],"title":"[Response
Ops][Event Log] Updating event log mappings if data stream and index
template already
exist","number":193205,"url":"https://github.com/elastic/kibana/pull/193205","mergeCommit":{"message":"[Response
Ops][Event Log] Updating event log mappings if data stream and index
template already exist (#193205)\n\nResolves
https://github.com/elastic/kibana/issues/192682\r\n\r\n##
Summary\r\n\r\nAs of 8.8, we started writing all event log documents to
the\r\n`.kibana-event-log-ds` index. Prior to this, we created a new
index\r\ntemplate and data stream for every version
(`.kibana-event-log-8.7` for\r\nexample) so any mapping updates that
were added for the version were\r\ncreated in the new index on
upgrade.\r\n\r\nWith the static index name and serverless, we need a way
to update\r\nmappings in existing indices. This PR uses the same
mechanism that we\r\nuse for the alerts index to update the index
template mappings and the\r\nmappings for the concrete backing indices
of a datastream.\r\n\r\n## To Verify\r\n\r\nRun ES and Kibana in `main`
to test the upgrade path for serverless \r\na. Check out `main`, run ES:
`yarn es snapshot --license trial --ssl
-E\r\npath.data=../test_el_upgrade` and Kibana `yarn start --ssl`\r\n b.
Create a rule and let it run to populate the event log index\r\nc.
Switch to this PR branch. Make a mapping update to the event
log\r\nindex:\r\n\r\n```\r\n---
a/x-pack/plugins/event_log/generated/mappings.json\r\n+++
b/x-pack/plugins/event_log/generated/mappings.json\r\n@@ -172,6 +172,9
@@\r\n },\r\n \"rule\": {\r\n \"properties\": {\r\n+ \"test\": {\r\n+
\"type\": \"keyword\"\r\n+ },\r\n \"author\": {\r\n \"ignore_above\":
1024,\r\n \"type\": \"keyword\",\r\n```\r\n d. Start ES and Kibana with
the same commands as above\r\ne. Verify that the `.kibana-event-log-ds`
index is created and has the\r\nupdated
mapping:\r\n-\r\nhttps://localhost:5601/app/management/data/index_management/templates/.kibana-event-log-template\r\n-\r\nhttps://localhost:5601/app/management/data/index_management/indices/index_details?indexName=.ds-.kibana-event-log-ds-2024.09.17-000001&filter=.kibana-&includeHiddenIndices=true&tab=mappings\r\n\r\nI
also verified the following:\r\n1. Run ES and Kibana in 8.7 to test the
upgrade path from 8.7 (when\r\nevent log indices were versioned) to
now\r\n2. Run ES and Kibana in 8.15 to test the upgrade path from the
previous\r\nrelease to now\r\n\r\nHowever, I had to create an 8.x branch
and cherry pick this commit\r\nbecause `main` is now on 9.0 and we can't
upgrade directly from older\r\n8.x version to
9.0!\r\n\r\n---------\r\n\r\nCo-authored-by: Elastic Machine
<elasticmachine@users.noreply.github.com>\r\nCo-authored-by:
kibanamachine
<42973632+kibanamachine@users.noreply.github.com>","sha":"e2798def07d50595806748dd64cccaa216c5e234"}},"sourceBranch":"main","suggestedTargetBranches":["8.x"],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/193205","number":193205,"mergeCommit":{"message":"[Response
Ops][Event Log] Updating event log mappings if data stream and index
template already exist (#193205)\n\nResolves
https://github.com/elastic/kibana/issues/192682\r\n\r\n##
Summary\r\n\r\nAs of 8.8, we started writing all event log documents to
the\r\n`.kibana-event-log-ds` index. Prior to this, we created a new
index\r\ntemplate and data stream for every version
(`.kibana-event-log-8.7` for\r\nexample) so any mapping updates that
were added for the version were\r\ncreated in the new index on
upgrade.\r\n\r\nWith the static index name and serverless, we need a way
to update\r\nmappings in existing indices. This PR uses the same
mechanism that we\r\nuse for the alerts index to update the index
template mappings and the\r\nmappings for the concrete backing indices
of a datastream.\r\n\r\n## To Verify\r\n\r\nRun ES and Kibana in `main`
to test the upgrade path for serverless \r\na. Check out `main`, run ES:
`yarn es snapshot --license trial --ssl
-E\r\npath.data=../test_el_upgrade` and Kibana `yarn start --ssl`\r\n b.
Create a rule and let it run to populate the event log index\r\nc.
Switch to this PR branch. Make a mapping update to the event
log\r\nindex:\r\n\r\n```\r\n---
a/x-pack/plugins/event_log/generated/mappings.json\r\n+++
b/x-pack/plugins/event_log/generated/mappings.json\r\n@@ -172,6 +172,9
@@\r\n },\r\n \"rule\": {\r\n \"properties\": {\r\n+ \"test\": {\r\n+
\"type\": \"keyword\"\r\n+ },\r\n \"author\": {\r\n \"ignore_above\":
1024,\r\n \"type\": \"keyword\",\r\n```\r\n d. Start ES and Kibana with
the same commands as above\r\ne. Verify that the `.kibana-event-log-ds`
index is created and has the\r\nupdated
mapping:\r\n-\r\nhttps://localhost:5601/app/management/data/index_management/templates/.kibana-event-log-template\r\n-\r\nhttps://localhost:5601/app/management/data/index_management/indices/index_details?indexName=.ds-.kibana-event-log-ds-2024.09.17-000001&filter=.kibana-&includeHiddenIndices=true&tab=mappings\r\n\r\nI
also verified the following:\r\n1. Run ES and Kibana in 8.7 to test the
upgrade path from 8.7 (when\r\nevent log indices were versioned) to
now\r\n2. Run ES and Kibana in 8.15 to test the upgrade path from the
previous\r\nrelease to now\r\n\r\nHowever, I had to create an 8.x branch
and cherry pick this commit\r\nbecause `main` is now on 9.0 and we can't
upgrade directly from older\r\n8.x version to
9.0!\r\n\r\n---------\r\n\r\nCo-authored-by: Elastic Machine
<elasticmachine@users.noreply.github.com>\r\nCo-authored-by:
kibanamachine
<42973632+kibanamachine@users.noreply.github.com>","sha":"e2798def07d50595806748dd64cccaa216c5e234"}},{"branch":"8.x","label":"v8.16.0","branchLabelMappingKey":"^v8.16.0$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->

Co-authored-by: Ying Mao <ying.mao@elastic.co>
This commit is contained in:
Kibana Machine 2024-09-21 01:30:32 +10:00 committed by GitHub
parent f94a88f0d1
commit 2933ad6278
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 523 additions and 15 deletions

View file

@ -0,0 +1,12 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
module.exports = {
preset: '@kbn/test/jest_integration',
rootDir: '../../..',
roots: ['<rootDir>/x-pack/plugins/event_log'],
};

View file

@ -13,8 +13,10 @@ const createClusterClientMock = () => {
indexDocuments: jest.fn(),
doesIndexTemplateExist: jest.fn(),
createIndexTemplate: jest.fn(),
updateIndexTemplate: jest.fn(),
doesDataStreamExist: jest.fn(),
createDataStream: jest.fn(),
updateConcreteIndices: jest.fn(),
getExistingLegacyIndexTemplates: jest.fn(),
setLegacyIndexTemplateToHidden: jest.fn(),
getExistingIndices: jest.fn(),

View file

@ -215,6 +215,117 @@ describe('createIndexTemplate', () => {
});
});
describe('updateIndexTemplate', () => {
test('should call cluster with given template', async () => {
clusterClient.indices.simulateTemplate.mockImplementationOnce(async () => ({
template: {
aliases: {
alias_name_1: {
is_hidden: true,
},
alias_name_2: {
is_hidden: true,
},
},
settings: {
hidden: true,
number_of_shards: 1,
auto_expand_replicas: '0-1',
},
mappings: { dynamic: false, properties: { '@timestamp': { type: 'date' } } },
},
}));
await clusterClientAdapter.updateIndexTemplate('foo', { args: true });
expect(clusterClient.indices.simulateTemplate).toHaveBeenCalledWith({
name: 'foo',
body: { args: true },
});
expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledWith({
name: 'foo',
body: { args: true },
});
});
test(`should throw error if simulate mappings response is empty`, async () => {
clusterClient.indices.simulateTemplate.mockImplementationOnce(async () => ({
template: {
aliases: {
alias_name_1: {
is_hidden: true,
},
alias_name_2: {
is_hidden: true,
},
},
settings: {
hidden: true,
number_of_shards: 1,
auto_expand_replicas: '0-1',
},
mappings: {},
},
}));
await expect(() =>
clusterClientAdapter.updateIndexTemplate('foo', { name: 'template', args: true })
).rejects.toThrowErrorMatchingInlineSnapshot(
`"No mappings would be generated for template, possibly due to failed/misconfigured bootstrapping"`
);
expect(logger.error).toHaveBeenCalledWith(
`Error updating index template foo: No mappings would be generated for template, possibly due to failed/misconfigured bootstrapping`
);
});
test(`should throw error if simulateTemplate throws error`, async () => {
clusterClient.indices.simulateTemplate.mockImplementationOnce(() => {
throw new Error('failed to simulate');
});
await expect(() =>
clusterClientAdapter.updateIndexTemplate('foo', { name: 'template', args: true })
).rejects.toThrowErrorMatchingInlineSnapshot(`"failed to simulate"`);
expect(logger.error).toHaveBeenCalledWith(
`Error updating index template foo: failed to simulate`
);
});
test(`should throw error if putIndexTemplate throws error`, async () => {
clusterClient.indices.simulateTemplate.mockImplementationOnce(async () => ({
template: {
aliases: {
alias_name_1: {
is_hidden: true,
},
alias_name_2: {
is_hidden: true,
},
},
settings: {
hidden: true,
number_of_shards: 1,
auto_expand_replicas: '0-1',
},
mappings: { dynamic: false, properties: { '@timestamp': { type: 'date' } } },
},
}));
clusterClient.indices.putIndexTemplate.mockImplementationOnce(() => {
throw new Error('failed to update index template');
});
await expect(() =>
clusterClientAdapter.updateIndexTemplate('foo', { name: 'template', args: true })
).rejects.toThrowErrorMatchingInlineSnapshot(`"failed to update index template"`);
expect(logger.error).toHaveBeenCalledWith(
`Error updating index template foo: failed to update index template`
);
});
});
describe('getExistingLegacyIndexTemplates', () => {
test('should call cluster with given index template pattern', async () => {
await clusterClientAdapter.getExistingLegacyIndexTemplates('foo*');
@ -497,7 +608,7 @@ describe('doesDataStreamExist', () => {
});
});
describe('createIndex', () => {
describe('createDataStream', () => {
test('should call cluster with proper arguments', async () => {
await clusterClientAdapter.createDataStream('foo');
expect(clusterClient.indices.createDataStream).toHaveBeenCalledWith({
@ -526,6 +637,95 @@ describe('createIndex', () => {
});
});
describe('updateConcreteIndices', () => {
test('should call cluster with proper arguments', async () => {
clusterClient.indices.simulateIndexTemplate.mockImplementationOnce(async () => ({
template: {
aliases: { alias_name_1: { is_hidden: true } },
settings: {
hidden: true,
number_of_shards: 1,
auto_expand_replicas: '0-1',
},
mappings: { dynamic: false, properties: { '@timestamp': { type: 'date' } } },
},
}));
await clusterClientAdapter.updateConcreteIndices('foo');
expect(clusterClient.indices.simulateIndexTemplate).toHaveBeenCalledWith({
name: 'foo',
});
expect(clusterClient.indices.putMapping).toHaveBeenCalledWith({
index: 'foo',
body: { dynamic: false, properties: { '@timestamp': { type: 'date' } } },
});
});
test('should not update mapping if simulate response does not contain mappings', async () => {
// @ts-ignore
clusterClient.indices.simulateIndexTemplate.mockImplementationOnce(async () => ({
template: {
aliases: { alias_name_1: { is_hidden: true } },
settings: {
hidden: true,
number_of_shards: 1,
auto_expand_replicas: '0-1',
},
},
}));
await clusterClientAdapter.updateConcreteIndices('foo');
expect(clusterClient.indices.simulateIndexTemplate).toHaveBeenCalledWith({
name: 'foo',
});
expect(clusterClient.indices.putMapping).not.toHaveBeenCalled();
});
test('should throw error if simulateIndexTemplate throws error', async () => {
clusterClient.indices.simulateIndexTemplate.mockImplementationOnce(() => {
throw new Error('failed to simulate');
});
await expect(() =>
clusterClientAdapter.updateConcreteIndices('foo')
).rejects.toThrowErrorMatchingInlineSnapshot(`"failed to simulate"`);
expect(clusterClient.indices.putMapping).not.toHaveBeenCalled();
expect(logger.error).toHaveBeenCalledWith(
`Error updating index mappings for foo: failed to simulate`
);
});
test('should throw error if putMapping throws error', async () => {
clusterClient.indices.simulateIndexTemplate.mockImplementationOnce(async () => ({
template: {
aliases: { alias_name_1: { is_hidden: true } },
settings: {
hidden: true,
number_of_shards: 1,
auto_expand_replicas: '0-1',
},
mappings: { dynamic: false, properties: { '@timestamp': { type: 'date' } } },
},
}));
clusterClient.indices.putMapping.mockImplementationOnce(() => {
throw new Error('failed to put mappings');
});
await expect(() =>
clusterClientAdapter.updateConcreteIndices('foo')
).rejects.toThrowErrorMatchingInlineSnapshot(`"failed to put mappings"`);
expect(clusterClient.indices.putMapping).toHaveBeenCalledWith({
index: 'foo',
body: { dynamic: false, properties: { '@timestamp': { type: 'date' } } },
});
expect(logger.error).toHaveBeenCalledWith(
`Error updating index mappings for foo: failed to put mappings`
);
});
});
describe('queryEventsBySavedObject', () => {
const DEFAULT_OPTIONS = queryOptionsSchema.validate({});

View file

@ -7,7 +7,7 @@
import { Subject } from 'rxjs';
import { bufferTime, filter as rxFilter, concatMap } from 'rxjs';
import { reject, isUndefined, isNumber, pick } from 'lodash';
import { reject, isUndefined, isNumber, pick, isEmpty, get } from 'lodash';
import type { PublicMethodsOf } from '@kbn/utility-types';
import { Logger, ElasticsearchClient } from '@kbn/core/server';
import util from 'util';
@ -213,6 +213,28 @@ export class ClusterClientAdapter<TDoc extends { body: AliasAny; index: string }
}
}
public async updateIndexTemplate(name: string, template: Record<string, unknown>): Promise<void> {
this.logger.info(`Updating index template ${name}`);
try {
const esClient = await this.elasticsearchClientPromise;
// Simulate the index template to proactively identify any issues with the mappings
const simulateResponse = await esClient.indices.simulateTemplate({ name, body: template });
const mappings: estypes.MappingTypeMapping = simulateResponse.template.mappings;
if (isEmpty(mappings)) {
throw new Error(
`No mappings would be generated for ${template.name}, possibly due to failed/misconfigured bootstrapping`
);
}
await esClient.indices.putIndexTemplate({ name, body: template });
} catch (err) {
this.logger.error(`Error updating index template ${name}: ${err.message}`);
throw err;
}
}
public async getExistingLegacyIndexTemplates(
indexTemplatePattern: string
): Promise<estypes.IndicesGetTemplateResponse> {
@ -335,7 +357,7 @@ export class ClusterClientAdapter<TDoc extends { body: AliasAny; index: string }
}
}
public async createDataStream(name: string, body: Record<string, unknown> = {}): Promise<void> {
public async createDataStream(name: string): Promise<void> {
this.logger.info(`Creating datastream ${name}`);
try {
const esClient = await this.elasticsearchClientPromise;
@ -347,6 +369,23 @@ export class ClusterClientAdapter<TDoc extends { body: AliasAny; index: string }
}
}
public async updateConcreteIndices(name: string): Promise<void> {
this.logger.info(`Updating concrete index mappings for ${name}`);
try {
const esClient = await this.elasticsearchClientPromise;
const simulatedIndexMapping = await esClient.indices.simulateIndexTemplate({ name });
const simulatedMapping = get(simulatedIndexMapping, ['template', 'mappings']);
if (simulatedMapping != null) {
await esClient.indices.putMapping({ index: name, body: simulatedMapping });
this.logger.debug(`Successfully updated concrete index mappings for ${name}`);
}
} catch (err) {
this.logger.error(`Error updating index mappings for ${name}: ${err.message}`);
throw err;
}
}
public async queryEventsBySavedObjects(
queryOptions: FindEventsOptionsBySavedObjectFilter
): Promise<QueryEventsBySavedObjectResult> {

View file

@ -18,7 +18,7 @@ describe('initializeEs', () => {
esContext.esAdapter.getExistingIndexAliases.mockResolvedValue({});
});
test(`should update existing index templates if any exist and are not hidden`, async () => {
test(`should update existing index templates to hidden if any exist and are not hidden`, async () => {
const testTemplate = {
order: 0,
index_patterns: ['foo-bar-*'],
@ -393,14 +393,16 @@ describe('initializeEs', () => {
await initializeEs(esContext);
expect(esContext.esAdapter.doesIndexTemplateExist).toHaveBeenCalled();
expect(esContext.esAdapter.createIndexTemplate).toHaveBeenCalled();
expect(esContext.esAdapter.updateIndexTemplate).not.toHaveBeenCalled();
});
test(`shouldn't create index template if it already exists`, async () => {
test(`should update index template if it already exists`, async () => {
esContext.esAdapter.doesIndexTemplateExist.mockResolvedValue(true);
await initializeEs(esContext);
expect(esContext.esAdapter.doesIndexTemplateExist).toHaveBeenCalled();
expect(esContext.esAdapter.createIndexTemplate).not.toHaveBeenCalled();
expect(esContext.esAdapter.updateIndexTemplate).toHaveBeenCalled();
});
test(`should create data stream if it doesn't exist`, async () => {
@ -409,14 +411,16 @@ describe('initializeEs', () => {
await initializeEs(esContext);
expect(esContext.esAdapter.doesDataStreamExist).toHaveBeenCalled();
expect(esContext.esAdapter.createDataStream).toHaveBeenCalled();
expect(esContext.esAdapter.updateConcreteIndices).not.toHaveBeenCalled();
});
test(`shouldn't create data stream if it already exists`, async () => {
test(`should update indices of data stream if it already exists`, async () => {
esContext.esAdapter.doesDataStreamExist.mockResolvedValue(true);
await initializeEs(esContext);
expect(esContext.esAdapter.doesDataStreamExist).toHaveBeenCalled();
expect(esContext.esAdapter.createDataStream).not.toHaveBeenCalled();
expect(esContext.esAdapter.updateConcreteIndices).toHaveBeenCalled();
});
});

View file

@ -216,12 +216,17 @@ class EsInitializationSteps {
const exists = await this.esContext.esAdapter.doesIndexTemplateExist(
this.esContext.esNames.indexTemplate
);
const templateBody = getIndexTemplate(this.esContext.esNames);
if (!exists) {
const templateBody = getIndexTemplate(this.esContext.esNames);
await this.esContext.esAdapter.createIndexTemplate(
this.esContext.esNames.indexTemplate,
templateBody
);
} else {
await this.esContext.esAdapter.updateIndexTemplate(
this.esContext.esNames.indexTemplate,
templateBody
);
}
}
@ -230,14 +235,10 @@ class EsInitializationSteps {
this.esContext.esNames.dataStream
);
if (!exists) {
await this.esContext.esAdapter.createDataStream(this.esContext.esNames.dataStream, {
aliases: {
[this.esContext.esNames.dataStream]: {
is_write_index: true,
is_hidden: true,
},
},
});
await this.esContext.esAdapter.createDataStream(this.esContext.esNames.dataStream);
} else {
// apply current mappings to existing data stream
await this.esContext.esAdapter.updateConcreteIndices(this.esContext.esNames.dataStream);
}
}
}

View file

@ -0,0 +1,173 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { type ElasticsearchClient } from '@kbn/core/server';
import { setupKibanaServer, setupTestServers } from './lib/setup_test_servers';
import { IEvent } from '../types';
import { EsContextCtorParams } from '../es/context';
const { createEsContext: createEsContextMock } = jest.requireMock('../es');
jest.mock('../es', () => {
const actual = jest.requireActual('../es');
return {
...actual,
createEsContext: jest.fn().mockImplementation((opts) => {
return new actual.createEsContext(opts);
}),
};
});
describe('update existing event log mappings on startup', () => {
it('should update mappings for existing event log indices', async () => {
const setupResult = await setupTestServers();
const esServer = setupResult.esServer;
let kibanaServer = setupResult.kibanaServer;
expect(createEsContextMock).toHaveBeenCalledTimes(1);
let createEsContextOpts: EsContextCtorParams = createEsContextMock.mock.calls[0][0];
let infoLogSpy = jest.spyOn(createEsContextOpts.logger, 'info');
await retry(async () => {
expect(infoLogSpy).toHaveBeenCalledWith(`Creating datastream .kibana-event-log-ds`);
expect(infoLogSpy).not.toHaveBeenCalledWith(
`Updating concrete index mappings for .kibana-event-log-ds`
);
});
await injectEventLogDoc(kibanaServer.coreStart.elasticsearch.client.asInternalUser, {
'@timestamp': '2024-09-19T20:38:47.124Z',
event: {
provider: 'alerting',
action: 'execute',
kind: 'alert',
category: ['AlertingExample'],
start: '2024-09-19T20:38:46.963Z',
outcome: 'success',
end: '2024-09-19T20:38:47.124Z',
duration: '161000000',
},
kibana: {
alert: {
rule: {
rule_type_id: 'example.always-firing',
consumer: 'alerts',
execution: {
uuid: '578f0ca3-aa08-4700-aed0-236c888c6cae',
metrics: {
number_of_triggered_actions: 0,
number_of_generated_actions: 0,
alert_counts: {
active: 5,
new: 5,
recovered: 5,
},
number_of_delayed_alerts: 0,
number_of_searches: 0,
es_search_duration_ms: 0,
total_search_duration_ms: 0,
claim_to_start_duration_ms: 26,
total_run_duration_ms: 187,
prepare_rule_duration_ms: 18,
rule_type_run_duration_ms: 0,
process_alerts_duration_ms: 1,
persist_alerts_duration_ms: 64,
trigger_actions_duration_ms: 0,
process_rule_duration_ms: 69,
},
},
},
},
saved_objects: [
{
rel: 'primary',
type: 'alert',
id: '3389d834-edc2-4245-a319-3ff689f5bf3b',
type_id: 'example.always-firing',
},
],
space_ids: ['default'],
task: {
scheduled: '2024-09-19T20:38:46.797Z',
schedule_delay: 166000000,
},
alerting: {
outcome: 'success',
status: 'active',
},
server_uuid: '5b2de169-2785-441b-ae8c-186a1936b17d',
version: '9.0.0',
},
rule: {
id: '3389d834-edc2-4245-a319-3ff689f5bf3b',
license: 'basic',
category: 'example.always-firing',
ruleset: 'AlertingExample',
name: 'e',
},
message: "rule executed: example.always-firing:3389d834-edc2-4245-a319-3ff689f5bf3b: 'e'",
ecs: {
version: '1.8.0',
},
});
if (kibanaServer) {
await kibanaServer.stop();
}
infoLogSpy.mockRestore();
const restartKb = await setupKibanaServer();
kibanaServer = restartKb.kibanaServer;
expect(createEsContextMock).toHaveBeenCalledTimes(2);
createEsContextOpts = createEsContextMock.mock.calls[1][0];
infoLogSpy = jest.spyOn(createEsContextOpts.logger, 'info');
const debugLogSpy = jest.spyOn(createEsContextOpts.logger, 'debug');
await retry(async () => {
expect(infoLogSpy).toHaveBeenCalledWith(
`Updating concrete index mappings for .kibana-event-log-ds`
);
expect(debugLogSpy).toHaveBeenCalledWith(
`Successfully updated concrete index mappings for .kibana-event-log-ds`
);
});
if (kibanaServer) {
await kibanaServer.stop();
}
if (esServer) {
await esServer.stop();
}
});
});
async function injectEventLogDoc(esClient: ElasticsearchClient, doc: IEvent) {
await esClient.index({
index: '.kibana-event-log-ds',
document: doc,
});
}
interface RetryOpts {
times: number;
intervalMs: number;
}
async function retry<T>(cb: () => Promise<T>, options: RetryOpts = { times: 60, intervalMs: 500 }) {
let attempt = 1;
while (true) {
try {
return await cb();
} catch (e) {
if (attempt >= options.times) {
throw e;
}
}
attempt++;
await new Promise((resolve) => setTimeout(resolve, options.intervalMs));
}
}

View file

@ -0,0 +1,76 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import deepmerge from 'deepmerge';
import { createTestServers, createRootWithCorePlugins } from '@kbn/core-test-helpers-kbn-server';
function createRoot(settings = {}) {
return createRootWithCorePlugins(
deepmerge(
{
logging: {
root: {
level: 'warn',
},
loggers: [
{
name: 'plugins.eventLog',
level: 'all',
},
],
},
},
settings
),
{ oss: false }
);
}
export async function setupTestServers(settings = {}) {
const { startES } = createTestServers({
adjustTimeout: (t) => jest.setTimeout(t),
settings: {
es: {
license: 'trial',
},
},
});
const esServer = await startES();
const root = createRoot(settings);
await root.preboot();
const coreSetup = await root.setup();
const coreStart = await root.start();
return {
esServer,
kibanaServer: {
root,
coreSetup,
coreStart,
stop: async () => await root.shutdown(),
},
};
}
export async function setupKibanaServer(settings = {}) {
const root = createRoot(settings);
await root.preboot();
const coreSetup = await root.setup();
const coreStart = await root.start();
return {
kibanaServer: {
root,
coreSetup,
coreStart,
stop: async () => await root.shutdown(),
},
};
}

View file

@ -21,6 +21,7 @@
"@kbn/std",
"@kbn/safer-lodash-set",
"@kbn/serverless",
"@kbn/core-test-helpers-kbn-server",
],
"exclude": [
"target/**/*",