mirror of
https://github.com/elastic/kibana.git
synced 2025-04-24 01:38:56 -04:00
[8.x] [Response Ops][Event Log] Updating event log mappings if data stream and index template already exist (#193205) (#193589)
# Backport This will backport the following commits from `main` to `8.x`: - [[Response Ops][Event Log] Updating event log mappings if data stream and index template already exist (#193205)](https://github.com/elastic/kibana/pull/193205) <!--- Backport version: 9.4.3 --> ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) <!--BACKPORT [{"author":{"name":"Ying Mao","email":"ying.mao@elastic.co"},"sourceCommit":{"committedDate":"2024-09-20T13:55:48Z","message":"[Response Ops][Event Log] Updating event log mappings if data stream and index template already exist (#193205)\n\nResolves https://github.com/elastic/kibana/issues/192682\r\n\r\n## Summary\r\n\r\nAs of 8.8, we started writing all event log documents to the\r\n`.kibana-event-log-ds` index. Prior to this, we created a new index\r\ntemplate and data stream for every version (`.kibana-event-log-8.7` for\r\nexample) so any mapping updates that were added for the version were\r\ncreated in the new index on upgrade.\r\n\r\nWith the static index name and serverless, we need a way to update\r\nmappings in existing indices. This PR uses the same mechanism that we\r\nuse for the alerts index to update the index template mappings and the\r\nmappings for the concrete backing indices of a datastream.\r\n\r\n## To Verify\r\n\r\nRun ES and Kibana in `main` to test the upgrade path for serverless \r\na. Check out `main`, run ES: `yarn es snapshot --license trial --ssl -E\r\npath.data=../test_el_upgrade` and Kibana `yarn start --ssl`\r\n b. Create a rule and let it run to populate the event log index\r\nc. Switch to this PR branch. Make a mapping update to the event log\r\nindex:\r\n\r\n```\r\n--- a/x-pack/plugins/event_log/generated/mappings.json\r\n+++ b/x-pack/plugins/event_log/generated/mappings.json\r\n@@ -172,6 +172,9 @@\r\n },\r\n \"rule\": {\r\n \"properties\": {\r\n+ \"test\": {\r\n+ \"type\": \"keyword\"\r\n+ },\r\n \"author\": {\r\n \"ignore_above\": 1024,\r\n \"type\": \"keyword\",\r\n```\r\n d. Start ES and Kibana with the same commands as above\r\ne. Verify that the `.kibana-event-log-ds` index is created and has the\r\nupdated mapping:\r\n-\r\nhttps://localhost:5601/app/management/data/index_management/templates/.kibana-event-log-template\r\n-\r\nhttps://localhost:5601/app/management/data/index_management/indices/index_details?indexName=.ds-.kibana-event-log-ds-2024.09.17-000001&filter=.kibana-&includeHiddenIndices=true&tab=mappings\r\n\r\nI also verified the following:\r\n1. Run ES and Kibana in 8.7 to test the upgrade path from 8.7 (when\r\nevent log indices were versioned) to now\r\n2. Run ES and Kibana in 8.15 to test the upgrade path from the previous\r\nrelease to now\r\n\r\nHowever, I had to create an 8.x branch and cherry pick this commit\r\nbecause `main` is now on 9.0 and we can't upgrade directly from older\r\n8.x version to 9.0!\r\n\r\n---------\r\n\r\nCo-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>\r\nCo-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>","sha":"e2798def07d50595806748dd64cccaa216c5e234","branchLabelMapping":{"^v9.0.0$":"main","^v8.16.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","Team:ResponseOps","v9.0.0","Feature:EventLog","backport:prev-minor","v8.16.0"],"title":"[Response Ops][Event Log] Updating event log mappings if data stream and index template already exist","number":193205,"url":"https://github.com/elastic/kibana/pull/193205","mergeCommit":{"message":"[Response Ops][Event Log] Updating event log mappings if data stream and index template already exist (#193205)\n\nResolves https://github.com/elastic/kibana/issues/192682\r\n\r\n## Summary\r\n\r\nAs of 8.8, we started writing all event log documents to the\r\n`.kibana-event-log-ds` index. Prior to this, we created a new index\r\ntemplate and data stream for every version (`.kibana-event-log-8.7` for\r\nexample) so any mapping updates that were added for the version were\r\ncreated in the new index on upgrade.\r\n\r\nWith the static index name and serverless, we need a way to update\r\nmappings in existing indices. This PR uses the same mechanism that we\r\nuse for the alerts index to update the index template mappings and the\r\nmappings for the concrete backing indices of a datastream.\r\n\r\n## To Verify\r\n\r\nRun ES and Kibana in `main` to test the upgrade path for serverless \r\na. Check out `main`, run ES: `yarn es snapshot --license trial --ssl -E\r\npath.data=../test_el_upgrade` and Kibana `yarn start --ssl`\r\n b. Create a rule and let it run to populate the event log index\r\nc. Switch to this PR branch. Make a mapping update to the event log\r\nindex:\r\n\r\n```\r\n--- a/x-pack/plugins/event_log/generated/mappings.json\r\n+++ b/x-pack/plugins/event_log/generated/mappings.json\r\n@@ -172,6 +172,9 @@\r\n },\r\n \"rule\": {\r\n \"properties\": {\r\n+ \"test\": {\r\n+ \"type\": \"keyword\"\r\n+ },\r\n \"author\": {\r\n \"ignore_above\": 1024,\r\n \"type\": \"keyword\",\r\n```\r\n d. Start ES and Kibana with the same commands as above\r\ne. Verify that the `.kibana-event-log-ds` index is created and has the\r\nupdated mapping:\r\n-\r\nhttps://localhost:5601/app/management/data/index_management/templates/.kibana-event-log-template\r\n-\r\nhttps://localhost:5601/app/management/data/index_management/indices/index_details?indexName=.ds-.kibana-event-log-ds-2024.09.17-000001&filter=.kibana-&includeHiddenIndices=true&tab=mappings\r\n\r\nI also verified the following:\r\n1. Run ES and Kibana in 8.7 to test the upgrade path from 8.7 (when\r\nevent log indices were versioned) to now\r\n2. Run ES and Kibana in 8.15 to test the upgrade path from the previous\r\nrelease to now\r\n\r\nHowever, I had to create an 8.x branch and cherry pick this commit\r\nbecause `main` is now on 9.0 and we can't upgrade directly from older\r\n8.x version to 9.0!\r\n\r\n---------\r\n\r\nCo-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>\r\nCo-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>","sha":"e2798def07d50595806748dd64cccaa216c5e234"}},"sourceBranch":"main","suggestedTargetBranches":["8.x"],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/193205","number":193205,"mergeCommit":{"message":"[Response Ops][Event Log] Updating event log mappings if data stream and index template already exist (#193205)\n\nResolves https://github.com/elastic/kibana/issues/192682\r\n\r\n## Summary\r\n\r\nAs of 8.8, we started writing all event log documents to the\r\n`.kibana-event-log-ds` index. Prior to this, we created a new index\r\ntemplate and data stream for every version (`.kibana-event-log-8.7` for\r\nexample) so any mapping updates that were added for the version were\r\ncreated in the new index on upgrade.\r\n\r\nWith the static index name and serverless, we need a way to update\r\nmappings in existing indices. This PR uses the same mechanism that we\r\nuse for the alerts index to update the index template mappings and the\r\nmappings for the concrete backing indices of a datastream.\r\n\r\n## To Verify\r\n\r\nRun ES and Kibana in `main` to test the upgrade path for serverless \r\na. Check out `main`, run ES: `yarn es snapshot --license trial --ssl -E\r\npath.data=../test_el_upgrade` and Kibana `yarn start --ssl`\r\n b. Create a rule and let it run to populate the event log index\r\nc. Switch to this PR branch. Make a mapping update to the event log\r\nindex:\r\n\r\n```\r\n--- a/x-pack/plugins/event_log/generated/mappings.json\r\n+++ b/x-pack/plugins/event_log/generated/mappings.json\r\n@@ -172,6 +172,9 @@\r\n },\r\n \"rule\": {\r\n \"properties\": {\r\n+ \"test\": {\r\n+ \"type\": \"keyword\"\r\n+ },\r\n \"author\": {\r\n \"ignore_above\": 1024,\r\n \"type\": \"keyword\",\r\n```\r\n d. Start ES and Kibana with the same commands as above\r\ne. Verify that the `.kibana-event-log-ds` index is created and has the\r\nupdated mapping:\r\n-\r\nhttps://localhost:5601/app/management/data/index_management/templates/.kibana-event-log-template\r\n-\r\nhttps://localhost:5601/app/management/data/index_management/indices/index_details?indexName=.ds-.kibana-event-log-ds-2024.09.17-000001&filter=.kibana-&includeHiddenIndices=true&tab=mappings\r\n\r\nI also verified the following:\r\n1. Run ES and Kibana in 8.7 to test the upgrade path from 8.7 (when\r\nevent log indices were versioned) to now\r\n2. Run ES and Kibana in 8.15 to test the upgrade path from the previous\r\nrelease to now\r\n\r\nHowever, I had to create an 8.x branch and cherry pick this commit\r\nbecause `main` is now on 9.0 and we can't upgrade directly from older\r\n8.x version to 9.0!\r\n\r\n---------\r\n\r\nCo-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>\r\nCo-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>","sha":"e2798def07d50595806748dd64cccaa216c5e234"}},{"branch":"8.x","label":"v8.16.0","branchLabelMappingKey":"^v8.16.0$","isSourceBranch":false,"state":"NOT_CREATED"}]}] BACKPORT--> Co-authored-by: Ying Mao <ying.mao@elastic.co>
This commit is contained in:
parent
f94a88f0d1
commit
2933ad6278
9 changed files with 523 additions and 15 deletions
12
x-pack/plugins/event_log/jest.integration.config.js
Normal file
12
x-pack/plugins/event_log/jest.integration.config.js
Normal file
|
@ -0,0 +1,12 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
module.exports = {
|
||||
preset: '@kbn/test/jest_integration',
|
||||
rootDir: '../../..',
|
||||
roots: ['<rootDir>/x-pack/plugins/event_log'],
|
||||
};
|
|
@ -13,8 +13,10 @@ const createClusterClientMock = () => {
|
|||
indexDocuments: jest.fn(),
|
||||
doesIndexTemplateExist: jest.fn(),
|
||||
createIndexTemplate: jest.fn(),
|
||||
updateIndexTemplate: jest.fn(),
|
||||
doesDataStreamExist: jest.fn(),
|
||||
createDataStream: jest.fn(),
|
||||
updateConcreteIndices: jest.fn(),
|
||||
getExistingLegacyIndexTemplates: jest.fn(),
|
||||
setLegacyIndexTemplateToHidden: jest.fn(),
|
||||
getExistingIndices: jest.fn(),
|
||||
|
|
|
@ -215,6 +215,117 @@ describe('createIndexTemplate', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('updateIndexTemplate', () => {
|
||||
test('should call cluster with given template', async () => {
|
||||
clusterClient.indices.simulateTemplate.mockImplementationOnce(async () => ({
|
||||
template: {
|
||||
aliases: {
|
||||
alias_name_1: {
|
||||
is_hidden: true,
|
||||
},
|
||||
alias_name_2: {
|
||||
is_hidden: true,
|
||||
},
|
||||
},
|
||||
settings: {
|
||||
hidden: true,
|
||||
number_of_shards: 1,
|
||||
auto_expand_replicas: '0-1',
|
||||
},
|
||||
mappings: { dynamic: false, properties: { '@timestamp': { type: 'date' } } },
|
||||
},
|
||||
}));
|
||||
|
||||
await clusterClientAdapter.updateIndexTemplate('foo', { args: true });
|
||||
|
||||
expect(clusterClient.indices.simulateTemplate).toHaveBeenCalledWith({
|
||||
name: 'foo',
|
||||
body: { args: true },
|
||||
});
|
||||
expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledWith({
|
||||
name: 'foo',
|
||||
body: { args: true },
|
||||
});
|
||||
});
|
||||
|
||||
test(`should throw error if simulate mappings response is empty`, async () => {
|
||||
clusterClient.indices.simulateTemplate.mockImplementationOnce(async () => ({
|
||||
template: {
|
||||
aliases: {
|
||||
alias_name_1: {
|
||||
is_hidden: true,
|
||||
},
|
||||
alias_name_2: {
|
||||
is_hidden: true,
|
||||
},
|
||||
},
|
||||
settings: {
|
||||
hidden: true,
|
||||
number_of_shards: 1,
|
||||
auto_expand_replicas: '0-1',
|
||||
},
|
||||
mappings: {},
|
||||
},
|
||||
}));
|
||||
|
||||
await expect(() =>
|
||||
clusterClientAdapter.updateIndexTemplate('foo', { name: 'template', args: true })
|
||||
).rejects.toThrowErrorMatchingInlineSnapshot(
|
||||
`"No mappings would be generated for template, possibly due to failed/misconfigured bootstrapping"`
|
||||
);
|
||||
|
||||
expect(logger.error).toHaveBeenCalledWith(
|
||||
`Error updating index template foo: No mappings would be generated for template, possibly due to failed/misconfigured bootstrapping`
|
||||
);
|
||||
});
|
||||
|
||||
test(`should throw error if simulateTemplate throws error`, async () => {
|
||||
clusterClient.indices.simulateTemplate.mockImplementationOnce(() => {
|
||||
throw new Error('failed to simulate');
|
||||
});
|
||||
|
||||
await expect(() =>
|
||||
clusterClientAdapter.updateIndexTemplate('foo', { name: 'template', args: true })
|
||||
).rejects.toThrowErrorMatchingInlineSnapshot(`"failed to simulate"`);
|
||||
|
||||
expect(logger.error).toHaveBeenCalledWith(
|
||||
`Error updating index template foo: failed to simulate`
|
||||
);
|
||||
});
|
||||
|
||||
test(`should throw error if putIndexTemplate throws error`, async () => {
|
||||
clusterClient.indices.simulateTemplate.mockImplementationOnce(async () => ({
|
||||
template: {
|
||||
aliases: {
|
||||
alias_name_1: {
|
||||
is_hidden: true,
|
||||
},
|
||||
alias_name_2: {
|
||||
is_hidden: true,
|
||||
},
|
||||
},
|
||||
settings: {
|
||||
hidden: true,
|
||||
number_of_shards: 1,
|
||||
auto_expand_replicas: '0-1',
|
||||
},
|
||||
mappings: { dynamic: false, properties: { '@timestamp': { type: 'date' } } },
|
||||
},
|
||||
}));
|
||||
clusterClient.indices.putIndexTemplate.mockImplementationOnce(() => {
|
||||
throw new Error('failed to update index template');
|
||||
});
|
||||
|
||||
await expect(() =>
|
||||
clusterClientAdapter.updateIndexTemplate('foo', { name: 'template', args: true })
|
||||
).rejects.toThrowErrorMatchingInlineSnapshot(`"failed to update index template"`);
|
||||
|
||||
expect(logger.error).toHaveBeenCalledWith(
|
||||
`Error updating index template foo: failed to update index template`
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getExistingLegacyIndexTemplates', () => {
|
||||
test('should call cluster with given index template pattern', async () => {
|
||||
await clusterClientAdapter.getExistingLegacyIndexTemplates('foo*');
|
||||
|
@ -497,7 +608,7 @@ describe('doesDataStreamExist', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('createIndex', () => {
|
||||
describe('createDataStream', () => {
|
||||
test('should call cluster with proper arguments', async () => {
|
||||
await clusterClientAdapter.createDataStream('foo');
|
||||
expect(clusterClient.indices.createDataStream).toHaveBeenCalledWith({
|
||||
|
@ -526,6 +637,95 @@ describe('createIndex', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('updateConcreteIndices', () => {
|
||||
test('should call cluster with proper arguments', async () => {
|
||||
clusterClient.indices.simulateIndexTemplate.mockImplementationOnce(async () => ({
|
||||
template: {
|
||||
aliases: { alias_name_1: { is_hidden: true } },
|
||||
settings: {
|
||||
hidden: true,
|
||||
number_of_shards: 1,
|
||||
auto_expand_replicas: '0-1',
|
||||
},
|
||||
mappings: { dynamic: false, properties: { '@timestamp': { type: 'date' } } },
|
||||
},
|
||||
}));
|
||||
|
||||
await clusterClientAdapter.updateConcreteIndices('foo');
|
||||
expect(clusterClient.indices.simulateIndexTemplate).toHaveBeenCalledWith({
|
||||
name: 'foo',
|
||||
});
|
||||
expect(clusterClient.indices.putMapping).toHaveBeenCalledWith({
|
||||
index: 'foo',
|
||||
body: { dynamic: false, properties: { '@timestamp': { type: 'date' } } },
|
||||
});
|
||||
});
|
||||
|
||||
test('should not update mapping if simulate response does not contain mappings', async () => {
|
||||
// @ts-ignore
|
||||
clusterClient.indices.simulateIndexTemplate.mockImplementationOnce(async () => ({
|
||||
template: {
|
||||
aliases: { alias_name_1: { is_hidden: true } },
|
||||
settings: {
|
||||
hidden: true,
|
||||
number_of_shards: 1,
|
||||
auto_expand_replicas: '0-1',
|
||||
},
|
||||
},
|
||||
}));
|
||||
|
||||
await clusterClientAdapter.updateConcreteIndices('foo');
|
||||
expect(clusterClient.indices.simulateIndexTemplate).toHaveBeenCalledWith({
|
||||
name: 'foo',
|
||||
});
|
||||
expect(clusterClient.indices.putMapping).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test('should throw error if simulateIndexTemplate throws error', async () => {
|
||||
clusterClient.indices.simulateIndexTemplate.mockImplementationOnce(() => {
|
||||
throw new Error('failed to simulate');
|
||||
});
|
||||
|
||||
await expect(() =>
|
||||
clusterClientAdapter.updateConcreteIndices('foo')
|
||||
).rejects.toThrowErrorMatchingInlineSnapshot(`"failed to simulate"`);
|
||||
|
||||
expect(clusterClient.indices.putMapping).not.toHaveBeenCalled();
|
||||
expect(logger.error).toHaveBeenCalledWith(
|
||||
`Error updating index mappings for foo: failed to simulate`
|
||||
);
|
||||
});
|
||||
|
||||
test('should throw error if putMapping throws error', async () => {
|
||||
clusterClient.indices.simulateIndexTemplate.mockImplementationOnce(async () => ({
|
||||
template: {
|
||||
aliases: { alias_name_1: { is_hidden: true } },
|
||||
settings: {
|
||||
hidden: true,
|
||||
number_of_shards: 1,
|
||||
auto_expand_replicas: '0-1',
|
||||
},
|
||||
mappings: { dynamic: false, properties: { '@timestamp': { type: 'date' } } },
|
||||
},
|
||||
}));
|
||||
clusterClient.indices.putMapping.mockImplementationOnce(() => {
|
||||
throw new Error('failed to put mappings');
|
||||
});
|
||||
|
||||
await expect(() =>
|
||||
clusterClientAdapter.updateConcreteIndices('foo')
|
||||
).rejects.toThrowErrorMatchingInlineSnapshot(`"failed to put mappings"`);
|
||||
|
||||
expect(clusterClient.indices.putMapping).toHaveBeenCalledWith({
|
||||
index: 'foo',
|
||||
body: { dynamic: false, properties: { '@timestamp': { type: 'date' } } },
|
||||
});
|
||||
expect(logger.error).toHaveBeenCalledWith(
|
||||
`Error updating index mappings for foo: failed to put mappings`
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('queryEventsBySavedObject', () => {
|
||||
const DEFAULT_OPTIONS = queryOptionsSchema.validate({});
|
||||
|
||||
|
|
|
@ -7,7 +7,7 @@
|
|||
|
||||
import { Subject } from 'rxjs';
|
||||
import { bufferTime, filter as rxFilter, concatMap } from 'rxjs';
|
||||
import { reject, isUndefined, isNumber, pick } from 'lodash';
|
||||
import { reject, isUndefined, isNumber, pick, isEmpty, get } from 'lodash';
|
||||
import type { PublicMethodsOf } from '@kbn/utility-types';
|
||||
import { Logger, ElasticsearchClient } from '@kbn/core/server';
|
||||
import util from 'util';
|
||||
|
@ -213,6 +213,28 @@ export class ClusterClientAdapter<TDoc extends { body: AliasAny; index: string }
|
|||
}
|
||||
}
|
||||
|
||||
public async updateIndexTemplate(name: string, template: Record<string, unknown>): Promise<void> {
|
||||
this.logger.info(`Updating index template ${name}`);
|
||||
|
||||
try {
|
||||
const esClient = await this.elasticsearchClientPromise;
|
||||
|
||||
// Simulate the index template to proactively identify any issues with the mappings
|
||||
const simulateResponse = await esClient.indices.simulateTemplate({ name, body: template });
|
||||
const mappings: estypes.MappingTypeMapping = simulateResponse.template.mappings;
|
||||
|
||||
if (isEmpty(mappings)) {
|
||||
throw new Error(
|
||||
`No mappings would be generated for ${template.name}, possibly due to failed/misconfigured bootstrapping`
|
||||
);
|
||||
}
|
||||
await esClient.indices.putIndexTemplate({ name, body: template });
|
||||
} catch (err) {
|
||||
this.logger.error(`Error updating index template ${name}: ${err.message}`);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
public async getExistingLegacyIndexTemplates(
|
||||
indexTemplatePattern: string
|
||||
): Promise<estypes.IndicesGetTemplateResponse> {
|
||||
|
@ -335,7 +357,7 @@ export class ClusterClientAdapter<TDoc extends { body: AliasAny; index: string }
|
|||
}
|
||||
}
|
||||
|
||||
public async createDataStream(name: string, body: Record<string, unknown> = {}): Promise<void> {
|
||||
public async createDataStream(name: string): Promise<void> {
|
||||
this.logger.info(`Creating datastream ${name}`);
|
||||
try {
|
||||
const esClient = await this.elasticsearchClientPromise;
|
||||
|
@ -347,6 +369,23 @@ export class ClusterClientAdapter<TDoc extends { body: AliasAny; index: string }
|
|||
}
|
||||
}
|
||||
|
||||
public async updateConcreteIndices(name: string): Promise<void> {
|
||||
this.logger.info(`Updating concrete index mappings for ${name}`);
|
||||
try {
|
||||
const esClient = await this.elasticsearchClientPromise;
|
||||
const simulatedIndexMapping = await esClient.indices.simulateIndexTemplate({ name });
|
||||
const simulatedMapping = get(simulatedIndexMapping, ['template', 'mappings']);
|
||||
|
||||
if (simulatedMapping != null) {
|
||||
await esClient.indices.putMapping({ index: name, body: simulatedMapping });
|
||||
this.logger.debug(`Successfully updated concrete index mappings for ${name}`);
|
||||
}
|
||||
} catch (err) {
|
||||
this.logger.error(`Error updating index mappings for ${name}: ${err.message}`);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
public async queryEventsBySavedObjects(
|
||||
queryOptions: FindEventsOptionsBySavedObjectFilter
|
||||
): Promise<QueryEventsBySavedObjectResult> {
|
||||
|
|
|
@ -18,7 +18,7 @@ describe('initializeEs', () => {
|
|||
esContext.esAdapter.getExistingIndexAliases.mockResolvedValue({});
|
||||
});
|
||||
|
||||
test(`should update existing index templates if any exist and are not hidden`, async () => {
|
||||
test(`should update existing index templates to hidden if any exist and are not hidden`, async () => {
|
||||
const testTemplate = {
|
||||
order: 0,
|
||||
index_patterns: ['foo-bar-*'],
|
||||
|
@ -393,14 +393,16 @@ describe('initializeEs', () => {
|
|||
await initializeEs(esContext);
|
||||
expect(esContext.esAdapter.doesIndexTemplateExist).toHaveBeenCalled();
|
||||
expect(esContext.esAdapter.createIndexTemplate).toHaveBeenCalled();
|
||||
expect(esContext.esAdapter.updateIndexTemplate).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test(`shouldn't create index template if it already exists`, async () => {
|
||||
test(`should update index template if it already exists`, async () => {
|
||||
esContext.esAdapter.doesIndexTemplateExist.mockResolvedValue(true);
|
||||
|
||||
await initializeEs(esContext);
|
||||
expect(esContext.esAdapter.doesIndexTemplateExist).toHaveBeenCalled();
|
||||
expect(esContext.esAdapter.createIndexTemplate).not.toHaveBeenCalled();
|
||||
expect(esContext.esAdapter.updateIndexTemplate).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test(`should create data stream if it doesn't exist`, async () => {
|
||||
|
@ -409,14 +411,16 @@ describe('initializeEs', () => {
|
|||
await initializeEs(esContext);
|
||||
expect(esContext.esAdapter.doesDataStreamExist).toHaveBeenCalled();
|
||||
expect(esContext.esAdapter.createDataStream).toHaveBeenCalled();
|
||||
expect(esContext.esAdapter.updateConcreteIndices).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test(`shouldn't create data stream if it already exists`, async () => {
|
||||
test(`should update indices of data stream if it already exists`, async () => {
|
||||
esContext.esAdapter.doesDataStreamExist.mockResolvedValue(true);
|
||||
|
||||
await initializeEs(esContext);
|
||||
expect(esContext.esAdapter.doesDataStreamExist).toHaveBeenCalled();
|
||||
expect(esContext.esAdapter.createDataStream).not.toHaveBeenCalled();
|
||||
expect(esContext.esAdapter.updateConcreteIndices).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
|
|
|
@ -216,12 +216,17 @@ class EsInitializationSteps {
|
|||
const exists = await this.esContext.esAdapter.doesIndexTemplateExist(
|
||||
this.esContext.esNames.indexTemplate
|
||||
);
|
||||
const templateBody = getIndexTemplate(this.esContext.esNames);
|
||||
if (!exists) {
|
||||
const templateBody = getIndexTemplate(this.esContext.esNames);
|
||||
await this.esContext.esAdapter.createIndexTemplate(
|
||||
this.esContext.esNames.indexTemplate,
|
||||
templateBody
|
||||
);
|
||||
} else {
|
||||
await this.esContext.esAdapter.updateIndexTemplate(
|
||||
this.esContext.esNames.indexTemplate,
|
||||
templateBody
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -230,14 +235,10 @@ class EsInitializationSteps {
|
|||
this.esContext.esNames.dataStream
|
||||
);
|
||||
if (!exists) {
|
||||
await this.esContext.esAdapter.createDataStream(this.esContext.esNames.dataStream, {
|
||||
aliases: {
|
||||
[this.esContext.esNames.dataStream]: {
|
||||
is_write_index: true,
|
||||
is_hidden: true,
|
||||
},
|
||||
},
|
||||
});
|
||||
await this.esContext.esAdapter.createDataStream(this.esContext.esNames.dataStream);
|
||||
} else {
|
||||
// apply current mappings to existing data stream
|
||||
await this.esContext.esAdapter.updateConcreteIndices(this.esContext.esNames.dataStream);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -0,0 +1,173 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { type ElasticsearchClient } from '@kbn/core/server';
|
||||
import { setupKibanaServer, setupTestServers } from './lib/setup_test_servers';
|
||||
import { IEvent } from '../types';
|
||||
import { EsContextCtorParams } from '../es/context';
|
||||
|
||||
const { createEsContext: createEsContextMock } = jest.requireMock('../es');
|
||||
jest.mock('../es', () => {
|
||||
const actual = jest.requireActual('../es');
|
||||
return {
|
||||
...actual,
|
||||
createEsContext: jest.fn().mockImplementation((opts) => {
|
||||
return new actual.createEsContext(opts);
|
||||
}),
|
||||
};
|
||||
});
|
||||
|
||||
describe('update existing event log mappings on startup', () => {
|
||||
it('should update mappings for existing event log indices', async () => {
|
||||
const setupResult = await setupTestServers();
|
||||
const esServer = setupResult.esServer;
|
||||
let kibanaServer = setupResult.kibanaServer;
|
||||
|
||||
expect(createEsContextMock).toHaveBeenCalledTimes(1);
|
||||
let createEsContextOpts: EsContextCtorParams = createEsContextMock.mock.calls[0][0];
|
||||
let infoLogSpy = jest.spyOn(createEsContextOpts.logger, 'info');
|
||||
|
||||
await retry(async () => {
|
||||
expect(infoLogSpy).toHaveBeenCalledWith(`Creating datastream .kibana-event-log-ds`);
|
||||
expect(infoLogSpy).not.toHaveBeenCalledWith(
|
||||
`Updating concrete index mappings for .kibana-event-log-ds`
|
||||
);
|
||||
});
|
||||
|
||||
await injectEventLogDoc(kibanaServer.coreStart.elasticsearch.client.asInternalUser, {
|
||||
'@timestamp': '2024-09-19T20:38:47.124Z',
|
||||
event: {
|
||||
provider: 'alerting',
|
||||
action: 'execute',
|
||||
kind: 'alert',
|
||||
category: ['AlertingExample'],
|
||||
start: '2024-09-19T20:38:46.963Z',
|
||||
outcome: 'success',
|
||||
end: '2024-09-19T20:38:47.124Z',
|
||||
duration: '161000000',
|
||||
},
|
||||
kibana: {
|
||||
alert: {
|
||||
rule: {
|
||||
rule_type_id: 'example.always-firing',
|
||||
consumer: 'alerts',
|
||||
execution: {
|
||||
uuid: '578f0ca3-aa08-4700-aed0-236c888c6cae',
|
||||
metrics: {
|
||||
number_of_triggered_actions: 0,
|
||||
number_of_generated_actions: 0,
|
||||
alert_counts: {
|
||||
active: 5,
|
||||
new: 5,
|
||||
recovered: 5,
|
||||
},
|
||||
number_of_delayed_alerts: 0,
|
||||
number_of_searches: 0,
|
||||
es_search_duration_ms: 0,
|
||||
total_search_duration_ms: 0,
|
||||
claim_to_start_duration_ms: 26,
|
||||
total_run_duration_ms: 187,
|
||||
prepare_rule_duration_ms: 18,
|
||||
rule_type_run_duration_ms: 0,
|
||||
process_alerts_duration_ms: 1,
|
||||
persist_alerts_duration_ms: 64,
|
||||
trigger_actions_duration_ms: 0,
|
||||
process_rule_duration_ms: 69,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
saved_objects: [
|
||||
{
|
||||
rel: 'primary',
|
||||
type: 'alert',
|
||||
id: '3389d834-edc2-4245-a319-3ff689f5bf3b',
|
||||
type_id: 'example.always-firing',
|
||||
},
|
||||
],
|
||||
space_ids: ['default'],
|
||||
task: {
|
||||
scheduled: '2024-09-19T20:38:46.797Z',
|
||||
schedule_delay: 166000000,
|
||||
},
|
||||
alerting: {
|
||||
outcome: 'success',
|
||||
status: 'active',
|
||||
},
|
||||
server_uuid: '5b2de169-2785-441b-ae8c-186a1936b17d',
|
||||
version: '9.0.0',
|
||||
},
|
||||
rule: {
|
||||
id: '3389d834-edc2-4245-a319-3ff689f5bf3b',
|
||||
license: 'basic',
|
||||
category: 'example.always-firing',
|
||||
ruleset: 'AlertingExample',
|
||||
name: 'e',
|
||||
},
|
||||
message: "rule executed: example.always-firing:3389d834-edc2-4245-a319-3ff689f5bf3b: 'e'",
|
||||
ecs: {
|
||||
version: '1.8.0',
|
||||
},
|
||||
});
|
||||
|
||||
if (kibanaServer) {
|
||||
await kibanaServer.stop();
|
||||
}
|
||||
infoLogSpy.mockRestore();
|
||||
|
||||
const restartKb = await setupKibanaServer();
|
||||
kibanaServer = restartKb.kibanaServer;
|
||||
|
||||
expect(createEsContextMock).toHaveBeenCalledTimes(2);
|
||||
createEsContextOpts = createEsContextMock.mock.calls[1][0];
|
||||
infoLogSpy = jest.spyOn(createEsContextOpts.logger, 'info');
|
||||
const debugLogSpy = jest.spyOn(createEsContextOpts.logger, 'debug');
|
||||
|
||||
await retry(async () => {
|
||||
expect(infoLogSpy).toHaveBeenCalledWith(
|
||||
`Updating concrete index mappings for .kibana-event-log-ds`
|
||||
);
|
||||
expect(debugLogSpy).toHaveBeenCalledWith(
|
||||
`Successfully updated concrete index mappings for .kibana-event-log-ds`
|
||||
);
|
||||
});
|
||||
|
||||
if (kibanaServer) {
|
||||
await kibanaServer.stop();
|
||||
}
|
||||
if (esServer) {
|
||||
await esServer.stop();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
async function injectEventLogDoc(esClient: ElasticsearchClient, doc: IEvent) {
|
||||
await esClient.index({
|
||||
index: '.kibana-event-log-ds',
|
||||
document: doc,
|
||||
});
|
||||
}
|
||||
|
||||
interface RetryOpts {
|
||||
times: number;
|
||||
intervalMs: number;
|
||||
}
|
||||
|
||||
async function retry<T>(cb: () => Promise<T>, options: RetryOpts = { times: 60, intervalMs: 500 }) {
|
||||
let attempt = 1;
|
||||
while (true) {
|
||||
try {
|
||||
return await cb();
|
||||
} catch (e) {
|
||||
if (attempt >= options.times) {
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
attempt++;
|
||||
await new Promise((resolve) => setTimeout(resolve, options.intervalMs));
|
||||
}
|
||||
}
|
|
@ -0,0 +1,76 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import deepmerge from 'deepmerge';
|
||||
import { createTestServers, createRootWithCorePlugins } from '@kbn/core-test-helpers-kbn-server';
|
||||
|
||||
function createRoot(settings = {}) {
|
||||
return createRootWithCorePlugins(
|
||||
deepmerge(
|
||||
{
|
||||
logging: {
|
||||
root: {
|
||||
level: 'warn',
|
||||
},
|
||||
loggers: [
|
||||
{
|
||||
name: 'plugins.eventLog',
|
||||
level: 'all',
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
settings
|
||||
),
|
||||
{ oss: false }
|
||||
);
|
||||
}
|
||||
export async function setupTestServers(settings = {}) {
|
||||
const { startES } = createTestServers({
|
||||
adjustTimeout: (t) => jest.setTimeout(t),
|
||||
settings: {
|
||||
es: {
|
||||
license: 'trial',
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const esServer = await startES();
|
||||
|
||||
const root = createRoot(settings);
|
||||
|
||||
await root.preboot();
|
||||
const coreSetup = await root.setup();
|
||||
const coreStart = await root.start();
|
||||
|
||||
return {
|
||||
esServer,
|
||||
kibanaServer: {
|
||||
root,
|
||||
coreSetup,
|
||||
coreStart,
|
||||
stop: async () => await root.shutdown(),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export async function setupKibanaServer(settings = {}) {
|
||||
const root = createRoot(settings);
|
||||
|
||||
await root.preboot();
|
||||
const coreSetup = await root.setup();
|
||||
const coreStart = await root.start();
|
||||
|
||||
return {
|
||||
kibanaServer: {
|
||||
root,
|
||||
coreSetup,
|
||||
coreStart,
|
||||
stop: async () => await root.shutdown(),
|
||||
},
|
||||
};
|
||||
}
|
|
@ -21,6 +21,7 @@
|
|||
"@kbn/std",
|
||||
"@kbn/safer-lodash-set",
|
||||
"@kbn/serverless",
|
||||
"@kbn/core-test-helpers-kbn-server",
|
||||
],
|
||||
"exclude": [
|
||||
"target/**/*",
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue