[8.x] [IndexAdapter] Extract index-adapter package from data-stream-adapter (#199575) (#199848)

# Backport

This will backport the following commits from `main` to `8.x`:
- [[IndexAdapter] Extract index-adapter package from data-stream-adapter
(#199575)](https://github.com/elastic/kibana/pull/199575)

<!--- Backport version: 8.9.8 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Sergi
Massaneda","email":"sergi.massaneda@elastic.co"},"sourceCommit":{"committedDate":"2024-11-12T17:16:32Z","message":"[IndexAdapter]
Extract index-adapter package from data-stream-adapter (#199575)\n\n##
Summary\r\n\r\nExtracts `IndexAdapter` from `DataStreamAdapter`
and\r\n`IndexPatternAdapter` from `DataStreamSpaceAdapter`.\r\n\r\nThere
are no breaking changes for the _data-stream-adapter_ package;
the\r\nbehavior of both the `DataStreamAdapter` and
`DataStreamSpaceAdapter`\r\nremains unchanged.\r\n\r\nThe new
_index-adapter_ package exports `IndexAdapter`
and\r\n`IndexPatternAdapter` to manage individual indices without using
data\r\nstreams.\r\n\r\nThis is needed for SIEM rule
migrations.\r\n\r\n---------\r\n\r\nCo-authored-by: kibanamachine
<42973632+kibanamachine@users.noreply.github.com>\r\nCo-authored-by:
Elastic Machine
<elasticmachine@users.noreply.github.com>","sha":"9a9f02c9315beda4089b1ef16089747c080bc345","branchLabelMapping":{"^v9.0.0$":"main","^v8.17.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","v9.0.0","Team:Threat
Hunting","Team:
SecuritySolution","backport:prev-minor","v8.18.0"],"number":199575,"url":"https://github.com/elastic/kibana/pull/199575","mergeCommit":{"message":"[IndexAdapter]
Extract index-adapter package from data-stream-adapter (#199575)\n\n##
Summary\r\n\r\nExtracts `IndexAdapter` from `DataStreamAdapter`
and\r\n`IndexPatternAdapter` from `DataStreamSpaceAdapter`.\r\n\r\nThere
are no breaking changes for the _data-stream-adapter_ package;
the\r\nbehavior of both the `DataStreamAdapter` and
`DataStreamSpaceAdapter`\r\nremains unchanged.\r\n\r\nThe new
_index-adapter_ package exports `IndexAdapter`
and\r\n`IndexPatternAdapter` to manage individual indices without using
data\r\nstreams.\r\n\r\nThis is needed for SIEM rule
migrations.\r\n\r\n---------\r\n\r\nCo-authored-by: kibanamachine
<42973632+kibanamachine@users.noreply.github.com>\r\nCo-authored-by:
Elastic Machine
<elasticmachine@users.noreply.github.com>","sha":"9a9f02c9315beda4089b1ef16089747c080bc345"}},"sourceBranch":"main","suggestedTargetBranches":["8.18"],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","labelRegex":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/199575","number":199575,"mergeCommit":{"message":"[IndexAdapter]
Extract index-adapter package from data-stream-adapter (#199575)\n\n##
Summary\r\n\r\nExtracts `IndexAdapter` from `DataStreamAdapter`
and\r\n`IndexPatternAdapter` from `DataStreamSpaceAdapter`.\r\n\r\nThere
are no breaking changes for the _data-stream-adapter_ package;
the\r\nbehavior of both the `DataStreamAdapter` and
`DataStreamSpaceAdapter`\r\nremains unchanged.\r\n\r\nThe new
_index-adapter_ package exports `IndexAdapter`
and\r\n`IndexPatternAdapter` to manage individual indices without using
data\r\nstreams.\r\n\r\nThis is needed for SIEM rule
migrations.\r\n\r\n---------\r\n\r\nCo-authored-by: kibanamachine
<42973632+kibanamachine@users.noreply.github.com>\r\nCo-authored-by:
Elastic Machine
<elasticmachine@users.noreply.github.com>","sha":"9a9f02c9315beda4089b1ef16089747c080bc345"}},{"branch":"8.18","label":"v8.18.0","labelRegex":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->
This commit is contained in:
Sergi Massaneda 2024-11-12 20:52:41 +01:00 committed by GitHub
parent 7ecc654b10
commit 953bc0feb3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
34 changed files with 858 additions and 221 deletions

View file

@ -567,6 +567,7 @@
"@kbn/i18n-react": "link:packages/kbn-i18n-react",
"@kbn/iframe-embedded-plugin": "link:x-pack/test/functional_embedded/plugins/iframe_embedded",
"@kbn/image-embeddable-plugin": "link:src/plugins/image_embeddable",
"@kbn/index-adapter": "link:packages/kbn-index-adapter",
"@kbn/index-lifecycle-management-plugin": "link:x-pack/plugins/index_lifecycle_management",
"@kbn/index-management-plugin": "link:x-pack/plugins/index_management",
"@kbn/index-management-shared-types": "link:x-pack/packages/index-management/index_management_shared_types",

View file

@ -9,13 +9,13 @@
export { DataStreamAdapter } from './src/data_stream_adapter';
export { DataStreamSpacesAdapter } from './src/data_stream_spaces_adapter';
export { retryTransientEsErrors } from './src/retry_transient_es_errors';
export { ecsFieldMap, type EcsFieldMap } from './src/field_maps/ecs_field_map';
export { retryTransientEsErrors, ecsFieldMap } from '@kbn/index-adapter';
export type {
DataStreamAdapterParams,
SetComponentTemplateParams,
SetIndexTemplateParams,
InstallParams,
} from './src/data_stream_adapter';
export * from './src/field_maps/types';
EcsFieldMap,
} from '@kbn/index-adapter';
export * from '@kbn/index-adapter/src/field_maps/types';

View file

@ -1,5 +1,6 @@
{
"type": "shared-common",
"type": "shared-server",
"id": "@kbn/data-stream-adapter",
"owner": "@elastic/security-threat-hunting-explore"
"owner": "@elastic/security-threat-hunting",
"visibility": "shared"
}

View file

@ -136,10 +136,11 @@ describe('createOrUpdateDataStream', () => {
it(`should create data stream if not exists`, async () => {
esClient.indices.getDataStream.mockResolvedValueOnce({ data_streams: [] });
await createDataStream({
await createOrUpdateDataStream({
esClient,
logger,
name,
totalFieldsLimit,
});
expect(esClient.indices.createDataStream).toHaveBeenCalledWith({ name });

View file

@ -11,7 +11,7 @@ import type { IndicesDataStream } from '@elastic/elasticsearch/lib/api/types';
import type { IndicesSimulateIndexTemplateResponse } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import type { Logger, ElasticsearchClient } from '@kbn/core/server';
import { get } from 'lodash';
import { retryTransientEsErrors } from './retry_transient_es_errors';
import { retryTransientEsErrors } from '@kbn/index-adapter';
interface UpdateIndexMappingsOpts {
logger: Logger;
@ -168,7 +168,7 @@ export async function createDataStream({
esClient,
name,
}: CreateDataStreamParams): Promise<void> {
logger.info(`Creating data stream - ${name}`);
logger.debug(`Checking data stream exists - ${name}`);
// check if data stream exists
let dataStreamExists = false;
@ -189,6 +189,7 @@ export async function createDataStream({
if (dataStreamExists) {
return;
}
logger.info(`Installing data stream - ${name}`);
try {
await retryTransientEsErrors(() => esClient.indices.createDataStream({ name }), { logger });

View file

@ -7,146 +7,23 @@
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import type {
ClusterPutComponentTemplateRequest,
IndicesIndexSettings,
IndicesPutIndexTemplateIndexTemplateMapping,
IndicesPutIndexTemplateRequest,
} from '@elastic/elasticsearch/lib/api/types';
import type { Logger, ElasticsearchClient } from '@kbn/core/server';
import type { Subject } from 'rxjs';
import type { FieldMap } from './field_maps/types';
import { createOrUpdateComponentTemplate } from './create_or_update_component_template';
import { IndexAdapter, SetIndexTemplateParams, type InstallParams } from '@kbn/index-adapter';
import { createOrUpdateDataStream } from './create_or_update_data_stream';
import { createOrUpdateIndexTemplate } from './create_or_update_index_template';
import { InstallShutdownError, installWithTimeout } from './install_with_timeout';
import { getComponentTemplate, getIndexTemplate } from './resource_installer_utils';
export interface DataStreamAdapterParams {
kibanaVersion: string;
totalFieldsLimit?: number;
}
export interface SetComponentTemplateParams {
name: string;
fieldMap: FieldMap;
settings?: IndicesIndexSettings;
dynamic?: 'strict' | boolean;
}
export interface SetIndexTemplateParams {
name: string;
componentTemplateRefs?: string[];
namespace?: string;
template?: IndicesPutIndexTemplateIndexTemplateMapping;
hidden?: boolean;
}
export interface GetInstallFnParams {
logger: Logger;
pluginStop$: Subject<void>;
tasksTimeoutMs?: number;
}
export interface InstallParams {
logger: Logger;
esClient: ElasticsearchClient | Promise<ElasticsearchClient>;
pluginStop$: Subject<void>;
tasksTimeoutMs?: number;
}
const DEFAULT_FIELDS_LIMIT = 2500;
export class DataStreamAdapter {
protected readonly kibanaVersion: string;
protected readonly totalFieldsLimit: number;
protected componentTemplates: ClusterPutComponentTemplateRequest[] = [];
protected indexTemplates: IndicesPutIndexTemplateRequest[] = [];
protected installed: boolean;
constructor(protected readonly name: string, options: DataStreamAdapterParams) {
this.installed = false;
this.kibanaVersion = options.kibanaVersion;
this.totalFieldsLimit = options.totalFieldsLimit ?? DEFAULT_FIELDS_LIMIT;
}
public setComponentTemplate(params: SetComponentTemplateParams) {
if (this.installed) {
throw new Error('Cannot set component template after install');
}
this.componentTemplates.push(getComponentTemplate(params));
}
export class DataStreamAdapter extends IndexAdapter {
public setIndexTemplate(params: SetIndexTemplateParams) {
if (this.installed) {
throw new Error('Cannot set index template after install');
}
this.indexTemplates.push(
getIndexTemplate({
...params,
indexPatterns: [this.name],
kibanaVersion: this.kibanaVersion,
totalFieldsLimit: this.totalFieldsLimit,
})
);
super.setIndexTemplate({ ...params, isDataStream: true });
}
protected getInstallFn({ logger, pluginStop$, tasksTimeoutMs }: GetInstallFnParams) {
return async (promise: Promise<void>, description?: string): Promise<void> => {
try {
await installWithTimeout({
installFn: () => promise,
description,
timeoutMs: tasksTimeoutMs,
pluginStop$,
});
} catch (err) {
if (err instanceof InstallShutdownError) {
logger.info(err.message);
} else {
throw err;
}
}
};
}
public async install({
logger,
esClient: esClientToResolve,
pluginStop$,
tasksTimeoutMs,
}: InstallParams) {
public async install(params: InstallParams) {
this.installed = true;
const { logger, pluginStop$, tasksTimeoutMs } = params;
const esClient = await params.esClient;
await this.installTemplates(params);
const esClient = await esClientToResolve;
const installFn = this.getInstallFn({ logger, pluginStop$, tasksTimeoutMs });
// Install component templates in parallel
await Promise.all(
this.componentTemplates.map((componentTemplate) =>
installFn(
createOrUpdateComponentTemplate({
template: componentTemplate,
esClient,
logger,
totalFieldsLimit: this.totalFieldsLimit,
}),
`${componentTemplate.name} component template`
)
)
);
// Install index templates in parallel
await Promise.all(
this.indexTemplates.map((indexTemplate) =>
installFn(
createOrUpdateIndexTemplate({
template: indexTemplate,
esClient,
logger,
}),
`${indexTemplate.name} index template`
)
)
);
// create data stream when everything is ready
await installFn(
createOrUpdateDataStream({

View file

@ -7,60 +7,27 @@
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import { createOrUpdateComponentTemplate } from './create_or_update_component_template';
import { createDataStream, updateDataStreams } from './create_or_update_data_stream';
import { createOrUpdateIndexTemplate } from './create_or_update_index_template';
import {
DataStreamAdapter,
type DataStreamAdapterParams,
IndexPatternAdapter,
type SetIndexTemplateParams,
type InstallParams,
} from './data_stream_adapter';
type InstallIndex,
} from '@kbn/index-adapter';
import { createDataStream, updateDataStreams } from './create_or_update_data_stream';
export class DataStreamSpacesAdapter extends DataStreamAdapter {
private installedSpaceDataStreamName: Map<string, Promise<string>>;
private _installSpace?: (spaceId: string) => Promise<string>;
constructor(private readonly prefix: string, options: DataStreamAdapterParams) {
super(`${prefix}-*`, options); // make indexTemplate `indexPatterns` match all data stream space names
this.installedSpaceDataStreamName = new Map();
export class DataStreamSpacesAdapter extends IndexPatternAdapter {
public setIndexTemplate(params: SetIndexTemplateParams) {
super.setIndexTemplate({ ...params, isDataStream: true });
}
public async install({
logger,
esClient: esClientToResolve,
pluginStop$,
tasksTimeoutMs,
}: InstallParams) {
this.installed = true;
protected async _install(params: InstallParams): Promise<InstallIndex> {
const { logger, pluginStop$, tasksTimeoutMs } = params;
const esClient = await esClientToResolve;
await this.installTemplates(params);
const esClient = await params.esClient;
const installFn = this.getInstallFn({ logger, pluginStop$, tasksTimeoutMs });
// Install component templates in parallel
await Promise.all(
this.componentTemplates.map((componentTemplate) =>
installFn(
createOrUpdateComponentTemplate({
template: componentTemplate,
esClient,
logger,
totalFieldsLimit: this.totalFieldsLimit,
}),
`create or update ${componentTemplate.name} component template`
)
)
);
// Install index templates in parallel
await Promise.all(
this.indexTemplates.map((indexTemplate) =>
installFn(
createOrUpdateIndexTemplate({ template: indexTemplate, esClient, logger }),
`create or update ${indexTemplate.name} index template`
)
)
);
// Update existing space data streams
await installFn(
updateDataStreams({
@ -72,31 +39,21 @@ export class DataStreamSpacesAdapter extends DataStreamAdapter {
`update space data streams`
);
// define function to install data stream for spaces on demand
this._installSpace = async (spaceId: string) => {
const existingInstallPromise = this.installedSpaceDataStreamName.get(spaceId);
if (existingInstallPromise) {
return existingInstallPromise;
}
const name = `${this.prefix}-${spaceId}`;
const installPromise = installFn(
createDataStream({ name, esClient, logger }),
`create ${name} data stream`
).then(() => name);
this.installedSpaceDataStreamName.set(spaceId, installPromise);
return installPromise;
};
// define function to install data stream on demand
return async (name: string) =>
installFn(createDataStream({ name, esClient, logger }), `create ${name} data stream`);
}
/**
* Method to create the data stream for a given space ID.
* It resolves with the full data stream name.
*/
public async installSpace(spaceId: string): Promise<string> {
if (!this._installSpace) {
throw new Error('Cannot installSpace before install');
}
return this._installSpace(spaceId);
await this.createIndex(spaceId);
return this.getIndexName(spaceId);
}
public async getInstalledSpaceName(spaceId: string): Promise<string | undefined> {
return this.installedSpaceDataStreamName.get(spaceId);
return this.getInstalledIndexName(spaceId);
}
}

View file

@ -5,18 +5,14 @@
"types": [
"jest",
"node",
"react",
"@emotion/react/types/css-prop",
"@testing-library/jest-dom",
"@testing-library/react"
]
},
"include": ["**/*.ts", "**/*.tsx"],
"include": ["**/*.ts"],
"kbn_references": [
"@kbn/core",
"@kbn/std",
"@kbn/safer-lodash-set",
"@kbn/logging-mocks",
"@kbn/index-adapter",
],
"exclude": [
"target/**/*"
],
"exclude": ["target/**/*"]
}

View file

@ -0,0 +1,59 @@
# @kbn/index-adapter
Utility library for Elasticsearch index management.
## IndexAdapter
Manage single index. Example:
```
// Setup
const indexAdapter = new IndexAdapter('my-awesome-index', { kibanaVersion: '8.12.1' });
indexAdapter.setComponentTemplate({
name: 'awesome-component-template',
fieldMap: {
'awesome.field1: { type: 'keyword', required: true },
'awesome.nested.field2: { type: 'number', required: false },
// ...
},
});
indexAdapter.setIndexTemplate({
name: 'awesome-index-template',
componentTemplateRefs: ['awesome-component-template', 'ecs-component-template'],
});
// Start
await indexAdapter.install({ logger, esClient, pluginStop$ }); // Installs templates and the 'my-awesome-index' index, or updates existing.
```
## IndexPatternAdapter
Manage index patterns. Example:
```
// Setup
const indexPatternAdapter = new IndexPatternAdapter('my-awesome-index', { kibanaVersion: '8.12.1' });
indexPatternAdapter.setComponentTemplate({
name: 'awesome-component-template',
fieldMap: {
'awesome.field1: { type: 'keyword', required: true },
'awesome.nested.field2: { type: 'number', required: false },
// ...
},
});
indexPatternAdapter.setIndexTemplate({
name: 'awesome-index-template',
componentTemplateRefs: ['awesome-component-template', 'ecs-component-template'],
});
// Start
indexPatternAdapter.install({ logger, esClient, pluginStop$ }); // Installs/updates templates for the index pattern 'my-awesome-index-*', and updates mappings of all specific indices
// Create a specific index on the fly
await indexPatternAdapter.installIndex('12345'); // creates 'my-awesome-index-12345' index if it does not exist.
```

View file

@ -0,0 +1,23 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
export { IndexAdapter } from './src/index_adapter';
export { IndexPatternAdapter, type InstallIndex } from './src/index_pattern_adapter';
export { retryTransientEsErrors } from './src/retry_transient_es_errors';
export { ecsFieldMap, type EcsFieldMap } from './src/field_maps/ecs_field_map';
export { createOrUpdateIndexTemplate } from './src/create_or_update_index_template';
export { createOrUpdateComponentTemplate } from './src/create_or_update_component_template';
export type {
SetComponentTemplateParams,
SetIndexTemplateParams,
IndexAdapterParams,
InstallParams,
} from './src/index_adapter';
export * from './src/field_maps/types';

View file

@ -0,0 +1,14 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
module.exports = {
preset: '@kbn/test',
rootDir: '../..',
roots: ['<rootDir>/packages/kbn-index-adapter'],
};

View file

@ -0,0 +1,6 @@
{
"type": "shared-server",
"id": "@kbn/index-adapter",
"owner": "@elastic/security-threat-hunting",
"visibility": "shared"
}

View file

@ -0,0 +1,7 @@
{
"name": "@kbn/index-adapter",
"version": "1.0.0",
"description": "Utility library for Elasticsearch index management",
"license": "Elastic License 2.0 OR AGPL-3.0-only OR SSPL-1.0",
"private": true
}

View file

@ -0,0 +1,166 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import { elasticsearchServiceMock, loggingSystemMock } from '@kbn/core/server/mocks';
import { updateIndices, createIndex, createOrUpdateIndex } from './create_or_update_index';
const logger = loggingSystemMock.createLogger();
const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
esClient.indices.putMapping.mockResolvedValue({ acknowledged: true });
esClient.indices.putSettings.mockResolvedValue({ acknowledged: true });
const simulateIndexTemplateResponse = { template: { mappings: {}, settings: {}, aliases: {} } };
esClient.indices.simulateIndexTemplate.mockResolvedValue(simulateIndexTemplateResponse);
const name = 'test_index_name';
const totalFieldsLimit = 1000;
describe('updateIndices', () => {
beforeEach(() => {
jest.clearAllMocks();
});
it(`should update indices`, async () => {
const indexName = 'test_index_name-default';
esClient.indices.get.mockResolvedValueOnce({ [indexName]: {} });
await updateIndices({
esClient,
logger,
name,
totalFieldsLimit,
});
expect(esClient.indices.get).toHaveBeenCalledWith({
index: name,
expand_wildcards: 'all',
});
expect(esClient.indices.putSettings).toHaveBeenCalledWith({
index: indexName,
body: { 'index.mapping.total_fields.limit': totalFieldsLimit },
});
expect(esClient.indices.simulateIndexTemplate).toHaveBeenCalledWith({
name: indexName,
});
expect(esClient.indices.putMapping).toHaveBeenCalledWith({
index: indexName,
body: simulateIndexTemplateResponse.template.mappings,
});
});
it(`should update multiple indices`, async () => {
const indexName1 = 'test_index_name-1';
const indexName2 = 'test_index_name-2';
esClient.indices.get.mockResolvedValueOnce({ [indexName1]: {}, [indexName2]: {} });
await updateIndices({
esClient,
logger,
name,
totalFieldsLimit,
});
expect(esClient.indices.putSettings).toHaveBeenCalledTimes(2);
expect(esClient.indices.simulateIndexTemplate).toHaveBeenCalledTimes(2);
expect(esClient.indices.putMapping).toHaveBeenCalledTimes(2);
});
it(`should not update indices when not exist`, async () => {
esClient.indices.get.mockResolvedValueOnce({});
await updateIndices({
esClient,
logger,
name,
totalFieldsLimit,
});
expect(esClient.indices.putSettings).not.toHaveBeenCalled();
expect(esClient.indices.simulateIndexTemplate).not.toHaveBeenCalled();
expect(esClient.indices.putMapping).not.toHaveBeenCalled();
});
});
describe('createIndex', () => {
beforeEach(() => {
jest.clearAllMocks();
});
it(`should create index`, async () => {
esClient.indices.exists.mockResolvedValueOnce(false);
await createIndex({
esClient,
logger,
name,
});
expect(esClient.indices.exists).toHaveBeenCalledWith({ index: name, expand_wildcards: 'all' });
expect(esClient.indices.create).toHaveBeenCalledWith({ index: name });
});
it(`should not create index if already exists`, async () => {
esClient.indices.exists.mockResolvedValueOnce(true);
await createIndex({
esClient,
logger,
name,
});
expect(esClient.indices.exists).toHaveBeenCalledWith({ index: name, expand_wildcards: 'all' });
expect(esClient.indices.create).not.toHaveBeenCalled();
});
});
describe('createOrUpdateIndex', () => {
beforeEach(() => {
jest.clearAllMocks();
});
it(`should create index if not exists`, async () => {
esClient.indices.exists.mockResolvedValueOnce(false);
await createOrUpdateIndex({
esClient,
logger,
name,
totalFieldsLimit,
});
expect(esClient.indices.create).toHaveBeenCalledWith({ index: name });
});
it(`should update index if already exists`, async () => {
esClient.indices.exists.mockResolvedValueOnce(true);
await createOrUpdateIndex({
esClient,
logger,
name,
totalFieldsLimit,
});
expect(esClient.indices.exists).toHaveBeenCalledWith({ index: name, expand_wildcards: 'all' });
expect(esClient.indices.putSettings).toHaveBeenCalledWith({
index: name,
body: { 'index.mapping.total_fields.limit': totalFieldsLimit },
});
expect(esClient.indices.simulateIndexTemplate).toHaveBeenCalledWith({
name,
});
expect(esClient.indices.putMapping).toHaveBeenCalledWith({
index: name,
body: simulateIndexTemplateResponse.template.mappings,
});
});
});

View file

@ -0,0 +1,237 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import type { IndexName } from '@elastic/elasticsearch/lib/api/types';
import type { IndicesSimulateIndexTemplateResponse } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import type { Logger, ElasticsearchClient } from '@kbn/core/server';
import { get } from 'lodash';
import { retryTransientEsErrors } from './retry_transient_es_errors';
interface UpdateIndexMappingsOpts {
logger: Logger;
esClient: ElasticsearchClient;
indexNames: string[];
totalFieldsLimit: number;
}
interface UpdateIndexOpts {
logger: Logger;
esClient: ElasticsearchClient;
indexName: string;
totalFieldsLimit: number;
}
const updateTotalFieldLimitSetting = async ({
logger,
esClient,
indexName,
totalFieldsLimit,
}: UpdateIndexOpts) => {
logger.debug(`Updating total field limit setting for ${indexName} data stream.`);
try {
const body = { 'index.mapping.total_fields.limit': totalFieldsLimit };
await retryTransientEsErrors(() => esClient.indices.putSettings({ index: indexName, body }), {
logger,
});
} catch (err) {
logger.error(
`Failed to PUT index.mapping.total_fields.limit settings for ${indexName}: ${err.message}`
);
throw err;
}
};
// This will update the mappings but *not* the settings. This
// is due to the fact settings can be classed as dynamic and static, and static
// updates will fail on an index that isn't closed. New settings *will* be applied as part
// of the ILM policy rollovers. More info: https://github.com/elastic/kibana/pull/113389#issuecomment-940152654
const updateMapping = async ({ logger, esClient, indexName }: UpdateIndexOpts) => {
logger.debug(`Updating mappings for ${indexName} data stream.`);
let simulatedIndexMapping: IndicesSimulateIndexTemplateResponse;
try {
simulatedIndexMapping = await retryTransientEsErrors(
() => esClient.indices.simulateIndexTemplate({ name: indexName }),
{ logger }
);
} catch (err) {
logger.error(
`Ignored PUT mappings for ${indexName}; error generating simulated mappings: ${err.message}`
);
return;
}
const simulatedMapping = get(simulatedIndexMapping, ['template', 'mappings']);
if (simulatedMapping == null) {
logger.error(`Ignored PUT mappings for ${indexName}; simulated mappings were empty`);
return;
}
try {
await retryTransientEsErrors(
() => esClient.indices.putMapping({ index: indexName, body: simulatedMapping }),
{ logger }
);
} catch (err) {
logger.error(`Failed to PUT mapping for ${indexName}: ${err.message}`);
throw err;
}
};
/**
* Updates the data stream mapping and total field limit setting
*/
const updateIndexMappings = async ({
logger,
esClient,
totalFieldsLimit,
indexNames,
}: UpdateIndexMappingsOpts) => {
// Update total field limit setting of found indices
// Other index setting changes are not updated at this time
await Promise.all(
indexNames.map((indexName) =>
updateTotalFieldLimitSetting({ logger, esClient, totalFieldsLimit, indexName })
)
);
// Update mappings of the found indices.
await Promise.all(
indexNames.map((indexName) => updateMapping({ logger, esClient, totalFieldsLimit, indexName }))
);
};
export interface CreateOrUpdateIndexParams {
name: string;
logger: Logger;
esClient: ElasticsearchClient;
totalFieldsLimit: number;
}
export async function createOrUpdateIndex({
logger,
esClient,
name,
totalFieldsLimit,
}: CreateOrUpdateIndexParams): Promise<void> {
logger.info(`Creating index - ${name}`);
// check if index exists
let indexExists = false;
try {
indexExists = await retryTransientEsErrors(
() => esClient.indices.exists({ index: name, expand_wildcards: 'all' }),
{ logger }
);
} catch (error) {
if (error?.statusCode !== 404) {
logger.error(`Error fetching index for ${name} - ${error.message}`);
throw error;
}
}
// if a index exists, update the underlying mapping
if (indexExists) {
await updateIndexMappings({
logger,
esClient,
indexNames: [name],
totalFieldsLimit,
});
} else {
try {
await retryTransientEsErrors(() => esClient.indices.create({ index: name }), { logger });
} catch (error) {
if (error?.meta?.body?.error?.type !== 'resource_already_exists_exception') {
logger.error(`Error creating index ${name} - ${error.message}`);
throw error;
}
}
}
}
export interface CreateIndexParams {
name: string;
logger: Logger;
esClient: ElasticsearchClient;
}
export async function createIndex({ logger, esClient, name }: CreateIndexParams): Promise<void> {
logger.debug(`Checking existence of index - ${name}`);
// check if index exists
let indexExists = false;
try {
indexExists = await retryTransientEsErrors(
() => esClient.indices.exists({ index: name, expand_wildcards: 'all' }),
{
logger,
}
);
} catch (error) {
if (error?.statusCode !== 404) {
logger.error(`Error fetching index for ${name} - ${error.message}`);
throw error;
}
}
// return if index already created
if (indexExists) {
return;
}
logger.info(`Creating index - ${name}`);
try {
await retryTransientEsErrors(() => esClient.indices.create({ index: name }), { logger });
} catch (error) {
if (error?.meta?.body?.error?.type !== 'resource_already_exists_exception') {
logger.error(`Error creating index ${name} - ${error.message}`);
throw error;
}
}
}
export interface CreateOrUpdateSpacesIndexParams {
name: string;
logger: Logger;
esClient: ElasticsearchClient;
totalFieldsLimit: number;
}
export async function updateIndices({
logger,
esClient,
name,
totalFieldsLimit,
}: CreateOrUpdateSpacesIndexParams): Promise<void> {
logger.info(`Updating indices - ${name}`);
// check if data stream exists
let indices: IndexName[] = [];
try {
const response = await retryTransientEsErrors(
() => esClient.indices.get({ index: name, expand_wildcards: 'all' }),
{ logger }
);
indices = Object.keys(response);
} catch (error) {
if (error?.statusCode !== 404) {
logger.error(`Error fetching indices for ${name} - ${error.message}`);
throw error;
}
}
if (indices.length > 0) {
await updateIndexMappings({
logger,
esClient,
totalFieldsLimit,
indexNames: indices,
});
}
}

View file

@ -0,0 +1,158 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import type {
ClusterPutComponentTemplateRequest,
IndicesPutIndexTemplateRequest,
} from '@elastic/elasticsearch/lib/api/types';
import type { Logger, ElasticsearchClient } from '@kbn/core/server';
import type { Subject } from 'rxjs';
import { createOrUpdateComponentTemplate } from './create_or_update_component_template';
import { createOrUpdateIndex } from './create_or_update_index';
import { createOrUpdateIndexTemplate } from './create_or_update_index_template';
import { InstallShutdownError, installWithTimeout } from './install_with_timeout';
import {
getComponentTemplate,
getIndexTemplate,
type GetComponentTemplateOpts,
type GetIndexTemplateOpts,
} from './resource_installer_utils';
export interface IndexAdapterParams {
kibanaVersion: string;
totalFieldsLimit?: number;
}
export type SetComponentTemplateParams = GetComponentTemplateOpts;
export type SetIndexTemplateParams = Omit<
GetIndexTemplateOpts,
'indexPatterns' | 'kibanaVersion' | 'totalFieldsLimit'
>;
export interface GetInstallFnParams {
logger: Logger;
pluginStop$: Subject<void>;
tasksTimeoutMs?: number;
}
export interface InstallParams {
logger: Logger;
esClient: ElasticsearchClient | Promise<ElasticsearchClient>;
pluginStop$: Subject<void>;
tasksTimeoutMs?: number;
}
const DEFAULT_FIELDS_LIMIT = 2500;
export class IndexAdapter {
protected readonly kibanaVersion: string;
protected readonly totalFieldsLimit: number;
protected componentTemplates: ClusterPutComponentTemplateRequest[] = [];
protected indexTemplates: IndicesPutIndexTemplateRequest[] = [];
protected installed: boolean;
constructor(protected readonly name: string, options: IndexAdapterParams) {
this.installed = false;
this.kibanaVersion = options.kibanaVersion;
this.totalFieldsLimit = options.totalFieldsLimit ?? DEFAULT_FIELDS_LIMIT;
}
public setComponentTemplate(params: SetComponentTemplateParams) {
if (this.installed) {
throw new Error('Cannot set component template after install');
}
this.componentTemplates.push(getComponentTemplate(params));
}
public setIndexTemplate(params: SetIndexTemplateParams) {
if (this.installed) {
throw new Error('Cannot set index template after install');
}
this.indexTemplates.push(
getIndexTemplate({
...params,
indexPatterns: [this.name],
kibanaVersion: this.kibanaVersion,
totalFieldsLimit: this.totalFieldsLimit,
})
);
}
protected getInstallFn({ logger, pluginStop$, tasksTimeoutMs }: GetInstallFnParams) {
return async (promise: Promise<void>, description?: string): Promise<void> => {
try {
await installWithTimeout({
installFn: () => promise,
description,
timeoutMs: tasksTimeoutMs,
pluginStop$,
});
} catch (err) {
if (err instanceof InstallShutdownError) {
logger.info(err.message);
} else {
throw err;
}
}
};
}
protected async installTemplates(params: InstallParams) {
const { logger, pluginStop$, tasksTimeoutMs } = params;
const esClient = await params.esClient;
const installFn = this.getInstallFn({ logger, pluginStop$, tasksTimeoutMs });
// Install component templates in parallel
await Promise.all(
this.componentTemplates.map((componentTemplate) =>
installFn(
createOrUpdateComponentTemplate({
template: componentTemplate,
esClient,
logger,
totalFieldsLimit: this.totalFieldsLimit,
}),
`create or update ${componentTemplate.name} component template`
)
)
);
// Install index templates in parallel
await Promise.all(
this.indexTemplates.map((indexTemplate) =>
installFn(
createOrUpdateIndexTemplate({
template: indexTemplate,
esClient,
logger,
}),
`create or update ${indexTemplate.name} index template`
)
)
);
}
public async install(params: InstallParams) {
this.installed = true;
const { logger, pluginStop$, tasksTimeoutMs } = params;
const esClient = await params.esClient;
await this.installTemplates(params);
const installFn = this.getInstallFn({ logger, pluginStop$, tasksTimeoutMs });
// create index when everything is ready
await installFn(
createOrUpdateIndex({
name: this.name,
esClient,
logger,
totalFieldsLimit: this.totalFieldsLimit,
}),
`${this.name} index`
);
}
}

View file

@ -0,0 +1,97 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import { createIndex, updateIndices } from './create_or_update_index';
import { IndexAdapter, type IndexAdapterParams, type InstallParams } from './index_adapter';
export type InstallIndex = (indexSuffix: string) => Promise<void>;
export class IndexPatternAdapter extends IndexAdapter {
protected installationPromises: Map<string, Promise<void>>;
protected installIndexPromise?: Promise<InstallIndex>;
constructor(protected readonly prefix: string, options: IndexAdapterParams) {
super(`${prefix}-*`, options); // make indexTemplate `indexPatterns` match all index names
this.installationPromises = new Map();
}
/** Method to create/update the templates, update existing indices and setup internal state for the adapter. */
public async install(params: InstallParams): Promise<void> {
this.installIndexPromise = this._install(params);
await this.installIndexPromise;
}
protected async _install(params: InstallParams): Promise<InstallIndex> {
const { logger, pluginStop$, tasksTimeoutMs } = params;
await this.installTemplates(params);
const esClient = await params.esClient;
const installFn = this.getInstallFn({ logger, pluginStop$, tasksTimeoutMs });
// Update existing specific indices
await installFn(
updateIndices({
name: this.name, // `${prefix}-*`
esClient,
logger,
totalFieldsLimit: this.totalFieldsLimit,
}),
`update specific indices`
);
// Define the function to create concrete indices on demand
return async (name: string) =>
installFn(createIndex({ name, esClient, logger }), `create ${name} index`);
}
/**
* Method to create the index for a given index suffix.
* Stores the installations promises to avoid concurrent installations for the same index.
* Index creation will only be attempted once per index suffix and existence will be checked before creating.
*/
public async createIndex(indexSuffix: string): Promise<void> {
if (!this.installIndexPromise) {
throw new Error('Cannot installIndex before install');
}
const existingInstallation = this.installationPromises.get(indexSuffix);
if (existingInstallation) {
return existingInstallation;
}
const indexName = this.getIndexName(indexSuffix);
// Awaits for installIndexPromise to resolve to ensure templates are installed before the specific index is created.
// This is a safety measure since the initial `install` call may not be awaited from the plugin lifecycle caller.
// However, the promise will most likely be already fulfilled by the time `createIndex` is called, so this is a no-op.
const installation = this.installIndexPromise
.then((installIndex) => installIndex(indexName))
.catch((err) => {
this.installationPromises.delete(indexSuffix);
throw err;
});
this.installationPromises.set(indexSuffix, installation);
return installation;
}
/** Method to get the full index name for a given index suffix. */
public getIndexName(indexSuffix: string): string {
return `${this.prefix}-${indexSuffix}`;
}
/** Method to get the full index name for a given index suffix. It returns undefined if the index does not exist. */
public async getInstalledIndexName(indexSuffix: string): Promise<string | undefined> {
const existingInstallation = this.installationPromises.get(indexSuffix);
if (!existingInstallation) {
return undefined;
}
return existingInstallation.then(() => this.getIndexName(indexSuffix)).catch(() => undefined);
}
}

View file

@ -24,7 +24,6 @@ describe('getIndexTemplate', () => {
expect(indexTemplate).toEqual({
name: defaultParams.name,
body: {
data_stream: { hidden: true },
index_patterns: defaultParams.indexPatterns,
composed_of: defaultParams.componentTemplateRefs,
template: {
@ -57,8 +56,17 @@ describe('getIndexTemplate', () => {
});
});
it('should create data stream index template with given parameters and defaults', () => {
const indexTemplate = getIndexTemplate({ ...defaultParams, isDataStream: true });
expect(indexTemplate.body).toEqual(
expect.objectContaining({
data_stream: { hidden: true },
})
);
});
it('should create not hidden index template', () => {
const { body } = getIndexTemplate({ ...defaultParams, hidden: false });
const { body } = getIndexTemplate({ ...defaultParams, isDataStream: true, hidden: false });
expect(body?.data_stream?.hidden).toEqual(false);
expect(body?.template?.settings?.hidden).toEqual(false);
});

View file

@ -19,7 +19,7 @@ import type {
import type { FieldMap } from './field_maps/types';
import { mappingFromFieldMap } from './field_maps/mapping_from_field_map';
interface GetComponentTemplateOpts {
export interface GetComponentTemplateOpts {
name: string;
fieldMap: FieldMap;
settings?: IndicesIndexSettings;
@ -47,7 +47,7 @@ export const getComponentTemplate = ({
},
});
interface GetIndexTemplateOpts {
export interface GetIndexTemplateOpts {
name: string;
indexPatterns: string[];
kibanaVersion: string;
@ -56,6 +56,7 @@ interface GetIndexTemplateOpts {
namespace?: string;
template?: IndicesPutIndexTemplateIndexTemplateMapping;
hidden?: boolean;
isDataStream?: boolean;
}
export const getIndexTemplate = ({
@ -67,6 +68,7 @@ export const getIndexTemplate = ({
namespace = 'default',
template = {},
hidden = true,
isDataStream = false,
}: GetIndexTemplateOpts): IndicesPutIndexTemplateRequest => {
const indexMetadata: Metadata = {
kibana: {
@ -79,7 +81,7 @@ export const getIndexTemplate = ({
return {
name,
body: {
data_stream: { hidden },
...(isDataStream && { data_stream: { hidden } }),
index_patterns: indexPatterns,
composed_of: componentTemplateRefs,
template: {

View file

@ -0,0 +1,20 @@
{
"extends": "../../tsconfig.base.json",
"compilerOptions": {
"outDir": "target/types",
"types": [
"jest",
"node",
]
},
"include": ["**/*.ts"],
"kbn_references": [
"@kbn/core",
"@kbn/std",
"@kbn/safer-lodash-set",
"@kbn/logging-mocks",
],
"exclude": [
"target/**/*"
],
}

View file

@ -1034,6 +1034,8 @@
"@kbn/import-locator/*": ["packages/kbn-import-locator/*"],
"@kbn/import-resolver": ["packages/kbn-import-resolver"],
"@kbn/import-resolver/*": ["packages/kbn-import-resolver/*"],
"@kbn/index-adapter": ["packages/kbn-index-adapter"],
"@kbn/index-adapter/*": ["packages/kbn-index-adapter/*"],
"@kbn/index-lifecycle-management-plugin": ["x-pack/plugins/index_lifecycle_management"],
"@kbn/index-lifecycle-management-plugin/*": ["x-pack/plugins/index_lifecycle_management/*"],
"@kbn/index-management-plugin": ["x-pack/plugins/index_management"],

View file

@ -5326,6 +5326,10 @@
version "0.0.0"
uid ""
"@kbn/index-adapter@link:packages/kbn-index-adapter":
version "0.0.0"
uid ""
"@kbn/index-lifecycle-management-plugin@link:x-pack/plugins/index_lifecycle_management":
version "0.0.0"
uid ""