mirror of
https://github.com/elastic/kibana.git
synced 2025-04-23 09:19:04 -04:00
# Backport This will backport the following commits from `main` to `8.x`: - [[IndexAdapter] Extract index-adapter package from data-stream-adapter (#199575)](https://github.com/elastic/kibana/pull/199575) <!--- Backport version: 8.9.8 --> ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) <!--BACKPORT [{"author":{"name":"Sergi Massaneda","email":"sergi.massaneda@elastic.co"},"sourceCommit":{"committedDate":"2024-11-12T17:16:32Z","message":"[IndexAdapter] Extract index-adapter package from data-stream-adapter (#199575)\n\n## Summary\r\n\r\nExtracts `IndexAdapter` from `DataStreamAdapter` and\r\n`IndexPatternAdapter` from `DataStreamSpaceAdapter`.\r\n\r\nThere are no breaking changes for the _data-stream-adapter_ package; the\r\nbehavior of both the `DataStreamAdapter` and `DataStreamSpaceAdapter`\r\nremains unchanged.\r\n\r\nThe new _index-adapter_ package exports `IndexAdapter` and\r\n`IndexPatternAdapter` to manage individual indices without using data\r\nstreams.\r\n\r\nThis is needed for SIEM rule migrations.\r\n\r\n---------\r\n\r\nCo-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>\r\nCo-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>","sha":"9a9f02c9315beda4089b1ef16089747c080bc345","branchLabelMapping":{"^v9.0.0$":"main","^v8.17.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","v9.0.0","Team:Threat Hunting","Team: SecuritySolution","backport:prev-minor","v8.18.0"],"number":199575,"url":"https://github.com/elastic/kibana/pull/199575","mergeCommit":{"message":"[IndexAdapter] Extract index-adapter package from data-stream-adapter (#199575)\n\n## Summary\r\n\r\nExtracts `IndexAdapter` from `DataStreamAdapter` and\r\n`IndexPatternAdapter` from `DataStreamSpaceAdapter`.\r\n\r\nThere are no breaking changes for the _data-stream-adapter_ package; the\r\nbehavior of both the `DataStreamAdapter` and `DataStreamSpaceAdapter`\r\nremains unchanged.\r\n\r\nThe new _index-adapter_ package exports `IndexAdapter` and\r\n`IndexPatternAdapter` to manage individual indices without using data\r\nstreams.\r\n\r\nThis is needed for SIEM rule migrations.\r\n\r\n---------\r\n\r\nCo-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>\r\nCo-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>","sha":"9a9f02c9315beda4089b1ef16089747c080bc345"}},"sourceBranch":"main","suggestedTargetBranches":["8.18"],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","labelRegex":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/199575","number":199575,"mergeCommit":{"message":"[IndexAdapter] Extract index-adapter package from data-stream-adapter (#199575)\n\n## Summary\r\n\r\nExtracts `IndexAdapter` from `DataStreamAdapter` and\r\n`IndexPatternAdapter` from `DataStreamSpaceAdapter`.\r\n\r\nThere are no breaking changes for the _data-stream-adapter_ package; the\r\nbehavior of both the `DataStreamAdapter` and `DataStreamSpaceAdapter`\r\nremains unchanged.\r\n\r\nThe new _index-adapter_ package exports `IndexAdapter` and\r\n`IndexPatternAdapter` to manage individual indices without using data\r\nstreams.\r\n\r\nThis is needed for SIEM rule migrations.\r\n\r\n---------\r\n\r\nCo-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>\r\nCo-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>","sha":"9a9f02c9315beda4089b1ef16089747c080bc345"}},{"branch":"8.18","label":"v8.18.0","labelRegex":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"}]}] BACKPORT-->
This commit is contained in:
parent
7ecc654b10
commit
953bc0feb3
34 changed files with 858 additions and 221 deletions
|
@ -567,6 +567,7 @@
|
|||
"@kbn/i18n-react": "link:packages/kbn-i18n-react",
|
||||
"@kbn/iframe-embedded-plugin": "link:x-pack/test/functional_embedded/plugins/iframe_embedded",
|
||||
"@kbn/image-embeddable-plugin": "link:src/plugins/image_embeddable",
|
||||
"@kbn/index-adapter": "link:packages/kbn-index-adapter",
|
||||
"@kbn/index-lifecycle-management-plugin": "link:x-pack/plugins/index_lifecycle_management",
|
||||
"@kbn/index-management-plugin": "link:x-pack/plugins/index_management",
|
||||
"@kbn/index-management-shared-types": "link:x-pack/packages/index-management/index_management_shared_types",
|
||||
|
|
|
@ -9,13 +9,13 @@
|
|||
|
||||
export { DataStreamAdapter } from './src/data_stream_adapter';
|
||||
export { DataStreamSpacesAdapter } from './src/data_stream_spaces_adapter';
|
||||
export { retryTransientEsErrors } from './src/retry_transient_es_errors';
|
||||
export { ecsFieldMap, type EcsFieldMap } from './src/field_maps/ecs_field_map';
|
||||
|
||||
export { retryTransientEsErrors, ecsFieldMap } from '@kbn/index-adapter';
|
||||
export type {
|
||||
DataStreamAdapterParams,
|
||||
SetComponentTemplateParams,
|
||||
SetIndexTemplateParams,
|
||||
InstallParams,
|
||||
} from './src/data_stream_adapter';
|
||||
export * from './src/field_maps/types';
|
||||
EcsFieldMap,
|
||||
} from '@kbn/index-adapter';
|
||||
|
||||
export * from '@kbn/index-adapter/src/field_maps/types';
|
||||
|
|
|
@ -1,5 +1,6 @@
|
|||
{
|
||||
"type": "shared-common",
|
||||
"type": "shared-server",
|
||||
"id": "@kbn/data-stream-adapter",
|
||||
"owner": "@elastic/security-threat-hunting-explore"
|
||||
"owner": "@elastic/security-threat-hunting",
|
||||
"visibility": "shared"
|
||||
}
|
||||
|
|
|
@ -136,10 +136,11 @@ describe('createOrUpdateDataStream', () => {
|
|||
it(`should create data stream if not exists`, async () => {
|
||||
esClient.indices.getDataStream.mockResolvedValueOnce({ data_streams: [] });
|
||||
|
||||
await createDataStream({
|
||||
await createOrUpdateDataStream({
|
||||
esClient,
|
||||
logger,
|
||||
name,
|
||||
totalFieldsLimit,
|
||||
});
|
||||
|
||||
expect(esClient.indices.createDataStream).toHaveBeenCalledWith({ name });
|
||||
|
|
|
@ -11,7 +11,7 @@ import type { IndicesDataStream } from '@elastic/elasticsearch/lib/api/types';
|
|||
import type { IndicesSimulateIndexTemplateResponse } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
|
||||
import type { Logger, ElasticsearchClient } from '@kbn/core/server';
|
||||
import { get } from 'lodash';
|
||||
import { retryTransientEsErrors } from './retry_transient_es_errors';
|
||||
import { retryTransientEsErrors } from '@kbn/index-adapter';
|
||||
|
||||
interface UpdateIndexMappingsOpts {
|
||||
logger: Logger;
|
||||
|
@ -168,7 +168,7 @@ export async function createDataStream({
|
|||
esClient,
|
||||
name,
|
||||
}: CreateDataStreamParams): Promise<void> {
|
||||
logger.info(`Creating data stream - ${name}`);
|
||||
logger.debug(`Checking data stream exists - ${name}`);
|
||||
|
||||
// check if data stream exists
|
||||
let dataStreamExists = false;
|
||||
|
@ -189,6 +189,7 @@ export async function createDataStream({
|
|||
if (dataStreamExists) {
|
||||
return;
|
||||
}
|
||||
logger.info(`Installing data stream - ${name}`);
|
||||
|
||||
try {
|
||||
await retryTransientEsErrors(() => esClient.indices.createDataStream({ name }), { logger });
|
||||
|
|
|
@ -7,146 +7,23 @@
|
|||
* License v3.0 only", or the "Server Side Public License, v 1".
|
||||
*/
|
||||
|
||||
import type {
|
||||
ClusterPutComponentTemplateRequest,
|
||||
IndicesIndexSettings,
|
||||
IndicesPutIndexTemplateIndexTemplateMapping,
|
||||
IndicesPutIndexTemplateRequest,
|
||||
} from '@elastic/elasticsearch/lib/api/types';
|
||||
import type { Logger, ElasticsearchClient } from '@kbn/core/server';
|
||||
import type { Subject } from 'rxjs';
|
||||
import type { FieldMap } from './field_maps/types';
|
||||
import { createOrUpdateComponentTemplate } from './create_or_update_component_template';
|
||||
import { IndexAdapter, SetIndexTemplateParams, type InstallParams } from '@kbn/index-adapter';
|
||||
import { createOrUpdateDataStream } from './create_or_update_data_stream';
|
||||
import { createOrUpdateIndexTemplate } from './create_or_update_index_template';
|
||||
import { InstallShutdownError, installWithTimeout } from './install_with_timeout';
|
||||
import { getComponentTemplate, getIndexTemplate } from './resource_installer_utils';
|
||||
|
||||
export interface DataStreamAdapterParams {
|
||||
kibanaVersion: string;
|
||||
totalFieldsLimit?: number;
|
||||
}
|
||||
export interface SetComponentTemplateParams {
|
||||
name: string;
|
||||
fieldMap: FieldMap;
|
||||
settings?: IndicesIndexSettings;
|
||||
dynamic?: 'strict' | boolean;
|
||||
}
|
||||
export interface SetIndexTemplateParams {
|
||||
name: string;
|
||||
componentTemplateRefs?: string[];
|
||||
namespace?: string;
|
||||
template?: IndicesPutIndexTemplateIndexTemplateMapping;
|
||||
hidden?: boolean;
|
||||
}
|
||||
|
||||
export interface GetInstallFnParams {
|
||||
logger: Logger;
|
||||
pluginStop$: Subject<void>;
|
||||
tasksTimeoutMs?: number;
|
||||
}
|
||||
export interface InstallParams {
|
||||
logger: Logger;
|
||||
esClient: ElasticsearchClient | Promise<ElasticsearchClient>;
|
||||
pluginStop$: Subject<void>;
|
||||
tasksTimeoutMs?: number;
|
||||
}
|
||||
|
||||
const DEFAULT_FIELDS_LIMIT = 2500;
|
||||
|
||||
export class DataStreamAdapter {
|
||||
protected readonly kibanaVersion: string;
|
||||
protected readonly totalFieldsLimit: number;
|
||||
protected componentTemplates: ClusterPutComponentTemplateRequest[] = [];
|
||||
protected indexTemplates: IndicesPutIndexTemplateRequest[] = [];
|
||||
protected installed: boolean;
|
||||
|
||||
constructor(protected readonly name: string, options: DataStreamAdapterParams) {
|
||||
this.installed = false;
|
||||
this.kibanaVersion = options.kibanaVersion;
|
||||
this.totalFieldsLimit = options.totalFieldsLimit ?? DEFAULT_FIELDS_LIMIT;
|
||||
}
|
||||
|
||||
public setComponentTemplate(params: SetComponentTemplateParams) {
|
||||
if (this.installed) {
|
||||
throw new Error('Cannot set component template after install');
|
||||
}
|
||||
this.componentTemplates.push(getComponentTemplate(params));
|
||||
}
|
||||
|
||||
export class DataStreamAdapter extends IndexAdapter {
|
||||
public setIndexTemplate(params: SetIndexTemplateParams) {
|
||||
if (this.installed) {
|
||||
throw new Error('Cannot set index template after install');
|
||||
}
|
||||
this.indexTemplates.push(
|
||||
getIndexTemplate({
|
||||
...params,
|
||||
indexPatterns: [this.name],
|
||||
kibanaVersion: this.kibanaVersion,
|
||||
totalFieldsLimit: this.totalFieldsLimit,
|
||||
})
|
||||
);
|
||||
super.setIndexTemplate({ ...params, isDataStream: true });
|
||||
}
|
||||
|
||||
protected getInstallFn({ logger, pluginStop$, tasksTimeoutMs }: GetInstallFnParams) {
|
||||
return async (promise: Promise<void>, description?: string): Promise<void> => {
|
||||
try {
|
||||
await installWithTimeout({
|
||||
installFn: () => promise,
|
||||
description,
|
||||
timeoutMs: tasksTimeoutMs,
|
||||
pluginStop$,
|
||||
});
|
||||
} catch (err) {
|
||||
if (err instanceof InstallShutdownError) {
|
||||
logger.info(err.message);
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
public async install({
|
||||
logger,
|
||||
esClient: esClientToResolve,
|
||||
pluginStop$,
|
||||
tasksTimeoutMs,
|
||||
}: InstallParams) {
|
||||
public async install(params: InstallParams) {
|
||||
this.installed = true;
|
||||
const { logger, pluginStop$, tasksTimeoutMs } = params;
|
||||
const esClient = await params.esClient;
|
||||
|
||||
await this.installTemplates(params);
|
||||
|
||||
const esClient = await esClientToResolve;
|
||||
const installFn = this.getInstallFn({ logger, pluginStop$, tasksTimeoutMs });
|
||||
|
||||
// Install component templates in parallel
|
||||
await Promise.all(
|
||||
this.componentTemplates.map((componentTemplate) =>
|
||||
installFn(
|
||||
createOrUpdateComponentTemplate({
|
||||
template: componentTemplate,
|
||||
esClient,
|
||||
logger,
|
||||
totalFieldsLimit: this.totalFieldsLimit,
|
||||
}),
|
||||
`${componentTemplate.name} component template`
|
||||
)
|
||||
)
|
||||
);
|
||||
|
||||
// Install index templates in parallel
|
||||
await Promise.all(
|
||||
this.indexTemplates.map((indexTemplate) =>
|
||||
installFn(
|
||||
createOrUpdateIndexTemplate({
|
||||
template: indexTemplate,
|
||||
esClient,
|
||||
logger,
|
||||
}),
|
||||
`${indexTemplate.name} index template`
|
||||
)
|
||||
)
|
||||
);
|
||||
|
||||
// create data stream when everything is ready
|
||||
await installFn(
|
||||
createOrUpdateDataStream({
|
||||
|
|
|
@ -7,60 +7,27 @@
|
|||
* License v3.0 only", or the "Server Side Public License, v 1".
|
||||
*/
|
||||
|
||||
import { createOrUpdateComponentTemplate } from './create_or_update_component_template';
|
||||
import { createDataStream, updateDataStreams } from './create_or_update_data_stream';
|
||||
import { createOrUpdateIndexTemplate } from './create_or_update_index_template';
|
||||
import {
|
||||
DataStreamAdapter,
|
||||
type DataStreamAdapterParams,
|
||||
IndexPatternAdapter,
|
||||
type SetIndexTemplateParams,
|
||||
type InstallParams,
|
||||
} from './data_stream_adapter';
|
||||
type InstallIndex,
|
||||
} from '@kbn/index-adapter';
|
||||
import { createDataStream, updateDataStreams } from './create_or_update_data_stream';
|
||||
|
||||
export class DataStreamSpacesAdapter extends DataStreamAdapter {
|
||||
private installedSpaceDataStreamName: Map<string, Promise<string>>;
|
||||
private _installSpace?: (spaceId: string) => Promise<string>;
|
||||
|
||||
constructor(private readonly prefix: string, options: DataStreamAdapterParams) {
|
||||
super(`${prefix}-*`, options); // make indexTemplate `indexPatterns` match all data stream space names
|
||||
this.installedSpaceDataStreamName = new Map();
|
||||
export class DataStreamSpacesAdapter extends IndexPatternAdapter {
|
||||
public setIndexTemplate(params: SetIndexTemplateParams) {
|
||||
super.setIndexTemplate({ ...params, isDataStream: true });
|
||||
}
|
||||
|
||||
public async install({
|
||||
logger,
|
||||
esClient: esClientToResolve,
|
||||
pluginStop$,
|
||||
tasksTimeoutMs,
|
||||
}: InstallParams) {
|
||||
this.installed = true;
|
||||
protected async _install(params: InstallParams): Promise<InstallIndex> {
|
||||
const { logger, pluginStop$, tasksTimeoutMs } = params;
|
||||
|
||||
const esClient = await esClientToResolve;
|
||||
await this.installTemplates(params);
|
||||
|
||||
const esClient = await params.esClient;
|
||||
const installFn = this.getInstallFn({ logger, pluginStop$, tasksTimeoutMs });
|
||||
|
||||
// Install component templates in parallel
|
||||
await Promise.all(
|
||||
this.componentTemplates.map((componentTemplate) =>
|
||||
installFn(
|
||||
createOrUpdateComponentTemplate({
|
||||
template: componentTemplate,
|
||||
esClient,
|
||||
logger,
|
||||
totalFieldsLimit: this.totalFieldsLimit,
|
||||
}),
|
||||
`create or update ${componentTemplate.name} component template`
|
||||
)
|
||||
)
|
||||
);
|
||||
|
||||
// Install index templates in parallel
|
||||
await Promise.all(
|
||||
this.indexTemplates.map((indexTemplate) =>
|
||||
installFn(
|
||||
createOrUpdateIndexTemplate({ template: indexTemplate, esClient, logger }),
|
||||
`create or update ${indexTemplate.name} index template`
|
||||
)
|
||||
)
|
||||
);
|
||||
|
||||
// Update existing space data streams
|
||||
await installFn(
|
||||
updateDataStreams({
|
||||
|
@ -72,31 +39,21 @@ export class DataStreamSpacesAdapter extends DataStreamAdapter {
|
|||
`update space data streams`
|
||||
);
|
||||
|
||||
// define function to install data stream for spaces on demand
|
||||
this._installSpace = async (spaceId: string) => {
|
||||
const existingInstallPromise = this.installedSpaceDataStreamName.get(spaceId);
|
||||
if (existingInstallPromise) {
|
||||
return existingInstallPromise;
|
||||
}
|
||||
const name = `${this.prefix}-${spaceId}`;
|
||||
const installPromise = installFn(
|
||||
createDataStream({ name, esClient, logger }),
|
||||
`create ${name} data stream`
|
||||
).then(() => name);
|
||||
|
||||
this.installedSpaceDataStreamName.set(spaceId, installPromise);
|
||||
return installPromise;
|
||||
};
|
||||
// define function to install data stream on demand
|
||||
return async (name: string) =>
|
||||
installFn(createDataStream({ name, esClient, logger }), `create ${name} data stream`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Method to create the data stream for a given space ID.
|
||||
* It resolves with the full data stream name.
|
||||
*/
|
||||
public async installSpace(spaceId: string): Promise<string> {
|
||||
if (!this._installSpace) {
|
||||
throw new Error('Cannot installSpace before install');
|
||||
}
|
||||
return this._installSpace(spaceId);
|
||||
await this.createIndex(spaceId);
|
||||
return this.getIndexName(spaceId);
|
||||
}
|
||||
|
||||
public async getInstalledSpaceName(spaceId: string): Promise<string | undefined> {
|
||||
return this.installedSpaceDataStreamName.get(spaceId);
|
||||
return this.getInstalledIndexName(spaceId);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -5,18 +5,14 @@
|
|||
"types": [
|
||||
"jest",
|
||||
"node",
|
||||
"react",
|
||||
"@emotion/react/types/css-prop",
|
||||
"@testing-library/jest-dom",
|
||||
"@testing-library/react"
|
||||
]
|
||||
},
|
||||
"include": ["**/*.ts", "**/*.tsx"],
|
||||
"include": ["**/*.ts"],
|
||||
"kbn_references": [
|
||||
"@kbn/core",
|
||||
"@kbn/std",
|
||||
"@kbn/safer-lodash-set",
|
||||
"@kbn/logging-mocks",
|
||||
"@kbn/index-adapter",
|
||||
],
|
||||
"exclude": [
|
||||
"target/**/*"
|
||||
],
|
||||
"exclude": ["target/**/*"]
|
||||
}
|
||||
|
|
59
packages/kbn-index-adapter/README.md
Normal file
59
packages/kbn-index-adapter/README.md
Normal file
|
@ -0,0 +1,59 @@
|
|||
# @kbn/index-adapter
|
||||
|
||||
Utility library for Elasticsearch index management.
|
||||
|
||||
## IndexAdapter
|
||||
|
||||
Manage single index. Example:
|
||||
|
||||
```
|
||||
// Setup
|
||||
const indexAdapter = new IndexAdapter('my-awesome-index', { kibanaVersion: '8.12.1' });
|
||||
|
||||
indexAdapter.setComponentTemplate({
|
||||
name: 'awesome-component-template',
|
||||
fieldMap: {
|
||||
'awesome.field1: { type: 'keyword', required: true },
|
||||
'awesome.nested.field2: { type: 'number', required: false },
|
||||
// ...
|
||||
},
|
||||
});
|
||||
|
||||
indexAdapter.setIndexTemplate({
|
||||
name: 'awesome-index-template',
|
||||
componentTemplateRefs: ['awesome-component-template', 'ecs-component-template'],
|
||||
});
|
||||
|
||||
// Start
|
||||
await indexAdapter.install({ logger, esClient, pluginStop$ }); // Installs templates and the 'my-awesome-index' index, or updates existing.
|
||||
```
|
||||
|
||||
|
||||
## IndexPatternAdapter
|
||||
|
||||
Manage index patterns. Example:
|
||||
|
||||
```
|
||||
// Setup
|
||||
const indexPatternAdapter = new IndexPatternAdapter('my-awesome-index', { kibanaVersion: '8.12.1' });
|
||||
|
||||
indexPatternAdapter.setComponentTemplate({
|
||||
name: 'awesome-component-template',
|
||||
fieldMap: {
|
||||
'awesome.field1: { type: 'keyword', required: true },
|
||||
'awesome.nested.field2: { type: 'number', required: false },
|
||||
// ...
|
||||
},
|
||||
});
|
||||
|
||||
indexPatternAdapter.setIndexTemplate({
|
||||
name: 'awesome-index-template',
|
||||
componentTemplateRefs: ['awesome-component-template', 'ecs-component-template'],
|
||||
});
|
||||
|
||||
// Start
|
||||
indexPatternAdapter.install({ logger, esClient, pluginStop$ }); // Installs/updates templates for the index pattern 'my-awesome-index-*', and updates mappings of all specific indices
|
||||
|
||||
// Create a specific index on the fly
|
||||
await indexPatternAdapter.installIndex('12345'); // creates 'my-awesome-index-12345' index if it does not exist.
|
||||
```
|
23
packages/kbn-index-adapter/index.ts
Normal file
23
packages/kbn-index-adapter/index.ts
Normal file
|
@ -0,0 +1,23 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the "Elastic License
|
||||
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
|
||||
* Public License v 1"; you may not use this file except in compliance with, at
|
||||
* your election, the "Elastic License 2.0", the "GNU Affero General Public
|
||||
* License v3.0 only", or the "Server Side Public License, v 1".
|
||||
*/
|
||||
|
||||
export { IndexAdapter } from './src/index_adapter';
|
||||
export { IndexPatternAdapter, type InstallIndex } from './src/index_pattern_adapter';
|
||||
export { retryTransientEsErrors } from './src/retry_transient_es_errors';
|
||||
export { ecsFieldMap, type EcsFieldMap } from './src/field_maps/ecs_field_map';
|
||||
export { createOrUpdateIndexTemplate } from './src/create_or_update_index_template';
|
||||
export { createOrUpdateComponentTemplate } from './src/create_or_update_component_template';
|
||||
|
||||
export type {
|
||||
SetComponentTemplateParams,
|
||||
SetIndexTemplateParams,
|
||||
IndexAdapterParams,
|
||||
InstallParams,
|
||||
} from './src/index_adapter';
|
||||
export * from './src/field_maps/types';
|
14
packages/kbn-index-adapter/jest.config.js
Normal file
14
packages/kbn-index-adapter/jest.config.js
Normal file
|
@ -0,0 +1,14 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the "Elastic License
|
||||
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
|
||||
* Public License v 1"; you may not use this file except in compliance with, at
|
||||
* your election, the "Elastic License 2.0", the "GNU Affero General Public
|
||||
* License v3.0 only", or the "Server Side Public License, v 1".
|
||||
*/
|
||||
|
||||
module.exports = {
|
||||
preset: '@kbn/test',
|
||||
rootDir: '../..',
|
||||
roots: ['<rootDir>/packages/kbn-index-adapter'],
|
||||
};
|
6
packages/kbn-index-adapter/kibana.jsonc
Normal file
6
packages/kbn-index-adapter/kibana.jsonc
Normal file
|
@ -0,0 +1,6 @@
|
|||
{
|
||||
"type": "shared-server",
|
||||
"id": "@kbn/index-adapter",
|
||||
"owner": "@elastic/security-threat-hunting",
|
||||
"visibility": "shared"
|
||||
}
|
7
packages/kbn-index-adapter/package.json
Normal file
7
packages/kbn-index-adapter/package.json
Normal file
|
@ -0,0 +1,7 @@
|
|||
{
|
||||
"name": "@kbn/index-adapter",
|
||||
"version": "1.0.0",
|
||||
"description": "Utility library for Elasticsearch index management",
|
||||
"license": "Elastic License 2.0 OR AGPL-3.0-only OR SSPL-1.0",
|
||||
"private": true
|
||||
}
|
166
packages/kbn-index-adapter/src/create_or_update_index.test.ts
Normal file
166
packages/kbn-index-adapter/src/create_or_update_index.test.ts
Normal file
|
@ -0,0 +1,166 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the "Elastic License
|
||||
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
|
||||
* Public License v 1"; you may not use this file except in compliance with, at
|
||||
* your election, the "Elastic License 2.0", the "GNU Affero General Public
|
||||
* License v3.0 only", or the "Server Side Public License, v 1".
|
||||
*/
|
||||
|
||||
import { elasticsearchServiceMock, loggingSystemMock } from '@kbn/core/server/mocks';
|
||||
import { updateIndices, createIndex, createOrUpdateIndex } from './create_or_update_index';
|
||||
|
||||
const logger = loggingSystemMock.createLogger();
|
||||
const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
|
||||
|
||||
esClient.indices.putMapping.mockResolvedValue({ acknowledged: true });
|
||||
esClient.indices.putSettings.mockResolvedValue({ acknowledged: true });
|
||||
|
||||
const simulateIndexTemplateResponse = { template: { mappings: {}, settings: {}, aliases: {} } };
|
||||
esClient.indices.simulateIndexTemplate.mockResolvedValue(simulateIndexTemplateResponse);
|
||||
|
||||
const name = 'test_index_name';
|
||||
const totalFieldsLimit = 1000;
|
||||
|
||||
describe('updateIndices', () => {
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
it(`should update indices`, async () => {
|
||||
const indexName = 'test_index_name-default';
|
||||
esClient.indices.get.mockResolvedValueOnce({ [indexName]: {} });
|
||||
|
||||
await updateIndices({
|
||||
esClient,
|
||||
logger,
|
||||
name,
|
||||
totalFieldsLimit,
|
||||
});
|
||||
|
||||
expect(esClient.indices.get).toHaveBeenCalledWith({
|
||||
index: name,
|
||||
expand_wildcards: 'all',
|
||||
});
|
||||
|
||||
expect(esClient.indices.putSettings).toHaveBeenCalledWith({
|
||||
index: indexName,
|
||||
body: { 'index.mapping.total_fields.limit': totalFieldsLimit },
|
||||
});
|
||||
expect(esClient.indices.simulateIndexTemplate).toHaveBeenCalledWith({
|
||||
name: indexName,
|
||||
});
|
||||
expect(esClient.indices.putMapping).toHaveBeenCalledWith({
|
||||
index: indexName,
|
||||
body: simulateIndexTemplateResponse.template.mappings,
|
||||
});
|
||||
});
|
||||
|
||||
it(`should update multiple indices`, async () => {
|
||||
const indexName1 = 'test_index_name-1';
|
||||
const indexName2 = 'test_index_name-2';
|
||||
esClient.indices.get.mockResolvedValueOnce({ [indexName1]: {}, [indexName2]: {} });
|
||||
|
||||
await updateIndices({
|
||||
esClient,
|
||||
logger,
|
||||
name,
|
||||
totalFieldsLimit,
|
||||
});
|
||||
|
||||
expect(esClient.indices.putSettings).toHaveBeenCalledTimes(2);
|
||||
expect(esClient.indices.simulateIndexTemplate).toHaveBeenCalledTimes(2);
|
||||
expect(esClient.indices.putMapping).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it(`should not update indices when not exist`, async () => {
|
||||
esClient.indices.get.mockResolvedValueOnce({});
|
||||
|
||||
await updateIndices({
|
||||
esClient,
|
||||
logger,
|
||||
name,
|
||||
totalFieldsLimit,
|
||||
});
|
||||
|
||||
expect(esClient.indices.putSettings).not.toHaveBeenCalled();
|
||||
expect(esClient.indices.simulateIndexTemplate).not.toHaveBeenCalled();
|
||||
expect(esClient.indices.putMapping).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('createIndex', () => {
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
it(`should create index`, async () => {
|
||||
esClient.indices.exists.mockResolvedValueOnce(false);
|
||||
|
||||
await createIndex({
|
||||
esClient,
|
||||
logger,
|
||||
name,
|
||||
});
|
||||
|
||||
expect(esClient.indices.exists).toHaveBeenCalledWith({ index: name, expand_wildcards: 'all' });
|
||||
expect(esClient.indices.create).toHaveBeenCalledWith({ index: name });
|
||||
});
|
||||
|
||||
it(`should not create index if already exists`, async () => {
|
||||
esClient.indices.exists.mockResolvedValueOnce(true);
|
||||
|
||||
await createIndex({
|
||||
esClient,
|
||||
logger,
|
||||
name,
|
||||
});
|
||||
|
||||
expect(esClient.indices.exists).toHaveBeenCalledWith({ index: name, expand_wildcards: 'all' });
|
||||
expect(esClient.indices.create).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('createOrUpdateIndex', () => {
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
it(`should create index if not exists`, async () => {
|
||||
esClient.indices.exists.mockResolvedValueOnce(false);
|
||||
|
||||
await createOrUpdateIndex({
|
||||
esClient,
|
||||
logger,
|
||||
name,
|
||||
totalFieldsLimit,
|
||||
});
|
||||
|
||||
expect(esClient.indices.create).toHaveBeenCalledWith({ index: name });
|
||||
});
|
||||
|
||||
it(`should update index if already exists`, async () => {
|
||||
esClient.indices.exists.mockResolvedValueOnce(true);
|
||||
|
||||
await createOrUpdateIndex({
|
||||
esClient,
|
||||
logger,
|
||||
name,
|
||||
totalFieldsLimit,
|
||||
});
|
||||
|
||||
expect(esClient.indices.exists).toHaveBeenCalledWith({ index: name, expand_wildcards: 'all' });
|
||||
|
||||
expect(esClient.indices.putSettings).toHaveBeenCalledWith({
|
||||
index: name,
|
||||
body: { 'index.mapping.total_fields.limit': totalFieldsLimit },
|
||||
});
|
||||
expect(esClient.indices.simulateIndexTemplate).toHaveBeenCalledWith({
|
||||
name,
|
||||
});
|
||||
expect(esClient.indices.putMapping).toHaveBeenCalledWith({
|
||||
index: name,
|
||||
body: simulateIndexTemplateResponse.template.mappings,
|
||||
});
|
||||
});
|
||||
});
|
237
packages/kbn-index-adapter/src/create_or_update_index.ts
Normal file
237
packages/kbn-index-adapter/src/create_or_update_index.ts
Normal file
|
@ -0,0 +1,237 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the "Elastic License
|
||||
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
|
||||
* Public License v 1"; you may not use this file except in compliance with, at
|
||||
* your election, the "Elastic License 2.0", the "GNU Affero General Public
|
||||
* License v3.0 only", or the "Server Side Public License, v 1".
|
||||
*/
|
||||
|
||||
import type { IndexName } from '@elastic/elasticsearch/lib/api/types';
|
||||
import type { IndicesSimulateIndexTemplateResponse } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
|
||||
import type { Logger, ElasticsearchClient } from '@kbn/core/server';
|
||||
import { get } from 'lodash';
|
||||
import { retryTransientEsErrors } from './retry_transient_es_errors';
|
||||
|
||||
interface UpdateIndexMappingsOpts {
|
||||
logger: Logger;
|
||||
esClient: ElasticsearchClient;
|
||||
indexNames: string[];
|
||||
totalFieldsLimit: number;
|
||||
}
|
||||
|
||||
interface UpdateIndexOpts {
|
||||
logger: Logger;
|
||||
esClient: ElasticsearchClient;
|
||||
indexName: string;
|
||||
totalFieldsLimit: number;
|
||||
}
|
||||
|
||||
const updateTotalFieldLimitSetting = async ({
|
||||
logger,
|
||||
esClient,
|
||||
indexName,
|
||||
totalFieldsLimit,
|
||||
}: UpdateIndexOpts) => {
|
||||
logger.debug(`Updating total field limit setting for ${indexName} data stream.`);
|
||||
|
||||
try {
|
||||
const body = { 'index.mapping.total_fields.limit': totalFieldsLimit };
|
||||
await retryTransientEsErrors(() => esClient.indices.putSettings({ index: indexName, body }), {
|
||||
logger,
|
||||
});
|
||||
} catch (err) {
|
||||
logger.error(
|
||||
`Failed to PUT index.mapping.total_fields.limit settings for ${indexName}: ${err.message}`
|
||||
);
|
||||
throw err;
|
||||
}
|
||||
};
|
||||
|
||||
// This will update the mappings but *not* the settings. This
|
||||
// is due to the fact settings can be classed as dynamic and static, and static
|
||||
// updates will fail on an index that isn't closed. New settings *will* be applied as part
|
||||
// of the ILM policy rollovers. More info: https://github.com/elastic/kibana/pull/113389#issuecomment-940152654
|
||||
const updateMapping = async ({ logger, esClient, indexName }: UpdateIndexOpts) => {
|
||||
logger.debug(`Updating mappings for ${indexName} data stream.`);
|
||||
|
||||
let simulatedIndexMapping: IndicesSimulateIndexTemplateResponse;
|
||||
try {
|
||||
simulatedIndexMapping = await retryTransientEsErrors(
|
||||
() => esClient.indices.simulateIndexTemplate({ name: indexName }),
|
||||
{ logger }
|
||||
);
|
||||
} catch (err) {
|
||||
logger.error(
|
||||
`Ignored PUT mappings for ${indexName}; error generating simulated mappings: ${err.message}`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const simulatedMapping = get(simulatedIndexMapping, ['template', 'mappings']);
|
||||
|
||||
if (simulatedMapping == null) {
|
||||
logger.error(`Ignored PUT mappings for ${indexName}; simulated mappings were empty`);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await retryTransientEsErrors(
|
||||
() => esClient.indices.putMapping({ index: indexName, body: simulatedMapping }),
|
||||
{ logger }
|
||||
);
|
||||
} catch (err) {
|
||||
logger.error(`Failed to PUT mapping for ${indexName}: ${err.message}`);
|
||||
throw err;
|
||||
}
|
||||
};
|
||||
/**
|
||||
* Updates the data stream mapping and total field limit setting
|
||||
*/
|
||||
const updateIndexMappings = async ({
|
||||
logger,
|
||||
esClient,
|
||||
totalFieldsLimit,
|
||||
indexNames,
|
||||
}: UpdateIndexMappingsOpts) => {
|
||||
// Update total field limit setting of found indices
|
||||
// Other index setting changes are not updated at this time
|
||||
await Promise.all(
|
||||
indexNames.map((indexName) =>
|
||||
updateTotalFieldLimitSetting({ logger, esClient, totalFieldsLimit, indexName })
|
||||
)
|
||||
);
|
||||
// Update mappings of the found indices.
|
||||
await Promise.all(
|
||||
indexNames.map((indexName) => updateMapping({ logger, esClient, totalFieldsLimit, indexName }))
|
||||
);
|
||||
};
|
||||
|
||||
export interface CreateOrUpdateIndexParams {
|
||||
name: string;
|
||||
logger: Logger;
|
||||
esClient: ElasticsearchClient;
|
||||
totalFieldsLimit: number;
|
||||
}
|
||||
|
||||
export async function createOrUpdateIndex({
|
||||
logger,
|
||||
esClient,
|
||||
name,
|
||||
totalFieldsLimit,
|
||||
}: CreateOrUpdateIndexParams): Promise<void> {
|
||||
logger.info(`Creating index - ${name}`);
|
||||
|
||||
// check if index exists
|
||||
let indexExists = false;
|
||||
try {
|
||||
indexExists = await retryTransientEsErrors(
|
||||
() => esClient.indices.exists({ index: name, expand_wildcards: 'all' }),
|
||||
{ logger }
|
||||
);
|
||||
} catch (error) {
|
||||
if (error?.statusCode !== 404) {
|
||||
logger.error(`Error fetching index for ${name} - ${error.message}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// if a index exists, update the underlying mapping
|
||||
if (indexExists) {
|
||||
await updateIndexMappings({
|
||||
logger,
|
||||
esClient,
|
||||
indexNames: [name],
|
||||
totalFieldsLimit,
|
||||
});
|
||||
} else {
|
||||
try {
|
||||
await retryTransientEsErrors(() => esClient.indices.create({ index: name }), { logger });
|
||||
} catch (error) {
|
||||
if (error?.meta?.body?.error?.type !== 'resource_already_exists_exception') {
|
||||
logger.error(`Error creating index ${name} - ${error.message}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export interface CreateIndexParams {
|
||||
name: string;
|
||||
logger: Logger;
|
||||
esClient: ElasticsearchClient;
|
||||
}
|
||||
|
||||
export async function createIndex({ logger, esClient, name }: CreateIndexParams): Promise<void> {
|
||||
logger.debug(`Checking existence of index - ${name}`);
|
||||
|
||||
// check if index exists
|
||||
let indexExists = false;
|
||||
try {
|
||||
indexExists = await retryTransientEsErrors(
|
||||
() => esClient.indices.exists({ index: name, expand_wildcards: 'all' }),
|
||||
{
|
||||
logger,
|
||||
}
|
||||
);
|
||||
} catch (error) {
|
||||
if (error?.statusCode !== 404) {
|
||||
logger.error(`Error fetching index for ${name} - ${error.message}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// return if index already created
|
||||
if (indexExists) {
|
||||
return;
|
||||
}
|
||||
|
||||
logger.info(`Creating index - ${name}`);
|
||||
try {
|
||||
await retryTransientEsErrors(() => esClient.indices.create({ index: name }), { logger });
|
||||
} catch (error) {
|
||||
if (error?.meta?.body?.error?.type !== 'resource_already_exists_exception') {
|
||||
logger.error(`Error creating index ${name} - ${error.message}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export interface CreateOrUpdateSpacesIndexParams {
|
||||
name: string;
|
||||
logger: Logger;
|
||||
esClient: ElasticsearchClient;
|
||||
totalFieldsLimit: number;
|
||||
}
|
||||
|
||||
export async function updateIndices({
|
||||
logger,
|
||||
esClient,
|
||||
name,
|
||||
totalFieldsLimit,
|
||||
}: CreateOrUpdateSpacesIndexParams): Promise<void> {
|
||||
logger.info(`Updating indices - ${name}`);
|
||||
|
||||
// check if data stream exists
|
||||
let indices: IndexName[] = [];
|
||||
try {
|
||||
const response = await retryTransientEsErrors(
|
||||
() => esClient.indices.get({ index: name, expand_wildcards: 'all' }),
|
||||
{ logger }
|
||||
);
|
||||
indices = Object.keys(response);
|
||||
} catch (error) {
|
||||
if (error?.statusCode !== 404) {
|
||||
logger.error(`Error fetching indices for ${name} - ${error.message}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
if (indices.length > 0) {
|
||||
await updateIndexMappings({
|
||||
logger,
|
||||
esClient,
|
||||
totalFieldsLimit,
|
||||
indexNames: indices,
|
||||
});
|
||||
}
|
||||
}
|
158
packages/kbn-index-adapter/src/index_adapter.ts
Normal file
158
packages/kbn-index-adapter/src/index_adapter.ts
Normal file
|
@ -0,0 +1,158 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the "Elastic License
|
||||
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
|
||||
* Public License v 1"; you may not use this file except in compliance with, at
|
||||
* your election, the "Elastic License 2.0", the "GNU Affero General Public
|
||||
* License v3.0 only", or the "Server Side Public License, v 1".
|
||||
*/
|
||||
|
||||
import type {
|
||||
ClusterPutComponentTemplateRequest,
|
||||
IndicesPutIndexTemplateRequest,
|
||||
} from '@elastic/elasticsearch/lib/api/types';
|
||||
import type { Logger, ElasticsearchClient } from '@kbn/core/server';
|
||||
import type { Subject } from 'rxjs';
|
||||
import { createOrUpdateComponentTemplate } from './create_or_update_component_template';
|
||||
import { createOrUpdateIndex } from './create_or_update_index';
|
||||
import { createOrUpdateIndexTemplate } from './create_or_update_index_template';
|
||||
import { InstallShutdownError, installWithTimeout } from './install_with_timeout';
|
||||
import {
|
||||
getComponentTemplate,
|
||||
getIndexTemplate,
|
||||
type GetComponentTemplateOpts,
|
||||
type GetIndexTemplateOpts,
|
||||
} from './resource_installer_utils';
|
||||
|
||||
export interface IndexAdapterParams {
|
||||
kibanaVersion: string;
|
||||
totalFieldsLimit?: number;
|
||||
}
|
||||
export type SetComponentTemplateParams = GetComponentTemplateOpts;
|
||||
export type SetIndexTemplateParams = Omit<
|
||||
GetIndexTemplateOpts,
|
||||
'indexPatterns' | 'kibanaVersion' | 'totalFieldsLimit'
|
||||
>;
|
||||
export interface GetInstallFnParams {
|
||||
logger: Logger;
|
||||
pluginStop$: Subject<void>;
|
||||
tasksTimeoutMs?: number;
|
||||
}
|
||||
export interface InstallParams {
|
||||
logger: Logger;
|
||||
esClient: ElasticsearchClient | Promise<ElasticsearchClient>;
|
||||
pluginStop$: Subject<void>;
|
||||
tasksTimeoutMs?: number;
|
||||
}
|
||||
|
||||
const DEFAULT_FIELDS_LIMIT = 2500;
|
||||
|
||||
export class IndexAdapter {
|
||||
protected readonly kibanaVersion: string;
|
||||
protected readonly totalFieldsLimit: number;
|
||||
protected componentTemplates: ClusterPutComponentTemplateRequest[] = [];
|
||||
protected indexTemplates: IndicesPutIndexTemplateRequest[] = [];
|
||||
protected installed: boolean;
|
||||
|
||||
constructor(protected readonly name: string, options: IndexAdapterParams) {
|
||||
this.installed = false;
|
||||
this.kibanaVersion = options.kibanaVersion;
|
||||
this.totalFieldsLimit = options.totalFieldsLimit ?? DEFAULT_FIELDS_LIMIT;
|
||||
}
|
||||
|
||||
public setComponentTemplate(params: SetComponentTemplateParams) {
|
||||
if (this.installed) {
|
||||
throw new Error('Cannot set component template after install');
|
||||
}
|
||||
this.componentTemplates.push(getComponentTemplate(params));
|
||||
}
|
||||
|
||||
public setIndexTemplate(params: SetIndexTemplateParams) {
|
||||
if (this.installed) {
|
||||
throw new Error('Cannot set index template after install');
|
||||
}
|
||||
this.indexTemplates.push(
|
||||
getIndexTemplate({
|
||||
...params,
|
||||
indexPatterns: [this.name],
|
||||
kibanaVersion: this.kibanaVersion,
|
||||
totalFieldsLimit: this.totalFieldsLimit,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
protected getInstallFn({ logger, pluginStop$, tasksTimeoutMs }: GetInstallFnParams) {
|
||||
return async (promise: Promise<void>, description?: string): Promise<void> => {
|
||||
try {
|
||||
await installWithTimeout({
|
||||
installFn: () => promise,
|
||||
description,
|
||||
timeoutMs: tasksTimeoutMs,
|
||||
pluginStop$,
|
||||
});
|
||||
} catch (err) {
|
||||
if (err instanceof InstallShutdownError) {
|
||||
logger.info(err.message);
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
protected async installTemplates(params: InstallParams) {
|
||||
const { logger, pluginStop$, tasksTimeoutMs } = params;
|
||||
const esClient = await params.esClient;
|
||||
const installFn = this.getInstallFn({ logger, pluginStop$, tasksTimeoutMs });
|
||||
|
||||
// Install component templates in parallel
|
||||
await Promise.all(
|
||||
this.componentTemplates.map((componentTemplate) =>
|
||||
installFn(
|
||||
createOrUpdateComponentTemplate({
|
||||
template: componentTemplate,
|
||||
esClient,
|
||||
logger,
|
||||
totalFieldsLimit: this.totalFieldsLimit,
|
||||
}),
|
||||
`create or update ${componentTemplate.name} component template`
|
||||
)
|
||||
)
|
||||
);
|
||||
|
||||
// Install index templates in parallel
|
||||
await Promise.all(
|
||||
this.indexTemplates.map((indexTemplate) =>
|
||||
installFn(
|
||||
createOrUpdateIndexTemplate({
|
||||
template: indexTemplate,
|
||||
esClient,
|
||||
logger,
|
||||
}),
|
||||
`create or update ${indexTemplate.name} index template`
|
||||
)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
public async install(params: InstallParams) {
|
||||
this.installed = true;
|
||||
const { logger, pluginStop$, tasksTimeoutMs } = params;
|
||||
const esClient = await params.esClient;
|
||||
|
||||
await this.installTemplates(params);
|
||||
|
||||
const installFn = this.getInstallFn({ logger, pluginStop$, tasksTimeoutMs });
|
||||
|
||||
// create index when everything is ready
|
||||
await installFn(
|
||||
createOrUpdateIndex({
|
||||
name: this.name,
|
||||
esClient,
|
||||
logger,
|
||||
totalFieldsLimit: this.totalFieldsLimit,
|
||||
}),
|
||||
`${this.name} index`
|
||||
);
|
||||
}
|
||||
}
|
97
packages/kbn-index-adapter/src/index_pattern_adapter.ts
Normal file
97
packages/kbn-index-adapter/src/index_pattern_adapter.ts
Normal file
|
@ -0,0 +1,97 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the "Elastic License
|
||||
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
|
||||
* Public License v 1"; you may not use this file except in compliance with, at
|
||||
* your election, the "Elastic License 2.0", the "GNU Affero General Public
|
||||
* License v3.0 only", or the "Server Side Public License, v 1".
|
||||
*/
|
||||
|
||||
import { createIndex, updateIndices } from './create_or_update_index';
|
||||
import { IndexAdapter, type IndexAdapterParams, type InstallParams } from './index_adapter';
|
||||
|
||||
export type InstallIndex = (indexSuffix: string) => Promise<void>;
|
||||
|
||||
export class IndexPatternAdapter extends IndexAdapter {
|
||||
protected installationPromises: Map<string, Promise<void>>;
|
||||
protected installIndexPromise?: Promise<InstallIndex>;
|
||||
|
||||
constructor(protected readonly prefix: string, options: IndexAdapterParams) {
|
||||
super(`${prefix}-*`, options); // make indexTemplate `indexPatterns` match all index names
|
||||
this.installationPromises = new Map();
|
||||
}
|
||||
|
||||
/** Method to create/update the templates, update existing indices and setup internal state for the adapter. */
|
||||
public async install(params: InstallParams): Promise<void> {
|
||||
this.installIndexPromise = this._install(params);
|
||||
await this.installIndexPromise;
|
||||
}
|
||||
|
||||
protected async _install(params: InstallParams): Promise<InstallIndex> {
|
||||
const { logger, pluginStop$, tasksTimeoutMs } = params;
|
||||
|
||||
await this.installTemplates(params);
|
||||
|
||||
const esClient = await params.esClient;
|
||||
const installFn = this.getInstallFn({ logger, pluginStop$, tasksTimeoutMs });
|
||||
|
||||
// Update existing specific indices
|
||||
await installFn(
|
||||
updateIndices({
|
||||
name: this.name, // `${prefix}-*`
|
||||
esClient,
|
||||
logger,
|
||||
totalFieldsLimit: this.totalFieldsLimit,
|
||||
}),
|
||||
`update specific indices`
|
||||
);
|
||||
|
||||
// Define the function to create concrete indices on demand
|
||||
return async (name: string) =>
|
||||
installFn(createIndex({ name, esClient, logger }), `create ${name} index`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Method to create the index for a given index suffix.
|
||||
* Stores the installations promises to avoid concurrent installations for the same index.
|
||||
* Index creation will only be attempted once per index suffix and existence will be checked before creating.
|
||||
*/
|
||||
public async createIndex(indexSuffix: string): Promise<void> {
|
||||
if (!this.installIndexPromise) {
|
||||
throw new Error('Cannot installIndex before install');
|
||||
}
|
||||
|
||||
const existingInstallation = this.installationPromises.get(indexSuffix);
|
||||
if (existingInstallation) {
|
||||
return existingInstallation;
|
||||
}
|
||||
const indexName = this.getIndexName(indexSuffix);
|
||||
|
||||
// Awaits for installIndexPromise to resolve to ensure templates are installed before the specific index is created.
|
||||
// This is a safety measure since the initial `install` call may not be awaited from the plugin lifecycle caller.
|
||||
// However, the promise will most likely be already fulfilled by the time `createIndex` is called, so this is a no-op.
|
||||
const installation = this.installIndexPromise
|
||||
.then((installIndex) => installIndex(indexName))
|
||||
.catch((err) => {
|
||||
this.installationPromises.delete(indexSuffix);
|
||||
throw err;
|
||||
});
|
||||
|
||||
this.installationPromises.set(indexSuffix, installation);
|
||||
return installation;
|
||||
}
|
||||
|
||||
/** Method to get the full index name for a given index suffix. */
|
||||
public getIndexName(indexSuffix: string): string {
|
||||
return `${this.prefix}-${indexSuffix}`;
|
||||
}
|
||||
|
||||
/** Method to get the full index name for a given index suffix. It returns undefined if the index does not exist. */
|
||||
public async getInstalledIndexName(indexSuffix: string): Promise<string | undefined> {
|
||||
const existingInstallation = this.installationPromises.get(indexSuffix);
|
||||
if (!existingInstallation) {
|
||||
return undefined;
|
||||
}
|
||||
return existingInstallation.then(() => this.getIndexName(indexSuffix)).catch(() => undefined);
|
||||
}
|
||||
}
|
|
@ -24,7 +24,6 @@ describe('getIndexTemplate', () => {
|
|||
expect(indexTemplate).toEqual({
|
||||
name: defaultParams.name,
|
||||
body: {
|
||||
data_stream: { hidden: true },
|
||||
index_patterns: defaultParams.indexPatterns,
|
||||
composed_of: defaultParams.componentTemplateRefs,
|
||||
template: {
|
||||
|
@ -57,8 +56,17 @@ describe('getIndexTemplate', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('should create data stream index template with given parameters and defaults', () => {
|
||||
const indexTemplate = getIndexTemplate({ ...defaultParams, isDataStream: true });
|
||||
expect(indexTemplate.body).toEqual(
|
||||
expect.objectContaining({
|
||||
data_stream: { hidden: true },
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it('should create not hidden index template', () => {
|
||||
const { body } = getIndexTemplate({ ...defaultParams, hidden: false });
|
||||
const { body } = getIndexTemplate({ ...defaultParams, isDataStream: true, hidden: false });
|
||||
expect(body?.data_stream?.hidden).toEqual(false);
|
||||
expect(body?.template?.settings?.hidden).toEqual(false);
|
||||
});
|
|
@ -19,7 +19,7 @@ import type {
|
|||
import type { FieldMap } from './field_maps/types';
|
||||
import { mappingFromFieldMap } from './field_maps/mapping_from_field_map';
|
||||
|
||||
interface GetComponentTemplateOpts {
|
||||
export interface GetComponentTemplateOpts {
|
||||
name: string;
|
||||
fieldMap: FieldMap;
|
||||
settings?: IndicesIndexSettings;
|
||||
|
@ -47,7 +47,7 @@ export const getComponentTemplate = ({
|
|||
},
|
||||
});
|
||||
|
||||
interface GetIndexTemplateOpts {
|
||||
export interface GetIndexTemplateOpts {
|
||||
name: string;
|
||||
indexPatterns: string[];
|
||||
kibanaVersion: string;
|
||||
|
@ -56,6 +56,7 @@ interface GetIndexTemplateOpts {
|
|||
namespace?: string;
|
||||
template?: IndicesPutIndexTemplateIndexTemplateMapping;
|
||||
hidden?: boolean;
|
||||
isDataStream?: boolean;
|
||||
}
|
||||
|
||||
export const getIndexTemplate = ({
|
||||
|
@ -67,6 +68,7 @@ export const getIndexTemplate = ({
|
|||
namespace = 'default',
|
||||
template = {},
|
||||
hidden = true,
|
||||
isDataStream = false,
|
||||
}: GetIndexTemplateOpts): IndicesPutIndexTemplateRequest => {
|
||||
const indexMetadata: Metadata = {
|
||||
kibana: {
|
||||
|
@ -79,7 +81,7 @@ export const getIndexTemplate = ({
|
|||
return {
|
||||
name,
|
||||
body: {
|
||||
data_stream: { hidden },
|
||||
...(isDataStream && { data_stream: { hidden } }),
|
||||
index_patterns: indexPatterns,
|
||||
composed_of: componentTemplateRefs,
|
||||
template: {
|
20
packages/kbn-index-adapter/tsconfig.json
Normal file
20
packages/kbn-index-adapter/tsconfig.json
Normal file
|
@ -0,0 +1,20 @@
|
|||
{
|
||||
"extends": "../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"outDir": "target/types",
|
||||
"types": [
|
||||
"jest",
|
||||
"node",
|
||||
]
|
||||
},
|
||||
"include": ["**/*.ts"],
|
||||
"kbn_references": [
|
||||
"@kbn/core",
|
||||
"@kbn/std",
|
||||
"@kbn/safer-lodash-set",
|
||||
"@kbn/logging-mocks",
|
||||
],
|
||||
"exclude": [
|
||||
"target/**/*"
|
||||
],
|
||||
}
|
|
@ -1034,6 +1034,8 @@
|
|||
"@kbn/import-locator/*": ["packages/kbn-import-locator/*"],
|
||||
"@kbn/import-resolver": ["packages/kbn-import-resolver"],
|
||||
"@kbn/import-resolver/*": ["packages/kbn-import-resolver/*"],
|
||||
"@kbn/index-adapter": ["packages/kbn-index-adapter"],
|
||||
"@kbn/index-adapter/*": ["packages/kbn-index-adapter/*"],
|
||||
"@kbn/index-lifecycle-management-plugin": ["x-pack/plugins/index_lifecycle_management"],
|
||||
"@kbn/index-lifecycle-management-plugin/*": ["x-pack/plugins/index_lifecycle_management/*"],
|
||||
"@kbn/index-management-plugin": ["x-pack/plugins/index_management"],
|
||||
|
|
|
@ -5326,6 +5326,10 @@
|
|||
version "0.0.0"
|
||||
uid ""
|
||||
|
||||
"@kbn/index-adapter@link:packages/kbn-index-adapter":
|
||||
version "0.0.0"
|
||||
uid ""
|
||||
|
||||
"@kbn/index-lifecycle-management-plugin@link:x-pack/plugins/index_lifecycle_management":
|
||||
version "0.0.0"
|
||||
uid ""
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue