mirror of
https://github.com/elastic/kibana.git
synced 2025-04-23 17:28:26 -04:00
[Security Solution] Data quality dashboard persistence (#173185)
## Summary issue https://github.com/elastic/security-team/issues/7382 ### Data Stream Adapter This PR introduces the `@kbn/data-stream-adapter` package, which is a utility library to facilitate Data Stream creation and maintenance in Kibana, it was inspired by the data stream implementation in the Alerts plugin. The library has two exports: - `DataStreamSpacesAdapter`: to manage space data streams. It uses the `name-of-the-data-stream-<spaceId>` naming pattern. - `DataStreamAdapter`: to manage single (not space-aware) data streams. Usage examples in the package [README](450be0369d/packages/kbn-data-stream-adapter/README.md) ### Data Quality Dashboard The `DataStreamSpacesAdapter` has been integrated into the data quality dashboard to store all the quality checks users perform. The information stored is the metadata (also used for telemetry) and the actual data rendered in the tables. FieldMap definition [here](450be0369d/x-pack/plugins/ecs_data_quality_dashboard/server/lib/data_stream/results_field_map.ts) ### Demo311a0bf5-004b-46d7-8140-52a233361c91 --------- Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Philippe Oberti <philippe.oberti@elastic.co> Co-authored-by: Garrett Spong <spong@users.noreply.github.com> Co-authored-by: Efe Gürkan YALAMAN <efeguerkan.yalaman@elastic.co> Co-authored-by: Tiago Costa <tiago.costa@elastic.co> Co-authored-by: Sander Philipse <94373878+sphilipse@users.noreply.github.com> Co-authored-by: JD Kurma <JDKurma@gmail.com> Co-authored-by: Jan Monschke <jan.monschke@elastic.co> Co-authored-by: Patryk Kopyciński <contact@patrykkopycinski.com> Co-authored-by: Khristinin Nikita <nikita.khristinin@elastic.co> Co-authored-by: Marco Liberati <dej611@users.noreply.github.com> Co-authored-by: Julia Rechkunova <julia.rechkunova@elastic.co> Co-authored-by: Stratoula Kalafateli <efstratia.kalafateli@elastic.co> Co-authored-by: Davis McPhee <davis.mcphee@elastic.co> Co-authored-by: Eyo O. Eyo <7893459+eokoneyo@users.noreply.github.com> Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com> Co-authored-by: Marta Bondyra <4283304+mbondyra@users.noreply.github.com> Co-authored-by: Søren Louv-Jansen <soren.louv@elastic.co> Co-authored-by: Dzmitry Lemechko <dzmitry.lemechko@elastic.co> Co-authored-by: Candace Park <56409205+parkiino@users.noreply.github.com>
This commit is contained in:
Sergi Massaneda
GitHub
parent
256d12e0a5
commit
a63bb6add0
86 changed files with 4229 additions and 121 deletions
|
|
@ -1121,6 +1121,7 @@ module.exports = {
|
|||
'x-pack/plugins/security_solution_serverless/**/*.{js,mjs,ts,tsx}',
|
||||
'x-pack/plugins/timelines/**/*.{js,mjs,ts,tsx}',
|
||||
'x-pack/plugins/cases/**/*.{js,mjs,ts,tsx}',
|
||||
'packages/kbn-data-stream-adapter/**/*.{js,mjs,ts,tsx}',
|
||||
],
|
||||
plugins: ['eslint-plugin-node', 'react'],
|
||||
env: {
|
||||
|
|
@ -1218,6 +1219,8 @@ module.exports = {
|
|||
'x-pack/plugins/security_solution_ess/**/*.{js,mjs,ts,tsx}',
|
||||
'x-pack/plugins/security_solution_serverless/**/*.{js,mjs,ts,tsx}',
|
||||
'x-pack/plugins/cases/**/*.{js,mjs,ts,tsx}',
|
||||
'x-pack/plugins/ecs_data_quality_dashboard/**/*.{js,mjs,ts,tsx}',
|
||||
'packages/kbn-data-stream-adapter/**/*.{js,mjs,ts,tsx}',
|
||||
],
|
||||
rules: {
|
||||
'@typescript-eslint/consistent-type-imports': 'error',
|
||||
|
|
|
|||
1
.github/CODEOWNERS
vendored
1
.github/CODEOWNERS
vendored
|
|
@ -319,6 +319,7 @@ x-pack/packages/kbn-data-forge @elastic/obs-ux-management-team
|
|||
src/plugins/data @elastic/kibana-visualizations @elastic/kibana-data-discovery
|
||||
test/plugin_functional/plugins/data_search @elastic/kibana-data-discovery
|
||||
packages/kbn-data-service @elastic/kibana-visualizations @elastic/kibana-data-discovery
|
||||
packages/kbn-data-stream-adapter @elastic/security-threat-hunting-explore
|
||||
src/plugins/data_view_editor @elastic/kibana-data-discovery
|
||||
examples/data_view_field_editor_example @elastic/kibana-data-discovery
|
||||
src/plugins/data_view_field_editor @elastic/kibana-data-discovery
|
||||
|
|
|
|||
|
|
@ -369,6 +369,7 @@
|
|||
"@kbn/data-plugin": "link:src/plugins/data",
|
||||
"@kbn/data-search-plugin": "link:test/plugin_functional/plugins/data_search",
|
||||
"@kbn/data-service": "link:packages/kbn-data-service",
|
||||
"@kbn/data-stream-adapter": "link:packages/kbn-data-stream-adapter",
|
||||
"@kbn/data-view-editor-plugin": "link:src/plugins/data_view_editor",
|
||||
"@kbn/data-view-field-editor-example-plugin": "link:examples/data_view_field_editor_example",
|
||||
"@kbn/data-view-field-editor-plugin": "link:src/plugins/data_view_field_editor",
|
||||
|
|
|
|||
69
packages/kbn-data-stream-adapter/README.md
Normal file
69
packages/kbn-data-stream-adapter/README.md
Normal file
|
|
@ -0,0 +1,69 @@
|
|||
# @kbn/data-stream-adapter
|
||||
|
||||
Utility library for Elasticsearch data stream management.
|
||||
|
||||
## DataStreamAdapter
|
||||
|
||||
Manage single data streams. Example:
|
||||
|
||||
```
|
||||
// Setup
|
||||
const dataStream = new DataStreamAdapter('my-awesome-datastream', { kibanaVersion: '8.12.1' });
|
||||
|
||||
dataStream.setComponentTemplate({
|
||||
name: 'awesome-component-template',
|
||||
fieldMap: {
|
||||
'awesome.field1: { type: 'keyword', required: true },
|
||||
'awesome.nested.field2: { type: 'number', required: false },
|
||||
// ...
|
||||
},
|
||||
});
|
||||
|
||||
dataStream.setIndexTemplate({
|
||||
name: 'awesome-index-template',
|
||||
componentTemplateRefs: ['awesome-component-template', 'ecs-component-template'],
|
||||
template: {
|
||||
lifecycle: {
|
||||
data_retention: '5d',
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
// Start
|
||||
await dataStream.install({ logger, esClient, pluginStop$ }); // Installs templates and the data stream, or updates existing.
|
||||
```
|
||||
|
||||
|
||||
## DataStreamSpacesAdapter
|
||||
|
||||
Manage data streams per space. Example:
|
||||
|
||||
```
|
||||
// Setup
|
||||
const spacesDataStream = new DataStreamSpacesAdapter('my-awesome-datastream', { kibanaVersion: '8.12.1' });
|
||||
|
||||
spacesDataStream.setComponentTemplate({
|
||||
name: 'awesome-component-template',
|
||||
fieldMap: {
|
||||
'awesome.field1: { type: 'keyword', required: true },
|
||||
'awesome.nested.field2: { type: 'number', required: false },
|
||||
// ...
|
||||
},
|
||||
});
|
||||
|
||||
spacesDataStream.setIndexTemplate({
|
||||
name: 'awesome-index-template',
|
||||
componentTemplateRefs: ['awesome-component-template', 'ecs-component-template'],
|
||||
template: {
|
||||
lifecycle: {
|
||||
data_retention: '5d',
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
// Start
|
||||
await spacesDataStream.install({ logger, esClient, pluginStop$ }); // Installs templates and updates existing data streams.
|
||||
|
||||
// Create a space data stream on the fly
|
||||
await spacesDataStream.installSpace('space2'); // creates 'my-awesome-datastream-space2' data stream if it does not exist.
|
||||
```
|
||||
20
packages/kbn-data-stream-adapter/index.ts
Normal file
20
packages/kbn-data-stream-adapter/index.ts
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
export { DataStreamAdapter } from './src/data_stream_adapter';
|
||||
export { DataStreamSpacesAdapter } from './src/data_stream_spaces_adapter';
|
||||
export { retryTransientEsErrors } from './src/retry_transient_es_errors';
|
||||
export { ecsFieldMap, type EcsFieldMap } from './src/field_maps/ecs_field_map';
|
||||
|
||||
export type {
|
||||
DataStreamAdapterParams,
|
||||
SetComponentTemplateParams,
|
||||
SetIndexTemplateParams,
|
||||
InstallParams,
|
||||
} from './src/data_stream_adapter';
|
||||
export * from './src/field_maps/types';
|
||||
13
packages/kbn-data-stream-adapter/jest.config.js
Normal file
13
packages/kbn-data-stream-adapter/jest.config.js
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
module.exports = {
|
||||
preset: '@kbn/test',
|
||||
rootDir: '../..',
|
||||
roots: ['<rootDir>/packages/kbn-data-stream-adapter'],
|
||||
};
|
||||
5
packages/kbn-data-stream-adapter/kibana.jsonc
Normal file
5
packages/kbn-data-stream-adapter/kibana.jsonc
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"type": "shared-common",
|
||||
"id": "@kbn/data-stream-adapter",
|
||||
"owner": "@elastic/security-threat-hunting-explore"
|
||||
}
|
||||
7
packages/kbn-data-stream-adapter/package.json
Normal file
7
packages/kbn-data-stream-adapter/package.json
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
{
|
||||
"name": "@kbn/data-stream-adapter",
|
||||
"version": "1.0.0",
|
||||
"description": "Utility library for Elasticsearch Data Stream management",
|
||||
"license": "SSPL-1.0 OR Elastic License 2.0",
|
||||
"private": true
|
||||
}
|
||||
|
|
@ -0,0 +1,290 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import { elasticsearchServiceMock, loggingSystemMock } from '@kbn/core/server/mocks';
|
||||
import type { DiagnosticResult } from '@elastic/elasticsearch';
|
||||
import { errors as EsErrors } from '@elastic/elasticsearch';
|
||||
import { createOrUpdateComponentTemplate } from './create_or_update_component_template';
|
||||
|
||||
const randomDelayMultiplier = 0.01;
|
||||
const logger = loggingSystemMock.createLogger();
|
||||
const clusterClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
|
||||
|
||||
const componentTemplate = {
|
||||
name: 'test-mappings',
|
||||
_meta: {
|
||||
managed: true,
|
||||
},
|
||||
template: {
|
||||
settings: {
|
||||
number_of_shards: 1,
|
||||
'index.mapping.total_fields.limit': 1500,
|
||||
},
|
||||
mappings: {
|
||||
dynamic: false,
|
||||
properties: {
|
||||
foo: {
|
||||
ignore_above: 1024,
|
||||
type: 'keyword',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
describe('createOrUpdateComponentTemplate', () => {
|
||||
beforeEach(() => {
|
||||
jest.resetAllMocks();
|
||||
jest.spyOn(global.Math, 'random').mockReturnValue(randomDelayMultiplier);
|
||||
});
|
||||
|
||||
it(`should call esClient to put component template`, async () => {
|
||||
await createOrUpdateComponentTemplate({
|
||||
logger,
|
||||
esClient: clusterClient,
|
||||
template: componentTemplate,
|
||||
totalFieldsLimit: 2500,
|
||||
});
|
||||
|
||||
expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledWith(componentTemplate);
|
||||
});
|
||||
|
||||
it(`should retry on transient ES errors`, async () => {
|
||||
clusterClient.cluster.putComponentTemplate
|
||||
.mockRejectedValueOnce(new EsErrors.ConnectionError('foo'))
|
||||
.mockRejectedValueOnce(new EsErrors.TimeoutError('timeout'))
|
||||
.mockResolvedValue({ acknowledged: true });
|
||||
await createOrUpdateComponentTemplate({
|
||||
logger,
|
||||
esClient: clusterClient,
|
||||
template: componentTemplate,
|
||||
totalFieldsLimit: 2500,
|
||||
});
|
||||
|
||||
expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
it(`should log and throw error if max retries exceeded`, async () => {
|
||||
clusterClient.cluster.putComponentTemplate.mockRejectedValue(
|
||||
new EsErrors.ConnectionError('foo')
|
||||
);
|
||||
await expect(() =>
|
||||
createOrUpdateComponentTemplate({
|
||||
logger,
|
||||
esClient: clusterClient,
|
||||
template: componentTemplate,
|
||||
totalFieldsLimit: 2500,
|
||||
})
|
||||
).rejects.toThrowErrorMatchingInlineSnapshot(`"foo"`);
|
||||
|
||||
expect(logger.error).toHaveBeenCalledWith(
|
||||
`Error installing component template test-mappings - foo`
|
||||
);
|
||||
expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4);
|
||||
});
|
||||
|
||||
it(`should log and throw error if ES throws error`, async () => {
|
||||
clusterClient.cluster.putComponentTemplate.mockRejectedValue(new Error('generic error'));
|
||||
|
||||
await expect(() =>
|
||||
createOrUpdateComponentTemplate({
|
||||
logger,
|
||||
esClient: clusterClient,
|
||||
template: componentTemplate,
|
||||
totalFieldsLimit: 2500,
|
||||
})
|
||||
).rejects.toThrowErrorMatchingInlineSnapshot(`"generic error"`);
|
||||
|
||||
expect(logger.error).toHaveBeenCalledWith(
|
||||
`Error installing component template test-mappings - generic error`
|
||||
);
|
||||
});
|
||||
|
||||
it(`should update index template field limit and retry if putTemplate throws error with field limit error`, async () => {
|
||||
clusterClient.cluster.putComponentTemplate.mockRejectedValueOnce(
|
||||
new EsErrors.ResponseError({
|
||||
body: 'illegal_argument_exception: Limit of total fields [1800] has been exceeded',
|
||||
} as DiagnosticResult)
|
||||
);
|
||||
const existingIndexTemplate = {
|
||||
name: 'test-template',
|
||||
index_template: {
|
||||
index_patterns: ['test*'],
|
||||
composed_of: ['test-mappings'],
|
||||
template: {
|
||||
settings: {
|
||||
auto_expand_replicas: '0-1',
|
||||
hidden: true,
|
||||
'index.lifecycle': {
|
||||
name: '.alerts-ilm-policy',
|
||||
rollover_alias: `.alerts-empty-default`,
|
||||
},
|
||||
'index.mapping.total_fields.limit': 1800,
|
||||
},
|
||||
mappings: {
|
||||
dynamic: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
clusterClient.indices.getIndexTemplate.mockResolvedValueOnce({
|
||||
index_templates: [existingIndexTemplate],
|
||||
});
|
||||
|
||||
await createOrUpdateComponentTemplate({
|
||||
logger,
|
||||
esClient: clusterClient,
|
||||
template: componentTemplate,
|
||||
totalFieldsLimit: 2500,
|
||||
});
|
||||
|
||||
expect(logger.info).toHaveBeenCalledWith('Updating total_fields.limit from 1800 to 2500');
|
||||
expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(2);
|
||||
expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledTimes(1);
|
||||
expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledWith({
|
||||
name: existingIndexTemplate.name,
|
||||
body: {
|
||||
...existingIndexTemplate.index_template,
|
||||
template: {
|
||||
...existingIndexTemplate.index_template.template,
|
||||
settings: {
|
||||
...existingIndexTemplate.index_template.template?.settings,
|
||||
'index.mapping.total_fields.limit': 2500,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it(`should update index template field limit and retry if putTemplate throws error with field limit error when there are malformed index templates`, async () => {
|
||||
clusterClient.cluster.putComponentTemplate.mockRejectedValueOnce(
|
||||
new EsErrors.ResponseError({
|
||||
body: 'illegal_argument_exception: Limit of total fields [1800] has been exceeded',
|
||||
} as DiagnosticResult)
|
||||
);
|
||||
const existingIndexTemplate = {
|
||||
name: 'test-template',
|
||||
index_template: {
|
||||
index_patterns: ['test*'],
|
||||
composed_of: ['test-mappings'],
|
||||
template: {
|
||||
settings: {
|
||||
auto_expand_replicas: '0-1',
|
||||
hidden: true,
|
||||
'index.lifecycle': {
|
||||
name: '.alerts-ilm-policy',
|
||||
rollover_alias: `.alerts-empty-default`,
|
||||
},
|
||||
'index.mapping.total_fields.limit': 1800,
|
||||
},
|
||||
mappings: {
|
||||
dynamic: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
clusterClient.indices.getIndexTemplate.mockResolvedValueOnce({
|
||||
index_templates: [
|
||||
existingIndexTemplate,
|
||||
{
|
||||
name: 'lyndon',
|
||||
// @ts-expect-error
|
||||
index_template: {
|
||||
index_patterns: ['intel*'],
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'sample_ds',
|
||||
// @ts-expect-error
|
||||
index_template: {
|
||||
index_patterns: ['sample_ds-*'],
|
||||
data_stream: {
|
||||
hidden: false,
|
||||
allow_custom_routing: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
await createOrUpdateComponentTemplate({
|
||||
logger,
|
||||
esClient: clusterClient,
|
||||
template: componentTemplate,
|
||||
totalFieldsLimit: 2500,
|
||||
});
|
||||
|
||||
expect(logger.info).toHaveBeenCalledWith('Updating total_fields.limit from 1800 to 2500');
|
||||
expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(2);
|
||||
expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledTimes(1);
|
||||
expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledWith({
|
||||
name: existingIndexTemplate.name,
|
||||
body: {
|
||||
...existingIndexTemplate.index_template,
|
||||
template: {
|
||||
...existingIndexTemplate.index_template.template,
|
||||
settings: {
|
||||
...existingIndexTemplate.index_template.template?.settings,
|
||||
'index.mapping.total_fields.limit': 2500,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it(`should retry getIndexTemplate and putIndexTemplate on transient ES errors`, async () => {
|
||||
clusterClient.cluster.putComponentTemplate.mockRejectedValueOnce(
|
||||
new EsErrors.ResponseError({
|
||||
body: 'illegal_argument_exception: Limit of total fields [1800] has been exceeded',
|
||||
} as DiagnosticResult)
|
||||
);
|
||||
const existingIndexTemplate = {
|
||||
name: 'test-template',
|
||||
index_template: {
|
||||
index_patterns: ['test*'],
|
||||
composed_of: ['test-mappings'],
|
||||
template: {
|
||||
settings: {
|
||||
auto_expand_replicas: '0-1',
|
||||
hidden: true,
|
||||
'index.lifecycle': {
|
||||
name: '.alerts-ilm-policy',
|
||||
rollover_alias: `.alerts-empty-default`,
|
||||
},
|
||||
'index.mapping.total_fields.limit': 1800,
|
||||
},
|
||||
mappings: {
|
||||
dynamic: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
clusterClient.indices.getIndexTemplate
|
||||
.mockRejectedValueOnce(new EsErrors.ConnectionError('foo'))
|
||||
.mockRejectedValueOnce(new EsErrors.TimeoutError('timeout'))
|
||||
.mockResolvedValueOnce({
|
||||
index_templates: [existingIndexTemplate],
|
||||
});
|
||||
clusterClient.indices.putIndexTemplate
|
||||
.mockRejectedValueOnce(new EsErrors.ConnectionError('foo'))
|
||||
.mockRejectedValueOnce(new EsErrors.TimeoutError('timeout'))
|
||||
.mockResolvedValue({ acknowledged: true });
|
||||
await createOrUpdateComponentTemplate({
|
||||
logger,
|
||||
esClient: clusterClient,
|
||||
template: componentTemplate,
|
||||
totalFieldsLimit: 2500,
|
||||
});
|
||||
|
||||
expect(logger.info).toHaveBeenCalledWith('Updating total_fields.limit from 1800 to 2500');
|
||||
expect(clusterClient.indices.getIndexTemplate).toHaveBeenCalledTimes(3);
|
||||
expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,118 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import type {
|
||||
ClusterPutComponentTemplateRequest,
|
||||
IndicesGetIndexTemplateIndexTemplateItem,
|
||||
} from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
|
||||
import type { Logger, ElasticsearchClient } from '@kbn/core/server';
|
||||
import { asyncForEach } from '@kbn/std';
|
||||
import { retryTransientEsErrors } from './retry_transient_es_errors';
|
||||
|
||||
interface CreateOrUpdateComponentTemplateOpts {
|
||||
logger: Logger;
|
||||
esClient: ElasticsearchClient;
|
||||
template: ClusterPutComponentTemplateRequest;
|
||||
totalFieldsLimit: number;
|
||||
}
|
||||
|
||||
const putIndexTemplateTotalFieldsLimitUsingComponentTemplate = async (
|
||||
esClient: ElasticsearchClient,
|
||||
componentTemplateName: string,
|
||||
totalFieldsLimit: number,
|
||||
logger: Logger
|
||||
) => {
|
||||
// Get all index templates and filter down to just the ones referencing this component template
|
||||
const { index_templates: indexTemplates } = await retryTransientEsErrors(
|
||||
() => esClient.indices.getIndexTemplate(),
|
||||
{ logger }
|
||||
);
|
||||
const indexTemplatesUsingComponentTemplate = (indexTemplates ?? []).filter(
|
||||
(indexTemplate: IndicesGetIndexTemplateIndexTemplateItem) =>
|
||||
(indexTemplate.index_template?.composed_of ?? []).includes(componentTemplateName)
|
||||
);
|
||||
|
||||
await asyncForEach(
|
||||
indexTemplatesUsingComponentTemplate,
|
||||
async (template: IndicesGetIndexTemplateIndexTemplateItem) => {
|
||||
await retryTransientEsErrors(
|
||||
() =>
|
||||
esClient.indices.putIndexTemplate({
|
||||
name: template.name,
|
||||
body: {
|
||||
...template.index_template,
|
||||
template: {
|
||||
...template.index_template.template,
|
||||
settings: {
|
||||
...template.index_template.template?.settings,
|
||||
'index.mapping.total_fields.limit': totalFieldsLimit,
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
{ logger }
|
||||
);
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
const createOrUpdateComponentTemplateHelper = async (
|
||||
esClient: ElasticsearchClient,
|
||||
template: ClusterPutComponentTemplateRequest,
|
||||
totalFieldsLimit: number,
|
||||
logger: Logger
|
||||
) => {
|
||||
try {
|
||||
await retryTransientEsErrors(() => esClient.cluster.putComponentTemplate(template), { logger });
|
||||
} catch (error) {
|
||||
const limitErrorMatch = error.message.match(
|
||||
/Limit of total fields \[(\d+)\] has been exceeded/
|
||||
);
|
||||
if (limitErrorMatch != null) {
|
||||
// This error message occurs when there is an index template using this component template
|
||||
// that contains a field limit setting that using this component template exceeds
|
||||
// Specifically, this can happen for the ECS component template when we add new fields
|
||||
// to adhere to the ECS spec. Individual index templates specify field limits so if the
|
||||
// number of new ECS fields pushes the composed mapping above the limit, this error will
|
||||
// occur. We have to update the field limit inside the index template now otherwise we
|
||||
// can never update the component template
|
||||
|
||||
logger.info(`Updating total_fields.limit from ${limitErrorMatch[1]} to ${totalFieldsLimit}`);
|
||||
|
||||
await putIndexTemplateTotalFieldsLimitUsingComponentTemplate(
|
||||
esClient,
|
||||
template.name,
|
||||
totalFieldsLimit,
|
||||
logger
|
||||
);
|
||||
|
||||
// Try to update the component template again
|
||||
await retryTransientEsErrors(() => esClient.cluster.putComponentTemplate(template), {
|
||||
logger,
|
||||
});
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
export const createOrUpdateComponentTemplate = async ({
|
||||
logger,
|
||||
esClient,
|
||||
template,
|
||||
totalFieldsLimit,
|
||||
}: CreateOrUpdateComponentTemplateOpts) => {
|
||||
logger.info(`Installing component template ${template.name}`);
|
||||
|
||||
try {
|
||||
await createOrUpdateComponentTemplateHelper(esClient, template, totalFieldsLimit, logger);
|
||||
} catch (err) {
|
||||
logger.error(`Error installing component template ${template.name} - ${err.message}`);
|
||||
throw err;
|
||||
}
|
||||
};
|
||||
|
|
@ -0,0 +1,172 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import type { IndicesDataStream } from '@elastic/elasticsearch/lib/api/types';
|
||||
import { elasticsearchServiceMock, loggingSystemMock } from '@kbn/core/server/mocks';
|
||||
import {
|
||||
updateDataStreams,
|
||||
createDataStream,
|
||||
createOrUpdateDataStream,
|
||||
} from './create_or_update_data_stream';
|
||||
|
||||
const logger = loggingSystemMock.createLogger();
|
||||
const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
|
||||
|
||||
esClient.indices.putMapping.mockResolvedValue({ acknowledged: true });
|
||||
esClient.indices.putSettings.mockResolvedValue({ acknowledged: true });
|
||||
|
||||
const simulateIndexTemplateResponse = { template: { mappings: { is_managed: true } } };
|
||||
esClient.indices.simulateIndexTemplate.mockResolvedValue(simulateIndexTemplateResponse);
|
||||
|
||||
const name = 'test_data_stream';
|
||||
const totalFieldsLimit = 1000;
|
||||
|
||||
describe('updateDataStreams', () => {
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
it(`should update data streams`, async () => {
|
||||
const dataStreamName = 'test_data_stream-default';
|
||||
esClient.indices.getDataStream.mockResolvedValueOnce({
|
||||
data_streams: [{ name: dataStreamName } as IndicesDataStream],
|
||||
});
|
||||
|
||||
await updateDataStreams({
|
||||
esClient,
|
||||
logger,
|
||||
name,
|
||||
totalFieldsLimit,
|
||||
});
|
||||
|
||||
expect(esClient.indices.getDataStream).toHaveBeenCalledWith({ name, expand_wildcards: 'all' });
|
||||
|
||||
expect(esClient.indices.putSettings).toHaveBeenCalledWith({
|
||||
index: dataStreamName,
|
||||
body: { 'index.mapping.total_fields.limit': totalFieldsLimit },
|
||||
});
|
||||
expect(esClient.indices.simulateIndexTemplate).toHaveBeenCalledWith({
|
||||
name: dataStreamName,
|
||||
});
|
||||
expect(esClient.indices.putMapping).toHaveBeenCalledWith({
|
||||
index: dataStreamName,
|
||||
body: simulateIndexTemplateResponse.template.mappings,
|
||||
});
|
||||
});
|
||||
|
||||
it(`should update multiple data streams`, async () => {
|
||||
const dataStreamName1 = 'test_data_stream-1';
|
||||
const dataStreamName2 = 'test_data_stream-2';
|
||||
esClient.indices.getDataStream.mockResolvedValueOnce({
|
||||
data_streams: [{ name: dataStreamName1 }, { name: dataStreamName2 }] as IndicesDataStream[],
|
||||
});
|
||||
|
||||
await updateDataStreams({
|
||||
esClient,
|
||||
logger,
|
||||
name,
|
||||
totalFieldsLimit,
|
||||
});
|
||||
|
||||
expect(esClient.indices.putSettings).toHaveBeenCalledTimes(2);
|
||||
expect(esClient.indices.simulateIndexTemplate).toHaveBeenCalledTimes(2);
|
||||
expect(esClient.indices.putMapping).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it(`should not update data streams when not exist`, async () => {
|
||||
esClient.indices.getDataStream.mockResolvedValueOnce({ data_streams: [] });
|
||||
|
||||
await updateDataStreams({
|
||||
esClient,
|
||||
logger,
|
||||
name,
|
||||
totalFieldsLimit,
|
||||
});
|
||||
|
||||
expect(esClient.indices.putSettings).not.toHaveBeenCalled();
|
||||
expect(esClient.indices.simulateIndexTemplate).not.toHaveBeenCalled();
|
||||
expect(esClient.indices.putMapping).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('createDataStream', () => {
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
it(`should create data stream`, async () => {
|
||||
esClient.indices.getDataStream.mockResolvedValueOnce({ data_streams: [] });
|
||||
|
||||
await createDataStream({
|
||||
esClient,
|
||||
logger,
|
||||
name,
|
||||
});
|
||||
|
||||
expect(esClient.indices.createDataStream).toHaveBeenCalledWith({ name });
|
||||
});
|
||||
|
||||
it(`should not create data stream if already exists`, async () => {
|
||||
esClient.indices.getDataStream.mockResolvedValueOnce({
|
||||
data_streams: [{ name: 'test_data_stream-default' } as IndicesDataStream],
|
||||
});
|
||||
|
||||
await createDataStream({
|
||||
esClient,
|
||||
logger,
|
||||
name,
|
||||
});
|
||||
|
||||
expect(esClient.indices.createDataStream).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('createOrUpdateDataStream', () => {
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
it(`should create data stream if not exists`, async () => {
|
||||
esClient.indices.getDataStream.mockResolvedValueOnce({ data_streams: [] });
|
||||
|
||||
await createDataStream({
|
||||
esClient,
|
||||
logger,
|
||||
name,
|
||||
});
|
||||
|
||||
expect(esClient.indices.createDataStream).toHaveBeenCalledWith({ name });
|
||||
});
|
||||
|
||||
it(`should update data stream if already exists`, async () => {
|
||||
esClient.indices.getDataStream.mockResolvedValueOnce({
|
||||
data_streams: [{ name: 'test_data_stream-default' } as IndicesDataStream],
|
||||
});
|
||||
|
||||
await createOrUpdateDataStream({
|
||||
esClient,
|
||||
logger,
|
||||
name,
|
||||
totalFieldsLimit,
|
||||
});
|
||||
|
||||
expect(esClient.indices.getDataStream).toHaveBeenCalledWith({ name, expand_wildcards: 'all' });
|
||||
|
||||
expect(esClient.indices.putSettings).toHaveBeenCalledWith({
|
||||
index: name,
|
||||
body: { 'index.mapping.total_fields.limit': totalFieldsLimit },
|
||||
});
|
||||
expect(esClient.indices.simulateIndexTemplate).toHaveBeenCalledWith({
|
||||
name,
|
||||
});
|
||||
expect(esClient.indices.putMapping).toHaveBeenCalledWith({
|
||||
index: name,
|
||||
body: simulateIndexTemplateResponse.template.mappings,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,239 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import type { IndicesDataStream } from '@elastic/elasticsearch/lib/api/types';
|
||||
import type { IndicesSimulateIndexTemplateResponse } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
|
||||
import type { Logger, ElasticsearchClient } from '@kbn/core/server';
|
||||
import { get } from 'lodash';
|
||||
import { retryTransientEsErrors } from './retry_transient_es_errors';
|
||||
|
||||
interface UpdateIndexMappingsOpts {
|
||||
logger: Logger;
|
||||
esClient: ElasticsearchClient;
|
||||
indexNames: string[];
|
||||
totalFieldsLimit: number;
|
||||
}
|
||||
|
||||
interface UpdateIndexOpts {
|
||||
logger: Logger;
|
||||
esClient: ElasticsearchClient;
|
||||
indexName: string;
|
||||
totalFieldsLimit: number;
|
||||
}
|
||||
|
||||
const updateTotalFieldLimitSetting = async ({
|
||||
logger,
|
||||
esClient,
|
||||
indexName,
|
||||
totalFieldsLimit,
|
||||
}: UpdateIndexOpts) => {
|
||||
logger.debug(`Updating total field limit setting for ${indexName} data stream.`);
|
||||
|
||||
try {
|
||||
const body = { 'index.mapping.total_fields.limit': totalFieldsLimit };
|
||||
await retryTransientEsErrors(() => esClient.indices.putSettings({ index: indexName, body }), {
|
||||
logger,
|
||||
});
|
||||
} catch (err) {
|
||||
logger.error(
|
||||
`Failed to PUT index.mapping.total_fields.limit settings for ${indexName}: ${err.message}`
|
||||
);
|
||||
throw err;
|
||||
}
|
||||
};
|
||||
|
||||
// This will update the mappings but *not* the settings. This
|
||||
// is due to the fact settings can be classed as dynamic and static, and static
|
||||
// updates will fail on an index that isn't closed. New settings *will* be applied as part
|
||||
// of the ILM policy rollovers. More info: https://github.com/elastic/kibana/pull/113389#issuecomment-940152654
|
||||
const updateMapping = async ({ logger, esClient, indexName }: UpdateIndexOpts) => {
|
||||
logger.debug(`Updating mappings for ${indexName} data stream.`);
|
||||
|
||||
let simulatedIndexMapping: IndicesSimulateIndexTemplateResponse;
|
||||
try {
|
||||
simulatedIndexMapping = await retryTransientEsErrors(
|
||||
() => esClient.indices.simulateIndexTemplate({ name: indexName }),
|
||||
{ logger }
|
||||
);
|
||||
} catch (err) {
|
||||
logger.error(
|
||||
`Ignored PUT mappings for ${indexName}; error generating simulated mappings: ${err.message}`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const simulatedMapping = get(simulatedIndexMapping, ['template', 'mappings']);
|
||||
|
||||
if (simulatedMapping == null) {
|
||||
logger.error(`Ignored PUT mappings for ${indexName}; simulated mappings were empty`);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await retryTransientEsErrors(
|
||||
() => esClient.indices.putMapping({ index: indexName, body: simulatedMapping }),
|
||||
{ logger }
|
||||
);
|
||||
} catch (err) {
|
||||
logger.error(`Failed to PUT mapping for ${indexName}: ${err.message}`);
|
||||
throw err;
|
||||
}
|
||||
};
|
||||
/**
|
||||
* Updates the data stream mapping and total field limit setting
|
||||
*/
|
||||
const updateDataStreamMappings = async ({
|
||||
logger,
|
||||
esClient,
|
||||
totalFieldsLimit,
|
||||
indexNames,
|
||||
}: UpdateIndexMappingsOpts) => {
|
||||
// Update total field limit setting of found indices
|
||||
// Other index setting changes are not updated at this time
|
||||
await Promise.all(
|
||||
indexNames.map((indexName) =>
|
||||
updateTotalFieldLimitSetting({ logger, esClient, totalFieldsLimit, indexName })
|
||||
)
|
||||
);
|
||||
// Update mappings of the found indices.
|
||||
await Promise.all(
|
||||
indexNames.map((indexName) => updateMapping({ logger, esClient, totalFieldsLimit, indexName }))
|
||||
);
|
||||
};
|
||||
|
||||
export interface CreateOrUpdateDataStreamParams {
|
||||
name: string;
|
||||
logger: Logger;
|
||||
esClient: ElasticsearchClient;
|
||||
totalFieldsLimit: number;
|
||||
}
|
||||
|
||||
export async function createOrUpdateDataStream({
|
||||
logger,
|
||||
esClient,
|
||||
name,
|
||||
totalFieldsLimit,
|
||||
}: CreateOrUpdateDataStreamParams): Promise<void> {
|
||||
logger.info(`Creating data stream - ${name}`);
|
||||
|
||||
// check if data stream exists
|
||||
let dataStreamExists = false;
|
||||
try {
|
||||
const response = await retryTransientEsErrors(
|
||||
() => esClient.indices.getDataStream({ name, expand_wildcards: 'all' }),
|
||||
{ logger }
|
||||
);
|
||||
dataStreamExists = response.data_streams.length > 0;
|
||||
} catch (error) {
|
||||
if (error?.statusCode !== 404) {
|
||||
logger.error(`Error fetching data stream for ${name} - ${error.message}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// if a data stream exists, update the underlying mapping
|
||||
if (dataStreamExists) {
|
||||
await updateDataStreamMappings({
|
||||
logger,
|
||||
esClient,
|
||||
indexNames: [name],
|
||||
totalFieldsLimit,
|
||||
});
|
||||
} else {
|
||||
try {
|
||||
await retryTransientEsErrors(() => esClient.indices.createDataStream({ name }), { logger });
|
||||
} catch (error) {
|
||||
if (error?.meta?.body?.error?.type !== 'resource_already_exists_exception') {
|
||||
logger.error(`Error creating data stream ${name} - ${error.message}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export interface CreateDataStreamParams {
|
||||
name: string;
|
||||
logger: Logger;
|
||||
esClient: ElasticsearchClient;
|
||||
}
|
||||
|
||||
export async function createDataStream({
|
||||
logger,
|
||||
esClient,
|
||||
name,
|
||||
}: CreateDataStreamParams): Promise<void> {
|
||||
logger.info(`Creating data stream - ${name}`);
|
||||
|
||||
// check if data stream exists
|
||||
let dataStreamExists = false;
|
||||
try {
|
||||
const response = await retryTransientEsErrors(
|
||||
() => esClient.indices.getDataStream({ name, expand_wildcards: 'all' }),
|
||||
{ logger }
|
||||
);
|
||||
dataStreamExists = response.data_streams.length > 0;
|
||||
} catch (error) {
|
||||
if (error?.statusCode !== 404) {
|
||||
logger.error(`Error fetching data stream for ${name} - ${error.message}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// return if data stream already created
|
||||
if (dataStreamExists) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await retryTransientEsErrors(() => esClient.indices.createDataStream({ name }), { logger });
|
||||
} catch (error) {
|
||||
if (error?.meta?.body?.error?.type !== 'resource_already_exists_exception') {
|
||||
logger.error(`Error creating data stream ${name} - ${error.message}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export interface CreateOrUpdateSpacesDataStreamParams {
|
||||
name: string;
|
||||
logger: Logger;
|
||||
esClient: ElasticsearchClient;
|
||||
totalFieldsLimit: number;
|
||||
}
|
||||
|
||||
export async function updateDataStreams({
|
||||
logger,
|
||||
esClient,
|
||||
name,
|
||||
totalFieldsLimit,
|
||||
}: CreateOrUpdateSpacesDataStreamParams): Promise<void> {
|
||||
logger.info(`Updating data streams - ${name}`);
|
||||
|
||||
// check if data stream exists
|
||||
let dataStreams: IndicesDataStream[] = [];
|
||||
try {
|
||||
const response = await retryTransientEsErrors(
|
||||
() => esClient.indices.getDataStream({ name, expand_wildcards: 'all' }),
|
||||
{ logger }
|
||||
);
|
||||
dataStreams = response.data_streams;
|
||||
} catch (error) {
|
||||
if (error?.statusCode !== 404) {
|
||||
logger.error(`Error fetching data stream for ${name} - ${error.message}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
if (dataStreams.length > 0) {
|
||||
await updateDataStreamMappings({
|
||||
logger,
|
||||
esClient,
|
||||
totalFieldsLimit,
|
||||
indexNames: dataStreams.map((dataStream) => dataStream.name),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,167 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import { elasticsearchServiceMock, loggingSystemMock } from '@kbn/core/server/mocks';
|
||||
import { errors as EsErrors } from '@elastic/elasticsearch';
|
||||
import { createOrUpdateIndexTemplate } from './create_or_update_index_template';
|
||||
|
||||
const randomDelayMultiplier = 0.01;
|
||||
const logger = loggingSystemMock.createLogger();
|
||||
const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
|
||||
|
||||
const getIndexTemplate = (namespace: string = 'default', useDataStream: boolean = false) => ({
|
||||
name: `.alerts-test.alerts-${namespace}-index-template`,
|
||||
body: {
|
||||
_meta: {
|
||||
kibana: {
|
||||
version: '8.6.1',
|
||||
},
|
||||
managed: true,
|
||||
namespace,
|
||||
},
|
||||
composed_of: ['mappings1', 'framework-mappings'],
|
||||
index_patterns: [`.internal.alerts-test.alerts-${namespace}-*`],
|
||||
template: {
|
||||
mappings: {
|
||||
_meta: {
|
||||
kibana: {
|
||||
version: '8.6.1',
|
||||
},
|
||||
managed: true,
|
||||
namespace,
|
||||
},
|
||||
dynamic: false,
|
||||
},
|
||||
settings: {
|
||||
auto_expand_replicas: '0-1',
|
||||
hidden: true,
|
||||
...(useDataStream
|
||||
? {}
|
||||
: {
|
||||
'index.lifecycle': {
|
||||
name: 'test-ilm-policy',
|
||||
rollover_alias: `.alerts-test.alerts-${namespace}`,
|
||||
},
|
||||
}),
|
||||
'index.mapping.ignore_malformed': true,
|
||||
'index.mapping.total_fields.limit': 2500,
|
||||
},
|
||||
},
|
||||
priority: namespace.length,
|
||||
},
|
||||
});
|
||||
|
||||
const simulateTemplateResponse = {
|
||||
template: {
|
||||
aliases: {
|
||||
alias_name_1: {
|
||||
is_hidden: true,
|
||||
},
|
||||
alias_name_2: {
|
||||
is_hidden: true,
|
||||
},
|
||||
},
|
||||
mappings: { enabled: false },
|
||||
settings: {},
|
||||
},
|
||||
};
|
||||
|
||||
describe('createOrUpdateIndexTemplate', () => {
|
||||
beforeEach(() => {
|
||||
jest.resetAllMocks();
|
||||
jest.spyOn(global.Math, 'random').mockReturnValue(randomDelayMultiplier);
|
||||
});
|
||||
|
||||
it(`should call esClient to put index template`, async () => {
|
||||
esClient.indices.simulateTemplate.mockImplementation(async () => simulateTemplateResponse);
|
||||
await createOrUpdateIndexTemplate({ logger, esClient, template: getIndexTemplate() });
|
||||
|
||||
expect(esClient.indices.simulateTemplate).toHaveBeenCalledWith(getIndexTemplate());
|
||||
expect(esClient.indices.putIndexTemplate).toHaveBeenCalledWith(getIndexTemplate());
|
||||
});
|
||||
|
||||
it(`should retry on transient ES errors`, async () => {
|
||||
esClient.indices.simulateTemplate.mockImplementation(async () => simulateTemplateResponse);
|
||||
esClient.indices.putIndexTemplate
|
||||
.mockRejectedValueOnce(new EsErrors.ConnectionError('foo'))
|
||||
.mockRejectedValueOnce(new EsErrors.TimeoutError('timeout'))
|
||||
.mockResolvedValue({ acknowledged: true });
|
||||
await createOrUpdateIndexTemplate({ logger, esClient, template: getIndexTemplate() });
|
||||
|
||||
expect(esClient.indices.putIndexTemplate).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
it(`should retry simulateTemplate on transient ES errors`, async () => {
|
||||
esClient.indices.simulateTemplate
|
||||
.mockRejectedValueOnce(new EsErrors.ConnectionError('foo'))
|
||||
.mockRejectedValueOnce(new EsErrors.TimeoutError('timeout'))
|
||||
.mockImplementation(async () => simulateTemplateResponse);
|
||||
esClient.indices.putIndexTemplate.mockResolvedValue({ acknowledged: true });
|
||||
await createOrUpdateIndexTemplate({ logger, esClient, template: getIndexTemplate });
|
||||
|
||||
expect(esClient.indices.simulateTemplate).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
it(`should log and throw error if max retries exceeded`, async () => {
|
||||
esClient.indices.simulateTemplate.mockImplementation(async () => simulateTemplateResponse);
|
||||
esClient.indices.putIndexTemplate.mockRejectedValue(new EsErrors.ConnectionError('foo'));
|
||||
await expect(() =>
|
||||
createOrUpdateIndexTemplate({ logger, esClient, template: getIndexTemplate() })
|
||||
).rejects.toThrowErrorMatchingInlineSnapshot(`"foo"`);
|
||||
|
||||
expect(logger.error).toHaveBeenCalledWith(
|
||||
`Error installing index template .alerts-test.alerts-default-index-template - foo`,
|
||||
expect.any(Error)
|
||||
);
|
||||
expect(esClient.indices.putIndexTemplate).toHaveBeenCalledTimes(4);
|
||||
});
|
||||
|
||||
it(`should log and throw error if ES throws error`, async () => {
|
||||
esClient.indices.simulateTemplate.mockImplementation(async () => simulateTemplateResponse);
|
||||
esClient.indices.putIndexTemplate.mockRejectedValue(new Error('generic error'));
|
||||
|
||||
await expect(() =>
|
||||
createOrUpdateIndexTemplate({ logger, esClient, template: getIndexTemplate() })
|
||||
).rejects.toThrowErrorMatchingInlineSnapshot(`"generic error"`);
|
||||
|
||||
expect(logger.error).toHaveBeenCalledWith(
|
||||
`Error installing index template .alerts-test.alerts-default-index-template - generic error`,
|
||||
expect.any(Error)
|
||||
);
|
||||
});
|
||||
|
||||
it(`should log and return without updating template if simulate throws error`, async () => {
|
||||
esClient.indices.simulateTemplate.mockRejectedValue(new Error('simulate error'));
|
||||
esClient.indices.putIndexTemplate.mockRejectedValue(new Error('generic error'));
|
||||
|
||||
await createOrUpdateIndexTemplate({ logger, esClient, template: getIndexTemplate() });
|
||||
|
||||
expect(logger.error).toHaveBeenCalledWith(
|
||||
`Failed to simulate index template mappings for .alerts-test.alerts-default-index-template; not applying mappings - simulate error`,
|
||||
expect.any(Error)
|
||||
);
|
||||
expect(esClient.indices.putIndexTemplate).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it(`should throw error if simulate returns empty mappings`, async () => {
|
||||
esClient.indices.simulateTemplate.mockImplementationOnce(async () => ({
|
||||
...simulateTemplateResponse,
|
||||
template: {
|
||||
...simulateTemplateResponse.template,
|
||||
mappings: {},
|
||||
},
|
||||
}));
|
||||
|
||||
await expect(() =>
|
||||
createOrUpdateIndexTemplate({ logger, esClient, template: getIndexTemplate() })
|
||||
).rejects.toThrowErrorMatchingInlineSnapshot(
|
||||
`"No mappings would be generated for .alerts-test.alerts-default-index-template, possibly due to failed/misconfigured bootstrapping"`
|
||||
);
|
||||
expect(esClient.indices.putIndexTemplate).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,66 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import type {
|
||||
IndicesPutIndexTemplateRequest,
|
||||
MappingTypeMapping,
|
||||
} from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
|
||||
import type { Logger, ElasticsearchClient } from '@kbn/core/server';
|
||||
import { isEmpty } from 'lodash/fp';
|
||||
import { retryTransientEsErrors } from './retry_transient_es_errors';
|
||||
|
||||
interface CreateOrUpdateIndexTemplateOpts {
|
||||
logger: Logger;
|
||||
esClient: ElasticsearchClient;
|
||||
template: IndicesPutIndexTemplateRequest;
|
||||
}
|
||||
|
||||
/**
|
||||
* Installs index template that uses installed component template
|
||||
* Prior to installation, simulates the installation to check for possible
|
||||
* conflicts. Simulate should return an empty mapping if a template
|
||||
* conflicts with an already installed template.
|
||||
*/
|
||||
export const createOrUpdateIndexTemplate = async ({
|
||||
logger,
|
||||
esClient,
|
||||
template,
|
||||
}: CreateOrUpdateIndexTemplateOpts) => {
|
||||
logger.info(`Installing index template ${template.name}`);
|
||||
|
||||
let mappings: MappingTypeMapping = {};
|
||||
try {
|
||||
// Simulate the index template to proactively identify any issues with the mapping
|
||||
const simulateResponse = await retryTransientEsErrors(
|
||||
() => esClient.indices.simulateTemplate(template),
|
||||
{ logger }
|
||||
);
|
||||
mappings = simulateResponse.template.mappings;
|
||||
} catch (err) {
|
||||
logger.error(
|
||||
`Failed to simulate index template mappings for ${template.name}; not applying mappings - ${err.message}`,
|
||||
err
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (isEmpty(mappings)) {
|
||||
throw new Error(
|
||||
`No mappings would be generated for ${template.name}, possibly due to failed/misconfigured bootstrapping`
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
await retryTransientEsErrors(() => esClient.indices.putIndexTemplate(template), {
|
||||
logger,
|
||||
});
|
||||
} catch (err) {
|
||||
logger.error(`Error installing index template ${template.name} - ${err.message}`, err);
|
||||
throw err;
|
||||
}
|
||||
};
|
||||
160
packages/kbn-data-stream-adapter/src/data_stream_adapter.ts
Normal file
160
packages/kbn-data-stream-adapter/src/data_stream_adapter.ts
Normal file
|
|
@ -0,0 +1,160 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import type {
|
||||
ClusterPutComponentTemplateRequest,
|
||||
IndicesIndexSettings,
|
||||
IndicesPutIndexTemplateIndexTemplateMapping,
|
||||
IndicesPutIndexTemplateRequest,
|
||||
} from '@elastic/elasticsearch/lib/api/types';
|
||||
import type { Logger, ElasticsearchClient } from '@kbn/core/server';
|
||||
import type { Subject } from 'rxjs';
|
||||
import type { FieldMap } from './field_maps/types';
|
||||
import { createOrUpdateComponentTemplate } from './create_or_update_component_template';
|
||||
import { createOrUpdateDataStream } from './create_or_update_data_stream';
|
||||
import { createOrUpdateIndexTemplate } from './create_or_update_index_template';
|
||||
import { InstallShutdownError, installWithTimeout } from './install_with_timeout';
|
||||
import { getComponentTemplate, getIndexTemplate } from './resource_installer_utils';
|
||||
|
||||
export interface DataStreamAdapterParams {
|
||||
kibanaVersion: string;
|
||||
totalFieldsLimit?: number;
|
||||
}
|
||||
export interface SetComponentTemplateParams {
|
||||
name: string;
|
||||
fieldMap: FieldMap;
|
||||
settings?: IndicesIndexSettings;
|
||||
dynamic?: 'strict' | boolean;
|
||||
}
|
||||
export interface SetIndexTemplateParams {
|
||||
name: string;
|
||||
componentTemplateRefs?: string[];
|
||||
namespace?: string;
|
||||
template?: IndicesPutIndexTemplateIndexTemplateMapping;
|
||||
hidden?: boolean;
|
||||
}
|
||||
|
||||
export interface GetInstallFnParams {
|
||||
logger: Logger;
|
||||
pluginStop$: Subject<void>;
|
||||
tasksTimeoutMs?: number;
|
||||
}
|
||||
export interface InstallParams {
|
||||
logger: Logger;
|
||||
esClient: ElasticsearchClient | Promise<ElasticsearchClient>;
|
||||
pluginStop$: Subject<void>;
|
||||
tasksTimeoutMs?: number;
|
||||
}
|
||||
|
||||
const DEFAULT_FIELDS_LIMIT = 2500;
|
||||
|
||||
export class DataStreamAdapter {
|
||||
protected readonly kibanaVersion: string;
|
||||
protected readonly totalFieldsLimit: number;
|
||||
protected componentTemplates: ClusterPutComponentTemplateRequest[] = [];
|
||||
protected indexTemplates: IndicesPutIndexTemplateRequest[] = [];
|
||||
protected installed: boolean;
|
||||
|
||||
constructor(protected readonly name: string, options: DataStreamAdapterParams) {
|
||||
this.installed = false;
|
||||
this.kibanaVersion = options.kibanaVersion;
|
||||
this.totalFieldsLimit = options.totalFieldsLimit ?? DEFAULT_FIELDS_LIMIT;
|
||||
}
|
||||
|
||||
public setComponentTemplate(params: SetComponentTemplateParams) {
|
||||
if (this.installed) {
|
||||
throw new Error('Cannot set component template after install');
|
||||
}
|
||||
this.componentTemplates.push(getComponentTemplate(params));
|
||||
}
|
||||
|
||||
public setIndexTemplate(params: SetIndexTemplateParams) {
|
||||
if (this.installed) {
|
||||
throw new Error('Cannot set index template after install');
|
||||
}
|
||||
this.indexTemplates.push(
|
||||
getIndexTemplate({
|
||||
...params,
|
||||
indexPatterns: [this.name],
|
||||
kibanaVersion: this.kibanaVersion,
|
||||
totalFieldsLimit: this.totalFieldsLimit,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
protected getInstallFn({ logger, pluginStop$, tasksTimeoutMs }: GetInstallFnParams) {
|
||||
return async (promise: Promise<void>, description?: string): Promise<void> => {
|
||||
try {
|
||||
await installWithTimeout({
|
||||
installFn: () => promise,
|
||||
description,
|
||||
timeoutMs: tasksTimeoutMs,
|
||||
pluginStop$,
|
||||
});
|
||||
} catch (err) {
|
||||
if (err instanceof InstallShutdownError) {
|
||||
logger.info(err.message);
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
public async install({
|
||||
logger,
|
||||
esClient: esClientToResolve,
|
||||
pluginStop$,
|
||||
tasksTimeoutMs,
|
||||
}: InstallParams) {
|
||||
this.installed = true;
|
||||
|
||||
const esClient = await esClientToResolve;
|
||||
const installFn = this.getInstallFn({ logger, pluginStop$, tasksTimeoutMs });
|
||||
|
||||
// Install component templates in parallel
|
||||
await Promise.all(
|
||||
this.componentTemplates.map((componentTemplate) =>
|
||||
installFn(
|
||||
createOrUpdateComponentTemplate({
|
||||
template: componentTemplate,
|
||||
esClient,
|
||||
logger,
|
||||
totalFieldsLimit: this.totalFieldsLimit,
|
||||
}),
|
||||
`${componentTemplate.name} component template`
|
||||
)
|
||||
)
|
||||
);
|
||||
|
||||
// Install index templates in parallel
|
||||
await Promise.all(
|
||||
this.indexTemplates.map((indexTemplate) =>
|
||||
installFn(
|
||||
createOrUpdateIndexTemplate({
|
||||
template: indexTemplate,
|
||||
esClient,
|
||||
logger,
|
||||
}),
|
||||
`${indexTemplate.name} index template`
|
||||
)
|
||||
)
|
||||
);
|
||||
|
||||
// create data stream when everything is ready
|
||||
await installFn(
|
||||
createOrUpdateDataStream({
|
||||
name: this.name,
|
||||
esClient,
|
||||
logger,
|
||||
totalFieldsLimit: this.totalFieldsLimit,
|
||||
}),
|
||||
`${this.name} data stream`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,100 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
import { createOrUpdateComponentTemplate } from './create_or_update_component_template';
|
||||
import { createDataStream, updateDataStreams } from './create_or_update_data_stream';
|
||||
import { createOrUpdateIndexTemplate } from './create_or_update_index_template';
|
||||
import {
|
||||
DataStreamAdapter,
|
||||
type DataStreamAdapterParams,
|
||||
type InstallParams,
|
||||
} from './data_stream_adapter';
|
||||
|
||||
export class DataStreamSpacesAdapter extends DataStreamAdapter {
|
||||
private installedSpaceDataStreamName: Map<string, Promise<string>>;
|
||||
private _installSpace?: (spaceId: string) => Promise<string>;
|
||||
|
||||
constructor(private readonly prefix: string, options: DataStreamAdapterParams) {
|
||||
super(`${prefix}-*`, options); // make indexTemplate `indexPatterns` match all data stream space names
|
||||
this.installedSpaceDataStreamName = new Map();
|
||||
}
|
||||
|
||||
public async install({
|
||||
logger,
|
||||
esClient: esClientToResolve,
|
||||
pluginStop$,
|
||||
tasksTimeoutMs,
|
||||
}: InstallParams) {
|
||||
this.installed = true;
|
||||
|
||||
const esClient = await esClientToResolve;
|
||||
const installFn = this.getInstallFn({ logger, pluginStop$, tasksTimeoutMs });
|
||||
|
||||
// Install component templates in parallel
|
||||
await Promise.all(
|
||||
this.componentTemplates.map((componentTemplate) =>
|
||||
installFn(
|
||||
createOrUpdateComponentTemplate({
|
||||
template: componentTemplate,
|
||||
esClient,
|
||||
logger,
|
||||
totalFieldsLimit: this.totalFieldsLimit,
|
||||
}),
|
||||
`create or update ${componentTemplate.name} component template`
|
||||
)
|
||||
)
|
||||
);
|
||||
|
||||
// Install index templates in parallel
|
||||
await Promise.all(
|
||||
this.indexTemplates.map((indexTemplate) =>
|
||||
installFn(
|
||||
createOrUpdateIndexTemplate({ template: indexTemplate, esClient, logger }),
|
||||
`create or update ${indexTemplate.name} index template`
|
||||
)
|
||||
)
|
||||
);
|
||||
|
||||
// Update existing space data streams
|
||||
await installFn(
|
||||
updateDataStreams({
|
||||
name: `${this.prefix}-*`,
|
||||
esClient,
|
||||
logger,
|
||||
totalFieldsLimit: this.totalFieldsLimit,
|
||||
}),
|
||||
`update space data streams`
|
||||
);
|
||||
|
||||
// define function to install data stream for spaces on demand
|
||||
this._installSpace = async (spaceId: string) => {
|
||||
const existingInstallPromise = this.installedSpaceDataStreamName.get(spaceId);
|
||||
if (existingInstallPromise) {
|
||||
return existingInstallPromise;
|
||||
}
|
||||
const name = `${this.prefix}-${spaceId}`;
|
||||
const installPromise = installFn(
|
||||
createDataStream({ name, esClient, logger }),
|
||||
`create ${name} data stream`
|
||||
).then(() => name);
|
||||
|
||||
this.installedSpaceDataStreamName.set(spaceId, installPromise);
|
||||
return installPromise;
|
||||
};
|
||||
}
|
||||
|
||||
public async installSpace(spaceId: string): Promise<string> {
|
||||
if (!this._installSpace) {
|
||||
throw new Error('Cannot installSpace before install');
|
||||
}
|
||||
return this._installSpace(spaceId);
|
||||
}
|
||||
|
||||
public async getInstalledSpaceName(spaceId: string): Promise<string | undefined> {
|
||||
return this.installedSpaceDataStreamName.get(spaceId);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,88 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import { EcsFlat } from '@kbn/ecs';
|
||||
import type { EcsMetadata, FieldMap } from './types';
|
||||
|
||||
const EXCLUDED_TYPES = ['constant_keyword'];
|
||||
|
||||
// ECS fields that have reached Stage 2 in the RFC process
|
||||
// are included in the generated Yaml but are still considered
|
||||
// experimental. Some are correctly marked as beta but most are
|
||||
// not.
|
||||
|
||||
// More about the RFC stages here: https://elastic.github.io/ecs/stages.html
|
||||
|
||||
// The following RFCS are currently in stage 2:
|
||||
// https://github.com/elastic/ecs/blob/main/rfcs/text/0027-faas-fields.md
|
||||
// https://github.com/elastic/ecs/blob/main/rfcs/text/0035-tty-output.md
|
||||
// https://github.com/elastic/ecs/blob/main/rfcs/text/0037-host-metrics.md
|
||||
// https://github.com/elastic/ecs/blob/main/rfcs/text/0040-volume-device.md
|
||||
|
||||
// Fields from these RFCs that are not already in the ECS component template
|
||||
// as of 8.11 are manually identified as experimental below.
|
||||
// The next time this list is updated, we should check the above list of RFCs to
|
||||
// see if any have moved to Stage 3 and remove them from the list and check if
|
||||
// there are any new stage 2 RFCs with fields we should exclude as experimental.
|
||||
|
||||
const EXPERIMENTAL_FIELDS = [
|
||||
'faas.trigger', // this was previously mapped as nested but changed to object
|
||||
'faas.trigger.request_id',
|
||||
'faas.trigger.type',
|
||||
'host.cpu.system.norm.pct',
|
||||
'host.cpu.user.norm.pct',
|
||||
'host.fsstats.total_size.total',
|
||||
'host.fsstats.total_size.used',
|
||||
'host.fsstats.total_size.used.pct',
|
||||
'host.load.norm.1',
|
||||
'host.load.norm.5',
|
||||
'host.load.norm.15',
|
||||
'host.memory.actual.used.bytes',
|
||||
'host.memory.actual.used.pct',
|
||||
'host.memory.total',
|
||||
'process.io.bytes',
|
||||
'volume.bus_type',
|
||||
'volume.default_access',
|
||||
'volume.device_name',
|
||||
'volume.device_type',
|
||||
'volume.dos_name',
|
||||
'volume.file_system_type',
|
||||
'volume.mount_name',
|
||||
'volume.nt_name',
|
||||
'volume.product_id',
|
||||
'volume.product_name',
|
||||
'volume.removable',
|
||||
'volume.serial_number',
|
||||
'volume.size',
|
||||
'volume.vendor_id',
|
||||
'volume.vendor_name',
|
||||
'volume.writable',
|
||||
];
|
||||
|
||||
export const ecsFieldMap: FieldMap = Object.fromEntries(
|
||||
Object.entries(EcsFlat)
|
||||
.filter(
|
||||
([key, value]) => !EXCLUDED_TYPES.includes(value.type) && !EXPERIMENTAL_FIELDS.includes(key)
|
||||
)
|
||||
.map(([key, _]) => {
|
||||
const value: EcsMetadata = EcsFlat[key as keyof typeof EcsFlat];
|
||||
return [
|
||||
key,
|
||||
{
|
||||
type: value.type,
|
||||
array: value.normalize.includes('array'),
|
||||
required: !!value.required,
|
||||
...(value.scaling_factor ? { scaling_factor: value.scaling_factor } : {}),
|
||||
...(value.ignore_above ? { ignore_above: value.ignore_above } : {}),
|
||||
...(value.multi_fields ? { multi_fields: value.multi_fields } : {}),
|
||||
},
|
||||
];
|
||||
})
|
||||
);
|
||||
|
||||
export type EcsFieldMap = typeof ecsFieldMap;
|
||||
|
|
@ -0,0 +1,393 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import { alertFieldMap, legacyAlertFieldMap, type FieldMap } from '@kbn/alerts-as-data-utils';
|
||||
import { mappingFromFieldMap } from './mapping_from_field_map';
|
||||
|
||||
export const testFieldMap: FieldMap = {
|
||||
date_field: {
|
||||
type: 'date',
|
||||
array: false,
|
||||
required: true,
|
||||
},
|
||||
keyword_field: {
|
||||
type: 'keyword',
|
||||
array: false,
|
||||
required: false,
|
||||
ignore_above: 1024,
|
||||
},
|
||||
long_field: {
|
||||
type: 'long',
|
||||
array: false,
|
||||
required: false,
|
||||
},
|
||||
multifield_field: {
|
||||
type: 'keyword',
|
||||
array: false,
|
||||
required: false,
|
||||
ignore_above: 1024,
|
||||
multi_fields: [
|
||||
{
|
||||
flat_name: 'multifield_field.text',
|
||||
name: 'text',
|
||||
type: 'match_only_text',
|
||||
},
|
||||
],
|
||||
},
|
||||
geopoint_field: {
|
||||
type: 'geo_point',
|
||||
array: false,
|
||||
required: false,
|
||||
},
|
||||
ip_field: {
|
||||
type: 'ip',
|
||||
array: false,
|
||||
required: false,
|
||||
},
|
||||
array_field: {
|
||||
type: 'keyword',
|
||||
array: true,
|
||||
required: false,
|
||||
ignore_above: 1024,
|
||||
},
|
||||
nested_array_field: {
|
||||
type: 'nested',
|
||||
array: false,
|
||||
required: false,
|
||||
},
|
||||
'nested_array_field.field1': {
|
||||
type: 'keyword',
|
||||
array: false,
|
||||
required: false,
|
||||
ignore_above: 1024,
|
||||
},
|
||||
'nested_array_field.field2': {
|
||||
type: 'keyword',
|
||||
array: false,
|
||||
required: false,
|
||||
ignore_above: 1024,
|
||||
},
|
||||
scaled_float_field: {
|
||||
type: 'scaled_float',
|
||||
array: false,
|
||||
required: false,
|
||||
scaling_factor: 1000,
|
||||
},
|
||||
constant_keyword_field: {
|
||||
type: 'constant_keyword',
|
||||
array: false,
|
||||
required: false,
|
||||
},
|
||||
'parent_field.child1': {
|
||||
type: 'keyword',
|
||||
array: false,
|
||||
required: false,
|
||||
ignore_above: 1024,
|
||||
},
|
||||
'parent_field.child2': {
|
||||
type: 'keyword',
|
||||
array: false,
|
||||
required: false,
|
||||
ignore_above: 1024,
|
||||
},
|
||||
unmapped_object: {
|
||||
type: 'object',
|
||||
required: false,
|
||||
enabled: false,
|
||||
},
|
||||
formatted_field: {
|
||||
type: 'date_range',
|
||||
required: false,
|
||||
format: 'epoch_millis||strict_date_optional_time',
|
||||
},
|
||||
};
|
||||
export const expectedTestMapping = {
|
||||
properties: {
|
||||
array_field: {
|
||||
ignore_above: 1024,
|
||||
type: 'keyword',
|
||||
},
|
||||
constant_keyword_field: {
|
||||
type: 'constant_keyword',
|
||||
},
|
||||
date_field: {
|
||||
type: 'date',
|
||||
},
|
||||
multifield_field: {
|
||||
fields: {
|
||||
text: {
|
||||
type: 'match_only_text',
|
||||
},
|
||||
},
|
||||
ignore_above: 1024,
|
||||
type: 'keyword',
|
||||
},
|
||||
geopoint_field: {
|
||||
type: 'geo_point',
|
||||
},
|
||||
ip_field: {
|
||||
type: 'ip',
|
||||
},
|
||||
keyword_field: {
|
||||
ignore_above: 1024,
|
||||
type: 'keyword',
|
||||
},
|
||||
long_field: {
|
||||
type: 'long',
|
||||
},
|
||||
nested_array_field: {
|
||||
properties: {
|
||||
field1: {
|
||||
ignore_above: 1024,
|
||||
type: 'keyword',
|
||||
},
|
||||
field2: {
|
||||
ignore_above: 1024,
|
||||
type: 'keyword',
|
||||
},
|
||||
},
|
||||
type: 'nested',
|
||||
},
|
||||
parent_field: {
|
||||
properties: {
|
||||
child1: {
|
||||
ignore_above: 1024,
|
||||
type: 'keyword',
|
||||
},
|
||||
child2: {
|
||||
ignore_above: 1024,
|
||||
type: 'keyword',
|
||||
},
|
||||
},
|
||||
},
|
||||
scaled_float_field: {
|
||||
scaling_factor: 1000,
|
||||
type: 'scaled_float',
|
||||
},
|
||||
unmapped_object: {
|
||||
enabled: false,
|
||||
type: 'object',
|
||||
},
|
||||
formatted_field: {
|
||||
type: 'date_range',
|
||||
format: 'epoch_millis||strict_date_optional_time',
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
describe('mappingFromFieldMap', () => {
|
||||
it('correctly creates mapping from field map', () => {
|
||||
expect(mappingFromFieldMap(testFieldMap)).toEqual({
|
||||
dynamic: 'strict',
|
||||
...expectedTestMapping,
|
||||
});
|
||||
expect(mappingFromFieldMap(alertFieldMap)).toEqual({
|
||||
dynamic: 'strict',
|
||||
properties: {
|
||||
'@timestamp': {
|
||||
ignore_malformed: false,
|
||||
type: 'date',
|
||||
},
|
||||
event: {
|
||||
properties: {
|
||||
action: {
|
||||
type: 'keyword',
|
||||
},
|
||||
kind: {
|
||||
type: 'keyword',
|
||||
},
|
||||
},
|
||||
},
|
||||
kibana: {
|
||||
properties: {
|
||||
alert: {
|
||||
properties: {
|
||||
action_group: {
|
||||
type: 'keyword',
|
||||
},
|
||||
case_ids: {
|
||||
type: 'keyword',
|
||||
},
|
||||
duration: {
|
||||
properties: {
|
||||
us: {
|
||||
type: 'long',
|
||||
},
|
||||
},
|
||||
},
|
||||
end: {
|
||||
type: 'date',
|
||||
},
|
||||
flapping: {
|
||||
type: 'boolean',
|
||||
},
|
||||
flapping_history: {
|
||||
type: 'boolean',
|
||||
},
|
||||
maintenance_window_ids: {
|
||||
type: 'keyword',
|
||||
},
|
||||
instance: {
|
||||
properties: {
|
||||
id: {
|
||||
type: 'keyword',
|
||||
},
|
||||
},
|
||||
},
|
||||
last_detected: {
|
||||
type: 'date',
|
||||
},
|
||||
reason: {
|
||||
fields: {
|
||||
text: {
|
||||
type: 'match_only_text',
|
||||
},
|
||||
},
|
||||
type: 'keyword',
|
||||
},
|
||||
rule: {
|
||||
properties: {
|
||||
category: {
|
||||
type: 'keyword',
|
||||
},
|
||||
consumer: {
|
||||
type: 'keyword',
|
||||
},
|
||||
execution: {
|
||||
properties: {
|
||||
uuid: {
|
||||
type: 'keyword',
|
||||
},
|
||||
},
|
||||
},
|
||||
name: {
|
||||
type: 'keyword',
|
||||
},
|
||||
parameters: {
|
||||
type: 'flattened',
|
||||
ignore_above: 4096,
|
||||
},
|
||||
producer: {
|
||||
type: 'keyword',
|
||||
},
|
||||
revision: {
|
||||
type: 'long',
|
||||
},
|
||||
rule_type_id: {
|
||||
type: 'keyword',
|
||||
},
|
||||
tags: {
|
||||
type: 'keyword',
|
||||
},
|
||||
uuid: {
|
||||
type: 'keyword',
|
||||
},
|
||||
},
|
||||
},
|
||||
start: {
|
||||
type: 'date',
|
||||
},
|
||||
status: {
|
||||
type: 'keyword',
|
||||
},
|
||||
time_range: {
|
||||
type: 'date_range',
|
||||
format: 'epoch_millis||strict_date_optional_time',
|
||||
},
|
||||
url: {
|
||||
ignore_above: 2048,
|
||||
index: false,
|
||||
type: 'keyword',
|
||||
},
|
||||
uuid: {
|
||||
type: 'keyword',
|
||||
},
|
||||
workflow_assignee_ids: {
|
||||
type: 'keyword',
|
||||
},
|
||||
workflow_status: {
|
||||
type: 'keyword',
|
||||
},
|
||||
workflow_tags: {
|
||||
type: 'keyword',
|
||||
},
|
||||
},
|
||||
},
|
||||
space_ids: {
|
||||
type: 'keyword',
|
||||
},
|
||||
version: {
|
||||
type: 'version',
|
||||
},
|
||||
},
|
||||
},
|
||||
tags: {
|
||||
type: 'keyword',
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(mappingFromFieldMap(legacyAlertFieldMap)).toEqual({
|
||||
dynamic: 'strict',
|
||||
properties: {
|
||||
kibana: {
|
||||
properties: {
|
||||
alert: {
|
||||
properties: {
|
||||
risk_score: { type: 'float' },
|
||||
rule: {
|
||||
properties: {
|
||||
author: { type: 'keyword' },
|
||||
created_at: { type: 'date' },
|
||||
created_by: { type: 'keyword' },
|
||||
description: { type: 'keyword' },
|
||||
enabled: { type: 'keyword' },
|
||||
from: { type: 'keyword' },
|
||||
interval: { type: 'keyword' },
|
||||
license: { type: 'keyword' },
|
||||
note: { type: 'keyword' },
|
||||
references: { type: 'keyword' },
|
||||
rule_id: { type: 'keyword' },
|
||||
rule_name_override: { type: 'keyword' },
|
||||
to: { type: 'keyword' },
|
||||
type: { type: 'keyword' },
|
||||
updated_at: { type: 'date' },
|
||||
updated_by: { type: 'keyword' },
|
||||
version: { type: 'keyword' },
|
||||
},
|
||||
},
|
||||
severity: { type: 'keyword' },
|
||||
suppression: {
|
||||
properties: {
|
||||
docs_count: { type: 'long' },
|
||||
end: { type: 'date' },
|
||||
terms: {
|
||||
properties: { field: { type: 'keyword' }, value: { type: 'keyword' } },
|
||||
},
|
||||
start: { type: 'date' },
|
||||
},
|
||||
},
|
||||
system_status: { type: 'keyword' },
|
||||
workflow_reason: { type: 'keyword' },
|
||||
workflow_status_updated_at: { type: 'date' },
|
||||
workflow_user: { type: 'keyword' },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
ecs: { properties: { version: { type: 'keyword' } } },
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('uses dynamic setting if specified', () => {
|
||||
expect(mappingFromFieldMap(testFieldMap, true)).toEqual({
|
||||
dynamic: true,
|
||||
...expectedTestMapping,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,54 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
|
||||
import { set } from '@kbn/safer-lodash-set';
|
||||
import type { FieldMap, MultiField } from './types';
|
||||
|
||||
export function mappingFromFieldMap(
|
||||
fieldMap: FieldMap,
|
||||
dynamic: 'strict' | boolean = 'strict'
|
||||
): MappingTypeMapping {
|
||||
const mappings = {
|
||||
dynamic,
|
||||
properties: {},
|
||||
};
|
||||
|
||||
const fields = Object.keys(fieldMap).map((key: string) => {
|
||||
const field = fieldMap[key];
|
||||
return {
|
||||
name: key,
|
||||
...field,
|
||||
};
|
||||
});
|
||||
|
||||
fields.forEach((field) => {
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
const { name, required, array, multi_fields, ...rest } = field;
|
||||
const mapped = multi_fields
|
||||
? {
|
||||
...rest,
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
fields: multi_fields.reduce((acc, multi_field: MultiField) => {
|
||||
acc[multi_field.name] = {
|
||||
type: multi_field.type,
|
||||
};
|
||||
return acc;
|
||||
}, {} as Record<string, unknown>),
|
||||
}
|
||||
: rest;
|
||||
|
||||
set(mappings.properties, field.name.split('.').join('.properties.'), mapped);
|
||||
|
||||
if (name === '@timestamp') {
|
||||
set(mappings.properties, `${name}.ignore_malformed`, false);
|
||||
}
|
||||
});
|
||||
|
||||
return mappings;
|
||||
}
|
||||
56
packages/kbn-data-stream-adapter/src/field_maps/types.ts
Normal file
56
packages/kbn-data-stream-adapter/src/field_maps/types.ts
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
export interface AllowedValue {
|
||||
description?: string;
|
||||
name?: string;
|
||||
}
|
||||
|
||||
export interface MultiField {
|
||||
flat_name: string;
|
||||
name: string;
|
||||
type: string;
|
||||
}
|
||||
|
||||
export interface EcsMetadata {
|
||||
allowed_values?: AllowedValue[];
|
||||
dashed_name: string;
|
||||
description: string;
|
||||
doc_values?: boolean;
|
||||
example?: string | number | boolean;
|
||||
flat_name: string;
|
||||
ignore_above?: number;
|
||||
index?: boolean;
|
||||
level: string;
|
||||
multi_fields?: MultiField[];
|
||||
name: string;
|
||||
normalize: string[];
|
||||
required?: boolean;
|
||||
scaling_factor?: number;
|
||||
short: string;
|
||||
type: string;
|
||||
properties?: Record<string, { type: string }>;
|
||||
}
|
||||
|
||||
export interface FieldMap {
|
||||
[key: string]: {
|
||||
type: string;
|
||||
required: boolean;
|
||||
array?: boolean;
|
||||
doc_values?: boolean;
|
||||
enabled?: boolean;
|
||||
format?: string;
|
||||
ignore_above?: number;
|
||||
multi_fields?: MultiField[];
|
||||
index?: boolean;
|
||||
path?: string;
|
||||
scaling_factor?: number;
|
||||
dynamic?: boolean | 'strict';
|
||||
properties?: Record<string, { type: string }>;
|
||||
};
|
||||
}
|
||||
|
|
@ -0,0 +1,63 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import { loggerMock } from '@kbn/logging-mocks';
|
||||
|
||||
import { installWithTimeout } from './install_with_timeout';
|
||||
import { ReplaySubject, type Subject } from 'rxjs';
|
||||
|
||||
const logger = loggerMock.create();
|
||||
|
||||
describe('installWithTimeout', () => {
|
||||
let pluginStop$: Subject<void>;
|
||||
|
||||
beforeEach(() => {
|
||||
jest.resetAllMocks();
|
||||
pluginStop$ = new ReplaySubject(1);
|
||||
});
|
||||
|
||||
it(`should call installFn`, async () => {
|
||||
const installFn = jest.fn();
|
||||
await installWithTimeout({
|
||||
installFn,
|
||||
pluginStop$,
|
||||
timeoutMs: 10,
|
||||
});
|
||||
expect(installFn).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it(`should short-circuit installFn if it exceeds configured timeout`, async () => {
|
||||
await expect(() =>
|
||||
installWithTimeout({
|
||||
installFn: async () => {
|
||||
await new Promise((r) => setTimeout(r, 20));
|
||||
},
|
||||
pluginStop$,
|
||||
timeoutMs: 10,
|
||||
})
|
||||
).rejects.toThrowErrorMatchingInlineSnapshot(
|
||||
`"Failure during installation. Timeout: it took more than 10ms"`
|
||||
);
|
||||
});
|
||||
|
||||
it(`should short-circuit installFn if pluginStop$ signal is received`, async () => {
|
||||
pluginStop$.next();
|
||||
await expect(() =>
|
||||
installWithTimeout({
|
||||
installFn: async () => {
|
||||
await new Promise((r) => setTimeout(r, 5));
|
||||
logger.info(`running`);
|
||||
},
|
||||
pluginStop$,
|
||||
timeoutMs: 10,
|
||||
})
|
||||
).rejects.toThrowErrorMatchingInlineSnapshot(
|
||||
`"Server is stopping; must stop all async operations"`
|
||||
);
|
||||
});
|
||||
});
|
||||
67
packages/kbn-data-stream-adapter/src/install_with_timeout.ts
Normal file
67
packages/kbn-data-stream-adapter/src/install_with_timeout.ts
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import { firstValueFrom, type Observable } from 'rxjs';
|
||||
|
||||
const INSTALLATION_TIMEOUT = 20 * 60 * 1000; // 20 minutes
|
||||
|
||||
interface InstallWithTimeoutOpts {
|
||||
description?: string;
|
||||
installFn: () => Promise<void>;
|
||||
pluginStop$: Observable<void>;
|
||||
timeoutMs?: number;
|
||||
}
|
||||
|
||||
export class InstallShutdownError extends Error {
|
||||
constructor() {
|
||||
super('Server is stopping; must stop all async operations');
|
||||
Object.setPrototypeOf(this, InstallShutdownError.prototype);
|
||||
}
|
||||
}
|
||||
|
||||
export const installWithTimeout = async ({
|
||||
description,
|
||||
installFn,
|
||||
pluginStop$,
|
||||
timeoutMs = INSTALLATION_TIMEOUT,
|
||||
}: InstallWithTimeoutOpts): Promise<void> => {
|
||||
try {
|
||||
let timeoutId: NodeJS.Timeout;
|
||||
const install = async (): Promise<void> => {
|
||||
await installFn();
|
||||
if (timeoutId) {
|
||||
clearTimeout(timeoutId);
|
||||
}
|
||||
};
|
||||
|
||||
const throwTimeoutException = (): Promise<void> => {
|
||||
return new Promise((_, reject) => {
|
||||
timeoutId = setTimeout(() => {
|
||||
const msg = `Timeout: it took more than ${timeoutMs}ms`;
|
||||
reject(new Error(msg));
|
||||
}, timeoutMs);
|
||||
|
||||
firstValueFrom(pluginStop$).then(() => {
|
||||
clearTimeout(timeoutId);
|
||||
reject(new InstallShutdownError());
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
await Promise.race([install(), throwTimeoutException()]);
|
||||
} catch (e) {
|
||||
if (e instanceof InstallShutdownError) {
|
||||
throw e;
|
||||
} else {
|
||||
const reason = e?.message || 'Unknown reason';
|
||||
throw new Error(
|
||||
`Failure during installation${description ? ` of ${description}` : ''}. ${reason}`
|
||||
);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
|
@ -0,0 +1,170 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import { getIndexTemplate, getComponentTemplate } from './resource_installer_utils';
|
||||
|
||||
describe('getIndexTemplate', () => {
|
||||
const defaultParams = {
|
||||
name: 'indexTemplateName',
|
||||
kibanaVersion: '8.12.1',
|
||||
indexPatterns: ['indexPattern1', 'indexPattern2'],
|
||||
componentTemplateRefs: ['template1', 'template2'],
|
||||
totalFieldsLimit: 2500,
|
||||
};
|
||||
|
||||
it('should create index template with given parameters and defaults', () => {
|
||||
const indexTemplate = getIndexTemplate(defaultParams);
|
||||
|
||||
expect(indexTemplate).toEqual({
|
||||
name: defaultParams.name,
|
||||
body: {
|
||||
data_stream: { hidden: true },
|
||||
index_patterns: defaultParams.indexPatterns,
|
||||
composed_of: defaultParams.componentTemplateRefs,
|
||||
template: {
|
||||
settings: {
|
||||
hidden: true,
|
||||
auto_expand_replicas: '0-1',
|
||||
'index.mapping.ignore_malformed': true,
|
||||
'index.mapping.total_fields.limit': defaultParams.totalFieldsLimit,
|
||||
},
|
||||
mappings: {
|
||||
dynamic: false,
|
||||
_meta: {
|
||||
kibana: {
|
||||
version: defaultParams.kibanaVersion,
|
||||
},
|
||||
managed: true,
|
||||
namespace: 'default',
|
||||
},
|
||||
},
|
||||
},
|
||||
_meta: {
|
||||
kibana: {
|
||||
version: defaultParams.kibanaVersion,
|
||||
},
|
||||
managed: true,
|
||||
namespace: 'default',
|
||||
},
|
||||
priority: 7,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('should create not hidden index template', () => {
|
||||
const { body } = getIndexTemplate({ ...defaultParams, hidden: false });
|
||||
expect(body?.data_stream?.hidden).toEqual(false);
|
||||
expect(body?.template?.settings?.hidden).toEqual(false);
|
||||
});
|
||||
|
||||
it('should create index template with custom namespace', () => {
|
||||
const { body } = getIndexTemplate({ ...defaultParams, namespace: 'custom-namespace' });
|
||||
expect(body?._meta?.namespace).toEqual('custom-namespace');
|
||||
expect(body?.priority).toEqual(16);
|
||||
});
|
||||
|
||||
it('should create index template with template overrides', () => {
|
||||
const { body } = getIndexTemplate({
|
||||
...defaultParams,
|
||||
template: {
|
||||
settings: {
|
||||
number_of_shards: 1,
|
||||
},
|
||||
mappings: {
|
||||
dynamic: true,
|
||||
},
|
||||
lifecycle: {
|
||||
data_retention: '30d',
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(body?.template?.settings).toEqual({
|
||||
hidden: true,
|
||||
auto_expand_replicas: '0-1',
|
||||
'index.mapping.ignore_malformed': true,
|
||||
'index.mapping.total_fields.limit': defaultParams.totalFieldsLimit,
|
||||
number_of_shards: 1,
|
||||
});
|
||||
|
||||
expect(body?.template?.mappings).toEqual({
|
||||
dynamic: true,
|
||||
_meta: {
|
||||
kibana: {
|
||||
version: defaultParams.kibanaVersion,
|
||||
},
|
||||
managed: true,
|
||||
namespace: 'default',
|
||||
},
|
||||
});
|
||||
|
||||
expect(body?.template?.lifecycle).toEqual({
|
||||
data_retention: '30d',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('getComponentTemplate', () => {
|
||||
const defaultParams = {
|
||||
name: 'componentTemplateName',
|
||||
kibanaVersion: '8.12.1',
|
||||
fieldMap: {
|
||||
field1: { type: 'text', required: true },
|
||||
field2: { type: 'keyword', required: false },
|
||||
},
|
||||
};
|
||||
|
||||
it('should create component template with given parameters and defaults', () => {
|
||||
const componentTemplate = getComponentTemplate(defaultParams);
|
||||
|
||||
expect(componentTemplate).toEqual({
|
||||
name: defaultParams.name,
|
||||
_meta: {
|
||||
managed: true,
|
||||
},
|
||||
template: {
|
||||
settings: {
|
||||
number_of_shards: 1,
|
||||
'index.mapping.total_fields.limit': 1500,
|
||||
},
|
||||
mappings: {
|
||||
dynamic: 'strict',
|
||||
properties: {
|
||||
field1: {
|
||||
type: 'text',
|
||||
},
|
||||
field2: {
|
||||
type: 'keyword',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('should create component template with custom settings', () => {
|
||||
const { template } = getComponentTemplate({
|
||||
...defaultParams,
|
||||
settings: {
|
||||
number_of_shards: 1,
|
||||
number_of_replicas: 1,
|
||||
},
|
||||
});
|
||||
|
||||
expect(template.settings).toEqual({
|
||||
number_of_shards: 1,
|
||||
number_of_replicas: 1,
|
||||
'index.mapping.total_fields.limit': 1500,
|
||||
});
|
||||
});
|
||||
|
||||
it('should create component template with custom dynamic', () => {
|
||||
const { template } = getComponentTemplate({ ...defaultParams, dynamic: true });
|
||||
expect(template.mappings?.dynamic).toEqual(true);
|
||||
});
|
||||
});
|
||||
106
packages/kbn-data-stream-adapter/src/resource_installer_utils.ts
Normal file
106
packages/kbn-data-stream-adapter/src/resource_installer_utils.ts
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import type {
|
||||
IndicesPutIndexTemplateRequest,
|
||||
Metadata,
|
||||
} from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
|
||||
import type {
|
||||
ClusterPutComponentTemplateRequest,
|
||||
IndicesIndexSettings,
|
||||
IndicesPutIndexTemplateIndexTemplateMapping,
|
||||
} from '@elastic/elasticsearch/lib/api/types';
|
||||
import type { FieldMap } from './field_maps/types';
|
||||
import { mappingFromFieldMap } from './field_maps/mapping_from_field_map';
|
||||
|
||||
interface GetComponentTemplateOpts {
|
||||
name: string;
|
||||
fieldMap: FieldMap;
|
||||
settings?: IndicesIndexSettings;
|
||||
dynamic?: 'strict' | boolean;
|
||||
}
|
||||
|
||||
export const getComponentTemplate = ({
|
||||
name,
|
||||
fieldMap,
|
||||
settings,
|
||||
dynamic = 'strict',
|
||||
}: GetComponentTemplateOpts): ClusterPutComponentTemplateRequest => ({
|
||||
name,
|
||||
_meta: {
|
||||
managed: true,
|
||||
},
|
||||
template: {
|
||||
settings: {
|
||||
number_of_shards: 1,
|
||||
'index.mapping.total_fields.limit':
|
||||
Math.ceil(Object.keys(fieldMap).length / 1000) * 1000 + 500,
|
||||
...settings,
|
||||
},
|
||||
mappings: mappingFromFieldMap(fieldMap, dynamic),
|
||||
},
|
||||
});
|
||||
|
||||
interface GetIndexTemplateOpts {
|
||||
name: string;
|
||||
indexPatterns: string[];
|
||||
kibanaVersion: string;
|
||||
totalFieldsLimit: number;
|
||||
componentTemplateRefs?: string[];
|
||||
namespace?: string;
|
||||
template?: IndicesPutIndexTemplateIndexTemplateMapping;
|
||||
hidden?: boolean;
|
||||
}
|
||||
|
||||
export const getIndexTemplate = ({
|
||||
name,
|
||||
indexPatterns,
|
||||
kibanaVersion,
|
||||
totalFieldsLimit,
|
||||
componentTemplateRefs,
|
||||
namespace = 'default',
|
||||
template = {},
|
||||
hidden = true,
|
||||
}: GetIndexTemplateOpts): IndicesPutIndexTemplateRequest => {
|
||||
const indexMetadata: Metadata = {
|
||||
kibana: {
|
||||
version: kibanaVersion,
|
||||
},
|
||||
managed: true,
|
||||
namespace,
|
||||
};
|
||||
|
||||
return {
|
||||
name,
|
||||
body: {
|
||||
data_stream: { hidden },
|
||||
index_patterns: indexPatterns,
|
||||
composed_of: componentTemplateRefs,
|
||||
template: {
|
||||
...template,
|
||||
settings: {
|
||||
hidden,
|
||||
auto_expand_replicas: '0-1',
|
||||
'index.mapping.ignore_malformed': true,
|
||||
'index.mapping.total_fields.limit': totalFieldsLimit,
|
||||
...template.settings,
|
||||
},
|
||||
mappings: {
|
||||
dynamic: false,
|
||||
_meta: indexMetadata,
|
||||
...template.mappings,
|
||||
},
|
||||
},
|
||||
_meta: indexMetadata,
|
||||
|
||||
// By setting the priority to namespace.length, we ensure that if one namespace is a prefix of another namespace
|
||||
// then newly created indices will use the matching template with the *longest* namespace
|
||||
priority: namespace.length,
|
||||
},
|
||||
};
|
||||
};
|
||||
|
|
@ -0,0 +1,78 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
import { loggingSystemMock } from '@kbn/core/server/mocks';
|
||||
import { errors as EsErrors, type DiagnosticResult } from '@elastic/elasticsearch';
|
||||
import { retryTransientEsErrors } from './retry_transient_es_errors';
|
||||
|
||||
const mockLogger = loggingSystemMock.createLogger();
|
||||
|
||||
// mock setTimeout to avoid waiting in tests and prevent test flakiness
|
||||
global.setTimeout = jest.fn((cb) => jest.fn(cb())) as unknown as typeof global.setTimeout;
|
||||
|
||||
describe('retryTransientEsErrors', () => {
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ error: new EsErrors.ConnectionError('test error'), errorType: 'ConnectionError' },
|
||||
{
|
||||
error: new EsErrors.NoLivingConnectionsError('test error', {} as DiagnosticResult),
|
||||
errorType: 'NoLivingConnectionsError',
|
||||
},
|
||||
{ error: new EsErrors.TimeoutError('test error'), errorType: 'TimeoutError' },
|
||||
{
|
||||
error: new EsErrors.ResponseError({ statusCode: 503 } as DiagnosticResult),
|
||||
errorType: 'ResponseError (Unavailable)',
|
||||
},
|
||||
{
|
||||
error: new EsErrors.ResponseError({ statusCode: 408 } as DiagnosticResult),
|
||||
errorType: 'ResponseError (RequestTimeout)',
|
||||
},
|
||||
{
|
||||
error: new EsErrors.ResponseError({ statusCode: 410 } as DiagnosticResult),
|
||||
errorType: 'ResponseError (Gone)',
|
||||
},
|
||||
])('should retry $errorType', async ({ error }) => {
|
||||
const mockFn = jest.fn();
|
||||
mockFn.mockRejectedValueOnce(error);
|
||||
mockFn.mockResolvedValueOnce('success');
|
||||
|
||||
const result = await retryTransientEsErrors(mockFn, { logger: mockLogger });
|
||||
|
||||
expect(result).toEqual('success');
|
||||
expect(mockFn).toHaveBeenCalledTimes(2);
|
||||
expect(mockLogger.warn).toHaveBeenCalledTimes(1);
|
||||
expect(mockLogger.error).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should throw non-transient errors', async () => {
|
||||
const error = new EsErrors.ResponseError({ statusCode: 429 } as DiagnosticResult);
|
||||
const mockFn = jest.fn();
|
||||
mockFn.mockRejectedValueOnce(error);
|
||||
|
||||
await expect(retryTransientEsErrors(mockFn, { logger: mockLogger })).rejects.toEqual(error);
|
||||
|
||||
expect(mockFn).toHaveBeenCalledTimes(1);
|
||||
expect(mockLogger.warn).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should throw if max retries exceeded', async () => {
|
||||
const error = new EsErrors.ConnectionError('test error');
|
||||
const mockFn = jest.fn();
|
||||
mockFn.mockRejectedValueOnce(error);
|
||||
mockFn.mockRejectedValueOnce(error);
|
||||
|
||||
await expect(
|
||||
retryTransientEsErrors(mockFn, { logger: mockLogger, attempt: 2 })
|
||||
).rejects.toEqual(error);
|
||||
|
||||
expect(mockFn).toHaveBeenCalledTimes(2);
|
||||
expect(mockLogger.warn).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,54 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import type { Logger } from '@kbn/core/server';
|
||||
import { errors as EsErrors } from '@elastic/elasticsearch';
|
||||
|
||||
const MAX_ATTEMPTS = 3;
|
||||
|
||||
const retryResponseStatuses = [
|
||||
503, // ServiceUnavailable
|
||||
408, // RequestTimeout
|
||||
410, // Gone
|
||||
];
|
||||
|
||||
const isRetryableError = (e: Error) =>
|
||||
e instanceof EsErrors.NoLivingConnectionsError ||
|
||||
e instanceof EsErrors.ConnectionError ||
|
||||
e instanceof EsErrors.TimeoutError ||
|
||||
(e instanceof EsErrors.ResponseError &&
|
||||
e?.statusCode &&
|
||||
retryResponseStatuses.includes(e.statusCode));
|
||||
|
||||
const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
export const retryTransientEsErrors = async <T>(
|
||||
esCall: () => Promise<T>,
|
||||
{ logger, attempt = 0 }: { logger: Logger; attempt?: number }
|
||||
): Promise<T> => {
|
||||
try {
|
||||
return await esCall();
|
||||
} catch (e) {
|
||||
if (attempt < MAX_ATTEMPTS && isRetryableError(e)) {
|
||||
const retryCount = attempt + 1;
|
||||
const retryDelaySec: number = Math.min(Math.pow(2, retryCount), 30); // 2s, 4s, 8s, 16s, 30s, 30s, 30s...
|
||||
|
||||
logger.warn(
|
||||
`Retrying Elasticsearch operation after [${retryDelaySec}s] due to error: ${e.toString()} ${
|
||||
e.stack
|
||||
}`
|
||||
);
|
||||
|
||||
// delay with some randomness
|
||||
await delay(retryDelaySec * 1000 * Math.random());
|
||||
return retryTransientEsErrors(esCall, { logger, attempt: retryCount });
|
||||
}
|
||||
|
||||
throw e;
|
||||
}
|
||||
};
|
||||
24
packages/kbn-data-stream-adapter/tsconfig.json
Normal file
24
packages/kbn-data-stream-adapter/tsconfig.json
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
{
|
||||
"extends": "../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"outDir": "target/types",
|
||||
"types": [
|
||||
"jest",
|
||||
"node",
|
||||
"react",
|
||||
"@emotion/react/types/css-prop",
|
||||
"@testing-library/jest-dom",
|
||||
"@testing-library/react"
|
||||
]
|
||||
},
|
||||
"include": ["**/*.ts", "**/*.tsx"],
|
||||
"kbn_references": [
|
||||
"@kbn/core",
|
||||
"@kbn/std",
|
||||
"@kbn/ecs",
|
||||
"@kbn/alerts-as-data-utils",
|
||||
"@kbn/safer-lodash-set",
|
||||
"@kbn/logging-mocks",
|
||||
],
|
||||
"exclude": ["target/**/*"]
|
||||
}
|
||||
|
|
@ -632,6 +632,8 @@
|
|||
"@kbn/data-search-plugin/*": ["test/plugin_functional/plugins/data_search/*"],
|
||||
"@kbn/data-service": ["packages/kbn-data-service"],
|
||||
"@kbn/data-service/*": ["packages/kbn-data-service/*"],
|
||||
"@kbn/data-stream-adapter": ["packages/kbn-data-stream-adapter"],
|
||||
"@kbn/data-stream-adapter/*": ["packages/kbn-data-stream-adapter/*"],
|
||||
"@kbn/data-view-editor-plugin": ["src/plugins/data_view_editor"],
|
||||
"@kbn/data-view-editor-plugin/*": ["src/plugins/data_view_editor/*"],
|
||||
"@kbn/data-view-field-editor-example-plugin": ["examples/data_view_field_editor_example"],
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import { notificationServiceMock } from '@kbn/core-notifications-browser-mocks';
|
||||
import { renderHook } from '@testing-library/react-hooks';
|
||||
import React from 'react';
|
||||
|
||||
|
|
@ -13,6 +14,7 @@ import { DataQualityProvider, useDataQualityContext } from '.';
|
|||
const mockReportDataQualityIndexChecked = jest.fn();
|
||||
const mockReportDataQualityCheckAllClicked = jest.fn();
|
||||
const mockHttpFetch = jest.fn();
|
||||
const { toasts } = notificationServiceMock.createSetupContract();
|
||||
const mockTelemetryEvents = {
|
||||
reportDataQualityIndexChecked: mockReportDataQualityIndexChecked,
|
||||
reportDataQualityCheckAllCompleted: mockReportDataQualityCheckAllClicked,
|
||||
|
|
@ -22,6 +24,7 @@ const ContextWrapper: React.FC = ({ children }) => (
|
|||
httpFetch={mockHttpFetch}
|
||||
telemetryEvents={mockTelemetryEvents}
|
||||
isILMAvailable={true}
|
||||
toasts={toasts}
|
||||
>
|
||||
{children}
|
||||
</DataQualityProvider>
|
||||
|
|
|
|||
|
|
@ -5,14 +5,16 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { HttpHandler } from '@kbn/core-http-browser';
|
||||
import React, { useMemo } from 'react';
|
||||
import { TelemetryEvents } from '../../types';
|
||||
import type { HttpHandler } from '@kbn/core-http-browser';
|
||||
import type { IToasts } from '@kbn/core-notifications-browser';
|
||||
import type { TelemetryEvents } from '../../types';
|
||||
|
||||
interface DataQualityProviderProps {
|
||||
httpFetch: HttpHandler;
|
||||
isILMAvailable: boolean;
|
||||
telemetryEvents: TelemetryEvents;
|
||||
toasts: IToasts;
|
||||
}
|
||||
|
||||
const DataQualityContext = React.createContext<DataQualityProviderProps | undefined>(undefined);
|
||||
|
|
@ -20,16 +22,18 @@ const DataQualityContext = React.createContext<DataQualityProviderProps | undefi
|
|||
export const DataQualityProvider: React.FC<DataQualityProviderProps> = ({
|
||||
children,
|
||||
httpFetch,
|
||||
toasts,
|
||||
isILMAvailable,
|
||||
telemetryEvents,
|
||||
}) => {
|
||||
const value = useMemo(
|
||||
() => ({
|
||||
httpFetch,
|
||||
toasts,
|
||||
isILMAvailable,
|
||||
telemetryEvents,
|
||||
}),
|
||||
[httpFetch, isILMAvailable, telemetryEvents]
|
||||
[httpFetch, toasts, isILMAvailable, telemetryEvents]
|
||||
);
|
||||
|
||||
return <DataQualityContext.Provider value={value}>{children}</DataQualityContext.Provider>;
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ import { useAddToNewCase } from '../../use_add_to_new_case';
|
|||
import { useMappings } from '../../use_mappings';
|
||||
import { useUnallowedValues } from '../../use_unallowed_values';
|
||||
import { useDataQualityContext } from '../data_quality_context';
|
||||
import { getSizeInBytes } from '../../helpers';
|
||||
import { getSizeInBytes, postResult } from '../../helpers';
|
||||
|
||||
const EMPTY_MARKDOWN_COMMENTS: string[] = [];
|
||||
|
||||
|
|
@ -104,7 +104,7 @@ const IndexPropertiesComponent: React.FC<Props> = ({
|
|||
updatePatternRollup,
|
||||
}) => {
|
||||
const { error: mappingsError, indexes, loading: loadingMappings } = useMappings(indexName);
|
||||
const { telemetryEvents, isILMAvailable } = useDataQualityContext();
|
||||
const { telemetryEvents, isILMAvailable, httpFetch, toasts } = useDataQualityContext();
|
||||
|
||||
const requestItems = useMemo(
|
||||
() =>
|
||||
|
|
@ -249,7 +249,7 @@ const IndexPropertiesComponent: React.FC<Props> = ({
|
|||
})
|
||||
: EMPTY_MARKDOWN_COMMENTS;
|
||||
|
||||
updatePatternRollup({
|
||||
const updatedRollup = {
|
||||
...patternRollup,
|
||||
results: {
|
||||
...patternRollup.results,
|
||||
|
|
@ -264,10 +264,11 @@ const IndexPropertiesComponent: React.FC<Props> = ({
|
|||
sameFamily: indexSameFamily,
|
||||
},
|
||||
},
|
||||
});
|
||||
};
|
||||
updatePatternRollup(updatedRollup);
|
||||
|
||||
if (indexId && requestTime != null && requestTime > 0 && partitionedFieldMetadata) {
|
||||
telemetryEvents.reportDataQualityIndexChecked?.({
|
||||
const checkMetadata = {
|
||||
batchId: uuidv4(),
|
||||
ecsVersion: EcsVersion,
|
||||
errorCount: error ? 1 : 0,
|
||||
|
|
@ -276,7 +277,10 @@ const IndexPropertiesComponent: React.FC<Props> = ({
|
|||
indexName,
|
||||
isCheckAll: false,
|
||||
numberOfDocuments: docsCount,
|
||||
numberOfFields: partitionedFieldMetadata.all.length,
|
||||
numberOfIncompatibleFields: indexIncompatible,
|
||||
numberOfEcsFields: partitionedFieldMetadata.ecsCompliant.length,
|
||||
numberOfCustomFields: partitionedFieldMetadata.custom.length,
|
||||
numberOfIndices: 1,
|
||||
numberOfIndicesChecked: 1,
|
||||
numberOfSameFamily: indexSameFamily,
|
||||
|
|
@ -289,7 +293,11 @@ const IndexPropertiesComponent: React.FC<Props> = ({
|
|||
unallowedValueFields: getIncompatibleValuesFields(
|
||||
partitionedFieldMetadata.incompatible
|
||||
),
|
||||
});
|
||||
};
|
||||
telemetryEvents.reportDataQualityIndexChecked?.(checkMetadata);
|
||||
|
||||
const result = { meta: checkMetadata, rollup: updatedRollup };
|
||||
postResult({ result, httpFetch, toasts, abortController: new AbortController() });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -297,6 +305,7 @@ const IndexPropertiesComponent: React.FC<Props> = ({
|
|||
docsCount,
|
||||
formatBytes,
|
||||
formatNumber,
|
||||
httpFetch,
|
||||
ilmPhase,
|
||||
indexId,
|
||||
indexName,
|
||||
|
|
@ -309,6 +318,7 @@ const IndexPropertiesComponent: React.FC<Props> = ({
|
|||
patternRollup,
|
||||
requestTime,
|
||||
telemetryEvents,
|
||||
toasts,
|
||||
unallowedValuesError,
|
||||
updatePatternRollup,
|
||||
]);
|
||||
|
|
|
|||
|
|
@ -35,6 +35,9 @@ import {
|
|||
getTotalSizeInBytes,
|
||||
hasValidTimestampMapping,
|
||||
isMappingCompatible,
|
||||
postResult,
|
||||
getResults,
|
||||
ResultData,
|
||||
} from './helpers';
|
||||
import {
|
||||
hostNameWithTextMapping,
|
||||
|
|
@ -77,6 +80,8 @@ import {
|
|||
PatternRollup,
|
||||
UnallowedValueCount,
|
||||
} from './types';
|
||||
import { httpServiceMock } from '@kbn/core-http-browser-mocks';
|
||||
import { notificationServiceMock } from '@kbn/core-notifications-browser-mocks';
|
||||
|
||||
const ecsMetadata: Record<string, EcsMetadata> = EcsFlat as unknown as Record<string, EcsMetadata>;
|
||||
|
||||
|
|
@ -1489,4 +1494,81 @@ describe('helpers', () => {
|
|||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('postResult', () => {
|
||||
const { fetch } = httpServiceMock.createStartContract();
|
||||
const { toasts } = notificationServiceMock.createStartContract();
|
||||
beforeEach(() => {
|
||||
fetch.mockClear();
|
||||
});
|
||||
|
||||
test('it posts the result', async () => {
|
||||
const result = { meta: {}, rollup: {} } as unknown as ResultData;
|
||||
await postResult({
|
||||
httpFetch: fetch,
|
||||
result,
|
||||
abortController: new AbortController(),
|
||||
toasts,
|
||||
});
|
||||
|
||||
expect(fetch).toHaveBeenCalledWith(
|
||||
'/internal/ecs_data_quality_dashboard/results',
|
||||
expect.objectContaining({
|
||||
method: 'POST',
|
||||
body: JSON.stringify(result),
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
test('it throws error', async () => {
|
||||
const result = { meta: {}, rollup: {} } as unknown as ResultData;
|
||||
fetch.mockRejectedValueOnce('test-error');
|
||||
await postResult({
|
||||
httpFetch: fetch,
|
||||
result,
|
||||
abortController: new AbortController(),
|
||||
toasts,
|
||||
});
|
||||
expect(toasts.addError).toHaveBeenCalledWith('test-error', { title: expect.any(String) });
|
||||
});
|
||||
});
|
||||
|
||||
describe('getResults', () => {
|
||||
const { fetch } = httpServiceMock.createStartContract();
|
||||
const { toasts } = notificationServiceMock.createStartContract();
|
||||
beforeEach(() => {
|
||||
fetch.mockClear();
|
||||
});
|
||||
|
||||
test('it gets the results', async () => {
|
||||
await getResults({
|
||||
httpFetch: fetch,
|
||||
abortController: new AbortController(),
|
||||
patterns: ['auditbeat-*', 'packetbeat-*'],
|
||||
toasts,
|
||||
});
|
||||
|
||||
expect(fetch).toHaveBeenCalledWith(
|
||||
'/internal/ecs_data_quality_dashboard/results',
|
||||
expect.objectContaining({
|
||||
method: 'GET',
|
||||
query: { patterns: 'auditbeat-*,packetbeat-*' },
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it('should catch error', async () => {
|
||||
fetch.mockRejectedValueOnce('test-error');
|
||||
|
||||
const results = await getResults({
|
||||
httpFetch: fetch,
|
||||
abortController: new AbortController(),
|
||||
patterns: ['auditbeat-*', 'packetbeat-*'],
|
||||
toasts,
|
||||
});
|
||||
|
||||
expect(toasts.addError).toHaveBeenCalledWith('test-error', { title: expect.any(String) });
|
||||
expect(results).toEqual([]);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -5,11 +5,13 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { HttpHandler } from '@kbn/core-http-browser';
|
||||
import type {
|
||||
IlmExplainLifecycleLifecycleExplain,
|
||||
IndicesStatsIndicesStats,
|
||||
} from '@elastic/elasticsearch/lib/api/types';
|
||||
import { has, sortBy } from 'lodash/fp';
|
||||
import { IToasts } from '@kbn/core-notifications-browser';
|
||||
import { getIlmPhase } from './data_quality_panel/pattern/helpers';
|
||||
import { getFillColor } from './data_quality_panel/tabs/summary_tab/helpers';
|
||||
|
||||
|
|
@ -17,6 +19,7 @@ import * as i18n from './translations';
|
|||
|
||||
import type {
|
||||
DataQualityCheckResult,
|
||||
DataQualityIndexCheckedParams,
|
||||
EcsMetadata,
|
||||
EnrichedFieldMetadata,
|
||||
ErrorSummary,
|
||||
|
|
@ -443,3 +446,58 @@ export const getErrorSummaries = (
|
|||
[]
|
||||
);
|
||||
};
|
||||
|
||||
export const RESULTS_API_ROUTE = '/internal/ecs_data_quality_dashboard/results';
|
||||
|
||||
export interface ResultData {
|
||||
meta: DataQualityIndexCheckedParams;
|
||||
rollup: PatternRollup;
|
||||
}
|
||||
|
||||
export async function postResult({
|
||||
result,
|
||||
httpFetch,
|
||||
toasts,
|
||||
abortController,
|
||||
}: {
|
||||
result: ResultData;
|
||||
httpFetch: HttpHandler;
|
||||
toasts: IToasts;
|
||||
abortController: AbortController;
|
||||
}): Promise<void> {
|
||||
try {
|
||||
await httpFetch<void>(RESULTS_API_ROUTE, {
|
||||
method: 'POST',
|
||||
signal: abortController.signal,
|
||||
version: INTERNAL_API_VERSION,
|
||||
body: JSON.stringify(result),
|
||||
});
|
||||
} catch (err) {
|
||||
toasts.addError(err, { title: i18n.POST_RESULT_ERROR_TITLE });
|
||||
}
|
||||
}
|
||||
|
||||
export async function getResults({
|
||||
patterns,
|
||||
httpFetch,
|
||||
toasts,
|
||||
abortController,
|
||||
}: {
|
||||
patterns: string[];
|
||||
httpFetch: HttpHandler;
|
||||
toasts: IToasts;
|
||||
abortController: AbortController;
|
||||
}): Promise<ResultData[]> {
|
||||
try {
|
||||
const results = await httpFetch<ResultData[]>(RESULTS_API_ROUTE, {
|
||||
method: 'GET',
|
||||
signal: abortController.signal,
|
||||
version: INTERNAL_API_VERSION,
|
||||
query: { patterns: patterns.join(',') },
|
||||
});
|
||||
return results;
|
||||
} catch (err) {
|
||||
toasts.addError(err, { title: i18n.GET_RESULTS_ERROR_TITLE });
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,6 +11,9 @@ import React from 'react';
|
|||
|
||||
import { TestProviders } from './mock/test_providers/test_providers';
|
||||
import { DataQualityPanel } from '.';
|
||||
import { notificationServiceMock } from '@kbn/core-notifications-browser-mocks';
|
||||
|
||||
const { toasts } = notificationServiceMock.createSetupContract();
|
||||
|
||||
describe('DataQualityPanel', () => {
|
||||
describe('when ILM phases are provided', () => {
|
||||
|
|
@ -20,7 +23,6 @@ describe('DataQualityPanel', () => {
|
|||
render(
|
||||
<TestProviders>
|
||||
<DataQualityPanel
|
||||
addSuccessToast={jest.fn()}
|
||||
canUserCreateAndReadCases={jest.fn()}
|
||||
defaultBytesFormat={''}
|
||||
defaultNumberFormat={''}
|
||||
|
|
@ -35,6 +37,7 @@ describe('DataQualityPanel', () => {
|
|||
reportDataQualityIndexChecked={jest.fn()}
|
||||
setLastChecked={jest.fn()}
|
||||
baseTheme={DARK_THEME}
|
||||
toasts={toasts}
|
||||
/>
|
||||
</TestProviders>
|
||||
);
|
||||
|
|
@ -56,7 +59,6 @@ describe('DataQualityPanel', () => {
|
|||
render(
|
||||
<TestProviders>
|
||||
<DataQualityPanel
|
||||
addSuccessToast={jest.fn()}
|
||||
canUserCreateAndReadCases={jest.fn()}
|
||||
defaultBytesFormat={''}
|
||||
defaultNumberFormat={''}
|
||||
|
|
@ -71,6 +73,7 @@ describe('DataQualityPanel', () => {
|
|||
reportDataQualityIndexChecked={jest.fn()}
|
||||
setLastChecked={jest.fn()}
|
||||
baseTheme={DARK_THEME}
|
||||
toasts={toasts}
|
||||
/>
|
||||
</TestProviders>
|
||||
);
|
||||
|
|
|
|||
|
|
@ -19,13 +19,14 @@ import type {
|
|||
} from '@elastic/charts';
|
||||
import React, { useCallback, useMemo } from 'react';
|
||||
|
||||
import type { IToasts } from '@kbn/core-notifications-browser';
|
||||
import { Body } from './data_quality_panel/body';
|
||||
import { DataQualityProvider } from './data_quality_panel/data_quality_context';
|
||||
import { EMPTY_STAT } from './helpers';
|
||||
import { ReportDataQualityCheckAllCompleted, ReportDataQualityIndexChecked } from './types';
|
||||
|
||||
interface Props {
|
||||
addSuccessToast: (toast: { title: string }) => void;
|
||||
toasts: IToasts;
|
||||
baseTheme: Theme;
|
||||
canUserCreateAndReadCases: () => boolean;
|
||||
defaultNumberFormat: string;
|
||||
|
|
@ -66,7 +67,7 @@ interface Props {
|
|||
|
||||
/** Renders the `Data Quality` dashboard content */
|
||||
const DataQualityPanelComponent: React.FC<Props> = ({
|
||||
addSuccessToast,
|
||||
toasts,
|
||||
baseTheme,
|
||||
canUserCreateAndReadCases,
|
||||
defaultBytesFormat,
|
||||
|
|
@ -103,11 +104,19 @@ const DataQualityPanelComponent: React.FC<Props> = ({
|
|||
[reportDataQualityCheckAllCompleted, reportDataQualityIndexChecked]
|
||||
);
|
||||
|
||||
const addSuccessToast = useCallback(
|
||||
(toast: { title: string }) => {
|
||||
toasts.addSuccess(toast);
|
||||
},
|
||||
[toasts]
|
||||
);
|
||||
|
||||
return (
|
||||
<DataQualityProvider
|
||||
httpFetch={httpFetch}
|
||||
telemetryEvents={telemetryEvents}
|
||||
isILMAvailable={isILMAvailable}
|
||||
toasts={toasts}
|
||||
>
|
||||
<Body
|
||||
addSuccessToast={addSuccessToast}
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@
|
|||
|
||||
import { actionTypeRegistryMock } from '@kbn/triggers-actions-ui-plugin/public/application/action_type_registry.mock';
|
||||
import { httpServiceMock } from '@kbn/core-http-browser-mocks';
|
||||
import { notificationServiceMock } from '@kbn/core-notifications-browser-mocks';
|
||||
import { AssistantAvailability, AssistantProvider } from '@kbn/elastic-assistant';
|
||||
import { I18nProvider } from '@kbn/i18n-react';
|
||||
import { euiDarkVars } from '@kbn/ui-theme';
|
||||
|
|
@ -26,6 +27,7 @@ window.scrollTo = jest.fn();
|
|||
/** A utility for wrapping children in the providers required to run tests */
|
||||
export const TestProvidersComponent: React.FC<Props> = ({ children, isILMAvailable = true }) => {
|
||||
const http = httpServiceMock.createSetupContract({ basePath: '/test' });
|
||||
const { toasts } = notificationServiceMock.createSetupContract();
|
||||
const actionTypeRegistry = actionTypeRegistryMock.create();
|
||||
const mockGetInitialConversations = jest.fn(() => ({}));
|
||||
const mockGetComments = jest.fn(() => []);
|
||||
|
|
@ -79,6 +81,7 @@ export const TestProvidersComponent: React.FC<Props> = ({ children, isILMAvailab
|
|||
>
|
||||
<DataQualityProvider
|
||||
httpFetch={http.fetch}
|
||||
toasts={toasts}
|
||||
isILMAvailable={isILMAvailable}
|
||||
telemetryEvents={mockTelemetryEvents}
|
||||
>
|
||||
|
|
|
|||
|
|
@ -292,3 +292,13 @@ export const WARM_PATTERN_TOOLTIP = ({ indices, pattern }: { indices: number; pa
|
|||
defaultMessage:
|
||||
'{indices} {indices, plural, =1 {index} other {indices}} matching the {pattern} pattern {indices, plural, =1 {is} other {are}} warm. Warm indices are no longer being updated but are still being queried.',
|
||||
});
|
||||
|
||||
export const POST_RESULT_ERROR_TITLE = i18n.translate(
|
||||
'securitySolutionPackages.ecsDataQualityDashboard.postResultErrorTitle',
|
||||
{ defaultMessage: 'Error writing saved data quality check results' }
|
||||
);
|
||||
|
||||
export const GET_RESULTS_ERROR_TITLE = i18n.translate(
|
||||
'securitySolutionPackages.ecsDataQualityDashboard.getResultErrorTitle',
|
||||
{ defaultMessage: 'Error reading saved data quality check results' }
|
||||
);
|
||||
|
|
|
|||
|
|
@ -142,19 +142,7 @@ export interface IndexToCheck {
|
|||
indexName: string;
|
||||
}
|
||||
|
||||
export type OnCheckCompleted = ({
|
||||
batchId,
|
||||
checkAllStartTime,
|
||||
error,
|
||||
formatBytes,
|
||||
formatNumber,
|
||||
indexName,
|
||||
isLastCheck,
|
||||
partitionedFieldMetadata,
|
||||
pattern,
|
||||
version,
|
||||
requestTime,
|
||||
}: {
|
||||
export type OnCheckCompleted = (param: {
|
||||
batchId: string;
|
||||
checkAllStartTime: number;
|
||||
error: string | null;
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import { DataQualityProvider } from '../data_quality_panel/data_quality_context'
|
|||
import { mockIlmExplain } from '../mock/ilm_explain/mock_ilm_explain';
|
||||
import { ERROR_LOADING_ILM_EXPLAIN } from '../translations';
|
||||
import { useIlmExplain, UseIlmExplain } from '.';
|
||||
import { notificationServiceMock } from '@kbn/core-notifications-browser-mocks';
|
||||
|
||||
const mockHttpFetch = jest.fn();
|
||||
const mockReportDataQualityIndexChecked = jest.fn();
|
||||
|
|
@ -20,6 +21,7 @@ const mockTelemetryEvents = {
|
|||
reportDataQualityIndexChecked: mockReportDataQualityIndexChecked,
|
||||
reportDataQualityCheckAllCompleted: mockReportDataQualityCheckAllClicked,
|
||||
};
|
||||
const { toasts } = notificationServiceMock.createSetupContract();
|
||||
const ContextWrapper: React.FC<{ children: React.ReactNode; isILMAvailable: boolean }> = ({
|
||||
children,
|
||||
isILMAvailable = true,
|
||||
|
|
@ -28,6 +30,7 @@ const ContextWrapper: React.FC<{ children: React.ReactNode; isILMAvailable: bool
|
|||
httpFetch={mockHttpFetch}
|
||||
telemetryEvents={mockTelemetryEvents}
|
||||
isILMAvailable={isILMAvailable}
|
||||
toasts={toasts}
|
||||
>
|
||||
{children}
|
||||
</DataQualityProvider>
|
||||
|
|
@ -76,6 +79,7 @@ describe('useIlmExplain', () => {
|
|||
httpFetch={mockHttpFetch}
|
||||
telemetryEvents={mockTelemetryEvents}
|
||||
isILMAvailable={false}
|
||||
toasts={toasts}
|
||||
>
|
||||
{children}
|
||||
</DataQualityProvider>
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import { DataQualityProvider } from '../data_quality_panel/data_quality_context'
|
|||
import { mockMappingsResponse } from '../mock/mappings_response/mock_mappings_response';
|
||||
import { ERROR_LOADING_MAPPINGS } from '../translations';
|
||||
import { useMappings, UseMappings } from '.';
|
||||
import { notificationServiceMock } from '@kbn/core-notifications-browser-mocks';
|
||||
|
||||
const mockHttpFetch = jest.fn();
|
||||
const mockReportDataQualityIndexChecked = jest.fn();
|
||||
|
|
@ -20,12 +21,14 @@ const mockTelemetryEvents = {
|
|||
reportDataQualityIndexChecked: mockReportDataQualityIndexChecked,
|
||||
reportDataQualityCheckAllCompleted: mockReportDataQualityCheckAllClicked,
|
||||
};
|
||||
const { toasts } = notificationServiceMock.createSetupContract();
|
||||
|
||||
const ContextWrapper: React.FC = ({ children }) => (
|
||||
<DataQualityProvider
|
||||
httpFetch={mockHttpFetch}
|
||||
telemetryEvents={mockTelemetryEvents}
|
||||
isILMAvailable={true}
|
||||
toasts={toasts}
|
||||
>
|
||||
{children}
|
||||
</DataQualityProvider>
|
||||
|
|
|
|||
|
|
@ -13,7 +13,6 @@ import {
|
|||
getTotalIndices,
|
||||
getTotalIndicesChecked,
|
||||
getTotalSameFamily,
|
||||
onPatternRollupUpdated,
|
||||
updateResultOnCheckCompleted,
|
||||
} from './helpers';
|
||||
import { auditbeatWithAllResults } from '../mock/pattern_rollup/mock_auditbeat_pattern_rollup';
|
||||
|
|
@ -166,21 +165,6 @@ describe('helpers', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('onPatternRollupUpdated', () => {
|
||||
test('it returns a new collection with the updated rollup', () => {
|
||||
const before: Record<string, PatternRollup> = {
|
||||
'auditbeat-*': auditbeatWithAllResults,
|
||||
};
|
||||
|
||||
expect(
|
||||
onPatternRollupUpdated({
|
||||
patternRollup: mockPacketbeatPatternRollup,
|
||||
patternRollups: before,
|
||||
})
|
||||
).toEqual(patternRollups);
|
||||
});
|
||||
});
|
||||
|
||||
describe('updateResultOnCheckCompleted', () => {
|
||||
const packetbeatStats861: IndicesStatsIndicesStats =
|
||||
mockPacketbeatPatternRollup.stats != null
|
||||
|
|
|
|||
|
|
@ -87,17 +87,6 @@ export const getTotalIndicesChecked = (patternRollups: Record<string, PatternRol
|
|||
);
|
||||
};
|
||||
|
||||
export const onPatternRollupUpdated = ({
|
||||
patternRollup,
|
||||
patternRollups,
|
||||
}: {
|
||||
patternRollup: PatternRollup;
|
||||
patternRollups: Record<string, PatternRollup>;
|
||||
}): Record<string, PatternRollup> => ({
|
||||
...patternRollups,
|
||||
[patternRollup.pattern]: patternRollup,
|
||||
});
|
||||
|
||||
export const updateResultOnCheckCompleted = ({
|
||||
error,
|
||||
formatBytes,
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@
|
|||
import { useCallback, useEffect, useMemo, useState } from 'react';
|
||||
import { EcsVersion } from '@kbn/ecs';
|
||||
|
||||
import { isEmpty } from 'lodash/fp';
|
||||
import {
|
||||
getTotalDocsCount,
|
||||
getTotalIncompatible,
|
||||
|
|
@ -15,12 +16,18 @@ import {
|
|||
getTotalIndicesChecked,
|
||||
getTotalSameFamily,
|
||||
getTotalSizeInBytes,
|
||||
onPatternRollupUpdated,
|
||||
updateResultOnCheckCompleted,
|
||||
} from './helpers';
|
||||
|
||||
import type { OnCheckCompleted, PatternRollup } from '../types';
|
||||
import { getDocsCount, getIndexId, getSizeInBytes, getTotalPatternSameFamily } from '../helpers';
|
||||
import {
|
||||
getDocsCount,
|
||||
getIndexId,
|
||||
getResults,
|
||||
getSizeInBytes,
|
||||
getTotalPatternSameFamily,
|
||||
postResult,
|
||||
} from '../helpers';
|
||||
import { getIlmPhase, getIndexIncompatible } from '../data_quality_panel/pattern/helpers';
|
||||
import { useDataQualityContext } from '../data_quality_panel/data_quality_context';
|
||||
import {
|
||||
|
|
@ -53,15 +60,60 @@ interface UseResultsRollup {
|
|||
updatePatternRollup: (patternRollup: PatternRollup) => void;
|
||||
}
|
||||
|
||||
const useStoredPatternRollups = (patterns: string[]) => {
|
||||
const { httpFetch, toasts } = useDataQualityContext();
|
||||
const [storedRollups, setStoredRollups] = useState<Record<string, PatternRollup>>({});
|
||||
|
||||
useEffect(() => {
|
||||
if (isEmpty(patterns)) {
|
||||
return;
|
||||
}
|
||||
|
||||
let ignore = false;
|
||||
const abortController = new AbortController();
|
||||
const fetchStoredRollups = async () => {
|
||||
const results = await getResults({ httpFetch, abortController, patterns, toasts });
|
||||
if (results?.length && !ignore) {
|
||||
setStoredRollups(Object.fromEntries(results.map(({ rollup }) => [rollup.pattern, rollup])));
|
||||
}
|
||||
};
|
||||
|
||||
fetchStoredRollups();
|
||||
return () => {
|
||||
ignore = true;
|
||||
};
|
||||
}, [httpFetch, patterns, toasts]);
|
||||
|
||||
return storedRollups;
|
||||
};
|
||||
|
||||
export const useResultsRollup = ({ ilmPhases, patterns }: Props): UseResultsRollup => {
|
||||
const { httpFetch, toasts } = useDataQualityContext();
|
||||
const [patternIndexNames, setPatternIndexNames] = useState<Record<string, string[]>>({});
|
||||
const [patternRollups, setPatternRollups] = useState<Record<string, PatternRollup>>({});
|
||||
|
||||
const storedPatternsRollups = useStoredPatternRollups(patterns);
|
||||
|
||||
useEffect(() => {
|
||||
if (!isEmpty(storedPatternsRollups)) {
|
||||
setPatternRollups((current) => ({ ...current, ...storedPatternsRollups }));
|
||||
}
|
||||
}, [storedPatternsRollups]);
|
||||
|
||||
const updatePatternRollups = useCallback(
|
||||
(updateRollups: (current: Record<string, PatternRollup>) => Record<string, PatternRollup>) => {
|
||||
setPatternRollups((current) => updateRollups(current));
|
||||
},
|
||||
[]
|
||||
);
|
||||
|
||||
const { telemetryEvents, isILMAvailable } = useDataQualityContext();
|
||||
const updatePatternRollup = useCallback((patternRollup: PatternRollup) => {
|
||||
setPatternRollups((current) =>
|
||||
onPatternRollupUpdated({ patternRollup, patternRollups: current })
|
||||
);
|
||||
}, []);
|
||||
const updatePatternRollup = useCallback(
|
||||
(patternRollup: PatternRollup) => {
|
||||
updatePatternRollups((current) => ({ ...current, [patternRollup.pattern]: patternRollup }));
|
||||
},
|
||||
[updatePatternRollups]
|
||||
);
|
||||
|
||||
const totalDocsCount = useMemo(() => getTotalDocsCount(patternRollups), [patternRollups]);
|
||||
const totalIncompatible = useMemo(() => getTotalIncompatible(patternRollups), [patternRollups]);
|
||||
|
|
@ -75,10 +127,7 @@ export const useResultsRollup = ({ ilmPhases, patterns }: Props): UseResultsRoll
|
|||
|
||||
const updatePatternIndexNames = useCallback(
|
||||
({ indexNames, pattern }: { indexNames: string[]; pattern: string }) => {
|
||||
setPatternIndexNames((current) => ({
|
||||
...current,
|
||||
[pattern]: indexNames,
|
||||
}));
|
||||
setPatternIndexNames((current) => ({ ...current, [pattern]: indexNames }));
|
||||
},
|
||||
[]
|
||||
);
|
||||
|
|
@ -96,11 +145,8 @@ export const useResultsRollup = ({ ilmPhases, patterns }: Props): UseResultsRoll
|
|||
requestTime,
|
||||
isLastCheck,
|
||||
}) => {
|
||||
const indexId = getIndexId({ indexName, stats: patternRollups[pattern].stats });
|
||||
const ilmExplain = patternRollups[pattern].ilmExplain;
|
||||
|
||||
setPatternRollups((current) => {
|
||||
const updated = updateResultOnCheckCompleted({
|
||||
setPatternRollups((currentPatternRollups) => {
|
||||
const updatedRollups = updateResultOnCheckCompleted({
|
||||
error,
|
||||
formatBytes,
|
||||
formatNumber,
|
||||
|
|
@ -108,19 +154,23 @@ export const useResultsRollup = ({ ilmPhases, patterns }: Props): UseResultsRoll
|
|||
isILMAvailable,
|
||||
partitionedFieldMetadata,
|
||||
pattern,
|
||||
patternRollups: current,
|
||||
patternRollups: currentPatternRollups,
|
||||
});
|
||||
|
||||
const updatedRollup = updatedRollups[pattern];
|
||||
const { stats, results, ilmExplain } = updatedRollup;
|
||||
const indexId = getIndexId({ indexName, stats });
|
||||
|
||||
if (
|
||||
indexId != null &&
|
||||
updated[pattern].stats &&
|
||||
updated[pattern].results &&
|
||||
stats &&
|
||||
results &&
|
||||
ilmExplain &&
|
||||
requestTime != null &&
|
||||
requestTime > 0 &&
|
||||
partitionedFieldMetadata &&
|
||||
ilmExplain
|
||||
partitionedFieldMetadata
|
||||
) {
|
||||
telemetryEvents.reportDataQualityIndexChecked?.({
|
||||
const metadata = {
|
||||
batchId,
|
||||
ecsVersion: EcsVersion,
|
||||
errorCount: error ? 1 : 0,
|
||||
|
|
@ -128,16 +178,19 @@ export const useResultsRollup = ({ ilmPhases, patterns }: Props): UseResultsRoll
|
|||
indexId,
|
||||
indexName,
|
||||
isCheckAll: true,
|
||||
numberOfDocuments: getDocsCount({ indexName, stats: updated[pattern].stats }),
|
||||
numberOfDocuments: getDocsCount({ indexName, stats }),
|
||||
numberOfFields: partitionedFieldMetadata.all.length,
|
||||
numberOfIncompatibleFields: getIndexIncompatible({
|
||||
indexName,
|
||||
results: updated[pattern].results,
|
||||
results,
|
||||
}),
|
||||
numberOfEcsFields: partitionedFieldMetadata.ecsCompliant.length,
|
||||
numberOfCustomFields: partitionedFieldMetadata.custom.length,
|
||||
numberOfIndices: 1,
|
||||
numberOfIndicesChecked: 1,
|
||||
numberOfSameFamily: getTotalPatternSameFamily(updated[pattern].results),
|
||||
numberOfSameFamily: getTotalPatternSameFamily(results),
|
||||
sameFamilyFields: getSameFamilyFields(partitionedFieldMetadata.sameFamily),
|
||||
sizeInBytes: getSizeInBytes({ stats: updated[pattern].stats, indexName }),
|
||||
sizeInBytes: getSizeInBytes({ stats, indexName }),
|
||||
timeConsumedMs: requestTime,
|
||||
unallowedMappingFields: getIncompatibleMappingsFields(
|
||||
partitionedFieldMetadata.incompatible
|
||||
|
|
@ -145,7 +198,11 @@ export const useResultsRollup = ({ ilmPhases, patterns }: Props): UseResultsRoll
|
|||
unallowedValueFields: getIncompatibleValuesFields(
|
||||
partitionedFieldMetadata.incompatible
|
||||
),
|
||||
});
|
||||
};
|
||||
telemetryEvents.reportDataQualityIndexChecked?.(metadata);
|
||||
|
||||
const result = { meta: metadata, rollup: updatedRollup };
|
||||
postResult({ result, httpFetch, toasts, abortController: new AbortController() });
|
||||
}
|
||||
|
||||
if (isLastCheck) {
|
||||
|
|
@ -153,19 +210,19 @@ export const useResultsRollup = ({ ilmPhases, patterns }: Props): UseResultsRoll
|
|||
batchId,
|
||||
ecsVersion: EcsVersion,
|
||||
isCheckAll: true,
|
||||
numberOfDocuments: getTotalDocsCount(updated),
|
||||
numberOfIncompatibleFields: getTotalIncompatible(updated),
|
||||
numberOfIndices: getTotalIndices(updated),
|
||||
numberOfIndicesChecked: getTotalIndicesChecked(updated),
|
||||
numberOfSameFamily: getTotalSameFamily(updated),
|
||||
sizeInBytes: getTotalSizeInBytes(updated),
|
||||
numberOfDocuments: getTotalDocsCount(updatedRollups),
|
||||
numberOfIncompatibleFields: getTotalIncompatible(updatedRollups),
|
||||
numberOfIndices: getTotalIndices(updatedRollups),
|
||||
numberOfIndicesChecked: getTotalIndicesChecked(updatedRollups),
|
||||
numberOfSameFamily: getTotalSameFamily(updatedRollups),
|
||||
sizeInBytes: getTotalSizeInBytes(updatedRollups),
|
||||
timeConsumedMs: Date.now() - checkAllStartTime,
|
||||
});
|
||||
}
|
||||
return updated;
|
||||
return updatedRollups;
|
||||
});
|
||||
},
|
||||
[isILMAvailable, patternRollups, telemetryEvents]
|
||||
[httpFetch, isILMAvailable, telemetryEvents, toasts]
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import { DataQualityProvider } from '../data_quality_panel/data_quality_context'
|
|||
import { mockStatsGreenIndex } from '../mock/stats/mock_stats_green_index';
|
||||
import { ERROR_LOADING_STATS } from '../translations';
|
||||
import { useStats, UseStats } from '.';
|
||||
import { notificationServiceMock } from '@kbn/core-notifications-browser-mocks';
|
||||
|
||||
const mockHttpFetch = jest.fn();
|
||||
const mockReportDataQualityIndexChecked = jest.fn();
|
||||
|
|
@ -20,12 +21,14 @@ const mockTelemetryEvents = {
|
|||
reportDataQualityIndexChecked: mockReportDataQualityIndexChecked,
|
||||
reportDataQualityCheckAllCompleted: mockReportDataQualityCheckAllClicked,
|
||||
};
|
||||
const { toasts } = notificationServiceMock.createSetupContract();
|
||||
|
||||
const ContextWrapper: React.FC = ({ children }) => (
|
||||
<DataQualityProvider
|
||||
httpFetch={mockHttpFetch}
|
||||
telemetryEvents={mockTelemetryEvents}
|
||||
isILMAvailable={true}
|
||||
toasts={toasts}
|
||||
>
|
||||
{children}
|
||||
</DataQualityProvider>
|
||||
|
|
@ -36,6 +39,7 @@ const ContextWrapperILMNotAvailable: React.FC = ({ children }) => (
|
|||
httpFetch={mockHttpFetch}
|
||||
telemetryEvents={mockTelemetryEvents}
|
||||
isILMAvailable={false}
|
||||
toasts={toasts}
|
||||
>
|
||||
{children}
|
||||
</DataQualityProvider>
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ import { mockUnallowedValuesResponse } from '../mock/unallowed_values/mock_unall
|
|||
import { ERROR_LOADING_UNALLOWED_VALUES } from '../translations';
|
||||
import { EcsMetadata, UnallowedValueRequestItem } from '../types';
|
||||
import { useUnallowedValues, UseUnallowedValues } from '.';
|
||||
import { notificationServiceMock } from '@kbn/core-notifications-browser-mocks';
|
||||
|
||||
const mockHttpFetch = jest.fn();
|
||||
const mockReportDataQualityIndexChecked = jest.fn();
|
||||
|
|
@ -23,12 +24,14 @@ const mockTelemetryEvents = {
|
|||
reportDataQualityIndexChecked: mockReportDataQualityIndexChecked,
|
||||
reportDataQualityCheckAllCompleted: mockReportDataQualityCheckAllClicked,
|
||||
};
|
||||
const { toasts } = notificationServiceMock.createSetupContract();
|
||||
|
||||
const ContextWrapper: React.FC = ({ children }) => (
|
||||
<DataQualityProvider
|
||||
httpFetch={mockHttpFetch}
|
||||
telemetryEvents={mockTelemetryEvents}
|
||||
isILMAvailable={true}
|
||||
toasts={toasts}
|
||||
>
|
||||
{children}
|
||||
</DataQualityProvider>
|
||||
|
|
|
|||
|
|
@ -25,5 +25,7 @@
|
|||
"@kbn/elastic-assistant",
|
||||
"@kbn/triggers-actions-ui-plugin",
|
||||
"@kbn/core",
|
||||
"@kbn/core-notifications-browser",
|
||||
"@kbn/core-notifications-browser-mocks",
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,4 +13,5 @@ export const GET_INDEX_STATS = `${BASE_PATH}/stats/{pattern}`;
|
|||
export const GET_INDEX_MAPPINGS = `${BASE_PATH}/mappings/{pattern}`;
|
||||
export const GET_UNALLOWED_FIELD_VALUES = `${BASE_PATH}/unallowed_field_values`;
|
||||
export const GET_ILM_EXPLAIN = `${BASE_PATH}/ilm_explain/{pattern}`;
|
||||
export const RESULTS_ROUTE_PATH = `${BASE_PATH}/results`;
|
||||
export const INTERNAL_API_VERSION = '1';
|
||||
|
|
|
|||
|
|
@ -8,7 +8,10 @@
|
|||
"server": true,
|
||||
"browser": false,
|
||||
"requiredPlugins": [
|
||||
"data"
|
||||
"data",
|
||||
],
|
||||
"optionalPlugins": [
|
||||
"spaces",
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -39,6 +39,9 @@ const createAppClientMock = () => ({});
|
|||
const createRequestContextMock = (clients: MockClients = createMockClients()) => {
|
||||
return {
|
||||
core: clients.core,
|
||||
dataQualityDashboard: {
|
||||
getResultsIndexName: jest.fn(() => Promise.resolve('mock_results_index_name')),
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import { SearchRequest } from '@elastic/elasticsearch/lib/api/types';
|
||||
import type { SearchRequest } from '@elastic/elasticsearch/lib/api/types';
|
||||
|
||||
export const getRequestBody = ({
|
||||
indexPattern,
|
||||
|
|
|
|||
|
|
@ -5,11 +5,11 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import {
|
||||
import type {
|
||||
MsearchMultisearchHeader,
|
||||
MsearchMultisearchBody,
|
||||
} from '@elastic/elasticsearch/lib/api/types';
|
||||
import { AllowedValuesInputs } from '../schemas/get_unallowed_field_values';
|
||||
import type { AllowedValuesInputs } from '../schemas/get_unallowed_field_values';
|
||||
|
||||
export const getMSearchRequestHeader = (indexName: string): MsearchMultisearchHeader => ({
|
||||
expand_wildcards: ['open'],
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import { PluginInitializerContext } from '@kbn/core/server';
|
||||
import type { PluginInitializerContext } from '@kbn/core/server';
|
||||
|
||||
// This exports static code and TypeScript types,
|
||||
// as well as, Kibana Platform `plugin()` initializer.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,121 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { ResultsDataStream } from './results_data_stream';
|
||||
import { Subject } from 'rxjs';
|
||||
import type { InstallParams } from '@kbn/data-stream-adapter';
|
||||
import { DataStreamSpacesAdapter } from '@kbn/data-stream-adapter';
|
||||
import { elasticsearchServiceMock } from '@kbn/core-elasticsearch-server-mocks';
|
||||
import { loggerMock } from '@kbn/logging-mocks';
|
||||
|
||||
jest.mock('@kbn/data-stream-adapter');
|
||||
|
||||
const MockedDataStreamSpacesAdapter = DataStreamSpacesAdapter as unknown as jest.MockedClass<
|
||||
typeof DataStreamSpacesAdapter
|
||||
>;
|
||||
|
||||
const esClient = elasticsearchServiceMock.createStart().client.asInternalUser;
|
||||
|
||||
describe('ResultsDataStream', () => {
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
describe('constructor', () => {
|
||||
it('should create DataStreamSpacesAdapter', () => {
|
||||
new ResultsDataStream({ kibanaVersion: '8.13.0' });
|
||||
expect(MockedDataStreamSpacesAdapter).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('should create component templates', () => {
|
||||
new ResultsDataStream({ kibanaVersion: '8.13.0' });
|
||||
const [dataStreamSpacesAdapter] = MockedDataStreamSpacesAdapter.mock.instances;
|
||||
expect(dataStreamSpacesAdapter.setComponentTemplate).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ name: '.kibana-data-quality-dashboard-ecs-mappings' })
|
||||
);
|
||||
expect(dataStreamSpacesAdapter.setComponentTemplate).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ name: '.kibana-data-quality-dashboard-results-mappings' })
|
||||
);
|
||||
});
|
||||
|
||||
it('should create index templates', () => {
|
||||
new ResultsDataStream({ kibanaVersion: '8.13.0' });
|
||||
const [dataStreamSpacesAdapter] = MockedDataStreamSpacesAdapter.mock.instances;
|
||||
expect(dataStreamSpacesAdapter.setIndexTemplate).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ name: '.kibana-data-quality-dashboard-results-index-template' })
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('install', () => {
|
||||
it('should install data stream', async () => {
|
||||
const resultsDataStream = new ResultsDataStream({ kibanaVersion: '8.13.0' });
|
||||
const params: InstallParams = {
|
||||
esClient,
|
||||
logger: loggerMock.create(),
|
||||
pluginStop$: new Subject(),
|
||||
};
|
||||
await resultsDataStream.install(params);
|
||||
const [dataStreamSpacesAdapter] = MockedDataStreamSpacesAdapter.mock.instances;
|
||||
expect(dataStreamSpacesAdapter.install).toHaveBeenCalledWith(params);
|
||||
});
|
||||
|
||||
it('should log error', async () => {
|
||||
const resultsDataStream = new ResultsDataStream({ kibanaVersion: '8.13.0' });
|
||||
const params: InstallParams = {
|
||||
esClient,
|
||||
logger: loggerMock.create(),
|
||||
pluginStop$: new Subject(),
|
||||
};
|
||||
const [dataStreamSpacesAdapter] = MockedDataStreamSpacesAdapter.mock.instances;
|
||||
const error = new Error('test-error');
|
||||
(dataStreamSpacesAdapter.install as jest.Mock).mockRejectedValueOnce(error);
|
||||
|
||||
await resultsDataStream.install(params);
|
||||
expect(params.logger.error).toHaveBeenCalledWith(expect.any(String), error);
|
||||
});
|
||||
});
|
||||
|
||||
describe('installSpace', () => {
|
||||
it('should install space', async () => {
|
||||
const resultsDataStream = new ResultsDataStream({ kibanaVersion: '8.13.0' });
|
||||
const params: InstallParams = {
|
||||
esClient,
|
||||
logger: loggerMock.create(),
|
||||
pluginStop$: new Subject(),
|
||||
};
|
||||
const [dataStreamSpacesAdapter] = MockedDataStreamSpacesAdapter.mock.instances;
|
||||
(dataStreamSpacesAdapter.install as jest.Mock).mockResolvedValueOnce(undefined);
|
||||
|
||||
await resultsDataStream.install(params);
|
||||
await resultsDataStream.installSpace('space1');
|
||||
|
||||
expect(dataStreamSpacesAdapter.getInstalledSpaceName).toHaveBeenCalledWith('space1');
|
||||
expect(dataStreamSpacesAdapter.installSpace).toHaveBeenCalledWith('space1');
|
||||
});
|
||||
|
||||
it('should not install space if install not executed', async () => {
|
||||
const resultsDataStream = new ResultsDataStream({ kibanaVersion: '8.13.0' });
|
||||
expect(resultsDataStream.installSpace('space1')).rejects.toThrowError();
|
||||
});
|
||||
|
||||
it('should throw error if main install had error', async () => {
|
||||
const resultsDataStream = new ResultsDataStream({ kibanaVersion: '8.13.0' });
|
||||
const params: InstallParams = {
|
||||
esClient,
|
||||
logger: loggerMock.create(),
|
||||
pluginStop$: new Subject(),
|
||||
};
|
||||
const [dataStreamSpacesAdapter] = MockedDataStreamSpacesAdapter.mock.instances;
|
||||
const error = new Error('test-error');
|
||||
(dataStreamSpacesAdapter.install as jest.Mock).mockRejectedValueOnce(error);
|
||||
await resultsDataStream.install(params);
|
||||
|
||||
expect(resultsDataStream.installSpace('space1')).rejects.toThrowError(error);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,67 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { DataStreamSpacesAdapter, ecsFieldMap, type InstallParams } from '@kbn/data-stream-adapter';
|
||||
import { resultsFieldMap } from './results_field_map';
|
||||
|
||||
const TOTAL_FIELDS_LIMIT = 2500;
|
||||
|
||||
const RESULTS_DATA_STREAM_NAME = '.kibana-data-quality-dashboard-results';
|
||||
|
||||
const RESULTS_INDEX_TEMPLATE_NAME = '.kibana-data-quality-dashboard-results-index-template';
|
||||
const RESULTS_COMPONENT_TEMPLATE_NAME = '.kibana-data-quality-dashboard-results-mappings';
|
||||
const ECS_COMPONENT_TEMPLATE_NAME = '.kibana-data-quality-dashboard-ecs-mappings';
|
||||
|
||||
export class ResultsDataStream {
|
||||
private readonly dataStream: DataStreamSpacesAdapter;
|
||||
private installPromise?: Promise<void>;
|
||||
|
||||
constructor({ kibanaVersion }: { kibanaVersion: string }) {
|
||||
this.dataStream = new DataStreamSpacesAdapter(RESULTS_DATA_STREAM_NAME, {
|
||||
kibanaVersion,
|
||||
totalFieldsLimit: TOTAL_FIELDS_LIMIT,
|
||||
});
|
||||
this.dataStream.setComponentTemplate({
|
||||
name: ECS_COMPONENT_TEMPLATE_NAME,
|
||||
fieldMap: ecsFieldMap,
|
||||
});
|
||||
this.dataStream.setComponentTemplate({
|
||||
name: RESULTS_COMPONENT_TEMPLATE_NAME,
|
||||
fieldMap: resultsFieldMap,
|
||||
});
|
||||
|
||||
this.dataStream.setIndexTemplate({
|
||||
name: RESULTS_INDEX_TEMPLATE_NAME,
|
||||
componentTemplateRefs: [RESULTS_COMPONENT_TEMPLATE_NAME, ECS_COMPONENT_TEMPLATE_NAME],
|
||||
});
|
||||
}
|
||||
|
||||
async install(params: InstallParams) {
|
||||
try {
|
||||
this.installPromise = this.dataStream.install(params);
|
||||
await this.installPromise;
|
||||
} catch (err) {
|
||||
params.logger.error(
|
||||
`Error installing results data stream. Data quality dashboard persistence may be impacted.- ${err.message}`,
|
||||
err
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async installSpace(spaceId: string): Promise<string> {
|
||||
if (!this.installPromise) {
|
||||
throw new Error('Results data stream not installed');
|
||||
}
|
||||
// wait for install to complete, may reject if install failed, routes should handle this
|
||||
await this.installPromise;
|
||||
let dataStreamName = await this.dataStream.getInstalledSpaceName(spaceId);
|
||||
if (!dataStreamName) {
|
||||
dataStreamName = await this.dataStream.installSpace(spaceId);
|
||||
}
|
||||
return dataStreamName;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { FieldMap } from '@kbn/data-stream-adapter';
|
||||
|
||||
export const resultsFieldMap: FieldMap = {
|
||||
'meta.batchId': { type: 'keyword', required: true },
|
||||
'meta.ecsVersion': { type: 'keyword', required: true },
|
||||
'meta.errorCount': { type: 'long', required: true },
|
||||
'meta.ilmPhase': { type: 'keyword', required: true },
|
||||
'meta.indexId': { type: 'keyword', required: true },
|
||||
'meta.indexName': { type: 'keyword', required: true },
|
||||
'meta.isCheckAll': { type: 'boolean', required: true },
|
||||
'meta.numberOfDocuments': { type: 'long', required: true },
|
||||
'meta.numberOfFields': { type: 'long', required: true },
|
||||
'meta.numberOfIncompatibleFields': { type: 'long', required: true },
|
||||
'meta.numberOfEcsFields': { type: 'long', required: true },
|
||||
'meta.numberOfCustomFields': { type: 'long', required: true },
|
||||
'meta.numberOfIndices': { type: 'long', required: true },
|
||||
'meta.numberOfIndicesChecked': { type: 'long', required: true },
|
||||
'meta.numberOfSameFamily': { type: 'long', required: true },
|
||||
'meta.sameFamilyFields': { type: 'keyword', required: true, array: true },
|
||||
'meta.sizeInBytes': { type: 'long', required: true },
|
||||
'meta.timeConsumedMs': { type: 'long', required: true },
|
||||
'meta.unallowedMappingFields': { type: 'keyword', required: true, array: true },
|
||||
'meta.unallowedValueFields': { type: 'keyword', required: true, array: true },
|
||||
'rollup.docsCount': { type: 'long', required: true },
|
||||
'rollup.error': { type: 'text', required: false },
|
||||
'rollup.ilmExplainPhaseCounts': { type: 'object', required: false },
|
||||
'rollup.indices': { type: 'long', required: true },
|
||||
'rollup.pattern': { type: 'keyword', required: true },
|
||||
'rollup.sizeInBytes': { type: 'long', required: true },
|
||||
'rollup.ilmExplain': { type: 'object', required: true, array: true },
|
||||
'rollup.stats': { type: 'object', required: true, array: true },
|
||||
'rollup.results': { type: 'object', required: true, array: true },
|
||||
};
|
||||
|
|
@ -5,7 +5,7 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import { IlmExplainLifecycleResponse } from '@elastic/elasticsearch/lib/api/types';
|
||||
import type { IlmExplainLifecycleResponse } from '@elastic/elasticsearch/lib/api/types';
|
||||
import type { IScopedClusterClient } from '@kbn/core/server';
|
||||
|
||||
export const fetchILMExplain = (
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import { IndicesGetMappingIndexMappingRecord } from '@elastic/elasticsearch/lib/api/types';
|
||||
import type { IndicesGetMappingIndexMappingRecord } from '@elastic/elasticsearch/lib/api/types';
|
||||
import type { IScopedClusterClient } from '@kbn/core/server';
|
||||
|
||||
export const fetchMappings = (
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import { IndicesStatsResponse } from '@elastic/elasticsearch/lib/api/types';
|
||||
import type { IndicesStatsResponse } from '@elastic/elasticsearch/lib/api/types';
|
||||
import type { IScopedClusterClient } from '@kbn/core/server';
|
||||
|
||||
export const fetchStats = (
|
||||
|
|
|
|||
|
|
@ -6,12 +6,12 @@
|
|||
*/
|
||||
import type { ElasticsearchClient } from '@kbn/core/server';
|
||||
|
||||
import { MsearchRequestItem } from '@elastic/elasticsearch/lib/api/types';
|
||||
import type { MsearchRequestItem } from '@elastic/elasticsearch/lib/api/types';
|
||||
import {
|
||||
getMSearchRequestBody,
|
||||
getMSearchRequestHeader,
|
||||
} from '../helpers/get_unallowed_field_requests';
|
||||
import { GetUnallowedFieldValuesInputs } from '../schemas/get_unallowed_field_values';
|
||||
import type { GetUnallowedFieldValuesInputs } from '../schemas/get_unallowed_field_values';
|
||||
|
||||
export const getUnallowedFieldValues = (
|
||||
esClient: ElasticsearchClient,
|
||||
|
|
|
|||
|
|
@ -5,34 +5,77 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import { PluginInitializerContext, CoreSetup, CoreStart, Plugin, Logger } from '@kbn/core/server';
|
||||
import type {
|
||||
PluginInitializerContext,
|
||||
CoreSetup,
|
||||
CoreStart,
|
||||
Plugin,
|
||||
Logger,
|
||||
} from '@kbn/core/server';
|
||||
|
||||
import { EcsDataQualityDashboardPluginSetup, EcsDataQualityDashboardPluginStart } from './types';
|
||||
import { ReplaySubject, type Subject } from 'rxjs';
|
||||
import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants';
|
||||
import type {
|
||||
EcsDataQualityDashboardPluginSetup,
|
||||
EcsDataQualityDashboardPluginStart,
|
||||
PluginSetupDependencies,
|
||||
DataQualityDashboardRequestHandlerContext,
|
||||
} from './types';
|
||||
import {
|
||||
getILMExplainRoute,
|
||||
getIndexMappingsRoute,
|
||||
getIndexStatsRoute,
|
||||
getUnallowedFieldValuesRoute,
|
||||
resultsRoutes,
|
||||
} from './routes';
|
||||
import { ResultsDataStream } from './lib/data_stream/results_data_stream';
|
||||
|
||||
export class EcsDataQualityDashboardPlugin
|
||||
implements Plugin<EcsDataQualityDashboardPluginSetup, EcsDataQualityDashboardPluginStart>
|
||||
{
|
||||
private readonly logger: Logger;
|
||||
private readonly resultsDataStream: ResultsDataStream;
|
||||
private pluginStop$: Subject<void>;
|
||||
|
||||
constructor(initializerContext: PluginInitializerContext) {
|
||||
this.logger = initializerContext.logger.get();
|
||||
this.pluginStop$ = new ReplaySubject(1);
|
||||
this.resultsDataStream = new ResultsDataStream({
|
||||
kibanaVersion: initializerContext.env.packageInfo.version,
|
||||
});
|
||||
}
|
||||
|
||||
public setup(core: CoreSetup) {
|
||||
this.logger.debug('ecsDataQualityDashboard: Setup'); // this would be deleted when plugin is removed
|
||||
const router = core.http.createRouter(); // this would be deleted when plugin is removed
|
||||
public setup(core: CoreSetup, plugins: PluginSetupDependencies) {
|
||||
this.logger.debug('ecsDataQualityDashboard: Setup');
|
||||
|
||||
// TODO: Uncomment https://github.com/elastic/kibana/pull/173185#issuecomment-1908034302
|
||||
// this.resultsDataStream.install({
|
||||
// esClient: core
|
||||
// .getStartServices()
|
||||
// .then(([{ elasticsearch }]) => elasticsearch.client.asInternalUser),
|
||||
// logger: this.logger,
|
||||
// pluginStop$: this.pluginStop$,
|
||||
// });
|
||||
|
||||
core.http.registerRouteHandlerContext<
|
||||
DataQualityDashboardRequestHandlerContext,
|
||||
'dataQualityDashboard'
|
||||
>('dataQualityDashboard', (_context, request) => {
|
||||
const spaceId = plugins.spaces?.spacesService.getSpaceId(request) ?? DEFAULT_SPACE_ID;
|
||||
return {
|
||||
spaceId,
|
||||
getResultsIndexName: async () => this.resultsDataStream.installSpace(spaceId),
|
||||
};
|
||||
});
|
||||
|
||||
const router = core.http.createRouter<DataQualityDashboardRequestHandlerContext>();
|
||||
|
||||
// Register server side APIs
|
||||
getIndexMappingsRoute(router, this.logger);
|
||||
getIndexStatsRoute(router, this.logger);
|
||||
getUnallowedFieldValuesRoute(router, this.logger);
|
||||
getILMExplainRoute(router, this.logger);
|
||||
resultsRoutes(router, this.logger);
|
||||
return {};
|
||||
}
|
||||
|
||||
|
|
@ -41,5 +84,8 @@ export class EcsDataQualityDashboardPlugin
|
|||
return {};
|
||||
}
|
||||
|
||||
public stop() {}
|
||||
public stop() {
|
||||
this.pluginStop$.next();
|
||||
this.pluginStop$.complete();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ import { serverMock } from '../__mocks__/server';
|
|||
import { requestMock } from '../__mocks__/request';
|
||||
import { requestContextMock } from '../__mocks__/request_context';
|
||||
import { getILMExplainRoute } from './get_ilm_explain';
|
||||
import { loggerMock, MockedLogger } from '@kbn/logging-mocks';
|
||||
import { loggerMock, type MockedLogger } from '@kbn/logging-mocks';
|
||||
|
||||
jest.mock('../lib', () => ({
|
||||
fetchILMExplain: jest.fn(),
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import { IRouter, Logger } from '@kbn/core/server';
|
||||
import type { IRouter, Logger } from '@kbn/core/server';
|
||||
|
||||
import { GET_ILM_EXPLAIN, INTERNAL_API_VERSION } from '../../common/constants';
|
||||
import { fetchILMExplain } from '../lib';
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ import { serverMock } from '../__mocks__/server';
|
|||
import { requestMock } from '../__mocks__/request';
|
||||
import { requestContextMock } from '../__mocks__/request_context';
|
||||
import { getIndexMappingsRoute } from './get_index_mappings';
|
||||
import { loggerMock, MockedLogger } from '@kbn/logging-mocks';
|
||||
import { loggerMock, type MockedLogger } from '@kbn/logging-mocks';
|
||||
|
||||
jest.mock('../lib', () => ({
|
||||
fetchMappings: jest.fn(),
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import { IRouter, Logger } from '@kbn/core/server';
|
||||
import type { IRouter, Logger } from '@kbn/core/server';
|
||||
|
||||
import { fetchMappings } from '../lib';
|
||||
import { buildResponse } from '../lib/build_response';
|
||||
|
|
|
|||
|
|
@ -12,7 +12,8 @@ import { serverMock } from '../__mocks__/server';
|
|||
import { requestMock } from '../__mocks__/request';
|
||||
import { requestContextMock } from '../__mocks__/request_context';
|
||||
import { getIndexStatsRoute } from './get_index_stats';
|
||||
import { loggerMock, MockedLogger } from '@kbn/logging-mocks';
|
||||
import type { MockedLogger } from '@kbn/logging-mocks';
|
||||
import { loggerMock } from '@kbn/logging-mocks';
|
||||
|
||||
jest.mock('../lib', () => ({
|
||||
fetchStats: jest.fn(),
|
||||
|
|
|
|||
|
|
@ -5,9 +5,9 @@
|
|||
* 2.0.
|
||||
*/
|
||||
import { i18n } from '@kbn/i18n';
|
||||
import { IRouter, Logger } from '@kbn/core/server';
|
||||
import type { IRouter, Logger } from '@kbn/core/server';
|
||||
|
||||
import { IndicesStatsIndicesStats } from '@elastic/elasticsearch/lib/api/types';
|
||||
import type { IndicesStatsIndicesStats } from '@elastic/elasticsearch/lib/api/types';
|
||||
import { fetchStats, fetchAvailableIndices } from '../lib';
|
||||
import { buildResponse } from '../lib/build_response';
|
||||
import { GET_INDEX_STATS, INTERNAL_API_VERSION } from '../../common/constants';
|
||||
|
|
|
|||
|
|
@ -12,7 +12,8 @@ import { serverMock } from '../__mocks__/server';
|
|||
import { requestMock } from '../__mocks__/request';
|
||||
import { requestContextMock } from '../__mocks__/request_context';
|
||||
import { getUnallowedFieldValuesRoute } from './get_unallowed_field_values';
|
||||
import { loggerMock, MockedLogger } from '@kbn/logging-mocks';
|
||||
import type { MockedLogger } from '@kbn/logging-mocks';
|
||||
import { loggerMock } from '@kbn/logging-mocks';
|
||||
|
||||
jest.mock('../lib', () => ({
|
||||
getUnallowedFieldValues: jest.fn(),
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import { IRouter, Logger } from '@kbn/core/server';
|
||||
import type { IRouter, Logger } from '@kbn/core/server';
|
||||
|
||||
import { getUnallowedFieldValues } from '../lib';
|
||||
import { buildResponse } from '../lib/build_response';
|
||||
|
|
|
|||
|
|
@ -8,3 +8,4 @@ export { getIndexMappingsRoute } from './get_index_mappings';
|
|||
export { getIndexStatsRoute } from './get_index_stats';
|
||||
export { getUnallowedFieldValuesRoute } from './get_unallowed_field_values';
|
||||
export { getILMExplainRoute } from './get_ilm_explain';
|
||||
export { resultsRoutes } from './results';
|
||||
|
|
|
|||
|
|
@ -0,0 +1,211 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
import { RESULTS_ROUTE_PATH } from '../../../common/constants';
|
||||
|
||||
import { serverMock } from '../../__mocks__/server';
|
||||
import { requestMock } from '../../__mocks__/request';
|
||||
import { requestContextMock } from '../../__mocks__/request_context';
|
||||
import type { LatestAggResponseBucket } from './get_results';
|
||||
import { getResultsRoute, getQuery } from './get_results';
|
||||
import { loggerMock, type MockedLogger } from '@kbn/logging-mocks';
|
||||
import { resultBody, resultDocument } from './results.mock';
|
||||
import type {
|
||||
SearchResponse,
|
||||
SecurityHasPrivilegesResponse,
|
||||
} from '@elastic/elasticsearch/lib/api/types';
|
||||
import type { ResultDocument } from '../../schemas/result';
|
||||
|
||||
const searchResponse = {
|
||||
aggregations: {
|
||||
latest: {
|
||||
buckets: [
|
||||
{
|
||||
key: 'logs-*',
|
||||
latest_doc: { hits: { hits: [{ _source: resultDocument }] } },
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
} as unknown as SearchResponse<
|
||||
ResultDocument,
|
||||
Record<string, { buckets: LatestAggResponseBucket[] }>
|
||||
>;
|
||||
|
||||
// TODO: https://github.com/elastic/kibana/pull/173185#issuecomment-1908034302
|
||||
describe.skip('getResultsRoute route', () => {
|
||||
describe('querying', () => {
|
||||
let server: ReturnType<typeof serverMock.create>;
|
||||
let { context } = requestContextMock.createTools();
|
||||
let logger: MockedLogger;
|
||||
|
||||
const req = requestMock.create({
|
||||
method: 'get',
|
||||
path: RESULTS_ROUTE_PATH,
|
||||
query: { patterns: 'logs-*,alerts-*' },
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
|
||||
server = serverMock.create();
|
||||
logger = loggerMock.create();
|
||||
|
||||
({ context } = requestContextMock.createTools());
|
||||
|
||||
context.core.elasticsearch.client.asCurrentUser.security.hasPrivileges.mockResolvedValue({
|
||||
index: { 'logs-*': { all: true }, 'alerts-*': { all: true } },
|
||||
} as unknown as SecurityHasPrivilegesResponse);
|
||||
|
||||
getResultsRoute(server.router, logger);
|
||||
});
|
||||
|
||||
it('gets result', async () => {
|
||||
const mockSearch = context.core.elasticsearch.client.asInternalUser.search;
|
||||
mockSearch.mockResolvedValueOnce(searchResponse);
|
||||
|
||||
const response = await server.inject(req, requestContextMock.convertContext(context));
|
||||
expect(mockSearch).toHaveBeenCalled();
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
expect(response.body).toEqual([{ '@timestamp': expect.any(Number), ...resultBody }]);
|
||||
});
|
||||
|
||||
it('handles results data stream error', async () => {
|
||||
const errorMessage = 'Installation Error!';
|
||||
context.dataQualityDashboard.getResultsIndexName.mockRejectedValueOnce(
|
||||
new Error(errorMessage)
|
||||
);
|
||||
const response = await server.inject(req, requestContextMock.convertContext(context));
|
||||
expect(response.status).toEqual(503);
|
||||
expect(response.body).toEqual({
|
||||
message: expect.stringContaining(errorMessage),
|
||||
status_code: 503,
|
||||
});
|
||||
expect(logger.error).toHaveBeenCalledWith(expect.stringContaining(errorMessage));
|
||||
});
|
||||
|
||||
it('handles error', async () => {
|
||||
const errorMessage = 'Error!';
|
||||
const mockSearch = context.core.elasticsearch.client.asInternalUser.search;
|
||||
mockSearch.mockRejectedValueOnce({ message: errorMessage });
|
||||
|
||||
const response = await server.inject(req, requestContextMock.convertContext(context));
|
||||
expect(response.status).toEqual(500);
|
||||
expect(response.body).toEqual({ message: errorMessage, status_code: 500 });
|
||||
});
|
||||
});
|
||||
|
||||
describe('request pattern authorization', () => {
|
||||
let server: ReturnType<typeof serverMock.create>;
|
||||
let { context } = requestContextMock.createTools();
|
||||
let logger: MockedLogger;
|
||||
|
||||
const req = requestMock.create({
|
||||
method: 'get',
|
||||
path: RESULTS_ROUTE_PATH,
|
||||
query: { patterns: 'logs-*,alerts-*' },
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
|
||||
server = serverMock.create();
|
||||
logger = loggerMock.create();
|
||||
|
||||
({ context } = requestContextMock.createTools());
|
||||
|
||||
context.core.elasticsearch.client.asInternalUser.search.mockResolvedValue(searchResponse);
|
||||
|
||||
context.core.elasticsearch.client.asCurrentUser.security.hasPrivileges.mockResolvedValue({
|
||||
index: { 'logs-*': { all: true }, 'alerts-*': { all: true } },
|
||||
} as unknown as SecurityHasPrivilegesResponse);
|
||||
|
||||
getResultsRoute(server.router, logger);
|
||||
});
|
||||
|
||||
it('should authorize pattern', async () => {
|
||||
const mockHasPrivileges =
|
||||
context.core.elasticsearch.client.asCurrentUser.security.hasPrivileges;
|
||||
mockHasPrivileges.mockResolvedValueOnce({
|
||||
index: { 'logs-*': { all: true }, 'alerts-*': { all: true } },
|
||||
} as unknown as SecurityHasPrivilegesResponse);
|
||||
|
||||
const response = await server.inject(req, requestContextMock.convertContext(context));
|
||||
expect(mockHasPrivileges).toHaveBeenCalledWith({
|
||||
index: [
|
||||
{ names: ['logs-*', 'alerts-*'], privileges: ['all', 'read', 'view_index_metadata'] },
|
||||
],
|
||||
});
|
||||
expect(context.core.elasticsearch.client.asInternalUser.search).toHaveBeenCalled();
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
expect(response.body).toEqual([{ '@timestamp': expect.any(Number), ...resultBody }]);
|
||||
});
|
||||
|
||||
it('should search authorized patterns only', async () => {
|
||||
const mockHasPrivileges =
|
||||
context.core.elasticsearch.client.asCurrentUser.security.hasPrivileges;
|
||||
mockHasPrivileges.mockResolvedValueOnce({
|
||||
index: { 'logs-*': { all: false }, 'alerts-*': { all: true } },
|
||||
} as unknown as SecurityHasPrivilegesResponse);
|
||||
|
||||
const response = await server.inject(req, requestContextMock.convertContext(context));
|
||||
expect(context.core.elasticsearch.client.asInternalUser.search).toHaveBeenCalledWith({
|
||||
index: expect.any(String),
|
||||
...getQuery(['alerts-*']),
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
});
|
||||
|
||||
it('should not search unauthorized patterns', async () => {
|
||||
const mockHasPrivileges =
|
||||
context.core.elasticsearch.client.asCurrentUser.security.hasPrivileges;
|
||||
mockHasPrivileges.mockResolvedValueOnce({
|
||||
index: { 'logs-*': { all: false }, 'alerts-*': { all: false } },
|
||||
} as unknown as SecurityHasPrivilegesResponse);
|
||||
|
||||
const response = await server.inject(req, requestContextMock.convertContext(context));
|
||||
expect(context.core.elasticsearch.client.asInternalUser.search).not.toHaveBeenCalled();
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
expect(response.body).toEqual([]);
|
||||
});
|
||||
|
||||
it('handles pattern authorization error', async () => {
|
||||
const errorMessage = 'Error!';
|
||||
const mockHasPrivileges =
|
||||
context.core.elasticsearch.client.asCurrentUser.security.hasPrivileges;
|
||||
mockHasPrivileges.mockRejectedValueOnce({ message: errorMessage });
|
||||
|
||||
const response = await server.inject(req, requestContextMock.convertContext(context));
|
||||
expect(response.status).toEqual(500);
|
||||
expect(response.body).toEqual({ message: errorMessage, status_code: 500 });
|
||||
});
|
||||
});
|
||||
|
||||
describe('request validation', () => {
|
||||
let server: ReturnType<typeof serverMock.create>;
|
||||
let logger: MockedLogger;
|
||||
beforeEach(() => {
|
||||
server = serverMock.create();
|
||||
logger = loggerMock.create();
|
||||
getResultsRoute(server.router, logger);
|
||||
});
|
||||
|
||||
test('disallows invalid query param', () => {
|
||||
const req = requestMock.create({
|
||||
method: 'get',
|
||||
path: RESULTS_ROUTE_PATH,
|
||||
query: {},
|
||||
});
|
||||
const result = server.validate(req);
|
||||
|
||||
expect(result.badRequest).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,113 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { IRouter, Logger } from '@kbn/core/server';
|
||||
|
||||
import { RESULTS_ROUTE_PATH, INTERNAL_API_VERSION } from '../../../common/constants';
|
||||
import { buildResponse } from '../../lib/build_response';
|
||||
import { buildRouteValidation } from '../../schemas/common';
|
||||
import { GetResultQuery } from '../../schemas/result';
|
||||
import type { Result, ResultDocument } from '../../schemas/result';
|
||||
import { API_DEFAULT_ERROR_MESSAGE } from '../../translations';
|
||||
import type { DataQualityDashboardRequestHandlerContext } from '../../types';
|
||||
import { createResultFromDocument } from './parser';
|
||||
import { API_RESULTS_INDEX_NOT_AVAILABLE } from './translations';
|
||||
|
||||
export const getQuery = (patterns: string[]) => ({
|
||||
size: 0,
|
||||
query: {
|
||||
bool: { filter: [{ terms: { 'rollup.pattern': patterns } }] },
|
||||
},
|
||||
aggs: {
|
||||
latest: {
|
||||
terms: { field: 'rollup.pattern', size: 10000 }, // big enough to get all patterns, but under `index.max_terms_count` (default 65536)
|
||||
aggs: { latest_doc: { top_hits: { size: 1, sort: [{ '@timestamp': { order: 'desc' } }] } } },
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
export interface LatestAggResponseBucket {
|
||||
key: string;
|
||||
latest_doc: { hits: { hits: Array<{ _source: ResultDocument }> } };
|
||||
}
|
||||
|
||||
export const getResultsRoute = (
|
||||
router: IRouter<DataQualityDashboardRequestHandlerContext>,
|
||||
logger: Logger
|
||||
) => {
|
||||
router.versioned
|
||||
.get({
|
||||
path: RESULTS_ROUTE_PATH,
|
||||
access: 'internal',
|
||||
options: { tags: ['access:securitySolution'] },
|
||||
})
|
||||
.addVersion(
|
||||
{
|
||||
version: INTERNAL_API_VERSION,
|
||||
validate: { request: { query: buildRouteValidation(GetResultQuery) } },
|
||||
},
|
||||
async (context, request, response) => {
|
||||
// TODO: https://github.com/elastic/kibana/pull/173185#issuecomment-1908034302
|
||||
return response.ok({ body: [] });
|
||||
|
||||
// eslint-disable-next-line no-unreachable
|
||||
const services = await context.resolve(['core', 'dataQualityDashboard']);
|
||||
const resp = buildResponse(response);
|
||||
|
||||
let index: string;
|
||||
try {
|
||||
index = await services.dataQualityDashboard.getResultsIndexName();
|
||||
} catch (err) {
|
||||
logger.error(`[GET results] Error retrieving results index name: ${err.message}`);
|
||||
return resp.error({
|
||||
body: `${API_RESULTS_INDEX_NOT_AVAILABLE}: ${err.message}`,
|
||||
statusCode: 503,
|
||||
});
|
||||
}
|
||||
|
||||
try {
|
||||
// Confirm user has authorization for the requested patterns
|
||||
const { patterns } = request.query;
|
||||
const userEsClient = services.core.elasticsearch.client.asCurrentUser;
|
||||
const privileges = await userEsClient.security.hasPrivileges({
|
||||
index: [
|
||||
{ names: patterns.split(','), privileges: ['all', 'read', 'view_index_metadata'] },
|
||||
],
|
||||
});
|
||||
const authorizedPatterns = Object.keys(privileges.index).filter((pattern) =>
|
||||
Object.values(privileges.index[pattern]).some((v) => v === true)
|
||||
);
|
||||
if (authorizedPatterns.length === 0) {
|
||||
return response.ok({ body: [] });
|
||||
}
|
||||
|
||||
// Get the latest result of each pattern
|
||||
const query = { index, ...getQuery(authorizedPatterns) };
|
||||
const internalEsClient = services.core.elasticsearch.client.asInternalUser;
|
||||
|
||||
const { aggregations } = await internalEsClient.search<
|
||||
ResultDocument,
|
||||
Record<string, { buckets: LatestAggResponseBucket[] }>
|
||||
>(query);
|
||||
|
||||
const results: Result[] =
|
||||
aggregations?.latest?.buckets.map((bucket) =>
|
||||
createResultFromDocument(bucket.latest_doc.hits.hits[0]._source)
|
||||
) ?? [];
|
||||
|
||||
return response.ok({ body: results });
|
||||
} catch (err) {
|
||||
logger.error(JSON.stringify(err));
|
||||
|
||||
return resp.error({
|
||||
body: err.message ?? API_DEFAULT_ERROR_MESSAGE,
|
||||
statusCode: err.statusCode ?? 500,
|
||||
});
|
||||
}
|
||||
}
|
||||
);
|
||||
};
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { IRouter, Logger } from '@kbn/core/server';
|
||||
|
||||
import { postResultsRoute } from './post_results';
|
||||
import { getResultsRoute } from './get_results';
|
||||
import type { DataQualityDashboardRequestHandlerContext } from '../../types';
|
||||
|
||||
export const resultsRoutes = (
|
||||
router: IRouter<DataQualityDashboardRequestHandlerContext>,
|
||||
logger: Logger
|
||||
) => {
|
||||
postResultsRoute(router, logger);
|
||||
getResultsRoute(router, logger);
|
||||
};
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { createDocumentFromResult, createResultFromDocument } from './parser';
|
||||
import { resultBody, resultDocument } from './results.mock';
|
||||
|
||||
describe('createDocumentFromResult', () => {
|
||||
it('should create document from result', () => {
|
||||
const document = createDocumentFromResult(resultBody);
|
||||
expect(document).toEqual({ ...resultDocument, '@timestamp': expect.any(Number) });
|
||||
});
|
||||
});
|
||||
|
||||
describe('createResultFromDocument', () => {
|
||||
it('should create document from result', () => {
|
||||
const result = createResultFromDocument(resultDocument);
|
||||
expect(result).toEqual({ ...resultBody, '@timestamp': expect.any(Number) });
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
import type { Result, ResultDocument, IndexArray, IndexObject } from '../../schemas/result';
|
||||
|
||||
export const createDocumentFromResult = (result: Result): ResultDocument => {
|
||||
const { rollup } = result;
|
||||
const document: ResultDocument = {
|
||||
...result,
|
||||
'@timestamp': Date.now(),
|
||||
rollup: {
|
||||
...rollup,
|
||||
ilmExplain: indexObjectToIndexArray(rollup.ilmExplain),
|
||||
stats: indexObjectToIndexArray(rollup.stats),
|
||||
results: indexObjectToIndexArray(rollup.results),
|
||||
},
|
||||
};
|
||||
|
||||
return document;
|
||||
};
|
||||
|
||||
export const createResultFromDocument = (document: ResultDocument): Result => {
|
||||
const { rollup } = document;
|
||||
const result = {
|
||||
...document,
|
||||
rollup: {
|
||||
...rollup,
|
||||
ilmExplain: indexArrayToIndexObject(rollup.ilmExplain),
|
||||
stats: indexArrayToIndexObject(rollup.stats),
|
||||
results: indexArrayToIndexObject(rollup.results),
|
||||
},
|
||||
};
|
||||
|
||||
return result;
|
||||
};
|
||||
|
||||
// ES parses object keys containing `.` as nested dot-separated field names (e.g. `event.name`).
|
||||
// we need to convert documents containing objects with "indexName" keys (e.g. `.index-name-checked`)
|
||||
// to object arrays so they can be stored correctly, we keep the key in the `_indexName` field.
|
||||
const indexObjectToIndexArray = (obj: IndexObject): IndexArray =>
|
||||
Object.entries(obj).map(([key, value]) => ({ ...value, _indexName: key }));
|
||||
|
||||
// convert index arrays back to objects with indexName as key
|
||||
const indexArrayToIndexObject = (arr: IndexArray): IndexObject =>
|
||||
Object.fromEntries(arr.map(({ _indexName, ...value }) => [_indexName, value]));
|
||||
|
|
@ -0,0 +1,180 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
import { RESULTS_ROUTE_PATH } from '../../../common/constants';
|
||||
|
||||
import { serverMock } from '../../__mocks__/server';
|
||||
import { requestMock } from '../../__mocks__/request';
|
||||
import { requestContextMock } from '../../__mocks__/request_context';
|
||||
import { postResultsRoute } from './post_results';
|
||||
import { loggerMock, type MockedLogger } from '@kbn/logging-mocks';
|
||||
import type {
|
||||
SecurityHasPrivilegesResponse,
|
||||
WriteResponseBase,
|
||||
} from '@elastic/elasticsearch/lib/api/types';
|
||||
import { resultBody, resultDocument } from './results.mock';
|
||||
|
||||
// TODO: https://github.com/elastic/kibana/pull/173185#issuecomment-1908034302
|
||||
describe.skip('postResultsRoute route', () => {
|
||||
describe('indexation', () => {
|
||||
let server: ReturnType<typeof serverMock.create>;
|
||||
let { context } = requestContextMock.createTools();
|
||||
let logger: MockedLogger;
|
||||
|
||||
const req = requestMock.create({ method: 'post', path: RESULTS_ROUTE_PATH, body: resultBody });
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
|
||||
server = serverMock.create();
|
||||
logger = loggerMock.create();
|
||||
|
||||
({ context } = requestContextMock.createTools());
|
||||
|
||||
context.core.elasticsearch.client.asCurrentUser.security.hasPrivileges.mockResolvedValue({
|
||||
has_all_requested: true,
|
||||
} as unknown as SecurityHasPrivilegesResponse);
|
||||
|
||||
postResultsRoute(server.router, logger);
|
||||
});
|
||||
|
||||
it('indexes result', async () => {
|
||||
const mockIndex = context.core.elasticsearch.client.asInternalUser.index;
|
||||
mockIndex.mockResolvedValueOnce({ result: 'created' } as WriteResponseBase);
|
||||
|
||||
const response = await server.inject(req, requestContextMock.convertContext(context));
|
||||
expect(mockIndex).toHaveBeenCalledWith({
|
||||
body: { ...resultDocument, '@timestamp': expect.any(Number) },
|
||||
index: await context.dataQualityDashboard.getResultsIndexName(),
|
||||
});
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
expect(response.body).toEqual({ result: 'created' });
|
||||
});
|
||||
|
||||
it('handles results data stream error', async () => {
|
||||
const errorMessage = 'Installation Error!';
|
||||
context.dataQualityDashboard.getResultsIndexName.mockRejectedValueOnce(
|
||||
new Error(errorMessage)
|
||||
);
|
||||
const response = await server.inject(req, requestContextMock.convertContext(context));
|
||||
expect(response.status).toEqual(503);
|
||||
expect(response.body).toEqual({
|
||||
message: expect.stringContaining(errorMessage),
|
||||
status_code: 503,
|
||||
});
|
||||
expect(logger.error).toHaveBeenCalledWith(expect.stringContaining(errorMessage));
|
||||
});
|
||||
|
||||
it('handles index error', async () => {
|
||||
const errorMessage = 'Error!';
|
||||
const mockIndex = context.core.elasticsearch.client.asInternalUser.index;
|
||||
mockIndex.mockRejectedValueOnce({ message: errorMessage });
|
||||
|
||||
const response = await server.inject(req, requestContextMock.convertContext(context));
|
||||
expect(response.status).toEqual(500);
|
||||
expect(response.body).toEqual({ message: errorMessage, status_code: 500 });
|
||||
});
|
||||
});
|
||||
|
||||
describe('request pattern authorization', () => {
|
||||
let server: ReturnType<typeof serverMock.create>;
|
||||
let { context } = requestContextMock.createTools();
|
||||
let logger: MockedLogger;
|
||||
|
||||
const req = requestMock.create({ method: 'post', path: RESULTS_ROUTE_PATH, body: resultBody });
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
|
||||
server = serverMock.create();
|
||||
logger = loggerMock.create();
|
||||
|
||||
({ context } = requestContextMock.createTools());
|
||||
|
||||
context.core.elasticsearch.client.asInternalUser.index.mockResolvedValueOnce({
|
||||
result: 'created',
|
||||
} as WriteResponseBase);
|
||||
|
||||
postResultsRoute(server.router, logger);
|
||||
});
|
||||
|
||||
it('should authorize pattern', async () => {
|
||||
const mockHasPrivileges =
|
||||
context.core.elasticsearch.client.asCurrentUser.security.hasPrivileges;
|
||||
mockHasPrivileges.mockResolvedValueOnce({
|
||||
has_all_requested: true,
|
||||
} as unknown as SecurityHasPrivilegesResponse);
|
||||
|
||||
const response = await server.inject(req, requestContextMock.convertContext(context));
|
||||
expect(mockHasPrivileges).toHaveBeenCalledWith({
|
||||
index: [
|
||||
{
|
||||
names: [resultBody.rollup.pattern],
|
||||
privileges: ['all', 'read', 'view_index_metadata'],
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(context.core.elasticsearch.client.asInternalUser.index).toHaveBeenCalled();
|
||||
expect(response.status).toEqual(200);
|
||||
expect(response.body).toEqual({ result: 'created' });
|
||||
});
|
||||
|
||||
it('should not index unauthorized pattern', async () => {
|
||||
const mockHasPrivileges =
|
||||
context.core.elasticsearch.client.asCurrentUser.security.hasPrivileges;
|
||||
mockHasPrivileges.mockResolvedValueOnce({
|
||||
has_all_requested: false,
|
||||
} as unknown as SecurityHasPrivilegesResponse);
|
||||
|
||||
const response = await server.inject(req, requestContextMock.convertContext(context));
|
||||
expect(mockHasPrivileges).toHaveBeenCalledWith({
|
||||
index: [
|
||||
{
|
||||
names: [resultBody.rollup.pattern],
|
||||
privileges: ['all', 'read', 'view_index_metadata'],
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(context.core.elasticsearch.client.asInternalUser.index).not.toHaveBeenCalled();
|
||||
|
||||
expect(response.status).toEqual(200);
|
||||
expect(response.body).toEqual({ result: 'noop' });
|
||||
});
|
||||
|
||||
it('handles pattern authorization error', async () => {
|
||||
const errorMessage = 'Error!';
|
||||
const mockHasPrivileges =
|
||||
context.core.elasticsearch.client.asCurrentUser.security.hasPrivileges;
|
||||
mockHasPrivileges.mockRejectedValueOnce({ message: errorMessage });
|
||||
|
||||
const response = await server.inject(req, requestContextMock.convertContext(context));
|
||||
expect(response.status).toEqual(500);
|
||||
expect(response.body).toEqual({ message: errorMessage, status_code: 500 });
|
||||
});
|
||||
});
|
||||
|
||||
describe('request validation', () => {
|
||||
let server: ReturnType<typeof serverMock.create>;
|
||||
let logger: MockedLogger;
|
||||
beforeEach(() => {
|
||||
server = serverMock.create();
|
||||
logger = loggerMock.create();
|
||||
postResultsRoute(server.router, logger);
|
||||
});
|
||||
|
||||
test('disallows invalid pattern', () => {
|
||||
const req = requestMock.create({
|
||||
method: 'post',
|
||||
path: RESULTS_ROUTE_PATH,
|
||||
body: { rollup: resultBody.rollup },
|
||||
});
|
||||
const result = server.validate(req);
|
||||
|
||||
expect(result.badRequest).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,80 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { IRouter, Logger } from '@kbn/core/server';
|
||||
|
||||
import { RESULTS_ROUTE_PATH, INTERNAL_API_VERSION } from '../../../common/constants';
|
||||
import { buildResponse } from '../../lib/build_response';
|
||||
import { buildRouteValidation } from '../../schemas/common';
|
||||
import { PostResultBody } from '../../schemas/result';
|
||||
import { API_DEFAULT_ERROR_MESSAGE } from '../../translations';
|
||||
import type { DataQualityDashboardRequestHandlerContext } from '../../types';
|
||||
import { createDocumentFromResult } from './parser';
|
||||
import { API_RESULTS_INDEX_NOT_AVAILABLE } from './translations';
|
||||
|
||||
export const postResultsRoute = (
|
||||
router: IRouter<DataQualityDashboardRequestHandlerContext>,
|
||||
logger: Logger
|
||||
) => {
|
||||
router.versioned
|
||||
.post({
|
||||
path: RESULTS_ROUTE_PATH,
|
||||
access: 'internal',
|
||||
options: { tags: ['access:securitySolution'] },
|
||||
})
|
||||
.addVersion(
|
||||
{
|
||||
version: INTERNAL_API_VERSION,
|
||||
validate: { request: { body: buildRouteValidation(PostResultBody) } },
|
||||
},
|
||||
async (context, request, response) => {
|
||||
// TODO: https://github.com/elastic/kibana/pull/173185#issuecomment-1908034302
|
||||
return response.ok({ body: { result: 'noop' } });
|
||||
|
||||
// eslint-disable-next-line no-unreachable
|
||||
const services = await context.resolve(['core', 'dataQualityDashboard']);
|
||||
const resp = buildResponse(response);
|
||||
|
||||
let index: string;
|
||||
try {
|
||||
index = await services.dataQualityDashboard.getResultsIndexName();
|
||||
} catch (err) {
|
||||
logger.error(`[POST result] Error retrieving results index name: ${err.message}`);
|
||||
return resp.error({
|
||||
body: `${API_RESULTS_INDEX_NOT_AVAILABLE}: ${err.message}`,
|
||||
statusCode: 503,
|
||||
});
|
||||
}
|
||||
|
||||
try {
|
||||
// Confirm user has authorization for the pattern payload
|
||||
const { pattern } = request.body.rollup;
|
||||
const userEsClient = services.core.elasticsearch.client.asCurrentUser;
|
||||
const privileges = await userEsClient.security.hasPrivileges({
|
||||
index: [{ names: [pattern], privileges: ['all', 'read', 'view_index_metadata'] }],
|
||||
});
|
||||
if (!privileges.has_all_requested) {
|
||||
return response.ok({ body: { result: 'noop' } });
|
||||
}
|
||||
|
||||
// Index the result
|
||||
const document = createDocumentFromResult(request.body);
|
||||
const esClient = services.core.elasticsearch.client.asInternalUser;
|
||||
const outcome = await esClient.index({ index, body: document });
|
||||
|
||||
return response.ok({ body: { result: outcome.result } });
|
||||
} catch (err) {
|
||||
logger.error(JSON.stringify(err));
|
||||
|
||||
return resp.error({
|
||||
body: err.message ?? API_DEFAULT_ERROR_MESSAGE,
|
||||
statusCode: err.statusCode ?? 500,
|
||||
});
|
||||
}
|
||||
}
|
||||
);
|
||||
};
|
||||
|
|
@ -0,0 +1,202 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { ResultDocument } from '../../schemas/result';
|
||||
|
||||
export const resultDocument: ResultDocument = {
|
||||
'@timestamp': 1622767273955,
|
||||
meta: {
|
||||
batchId: 'aae36cd8-3825-4ad1-baa4-79bdf4617f8a',
|
||||
ecsVersion: '8.6.1',
|
||||
errorCount: 0,
|
||||
ilmPhase: 'hot',
|
||||
indexId: 'aO29KOwtQ3Snf-Pit5Wf4w',
|
||||
indexName: '.internal.alerts-security.alerts-default-000001',
|
||||
isCheckAll: true,
|
||||
numberOfDocuments: 20,
|
||||
numberOfFields: 1726,
|
||||
numberOfIncompatibleFields: 2,
|
||||
numberOfEcsFields: 1440,
|
||||
numberOfCustomFields: 284,
|
||||
numberOfIndices: 1,
|
||||
numberOfIndicesChecked: 1,
|
||||
numberOfSameFamily: 0,
|
||||
sameFamilyFields: [],
|
||||
sizeInBytes: 506471,
|
||||
timeConsumedMs: 85,
|
||||
unallowedMappingFields: [],
|
||||
unallowedValueFields: ['event.category', 'event.outcome'],
|
||||
},
|
||||
rollup: {
|
||||
docsCount: 20,
|
||||
error: null,
|
||||
ilmExplain: [
|
||||
{
|
||||
_indexName: '.internal.alerts-security.alerts-default-000001',
|
||||
index: '.internal.alerts-security.alerts-default-000001',
|
||||
managed: true,
|
||||
policy: '.alerts-ilm-policy',
|
||||
index_creation_date_millis: 1700757268526,
|
||||
time_since_index_creation: '20.99d',
|
||||
lifecycle_date_millis: 1700757268526,
|
||||
age: '20.99d',
|
||||
phase: 'hot',
|
||||
phase_time_millis: 1700757270294,
|
||||
action: 'rollover',
|
||||
action_time_millis: 1700757273955,
|
||||
step: 'check-rollover-ready',
|
||||
step_time_millis: 1700757273955,
|
||||
phase_execution: {
|
||||
policy: '.alerts-ilm-policy',
|
||||
phase_definition: {
|
||||
min_age: '0ms',
|
||||
actions: {
|
||||
rollover: {
|
||||
max_age: '30d',
|
||||
max_primary_shard_size: '50gb',
|
||||
},
|
||||
},
|
||||
},
|
||||
version: 1,
|
||||
modified_date_in_millis: 1700757266723,
|
||||
},
|
||||
},
|
||||
],
|
||||
ilmExplainPhaseCounts: {
|
||||
hot: 1,
|
||||
warm: 0,
|
||||
cold: 0,
|
||||
frozen: 0,
|
||||
unmanaged: 0,
|
||||
},
|
||||
indices: 1,
|
||||
pattern: '.alerts-security.alerts-default',
|
||||
results: [
|
||||
{
|
||||
_indexName: '.internal.alerts-security.alerts-default-000001',
|
||||
docsCount: 20,
|
||||
error: null,
|
||||
ilmPhase: 'hot',
|
||||
incompatible: 2,
|
||||
indexName: '.internal.alerts-security.alerts-default-000001',
|
||||
markdownComments: [
|
||||
'### .internal.alerts-security.alerts-default-000001\n',
|
||||
'| Result | Index | Docs | Incompatible fields | ILM Phase | Size |\n|--------|-------|------|---------------------|-----------|------|\n| ❌ | .internal.alerts-security.alerts-default-000001 | 20 (100,0 %) | 2 | `hot` | 494.6KB |\n\n',
|
||||
'### **Incompatible fields** `2` **Same family** `0` **Custom fields** `284` **ECS compliant fields** `1440` **All fields** `1726`\n',
|
||||
"#### 2 incompatible fields\n\nFields are incompatible with ECS when index mappings, or the values of the fields in the index, don't conform to the Elastic Common Schema (ECS), version 8.6.1.\n\n❌ Detection engine rules referencing these fields may not match them correctly\n❌ Pages may not display some events or fields due to unexpected field mappings or values\n❌ Mappings or field values that don't comply with ECS are not supported\n",
|
||||
'\n\n#### Incompatible field values - .internal.alerts-security.alerts-default-000001\n\n\n| Field | ECS values (expected) | Document values (actual) | \n|-------|-----------------------|--------------------------|\n| event.category | `authentication`, `configuration`, `database`, `driver`, `email`, `file`, `host`, `iam`, `intrusion_detection`, `malware`, `network`, `package`, `process`, `registry`, `session`, `threat`, `vulnerability`, `web` | `behavior` (1) |\n| event.outcome | `failure`, `success`, `unknown` | `` (12) |\n\n',
|
||||
],
|
||||
pattern: '.alerts-security.alerts-default',
|
||||
sameFamily: 0,
|
||||
},
|
||||
],
|
||||
sizeInBytes: 506471,
|
||||
stats: [
|
||||
{
|
||||
_indexName: '.internal.alerts-security.alerts-default-000001',
|
||||
uuid: 'aO29KOwtQ3Snf-Pit5Wf4w',
|
||||
health: 'green',
|
||||
status: 'open',
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
export const resultBody = {
|
||||
meta: {
|
||||
batchId: 'aae36cd8-3825-4ad1-baa4-79bdf4617f8a',
|
||||
ecsVersion: '8.6.1',
|
||||
errorCount: 0,
|
||||
ilmPhase: 'hot',
|
||||
indexId: 'aO29KOwtQ3Snf-Pit5Wf4w',
|
||||
indexName: '.internal.alerts-security.alerts-default-000001',
|
||||
isCheckAll: true,
|
||||
numberOfDocuments: 20,
|
||||
numberOfFields: 1726,
|
||||
numberOfIncompatibleFields: 2,
|
||||
numberOfEcsFields: 1440,
|
||||
numberOfCustomFields: 284,
|
||||
numberOfIndices: 1,
|
||||
numberOfIndicesChecked: 1,
|
||||
numberOfSameFamily: 0,
|
||||
sameFamilyFields: [],
|
||||
sizeInBytes: 506471,
|
||||
timeConsumedMs: 85,
|
||||
unallowedMappingFields: [],
|
||||
unallowedValueFields: ['event.category', 'event.outcome'],
|
||||
},
|
||||
rollup: {
|
||||
docsCount: 20,
|
||||
error: null,
|
||||
ilmExplain: {
|
||||
'.internal.alerts-security.alerts-default-000001': {
|
||||
index: '.internal.alerts-security.alerts-default-000001',
|
||||
managed: true,
|
||||
policy: '.alerts-ilm-policy',
|
||||
index_creation_date_millis: 1700757268526,
|
||||
time_since_index_creation: '20.99d',
|
||||
lifecycle_date_millis: 1700757268526,
|
||||
age: '20.99d',
|
||||
phase: 'hot',
|
||||
phase_time_millis: 1700757270294,
|
||||
action: 'rollover',
|
||||
action_time_millis: 1700757273955,
|
||||
step: 'check-rollover-ready',
|
||||
step_time_millis: 1700757273955,
|
||||
phase_execution: {
|
||||
policy: '.alerts-ilm-policy',
|
||||
phase_definition: {
|
||||
min_age: '0ms',
|
||||
actions: {
|
||||
rollover: {
|
||||
max_age: '30d',
|
||||
max_primary_shard_size: '50gb',
|
||||
},
|
||||
},
|
||||
},
|
||||
version: 1,
|
||||
modified_date_in_millis: 1700757266723,
|
||||
},
|
||||
},
|
||||
},
|
||||
ilmExplainPhaseCounts: {
|
||||
hot: 1,
|
||||
warm: 0,
|
||||
cold: 0,
|
||||
frozen: 0,
|
||||
unmanaged: 0,
|
||||
},
|
||||
indices: 1,
|
||||
pattern: '.alerts-security.alerts-default',
|
||||
results: {
|
||||
'.internal.alerts-security.alerts-default-000001': {
|
||||
docsCount: 20,
|
||||
error: null,
|
||||
ilmPhase: 'hot',
|
||||
incompatible: 2,
|
||||
indexName: '.internal.alerts-security.alerts-default-000001',
|
||||
markdownComments: [
|
||||
'### .internal.alerts-security.alerts-default-000001\n',
|
||||
'| Result | Index | Docs | Incompatible fields | ILM Phase | Size |\n|--------|-------|------|---------------------|-----------|------|\n| ❌ | .internal.alerts-security.alerts-default-000001 | 20 (100,0 %) | 2 | `hot` | 494.6KB |\n\n',
|
||||
'### **Incompatible fields** `2` **Same family** `0` **Custom fields** `284` **ECS compliant fields** `1440` **All fields** `1726`\n',
|
||||
"#### 2 incompatible fields\n\nFields are incompatible with ECS when index mappings, or the values of the fields in the index, don't conform to the Elastic Common Schema (ECS), version 8.6.1.\n\n❌ Detection engine rules referencing these fields may not match them correctly\n❌ Pages may not display some events or fields due to unexpected field mappings or values\n❌ Mappings or field values that don't comply with ECS are not supported\n",
|
||||
'\n\n#### Incompatible field values - .internal.alerts-security.alerts-default-000001\n\n\n| Field | ECS values (expected) | Document values (actual) | \n|-------|-----------------------|--------------------------|\n| event.category | `authentication`, `configuration`, `database`, `driver`, `email`, `file`, `host`, `iam`, `intrusion_detection`, `malware`, `network`, `package`, `process`, `registry`, `session`, `threat`, `vulnerability`, `web` | `behavior` (1) |\n| event.outcome | `failure`, `success`, `unknown` | `` (12) |\n\n',
|
||||
],
|
||||
pattern: '.alerts-security.alerts-default',
|
||||
sameFamily: 0,
|
||||
},
|
||||
},
|
||||
sizeInBytes: 506471,
|
||||
stats: {
|
||||
'.internal.alerts-security.alerts-default-000001': {
|
||||
uuid: 'aO29KOwtQ3Snf-Pit5Wf4w',
|
||||
health: 'green',
|
||||
status: 'open',
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { i18n } from '@kbn/i18n';
|
||||
|
||||
export const API_RESULTS_INDEX_NOT_AVAILABLE = i18n.translate(
|
||||
'xpack.ecsDataQualityDashboard.api.results.indexNotAvailable',
|
||||
{
|
||||
defaultMessage: 'Data Quality Dashboard result persistence not available',
|
||||
}
|
||||
);
|
||||
|
|
@ -7,7 +7,7 @@
|
|||
|
||||
import { fold } from 'fp-ts/lib/Either';
|
||||
import { pipe } from 'fp-ts/lib/pipeable';
|
||||
import * as rt from 'io-ts';
|
||||
import type * as rt from 'io-ts';
|
||||
import { exactCheck, formatErrors } from '@kbn/securitysolution-io-ts-utils';
|
||||
import type {
|
||||
RouteValidationFunction,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,70 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import * as t from 'io-ts';
|
||||
|
||||
export const ResultMeta = t.type({
|
||||
batchId: t.string,
|
||||
ecsVersion: t.string,
|
||||
errorCount: t.number,
|
||||
ilmPhase: t.string,
|
||||
indexId: t.string,
|
||||
indexName: t.string,
|
||||
isCheckAll: t.boolean,
|
||||
numberOfDocuments: t.number,
|
||||
numberOfFields: t.number,
|
||||
numberOfIncompatibleFields: t.number,
|
||||
numberOfEcsFields: t.number,
|
||||
numberOfCustomFields: t.number,
|
||||
numberOfIndices: t.number,
|
||||
numberOfIndicesChecked: t.number,
|
||||
numberOfSameFamily: t.number,
|
||||
sameFamilyFields: t.array(t.string),
|
||||
sizeInBytes: t.number,
|
||||
timeConsumedMs: t.number,
|
||||
unallowedMappingFields: t.array(t.string),
|
||||
unallowedValueFields: t.array(t.string),
|
||||
});
|
||||
export type ResultMeta = t.TypeOf<typeof ResultMeta>;
|
||||
|
||||
export const ResultRollup = t.type({
|
||||
docsCount: t.number,
|
||||
error: t.union([t.string, t.null]),
|
||||
indices: t.number,
|
||||
pattern: t.string,
|
||||
sizeInBytes: t.number,
|
||||
ilmExplainPhaseCounts: t.record(t.string, t.number),
|
||||
ilmExplain: t.record(t.string, t.UnknownRecord),
|
||||
stats: t.record(t.string, t.UnknownRecord),
|
||||
results: t.record(t.string, t.UnknownRecord),
|
||||
});
|
||||
export type ResultRollup = t.TypeOf<typeof ResultRollup>;
|
||||
|
||||
export const Result = t.type({
|
||||
meta: ResultMeta,
|
||||
rollup: ResultRollup,
|
||||
});
|
||||
export type Result = t.TypeOf<typeof Result>;
|
||||
|
||||
export type IndexArray = Array<{ _indexName: string } & Record<string, unknown>>;
|
||||
export type IndexObject = Record<string, Record<string, unknown>>;
|
||||
|
||||
export type ResultDocument = Omit<Result, 'rollup'> & {
|
||||
'@timestamp': number;
|
||||
rollup: Omit<ResultRollup, 'stats' | 'results' | 'ilmExplain'> & {
|
||||
stats: IndexArray;
|
||||
results: IndexArray;
|
||||
ilmExplain: IndexArray;
|
||||
};
|
||||
};
|
||||
|
||||
// Routes validation schemas
|
||||
|
||||
export const GetResultQuery = t.type({ patterns: t.string });
|
||||
export type GetResultQuery = t.TypeOf<typeof GetResultQuery>;
|
||||
|
||||
export const PostResultBody = Result;
|
||||
|
|
@ -5,8 +5,22 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { CustomRequestHandlerContext } from '@kbn/core-http-request-handler-context-server';
|
||||
import type { SpacesPluginSetup } from '@kbn/spaces-plugin/server';
|
||||
|
||||
/** The plugin setup interface */
|
||||
export interface EcsDataQualityDashboardPluginSetup {} // eslint-disable-line @typescript-eslint/no-empty-interface
|
||||
|
||||
/** The plugin start interface */
|
||||
export interface EcsDataQualityDashboardPluginStart {} // eslint-disable-line @typescript-eslint/no-empty-interface
|
||||
|
||||
export interface PluginSetupDependencies {
|
||||
spaces?: SpacesPluginSetup;
|
||||
}
|
||||
|
||||
export type DataQualityDashboardRequestHandlerContext = CustomRequestHandlerContext<{
|
||||
dataQualityDashboard: {
|
||||
spaceId: string;
|
||||
getResultsIndexName: () => Promise<string>;
|
||||
};
|
||||
}>;
|
||||
|
|
|
|||
|
|
@ -21,6 +21,9 @@
|
|||
"@kbn/i18n",
|
||||
"@kbn/core-http-router-server-mocks",
|
||||
"@kbn/logging-mocks",
|
||||
"@kbn/data-stream-adapter",
|
||||
"@kbn/spaces-plugin",
|
||||
"@kbn/core-elasticsearch-server-mocks",
|
||||
],
|
||||
"exclude": [
|
||||
"target/**/*",
|
||||
|
|
|
|||
|
|
@ -77,6 +77,13 @@ export const dataQualityIndexCheckedEvent: DataQualityTelemetryIndexCheckedEvent
|
|||
optional: true,
|
||||
},
|
||||
},
|
||||
numberOfFields: {
|
||||
type: 'integer',
|
||||
_meta: {
|
||||
description: 'Total number of fields',
|
||||
optional: true,
|
||||
},
|
||||
},
|
||||
numberOfIncompatibleFields: {
|
||||
type: 'integer',
|
||||
_meta: {
|
||||
|
|
@ -84,6 +91,20 @@ export const dataQualityIndexCheckedEvent: DataQualityTelemetryIndexCheckedEvent
|
|||
optional: true,
|
||||
},
|
||||
},
|
||||
numberOfEcsFields: {
|
||||
type: 'integer',
|
||||
_meta: {
|
||||
description: 'Number of ecs compatible fields',
|
||||
optional: true,
|
||||
},
|
||||
},
|
||||
numberOfCustomFields: {
|
||||
type: 'integer',
|
||||
_meta: {
|
||||
description: 'Number of custom fields',
|
||||
optional: true,
|
||||
},
|
||||
},
|
||||
numberOfDocuments: {
|
||||
type: 'integer',
|
||||
_meta: {
|
||||
|
|
@ -187,6 +208,13 @@ export const dataQualityCheckAllClickedEvent: DataQualityTelemetryCheckAllComple
|
|||
optional: true,
|
||||
},
|
||||
},
|
||||
numberOfFields: {
|
||||
type: 'integer',
|
||||
_meta: {
|
||||
description: 'Total number of fields',
|
||||
optional: true,
|
||||
},
|
||||
},
|
||||
numberOfIncompatibleFields: {
|
||||
type: 'integer',
|
||||
_meta: {
|
||||
|
|
@ -194,6 +222,20 @@ export const dataQualityCheckAllClickedEvent: DataQualityTelemetryCheckAllComple
|
|||
optional: true,
|
||||
},
|
||||
},
|
||||
numberOfEcsFields: {
|
||||
type: 'integer',
|
||||
_meta: {
|
||||
description: 'Number of ecs compatible fields',
|
||||
optional: true,
|
||||
},
|
||||
},
|
||||
numberOfCustomFields: {
|
||||
type: 'integer',
|
||||
_meta: {
|
||||
description: 'Number of custom fields',
|
||||
optional: true,
|
||||
},
|
||||
},
|
||||
numberOfDocuments: {
|
||||
type: 'integer',
|
||||
_meta: {
|
||||
|
|
|
|||
|
|
@ -23,7 +23,10 @@ export interface ReportDataQualityCheckAllCompletedParams {
|
|||
ecsVersion?: string;
|
||||
isCheckAll?: boolean;
|
||||
numberOfDocuments?: number;
|
||||
numberOfFields?: number;
|
||||
numberOfIncompatibleFields?: number;
|
||||
numberOfEcsFields?: number;
|
||||
numberOfCustomFields?: number;
|
||||
numberOfIndices?: number;
|
||||
numberOfIndicesChecked?: number;
|
||||
numberOfSameFamily?: number;
|
||||
|
|
|
|||
|
|
@ -136,12 +136,6 @@ const DataQualityComponent: React.FC = () => {
|
|||
const { baseTheme, theme } = useThemes();
|
||||
const toasts = useToasts();
|
||||
|
||||
const addSuccessToast = useCallback(
|
||||
(toast: { title: string }) => {
|
||||
toasts.addSuccess(toast);
|
||||
},
|
||||
[toasts]
|
||||
);
|
||||
const [defaultBytesFormat] = useUiSetting$<string>(DEFAULT_BYTES_FORMAT);
|
||||
const [defaultNumberFormat] = useUiSetting$<string>(DEFAULT_NUMBER_FORMAT);
|
||||
const labelInputId = useGeneratedHtmlId({ prefix: 'labelInput' });
|
||||
|
|
@ -280,7 +274,6 @@ const DataQualityComponent: React.FC = () => {
|
|||
</HeaderPage>
|
||||
|
||||
<DataQualityPanel
|
||||
addSuccessToast={addSuccessToast}
|
||||
baseTheme={baseTheme}
|
||||
canUserCreateAndReadCases={canUserCreateAndReadCases}
|
||||
defaultBytesFormat={defaultBytesFormat}
|
||||
|
|
@ -299,6 +292,7 @@ const DataQualityComponent: React.FC = () => {
|
|||
setLastChecked={setLastChecked}
|
||||
startDate={startDate}
|
||||
theme={theme}
|
||||
toasts={toasts}
|
||||
/>
|
||||
</SecuritySolutionPageWrapper>
|
||||
) : (
|
||||
|
|
|
|||
|
|
@ -4329,6 +4329,10 @@
|
|||
version "0.0.0"
|
||||
uid ""
|
||||
|
||||
"@kbn/data-stream-adapter@link:packages/kbn-data-stream-adapter":
|
||||
version "0.0.0"
|
||||
uid ""
|
||||
|
||||
"@kbn/data-view-editor-plugin@link:src/plugins/data_view_editor":
|
||||
version "0.0.0"
|
||||
uid ""
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue