github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 09:48:58 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Fleet] Fix datastream settings on package upgrade and package policy creation (#147368)

Browse source
This commit is contained in:
Nicolas Chaulet 2022-12-13 20:52:07 -05:00 • committed by GitHub
parent 5d74e8307a
commit 5a9c04b24f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 190 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

44
x-pack/plugins/fleet/common/services/package_to_package_policy.test.ts
View file

@ -360,6 +360,50 @@ describe('Fleet - packageToPackagePolicy', () => {
});
});
it('returns package policy with experimental datastream features', () => {
expect(
packageToPackagePolicy(
{
...mockPackage,
savedObject: {
attributes: {
experimental_data_stream_features: [
{
data_stream: 'metrics-test.testdataset',
features: {
synthetic_source: true,
tsdb: true,
},
},
],
},
} as any,
},
'1'
)
).toEqual({
policy_id: '1',
namespace: '',
enabled: true,
inputs: [],
name: 'mock-package-1',
package: {
name: 'mock-package',
title: 'Mock package',
version: '0.0.0',
experimental_data_stream_features: [
{
data_stream: 'metrics-test.testdataset',
features: {
synthetic_source: true,
tsdb: true,
},
},
],
},
});
});
it('returns package policy with custom name', () => {
expect(packageToPackagePolicy(mockPackage, '1', 'default', 'pkgPolicy-1')).toEqual({
policy_id: '1',

8
x-pack/plugins/fleet/common/services/package_to_package_policy.ts
View file

@ -205,6 +205,11 @@ export const packageToPackagePolicy = (
description?: string,
integrationToEnable?: string
): NewPackagePolicy => {
const experimentalDataStreamFeatures =
'savedObject' in packageInfo
? packageInfo.savedObject?.attributes?.experimental_data_stream_features
: undefined;
const packagePolicy: NewPackagePolicy = {
name: packagePolicyName || `${packageInfo.name}-1`,
namespace,
@ -213,6 +218,9 @@ export const packageToPackagePolicy = (
name: packageInfo.name,
title: packageInfo.title,
version: packageInfo.version,
...(experimentalDataStreamFeatures
? { experimental_data_stream_features: experimentalDataStreamFeatures }
: undefined),
},
enabled: true,
policy_id: agentPolicyId,

51
x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/experimental_datastream_settings.test.tsx
View file

@ -6,6 +6,7 @@
*/
import React from 'react';
import { act, fireEvent } from '@testing-library/react';
import { createFleetTestRendererMock } from '../../../../../../../../mock';
@ -131,5 +132,55 @@ describe('ExperimentDatastreamSettings', () => {
expect(syntheticSourceSwitch).not.toBeEnabled();
expect(mockSetNewExperimentalDataFeatures).not.toBeCalled();
});
it('should not mutate original experimental feature if a user change one', () => {
const mockSetNewExperimentalDataFeatures = jest.fn();
const experimentalDataFeatures = [
{
data_stream: 'logs-test',
features: {
synthetic_source: true,
tsdb: false,
},
},
];
const res = createFleetTestRendererMock().render(
<ExperimentDatastreamSettings
registryDataStream={
{
type: 'logs',
dataset: 'test',
} as unknown as RegistryDataStream
}
experimentalDataFeatures={experimentalDataFeatures}
setNewExperimentalDataFeatures={mockSetNewExperimentalDataFeatures}
/>
);
act(() => {
fireEvent.click(
res.getByTestId('packagePolicyEditor.syntheticSourceExperimentalFeature.switch')
);
});
expect(mockSetNewExperimentalDataFeatures).toBeCalled();
expect(mockSetNewExperimentalDataFeatures.mock.calls[0][0]).toEqual([
{
data_stream: 'logs-test',
features: {
synthetic_source: false,
tsdb: false,
},
},
]);
expect(experimentalDataFeatures).toEqual([
{
data_stream: 'logs-test',
features: {
synthetic_source: true,
tsdb: false,
},
},
]);
});
});
});

4
x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/experimental_datastream_settings.tsx
View file

@ -73,14 +73,14 @@ export const ExperimentDatastreamSettings: React.FunctionComponent<Props> = ({
const onIndexingSettingChange = (
features: Partial<Record<ExperimentalIndexingFeature, boolean>>
) => {
const newExperimentalDataStreamFeatures = [...(experimentalDataFeatures ?? [])];
const newExperimentalDataStreamFeatures =
experimentalDataFeatures?.map((feature) => ({ ...feature })) ?? [];
const dataStream = getRegistryDataStreamAssetBaseName(registryDataStream);
const existingSettingRecord = newExperimentalDataStreamFeatures.find(
(x) => x.data_stream === dataStream
);
if (existingSettingRecord) {
existingSettingRecord.features = {
...existingSettingRecord.features,

40
x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts
View file

@ -122,4 +122,44 @@ describe('EPM index template install', () => {
expect(packageTemplate.mappings).toHaveProperty('_source');
expect(packageTemplate.mappings._source).toEqual({ mode: 'synthetic' });
});
it('tests prepareTemplate to not set source mode to synthetics if specified but user disabled it', async () => {
const dataStreamDatasetIsPrefixTrue = {
type: 'metrics',
dataset: 'package.dataset',
title: 'test data stream',
release: 'experimental',
package: 'package',
path: 'path',
ingest_pipeline: 'default',
dataset_is_prefix: true,
elasticsearch: {
source_mode: 'synthetic',
},
} as RegistryDataStream;
const pkg = {
name: 'package',
version: '0.0.1',
};
const { componentTemplates } = prepareTemplate({
pkg,
dataStream: dataStreamDatasetIsPrefixTrue,
experimentalDataStreamFeature: {
data_stream: 'metrics-package.dataset',
features: {
synthetic_source: false,
tsdb: false,
},
},
});
const packageTemplate = componentTemplates['metrics-package.dataset@package'].template;
if (!('mappings' in packageTemplate)) {
throw new Error('no mappings on package template');
}
expect(packageTemplate.mappings).not.toHaveProperty('_source');
});
});

31
x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts
View file

@ -23,6 +23,7 @@ import {
getPipelineNameForDatastream,
getFileDataIndexName,
getFileMetadataIndexName,
getRegistryDataStreamAssetBaseName,
} from '../../../../../common/services';
import type {
RegistryDataStream,
@ -35,6 +36,7 @@ import type {
TemplateMap,
EsAssetReference,
PackageInfo,
ExperimentalDataStreamFeature,
} from '../../../../types';
import { loadFieldsFromYaml, processFields } from '../../fields/field';
import { getAsset, getPathParts } from '../../archive';
@ -61,7 +63,8 @@ const FLEET_COMPONENT_TEMPLATE_NAMES = FLEET_COMPONENT_TEMPLATES.map((tmpl) => t
export const prepareToInstallTemplates = (
installablePackage: InstallablePackage,
paths: string[],
esReferences: EsAssetReference[]
esReferences: EsAssetReference[],
experimentalDataStreamFeatures: ExperimentalDataStreamFeature[] = []
): {
assetsToAdd: EsAssetReference[];
assetsToRemove: EsAssetReference[];
@ -78,9 +81,15 @@ export const prepareToInstallTemplates = (
const dataStreams = installablePackage.data_streams;
if (!dataStreams) return { assetsToAdd: [], assetsToRemove, install: () => Promise.resolve([]) };
const templates = dataStreams.map((dataStream) =>
prepareTemplate({ pkg: installablePackage, dataStream })
);
const templates = dataStreams.map((dataStream) => {
const experimentalDataStreamFeature = experimentalDataStreamFeatures.find(
(datastreamFeature) =>
datastreamFeature.data_stream === getRegistryDataStreamAssetBaseName(dataStream)
);
return prepareTemplate({ pkg: installablePackage, dataStream, experimentalDataStreamFeature });
});
const assetsToAdd = getAllTemplateRefs(templates.map((template) => template.indexTemplate));
return {
@ -243,6 +252,7 @@ export function buildComponentTemplates(params: {
packageName: string;
pipelineName?: string;
defaultSettings: IndexTemplate['template']['settings'];
experimentalDataStreamFeature?: ExperimentalDataStreamFeature;
}) {
const {
templateName,
@ -271,6 +281,10 @@ export function buildComponentTemplates(params: {
(dynampingTemplate) => Object.keys(dynampingTemplate)[0]
);
const sourceModeSynthetic =
params.experimentalDataStreamFeature?.features.synthetic_source !== false &&
(params.experimentalDataStreamFeature?.features.synthetic_source === true ||
registryElasticsearch?.source_mode === 'synthetic');
templatesMap[packageTemplateName] = {
template: {
settings: {
@ -291,13 +305,11 @@ export function buildComponentTemplates(params: {
properties: mappingsProperties,
dynamic_templates: mappingsDynamicTemplates.length ? mappingsDynamicTemplates : undefined,
...omit(indexTemplateMappings, 'properties', 'dynamic_templates', '_source'),
...(indexTemplateMappings?._source || registryElasticsearch?.source_mode
...(indexTemplateMappings?._source || sourceModeSynthetic
? {
_source: {
...indexTemplateMappings?._source,
...(registryElasticsearch?.source_mode === 'synthetic'
? { mode: 'synthetic' }
: {}),
...(sourceModeSynthetic ? { mode: 'synthetic' } : {}),
},
}
: {}),
@ -465,9 +477,11 @@ export async function ensureAliasHasWriteIndex(opts: {
export function prepareTemplate({
pkg,
dataStream,
experimentalDataStreamFeature,
}: {
pkg: Pick<PackageInfo, 'name' | 'version' | 'type'>;
dataStream: RegistryDataStream;
experimentalDataStreamFeature?: ExperimentalDataStreamFeature;
}): { componentTemplates: TemplateMap; indexTemplate: IndexTemplateEntry } {
const { name: packageName, version: packageVersion } = pkg;
const fields = loadFieldsFromYaml(pkg, dataStream.path);
@ -494,6 +508,7 @@ export function prepareTemplate({
templateName,
pipelineName,
registryElasticsearch: dataStream.elasticsearch,
experimentalDataStreamFeature,
});
const template = getTemplate({

10
x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts
View file

@ -180,8 +180,16 @@ export async function _installPackage({
* This split of prepare/install could be extended to all asset types. Besides performance, it also allows us to
* more easily write unit tests against the asset generation code without needing to mock ES responses.
*/
const experimentalDataStreamFeatures =
installedPkg?.attributes?.experimental_data_stream_features ?? [];
const preparedIngestPipelines = prepareToInstallPipelines(packageInfo, paths);
const preparedIndexTemplates = prepareToInstallTemplates(packageInfo, paths, esReferences);
const preparedIndexTemplates = prepareToInstallTemplates(
packageInfo,
paths,
esReferences,
experimentalDataStreamFeatures
);
// Update the references for the templates and ingest pipelines together. Need to be done togther to avoid race
// conditions on updating the installed_es field at the same time

12
x-pack/plugins/fleet/server/services/package_policy.ts
View file

@ -211,6 +211,9 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
SAVED_OBJECT_TYPE,
{
...packagePolicy,
...(packagePolicy.package
? { package: omit(packagePolicy.package, 'experimental_data_stream_features') }
: {}),
inputs,
elasticsearch,
revision: 1,
@ -285,6 +288,9 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
id: packagePolicyId,
attributes: {
...pkgPolicyWithoutId,
...(packagePolicy.package
? { package: omit(packagePolicy.package, 'experimental_data_stream_features') }
: {}),
inputs,
elasticsearch,
policy_id: agentPolicyId,
@ -526,6 +532,9 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
id,
{
...restOfPackagePolicy,
...(restOfPackagePolicy.package
? { package: omit(restOfPackagePolicy.package, 'experimental_data_stream_features') }
: {}),
inputs,
elasticsearch,
revision: oldPackagePolicy.revision + 1,
@ -620,6 +629,9 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
id,
attributes: {
...restOfPackagePolicy,
...(restOfPackagePolicy.package
? { package: omit(restOfPackagePolicy.package, 'experimental_data_stream_features') }
: {}),
inputs,
elasticsearch,
revision: oldPackagePolicy.revision + 1,

1
x-pack/plugins/fleet/server/types/index.tsx
View file

@ -87,6 +87,7 @@ export type {
PackageVerificationStatus,
BulkInstallPackageInfo,
PackageAssetReference,
ExperimentalDataStreamFeature,
} from '../../common/types';
export { ElasticsearchAssetType, KibanaAssetType, KibanaSavedObjectType } from '../../common/types';
export { dataTypes } from '../../common/constants';