github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 01:38:56 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Fleet] Fix transforms with new specs not reinstalled when version is same (#155453)

Browse source 
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Quynh Nguyen (Quinn) 2023-04-25 15:15:15 -05:00 • committed by GitHub
parent 933bca2537
commit bfe5aeed48
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 217 additions and 150 deletions
Download patch file Download diff file Expand all files Collapse all files

1
x-pack/plugins/fleet/server/services/api_keys/transform_api_keys.ts
View file

@ -96,7 +96,6 @@ export async function generateTransformSecondaryAuthHeaders({
role_descriptors: {},
}
);
logger.debug(`Created api_key name: ${name}`);
let encodedApiKey: TransformAPIKey['encoded'] | null = null;

119
x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts
View file

@ -146,6 +146,7 @@ const processTransformAssetsPerModule = (
installNameSuffix: string,
transformPaths: string[],
previousInstalledTransformEsAssets: EsAssetReference[] = [],
force?: boolean,
username?: string
) => {
const transformsSpecifications = new Map();
@ -244,8 +245,14 @@ const processTransformAssetsPerModule = (
`default-${transformVersion}`
);
const currentTransformSameAsPrev =
previousInstalledTransformEsAssets.find((t) => t.id === installationName) !== undefined;
// Here, we track if fleet_transform_version (not package version) has changed based on installation name
// if version has changed, install transform and update es assets
// else, don't delete the dest index and install transform as it can be an expensive operation
const matchingTransformFromPrevInstall = previousInstalledTransformEsAssets.find(
(t) => t.id === installationName
);
const currentTransformSameAsPrev = matchingTransformFromPrevInstall !== undefined;
if (previousInstalledTransformEsAssets.length === 0) {
aliasesRefs.push(...aliasNames);
transforms.push({
@ -258,30 +265,36 @@ const processTransformAssetsPerModule = (
});
transformsSpecifications.get(transformModuleId)?.set('transformVersionChanged', true);
} else {
if (!currentTransformSameAsPrev) {
// If upgrading from old json schema to new yml schema
// We need to make sure to delete those transforms by matching the legacy naming convention
const versionFromOldJsonSchema = previousInstalledTransformEsAssets.find((t) =>
t.id.startsWith(
getLegacyTransformNameForInstallation(
installablePackage,
`${transformModuleId}/default.json`
if (force || !currentTransformSameAsPrev) {
// If we are reinstalling the package (i.e. force = true),
// force delete old transforms so we can reinstall the same transforms again
if (force && matchingTransformFromPrevInstall) {
transformsToRemoveWithDestIndex.push(matchingTransformFromPrevInstall);
} else {
// If upgrading from old json schema to new yml schema
// We need to make sure to delete those transforms by matching the legacy naming convention
const versionFromOldJsonSchema = previousInstalledTransformEsAssets.find((t) =>
t.id.startsWith(
getLegacyTransformNameForInstallation(
installablePackage,
`${transformModuleId}/default.json`
)
)
)
);
);
if (versionFromOldJsonSchema !== undefined) {
transformsToRemoveWithDestIndex.push(versionFromOldJsonSchema);
}
if (versionFromOldJsonSchema !== undefined) {
transformsToRemoveWithDestIndex.push(versionFromOldJsonSchema);
}
// If upgrading from yml to newer version of yaml
// Match using new naming convention
const installNameWithoutVersion = installationName.split(transformVersion)[0];
const prevVersion = previousInstalledTransformEsAssets.find((t) =>
t.id.startsWith(installNameWithoutVersion)
);
if (prevVersion !== undefined) {
transformsToRemove.push(prevVersion);
// If upgrading from yml to newer version of yaml
// Match using new naming convention
const installNameWithoutVersion = installationName.split(transformVersion)[0];
const prevVersion = previousInstalledTransformEsAssets.find((t) =>
t.id.startsWith(installNameWithoutVersion)
);
if (prevVersion !== undefined) {
transformsToRemove.push(prevVersion);
}
}
transforms.push({
transformModuleId,
@ -387,6 +400,7 @@ const installTransformsAssets = async (
logger: Logger,
esReferences: EsAssetReference[] = [],
previousInstalledTransformEsAssets: EsAssetReference[] = [],
force?: boolean,
authorizationHeader?: HTTPAuthorizationHeader | null
) => {
let installedTransforms: EsAssetReference[] = [];
@ -408,20 +422,24 @@ const installTransformsAssets = async (
installNameSuffix,
transformPaths,
previousInstalledTransformEsAssets,
force,
username
);
// By default, for internal Elastic packages that touch system indices, we want to run as internal user
// so we set runAsKibanaSystem: true by default (e.g. when run_as_kibana_system set to true/not defined in yml file).
// If package should be installed as the logged in user, set run_as_kibana_system: false,
// and pass es-secondary-authorization in header when creating the transforms.
const secondaryAuth = await generateTransformSecondaryAuthHeaders({
authorizationHeader,
logger,
pkgName: installablePackage.name,
pkgVersion: installablePackage.version,
username,
});
// generate api key, and pass es-secondary-authorization in header when creating the transforms.
const secondaryAuth = transforms.some((t) => t.runAsKibanaSystem === false)
? await generateTransformSecondaryAuthHeaders({
authorizationHeader,
logger,
pkgName: installablePackage.name,
pkgVersion: installablePackage.version,
username,
})
: // No need to generate api key/secondary auth if all transforms are run as kibana_system user
undefined;
// delete all previous transform
await Promise.all([
@ -567,15 +585,34 @@ const installTransformsAssets = async (
return { installedTransforms, esReferences };
};
export const installTransforms = async (
installablePackage: InstallablePackage,
paths: string[],
esClient: ElasticsearchClient,
savedObjectsClient: SavedObjectsClientContract,
logger: Logger,
esReferences?: EsAssetReference[],
authorizationHeader?: HTTPAuthorizationHeader | null
) => {
interface InstallTransformsParams {
installablePackage: InstallablePackage;
paths: string[];
esClient: ElasticsearchClient;
savedObjectsClient: SavedObjectsClientContract;
logger: Logger;
esReferences?: EsAssetReference[];
/**
* Force transforms to install again even though fleet_transform_version might be same
* Should be true when package is re-installing
*/
force?: boolean;
/**
* Authorization header parsed from original Kibana request, used to generate API key from user
* to pass in secondary authorization info to transform
*/
authorizationHeader?: HTTPAuthorizationHeader | null;
}
export const installTransforms = async ({
installablePackage,
paths,
esClient,
savedObjectsClient,
logger,
force,
esReferences,
authorizationHeader,
}: InstallTransformsParams) => {
const transformPaths = paths.filter((path) => isTransform(path));
const installation = await getInstallation({
@ -613,6 +650,7 @@ export const installTransforms = async (
);
}
// If package contains yml transform specifications
return await installTransformsAssets(
installablePackage,
installNameSuffix,
@ -622,6 +660,7 @@ export const installTransforms = async (
logger,
esReferences,
previousInstalledTransformEsAssets,
force,
authorizationHeader
);
};

48
x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/legacy_transforms.test.ts
View file

@ -122,8 +122,8 @@ describe('test transform install with legacy schema', () => {
],
});
await installTransforms(
{
await installTransforms({
installablePackage: {
name: 'endpoint',
version: '0.16.0-dev.0',
data_streams: [
@ -157,16 +157,16 @@ describe('test transform install with legacy schema', () => {
},
],
} as unknown as RegistryPackage,
[
paths: [
'endpoint-0.16.0-dev.0/data_stream/policy/elasticsearch/ingest_pipeline/default.json',
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata/default.json',
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/default.json',
],
esClient,
savedObjectsClient,
loggerMock.create(),
previousInstallation.installed_es
);
logger: loggerMock.create(),
esReferences: previousInstallation.installed_es,
});
expect(esClient.transform.getTransform.mock.calls).toEqual([
[
@ -320,8 +320,8 @@ describe('test transform install with legacy schema', () => {
} as unknown as SavedObject<Installation>)
);
await installTransforms(
{
await installTransforms({
installablePackage: {
name: 'endpoint',
version: '0.16.0-dev.0',
data_streams: [
@ -341,12 +341,12 @@ describe('test transform install with legacy schema', () => {
},
],
} as unknown as RegistryPackage,
['endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/default.json'],
paths: ['endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/default.json'],
esClient,
savedObjectsClient,
loggerMock.create(),
previousInstallation.installed_es
);
logger: loggerMock.create(),
esReferences: previousInstallation.installed_es,
});
const meta = getESAssetMetadata({ packageName: 'endpoint' });
@ -422,8 +422,8 @@ describe('test transform install with legacy schema', () => {
],
});
await installTransforms(
{
await installTransforms({
installablePackage: {
name: 'endpoint',
version: '0.16.0-dev.0',
data_streams: [
@ -457,12 +457,12 @@ describe('test transform install with legacy schema', () => {
},
],
} as unknown as RegistryPackage,
[],
paths: [],
esClient,
savedObjectsClient,
loggerMock.create(),
previousInstallation.installed_es
);
logger: loggerMock.create(),
esReferences: previousInstallation.installed_es,
});
expect(esClient.transform.getTransform.mock.calls).toEqual([
[
@ -556,8 +556,8 @@ describe('test transform install with legacy schema', () => {
)
);
await installTransforms(
{
await installTransforms({
installablePackage: {
name: 'endpoint',
version: '0.16.0-dev.0',
data_streams: [
@ -577,12 +577,12 @@ describe('test transform install with legacy schema', () => {
},
],
} as unknown as RegistryPackage,
['endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/default.json'],
paths: ['endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/default.json'],
esClient,
savedObjectsClient,
loggerMock.create(),
previousInstallation.installed_es
);
logger: loggerMock.create(),
esReferences: previousInstallation.installed_es,
});
const meta = getESAssetMetadata({ packageName: 'endpoint' });

13
x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/reauthorize.ts
View file

@ -26,6 +26,7 @@ interface FleetTransformMetadata {
managed_by?: string;
installed_by?: string;
last_authorized_by?: string;
run_as_kibana_system?: boolean;
transformId: string;
}
@ -114,10 +115,14 @@ export async function handleTransformReauthorizeAndStart({
)
)
);
const transformsMetadata: FleetTransformMetadata[] = transformInfos.flat().map((t) => {
const transform = t.transforms?.[0];
return { ...transform._meta, transformId: transform?.id };
});
const transformsMetadata: FleetTransformMetadata[] = transformInfos
.flat()
.map<FleetTransformMetadata>((t) => {
const transform = t.transforms?.[0];
return { ...transform._meta, transformId: transform?.id };
})
.filter((t) => t?.run_as_kibana_system === false);
const shouldInstallSequentially =
uniqBy(transformsMetadata, 'order').length === transforms.length;

54
x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transform_utils.test.ts
View file

@ -10,38 +10,50 @@ import { getDestinationIndexAliases } from './transform_utils';
describe('test transform_utils', () => {
describe('getDestinationIndexAliases()', function () {
test('return transform alias settings when input is an object', () => {
const aliasSettings = {
'.alerts-security.host-risk-score-latest.latest': { move_on_creation: true },
'.alerts-security.host-risk-score-latest.all': { move_on_creation: false },
};
expect(getDestinationIndexAliases(aliasSettings)).toStrictEqual([
{ alias: '.alerts-security.host-risk-score-latest.latest', move_on_creation: true },
{ alias: '.alerts-security.host-risk-score-latest.all', move_on_creation: false },
expect(
getDestinationIndexAliases({
'alias1.latest': { move_on_creation: true },
'alias1.all': { move_on_creation: false },
})
).toStrictEqual([
{ alias: 'alias1.latest', move_on_creation: true },
{ alias: 'alias1.all', move_on_creation: false },
]);
expect(
getDestinationIndexAliases({
'alias1.latest': null,
'alias1.all': { move_on_creation: false },
alias2: { move_on_creation: true },
alias3: undefined,
alias4: '',
alias5: 'invalid string',
})
).toStrictEqual([
{ alias: 'alias1.latest', move_on_creation: false },
{ alias: 'alias1.all', move_on_creation: false },
{ alias: 'alias2', move_on_creation: true },
{ alias: 'alias3', move_on_creation: false },
{ alias: 'alias4', move_on_creation: false },
{ alias: 'alias5', move_on_creation: false },
]);
});
test('return transform alias settings when input is an array', () => {
const aliasSettings = [
'.alerts-security.host-risk-score-latest.latest',
'.alerts-security.host-risk-score-latest.all',
];
const aliasSettings = ['alias1.latest', 'alias1.all'];
expect(getDestinationIndexAliases(aliasSettings)).toStrictEqual([
{ alias: '.alerts-security.host-risk-score-latest.latest', move_on_creation: true },
{ alias: '.alerts-security.host-risk-score-latest.all', move_on_creation: false },
{ alias: 'alias1.latest', move_on_creation: true },
{ alias: 'alias1.all', move_on_creation: false },
]);
});
test('return transform alias settings when input is a string', () => {
expect(
getDestinationIndexAliases('.alerts-security.host-risk-score-latest.latest')
).toStrictEqual([
{ alias: '.alerts-security.host-risk-score-latest.latest', move_on_creation: true },
expect(getDestinationIndexAliases('alias1.latest')).toStrictEqual([
{ alias: 'alias1.latest', move_on_creation: true },
]);
expect(
getDestinationIndexAliases('.alerts-security.host-risk-score-latest.all')
).toStrictEqual([
{ alias: '.alerts-security.host-risk-score-latest.all', move_on_creation: false },
expect(getDestinationIndexAliases('alias1.all')).toStrictEqual([
{ alias: 'alias1.all', move_on_creation: false },
]);
});

7
x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transform_utils.ts
View file

@ -19,11 +19,14 @@ export const getDestinationIndexAliases = (aliasSettings: unknown): TransformAli
if (!aliasSettings) return aliases;
// If in form of
// Can be in form of {
// 'alias1': null,
// 'alias2': { move_on_creation: false }
// }
if (isPopulatedObject<string, { move_on_creation?: boolean }>(aliasSettings)) {
Object.keys(aliasSettings).forEach((alias) => {
if (aliasSettings.hasOwnProperty(alias) && typeof alias === 'string') {
const moveOnCreation = aliasSettings[alias].move_on_creation === true;
const moveOnCreation = aliasSettings[alias]?.move_on_creation === true;
aliases.push({ alias, move_on_creation: moveOnCreation });
}
});

74
x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transforms.test.ts
View file

@ -241,21 +241,21 @@ _meta:
],
});
await installTransforms(
{
await installTransforms({
installablePackage: {
name: 'endpoint',
version: '0.16.0-dev.0',
} as unknown as RegistryPackage,
[
paths: [
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/fields/fields.yml',
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/manifest.yml',
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml',
],
esClient,
savedObjectsClient,
loggerMock.create(),
previousInstallation.installed_es
);
logger: loggerMock.create(),
esReferences: previousInstallation.installed_es,
});
// Stop and delete previously installed transforms
expect(esClient.transform.stopTransform.mock.calls).toEqual([
@ -516,21 +516,21 @@ _meta:
],
});
await installTransforms(
{
await installTransforms({
installablePackage: {
name: 'endpoint',
version: '0.16.0-dev.0',
} as unknown as RegistryPackage,
[
paths: [
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/fields/fields.yml',
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/manifest.yml',
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml',
],
esClient,
savedObjectsClient,
loggerMock.create(),
previousInstallation.installed_es
);
logger: loggerMock.create(),
esReferences: previousInstallation.installed_es,
});
// Stop and delete previously installed transforms
expect(esClient.transform.stopTransform.mock.calls).toEqual([
@ -798,20 +798,20 @@ _meta:
],
});
await installTransforms(
{
await installTransforms({
installablePackage: {
name: 'endpoint',
version: '0.16.0-dev.0',
} as unknown as RegistryPackage,
[
paths: [
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/fields/fields.yml',
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml',
],
esClient,
savedObjectsClient,
loggerMock.create(),
previousInstallation.installed_es
);
logger: loggerMock.create(),
esReferences: previousInstallation.installed_es,
});
// Stop and delete previously installed transforms
expect(esClient.transform.stopTransform.mock.calls).toEqual([
@ -1015,21 +1015,21 @@ _meta:
} as unknown as SavedObject<Installation>)
);
await installTransforms(
{
await installTransforms({
installablePackage: {
name: 'endpoint',
version: '0.16.0-dev.0',
} as unknown as RegistryPackage,
[
paths: [
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/manifest.yml',
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml',
],
esClient,
savedObjectsClient,
loggerMock.create(),
previousInstallation.installed_es,
authorizationHeader
);
logger: loggerMock.create(),
esReferences: previousInstallation.installed_es,
authorizationHeader,
});
expect(esClient.transform.putTransform.mock.calls).toEqual([[expectedData.TRANSFORM]]);
// Does not start transform because start is set to false in manifest.yml
@ -1108,20 +1108,20 @@ _meta:
})
);
await installTransforms(
{
await installTransforms({
installablePackage: {
name: 'endpoint',
version: '0.16.0-dev.0',
} as unknown as RegistryPackage,
[
paths: [
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/manifest.yml',
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml',
],
esClient,
savedObjectsClient,
loggerMock.create(),
previousInstallation.installed_es
);
logger: loggerMock.create(),
esReferences: previousInstallation.installed_es,
});
expect(esClient.indices.create.mock.calls).toEqual([]);
@ -1189,21 +1189,21 @@ _meta:
} as unknown as SavedObject<Installation>)
);
await installTransforms(
{
await installTransforms({
installablePackage: {
name: 'endpoint',
version: '0.16.0-dev.0',
} as unknown as RegistryPackage,
[
paths: [
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/fields/fields.yml',
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/manifest.yml',
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml',
],
esClient,
savedObjectsClient,
loggerMock.create(),
previousInstallation.installed_es
);
logger: loggerMock.create(),
esReferences: previousInstallation.installed_es,
});
// Transform from old version is neither stopped nor deleted
expect(esClient.transform.stopTransform.mock.calls).toEqual([]);

22
x-pack/plugins/fleet/server/services/epm/package_service.test.ts
View file

@ -132,15 +132,19 @@ function getTest(
args: [pkg, paths],
spy: jest.spyOn(epmTransformsInstall, 'installTransforms'),
spyArgs: [
pkg,
paths,
mocks.esClient,
mocks.soClient,
mocks.logger,
// Undefined es references
undefined,
// Undefined secondary authorization
undefined,
{
installablePackage: pkg,
paths,
esClient: mocks.esClient,
savedObjectsClient: mocks.soClient,
logger: mocks.logger,
// package reinstall means we need to force transforms to reinstall
force: true,
// Undefined es references
esReferences: undefined,
// Undefined secondary authorization
authorizationHeader: undefined,
},
],
spyResponse: {
installedTransforms: [

17
x-pack/plugins/fleet/server/services/epm/package_service.ts
View file

@ -209,15 +209,16 @@ class PackageClientImpl implements PackageClient {
async #reinstallTransforms(packageInfo: InstallablePackage, paths: string[]) {
const authorizationHeader = await this.getAuthorizationHeader();
const { installedTransforms } = await installTransforms(
packageInfo,
const { installedTransforms } = await installTransforms({
installablePackage: packageInfo,
paths,
this.internalEsClient,
this.internalSoClient,
this.logger,
undefined,
authorizationHeader
);
esClient: this.internalEsClient,
savedObjectsClient: this.internalSoClient,
logger: this.logger,
force: true,
esReferences: undefined,
authorizationHeader,
});
return installedTransforms;
}

11
x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts
View file

@ -75,6 +75,7 @@ export async function _installPackage({
installType,
installSource,
spaceId,
force,
verificationResult,
authorizationHeader,
}: {
@ -90,6 +91,7 @@ export async function _installPackage({
installType: InstallType;
installSource: InstallSource;
spaceId: string;
force?: boolean;
verificationResult?: PackageVerificationResult;
authorizationHeader?: HTTPAuthorizationHeader | null;
}): Promise<AssetReference[]> {
@ -249,15 +251,16 @@ export async function _installPackage({
);
({ esReferences } = await withPackageSpan('Install transforms', () =>
installTransforms(
packageInfo,
installTransforms({
installablePackage: packageInfo,
paths,
esClient,
savedObjectsClient,
logger,
esReferences,
authorizationHeader
)
force,
authorizationHeader,
})
));
// If this is an update or retrying an update, delete the previous version's pipelines

1
x-pack/plugins/fleet/server/services/epm/packages/install.ts
View file

@ -511,6 +511,7 @@ async function installPackageCommon(options: {
verificationResult,
installSource,
authorizationHeader,
force,
})
.then(async (assets) => {
await removeOldAssets({