github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 01:13:23 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Fleet] Use streaming for package install instead of an assetsMap with everything loaded in memory (#211961)

Browse source
This commit is contained in:
Nicolas Chaulet 2025-03-04 13:15:34 -05:00 • committed by GitHub
parent 4447a7050a
commit d3d44defa4
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
32 changed files with 701 additions and 548 deletions
Download patch file Download diff file Expand all files Collapse all files

10
x-pack/platform/plugins/shared/fleet/common/types/models/epm.ts
View file

@ -132,17 +132,15 @@ export interface ArchiveEntry {
}
export interface ArchiveIterator {
traverseEntries: (onEntry: (entry: ArchiveEntry) => Promise<void>) => Promise<void>;
traverseEntries: (
onEntry: (entry: ArchiveEntry) => Promise<void>,
readBuffer?: (path: string) => boolean
) => Promise<void>;
getPaths: () => Promise<string[]>;
}
export interface PackageInstallContext {
packageInfo: InstallablePackage;
/**
* @deprecated Use `archiveIterator` to access the package archive entries
* without loading them all into memory at once.
*/
assetsMap: AssetsMap;
paths: string[];
archiveIterator: ArchiveIterator;
}

1
x-pack/platform/plugins/shared/fleet/server/routes/epm/kibana_assets_handler.ts
View file

@ -68,7 +68,6 @@ export const installPackageKibanaAssetsHandler: FleetRequestHandler<
packageInstallContext: {
packageInfo,
paths: installedPkgWithAssets.paths,
assetsMap: installedPkgWithAssets.assetsMap,
archiveIterator: createArchiveIteratorFromMap(installedPkgWithAssets.assetsMap),
},
});

9
x-pack/platform/plugins/shared/fleet/server/services/epm/archive/archive_iterator.ts
View file

@ -74,10 +74,15 @@ export const createArchiveIterator = (
*/
export const createArchiveIteratorFromMap = (assetsMap: AssetsMap): ArchiveIterator => {
const traverseEntries = async (
onEntry: (entry: ArchiveEntry) => Promise<void>
onEntry: (entry: ArchiveEntry) => Promise<void>,
readBuffer?: (path: string) => boolean
): Promise<void> => {
for (const [path, buffer] of assetsMap) {
await onEntry({ path, buffer });
if (readBuffer && !readBuffer(path)) {
await onEntry({ path });
} else {
await onEntry({ path, buffer });
}
}
};

23
x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/datastream_ilm/install.ts
View file

@ -12,7 +12,11 @@ import {
ElasticsearchAssetType,
type PackageInstallContext,
} from '../../../../../common/types/models';
import type { EsAssetReference, RegistryDataStream } from '../../../../../common/types/models';
import type {
AssetsMap,
EsAssetReference,
RegistryDataStream,
} from '../../../../../common/types/models';
import { updateEsAssetReferences } from '../../packages/es_assets_reference';
import { getAssetFromAssetsMap } from '../../archive';
@ -40,7 +44,7 @@ export const installIlmForDataStream = async (
logger: Logger,
esReferences: EsAssetReference[]
) => {
const { packageInfo: registryPackage, paths, assetsMap } = packageInstallContext;
const { packageInfo: registryPackage, paths } = packageInstallContext;
const previousInstalledIlmEsAssets = esReferences.filter(
({ type }) => type === ElasticsearchAssetType.dataStreamIlmPolicy
);
@ -72,6 +76,19 @@ export const installIlmForDataStream = async (
};
const dataStreamIlmPaths = paths.filter((path) => isDataStreamIlm(path));
const dataStreamIlmAssetsMap: AssetsMap = new Map();
await packageInstallContext.archiveIterator.traverseEntries(
async (entry) => {
if (!entry.buffer) {
return;
}
dataStreamIlmAssetsMap.set(entry.path, entry.buffer);
},
(path) => dataStreamIlmPaths.includes(path)
);
let installedIlms: EsAssetReference[] = [];
if (dataStreamIlmPaths.length > 0) {
const ilmPathDatasets = dataStreams.reduce<IlmPathDataset[]>((acc, dataStream) => {
@ -103,7 +120,7 @@ export const installIlmForDataStream = async (
const ilmInstallations: IlmInstallation[] = ilmPathDatasets.map(
(ilmPathDataset: IlmPathDataset) => {
const content = JSON.parse(
getAssetFromAssetsMap(assetsMap, ilmPathDataset.path).toString('utf-8')
getAssetFromAssetsMap(dataStreamIlmAssetsMap, ilmPathDataset.path).toString('utf-8')
);
content.policy._meta = getESAssetMetadata({ packageName: registryPackage.name });

18
x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/ilm/install.ts
View file

@ -16,7 +16,7 @@ import { updateEsAssetReferences } from '../../packages/es_assets_reference';
import { getESAssetMetadata } from '../meta';
import { retryTransientEsErrors } from '../retry';
import { PackageInvalidArchiveError } from '../../../../errors';
import type { PackageInstallContext } from '../../../../../common/types';
import type { AssetsMap, PackageInstallContext } from '../../../../../common/types';
import { MAX_CONCURRENT_ILM_POLICIES_OPERATIONS } from '../../../../constants';
export async function installILMPolicy(
@ -30,10 +30,20 @@ export async function installILMPolicy(
const ilmPaths = packageInstallContext.paths.filter((path) => isILMPolicy(path));
if (!ilmPaths.length) return esReferences;
const ilmAssetsMap: AssetsMap = new Map();
await packageInstallContext.archiveIterator.traverseEntries(
async (entry) => {
if (!entry.buffer) {
return;
}
ilmAssetsMap.set(entry.path, entry.buffer);
},
(path) => ilmPaths.includes(path)
);
const ilmPolicies = ilmPaths.map((path) => {
const body = JSON.parse(
getAssetFromAssetsMap(packageInstallContext.assetsMap, path).toString('utf-8')
);
const body = JSON.parse(getAssetFromAssetsMap(ilmAssetsMap, path).toString('utf-8'));
body.policy._meta = getESAssetMetadata({ packageName: packageInfo.name });

30
x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.test.ts
View file

@ -7,6 +7,8 @@
import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks';
import { loggerMock } from '@kbn/logging-mocks';
import { createArchiveIteratorFromMap } from '../../archive/archive_iterator';
import { prepareToInstallPipelines } from './install';
jest.mock('../../archive/cache');
@ -25,10 +27,10 @@ describe('Install pipeline tests', () => {
path: '/datasettest',
},
],
},
} as any,
paths: [],
assetsMap: new Map(),
} as any);
archiveIterator: createArchiveIteratorFromMap(new Map()),
});
expect(res.assetsToAdd).toEqual([{ id: 'logs-datasettest-1.0.0', type: 'ingest_pipeline' }]);
const esClient = elasticsearchClientMock.createInternalClient();
@ -64,16 +66,18 @@ describe('Install pipeline tests', () => {
'packagetest-1.0.0/data_stream/datasettest/elasticsearch/ingest_pipeline/default.yml',
'packagetest-1.0.0/data_stream/datasettest/elasticsearch/ingest_pipeline/standard.yml',
],
assetsMap: new Map([
[
'packagetest-1.0.0/data_stream/datasettest/elasticsearch/ingest_pipeline/default.yml',
Buffer.from('description: test'),
],
[
'packagetest-1.0.0/data_stream/datasettest/elasticsearch/ingest_pipeline/standard.yml',
Buffer.from('description: test'),
],
]),
archiveIterator: createArchiveIteratorFromMap(
new Map([
[
'packagetest-1.0.0/data_stream/datasettest/elasticsearch/ingest_pipeline/default.yml',
Buffer.from('description: test'),
],
[
'packagetest-1.0.0/data_stream/datasettest/elasticsearch/ingest_pipeline/standard.yml',
Buffer.from('description: test'),
],
])
),
} as any);
expect(res.assetsToAdd).toEqual([
{ id: 'logs-datasettest-1.0.0', type: 'ingest_pipeline' },

17
x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts
View file

@ -25,7 +25,7 @@ import {
FLEET_EVENT_INGESTED_PIPELINE_CONTENT,
} from '../../../../constants';
import { getPipelineNameForDatastream } from '../../../../../common/services';
import type { ArchiveEntry, PackageInstallContext } from '../../../../../common/types';
import type { ArchiveEntry, AssetsMap, PackageInstallContext } from '../../../../../common/types';
import { appendMetadataToIngestPipeline } from '../meta';
import { retryTransientEsErrors } from '../retry';
@ -157,6 +157,19 @@ export async function installAllPipelines({
> = [];
const substitutions: RewriteSubstitution[] = [];
const pipelineAssetsMap: AssetsMap = new Map();
await packageInstallContext.archiveIterator.traverseEntries(
async (entry) => {
if (!entry.buffer) {
return;
}
pipelineAssetsMap.set(entry.path, entry.buffer);
},
(path) => pipelinePaths.includes(path)
);
let datastreamPipelineCreated = false;
pipelinePaths.forEach((path) => {
const { name, extension } = getNameAndExtension(path);
@ -169,7 +182,7 @@ export async function installAllPipelines({
dataStream,
packageVersion: packageInstallContext.packageInfo.version,
});
const content = getAssetFromAssetsMap(packageInstallContext.assetsMap, path).toString('utf-8');
const content = getAssetFromAssetsMap(pipelineAssetsMap, path).toString('utf-8');
pipelinesInfos.push({
nameForInstallation,
shouldInstallCustomPipelines: dataStream && isMainPipeline,

16
x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/ml_model/install.ts
View file

@ -13,7 +13,7 @@ import {
ElasticsearchAssetType,
type PackageInstallContext,
} from '../../../../../common/types/models';
import type { EsAssetReference } from '../../../../../common/types/models';
import type { AssetsMap, EsAssetReference } from '../../../../../common/types/models';
import { retryTransientEsErrors } from '../retry';
@ -34,9 +34,19 @@ export const installMlModel = async (
const mlModelPath = packageInstallContext.paths.find((path) => isMlModel(path));
if (mlModelPath !== undefined) {
const content = getAssetFromAssetsMap(packageInstallContext.assetsMap, mlModelPath).toString(
'utf-8'
const mlModelAssetsMap: AssetsMap = new Map();
await packageInstallContext.archiveIterator.traverseEntries(
async (entry) => {
if (!entry.buffer) {
return;
}
mlModelAssetsMap.set(entry.path, entry.buffer);
},
(path) => path === mlModelPath
);
const content = getAssetFromAssetsMap(mlModelAssetsMap, mlModelPath).toString('utf-8');
const pathParts = mlModelPath.split('/');
const modelId = pathParts[pathParts.length - 1].replace('.json', '');

36
x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/template/install.test.ts
View file

@ -10,6 +10,7 @@ import { createAppContextStartContractMock } from '../../../../mocks';
import { appContextService } from '../../..';
import { loadDatastreamsFieldsFromYaml } from '../../fields/field';
import type { PackageInstallContext, RegistryDataStream } from '../../../../../common/types';
import { createArchiveIteratorFromMap } from '../../archive/archive_iterator';
import { prepareTemplate, prepareToInstallTemplates } from './install';
@ -53,7 +54,11 @@ describe('EPM index template install', () => {
const templatePriorityDatasetIsPrefixUnset = 200;
const {
indexTemplate: { indexTemplate },
} = prepareTemplate({ packageInstallContext, dataStream: dataStreamDatasetIsPrefixUnset });
} = await prepareTemplate({
packageInstallContext,
fieldAssetsMap: new Map(),
dataStream: dataStreamDatasetIsPrefixUnset,
});
expect(indexTemplate.priority).toBe(templatePriorityDatasetIsPrefixUnset);
expect(indexTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixUnset]);
});
@ -74,7 +79,11 @@ describe('EPM index template install', () => {
const templatePriorityDatasetIsPrefixFalse = 200;
const {
indexTemplate: { indexTemplate },
} = prepareTemplate({ packageInstallContext, dataStream: dataStreamDatasetIsPrefixFalse });
} = prepareTemplate({
packageInstallContext,
fieldAssetsMap: new Map(),
dataStream: dataStreamDatasetIsPrefixFalse,
});
expect(indexTemplate.priority).toBe(templatePriorityDatasetIsPrefixFalse);
expect(indexTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixFalse]);
@ -96,7 +105,11 @@ describe('EPM index template install', () => {
const templatePriorityDatasetIsPrefixTrue = 150;
const {
indexTemplate: { indexTemplate },
} = prepareTemplate({ packageInstallContext, dataStream: dataStreamDatasetIsPrefixTrue });
} = prepareTemplate({
packageInstallContext,
fieldAssetsMap: new Map(),
dataStream: dataStreamDatasetIsPrefixTrue,
});
expect(indexTemplate.priority).toBe(templatePriorityDatasetIsPrefixTrue);
expect(indexTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixTrue]);
@ -119,6 +132,7 @@ describe('EPM index template install', () => {
const { componentTemplates } = prepareTemplate({
packageInstallContext,
fieldAssetsMap: new Map(),
dataStream: dataStreamDatasetIsPrefixTrue,
});
@ -149,6 +163,7 @@ describe('EPM index template install', () => {
const { componentTemplates } = prepareTemplate({
packageInstallContext,
fieldAssetsMap: new Map(),
dataStream: dataStreamDatasetIsPrefixTrue,
});
@ -180,6 +195,7 @@ describe('EPM index template install', () => {
const { componentTemplates } = prepareTemplate({
packageInstallContext,
dataStream: dataStreamDatasetIsPrefixTrue,
fieldAssetsMap: new Map(),
experimentalDataStreamFeature: {
data_stream: 'metrics-package.dataset',
features: {
@ -218,6 +234,7 @@ describe('EPM index template install', () => {
const { componentTemplates } = prepareTemplate({
packageInstallContext,
dataStream: dataStreamDatasetIsPrefixTrue,
fieldAssetsMap: new Map(),
experimentalDataStreamFeature: {
data_stream: 'metrics-package.dataset',
features: {
@ -255,6 +272,7 @@ describe('EPM index template install', () => {
const { indexTemplate } = prepareTemplate({
packageInstallContext,
fieldAssetsMap: new Map(),
dataStream: dataStreamDatasetIsPrefixTrue,
});
@ -286,6 +304,7 @@ describe('EPM index template install', () => {
const { componentTemplates } = prepareTemplate({
packageInstallContext,
dataStream,
fieldAssetsMap: new Map(),
});
const packageTemplate = componentTemplates['logs-package.dataset@package'].template;
@ -313,6 +332,7 @@ describe('EPM index template install', () => {
const { componentTemplates } = prepareTemplate({
packageInstallContext,
dataStream,
fieldAssetsMap: new Map(),
});
const packageTemplate = componentTemplates['logs-package.dataset@package'].template;
@ -350,6 +370,7 @@ describe('EPM index template install', () => {
const { componentTemplates } = prepareTemplate({
packageInstallContext,
fieldAssetsMap: new Map(),
dataStream,
});
@ -388,6 +409,7 @@ describe('EPM index template install', () => {
const { componentTemplates } = prepareTemplate({
packageInstallContext,
fieldAssetsMap: new Map(),
dataStream,
});
@ -428,6 +450,7 @@ describe('EPM index template install', () => {
const { componentTemplates } = prepareTemplate({
packageInstallContext,
fieldAssetsMap: new Map(),
dataStream,
});
@ -472,6 +495,7 @@ describe('EPM index template install', () => {
const { componentTemplates } = prepareTemplate({
packageInstallContext,
fieldAssetsMap: new Map(),
dataStream,
});
@ -507,6 +531,7 @@ describe('EPM index template install', () => {
const { componentTemplates } = prepareTemplate({
packageInstallContext,
fieldAssetsMap: new Map(),
dataStream,
});
@ -515,7 +540,7 @@ describe('EPM index template install', () => {
expect(packageTemplate).not.toHaveProperty('lifecycle');
});
test('test prepareToInstallTemplates does not include stack component templates in tracked assets', () => {
test('test prepareToInstallTemplates does not include stack component templates in tracked assets', async () => {
const dataStreamDatasetIsPrefixUnset = {
type: 'logs',
dataset: 'package.dataset',
@ -526,13 +551,14 @@ describe('EPM index template install', () => {
ingest_pipeline: 'default',
} as RegistryDataStream;
const { assetsToAdd } = prepareToInstallTemplates(
const { assetsToAdd } = await prepareToInstallTemplates(
{
packageInfo: {
name: 'package',
version: '0.0.1',
data_streams: [dataStreamDatasetIsPrefixUnset],
},
archiveIterator: createArchiveIteratorFromMap(new Map()),
} as PackageInstallContext,
[],
[]

62
x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/template/install.ts
View file

@ -32,7 +32,7 @@ import type {
ExperimentalDataStreamFeature,
} from '../../../../types';
import type { Fields } from '../../fields/field';
import { loadDatastreamsFieldsFromYaml, processFields } from '../../fields/field';
import { isFields, loadDatastreamsFieldsFromYaml, processFields } from '../../fields/field';
import { getAssetFromAssetsMap, getPathParts } from '../../archive';
import {
FLEET_COMPONENT_TEMPLATES,
@ -48,7 +48,7 @@ import {
forEachMappings,
} from '../../../experimental_datastream_features_helper';
import { appContextService } from '../../../app_context';
import type { PackageInstallContext } from '../../../../../common/types';
import type { AssetsMap, PackageInstallContext } from '../../../../../common/types';
import {
generateMappings,
@ -62,16 +62,16 @@ import { isUserSettingsTemplate } from './utils';
const FLEET_COMPONENT_TEMPLATE_NAMES = FLEET_COMPONENT_TEMPLATES.map((tmpl) => tmpl.name);
export const prepareToInstallTemplates = (
export const prepareToInstallTemplates = async (
packageInstallContext: PackageInstallContext,
esReferences: EsAssetReference[],
experimentalDataStreamFeatures: ExperimentalDataStreamFeature[] = [],
onlyForDataStreams?: RegistryDataStream[]
): {
): Promise<{
assetsToAdd: EsAssetReference[];
assetsToRemove: EsAssetReference[];
install: (esClient: ElasticsearchClient, logger: Logger) => Promise<IndexTemplateEntry[]>;
} => {
}> => {
const { packageInfo } = packageInstallContext;
// remove package installation's references to index templates
const assetsToRemove = esReferences.filter(
@ -80,6 +80,13 @@ export const prepareToInstallTemplates = (
type === ElasticsearchAssetType.componentTemplate
);
const fieldAssetsMap: AssetsMap = new Map();
await packageInstallContext.archiveIterator.traverseEntries(async (entry) => {
if (entry.buffer) {
fieldAssetsMap.set(entry.path, entry.buffer);
}
}, isFields);
// build templates per data stream from yml files
const dataStreams = onlyForDataStreams || packageInfo.data_streams;
if (!dataStreams) return { assetsToAdd: [], assetsToRemove, install: () => Promise.resolve([]) };
@ -90,7 +97,12 @@ export const prepareToInstallTemplates = (
datastreamFeature.data_stream === getRegistryDataStreamAssetBaseName(dataStream)
);
return prepareTemplate({ packageInstallContext, dataStream, experimentalDataStreamFeature });
return prepareTemplate({
packageInstallContext,
fieldAssetsMap,
dataStream,
experimentalDataStreamFeature,
});
});
const assetsToAdd = getAllTemplateRefs(templates.map((template) => template.indexTemplate));
@ -131,14 +143,23 @@ const installPreBuiltTemplates = async (
) => {
const templatePaths = packageInstallContext.paths.filter((path) => isTemplate(path));
try {
const templateAssetsMap: AssetsMap = new Map();
await packageInstallContext.archiveIterator.traverseEntries(
async (entry) => {
if (!entry.buffer) {
return;
}
templateAssetsMap.set(entry.path, entry.buffer);
},
(path) => templatePaths.includes(path)
);
await pMap(
templatePaths,
async (path) => {
const { file } = getPathParts(path);
const templateName = file.substr(0, file.lastIndexOf('.'));
const content = JSON.parse(
getAssetFromAssetsMap(packageInstallContext.assetsMap, path).toString('utf8')
);
const content = JSON.parse(getAssetFromAssetsMap(templateAssetsMap, path).toString('utf8'));
const esClientParams = { name: templateName, body: content };
const esClientRequestOptions = { ignore: [404] };
@ -175,14 +196,23 @@ const installPreBuiltComponentTemplates = async (
) => {
const templatePaths = packageInstallContext.paths.filter((path) => isComponentTemplate(path));
try {
const templateAssetsMap: AssetsMap = new Map();
await packageInstallContext.archiveIterator.traverseEntries(
async (entry) => {
if (!entry.buffer) {
return;
}
templateAssetsMap.set(entry.path, entry.buffer);
},
(path) => templatePaths.includes(path)
);
await pMap(
templatePaths,
async (path) => {
const { file } = getPathParts(path);
const templateName = file.substr(0, file.lastIndexOf('.'));
const content = JSON.parse(
getAssetFromAssetsMap(packageInstallContext.assetsMap, path).toString('utf8')
);
const content = JSON.parse(getAssetFromAssetsMap(templateAssetsMap, path).toString('utf8'));
const esClientParams = {
name: templateName,
@ -573,15 +603,21 @@ function countFields(fields: Fields): number {
export function prepareTemplate({
packageInstallContext,
fieldAssetsMap,
dataStream,
experimentalDataStreamFeature,
}: {
packageInstallContext: PackageInstallContext;
fieldAssetsMap: AssetsMap;
dataStream: RegistryDataStream;
experimentalDataStreamFeature?: ExperimentalDataStreamFeature;
}): { componentTemplates: TemplateMap; indexTemplate: IndexTemplateEntry } {
const { name: packageName, version: packageVersion } = packageInstallContext.packageInfo;
const fields = loadDatastreamsFieldsFromYaml(packageInstallContext, dataStream.path);
const fields = loadDatastreamsFieldsFromYaml(
packageInstallContext,
fieldAssetsMap,
dataStream.path
);
const isIndexModeTimeSeries =
dataStream.elasticsearch?.index_mode === 'time_series' ||

47
x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/transform/install.ts
View file

@ -40,6 +40,7 @@ import type {
ESAssetMetadata,
IndexTemplate,
RegistryElasticsearch,
AssetsMap,
} from '../../../../../common/types/models';
import { getInstallation } from '../../packages';
import { retryTransientEsErrors } from '../retry';
@ -93,6 +94,17 @@ const installLegacyTransformsAssets = async (
let installedTransforms: EsAssetReference[] = [];
if (transformPaths.length > 0) {
const transformAssetsMap: AssetsMap = new Map();
await packageInstallContext.archiveIterator.traverseEntries(
async (entry) => {
if (!entry.buffer) {
return;
}
transformAssetsMap.set(entry.path, entry.buffer);
},
(path) => transformPaths.includes(path)
);
const transformRefs = transformPaths.reduce<EsAssetReference[]>((acc, path) => {
acc.push({
id: getLegacyTransformNameForInstallation(
@ -117,9 +129,7 @@ const installLegacyTransformsAssets = async (
);
const transforms: TransformInstallation[] = transformPaths.map((path: string) => {
const content = JSON.parse(
getAssetFromAssetsMap(packageInstallContext.assetsMap, path).toString('utf-8')
);
const content = JSON.parse(getAssetFromAssetsMap(transformAssetsMap, path).toString('utf-8'));
content._meta = getESAssetMetadata({ packageName: packageInstallContext.packageInfo.name });
return {
@ -153,7 +163,7 @@ const installLegacyTransformsAssets = async (
return { installedTransforms, esReferences };
};
const processTransformAssetsPerModule = (
const processTransformAssetsPerModule = async (
packageInstallContext: PackageInstallContext,
installNameSuffix: string,
transformPaths: string[],
@ -161,7 +171,7 @@ const processTransformAssetsPerModule = (
force?: boolean,
username?: string
) => {
const { assetsMap, packageInfo: installablePackage } = packageInstallContext;
const { packageInfo: installablePackage } = packageInstallContext;
const transformsSpecifications = new Map();
const destinationIndexTemplates: DestinationIndexTemplateInstallation[] = [];
const transforms: TransformInstallation[] = [];
@ -170,6 +180,17 @@ const processTransformAssetsPerModule = (
const transformsToRemoveWithDestIndex: EsAssetReference[] = [];
const indicesToAddRefs: EsAssetReference[] = [];
const transformAssetsMap: AssetsMap = new Map();
await packageInstallContext.archiveIterator.traverseEntries(
async (entry) => {
if (!entry.buffer) {
return;
}
transformAssetsMap.set(entry.path, entry.buffer);
},
(path) => transformPaths.includes(path)
);
transformPaths.forEach((path: string) => {
const { transformModuleId, fileName } = getTransformFolderAndFileNames(
installablePackage,
@ -182,7 +203,7 @@ const processTransformAssetsPerModule = (
}
const packageAssets = transformsSpecifications.get(transformModuleId);
const content = load(getAssetFromAssetsMap(assetsMap, path).toString('utf-8'));
const content = load(getAssetFromAssetsMap(transformAssetsMap, path).toString('utf-8'));
// Handling fields.yml and all other files within 'fields' folder
if (fileName === TRANSFORM_SPECS_TYPES.FIELDS || isFields(path)) {
@ -387,6 +408,12 @@ const processTransformAssetsPerModule = (
version: t.transformVersion,
}));
const fieldAssetsMap: AssetsMap = new Map();
await packageInstallContext.archiveIterator.traverseEntries(async (entry) => {
if (entry.buffer) {
fieldAssetsMap.set(entry.path, entry.buffer);
}
}, isFields);
// Load and generate mappings
for (const destinationIndexTemplate of destinationIndexTemplates) {
if (!destinationIndexTemplate.transformModuleId) {
@ -397,7 +424,11 @@ const processTransformAssetsPerModule = (
.get(destinationIndexTemplate.transformModuleId)
?.set(
'mappings',
loadMappingForTransform(packageInstallContext, destinationIndexTemplate.transformModuleId)
loadMappingForTransform(
packageInstallContext,
fieldAssetsMap,
destinationIndexTemplate.transformModuleId
)
);
}
@ -441,7 +472,7 @@ const installTransformsAssets = async (
transformsSpecifications,
transformsToRemove,
transformsToRemoveWithDestIndex,
} = processTransformAssetsPerModule(
} = await processTransformAssetsPerModule(
packageInstallContext,
installNameSuffix,
transformPaths,

33
x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/transform/legacy_transforms.test.ts
View file

@ -26,6 +26,7 @@ import { appContextService } from '../../../app_context';
import { getESAssetMetadata } from '../meta';
import { createArchiveIteratorFromMap } from '../../archive/archive_iterator';
import type { PackageInstallContext } from '../../../../../common/types';
import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../constants';
@ -167,6 +168,22 @@ describe('test transform install with legacy schema', () => {
Buffer.from('{"content": "data"}'),
],
]),
archiveIterator: createArchiveIteratorFromMap(
new Map([
[
'endpoint-0.16.0-dev.0/data_stream/policy/elasticsearch/ingest_pipeline/default.json',
Buffer.from('{"content": "data"}'),
],
[
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata/default.json',
Buffer.from('{"content": "data"}'),
],
[
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/default.json',
Buffer.from('{"content": "data"}'),
],
])
),
} as unknown as PackageInstallContext,
esClient,
savedObjectsClient,
@ -339,6 +356,14 @@ describe('test transform install with legacy schema', () => {
Buffer.from('{"content": "data"}'),
],
]),
archiveIterator: createArchiveIteratorFromMap(
new Map([
[
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/default.json',
Buffer.from('{"content": "data"}'),
],
])
),
} as unknown as PackageInstallContext,
esClient,
savedObjectsClient,
@ -566,6 +591,14 @@ describe('test transform install with legacy schema', () => {
Buffer.from('{"content": "data"}'),
],
]),
archiveIterator: createArchiveIteratorFromMap(
new Map([
[
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/default.json',
Buffer.from('{"content": "data"}'),
],
])
),
} as unknown as PackageInstallContext,
esClient,
savedObjectsClient,

51
x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/transform/mappings.test.ts
View file

@ -14,10 +14,10 @@ describe('loadMappingForTransform', () => {
const fields = loadMappingForTransform(
{
packageInfo: {} as any,
assetsMap: new Map(),
archiveIterator: createArchiveIteratorFromMap(new Map()),
paths: [],
},
new Map(),
'test'
);
@ -28,36 +28,37 @@ describe('loadMappingForTransform', () => {
const fields = loadMappingForTransform(
{
packageInfo: {} as any,
assetsMap: new Map([
[
'/package/ti_opencti/2.1.0/elasticsearch/transform/latest_ioc/fields/ecs.yml',
Buffer.from(
`
- description: Description of the threat feed in a UI friendly format.
name: threat.feed.description
type: keyword
- description: The name of the threat feed in UI friendly format.
name: threat.feed.name
type: keyword`
),
],
[
'/package/ti_opencti/2.1.0/elasticsearch/transform/latest_ioc/fields/ecs-extra.yml',
Buffer.from(
`
- description: The display name indicator in an UI friendly format
level: extended
name: threat.indicator.name
type: keyword`
),
],
]),
archiveIterator: createArchiveIteratorFromMap(new Map()),
paths: [
'/package/ti_opencti/2.1.0/elasticsearch/transform/latest_ioc/fields/ecs.yml',
'/package/ti_opencti/2.1.0/elasticsearch/transform/latest_ioc/fields/ecs-extra.yml',
],
},
new Map([
[
'/package/ti_opencti/2.1.0/elasticsearch/transform/latest_ioc/fields/ecs.yml',
Buffer.from(
`
- description: Description of the threat feed in a UI friendly format.
name: threat.feed.description
type: keyword
- description: The name of the threat feed in UI friendly format.
name: threat.feed.name
type: keyword`
),
],
[
'/package/ti_opencti/2.1.0/elasticsearch/transform/latest_ioc/fields/ecs-extra.yml',
Buffer.from(
`
- description: The display name indicator in an UI friendly format
level: extended
name: threat.indicator.name
type: keyword`
),
],
]),
'latest_ioc'
);

9
x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/transform/mappings.ts
View file

@ -4,15 +4,20 @@
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { type PackageInstallContext } from '../../../../../common/types/models';
import type { AssetsMap, PackageInstallContext } from '../../../../../common/types/models';
import { loadTransformFieldsFromYaml, processFields } from '../../fields/field';
import { generateMappings } from '../template/template';
export function loadMappingForTransform(
packageInstallContext: PackageInstallContext,
fieldAssetsMap: AssetsMap,
transformModuleId: string
) {
const fields = loadTransformFieldsFromYaml(packageInstallContext, transformModuleId);
const fields = loadTransformFieldsFromYaml(
packageInstallContext,
fieldAssetsMap,
transformModuleId
);
const validFields = processFields(fields);
return generateMappings(validFields);
}

173
x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/transform/transforms.test.ts
View file

@ -22,6 +22,7 @@ import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../constants';
import { getESAssetMetadata } from '../meta';
import { createArchiveIteratorFromMap } from '../../archive/archive_iterator';
import { createAppContextStartContractMock } from '../../../../mocks';
import type { PackageInstallContext } from '../../../../../common/types';
@ -268,28 +269,30 @@ _meta:
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/manifest.yml',
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml',
],
assetsMap: new Map([
[
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/fields/beats.yml',
sourceData.BEATS_FIELDS,
],
[
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/fields/agent.yml',
sourceData.AGENT_FIELDS,
],
[
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/fields/fields.yml',
sourceData.FIELDS,
],
[
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/manifest.yml',
sourceData.MANIFEST,
],
[
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml',
sourceData.TRANSFORM,
],
]),
archiveIterator: createArchiveIteratorFromMap(
new Map([
[
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/fields/beats.yml',
Buffer.from(sourceData.BEATS_FIELDS),
],
[
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/fields/agent.yml',
Buffer.from(sourceData.AGENT_FIELDS),
],
[
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/fields/fields.yml',
Buffer.from(sourceData.FIELDS),
],
[
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/manifest.yml',
Buffer.from(sourceData.MANIFEST),
],
[
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml',
Buffer.from(sourceData.TRANSFORM),
],
])
),
} as unknown as PackageInstallContext,
esClient,
savedObjectsClient,
@ -570,20 +573,22 @@ _meta:
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/manifest.yml',
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml',
],
assetsMap: new Map([
[
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/fields/fields.yml',
Buffer.from(sourceData.FIELDS),
],
[
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/manifest.yml',
Buffer.from(sourceData.MANIFEST),
],
[
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml',
Buffer.from(sourceData.TRANSFORM),
],
]),
archiveIterator: createArchiveIteratorFromMap(
new Map([
[
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/fields/fields.yml',
Buffer.from(sourceData.FIELDS),
],
[
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/manifest.yml',
Buffer.from(sourceData.MANIFEST),
],
[
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml',
Buffer.from(sourceData.TRANSFORM),
],
])
),
} as unknown as PackageInstallContext,
esClient,
savedObjectsClient,
@ -852,16 +857,18 @@ _meta:
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/fields/fields.yml',
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml',
],
assetsMap: new Map([
[
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/fields/fields.yml',
Buffer.from(sourceData.FIELDS),
],
[
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml',
Buffer.from(sourceData.TRANSFORM),
],
]),
archiveIterator: createArchiveIteratorFromMap(
new Map([
[
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/fields/fields.yml',
Buffer.from(sourceData.FIELDS),
],
[
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml',
Buffer.from(sourceData.TRANSFORM),
],
]) as any
),
} as unknown as PackageInstallContext,
esClient,
savedObjectsClient,
@ -1080,16 +1087,18 @@ _meta:
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/manifest.yml',
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml',
],
assetsMap: new Map([
[
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/manifest.yml',
sourceData.MANIFEST,
],
[
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml',
sourceData.TRANSFORM,
],
]),
archiveIterator: createArchiveIteratorFromMap(
new Map([
[
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/manifest.yml',
sourceData.MANIFEST,
],
[
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml',
sourceData.TRANSFORM,
],
]) as any
),
} as unknown as PackageInstallContext,
esClient,
savedObjectsClient,
@ -1184,16 +1193,18 @@ _meta:
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/manifest.yml',
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml',
],
assetsMap: new Map([
[
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/manifest.yml',
sourceData.MANIFEST,
],
[
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml',
sourceData.TRANSFORM,
],
]),
archiveIterator: createArchiveIteratorFromMap(
new Map([
[
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/manifest.yml',
sourceData.MANIFEST,
],
[
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml',
sourceData.TRANSFORM,
],
]) as any
),
} as unknown as PackageInstallContext,
esClient,
savedObjectsClient,
@ -1279,20 +1290,22 @@ _meta:
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/manifest.yml',
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml',
],
assetsMap: new Map([
[
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/fields/fields.yml',
sourceData.FIELDS,
],
[
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/manifest.yml',
sourceData.MANIFEST,
],
[
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml',
sourceData.TRANSFORM,
],
]),
archiveIterator: createArchiveIteratorFromMap(
new Map([
[
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/fields/fields.yml',
sourceData.FIELDS,
],
[
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/manifest.yml',
sourceData.MANIFEST,
],
[
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml',
sourceData.TRANSFORM,
],
]) as any
),
} as unknown as PackageInstallContext,
esClient,
savedObjectsClient,

8
x-pack/platform/plugins/shared/fleet/server/services/epm/fields/field.ts
View file

@ -7,7 +7,7 @@
import { load } from 'js-yaml';
import type { PackageInstallContext } from '../../../../common/types';
import type { AssetsMap, PackageInstallContext } from '../../../../common/types';
import { getAssetsDataFromAssetsMap } from '../packages/assets';
// This should become a copy of https://github.com/elastic/beats/blob/d9a4c9c240a9820fab15002592e5bb6db318543b/libbeat/mapping/field.go#L39
@ -310,12 +310,13 @@ function combineFilter(...filters: Array<(path: string) => boolean>) {
export const loadDatastreamsFieldsFromYaml = (
packageInstallContext: PackageInstallContext,
fieldAssetsMap: AssetsMap,
datasetName?: string
): Field[] => {
// Fetch all field definition files
const fieldDefinitionFiles = getAssetsDataFromAssetsMap(
packageInstallContext.packageInfo,
packageInstallContext.assetsMap,
fieldAssetsMap,
isFields,
datasetName
);
@ -334,12 +335,13 @@ export const loadDatastreamsFieldsFromYaml = (
export const loadTransformFieldsFromYaml = (
packageInstallContext: PackageInstallContext,
fieldAssetsMap: AssetsMap,
transformName: string
): Field[] => {
// Fetch all field definition files
const fieldDefinitionFiles = getAssetsDataFromAssetsMap(
packageInstallContext.packageInfo,
packageInstallContext.assetsMap,
fieldAssetsMap,
combineFilter(isFields, filterForTransformAssets(transformName))
);
return fieldDefinitionFiles.reduce<Field[]>((acc, file) => {

159
x-pack/platform/plugins/shared/fleet/server/services/epm/kibana/assets/install.ts
View file

@ -17,9 +17,9 @@ import type {
Logger,
} from '@kbn/core/server';
import { createListStream } from '@kbn/utils';
import { partition, chunk } from 'lodash';
import { partition, chunk, once } from 'lodash';
import { getAssetFromAssetsMap, getPathParts } from '../../archive';
import { getPathParts } from '../../archive';
import { KibanaAssetType, KibanaSavedObjectType } from '../../../../types';
import type { AssetReference, Installation, PackageSpecTags } from '../../../../types';
import type { KibanaAssetReference, PackageInstallContext } from '../../../../../common/types';
@ -28,7 +28,7 @@ import {
getIndexPatternSavedObjects,
makeManagedIndexPatternsGlobal,
} from '../index_pattern/install';
import { kibanaAssetsToAssetsRef, saveKibanaAssetsRefs } from '../../packages/install';
import { saveKibanaAssetsRefs } from '../../packages/install';
import { deleteKibanaSavedObjectsAssets } from '../../packages/remove';
import { FleetError, KibanaSOReferenceError } from '../../../../errors';
import { withPackageSpan } from '../../packages/utils';
@ -38,7 +38,7 @@ import { appContextService } from '../../..';
import { tagKibanaAssets } from './tag_assets';
import { getSpaceAwareSaveobjectsClients } from './saved_objects';
const MAX_ASSETS_TO_INSTALL_IN_PARALLEL = 1000;
const MAX_ASSETS_TO_INSTALL_IN_PARALLEL = 200;
type SavedObjectsImporterContract = Pick<ISavedObjectsImporter, 'import' | 'resolveImportErrors'>;
const formatImportErrorsForLog = (errors: SavedObjectsImportFailure[]) =>
@ -111,42 +111,51 @@ export async function installKibanaAssets(options: {
savedObjectsImporter: SavedObjectsImporterContract;
logger: Logger;
pkgName: string;
kibanaAssets: Record<KibanaAssetType, ArchiveAsset[]>;
kibanaAssetsArchiveIterator: ReturnType<typeof getKibanaAssetsArchiveIterator>;
}): Promise<SavedObjectsImportSuccess[]> {
const { kibanaAssets, savedObjectsClient, savedObjectsImporter, logger } = options;
const { kibanaAssetsArchiveIterator, savedObjectsClient, savedObjectsImporter, logger } = options;
const assetsToInstall = Object.entries(kibanaAssets).flatMap(([assetType, assets]) => {
if (!validKibanaAssetTypes.has(assetType as KibanaAssetType)) {
return [];
}
let assetsToInstall: ArchiveAsset[] = [];
let res: SavedObjectsImportSuccess[] = [];
if (!assets.length) {
return [];
}
const installManagedIndexPatternOnce = once(() =>
installManagedIndexPattern({
savedObjectsClient,
savedObjectsImporter,
})
);
const assetFilter = AssetFilters[assetType];
if (assetFilter) {
return assetFilter(assets);
}
async function flushAssetsToInstall() {
await installManagedIndexPatternOnce();
return assets;
});
if (!assetsToInstall.length) {
return [];
const installedAssets = await installKibanaSavedObjects({
logger,
savedObjectsImporter,
kibanaAssets: assetsToInstall,
assetsChunkSize: MAX_ASSETS_TO_INSTALL_IN_PARALLEL,
});
assetsToInstall = [];
res = [...res, ...installedAssets];
}
await installManagedIndexPattern({
savedObjectsClient,
savedObjectsImporter,
await kibanaAssetsArchiveIterator(async ({ assetType, asset }) => {
const assetFilter = AssetFilters[assetType];
if (assetFilter) {
assetsToInstall = [...assetsToInstall, ...assetFilter([asset])];
} else {
assetsToInstall.push(asset);
}
if (assetsToInstall.length >= MAX_ASSETS_TO_INSTALL_IN_PARALLEL) {
await flushAssetsToInstall();
}
});
return await installKibanaSavedObjects({
logger,
savedObjectsImporter,
kibanaAssets: assetsToInstall,
assetsChunkSize: MAX_ASSETS_TO_INSTALL_IN_PARALLEL,
});
if (assetsToInstall.length) {
await flushAssetsToInstall();
}
return res;
}
export async function installManagedIndexPattern({
@ -267,50 +276,41 @@ export async function installKibanaAssetsAndReferences({
const { savedObjectsImporter, savedObjectTagAssignmentService, savedObjectTagClient } =
getSpaceAwareSaveobjectsClients(spaceId);
// This is where the memory consumption is rising up in the first place
const kibanaAssets = getKibanaAssets(packageInstallContext);
const kibanaAssetsArchiveIterator = getKibanaAssetsArchiveIterator(packageInstallContext);
if (installedPkg) {
await deleteKibanaSavedObjectsAssets({ installedPkg, spaceId });
}
let installedKibanaAssetsRefs: KibanaAssetReference[] = [];
if (!installAsAdditionalSpace) {
// save new kibana refs before installing the assets
installedKibanaAssetsRefs = await saveKibanaAssetsRefs(
savedObjectsClient,
pkgName,
kibanaAssetsToAssetsRef(kibanaAssets)
);
}
const importedAssets = await installKibanaAssets({
savedObjectsClient,
logger,
savedObjectsImporter,
pkgName,
kibanaAssets,
kibanaAssetsArchiveIterator,
});
if (installAsAdditionalSpace) {
const assets = importedAssets.map(
({ id, type, destinationId }) =>
({
id: destinationId ?? id,
originId: id,
type,
} as KibanaAssetReference)
);
installedKibanaAssetsRefs = await saveKibanaAssetsRefs(
savedObjectsClient,
pkgName,
assets,
installedPkg && installedPkg.attributes.installed_kibana_space_id === spaceId
? false
: installAsAdditionalSpace
);
}
const assets = importedAssets.map(
({ id, type, destinationId }) =>
({
id: destinationId ?? id,
...(destinationId ? { originId: id } : {}),
type,
} as KibanaAssetReference)
);
installedKibanaAssetsRefs = await saveKibanaAssetsRefs(
savedObjectsClient,
pkgName,
assets,
installedPkg && installedPkg.attributes.installed_kibana_space_id === spaceId
? false
: installAsAdditionalSpace
);
await withPackageSpan('Create and assign package tags', () =>
tagKibanaAssets({
savedObjectTagAssignmentService,
savedObjectTagClient,
kibanaAssets,
pkgTitle,
pkgName,
spaceId,
@ -355,25 +355,32 @@ export const isKibanaAssetType = (path: string) => {
return parts.service === 'kibana' && (kibanaAssetTypes as string[]).includes(parts.type);
};
export function getKibanaAssets(
packageInstallContext: PackageInstallContext
): Record<KibanaAssetType, ArchiveAsset[]> {
const result = Object.fromEntries<ArchiveAsset[]>(
kibanaAssetTypes.map((type) => [type, []])
) as Record<KibanaAssetType, ArchiveAsset[]>;
function getKibanaAssetsArchiveIterator(packageInstallContext: PackageInstallContext) {
return (
onEntry: (entry: {
path: string;
asset: ArchiveAsset;
assetType: KibanaAssetType;
}) => Promise<void>
) => {
return packageInstallContext.archiveIterator.traverseEntries(async (entry) => {
if (!entry.buffer) {
return;
}
packageInstallContext.paths.filter(isKibanaAssetType).forEach((path) => {
const buffer = getAssetFromAssetsMap(packageInstallContext.assetsMap, path);
const asset = JSON.parse(buffer.toString('utf8'));
const asset = JSON.parse(entry.buffer.toString('utf8'));
const assetType = getPathParts(path).type as KibanaAssetType;
const soType = KibanaSavedObjectTypeMapping[assetType];
if (asset.type === soType) {
result[assetType].push(asset);
}
});
const assetType = getPathParts(entry.path).type as KibanaAssetType;
const soType = KibanaSavedObjectTypeMapping[assetType];
if (!validKibanaAssetTypes.has(assetType)) {
return;
}
return result;
if (asset.type === soType) {
await onEntry({ path: entry.path, assetType, asset });
}
}, isKibanaAssetType);
};
}
const isImportConflictError = (e: SavedObjectsImportFailure) => e?.error?.type === 'conflict';

226
x-pack/platform/plugins/shared/fleet/server/services/epm/kibana/assets/tag_assets.test.ts
View file

@ -42,16 +42,15 @@ describe('tagKibanaAssets', () => {
savedObjectTagClient.create.mockImplementation(({ name }: { name: string }) =>
Promise.resolve({ id: name.toLowerCase(), name })
);
const kibanaAssets = { dashboard: [{ id: 'dashboard1', type: 'dashboard' }] } as any;
const importedAssets = [{ id: 'dashboard1', type: 'dashboard' }] as any;
await tagKibanaAssets({
savedObjectTagAssignmentService,
savedObjectTagClient,
kibanaAssets,
pkgTitle: 'System',
pkgName: 'system',
spaceId: 'default',
importedAssets: [],
importedAssets,
});
expect(savedObjectTagClient.create).toHaveBeenCalledWith(
@ -72,7 +71,7 @@ describe('tagKibanaAssets', () => {
);
expect(savedObjectTagAssignmentService.updateTagAssignments).toHaveBeenCalledWith({
tags: ['fleet-managed-default', 'fleet-pkg-system-default'],
assign: kibanaAssets.dashboard,
assign: importedAssets,
unassign: [],
refresh: false,
});
@ -80,22 +79,21 @@ describe('tagKibanaAssets', () => {
it('should only assign Managed and System tags when tags already exist', async () => {
savedObjectTagClient.get.mockResolvedValue({ name: '', color: '', description: '' });
const kibanaAssets = { dashboard: [{ id: 'dashboard1', type: 'dashboard' }] } as any;
const importedAssets = [{ id: 'dashboard1', type: 'dashboard' }] as any;
await tagKibanaAssets({
savedObjectTagAssignmentService,
savedObjectTagClient,
kibanaAssets,
importedAssets,
pkgTitle: 'System',
pkgName: 'system',
spaceId: 'default',
importedAssets: [],
});
expect(savedObjectTagClient.create).not.toHaveBeenCalled();
expect(savedObjectTagAssignmentService.updateTagAssignments).toHaveBeenCalledWith({
tags: ['fleet-managed-default', 'fleet-pkg-system-default'],
assign: kibanaAssets.dashboard,
assign: importedAssets,
unassign: [],
refresh: false,
});
@ -103,16 +101,16 @@ describe('tagKibanaAssets', () => {
it('should use destinationId instead of original SO id if imported asset has it', async () => {
savedObjectTagClient.get.mockResolvedValue({ name: '', color: '', description: '' });
const kibanaAssets = { dashboard: [{ id: 'dashboard1', type: 'dashboard' }] } as any;
await tagKibanaAssets({
savedObjectTagAssignmentService,
savedObjectTagClient,
kibanaAssets,
pkgTitle: 'System',
pkgName: 'system',
spaceId: 'default',
importedAssets: [{ id: 'dashboard1', destinationId: 'destination1' } as any],
importedAssets: [
{ id: 'dashboard1', destinationId: 'destination1', type: 'dashboard' } as any,
],
});
expect(savedObjectTagClient.create).not.toHaveBeenCalled();
@ -129,33 +127,32 @@ describe('tagKibanaAssets', () => {
savedObjectTagClient.create.mockImplementation(({ name }: { name: string }) =>
Promise.resolve({ id: name.toLowerCase(), name })
);
const kibanaAssets = {
dashboard: [{ id: 'dashboard1', type: 'dashboard' }],
search: [{ id: 's1', type: 'search' }],
config: [{ id: 'c1', type: 'config' }],
visualization: [{ id: 'v1', type: 'visualization' }],
osquery_pack_asset: [{ id: 'osquery-pack-asset1', type: 'osquery-pack-asset' }],
osquery_saved_query: [{ id: 'osquery_saved_query1', type: 'osquery_saved_query' }],
} as any;
const importedAssets = [
{ id: 'dashboard1', type: 'dashboard' },
{ id: 's1', type: 'search' },
{ id: 'c1', type: 'config' },
{ id: 'v1', type: 'visualization' },
{ id: 'osquery-pack-asset1', type: 'osquery-pack-asset' },
{ id: 'osquery_saved_query1', type: 'osquery-saved-query' },
] as any;
await tagKibanaAssets({
savedObjectTagAssignmentService,
savedObjectTagClient,
kibanaAssets,
importedAssets,
pkgTitle: 'System',
pkgName: 'system',
spaceId: 'default',
importedAssets: [],
});
expect(savedObjectTagAssignmentService.updateTagAssignments).toHaveBeenCalledWith({
tags: ['fleet-managed-default', 'fleet-pkg-system-default'],
assign: [
...kibanaAssets.dashboard,
...kibanaAssets.search,
...kibanaAssets.visualization,
...kibanaAssets.osquery_pack_asset,
...kibanaAssets.osquery_saved_query,
{ id: 'dashboard1', type: 'dashboard' },
{ id: 's1', type: 'search' },
{ id: 'v1', type: 'visualization' },
{ id: 'osquery-pack-asset1', type: 'osquery-pack-asset' },
{ id: 'osquery_saved_query1', type: 'osquery-saved-query' },
],
unassign: [],
refresh: false,
@ -163,16 +160,15 @@ describe('tagKibanaAssets', () => {
});
it('should do nothing if no taggable assets', async () => {
const kibanaAssets = { config: [{ id: 'c1', type: 'config' }] } as any;
const importedAssets = [{ id: 'c1', type: 'config' }] as any;
await tagKibanaAssets({
savedObjectTagAssignmentService,
savedObjectTagClient,
kibanaAssets,
pkgTitle: 'System',
pkgName: 'system',
spaceId: 'default',
importedAssets: [],
importedAssets,
});
expect(savedObjectTagAssignmentService.updateTagAssignments).not.toHaveBeenCalled();
@ -188,16 +184,15 @@ describe('tagKibanaAssets', () => {
savedObjectTagClient.create.mockImplementation(({ name }: { name: string }) =>
Promise.resolve({ id: name.toLowerCase(), name })
);
const kibanaAssets = { dashboard: [{ id: 'dashboard1', type: 'dashboard' }] } as any;
const importedAssets = [{ id: 'dashboard1', type: 'dashboard' }] as any;
await tagKibanaAssets({
savedObjectTagAssignmentService,
savedObjectTagClient,
kibanaAssets,
importedAssets,
pkgTitle: 'System',
pkgName: 'system',
spaceId: 'default',
importedAssets: [],
});
expect(savedObjectTagClient.create).not.toHaveBeenCalledWith(
@ -219,7 +214,7 @@ describe('tagKibanaAssets', () => {
);
expect(savedObjectTagAssignmentService.updateTagAssignments).toHaveBeenCalledWith({
tags: ['managed', 'fleet-pkg-system-default'],
assign: kibanaAssets.dashboard,
assign: importedAssets,
unassign: [],
refresh: false,
});
@ -235,16 +230,15 @@ describe('tagKibanaAssets', () => {
savedObjectTagClient.create.mockImplementation(({ name }: { name: string }) =>
Promise.resolve({ id: name.toLowerCase(), name })
);
const kibanaAssets = { dashboard: [{ id: 'dashboard1', type: 'dashboard' }] } as any;
const importedAssets = [{ id: 'dashboard1', type: 'dashboard' }] as any;
await tagKibanaAssets({
savedObjectTagAssignmentService,
savedObjectTagClient,
kibanaAssets,
pkgTitle: 'System',
pkgName: 'system',
spaceId: 'default',
importedAssets: [],
importedAssets,
});
expect(savedObjectTagClient.create).toHaveBeenCalledWith(
@ -266,7 +260,7 @@ describe('tagKibanaAssets', () => {
);
expect(savedObjectTagAssignmentService.updateTagAssignments).toHaveBeenCalledWith({
tags: ['fleet-managed-default', 'system'],
assign: kibanaAssets.dashboard,
assign: importedAssets,
unassign: [],
refresh: false,
});
@ -283,22 +277,21 @@ describe('tagKibanaAssets', () => {
savedObjectTagClient.create.mockImplementation(({ name }: { name: string }) =>
Promise.resolve({ id: name.toLowerCase(), name })
);
const kibanaAssets = { dashboard: [{ id: 'dashboard1', type: 'dashboard' }] } as any;
const importedAssets = [{ id: 'dashboard1', type: 'dashboard' }] as any;
await tagKibanaAssets({
savedObjectTagAssignmentService,
savedObjectTagClient,
kibanaAssets,
importedAssets,
pkgTitle: 'System',
pkgName: 'system',
spaceId: 'default',
importedAssets: [],
});
expect(savedObjectTagClient.create).not.toHaveBeenCalled();
expect(savedObjectTagAssignmentService.updateTagAssignments).toHaveBeenCalledWith({
tags: ['managed', 'system'],
assign: kibanaAssets.dashboard,
assign: importedAssets,
unassign: [],
refresh: false,
});
@ -309,14 +302,12 @@ describe('tagKibanaAssets', () => {
savedObjectTagClient.create.mockImplementation(({ name }: { name: string }) =>
Promise.resolve({ id: name.toLowerCase(), name })
);
const kibanaAssets = {
dashboard: [
{ id: 'dashboard1', type: 'dashboard' },
{ id: 'dashboard2', type: 'dashboard' },
{ id: 'search_id1', type: 'search' },
{ id: 'search_id2', type: 'search' },
],
} as any;
const importedAssets = [
{ id: 'dashboard1', type: 'dashboard' },
{ id: 'dashboard2', type: 'dashboard' },
{ id: 'search_id1', type: 'search' },
{ id: 'search_id2', type: 'search' },
] as any;
const assetTags = [
{
text: 'Foo',
@ -326,11 +317,10 @@ describe('tagKibanaAssets', () => {
await tagKibanaAssets({
savedObjectTagAssignmentService,
savedObjectTagClient,
kibanaAssets,
pkgTitle: 'TestPackage',
pkgName: 'test-pkg',
spaceId: 'default',
importedAssets: [],
importedAssets,
assetTags,
});
expect(savedObjectTagClient.create).toHaveBeenCalledTimes(3);
@ -412,23 +402,20 @@ describe('tagKibanaAssets', () => {
savedObjectTagClient.create.mockImplementation(({ name }: { name: string }) =>
Promise.resolve({ id: name.toLowerCase(), name })
);
const kibanaAssets = {
dashboard: [
{ id: 'dashboard1', type: 'dashboard' },
{ id: 'dashboard2', type: 'dashboard' },
{ id: 'search_id1', type: 'search' },
{ id: 'search_id2', type: 'search' },
],
} as any;
const importedAssets = [
{ id: 'dashboard1', type: 'dashboard' },
{ id: 'dashboard2', type: 'dashboard' },
{ id: 'search_id1', type: 'search' },
{ id: 'search_id2', type: 'search' },
] as any;
const assetTags = [{ text: 'Bar', asset_ids: ['dashboard1', 'search_id1'] }];
await tagKibanaAssets({
savedObjectTagAssignmentService,
savedObjectTagClient,
kibanaAssets,
importedAssets,
pkgTitle: 'TestPackage',
pkgName: 'test-pkg',
spaceId: 'default',
importedAssets: [],
assetTags,
});
expect(savedObjectTagClient.create).toHaveBeenCalledTimes(3);
@ -510,13 +497,11 @@ describe('tagKibanaAssets', () => {
savedObjectTagClient.create.mockImplementation(({ name }: { name: string }) =>
Promise.resolve({ id: name.toLowerCase(), name })
);
const kibanaAssets = {
dashboard: [
{ id: 'dashboard1', type: 'dashboard' },
{ id: 'dashboard2', type: 'dashboard' },
{ id: 'search_id1', type: 'search' },
],
} as any;
const importedAssets = [
{ id: 'dashboard1', type: 'dashboard' },
{ id: 'dashboard2', type: 'dashboard' },
{ id: 'search_id1', type: 'search' },
] as any;
const assetTags = [
{
text: 'myCustomTag',
@ -527,11 +512,10 @@ describe('tagKibanaAssets', () => {
await tagKibanaAssets({
savedObjectTagAssignmentService,
savedObjectTagClient,
kibanaAssets,
importedAssets,
pkgTitle: 'TestPackage',
pkgName: 'test-pkg',
spaceId: 'default',
importedAssets: [],
assetTags,
});
expect(savedObjectTagClient.create).toHaveBeenCalledTimes(3);
@ -610,14 +594,12 @@ describe('tagKibanaAssets', () => {
savedObjectTagClient.create.mockImplementation(({ name }: { name: string }) =>
Promise.resolve({ id: name.toLowerCase(), name })
);
const kibanaAssets = {
dashboard: [
{ id: 'dashboard1', type: 'dashboard' },
{ id: 'dashboard2', type: 'dashboard' },
{ id: 'search_id1', type: 'search' },
{ id: 'search_id2', type: 'search' },
],
} as any;
const importedAssets = [
{ id: 'dashboard1', type: 'dashboard' },
{ id: 'dashboard2', type: 'dashboard' },
{ id: 'search_id1', type: 'search' },
{ id: 'search_id2', type: 'search' },
] as any;
const assetTags = [
{
text: 'Foo',
@ -633,11 +615,10 @@ describe('tagKibanaAssets', () => {
await tagKibanaAssets({
savedObjectTagAssignmentService,
savedObjectTagClient,
kibanaAssets,
pkgTitle: 'TestPackage',
pkgName: 'test-pkg',
spaceId: 'default',
importedAssets: [],
importedAssets,
assetTags,
});
expect(savedObjectTagClient.create).not.toHaveBeenCalled();
@ -657,14 +638,12 @@ describe('tagKibanaAssets', () => {
savedObjectTagClient.create.mockImplementation(({ name }: { name: string }) =>
Promise.resolve({ id: name.toLowerCase(), name })
);
const kibanaAssets = {
dashboard: [
{ id: 'dashboard1', type: 'dashboard' },
{ id: 'dashboard2', type: 'dashboard' },
{ id: 'search_id1', type: 'search' },
{ id: 'search_id2', type: 'search' },
],
} as any;
const importedAssets = [
{ id: 'dashboard1', type: 'dashboard' },
{ id: 'dashboard2', type: 'dashboard' },
{ id: 'search_id1', type: 'search' },
{ id: 'search_id2', type: 'search' },
] as any;
const assetTags = [
{
text: 'foo',
@ -674,11 +653,10 @@ describe('tagKibanaAssets', () => {
await tagKibanaAssets({
savedObjectTagAssignmentService,
savedObjectTagClient,
kibanaAssets,
importedAssets,
pkgTitle: 'TestPackage',
pkgName: 'test-pkg',
spaceId: 'default',
importedAssets: [],
assetTags,
});
expect(savedObjectTagClient.create).toHaveBeenCalledTimes(2);
@ -702,14 +680,12 @@ describe('tagKibanaAssets', () => {
savedObjectTagClient.create.mockImplementation(({ name }: { name: string }) =>
Promise.resolve({ id: name.toLowerCase(), name })
);
const kibanaAssets = {
dashboard: [
{ id: 'dashboard1', type: 'dashboard' },
{ id: 'dashboard2', type: 'dashboard' },
{ id: 'search_id1', type: 'search' },
{ id: 'search_id2', type: 'search' },
],
} as any;
const importedAssets = [
{ id: 'dashboard1', type: 'dashboard' },
{ id: 'dashboard2', type: 'dashboard' },
{ id: 'search_id1', type: 'search' },
{ id: 'search_id2', type: 'search' },
] as any;
const assetTags = [
{
text: 'Security Solution',
@ -719,11 +695,10 @@ describe('tagKibanaAssets', () => {
await tagKibanaAssets({
savedObjectTagAssignmentService,
savedObjectTagClient,
kibanaAssets,
importedAssets,
pkgTitle: 'TestPackage',
pkgName: 'test-pkg',
spaceId: 'default',
importedAssets: [],
assetTags,
});
expect(savedObjectTagClient.create).toHaveBeenCalledWith(
@ -753,14 +728,12 @@ describe('tagKibanaAssets', () => {
savedObjectTagClient.create.mockImplementation(({ name }: { name: string }) =>
Promise.resolve({ id: name.toLowerCase(), name })
);
const kibanaAssets = {
dashboard: [
{ id: 'dashboard1', type: 'dashboard' },
{ id: 'dashboard2', type: 'dashboard' },
{ id: 'search_id1', type: 'search' },
{ id: 'search_id2', type: 'search' },
],
} as any;
const importedAssets = [
{ id: 'dashboard1', type: 'dashboard' },
{ id: 'dashboard2', type: 'dashboard' },
{ id: 'search_id1', type: 'search' },
{ id: 'search_id2', type: 'search' },
] as any;
const assetTags = [
{
text: 'Security Solution',
@ -770,11 +743,10 @@ describe('tagKibanaAssets', () => {
await tagKibanaAssets({
savedObjectTagAssignmentService,
savedObjectTagClient,
kibanaAssets,
importedAssets,
pkgTitle: 'TestPackage',
pkgName: 'test-pkg',
spaceId: 'my-secondary-space',
importedAssets: [],
assetTags,
});
expect(savedObjectTagClient.create).toHaveBeenCalledWith(managedTagPayloadArg1, {
@ -815,23 +787,20 @@ describe('tagKibanaAssets', () => {
savedObjectTagClient.create.mockImplementation(({ name }: { name: string }) =>
Promise.resolve({ id: name.toLowerCase(), name })
);
const kibanaAssets = {
dashboard: [
{ id: 'dashboard1', type: 'dashboard' },
{ id: 'dashboard2', type: 'dashboard' },
{ id: 'search_id1', type: 'search' },
{ id: 'search_id2', type: 'search' },
],
} as any;
const importedAssets = [
{ id: 'dashboard1', type: 'dashboard' },
{ id: 'dashboard2', type: 'dashboard' },
{ id: 'search_id1', type: 'search' },
{ id: 'search_id2', type: 'search' },
] as any;
await tagKibanaAssets({
savedObjectTagAssignmentService,
savedObjectTagClient,
kibanaAssets,
importedAssets,
pkgTitle: 'TestPackage',
pkgName: 'test-pkg',
spaceId: 'default',
importedAssets: [],
assetTags: [],
});
expect(savedObjectTagClient.create).toHaveBeenCalledTimes(2);
@ -843,14 +812,12 @@ describe('tagKibanaAssets', () => {
savedObjectTagClient.create.mockImplementation(({ name }: { name: string }) =>
Promise.resolve({ id: name.toLowerCase(), name })
);
const kibanaAssets = {
dashboard: [
{ id: 'dashboard1', type: 'dashboard' },
{ id: 'dashboard2', type: 'dashboard' },
{ id: 'search_id1', type: 'search' },
{ id: 'search_id2', type: 'search' },
],
} as any;
const importedAssets = [
{ id: 'dashboard1', type: 'dashboard' },
{ id: 'dashboard2', type: 'dashboard' },
{ id: 'search_id1', type: 'search' },
{ id: 'search_id2', type: 'search' },
] as any;
const assetTags = [
{
text: 'Foo',
@ -861,11 +828,10 @@ describe('tagKibanaAssets', () => {
await tagKibanaAssets({
savedObjectTagAssignmentService,
savedObjectTagClient,
kibanaAssets,
importedAssets,
pkgTitle: 'TestPackage',
pkgName: 'test-pkg',
spaceId: 'default',
importedAssets: [],
assetTags,
});
expect(savedObjectTagClient.create).toHaveBeenCalledTimes(3);

43
x-pack/platform/plugins/shared/fleet/server/services/epm/kibana/assets/tag_assets.ts
View file

@ -6,7 +6,7 @@
*/
import { v5 as uuidv5 } from 'uuid';
import { uniqBy } from 'lodash';
import { omit, uniqBy } from 'lodash';
import pMap from 'p-map';
import type { SavedObjectsImportSuccess } from '@kbn/core-saved-objects-common';
import { taggableTypes } from '@kbn/saved-objects-tagging-plugin/common/constants';
@ -14,14 +14,11 @@ import type { IAssignmentService } from '@kbn/saved-objects-tagging-plugin/serve
import type { ITagsClient } from '@kbn/saved-objects-tagging-plugin/common/types';
import { MAX_CONCURRENT_PACKAGE_ASSETS } from '../../../../constants';
import type { KibanaAssetType } from '../../../../../common';
import type { PackageSpecTags } from '../../../../types';
import { appContextService } from '../../../app_context';
import type { ArchiveAsset } from './install';
import { KibanaSavedObjectTypeMapping } from './install';
interface ObjectReference {
type: string;
id: string;
@ -85,7 +82,6 @@ const getRandomColor = () => {
interface TagAssetsParams {
savedObjectTagAssignmentService: IAssignmentService;
savedObjectTagClient: ITagsClient;
kibanaAssets: Record<KibanaAssetType, ArchiveAsset[]>;
pkgTitle: string;
pkgName: string;
spaceId: string;
@ -93,14 +89,15 @@ interface TagAssetsParams {
assetTags?: PackageSpecTags[];
}
export async function tagKibanaAssets(opts: TagAssetsParams) {
const { savedObjectTagAssignmentService, kibanaAssets, importedAssets } = opts;
const getNewId = (asset: SavedObjectsImportSuccess) =>
asset?.destinationId ? asset.destinationId : asset.id;
const getNewId = (assetId: string) =>
importedAssets.find((imported) => imported.id === assetId)?.destinationId ?? assetId;
const taggableAssets = getTaggableAssets(kibanaAssets).map((asset) => ({
...asset,
id: getNewId(asset.id),
export async function tagKibanaAssets(opts: TagAssetsParams) {
const { savedObjectTagAssignmentService, importedAssets } = opts;
const taggableAssets = getTaggableAssets(importedAssets).map((asset) => ({
...omit(asset, 'destinationId'),
id: getNewId(asset),
}));
if (taggableAssets.length > 0) {
const [managedTagId, packageTagId] = await Promise.all([
@ -150,18 +147,8 @@ export async function tagKibanaAssets(opts: TagAssetsParams) {
}
}
function getTaggableAssets(kibanaAssets: TagAssetsParams['kibanaAssets']) {
return Object.entries(kibanaAssets).flatMap(([assetType, assets]) => {
if (!taggableTypes.includes(KibanaSavedObjectTypeMapping[assetType as KibanaAssetType])) {
return [];
}
if (!assets.length) {
return [];
}
return assets;
});
function getTaggableAssets(importedAssets: SavedObjectsImportSuccess[]) {
return importedAssets.filter((asset) => taggableTypes.includes(asset.type));
}
async function ensureManagedTag(
@ -219,7 +206,7 @@ async function ensurePackageTag(
// Ensure that asset tags coming from the kibana/tags.yml file are correctly parsed and created
async function getPackageSpecTags(
taggableAssets: ArchiveAsset[],
taggableAssets: SavedObjectsImportSuccess[],
opts: Pick<TagAssetsParams, 'spaceId' | 'savedObjectTagClient' | 'pkgName' | 'assetTags'>
): Promise<PackageSpecTagsAssets[]> {
const { spaceId, savedObjectTagClient, pkgName, assetTags } = opts;
@ -254,7 +241,7 @@ async function getPackageSpecTags(
// Get all the assets of types defined in tag.asset_types from taggable kibanaAssets
const getAssetTypesObjectReferences = (
assetTypes: string[] | undefined,
taggableAssets: ArchiveAsset[]
taggableAssets: SavedObjectsImportSuccess[]
): ObjectReference[] => {
if (!assetTypes || assetTypes.length === 0) return [];
@ -268,7 +255,7 @@ const getAssetTypesObjectReferences = (
// Get the references to ids defined in tag.asset_ids from taggable kibanaAssets
const getAssetIdsObjectReferences = (
assetIds: string[] | undefined,
taggableAssets: ArchiveAsset[]
taggableAssets: SavedObjectsImportSuccess[]
): ObjectReference[] => {
if (!assetIds || assetIds.length === 0) return [];

1
x-pack/platform/plugins/shared/fleet/server/services/epm/package_service.ts
View file

@ -414,7 +414,6 @@ class PackageClientImpl implements PackageClient {
const { installedTransforms } = await installTransforms({
packageInstallContext: {
assetsMap,
packageInfo,
paths,
archiveIterator,

11
x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install.ts
View file

@ -492,17 +492,16 @@ async function installPackageFromRegistry({
}
// get latest package version and requested version in parallel for performance
const [latestPackage, { paths, packageInfo, assetsMap, archiveIterator, verificationResult }] =
const [latestPackage, { paths, packageInfo, archiveIterator, verificationResult }] =
await Promise.all([
latestPkg ? Promise.resolve(latestPkg) : queryLatest(),
Registry.getPackage(pkgName, pkgVersion, {
ignoreUnverified: force && !neverIgnoreVerificationError,
useStreaming,
useStreaming: true,
}),
]);
const packageInstallContext: PackageInstallContext = {
packageInfo,
assetsMap,
paths,
archiveIterator,
};
@ -831,7 +830,7 @@ async function installPackageByUpload({
packageInfo,
});
const { paths, assetsMap, archiveIterator } = await unpackBufferToAssetsMap({
const { paths, archiveIterator } = await unpackBufferToAssetsMap({
archiveBuffer,
contentType,
useStreaming,
@ -839,7 +838,6 @@ async function installPackageByUpload({
const packageInstallContext: PackageInstallContext = {
packageInfo: { ...packageInfo, version: pkgVersion },
assetsMap,
paths,
archiveIterator,
};
@ -1036,7 +1034,6 @@ export async function installCustomPackage(
const archiveIterator = createArchiveIteratorFromMap(assetsMap);
const packageInstallContext: PackageInstallContext = {
assetsMap,
paths,
packageInfo,
archiveIterator,
@ -1374,7 +1371,6 @@ export async function installAssetsForInputPackagePolicy(opts: {
const archiveIterator = createArchiveIteratorFromMap(pkg.assetsMap);
packageInstallContext = {
assetsMap: pkg.assetsMap,
packageInfo: pkg.packageInfo,
paths: pkg.paths,
archiveIterator,
@ -1382,7 +1378,6 @@ export async function installAssetsForInputPackagePolicy(opts: {
} else {
const archiveIterator = createArchiveIteratorFromMap(installedPkgWithAssets.assetsMap);
packageInstallContext = {
assetsMap: installedPkgWithAssets.assetsMap,
packageInfo: installedPkgWithAssets.packageInfo,
paths: installedPkgWithAssets.paths,
archiveIterator,

2
x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_index_template_pipeline.ts
View file

@ -55,7 +55,7 @@ export async function installIndexTemplatesAndPipelines({
packageInstallContext,
onlyForDataStreams
);
const preparedIndexTemplates = prepareToInstallTemplates(
const preparedIndexTemplates = await prepareToInstallTemplates(
packageInstallContext,
esReferences,
experimentalDataStreamFeatures,

5
x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.test.ts
View file

@ -111,7 +111,6 @@ describe('_stateMachineInstallPackage', () => {
esClient,
logger: loggerMock.create(),
packageInstallContext: {
assetsMap: new Map(),
archiveIterator: createArchiveIteratorFromMap(new Map()),
paths: [],
packageInfo: {
@ -174,7 +173,6 @@ describe('_stateMachineInstallPackage', () => {
esClient,
logger: loggerMock.create(),
packageInstallContext: {
assetsMap: new Map(),
archiveIterator: createArchiveIteratorFromMap(new Map()),
paths: [],
packageInfo: {
@ -211,7 +209,6 @@ describe('_stateMachineInstallPackage', () => {
esClient,
logger: loggerMock.create(),
packageInstallContext: {
assetsMap: new Map(),
archiveIterator: createArchiveIteratorFromMap(new Map()),
paths: [],
packageInfo: {
@ -261,7 +258,6 @@ describe('_stateMachineInstallPackage', () => {
esClient,
logger: loggerMock.create(),
packageInstallContext: {
assetsMap: new Map(),
archiveIterator: createArchiveIteratorFromMap(new Map()),
paths: [],
packageInfo: {
@ -341,7 +337,6 @@ describe('_stateMachineInstallPackage', () => {
conditions: { kibana: { version: 'x.y.z' } },
owner: { github: 'elastic/fleet' },
} as any,
assetsMap: new Map(),
archiveIterator: createArchiveIteratorFromMap(new Map()),
paths: [],
},

4
x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_create_restart_installation.test.ts
View file

@ -85,7 +85,6 @@ describe('stepCreateRestartInstallation', () => {
esClient,
logger,
packageInstallContext: {
assetsMap: new Map(),
archiveIterator: createArchiveIteratorFromMap(new Map()),
paths: [],
packageInfo: {
@ -122,7 +121,6 @@ describe('stepCreateRestartInstallation', () => {
esClient,
logger,
packageInstallContext: {
assetsMap: new Map(),
archiveIterator: createArchiveIteratorFromMap(new Map()),
paths: [],
packageInfo: {
@ -167,7 +165,6 @@ describe('stepCreateRestartInstallation', () => {
esClient,
logger,
packageInstallContext: {
assetsMap: new Map(),
archiveIterator: createArchiveIteratorFromMap(new Map()),
paths: [],
packageInfo: {
@ -212,7 +209,6 @@ describe('stepCreateRestartInstallation', () => {
esClient,
logger,
packageInstallContext: {
assetsMap: new Map(),
archiveIterator: createArchiveIteratorFromMap(new Map()),
paths: [],
packageInfo: {

1
x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_install_ilm_policies.test.ts
View file

@ -241,7 +241,6 @@ describe('stepInstallILMPolicies', () => {
conditions: { kibana: { version: 'x.y.z' } },
owner: { github: 'elastic/fleet' },
} as any,
assetsMap: new Map(),
archiveIterator: createArchiveIteratorFromMap(new Map()),
paths: [],
},

4
x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_install_kibana_assets.test.ts
View file

@ -74,7 +74,6 @@ const packageInstallContext = {
owner: { github: 'elastic/fleet' },
} as any,
paths: ['some/path/1', 'some/path/2'],
assetsMap: new Map(),
archiveIterator: createArchiveIteratorFromMap(new Map()),
};
@ -99,7 +98,6 @@ describe('stepInstallKibanaAssets', () => {
esClient,
logger: loggerMock.create(),
packageInstallContext: {
assetsMap: new Map(),
archiveIterator: createArchiveIteratorFromMap(new Map()),
paths: [],
packageInfo: {
@ -139,7 +137,6 @@ describe('stepInstallKibanaAssets', () => {
esClient,
logger: loggerMock.create(),
packageInstallContext: {
assetsMap: new Map(),
archiveIterator: createArchiveIteratorFromMap(new Map()),
paths: [],
packageInfo: {
@ -186,7 +183,6 @@ describe('stepInstallKibanaAssetsWithStreaming', () => {
esClient,
logger: loggerMock.create(),
packageInstallContext: {
assetsMap,
archiveIterator,
paths: [],
packageInfo: {

69
x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.ts
View file

@ -6,7 +6,7 @@
*/
import { ASSETS_SAVED_OBJECT_TYPE } from '../../../../../constants';
import type { PackageAssetReference } from '../../../../../types';
import type { AssetsMap, KibanaAssetType, PackageAssetReference } from '../../../../../types';
import { removeArchiveEntries, saveArchiveEntriesFromAssetsMap } from '../../../archive/storage';
@ -14,43 +14,52 @@ import { withPackageSpan } from '../../utils';
import type { InstallContext } from '../_state_machine_package_install';
import { INSTALL_STATES } from '../../../../../../common/types';
import { isKibanaAssetType } from '../../../kibana/assets/install';
import { getPathParts } from '../../../archive';
export async function stepSaveArchiveEntries(context: InstallContext) {
const { packageInstallContext, savedObjectsClient, installSource, useStreaming } = context;
const { packageInfo, archiveIterator } = packageInstallContext;
let assetsMap = packageInstallContext?.assetsMap;
let paths = packageInstallContext?.paths;
// For stream based installations, we don't want to save any assets but
// manifest.yaml due to the large number of assets in the package.
if (useStreaming) {
assetsMap = new Map();
await archiveIterator.traverseEntries(async (entry) => {
// Skip only kibana assets type
if (!isKibanaAssetType(entry.path)) {
assetsMap.set(entry.path, entry.buffer);
}
});
paths = Array.from(assetsMap.keys());
let assetsToSaveMap: AssetsMap = new Map();
let packageAssetRefs: PackageAssetReference[] = [];
async function flushAssets() {
const paths = Array.from(assetsToSaveMap.keys());
const packageAssetResults = await withPackageSpan('Update archive entries', () =>
saveArchiveEntriesFromAssetsMap({
savedObjectsClient,
assetsMap: assetsToSaveMap,
paths,
packageInfo,
installSource,
})
);
packageAssetRefs = [
...packageAssetRefs,
...packageAssetResults.saved_objects.map((result) => ({
id: result.id,
type: ASSETS_SAVED_OBJECT_TYPE as typeof ASSETS_SAVED_OBJECT_TYPE,
})),
];
assetsToSaveMap = new Map();
}
const packageAssetResults = await withPackageSpan('Update archive entries', () =>
saveArchiveEntriesFromAssetsMap({
savedObjectsClient,
assetsMap,
paths,
packageInfo,
installSource,
})
);
const packageAssetRefs: PackageAssetReference[] = packageAssetResults.saved_objects.map(
(result) => ({
id: result.id,
type: ASSETS_SAVED_OBJECT_TYPE,
})
);
await archiveIterator.traverseEntries(async (entry) => {
const assetType = getPathParts(entry.path).type as KibanaAssetType;
if (assetType === 'security_rule' && useStreaming) {
// Skip security rules to avoid storing to many things
} else {
assetsToSaveMap.set(entry.path, entry.buffer);
}
if (assetsToSaveMap.size > 100) {
await flushAssets();
}
});
await flushAssets();
return { packageAssetRefs };
}

2
x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_save_system_object.test.ts
View file

@ -66,7 +66,6 @@ describe('updateLatestExecutedState', () => {
esClient,
logger,
packageInstallContext: {
assetsMap: new Map(),
archiveIterator: createArchiveIteratorFromMap(new Map()),
paths: [],
packageInfo: {
@ -133,7 +132,6 @@ describe('updateLatestExecutedState', () => {
esClient,
logger,
packageInstallContext: {
assetsMap: new Map(),
archiveIterator: createArchiveIteratorFromMap(new Map()),
paths: [],
packageInfo: {

4
x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/update_latest_executed_state.test.ts
View file

@ -62,7 +62,6 @@ describe('updateLatestExecutedState', () => {
esClient,
logger,
packageInstallContext: {
assetsMap: new Map(),
archiveIterator: createArchiveIteratorFromMap(new Map()),
paths: [],
packageInfo: {
@ -118,7 +117,6 @@ describe('updateLatestExecutedState', () => {
esClient,
logger,
packageInstallContext: {
assetsMap: new Map(),
archiveIterator: createArchiveIteratorFromMap(new Map()),
paths: [],
packageInfo: {
@ -156,7 +154,6 @@ describe('updateLatestExecutedState', () => {
esClient,
logger,
packageInstallContext: {
assetsMap: new Map(),
archiveIterator: createArchiveIteratorFromMap(new Map()),
paths: [],
packageInfo: {
@ -202,7 +199,6 @@ describe('updateLatestExecutedState', () => {
esClient,
logger,
packageInstallContext: {
assetsMap: new Map(),
archiveIterator: createArchiveIteratorFromMap(new Map()),
paths: [],
packageInfo: {

2
x-pack/platform/plugins/shared/fleet/server/services/package_policies/experimental_datastream_features.ts
View file

@ -75,11 +75,11 @@ export async function handleExperimentalDatastreamFeatureOptIn({
);
return prepareTemplate({
packageInstallContext: {
assetsMap,
archiveIterator: createArchiveIteratorFromMap(assetsMap),
packageInfo,
paths,
},
fieldAssetsMap: assetsMap,
dataStream,
experimentalDataStreamFeature,
});

80
x-pack/test/fleet_api_integration/apis/epm/__snapshots__/bulk_get_assets.snap generated
View file

@ -98,6 +98,12 @@ Array [
"id": "metrics-all_assets.test_metrics@custom",
"type": "component_template",
},
Object {
"appLink": "",
"attributes": Object {},
"id": "sample_csp_rule_template",
"type": "csp-rule-template",
},
Object {
"appLink": "/app/dashboards#/view/sample_dashboard",
"attributes": Object {
@ -116,6 +122,14 @@ Array [
"id": "sample_dashboard2",
"type": "dashboard",
},
Object {
"appLink": "/app/management/kibana/dataViews/dataView/test-*",
"attributes": Object {
"title": "test-*",
},
"id": "test-*",
"type": "index-pattern",
},
Object {
"appLink": "/app/lens#/edit/sample_lens",
"attributes": Object {
@ -125,32 +139,6 @@ Array [
"id": "sample_lens",
"type": "lens",
},
Object {
"appLink": "/app/visualize#/edit/sample_visualization",
"attributes": Object {
"description": "sample visualization update",
"title": "sample vis title",
},
"id": "sample_visualization",
"type": "visualization",
},
Object {
"appLink": "/app/discover#/view/sample_search",
"attributes": Object {
"description": "",
"title": "All logs [Logs Kafka] ECS",
},
"id": "sample_search",
"type": "search",
},
Object {
"appLink": "/app/management/kibana/dataViews/dataView/test-*",
"attributes": Object {
"title": "test-*",
},
"id": "test-*",
"type": "index-pattern",
},
Object {
"appLink": "/app/ml/supplied_configurations/?_a=(supplied_configurations%3A(queryText%3A'Nginx%20access%20logs'))",
"attributes": Object {
@ -160,20 +148,6 @@ Array [
"id": "sample_ml_module",
"type": "ml-module",
},
Object {
"appLink": "",
"attributes": Object {
"description": "Identifies a suspicious parent child process relationship with cmd.exe descending from svchost.exe",
},
"id": "sample_security_rule",
"type": "security-rule",
},
Object {
"appLink": "",
"attributes": Object {},
"id": "sample_csp_rule_template",
"type": "csp-rule-template",
},
Object {
"appLink": "",
"attributes": Object {},
@ -188,6 +162,23 @@ Array [
"id": "sample_osquery_saved_query",
"type": "osquery-saved-query",
},
Object {
"appLink": "/app/discover#/view/sample_search",
"attributes": Object {
"description": "",
"title": "All logs [Logs Kafka] ECS",
},
"id": "sample_search",
"type": "search",
},
Object {
"appLink": "",
"attributes": Object {
"description": "Identifies a suspicious parent child process relationship with cmd.exe descending from svchost.exe",
},
"id": "sample_security_rule",
"type": "security-rule",
},
Object {
"appLink": "",
"attributes": Object {
@ -196,5 +187,14 @@ Array [
"id": "sample_tag",
"type": "tag",
},
Object {
"appLink": "/app/visualize#/edit/sample_visualization",
"attributes": Object {
"description": "sample visualization update",
"title": "sample vis title",
},
"id": "sample_visualization",
"type": "visualization",
},
]
`;

93
x-pack/test/fleet_api_integration/apis/epm/update_assets.ts
View file

@ -8,6 +8,7 @@
import expect from '@kbn/expect';
import { FLEET_INSTALL_FORMAT_VERSION } from '@kbn/fleet-plugin/server/constants';
import { sortBy } from 'lodash';
import { FtrProviderContext } from '../../../api_integration/ftr_provider_context';
import { skipIfNoDockerRegistry } from '../../helpers';
@ -336,50 +337,56 @@ export default function (providerContext: FtrProviderContext) {
id: 'all_assets',
});
expect(res.attributes).eql({
expect({
...res.attributes,
installed_kibana: sortBy(res.attributes.installed_kibana, ['id']),
}).eql({
installed_kibana_space_id: 'default',
installed_kibana: [
{
id: 'sample_dashboard',
type: 'dashboard',
},
{
id: 'sample_lens',
type: 'lens',
},
{
id: 'sample_visualization',
type: 'visualization',
},
{
id: 'sample_search2',
type: 'search',
},
{
id: 'sample_ml_module',
type: 'ml-module',
},
{
id: 'sample_security_rule',
type: 'security-rule',
},
{
id: 'sample_csp_rule_template2',
type: 'csp-rule-template',
},
{
id: 'sample_osquery_pack_asset',
type: 'osquery-pack-asset',
},
{
id: 'sample_osquery_saved_query',
type: 'osquery-saved-query',
},
{
id: 'sample_tag',
type: 'tag',
},
],
installed_kibana: sortBy(
[
{
id: 'sample_dashboard',
type: 'dashboard',
},
{
id: 'sample_lens',
type: 'lens',
},
{
id: 'sample_visualization',
type: 'visualization',
},
{
id: 'sample_search2',
type: 'search',
},
{
id: 'sample_ml_module',
type: 'ml-module',
},
{
id: 'sample_security_rule',
type: 'security-rule',
},
{
id: 'sample_csp_rule_template2',
type: 'csp-rule-template',
},
{
id: 'sample_osquery_pack_asset',
type: 'osquery-pack-asset',
},
{
id: 'sample_osquery_saved_query',
type: 'osquery-saved-query',
},
{
id: 'sample_tag',
type: 'tag',
},
],
'id'
),
installed_es: [
{
id: 'all_assets',