github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 17:28:26 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Fleet] Fetch only relevant assets for package policies operation (#213267)

Browse source
This commit is contained in:
Nicolas Chaulet 2025-03-06 12:48:15 -05:00 • committed by GitHub
parent 34fa3135a2
commit 9b729b3a6c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
14 changed files with 457 additions and 64 deletions
Download patch file Download diff file Expand all files Collapse all files

3
x-pack/platform/plugins/shared/fleet/common/types/models/epm.ts
View file

@ -126,6 +126,8 @@ export type InstallablePackage = RegistryPackage | ArchivePackage;
export type AssetsMap = Map<string, Buffer | undefined>;
export type PackagePolicyAssetsMap = AssetsMap & { __brand: 'PackagePolicyAssetsMap' };
export interface ArchiveEntry {
path: string;
buffer?: Buffer;
@ -723,6 +725,7 @@ export interface EsAssetReference {
export interface PackageAssetReference {
id: string;
path?: string; // Package installed prior to 9.1.0 will not have that property
type: typeof ASSETS_SAVED_OBJECT_TYPE;
}

13
x-pack/platform/plugins/shared/fleet/server/routes/package_policy/handlers.ts
View file

@ -60,7 +60,7 @@ import type {
SimplifiedInputs,
SimplifiedPackagePolicy,
} from '../../../common/services/simplified_package_policy_helper';
import { runWithCache } from '../../services/epm/packages/cache';
import { validateAgentlessInputs } from '../../../common/services/agentless_policy_helper';
import {
@ -544,11 +544,12 @@ export const dryRunUpgradePackagePolicyHandler: RequestHandler<
const body: UpgradePackagePolicyDryRunResponse = [];
const { packagePolicyIds } = request.body;
for (const id of packagePolicyIds) {
const result = await packagePolicyService.getUpgradeDryRunDiff(soClient, id);
body.push(result);
}
await runWithCache(async () => {
for (const id of packagePolicyIds) {
const result = await packagePolicyService.getUpgradeDryRunDiff(soClient, id);
body.push(result);
}
});
const firstFatalError = body.find((item) => item.statusCode && item.statusCode !== 200);

6
x-pack/platform/plugins/shared/fleet/server/services/epm/archive/storage.ts
View file

@ -121,7 +121,11 @@ export async function saveArchiveEntriesFromAssetsMap(opts: {
})
);
const results = await savedObjectsClient.bulkCreate<PackageAsset>(bulkBody, { refresh: false });
const results = await savedObjectsClient.bulkCreate<PackageAsset>(bulkBody, {
refresh: false,
overwrite: true,
});
return results;
}

29
x-pack/platform/plugins/shared/fleet/server/services/epm/packages/cache.ts
View file

@ -9,7 +9,7 @@ import { AsyncLocalStorage } from 'async_hooks';
import LRUCache from 'lru-cache';
import type { AssetsMap } from '../../../../common/types';
import type { AssetsMap, PackagePolicyAssetsMap } from '../../../../common/types';
import type { PackageInfo } from '../../../../common';
@ -17,12 +17,15 @@ const cacheStore = new AsyncLocalStorage<CacheSession>();
const PACKAGE_INFO_CACHE_SIZE = 20;
const PACKAGE_ASSETS_MAP_CACHE_SIZE = 1;
const AGENT_TEMPLATE_ASSETS_MAP_CACHE_SIZE = 5;
class CacheSession {
private _packageInfoCache?: LRUCache<string, PackageInfo>;
private _packageAssetsMap?: LRUCache<string, AssetsMap>;
private _agentTemplateAssetsMap?: LRUCache<string, PackagePolicyAssetsMap>;
getPackageInfoCache() {
if (!this._packageInfoCache) {
this._packageInfoCache = new LRUCache<string, PackageInfo>({
@ -40,6 +43,15 @@ class CacheSession {
}
return this._packageAssetsMap;
}
getAgentTemplateAssetsMapCache() {
if (!this._agentTemplateAssetsMap) {
this._agentTemplateAssetsMap = new LRUCache<string, PackagePolicyAssetsMap>({
max: AGENT_TEMPLATE_ASSETS_MAP_CACHE_SIZE,
});
}
return this._agentTemplateAssetsMap;
}
}
export function getPackageInfoCache(pkgName: string, pkgVersion: string) {
@ -65,6 +77,21 @@ export function setPackageAssetsMapCache(
?.set(`${pkgName}:${pkgVersion}`, assetsMap);
}
export function getAgentTemplateAssetsMapCache(pkgName: string, pkgVersion: string) {
return cacheStore.getStore()?.getAgentTemplateAssetsMapCache()?.get(`${pkgName}:${pkgVersion}`);
}
export function setAgentTemplateAssetsMapCache(
pkgName: string,
pkgVersion: string,
assetsMap: PackagePolicyAssetsMap
) {
return cacheStore
.getStore()
?.getAgentTemplateAssetsMapCache()
?.set(`${pkgName}:${pkgVersion}`, assetsMap);
}
export async function runWithCache<T = any>(cb: () => Promise<T>): Promise<T> {
const cache = new CacheSession();

163
x-pack/platform/plugins/shared/fleet/server/services/epm/packages/get.test.ts
View file

@ -8,7 +8,11 @@
import type { SavedObjectsClientContract, SavedObjectsFindResult } from '@kbn/core/server';
import { SavedObjectsErrorHelpers } from '@kbn/core/server';
import { savedObjectsClientMock, elasticsearchServiceMock } from '@kbn/core/server/mocks';
import {
savedObjectsClientMock,
elasticsearchServiceMock,
loggingSystemMock,
} from '@kbn/core/server/mocks';
import {
ASSETS_SAVED_OBJECT_TYPE,
@ -24,17 +28,32 @@ import { PackageNotFoundError } from '../../../errors';
import { getSettings } from '../../settings';
import { auditLoggingService } from '../../audit_logging';
import * as Registry from '../registry';
import { getEsPackage } from '../archive/storage';
import { createArchiveIteratorFromMap } from '../archive/archive_iterator';
import { getInstalledPackages, getPackageInfo, getPackages, getPackageUsageStats } from './get';
import {
getAgentTemplateAssetsMap,
getInstalledPackages,
getPackageInfo,
getPackages,
getPackageUsageStats,
} from './get';
jest.mock('../registry');
jest.mock('../../settings');
jest.mock('../../audit_logging');
jest.mock('../../data_streams');
jest.mock('../archive/storage', () => {
return {
...jest.requireActual('../archive/storage'),
getEsPackage: jest
.fn()
.mockImplementation((...args) =>
jest.requireActual('../archive/storage').getEsPackage(...args)
),
};
});
const MockRegistry = jest.mocked(Registry);
@ -1150,4 +1169,140 @@ owner: elastic`,
});
});
});
describe('getAgentTemplateAssetsMap', () => {
const assetsMap = new Map([
['test-1.0.0/LICENSE.txt', Buffer.from('')],
['test-1.0.0/changelog.yml', Buffer.from('')],
['test-1.0.0/manifest.yml', Buffer.from('')],
['test-1.0.0/docs/README.md', Buffer.from('')],
['test-1.0.0/img/logo_nginx.svg', Buffer.from('')],
['test-1.0.0/img/nginx-logs-access-error.png', Buffer.from('')],
['test-1.0.0/img/nginx-logs-overview.png', Buffer.from('')],
['test-1.0.0/img/nginx-metrics-overview.png', Buffer.from('')],
['test-1.0.0/data_stream/access/manifest.yml', Buffer.from('')],
['test-1.0.0/data_stream/access/sample_event.json', Buffer.from('')],
['test-1.0.0/data_stream/error/manifest.yml', Buffer.from('')],
['test-1.0.0/data_stream/error/sample_event.json', Buffer.from('')],
['test-1.0.0/data_stream/stubstatus/manifest.yml', Buffer.from('')],
['test-1.0.0/data_stream/stubstatus/sample_event.json', Buffer.from('')],
[
'test-1.0.0/kibana/dashboard/nginx-023d2930-f1a5-11e7-a9ef-93c69af7b129.json',
Buffer.from(''),
],
[
'test-1.0.0/kibana/dashboard/nginx-046212a0-a2a1-11e7-928f-5dbe6f6f5519.json',
Buffer.from(''),
],
[
'test-1.0.0/kibana/dashboard/nginx-55a9e6e0-a29e-11e7-928f-5dbe6f6f5519.json',
Buffer.from(''),
],
['test-1.0.0/kibana/ml_module/nginx-Logs-ml.json', Buffer.from('')],
['test-1.0.0/data_stream/access/fields/agent.yml', Buffer.from('')],
['test-1.0.0/data_stream/access/fields/base-fields.yml', Buffer.from('')],
['test-1.0.0/data_stream/access/fields/fields.yml', Buffer.from('')],
['test-1.0.0/data_stream/error/fields/agent.yml', Buffer.from('')],
['test-1.0.0/data_stream/error/fields/base-fields.yml', Buffer.from('')],
['test-1.0.0/data_stream/error/fields/fields.yml', Buffer.from('')],
['test-1.0.0/data_stream/stubstatus/fields/agent.yml', Buffer.from('')],
['test-1.0.0/data_stream/stubstatus/fields/base-fields.yml', Buffer.from('')],
['test-1.0.0/data_stream/stubstatus/fields/ecs.yml', Buffer.from('')],
['test-1.0.0/data_stream/stubstatus/fields/fields.yml', Buffer.from('')],
['test-1.0.0/data_stream/access/agent/stream/httpjson.yml.hbs', Buffer.from('')],
['test-1.0.0/data_stream/access/agent/stream/stream.yml.hbs', Buffer.from('')],
['test-1.0.0/data_stream/access/elasticsearch/ingest_pipeline/default.yml', Buffer.from('')],
[
'test-1.0.0/data_stream/access/elasticsearch/ingest_pipeline/third-party.yml',
Buffer.from(''),
],
['test-1.0.0/data_stream/error/agent/stream/httpjson.yml.hbs', Buffer.from('')],
['test-1.0.0/data_stream/error/agent/stream/stream.yml.hbs', Buffer.from('')],
['test-1.0.0/data_stream/error/elasticsearch/ingest_pipeline/default.yml', Buffer.from('')],
[
'test-1.0.0/data_stream/error/elasticsearch/ingest_pipeline/third-party.yml',
Buffer.from(''),
],
['test-1.0.0/data_stream/stubstatus/agent/stream/stream.yml.hbs', Buffer.from('')],
]);
beforeEach(() => {
MockRegistry.getPackage.mockResolvedValue({
paths: [],
assetsMap,
archiveIterator: createArchiveIteratorFromMap(assetsMap),
packageInfo: {
name: 'test',
version: '1.0.0',
} as RegistryPackage,
});
});
it('should work with not installed package', async () => {
const savedObjectsClient = savedObjectsClientMock.create();
savedObjectsClient.get.mockRejectedValue(
SavedObjectsErrorHelpers.createGenericNotFoundError('not found')
);
const packagePolicyAssetsMap = await getAgentTemplateAssetsMap({
savedObjectsClient,
logger: loggingSystemMock.createLogger(),
packageInfo: {
name: 'test',
version: '1.0.0',
} as any,
});
expect([...packagePolicyAssetsMap.keys()]).toMatchInlineSnapshot(`
Array [
"test-1.0.0/manifest.yml",
"test-1.0.0/data_stream/access/manifest.yml",
"test-1.0.0/data_stream/error/manifest.yml",
"test-1.0.0/data_stream/stubstatus/manifest.yml",
"test-1.0.0/data_stream/access/agent/stream/httpjson.yml.hbs",
"test-1.0.0/data_stream/access/agent/stream/stream.yml.hbs",
"test-1.0.0/data_stream/error/agent/stream/httpjson.yml.hbs",
"test-1.0.0/data_stream/error/agent/stream/stream.yml.hbs",
"test-1.0.0/data_stream/stubstatus/agent/stream/stream.yml.hbs",
]
`);
});
it('should work installed package', async () => {
const savedObjectsClient = savedObjectsClientMock.create();
jest.mocked(getEsPackage).mockResolvedValueOnce({
assets_path: [...assetsMap.keys()].map((path) => ({ id: path, path, type: 'test' })),
} as any);
savedObjectsClient.get.mockResolvedValue({
attributes: [
{
assets_path: [...assetsMap.keys()].map((path) => ({ id: path, path, type: 'test' })),
},
],
} as any);
const packagePolicyAssetsMap = await getAgentTemplateAssetsMap({
savedObjectsClient,
logger: loggingSystemMock.createLogger(),
packageInfo: {
name: 'test',
version: '1.0.0',
} as any,
});
expect([...packagePolicyAssetsMap.keys()]).toMatchInlineSnapshot(`
Array [
"test-1.0.0/manifest.yml",
"test-1.0.0/data_stream/access/manifest.yml",
"test-1.0.0/data_stream/error/manifest.yml",
"test-1.0.0/data_stream/stubstatus/manifest.yml",
"test-1.0.0/data_stream/access/agent/stream/httpjson.yml.hbs",
"test-1.0.0/data_stream/access/agent/stream/stream.yml.hbs",
"test-1.0.0/data_stream/error/agent/stream/httpjson.yml.hbs",
"test-1.0.0/data_stream/error/agent/stream/stream.yml.hbs",
"test-1.0.0/data_stream/stubstatus/agent/stream/stream.yml.hbs",
]
`);
});
});
});

68
x-pack/platform/plugins/shared/fleet/server/services/epm/packages/get.ts
View file

@ -38,6 +38,7 @@ import type {
InstalledPackage,
PackageSpecManifest,
AssetsMap,
PackagePolicyAssetsMap,
} from '../../../../common/types';
import {
PACKAGES_SAVED_OBJECT_TYPE,
@ -67,7 +68,7 @@ import { getPackagePolicySavedObjectType } from '../../package_policy';
import { auditLoggingService } from '../../audit_logging';
import { getFilteredSearchPackages } from '../filtered_packages';
import { filterAssetPathForParseAndVerifyArchive } from '../archive/parse';
import { airGappedUtils } from '../airgapped';
import { createInstallableFrom } from '.';
@ -76,6 +77,8 @@ import {
setPackageAssetsMapCache,
getPackageInfoCache,
setPackageInfoCache,
getAgentTemplateAssetsMapCache,
setAgentTemplateAssetsMapCache,
} from './cache';
export { getFile } from '../registry';
@ -715,15 +718,23 @@ export async function getInstalledPackageWithAssets(options: {
pkgName: string;
logger?: Logger;
ignoreUnverified?: boolean;
assetsFilter?: (path: string) => boolean;
}) {
const installation = await getInstallation(options);
if (!installation) {
return;
}
const assetsReference =
(typeof options.assetsFilter !== 'undefined'
? installation.package_assets?.filter(({ path }) =>
typeof path !== 'undefined' ? options.assetsFilter!(path) : true
)
: installation.package_assets) ?? [];
const esPackage = await getEsPackage(
installation.name,
installation.version,
installation.package_assets ?? [],
assetsReference,
options.savedObjectsClient
);
@ -800,3 +811,56 @@ export async function getPackageAssetsMap({
throw error;
}
}
/**
* Return assets agent template assets map for package policies operation
*/
export async function getAgentTemplateAssetsMap({
savedObjectsClient,
packageInfo,
logger,
ignoreUnverified,
}: {
savedObjectsClient: SavedObjectsClientContract;
packageInfo: PackageInfo;
logger: Logger;
ignoreUnverified?: boolean;
}): Promise<PackagePolicyAssetsMap> {
const cache = getAgentTemplateAssetsMapCache(packageInfo.name, packageInfo.version);
if (cache) {
return cache;
}
const assetsFilter = (path: string) =>
filterAssetPathForParseAndVerifyArchive(path) || !!path.match(/\/agent\/.*\.hbs/);
const installedPackageWithAssets = await getInstalledPackageWithAssets({
savedObjectsClient,
pkgName: packageInfo.name,
logger,
assetsFilter,
});
try {
let assetsMap: PackagePolicyAssetsMap | undefined;
if (installedPackageWithAssets?.installation.version !== packageInfo.version) {
// Try to get from registry
const pkg = await Registry.getPackage(packageInfo.name, packageInfo.version, {
ignoreUnverified,
useStreaming: true,
});
assetsMap = new Map() as PackagePolicyAssetsMap;
await pkg.archiveIterator.traverseEntries(async (entry) => {
if (entry.buffer) {
assetsMap!.set(entry.path, entry.buffer);
}
}, assetsFilter);
} else {
assetsMap = installedPackageWithAssets.assetsMap as PackagePolicyAssetsMap;
}
setAgentTemplateAssetsMapCache(packageInfo.name, packageInfo.version, assetsMap);
return assetsMap as PackagePolicyAssetsMap;
} catch (error) {
logger.warn(`getAgentTemplateAssetsMap error: ${error}`);
throw error;
}
}

4
x-pack/platform/plugins/shared/fleet/server/services/epm/packages/get_template_inputs.ts
View file

@ -33,7 +33,7 @@ import { _sortYamlKeys } from '../../../../common/services/full_agent_policy_to_
import { getFullInputStreams } from '../../agent_policies/package_policies_to_agent_inputs';
import { getPackageInfo } from '.';
import { getPackageAssetsMap } from './get';
import { getAgentTemplateAssetsMap } from './get';
type Format = 'yml' | 'json';
@ -161,7 +161,7 @@ export async function getTemplateInputs(
}
}
const assetsMap = await getPackageAssetsMap({
const assetsMap = await getAgentTemplateAssetsMap({
logger: appContextService.getLogger(),
packageInfo,
savedObjectsClient: soClient,

14
x-pack/platform/plugins/shared/fleet/server/services/epm/packages/get_templates_inputs.test.ts
View file

@ -8,12 +8,12 @@
import { savedObjectsClientMock } from '@kbn/core-saved-objects-api-server-mocks';
import { createAppContextStartContractMock } from '../../../mocks';
import type { PackagePolicyInput } from '../../../../common/types';
import type { PackagePolicyAssetsMap, PackagePolicyInput } from '../../../../common/types';
import { appContextService } from '../..';
import { getTemplateInputs, templatePackagePolicyToFullInputStreams } from './get_template_inputs';
import REDIS_1_18_0_PACKAGE_INFO from './__fixtures__/redis_1_18_0_package_info.json';
import { getPackageAssetsMap, getPackageInfo } from './get';
import { getAgentTemplateAssetsMap, getPackageInfo } from './get';
import { REDIS_ASSETS_MAP } from './__fixtures__/redis_1_18_0_streams_template';
import { LOGS_2_3_0_ASSETS_MAP, LOGS_2_3_0_PACKAGE_INFO } from './__fixtures__/logs_2_3_0';
import { DOCKER_2_11_0_PACKAGE_INFO, DOCKER_2_11_0_ASSETS_MAP } from './__fixtures__/docker_2_11_0';
@ -324,19 +324,19 @@ describe('Fleet - templatePackagePolicyToFullInputStreams', () => {
describe('Fleet - getTemplateInputs', () => {
beforeEach(() => {
appContextService.start(createAppContextStartContractMock());
jest.mocked(getPackageAssetsMap).mockImplementation(async ({ packageInfo }) => {
jest.mocked(getAgentTemplateAssetsMap).mockImplementation(async ({ packageInfo }) => {
if (packageInfo.name === 'redis' && packageInfo.version === '1.18.0') {
return REDIS_ASSETS_MAP;
return REDIS_ASSETS_MAP as PackagePolicyAssetsMap;
}
if (packageInfo.name === 'log') {
return LOGS_2_3_0_ASSETS_MAP;
return LOGS_2_3_0_ASSETS_MAP as PackagePolicyAssetsMap;
}
if (packageInfo.name === 'docker') {
return DOCKER_2_11_0_ASSETS_MAP;
return DOCKER_2_11_0_ASSETS_MAP as PackagePolicyAssetsMap;
}
return new Map();
return new Map() as PackagePolicyAssetsMap;
});
jest.mocked(getPackageInfo).mockImplementation(async ({ pkgName, pkgVersion }) => {
const pkgInfo = packageInfoCache.get(`${pkgName}-${pkgVersion}`);

1
x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.test.ts
View file

@ -210,6 +210,7 @@ describe('stepSaveArchiveEntries', () => {
packageAssetRefs: [
{
id: 'test',
path: 'some/path',
type: 'epm-packages-assets',
},
],

1
x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.ts
View file

@ -40,6 +40,7 @@ export async function stepSaveArchiveEntries(context: InstallContext) {
...packageAssetRefs,
...packageAssetResults.saved_objects.map((result) => ({
id: result.id,
path: result.attributes?.asset_path,
type: ASSETS_SAVED_OBJECT_TYPE as typeof ASSETS_SAVED_OBJECT_TYPE,
})),
];

4
x-pack/platform/plugins/shared/fleet/server/services/package_policy.test.ts
View file

@ -50,6 +50,7 @@ import type {
NewPackagePolicyInput,
PackagePolicyPackage,
DeletePackagePoliciesResponse,
PackagePolicyAssetsMap,
} from '../../common/types';
import { packageToPackagePolicy } from '../../common/services';
@ -120,7 +121,7 @@ const ASSETS_MAP_FIXTURES = new Map([
{{/each}}
`),
],
]);
]) as PackagePolicyAssetsMap;
async function mockedGetInstallation(params: any) {
let pkg;
@ -188,6 +189,7 @@ jest.mock('./epm/registry', () => ({
jest.mock('./epm/packages/get', () => ({
getPackageAssetsMap: jest.fn().mockResolvedValue(new Map()),
getAgentTemplateAssetsMap: jest.fn().mockResolvedValue(new Map()),
}));
jest.mock('./agent_policy');

29
x-pack/platform/plugins/shared/fleet/server/services/package_policy.ts
View file

@ -70,8 +70,8 @@ import type {
ExperimentalDataStreamFeature,
DeletePackagePoliciesResponse,
PolicySecretReference,
AssetsMap,
AgentPolicy,
PackagePolicyAssetsMap,
} from '../../common/types';
import {
FleetError,
@ -148,7 +148,7 @@ import {
deleteSecretsIfNotReferenced as deleteSecrets,
isSecretStorageEnabled,
} from './secrets';
import { getPackageAssetsMap } from './epm/packages/get';
import { getAgentTemplateAssetsMap } from './epm/packages/get';
import { validateAgentPolicyOutputForIntegration } from './agent_policies/outputs_helpers';
import type { PackagePolicyClientFetchAllItemIdsOptions } from './package_policy_service';
import {
@ -174,12 +174,12 @@ async function getPkgInfoAssetsMap({
}) {
const packageInfosandAssetsMap = new Map<
string,
{ assetsMap: AssetsMap; pkgInfo: PackageInfo }
{ assetsMap: PackagePolicyAssetsMap; pkgInfo: PackageInfo }
>();
await pMap(
packageInfos,
async (pkgInfo) => {
const assetsMap = await getPackageAssetsMap({
const assetsMap = await getAgentTemplateAssetsMap({
logger,
packageInfo: pkgInfo,
savedObjectsClient,
@ -385,7 +385,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
inputs = enrichedPackagePolicy.inputs as PackagePolicyInput[];
}
const assetsMap = await getPackageAssetsMap({
const assetsMap = await getAgentTemplateAssetsMap({
logger,
packageInfo: pkgInfo,
savedObjectsClient: soClient,
@ -717,7 +717,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
`Package info and assets not found: ${packagePolicy.package.name}-${packagePolicy.package.version}`
);
}
const assetsMap = await getPackageAssetsMap({
const assetsMap = await getAgentTemplateAssetsMap({
logger: appContextService.getLogger(),
packageInfo: pkgInfo,
savedObjectsClient: soClient,
@ -1036,7 +1036,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
secretsToDelete = secretsRes.secretsToDelete;
inputs = restOfPackagePolicy.inputs as PackagePolicyInput[];
}
const assetsMap = await getPackageAssetsMap({
const assetsMap = await getAgentTemplateAssetsMap({
logger,
packageInfo: pkgInfo,
savedObjectsClient: soClient,
@ -1770,7 +1770,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
packageInfo,
packageToPackagePolicyInputs(packageInfo) as InputsOverride[]
);
const assetsMap = await getPackageAssetsMap({
const assetsMap = await getAgentTemplateAssetsMap({
logger: appContextService.getLogger(),
packageInfo,
savedObjectsClient: soClient,
@ -1809,7 +1809,8 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
({ packagePolicy, packageInfo, experimentalDataStreamFeatures } =
await this.getUpgradePackagePolicyInfo(soClient, id, packagePolicy, pkgVersion));
const assetsMap = await getPackageAssetsMap({
const assetsMap = await getAgentTemplateAssetsMap({
logger: appContextService.getLogger(),
packageInfo,
savedObjectsClient: soClient,
@ -1834,7 +1835,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
soClient: SavedObjectsClientContract,
packagePolicy: PackagePolicy,
packageInfo: PackageInfo,
assetsMap: AssetsMap
assetsMap: PackagePolicyAssetsMap
): Promise<UpgradePackagePolicyDryRunResponseItem> {
const updatedPackagePolicy = updatePackageInputs(
{
@ -2507,7 +2508,7 @@ export async function _compilePackagePolicyInputs(
pkgInfo: PackageInfo,
vars: PackagePolicy['vars'],
inputs: PackagePolicyInput[],
assetsMap: AssetsMap
assetsMap: PackagePolicyAssetsMap
): Promise<PackagePolicyInput[]> {
const inputsPromises = inputs.map(async (input) => {
const compiledInput = await _compilePackagePolicyInput(pkgInfo, vars, input, assetsMap);
@ -2526,7 +2527,7 @@ async function _compilePackagePolicyInput(
pkgInfo: PackageInfo,
vars: PackagePolicy['vars'],
input: PackagePolicyInput,
assetsMap: AssetsMap
assetsMap: PackagePolicyAssetsMap
) {
const packagePolicyTemplate = input.policy_template
? pkgInfo.policy_templates?.find(
@ -2575,7 +2576,7 @@ async function _compilePackageStreams(
pkgInfo: PackageInfo,
vars: PackagePolicy['vars'],
input: PackagePolicyInput,
assetsMap: AssetsMap
assetsMap: PackagePolicyAssetsMap
) {
const streamsPromises = input.streams.map((stream) =>
_compilePackageStream(pkgInfo, vars, input, stream, assetsMap)
@ -2646,7 +2647,7 @@ async function _compilePackageStream(
vars: PackagePolicy['vars'],
input: PackagePolicyInput,
streamIn: PackagePolicyInputStream,
assetsMap: AssetsMap
assetsMap: PackagePolicyAssetsMap
) {
let stream = streamIn;

29
x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts
View file

@ -542,6 +542,7 @@ const expectAssetsInstalled = ({
installed_es: sortBy(res.attributes.installed_es, (o: AssetReference) => o.type),
package_assets: sortBy(res.attributes.package_assets, (o: AssetReference) => o.type),
};
expect(sortedRes).eql({
installed_kibana: [
{
@ -663,114 +664,142 @@ const expectAssetsInstalled = ({
package_assets: [
{
id: '333a22a1-e639-5af5-ae62-907ffc83d603',
path: 'all_assets-0.1.0/data_stream/test_logs/elasticsearch/ilm_policy/all_assets.json',
type: 'epm-packages-assets',
},
{
id: '256f3dad-6870-56c3-80a1-8dfa11e2d568',
path: 'all_assets-0.1.0/data_stream/test_logs/elasticsearch/ingest_pipeline/default.yml',
type: 'epm-packages-assets',
},
{
id: '3fa0512f-bc01-5c2e-9df1-bc2f2a8259c8',
path: 'all_assets-0.1.0/data_stream/test_logs/elasticsearch/ingest_pipeline/pipeline1.yml',
type: 'epm-packages-assets',
},
{
id: 'ea334ad8-80c2-5acd-934b-2a377290bf97',
path: 'all_assets-0.1.0/data_stream/test_logs/elasticsearch/ingest_pipeline/pipeline2.yml',
type: 'epm-packages-assets',
},
{
id: '96c6eb85-fe2e-56c6-84be-5fda976796db',
path: 'all_assets-0.1.0/data_stream/test_logs/fields/ecs.yml',
type: 'epm-packages-assets',
},
{
id: '2d73a161-fa69-52d0-aa09-1bdc691b95bb',
path: 'all_assets-0.1.0/data_stream/test_logs/fields/fields.yml',
type: 'epm-packages-assets',
},
{
id: '0a00c2d2-ce63-5b9c-9aa0-0cf1938f7362',
path: 'all_assets-0.1.0/data_stream/test_logs/manifest.yml',
type: 'epm-packages-assets',
},
{
id: '691f0505-18c5-57a6-9f40-06e8affbdf7a',
path: 'all_assets-0.1.0/data_stream/test_metrics/elasticsearch/ilm_policy/all_assets.json',
type: 'epm-packages-assets',
},
{
id: 'b36e6dd0-58f7-5dd0-a286-8187e4019274',
path: 'all_assets-0.1.0/data_stream/test_metrics/fields/ecs.yml',
type: 'epm-packages-assets',
},
{
id: 'f839c76e-d194-555a-90a1-3265a45789e4',
path: 'all_assets-0.1.0/data_stream/test_metrics/fields/fields.yml',
type: 'epm-packages-assets',
},
{
id: '9af7bbb3-7d8a-50fa-acc9-9dde6f5efca2',
path: 'all_assets-0.1.0/data_stream/test_metrics/manifest.yml',
type: 'epm-packages-assets',
},
{
id: '1e97a20f-9d1c-529b-8ff2-da4e8ba8bb71',
path: 'all_assets-0.1.0/docs/README.md',
type: 'epm-packages-assets',
},
{
id: 'ed5d54d5-2516-5d49-9e61-9508b0152d2b',
path: 'all_assets-0.1.0/elasticsearch/ml_model/test/default.json',
type: 'epm-packages-assets',
},
{
id: 'bd5ff3c5-655e-5385-9918-b60ff3040aad',
path: 'all_assets-0.1.0/img/logo_overrides_64_color.svg',
type: 'epm-packages-assets',
},
{
id: '943d5767-41f5-57c3-ba02-48e0f6a837db',
path: 'all_assets-0.1.0/kibana/csp_rule_template/sample_csp_rule_template.json',
type: 'epm-packages-assets',
},
{
id: '0954ce3b-3165-5c1f-a4c0-56eb5f2fa487',
path: 'all_assets-0.1.0/kibana/dashboard/sample_dashboard.json',
type: 'epm-packages-assets',
},
{
id: '60d6d054-57e4-590f-a580-52bf3f5e7cca',
path: 'all_assets-0.1.0/kibana/dashboard/sample_dashboard2.json',
type: 'epm-packages-assets',
},
{
id: '47758dc2-979d-5fbe-a2bd-9eded68a5a43',
path: 'all_assets-0.1.0/kibana/index_pattern/invalid.json',
type: 'epm-packages-assets',
},
{
id: '318959c9-997b-5a14-b328-9fc7355b4b74',
path: 'all_assets-0.1.0/kibana/index_pattern/test_index_pattern.json',
type: 'epm-packages-assets',
},
{
id: 'e21b59b5-eb76-5ab0-bef2-1c8e379e6197',
path: 'all_assets-0.1.0/kibana/lens/sample_lens.json',
type: 'epm-packages-assets',
},
{
id: '4c758d70-ecf1-56b3-b704-6d8374841b34',
path: 'all_assets-0.1.0/kibana/ml_module/sample_ml_module.json',
type: 'epm-packages-assets',
},
{
id: '313ddb31-e70a-59e8-8287-310d4652a9b7',
path: 'all_assets-0.1.0/kibana/osquery_pack_asset/sample_osquery_pack_asset.json',
type: 'epm-packages-assets',
},
{
id: '24a74223-5fdb-52ca-9cb5-b2cdd2a42b07',
path: 'all_assets-0.1.0/kibana/osquery_saved_query/sample_osquery_saved_query.json',
type: 'epm-packages-assets',
},
{
id: 'e786cbd9-0f3b-5a0b-82a6-db25145ebf58',
path: 'all_assets-0.1.0/kibana/search/sample_search.json',
type: 'epm-packages-assets',
},
{
id: 'd8b175c3-0d42-5ec7-90c1-d1e4b307a4c2',
path: 'all_assets-0.1.0/kibana/security_rule/sample_security_rule.json',
type: 'epm-packages-assets',
},
{
id: 'b265a5e0-c00b-5eda-ac44-2ddbd36d9ad0',
path: 'all_assets-0.1.0/kibana/tag/sample_tag.json',
type: 'epm-packages-assets',
},
{
id: '53c94591-aa33-591d-8200-cd524c2a0561',
path: 'all_assets-0.1.0/kibana/visualization/sample_visualization.json',
type: 'epm-packages-assets',
},
{
id: 'b658d2d4-752e-54b8-afc2-4c76155c1466',
path: 'all_assets-0.1.0/manifest.yml',
type: 'epm-packages-assets',
},
],

157
x-pack/test/fleet_api_integration/apis/epm/update_assets.ts
View file

@ -340,6 +340,7 @@ export default function (providerContext: FtrProviderContext) {
expect({
...res.attributes,
installed_kibana: sortBy(res.attributes.installed_kibana, ['id']),
package_assets: sortBy(res.attributes.package_assets, ['id']),
}).eql({
installed_kibana_space_id: 'default',
installed_kibana: sortBy(
@ -467,32 +468,136 @@ export default function (providerContext: FtrProviderContext) {
test_metrics: 'metrics-all_assets.test_metrics-*',
},
package_assets: [
{ id: '3eb4c54a-638f-51b6-84e2-d53f5a666e37', type: 'epm-packages-assets' },
{ id: '4acfbf69-7a27-5c58-9c99-7c86843d958f', type: 'epm-packages-assets' },
{ id: '938655df-b339-523c-a9e4-123c89c0e1e1', type: 'epm-packages-assets' },
{ id: 'eec4606c-dbfa-565b-8e9c-fce1e641f3fc', type: 'epm-packages-assets' },
{ id: 'ef67e7e0-dca3-5a62-a42a-745db5ad7c1f', type: 'epm-packages-assets' },
{ id: '64239d25-be40-5e10-94b5-f6b74b8c5474', type: 'epm-packages-assets' },
{ id: '071b5113-4c9f-5ee9-aafe-d098a4c066f6', type: 'epm-packages-assets' },
{ id: '498d8215-2613-5399-9a13-fa4f0bf513e2', type: 'epm-packages-assets' },
{ id: 'd2f87071-c866-503a-8fcb-7b23a8c7afbf', type: 'epm-packages-assets' },
{ id: '5a080eba-f482-545c-8695-6ccbd426b2a2', type: 'epm-packages-assets' },
{ id: '28523a82-1328-578d-84cb-800970560200', type: 'epm-packages-assets' },
{ id: 'cc1e3e1d-f27b-5d05-86f6-6e4b9a47c7dc', type: 'epm-packages-assets' },
{ id: '5c3aa147-089c-5084-beca-53c00e72ac80', type: 'epm-packages-assets' },
{ id: '0c8c3c6a-90cb-5f0e-8359-d807785b046c', type: 'epm-packages-assets' },
{ id: '48e582df-b1d2-5f88-b6ea-ba1fafd3a569', type: 'epm-packages-assets' },
{ id: '7f97600c-d983-53e0-ae2a-a59bf35d7f0d', type: 'epm-packages-assets' },
{ id: 'bf3b0b65-9fdc-53c6-a9ca-e76140e56490', type: 'epm-packages-assets' },
{ id: '7f4c5aca-b4f5-5f0a-95af-051da37513fc', type: 'epm-packages-assets' },
{ id: '4281a436-45a8-54ab-9724-fda6849f789d', type: 'epm-packages-assets' },
{ id: 'cb0bbdd7-e043-508b-91c0-09e4cc0f5a3c', type: 'epm-packages-assets' },
{ id: '6a87d1a5-adf8-5a30-82c4-4c3b8298272b', type: 'epm-packages-assets' },
{ id: '2e56f08b-1d06-55ed-abee-4708e1ccf0aa', type: 'epm-packages-assets' },
{ id: '4035007b-9c33-5227-9803-2de8a17523b5', type: 'epm-packages-assets' },
{ id: 'e6ae7d31-6920-5408-9219-91ef1662044b', type: 'epm-packages-assets' },
{ id: 'c7bf1a39-e057-58a0-afde-fb4b48751d8c', type: 'epm-packages-assets' },
{ id: '8c665f28-a439-5f43-b5fd-8fda7b576735', type: 'epm-packages-assets' },
{
id: '071b5113-4c9f-5ee9-aafe-d098a4c066f6',
path: 'all_assets-0.2.0/data_stream/test_logs2/fields/ecs.yml',
type: 'epm-packages-assets',
},
{
id: '0c8c3c6a-90cb-5f0e-8359-d807785b046c',
path: 'all_assets-0.2.0/elasticsearch/ml_model/test/default.json',
type: 'epm-packages-assets',
},
{
id: '28523a82-1328-578d-84cb-800970560200',
path: 'all_assets-0.2.0/data_stream/test_metrics/fields/fields.yml',
type: 'epm-packages-assets',
},
{
id: '2e56f08b-1d06-55ed-abee-4708e1ccf0aa',
path: 'all_assets-0.2.0/kibana/search/sample_search2.json',
type: 'epm-packages-assets',
},
{
id: '3eb4c54a-638f-51b6-84e2-d53f5a666e37',
path: 'all_assets-0.2.0/data_stream/test_logs/elasticsearch/ilm_policy/all_assets.json',
type: 'epm-packages-assets',
},
{
id: '4035007b-9c33-5227-9803-2de8a17523b5',
path: 'all_assets-0.2.0/kibana/security_rule/sample_security_rule.json',
type: 'epm-packages-assets',
},
{
id: '4281a436-45a8-54ab-9724-fda6849f789d',
path: 'all_assets-0.2.0/kibana/ml_module/sample_ml_module.json',
type: 'epm-packages-assets',
},
{
id: '48e582df-b1d2-5f88-b6ea-ba1fafd3a569',
path: 'all_assets-0.2.0/img/logo_overrides_64_color.svg',
type: 'epm-packages-assets',
},
{
id: '498d8215-2613-5399-9a13-fa4f0bf513e2',
path: 'all_assets-0.2.0/data_stream/test_logs2/fields/fields.yml',
type: 'epm-packages-assets',
},
{
id: '4acfbf69-7a27-5c58-9c99-7c86843d958f',
path: 'all_assets-0.2.0/data_stream/test_logs/elasticsearch/ingest_pipeline/default.yml',
type: 'epm-packages-assets',
},
{
id: '5a080eba-f482-545c-8695-6ccbd426b2a2',
path: 'all_assets-0.2.0/data_stream/test_metrics/fields/ecs.yml',
type: 'epm-packages-assets',
},
{
id: '5c3aa147-089c-5084-beca-53c00e72ac80',
path: 'all_assets-0.2.0/docs/README.md',
type: 'epm-packages-assets',
},
{
id: '64239d25-be40-5e10-94b5-f6b74b8c5474',
path: 'all_assets-0.2.0/data_stream/test_logs/manifest.yml',
type: 'epm-packages-assets',
},
{
id: '6a87d1a5-adf8-5a30-82c4-4c3b8298272b',
path: 'all_assets-0.2.0/kibana/osquery_saved_query/sample_osquery_saved_query.json',
type: 'epm-packages-assets',
},
{
id: '7f4c5aca-b4f5-5f0a-95af-051da37513fc',
path: 'all_assets-0.2.0/kibana/lens/sample_lens.json',
type: 'epm-packages-assets',
},
{
id: '7f97600c-d983-53e0-ae2a-a59bf35d7f0d',
path: 'all_assets-0.2.0/kibana/csp_rule_template/sample_csp_rule_template.json',
type: 'epm-packages-assets',
},
{
id: '8c665f28-a439-5f43-b5fd-8fda7b576735',
path: 'all_assets-0.2.0/manifest.yml',
type: 'epm-packages-assets',
},
{
id: '938655df-b339-523c-a9e4-123c89c0e1e1',
path: 'all_assets-0.2.0/data_stream/test_logs/elasticsearch/ingest_pipeline/pipeline1.yml',
type: 'epm-packages-assets',
},
{
id: 'bf3b0b65-9fdc-53c6-a9ca-e76140e56490',
path: 'all_assets-0.2.0/kibana/dashboard/sample_dashboard.json',
type: 'epm-packages-assets',
},
{
id: 'c7bf1a39-e057-58a0-afde-fb4b48751d8c',
path: 'all_assets-0.2.0/kibana/visualization/sample_visualization.json',
type: 'epm-packages-assets',
},
{
id: 'cb0bbdd7-e043-508b-91c0-09e4cc0f5a3c',
path: 'all_assets-0.2.0/kibana/osquery_pack_asset/sample_osquery_pack_asset.json',
type: 'epm-packages-assets',
},
{
id: 'cc1e3e1d-f27b-5d05-86f6-6e4b9a47c7dc',
path: 'all_assets-0.2.0/data_stream/test_metrics/manifest.yml',
type: 'epm-packages-assets',
},
{
id: 'd2f87071-c866-503a-8fcb-7b23a8c7afbf',
path: 'all_assets-0.2.0/data_stream/test_logs2/manifest.yml',
type: 'epm-packages-assets',
},
{
id: 'e6ae7d31-6920-5408-9219-91ef1662044b',
path: 'all_assets-0.2.0/kibana/tag/sample_tag.json',
type: 'epm-packages-assets',
},
{
id: 'eec4606c-dbfa-565b-8e9c-fce1e641f3fc',
path: 'all_assets-0.2.0/data_stream/test_logs/fields/ecs.yml',
type: 'epm-packages-assets',
},
{
id: 'ef67e7e0-dca3-5a62-a42a-745db5ad7c1f',
path: 'all_assets-0.2.0/data_stream/test_logs/fields/fields.yml',
type: 'epm-packages-assets',
},
],
name: 'all_assets',
version: '0.2.0',