[Fleet] Tagging of integration assets (#137184)

* WIP saved object tagging

* fixing plugin usage

* added logic to create tags before assigning

* moved constants out

* fixed tests, added span

* added unit test to tagKibanaAssets

* fix test

* fix types

* fixed tests

* fixed tests

* fix types

* fix sot tests by loading empty kibana archive

* added tag checking to api integration test

* added refresh option to speed up tagging assets

* fixed tests

* workaround to prevent installing fleet packages in SOT functional tests

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Julia Bardi 2022-07-29 09:32:57 +02:00 committed by GitHub
parent 63236283cb
commit c6d4fb594b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 366 additions and 18 deletions

View file

@ -23,8 +23,14 @@ export interface GetAllTagsOptions {
asSystemRequest?: boolean;
}
export interface CreateTagOptions {
id?: string;
overwrite?: boolean;
refresh?: boolean | 'wait_for';
}
export interface ITagsClient {
create(attributes: TagAttributes): Promise<Tag>;
create(attributes: TagAttributes, options?: CreateTagOptions): Promise<Tag>;
get(id: string): Promise<Tag>;
getAll(options?: GetAllTagsOptions): Promise<Tag[]>;
delete(id: string): Promise<void>;

View file

@ -8,7 +8,7 @@
"server": true,
"ui": true,
"configPath": ["xpack", "fleet"],
"requiredPlugins": ["licensing", "data", "encryptedSavedObjects", "navigation", "customIntegrations", "share", "spaces", "security", "unifiedSearch"],
"requiredPlugins": ["licensing", "data", "encryptedSavedObjects", "navigation", "customIntegrations", "share", "spaces", "security", "unifiedSearch", "savedObjectsTagging"],
"optionalPlugins": ["features", "cloud", "usageCollection", "home", "globalSearch", "telemetry", "discover", "ingestPipelines"],
"extraPublicDirs": ["common"],
"requiredBundles": ["kibanaReact", "cloud", "esUiShared", "infra", "kibanaUtils", "usageCollection", "unifiedSearch"]

View file

@ -42,6 +42,8 @@ import type { CloudSetup } from '@kbn/cloud-plugin/server';
import type { SpacesPluginStart } from '@kbn/spaces-plugin/server';
import type { SavedObjectTaggingStart } from '@kbn/saved-objects-tagging-plugin/server';
import type { FleetConfigType } from '../common/types';
import type { FleetAuthz } from '../common';
import type { ExperimentalFeatures } from '../common/experimental_features';
@ -115,6 +117,7 @@ export interface FleetStartDeps {
encryptedSavedObjects: EncryptedSavedObjectsPluginStart;
security: SecurityPluginStart;
telemetry?: TelemetryPluginStart;
savedObjectsTagging: SavedObjectTaggingStart;
}
export interface FleetAppContext {
@ -128,6 +131,7 @@ export interface FleetAppContext {
configInitialValue: FleetConfigType;
experimentalFeatures: ExperimentalFeatures;
savedObjects: SavedObjectsServiceStart;
savedObjectsTagging?: SavedObjectTaggingStart;
isProductionMode: PluginInitializerContext['env']['mode']['prod'];
kibanaVersion: PluginInitializerContext['env']['packageInfo']['version'];
kibanaBranch: PluginInitializerContext['env']['packageInfo']['branch'];
@ -400,6 +404,7 @@ export class FleetPlugin
this.configInitialValue.enableExperimental || []
),
savedObjects: core.savedObjects,
savedObjectsTagging: plugins.savedObjectsTagging,
isProductionMode: this.isProductionMode,
kibanaVersion: this.kibanaVersion,
kibanaBranch: this.kibanaBranch,

View file

@ -26,6 +26,8 @@ import type { SecurityPluginStart, SecurityPluginSetup } from '@kbn/security-plu
import type { CloudSetup } from '@kbn/cloud-plugin/server';
import type { SavedObjectTaggingStart } from '@kbn/saved-objects-tagging-plugin/server';
import type { FleetConfigType } from '../../common/types';
import type { ExperimentalFeatures } from '../../common/experimental_features';
import type {
@ -58,6 +60,7 @@ class AppContextService {
private httpSetup?: HttpServiceSetup;
private externalCallbacks: ExternalCallbacksStorage = new Map();
private telemetryEventsSender: TelemetryEventsSender | undefined;
private savedObjectsTagging: SavedObjectTaggingStart | undefined;
public start(appContext: FleetAppContext) {
this.data = appContext.data;
@ -75,6 +78,7 @@ class AppContextService {
this.kibanaBranch = appContext.kibanaBranch;
this.httpSetup = appContext.httpSetup;
this.telemetryEventsSender = appContext.telemetryEventsSender;
this.savedObjectsTagging = appContext.savedObjectsTagging;
if (appContext.config$) {
this.config$ = appContext.config$;
@ -143,6 +147,13 @@ class AppContextService {
return this.savedObjects;
}
public getSavedObjectsTagging() {
if (!this.savedObjectsTagging) {
throw new Error('Saved object tagging start service not set.');
}
return this.savedObjectsTagging;
}
public getInternalUserSOClient(request: KibanaRequest) {
// soClient as kibana internal users, be careful on how you use it, security is not enabled
return appContextService.getSavedObjects().getScopedClient(request, {

View file

@ -18,6 +18,8 @@ import type { SavedObjectsImportSuccess, SavedObjectsImportFailure } from '@kbn/
import { createListStream } from '@kbn/utils';
import { partition } from 'lodash';
import type { IAssignmentService, ITagsClient } from '@kbn/saved-objects-tagging-plugin/server';
import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../../common';
import { getAsset, getPathParts } from '../../archive';
import { KibanaAssetType, KibanaSavedObjectType } from '../../../../types';
@ -27,6 +29,10 @@ import { indexPatternTypes, getIndexPatternSavedObjects } from '../index_pattern
import { saveKibanaAssetsRefs } from '../../packages/install';
import { deleteKibanaSavedObjectsAssets } from '../../packages/remove';
import { withPackageSpan } from '../../packages/utils';
import { tagKibanaAssets } from './tag_assets';
type SavedObjectsImporterContract = Pick<SavedObjectsImporter, 'import' | 'resolveImportErrors'>;
const formatImportErrorsForLog = (errors: SavedObjectsImportFailure[]) =>
JSON.stringify(
@ -128,15 +134,21 @@ export async function installKibanaAssets(options: {
export async function installKibanaAssetsAndReferences({
savedObjectsClient,
savedObjectsImporter,
savedObjectTagAssignmentService,
savedObjectTagClient,
logger,
pkgName,
pkgTitle,
paths,
installedPkg,
}: {
savedObjectsClient: SavedObjectsClientContract;
savedObjectsImporter: Pick<SavedObjectsImporter, 'import' | 'resolveImportErrors'>;
savedObjectTagAssignmentService: IAssignmentService;
savedObjectTagClient: ITagsClient;
logger: Logger;
pkgName: string;
pkgTitle: string;
paths: string[];
installedPkg?: SavedObject<Installation>;
}) {
@ -156,6 +168,16 @@ export async function installKibanaAssetsAndReferences({
kibanaAssets,
});
await withPackageSpan('Create and assign package tags', () =>
tagKibanaAssets({
savedObjectTagAssignmentService,
savedObjectTagClient,
kibanaAssets,
pkgTitle,
pkgName,
})
);
return installedKibanaAssetsRefs;
}

View file

@ -0,0 +1,127 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { tagKibanaAssets } from './tag_assets';
describe('tagKibanaAssets', () => {
const savedObjectTagAssignmentService = {
updateTagAssignments: jest.fn(),
} as any;
const savedObjectTagClient = {
getAll: jest.fn(),
create: jest.fn(),
} as any;
beforeEach(() => {
savedObjectTagAssignmentService.updateTagAssignments.mockReset();
savedObjectTagClient.getAll.mockReset();
savedObjectTagClient.create.mockReset();
});
it('should create Managed and System tags when tagKibanaAssets with System package', async () => {
savedObjectTagClient.getAll.mockResolvedValue([]);
savedObjectTagClient.create.mockImplementation(({ name }: { name: string }) =>
Promise.resolve({ id: name.toLowerCase(), name })
);
const kibanaAssets = { dashboard: [{ id: 'dashboard1', type: 'dashboard' }] } as any;
await tagKibanaAssets({
savedObjectTagAssignmentService,
savedObjectTagClient,
kibanaAssets,
pkgTitle: 'System',
pkgName: 'system',
});
expect(savedObjectTagClient.create).toHaveBeenCalledWith(
{
name: 'Managed',
description: '',
color: '#FFFFFF',
},
{ id: 'managed', overwrite: true, refresh: false }
);
expect(savedObjectTagClient.create).toHaveBeenCalledWith(
{
name: 'System',
description: '',
color: '#FFFFFF',
},
{ id: 'system', overwrite: true, refresh: false }
);
expect(savedObjectTagAssignmentService.updateTagAssignments).toHaveBeenCalledWith({
tags: ['managed', 'system'],
assign: kibanaAssets.dashboard,
unassign: [],
refresh: false,
});
});
it('should only assign Managed and System tags when tags already exist', async () => {
savedObjectTagClient.getAll.mockResolvedValue([
{ id: 'managed', name: 'Managed' },
{ id: 'system', name: 'System' },
]);
const kibanaAssets = { dashboard: [{ id: 'dashboard1', type: 'dashboard' }] } as any;
await tagKibanaAssets({
savedObjectTagAssignmentService,
savedObjectTagClient,
kibanaAssets,
pkgTitle: 'System',
pkgName: 'system',
});
expect(savedObjectTagClient.create).not.toHaveBeenCalled();
expect(savedObjectTagAssignmentService.updateTagAssignments).toHaveBeenCalledWith({
tags: ['managed', 'system'],
assign: kibanaAssets.dashboard,
unassign: [],
refresh: false,
});
});
it('should skip non taggable asset types', async () => {
savedObjectTagClient.getAll.mockResolvedValue([]);
savedObjectTagClient.create.mockImplementation(({ name }: { name: string }) =>
Promise.resolve({ id: name.toLowerCase(), name })
);
const kibanaAssets = {
dashboard: [{ id: 'dashboard1', type: 'dashboard' }],
search: [{ id: 's1', type: 'search' }],
visualization: [{ id: 'v1', type: 'visualization' }],
} as any;
await tagKibanaAssets({
savedObjectTagAssignmentService,
savedObjectTagClient,
kibanaAssets,
pkgTitle: 'System',
pkgName: 'system',
});
expect(savedObjectTagAssignmentService.updateTagAssignments).toHaveBeenCalledWith({
tags: ['managed', 'system'],
assign: [...kibanaAssets.dashboard, ...kibanaAssets.visualization],
unassign: [],
refresh: false,
});
});
it('should do nothing if no taggable assets', async () => {
const kibanaAssets = { search: [{ id: 's1', type: 'search' }] } as any;
await tagKibanaAssets({
savedObjectTagAssignmentService,
savedObjectTagClient,
kibanaAssets,
pkgTitle: 'System',
pkgName: 'system',
});
expect(savedObjectTagAssignmentService.updateTagAssignments).not.toHaveBeenCalled();
});
});

View file

@ -0,0 +1,80 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { taggableTypes } from '@kbn/saved-objects-tagging-plugin/common/constants';
import type { IAssignmentService, ITagsClient } from '@kbn/saved-objects-tagging-plugin/server';
import type { KibanaAssetType } from '../../../../../common';
import type { ArchiveAsset } from './install';
const TAG_COLOR = '#FFFFFF';
const MANAGED_TAG_NAME = 'Managed';
const MANAGED_TAG_ID = 'managed';
export async function tagKibanaAssets({
savedObjectTagAssignmentService,
savedObjectTagClient,
kibanaAssets,
pkgTitle,
pkgName,
}: {
savedObjectTagAssignmentService: IAssignmentService;
savedObjectTagClient: ITagsClient;
kibanaAssets: Record<KibanaAssetType, ArchiveAsset[]>;
pkgTitle: string;
pkgName: string;
}) {
const taggableAssets = Object.entries(kibanaAssets).flatMap(([assetType, assets]) => {
if (!taggableTypes.includes(assetType as KibanaAssetType)) {
return [];
}
if (!assets.length) {
return [];
}
return assets;
});
// no assets to tag
if (taggableAssets.length === 0) {
return;
}
const allTags = await savedObjectTagClient.getAll();
let managedTag = allTags.find((tag) => tag.name === MANAGED_TAG_NAME);
if (!managedTag) {
managedTag = await savedObjectTagClient.create(
{
name: MANAGED_TAG_NAME,
description: '',
color: TAG_COLOR,
},
{ id: MANAGED_TAG_ID, overwrite: true, refresh: false }
);
}
let packageTag = allTags.find((tag) => tag.name === pkgTitle);
if (!packageTag) {
packageTag = await savedObjectTagClient.create(
{
name: pkgTitle,
description: '',
color: TAG_COLOR,
},
{ id: pkgName, overwrite: true, refresh: false }
);
}
await savedObjectTagAssignmentService.updateTagAssignments({
tags: [managedTag.id, packageTag.id],
assign: taggableAssets,
unassign: [],
refresh: false,
});
}

View file

@ -14,6 +14,8 @@ import type {
} from '@kbn/core/server';
import { SavedObjectsErrorHelpers } from '@kbn/core/server';
import type { IAssignmentService, ITagsClient } from '@kbn/saved-objects-tagging-plugin/server';
import {
MAX_TIME_COMPLETE_INSTALL,
ASSETS_SAVED_OBJECT_TYPE,
@ -56,6 +58,8 @@ import { withPackageSpan } from './utils';
export async function _installPackage({
savedObjectsClient,
savedObjectsImporter,
savedObjectTagAssignmentService,
savedObjectTagClient,
esClient,
logger,
installedPkg,
@ -68,6 +72,8 @@ export async function _installPackage({
}: {
savedObjectsClient: SavedObjectsClientContract;
savedObjectsImporter: Pick<SavedObjectsImporter, 'import' | 'resolveImportErrors'>;
savedObjectTagAssignmentService: IAssignmentService;
savedObjectTagClient: ITagsClient;
esClient: ElasticsearchClient;
logger: Logger;
installedPkg?: SavedObject<Installation>;
@ -78,7 +84,7 @@ export async function _installPackage({
spaceId: string;
verificationResult?: PackageVerificationResult;
}): Promise<AssetReference[]> {
const { name: pkgName, version: pkgVersion } = packageInfo;
const { name: pkgName, version: pkgVersion, title: pkgTitle } = packageInfo;
try {
// if some installation already exists
@ -120,7 +126,10 @@ export async function _installPackage({
installKibanaAssetsAndReferences({
savedObjectsClient,
savedObjectsImporter,
savedObjectTagAssignmentService,
savedObjectTagClient,
pkgName,
pkgTitle,
paths,
installedPkg,
logger,

View file

@ -32,6 +32,10 @@ jest.mock('../../app_context', () => {
getSavedObjects: jest.fn(() => ({
createImporter: jest.fn(),
})),
getSavedObjectsTagging: jest.fn(() => ({
createInternalAssignmentService: jest.fn(),
createTagClient: jest.fn(),
})),
},
};
});

View file

@ -362,11 +362,21 @@ async function installPackageFromRegistry({
.getSavedObjects()
.createImporter(savedObjectsClient);
const savedObjectTagAssignmentService = appContextService
.getSavedObjectsTagging()
.createInternalAssignmentService({ client: savedObjectsClient });
const savedObjectTagClient = appContextService
.getSavedObjectsTagging()
.createTagClient({ client: savedObjectsClient });
// try installing the package, if there was an error, call error handler and rethrow
// @ts-expect-error status is string instead of InstallResult.status 'installed' | 'already_installed'
return await _installPackage({
savedObjectsClient,
savedObjectsImporter,
savedObjectTagAssignmentService,
savedObjectTagClient,
esClient,
logger,
installedPkg,
@ -477,10 +487,20 @@ async function installPackageByUpload({
.getSavedObjects()
.createImporter(savedObjectsClient);
const savedObjectTagAssignmentService = appContextService
.getSavedObjectsTagging()
.createInternalAssignmentService({ client: savedObjectsClient });
const savedObjectTagClient = appContextService
.getSavedObjectsTagging()
.createTagClient({ client: savedObjectsClient });
// @ts-expect-error status is string instead of InstallResult.status 'installed' | 'already_installed'
return await _installPackage({
savedObjectsClient,
savedObjectsImporter,
savedObjectTagAssignmentService,
savedObjectTagClient,
esClient,
logger,
installedPkg,

View file

@ -26,6 +26,7 @@ export interface UpdateTagAssignmentsOptions {
tags: string[];
assign: ObjectReference[];
unassign: ObjectReference[];
refresh?: boolean | 'wait_for';
}
export interface FindAssignableObjectsOptions {

View file

@ -154,21 +154,56 @@ describe('AssignmentService', () => {
});
expect(savedObjectClient.bulkUpdate).toHaveBeenCalledTimes(1);
expect(savedObjectClient.bulkUpdate).toHaveBeenCalledWith([
expect(savedObjectClient.bulkUpdate).toHaveBeenCalledWith(
[
{
type: 'dashboard',
id: 'dash-1',
attributes: {},
references: [createReference('tag', 'tag-1'), createReference('tag', 'tag-2')],
},
{
type: 'map',
id: 'map-1',
attributes: {},
references: [createReference('dashboard', 'dash-1')],
},
],
{ refresh: undefined }
);
});
});
it('calls `soClient.bulkUpdate` to update the references with refresh false', async () => {
savedObjectClient.bulkGet.mockResolvedValue({
saved_objects: [
createSavedObject({
type: 'dashboard',
id: 'dash-1',
references: [],
}),
],
});
getUpdatableSavedObjectTypesMock.mockResolvedValue(['dashboard']);
await service.updateTagAssignments({
tags: ['tag-1', 'tag-2'],
assign: [{ type: 'dashboard', id: 'dash-1' }],
unassign: [],
refresh: false,
});
expect(savedObjectClient.bulkUpdate).toHaveBeenCalledWith(
[
{
type: 'dashboard',
id: 'dash-1',
attributes: {},
references: [createReference('tag', 'tag-1'), createReference('tag', 'tag-2')],
},
{
type: 'map',
id: 'map-1',
attributes: {},
references: [createReference('dashboard', 'dash-1')],
},
]);
});
],
{ refresh: false }
);
});
describe('#findAssignableObjects', () => {

View file

@ -106,7 +106,12 @@ export class AssignmentService {
});
}
public async updateTagAssignments({ tags, assign, unassign }: UpdateTagAssignmentsOptions) {
public async updateTagAssignments({
tags,
assign,
unassign,
refresh,
}: UpdateTagAssignmentsOptions) {
const updatedTypes = uniq([...assign, ...unassign].map(({ type }) => type));
const untaggableTypes = difference(updatedTypes, taggableTypes);
@ -149,7 +154,7 @@ export class AssignmentService {
};
});
await this.soClient.bulkUpdate(updatedObjects);
await this.soClient.bulkUpdate(updatedObjects, { refresh });
}
}

View file

@ -58,7 +58,20 @@ describe('TagsClient', () => {
await tagsClient.create(attributes);
expect(soClient.create).toHaveBeenCalledTimes(1);
expect(soClient.create).toHaveBeenCalledWith('tag', attributes);
expect(soClient.create).toHaveBeenCalledWith('tag', attributes, undefined);
});
it('calls `soClient.create` with options', async () => {
const attributes = createAttributes();
await tagsClient.create(attributes, { id: '1', overwrite: true, refresh: false });
expect(soClient.create).toHaveBeenCalledTimes(1);
expect(soClient.create).toHaveBeenCalledWith('tag', attributes, {
id: '1',
overwrite: true,
refresh: false,
});
});
it('converts the object returned from the soClient to a `Tag`', async () => {

View file

@ -6,6 +6,7 @@
*/
import { SavedObjectsClientContract } from '@kbn/core/server';
import { CreateTagOptions } from '@kbn/saved-objects-tagging-oss-plugin/common/types';
import { TagSavedObject, TagAttributes, ITagsClient } from '../../../common/types';
import { tagSavedObjectTypeName } from '../../../common/constants';
import { TagValidationError } from './errors';
@ -24,12 +25,12 @@ export class TagsClient implements ITagsClient {
this.soClient = client;
}
public async create(attributes: TagAttributes) {
public async create(attributes: TagAttributes, options?: CreateTagOptions) {
const validation = validateTag(attributes);
if (!validation.valid) {
throw new TagValidationError('Error validating tag attributes', validation);
}
const raw = await this.soClient.create<TagAttributes>(this.type, attributes);
const raw = await this.soClient.create<TagAttributes>(this.type, attributes, options);
return savedObjectToTag(raw);
}

View file

@ -140,6 +140,11 @@ export default function (providerContext: FtrProviderContext) {
},
})
.expect(200);
const { body } = await supertest
.get(`/internal/saved_objects_tagging/tags/_find?page=1&perPage=10000`)
.expect(200);
expect(body.tags.find((tag: any) => tag.name === 'Managed').relationCount).to.be(6);
expect(body.tags.find((tag: any) => tag.name === 'For File Tests').relationCount).to.be(6);
});
it('should return a 400 with an empty namespace', async function () {

View file

@ -31,6 +31,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
serverArgs: [
...apiIntegrationConfig.get('kbnTestServer.serverArgs'),
'--server.xsrf.disableProtection=true',
`--xpack.fleet.registryUrl=http://localhost:12345`, // setting to invalid registry url to prevent installing preconfigured packages
],
},
};

View file

@ -33,7 +33,10 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
kbnTestServer: {
...kibanaFunctionalConfig.get('kbnTestServer'),
serverArgs: [...kibanaFunctionalConfig.get('kbnTestServer.serverArgs')],
serverArgs: [
...kibanaFunctionalConfig.get('kbnTestServer.serverArgs'),
`--xpack.fleet.registryUrl=http://localhost:12345`, // setting to invalid registry url to prevent installing preconfigured packages
],
},
};
}