[Fleet] Simplified package policy create API, enriching default values (#119739) (#120576)

* simplified api, enriching default values

* removed comment

* reverted changes in common models

* fixed handlers test

* separated create schema

* fixed test

* removed throwing error

* getting package info from registry only

* fix type

* fix test and new tests

* simplified update api

* updated open api spec

* fixed compiled_input

* added policy_template enrich

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

Co-authored-by: Julia Bardi <90178898+juliaElastic@users.noreply.github.com>
This commit is contained in:
Kibana Machine 2021-12-07 04:09:47 -05:00 committed by GitHub
parent 8f57ac7fea
commit ef882c219b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 744 additions and 84 deletions

View file

@ -2673,8 +2673,7 @@
},
"required": [
"name",
"version",
"title"
"version"
]
},
"namespace": {
@ -2713,8 +2712,7 @@
},
"required": [
"type",
"enabled",
"streams"
"enabled"
]
}
},
@ -2729,9 +2727,7 @@
}
},
"required": [
"output_id",
"inputs",
"policy_id",
"name"
]
},
@ -2858,19 +2854,86 @@
},
"update_package_policy": {
"title": "Update package policy",
"allOf": [
{
"type": "object",
"description": "",
"properties": {
"version": {
"type": "string"
},
"enabled": {
"type": "boolean"
},
"package": {
"type": "object",
"properties": {
"name": {
"type": "string"
},
"version": {
"type": "string"
},
"title": {
"type": "string"
}
},
"required": [
"name",
"title",
"version"
]
},
"namespace": {
"type": "string"
},
"output_id": {
"type": "string"
},
"inputs": {
"type": "array",
"items": {
"type": "object",
"properties": {
"type": {
"type": "string"
},
"enabled": {
"type": "boolean"
},
"processors": {
"type": "array",
"items": {
"type": "string"
}
},
"streams": {
"type": "array",
"items": {}
},
"config": {
"type": "object"
},
"vars": {
"type": "object"
}
},
"required": [
"type",
"enabled",
"streams"
]
}
},
{
"$ref": "#/components/schemas/new_package_policy"
"policy_id": {
"type": "string"
},
"name": {
"type": "string"
},
"description": {
"type": "string"
}
]
},
"required": null
},
"output": {
"title": "Output",

View file

@ -1683,7 +1683,6 @@ components:
required:
- name
- version
- title
namespace:
type: string
output_id:
@ -1711,7 +1710,6 @@ components:
required:
- type
- enabled
- streams
policy_id:
type: string
name:
@ -1719,9 +1717,7 @@ components:
description:
type: string
required:
- output_id
- inputs
- policy_id
- name
package_policy:
title: Package policy
@ -1801,12 +1797,61 @@ components:
items: {}
update_package_policy:
title: Update package policy
allOf:
- type: object
type: object
description: ''
properties:
version:
type: string
enabled:
type: boolean
package:
type: object
properties:
name:
type: string
version:
type: string
- $ref: '#/components/schemas/new_package_policy'
title:
type: string
required:
- name
- title
- version
namespace:
type: string
output_id:
type: string
inputs:
type: array
items:
type: object
properties:
type:
type: string
enabled:
type: boolean
processors:
type: array
items:
type: string
streams:
type: array
items: {}
config:
type: object
vars:
type: object
required:
- type
- enabled
- streams
policy_id:
type: string
name:
type: string
description:
type: string
required: null
output:
title: Output
type: object

View file

@ -16,7 +16,6 @@ properties:
required:
- name
- version
- title
namespace:
type: string
output_id:
@ -44,7 +43,6 @@ properties:
required:
- type
- enabled
- streams
policy_id:
type: string
name:
@ -52,7 +50,5 @@ properties:
description:
type: string
required:
- output_id
- inputs
- policy_id
- name

View file

@ -1,7 +1,56 @@
title: Update package policy
allOf:
- type: object
type: object
description: ''
properties:
version:
type: string
enabled:
type: boolean
package:
type: object
properties:
name:
type: string
version:
type: string
- $ref: ./new_package_policy.yaml
title:
type: string
required:
- name
- title
- version
namespace:
type: string
output_id:
type: string
inputs:
type: array
items:
type: object
properties:
type:
type: string
enabled:
type: boolean
processors:
type: array
items:
type: string
streams:
type: array
items: {}
config:
type: object
vars:
type: object
required:
- type
- enabled
- streams
policy_id:
type: string
name:
type: string
description:
type: string
required:

View file

@ -99,6 +99,7 @@ export const createPackagePolicyServiceMock = (): jest.Mocked<PackagePolicyServi
return {
_compilePackagePolicyInputs: jest.fn(),
buildPackagePolicyFromPackage: jest.fn(),
buildPackagePolicyFromPackageWithVersion: jest.fn(),
bulkCreate: jest.fn(),
create: jest.fn(),
delete: jest.fn(),
@ -112,6 +113,7 @@ export const createPackagePolicyServiceMock = (): jest.Mocked<PackagePolicyServi
upgrade: jest.fn(),
getUpgradeDryRunDiff: jest.fn(),
getUpgradePackagePolicyInfo: jest.fn(),
enrichPolicyWithDefaultsFromPackage: jest.fn(),
};
};

View file

@ -17,7 +17,12 @@ import type {
PostPackagePolicyCreateCallback,
PutPackagePolicyUpdateCallback,
} from '../..';
import type { CreatePackagePolicyRequestSchema } from '../../types/rest_spec';
import type {
CreatePackagePolicyRequestSchema,
UpdatePackagePolicyRequestSchema,
} from '../../types/rest_spec';
import type { PackagePolicy } from '../../types';
import { registerRoutes } from './index';
@ -72,6 +77,9 @@ jest.mock(
),
upgrade: jest.fn(),
getUpgradeDryRunDiff: jest.fn(),
enrichPolicyWithDefaultsFromPackage: jest
.fn()
.mockImplementation((soClient, newPolicy) => newPolicy),
},
};
}
@ -91,7 +99,7 @@ describe('When calling package policy', () => {
let context: ReturnType<typeof xpackMocks.createRequestHandlerContext>;
let response: ReturnType<typeof httpServerMock.createResponseFactory>;
beforeAll(() => {
beforeEach(() => {
routerMock = httpServiceMock.createRouter();
registerRoutes(routerMock);
});
@ -132,7 +140,7 @@ describe('When calling package policy', () => {
};
// Set the routeConfig and routeHandler to the Create API
beforeAll(() => {
beforeEach(() => {
[routeConfig, routeHandler] = routerMock.post.mock.calls.find(([{ path }]) =>
path.startsWith(PACKAGE_POLICY_API_ROUTES.CREATE_PATTERN)
)!;
@ -259,4 +267,148 @@ describe('When calling package policy', () => {
});
});
});
describe('update api handler', () => {
const getUpdateKibanaRequest = (
newData?: typeof UpdatePackagePolicyRequestSchema.body
): KibanaRequest<
typeof UpdatePackagePolicyRequestSchema.params,
undefined,
typeof UpdatePackagePolicyRequestSchema.body
> => {
return httpServerMock.createKibanaRequest<
typeof UpdatePackagePolicyRequestSchema.params,
undefined,
typeof UpdatePackagePolicyRequestSchema.body
>({
path: routeConfig.path,
method: 'put',
params: { packagePolicyId: '1' },
body: newData || {},
});
};
const existingPolicy = {
name: 'endpoint-1',
description: 'desc',
policy_id: '2',
enabled: true,
output_id: '3',
inputs: [
{
type: 'logfile',
enabled: true,
streams: [
{
enabled: true,
data_stream: {
type: 'logs',
dataset: 'apache.access',
},
id: '1',
},
],
},
],
namespace: 'default',
package: { name: 'endpoint', title: 'Elastic Endpoint', version: '0.5.0' },
vars: {
paths: {
value: ['/var/log/apache2/access.log*'],
type: 'text',
},
},
};
beforeEach(() => {
[routeConfig, routeHandler] = routerMock.put.mock.calls.find(([{ path }]) =>
path.startsWith(PACKAGE_POLICY_API_ROUTES.UPDATE_PATTERN)
)!;
});
beforeEach(() => {
packagePolicyServiceMock.update.mockImplementation((soClient, esClient, policyId, newData) =>
Promise.resolve(newData as PackagePolicy)
);
packagePolicyServiceMock.get.mockResolvedValue({
id: '1',
revision: 1,
created_at: '',
created_by: '',
updated_at: '',
updated_by: '',
...existingPolicy,
inputs: [
{
...existingPolicy.inputs[0],
compiled_input: '',
streams: [
{
...existingPolicy.inputs[0].streams[0],
compiled_stream: {},
},
],
},
],
});
});
it('should use existing package policy props if not provided by request', async () => {
const request = getUpdateKibanaRequest();
await routeHandler(context, request, response);
expect(response.ok).toHaveBeenCalledWith({
body: { item: existingPolicy },
});
});
it('should use request package policy props if provided by request', async () => {
const newData = {
name: 'endpoint-2',
description: '',
policy_id: '3',
enabled: false,
output_id: '',
inputs: [
{
type: 'metrics',
enabled: true,
streams: [
{
enabled: true,
data_stream: {
type: 'metrics',
dataset: 'apache.access',
},
id: '1',
},
],
},
],
namespace: 'namespace',
package: { name: 'endpoint', title: 'Elastic Endpoint', version: '0.6.0' },
vars: {
paths: {
value: ['/my/access.log*'],
type: 'text',
},
},
};
const request = getUpdateKibanaRequest(newData as any);
await routeHandler(context, request, response);
expect(response.ok).toHaveBeenCalledWith({
body: { item: newData },
});
});
it('should override props provided by request only', async () => {
const newData = {
namespace: 'namespace',
};
const request = getUpdateKibanaRequest(newData as any);
await routeHandler(context, request, response);
expect(response.ok).toHaveBeenCalledWith({
body: { item: { ...existingPolicy, namespace: 'namespace' } },
});
});
});
});

View file

@ -23,6 +23,7 @@ import type {
import type {
CreatePackagePolicyResponse,
DeletePackagePoliciesResponse,
NewPackagePolicy,
UpgradePackagePolicyDryRunResponse,
UpgradePackagePolicyResponse,
} from '../../../common';
@ -89,9 +90,14 @@ export const createPackagePolicyHandler: RequestHandler<
const user = appContextService.getSecurity()?.authc.getCurrentUser(request) || undefined;
const { force, ...newPolicy } = request.body;
try {
const newPackagePolicy = await packagePolicyService.enrichPolicyWithDefaultsFromPackage(
soClient,
newPolicy as NewPackagePolicy
);
const newData = await packagePolicyService.runExternalCallbacks(
'packagePolicyCreate',
newPolicy,
newPackagePolicy,
context,
request
);
@ -130,9 +136,33 @@ export const updatePackagePolicyHandler: RequestHandler<
throw Boom.notFound('Package policy not found');
}
let newData = { ...request.body };
const pkg = newData.package || packagePolicy.package;
const inputs = newData.inputs || packagePolicy.inputs;
const body = { ...request.body };
// removed fields not recognized by schema
const packagePolicyInputs = packagePolicy.inputs.map((input) => {
const newInput = {
...input,
streams: input.streams.map((stream) => {
const newStream = { ...stream };
delete newStream.compiled_stream;
return newStream;
}),
};
delete newInput.compiled_input;
return newInput;
});
// listing down accepted properties, because loaded packagePolicy contains some that are not accepted in update
let newData = {
...body,
name: body.name ?? packagePolicy.name,
description: body.description ?? packagePolicy.description,
namespace: body.namespace ?? packagePolicy.namespace,
policy_id: body.policy_id ?? packagePolicy.policy_id,
enabled: body.enabled ?? packagePolicy.enabled,
output_id: body.output_id ?? packagePolicy.output_id,
package: body.package ?? packagePolicy.package,
inputs: body.inputs ?? packagePolicyInputs,
vars: body.vars ?? packagePolicy.vars,
} as NewPackagePolicy;
try {
newData = await packagePolicyService.runExternalCallbacks(
@ -146,7 +176,7 @@ export const updatePackagePolicyHandler: RequestHandler<
soClient,
esClient,
request.params.packagePolicyId,
{ ...newData, package: pkg, inputs },
newData,
{ user },
packagePolicy.package?.version
);

View file

@ -33,8 +33,10 @@ import type {
InputsOverride,
NewPackagePolicy,
NewPackagePolicyInput,
PackagePolicyPackage,
RegistryPackage,
} from '../../common';
import { packageToPackagePolicy } from '../../common';
import { IngestManagerError } from '../errors';
@ -107,6 +109,11 @@ jest.mock('./epm/packages', () => {
};
});
jest.mock('../../common', () => ({
...jest.requireActual('../../common'),
packageToPackagePolicy: jest.fn(),
}));
jest.mock('./epm/registry');
jest.mock('./agent_policy', () => {
@ -125,6 +132,7 @@ jest.mock('./agent_policy', () => {
return agentPolicy;
},
bumpRevision: () => {},
getDefaultAgentPolicyId: () => Promise.resolve('1'),
},
};
});
@ -2815,6 +2823,216 @@ describe('Package policy service', () => {
});
});
});
describe('enrich package policy on create', () => {
beforeEach(() => {
(packageToPackagePolicy as jest.Mock).mockReturnValue({
package: { name: 'apache', title: 'Apache', version: '1.0.0' },
inputs: [
{
type: 'logfile',
policy_template: 'log',
enabled: true,
streams: [
{
enabled: true,
data_stream: {
type: 'logs',
dataset: 'apache.access',
},
},
],
},
],
vars: {
paths: {
value: ['/var/log/apache2/access.log*'],
type: 'text',
},
},
});
});
it('should enrich from epm with defaults', async () => {
const newPolicy = {
name: 'apache-1',
inputs: [{ type: 'logfile', enabled: false }],
package: { name: 'apache', version: '0.3.3' },
} as NewPackagePolicy;
const result = await packagePolicyService.enrichPolicyWithDefaultsFromPackage(
savedObjectsClientMock.create(),
newPolicy
);
expect(result).toEqual({
name: 'apache-1',
namespace: 'default',
description: '',
package: { name: 'apache', title: 'Apache', version: '1.0.0' },
enabled: true,
policy_id: '1',
output_id: '',
inputs: [
{
enabled: false,
type: 'logfile',
policy_template: 'log',
streams: [
{
enabled: false,
data_stream: {
type: 'logs',
dataset: 'apache.access',
},
},
],
},
],
vars: {
paths: {
value: ['/var/log/apache2/access.log*'],
type: 'text',
},
},
});
});
it('should enrich from epm with defaults using policy template', async () => {
(packageToPackagePolicy as jest.Mock).mockReturnValueOnce({
package: { name: 'aws', title: 'AWS', version: '1.0.0' },
inputs: [
{
type: 'aws/metrics',
policy_template: 'cloudtrail',
enabled: true,
streams: [
{
enabled: true,
data_stream: {
type: 'metrics',
dataset: 'cloudtrail',
},
},
],
},
{
type: 'aws/metrics',
policy_template: 'cloudwatch',
enabled: true,
streams: [
{
enabled: true,
data_stream: {
type: 'metrics',
dataset: 'cloudwatch',
},
},
],
},
],
});
const newPolicy = {
name: 'aws-1',
inputs: [{ type: 'aws/metrics', policy_template: 'cloudwatch', enabled: true }],
package: { name: 'aws', version: '1.0.0' },
} as NewPackagePolicy;
const result = await packagePolicyService.enrichPolicyWithDefaultsFromPackage(
savedObjectsClientMock.create(),
newPolicy
);
expect(result).toEqual({
name: 'aws-1',
namespace: 'default',
description: '',
package: { name: 'aws', title: 'AWS', version: '1.0.0' },
enabled: true,
policy_id: '1',
output_id: '',
inputs: [
{
type: 'aws/metrics',
policy_template: 'cloudwatch',
enabled: true,
streams: [
{
enabled: true,
data_stream: {
type: 'metrics',
dataset: 'cloudwatch',
},
},
],
},
],
});
});
it('should override defaults with new values', async () => {
const newPolicy = {
name: 'apache-2',
namespace: 'namespace',
description: 'desc',
enabled: false,
policy_id: '2',
output_id: '3',
inputs: [
{
type: 'logfile',
enabled: true,
streams: [
{
enabled: true,
data_stream: {
type: 'logs',
dataset: 'apache.error',
},
},
],
},
],
vars: {
paths: {
value: ['/my/access.log*'],
type: 'text',
},
},
package: { name: 'apache', version: '1.0.0' } as PackagePolicyPackage,
} as NewPackagePolicy;
const result = await packagePolicyService.enrichPolicyWithDefaultsFromPackage(
savedObjectsClientMock.create(),
newPolicy
);
expect(result).toEqual({
name: 'apache-2',
namespace: 'namespace',
description: 'desc',
package: { name: 'apache', title: 'Apache', version: '1.0.0' },
enabled: false,
policy_id: '2',
output_id: '3',
inputs: [
{
enabled: true,
type: 'logfile',
streams: [
{
enabled: true,
data_stream: {
type: 'logs',
dataset: 'apache.error',
},
},
],
},
],
vars: {
paths: {
value: ['/my/access.log*'],
type: 'text',
},
},
});
});
});
});
describe('_applyIndexPrivileges()', () => {

View file

@ -710,6 +710,67 @@ class PackagePolicyService {
}
}
public async enrichPolicyWithDefaultsFromPackage(
soClient: SavedObjectsClientContract,
newPolicy: NewPackagePolicy
): Promise<NewPackagePolicy> {
let newPackagePolicy: NewPackagePolicy = newPolicy;
if (newPolicy.package) {
const newPP = await this.buildPackagePolicyFromPackageWithVersion(
soClient,
newPolicy.package.name,
newPolicy.package.version
);
if (newPP) {
const inputs = newPolicy.inputs.map((input) => {
const defaultInput = newPP.inputs.find(
(i) =>
i.type === input.type &&
(!input.policy_template || input.policy_template === i.policy_template)
);
return {
...defaultInput,
enabled: input.enabled,
type: input.type,
// to propagate "enabled: false" to streams
streams: defaultInput?.streams?.map((stream) => ({
...stream,
enabled: input.enabled,
})),
} as NewPackagePolicyInput;
});
newPackagePolicy = {
...newPP,
name: newPolicy.name,
namespace: newPolicy.namespace ?? 'default',
description: newPolicy.description ?? '',
enabled: newPolicy.enabled ?? true,
policy_id:
newPolicy.policy_id ?? (await agentPolicyService.getDefaultAgentPolicyId(soClient)),
output_id: newPolicy.output_id ?? '',
inputs: newPolicy.inputs[0]?.streams ? newPolicy.inputs : inputs,
vars: newPolicy.vars || newPP.vars,
};
}
}
return newPackagePolicy;
}
public async buildPackagePolicyFromPackageWithVersion(
soClient: SavedObjectsClientContract,
pkgName: string,
pkgVersion: string
): Promise<NewPackagePolicy | undefined> {
const packageInfo = await getPackageInfo({
savedObjectsClient: soClient,
pkgName,
pkgVersion,
});
if (packageInfo) {
return packageToPackagePolicy(packageInfo, '', '');
}
}
public async buildPackagePolicyFromPackage(
soClient: SavedObjectsClientContract,
pkgName: string

View file

@ -28,6 +28,53 @@ const ConfigRecordSchema = schema.recordOf(
})
);
const PackagePolicyStreamsSchema = {
id: schema.maybe(schema.string()), // BWC < 7.11
enabled: schema.boolean(),
keep_enabled: schema.maybe(schema.boolean()),
data_stream: schema.object({
dataset: schema.string(),
type: schema.string(),
elasticsearch: schema.maybe(
schema.object({
privileges: schema.maybe(
schema.object({
indices: schema.maybe(schema.arrayOf(schema.string())),
})
),
})
),
}),
vars: schema.maybe(ConfigRecordSchema),
config: schema.maybe(
schema.recordOf(
schema.string(),
schema.object({
type: schema.maybe(schema.string()),
value: schema.maybe(schema.any()),
})
)
),
};
const PackagePolicyInputsSchema = {
type: schema.string(),
policy_template: schema.maybe(schema.string()),
enabled: schema.boolean(),
keep_enabled: schema.maybe(schema.boolean()),
vars: schema.maybe(ConfigRecordSchema),
config: schema.maybe(
schema.recordOf(
schema.string(),
schema.object({
type: schema.maybe(schema.string()),
value: schema.maybe(schema.any()),
})
)
),
streams: schema.arrayOf(schema.object(PackagePolicyStreamsSchema)),
};
const PackagePolicyBaseSchema = {
name: schema.string(),
description: schema.maybe(schema.string()),
@ -42,54 +89,7 @@ const PackagePolicyBaseSchema = {
})
),
output_id: schema.string(),
inputs: schema.arrayOf(
schema.object({
type: schema.string(),
policy_template: schema.maybe(schema.string()),
enabled: schema.boolean(),
keep_enabled: schema.maybe(schema.boolean()),
vars: schema.maybe(ConfigRecordSchema),
config: schema.maybe(
schema.recordOf(
schema.string(),
schema.object({
type: schema.maybe(schema.string()),
value: schema.maybe(schema.any()),
})
)
),
streams: schema.arrayOf(
schema.object({
id: schema.maybe(schema.string()), // BWC < 7.11
enabled: schema.boolean(),
keep_enabled: schema.maybe(schema.boolean()),
data_stream: schema.object({
dataset: schema.string(),
type: schema.string(),
elasticsearch: schema.maybe(
schema.object({
privileges: schema.maybe(
schema.object({
indices: schema.maybe(schema.arrayOf(schema.string())),
})
),
})
),
}),
vars: schema.maybe(ConfigRecordSchema),
config: schema.maybe(
schema.recordOf(
schema.string(),
schema.object({
type: schema.maybe(schema.string()),
value: schema.maybe(schema.any()),
})
)
),
})
),
})
),
inputs: schema.arrayOf(schema.object(PackagePolicyInputsSchema)),
vars: schema.maybe(ConfigRecordSchema),
};
@ -99,6 +99,47 @@ export const NewPackagePolicySchema = schema.object({
force: schema.maybe(schema.boolean()),
});
const CreatePackagePolicyProps = {
...PackagePolicyBaseSchema,
namespace: schema.maybe(NamespaceSchema),
policy_id: schema.maybe(schema.string()),
enabled: schema.maybe(schema.boolean()),
package: schema.maybe(
schema.object({
name: schema.string(),
title: schema.maybe(schema.string()),
version: schema.string(),
})
),
output_id: schema.maybe(schema.string()),
inputs: schema.arrayOf(
schema.object({
...PackagePolicyInputsSchema,
streams: schema.maybe(schema.arrayOf(schema.object(PackagePolicyStreamsSchema))),
})
),
};
export const CreatePackagePolicyRequestBodySchema = schema.object({
...CreatePackagePolicyProps,
id: schema.maybe(schema.string()),
force: schema.maybe(schema.boolean()),
});
export const UpdatePackagePolicyRequestBodySchema = schema.object({
...CreatePackagePolicyProps,
name: schema.maybe(schema.string()),
inputs: schema.maybe(
schema.arrayOf(
schema.object({
...PackagePolicyInputsSchema,
streams: schema.maybe(schema.arrayOf(schema.object(PackagePolicyStreamsSchema))),
})
)
),
version: schema.maybe(schema.string()),
});
export const UpdatePackagePolicySchema = schema.object({
...PackagePolicyBaseSchema,
version: schema.maybe(schema.string()),

View file

@ -7,7 +7,10 @@
import { schema } from '@kbn/config-schema';
import { NewPackagePolicySchema, UpdatePackagePolicySchema } from '../models';
import {
CreatePackagePolicyRequestBodySchema,
UpdatePackagePolicyRequestBodySchema,
} from '../models';
import { ListWithKuerySchema } from './index';
@ -22,12 +25,12 @@ export const GetOnePackagePolicyRequestSchema = {
};
export const CreatePackagePolicyRequestSchema = {
body: NewPackagePolicySchema,
body: CreatePackagePolicyRequestBodySchema,
};
export const UpdatePackagePolicyRequestSchema = {
...GetOnePackagePolicyRequestSchema,
body: UpdatePackagePolicySchema,
body: UpdatePackagePolicyRequestBodySchema,
};
export const DeletePackagePoliciesRequestSchema = {