github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 01:38:56 -04:00
Code Issues Projects Releases Packages Wiki Activity

[UII] Add internal api that allows to create agent policy and its package policies (#212977)

Browse source 
## Summary

Resolves #206488. This PR introduces a new internal API that allows an
agent policy and its package policies to be created in one request:

<details>
<summary>⤵️ Expand for console snippet ⤵️</summary>

```
POST kbn:/internal/fleet/agent_and_package_policies
{
  "id": "jens-awesome-policy",
  "name": "awesome policy",
  "description": "",
  "namespace": "default",
  "package_policies": [
    {
      "policy_ids": [
        "jens-awesome-policy"
      ],
      "package": {
        "name": "log",
        "version": "1.1.2"
      },
      "name": "log-for-awesome-policy",
      "description": "",
      "namespace": "",
      "inputs": {
        "logs-logfile": {
          "enabled": true,
          "streams": {
            "log.log": {
              "enabled": true,
              "vars": {
                "paths": [
                  "/tmp/some-path"
                ],
                "data_stream.dataset": "generic",
                "tags": [],
                "custom": ""
              }
            }
          }
        }
      }
    },
    {
      "id": "fixed-id-for-filestream",
      "package": {
        "name": "filestream",
        "version": "1.0.1"
      },
      "name": "filestream-1",
      "description": "",
      "namespace": "",
      "inputs": {
        "filestream-filestream": {
          "enabled": true,
          "streams": {
            "filestream.generic": {
              "enabled": true,
              "vars": {
                "paths": [
                  "/var/log/*.log"
                ],
                "data_stream.dataset": "filestream.generic",
                "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
                "exclude_files": [
                  "\\.gz$"
                ],
                "include_files": [],
                "tags": [],
                "recursive_glob": true,
                "clean_inactive": -1,
                "harvester_limit": 0,
                "fingerprint": true,
                "fingerprint_offset": 0,
                "fingerprint_length": 1024,
                "exclude_lines": [],
                "include_lines": []
              }
            }
          }
        }
      }
    }
  ]
}
```
</details>

If successful, the response will be the agent policy with the full
nested package policies.

`id`s can be specified or omitted for either the agent or package
policy. If necessary, the resulting package policy's `policy_id` /
`policy_ids` will be updated with final ID of the created agent policy.

If any of the package policies fail to be created for any reason, there
is a rollback mechanism to delete the agent and package policies that
were already created.

The API also supports any query params that are supported by the create
agent policy and create package policy endpoints:
```
sys_monitoring: boolean; // passed to agent policy creation
format: 'simplified' | 'legacy; // passed to package policy creation
```

### Dev note
The new `createAgentAndPackagePoliciesHandler()` is unique in that it
acts mostly as a passthrough to other handlers,
`createAgentPolicyHandler` and `createPackagePolicyHandler`. This means
all the checks (spaces, rbac, etc) performed on the other handlers are
triggered appropriately.

### Checklist

Check the PR satisfies following conditions. 

Reviewers should verify this PR satisfies this list as well.

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [x] The PR description includes the appropriate Release Notes section,
and the correct `release_note:*` label is applied per the
[guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)
This commit is contained in:
Jen Huang 2025-03-03 18:22:19 -08:00 • committed by GitHub
parent 037e8f58e0
commit 8854433830
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 487 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

1
x-pack/platform/plugins/shared/fleet/common/constants/routes.ts
View file

@ -82,6 +82,7 @@ export const AGENT_POLICY_API_ROUTES = {
LIST_OUTPUTS_PATTERN: `${AGENT_POLICY_API_ROOT}/outputs`,
INFO_OUTPUTS_PATTERN: `${AGENT_POLICY_API_ROOT}/{agentPolicyId}/outputs`,
AUTO_UPGRADE_AGENTS_STATUS_PATTERN: `${AGENT_POLICY_API_ROOT}/{agentPolicyId}/auto_upgrade_agents_status`,
CREATE_WITH_PACKAGE_POLICIES: `${INTERNAL_ROOT}/agent_and_package_policies`,
};
// Kubernetes Manifest API routes

3
x-pack/platform/plugins/shared/fleet/common/types/rest_spec/agent_policy.ts
View file

@ -50,6 +50,9 @@ export interface GetAutoUpgradeAgentsStatusResponse {
export interface CreateAgentPolicyRequest {
body: NewAgentPolicy;
query: {
sys_monitoring?: boolean;
};
}
export interface CreateAgentPolicyResponse {

4
x-pack/platform/plugins/shared/fleet/common/types/rest_spec/package_policy.ts
View file

@ -13,6 +13,7 @@ import type {
PackagePolicyPackage,
FullAgentPolicyInput,
} from '../models';
import type { inputsFormat } from '../../constants';
import type { BulkGetResult, ListResult, ListWithKuery } from './common';
@ -35,6 +36,9 @@ export interface GetOnePackagePolicyResponse {
export interface CreatePackagePolicyRequest {
body: NewPackagePolicy & { force?: boolean };
query: {
format?: typeof inputsFormat.Simplified | typeof inputsFormat.Legacy;
};
}
export interface CreatePackagePolicyResponse {

117
x-pack/platform/plugins/shared/fleet/server/routes/agent_policy/handlers.ts
View file

@ -17,7 +17,7 @@ import { inputsFormat } from '../../../common/constants';
import { HTTPAuthorizationHeader } from '../../../common/http_authorization_header';
import { fullAgentPolicyToYaml } from '../../../common/services';
import { appContextService, agentPolicyService } from '../../services';
import { appContextService, agentPolicyService, packagePolicyService } from '../../services';
import { type AgentClient, getLatestAvailableAgentVersion } from '../../services/agents';
import {
AGENTS_PREFIX,
@ -40,12 +40,14 @@ import type {
GetAgentPolicyOutputsRequestSchema,
GetListAgentPolicyOutputsRequestSchema,
GetAutoUpgradeAgentsStatusRequestSchema,
CreateAgentAndPackagePolicyRequestSchema,
} from '../../types';
import type {
GetAgentPoliciesResponse,
GetAgentPoliciesResponseItem,
GetOneAgentPolicyResponse,
CreateAgentPolicyRequest,
CreateAgentPolicyResponse,
UpdateAgentPolicyResponse,
CopyAgentPolicyResponse,
@ -56,6 +58,7 @@ import type {
BulkGetAgentPoliciesResponse,
GetAgentPolicyOutputsResponse,
GetListAgentPolicyOutputsResponse,
CreatePackagePolicyRequest,
} from '../../../common/types';
import { AgentPolicyNotFoundError, FleetUnauthorizedError, FleetError } from '../../errors';
import { createAgentPolicyWithPackages } from '../../services/agent_policy_create';
@ -64,6 +67,8 @@ import { packagePolicyToSimplifiedPackagePolicy } from '../../../common/services
import { FLEET_API_PRIVILEGES } from '../../constants/api_privileges';
import { getAutoUpgradeAgentsStatus } from '../../services/agents';
import { createPackagePolicyHandler } from '../package_policy/handlers';
export async function populateAssignedAgentsCount(
agentClient: AgentClient,
agentPolicies: AgentPolicy[]
@ -390,6 +395,116 @@ export const createAgentPolicyHandler: FleetRequestHandler<
}
};
export const createAgentAndPackagePoliciesHandler: FleetRequestHandler<
undefined,
TypeOf<typeof CreateAgentAndPackagePolicyRequestSchema.query>,
TypeOf<typeof CreateAgentAndPackagePolicyRequestSchema.body>
> = async (context, request, response) => {
const coreContext = await context.core;
const logger = appContextService.getLogger();
logger.debug('Creating agent and package policies');
// Try to create the agent policy
const { package_policies: packagePolicies, ...agentPolicyWithoutPackagePolicies } = request.body;
const agentPolicyRequest = {
...request,
body: agentPolicyWithoutPackagePolicies,
query: request.query satisfies CreateAgentPolicyRequest['query'],
};
const agentPolicyResult = await createAgentPolicyHandler(context, agentPolicyRequest, response);
const createdAgentPolicy: CreateAgentPolicyResponse['item'] = agentPolicyResult.options.body.item;
const createdPackagePolicyIds = [];
if (agentPolicyRequest.body.id && agentPolicyRequest.body.id !== createdAgentPolicy.id) {
logger.warn(
`Agent policy created with id ${createdAgentPolicy.id} instead of requested id ${agentPolicyRequest.body.id}`
);
}
// Try to create the package policies
try {
for (const packagePolicy of packagePolicies) {
// Extract the original agent policy ID from the request in order to replace it with the created agent policy ID
const {
policy_id: agentPolicyId,
policy_ids: agentPolicyIds,
...restOfPackagePolicy
} = packagePolicy;
// Warn if the requested agent policy ID does not match the created agent policy ID
if (agentPolicyId && agentPolicyId !== createdAgentPolicy.id) {
logger.warn(
`Creating package policy with agent policy ID ${createdAgentPolicy.id} instead of requested id ${agentPolicyId}`
);
}
if (
agentPolicyIds &&
agentPolicyIds.length > 0 &&
(!agentPolicyIds.includes(createdAgentPolicy.id) || agentPolicyIds.length > 1)
) {
logger.warn(
`Creating package policy with agent policy ID ${
createdAgentPolicy.id
} instead of requested id(s) ${agentPolicyIds.join(',')}`
);
}
const packagePolicyRequest = {
...request,
body: {
...restOfPackagePolicy,
policy_ids: [createdAgentPolicy.id],
},
query: request.query satisfies CreatePackagePolicyRequest['query'],
};
const packagePolicyResult = await createPackagePolicyHandler(
context,
packagePolicyRequest,
response
);
createdPackagePolicyIds.push(packagePolicyResult.options.body.item.id);
}
// Return the created agent policy with full package policy details
return getOneAgentPolicyHandler(
context,
{
...request,
body: {},
params: { agentPolicyId: createdAgentPolicy.id },
},
response
);
} catch (e) {
// If there is an error creating package policies, delete any created package policy
// and the parent agent policy
if (createdPackagePolicyIds.length > 0) {
await packagePolicyService.delete(
coreContext.savedObjects.client,
coreContext.elasticsearch.client.asInternalUser,
createdPackagePolicyIds,
{
force: true,
skipUnassignFromAgentPolicies: true,
}
);
}
if (createdAgentPolicy) {
await agentPolicyService.delete(
coreContext.savedObjects.client,
coreContext.elasticsearch.client.asInternalUser,
createdAgentPolicy.id,
{
force: true,
}
);
}
// Rethrow
throw e;
}
};
export const updateAgentPolicyHandler: FleetRequestHandler<
TypeOf<typeof UpdateAgentPolicyRequestSchema.params>,
TypeOf<typeof UpdateAgentPolicyRequestSchema.query>,

62
x-pack/platform/plugins/shared/fleet/server/routes/agent_policy/index.test.ts
View file

@ -22,6 +22,7 @@ import {
GetFullAgentPolicyResponseSchema,
DownloadFullAgentPolicyResponseSchema,
GetK8sManifestResponseScheme,
CreateAgentAndPackagePolicyRequestSchema,
} from '../../types';
import { ListResponseSchema } from '../schema/utils';
@ -40,6 +41,7 @@ import {
downloadK8sManifest,
getK8sManifest,
bulkGetAgentPoliciesHandler,
createAgentAndPackagePoliciesHandler,
} from './handlers';
jest.mock('./handlers', () => ({
@ -53,6 +55,7 @@ jest.mock('./handlers', () => ({
getFullAgentPolicy: jest.fn(),
getK8sManifest: jest.fn(),
bulkGetAgentPoliciesHandler: jest.fn(),
createAgentAndPackagePoliciesHandler: jest.fn(),
}));
jest.mock('../../services', () => ({
@ -412,6 +415,65 @@ describe('schema validation', () => {
expect(validationResp).toEqual(expectedResponse);
});
describe('create agent policy with package policies', () => {
const validRequestBody = {
name: 'Test Agent Policy',
namespace: 'default',
description: 'Test description',
package_policies: [
{
name: 'Test Package Policy',
namespace: 'default',
policy_ids: [],
enabled: true,
inputs: [],
},
],
};
it('should return valid response', async () => {
const expectedResponse = {
item: agentPolicy,
};
(createAgentAndPackagePoliciesHandler as jest.Mock).mockImplementation(
(ctx, request, res) => {
return res.ok({ body: expectedResponse });
}
);
await createAgentAndPackagePoliciesHandler(context, {} as any, response);
expect(response.ok).toHaveBeenCalledWith({
body: expectedResponse,
});
const validationResp = GetAgentPolicyResponseSchema.validate(expectedResponse);
expect(validationResp).toEqual(expectedResponse);
});
it('should validate request schema', async () => {
expect(() =>
CreateAgentAndPackagePolicyRequestSchema.body.validate(validRequestBody)
).not.toThrow();
const invalidRequest = {
id: 'policy-missing-name',
namespace: 'default',
package_policies: [
{
name: 'Test Package Policy',
namespace: 'default',
policy_ids: [],
enabled: true,
inputs: [],
},
],
};
expect(() =>
CreateAgentAndPackagePolicyRequestSchema.body.validate(invalidRequest)
).toThrow();
});
});
it('update agent policy should return valid response', async () => {
const expectedResponse = {
item: agentPolicy,

35
x-pack/platform/plugins/shared/fleet/server/routes/agent_policy/index.ts
View file

@ -35,6 +35,7 @@ import {
GetListAgentPolicyOutputsRequestSchema,
GetAutoUpgradeAgentsStatusRequestSchema,
GetAutoUpgradeAgentsStatusResponseSchema,
CreateAgentAndPackagePolicyRequestSchema,
} from '../../types';
import { K8S_API_ROUTES } from '../../../common/constants';
@ -58,6 +59,7 @@ import {
GetAgentPolicyOutputsHandler,
GetListAgentPolicyOutputsHandler,
getAutoUpgradeAgentsStatusHandler,
createAgentAndPackagePoliciesHandler,
} from './handlers';
export const registerRoutes = (router: FleetAuthzRouter, config: FleetConfigType) => {
@ -249,6 +251,39 @@ export const registerRoutes = (router: FleetAuthzRouter, config: FleetConfigType
createAgentPolicyHandler
);
// Create agent + package policies in a single request
// Used for agentless integrations
router.versioned
.post({
path: AGENT_POLICY_API_ROUTES.CREATE_WITH_PACKAGE_POLICIES,
security: {
authz: {
requiredPrivileges: [FLEET_API_PRIVILEGES.AGENT_POLICIES.ALL],
},
},
summary: `Create an agent policy and its package policies in one request`,
options: {
tags: ['oas-tag:Elastic Agent policies'],
},
})
.addVersion(
{
version: API_VERSIONS.public.v1,
validate: {
request: CreateAgentAndPackagePolicyRequestSchema,
response: {
200: {
body: () => GetAgentPolicyResponseSchema,
},
400: {
body: genericErrorResponse,
},
},
},
},
createAgentAndPackagePoliciesHandler
);
// Update
router.versioned
.put({

11
x-pack/platform/plugins/shared/fleet/server/types/rest_spec/agent_policy.ts
View file

@ -18,6 +18,7 @@ import { LEGACY_AGENT_POLICY_SAVED_OBJECT_TYPE, AGENT_POLICY_MAPPINGS } from '..
import { validateKuery } from '../../routes/utils/filter_utils';
import { BulkRequestBodySchema } from './common';
import { CreatePackagePolicyRequestSchema } from './package_policy';
export const GetAgentPoliciesRequestSchema = {
query: schema.object(
@ -118,6 +119,16 @@ export const CreateAgentPolicyRequestSchema = {
}),
};
export const CreateAgentAndPackagePolicyRequestSchema = {
body: CreateAgentPolicyRequestSchema.body.extends({
package_policies: schema.arrayOf(CreatePackagePolicyRequestSchema.body),
}),
query: schema.intersection([
CreateAgentPolicyRequestSchema.query,
CreatePackagePolicyRequestSchema.query,
]),
};
export const UpdateAgentPolicyRequestSchema = {
...GetOneAgentPolicyRequestSchema,
body: NewAgentPolicySchema.extends({

255
x-pack/test/fleet_api_integration/apis/agent_policy/agent_policy.ts
View file

@ -2037,5 +2037,260 @@ export default function (providerContext: FtrProviderContext) {
await supertest.delete(`/api/fleet/agents/agent-2`).set('kbn-xsrf', 'xx').expect(200);
});
});
describe('POST /internal/fleet/agent_and_package_policies', () => {
before(async () => {
await esArchiver.load('x-pack/test/functional/es_archives/fleet/empty_fleet_server');
await kibanaServer.savedObjects.cleanStandardList();
await fleetAndAgents.setup();
});
after(async () => {
await esArchiver.unload('x-pack/test/functional/es_archives/fleet/empty_fleet_server');
});
afterEach(async () => {
await kibanaServer.savedObjects.cleanStandardList();
});
it('should create agent and package policy successfully when not given ids', async () => {
const requestBody = {
name: 'Test Agent Policy',
namespace: 'default',
description: 'Test description',
package_policies: [
{
name: 'Test Package Policy',
namespace: 'default',
policy_ids: [],
enabled: true,
inputs: [
{
enabled: true,
streams: [],
type: 'single_input',
},
],
package: {
name: 'filetest',
title: 'For File Tests',
version: '0.1.0',
},
},
],
};
const {
body: { item: createdPolicy },
} = await supertest
.post('/internal/fleet/agent_and_package_policies')
.set('kbn-xsrf', 'xxxx')
.send(requestBody)
.expect(200);
expect(createdPolicy.name).to.eql('Test Agent Policy');
expect(createdPolicy.package_policies[0].name).to.eql('Test Package Policy');
expect(createdPolicy.package_policies[0].policy_ids).to.eql([createdPolicy.id]);
});
it('should create agent and package policy successfully when given ids', async () => {
const requestBody = {
id: 'test-agent-policy-with-id',
name: 'Test Agent Policy',
namespace: 'default',
description: 'Test description',
package_policies: [
{
id: 'test-package-policy-with-id',
name: 'Test Package Policy',
namespace: 'default',
policy_ids: ['test-agent-policy-with-id'],
enabled: true,
inputs: [
{
enabled: true,
streams: [],
type: 'single_input',
},
],
package: {
name: 'filetest',
title: 'For File Tests',
version: '0.1.0',
},
},
{
id: 'test-package-policy-with-id-2',
name: 'Test Package Policy 2',
namespace: 'default',
policy_ids: ['test-agent-policy-with-id'],
enabled: true,
inputs: [
{
enabled: true,
streams: [],
type: 'single_input',
},
],
package: {
name: 'filetest',
title: 'For File Tests',
version: '0.1.0',
},
},
],
};
const {
body: { item: createdPolicy },
} = await supertest
.post('/internal/fleet/agent_and_package_policies')
.set('kbn-xsrf', 'xxxx')
.send(requestBody)
.expect(200);
expect(createdPolicy.id).to.eql(requestBody.id);
expect(createdPolicy.package_policies[0].id).to.eql(requestBody.package_policies[0].id);
expect(createdPolicy.package_policies[0].policy_ids).to.eql(
requestBody.package_policies[0].policy_ids
);
expect(createdPolicy.package_policies[1].id).to.eql(requestBody.package_policies[1].id);
expect(createdPolicy.package_policies[1].policy_ids).to.eql(
requestBody.package_policies[1].policy_ids
);
});
it('should create agent and package policy with consistent ids when given a mix', async () => {
const requestBody = {
name: 'Test Agent Policy',
namespace: 'default',
description: 'Test description',
package_policies: [
{
id: 'test-package-policy-mixed-id',
name: 'Test Package Policy',
namespace: 'default',
policy_id: 'some-invalid-id',
enabled: true,
inputs: [
{
enabled: true,
streams: [],
type: 'single_input',
},
],
package: {
name: 'filetest',
title: 'For File Tests',
version: '0.1.0',
},
},
],
};
const {
body: { item: createdPolicy },
} = await supertest
.post('/internal/fleet/agent_and_package_policies')
.set('kbn-xsrf', 'xxxx')
.send(requestBody)
.expect(200);
expect(createdPolicy.name).to.eql('Test Agent Policy');
expect(createdPolicy.package_policies[0].id).to.eql(requestBody.package_policies[0].id);
expect(createdPolicy.package_policies[0].policy_id).to.be(createdPolicy.id);
expect(createdPolicy.package_policies[0].policy_ids).to.eql([createdPolicy.id]);
});
it('should delete created agent policy and package policies if create package policy fails', async () => {
const requestBody = {
id: 'test-agent-policy-for-rollback',
name: 'Test Agent Policy',
namespace: 'default',
description: 'Test description',
package_policies: [
{
id: 'test-package-policy-for-rollback-1',
name: 'Test Package Policy',
namespace: 'default',
policy_ids: ['test-agent-policy-for-rollback'],
enabled: true,
inputs: [
{
enabled: true,
streams: [],
type: 'single_input',
},
],
package: {
name: 'filetest',
title: 'For File Tests',
version: '0.1.0',
},
},
{
id: 'test-package-policy-for-rollback-2',
name: 'Test Package Policy 2',
namespace: 'default',
policy_ids: ['test-agent-policy-for-rollback'],
enabled: true,
inputs: [
{
enabled: true,
streams: [],
type: 'single_input',
},
],
package: {
name: 'filetest',
title: 'For File Tests',
version: '0.1.0',
},
},
{
id: 'test-package-policy-for-rollback-3',
name: 'Test Package Policy 3',
namespace: 'default',
policy_ids: ['test-agent-policy-for-rollback'],
enabled: true,
inputs: [
{
enabled: true,
streams: [],
type: 'single_input',
},
],
package: {
name: 'filetest',
title: 'For File Tests',
version: '0.1.0-badversion', // to trigger error
},
},
],
};
const response = await supertest
.post('/internal/fleet/agent_and_package_policies')
.set('kbn-xsrf', 'xxxx')
.send(requestBody);
expect(response.status).to.not.be(200);
expect(response.body.error).to.not.be.empty();
// Verify that the valid created policies were deleted
await supertest
.get(`/api/fleet/package_policies/${requestBody.package_policies[0].id}`)
.set('kbn-xsrf', 'xxxx')
.expect(404);
await supertest
.get(`/api/fleet/package_policies/${requestBody.package_policies[1].id}`)
.set('kbn-xsrf', 'xxxx')
.expect(404);
await supertest
.get(`/api/fleet/agent_policies/${requestBody.id}`)
.set('kbn-xsrf', 'xxxx')
.expect(404);
});
});
});
}