github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 01:38:56 -04:00
Code Issues Projects Releases Packages Wiki Activity

[UII] Support integrations having secrets with multiple values (#216918)

Browse source 
## Summary

Resolves [#205102](https://github.com/elastic/kibana/issues/205102).
This PR makes Fleet support having multiple values for secrets, i.e.
integrations with variables such as:
```yml
  - name: connection_string
    title: Connection String
    type: password
    secret: true
    multi: true
```

When a package policy has a multi-value secret, the variable containing
references to secrets will be saved with `ids: string[]`:
```js
"connection_string": {
  "type": "password",
  "value": {
    "ids": [
      "c9A385UBLd_jDJtMILH5",
      "ddA385UBLd_jDJtMILH5"
    ],
    "isSecretRef": true
  }
}
```

There is no change for secrets with single values, the reference will
still be saved with `id: string`. There is also no change to the
`secret_references` block.

The policy editor will display the multi-value secrets like this when
creating:
<img width="747" alt="image"
src="https://github.com/user-attachments/assets/1c7128b7-3716-43ec-86a8-16778d4cf30e"
/>

And when editing/replacing:
<img width="750" alt="image"
src="https://github.com/user-attachments/assets/296bed1f-d9f4-49af-a810-c23b42d77139"
/>

## Testing
1. Download and upload test package
[azure-1.20.5-next.zip](https://github.com/user-attachments/files/19574682/azure-1.20.5-next.zip),
which modifies `connection_string` to be multi-value secret and updates
associated agent handlebars templates
- You may get an error about integration name not found, I'm not sure
what that error is, but the package will still be uploaded
2. Test adding the above version of Azure package policy with multiple
connection strings
3. Check that the agent yaml compiles correctly
4. Test editing, deleting the policies etc

### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [x] The PR description includes the appropriate Release Notes section,
and the correct `release_note:*` label is applied per the
[guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)
This commit is contained in:
Jen Huang 2025-04-03 10:56:43 -07:00 • committed by GitHub
parent 7d3f672f2e
commit 7158e0201b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 558 additions and 54 deletions
Download patch file Download diff file Expand all files Collapse all files

40
x-pack/platform/plugins/shared/fleet/common/services/validate_package_policy.test.ts
View file

@ -1610,7 +1610,7 @@ describe('Fleet - validatePackagePolicyConfig', () => {
expect(res).toBeNull();
});
it('should accept a secret ref instead of a text value for a secret field', () => {
it('should accept a secret ref id instead of a text value for a secret field', () => {
const res = validatePackagePolicyConfig(
{
value: { isSecretRef: true, id: 'secret1' },
@ -1626,7 +1626,24 @@ describe('Fleet - validatePackagePolicyConfig', () => {
expect(res).toBeNull();
});
it('secret refs should always have an id', () => {
it('should accept secret ref ids instead of a text value for a secret field', () => {
const res = validatePackagePolicyConfig(
{
value: { isSecretRef: true, ids: ['secret1', 'secret2'] },
},
{
name: 'secret_variable',
type: 'text',
multi: true,
secret: true,
},
'secret_variable',
load
);
expect(res).toBeNull();
});
it('secret refs should always have an id or ids', () => {
const res = validatePackagePolicyConfig(
{
value: { isSecretRef: true },
@ -1640,7 +1657,7 @@ describe('Fleet - validatePackagePolicyConfig', () => {
load
);
expect(res).toEqual(['Secret reference is invalid, id must be a string']);
expect(res).toEqual(['Secret reference is invalid, id or ids must be provided']);
});
it('secret ref id should be a string', () => {
const res = validatePackagePolicyConfig(
@ -1658,6 +1675,23 @@ describe('Fleet - validatePackagePolicyConfig', () => {
expect(res).toEqual(['Secret reference is invalid, id must be a string']);
});
it('secret ref ids should all be strings', () => {
const res = validatePackagePolicyConfig(
{
value: { isSecretRef: true, ids: ['someid', 123] },
},
{
name: 'secret_variable',
type: 'text',
multi: true,
secret: true,
},
'secret_variable',
load
);
expect(res).toEqual(['Secret reference is invalid, ids must be an array of strings']);
});
});
describe('Dataset', () => {

35
x-pack/platform/plugins/shared/fleet/common/services/validate_package_policy.ts
View file

@ -377,19 +377,42 @@ export const validatePackagePolicyConfig = (
}
if (varDef.secret === true && parsedValue && parsedValue.isSecretRef === true) {
if (
parsedValue.id === undefined ||
parsedValue.id === '' ||
typeof parsedValue.id !== 'string'
) {
if (!parsedValue.id && (!parsedValue.ids || parsedValue.ids.length === 0)) {
errors.push(
i18n.translate('xpack.fleet.packagePolicyValidation.invalidSecretReference', {
defaultMessage: 'Secret reference is invalid, id or ids must be provided',
})
);
return errors;
}
if (parsedValue.id && parsedValue.ids) {
errors.push(
i18n.translate('xpack.fleet.packagePolicyValidation.invalidSecretReference', {
defaultMessage: 'Secret reference is invalid, id or ids cannot both be provided',
})
);
return errors;
}
if (parsedValue.id && typeof parsedValue.id !== 'string') {
errors.push(
i18n.translate('xpack.fleet.packagePolicyValidation.invalidSecretReference', {
defaultMessage: 'Secret reference is invalid, id must be a string',
})
);
return errors;
}
if (parsedValue.ids && !parsedValue.ids.every((id: string) => typeof id === 'string')) {
errors.push(
i18n.translate('xpack.fleet.packagePolicyValidation.invalidSecretReference', {
defaultMessage: 'Secret reference is invalid, ids must be an array of strings',
})
);
return errors;
}
return null;
}

6
x-pack/platform/plugins/shared/fleet/common/types/models/secret.ts
View file

@ -13,10 +13,10 @@ export interface SecretElasticDoc {
value: string;
}
// this replaces a var value with a reference to a secret
export interface VarSecretReference {
id: string;
export type VarSecretReference = {
isSecretRef: true;
}
} & ({ id: string } | { ids: string[] });
export interface SecretPath {
path: string[];
value: PackagePolicyConfigRecordEntry;

6
x-pack/platform/plugins/shared/fleet/server/services/epm/agent/agent.ts
View file

@ -115,7 +115,11 @@ function buildTemplateVariables(logger: Logger, variables: PackagePolicyConfigRe
varPart[lastKeyPart] = recordEntry.value ? `"${yamlKeyPlaceholder}"` : null;
yamlValues[yamlKeyPlaceholder] = recordEntry.value ? load(recordEntry.value) : null;
} else if (recordEntry.value && recordEntry.value.isSecretRef) {
varPart[lastKeyPart] = toCompiledSecretRef(recordEntry.value.id);
if (recordEntry.value.ids) {
varPart[lastKeyPart] = recordEntry.value.ids.map((id: string) => toCompiledSecretRef(id));
} else {
varPart[lastKeyPart] = toCompiledSecretRef(recordEntry.value.id);
}
} else {
varPart[lastKeyPart] = recordEntry.value;
}

243
x-pack/platform/plugins/shared/fleet/server/services/secrets.test.ts
View file

@ -59,6 +59,7 @@ describe('secrets', () => {
vars: [
{ name: 'pkg-secret-1', type: 'text', secret: true },
{ name: 'pkg-secret-2', type: 'text', secret: true },
{ name: 'pkg-multi-secret', type: 'text', multi: true, secret: true },
],
data_streams: [
{
@ -70,6 +71,7 @@ describe('secrets', () => {
vars: [
{ name: 'stream-secret-1', type: 'text', secret: true },
{ name: 'stream-secret-2', type: 'text', secret: true },
{ name: 'stream-multi-secret', type: 'text', multi: true, secret: true },
],
},
],
@ -96,6 +98,12 @@ describe('secrets', () => {
type: 'text',
secret: true,
},
{
name: 'input-multi-secret',
type: 'text',
multi: true,
secret: true,
},
{ name: 'foo-input3-var-name', type: 'text', multi: true },
],
},
@ -103,6 +111,7 @@ describe('secrets', () => {
},
],
} as unknown as PackageInfo;
it('policy with package level secret vars', () => {
const packagePolicy = {
vars: {
@ -112,6 +121,9 @@ describe('secrets', () => {
'pkg-secret-2': {
value: 'pkg-secret-2-val',
},
'pkg-multi-secret': {
value: ['pkg-multi-secret-val1', 'pkg-multi-secret-val2'],
},
},
inputs: [],
} as unknown as NewPackagePolicy;
@ -129,6 +141,12 @@ describe('secrets', () => {
value: 'pkg-secret-2-val',
},
},
{
path: ['vars', 'pkg-multi-secret'],
value: {
value: ['pkg-multi-secret-val1', 'pkg-multi-secret-val2'],
},
},
]);
});
it('policy with package level secret vars and only one set', () => {
@ -163,6 +181,9 @@ describe('secrets', () => {
'input-secret-2': {
value: 'input-secret-2-val',
},
'input-multi-secret': {
value: ['input-multi-secret-val1', 'input-multi-secret-val2'],
},
},
streams: [],
},
@ -178,6 +199,10 @@ describe('secrets', () => {
path: ['inputs', '0', 'vars', 'input-secret-2'],
value: { value: 'input-secret-2-val' },
},
{
path: ['inputs', '0', 'vars', 'input-multi-secret'],
value: { value: ['input-multi-secret-val1', 'input-multi-secret-val2'] },
},
]);
});
it('stream level secret vars', () => {
@ -199,6 +224,9 @@ describe('secrets', () => {
'stream-secret-2': {
value: 'stream-secret-2-value',
},
'stream-multi-secret': {
value: ['stream-multi-secret-val1', 'stream-multi-secret-val2'],
},
},
},
],
@ -215,6 +243,10 @@ describe('secrets', () => {
path: ['inputs', '0', 'streams', '0', 'vars', 'stream-secret-2'],
value: { value: 'stream-secret-2-value' },
},
{
path: ['inputs', '0', 'streams', '0', 'vars', 'stream-multi-secret'],
value: { value: ['stream-multi-secret-val1', 'stream-multi-secret-val2'] },
},
]);
});
@ -798,6 +830,15 @@ describe('secrets', () => {
},
},
},
{
path: ['somepath4'],
value: {
value: {
isSecretRef: true,
ids: ['secret-4-1', 'secret-4-2'],
},
},
},
];
expect(diffSecretPaths(paths, paths.slice().reverse())).toEqual({
@ -854,7 +895,7 @@ describe('secrets', () => {
noChange: [paths1[0]],
});
});
it('double secret modified', () => {
it('multiple secret modified', () => {
const paths1 = [
{
path: ['somepath1'],
@ -874,6 +915,15 @@ describe('secrets', () => {
},
},
},
{
path: ['somepath3'],
value: {
value: {
isSecretRef: true,
ids: ['secret-3-1', 'secret-3-2'],
},
},
},
];
const paths2 = [
@ -885,6 +935,10 @@ describe('secrets', () => {
path: ['somepath2'],
value: { value: 'newvalue2' },
},
{
path: ['somepath3'],
value: { value: ['newvalue3-1', 'newvalue3-2'] },
},
];
expect(diffSecretPaths(paths1, paths2)).toEqual({
@ -897,6 +951,10 @@ describe('secrets', () => {
path: ['somepath2'],
value: { value: 'newvalue2' },
},
{
path: ['somepath3'],
value: { value: ['newvalue3-1', 'newvalue3-2'] },
},
],
toDelete: [
{
@ -917,6 +975,15 @@ describe('secrets', () => {
},
},
},
{
path: ['somepath3'],
value: {
value: {
isSecretRef: true,
ids: ['secret-3-1', 'secret-3-2'],
},
},
},
],
noChange: [],
});
@ -980,6 +1047,7 @@ describe('secrets', () => {
{ name: 'pkg-secret-1', type: 'text', secret: true, required: true },
{ name: 'pkg-secret-2', type: 'text', secret: true, required: false },
{ name: 'dot-notation.pkg-secret-3', type: 'text', secret: true, required: false },
{ name: 'pkg-multi-secret', type: 'text', secret: true, multi: true },
],
data_streams: [
{
@ -995,6 +1063,12 @@ describe('secrets', () => {
secret: true,
required: false,
},
{
name: 'stream-multi-secret',
type: 'text',
secret: true,
multi: true,
},
],
},
],
@ -1016,6 +1090,12 @@ describe('secrets', () => {
secret: true,
required: false,
},
{
name: 'input-multi-secret',
type: 'text',
secret: true,
multi: true,
},
],
},
],
@ -1132,6 +1212,64 @@ describe('secrets', () => {
).toBe(true);
});
});
describe('when secrets have multiple values', () => {
it('handles multi-value secrets correctly', async () => {
const mockPackagePolicy = {
vars: {
'pkg-multi-secret': {
value: ['multi-secret-val1', 'multi-secret-val2'],
},
},
inputs: [
{
type: 'foo',
vars: {
'input-multi-secret': {
value: ['input-multi-secret-val1', 'input-multi-secret-val2'],
},
},
streams: [
{
data_stream: { type: 'foo', dataset: 'somedataset' },
vars: {
'stream-multi-secret': {
value: ['stream-multi-secret-val1', 'stream-multi-secret-val2'],
},
},
},
],
},
],
} as unknown as NewPackagePolicy;
const result = await extractAndWriteSecrets({
packagePolicy: mockPackagePolicy,
packageInfo: mockIntegrationPackage,
esClient: esClientMock,
});
expect(esClientMock.transport.request).toHaveBeenCalledTimes(6);
expect(result.secretReferences).toHaveLength(6);
expect(result.packagePolicy.vars!['pkg-multi-secret'].value.ids).toHaveLength(2);
expect(result.packagePolicy.vars!['pkg-multi-secret'].value.isSecretRef).toBe(true);
expect(result.packagePolicy.inputs[0].vars!['input-multi-secret'].value.ids).toHaveLength(
2
);
expect(result.packagePolicy.inputs[0].vars!['input-multi-secret'].value.isSecretRef).toBe(
true
);
expect(
result.packagePolicy.inputs[0].streams[0].vars!['stream-multi-secret'].value.ids
).toHaveLength(2);
expect(
result.packagePolicy.inputs[0].streams[0].vars!['stream-multi-secret'].value.isSecretRef
).toBe(true);
});
});
});
describe('extractAndUpdateSecrets', () => {
@ -1158,6 +1296,7 @@ describe('secrets', () => {
{ name: 'pkg-secret-1', type: 'text', secret: true, required: true },
{ name: 'pkg-secret-2', type: 'text', secret: true, required: false },
{ name: 'dot-notation.pkg-secret-3', type: 'text', secret: true, required: false },
{ name: 'pkg-multi-secret', type: 'text', secret: true, multi: true },
],
data_streams: [
{
@ -1173,6 +1312,12 @@ describe('secrets', () => {
secret: true,
required: false,
},
{
name: 'stream-multi-secret',
type: 'text',
secret: true,
multi: true,
},
],
},
],
@ -1194,6 +1339,12 @@ describe('secrets', () => {
secret: true,
required: false,
},
{
name: 'input-multi-secret',
type: 'text',
secret: true,
multi: true,
},
],
},
],
@ -1368,6 +1519,96 @@ describe('secrets', () => {
expect(result.secretsToDelete).toHaveLength(2);
});
});
describe('when secrets have multiple values', () => {
it('handles multi-value secrets correctly', async () => {
const oldPackagePolicy = {
vars: {
'pkg-multi-secret': {
value: { ids: ['id1', 'id2'], isSecretRef: true },
},
},
inputs: [
{
type: 'foo',
vars: {
'input-multi-secret': {
value: { ids: ['id3', 'id4'], isSecretRef: true },
},
},
streams: [
{
data_stream: { type: 'foo', dataset: 'somedataset' },
vars: {
'stream-multi-secret': {
value: { ids: ['id5', 'id6'], isSecretRef: true },
},
},
},
],
},
],
} as unknown as PackagePolicy;
const updatedPackagePolicy = {
vars: {
'pkg-multi-secret': {
value: ['multi-secret-val1', 'multi-secret-val2'],
},
},
inputs: [
{
type: 'foo',
vars: {
'input-multi-secret': {
value: ['input-multi-secret-val1', 'input-multi-secret-val2'],
},
},
streams: [
{
data_stream: { type: 'foo', dataset: 'somedataset' },
vars: {
'stream-multi-secret': {
value: ['stream-multi-secret-val1', 'stream-multi-secret-val2'],
},
},
},
],
},
],
} as unknown as UpdatePackagePolicy;
const result = await extractAndUpdateSecrets({
oldPackagePolicy,
packagePolicyUpdate: updatedPackagePolicy,
packageInfo: mockIntegrationPackage,
esClient: esClientMock,
});
expect(esClientMock.transport.request).toHaveBeenCalledTimes(6);
expect(result.secretReferences).toHaveLength(6);
expect(result.packagePolicyUpdate.vars!['pkg-multi-secret'].value.ids).toHaveLength(2);
expect(result.packagePolicyUpdate.vars!['pkg-multi-secret'].value.isSecretRef).toBe(true);
expect(
result.packagePolicyUpdate.inputs[0].vars!['input-multi-secret'].value.ids
).toHaveLength(2);
expect(
result.packagePolicyUpdate.inputs[0].vars!['input-multi-secret'].value.isSecretRef
).toBe(true);
expect(
result.packagePolicyUpdate.inputs[0].streams[0].vars!['stream-multi-secret'].value.ids
).toHaveLength(2);
expect(
result.packagePolicyUpdate.inputs[0].streams[0].vars!['stream-multi-secret'].value
.isSecretRef
).toBe(true);
expect(result.secretsToDelete).toHaveLength(6);
});
});
});
describe('diffSOSecretPaths', () => {

132
x-pack/platform/plugins/shared/fleet/server/services/secrets.ts
View file

@ -67,23 +67,31 @@ import { checkFleetServerVersionsForSecretsStorage } from './fleet_server';
export async function createSecrets(opts: {
esClient: ElasticsearchClient;
values: string[];
}): Promise<Secret[]> {
values: Array<string | string[]>;
}): Promise<Array<Secret | Secret[]>> {
const { esClient, values } = opts;
const logger = appContextService.getLogger();
const secretsResponse: Secret[] = await Promise.all(
const sendESRequest = (value: string): Promise<Secret> => {
return retryTransientEsErrors(
() =>
esClient.transport.request({
method: 'POST',
path: SECRETS_ENDPOINT_PATH,
body: { value },
}),
{ logger }
);
};
const secretsResponse: Array<Secret | Secret[]> = await Promise.all(
values.map(async (value) => {
try {
return await retryTransientEsErrors(
() =>
esClient.transport.request({
method: 'POST',
path: SECRETS_ENDPOINT_PATH,
body: { value },
}),
{ logger }
);
if (Array.isArray(value)) {
return await Promise.all(value.map(sendESRequest));
} else {
return await sendESRequest(value);
}
} catch (err) {
const msg = `Error creating secrets: ${err}`;
logger.error(msg);
@ -92,7 +100,7 @@ export async function createSecrets(opts: {
})
);
secretsResponse.forEach((item) => {
const writeLog = (item: Secret) => {
auditLoggingService.writeCustomAuditLog({
message: `secret created: ${item.id}`,
event: {
@ -102,6 +110,14 @@ export async function createSecrets(opts: {
outcome: 'success',
},
});
};
secretsResponse.forEach((item) => {
if (Array.isArray(item)) {
item.forEach(writeLog);
} else {
writeLog(item);
}
});
return secretsResponse;
@ -164,10 +180,19 @@ export async function findPackagePoliciesUsingSecrets(opts: {
// create a map of secret_references.id to package policy id
const packagePoliciesBySecretId = packagePolicies.items.reduce((acc, packagePolicy) => {
packagePolicy?.secret_references?.forEach((secretReference) => {
if (!acc[secretReference.id]) {
acc[secretReference.id] = [];
if (Array.isArray(secretReference)) {
secretReference.forEach(({ id }) => {
if (!acc[id]) {
acc[id] = [];
}
acc[id].push(packagePolicy.id);
});
} else {
if (!acc[secretReference.id]) {
acc[secretReference.id] = [];
}
acc[secretReference.id].push(packagePolicy.id);
}
acc[secretReference.id].push(packagePolicy.id);
});
return acc;
}, {} as Record<string, string[]>);
@ -256,7 +281,12 @@ export async function extractAndWriteSecrets(opts: {
return {
packagePolicy: policyWithSecretRefs,
secretReferences: secrets.map(({ id }) => ({ id })),
secretReferences: secrets.reduce((acc: PolicySecretReference[], secret) => {
if (Array.isArray(secret)) {
return [...acc, ...secret.map(({ id }) => ({ id }))];
}
return [...acc, { id: secret.id }];
}, []),
};
}
@ -294,8 +324,18 @@ export async function extractAndUpdateSecrets(opts: {
);
const secretReferences = [
...noChange.map((secretPath) => ({ id: secretPath.value.value.id })),
...createdSecrets.map(({ id }) => ({ id })),
...noChange.reduce((acc: PolicySecretReference[], secretPath) => {
if (secretPath.value.value.ids) {
return [...acc, ...secretPath.value.value.ids.map((id: string) => ({ id }))];
}
return [...acc, { id: secretPath.value.value.id }];
}, []),
...createdSecrets.reduce((acc: PolicySecretReference[], secret) => {
if (Array.isArray(secret)) {
return [...acc, ...secret.map(({ id }) => ({ id }))];
}
return [...acc, { id: secret.id }];
}, []),
];
const secretsToDelete: PolicySecretReference[] = [];
@ -305,7 +345,13 @@ export async function extractAndUpdateSecrets(opts: {
// it may be that secrets were not enabled at the time of creation
// in which case they are just stored as plain text
if (secretPath.value.value?.isSecretRef) {
secretsToDelete.push({ id: secretPath.value.value.id });
if (secretPath.value.value.ids) {
secretPath.value.value.ids.forEach((id: string) => {
secretsToDelete.push({ id });
});
} else {
secretsToDelete.push({ id: secretPath.value.value.id });
}
}
});
@ -325,8 +371,11 @@ function containsSecretVar(vars?: RegistryVarsEntry[]) {
}
// this is how secrets are stored on the package policy
function toVarSecretRef(id: string): VarSecretReference {
return { id, isSecretRef: true };
function toVarSecretRef(secret: Secret | Secret[]): VarSecretReference {
if (Array.isArray(secret)) {
return { ids: secret.map(({ id }) => id), isSecretRef: true };
}
return { id: secret.id, isSecretRef: true };
}
// this is how IDs are inserted into compiled templates
@ -612,7 +661,7 @@ function _getInputSecretVarDefsByPolicyTemplateAndType(packageInfo: PackageInfo)
*/
function getPolicyWithSecretReferences(
secretPaths: SecretPath[],
secrets: Secret[],
secrets: Array<Secret | Secret[]>,
packagePolicy: NewPackagePolicy
) {
const result = JSON.parse(JSON.stringify(packagePolicy));
@ -626,7 +675,7 @@ function getPolicyWithSecretReferences(
const isLast = secretPathComponentIndex === secretPath.path.length - 1;
if (isLast) {
acc[val].value = toVarSecretRef(secrets[secretPathIndex].id);
acc[val].value = toVarSecretRef(secrets[secretPathIndex]);
}
return acc[val];
@ -717,22 +766,30 @@ async function extractAndWriteSOSecrets<T>(opts: {
const secrets = await createSecrets({
esClient,
values: secretPaths.map(({ value }) => value as string),
values: secretPaths.map(({ value }) => value as string | string[]),
});
const objectWithSecretRefs = JSON.parse(JSON.stringify(soObject));
secretPaths.forEach((secretPath, i) => {
const pathWithoutPrefix = secretPath.path.replace('secrets.', '');
const maybeHash = get(secretHashes, pathWithoutPrefix);
const currentSecret = secrets[i];
set(objectWithSecretRefs, secretPath.path, {
id: secrets[i].id,
...(Array.isArray(currentSecret)
? { ids: currentSecret.map(({ id }) => id) }
: { id: currentSecret.id }),
...(typeof maybeHash === 'string' && { hash: maybeHash }),
});
});
return {
soObjectWithSecrets: objectWithSecretRefs,
secretReferences: secrets.map(({ id }) => ({ id })),
secretReferences: secrets.reduce((acc: PolicySecretReference[], secret) => {
if (Array.isArray(secret)) {
return [...acc, ...secret.map(({ id }) => ({ id }))];
}
return [...acc, { id: secret.id }];
}, []),
};
}
@ -769,16 +826,31 @@ async function extractAndUpdateSOSecrets<T>(opts: {
toCreate.forEach((secretPath, i) => {
const pathWithoutPrefix = secretPath.path.replace('secrets.', '');
const maybeHash = get(secretHashes, pathWithoutPrefix);
const currentSecret = createdSecrets[i];
set(soObjectWithSecretRefs, secretPath.path, {
id: createdSecrets[i].id,
...(Array.isArray(currentSecret)
? { ids: currentSecret.map(({ id }) => id) }
: { id: currentSecret.id }),
...(typeof maybeHash === 'string' && { hash: maybeHash }),
});
});
const secretReferences = [
...noChange.map((secretPath) => ({ id: (secretPath.value as { id: string }).id })),
...createdSecrets.map(({ id }) => ({ id })),
...noChange.reduce((acc: PolicySecretReference[], secretPath) => {
const currentValue = secretPath.value as { id: string } | { ids: string[] };
if ('ids' in currentValue) {
return [...acc, ...currentValue.ids.map((id: string) => ({ id }))];
} else {
return [...acc, { id: currentValue.id }];
}
}, []),
...createdSecrets.reduce((acc: PolicySecretReference[], secret) => {
if (Array.isArray(secret)) {
return [...acc, ...secret.map(({ id }) => ({ id }))];
}
return [...acc, { id: secret.id }];
}, []),
];
return {

6
x-pack/test/fleet_api_integration/apis/fixtures/test_packages/secrets/1.0.0/data_stream/log/agent/input/input.yml.hbs
View file

@ -1,4 +1,10 @@
package_var_secret: {{package_var_secret}}
{{#if package_var_multi_secret}}
package_var_multi_secret:
{{#each package_var_multi_secret}}
- {{this}}
{{/each}}
{{/if}}
package_var_non_secret: {{package_var_non_secret}}
input_var_secret: {{input_var_secret}}
input_var_non_secret: {{input_var_non_secret}}

6
x-pack/test/fleet_api_integration/apis/fixtures/test_packages/secrets/1.0.0/data_stream/log/agent/stream/stream.yml.hbs
View file

@ -1,5 +1,11 @@
config.version: "2"
package_var_secret: {{package_var_secret}}
{{#if package_var_multi_secret}}
package_var_multi_secret:
{{#each package_var_multi_secret}}
- {{this}}
{{/each}}
{{/if}}
package_var_non_secret: {{package_var_non_secret}}
input_var_secret: {{input_var_secret}}
input_var_non_secret: {{input_var_non_secret}}

7
x-pack/test/fleet_api_integration/apis/fixtures/test_packages/secrets/1.0.0/manifest.yml
View file

@ -32,6 +32,13 @@ vars:
required: true
show_user: true
secret: true
- name: package_var_multi_secret
type: password
title: Package Var Multi Secret
multi: true
required: false
show_user: true
secret: true
- name: package_var_non_secret
type: text
title: Package Var Non Secret

131
x-pack/test/fleet_api_integration/apis/policy_secrets.ts
View file

@ -345,6 +345,7 @@ export default function (providerContext: FtrProviderContext) {
vars: {
package_var_secret: 'package_secret_val',
package_var_non_secret: 'package_non_secret_val',
package_var_multi_secret: ['package_multi_secret_val_1', 'package_multi_secret_val_2'],
},
package: {
name: 'secrets',
@ -404,9 +405,12 @@ export default function (providerContext: FtrProviderContext) {
it('should correctly create the policy with secrets', async () => {
const packageVarId = packagePolicyWithSecrets.vars.package_var_secret.value.id;
expect(packageVarId).to.be.an('string');
const packageVarMultiIds = packagePolicyWithSecrets.vars.package_var_multi_secret.value.ids;
expect(packageVarMultiIds).to.be.an('array');
expect(packageVarMultiIds.length).to.eql(2);
const inputVarId = packagePolicyWithSecrets.inputs[0].vars.input_var_secret.value.id;
expect(inputVarId).to.be.an('string');
@ -417,6 +421,8 @@ export default function (providerContext: FtrProviderContext) {
expect(
arrayIdsEqual(packagePolicyWithSecrets.secret_references, [
{ id: packageVarId },
{ id: packageVarMultiIds[0] },
{ id: packageVarMultiIds[1] },
{ id: streamVarId },
{ id: inputVarId },
])
@ -425,6 +431,10 @@ export default function (providerContext: FtrProviderContext) {
const expectedCompiledStream = {
'config.version': '2',
package_var_secret: secretVar(packageVarId),
package_var_multi_secret: [
secretVar(packageVarMultiIds[0]),
secretVar(packageVarMultiIds[1]),
],
package_var_non_secret: 'package_non_secret_val',
input_var_secret: secretVar(inputVarId),
input_var_non_secret: 'input_non_secret_val',
@ -438,6 +448,10 @@ export default function (providerContext: FtrProviderContext) {
const expectedCompiledInput = {
package_var_secret: secretVar(packageVarId),
package_var_multi_secret: [
secretVar(packageVarMultiIds[0]),
secretVar(packageVarMultiIds[1]),
],
package_var_non_secret: 'package_non_secret_val',
input_var_secret: secretVar(inputVarId),
input_var_non_secret: 'input_non_secret_val',
@ -446,6 +460,9 @@ export default function (providerContext: FtrProviderContext) {
expect(packagePolicyWithSecrets.inputs[0].compiled_input).to.eql(expectedCompiledInput);
expect(packagePolicyWithSecrets.vars.package_var_secret.value.isSecretRef).to.eql(true);
expect(packagePolicyWithSecrets.vars.package_var_multi_secret.value.isSecretRef).to.eql(
true
);
expect(packagePolicyWithSecrets.inputs[0].vars.input_var_secret.value.isSecretRef).to.eql(
true
);
@ -458,12 +475,17 @@ export default function (providerContext: FtrProviderContext) {
const packagePolicy = await getPackagePolicyById(packagePolicyWithSecrets.id);
const packageVarId = packagePolicy.vars.package_var_secret.value.id;
const packageVarMultiIds = packagePolicy.vars.package_var_multi_secret.value.ids;
const inputVarId = packagePolicy.inputs[0].vars.input_var_secret.value.id;
const streamVarId = packagePolicy.inputs[0].streams[0].vars.stream_var_secret.value.id;
const expectedCompiledStream = {
'config.version': '2',
package_var_secret: secretVar(packageVarId),
package_var_multi_secret: [
secretVar(packageVarMultiIds[0]),
secretVar(packageVarMultiIds[1]),
],
package_var_non_secret: 'package_non_secret_val',
input_var_secret: secretVar(inputVarId),
input_var_non_secret: 'input_non_secret_val',
@ -473,6 +495,10 @@ export default function (providerContext: FtrProviderContext) {
const expectedCompiledInput = {
package_var_secret: secretVar(packageVarId),
package_var_multi_secret: [
secretVar(packageVarMultiIds[0]),
secretVar(packageVarMultiIds[1]),
],
package_var_non_secret: 'package_non_secret_val',
input_var_secret: secretVar(inputVarId),
input_var_non_secret: 'input_non_secret_val',
@ -481,6 +507,8 @@ export default function (providerContext: FtrProviderContext) {
expect(
arrayIdsEqual(packagePolicy.secret_references, [
{ id: packageVarId },
{ id: packageVarMultiIds[0] },
{ id: packageVarMultiIds[1] },
{ id: streamVarId },
{ id: inputVarId },
])
@ -490,6 +518,8 @@ export default function (providerContext: FtrProviderContext) {
expect(packagePolicy.inputs[0].compiled_input).to.eql(expectedCompiledInput);
expect(packagePolicy.vars.package_var_secret.value.isSecretRef).to.eql(true);
expect(packagePolicy.vars.package_var_secret.value.id).eql(packageVarId);
expect(packagePolicy.vars.package_var_multi_secret.value.isSecretRef).to.eql(true);
expect(packagePolicy.vars.package_var_multi_secret.value.ids).to.eql(packageVarMultiIds);
expect(packagePolicy.inputs[0].vars.input_var_secret.value.isSecretRef).to.eql(true);
expect(packagePolicy.inputs[0].vars.input_var_secret.value.id).eql(inputVarId);
expect(packagePolicy.inputs[0].streams[0].vars.stream_var_secret.value.isSecretRef).to.eql(
@ -500,19 +530,27 @@ export default function (providerContext: FtrProviderContext) {
it('should have correctly created the secrets', async () => {
const packageVarId = packagePolicyWithSecrets.vars.package_var_secret.value.id;
const packageVarMultiIds = packagePolicyWithSecrets.vars.package_var_multi_secret.value.ids;
const inputVarId = packagePolicyWithSecrets.inputs[0].vars.input_var_secret.value.id;
const streamVarId =
packagePolicyWithSecrets.inputs[0].streams[0].vars.stream_var_secret.value.id;
const searchRes = await getSecrets([packageVarId, inputVarId, streamVarId]);
const searchRes = await getSecrets([
packageVarId,
...packageVarMultiIds,
inputVarId,
streamVarId,
]);
expect(searchRes.hits.hits.length).to.eql(3);
expect(searchRes.hits.hits.length).to.eql(5);
const secretValuesById = searchRes.hits.hits.reduce((acc: any, secret: any) => {
acc[secret._id] = secret._source.value;
return acc;
}, {});
expect(secretValuesById[packageVarId]).to.eql('package_secret_val');
expect(secretValuesById[packageVarMultiIds[0]]).to.eql('package_multi_secret_val_1');
expect(secretValuesById[packageVarMultiIds[1]]).to.eql('package_multi_secret_val_2');
expect(secretValuesById[inputVarId]).to.eql('input_secret_val');
expect(secretValuesById[streamVarId]).to.eql('stream_secret_val');
});
@ -521,6 +559,7 @@ export default function (providerContext: FtrProviderContext) {
const { data: policyDoc } = await getLatestPolicyRevision(testAgentPolicy.id);
const packageVarId = packagePolicyWithSecrets.vars.package_var_secret.value.id;
const packageVarMultiIds = packagePolicyWithSecrets.vars.package_var_multi_secret.value.ids;
const inputVarId = packagePolicyWithSecrets.inputs[0].vars.input_var_secret.value.id;
const streamVarId =
packagePolicyWithSecrets.inputs[0].streams[0].vars.stream_var_secret.value.id;
@ -530,6 +569,12 @@ export default function (providerContext: FtrProviderContext) {
{
id: packageVarId,
},
{
id: packageVarMultiIds[0],
},
{
id: packageVarMultiIds[1],
},
{
id: inputVarId,
},
@ -540,6 +585,10 @@ export default function (providerContext: FtrProviderContext) {
).to.eql(true);
expect(policyDoc.inputs[0].package_var_secret).to.eql(secretVar(packageVarId));
expect(policyDoc.inputs[0].package_var_multi_secret).to.eql([
secretVar(packageVarMultiIds[0]),
secretVar(packageVarMultiIds[1]),
]);
expect(policyDoc.inputs[0].input_var_secret).to.eql(secretVar(inputVarId));
expect(policyDoc.inputs[0].streams![0].package_var_secret).to.eql(secretVar(packageVarId));
expect(policyDoc.inputs[0].streams![0].input_var_secret).to.eql(secretVar(inputVarId));
@ -552,6 +601,7 @@ export default function (providerContext: FtrProviderContext) {
const input = agentPolicy.inputs[0];
const packageVarId = packagePolicyWithSecrets.vars.package_var_secret.value.id;
const packageVarMultiIds = packagePolicyWithSecrets.vars.package_var_multi_secret.value.ids;
const inputVarId = packagePolicyWithSecrets.inputs[0].vars.input_var_secret.value.id;
const streamVarId =
packagePolicyWithSecrets.inputs[0].streams[0].vars.stream_var_secret.value.id;
@ -561,6 +611,12 @@ export default function (providerContext: FtrProviderContext) {
{
id: packageVarId,
},
{
id: packageVarMultiIds[0],
},
{
id: packageVarMultiIds[1],
},
{
id: inputVarId,
},
@ -571,6 +627,10 @@ export default function (providerContext: FtrProviderContext) {
).to.eql(true);
expect(input.package_var_secret).to.eql(secretVar(packageVarId));
expect(input.package_var_multi_secret).to.eql([
secretVar(packageVarMultiIds[0]),
secretVar(packageVarMultiIds[1]),
]);
expect(input.input_var_secret).to.eql(secretVar(inputVarId));
expect(input.streams[0].package_var_secret).to.eql(secretVar(packageVarId));
expect(input.streams[0].input_var_secret).to.eql(secretVar(inputVarId));
@ -599,6 +659,10 @@ export default function (providerContext: FtrProviderContext) {
const updatedPolicy = createdPolicyToUpdatePolicy(packagePolicyWithSecrets);
updatedPolicy.vars.package_var_secret.value = 'new_package_secret_val';
updatedPolicy.vars.package_var_multi_secret.value = [
'new_package_multi_secret_val_1',
'new_package_multi_secret_val_2',
];
const updateRes = await supertest
.put(`/api/fleet/package_policies/${packagePolicyWithSecrets.id}`)
@ -617,7 +681,11 @@ export default function (providerContext: FtrProviderContext) {
it('should allow secret values to be updated (single policy update API)', async () => {
const updatedPackageVarId = updatedPackagePolicy.vars.package_var_secret.value.id;
const updatedPackageVarMultiIds =
updatedPackagePolicy.vars.package_var_multi_secret.value.ids;
expect(updatedPackageVarId).to.be.a('string');
expect(updatedPackageVarMultiIds).to.be.an('array');
expect(updatedPackageVarMultiIds.length).to.eql(2);
const inputVarId = packagePolicyWithSecrets.inputs[0].vars.input_var_secret.value.id;
const streamVarId =
@ -626,6 +694,8 @@ export default function (providerContext: FtrProviderContext) {
expect(
arrayIdsEqual(updatedPackagePolicy.secret_references, [
{ id: updatedPackageVarId },
{ id: updatedPackageVarMultiIds[0] },
{ id: updatedPackageVarMultiIds[1] },
{ id: streamVarId },
{ id: inputVarId },
])
@ -634,6 +704,10 @@ export default function (providerContext: FtrProviderContext) {
expect(updatedPackagePolicy.inputs[0].streams[0].compiled_stream).to.eql({
'config.version': 2,
package_var_secret: secretVar(updatedPackageVarId),
package_var_multi_secret: [
secretVar(updatedPackageVarMultiIds[0]),
secretVar(updatedPackageVarMultiIds[1]),
],
package_var_non_secret: 'package_non_secret_val',
input_var_secret: secretVar(inputVarId),
input_var_non_secret: 'input_non_secret_val',
@ -643,6 +717,10 @@ export default function (providerContext: FtrProviderContext) {
expect(updatedPackagePolicy.inputs[0].compiled_input).to.eql({
package_var_secret: secretVar(updatedPackageVarId),
package_var_multi_secret: [
secretVar(updatedPackageVarMultiIds[0]),
secretVar(updatedPackageVarMultiIds[1]),
],
package_var_non_secret: 'package_non_secret_val',
input_var_secret: secretVar(inputVarId),
input_var_non_secret: 'input_non_secret_val',
@ -650,6 +728,10 @@ export default function (providerContext: FtrProviderContext) {
expect(updatedPackagePolicy.vars.package_var_secret.value.isSecretRef).to.eql(true);
expect(updatedPackagePolicy.vars.package_var_secret.value.id).eql(updatedPackageVarId);
expect(updatedPackagePolicy.vars.package_var_multi_secret.value.isSecretRef).to.eql(true);
expect(updatedPackagePolicy.vars.package_var_multi_secret.value.ids).to.eql(
updatedPackageVarMultiIds
);
expect(updatedPackagePolicy.inputs[0].vars.input_var_secret.value.isSecretRef).to.eql(true);
expect(updatedPackagePolicy.inputs[0].vars.input_var_secret.value.id).eql(inputVarId);
expect(
@ -662,7 +744,7 @@ export default function (providerContext: FtrProviderContext) {
it('should have correctly deleted unused secrets after update', async () => {
const searchRes = await getSecrets();
expect(searchRes.hits.hits.length).to.eql(3); // should have created 1 and deleted 1 doc
expect(searchRes.hits.hits.length).to.eql(5); // should have created 2 and deleted 2 docs
const secretValuesById = searchRes.hits.hits.reduce((acc: any, secret: any) => {
acc[secret._id] = secret._source.value;
@ -670,6 +752,8 @@ export default function (providerContext: FtrProviderContext) {
}, {});
const updatedPackageVarId = updatedPackagePolicy.vars.package_var_secret.value.id;
const updatedPackageVarMultiIds =
updatedPackagePolicy.vars.package_var_multi_secret.value.ids;
expect(updatedPackageVarId).to.be.a('string');
const inputVarId = packagePolicyWithSecrets.inputs[0].vars.input_var_secret.value.id;
@ -677,6 +761,12 @@ export default function (providerContext: FtrProviderContext) {
packagePolicyWithSecrets.inputs[0].streams[0].vars.stream_var_secret.value.id;
expect(secretValuesById[updatedPackageVarId]).to.eql('new_package_secret_val');
expect(secretValuesById[updatedPackageVarMultiIds[0]]).to.eql(
'new_package_multi_secret_val_1'
);
expect(secretValuesById[updatedPackageVarMultiIds[1]]).to.eql(
'new_package_multi_secret_val_2'
);
expect(secretValuesById[inputVarId]).to.eql('input_secret_val');
expect(secretValuesById[streamVarId]).to.eql('stream_secret_val');
});
@ -727,6 +817,7 @@ export default function (providerContext: FtrProviderContext) {
it('should not duplicate secrets after duplicating agent policy', async () => {
const packageVarId = duplicatedPackagePolicy.vars.package_var_secret.value.id;
const packageVarMultiIds = duplicatedPackagePolicy.vars.package_var_multi_secret.value.ids;
const inputVarId = duplicatedPackagePolicy.inputs[0].vars.input_var_secret.value.id;
const streamVarId =
duplicatedPackagePolicy.inputs[0].streams[0].vars.stream_var_secret.value.id;
@ -736,6 +827,12 @@ export default function (providerContext: FtrProviderContext) {
{
id: packageVarId,
},
{
id: packageVarMultiIds[0],
},
{
id: packageVarMultiIds[1],
},
{
id: inputVarId,
},
@ -746,6 +843,10 @@ export default function (providerContext: FtrProviderContext) {
).to.eql(true);
expect(policyDoc.inputs[0].package_var_secret).to.eql(secretVar(packageVarId));
expect(policyDoc.inputs[0].package_var_multi_secret).to.eql([
secretVar(packageVarMultiIds[0]),
secretVar(packageVarMultiIds[1]),
]);
expect(policyDoc.inputs[0].input_var_secret).to.eql(secretVar(inputVarId));
expect(policyDoc.inputs[0].streams![0].package_var_secret).to.eql(secretVar(packageVarId));
expect(policyDoc.inputs[0].streams![0].input_var_secret).to.eql(secretVar(inputVarId));
@ -753,7 +854,7 @@ export default function (providerContext: FtrProviderContext) {
const searchRes = await getSecrets();
expect(searchRes.hits.hits.length).to.eql(3);
expect(searchRes.hits.hits.length).to.eql(5);
const secretValuesById = searchRes.hits.hits.reduce((acc: any, secret: any) => {
acc[secret._id] = secret._source.value;
@ -761,6 +862,8 @@ export default function (providerContext: FtrProviderContext) {
}, {});
expect(secretValuesById[packageVarId]).to.eql('package_secret_val');
expect(secretValuesById[packageVarMultiIds[0]]).to.eql('package_multi_secret_val_1');
expect(secretValuesById[packageVarMultiIds[1]]).to.eql('package_multi_secret_val_2');
expect(secretValuesById[inputVarId]).to.eql('input_secret_val');
expect(secretValuesById[streamVarId]).to.eql('stream_secret_val');
});
@ -770,6 +873,10 @@ export default function (providerContext: FtrProviderContext) {
delete updatedPolicy.name;
updatedPolicy.vars.package_var_secret.value = 'new_package_secret_val_2';
updatedPolicy.vars.package_var_multi_secret.value = [
'new_package_multi_secret_val_3',
'new_package_multi_secret_val_4',
];
const updateRes = await supertest
.put(`/api/fleet/package_policies/${duplicatedPackagePolicy.id}`)
@ -779,15 +886,19 @@ export default function (providerContext: FtrProviderContext) {
const updatedPackagePolicy = updateRes.body.item;
const updatedPackageVarId = updatedPackagePolicy.vars.package_var_secret.value.id;
const updatedPackageVarMultiIds =
updatedPackagePolicy.vars.package_var_multi_secret.value.ids;
const packageVarSecretIds = [
const packageSecretIds = [
packagePolicyWithSecrets.vars.package_var_secret.value.id,
...packagePolicyWithSecrets.vars.package_var_multi_secret.value.ids,
updatedPackageVarId,
...updatedPackageVarMultiIds,
];
const searchRes = await getSecrets(packageVarSecretIds);
const searchRes = await getSecrets(packageSecretIds);
expect(searchRes.hits.hits.length).to.eql(2);
expect(searchRes.hits.hits.length).to.eql(6);
});
it('should not delete used secrets on delete of duplicated package policy', async () => {
@ -801,8 +912,8 @@ export default function (providerContext: FtrProviderContext) {
const searchRes = await getSecrets();
// should have deleted new_package_secret_val_2
expect(searchRes.hits.hits.length).to.eql(3);
// should have deleted new_package_secret_val_2 and new_package_multi_secret_val_3/4
expect(searchRes.hits.hits.length).to.eql(5);
});
});