[Fleet] making service_token an output secret (#171875)

## Summary

Related to https://github.com/elastic/kibana/issues/104986

Making remote ES output's service_token a secret.

fleet-server change here:
https://github.com/elastic/fleet-server/pull/3051#discussion_r1406183654

Steps to verify:
- Enable remote ES output and output secrets in `kibana.dev.yml`
locally:
 ```
xpack.fleet.enableExperimental: ['remoteESOutput',
'outputSecretsStorage']
```
- Start es, kibana, fleet-server locally and start a second es locally
 - see detailed steps here: https://github.com/elastic/fleet-server/pull/3051
- Create a remote ES output, verify that the service_token is stored as a secret reference
```
GET .kibana_ingest/_search?q=type:ingest-outputs
```
- Verify that the enrolled agent sends data to the remote ES successfully

<img width="561" alt="image" src="122d9800-a2ec-47f8-97a7-acf64b87172a">
<img width="549" alt="image" src="e1751bdd-5aaf-4f68-9f92-7076b306cdfe">



### Checklist

- [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios
This commit is contained in:
Julia Bardi 2023-11-28 15:20:36 +01:00 committed by GitHub
parent 503123105f
commit 517c815c48
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
21 changed files with 348 additions and 47 deletions

View file

@ -1844,6 +1844,14 @@
}
}
}
},
"service_token": {
"dynamic": false,
"properties": {
"id": {
"type": "keyword"
}
}
}
}
}

View file

@ -106,7 +106,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
"infrastructure-ui-source": "113182d6895764378dfe7fa9fa027244f3a457c4",
"ingest-agent-policies": "7633e578f60c074f8267bc50ec4763845e431437",
"ingest-download-sources": "279a68147e62e4d8858c09ad1cf03bd5551ce58d",
"ingest-outputs": "8546f1123ec30dcbd6f238f72729c5f1656a4d9b",
"ingest-outputs": "4dd3cb38a91c848df95336a24a5abde2c8560fd1",
"ingest-package-policies": "f4c2767e852b700a8b82678925b86bac08958b43",
"ingest_manager_settings": "64955ef1b7a9ffa894d4bb9cf863b5602bfa6885",
"inventory-view": "b8683c8e352a286b4aca1ab21003115a4800af83",

View file

@ -8257,6 +8257,50 @@
"type"
]
},
"output_create_request_remote_elasticsearch": {
"title": "remote_elasticsearch",
"type": "object",
"properties": {
"id": {
"type": "string"
},
"is_default": {
"type": "boolean"
},
"is_default_monitoring": {
"type": "boolean"
},
"name": {
"type": "string"
},
"type": {
"type": "string",
"enum": [
"remote_elasticsearch"
]
},
"hosts": {
"type": "array",
"items": {
"type": "string"
}
},
"service_token": {
"type": "string"
},
"secrets": {
"type": "object",
"properties": {
"service_token": {
"type": "string"
}
}
}
},
"required": [
"name"
]
},
"output_create_request": {
"title": "Output",
"oneOf": [
@ -8268,6 +8312,9 @@
},
{
"$ref": "#/components/schemas/output_create_request_logstash"
},
{
"$ref": "#/components/schemas/output_create_request_remote_elasticsearch"
}
],
"discriminator": {
@ -8275,7 +8322,8 @@
"mapping": {
"elasticsearch": "#/components/schemas/output_create_request_elasticsearch",
"kafka": "#/components/schemas/output_create_request_kafka",
"logstash": "#/components/schemas/output_create_request_logstash"
"logstash": "#/components/schemas/output_create_request_logstash",
"remote_elasticsearch": "#/components/schemas/output_create_request_remote_elasticsearch"
}
}
},

View file

@ -5331,18 +5331,49 @@ components:
- name
- hosts
- type
output_create_request_remote_elasticsearch:
title: remote_elasticsearch
type: object
properties:
id:
type: string
is_default:
type: boolean
is_default_monitoring:
type: boolean
name:
type: string
type:
type: string
enum:
- remote_elasticsearch
hosts:
type: array
items:
type: string
service_token:
type: string
secrets:
type: object
properties:
service_token:
type: string
required:
- name
output_create_request:
title: Output
oneOf:
- $ref: '#/components/schemas/output_create_request_elasticsearch'
- $ref: '#/components/schemas/output_create_request_kafka'
- $ref: '#/components/schemas/output_create_request_logstash'
- $ref: '#/components/schemas/output_create_request_remote_elasticsearch'
discriminator:
propertyName: type
mapping:
elasticsearch: '#/components/schemas/output_create_request_elasticsearch'
kafka: '#/components/schemas/output_create_request_kafka'
logstash: '#/components/schemas/output_create_request_logstash'
remote_elasticsearch: '#/components/schemas/output_create_request_remote_elasticsearch'
output_update_request_elasticsearch:
title: elasticsearch
type: object

View file

@ -3,9 +3,11 @@ oneOf:
- $ref: './output_create_request_elasticsearch.yaml'
- $ref: './output_create_request_kafka.yaml'
- $ref: './output_create_request_logstash.yaml'
- $ref: './output_create_request_remote_elasticsearch.yaml'
discriminator:
propertyName: type
mapping:
elasticsearch: './output_create_request_elasticsearch.yaml'
kafka: './output_create_request_kafka.yaml'
logstash: './output_create_request_logstash.yaml'
remote_elasticsearch: './output_create_request_remote_elasticsearch.yaml'

View file

@ -0,0 +1,27 @@
title: remote_elasticsearch
type: object
properties:
id:
type: string
is_default:
type: boolean
is_default_monitoring:
type: boolean
name:
type: string
type:
type: string
enum: ['remote_elasticsearch']
hosts:
type: array
items:
type: string
service_token:
type: string
secrets:
type: object
properties:
service_token:
type: string
required:
- name

View file

@ -43,15 +43,7 @@ interface NewBaseOutput {
proxy_id?: string | null;
shipper?: ShipperOutput | null;
allow_edit?: string[];
secrets?: {
ssl?: {
key?:
| string
| {
id: string;
};
};
};
secrets?: {};
}
export interface NewElasticsearchOutput extends NewBaseOutput {
@ -61,10 +53,26 @@ export interface NewElasticsearchOutput extends NewBaseOutput {
export interface NewRemoteElasticsearchOutput extends NewBaseOutput {
type: OutputType['RemoteElasticsearch'];
service_token?: string;
secrets?: {
service_token?:
| string
| {
id: string;
};
};
}
export interface NewLogstashOutput extends NewBaseOutput {
type: OutputType['Logstash'];
secrets?: {
ssl?: {
key?:
| string
| {
id: string;
};
};
};
}
export type NewOutput =

View file

@ -191,7 +191,9 @@ describe('EditOutputFlyout', () => {
});
it('should render the flyout if the output provided is a remote ES output', async () => {
jest.spyOn(ExperimentalFeaturesService, 'get').mockReturnValue({ remoteESOutput: true });
jest
.spyOn(ExperimentalFeaturesService, 'get')
.mockReturnValue({ remoteESOutput: true, outputSecretsStorage: true });
const { utils } = renderFlyout({
type: 'remote_elasticsearch',
name: 'remote es output',
@ -208,6 +210,8 @@ describe('EditOutputFlyout', () => {
expect(utils.queryByTestId('settingsOutputsFlyout.typeInput')?.textContent).toContain(
'Remote Elasticsearch'
);
expect(utils.queryByTestId('serviceTokenSecretInput')).not.toBeNull();
});
it('should not display remote ES output in type lists if serverless', async () => {

View file

@ -272,7 +272,13 @@ export const EditOutputFlyout: React.FunctionComponent<EditOutputFlyoutProps> =
const renderRemoteElasticsearchSection = () => {
if (isRemoteESOutputEnabled) {
return <OutputFormRemoteEsSection inputs={inputs} />;
return (
<OutputFormRemoteEsSection
inputs={inputs}
useSecretsStorage={useSecretsStorage}
onUsePlainText={onUsePlainText}
/>
);
}
return null;
};

View file

@ -13,13 +13,16 @@ import { i18n } from '@kbn/i18n';
import { MultiRowInput } from '../multi_row_input';
import type { OutputFormInputsType } from './use_output_form';
import { SecretFormRow } from './output_form_secret_form_row';
interface Props {
inputs: OutputFormInputsType;
useSecretsStorage: boolean;
onUsePlainText: () => void;
}
export const OutputFormRemoteEsSection: React.FunctionComponent<Props> = (props) => {
const { inputs } = props;
const { inputs, useSecretsStorage, onUsePlainText } = props;
return (
<>
@ -38,27 +41,50 @@ export const OutputFormRemoteEsSection: React.FunctionComponent<Props> = (props)
isUrl
/>
<EuiSpacer size="m" />
<EuiFormRow
fullWidth
label={
<FormattedMessage
id="xpack.fleet.settings.editOutputFlyout.serviceTokenLabel"
defaultMessage="Service Token"
/>
}
{...inputs.serviceTokenInput.formRowProps}
>
<EuiFieldText
{inputs.serviceTokenInput.value || !useSecretsStorage ? (
<EuiFormRow
fullWidth
{...inputs.serviceTokenInput.props}
placeholder={i18n.translate(
'xpack.fleet.settings.editOutputFlyout.remoteESHostPlaceholder',
{
defaultMessage: 'Specify service token',
}
)}
/>
</EuiFormRow>
label={
<FormattedMessage
id="xpack.fleet.settings.editOutputFlyout.serviceTokenLabel"
defaultMessage="Service Token"
/>
}
{...inputs.serviceTokenInput.formRowProps}
>
<EuiFieldText
fullWidth
{...inputs.serviceTokenInput.props}
placeholder={i18n.translate(
'xpack.fleet.settings.editOutputFlyout.remoteESHostPlaceholder',
{
defaultMessage: 'Specify service token',
}
)}
/>
</EuiFormRow>
) : (
<SecretFormRow
fullWidth
title={i18n.translate('xpack.fleet.settings.editOutputFlyout.serviceTokenLabel', {
defaultMessage: 'Service Token',
})}
{...inputs.serviceTokenSecretInput.formRowProps}
onUsePlainText={onUsePlainText}
>
<EuiFieldText
data-test-subj="serviceTokenSecretInput"
fullWidth
{...inputs.serviceTokenSecretInput.props}
placeholder={i18n.translate(
'xpack.fleet.settings.editOutputFlyout.remoteESHostPlaceholder',
{
defaultMessage: 'Specify service token',
}
)}
/>
</SecretFormRow>
)}
<EuiSpacer size="m" />
<EuiCallOut
title={

View file

@ -270,6 +270,8 @@ export function validateServiceToken(value: string) {
}
}
export const validateServiceTokenSecret = toSecretValidator(validateServiceToken);
export function validateSSLCertificate(value: string) {
if (!value || value === '') {
return [

View file

@ -55,6 +55,7 @@ import {
validateYamlConfig,
validateCATrustedFingerPrint,
validateServiceToken,
validateServiceTokenSecret,
validateSSLCertificate,
validateSSLKey,
validateSSLKeySecret,
@ -88,6 +89,7 @@ export interface OutputFormInputsType {
defaultMonitoringOutputInput: ReturnType<typeof useSwitchInput>;
caTrustedFingerprintInput: ReturnType<typeof useInput>;
serviceTokenInput: ReturnType<typeof useInput>;
serviceTokenSecretInput: ReturnType<typeof useSecretInput>;
sslCertificateInput: ReturnType<typeof useInput>;
sslKeyInput: ReturnType<typeof useInput>;
sslKeySecretInput: ReturnType<typeof useSecretInput>;
@ -215,6 +217,12 @@ export function useOutputForm(onSucess: () => void, output?: Output) {
validateServiceToken,
isDisabled('service_token')
);
const serviceTokenSecretInput = useSecretInput(
(output as NewRemoteElasticsearchOutput)?.secrets?.service_token ?? '',
validateServiceTokenSecret,
isDisabled('service_token')
);
/*
Shipper feature flag - currently depends on the content of the yaml
# Enables the shipper:
@ -293,7 +301,7 @@ export function useOutputForm(onSucess: () => void, output?: Output) {
const sslKeyInput = useInput(output?.ssl?.key ?? '', validateSSLKey, isSSLEditable);
const sslKeySecretInput = useSecretInput(
output?.secrets?.ssl?.key,
(output as NewLogstashOutput)?.secrets?.ssl?.key,
validateSSLKeySecret,
isSSLEditable
);
@ -503,6 +511,7 @@ export function useOutputForm(onSucess: () => void, output?: Output) {
defaultMonitoringOutputInput,
caTrustedFingerprintInput,
serviceTokenInput,
serviceTokenSecretInput,
sslCertificateInput,
sslKeyInput,
sslKeySecretInput,
@ -562,6 +571,7 @@ export function useOutputForm(onSucess: () => void, output?: Output) {
const additionalYamlConfigValid = additionalYamlConfigInput.validate();
const caTrustedFingerprintValid = caTrustedFingerprintInput.validate();
const serviceTokenValid = serviceTokenInput.validate();
const serviceTokenSecretValid = serviceTokenSecretInput.validate();
const sslCertificateValid = sslCertificateInput.validate();
const sslKeyValid = sslKeyInput.validate();
const sslKeySecretValid = sslKeySecretInput.validate();
@ -607,7 +617,11 @@ export function useOutputForm(onSucess: () => void, output?: Output) {
}
if (isRemoteElasticsearch) {
return (
elasticsearchUrlsValid && additionalYamlConfigValid && nameInputValid && serviceTokenValid
elasticsearchUrlsValid &&
additionalYamlConfigValid &&
nameInputValid &&
((serviceTokenInput.value && serviceTokenValid) ||
(serviceTokenSecretInput.value && serviceTokenSecretValid))
);
} else {
// validate ES
@ -637,6 +651,7 @@ export function useOutputForm(onSucess: () => void, output?: Output) {
additionalYamlConfigInput,
caTrustedFingerprintInput,
serviceTokenInput,
serviceTokenSecretInput,
sslCertificateInput,
sslKeyInput,
sslKeySecretInput,
@ -852,6 +867,12 @@ export function useOutputForm(onSucess: () => void, output?: Output) {
is_default_monitoring: defaultMonitoringOutputInput.value,
config_yaml: additionalYamlConfigInput.value,
service_token: serviceTokenInput.value,
...(!serviceTokenInput.value &&
serviceTokenSecretInput.value && {
secrets: {
service_token: serviceTokenSecretInput.value,
},
}),
proxy_id: proxyIdValue,
...shipperParams,
} as NewRemoteElasticsearchOutput;
@ -958,6 +979,7 @@ export function useOutputForm(onSucess: () => void, output?: Output) {
elasticsearchUrlInput.value,
caTrustedFingerprintInput.value,
serviceTokenInput.value,
serviceTokenSecretInput.value,
confirm,
notifications.toasts,
]);

View file

@ -88,4 +88,41 @@ describe('output handler', () => {
expect(res).toEqual({ body: { item: { id: 'output1' } } });
});
it('should return error if both service_token and secrets.service_token is provided for remote_elasticsearch output', async () => {
jest
.spyOn(appContextService, 'getCloud')
.mockReturnValue({ isServerlessEnabled: false } as any);
const res = await postOutputHandler(
mockContext,
{
body: {
type: 'remote_elasticsearch',
service_token: 'token1',
secrets: { service_token: 'token2' },
},
} as any,
mockResponse as any
);
expect(res).toEqual({
body: { message: 'Cannot specify both service_token and secrets.service_token' },
statusCode: 400,
});
});
it('should return ok if one of service_token and secrets.service_token is provided for remote_elasticsearch output', async () => {
jest
.spyOn(appContextService, 'getCloud')
.mockReturnValue({ isServerlessEnabled: false } as any);
const res = await postOutputHandler(
mockContext,
{ body: { type: 'remote_elasticsearch', secrets: { service_token: 'token2' } } } as any,
mockResponse as any
);
expect(res).toEqual({ body: { item: { id: 'output1' } } });
});
});

View file

@ -37,9 +37,20 @@ function ensureNoDuplicateSecrets(output: Partial<Output>) {
if (output.type === outputType.Kafka && output?.password && output?.secrets?.password) {
throw Boom.badRequest('Cannot specify both password and secrets.password');
}
if (output.ssl?.key && output.secrets?.ssl?.key) {
if (
(output.type === outputType.Kafka || output.type === outputType.Logstash) &&
output.ssl?.key &&
output.secrets?.ssl?.key
) {
throw Boom.badRequest('Cannot specify both ssl.key and secrets.ssl.key');
}
if (
output.type === outputType.RemoteElasticsearch &&
output.service_token &&
output.secrets?.service_token
) {
throw Boom.badRequest('Cannot specify both service_token and secrets.service_token');
}
}
export const getOutputsHandler: RequestHandler = async (context, request, response) => {

View file

@ -270,6 +270,12 @@ const getSavedObjectTypes = (): { [key: string]: SavedObjectsType } => ({
},
},
},
service_token: {
dynamic: false,
properties: {
id: { type: 'keyword' },
},
},
},
},
},

View file

@ -205,7 +205,7 @@ async function updateRelatedSavedObject(
outputService.update(soClient, esClient, output.id, {
...omit(output, 'id'),
proxy_id: null,
});
} as Partial<Output>);
},
{ concurrency: 20 }
);

View file

@ -82,7 +82,7 @@ export async function createOrUpdatePreconfiguredOutputs(
ca_sha256: outputData.ca_sha256 ?? null,
ca_trusted_fingerprint: outputData.ca_trusted_fingerprint ?? null,
ssl: outputData.ssl ?? null,
};
} as NewOutput;
if (!data.hosts || data.hosts.length === 0) {
data.hosts = outputService.getDefaultESHosts();

View file

@ -10,7 +10,13 @@ import type { ElasticsearchClient, SavedObjectsClientContract } from '@kbn/core/
import { keyBy } from 'lodash';
import { set } from '@kbn/safer-lodash-set';
import type { KafkaOutput, Output, OutputSecretPath } from '../../common/types';
import type {
KafkaOutput,
NewLogstashOutput,
NewRemoteElasticsearchOutput,
Output,
OutputSecretPath,
} from '../../common/types';
import { packageHasNoPolicyTemplates } from '../../common/services/policy_template';
@ -280,11 +286,14 @@ function getOutputSecretPaths(
): OutputSecretPath[] {
const outputSecretPaths: OutputSecretPath[] = [];
if ((outputType === 'kafka' || outputType === 'logstash') && output.secrets?.ssl?.key) {
outputSecretPaths.push({
path: 'secrets.ssl.key',
value: output.secrets.ssl.key,
});
if (outputType === 'logstash') {
const logstashOutput = output as NewLogstashOutput;
if (logstashOutput?.secrets?.ssl?.key) {
outputSecretPaths.push({
path: 'secrets.ssl.key',
value: logstashOutput.secrets.ssl.key,
});
}
}
if (outputType === 'kafka') {
@ -295,6 +304,22 @@ function getOutputSecretPaths(
value: kafkaOutput.secrets.password,
});
}
if (kafkaOutput?.secrets?.ssl?.key) {
outputSecretPaths.push({
path: 'secrets.ssl.key',
value: kafkaOutput.secrets.ssl.key,
});
}
}
if (outputType === 'remote_elasticsearch') {
const remoteESOutput = output as NewRemoteElasticsearchOutput;
if (remoteESOutput.secrets?.service_token) {
outputSecretPaths.push({
path: 'secrets.service_token',
value: remoteESOutput.secrets.service_token,
});
}
}
return outputSecretPaths;
@ -340,6 +365,15 @@ export function getOutputSecretReferences(output: Output): PolicySecretReference
});
}
if (
output.type === 'remote_elasticsearch' &&
typeof output?.secrets?.service_token === 'object'
) {
outputSecretPaths.push({
id: output.secrets.service_token.id,
});
}
return outputSecretPaths;
}

View file

@ -130,13 +130,23 @@ const ElasticSearchUpdateSchema = {
export const RemoteElasticSearchSchema = {
...ElasticSearchSchema,
type: schema.literal(outputType.RemoteElasticsearch),
service_token: schema.string(),
service_token: schema.maybe(schema.string()),
secrets: schema.maybe(
schema.object({
service_token: schema.maybe(secretRefSchema),
})
),
};
const RemoteElasticSearchUpdateSchema = {
...ElasticSearchUpdateSchema,
type: schema.maybe(schema.literal(outputType.RemoteElasticsearch)),
service_token: schema.maybe(schema.string()),
secrets: schema.maybe(
schema.object({
service_token: schema.maybe(secretRefSchema),
})
),
};
/**

View file

@ -154,7 +154,9 @@ interface OutputSoElasticsearchAttributes extends OutputSoBaseAttributes {
export interface OutputSoRemoteElasticsearchAttributes extends OutputSoBaseAttributes {
type: OutputType['RemoteElasticsearch'];
service_token?: string;
secrets?: {};
secrets?: {
service_token?: { id: string };
};
}
interface OutputSoLogstashAttributes extends OutputSoBaseAttributes {

View file

@ -1132,6 +1132,23 @@ export default function (providerContext: FtrProviderContext) {
// @ts-ignore _source unknown type
expect(secret._source.value).to.equal('pass');
});
it('should create service_token secret correctly', async function () {
const res = await supertest
.post(`/api/fleet/outputs`)
.set('kbn-xsrf', 'xxxx')
.send({
name: 'Remote Elasticsearch With Service Token Secret',
type: 'remote_elasticsearch',
hosts: ['https://remote-es:9200'],
secrets: { service_token: 'token' },
});
const secretId = res.body.item.secrets.service_token.id;
const secret = await getSecretById(secretId);
// @ts-ignore _source unknown type
expect(secret._source.value).to.equal('token');
});
});
describe('DELETE /outputs/{outputId}', () => {