github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-06-28 11:05:39 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Logs onboarding] Generate elastic-agent.yml file for system logs (#162972)

Browse source 
Closes https://github.com/elastic/kibana/issues/154929.

This PR along with https://github.com/elastic/kibana/pull/162654,
https://github.com/elastic/kibana/pull/162706 and
https://github.com/elastic/kibana/pull/162600 completes the work
required for collect system logs.

### Changes
- `ObservabilityOnboardingType` now could be `logFiles | systemLogs`.
This help us to identify (without changing the script) whether we need
to retrieve the yaml configuration for customLogs or for systemLogs.
- Added `generateSystemLogsYml` which generates a specific configuration
for system logs.
- `get_has_logs.ts` was modified so we are querying the proper index
depending on the type of logs.

#### Demo


47eca890-37b2-401e-9e41-67c978ab50ad
This commit is contained in:
Yngrid Coello 2023-08-04 13:13:05 +02:00 • committed by GitHub
parent d082d7c678
commit c54d29737d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
19 changed files with 416 additions and 127 deletions
Download patch file Download diff file Expand all files Collapse all files

1
x-pack/plugins/observability_onboarding/public/components/app/custom_logs/wizard/install_elastic_agent.tsx
View file

@ -91,6 +91,7 @@ export function InstallElasticAgent() {
params: {
body: {
name: datasetName,
type: 'logFiles',
state: {
datasetName,
serviceName,

30
x-pack/plugins/observability_onboarding/public/components/app/system_logs/install_elastic_agent.tsx
View file

@ -40,8 +40,7 @@ export function InstallElasticAgent() {
const [elasticAgentPlatform, setElasticAgentPlatform] =
useState<ElasticAgentPlatform>('linux-tar');
const datasetName = 'elastic-agent';
const namespace = 'default';
const datasetName = 'system-logs';
function onBack() {
navigateToKibanaUrl('/app/observabilityOnboarding');
@ -83,10 +82,7 @@ export function InstallElasticAgent() {
params: {
body: {
name: datasetName,
state: {
datasetName,
namespace,
},
type: 'systemLogs',
},
},
});
@ -95,26 +91,6 @@ export function InstallElasticAgent() {
[monitoringRole?.hasPrivileges]
);
const { status: saveOnboardingStateDataStatus } = useFetcher((callApi) => {
const { onboardingId } = getState();
if (onboardingId) {
return callApi(
'PUT /internal/observability_onboarding/flow/{onboardingId}',
{
params: {
path: { onboardingId },
body: {
state: {
datasetName,
namespace,
},
},
},
}
);
}
}, []);
const { apiKeyEncoded, onboardingId } = installShipperSetup ?? getState();
const { data: yamlConfig = '', status: yamlConfigStatus } = useFetcher(
@ -132,7 +108,7 @@ export function InstallElasticAgent() {
[
apiKeyEncoded,
onboardingId,
saveOnboardingStateDataStatus === FETCH_STATUS.SUCCESS,
installShipperSetupStatus === FETCH_STATUS.SUCCESS,
]
);

8
x-pack/plugins/observability_onboarding/server/routes/elastic_agent/__snapshots__/generate_yml.test.ts.snap → x-pack/plugins/observability_onboarding/server/routes/elastic_agent/custom_logs/__snapshots__/generate_custom_logs_yml.test.ts.snap generated
View file

@ -1,6 +1,6 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`generateYml should return a basic yml configuration 1`] = `
exports[`generateCustomLogsYml should return a basic yml configuration 1`] = `
"outputs:
default:
type: elasticsearch
@ -21,7 +21,7 @@ inputs:
"
`;
exports[`generateYml should return a yml configuration with customConfigurations 1`] = `
exports[`generateCustomLogsYml should return a yml configuration with customConfigurations 1`] = `
"outputs:
default:
type: elasticsearch
@ -47,7 +47,7 @@ agent.monitoring:
"
`;
exports[`generateYml should return a yml configuration with multiple logFilePaths 1`] = `
exports[`generateCustomLogsYml should return a yml configuration with multiple logFilePaths 1`] = `
"outputs:
default:
type: elasticsearch
@ -69,7 +69,7 @@ inputs:
"
`;
exports[`generateYml should return a yml configuration with service name 1`] = `
exports[`generateCustomLogsYml should return a yml configuration with service name 1`] = `
"outputs:
default:
type: elasticsearch

12
x-pack/plugins/observability_onboarding/server/routes/elastic_agent/generate_yml.test.ts → x-pack/plugins/observability_onboarding/server/routes/elastic_agent/custom_logs/generate_custom_logs_yml.test.ts
View file

@ -6,7 +6,7 @@
*/
import { dump } from 'js-yaml';
import { generateYml } from './generate_yml';
import { generateCustomLogsYml } from './generate_custom_logs_yml';
const baseMockConfig = {
datasetName: 'my-dataset',
@ -17,9 +17,9 @@ const baseMockConfig = {
logfileId: 'my-logs-id',
};
describe('generateYml', () => {
describe('generateCustomLogsYml', () => {
it('should return a basic yml configuration', () => {
const result = generateYml(baseMockConfig);
const result = generateCustomLogsYml(baseMockConfig);
expect(result).toMatchSnapshot();
});
@ -29,7 +29,7 @@ describe('generateYml', () => {
logFilePaths: ['/my-service-1.logs', '/my-service-2.logs'],
};
const result = generateYml(mockConfig);
const result = generateCustomLogsYml(mockConfig);
expect(result).toMatchSnapshot();
});
@ -39,7 +39,7 @@ describe('generateYml', () => {
serviceName: 'my-service',
};
const result = generateYml(mockConfig);
const result = generateCustomLogsYml(mockConfig);
expect(result).toMatchSnapshot();
});
@ -57,7 +57,7 @@ describe('generateYml', () => {
}),
};
const result = generateYml(mockConfig);
const result = generateCustomLogsYml(mockConfig);
expect(result).toMatchSnapshot();
});
});

2
x-pack/plugins/observability_onboarding/server/routes/elastic_agent/generate_yml.ts → x-pack/plugins/observability_onboarding/server/routes/elastic_agent/custom_logs/generate_custom_logs_yml.ts
View file

@ -7,7 +7,7 @@
import { dump, load } from 'js-yaml';
export const generateYml = ({
export const generateCustomLogsYml = ({
datasetName = '',
serviceName,
namespace = '',

26
x-pack/plugins/observability_onboarding/server/routes/elastic_agent/route.ts
View file

@ -6,11 +6,13 @@
*/
import * as t from 'io-ts';
import { v4 as uuidv4 } from 'uuid';
import { getAuthenticationAPIKey } from '../../lib/get_authentication_api_key';
import { createObservabilityOnboardingServerRoute } from '../create_observability_onboarding_server_route';
import { generateYml } from './generate_yml';
import { getFallbackESUrl } from '../../lib/get_fallback_urls';
import { getObservabilityOnboardingFlow } from '../../lib/state';
import { createObservabilityOnboardingServerRoute } from '../create_observability_onboarding_server_route';
import { generateCustomLogsYml } from './custom_logs/generate_custom_logs_yml';
import { generateSystemLogsYml } from './system_logs/generate_system_logs_yml';
const generateConfig = createObservabilityOnboardingServerRoute({
endpoint: 'GET /internal/observability_onboarding/elastic_agent/config',
@ -43,17 +45,23 @@ const generateConfig = createObservabilityOnboardingServerRoute({
savedObjectId: onboardingId,
});
const yaml = generateYml({
datasetName: savedState?.state?.datasetName,
customConfigurations: savedState?.state?.customConfigurations,
logFilePaths: savedState?.state?.logFilePaths,
namespace: savedState?.state?.namespace,
const yaml =
savedState?.type === 'systemLogs'
? generateSystemLogsYml({
...savedState?.state,
apiKey: authApiKey
? `${authApiKey?.apiKeyId}:${authApiKey?.apiKey}`
: '$API_KEY',
esHost: elasticsearchUrl,
logfileId: `custom-logs-${Date.now()}`,
serviceName: savedState?.state?.serviceName,
uuid: uuidv4(),
})
: generateCustomLogsYml({
...savedState?.state,
apiKey: authApiKey
? `${authApiKey?.apiKeyId}:${authApiKey?.apiKey}`
: '$API_KEY',
esHost: elasticsearchUrl,
logfileId: `custom-logs-${uuidv4()}`,
});
return yaml;

48
x-pack/plugins/observability_onboarding/server/routes/elastic_agent/system_logs/__snapshots__/generate_system_logs_yml.test.ts.snap generated Normal file
View file

@ -0,0 +1,48 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`generateSystemLogsYml should return system logs oriented yml configuration 1`] = `
"outputs:
default:
type: elasticsearch
hosts:
- 'http://localhost:9200'
api_key: 'elastic:changeme'
inputs:
- id: system-logs-8df0ff52-6f3b-4b5a-a2da-f06c55d111d1
type: logfile
data_stream:
namespace: default
streams:
- id: logfile-system.auth-8df0ff52-6f3b-4b5a-a2da-f06c55d111d1
data_stream:
dataset: system.auth
type: logs
paths:
- /var/log/auth.log*
- /var/log/secure*
exclude_files:
- .gz$
multiline:
pattern: ^s
match: after
tags:
- system-auth
processors:
- add_locale: null
- id: logfile-system.syslog-8df0ff52-6f3b-4b5a-a2da-f06c55d111d1
data_stream:
dataset: system.syslog
type: logs
paths:
- /var/log/messages*
- /var/log/syslog*
- /var/log/system*
exclude_files:
- .gz$
multiline:
pattern: ^s
match: after
processors:
- add_locale: null
"
`;

22
x-pack/plugins/observability_onboarding/server/routes/elastic_agent/system_logs/generate_system_logs_yml.test.ts Normal file
View file

@ -0,0 +1,22 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { generateSystemLogsYml } from './generate_system_logs_yml';
const baseMockConfig = {
namespace: 'default',
apiKey: 'elastic:changeme',
esHost: ['http://localhost:9200'],
uuid: '8df0ff52-6f3b-4b5a-a2da-f06c55d111d1',
};
describe('generateSystemLogsYml', () => {
it('should return system logs oriented yml configuration', () => {
const result = generateSystemLogsYml(baseMockConfig);
expect(result).toMatchSnapshot();
});
});

82
x-pack/plugins/observability_onboarding/server/routes/elastic_agent/system_logs/generate_system_logs_yml.ts Normal file
View file

@ -0,0 +1,82 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { dump } from 'js-yaml';
export const generateSystemLogsYml = ({
namespace = 'default',
apiKey,
esHost,
uuid,
}: {
namespace?: string;
apiKey: string;
esHost: string[];
uuid: string;
}) => {
return dump({
outputs: {
default: {
type: 'elasticsearch',
hosts: esHost,
api_key: apiKey,
},
},
inputs: [
{
id: `system-logs-${uuid}`,
type: 'logfile',
data_stream: {
namespace,
},
streams: [
{
id: `logfile-system.auth-${uuid}`,
data_stream: {
dataset: 'system.auth',
type: 'logs',
},
paths: ['/var/log/auth.log*', '/var/log/secure*'],
exclude_files: ['.gz$'],
multiline: {
pattern: '^s',
match: 'after',
},
tags: ['system-auth'],
processors: [
{
add_locale: null,
},
],
},
{
id: `logfile-system.syslog-${uuid}`,
data_stream: {
dataset: 'system.syslog',
type: 'logs',
},
paths: [
'/var/log/messages*',
'/var/log/syslog*',
'/var/log/system*',
],
exclude_files: ['.gz$'],
multiline: {
pattern: '^s',
match: 'after',
},
processors: [
{
add_locale: null,
},
],
},
],
},
],
});
};

25
x-pack/plugins/observability_onboarding/server/routes/flow/get_has_logs.ts
View file

@ -6,19 +6,34 @@
*/
import { ElasticsearchClient } from '@kbn/core/server';
import {
LogFilesState,
ObservabilityOnboardingType,
SystemLogsState,
} from '../../saved_objects/observability_onboarding_status';
export async function getHasLogs({
dataset,
namespace,
type,
state,
esClient,
}: {
dataset: string;
namespace: string;
type: ObservabilityOnboardingType;
state?: LogFilesState | SystemLogsState;
esClient: ElasticsearchClient;
}) {
if (!state) {
return false;
}
try {
const { namespace } = state;
const index =
type === 'logFiles'
? `logs-${(state as LogFilesState).datasetName}-${namespace}`
: `logs-system.syslog-${namespace}`;
const { hits } = await esClient.search({
index: `logs-${dataset}-${namespace}`,
index,
terminate_after: 1,
});
const total = hits.total as { value: number };

9
x-pack/plugins/observability_onboarding/server/routes/flow/route.ts
View file

@ -148,16 +148,13 @@ const getProgressRoute = createObservabilityOnboardingServerRoute({
const esClient =
coreStart.elasticsearch.client.asScoped(request).asCurrentUser;
const dataset = savedObservabilityOnboardingState.state
?.datasetName as string;
const namespace = savedObservabilityOnboardingState.state
?.namespace as string;
const type = savedObservabilityOnboardingState.type;
if (progress['ea-status']?.status === 'complete') {
try {
const hasLogs = await getHasLogs({
dataset,
namespace,
type,
state: savedObservabilityOnboardingState.state,
esClient,
});
if (hasLogs) {

2
x-pack/plugins/observability_onboarding/server/routes/logs/api_key/create_shipper_api_key.ts
View file

@ -15,7 +15,7 @@ export function createShipperApiKey(
// Based on https://www.elastic.co/guide/en/fleet/master/grant-access-to-elasticsearch.html#create-api-key-standalone-agent
return esClient.security.createApiKey({
body: {
name: `standalone_agent_custom_logs_${name}`,
name: `standalone_agent_logs_onboarding_${name}`,
metadata: { application: 'logs' },
role_descriptors: {
standalone_agent: {

11
x-pack/plugins/observability_onboarding/server/routes/logs/route.ts
View file

@ -66,6 +66,9 @@ const createFlowRoute = createObservabilityOnboardingServerRoute({
t.type({
name: t.string,
}),
t.type({
type: t.union([t.literal('logFiles'), t.literal('systemLogs')]),
}),
t.partial({
state: t.record(t.string, t.unknown),
}),
@ -77,7 +80,7 @@ const createFlowRoute = createObservabilityOnboardingServerRoute({
const {
context,
params: {
body: { name, state },
body: { name, type, state },
},
core,
request,
@ -91,13 +94,15 @@ const createFlowRoute = createObservabilityOnboardingServerRoute({
name
);
const generatedState =
type === 'systemLogs' ? { namespace: 'default' } : state;
const savedObjectsClient = coreStart.savedObjects.getScopedClient(request);
const { id } = await saveObservabilityOnboardingFlow({
savedObjectsClient,
observabilityOnboardingState: {
type: 'logFiles',
state: state as ObservabilityOnboardingFlow['state'],
type,
state: generatedState as ObservabilityOnboardingFlow['state'],
progress: {},
},
});

13
x-pack/plugins/observability_onboarding/server/saved_objects/observability_onboarding_status.ts
View file

@ -17,10 +17,19 @@ export interface LogFilesState {
namespace: string;
}
type ObservabilityOnboardingFlowState = LogFilesState | undefined;
export interface SystemLogsState {
namespace: string;
}
export type ObservabilityOnboardingType = 'logFiles' | 'systemLogs';
type ObservabilityOnboardingFlowState =
| LogFilesState
| SystemLogsState
| undefined;
export interface ObservabilityOnboardingFlow {
type: 'logFiles';
type: ObservabilityOnboardingType;
state: ObservabilityOnboardingFlowState;
progress: Record<
string,

74
x-pack/test/observability_onboarding_api_integration/tests/elastic_agent/config.spec.ts
View file

@ -31,25 +31,6 @@ export default function ApiTest({ getService }: FtrProviderContext) {
const logFilepath = '/my-logs.log';
const serviceName = 'my-service';
before(async () => {
const req = await observabilityOnboardingApiClient.logMonitoringUser({
endpoint: 'POST /internal/observability_onboarding/logs/flow',
params: {
body: {
name: 'name',
state: {
datasetName,
namespace,
logFilePaths: [logFilepath],
serviceName,
},
},
},
});
onboardingId = req.body.onboardingId;
});
describe("when onboardingId doesn't exists", () => {
it('should return input properties empty', async () => {
const req = await callApi({
@ -66,6 +47,27 @@ export default function ApiTest({ getService }: FtrProviderContext) {
});
describe('when onboardingId exists', () => {
describe('and onboarding type is logFiles', () => {
before(async () => {
const req = await observabilityOnboardingApiClient.logMonitoringUser({
endpoint: 'POST /internal/observability_onboarding/logs/flow',
params: {
body: {
type: 'logFiles',
name: 'name',
state: {
datasetName,
namespace,
logFilePaths: [logFilepath],
serviceName,
},
},
},
});
onboardingId = req.body.onboardingId;
});
it('should return input properties configured', async () => {
const req = await callApi({
onboardingId,
@ -77,6 +79,40 @@ export default function ApiTest({ getService }: FtrProviderContext) {
expect(ymlConfig.inputs[0].data_stream.namespace).to.be(namespace);
expect(ymlConfig.inputs[0].streams[0].data_stream.dataset).to.be(datasetName);
expect(ymlConfig.inputs[0].streams[0].paths).to.be.eql([logFilepath]);
expect(ymlConfig.inputs[0].streams[0].processors[0].add_fields.fields.name).to.be.eql(
serviceName
);
});
});
describe('and onboarding type is systemLogs', () => {
before(async () => {
const req = await observabilityOnboardingApiClient.logMonitoringUser({
endpoint: 'POST /internal/observability_onboarding/logs/flow',
params: {
body: {
type: 'systemLogs',
name: 'name',
},
},
});
onboardingId = req.body.onboardingId;
});
it('should return input properties configured', async () => {
const req = await callApi({
onboardingId,
});
expect(req.status).to.be(200);
const ymlConfig = load(req.text);
expect(ymlConfig.inputs[0].data_stream.namespace).to.be('default');
expect(ymlConfig.inputs[0].streams.length).to.be(2);
expect(ymlConfig.inputs[0].streams[0].data_stream.dataset).to.be('system.auth');
expect(ymlConfig.inputs[0].streams[1].data_stream.dataset).to.be('system.syslog');
});
});
});
});

6
x-pack/test/observability_onboarding_api_integration/tests/flow/progress/es_utils.ts
View file

@ -15,7 +15,7 @@ export function createLogDoc({
}: {
time: number;
logFilepath: string;
serviceName: string;
serviceName?: string;
namespace: string;
datasetName: string;
message: string;
@ -30,9 +30,13 @@ export function createLogDoc({
path: logFilepath,
},
},
...(serviceName
? {
service: {
name: serviceName,
},
}
: {}),
data_stream: {
namespace,
type: 'logs',

68
x-pack/test/observability_onboarding_api_integration/tests/flow/progress/progress.spec.ts
View file

@ -44,6 +44,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
endpoint: 'POST /internal/observability_onboarding/logs/flow',
params: {
body: {
type: 'logFiles',
name: 'name',
state: {
datasetName,
@ -131,6 +132,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
});
describe('when logs have been ingested', () => {
describe('and onboarding type is logFiles', () => {
before(async () => {
await es.indices.createDataStream({
name: `logs-${datasetName}-${namespace}`,
@ -168,6 +170,72 @@ export default function ApiTest({ getService }: FtrProviderContext) {
});
});
});
describe('and onboarding type is systemLogs', () => {
let systemLogsOnboardingId: string;
before(async () => {
const req = await observabilityOnboardingApiClient.logMonitoringUser({
endpoint: 'POST /internal/observability_onboarding/logs/flow',
params: {
body: {
type: 'systemLogs',
name: 'name',
},
},
});
systemLogsOnboardingId = req.body.onboardingId;
await observabilityOnboardingApiClient.logMonitoringUser({
endpoint: 'POST /internal/observability_onboarding/flow/{id}/step/{name}',
params: {
path: {
id: systemLogsOnboardingId,
name: 'ea-status',
},
body: {
status: 'complete',
},
},
});
await es.indices.createDataStream({
name: `logs-system.syslog-${namespace}`,
});
const doc = createLogDoc({
time: new Date('06/28/2023').getTime(),
logFilepath: '/var/log/system.log',
namespace,
datasetName: 'system.syslog',
message: 'This is a system log message',
});
await es.bulk({
body: [{ create: { _index: `logs-system.syslog-${namespace}` } }, doc],
refresh: 'wait_for',
});
});
it('should return log-ingest as complete', async () => {
const request = await callApi({
onboardingId: systemLogsOnboardingId,
});
expect(request.status).to.be(200);
const logsIngestProgress = request.body.progress['logs-ingest'];
expect(logsIngestProgress).to.have.property('status', 'complete');
});
after(async () => {
await es.indices.deleteDataStream({
name: `logs-system.syslog-${namespace}`,
});
});
});
});
});
});
});

25
x-pack/test/observability_onboarding_api_integration/tests/logs/create.spec.ts
View file

@ -21,6 +21,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
endpoint: 'POST /internal/observability_onboarding/logs/flow',
params: {
body: {
type: 'logFiles',
name: 'name',
state,
},
@ -28,11 +29,12 @@ export default function ApiTest({ getService }: FtrProviderContext) {
});
}
async function callApiWithPrivileges(state = {}) {
async function callApiWithPrivileges(type: 'logFiles' | 'systemLogs', state = {}) {
return await observabilityOnboardingApiClient.logMonitoringUser({
endpoint: 'POST /internal/observability_onboarding/logs/flow',
params: {
body: {
type,
name: 'name',
state,
},
@ -54,14 +56,14 @@ export default function ApiTest({ getService }: FtrProviderContext) {
describe('when required privileges are set', () => {
it('returns a flow id and apiKey encoded', async () => {
const request = await callApiWithPrivileges();
const request = await callApiWithPrivileges('logFiles');
expect(request.status).to.be(200);
expect(request.body.apiKeyEncoded).to.not.empty();
expect(request.body.onboardingId).to.not.empty();
});
it('saves the expected state', async () => {
it('saves the expected state for logFiles', async () => {
const state = {
datasetName: 'my-dataset',
serviceName: 'my-service',
@ -69,7 +71,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
logFilePaths: 'my-service-logs.log',
};
const request = await callApiWithPrivileges(state);
const request = await callApiWithPrivileges('logFiles', state);
const savedState = await kibanaServer.savedObjects.get({
type: OBSERVABILITY_ONBOARDING_STATE_SAVED_OBJECT_TYPE,
@ -78,6 +80,21 @@ export default function ApiTest({ getService }: FtrProviderContext) {
expect(savedState.attributes).to.be.eql({ type: 'logFiles', state, progress: {} });
});
it('saves the expected state for systemLogs', async () => {
const state = {
namespace: 'default',
};
const request = await callApiWithPrivileges('systemLogs');
const savedState = await kibanaServer.savedObjects.get({
type: OBSERVABILITY_ONBOARDING_STATE_SAVED_OBJECT_TYPE,
id: request.body.onboardingId,
});
expect(savedState.attributes).to.be.eql({ type: 'systemLogs', state, progress: {} });
});
});
});
}

1
x-pack/test/observability_onboarding_api_integration/tests/logs/update_step_progress.spec.ts
View file

@ -67,6 +67,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
endpoint: 'POST /internal/observability_onboarding/logs/flow',
params: {
body: {
type: 'logFiles',
name: 'name',
state: {},
},