mirror of
https://github.com/elastic/kibana.git
synced 2025-04-24 01:38:56 -04:00
This fixes the message reconstruction for the pre-ECS formats of several filebeat modules by adding appropriate rules.
This commit is contained in:
Felix Stürmer
GitHub
parent
7a4a570498
commit
b4cb05bfaa
17 changed files with 1813 additions and 8 deletions
|
|
@ -82,6 +82,7 @@ const hoveredFieldStyle = css`
|
|||
const wrappedFieldStyle = css`
|
||||
overflow: visible;
|
||||
white-space: pre-wrap;
|
||||
word-break: break-all;
|
||||
`;
|
||||
|
||||
const unwrappedFieldStyle = css`
|
||||
|
|
|
|||
|
|
@ -0,0 +1,400 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import { compileFormattingRules } from '../message';
|
||||
import { filebeatHaproxyRules } from './filebeat_haproxy';
|
||||
|
||||
const { format } = compileFormattingRules(filebeatHaproxyRules);
|
||||
|
||||
describe('Filebeat Rules', () => {
|
||||
describe('in pre-ECS format', () => {
|
||||
test('haproxy default log', () => {
|
||||
const flattenedDocument = {
|
||||
'event.dataset': 'haproxy.log',
|
||||
'fileset.module': 'haproxy',
|
||||
'fileset.name': 'log',
|
||||
'haproxy.client.ip': '1.2.3.4',
|
||||
'haproxy.client.port': '40780',
|
||||
'haproxy.destination.ip': '1.2.3.4',
|
||||
'haproxy.destination.port': '5000',
|
||||
'haproxy.frontend_name': 'main',
|
||||
'haproxy.geoip.continent_name': 'North America',
|
||||
'haproxy.geoip.country_iso_code': 'US',
|
||||
'haproxy.geoip.location.lat': 37.751,
|
||||
'haproxy.geoip.location.lon': -97.822,
|
||||
'haproxy.mode': 'HTTP',
|
||||
'haproxy.pid': '24551',
|
||||
'haproxy.process_name': 'haproxy',
|
||||
'haproxy.source': '1.2.3.4',
|
||||
'input.type': 'log',
|
||||
offset: 0,
|
||||
'prospector.type': 'log',
|
||||
};
|
||||
|
||||
expect(format(flattenedDocument)).toMatchInlineSnapshot(`
|
||||
Array [
|
||||
Object {
|
||||
"constant": "[HAProxy] ",
|
||||
},
|
||||
Object {
|
||||
"field": "haproxy.client.ip",
|
||||
"highlights": Array [],
|
||||
"value": "1.2.3.4",
|
||||
},
|
||||
Object {
|
||||
"constant": ":",
|
||||
},
|
||||
Object {
|
||||
"field": "haproxy.client.port",
|
||||
"highlights": Array [],
|
||||
"value": "40780",
|
||||
},
|
||||
Object {
|
||||
"constant": " ",
|
||||
},
|
||||
Object {
|
||||
"field": "haproxy.frontend_name",
|
||||
"highlights": Array [],
|
||||
"value": "main",
|
||||
},
|
||||
Object {
|
||||
"constant": " ",
|
||||
},
|
||||
]
|
||||
`);
|
||||
});
|
||||
|
||||
test('haproxy tcp log', () => {
|
||||
const flattenedDocument = {
|
||||
'event.dataset': 'haproxy.log',
|
||||
'fileset.module': 'haproxy',
|
||||
'fileset.name': 'log',
|
||||
'haproxy.backend_name': 'app',
|
||||
'haproxy.backend_queue': 0,
|
||||
'haproxy.bytes_read': 212,
|
||||
'haproxy.client.ip': '127.0.0.1',
|
||||
'haproxy.client.port': 40962,
|
||||
'haproxy.connection_wait_time_ms': -1,
|
||||
'haproxy.connections.active': 1,
|
||||
'haproxy.connections.backend': 0,
|
||||
'haproxy.connections.frontend': 1,
|
||||
'haproxy.connections.retries': 0,
|
||||
'haproxy.connections.server': 0,
|
||||
'haproxy.frontend_name': 'main',
|
||||
'haproxy.pid': 25457,
|
||||
'haproxy.process_name': 'haproxy',
|
||||
'haproxy.server_name': '<NOSRV>',
|
||||
'haproxy.server_queue': 0,
|
||||
'haproxy.source': '127.0.0.1',
|
||||
'haproxy.tcp.processing_time_ms': 0,
|
||||
'haproxy.termination_state': 'SC',
|
||||
'haproxy.total_waiting_time_ms': -1,
|
||||
'input.type': 'log',
|
||||
offset: 0,
|
||||
'prospector.type': 'log',
|
||||
};
|
||||
|
||||
expect(format(flattenedDocument)).toMatchInlineSnapshot(`
|
||||
Array [
|
||||
Object {
|
||||
"constant": "[HAProxy][tcp] ",
|
||||
},
|
||||
Object {
|
||||
"field": "haproxy.client.ip",
|
||||
"highlights": Array [],
|
||||
"value": "127.0.0.1",
|
||||
},
|
||||
Object {
|
||||
"constant": ":",
|
||||
},
|
||||
Object {
|
||||
"field": "haproxy.client.port",
|
||||
"highlights": Array [],
|
||||
"value": "40962",
|
||||
},
|
||||
Object {
|
||||
"constant": " ",
|
||||
},
|
||||
Object {
|
||||
"field": "haproxy.frontend_name",
|
||||
"highlights": Array [],
|
||||
"value": "main",
|
||||
},
|
||||
Object {
|
||||
"constant": " -> ",
|
||||
},
|
||||
Object {
|
||||
"field": "haproxy.backend_name",
|
||||
"highlights": Array [],
|
||||
"value": "app",
|
||||
},
|
||||
Object {
|
||||
"constant": "/",
|
||||
},
|
||||
Object {
|
||||
"field": "haproxy.server_name",
|
||||
"highlights": Array [],
|
||||
"value": "<NOSRV>",
|
||||
},
|
||||
Object {
|
||||
"constant": " ",
|
||||
},
|
||||
Object {
|
||||
"field": "haproxy.connections.active",
|
||||
"highlights": Array [],
|
||||
"value": "1",
|
||||
},
|
||||
Object {
|
||||
"constant": "/",
|
||||
},
|
||||
Object {
|
||||
"field": "haproxy.connections.frontend",
|
||||
"highlights": Array [],
|
||||
"value": "1",
|
||||
},
|
||||
Object {
|
||||
"constant": "/",
|
||||
},
|
||||
Object {
|
||||
"field": "haproxy.connections.backend",
|
||||
"highlights": Array [],
|
||||
"value": "0",
|
||||
},
|
||||
Object {
|
||||
"constant": "/",
|
||||
},
|
||||
Object {
|
||||
"field": "haproxy.connections.server",
|
||||
"highlights": Array [],
|
||||
"value": "0",
|
||||
},
|
||||
Object {
|
||||
"constant": "/",
|
||||
},
|
||||
Object {
|
||||
"field": "haproxy.connections.retries",
|
||||
"highlights": Array [],
|
||||
"value": "0",
|
||||
},
|
||||
Object {
|
||||
"constant": " ",
|
||||
},
|
||||
Object {
|
||||
"field": "haproxy.server_queue",
|
||||
"highlights": Array [],
|
||||
"value": "0",
|
||||
},
|
||||
Object {
|
||||
"constant": "/",
|
||||
},
|
||||
Object {
|
||||
"field": "haproxy.backend_queue",
|
||||
"highlights": Array [],
|
||||
"value": "0",
|
||||
},
|
||||
]
|
||||
`);
|
||||
});
|
||||
|
||||
test('haproxy http log', () => {
|
||||
const flattenedDocument = {
|
||||
'event.dataset': 'haproxy.log',
|
||||
'fileset.module': 'haproxy',
|
||||
'fileset.name': 'log',
|
||||
'haproxy.backend_name': 'docs_microservice',
|
||||
'haproxy.backend_queue': 0,
|
||||
'haproxy.bytes_read': 168,
|
||||
'haproxy.client.ip': '1.2.3.4',
|
||||
'haproxy.client.port': 38862,
|
||||
'haproxy.connection_wait_time_ms': 1,
|
||||
'haproxy.connections.active': 6,
|
||||
'haproxy.connections.backend': 0,
|
||||
'haproxy.connections.frontend': 6,
|
||||
'haproxy.connections.retries': 0,
|
||||
'haproxy.connections.server': 0,
|
||||
'haproxy.frontend_name': 'incoming~',
|
||||
'haproxy.geoip.continent_name': 'North America',
|
||||
'haproxy.geoip.country_iso_code': 'US',
|
||||
'haproxy.geoip.location.lat': 37.751,
|
||||
'haproxy.geoip.location.lon': -97.822,
|
||||
'haproxy.http.request.captured_cookie': '-',
|
||||
'haproxy.http.request.raw_request_line':
|
||||
'GET /component---src-pages-index-js-4b15624544f97cf0bb8f.js HTTP/1.1',
|
||||
'haproxy.http.request.time_active_ms': 2,
|
||||
'haproxy.http.request.time_wait_ms': 0,
|
||||
'haproxy.http.request.time_wait_without_data_ms': 0,
|
||||
'haproxy.http.response.captured_cookie': '-',
|
||||
'haproxy.http.response.status_code': 304,
|
||||
'haproxy.pid': 32450,
|
||||
'haproxy.process_name': 'haproxy',
|
||||
'haproxy.server_name': 'docs',
|
||||
'haproxy.server_queue': 0,
|
||||
'haproxy.termination_state': '----',
|
||||
'haproxy.total_waiting_time_ms': 0,
|
||||
'input.type': 'log',
|
||||
offset: 0,
|
||||
'prospector.type': 'log',
|
||||
};
|
||||
|
||||
expect(format(flattenedDocument)).toMatchInlineSnapshot(`
|
||||
Array [
|
||||
Object {
|
||||
"constant": "[HAProxy][http] ",
|
||||
},
|
||||
Object {
|
||||
"field": "haproxy.client.ip",
|
||||
"highlights": Array [],
|
||||
"value": "1.2.3.4",
|
||||
},
|
||||
Object {
|
||||
"constant": ":",
|
||||
},
|
||||
Object {
|
||||
"field": "haproxy.client.port",
|
||||
"highlights": Array [],
|
||||
"value": "38862",
|
||||
},
|
||||
Object {
|
||||
"constant": " ",
|
||||
},
|
||||
Object {
|
||||
"field": "haproxy.frontend_name",
|
||||
"highlights": Array [],
|
||||
"value": "incoming~",
|
||||
},
|
||||
Object {
|
||||
"constant": " -> ",
|
||||
},
|
||||
Object {
|
||||
"field": "haproxy.backend_name",
|
||||
"highlights": Array [],
|
||||
"value": "docs_microservice",
|
||||
},
|
||||
Object {
|
||||
"constant": "/",
|
||||
},
|
||||
Object {
|
||||
"field": "haproxy.server_name",
|
||||
"highlights": Array [],
|
||||
"value": "docs",
|
||||
},
|
||||
Object {
|
||||
"constant": " \\"",
|
||||
},
|
||||
Object {
|
||||
"field": "haproxy.http.request.raw_request_line",
|
||||
"highlights": Array [],
|
||||
"value": "GET /component---src-pages-index-js-4b15624544f97cf0bb8f.js HTTP/1.1",
|
||||
},
|
||||
Object {
|
||||
"constant": "\\" ",
|
||||
},
|
||||
Object {
|
||||
"field": "haproxy.http.response.status_code",
|
||||
"highlights": Array [],
|
||||
"value": "304",
|
||||
},
|
||||
Object {
|
||||
"constant": " ",
|
||||
},
|
||||
Object {
|
||||
"field": "haproxy.http.request.time_wait_ms",
|
||||
"highlights": Array [],
|
||||
"value": "0",
|
||||
},
|
||||
Object {
|
||||
"constant": "/",
|
||||
},
|
||||
Object {
|
||||
"field": "haproxy.total_waiting_time_ms",
|
||||
"highlights": Array [],
|
||||
"value": "0",
|
||||
},
|
||||
Object {
|
||||
"constant": "/",
|
||||
},
|
||||
Object {
|
||||
"field": "haproxy.connection_wait_time_ms",
|
||||
"highlights": Array [],
|
||||
"value": "1",
|
||||
},
|
||||
Object {
|
||||
"constant": "/",
|
||||
},
|
||||
Object {
|
||||
"field": "haproxy.http.request.time_wait_without_data_ms",
|
||||
"highlights": Array [],
|
||||
"value": "0",
|
||||
},
|
||||
Object {
|
||||
"constant": "/",
|
||||
},
|
||||
Object {
|
||||
"field": "haproxy.http.request.time_active_ms",
|
||||
"highlights": Array [],
|
||||
"value": "2",
|
||||
},
|
||||
Object {
|
||||
"constant": " ",
|
||||
},
|
||||
Object {
|
||||
"field": "haproxy.connections.active",
|
||||
"highlights": Array [],
|
||||
"value": "6",
|
||||
},
|
||||
Object {
|
||||
"constant": "/",
|
||||
},
|
||||
Object {
|
||||
"field": "haproxy.connections.frontend",
|
||||
"highlights": Array [],
|
||||
"value": "6",
|
||||
},
|
||||
Object {
|
||||
"constant": "/",
|
||||
},
|
||||
Object {
|
||||
"field": "haproxy.connections.backend",
|
||||
"highlights": Array [],
|
||||
"value": "0",
|
||||
},
|
||||
Object {
|
||||
"constant": "/",
|
||||
},
|
||||
Object {
|
||||
"field": "haproxy.connections.server",
|
||||
"highlights": Array [],
|
||||
"value": "0",
|
||||
},
|
||||
Object {
|
||||
"constant": "/",
|
||||
},
|
||||
Object {
|
||||
"field": "haproxy.connections.retries",
|
||||
"highlights": Array [],
|
||||
"value": "0",
|
||||
},
|
||||
Object {
|
||||
"constant": " ",
|
||||
},
|
||||
Object {
|
||||
"field": "haproxy.server_queue",
|
||||
"highlights": Array [],
|
||||
"value": "0",
|
||||
},
|
||||
Object {
|
||||
"constant": "/",
|
||||
},
|
||||
Object {
|
||||
"field": "haproxy.backend_queue",
|
||||
"highlights": Array [],
|
||||
"value": "0",
|
||||
},
|
||||
]
|
||||
`);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,200 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
const commonFrontendFields = [
|
||||
{
|
||||
field: 'haproxy.client.ip',
|
||||
},
|
||||
{
|
||||
constant: ':',
|
||||
},
|
||||
{
|
||||
field: 'haproxy.client.port',
|
||||
},
|
||||
{
|
||||
constant: ' ',
|
||||
},
|
||||
{
|
||||
field: 'haproxy.frontend_name',
|
||||
},
|
||||
];
|
||||
|
||||
const commonBackendFields = [
|
||||
{
|
||||
constant: ' -> ',
|
||||
},
|
||||
{
|
||||
field: 'haproxy.backend_name',
|
||||
},
|
||||
{
|
||||
constant: '/',
|
||||
},
|
||||
{
|
||||
field: 'haproxy.server_name',
|
||||
},
|
||||
];
|
||||
|
||||
const commonConnectionStatsFields = [
|
||||
{
|
||||
field: 'haproxy.connections.active',
|
||||
},
|
||||
{
|
||||
constant: '/',
|
||||
},
|
||||
{
|
||||
field: 'haproxy.connections.frontend',
|
||||
},
|
||||
{
|
||||
constant: '/',
|
||||
},
|
||||
{
|
||||
field: 'haproxy.connections.backend',
|
||||
},
|
||||
{
|
||||
constant: '/',
|
||||
},
|
||||
{
|
||||
field: 'haproxy.connections.server',
|
||||
},
|
||||
{
|
||||
constant: '/',
|
||||
},
|
||||
{
|
||||
field: 'haproxy.connections.retries',
|
||||
},
|
||||
];
|
||||
|
||||
const commonQueueStatsFields = [
|
||||
{
|
||||
field: 'haproxy.server_queue',
|
||||
},
|
||||
{
|
||||
constant: '/',
|
||||
},
|
||||
{
|
||||
field: 'haproxy.backend_queue',
|
||||
},
|
||||
];
|
||||
|
||||
export const filebeatHaproxyRules = [
|
||||
{
|
||||
// pre-ECS
|
||||
when: {
|
||||
exists: ['haproxy.http.request.raw_request_line'],
|
||||
},
|
||||
format: [
|
||||
{
|
||||
constant: '[HAProxy][http] ',
|
||||
},
|
||||
...commonFrontendFields,
|
||||
...commonBackendFields,
|
||||
{
|
||||
constant: ' "',
|
||||
},
|
||||
{
|
||||
field: 'haproxy.http.request.raw_request_line',
|
||||
},
|
||||
{
|
||||
constant: '" ',
|
||||
},
|
||||
{
|
||||
field: 'haproxy.http.response.status_code',
|
||||
},
|
||||
{
|
||||
constant: ' ',
|
||||
},
|
||||
{
|
||||
field: 'haproxy.http.request.time_wait_ms',
|
||||
},
|
||||
{
|
||||
constant: '/',
|
||||
},
|
||||
{
|
||||
field: 'haproxy.total_waiting_time_ms',
|
||||
},
|
||||
{
|
||||
constant: '/',
|
||||
},
|
||||
{
|
||||
field: 'haproxy.connection_wait_time_ms',
|
||||
},
|
||||
{
|
||||
constant: '/',
|
||||
},
|
||||
{
|
||||
field: 'haproxy.http.request.time_wait_without_data_ms',
|
||||
},
|
||||
{
|
||||
constant: '/',
|
||||
},
|
||||
{
|
||||
field: 'haproxy.http.request.time_active_ms',
|
||||
},
|
||||
{
|
||||
constant: ' ',
|
||||
},
|
||||
...commonConnectionStatsFields,
|
||||
{
|
||||
constant: ' ',
|
||||
},
|
||||
...commonQueueStatsFields,
|
||||
],
|
||||
},
|
||||
{
|
||||
// pre-ECS
|
||||
when: {
|
||||
exists: ['haproxy.connections.active'],
|
||||
},
|
||||
format: [
|
||||
{
|
||||
constant: '[HAProxy][tcp] ',
|
||||
},
|
||||
...commonFrontendFields,
|
||||
...commonBackendFields,
|
||||
{
|
||||
constant: ' ',
|
||||
},
|
||||
...commonConnectionStatsFields,
|
||||
{
|
||||
constant: ' ',
|
||||
},
|
||||
...commonQueueStatsFields,
|
||||
],
|
||||
},
|
||||
{
|
||||
// pre-ECS
|
||||
when: {
|
||||
exists: ['haproxy.error_message'],
|
||||
},
|
||||
format: [
|
||||
{
|
||||
constant: '[HAProxy] ',
|
||||
},
|
||||
...commonFrontendFields,
|
||||
{
|
||||
constant: ' ',
|
||||
},
|
||||
{
|
||||
field: 'haproxy.error_message',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
// pre-ECS
|
||||
when: {
|
||||
exists: ['haproxy.frontend_name'],
|
||||
},
|
||||
format: [
|
||||
{
|
||||
constant: '[HAProxy] ',
|
||||
},
|
||||
...commonFrontendFields,
|
||||
{
|
||||
constant: ' ',
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
|
|
@ -0,0 +1,147 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import { compileFormattingRules } from '../message';
|
||||
import { filebeatIcingaRules } from './filebeat_icinga';
|
||||
|
||||
const { format } = compileFormattingRules(filebeatIcingaRules);
|
||||
|
||||
describe('Filebeat Rules', () => {
|
||||
describe('in pre-ECS format', () => {
|
||||
test('icinga debug log', () => {
|
||||
const flattenedDocument = {
|
||||
'@timestamp': '2017-04-04T11:43:09.000Z',
|
||||
'event.dataset': 'icinga.debug',
|
||||
'fileset.module': 'icinga',
|
||||
'fileset.name': 'debug',
|
||||
'icinga.debug.facility': 'GraphiteWriter',
|
||||
'icinga.debug.message':
|
||||
"Add to metric list:'icinga2.demo.services.procs.procs.perfdata.procs.warn 250 1491306189'.",
|
||||
'icinga.debug.severity': 'debug',
|
||||
'input.type': 'log',
|
||||
offset: 0,
|
||||
'prospector.type': 'log',
|
||||
};
|
||||
|
||||
expect(format(flattenedDocument)).toMatchInlineSnapshot(`
|
||||
Array [
|
||||
Object {
|
||||
"constant": "[Icinga][",
|
||||
},
|
||||
Object {
|
||||
"field": "icinga.debug.facility",
|
||||
"highlights": Array [],
|
||||
"value": "GraphiteWriter",
|
||||
},
|
||||
Object {
|
||||
"constant": "][",
|
||||
},
|
||||
Object {
|
||||
"field": "icinga.debug.severity",
|
||||
"highlights": Array [],
|
||||
"value": "debug",
|
||||
},
|
||||
Object {
|
||||
"constant": "] ",
|
||||
},
|
||||
Object {
|
||||
"field": "icinga.debug.message",
|
||||
"highlights": Array [],
|
||||
"value": "Add to metric list:'icinga2.demo.services.procs.procs.perfdata.procs.warn 250 1491306189'.",
|
||||
},
|
||||
]
|
||||
`);
|
||||
});
|
||||
|
||||
test('icinga main log', () => {
|
||||
const flattenedDocument = {
|
||||
'@timestamp': '2017-04-04T09:16:34.000Z',
|
||||
'event.dataset': 'icinga.main',
|
||||
'fileset.module': 'icinga',
|
||||
'fileset.name': 'main',
|
||||
'icinga.main.facility': 'Notification',
|
||||
'icinga.main.message':
|
||||
"Sending 'Recovery' notification 'demo!load!mail-icingaadmin for user 'on-call'",
|
||||
'icinga.main.severity': 'information',
|
||||
'input.type': 'log',
|
||||
offset: 0,
|
||||
'prospector.type': 'log',
|
||||
};
|
||||
|
||||
expect(format(flattenedDocument)).toMatchInlineSnapshot(`
|
||||
Array [
|
||||
Object {
|
||||
"constant": "[Icinga][",
|
||||
},
|
||||
Object {
|
||||
"field": "icinga.main.facility",
|
||||
"highlights": Array [],
|
||||
"value": "Notification",
|
||||
},
|
||||
Object {
|
||||
"constant": "][",
|
||||
},
|
||||
Object {
|
||||
"field": "icinga.main.severity",
|
||||
"highlights": Array [],
|
||||
"value": "information",
|
||||
},
|
||||
Object {
|
||||
"constant": "] ",
|
||||
},
|
||||
Object {
|
||||
"field": "icinga.main.message",
|
||||
"highlights": Array [],
|
||||
"value": "Sending 'Recovery' notification 'demo!load!mail-icingaadmin for user 'on-call'",
|
||||
},
|
||||
]
|
||||
`);
|
||||
});
|
||||
|
||||
test('icinga startup log', () => {
|
||||
const flattenedDocument = {
|
||||
'event.dataset': 'icinga.startup',
|
||||
'fileset.module': 'icinga',
|
||||
'fileset.name': 'startup',
|
||||
'icinga.startup.facility': 'cli',
|
||||
'icinga.startup.message': 'Icinga application loader (version: r2.6.3-1)',
|
||||
'icinga.startup.severity': 'information',
|
||||
'input.type': 'log',
|
||||
offset: 0,
|
||||
'prospector.type': 'log',
|
||||
};
|
||||
|
||||
expect(format(flattenedDocument)).toMatchInlineSnapshot(`
|
||||
Array [
|
||||
Object {
|
||||
"constant": "[Icinga][",
|
||||
},
|
||||
Object {
|
||||
"field": "icinga.startup.facility",
|
||||
"highlights": Array [],
|
||||
"value": "cli",
|
||||
},
|
||||
Object {
|
||||
"constant": "][",
|
||||
},
|
||||
Object {
|
||||
"field": "icinga.startup.severity",
|
||||
"highlights": Array [],
|
||||
"value": "information",
|
||||
},
|
||||
Object {
|
||||
"constant": "] ",
|
||||
},
|
||||
Object {
|
||||
"field": "icinga.startup.message",
|
||||
"highlights": Array [],
|
||||
"value": "Icinga application loader (version: r2.6.3-1)",
|
||||
},
|
||||
]
|
||||
`);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,86 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
export const filebeatIcingaRules = [
|
||||
{
|
||||
// pre-ECS
|
||||
when: {
|
||||
exists: ['icinga.main.message'],
|
||||
},
|
||||
format: [
|
||||
{
|
||||
constant: '[Icinga][',
|
||||
},
|
||||
{
|
||||
field: 'icinga.main.facility',
|
||||
},
|
||||
{
|
||||
constant: '][',
|
||||
},
|
||||
{
|
||||
field: 'icinga.main.severity',
|
||||
},
|
||||
{
|
||||
constant: '] ',
|
||||
},
|
||||
{
|
||||
field: 'icinga.main.message',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
// pre-ECS
|
||||
when: {
|
||||
exists: ['icinga.debug.message'],
|
||||
},
|
||||
format: [
|
||||
{
|
||||
constant: '[Icinga][',
|
||||
},
|
||||
{
|
||||
field: 'icinga.debug.facility',
|
||||
},
|
||||
{
|
||||
constant: '][',
|
||||
},
|
||||
{
|
||||
field: 'icinga.debug.severity',
|
||||
},
|
||||
{
|
||||
constant: '] ',
|
||||
},
|
||||
{
|
||||
field: 'icinga.debug.message',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
// pre-ECS
|
||||
when: {
|
||||
exists: ['icinga.startup.message'],
|
||||
},
|
||||
format: [
|
||||
{
|
||||
constant: '[Icinga][',
|
||||
},
|
||||
{
|
||||
field: 'icinga.startup.facility',
|
||||
},
|
||||
{
|
||||
constant: '][',
|
||||
},
|
||||
{
|
||||
field: 'icinga.startup.severity',
|
||||
},
|
||||
{
|
||||
constant: '] ',
|
||||
},
|
||||
{
|
||||
field: 'icinga.startup.message',
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
|
|
@ -0,0 +1,279 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import { compileFormattingRules } from '../message';
|
||||
import { filebeatIisRules } from './filebeat_iis';
|
||||
|
||||
const { format } = compileFormattingRules(filebeatIisRules);
|
||||
|
||||
describe('Filebeat Rules', () => {
|
||||
describe('in pre-ECS format', () => {
|
||||
test('iis access log', () => {
|
||||
const flattenedDocument = {
|
||||
'@timestamp': '2018-01-01T08:09:10.000Z',
|
||||
'event.dataset': 'iis.access',
|
||||
'fileset.module': 'iis',
|
||||
'fileset.name': 'access',
|
||||
'iis.access.geoip.city_name': 'Berlin',
|
||||
'iis.access.geoip.continent_name': 'Europe',
|
||||
'iis.access.geoip.country_iso_code': 'DE',
|
||||
'iis.access.geoip.location.lat': 52.4908,
|
||||
'iis.access.geoip.location.lon': 13.3275,
|
||||
'iis.access.geoip.region_iso_code': 'DE-BE',
|
||||
'iis.access.geoip.region_name': 'Land Berlin',
|
||||
'iis.access.method': 'GET',
|
||||
'iis.access.port': '80',
|
||||
'iis.access.query_string': 'q=100',
|
||||
'iis.access.referrer': '-',
|
||||
'iis.access.remote_ip': '85.181.35.98',
|
||||
'iis.access.request_time_ms': '123',
|
||||
'iis.access.response_code': '200',
|
||||
'iis.access.server_ip': '127.0.0.1',
|
||||
'iis.access.sub_status': '0',
|
||||
'iis.access.url': '/',
|
||||
'iis.access.user_agent.device': 'Other',
|
||||
'iis.access.user_agent.major': '57',
|
||||
'iis.access.user_agent.minor': '0',
|
||||
'iis.access.user_agent.name': 'Firefox',
|
||||
'iis.access.user_agent.original':
|
||||
'Mozilla/5.0+(Windows+NT+6.1;+Win64;+x64;+rv:57.0)+Gecko/20100101+Firefox/57.0',
|
||||
'iis.access.user_agent.os': 'Windows',
|
||||
'iis.access.user_agent.os_name': 'Windows',
|
||||
'iis.access.user_name': '-',
|
||||
'iis.access.win32_status': '0',
|
||||
'input.type': 'log',
|
||||
offset: 257,
|
||||
'prospector.type': 'log',
|
||||
};
|
||||
|
||||
expect(format(flattenedDocument)).toMatchInlineSnapshot(`
|
||||
Array [
|
||||
Object {
|
||||
"constant": "[IIS][access] ",
|
||||
},
|
||||
Object {
|
||||
"field": "iis.access.remote_ip",
|
||||
"highlights": Array [],
|
||||
"value": "85.181.35.98",
|
||||
},
|
||||
Object {
|
||||
"constant": " ",
|
||||
},
|
||||
Object {
|
||||
"field": "iis.access.user_name",
|
||||
"highlights": Array [],
|
||||
"value": "-",
|
||||
},
|
||||
Object {
|
||||
"constant": " \\"",
|
||||
},
|
||||
Object {
|
||||
"field": "iis.access.method",
|
||||
"highlights": Array [],
|
||||
"value": "GET",
|
||||
},
|
||||
Object {
|
||||
"constant": " ",
|
||||
},
|
||||
Object {
|
||||
"field": "iis.access.url",
|
||||
"highlights": Array [],
|
||||
"value": "/",
|
||||
},
|
||||
Object {
|
||||
"constant": " HTTP/",
|
||||
},
|
||||
Object {
|
||||
"field": "iis.access.http_version",
|
||||
"highlights": Array [],
|
||||
"value": "undefined",
|
||||
},
|
||||
Object {
|
||||
"constant": "\\" ",
|
||||
},
|
||||
Object {
|
||||
"field": "iis.access.response_code",
|
||||
"highlights": Array [],
|
||||
"value": "200",
|
||||
},
|
||||
Object {
|
||||
"constant": " ",
|
||||
},
|
||||
Object {
|
||||
"field": "iis.access.body_sent.bytes",
|
||||
"highlights": Array [],
|
||||
"value": "undefined",
|
||||
},
|
||||
]
|
||||
`);
|
||||
});
|
||||
|
||||
test('iis 7.5 access log', () => {
|
||||
const flattenedDocument = {
|
||||
'@timestamp': '2018-08-28T18:24:25.000Z',
|
||||
'event.dataset': 'iis.access',
|
||||
'fileset.module': 'iis',
|
||||
'fileset.name': 'access',
|
||||
'iis.access.method': 'GET',
|
||||
'iis.access.port': '80',
|
||||
'iis.access.query_string': '-',
|
||||
'iis.access.remote_ip': '10.100.118.31',
|
||||
'iis.access.request_time_ms': '792',
|
||||
'iis.access.response_code': '404',
|
||||
'iis.access.server_ip': '10.100.220.70',
|
||||
'iis.access.sub_status': '4',
|
||||
'iis.access.url': '/',
|
||||
'iis.access.user_agent.device': 'Other',
|
||||
'iis.access.user_agent.name': 'Other',
|
||||
'iis.access.user_agent.original':
|
||||
'Mozilla/4.0+(compatible;+MSIE+7.0;+Windows+NT+6.3;+WOW64;+Trident/7.0;+.NET4.0E;+.NET4.0C;+.NET+CLR+3.5.30729;+.NET+CLR[+2.0.50727](tel:+2050727);+.NET+CLR+3.0.30729)',
|
||||
'iis.access.user_agent.os': 'Windows',
|
||||
'iis.access.user_agent.os_name': 'Windows',
|
||||
'iis.access.user_name': '-',
|
||||
'iis.access.win32_status': '2',
|
||||
'input.type': 'log',
|
||||
offset: 244,
|
||||
'prospector.type': 'log',
|
||||
};
|
||||
|
||||
expect(format(flattenedDocument)).toMatchInlineSnapshot(`
|
||||
Array [
|
||||
Object {
|
||||
"constant": "[IIS][access] ",
|
||||
},
|
||||
Object {
|
||||
"field": "iis.access.remote_ip",
|
||||
"highlights": Array [],
|
||||
"value": "10.100.118.31",
|
||||
},
|
||||
Object {
|
||||
"constant": " ",
|
||||
},
|
||||
Object {
|
||||
"field": "iis.access.user_name",
|
||||
"highlights": Array [],
|
||||
"value": "-",
|
||||
},
|
||||
Object {
|
||||
"constant": " \\"",
|
||||
},
|
||||
Object {
|
||||
"field": "iis.access.method",
|
||||
"highlights": Array [],
|
||||
"value": "GET",
|
||||
},
|
||||
Object {
|
||||
"constant": " ",
|
||||
},
|
||||
Object {
|
||||
"field": "iis.access.url",
|
||||
"highlights": Array [],
|
||||
"value": "/",
|
||||
},
|
||||
Object {
|
||||
"constant": " HTTP/",
|
||||
},
|
||||
Object {
|
||||
"field": "iis.access.http_version",
|
||||
"highlights": Array [],
|
||||
"value": "undefined",
|
||||
},
|
||||
Object {
|
||||
"constant": "\\" ",
|
||||
},
|
||||
Object {
|
||||
"field": "iis.access.response_code",
|
||||
"highlights": Array [],
|
||||
"value": "404",
|
||||
},
|
||||
Object {
|
||||
"constant": " ",
|
||||
},
|
||||
Object {
|
||||
"field": "iis.access.body_sent.bytes",
|
||||
"highlights": Array [],
|
||||
"value": "undefined",
|
||||
},
|
||||
]
|
||||
`);
|
||||
});
|
||||
|
||||
test('iis error log', () => {
|
||||
const flattenedDocument = {
|
||||
'@timestamp': '2018-01-01T08:09:10.000Z',
|
||||
'event.dataset': 'iis.error',
|
||||
'fileset.module': 'iis',
|
||||
'fileset.name': 'error',
|
||||
'iis.error.http_version': '1.1',
|
||||
'iis.error.method': 'GET',
|
||||
'iis.error.queue_name': '-',
|
||||
'iis.error.reason_phrase': 'ConnLimit',
|
||||
'iis.error.remote_ip': '172.31.77.6',
|
||||
'iis.error.remote_port': '2094',
|
||||
'iis.error.response_code': '503',
|
||||
'iis.error.server_ip': '172.31.77.6',
|
||||
'iis.error.server_port': '80',
|
||||
'iis.error.url': '/qos/1kbfile.txt',
|
||||
'input.type': 'log',
|
||||
offset: 186,
|
||||
'prospector.type': 'log',
|
||||
};
|
||||
|
||||
expect(format(flattenedDocument)).toMatchInlineSnapshot(`
|
||||
Array [
|
||||
Object {
|
||||
"constant": "[IIS][error] ",
|
||||
},
|
||||
Object {
|
||||
"field": "iis.error.remote_ip",
|
||||
"highlights": Array [],
|
||||
"value": "172.31.77.6",
|
||||
},
|
||||
Object {
|
||||
"constant": " \\"",
|
||||
},
|
||||
Object {
|
||||
"field": "iis.error.method",
|
||||
"highlights": Array [],
|
||||
"value": "GET",
|
||||
},
|
||||
Object {
|
||||
"constant": " ",
|
||||
},
|
||||
Object {
|
||||
"field": "iis.error.url",
|
||||
"highlights": Array [],
|
||||
"value": "/qos/1kbfile.txt",
|
||||
},
|
||||
Object {
|
||||
"constant": " HTTP/",
|
||||
},
|
||||
Object {
|
||||
"field": "iis.error.http_version",
|
||||
"highlights": Array [],
|
||||
"value": "1.1",
|
||||
},
|
||||
Object {
|
||||
"constant": "\\" ",
|
||||
},
|
||||
Object {
|
||||
"field": "iis.error.response_code",
|
||||
"highlights": Array [],
|
||||
"value": "503",
|
||||
},
|
||||
Object {
|
||||
"constant": " ",
|
||||
},
|
||||
Object {
|
||||
"field": "iis.error.reason_phrase",
|
||||
"highlights": Array [],
|
||||
"value": "ConnLimit",
|
||||
},
|
||||
]
|
||||
`);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,122 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
export const filebeatIisRules = [
|
||||
{
|
||||
// pre-ECS
|
||||
when: {
|
||||
exists: ['iis.access.method'],
|
||||
},
|
||||
format: [
|
||||
{
|
||||
constant: '[IIS][access] ',
|
||||
},
|
||||
{
|
||||
field: 'iis.access.remote_ip',
|
||||
},
|
||||
{
|
||||
constant: ' ',
|
||||
},
|
||||
{
|
||||
field: 'iis.access.user_name',
|
||||
},
|
||||
{
|
||||
constant: ' "',
|
||||
},
|
||||
{
|
||||
field: 'iis.access.method',
|
||||
},
|
||||
{
|
||||
constant: ' ',
|
||||
},
|
||||
{
|
||||
field: 'iis.access.url',
|
||||
},
|
||||
{
|
||||
constant: ' HTTP/',
|
||||
},
|
||||
{
|
||||
field: 'iis.access.http_version',
|
||||
},
|
||||
{
|
||||
constant: '" ',
|
||||
},
|
||||
{
|
||||
field: 'iis.access.response_code',
|
||||
},
|
||||
{
|
||||
constant: ' ',
|
||||
},
|
||||
{
|
||||
field: 'iis.access.body_sent.bytes',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
// pre-ECS
|
||||
when: {
|
||||
exists: ['iis.error.url'],
|
||||
},
|
||||
format: [
|
||||
{
|
||||
constant: '[IIS][error] ',
|
||||
},
|
||||
{
|
||||
field: 'iis.error.remote_ip',
|
||||
},
|
||||
{
|
||||
constant: ' "',
|
||||
},
|
||||
{
|
||||
field: 'iis.error.method',
|
||||
},
|
||||
{
|
||||
constant: ' ',
|
||||
},
|
||||
{
|
||||
field: 'iis.error.url',
|
||||
},
|
||||
{
|
||||
constant: ' HTTP/',
|
||||
},
|
||||
{
|
||||
field: 'iis.error.http_version',
|
||||
},
|
||||
{
|
||||
constant: '" ',
|
||||
},
|
||||
{
|
||||
field: 'iis.error.response_code',
|
||||
},
|
||||
{
|
||||
constant: ' ',
|
||||
},
|
||||
{
|
||||
field: 'iis.error.reason_phrase',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
// pre-ECS
|
||||
when: {
|
||||
exists: ['iis.error.reason_phrase'],
|
||||
},
|
||||
format: [
|
||||
{
|
||||
constant: '[IIS][error] ',
|
||||
},
|
||||
{
|
||||
field: 'iis.error.remote_ip',
|
||||
},
|
||||
{
|
||||
constant: ' ',
|
||||
},
|
||||
{
|
||||
field: 'iis.error.reason_phrase',
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
|
|
@ -0,0 +1,112 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import { compileFormattingRules } from '../message';
|
||||
import { filebeatLogstashRules } from './filebeat_logstash';
|
||||
|
||||
const { format } = compileFormattingRules(filebeatLogstashRules);
|
||||
|
||||
describe('Filebeat Rules', () => {
|
||||
describe('in pre-ECS format', () => {
|
||||
test('logstash log', () => {
|
||||
const flattenedDocument = {
|
||||
'@timestamp': '2017-10-23T14:20:12.046Z',
|
||||
'event.dataset': 'logstash.log',
|
||||
'fileset.module': 'logstash',
|
||||
'fileset.name': 'log',
|
||||
'input.type': 'log',
|
||||
'logstash.log.level': 'INFO',
|
||||
'logstash.log.message':
|
||||
'Initializing module {:module_name=>"fb_apache", :directory=>"/usr/share/logstash/modules/fb_apache/configuration"}',
|
||||
'logstash.log.module': 'logstash.modules.scaffold',
|
||||
offset: 0,
|
||||
'prospector.type': 'log',
|
||||
};
|
||||
|
||||
expect(format(flattenedDocument)).toMatchInlineSnapshot(`
|
||||
Array [
|
||||
Object {
|
||||
"constant": "[Logstash][",
|
||||
},
|
||||
Object {
|
||||
"field": "logstash.log.level",
|
||||
"highlights": Array [],
|
||||
"value": "INFO",
|
||||
},
|
||||
Object {
|
||||
"constant": "] ",
|
||||
},
|
||||
Object {
|
||||
"field": "logstash.log.module",
|
||||
"highlights": Array [],
|
||||
"value": "logstash.modules.scaffold",
|
||||
},
|
||||
Object {
|
||||
"constant": " - ",
|
||||
},
|
||||
Object {
|
||||
"field": "logstash.log.message",
|
||||
"highlights": Array [],
|
||||
"value": "Initializing module {:module_name=>\\"fb_apache\\", :directory=>\\"/usr/share/logstash/modules/fb_apache/configuration\\"}",
|
||||
},
|
||||
]
|
||||
`);
|
||||
});
|
||||
|
||||
test('logstash slowlog', () => {
|
||||
const flattenedDocument = {
|
||||
'@timestamp': '2017-10-30T09:57:58.243Z',
|
||||
'event.dataset': 'logstash.slowlog',
|
||||
'fileset.module': 'logstash',
|
||||
'fileset.name': 'slowlog',
|
||||
'input.type': 'log',
|
||||
'logstash.slowlog.event':
|
||||
'"{\\"@version\\":\\"1\\",\\"@timestamp\\":\\"2017-10-30T13:57:55.130Z\\",\\"host\\":\\"sashimi\\",\\"sequence\\":0,\\"message\\":\\"Hello world!\\"}"',
|
||||
'logstash.slowlog.level': 'WARN',
|
||||
'logstash.slowlog.message':
|
||||
'event processing time {:plugin_params=>{"time"=>3, "id"=>"e4e12a4e3082615c5427079bf4250dbfa338ebac10f8ea9912d7b98a14f56b8c"}, :took_in_nanos=>3027675106, :took_in_millis=>3027, :event=>"{\\"@version\\":\\"1\\",\\"@timestamp\\":\\"2017-10-30T13:57:55.130Z\\",\\"host\\":\\"sashimi\\",\\"sequence\\":0,\\"message\\":\\"Hello world!\\"}"}',
|
||||
'logstash.slowlog.module': 'slowlog.logstash.filters.sleep',
|
||||
'logstash.slowlog.plugin_name': 'sleep',
|
||||
'logstash.slowlog.plugin_params':
|
||||
'{"time"=>3, "id"=>"e4e12a4e3082615c5427079bf4250dbfa338ebac10f8ea9912d7b98a14f56b8c"}',
|
||||
'logstash.slowlog.plugin_type': 'filters',
|
||||
'logstash.slowlog.took_in_millis': 3027,
|
||||
'logstash.slowlog.took_in_nanos': 3027675106,
|
||||
offset: 0,
|
||||
'prospector.type': 'log',
|
||||
};
|
||||
|
||||
expect(format(flattenedDocument)).toMatchInlineSnapshot(`
|
||||
Array [
|
||||
Object {
|
||||
"constant": "[Logstash][",
|
||||
},
|
||||
Object {
|
||||
"field": "logstash.slowlog.level",
|
||||
"highlights": Array [],
|
||||
"value": "WARN",
|
||||
},
|
||||
Object {
|
||||
"constant": "] ",
|
||||
},
|
||||
Object {
|
||||
"field": "logstash.slowlog.module",
|
||||
"highlights": Array [],
|
||||
"value": "slowlog.logstash.filters.sleep",
|
||||
},
|
||||
Object {
|
||||
"constant": " - ",
|
||||
},
|
||||
Object {
|
||||
"field": "logstash.slowlog.message",
|
||||
"highlights": Array [],
|
||||
"value": "event processing time {:plugin_params=>{\\"time\\"=>3, \\"id\\"=>\\"e4e12a4e3082615c5427079bf4250dbfa338ebac10f8ea9912d7b98a14f56b8c\\"}, :took_in_nanos=>3027675106, :took_in_millis=>3027, :event=>\\"{\\\\\\"@version\\\\\\":\\\\\\"1\\\\\\",\\\\\\"@timestamp\\\\\\":\\\\\\"2017-10-30T13:57:55.130Z\\\\\\",\\\\\\"host\\\\\\":\\\\\\"sashimi\\\\\\",\\\\\\"sequence\\\\\\":0,\\\\\\"message\\\\\\":\\\\\\"Hello world!\\\\\\"}\\"}",
|
||||
},
|
||||
]
|
||||
`);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
export const filebeatLogstashRules = [
|
||||
{
|
||||
// pre-ECS
|
||||
when: {
|
||||
exists: ['logstash.log.message'],
|
||||
},
|
||||
format: [
|
||||
{
|
||||
constant: '[Logstash][',
|
||||
},
|
||||
{
|
||||
field: 'logstash.log.level',
|
||||
},
|
||||
{
|
||||
constant: '] ',
|
||||
},
|
||||
{
|
||||
field: 'logstash.log.module',
|
||||
},
|
||||
{
|
||||
constant: ' - ',
|
||||
},
|
||||
{
|
||||
field: 'logstash.log.message',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
// pre-ECS
|
||||
when: {
|
||||
exists: ['logstash.slowlog.message'],
|
||||
},
|
||||
format: [
|
||||
{
|
||||
constant: '[Logstash][',
|
||||
},
|
||||
{
|
||||
field: 'logstash.slowlog.level',
|
||||
},
|
||||
{
|
||||
constant: '] ',
|
||||
},
|
||||
{
|
||||
field: 'logstash.slowlog.module',
|
||||
},
|
||||
{
|
||||
constant: ' - ',
|
||||
},
|
||||
{
|
||||
field: 'logstash.slowlog.message',
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
|
|
@ -0,0 +1,52 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import { compileFormattingRules } from '../message';
|
||||
import { filebeatMongodbRules } from './filebeat_mongodb';
|
||||
|
||||
const { format } = compileFormattingRules(filebeatMongodbRules);
|
||||
|
||||
describe('Filebeat Rules', () => {
|
||||
describe('in pre-ECS format', () => {
|
||||
test('mongodb log', () => {
|
||||
const flattenedDocument = {
|
||||
'@timestamp': '2018-02-05T12:44:56.677Z',
|
||||
'event.dataset': 'mongodb.log',
|
||||
'fileset.module': 'mongodb',
|
||||
'fileset.name': 'log',
|
||||
'input.type': 'log',
|
||||
'mongodb.log.component': 'STORAGE',
|
||||
'mongodb.log.context': 'initandlisten',
|
||||
'mongodb.log.message':
|
||||
'wiredtiger_open config: create,cache_size=8G,session_max=20000,eviction=(threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),file_manager=(close_idle_time=100000),checkpoint=(wait=60,log_size=2GB),statistics_log=(wait=0),',
|
||||
'mongodb.log.severity': 'I',
|
||||
offset: 281,
|
||||
'prospector.type': 'log',
|
||||
};
|
||||
|
||||
expect(format(flattenedDocument)).toMatchInlineSnapshot(`
|
||||
Array [
|
||||
Object {
|
||||
"constant": "[MongoDB][",
|
||||
},
|
||||
Object {
|
||||
"field": "mongodb.log.component",
|
||||
"highlights": Array [],
|
||||
"value": "STORAGE",
|
||||
},
|
||||
Object {
|
||||
"constant": "] ",
|
||||
},
|
||||
Object {
|
||||
"field": "mongodb.log.message",
|
||||
"highlights": Array [],
|
||||
"value": "wiredtiger_open config: create,cache_size=8G,session_max=20000,eviction=(threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),file_manager=(close_idle_time=100000),checkpoint=(wait=60,log_size=2GB),statistics_log=(wait=0),",
|
||||
},
|
||||
]
|
||||
`);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
export const filebeatMongodbRules = [
|
||||
{
|
||||
// pre-ECS
|
||||
when: {
|
||||
exists: ['mongodb.log.message'],
|
||||
},
|
||||
format: [
|
||||
{
|
||||
constant: '[MongoDB][',
|
||||
},
|
||||
{
|
||||
field: 'mongodb.log.component',
|
||||
},
|
||||
{
|
||||
constant: '] ',
|
||||
},
|
||||
{
|
||||
field: 'mongodb.log.message',
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
|
|
@ -0,0 +1,77 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import { compileFormattingRules } from '../message';
|
||||
import { filebeatOsqueryRules } from './filebeat_osquery';
|
||||
|
||||
const { format } = compileFormattingRules(filebeatOsqueryRules);
|
||||
|
||||
describe('Filebeat Rules', () => {
|
||||
describe('in pre-ECS format', () => {
|
||||
test('osquery result log', () => {
|
||||
const flattenedDocument = {
|
||||
'@timestamp': '2017-12-28T14:40:08.000Z',
|
||||
'event.dataset': 'osquery.result',
|
||||
'fileset.module': 'osquery',
|
||||
'fileset.name': 'result',
|
||||
'input.type': 'log',
|
||||
offset: 0,
|
||||
'osquery.result.action': 'removed',
|
||||
'osquery.result.calendar_time': 'Thu Dec 28 14:40:08 2017 UTC',
|
||||
'osquery.result.columns': {
|
||||
blocks: '122061322',
|
||||
blocks_available: '75966945',
|
||||
blocks_free: '121274885',
|
||||
blocks_size: '4096',
|
||||
device: '/dev/disk1s4',
|
||||
device_alias: '/dev/disk1s4',
|
||||
flags: '345018372',
|
||||
inodes: '9223372036854775807',
|
||||
inodes_free: '9223372036854775804',
|
||||
path: '/private/var/vm',
|
||||
type: 'apfs',
|
||||
},
|
||||
'osquery.result.counter': '1',
|
||||
'osquery.result.decorations.host_uuid': '4AB2906D-5516-5794-AF54-86D1D7F533F3',
|
||||
'osquery.result.decorations.username': 'tsg',
|
||||
'osquery.result.epoch': '0',
|
||||
'osquery.result.host_identifier': '192-168-0-4.rdsnet.ro',
|
||||
'osquery.result.name': 'pack_it-compliance_mounts',
|
||||
'osquery.result.unix_time': '1514472008',
|
||||
'prospector.type': 'log',
|
||||
};
|
||||
|
||||
expect(format(flattenedDocument)).toMatchInlineSnapshot(`
|
||||
Array [
|
||||
Object {
|
||||
"constant": "[Osquery][",
|
||||
},
|
||||
Object {
|
||||
"field": "osquery.result.action",
|
||||
"highlights": Array [],
|
||||
"value": "removed",
|
||||
},
|
||||
Object {
|
||||
"constant": "] ",
|
||||
},
|
||||
Object {
|
||||
"field": "osquery.result.host_identifier",
|
||||
"highlights": Array [],
|
||||
"value": "192-168-0-4.rdsnet.ro",
|
||||
},
|
||||
Object {
|
||||
"constant": " ",
|
||||
},
|
||||
Object {
|
||||
"field": "osquery.result.columns",
|
||||
"highlights": Array [],
|
||||
"value": "{\\"blocks\\":\\"122061322\\",\\"blocks_available\\":\\"75966945\\",\\"blocks_free\\":\\"121274885\\",\\"blocks_size\\":\\"4096\\",\\"device\\":\\"/dev/disk1s4\\",\\"device_alias\\":\\"/dev/disk1s4\\",\\"flags\\":\\"345018372\\",\\"inodes\\":\\"9223372036854775807\\",\\"inodes_free\\":\\"9223372036854775804\\",\\"path\\":\\"/private/var/vm\\",\\"type\\":\\"apfs\\"}",
|
||||
},
|
||||
]
|
||||
`);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
export const filebeatOsqueryRules = [
|
||||
{
|
||||
// pre-ECS
|
||||
when: {
|
||||
exists: ['osquery.result.name'],
|
||||
},
|
||||
format: [
|
||||
{
|
||||
constant: '[Osquery][',
|
||||
},
|
||||
{
|
||||
field: 'osquery.result.action',
|
||||
},
|
||||
{
|
||||
constant: '] ',
|
||||
},
|
||||
{
|
||||
field: 'osquery.result.host_identifier',
|
||||
},
|
||||
{
|
||||
constant: ' ',
|
||||
},
|
||||
{
|
||||
field: 'osquery.result.columns',
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
|
|
@ -0,0 +1,124 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import { compileFormattingRules } from '../message';
|
||||
import { filebeatTraefikRules } from './filebeat_traefik';
|
||||
|
||||
const { format } = compileFormattingRules(filebeatTraefikRules);
|
||||
|
||||
describe('Filebeat Rules', () => {
|
||||
describe('in pre-ECS format', () => {
|
||||
test('traefik access log', () => {
|
||||
const flattenedDocument = {
|
||||
'@timestamp': '2017-10-02T20:22:08.000Z',
|
||||
'event.dataset': 'traefik.access',
|
||||
'fileset.module': 'traefik',
|
||||
'fileset.name': 'access',
|
||||
'input.type': 'log',
|
||||
offset: 280,
|
||||
'prospector.type': 'log',
|
||||
'traefik.access.backend_url': 'http://172.19.0.3:5601',
|
||||
'traefik.access.body_sent.bytes': 0,
|
||||
'traefik.access.duration': 3,
|
||||
'traefik.access.frontend_name': 'Host-host1',
|
||||
'traefik.access.geoip.city_name': 'Berlin',
|
||||
'traefik.access.geoip.continent_name': 'Europe',
|
||||
'traefik.access.geoip.country_iso_code': 'DE',
|
||||
'traefik.access.geoip.location.lat': 52.4908,
|
||||
'traefik.access.geoip.location.lon': 13.3275,
|
||||
'traefik.access.geoip.region_iso_code': 'DE-BE',
|
||||
'traefik.access.geoip.region_name': 'Land Berlin',
|
||||
'traefik.access.http_version': '1.1',
|
||||
'traefik.access.method': 'GET',
|
||||
'traefik.access.referrer': 'http://example.com/login',
|
||||
'traefik.access.remote_ip': '85.181.35.98',
|
||||
'traefik.access.request_count': 271,
|
||||
'traefik.access.response_code': '304',
|
||||
'traefik.access.url': '/ui/favicons/favicon.ico',
|
||||
'traefik.access.user_agent.device': 'Other',
|
||||
'traefik.access.user_agent.major': '61',
|
||||
'traefik.access.user_agent.minor': '0',
|
||||
'traefik.access.user_agent.name': 'Chrome',
|
||||
'traefik.access.user_agent.original':
|
||||
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36',
|
||||
'traefik.access.user_agent.os': 'Linux',
|
||||
'traefik.access.user_agent.os_name': 'Linux',
|
||||
'traefik.access.user_agent.patch': '3163',
|
||||
'traefik.access.user_identifier': '-',
|
||||
'traefik.access.user_name': '-',
|
||||
};
|
||||
|
||||
expect(format(flattenedDocument)).toMatchInlineSnapshot(`
|
||||
Array [
|
||||
Object {
|
||||
"constant": "[Traefik][access] ",
|
||||
},
|
||||
Object {
|
||||
"field": "traefik.access.remote_ip",
|
||||
"highlights": Array [],
|
||||
"value": "85.181.35.98",
|
||||
},
|
||||
Object {
|
||||
"constant": " ",
|
||||
},
|
||||
Object {
|
||||
"field": "traefik.access.frontend_name",
|
||||
"highlights": Array [],
|
||||
"value": "Host-host1",
|
||||
},
|
||||
Object {
|
||||
"constant": " -> ",
|
||||
},
|
||||
Object {
|
||||
"field": "traefik.access.backend_url",
|
||||
"highlights": Array [],
|
||||
"value": "http://172.19.0.3:5601",
|
||||
},
|
||||
Object {
|
||||
"constant": " \\"",
|
||||
},
|
||||
Object {
|
||||
"field": "traefik.access.method",
|
||||
"highlights": Array [],
|
||||
"value": "GET",
|
||||
},
|
||||
Object {
|
||||
"constant": " ",
|
||||
},
|
||||
Object {
|
||||
"field": "traefik.access.url",
|
||||
"highlights": Array [],
|
||||
"value": "/ui/favicons/favicon.ico",
|
||||
},
|
||||
Object {
|
||||
"constant": " HTTP/",
|
||||
},
|
||||
Object {
|
||||
"field": "traefik.access.http_version",
|
||||
"highlights": Array [],
|
||||
"value": "1.1",
|
||||
},
|
||||
Object {
|
||||
"constant": "\\" ",
|
||||
},
|
||||
Object {
|
||||
"field": "traefik.access.response_code",
|
||||
"highlights": Array [],
|
||||
"value": "304",
|
||||
},
|
||||
Object {
|
||||
"constant": " ",
|
||||
},
|
||||
Object {
|
||||
"field": "traefik.access.body_sent.bytes",
|
||||
"highlights": Array [],
|
||||
"value": "0",
|
||||
},
|
||||
]
|
||||
`);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
export const filebeatTraefikRules = [
|
||||
{
|
||||
// pre-ECS
|
||||
when: {
|
||||
exists: ['traefik.access.method'],
|
||||
},
|
||||
format: [
|
||||
{
|
||||
constant: '[Traefik][access] ',
|
||||
},
|
||||
{
|
||||
field: 'traefik.access.remote_ip',
|
||||
},
|
||||
{
|
||||
constant: ' ',
|
||||
},
|
||||
{
|
||||
field: 'traefik.access.frontend_name',
|
||||
},
|
||||
{
|
||||
constant: ' -> ',
|
||||
},
|
||||
{
|
||||
field: 'traefik.access.backend_url',
|
||||
},
|
||||
{
|
||||
constant: ' "',
|
||||
},
|
||||
{
|
||||
field: 'traefik.access.method',
|
||||
},
|
||||
{
|
||||
constant: ' ',
|
||||
},
|
||||
{
|
||||
field: 'traefik.access.url',
|
||||
},
|
||||
{
|
||||
constant: ' HTTP/',
|
||||
},
|
||||
{
|
||||
field: 'traefik.access.http_version',
|
||||
},
|
||||
{
|
||||
constant: '" ',
|
||||
},
|
||||
{
|
||||
field: 'traefik.access.response_code',
|
||||
},
|
||||
{
|
||||
constant: ' ',
|
||||
},
|
||||
{
|
||||
field: 'traefik.access.body_sent.bytes',
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
|
|
@ -6,10 +6,17 @@
|
|||
|
||||
import { filebeatApache2Rules } from './filebeat_apache2';
|
||||
import { filebeatAuditdRules } from './filebeat_auditd';
|
||||
import { filebeatHaproxyRules } from './filebeat_haproxy';
|
||||
import { filebeatIcingaRules } from './filebeat_icinga';
|
||||
import { filebeatIisRules } from './filebeat_iis';
|
||||
import { filebeatLogstashRules } from './filebeat_logstash';
|
||||
import { filebeatMongodbRules } from './filebeat_mongodb';
|
||||
import { filebeatMySQLRules } from './filebeat_mysql';
|
||||
import { filebeatNginxRules } from './filebeat_nginx';
|
||||
import { filebeatOsqueryRules } from './filebeat_osquery';
|
||||
import { filebeatRedisRules } from './filebeat_redis';
|
||||
import { filebeatSystemRules } from './filebeat_system';
|
||||
import { filebeatTraefikRules } from './filebeat_traefik';
|
||||
|
||||
import { genericRules } from './generic';
|
||||
|
||||
|
|
@ -20,6 +27,13 @@ export const builtinRules = [
|
|||
...filebeatSystemRules,
|
||||
...filebeatMySQLRules,
|
||||
...filebeatAuditdRules,
|
||||
...filebeatHaproxyRules,
|
||||
...filebeatIcingaRules,
|
||||
...filebeatIisRules,
|
||||
...filebeatLogstashRules,
|
||||
...filebeatMongodbRules,
|
||||
...filebeatOsqueryRules,
|
||||
...filebeatTraefikRules,
|
||||
...genericRules,
|
||||
{
|
||||
when: {
|
||||
|
|
|
|||
|
|
@ -4,6 +4,8 @@
|
|||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import stringify from 'json-stable-stringify';
|
||||
|
||||
import { InfraLogMessageSegment } from '../../../graphql/types';
|
||||
|
||||
export function compileFormattingRules(rules: LogMessageFormattingRule[]) {
|
||||
|
|
@ -125,13 +127,16 @@ const compileFieldReferenceFormattingInstruction = (
|
|||
'field' in formattingInstruction
|
||||
? {
|
||||
formattingFields: [formattingInstruction.field],
|
||||
format: (fields: Fields) => [
|
||||
{
|
||||
field: formattingInstruction.field,
|
||||
value: `${fields[formattingInstruction.field]}`,
|
||||
highlights: [],
|
||||
},
|
||||
],
|
||||
format: (fields: Fields) => {
|
||||
const value = fields[formattingInstruction.field];
|
||||
return [
|
||||
{
|
||||
field: formattingInstruction.field,
|
||||
value: typeof value === 'object' ? stringify(value) : `${value}`,
|
||||
highlights: [],
|
||||
},
|
||||
];
|
||||
},
|
||||
}
|
||||
: null;
|
||||
|
||||
|
|
@ -150,7 +155,7 @@ const compileConstantFormattingInstruction = (
|
|||
: null;
|
||||
|
||||
interface Fields {
|
||||
[fieldName: string]: string | number | boolean | null;
|
||||
[fieldName: string]: string | number | object | boolean | null;
|
||||
}
|
||||
|
||||
interface LogMessageFormattingRule {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue