github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 17:28:26 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Infra UI] ECS Migration (#28205) (#29965)

Browse source 
* updating fields

* Migrate to ESC event.dataset

* Migragte fields to ECS fields

* renaming variable

* Reverting back to host.name

* Changing from Top Hits to Terms Agg for getting node name; change host.name back to host.hostname for name.

* Changing back to host.name

* Moving from using the document source to an aggregation for node name

* Updating tests with new data format and data.

* removing unused fields

* adding test data for docker 6.6.0

* Adding docker tests for 6.6.0

* Fixing jest tests

* Fixing tests

* Adding the most critical line of code in the entire project

* Fix ECS-compatible apache rules and restore old ones

* Fix ECS-compatible nginx rules and restore old ones

* Add tests for apache2 in ECS and pre-ECS format

* Add tests for nginx in ECS and pre-ECS format

* removing console.log

* Fixing tests
This commit is contained in:
Chris Cowan 2019-02-04 10:31:54 -07:00 • committed by GitHub
parent d8efde6907
commit fe9748583e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
24 changed files with 45158 additions and 391 deletions
Download patch file Download diff file Expand all files Collapse all files

16
x-pack/plugins/infra/public/components/waffle/waffle_group_by_controls.tsx
View file

@ -47,16 +47,16 @@ const getOptions = (
[InfraNodeType.pod]: ['kubernetes.namespace', 'kubernetes.node.name'].map(mapFieldToOption),
[InfraNodeType.container]: [
'host.name',
'meta.cloud.availability_zone',
'meta.cloud.machine_type',
'meta.cloud.project_id',
'meta.cloud.provider',
'cloud.availability_zone',
'cloud.machine_type',
'cloud.project_id',
'cloud.provider',
].map(mapFieldToOption),
[InfraNodeType.host]: [
'meta.cloud.availability_zone',
'meta.cloud.machine_type',
'meta.cloud.project_id',
'meta.cloud.provider',
'cloud.availability_zone',
'cloud.machine_type',
'cloud.project_id',
'cloud.provider',
].map(mapFieldToOption),
};
}

3
x-pack/plugins/infra/server/lib/adapters/framework/adapter_types.ts
View file

@ -157,9 +157,6 @@ export interface InfraDateRangeAggregationResponse {
export interface InfraMetadataAggregationBucket {
key: string;
names?: {
buckets: InfraMetadataAggregationBucket[];
};
}
export interface InfraMetadataAggregationResponse {

52
x-pack/plugins/infra/server/lib/adapters/metadata/elasticsearch_metadata_adapter.ts
View file

@ -4,7 +4,7 @@
* you may not use this file except in compliance with the Elastic License.
*/
import { first, get } from 'lodash';
import { get } from 'lodash';
import { InfraSourceConfiguration } from '../../sources';
import {
InfraBackendFrameworkAdapter,
@ -37,22 +37,19 @@ export class ElasticsearchMetadataAdapter implements InfraMetadataAdapter {
},
},
},
size: 1,
_source: [NAME_FIELDS[nodeType]],
size: 0,
aggs: {
nodeName: {
terms: {
field: NAME_FIELDS[nodeType],
size: 1,
},
},
metrics: {
terms: {
field: 'metricset.module',
field: 'event.dataset',
size: 1000,
},
aggs: {
names: {
terms: {
field: 'metricset.name',
size: 1000,
},
},
},
},
},
},
@ -60,7 +57,7 @@ export class ElasticsearchMetadataAdapter implements InfraMetadataAdapter {
const response = await this.framework.callWithRequest<
any,
{ metrics?: InfraMetadataAggregationResponse }
{ metrics?: InfraMetadataAggregationResponse; nodeName?: InfraMetadataAggregationResponse }
>(req, 'search', metricQuery);
const buckets =
@ -68,11 +65,9 @@ export class ElasticsearchMetadataAdapter implements InfraMetadataAdapter {
? response.aggregations.metrics.buckets
: [];
const sampleDoc = first(response.hits.hits);
return {
id: nodeId,
name: get(sampleDoc, `_source.${NAME_FIELDS[nodeType]}`),
name: get(response, ['aggregations', 'nodeName', 'buckets', 0, 'key'], nodeId),
buckets,
};
}
@ -94,22 +89,19 @@ export class ElasticsearchMetadataAdapter implements InfraMetadataAdapter {
},
},
},
size: 1,
_source: [NAME_FIELDS[nodeType]],
size: 0,
aggs: {
nodeName: {
terms: {
field: NAME_FIELDS[nodeType],
size: 1,
},
},
metrics: {
terms: {
field: 'fileset.module',
field: 'event.dataset',
size: 1000,
},
aggs: {
names: {
terms: {
field: 'fileset.name',
size: 1000,
},
},
},
},
},
},
@ -117,7 +109,7 @@ export class ElasticsearchMetadataAdapter implements InfraMetadataAdapter {
const response = await this.framework.callWithRequest<
any,
{ metrics?: InfraMetadataAggregationResponse }
{ metrics?: InfraMetadataAggregationResponse; nodeName?: InfraMetadataAggregationResponse }
>(req, 'search', logQuery);
const buckets =
@ -125,11 +117,9 @@ export class ElasticsearchMetadataAdapter implements InfraMetadataAdapter {
? response.aggregations.metrics.buckets
: [];
const sampleDoc = first(response.hits.hits);
return {
id: nodeId,
name: get(sampleDoc, `_source.${NAME_FIELDS[nodeType]}`),
name: get(response, ['aggregations', 'nodeName', 'buckets', 0, 'key'], nodeId),
buckets,
};
}

8
x-pack/plugins/infra/server/lib/adapters/metrics/models/nginx/nginx_hits.ts
View file

@ -23,7 +23,7 @@ export const nginxHits: InfraMetricModelCreator = (timeField, indexPattern, inte
},
],
split_mode: 'filter',
filter: 'nginx.access.response_code:[200 TO 299]',
filter: 'http.response.status_code:[200 TO 299]',
},
{
id: '300s',
@ -34,7 +34,7 @@ export const nginxHits: InfraMetricModelCreator = (timeField, indexPattern, inte
},
],
split_mode: 'filter',
filter: 'nginx.access.response_code:[300 TO 399]',
filter: 'http.response.status_code:[300 TO 399]',
},
{
id: '400s',
@ -45,7 +45,7 @@ export const nginxHits: InfraMetricModelCreator = (timeField, indexPattern, inte
},
],
split_mode: 'filter',
filter: 'nginx.access.response_code:[400 TO 499]',
filter: 'http.response.status_code:[400 TO 499]',
},
{
id: '500s',
@ -56,7 +56,7 @@ export const nginxHits: InfraMetricModelCreator = (timeField, indexPattern, inte
},
],
split_mode: 'filter',
filter: 'nginx.access.response_code:[500 TO 599]',
filter: 'http.response.status_code:[500 TO 599]',
},
],
});

2
x-pack/plugins/infra/server/lib/adapters/nodes/constants.ts
View file

@ -11,5 +11,5 @@ export const NODE_REQUEST_PARTITION_FACTOR = 1.2;
export const NAME_FIELDS = {
[InfraNodeType.host]: 'host.name',
[InfraNodeType.pod]: 'kubernetes.pod.name',
[InfraNodeType.container]: 'docker.container.name',
[InfraNodeType.container]: 'container.name',
};

5
x-pack/plugins/infra/server/lib/adapters/nodes/lib/create_node_item.ts
View file

@ -9,7 +9,6 @@ import moment from 'moment';
import { InfraMetricType, InfraNode, InfraNodeMetric } from '../../../../graphql/types';
import { InfraBucket, InfraNodeRequestOptions } from '../adapter_types';
import { NAME_FIELDS } from '../constants';
import { getBucketSizeInSeconds } from './get_bucket_size_in_seconds';
// TODO: Break these function into seperate files and expand beyond just documnet count
@ -72,9 +71,9 @@ export function createNodeItem(
node: InfraBucket,
bucket: InfraBucket
): InfraNode {
const nodeDoc = get(node, ['nodeDetails', 'hits', 'hits', 0]);
const nodeDetails = get(node, ['nodeDetails', 'buckets', 0]);
return {
metric: createNodeMetrics(options, node, bucket),
path: [{ value: node.key, label: get(nodeDoc, `_source.${NAME_FIELDS[options.nodeType]}`) }],
path: [{ value: node.key, label: get(nodeDetails, 'key', node.key) }],
} as InfraNode;
}

6
x-pack/plugins/infra/server/lib/adapters/nodes/processors/common/nodes_processor.ts
View file

@ -26,7 +26,6 @@ const nodeTypeToField = (options: InfraProcesorRequestOptions): string => {
};
export const nodesProcessor = (options: InfraProcesorRequestOptions) => {
const { fields } = options.nodeOptions.sourceConfiguration;
return (doc: InfraESSearchBody) => {
const result = cloneDeep(doc);
const field = nodeTypeToField(options);
@ -43,10 +42,9 @@ export const nodesProcessor = (options: InfraProcesorRequestOptions) => {
set(result, 'aggs.waffle.aggs.nodes.aggs', {
nodeDetails: {
top_hits: {
terms: {
field: NAME_FIELDS[options.nodeType],
size: 1,
_source: { includes: [NAME_FIELDS[options.nodeType]] },
sort: [{ [fields.timestamp]: { order: 'desc' } }],
},
},
});

331
x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_apache2.test.ts
View file

@ -8,102 +8,243 @@ import { compileFormattingRules } from '../message';
import { filebeatApache2Rules } from './filebeat_apache2';
const { format } = compileFormattingRules(filebeatApache2Rules);
describe('Filebeat Rules', () => {
test('Apache2 Access', () => {
const event = {
'apache2.access': true,
'apache2.access.remote_ip': '192.168.1.42',
'apache2.access.user_name': 'admin',
'apache2.access.method': 'GET',
'apache2.access.url': '/faqs',
'apache2.access.http_version': '1.1',
'apache2.access.response_code': '200',
'apache2.access.body_sent.bytes': 1024,
};
const message = format(event);
expect(message).toEqual([
{
constant: '[Apache][access] ',
},
{
field: 'apache2.access.remote_ip',
highlights: [],
value: '192.168.1.42',
},
{
constant: ' ',
},
{
field: 'apache2.access.user_name',
highlights: [],
value: 'admin',
},
{
constant: ' "',
},
{
field: 'apache2.access.method',
highlights: [],
value: 'GET',
},
{
constant: ' ',
},
{
field: 'apache2.access.url',
highlights: [],
value: '/faqs',
},
{
constant: ' HTTP/',
},
{
field: 'apache2.access.http_version',
highlights: [],
value: '1.1',
},
{
constant: '" ',
},
{
field: 'apache2.access.response_code',
highlights: [],
value: '200',
},
{
constant: ' ',
},
{
field: 'apache2.access.body_sent.bytes',
highlights: [],
value: '1024',
},
]);
describe('in ECS format', () => {
test('Apache2 Access', () => {
const flattenedDocument = {
'@timestamp': '2016-12-26T16:22:13.000Z',
'ecs.version': '1.0.0-beta2',
'event.dataset': 'apache.access',
'event.module': 'apache',
'fileset.name': 'access',
'http.request.method': 'GET',
'http.request.referrer': '-',
'http.response.body.bytes': 499,
'http.response.status_code': 404,
'http.version': '1.1',
'input.type': 'log',
'log.offset': 73,
'service.type': 'apache',
'source.address': '192.168.33.1',
'source.ip': '192.168.33.1',
'url.original': '/hello',
'user.name': '-',
'user_agent.device': 'Other',
'user_agent.major': '50',
'user_agent.minor': '0',
'user_agent.name': 'Firefox',
'user_agent.original':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0',
'user_agent.os.full_name': 'Mac OS X 10.12',
'user_agent.os.major': '10',
'user_agent.os.minor': '12',
'user_agent.os.name': 'Mac OS X',
};
expect(format(flattenedDocument)).toMatchInlineSnapshot(`
Array [
Object {
"constant": "[Apache][access] ",
},
Object {
"field": "source.ip",
"highlights": Array [],
"value": "192.168.33.1",
},
Object {
"constant": " ",
},
Object {
"field": "user.name",
"highlights": Array [],
"value": "-",
},
Object {
"constant": " \\"",
},
Object {
"field": "http.request.method",
"highlights": Array [],
"value": "GET",
},
Object {
"constant": " ",
},
Object {
"field": "url.original",
"highlights": Array [],
"value": "/hello",
},
Object {
"constant": " HTTP/",
},
Object {
"field": "http.version",
"highlights": Array [],
"value": "1.1",
},
Object {
"constant": "\\" ",
},
Object {
"field": "http.response.status_code",
"highlights": Array [],
"value": "404",
},
Object {
"constant": " ",
},
Object {
"field": "http.response.body.bytes",
"highlights": Array [],
"value": "499",
},
]
`);
});
test('Apache2 Error', () => {
const flattenedDocument = {
'@timestamp': '2016-12-26T16:22:08.000Z',
'ecs.version': '1.0.0-beta2',
'event.dataset': 'apache.error',
'event.module': 'apache',
'fileset.name': 'error',
'input.type': 'log',
'log.level': 'error',
'log.offset': 0,
message: 'File does not exist: /var/www/favicon.ico',
'service.type': 'apache',
'source.address': '192.168.33.1',
'source.ip': '192.168.33.1',
};
expect(format(flattenedDocument)).toMatchInlineSnapshot(`
Array [
Object {
"constant": "[Apache][",
},
Object {
"field": "log.level",
"highlights": Array [],
"value": "error",
},
Object {
"constant": "] ",
},
Object {
"field": "message",
"highlights": Array [],
"value": "File does not exist: /var/www/favicon.ico",
},
]
`);
});
});
test('Apache2 Error', () => {
const event = {
'apache2.error.message':
'AH00489: Apache/2.4.18 (Ubuntu) configured -- resuming normal operations',
'apache2.error.level': 'notice',
};
const message = format(event);
expect(message).toEqual([
{
constant: '[Apache][',
},
{
field: 'apache2.error.level',
highlights: [],
value: 'notice',
},
{
constant: '] ',
},
{
field: 'apache2.error.message',
highlights: [],
value: 'AH00489: Apache/2.4.18 (Ubuntu) configured -- resuming normal operations',
},
]);
describe('in pre-ECS format', () => {
test('Apache2 Access', () => {
const flattenedDocument = {
'apache2.access': true,
'apache2.access.remote_ip': '192.168.1.42',
'apache2.access.user_name': 'admin',
'apache2.access.method': 'GET',
'apache2.access.url': '/faqs',
'apache2.access.http_version': '1.1',
'apache2.access.response_code': '200',
'apache2.access.body_sent.bytes': 1024,
};
expect(format(flattenedDocument)).toMatchInlineSnapshot(`
Array [
Object {
"constant": "[Apache][access] ",
},
Object {
"field": "apache2.access.remote_ip",
"highlights": Array [],
"value": "192.168.1.42",
},
Object {
"constant": " ",
},
Object {
"field": "apache2.access.user_name",
"highlights": Array [],
"value": "admin",
},
Object {
"constant": " \\"",
},
Object {
"field": "apache2.access.method",
"highlights": Array [],
"value": "GET",
},
Object {
"constant": " ",
},
Object {
"field": "apache2.access.url",
"highlights": Array [],
"value": "/faqs",
},
Object {
"constant": " HTTP/",
},
Object {
"field": "apache2.access.http_version",
"highlights": Array [],
"value": "1.1",
},
Object {
"constant": "\\" ",
},
Object {
"field": "apache2.access.response_code",
"highlights": Array [],
"value": "200",
},
Object {
"constant": " ",
},
Object {
"field": "apache2.access.body_sent.bytes",
"highlights": Array [],
"value": "1024",
},
]
`);
});
test('Apache2 Error', () => {
const flattenedDocument = {
'apache2.error.message':
'AH00489: Apache/2.4.18 (Ubuntu) configured -- resuming normal operations',
'apache2.error.level': 'notice',
};
expect(format(flattenedDocument)).toMatchInlineSnapshot(`
Array [
Object {
"constant": "[Apache][",
},
Object {
"field": "apache2.error.level",
"highlights": Array [],
"value": "notice",
},
Object {
"constant": "] ",
},
Object {
"field": "apache2.error.message",
"highlights": Array [],
"value": "AH00489: Apache/2.4.18 (Ubuntu) configured -- resuming normal operations",
},
]
`);
});
});
});

76
x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_apache2.ts
View file

@ -6,6 +6,59 @@
export const filebeatApache2Rules = [
{
// ECS
when: {
values: {
'event.dataset': 'apache.access',
},
},
format: [
{
constant: '[Apache][access] ',
},
{
field: 'source.ip',
},
{
constant: ' ',
},
{
field: 'user.name',
},
{
constant: ' "',
},
{
field: 'http.request.method',
},
{
constant: ' ',
},
{
field: 'url.original',
},
{
constant: ' HTTP/',
},
{
field: 'http.version',
},
{
constant: '" ',
},
{
field: 'http.response.status_code',
},
{
constant: ' ',
},
{
field: 'http.response.body.bytes',
},
],
},
{
// pre-ECS
when: {
exists: ['apache2.access'],
},
@ -55,6 +108,29 @@ export const filebeatApache2Rules = [
],
},
{
// ECS
when: {
values: {
'event.dataset': 'apache.error',
},
},
format: [
{
constant: '[Apache][',
},
{
field: 'log.level',
},
{
constant: '] ',
},
{
field: 'message',
},
],
},
{
// pre-ECS
when: {
exists: ['apache2.error.message'],
},

339
x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_nginx.test.ts
View file

@ -8,106 +8,249 @@ import { compileFormattingRules } from '../message';
import { filebeatNginxRules } from './filebeat_nginx';
const { format } = compileFormattingRules(filebeatNginxRules);
describe('Filebeat Rules', () => {
test('Nginx Access Rule', () => {
const event = {
'nginx.access': true,
'nginx.access.remote_ip': '192.168.1.42',
'nginx.access.user_name': 'admin',
'nginx.access.method': 'GET',
'nginx.access.url': '/faq',
'nginx.access.http_version': '1.1',
'nginx.access.body_sent.bytes': 1024,
'nginx.access.response_code': 200,
};
const message = format(event);
expect(message).toEqual([
{
constant: '[Nginx][access] ',
},
{
field: 'nginx.access.remote_ip',
highlights: [],
value: '192.168.1.42',
},
{
constant: ' ',
},
{
field: 'nginx.access.user_name',
highlights: [],
value: 'admin',
},
{
constant: ' "',
},
{
field: 'nginx.access.method',
highlights: [],
value: 'GET',
},
{
constant: ' ',
},
{
field: 'nginx.access.url',
highlights: [],
value: '/faq',
},
{
constant: ' HTTP/',
},
{
field: 'nginx.access.http_version',
highlights: [],
value: '1.1',
},
{
constant: '" ',
},
{
field: 'nginx.access.response_code',
highlights: [],
value: '200',
},
{
constant: ' ',
},
{
field: 'nginx.access.body_sent.bytes',
highlights: [],
value: '1024',
},
]);
describe('in ECS format', () => {
test('Nginx Access', () => {
const flattenedDocument = {
'@timestamp': '2017-05-29T19:02:48.000Z',
'ecs.version': '1.0.0-beta2',
'event.dataset': 'nginx.access',
'event.module': 'nginx',
'fileset.name': 'access',
'http.request.method': 'GET',
'http.request.referrer': '-',
'http.response.body.bytes': 612,
'http.response.status_code': 404,
'http.version': '1.1',
'input.type': 'log',
'log.offset': 183,
'service.type': 'nginx',
'source.ip': '172.17.0.1',
'url.original': '/stringpatch',
'user.name': '-',
'user_agent.device': 'Other',
'user_agent.major': '15',
'user_agent.minor': '0',
'user_agent.name': 'Firefox Alpha',
'user_agent.original':
'Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2',
'user_agent.os.full_name': 'Windows 7',
'user_agent.os.name': 'Windows 7',
'user_agent.patch': 'a2',
};
expect(format(flattenedDocument)).toMatchInlineSnapshot(`
Array [
Object {
"constant": "[Nginx][access] ",
},
Object {
"field": "source.ip",
"highlights": Array [],
"value": "172.17.0.1",
},
Object {
"constant": " ",
},
Object {
"field": "user.name",
"highlights": Array [],
"value": "-",
},
Object {
"constant": " \\"",
},
Object {
"field": "http.request.method",
"highlights": Array [],
"value": "GET",
},
Object {
"constant": " ",
},
Object {
"field": "url.original",
"highlights": Array [],
"value": "/stringpatch",
},
Object {
"constant": " HTTP/",
},
Object {
"field": "http.version",
"highlights": Array [],
"value": "1.1",
},
Object {
"constant": "\\" ",
},
Object {
"field": "http.response.status_code",
"highlights": Array [],
"value": "404",
},
Object {
"constant": " ",
},
Object {
"field": "http.response.body.bytes",
"highlights": Array [],
"value": "612",
},
]
`);
});
test('Nginx Error', () => {
const flattenedDocument = {
'@timestamp': '2016-10-25T14:49:34.000Z',
'ecs.version': '1.0.0-beta2',
'event.dataset': 'nginx.error',
'event.module': 'nginx',
'fileset.name': 'error',
'input.type': 'log',
'log.level': 'error',
'log.offset': 0,
message:
'open() "/usr/local/Cellar/nginx/1.10.2_1/html/favicon.ico" failed (2: No such file or directory), client: 127.0.0.1, server: localhost, request: "GET /favicon.ico HTTP/1.1", host: "localhost:8080", referrer: "http://localhost:8080/"',
'nginx.error.connection_id': 1,
'process.pid': 54053,
'process.thread.id': 0,
'service.type': 'nginx',
};
expect(format(flattenedDocument)).toMatchInlineSnapshot(`
Array [
Object {
"constant": "[Nginx]",
},
Object {
"constant": "[",
},
Object {
"field": "log.level",
"highlights": Array [],
"value": "error",
},
Object {
"constant": "] ",
},
Object {
"field": "message",
"highlights": Array [],
"value": "open() \\"/usr/local/Cellar/nginx/1.10.2_1/html/favicon.ico\\" failed (2: No such file or directory), client: 127.0.0.1, server: localhost, request: \\"GET /favicon.ico HTTP/1.1\\", host: \\"localhost:8080\\", referrer: \\"http://localhost:8080/\\"",
},
]
`);
});
});
test('Nginx Access Rule', () => {
const event = {
'nginx.error.message':
'connect() failed (111: Connection refused) while connecting to upstream, client: 127.0.0.1, server: localhost, request: "GET /php-status?json= HTTP/1.1", upstream: "fastcgi://[::1]:9000", host: "localhost"',
'nginx.error.level': 'error',
};
const message = format(event);
expect(message).toEqual([
{
constant: '[Nginx]',
},
{
constant: '[',
},
{
field: 'nginx.error.level',
highlights: [],
value: 'error',
},
{
constant: '] ',
},
{
field: 'nginx.error.message',
highlights: [],
value:
describe('in pre-ECS format', () => {
test('Nginx Access', () => {
const flattenedDocument = {
'nginx.access': true,
'nginx.access.remote_ip': '192.168.1.42',
'nginx.access.user_name': 'admin',
'nginx.access.method': 'GET',
'nginx.access.url': '/faq',
'nginx.access.http_version': '1.1',
'nginx.access.body_sent.bytes': 1024,
'nginx.access.response_code': 200,
};
expect(format(flattenedDocument)).toMatchInlineSnapshot(`
Array [
Object {
"constant": "[Nginx][access] ",
},
Object {
"field": "nginx.access.remote_ip",
"highlights": Array [],
"value": "192.168.1.42",
},
Object {
"constant": " ",
},
Object {
"field": "nginx.access.user_name",
"highlights": Array [],
"value": "admin",
},
Object {
"constant": " \\"",
},
Object {
"field": "nginx.access.method",
"highlights": Array [],
"value": "GET",
},
Object {
"constant": " ",
},
Object {
"field": "nginx.access.url",
"highlights": Array [],
"value": "/faq",
},
Object {
"constant": " HTTP/",
},
Object {
"field": "nginx.access.http_version",
"highlights": Array [],
"value": "1.1",
},
Object {
"constant": "\\" ",
},
Object {
"field": "nginx.access.response_code",
"highlights": Array [],
"value": "200",
},
Object {
"constant": " ",
},
Object {
"field": "nginx.access.body_sent.bytes",
"highlights": Array [],
"value": "1024",
},
]
`);
});
test('Nginx Error', () => {
const flattenedDocument = {
'nginx.error.message':
'connect() failed (111: Connection refused) while connecting to upstream, client: 127.0.0.1, server: localhost, request: "GET /php-status?json= HTTP/1.1", upstream: "fastcgi://[::1]:9000", host: "localhost"',
},
]);
'nginx.error.level': 'error',
};
expect(format(flattenedDocument)).toMatchInlineSnapshot(`
Array [
Object {
"constant": "[Nginx]",
},
Object {
"constant": "[",
},
Object {
"field": "nginx.error.level",
"highlights": Array [],
"value": "error",
},
Object {
"constant": "] ",
},
Object {
"field": "nginx.error.message",
"highlights": Array [],
"value": "connect() failed (111: Connection refused) while connecting to upstream, client: 127.0.0.1, server: localhost, request: \\"GET /php-status?json= HTTP/1.1\\", upstream: \\"fastcgi://[::1]:9000\\", host: \\"localhost\\"",
},
]
`);
});
});
});

79
x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_nginx.ts
View file

@ -6,6 +6,59 @@
export const filebeatNginxRules = [
{
// ECS
when: {
values: {
'event.dataset': 'nginx.access',
},
},
format: [
{
constant: '[Nginx][access] ',
},
{
field: 'source.ip',
},
{
constant: ' ',
},
{
field: 'user.name',
},
{
constant: ' "',
},
{
field: 'http.request.method',
},
{
constant: ' ',
},
{
field: 'url.original',
},
{
constant: ' HTTP/',
},
{
field: 'http.version',
},
{
constant: '" ',
},
{
field: 'http.response.status_code',
},
{
constant: ' ',
},
{
field: 'http.response.body.bytes',
},
],
},
{
// pre-ECS
when: {
exists: ['nginx.access'],
},
@ -55,6 +108,32 @@ export const filebeatNginxRules = [
],
},
{
// ECS
when: {
values: {
'event.dataset': 'nginx.error',
},
},
format: [
{
constant: '[Nginx]',
},
{
constant: '[',
},
{
field: 'log.level',
},
{
constant: '] ',
},
{
field: 'message',
},
],
},
{
// pre-ECS
when: {
exists: ['nginx.error.message'],
},

4
x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/index.ts
View file

@ -23,14 +23,14 @@ export const builtinRules = [
...genericRules,
{
when: {
exists: ['source'],
exists: ['log.path'],
},
format: [
{
constant: 'failed to format message from ',
},
{
field: 'source',
field: 'log.path',
},
],
},

12
x-pack/plugins/infra/server/lib/domains/metadata_domain/metadata_domain.ts
View file

@ -43,17 +43,7 @@ export class InfraMetadataDomain {
const pickMetadata = (buckets: InfraMetadataAggregationBucket[]): string[] => {
if (buckets) {
const metadata = buckets
.map(module => {
if (module.names) {
return module.names.buckets.map(name => {
return `${module.key}.${name.key}`;
});
} else {
return [];
}
})
.reduce((a: string[], b: string[]) => a.concat(b), []);
const metadata = buckets.map(bucket => bucket.key);
return metadata;
} else {
return [];

20
x-pack/test/api_integration/apis/infra/constants.ts Normal file
View file

@ -0,0 +1,20 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
export const DATES = {
'7.0.0': {
hosts: {
min: 1547571261002,
max: 1547571831033,
},
},
'6.6.0': {
docker: {
min: 1547578132289,
max: 1547579090048,
},
},
};

1
x-pack/test/api_integration/apis/infra/index.js
View file

@ -4,7 +4,6 @@
* you may not use this file except in compliance with the Elastic License.
*/
export default function ({ loadTestFile }) {
describe('InfraOps GraphQL Endpoints', () => {
loadTestFile(require.resolve('./metadata'));

70
x-pack/test/api_integration/apis/infra/metadata.ts
View file

@ -15,28 +15,56 @@ const metadataTests: KbnTestProvider = ({ getService }) => {
const client = getService('infraOpsGraphQLClient');
describe('metadata', () => {
before(() => esArchiver.load('infra/metrics_and_logs'));
after(() => esArchiver.unload('infra/metrics_and_logs'));
describe('7.0.0', () => {
before(() => esArchiver.load('infra/7.0.0/hosts'));
after(() => esArchiver.unload('infra/7.0.0/hosts'));
it('supports the metadata container query', () => {
return client
.query<MetadataQuery.Query>({
query: metadataQuery,
variables: {
sourceId: 'default',
nodeId: 'demo-stack-nginx-01',
nodeType: 'host',
},
})
.then(resp => {
const metadata = resp.data.source.metadataByNode;
if (metadata) {
expect(metadata.features.length).to.be(14);
expect(metadata.name).to.equal('demo-stack-nginx-01');
} else {
throw new Error('Metadata should never be empty');
}
});
it('hosts', () => {
return client
.query<MetadataQuery.Query>({
query: metadataQuery,
variables: {
sourceId: 'default',
nodeId: 'demo-stack-mysql-01',
nodeType: 'host',
},
})
.then(resp => {
const metadata = resp.data.source.metadataByNode;
if (metadata) {
expect(metadata.features.length).to.be(14);
expect(metadata.name).to.equal('demo-stack-mysql-01');
} else {
throw new Error('Metadata should never be empty');
}
});
});
});
describe('6.6.0', () => {
before(() => esArchiver.load('infra/6.6.0/docker'));
after(() => esArchiver.unload('infra/6.6.0/docker'));
it('docker', () => {
return client
.query<MetadataQuery.Query>({
query: metadataQuery,
variables: {
sourceId: 'default',
nodeId: '631f36a845514442b93c3fdd2dc91bcd8feb680b8ac5832c7fb8fdc167bb938e',
nodeType: 'container',
},
})
.then(resp => {
const metadata = resp.data.source.metadataByNode;
if (metadata) {
expect(metadata.features.length).to.be(10);
expect(metadata.name).to.equal('docker-autodiscovery_elasticsearch_1');
} else {
throw new Error('Metadata should never be empty');
}
});
});
});
});
};

23
x-pack/test/api_integration/apis/infra/metrics.ts
View file

@ -11,13 +11,16 @@ import { metricsQuery } from '../../../../plugins/infra/public/containers/metric
import { MetricsQuery } from '../../../../plugins/infra/public/graphql/types';
import { KbnTestProvider } from './types';
import { DATES } from './constants';
const { min, max } = DATES['7.0.0'].hosts;
const metricTests: KbnTestProvider = ({ getService }) => {
const esArchiver = getService('esArchiver');
const client = getService('infraOpsGraphQLClient');
describe('metrics', () => {
before(() => esArchiver.load('infra/metrics_and_logs'));
after(() => esArchiver.unload('infra/metrics_and_logs'));
before(() => esArchiver.load('infra/7.0.0/hosts'));
after(() => esArchiver.unload('infra/7.0.0/hosts'));
it('should basically work', () => {
return client
@ -27,11 +30,11 @@ const metricTests: KbnTestProvider = ({ getService }) => {
sourceId: 'default',
metrics: ['hostCpuUsage'],
timerange: {
to: 1539806283952,
from: 1539805341208,
to: max,
from: min,
interval: '>=1m',
},
nodeId: 'demo-stack-nginx-01',
nodeId: 'demo-stack-mysql-01',
nodeType: 'host',
},
})
@ -45,8 +48,8 @@ const metricTests: KbnTestProvider = ({ getService }) => {
expect(series).to.have.property('id', 'user');
expect(series).to.have.property('data');
const datapoint = last(series.data);
expect(datapoint).to.have.property('timestamp', 1539806220000);
expect(datapoint).to.have.property('value', 0.0065);
expect(datapoint).to.have.property('timestamp', 1547571720000);
expect(datapoint).to.have.property('value', 0.0018333333333333333);
});
});
@ -58,11 +61,11 @@ const metricTests: KbnTestProvider = ({ getService }) => {
sourceId: 'default',
metrics: ['hostCpuUsage', 'hostLoad'],
timerange: {
to: 1539806283952,
from: 1539805341208,
to: max,
from: min,
interval: '>=1m',
},
nodeId: 'demo-stack-nginx-01',
nodeId: 'demo-stack-mysql-01',
nodeType: 'host',
},
})

252
x-pack/test/api_integration/apis/infra/waffle.ts
View file

@ -11,116 +11,170 @@ import { waffleNodesQuery } from '../../../../plugins/infra/public/containers/wa
import { WaffleNodesQuery } from '../../../../plugins/infra/public/graphql/types';
import { KbnTestProvider } from './types';
import { DATES } from './constants';
const waffleTests: KbnTestProvider = ({ getService }) => {
const esArchiver = getService('esArchiver');
const client = getService('infraOpsGraphQLClient');
describe('waffle nodes', () => {
before(() => esArchiver.load('infra/metrics_and_logs'));
after(() => esArchiver.unload('infra/metrics_and_logs'));
describe('6.6.0', () => {
const { min, max } = DATES['6.6.0'].docker;
before(() => esArchiver.load('infra/6.6.0/docker'));
after(() => esArchiver.unload('infra/6.6.0/docker'));
it('should basically work', () => {
return client
.query<WaffleNodesQuery.Query>({
query: waffleNodesQuery,
variables: {
sourceId: 'default',
timerange: {
to: 1539806283952,
from: 1539805341208,
interval: '1m',
it('should basically work', () => {
return client
.query<WaffleNodesQuery.Query>({
query: waffleNodesQuery,
variables: {
sourceId: 'default',
timerange: {
to: max,
from: min,
interval: '1m',
},
metric: { type: 'cpu' },
path: [{ type: 'containers' }],
},
metric: { type: 'cpu' },
path: [{ type: 'hosts' }],
},
})
.then(resp => {
const { map } = resp.data.source;
expect(map).to.have.property('nodes');
if (map) {
const { nodes } = map;
expect(nodes.length).to.equal(6);
const firstNode = first(nodes);
expect(firstNode).to.have.property('path');
expect(firstNode.path.length).to.equal(1);
expect(first(firstNode.path)).to.have.property('value', 'demo-stack-apache-01');
expect(firstNode).to.have.property('metric');
expect(firstNode.metric).to.eql({
name: 'cpu',
value: 0.011,
avg: 0.012215686274509805,
max: 0.020999999999999998,
__typename: 'InfraNodeMetric',
});
}
});
})
.then(resp => {
const { map } = resp.data.source;
expect(map).to.have.property('nodes');
if (map) {
const { nodes } = map;
expect(nodes.length).to.equal(5);
const firstNode = first(nodes);
expect(firstNode).to.have.property('path');
expect(firstNode.path.length).to.equal(1);
expect(first(firstNode.path)).to.have.property(
'value',
'242fddb9d376bbf0e38025d81764847ee5ec0308adfa095918fd3266f9d06c6a'
);
expect(first(firstNode.path)).to.have.property(
'label',
'docker-autodiscovery_nginx_1'
);
expect(firstNode).to.have.property('metric');
expect(firstNode.metric).to.eql({
name: 'cpu',
value: 0,
max: 0,
avg: 0,
__typename: 'InfraNodeMetric',
});
}
});
});
});
it('should basically work with 1 grouping', () => {
return client
.query<WaffleNodesQuery.Query>({
query: waffleNodesQuery,
variables: {
sourceId: 'default',
timerange: {
to: 1539806283952,
from: 1539805341208,
interval: '1m',
},
metric: { type: 'cpu' },
path: [{ type: 'terms', field: 'meta.cloud.availability_zone' }, { type: 'hosts' }],
},
})
.then(resp => {
const { map } = resp.data.source;
expect(map).to.have.property('nodes');
if (map) {
const { nodes } = map;
expect(nodes.length).to.equal(6);
const firstNode = first(nodes);
expect(firstNode).to.have.property('path');
expect(firstNode.path.length).to.equal(2);
expect(first(firstNode.path)).to.have.property(
'value',
'projects/189716325846/zones/us-central1-f'
);
expect(last(firstNode.path)).to.have.property('value', 'demo-stack-apache-01');
}
});
});
describe('7.0.0', () => {
const { min, max } = DATES['7.0.0'].hosts;
before(() => esArchiver.load('infra/7.0.0/hosts'));
after(() => esArchiver.unload('infra/7.0.0/hosts'));
it('should basically work with 2 grouping', () => {
return client
.query<WaffleNodesQuery.Query>({
query: waffleNodesQuery,
variables: {
sourceId: 'default',
timerange: {
to: 1539806283952,
from: 1539805341208,
interval: '1m',
it('should basically work', () => {
return client
.query<WaffleNodesQuery.Query>({
query: waffleNodesQuery,
variables: {
sourceId: 'default',
timerange: {
to: max,
from: min,
interval: '1m',
},
metric: { type: 'cpu' },
path: [{ type: 'hosts' }],
},
metric: { type: 'cpu' },
path: [
{ type: 'terms', field: 'meta.cloud.provider' },
{ type: 'terms', field: 'meta.cloud.availability_zone' },
{ type: 'hosts' },
],
},
})
.then(resp => {
const { map } = resp.data.source;
expect(map).to.have.property('nodes');
if (map) {
const { nodes } = map;
expect(nodes.length).to.equal(6);
const firstNode = first(nodes);
expect(firstNode).to.have.property('path');
expect(firstNode.path.length).to.equal(3);
expect(first(firstNode.path)).to.have.property('value', 'gce');
expect(last(firstNode.path)).to.have.property('value', 'demo-stack-apache-01');
}
});
})
.then(resp => {
const { map } = resp.data.source;
expect(map).to.have.property('nodes');
if (map) {
const { nodes } = map;
expect(nodes.length).to.equal(1);
const firstNode = first(nodes);
expect(firstNode).to.have.property('path');
expect(firstNode.path.length).to.equal(1);
expect(first(firstNode.path)).to.have.property('value', 'demo-stack-mysql-01');
expect(first(firstNode.path)).to.have.property('label', 'demo-stack-mysql-01');
expect(firstNode).to.have.property('metric');
expect(firstNode.metric).to.eql({
name: 'cpu',
value: 0.0035,
avg: 0.009066666666666666,
max: 0.0684,
__typename: 'InfraNodeMetric',
});
}
});
});
it('should basically work with 1 grouping', () => {
return client
.query<WaffleNodesQuery.Query>({
query: waffleNodesQuery,
variables: {
sourceId: 'default',
timerange: {
to: max,
from: min,
interval: '1m',
},
metric: { type: 'cpu' },
path: [{ type: 'terms', field: 'cloud.availability_zone' }, { type: 'hosts' }],
},
})
.then(resp => {
const { map } = resp.data.source;
expect(map).to.have.property('nodes');
if (map) {
const { nodes } = map;
expect(nodes.length).to.equal(1);
const firstNode = first(nodes);
expect(firstNode).to.have.property('path');
expect(firstNode.path.length).to.equal(2);
expect(first(firstNode.path)).to.have.property('value', 'virtualbox');
expect(last(firstNode.path)).to.have.property('value', 'demo-stack-mysql-01');
}
});
});
it('should basically work with 2 grouping', () => {
return client
.query<WaffleNodesQuery.Query>({
query: waffleNodesQuery,
variables: {
sourceId: 'default',
timerange: {
to: max,
from: min,
interval: '1m',
},
metric: { type: 'cpu' },
path: [
{ type: 'terms', field: 'cloud.provider' },
{ type: 'terms', field: 'cloud.availability_zone' },
{ type: 'hosts' },
],
},
})
.then(resp => {
const { map } = resp.data.source;
expect(map).to.have.property('nodes');
if (map) {
const { nodes } = map;
expect(nodes.length).to.equal(1);
const firstNode = first(nodes);
expect(firstNode).to.have.property('path');
expect(firstNode.path.length).to.equal(3);
expect(first(firstNode.path)).to.have.property('value', 'vagrant');
expect(firstNode.path[1]).to.have.property('value', 'virtualbox');
expect(last(firstNode.path)).to.have.property('value', 'demo-stack-mysql-01');
}
});
});
});
});
};

BIN
x-pack/test/functional/es_archives/infra/6.6.0/docker/data.json.gz Normal file
View file

Binary file not shown.

15345
x-pack/test/functional/es_archives/infra/6.6.0/docker/mappings.json Normal file
View file

File diff suppressed because it is too large Load diff

BIN
x-pack/test/functional/es_archives/infra/7.0.0/hosts/data.json.gz Normal file
View file

Binary file not shown.

18153
x-pack/test/functional/es_archives/infra/7.0.0/hosts/mappings.json Normal file
View file

File diff suppressed because it is too large Load diff

BIN
x-pack/test/functional/es_archives/infra/legacy/data.json.gz Normal file
View file

Binary file not shown.

10752
x-pack/test/functional/es_archives/infra/legacy/mappings.json Normal file
View file

File diff suppressed because it is too large Load diff