github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 17:59:23 -04:00
Code Issues Projects Releases Packages Wiki Activity

[8.x] [EEM] Add built in definitions for hosts and containers (#193157) (#194921)

Browse source 
# Backport

This will backport the following commits from `main` to `8.x`:
- [[EEM] Add built in definitions for hosts and containers
(#193157)](https://github.com/elastic/kibana/pull/193157)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Milton
Hultgren","email":"milton.hultgren@elastic.co"},"sourceCommit":{"committedDate":"2024-09-18T14:03:17Z","message":"[EEM]
Add built in definitions for hosts and containers (#193157)\n\n###
Summary\r\n\r\nThis PR adds built in definitions for `hosts` and
`containers` based on\r\nECS data.\r\n\r\n### How to test\r\n\r\n1.
Check out this branch of Kibana\r\n2. Start ES and Kibana, verify that
start up works and that the two\r\ndefinitions are installed (by calling
`GET\r\n/internal/entities/definition` or checking that the transforms
are\r\ninstalled).\r\n3. Ingest some data for each definition to work
with, verify that you\r\nget data in `entities-host-*` and
`entities-container-*` and that the\r\ndata matches the expected shape
(metadata and
metrics[1])\r\n\r\n\r\n[[1]](https://github.com/elastic/kibana/pull/193157#issuecomment-2355609821)","sha":"26a50f71562ad713903e8543b7793f482f717935","branchLabelMapping":{"^v9.0.0$":"main","^v8.16.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","v9.0.0","backport:prev-minor","Feature:EEM"],"title":"[EEM]
Add built in definitions for hosts and
containers","number":193157,"url":"https://github.com/elastic/kibana/pull/193157","mergeCommit":{"message":"[EEM]
Add built in definitions for hosts and containers (#193157)\n\n###
Summary\r\n\r\nThis PR adds built in definitions for `hosts` and
`containers` based on\r\nECS data.\r\n\r\n### How to test\r\n\r\n1.
Check out this branch of Kibana\r\n2. Start ES and Kibana, verify that
start up works and that the two\r\ndefinitions are installed (by calling
`GET\r\n/internal/entities/definition` or checking that the transforms
are\r\ninstalled).\r\n3. Ingest some data for each definition to work
with, verify that you\r\nget data in `entities-host-*` and
`entities-container-*` and that the\r\ndata matches the expected shape
(metadata and
metrics[1])\r\n\r\n\r\n[[1]](https://github.com/elastic/kibana/pull/193157#issuecomment-2355609821)","sha":"26a50f71562ad713903e8543b7793f482f717935"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/193157","number":193157,"mergeCommit":{"message":"[EEM]
Add built in definitions for hosts and containers (#193157)\n\n###
Summary\r\n\r\nThis PR adds built in definitions for `hosts` and
`containers` based on\r\nECS data.\r\n\r\n### How to test\r\n\r\n1.
Check out this branch of Kibana\r\n2. Start ES and Kibana, verify that
start up works and that the two\r\ndefinitions are installed (by calling
`GET\r\n/internal/entities/definition` or checking that the transforms
are\r\ninstalled).\r\n3. Ingest some data for each definition to work
with, verify that you\r\nget data in `entities-host-*` and
`entities-container-*` and that the\r\ndata matches the expected shape
(metadata and
metrics[1])\r\n\r\n\r\n[[1]](https://github.com/elastic/kibana/pull/193157#issuecomment-2355609821)","sha":"26a50f71562ad713903e8543b7793f482f717935"}}]}]
BACKPORT-->

Co-authored-by: Milton Hultgren <milton.hultgren@elastic.co>
This commit is contained in:
Kibana Machine 2024-10-08 04:35:33 +11:00 • committed by GitHub
parent 07c493c462
commit 2ae4759ba8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 346 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

158
x-pack/plugins/entity_manager/server/lib/entities/built_in/containers_from_ecs_data.ts Normal file
View file

@ -0,0 +1,158 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { EntityDefinition, entityDefinitionSchema } from '@kbn/entities-schema';
import { BUILT_IN_ID_PREFIX } from './constants';
export const builtInContainersFromEcsEntityDefinition: EntityDefinition =
entityDefinitionSchema.parse({
id: `${BUILT_IN_ID_PREFIX}containers_from_ecs_data`,
managed: true,
version: '1.0.0',
name: 'Containers from ECS data',
description:
'This definition extracts container entities from common data streams by looking for the ECS field container.id',
type: 'container',
indexPatterns: ['filebeat-*', 'logs-*', 'metrics-*', 'metricbeat-*'],
identityFields: ['container.id'],
displayNameTemplate: '{{container.id}}',
history: {
timestampField: '@timestamp',
interval: '5m',
settings: {
frequency: '5m',
},
},
metadata: [
{
source: '_index',
destination: 'source_index',
},
{
source: 'data_stream.type',
destination: 'source_data_stream.type',
},
{
source: 'data_stream.dataset',
destination: 'source_data_stream.dataset',
},
'container.name',
'container.image.name',
'container.image.tag',
'container.runtime',
'host.name',
'host.ip',
'host.mac',
'host.architecture',
'host.os.family',
'host.os.kernel',
'host.os.name',
'host.os.platform',
'host.os.type',
'host.os.version',
'cloud.provider',
'cloud.region',
'cloud.availability_zone',
'cloud.instance.id',
'cloud.instance.name',
'cloud.machine.type',
'cloud.service.name',
'agent.name',
'agent.type',
'agent.ephemeral_id',
],
metrics: [
{
name: 'log_rate',
equation: 'A',
metrics: [
{
name: 'A',
aggregation: 'doc_count',
filter: 'log.level: * OR error.log.level: *',
},
],
},
{
name: 'error_log_rate',
equation: 'A',
metrics: [
{
name: 'A',
aggregation: 'doc_count',
filter: '(log.level: "error" OR "ERROR") OR (error.log.level: "error" OR "ERROR")',
},
],
},
{
name: 'cpu_usage_avg',
equation: 'A',
metrics: [
{
name: 'A',
aggregation: 'avg',
field: 'docker.cpu.total.pct',
},
],
},
{
name: 'memory_usage_avg',
equation: 'A',
metrics: [
{
name: 'A',
aggregation: 'avg',
field: 'docker.memory.usage.pct',
},
],
},
{
name: 'network_in_avg',
equation: 'A',
metrics: [
{
name: 'A',
aggregation: 'avg',
field: 'docker.network.in.bytes',
},
],
},
{
name: 'network_out_avg',
equation: 'A',
metrics: [
{
name: 'A',
aggregation: 'avg',
field: 'docker.network.out.bytes',
},
],
},
{
name: 'disk_read_avg',
equation: 'A',
metrics: [
{
name: 'A',
aggregation: 'avg',
field: 'docker.diskio.read.ops',
},
],
},
{
name: 'disk_write_avg',
equation: 'A',
metrics: [
{
name: 'A',
aggregation: 'avg',
field: 'docker.diskio.write.ops',
},
],
},
],
});

179
x-pack/plugins/entity_manager/server/lib/entities/built_in/hosts_from_ecs_data.ts Normal file
View file

@ -0,0 +1,179 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { EntityDefinition, entityDefinitionSchema } from '@kbn/entities-schema';
import { BUILT_IN_ID_PREFIX } from './constants';
export const builtInHostsFromEcsEntityDefinition: EntityDefinition = entityDefinitionSchema.parse({
id: `${BUILT_IN_ID_PREFIX}hosts_from_ecs_data`,
managed: true,
version: '1.0.0',
name: 'Hosts from ECS data',
description:
'This definition extracts host entities from common data streams by looking for the ECS field host.name',
type: 'host',
indexPatterns: ['filebeat-*', 'logs-*', 'metrics-*', 'metricbeat-*'],
identityFields: ['host.name'],
displayNameTemplate: '{{host.name}}',
history: {
timestampField: '@timestamp',
interval: '5m',
settings: {
frequency: '5m',
},
},
metadata: [
{
source: '_index',
destination: 'source_index',
},
{
source: 'data_stream.type',
destination: 'source_data_stream.type',
},
{
source: 'data_stream.dataset',
destination: 'source_data_stream.dataset',
},
'host.hostname',
'host.ip',
'host.mac',
'host.architecture',
'host.containerized',
'host.os.platform',
'host.os.name',
'host.os.type',
'host.os.codename',
'host.os.family',
'host.os.kernel',
'host.os.version',
'cloud.provider',
'cloud.region',
'cloud.availability_zone',
'cloud.instance.id',
'cloud.instance.name',
'cloud.service.name',
'cloud.machine.type',
'cloud.account.id',
'cloud.project.id',
'agent.id',
'agent.name',
'agent.type',
'agent.version',
],
metrics: [
{
name: 'log_rate',
equation: 'A',
metrics: [
{
name: 'A',
aggregation: 'doc_count',
filter: 'log.level: * OR error.log.level: *',
},
],
},
{
name: 'error_log_rate',
equation: 'A',
metrics: [
{
name: 'A',
aggregation: 'doc_count',
filter: '(log.level: "error" OR "ERROR") OR (error.log.level: "error" OR "ERROR")',
},
],
},
{
name: 'cpu_usage_avg',
equation: 'A',
metrics: [
{
name: 'A',
aggregation: 'avg',
field: 'system.cpu.total.norm.pct',
},
],
},
{
name: 'normalized_load_avg',
equation: 'A / B',
metrics: [
{
name: 'A',
aggregation: 'avg',
field: 'system.load.1',
},
{
name: 'B',
aggregation: 'max',
field: 'system.load.cores',
},
],
},
{
name: 'memory_usage_avg',
equation: 'A',
metrics: [
{
name: 'A',
aggregation: 'avg',
field: 'system.memory.actual.used.pct',
},
],
},
{
name: 'memory_free_avg',
equation: 'A - B',
metrics: [
{
name: 'A',
aggregation: 'max',
field: 'system.memory.total',
},
{
name: 'B',
aggregation: 'avg',
field: 'system.memory.actual.used.bytes',
},
],
},
{
name: 'disk_usage_max',
equation: 'A',
metrics: [
{
name: 'A',
aggregation: 'max',
field: 'system.filesystem.used.pct',
},
],
},
{
name: 'rx_avg',
equation: 'A * 8',
metrics: [
{
name: 'A',
aggregation: 'sum',
field: 'host.network.ingress.bytes',
},
],
},
{
name: 'tx_avg',
equation: 'A * 8',
metrics: [
{
name: 'A',
aggregation: 'sum',
field: 'host.network.egress.bytes',
},
],
},
],
});

10
x-pack/plugins/entity_manager/server/lib/entities/built_in/index.ts
View file

@ -6,8 +6,14 @@
*/
import { EntityDefinition } from '@kbn/entities-schema';
import { builtInServicesFromLogsEntityDefinition } from './services';
import { builtInServicesFromEcsEntityDefinition } from './services_from_ecs_data';
import { builtInHostsFromEcsEntityDefinition } from './hosts_from_ecs_data';
import { builtInContainersFromEcsEntityDefinition } from './containers_from_ecs_data';
export { BUILT_IN_ID_PREFIX } from './constants';
export const builtInDefinitions: EntityDefinition[] = [builtInServicesFromLogsEntityDefinition];
export const builtInDefinitions: EntityDefinition[] = [
builtInServicesFromEcsEntityDefinition,
builtInHostsFromEcsEntityDefinition,
builtInContainersFromEcsEntityDefinition,
];

2
x-pack/plugins/entity_manager/server/lib/entities/built_in/services.ts → x-pack/plugins/entity_manager/server/lib/entities/built_in/services_from_ecs_data.ts
View file

@ -18,7 +18,7 @@ const serviceTransactionFilter = (additionalFilters: string[] = []) => {
return [...baseFilters, ...additionalFilters].join(' AND ');
};
export const builtInServicesFromLogsEntityDefinition: EntityDefinition =
export const builtInServicesFromEcsEntityDefinition: EntityDefinition =
entityDefinitionSchema.parse({
version: '0.3.0',
id: `${BUILT_IN_ID_PREFIX}services_from_ecs_data`,