github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 17:59:23 -04:00
Code Issues Projects Releases Packages Wiki Activity

Merge remote-tracking branch 'upstream/master' into kbn-62016-createhref-basepath

Browse source
This commit is contained in:
pgayvallet 2020-04-07 16:30:57 +02:00
commit ab763a3d0a
22 changed files with 32371 additions and 9960 deletions
Download patch file Download diff file Expand all files Collapse all files

34
x-pack/legacy/plugins/apm/public/components/app/ServiceMap/Cytoscape.stories.tsx
View file

@ -75,27 +75,17 @@ storiesOf('app/ServiceMap/Cytoscape', module)
const cy = cytoscape();
const elements = [
{ data: { id: 'default' } },
{ data: { id: 'cache', label: 'cache', 'span.type': 'cache' } },
{ data: { id: 'database', label: 'database', 'span.type': 'db' } },
{ data: { id: 'cache', 'span.type': 'cache' } },
{ data: { id: 'database', 'span.type': 'db' } },
{
data: {
id: 'elasticsearch',
label: 'elasticsearch',
'span.type': 'db',
'span.subtype': 'elasticsearch'
}
},
{
data: { id: 'external', label: 'external', 'span.type': 'external' }
},
{
data: {
id: 'messaging',
label: 'messaging',
'span.type': 'messaging'
}
},
{ data: { id: 'external', 'span.type': 'external' } },
{ data: { id: 'messaging', 'span.type': 'messaging' } },
{
data: {
id: 'dotnet',
@ -119,11 +109,18 @@ storiesOf('app/ServiceMap/Cytoscape', module)
},
{
data: {
id: 'js-base',
'service.name': 'js-base service',
id: 'RUM (js-base)',
'service.name': 'RUM service',
'agent.name': 'js-base'
}
},
{
data: {
id: 'RUM (rum-js)',
'service.name': 'RUM service',
'agent.name': 'rum-js'
}
},
{
data: {
id: 'nodejs',
@ -163,7 +160,8 @@ storiesOf('app/ServiceMap/Cytoscape', module)
description={
<pre>
agent.name: {node.data('agent.name') || 'undefined'},
span.type: {node.data('span.type') || 'undefined'}
span.type: {node.data('span.type') || 'undefined'},
span.subtype: {node.data('span.subtype') || 'undefined'}
</pre>
}
icon={
@ -174,7 +172,7 @@ storiesOf('app/ServiceMap/Cytoscape', module)
width={80}
/>
}
title={node.data('label')}
title={node.data('id')}
/>
</EuiFlexItem>
))}

6
x-pack/legacy/plugins/apm/public/components/app/ServiceMap/Cytoscape.tsx
View file

@ -124,6 +124,12 @@ export function Cytoscape({
// Trigger a custom "data" event when data changes
useEffect(() => {
if (cy && elements.length > 0) {
const renderedElements = cy.elements('node,edge');
const latestElementIds = elements.map(el => el.data.id);
const absentElements = renderedElements.filter(
el => !latestElementIds.includes(el.id())
);
cy.remove(absentElements);
cy.add(elements);
cy.trigger('data');
}

2
x-pack/legacy/plugins/apm/public/components/app/ServiceMap/cytoscapeOptions.ts
View file

@ -83,7 +83,7 @@ const style: cytoscape.Stylesheet[] = [
style: {
'curve-style': 'taxi',
// @ts-ignore
'taxi-direction': 'rightward',
'taxi-direction': 'auto',
'line-color': lineColor,
'overlay-opacity': 0,
'target-arrow-color': lineColor,

12
x-pack/legacy/plugins/apm/public/components/app/ServiceMap/icons.ts
View file

@ -5,11 +5,12 @@
*/
import cytoscape from 'cytoscape';
import { isRumAgentName } from '../../../../../../../plugins/apm/common/agent_name';
import {
AGENT_NAME,
SERVICE_NAME,
SPAN_TYPE,
SPAN_SUBTYPE
SPAN_SUBTYPE,
SPAN_TYPE
} from '../../../../../../../plugins/apm/common/elasticsearch_fieldnames';
import databaseIcon from './icons/database.svg';
import defaultIconImport from './icons/default.svg';
@ -62,7 +63,12 @@ export function iconForNode(node: cytoscape.NodeSingular) {
const type = node.data(SPAN_TYPE);
if (node.data(SERVICE_NAME)) {
return serviceIcons[node.data(AGENT_NAME) as string];
const agentName = node.data(AGENT_NAME);
// RUM can have multiple names. Normalize it
const normalizedAgentName = isRumAgentName(agentName)
? 'js-base'
: agentName;
return serviceIcons[normalizedAgentName];
} else if (isIE11) {
return defaultIcon;
} else if (

7
x-pack/legacy/plugins/apm/public/components/shared/ErrorRateAlertTrigger/index.tsx
View file

@ -6,6 +6,7 @@
import React from 'react';
import { EuiFieldNumber } from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import { isFinite } from 'lodash';
import { ForLastExpression } from '../../../../../../../plugins/triggers_actions_ui/public';
import { ALERT_TYPES_CONFIG } from '../../../../../../../plugins/apm/common/alert_types';
import { ServiceAlertTrigger } from '../ServiceAlertTrigger';
@ -37,15 +38,17 @@ export function ErrorRateAlertTrigger(props: Props) {
...alertParams
};
const threshold = isFinite(params.threshold) ? params.threshold : '';
const fields = [
<PopoverExpression
title={i18n.translate('xpack.apm.errorRateAlertTrigger.isAbove', {
defaultMessage: 'is above'
})}
value={params.threshold.toString()}
value={threshold.toString()}
>
<EuiFieldNumber
value={alertParams.threshold ?? ''}
value={threshold}
step={0}
onChange={e =>
setAlertParams('threshold', parseInt(e.target.value, 10))

99
x-pack/legacy/plugins/siem/server/lib/index_fields/elasticsearch_adapter.test.ts
View file

@ -23,77 +23,108 @@ describe('Index Fields', () => {
).toEqual(
sortBy('name', [
{
description:
'Date/time when the event originated.\n\nThis is the date/time extracted from the event, typically representing when\nthe event was generated by the source.\n\nIf the event source has no original timestamp, this value is typically populated\nby the first time the event was received by the pipeline.\n\nRequired field for all events.',
example: '2016-05-23T08:05:34.853Z',
name: '@timestamp',
type: 'date',
searchable: true,
aggregatable: true,
category: 'base',
description:
'Date/time when the event originated. For log events this is the date/time when the event was generated, and not when it was read. Required field for all events.',
example: '2016-05-23T08:05:34.853Z',
indexes: ['auditbeat', 'filebeat', 'packetbeat'],
name: '@timestamp',
searchable: true,
type: 'date',
},
{
aggregatable: true,
category: 'agent',
description:
'Ephemeral identifier of this agent (if one exists). This id normally changes across restarts, but `agent.id` does not.',
example: '8a4f500f',
indexes: ['auditbeat'],
name: 'agent.ephemeral_id',
searchable: true,
description: 'Each document has an _id that uniquely identifies it',
example: 'Y-6TfmcB0WOhS6qyMv3s',
footnote: '',
group: 1,
level: 'core',
name: '_id',
required: true,
type: 'string',
searchable: true,
aggregatable: false,
readFromDocValues: true,
category: '_id',
indexes: ['auditbeat', 'filebeat', 'packetbeat'],
},
{
description:
'An index is like a ‘database’ in a relational database. It has a mapping which defines multiple types. An index is a logical namespace which maps to one or more primary shards and can have zero or more replica shards.',
example: 'auditbeat-8.0.0-2019.02.19-000001',
footnote: '',
group: 1,
level: 'core',
name: '_index',
required: true,
type: 'string',
searchable: true,
aggregatable: true,
readFromDocValues: true,
category: '_index',
indexes: ['auditbeat', 'filebeat', 'packetbeat'],
},
{
description:
'Ephemeral identifier of this agent (if one exists).\n\nThis id normally changes across restarts, but `agent.id` does not.',
example: '8a4f500f',
name: 'agent.ephemeral_id',
type: 'string',
searchable: true,
aggregatable: true,
category: 'agent',
indexes: ['filebeat'],
indexes: ['auditbeat'],
},
{
name: 'agent.hostname',
searchable: true,
type: 'string',
},
{
aggregatable: true,
category: 'agent',
indexes: ['filebeat'],
},
{
description:
'Unique identifier of this agent (if one exists). Example: For Beats this would be beat.id.',
'Unique identifier of this agent (if one exists).\n\nExample: For Beats this would be beat.id.',
example: '8a4f500d',
indexes: ['packetbeat'],
name: 'agent.id',
searchable: true,
type: 'string',
},
{
searchable: true,
aggregatable: true,
category: 'agent',
indexes: ['packetbeat'],
},
{
description:
'Name of the agent. This is a name that can be given to an agent. This can be helpful if for example two Filebeat instances are running on the same host but a human readable separation is needed on which Filebeat instance data is coming from. If no name is given, the name is often left empty.',
'Custom name of the agent.\n\nThis is a name that can be given to an agent. This can be helpful if for example\ntwo Filebeat instances are running on the same host but a human readable separation\nis needed on which Filebeat instance data is coming from.\n\nIf no name is given, the name is often left empty.',
example: 'foo',
indexes: ['auditbeat', 'filebeat'],
name: 'agent.name',
searchable: true,
type: 'string',
},
{
searchable: true,
aggregatable: true,
category: 'agent',
indexes: ['auditbeat', 'filebeat'],
},
{
description:
'Type of the agent. The agent type stays always the same and should be given by the agent used. In case of Filebeat the agent would always be Filebeat also if two Filebeat instances are run on the same machine.',
'Type of the agent.\n\nThe agent type stays always the same and should be given by the agent used.\nIn case of Filebeat the agent would always be Filebeat also if two Filebeat\ninstances are run on the same machine.',
example: 'filebeat',
indexes: ['auditbeat', 'packetbeat'],
name: 'agent.type',
searchable: true,
type: 'string',
},
{
searchable: true,
aggregatable: true,
category: 'agent',
indexes: ['auditbeat', 'packetbeat'],
},
{
description: 'Version of the agent.',
example: '6.0.0-rc2',
indexes: ['auditbeat', 'filebeat'],
name: 'agent.version',
searchable: true,
type: 'string',
searchable: true,
aggregatable: true,
category: 'agent',
indexes: ['auditbeat', 'filebeat'],
},
])
);

55
x-pack/legacy/plugins/siem/server/lib/index_fields/elasticsearch_adapter.ts
View file

@ -4,7 +4,7 @@
* you may not use this file except in compliance with the Elastic License.
*/
import { get } from 'lodash/fp';
import { isEmpty, get } from 'lodash/fp';
import { IndexField } from '../../graphql/types';
import {
@ -51,6 +51,23 @@ export class ElasticsearchIndexFieldAdapter implements FieldsAdapter {
}
}
const missingFields = [
{
name: '_id',
type: 'string',
searchable: true,
aggregatable: false,
readFromDocValues: true,
},
{
name: '_index',
type: 'string',
searchable: true,
aggregatable: true,
readFromDocValues: true,
},
];
export const formatIndexFields = (
responsesIndexFields: IndexFieldDescriptor[][],
indexesAlias: IndexAlias[]
@ -59,20 +76,23 @@ export const formatIndexFields = (
.reduce(
(accumulator: IndexField[], indexFields: IndexFieldDescriptor[], indexesAliasIdx: number) => [
...accumulator,
...indexFields.reduce((itemAccumulator: IndexField[], index: IndexFieldDescriptor) => {
const alias: IndexAlias = indexesAlias[indexesAliasIdx];
const splitName = index.name.split('.');
const category = baseCategoryFields.includes(splitName[0]) ? 'base' : splitName[0];
return [
...itemAccumulator,
{
...(hasDocumentation(alias, index.name) ? getDocumentation(alias, index.name) : {}),
...index,
category,
indexes: [alias],
} as IndexField,
];
}, []),
...[...missingFields, ...indexFields].reduce(
(itemAccumulator: IndexField[], index: IndexFieldDescriptor) => {
const alias: IndexAlias = indexesAlias[indexesAliasIdx];
const splitName = index.name.split('.');
const category = baseCategoryFields.includes(splitName[0]) ? 'base' : splitName[0];
return [
...itemAccumulator,
{
...(hasDocumentation(alias, index.name) ? getDocumentation(alias, index.name) : {}),
...index,
category,
indexes: [alias],
} as IndexField,
];
},
[]
),
],
[]
)
@ -84,7 +104,10 @@ export const formatIndexFields = (
...accumulator.slice(0, alreadyExistingIndexField),
{
...existingIndexField,
indexes: [...existingIndexField.indexes, ...indexfield.indexes],
description: isEmpty(existingIndexField.description)
? indexfield.description
: existingIndexField.description,
indexes: Array.from(new Set([...existingIndexField.indexes, ...indexfield.indexes])),
},
...accumulator.slice(alreadyExistingIndexField + 1),
];

10237
x-pack/legacy/plugins/siem/server/utils/beat_schema/8.0.0/auditbeat.ts
View file

File diff suppressed because it is too large Load diff

7630
x-pack/legacy/plugins/siem/server/utils/beat_schema/8.0.0/ecs.ts
View file

File diff suppressed because it is too large Load diff

15937
x-pack/legacy/plugins/siem/server/utils/beat_schema/8.0.0/filebeat.ts
View file

File diff suppressed because it is too large Load diff

7623
x-pack/legacy/plugins/siem/server/utils/beat_schema/8.0.0/packetbeat.ts
View file

File diff suppressed because it is too large Load diff

599
x-pack/legacy/plugins/siem/server/utils/beat_schema/index.test.ts
View file

@ -17,175 +17,94 @@ describe('Schema Beat', () => {
convertData[0].fields = isArray(convertData[0].fields)
? convertData[0].fields!.slice(0, 6)
: [];
expect(convertSchemaToAssociativeArray(convertData)).toEqual({
'@timestamp': {
description:
'Date/time when the event originated. For log events this is the date/time when the event was generated, and not when it was read. Required field for all events.',
'Date/time when the event originated.\n\nThis is the date/time extracted from the event, typically representing when\nthe event was generated by the source.\n\nIf the event source has no original timestamp, this value is typically populated\nby the first time the event was received by the pipeline.\n\nRequired field for all events.',
example: '2016-05-23T08:05:34.853Z',
name: '@timestamp',
type: 'date',
},
labels: {
description:
'Custom key/value pairs.\n\nCan be used to add meta information to events. Should not contain nested objects.\nAll values are stored as keyword.\n\nExample: `docker` and `k8s` labels.',
example: '{"application": "foo-bar", "env": "production"}',
name: 'labels',
type: 'object',
},
message: {
description:
'For log events the message field contains the log message, optimized\nfor viewing in a log viewer.\n\nFor structured logs without an original message field, other fields can be concatenated\nto form a human-readable summary of the event.\n\nIf multiple messages exist, they can be combined into one message.',
example: 'Hello World',
name: 'message',
type: 'text',
},
tags: {
description: 'List of keywords used to tag each event.',
example: '["production", "env2"]',
name: 'tags',
type: 'keyword',
},
labels: {
description:
'Key/value pairs. Can be used to add meta information to events. Should not contain nested objects. All values are stored as keyword. Example: `docker` and `k8s` labels.',
example: '{"env":"production","application":"foo-bar"}',
name: 'labels',
type: 'object',
},
message: {
description:
'For log events the message field contains the log message. In other use cases the message field can be used to concatenate different values which are then freely searchable. If multiple messages exist, they can be combined into one message.',
example: 'Hello World',
name: 'message',
type: 'text',
},
agent: {
description:
'The agent fields contain the data about the software entity, if any, that collects, detects, or observes events on a host, or takes measurements on a host. Examples include Beats. Agents may also run on observers. ECS agent.* fields shall be populated with details of the agent running on the host or observer where the event happened or the measurement was taken.',
'The agent fields contain the data about the software entity, if\nany, that collects, detects, or observes events on a host, or takes measurements\non a host.\n\nExamples include Beats. Agents may also run on observers. ECS agent.* fields\nshall be populated with details of the agent running on the host or observer\nwhere the event happened or the measurement was taken.',
name: 'agent',
type: 'group',
fields: {
'agent.version': {
description: 'Version of the agent.',
example: '6.0.0-rc2',
name: 'version',
'agent.ephemeral_id': {
description:
'Ephemeral identifier of this agent (if one exists).\n\nThis id normally changes across restarts, but `agent.id` does not.',
example: '8a4f500f',
name: 'ephemeral_id',
type: 'keyword',
},
'agent.id': {
description:
'Unique identifier of this agent (if one exists).\n\nExample: For Beats this would be beat.id.',
example: '8a4f500d',
name: 'id',
type: 'keyword',
},
'agent.name': {
description:
'Name of the agent. This is a name that can be given to an agent. This can be helpful if for example two Filebeat instances are running on the same host but a human readable separation is needed on which Filebeat instance data is coming from. If no name is given, the name is often left empty.',
'Custom name of the agent.\n\nThis is a name that can be given to an agent. This can be helpful if for example\ntwo Filebeat instances are running on the same host but a human readable separation\nis needed on which Filebeat instance data is coming from.\n\nIf no name is given, the name is often left empty.',
example: 'foo',
name: 'name',
type: 'keyword',
},
'agent.type': {
description:
'Type of the agent. The agent type stays always the same and should be given by the agent used. In case of Filebeat the agent would always be Filebeat also if two Filebeat instances are run on the same machine.',
'Type of the agent.\n\nThe agent type stays always the same and should be given by the agent used.\nIn case of Filebeat the agent would always be Filebeat also if two Filebeat\ninstances are run on the same machine.',
example: 'filebeat',
name: 'type',
type: 'keyword',
},
'agent.id': {
description:
'Unique identifier of this agent (if one exists). Example: For Beats this would be beat.id.',
example: '8a4f500d',
name: 'id',
type: 'keyword',
},
'agent.ephemeral_id': {
description:
'Ephemeral identifier of this agent (if one exists). This id normally changes across restarts, but `agent.id` does not.',
example: '8a4f500f',
name: 'ephemeral_id',
'agent.version': {
description: 'Version of the agent.',
example: '6.0.0-rc2',
name: 'version',
type: 'keyword',
},
},
},
client: {
as: {
description:
'A client is defined as the initiator of a network connection for events regarding sessions, connections, or bidirectional flow records. For TCP events, the client is the initiator of the TCP connection that sends the SYN packet(s). For other protocols, the client is generally the initiator or requestor in the network transaction. Some systems use the term "originator" to refer the client in TCP connections. The client fields describe details about the system acting as the client in the network event. Client fields are usually populated in conjunction with server fields. Client fields are generally not populated for packet-level events. Client / server representations can add semantic context to an exchange, which is helpful to visualize the data in certain situations. If your context falls in that category, you should still ensure that source and destination are filled appropriately.',
name: 'client',
'An autonomous system (AS) is a collection of connected Internet Protocol\n(IP) routing prefixes under the control of one or more network operators on\nbehalf of a single administrative entity or domain that presents a common, clearly\ndefined routing policy to the internet.',
name: 'as',
type: 'group',
fields: {
'client.address': {
'as.number': {
description:
'Some event client addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is.',
name: 'address',
type: 'keyword',
},
'client.ip': {
description:
'IP address of the client. Can be one or multiple IPv4 or IPv6 addresses.',
name: 'ip',
type: 'ip',
},
'client.port': {
description: 'Port of the client.',
name: 'port',
'Unique number allocated to the autonomous system. The autonomous\nsystem number (ASN) uniquely identifies each network on the Internet.',
example: 15169,
name: 'number',
type: 'long',
},
'client.mac': {
description: 'MAC address of the client.',
name: 'mac',
type: 'keyword',
},
'client.domain': {
description: 'Client domain.',
name: 'domain',
type: 'keyword',
},
'client.bytes': {
description: 'Bytes sent from the client to the server.',
example: 184,
format: 'bytes',
name: 'bytes',
type: 'long',
},
'client.packets': {
description: 'Packets sent from the client to the server.',
example: 12,
name: 'packets',
type: 'long',
},
'client.geo': {
description:
'Geo fields can carry data about a specific location related to an event or geo information derived from an IP field.',
name: 'geo',
type: 'group',
},
'client.geo.location': {
description: 'Longitude and latitude.',
example: '{ "lon": -73.614830, "lat": 45.505918 }',
name: 'location',
type: 'geo_point',
},
'client.geo.continent_name': {
description: 'Name of the continent.',
example: 'North America',
name: 'continent_name',
type: 'keyword',
},
'client.geo.country_name': {
description: 'Country name.',
example: 'Canada',
name: 'country_name',
type: 'keyword',
},
'client.geo.region_name': {
description: 'Region name.',
example: 'Quebec',
name: 'region_name',
type: 'keyword',
},
'client.geo.city_name': {
description: 'City name.',
example: 'Montreal',
name: 'city_name',
type: 'keyword',
},
'client.geo.country_iso_code': {
description: 'Country ISO code.',
example: 'CA',
name: 'country_iso_code',
type: 'keyword',
},
'client.geo.region_iso_code': {
description: 'Region ISO code.',
example: 'CA-QC',
name: 'region_iso_code',
type: 'keyword',
},
'client.geo.name': {
description:
'User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation.',
example: 'boston-dc',
name: 'name',
'as.organization.name': {
description: 'Organization name.',
example: 'Google LLC',
name: 'organization.name',
type: 'keyword',
},
},
@ -198,175 +117,94 @@ describe('Schema Beat', () => {
convertData[0].fields = isArray(convertData[0].fields)
? convertData[0].fields!.slice(0, 6)
: [];
expect(convertSchemaToAssociativeArray(convertData)).toEqual({
'@timestamp': {
description:
'Date/time when the event originated. For log events this is the date/time when the event was generated, and not when it was read. Required field for all events.',
'Date/time when the event originated.\n\nThis is the date/time extracted from the event, typically representing when\nthe event was generated by the source.\n\nIf the event source has no original timestamp, this value is typically populated\nby the first time the event was received by the pipeline.\n\nRequired field for all events.',
example: '2016-05-23T08:05:34.853Z',
name: '@timestamp',
type: 'date',
},
labels: {
description:
'Custom key/value pairs.\n\nCan be used to add meta information to events. Should not contain nested objects.\nAll values are stored as keyword.\n\nExample: `docker` and `k8s` labels.',
example: '{"application": "foo-bar", "env": "production"}',
name: 'labels',
type: 'object',
},
message: {
description:
'For log events the message field contains the log message, optimized\nfor viewing in a log viewer.\n\nFor structured logs without an original message field, other fields can be concatenated\nto form a human-readable summary of the event.\n\nIf multiple messages exist, they can be combined into one message.',
example: 'Hello World',
name: 'message',
type: 'text',
},
tags: {
description: 'List of keywords used to tag each event.',
example: '["production", "env2"]',
name: 'tags',
type: 'keyword',
},
labels: {
description:
'Key/value pairs. Can be used to add meta information to events. Should not contain nested objects. All values are stored as keyword. Example: `docker` and `k8s` labels.',
example: '{"env":"production","application":"foo-bar"}',
name: 'labels',
type: 'object',
},
message: {
description:
'For log events the message field contains the log message. In other use cases the message field can be used to concatenate different values which are then freely searchable. If multiple messages exist, they can be combined into one message.',
example: 'Hello World',
name: 'message',
type: 'text',
},
agent: {
description:
'The agent fields contain the data about the software entity, if any, that collects, detects, or observes events on a host, or takes measurements on a host. Examples include Beats. Agents may also run on observers. ECS agent.* fields shall be populated with details of the agent running on the host or observer where the event happened or the measurement was taken.',
'The agent fields contain the data about the software entity, if\nany, that collects, detects, or observes events on a host, or takes measurements\non a host.\n\nExamples include Beats. Agents may also run on observers. ECS agent.* fields\nshall be populated with details of the agent running on the host or observer\nwhere the event happened or the measurement was taken.',
name: 'agent',
type: 'group',
fields: {
'agent.version': {
description: 'Version of the agent.',
example: '6.0.0-rc2',
name: 'version',
'agent.ephemeral_id': {
description:
'Ephemeral identifier of this agent (if one exists).\n\nThis id normally changes across restarts, but `agent.id` does not.',
example: '8a4f500f',
name: 'ephemeral_id',
type: 'keyword',
},
'agent.id': {
description:
'Unique identifier of this agent (if one exists).\n\nExample: For Beats this would be beat.id.',
example: '8a4f500d',
name: 'id',
type: 'keyword',
},
'agent.name': {
description:
'Name of the agent. This is a name that can be given to an agent. This can be helpful if for example two Filebeat instances are running on the same host but a human readable separation is needed on which Filebeat instance data is coming from. If no name is given, the name is often left empty.',
'Custom name of the agent.\n\nThis is a name that can be given to an agent. This can be helpful if for example\ntwo Filebeat instances are running on the same host but a human readable separation\nis needed on which Filebeat instance data is coming from.\n\nIf no name is given, the name is often left empty.',
example: 'foo',
name: 'name',
type: 'keyword',
},
'agent.type': {
description:
'Type of the agent. The agent type stays always the same and should be given by the agent used. In case of Filebeat the agent would always be Filebeat also if two Filebeat instances are run on the same machine.',
'Type of the agent.\n\nThe agent type stays always the same and should be given by the agent used.\nIn case of Filebeat the agent would always be Filebeat also if two Filebeat\ninstances are run on the same machine.',
example: 'filebeat',
name: 'type',
type: 'keyword',
},
'agent.id': {
description:
'Unique identifier of this agent (if one exists). Example: For Beats this would be beat.id.',
example: '8a4f500d',
name: 'id',
type: 'keyword',
},
'agent.ephemeral_id': {
description:
'Ephemeral identifier of this agent (if one exists). This id normally changes across restarts, but `agent.id` does not.',
example: '8a4f500f',
name: 'ephemeral_id',
'agent.version': {
description: 'Version of the agent.',
example: '6.0.0-rc2',
name: 'version',
type: 'keyword',
},
},
},
client: {
as: {
description:
'A client is defined as the initiator of a network connection for events regarding sessions, connections, or bidirectional flow records. For TCP events, the client is the initiator of the TCP connection that sends the SYN packet(s). For other protocols, the client is generally the initiator or requestor in the network transaction. Some systems use the term "originator" to refer the client in TCP connections. The client fields describe details about the system acting as the client in the network event. Client fields are usually populated in conjunction with server fields. Client fields are generally not populated for packet-level events. Client / server representations can add semantic context to an exchange, which is helpful to visualize the data in certain situations. If your context falls in that category, you should still ensure that source and destination are filled appropriately.',
name: 'client',
'An autonomous system (AS) is a collection of connected Internet Protocol\n(IP) routing prefixes under the control of one or more network operators on\nbehalf of a single administrative entity or domain that presents a common, clearly\ndefined routing policy to the internet.',
name: 'as',
type: 'group',
fields: {
'client.address': {
'as.number': {
description:
'Some event client addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is.',
name: 'address',
type: 'keyword',
},
'client.ip': {
description:
'IP address of the client. Can be one or multiple IPv4 or IPv6 addresses.',
name: 'ip',
type: 'ip',
},
'client.port': {
description: 'Port of the client.',
name: 'port',
'Unique number allocated to the autonomous system. The autonomous\nsystem number (ASN) uniquely identifies each network on the Internet.',
example: 15169,
name: 'number',
type: 'long',
},
'client.mac': {
description: 'MAC address of the client.',
name: 'mac',
type: 'keyword',
},
'client.domain': {
description: 'Client domain.',
name: 'domain',
type: 'keyword',
},
'client.bytes': {
description: 'Bytes sent from the client to the server.',
example: 184,
format: 'bytes',
name: 'bytes',
type: 'long',
},
'client.packets': {
description: 'Packets sent from the client to the server.',
example: 12,
name: 'packets',
type: 'long',
},
'client.geo': {
description:
'Geo fields can carry data about a specific location related to an event or geo information derived from an IP field.',
name: 'geo',
type: 'group',
},
'client.geo.location': {
description: 'Longitude and latitude.',
example: '{ "lon": -73.614830, "lat": 45.505918 }',
name: 'location',
type: 'geo_point',
},
'client.geo.continent_name': {
description: 'Name of the continent.',
example: 'North America',
name: 'continent_name',
type: 'keyword',
},
'client.geo.country_name': {
description: 'Country name.',
example: 'Canada',
name: 'country_name',
type: 'keyword',
},
'client.geo.region_name': {
description: 'Region name.',
example: 'Quebec',
name: 'region_name',
type: 'keyword',
},
'client.geo.city_name': {
description: 'City name.',
example: 'Montreal',
name: 'city_name',
type: 'keyword',
},
'client.geo.country_iso_code': {
description: 'Country ISO code.',
example: 'CA',
name: 'country_iso_code',
type: 'keyword',
},
'client.geo.region_iso_code': {
description: 'Region ISO code.',
example: 'CA-QC',
name: 'region_iso_code',
type: 'keyword',
},
'client.geo.name': {
description:
'User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation.',
example: 'boston-dc',
name: 'name',
'as.organization.name': {
description: 'Organization name.',
example: 'Google LLC',
name: 'organization.name',
type: 'keyword',
},
},
@ -379,175 +217,94 @@ describe('Schema Beat', () => {
convertData[0].fields = isArray(convertData[0].fields)
? convertData[0].fields!.slice(0, 6)
: [];
expect(convertSchemaToAssociativeArray(convertData)).toEqual({
'@timestamp': {
description:
'Date/time when the event originated. For log events this is the date/time when the event was generated, and not when it was read. Required field for all events.',
'Date/time when the event originated.\n\nThis is the date/time extracted from the event, typically representing when\nthe event was generated by the source.\n\nIf the event source has no original timestamp, this value is typically populated\nby the first time the event was received by the pipeline.\n\nRequired field for all events.',
example: '2016-05-23T08:05:34.853Z',
name: '@timestamp',
type: 'date',
},
labels: {
description:
'Custom key/value pairs.\n\nCan be used to add meta information to events. Should not contain nested objects.\nAll values are stored as keyword.\n\nExample: `docker` and `k8s` labels.',
example: '{"application": "foo-bar", "env": "production"}',
name: 'labels',
type: 'object',
},
message: {
description:
'For log events the message field contains the log message, optimized\nfor viewing in a log viewer.\n\nFor structured logs without an original message field, other fields can be concatenated\nto form a human-readable summary of the event.\n\nIf multiple messages exist, they can be combined into one message.',
example: 'Hello World',
name: 'message',
type: 'text',
},
tags: {
description: 'List of keywords used to tag each event.',
example: '["production", "env2"]',
name: 'tags',
type: 'keyword',
},
labels: {
description:
'Key/value pairs. Can be used to add meta information to events. Should not contain nested objects. All values are stored as keyword. Example: `docker` and `k8s` labels.',
example: '{"env":"production","application":"foo-bar"}',
name: 'labels',
type: 'object',
},
message: {
description:
'For log events the message field contains the log message. In other use cases the message field can be used to concatenate different values which are then freely searchable. If multiple messages exist, they can be combined into one message.',
example: 'Hello World',
name: 'message',
type: 'text',
},
agent: {
description:
'The agent fields contain the data about the software entity, if any, that collects, detects, or observes events on a host, or takes measurements on a host. Examples include Beats. Agents may also run on observers. ECS agent.* fields shall be populated with details of the agent running on the host or observer where the event happened or the measurement was taken.',
'The agent fields contain the data about the software entity, if\nany, that collects, detects, or observes events on a host, or takes measurements\non a host.\n\nExamples include Beats. Agents may also run on observers. ECS agent.* fields\nshall be populated with details of the agent running on the host or observer\nwhere the event happened or the measurement was taken.',
name: 'agent',
type: 'group',
fields: {
'agent.version': {
description: 'Version of the agent.',
example: '6.0.0-rc2',
name: 'version',
'agent.ephemeral_id': {
description:
'Ephemeral identifier of this agent (if one exists).\n\nThis id normally changes across restarts, but `agent.id` does not.',
example: '8a4f500f',
name: 'ephemeral_id',
type: 'keyword',
},
'agent.id': {
description:
'Unique identifier of this agent (if one exists).\n\nExample: For Beats this would be beat.id.',
example: '8a4f500d',
name: 'id',
type: 'keyword',
},
'agent.name': {
description:
'Name of the agent. This is a name that can be given to an agent. This can be helpful if for example two Filebeat instances are running on the same host but a human readable separation is needed on which Filebeat instance data is coming from. If no name is given, the name is often left empty.',
'Custom name of the agent.\n\nThis is a name that can be given to an agent. This can be helpful if for example\ntwo Filebeat instances are running on the same host but a human readable separation\nis needed on which Filebeat instance data is coming from.\n\nIf no name is given, the name is often left empty.',
example: 'foo',
name: 'name',
type: 'keyword',
},
'agent.type': {
description:
'Type of the agent. The agent type stays always the same and should be given by the agent used. In case of Filebeat the agent would always be Filebeat also if two Filebeat instances are run on the same machine.',
'Type of the agent.\n\nThe agent type stays always the same and should be given by the agent used.\nIn case of Filebeat the agent would always be Filebeat also if two Filebeat\ninstances are run on the same machine.',
example: 'filebeat',
name: 'type',
type: 'keyword',
},
'agent.id': {
description:
'Unique identifier of this agent (if one exists). Example: For Beats this would be beat.id.',
example: '8a4f500d',
name: 'id',
type: 'keyword',
},
'agent.ephemeral_id': {
description:
'Ephemeral identifier of this agent (if one exists). This id normally changes across restarts, but `agent.id` does not.',
example: '8a4f500f',
name: 'ephemeral_id',
'agent.version': {
description: 'Version of the agent.',
example: '6.0.0-rc2',
name: 'version',
type: 'keyword',
},
},
},
client: {
as: {
description:
'A client is defined as the initiator of a network connection for events regarding sessions, connections, or bidirectional flow records. For TCP events, the client is the initiator of the TCP connection that sends the SYN packet(s). For other protocols, the client is generally the initiator or requestor in the network transaction. Some systems use the term "originator" to refer the client in TCP connections. The client fields describe details about the system acting as the client in the network event. Client fields are usually populated in conjunction with server fields. Client fields are generally not populated for packet-level events. Client / server representations can add semantic context to an exchange, which is helpful to visualize the data in certain situations. If your context falls in that category, you should still ensure that source and destination are filled appropriately.',
name: 'client',
'An autonomous system (AS) is a collection of connected Internet Protocol\n(IP) routing prefixes under the control of one or more network operators on\nbehalf of a single administrative entity or domain that presents a common, clearly\ndefined routing policy to the internet.',
name: 'as',
type: 'group',
fields: {
'client.address': {
'as.number': {
description:
'Some event client addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is.',
name: 'address',
type: 'keyword',
},
'client.ip': {
description:
'IP address of the client. Can be one or multiple IPv4 or IPv6 addresses.',
name: 'ip',
type: 'ip',
},
'client.port': {
description: 'Port of the client.',
name: 'port',
'Unique number allocated to the autonomous system. The autonomous\nsystem number (ASN) uniquely identifies each network on the Internet.',
example: 15169,
name: 'number',
type: 'long',
},
'client.mac': {
description: 'MAC address of the client.',
name: 'mac',
type: 'keyword',
},
'client.domain': {
description: 'Client domain.',
name: 'domain',
type: 'keyword',
},
'client.bytes': {
description: 'Bytes sent from the client to the server.',
example: 184,
format: 'bytes',
name: 'bytes',
type: 'long',
},
'client.packets': {
description: 'Packets sent from the client to the server.',
example: 12,
name: 'packets',
type: 'long',
},
'client.geo': {
description:
'Geo fields can carry data about a specific location related to an event or geo information derived from an IP field.',
name: 'geo',
type: 'group',
},
'client.geo.location': {
description: 'Longitude and latitude.',
example: '{ "lon": -73.614830, "lat": 45.505918 }',
name: 'location',
type: 'geo_point',
},
'client.geo.continent_name': {
description: 'Name of the continent.',
example: 'North America',
name: 'continent_name',
type: 'keyword',
},
'client.geo.country_name': {
description: 'Country name.',
example: 'Canada',
name: 'country_name',
type: 'keyword',
},
'client.geo.region_name': {
description: 'Region name.',
example: 'Quebec',
name: 'region_name',
type: 'keyword',
},
'client.geo.city_name': {
description: 'City name.',
example: 'Montreal',
name: 'city_name',
type: 'keyword',
},
'client.geo.country_iso_code': {
description: 'Country ISO code.',
example: 'CA',
name: 'country_iso_code',
type: 'keyword',
},
'client.geo.region_iso_code': {
description: 'Region ISO code.',
example: 'CA-QC',
name: 'region_iso_code',
type: 'keyword',
},
'client.geo.name': {
description:
'User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation.',
example: 'boston-dc',
name: 'name',
'as.organization.name': {
description: 'Organization name.',
example: 'Google LLC',
name: 'organization.name',
type: 'keyword',
},
},
@ -562,40 +319,58 @@ describe('Schema Beat', () => {
'_id',
'_index',
'@timestamp',
'tags',
'labels',
'message',
'tags',
'agent',
'as',
'client',
'cloud',
'code_signature',
'container',
'destination',
'dll',
'dns',
'ecs',
'error',
'event',
'file',
'geo',
'group',
'hash',
'host',
'http',
'interface',
'log',
'network',
'observer',
'organization',
'os',
'package',
'pe',
'process',
'registry',
'related',
'rule',
'server',
'service',
'source',
'threat',
'tls',
'tracing',
'url',
'user',
'user_agent',
'vlan',
'vulnerability',
'agent.hostname',
'beat.timezone',
'fields',
'beat.name',
'beat.hostname',
'timeseries.instance',
'cloud.project.id',
'cloud.image.id',
'meta.cloud.provider',
'meta.cloud.instance_id',
'meta.cloud.instance_name',
@ -605,55 +380,17 @@ describe('Schema Beat', () => {
'meta.cloud.region',
'docker',
'kubernetes',
'type',
'server.process.name',
'server.process.args',
'server.process.executable',
'server.process.working_directory',
'server.process.start',
'client.process.name',
'client.process.args',
'client.process.executable',
'client.process.working_directory',
'client.process.start',
'real_ip',
'transport',
'flow.final',
'flow.id',
'flow.vlan',
'flow_id',
'final',
'vlan',
'source.stats.net_bytes_total',
'source.stats.net_packets_total',
'dest.stats.net_bytes_total',
'dest.stats.net_packets_total',
'status',
'method',
'resource',
'path',
'query',
'params',
'notes',
'request',
'response',
'bytes_in',
'bytes_out',
'amqp',
'no_request',
'cassandra',
'dhcpv4',
'dns',
'icmp',
'memcache',
'mongodb',
'mysql',
'nfs',
'rpc',
'pgsql',
'redis',
'thrift',
'tls',
'jolokia.agent.version',
'jolokia.agent.id',
'jolokia.server.product',
'jolokia.server.version',
'jolokia.server.vendor',
'jolokia.url',
'jolokia.secured',
'auditd',
'geoip',
'socket',
'system.audit',
]);
});
});

22
x-pack/legacy/plugins/siem/server/utils/beat_schema/index.ts
View file

@ -106,19 +106,14 @@ export const getIndexSchemaDoc = memoize((index: string) => {
...extraSchemaField,
...convertSchemaToAssociativeArray(winlogbeatSchema),
};
} else if (index.match('ecs') != null) {
return {
...extraSchemaField,
...ecsSchema,
};
}
return {};
return {
...extraSchemaField,
...convertSchemaToAssociativeArray(ecsSchema),
};
});
export const hasDocumentation = (index: string, path: string): boolean => {
if (index === 'unknown') {
return false;
}
const splitPath = path.split('.');
const category = splitPath.length > 0 ? splitPath[0] : null;
if (category === null) {
@ -131,16 +126,13 @@ export const hasDocumentation = (index: string, path: string): boolean => {
};
export const getDocumentation = (index: string, path: string) => {
if (index === 'unknown') {
return '';
}
const splitPath = path.split('.');
const category = splitPath.length > 0 ? splitPath[0] : null;
if (category === null) {
return '';
return {};
}
if (splitPath.length > 1) {
return get([category, 'fields', path], getIndexSchemaDoc(index)) || '';
return get([category, 'fields', path], getIndexSchemaDoc(index)) || {};
}
return get(category, getIndexSchemaDoc(index)) || '';
return get(category, getIndexSchemaDoc(index)) || {};
};

39
x-pack/legacy/plugins/siem/server/utils/beat_schema/type.ts
View file

@ -12,25 +12,35 @@ export type IndexAlias = 'auditbeat' | 'filebeat' | 'packetbeat' | 'ecs' | 'winl
*/
export interface SchemaFields {
default_field: boolean;
default_fields: boolean;
definition: string;
deprecated: string;
description: string;
doc_values: boolean;
example: string | number | object;
example: string | number | object | boolean;
footnote: string;
format: string;
group: number;
index: boolean;
ignore_above: number;
input_format: string;
level: string;
migration: boolean;
multi_fields: object[];
name: string;
norms: boolean;
object_type: string;
object_type_mapping_type: string;
output_format: string;
output_precision: number;
overwrite: boolean;
path: string;
possible_values: string[] | number[];
release: string;
required: boolean;
reusable: object;
short: string;
title: string;
type: string;
fields: Array<Partial<SchemaFields>>;
@ -48,33 +58,6 @@ export interface SchemaItem {
export type Schema = Array<Partial<SchemaItem>>;
/*
* ECS Interface
*
*/
interface EcsField {
description: string;
example: string;
footnote: string;
group: number;
level: string;
name: string;
required: boolean;
type: string;
}
interface EcsNamespace {
description: string;
fields: Readonly<Record<string, EcsField>>;
group: number;
name: string;
title: string;
type: string;
}
export type EcsSchema = Readonly<Record<string, EcsNamespace>>;
/*
* Associative Array Output Interface
*

5
x-pack/plugins/actions/server/builtin_action_types/lib/send_email.ts
View file

@ -65,6 +65,11 @@ export async function sendEmail(logger: Logger, options: SendEmailOptions): Prom
transportConfig.host = host;
transportConfig.port = port;
transportConfig.secure = !!secure;
if (!transportConfig.secure) {
transportConfig.tls = {
rejectUnauthorized: false,
};
}
}
const nodemailerTransport = nodemailer.createTransport(transportConfig);

4
x-pack/plugins/apm/server/lib/service_map/get_service_map_service_node_info.ts
View file

@ -41,9 +41,7 @@ export async function getServiceMapServiceNodeInfo({
const filter: ESFilter[] = [
{ range: rangeFilter(start, end) },
{ term: { [SERVICE_NAME]: serviceName } },
...(environment
? [{ term: { [SERVICE_ENVIRONMENT]: SERVICE_ENVIRONMENT } }]
: [])
...(environment ? [{ term: { [SERVICE_ENVIRONMENT]: environment } }] : [])
];
const minutes = Math.abs((end - start) / (1000 * 60));

8
x-pack/plugins/data_enhanced/server/search/es_search_strategy.test.ts
View file

@ -71,7 +71,7 @@ describe('ES search strategy', () => {
expect(mockApiCaller.mock.calls[0][0]).toBe('transport.request');
const { method, path, body } = mockApiCaller.mock.calls[0][1];
expect(method).toBe('POST');
expect(path).toBe('logstash-*/_async_search');
expect(path).toBe('/logstash-*/_async_search');
expect(body).toEqual({ query: {} });
});
@ -94,7 +94,7 @@ describe('ES search strategy', () => {
expect(mockApiCaller.mock.calls[0][0]).toBe('transport.request');
const { method, path, body } = mockApiCaller.mock.calls[0][1];
expect(method).toBe('GET');
expect(path).toBe('_async_search/foo');
expect(path).toBe('/_async_search/foo');
expect(body).toEqual(undefined);
});
@ -117,7 +117,7 @@ describe('ES search strategy', () => {
expect(mockApiCaller.mock.calls[0][0]).toBe('transport.request');
const { method, path } = mockApiCaller.mock.calls[0][1];
expect(method).toBe('POST');
expect(path).toBe('foo-%E7%A8%8B/_async_search');
expect(path).toBe('/foo-%E7%A8%8B/_async_search');
});
it('calls the rollup API if the index is a rollup type', async () => {
@ -139,6 +139,6 @@ describe('ES search strategy', () => {
expect(mockApiCaller.mock.calls[0][0]).toBe('transport.request');
const { method, path } = mockApiCaller.mock.calls[0][1];
expect(method).toBe('POST');
expect(path).toBe('foo-%E7%A8%8B/_rollup_search');
expect(path).toBe('/foo-%E7%A8%8B/_rollup_search');
});
});

6
x-pack/plugins/data_enhanced/server/search/es_search_strategy.ts
View file

@ -45,7 +45,7 @@ export const enhancedEsSearchStrategyProvider: TSearchStrategyProvider<typeof ES
const cancel: ISearchCancel<typeof ES_SEARCH_STRATEGY> = async id => {
const method = 'DELETE';
const path = encodeURI(`_async_search/${id}`);
const path = encodeURI(`/_async_search/${id}`);
await caller('transport.request', { method, path });
};
@ -66,7 +66,7 @@ async function asyncSearch(
const { body = undefined, index = undefined, ...queryParams } = request.id ? {} : params;
const method = request.id ? 'GET' : 'POST';
const path = encodeURI(request.id ? `_async_search/${request.id}` : `${index}/_async_search`);
const path = encodeURI(request.id ? `/_async_search/${request.id}` : `/${index}/_async_search`);
// Wait up to 1s for the response to return
const query = toSnakeCase({ waitForCompletionTimeout: '1s', ...queryParams });
@ -87,7 +87,7 @@ async function rollupSearch(
) {
const { body, index, ...params } = request.params;
const method = 'POST';
const path = encodeURI(`${index}/_rollup_search`);
const path = encodeURI(`/${index}/_rollup_search`);
const query = toSnakeCase(params);
const rawResponse = await ((caller(

1
x-pack/plugins/ingest_manager/common/services/package_to_config.test.ts
View file

@ -24,6 +24,7 @@ describe('Ingest Manager - packageToConfig', () => {
visualization: [],
search: [],
'index-pattern': [],
map: [],
},
},
status: InstallationStatus.notInstalled,

1
x-pack/plugins/ingest_manager/common/types/models/epm.ts
View file

@ -28,6 +28,7 @@ export enum KibanaAssetType {
visualization = 'visualization',
search = 'search',
indexPattern = 'index-pattern',
map = 'map',
}
export enum ElasticsearchAssetType {

2
x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/epm/constants.tsx
View file

@ -24,6 +24,7 @@ export const AssetTitleMap: Record<AssetType, string> = {
search: 'Saved Search',
visualization: 'Visualization',
input: 'Agent input',
map: 'Map',
};
export const ServiceTitleMap: Record<ServiceName, string> = {
@ -36,6 +37,7 @@ export const AssetIcons: Record<KibanaAssetType, IconType> = {
'index-pattern': 'indexPatternApp',
search: 'searchProfilerApp',
visualization: 'visualizeApp',
map: 'mapApp',
};
export const ServiceIcons: Record<ServiceName, IconType> = {

2
x-pack/test/api_integration/apis/siem/sources.ts
View file

@ -30,7 +30,7 @@ export default function({ getService }: FtrProviderContext) {
.then(resp => {
const sourceStatus = resp.data.source.status;
// test data in x-pack/test/functional/es_archives/auditbeat_test_data/data.json.gz
expect(sourceStatus.indexFields.length).to.be(395);
expect(sourceStatus.indexFields.length).to.be(397);
expect(sourceStatus.indicesExist).to.be(true);
});
});