mirror of
https://github.com/elastic/kibana.git
synced 2025-04-24 09:48:58 -04:00
Adds Network ML module with four ML jobs for ECS network data (#96480)
* network module adds the network module with four ml jobs for the 7.13 release * Update datafeed_high_count_network_denies.json json formatting * update test added the security_network module to the list * renames module name change to security_network / Security: Network * formatting change hyphen char to underscores * fixes and name changes fixes to df queries, descriptions. created_by param * update tests tests need the security_network module added * formatting change hyphens to underscores * descriptions format descriptions * Update datafeed_high_count_network_events.json indentation fixes * Update x-pack/plugins/ml/server/models/data_recognizer/modules/security_network/manifest.json Co-authored-by: Lisa Cawley <lcawley@elastic.co> * Update x-pack/plugins/ml/server/models/data_recognizer/modules/security_network/ml/high_count_network_denies.json Co-authored-by: Lisa Cawley <lcawley@elastic.co> * Update datafeed_high_count_network_events.json change to a filter Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Lisa Cawley <lcawley@elastic.co>
This commit is contained in:
Craig Chamberlain
GitHub
parent
98f799953b
commit
bedf92f001
12 changed files with 301 additions and 1 deletions
|
|
@ -0,0 +1,3 @@
|
|||
{
|
||||
"icon": "logoSecurity"
|
||||
}
|
||||
|
|
@ -0,0 +1,63 @@
|
|||
{
|
||||
"id": "security_network",
|
||||
"title": "Security: Network",
|
||||
"description": "Detect anomalous network activity in your ECS-compatible network logs.",
|
||||
"type": "network data",
|
||||
"logoFile": "logo.json",
|
||||
"defaultIndexPattern": [
|
||||
"logs-*",
|
||||
"filebeat-*",
|
||||
"packetbeat-*"
|
||||
],
|
||||
"query": {
|
||||
"bool": {
|
||||
"filter": [
|
||||
{
|
||||
"term": {
|
||||
"event.category": "network"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"jobs": [
|
||||
{
|
||||
"id": "high_count_by_destination_country",
|
||||
"file": "high_count_by_destination_country.json"
|
||||
},
|
||||
{
|
||||
"id": "high_count_network_denies",
|
||||
"file": "high_count_network_denies.json"
|
||||
},
|
||||
{
|
||||
"id": "high_count_network_events",
|
||||
"file": "high_count_network_events.json"
|
||||
},
|
||||
{
|
||||
"id": "rare_destination_country",
|
||||
"file": "rare_destination_country.json"
|
||||
}
|
||||
],
|
||||
"datafeeds": [
|
||||
{
|
||||
"id": "datafeed_high_count_by_destination_country",
|
||||
"file": "datafeed_high_count_by_destination_country.json",
|
||||
"job_id": "high_count_by_destination_country"
|
||||
},
|
||||
{
|
||||
"id": "datafeed_high_count_network_denies",
|
||||
"file": "datafeed_high_count_network_denies.json",
|
||||
"job_id": "high_count_network_denies"
|
||||
},
|
||||
{
|
||||
"id": "datafeed_high_count_network_events",
|
||||
"file": "datafeed_high_count_network_events.json",
|
||||
"job_id": "high_count_network_events"
|
||||
},
|
||||
{
|
||||
"id": "datafeed_rare_destination_country",
|
||||
"file": "datafeed_rare_destination_country.json",
|
||||
"job_id": "rare_destination_country"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
{
|
||||
"job_id": "high_count_by_destination_country",
|
||||
"indices": [
|
||||
"logs-*",
|
||||
"filebeat-*",
|
||||
"packetbeat-*"
|
||||
],
|
||||
"max_empty_searches": 10,
|
||||
"query": {
|
||||
"bool": {
|
||||
"filter": [
|
||||
{
|
||||
"term": {
|
||||
"event.category": "network"
|
||||
}
|
||||
},
|
||||
{
|
||||
"exists": {
|
||||
"field": "destination.geo.country_name"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
{
|
||||
"job_id": "high_count_network_denies",
|
||||
"indices": [
|
||||
"logs-*",
|
||||
"filebeat-*",
|
||||
"packetbeat-*"
|
||||
],
|
||||
"max_empty_searches": 10,
|
||||
"query": {
|
||||
"bool": {
|
||||
"filter": [
|
||||
{
|
||||
"term": {
|
||||
"event.category": "network"
|
||||
}
|
||||
},
|
||||
{
|
||||
"term": {
|
||||
"event.outcome": "deny"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
{
|
||||
"job_id": "high_count_network_events",
|
||||
"indices": [
|
||||
"logs-*",
|
||||
"filebeat-*",
|
||||
"packetbeat-*"
|
||||
],
|
||||
"max_empty_searches": 10,
|
||||
"query": {
|
||||
"bool": {
|
||||
"filter": [
|
||||
{
|
||||
"term": {
|
||||
"event.category": "network"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
{
|
||||
"job_id": "rare_destination_country",
|
||||
"indices": [
|
||||
"logs-*",
|
||||
"filebeat-*",
|
||||
"packetbeat-*"
|
||||
],
|
||||
"max_empty_searches": 10,
|
||||
"query": {
|
||||
"bool": {
|
||||
"filter": [
|
||||
{
|
||||
"term": {
|
||||
"event.category": "network"
|
||||
}
|
||||
},
|
||||
{
|
||||
"exists": {
|
||||
"field": "destination.geo.country_name"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
{
|
||||
"job_type": "anomaly_detector",
|
||||
"description": "Security: Network - looks for an unusually large spike in network activity to one destination country in the network logs. This could be due to unusually large amounts of reconnaissance or enumeration traffic. Data exfiltration activity may also produce such a surge in traffic to a destination country which does not normally appear in network traffic or business work-flows. Malware instances and persistence mechanisms may communicate with command-and-control (C2) infrastructure in their country of origin, which may be an unusual destination country for the source network.",
|
||||
"groups": [
|
||||
"security",
|
||||
"network"
|
||||
],
|
||||
"analysis_config": {
|
||||
"bucket_span": "15m",
|
||||
"detectors": [
|
||||
{
|
||||
"detector_description": "high_non_zero_count by \"destination.geo.country_name\"",
|
||||
"function": "high_non_zero_count",
|
||||
"by_field_name": "destination.geo.country_name",
|
||||
"detector_index": 0
|
||||
}
|
||||
],
|
||||
"influencers": [
|
||||
"destination.geo.country_name",
|
||||
"destination.as.organization.name",
|
||||
"source.ip",
|
||||
"destination.ip"
|
||||
]
|
||||
},
|
||||
"allow_lazy_open": true,
|
||||
"analysis_limits": {
|
||||
"model_memory_limit": "32mb"
|
||||
},
|
||||
"data_description": {
|
||||
"time_field": "@timestamp"
|
||||
},
|
||||
"custom_settings": {
|
||||
"created_by": "ml-module-security-network"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
{
|
||||
"job_type": "anomaly_detector",
|
||||
"description": "Security: Network - looks for an unusually large spike in network traffic that was denied by network ACLs or firewall rules. Such a burst of denied traffic is usually either 1) a misconfigured application or firewall or 2) suspicious or malicious activity. Unsuccessful attempts at network transit, in order to connect to command-and-control (C2), or engage in data exfiltration, may produce a burst of failed connections. This could also be due to unusually large amounts of reconnaissance or enumeration traffic. Denial-of-service attacks or traffic floods may also produce such a surge in traffic.",
|
||||
"groups": [
|
||||
"security",
|
||||
"network"
|
||||
],
|
||||
"analysis_config": {
|
||||
"bucket_span": "15m",
|
||||
"detectors": [
|
||||
{
|
||||
"detector_description": "high_count",
|
||||
"function": "high_count",
|
||||
"detector_index": 0
|
||||
}
|
||||
],
|
||||
"influencers": [
|
||||
"destination.geo.country_name",
|
||||
"destination.as.organization.name",
|
||||
"source.ip",
|
||||
"destination.port"
|
||||
]
|
||||
},
|
||||
"allow_lazy_open": true,
|
||||
"analysis_limits": {
|
||||
"model_memory_limit": "32mb"
|
||||
},
|
||||
"data_description": {
|
||||
"time_field": "@timestamp"
|
||||
},
|
||||
"custom_settings": {
|
||||
"created_by": "ml-module-security-network"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
{
|
||||
"job_type": "anomaly_detector",
|
||||
"description": "Security: Network - looks for an unusually large spike in network traffic. Such a burst of traffic, if not caused by a surge in business activity, can be due to suspicious or malicious activity. Large-scale data exfiltration may produce a burst of network traffic; this could also be due to unusually large amounts of reconnaissance or enumeration traffic. Denial-of-service attacks or traffic floods may also produce such a surge in traffic.",
|
||||
"groups": [
|
||||
"security",
|
||||
"network"
|
||||
],
|
||||
"analysis_config": {
|
||||
"bucket_span": "15m",
|
||||
"detectors": [
|
||||
{
|
||||
"detector_description": "high_count",
|
||||
"function": "high_count",
|
||||
"detector_index": 0
|
||||
}
|
||||
],
|
||||
"influencers": [
|
||||
"destination.geo.country_name",
|
||||
"destination.as.organization.name",
|
||||
"source.ip",
|
||||
"destination.ip"
|
||||
]
|
||||
},
|
||||
"allow_lazy_open": true,
|
||||
"analysis_limits": {
|
||||
"model_memory_limit": "32mb"
|
||||
},
|
||||
"data_description": {
|
||||
"time_field": "@timestamp"
|
||||
},
|
||||
"custom_settings": {
|
||||
"created_by": "ml-module-security-network"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
{
|
||||
"job_type": "anomaly_detector",
|
||||
"description": "Security: Network - looks for an unusual destination country name in the network logs. This can be due to initial access, persistence, command-and-control, or exfiltration activity. For example, when a user clicks on a link in a phishing email or opens a malicious document, a request may be sent to download and run a payload from a server in a country which does not normally appear in network traffic or business work-flows. Malware instances and persistence mechanisms may communicate with command-and-control (C2) infrastructure in their country of origin, which may be an unusual destination country for the source network.",
|
||||
"groups": [
|
||||
"security",
|
||||
"network"
|
||||
],
|
||||
"analysis_config": {
|
||||
"bucket_span": "15m",
|
||||
"detectors": [
|
||||
{
|
||||
"detector_description": "rare by \"destination.geo.country_name\"",
|
||||
"function": "rare",
|
||||
"by_field_name": "destination.geo.country_name",
|
||||
"detector_index": 0
|
||||
}
|
||||
],
|
||||
"influencers": [
|
||||
"destination.geo.country_name",
|
||||
"destination.as.organization.name",
|
||||
"source.ip",
|
||||
"destination.ip"
|
||||
]
|
||||
},
|
||||
"allow_lazy_open": true,
|
||||
"analysis_limits": {
|
||||
"model_memory_limit": "32mb"
|
||||
},
|
||||
"data_description": {
|
||||
"time_field": "@timestamp"
|
||||
},
|
||||
"custom_settings": {
|
||||
"created_by": "ml-module-security-network"
|
||||
}
|
||||
}
|
||||
|
|
@ -27,6 +27,7 @@ const moduleIds = [
|
|||
'sample_data_ecommerce',
|
||||
'sample_data_weblogs',
|
||||
'security_linux',
|
||||
'security_network',
|
||||
'security_windows',
|
||||
'siem_auditbeat',
|
||||
'siem_auditbeat_auth',
|
||||
|
|
|
|||
|
|
@ -143,7 +143,7 @@ export default ({ getService }: FtrProviderContext) => {
|
|||
user: USER.ML_POWERUSER,
|
||||
expected: {
|
||||
responseCode: 200,
|
||||
moduleIds: ['security_linux', 'security_windows'],
|
||||
moduleIds: ['security_linux', 'security_network', 'security_windows'],
|
||||
},
|
||||
},
|
||||
];
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue