mirror of
https://github.com/elastic/kibana.git
synced 2025-04-24 01:38:56 -04:00
[ML] Update auditbeat modules for ECS (#29934)
* [ML] Initial commit for auditbeat hosts ECS Rename fields for ECS Rework dashboards due to bwc * [ML] Further auditbeat tidy up and consistency changes Custom urls should link to saved search, not discover Ensure savedSearchId is used for visualizations Ensure filter terms are consistent TODO Decide if we should rename to auditd module TODO Fix for new saved object format * [ML] Refinements for auditbeat host module Remove duplicated title from visState Shrink panel heights in row 1 * [ML] Refinements to auditbeat module Update module name from auditd to auditbeat Add useMargins true for dashboards Add filter to custom url for exists auditd.data.syscall not exists container.runtime event.module: auditd * [ML] Initial commit for auditbeat_process_docker_ecs Update for ECS using container.name (instead of container.id) container.runtime: docker process.executable event.module: auditd auditd.data.syscall exists TODOs Use auditd.message_type: syscall (instead of auditd.data.syscall) Possibly combine with auditbeat hosts saved objects (depending on host.name being shared) Possibly combine to single dashboard Test against live auditbeat data collection
This commit is contained in:
Sophie Chang
Pete Harverson
parent
0aecd79c17
commit
9ebddcc8a4
24 changed files with 546 additions and 0 deletions
|
|
@ -0,0 +1,12 @@
|
|||
{
|
||||
"title": "ML Auditbeat Docker: Process Event Rate (ECS)",
|
||||
"hits": 0,
|
||||
"description": "Investigate unusual process event rates in a docker container",
|
||||
"panelsJSON": "[{\"size_x\":6,\"size_y\":4,\"row\":1,\"col\":1,\"id\":\"ml_auditbeat_docker_process_event_rate_vis_ecs\",\"panelIndex\":\"1\",\"type\":\"visualization\"},{\"size_x\":6,\"size_y\":4,\"row\":1,\"col\":7,\"id\":\"ml_auditbeat_docker_process_event_rate_by_process_ecs\",\"panelIndex\":\"2\",\"type\":\"visualization\"},{\"size_x\":12,\"size_y\":8,\"row\":5,\"col\":1,\"panelIndex\":\"3\",\"type\":\"search\",\"id\":\"ml_auditbeat_docker_process_events_ecs\"}]",
|
||||
"optionsJSON": "{\"useMargins\":true}",
|
||||
"version": 1,
|
||||
"timeRestore": false,
|
||||
"kibanaSavedObjectMeta": {
|
||||
"searchSourceJSON": "{}"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
{
|
||||
"title": "ML Auditbeat Docker: Process Explorer (ECS)",
|
||||
"hits": 0,
|
||||
"description": "Dashboard to explore processes for a docker container",
|
||||
"panelsJSON": "[{\"size_x\": 6,\"size_y\": 4,\"row\": 1,\"col\": 1,\"id\": \"ml_auditbeat_docker_process_occurrence_ecs\",\"panelIndex\": \"1\",\"type\": \"visualization\"},{\"size_x\": 12,\"size_y\": 8,\"row\": 5,\"col\": 1,\"panelIndex\": \"2\",\"type\": \"search\",\"id\": \"ml_auditbeat_docker_process_events_ecs\"},{\"size_x\": 6,\"size_y\": 4,\"row\": 1,\"col\": 7,\"panelIndex\": \"3\",\"type\": \"visualization\",\"id\": \"ml_auditbeat_docker_process_event_rate_by_process_ecs\"}\n]",
|
||||
"optionsJSON": "{\"useMargins\": true}",
|
||||
"version": 1,
|
||||
"timeRestore": false,
|
||||
"kibanaSavedObjectMeta": {
|
||||
"searchSourceJSON": "{}"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
{
|
||||
"title": "ML Auditbeat Docker: Process Events (ECS)",
|
||||
"description": "Auditbeat process events in docker containers",
|
||||
"hits": 0,
|
||||
"columns": [
|
||||
"container.runtime",
|
||||
"container.name",
|
||||
"auditd.data.syscall",
|
||||
"process.executable",
|
||||
"process.title"
|
||||
],
|
||||
"sort": [
|
||||
"@timestamp",
|
||||
"desc"
|
||||
],
|
||||
"version": 1,
|
||||
"kibanaSavedObjectMeta": {
|
||||
"searchSourceJSON": "{\"index\":\"INDEX_PATTERN_ID\",\"highlightAll\":true,\"version\":true,\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[{\"meta\":{\"index\":\"INDEX_PATTERN_ID\",\"negate\":false,\"disabled\":false,\"alias\":null,\"type\":\"phrase\",\"key\":\"container.runtime\",\"value\":\"docker\",\"params\":{\"query\":\"docker\"}},\"query\":{\"match\":{\"container.runtime\":{\"query\":\"docker\",\"type\":\"phrase\"}}},\"$state\":{\"store\":\"appState\"}},{\"meta\":{\"index\":\"INDEX_PATTERN_ID\",\"negate\":false,\"disabled\":false,\"alias\":null,\"type\":\"phrase\",\"key\":\"event.module\",\"value\":\"auditd\",\"params\":{\"query\":\"auditd\"}},\"query\":{\"match\":{\"event.module\":{\"query\":\"auditd\",\"type\":\"phrase\"}}},\"$state\":{\"store\":\"appState\"}},{\"meta\":{\"index\":\"INDEX_PATTERN_ID\",\"negate\":false,\"disabled\":false,\"alias\":null,\"type\":\"exists\",\"key\":\"auditd.data.syscall\",\"value\":\"exists\"},\"exists\":{\"field\":\"auditd.data.syscall\"},\"$state\":{\"store\":\"appState\"}}]}"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
{
|
||||
"title": "ML Auditbeat Docker: Process Event Rate by Process (ECS)",
|
||||
"visState": "{\"type\": \"histogram\",\"params\": {\"type\": \"histogram\",\"grid\": {\"categoryLines\": false,\"style\": {\"color\": \"#eee\"}},\"categoryAxes\": [{\"id\": \"CategoryAxis-1\",\"type\": \"category\",\"position\": \"bottom\",\"show\": true,\"style\": {},\"scale\": { \"type\": \"linear\"},\"labels\": { \"show\": true, \"truncate\": 100},\"title\": {}}],\"valueAxes\": [{\"id\": \"ValueAxis-1\",\"name\": \"LeftAxis-1\",\"type\": \"value\",\"position\": \"left\",\"show\": true,\"style\": {},\"scale\": { \"type\": \"linear\", \"mode\": \"normal\"},\"labels\": { \"show\": true, \"rotate\": 0, \"filter\": false, \"truncate\": 100},\"title\": { \"text\": \"Count\"}}],\"seriesParams\": [{\"show\": \"true\",\"type\": \"histogram\",\"mode\": \"stacked\",\"data\": { \"label\": \"Count\", \"id\": \"1\"},\"valueAxis\": \"ValueAxis-1\",\"drawLinesBetweenPoints\": true,\"showCircles\": true}],\"addTooltip\": true,\"addLegend\": true,\"legendPosition\": \"right\",\"times\": [],\"addTimeMarker\": false},\"aggs\": [{\"id\": \"1\",\"enabled\": true,\"type\": \"count\",\"schema\": \"metric\",\"params\": {}},{\"id\": \"2\",\"enabled\": true,\"type\": \"date_histogram\",\"schema\": \"segment\",\"params\": {\"field\": \"@timestamp\",\"useNormalizedEsInterval\": true,\"interval\": \"auto\",\"time_zone\": \"UTC\",\"drop_partials\": false,\"customInterval\": \"2h\",\"min_doc_count\": 1,\"extended_bounds\": {}}},{\"id\": \"3\",\"enabled\": true,\"type\": \"terms\",\"schema\": \"group\",\"params\": {\"field\": \"process.executable\",\"size\": 10,\"order\": \"desc\",\"orderBy\": \"1\",\"otherBucket\": false,\"otherBucketLabel\": \"Other\",\"missingBucket\": false,\"missingBucketLabel\": \"Missing\"}}]}",
|
||||
"uiStateJSON": "{}",
|
||||
"description": "",
|
||||
"savedSearchId": "ml_auditbeat_docker_process_events_ecs",
|
||||
"version": 1,
|
||||
"kibanaSavedObjectMeta": {
|
||||
"searchSourceJSON": "{}"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
{
|
||||
"title": "ML Auditbeat Docker: Process Event Rate (ECS)",
|
||||
"visState": "{\"type\": \"line\",\"params\": {\"type\": \"line\",\"grid\": { \"categoryLines\": false, \"style\": { \"color\": \"#eee\" }},\"categoryAxes\": [ { \"id\": \"CategoryAxis-1\", \"type\": \"category\", \"position\": \"bottom\", \"show\": true, \"style\": {}, \"scale\": { \"type\": \"linear\" }, \"labels\": { \"show\": true, \"truncate\": 100 }, \"title\": {} }],\"valueAxes\": [ { \"id\": \"ValueAxis-1\", \"name\": \"LeftAxis-1\", \"type\": \"value\", \"position\": \"left\", \"show\": true, \"style\": {}, \"scale\": { \"type\": \"linear\", \"mode\": \"normal\" }, \"labels\": { \"show\": true, \"rotate\": 0, \"filter\": false, \"truncate\": 100 }, \"title\": { \"text\": \"Count\" } }],\"seriesParams\": [ { \"show\": \"true\", \"type\": \"line\", \"mode\": \"normal\", \"data\": { \"label\": \"Count\", \"id\": \"1\" }, \"valueAxis\": \"ValueAxis-1\", \"drawLinesBetweenPoints\": true, \"showCircles\": true }],\"addTooltip\": true,\"addLegend\": true,\"legendPosition\": \"right\",\"times\": [],\"addTimeMarker\": false},\"aggs\": [{ \"id\": \"1\", \"enabled\": true, \"type\": \"count\", \"schema\": \"metric\", \"params\": {}},{ \"id\": \"2\", \"enabled\": true, \"type\": \"date_histogram\", \"schema\": \"segment\", \"params\": { \"field\": \"@timestamp\", \"useNormalizedEsInterval\": true, \"interval\": \"auto\", \"time_zone\": \"UTC\", \"drop_partials\": false, \"customInterval\": \"2h\", \"min_doc_count\": 1, \"extended_bounds\": {} }},{ \"id\": \"3\", \"enabled\": true, \"type\": \"terms\", \"schema\": \"group\", \"params\": { \"field\": \"container.name\", \"size\": 10, \"order\": \"desc\", \"orderBy\": \"1\", \"otherBucket\": false, \"otherBucketLabel\": \"Other\", \"missingBucket\": false, \"missingBucketLabel\": \"Missing\" }}]}",
|
||||
"uiStateJSON": "{}",
|
||||
"description": "",
|
||||
"savedSearchId": "ml_auditbeat_docker_process_events_ecs",
|
||||
"version": 1,
|
||||
"kibanaSavedObjectMeta": {
|
||||
"searchSourceJSON": "{}"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
{
|
||||
"title": "ML Auditbeat Docker: Process Occurrence - experimental (ECS)",
|
||||
"visState": "{\"type\":\"vega\",\"params\":{\"spec\":\"{\\n $schema: https://vega.github.io/schema/vega-lite/v2.json\\n mark: {type: \\\"point\\\"}\\n data: {\\n url: {\\n index: \\\"INDEX_PATTERN_NAME\\\"\\n body: {\\n size: 10000\\n query: {\\n bool: {\\n must: [\\n %dashboard_context-must_clause%\\n {\\n exists: {field: \\\"process.executable\\\"}\\n }\\n {\\n function_score: {\\n random_score: {seed: 10, field: \\\"_seq_no\\\"}\\n }\\n }\\n {\\n range: {\\n @timestamp: {\\n %timefilter%: true\\n }\\n }\\n }\\n ]\\n must_not: [\\n \\\"%dashboard_context-must_not_clause%\\\"\\n ]\\n }\\n }\\n script_fields: {\\n process_exe: {\\n script: {source: \\\"params['_source']['process']['executable']\\\"}\\n }\\n }\\n _source: [\\\"@timestamp\\\", \\\"process_exe\\\"]\\n }\\n }\\n format: {property: \\\"hits.hits\\\"}\\n }\\n transform: [\\n {calculate: \\\"toDate(datum._source['@timestamp'])\\\", as: \\\"time\\\"}\\n ]\\n encoding: {\\n x: {\\n field: time\\n type: temporal\\n axis: {labels: true, ticks: true, title: false},\\n timeUnit: utcyearmonthdatehoursminutes\\n }\\n y: {\\n field: fields.process_exe\\n type: ordinal\\n sort: {op: \\\"count\\\", order: \\\"descending\\\"}\\n axis: {labels: true, title: \\\"occurrence of process.executable\\\", ticks: false}\\n }\\n }\\n config: {\\n style: {\\n point: {filled: true}\\n }\\n }\\n}\"},\"aggs\":[]}",
|
||||
"uiStateJSON": "{}",
|
||||
"description": "",
|
||||
"savedSearchId": "ml_auditbeat_docker_process_events_ecs",
|
||||
"version": 1,
|
||||
"kibanaSavedObjectMeta": {
|
||||
"searchSourceJSON": "{}"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"src": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgAgMAAAAOFJJnAAAADFBMVEUAAAAAAAABf3X////ZaOWRAAAAAXRSTlMAQObYZgAAAAFiS0dEAxEMTPIAAAAfSURBVBjTYwgNDXVqBBIMcEYAAwNTAwMD60hkYIQGAIQRIolX2EV0AAAAAElFTkSuQmCC",
|
||||
"height": 32,
|
||||
"width": 32
|
||||
}
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
{
|
||||
"id": "auditbeat_process_docker_ecs",
|
||||
"title": "Auditbeat docker processes",
|
||||
"description": "Detect unusual processes in docker containers from auditd data (ECS)",
|
||||
"type": "Auditbeat data",
|
||||
"logoFile": "logo.json",
|
||||
"defaultIndexPattern": "auditbeat-*",
|
||||
"query": {
|
||||
"bool": {
|
||||
"filter": [
|
||||
{ "term": { "event.module": "auditd" } },
|
||||
{ "term": { "container.runtime": "docker" } }
|
||||
],
|
||||
"must": {
|
||||
"exists": { "field": "auditd.data.syscall" }
|
||||
}
|
||||
}
|
||||
},
|
||||
"jobs": [
|
||||
{
|
||||
"id": "docker_high_count_process_events_ecs",
|
||||
"file": "docker_high_count_process_events_ecs.json"
|
||||
},
|
||||
{
|
||||
"id": "docker_rare_process_activity_ecs",
|
||||
"file": "docker_rare_process_activity_ecs.json"
|
||||
}
|
||||
],
|
||||
"datafeeds": [
|
||||
{
|
||||
"id": "datafeed-docker_high_count_process_events_ecs",
|
||||
"file": "datafeed_docker_high_count_process_events_ecs.json",
|
||||
"job_id": "docker_high_count_process_events_ecs"
|
||||
},
|
||||
{
|
||||
"id": "datafeed-docker_rare_process_activity_ecs",
|
||||
"file": "datafeed_docker_rare_process_activity_ecs.json",
|
||||
"job_id": "docker_rare_process_activity_ecs"
|
||||
}
|
||||
],
|
||||
"kibana": {
|
||||
"dashboard": [
|
||||
{
|
||||
"id": "ml_auditbeat_docker_process_event_rate_ecs",
|
||||
"file": "ml_auditbeat_docker_process_event_rate_ecs.json"
|
||||
},
|
||||
{
|
||||
"id": "ml_auditbeat_docker_process_explorer_ecs",
|
||||
"file": "ml_auditbeat_docker_process_explorer_ecs.json"
|
||||
}
|
||||
],
|
||||
"search": [
|
||||
{
|
||||
"id": "ml_auditbeat_docker_process_events_ecs",
|
||||
"file": "ml_auditbeat_docker_process_events_ecs.json"
|
||||
}
|
||||
],
|
||||
"visualization": [
|
||||
{
|
||||
"id": "ml_auditbeat_docker_process_event_rate_by_process_ecs",
|
||||
"file": "ml_auditbeat_docker_process_event_rate_by_process_ecs.json"
|
||||
},
|
||||
{
|
||||
"id": "ml_auditbeat_docker_process_event_rate_vis_ecs",
|
||||
"file": "ml_auditbeat_docker_process_event_rate_vis_ecs.json"
|
||||
},
|
||||
{
|
||||
"id": "ml_auditbeat_docker_process_occurrence_ecs",
|
||||
"file": "ml_auditbeat_docker_process_occurrence_ecs.json"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
{
|
||||
"job_id": "JOB_ID",
|
||||
"indexes": [
|
||||
"INDEX_PATTERN_NAME"
|
||||
],
|
||||
"query": {
|
||||
"bool": {
|
||||
"filter": [
|
||||
{ "term": { "event.module": "auditd" } },
|
||||
{ "term": { "container.runtime": "docker" } }
|
||||
],
|
||||
"must": {
|
||||
"exists": { "field": "auditd.data.syscall" }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
{
|
||||
"job_id": "JOB_ID",
|
||||
"indexes": [
|
||||
"INDEX_PATTERN_NAME"
|
||||
],
|
||||
"query": {
|
||||
"bool": {
|
||||
"filter": [
|
||||
{ "term": { "event.module": "auditd" } },
|
||||
{ "term": { "container.runtime": "docker" } }
|
||||
],
|
||||
"must": {
|
||||
"exists": { "field": "auditd.data.syscall" }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
{
|
||||
"job_type": "anomaly_detector",
|
||||
"description": "Auditbeat: Detect unusual increases in process execution rates in docker containers (ECS)",
|
||||
"groups": ["auditd"],
|
||||
"analysis_config": {
|
||||
"bucket_span": "10m",
|
||||
"detectors": [
|
||||
{
|
||||
"detector_description": "high_non_zero_count partition container.name",
|
||||
"function": "high_count",
|
||||
"partition_field_name": "container.name"
|
||||
}
|
||||
],
|
||||
"influencers": [
|
||||
"container.name",
|
||||
"process.executable"
|
||||
]
|
||||
},
|
||||
"analysis_limits": {
|
||||
"model_memory_limit": "256mb",
|
||||
"categorization_examples_limit": 4
|
||||
},
|
||||
"data_description": {
|
||||
"time_field": "@timestamp",
|
||||
"time_format": "epoch_ms"
|
||||
},
|
||||
"custom_settings": {
|
||||
"custom_urls": [
|
||||
{
|
||||
"url_name": "Process rate",
|
||||
"time_range": "1h",
|
||||
"url_value": "kibana#/dashboard/ml_auditbeat_docker_process_event_rate_ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!f,index:INDEX_PATTERN_ID,key:event.module,negate:!f,params:(query:auditd),type:phrase,value:auditd),query:(match:(event.module:(query:auditd,type:phrase)))),('$state':(store:appState),meta:(alias:!n,disabled:!f,index:INDEX_PATTERN_ID,key:container.runtime,negate:!f,params:(query:docker),type:phrase,value:docker),query:(match:(container.runtime:(query:docker,type:phrase)))),('$state':(store:appState),exists:(field:auditd.data.syscall),meta:(alias:!n,disabled:!f,index:INDEX_PATTERN_ID,key:auditd.data.syscall,negate:!f,type:exists,value:exists))),query:(language:lucene,query:\u0027container.name:\u0022$container.name$\u0022\u0027))"
|
||||
},
|
||||
{
|
||||
"url_name": "Raw data",
|
||||
"time_range": "1h",
|
||||
"url_value": "kibana#/discover/ml_auditbeat_docker_process_events_ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(index:\u0027INDEX_PATTERN_ID\u0027,query:(language:lucene,query:\u0027container.name:\u0022$container.name$\u0022\u0027))"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
{
|
||||
"job_type": "anomaly_detector",
|
||||
"description": "Auditbeat: Detect rare process executions in docker containers (ECS)",
|
||||
"groups": ["auditd"],
|
||||
"analysis_config": {
|
||||
"bucket_span": "10m",
|
||||
"detectors": [
|
||||
{
|
||||
"function": "rare",
|
||||
"by_field_name": "process.executable",
|
||||
"partition_field_name": "container.name"
|
||||
}
|
||||
],
|
||||
"influencers": [
|
||||
"container.name",
|
||||
"process.executable"
|
||||
]
|
||||
},
|
||||
"analysis_limits": {
|
||||
"model_memory_limit": "256mb"
|
||||
},
|
||||
"data_description": {
|
||||
"time_field": "@timestamp",
|
||||
"time_format": "epoch_ms"
|
||||
},
|
||||
"custom_settings": {
|
||||
"custom_urls": [
|
||||
{
|
||||
"url_name": "Process explorer",
|
||||
"time_range": "1h",
|
||||
"url_value": "kibana#/dashboard/ml_auditbeat_docker_process_explorer_ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!f,index:INDEX_PATTERN_ID,key:event.module,negate:!f,params:(query:auditd),type:phrase,value:auditd),query:(match:(event.module:(query:auditd,type:phrase)))),('$state':(store:appState),meta:(alias:!n,disabled:!f,index:INDEX_PATTERN_ID,key:container.runtime,negate:!f,params:(query:docker),type:phrase,value:docker),query:(match:(container.runtime:(query:docker,type:phrase)))),('$state':(store:appState),exists:(field:auditd.data.syscall),meta:(alias:!n,disabled:!f,index:INDEX_PATTERN_ID,key:auditd.data.syscall,negate:!f,type:exists,value:exists))),query:(language:lucene,query:\u0027container.name:\u0022$container.name$\u0022\u0027))"
|
||||
},
|
||||
{
|
||||
"url_name": "Raw data",
|
||||
"time_range": "1h",
|
||||
"url_value": "kibana#/discover/ml_auditbeat_docker_process_events_ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(index:\u0027INDEX_PATTERN_ID\u0027,query:(language:lucene,query:\u0027container.name:\u0022$container.name$\u0022 AND process.executable:\u0022$process.executable$\u0022\u0027))"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
{
|
||||
"title": "ML Auditbeat Hosts: Process Event Rate (ECS)",
|
||||
"hits": 0,
|
||||
"description": "Investigate unusual process event rates on a host",
|
||||
"panelsJSON": "[{\"size_x\":6,\"size_y\":4,\"row\":1,\"col\":1,\"id\":\"ml_auditbeat_hosts_process_event_rate_vis_ecs\",\"panelIndex\":\"1\",\"type\":\"visualization\"},{\"size_x\":6,\"size_y\":4,\"row\":1,\"col\":7,\"id\":\"ml_auditbeat_hosts_process_event_rate_by_process_ecs\",\"panelIndex\":\"2\",\"type\":\"visualization\"},{\"size_x\":12,\"size_y\":8,\"row\":5,\"col\":1,\"panelIndex\":\"3\",\"type\":\"search\",\"id\":\"ml_auditbeat_hosts_process_events_ecs\"}]",
|
||||
"optionsJSON": "{\"useMargins\":true}",
|
||||
"version": 1,
|
||||
"timeRestore": false,
|
||||
"kibanaSavedObjectMeta": {
|
||||
"searchSourceJSON": "{}"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
{
|
||||
"title": "ML Auditbeat Hosts: Process Explorer (ECS)",
|
||||
"hits": 0,
|
||||
"description": "Explore processes on a host",
|
||||
"panelsJSON": "[{\"size_x\": 6,\"size_y\": 4,\"row\": 1,\"col\": 1,\"id\": \"ml_auditbeat_hosts_process_occurrence_ecs\",\"panelIndex\": \"1\",\"type\": \"visualization\"},{\"size_x\": 12,\"size_y\": 8,\"row\": 5,\"col\": 1,\"panelIndex\": \"2\",\"type\": \"search\",\"id\": \"ml_auditbeat_hosts_process_events_ecs\"},{\"size_x\": 6,\"size_y\": 4,\"row\": 1,\"col\": 7,\"panelIndex\": \"3\",\"type\": \"visualization\",\"id\": \"ml_auditbeat_hosts_process_event_rate_by_process_ecs\"}\n]",
|
||||
"optionsJSON": "{\"useMargins\":true}",
|
||||
"version": 1,
|
||||
"timeRestore": false,
|
||||
"kibanaSavedObjectMeta": {
|
||||
"searchSourceJSON": "{}"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
{
|
||||
"title": "ML Auditbeat Hosts: Process Events (ECS)",
|
||||
"description": "Auditbeat auditd process events on host machines",
|
||||
"hits": 0,
|
||||
"columns": [
|
||||
"host.name",
|
||||
"auditd.data.syscall",
|
||||
"process.executable",
|
||||
"process.title"
|
||||
],
|
||||
"sort": [
|
||||
"@timestamp",
|
||||
"desc"
|
||||
],
|
||||
"version": 1,
|
||||
"kibanaSavedObjectMeta": {
|
||||
"searchSourceJSON": "{\"index\":\"INDEX_PATTERN_ID\",\"highlightAll\":true,\"version\":true,\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[{\"meta\":{\"index\":\"INDEX_PATTERN_ID\",\"negate\":true,\"disabled\":false,\"alias\":null,\"type\":\"exists\",\"key\":\"container.runtime\",\"value\":\"exists\"},\"exists\":{\"field\":\"container.runtime\"},\"$state\":{\"store\":\"appState\"}},{\"meta\":{\"index\":\"INDEX_PATTERN_ID\",\"negate\":false,\"disabled\":false,\"alias\":null,\"type\":\"phrase\",\"key\":\"event.module\",\"value\":\"auditd\",\"params\":{\"query\":\"auditd\"}},\"query\":{\"match\":{\"event.module\":{\"query\":\"auditd\",\"type\":\"phrase\"}}},\"$state\":{\"store\":\"appState\"}},{\"meta\":{\"index\":\"INDEX_PATTERN_ID\",\"negate\":false,\"disabled\":false,\"alias\":null,\"type\":\"exists\",\"key\":\"auditd.data.syscall\",\"value\":\"exists\"},\"exists\":{\"field\":\"auditd.data.syscall\"},\"$state\":{\"store\":\"appState\"}}]}"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
{
|
||||
"title": "ML Auditbeat Hosts: Process Event Rate by Process (ECS)",
|
||||
"visState": "{\"type\": \"histogram\",\"params\": {\"type\": \"histogram\",\"grid\": {\"categoryLines\": false,\"style\": {\"color\": \"#eee\"}},\"categoryAxes\": [{\"id\": \"CategoryAxis-1\",\"type\": \"category\",\"position\": \"bottom\",\"show\": true,\"style\": {},\"scale\": {\"type\": \"linear\"},\"labels\": {\"show\": true,\"truncate\": 100},\"title\": {}}],\"valueAxes\": [{\"id\": \"ValueAxis-1\",\"name\": \"LeftAxis-1\",\"type\": \"value\",\"position\": \"left\",\"show\": true,\"style\": {},\"scale\": {\"type\": \"linear\",\"mode\": \"normal\"},\"labels\": {\"show\": true,\"rotate\": 0,\"filter\": false,\"truncate\": 100},\"title\": {\"text\": \"Count\"}}],\"seriesParams\": [{\"show\": \"true\",\"type\": \"histogram\",\"mode\": \"stacked\",\"data\": {\"label\": \"Count\",\"id\": \"1\"},\"valueAxis\": \"ValueAxis-1\",\"drawLinesBetweenPoints\": true,\"showCircles\": true}],\"addTooltip\": true,\"addLegend\": true,\"legendPosition\": \"right\",\"times\": [],\"addTimeMarker\": false},\"aggs\": [{\"id\": \"1\",\"enabled\": true,\"type\": \"count\",\"schema\": \"metric\",\"params\": {}},{\"id\": \"2\",\"enabled\": true,\"type\": \"date_histogram\",\"schema\": \"segment\",\"params\": {\"field\": \"@timestamp\",\"useNormalizedEsInterval\": true,\"interval\": \"auto\",\"time_zone\": \"UTC\",\"drop_partials\": false,\"customInterval\": \"2h\",\"min_doc_count\": 1,\"extended_bounds\": {}}},{\"id\": \"3\",\"enabled\": true,\"type\": \"terms\",\"schema\": \"group\",\"params\": {\"field\": \"process.executable\",\"size\": 10,\"order\": \"desc\",\"orderBy\": \"1\",\"otherBucket\": false,\"otherBucketLabel\": \"Other\",\"missingBucket\": false,\"missingBucketLabel\": \"Missing\"}}]}",
|
||||
"uiStateJSON": "{}",
|
||||
"description": "",
|
||||
"savedSearchId": "ml_auditbeat_hosts_process_events_ecs",
|
||||
"version": 1,
|
||||
"kibanaSavedObjectMeta": {
|
||||
"searchSourceJSON": "{}"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
{
|
||||
"title": "ML Auditbeat Hosts: Process Event Rate (ECS)",
|
||||
"visState":"{\"type\": \"line\",\"params\": {\"type\": \"line\",\"grid\": {\"categoryLines\": false,\"style\": {\"color\": \"#eee\"}},\"categoryAxes\": [{\"id\": \"CategoryAxis-1\",\"type\": \"category\",\"position\": \"bottom\",\"show\": true,\"style\": {},\"scale\": {\"type\": \"linear\"},\"labels\": {\"show\": true,\"truncate\": 100},\"title\": {}}],\"valueAxes\": [{\"id\": \"ValueAxis-1\",\"name\": \"LeftAxis-1\",\"type\": \"value\",\"position\": \"left\",\"show\": true,\"style\": {},\"scale\": {\"type\": \"linear\",\"mode\": \"normal\"},\"labels\": {\"show\": true,\"rotate\": 0,\"filter\": false,\"truncate\": 100},\"title\": {\"text\": \"Count\"}}],\"seriesParams\": [{\"show\": \"true\",\"type\": \"line\",\"mode\": \"normal\",\"data\": {\"label\": \"Count\",\"id\": \"1\"},\"valueAxis\": \"ValueAxis-1\",\"drawLinesBetweenPoints\": true,\"showCircles\": true}],\"addTooltip\": true,\"addLegend\": true,\"legendPosition\": \"right\",\"times\": [],\"addTimeMarker\": false},\"aggs\": [{\"id\": \"1\",\"enabled\": true,\"type\": \"count\",\"schema\": \"metric\",\"params\": {}},{\"id\": \"2\",\"enabled\": true,\"type\": \"date_histogram\",\"schema\": \"segment\",\"params\": {\"field\": \"@timestamp\",\"useNormalizedEsInterval\": true,\"interval\": \"auto\",\"time_zone\": \"UTC\",\"drop_partials\": false,\"customInterval\": \"2h\",\"min_doc_count\": 1,\"extended_bounds\": {}}},{\"id\": \"3\",\"enabled\": true,\"type\": \"terms\",\"schema\": \"group\",\"params\": {\"field\": \"host.name\",\"size\": 10,\"order\": \"desc\",\"orderBy\": \"1\",\"otherBucket\": false,\"otherBucketLabel\": \"Other\",\"missingBucket\": false,\"missingBucketLabel\": \"Missing\"}}]}",
|
||||
"uiStateJSON": "{}",
|
||||
"description": "",
|
||||
"savedSearchId": "ml_auditbeat_hosts_process_events_ecs",
|
||||
"version": 1,
|
||||
"kibanaSavedObjectMeta": {
|
||||
"searchSourceJSON": "{}"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
{
|
||||
"title": "ML Auditbeat Hosts: Process Occurrence - experimental (ECS)",
|
||||
"visState": "{\"type\":\"vega\",\"params\":{\"spec\":\"{\\n $schema: https://vega.github.io/schema/vega-lite/v2.json\\n mark: {type: \\\"point\\\"}\\n data: {\\n url: {\\n index: \\\"INDEX_PATTERN_NAME\\\"\\n body: {\\n size: 10000\\n query: {\\n bool: {\\n must: [\\n %dashboard_context-must_clause%\\n {\\n exists: {field: \\\"process.executable\\\"}\\n }\\n {\\n function_score: {\\n random_score: {seed: 10, field: \\\"_seq_no\\\"}\\n }\\n }\\n {\\n range: {\\n @timestamp: {\\n %timefilter%: true\\n }\\n }\\n }\\n ]\\n must_not: [\\n \\\"%dashboard_context-must_not_clause%\\\"\\n ]\\n }\\n }\\n script_fields: {\\n process_exe: {\\n script: {source: \\\"params['_source']['process']['executable']\\\"}\\n }\\n }\\n _source: [\\\"@timestamp\\\", \\\"process_exe\\\"]\\n }\\n }\\n format: {property: \\\"hits.hits\\\"}\\n }\\n transform: [\\n {calculate: \\\"toDate(datum._source['@timestamp'])\\\", as: \\\"time\\\"}\\n ]\\n encoding: {\\n x: {\\n field: time\\n type: temporal\\n axis: {labels: true, ticks: true, title: false},\\n timeUnit: utcyearmonthdatehoursminutes\\n }\\n y: {\\n field: fields.process_exe\\n type: ordinal\\n sort: {op: \\\"count\\\", order: \\\"descending\\\"}\\n axis: {labels: true, title: \\\"occurrence of process.executable\\\", ticks: false}\\n }\\n }\\n config: {\\n style: {\\n point: {filled: true}\\n }\\n }\\n}\"},\"aggs\":[]}",
|
||||
"uiStateJSON": "{}",
|
||||
"description": "",
|
||||
"savedSearchId": "ml_auditbeat_hosts_process_events_ecs",
|
||||
"version": 1,
|
||||
"kibanaSavedObjectMeta": {
|
||||
"searchSourceJSON": "{}"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"src": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgAgMAAAAOFJJnAAAADFBMVEUAAAAAAAABf3X////ZaOWRAAAAAXRSTlMAQObYZgAAAAFiS0dEAxEMTPIAAAAfSURBVBjTYwgNDXVqBBIMcEYAAwNTAwMD60hkYIQGAIQRIolX2EV0AAAAAElFTkSuQmCC",
|
||||
"height": 32,
|
||||
"width": 32
|
||||
}
|
||||
|
|
@ -0,0 +1,75 @@
|
|||
{
|
||||
"id": "auditbeat_process_hosts_ecs",
|
||||
"title": "Auditbeat host processes",
|
||||
"description": "Detect unusual processes on hosts from auditd data (ECS)",
|
||||
"type": "Auditbeat data",
|
||||
"logoFile": "logo.json",
|
||||
"defaultIndexPattern": "auditbeat-*",
|
||||
"query": {
|
||||
"bool": {
|
||||
"filter": [
|
||||
{ "term": { "event.module": "auditd" } }
|
||||
],
|
||||
"must": {
|
||||
"exists": { "field": "auditd.data.syscall" }
|
||||
},
|
||||
"must_not": {
|
||||
"exists": { "field": "container.runtime" }
|
||||
}
|
||||
}
|
||||
},
|
||||
"jobs": [
|
||||
{
|
||||
"id": "hosts_high_count_process_events_ecs",
|
||||
"file": "hosts_high_count_process_events_ecs.json"
|
||||
},
|
||||
{
|
||||
"id": "hosts_rare_process_activity_ecs",
|
||||
"file": "hosts_rare_process_activity_ecs.json"
|
||||
}
|
||||
],
|
||||
"datafeeds": [
|
||||
{
|
||||
"id": "datafeed-hosts_high_count_process_events_ecs",
|
||||
"file": "datafeed_hosts_high_count_process_events_ecs.json",
|
||||
"job_id": "hosts_high_count_process_events_ecs"
|
||||
},
|
||||
{
|
||||
"id": "datafeed-hosts_rare_process_activity_ecs",
|
||||
"file": "datafeed_hosts_rare_process_activity_ecs.json",
|
||||
"job_id": "hosts_rare_process_activity_ecs"
|
||||
}
|
||||
],
|
||||
"kibana": {
|
||||
"dashboard": [
|
||||
{
|
||||
"id": "ml_auditbeat_hosts_process_event_rate_ecs",
|
||||
"file": "ml_auditbeat_hosts_process_event_rate_ecs.json"
|
||||
},
|
||||
{
|
||||
"id": "ml_auditbeat_hosts_process_explorer_ecs",
|
||||
"file": "ml_auditbeat_hosts_process_explorer_ecs.json"
|
||||
}
|
||||
],
|
||||
"search": [
|
||||
{
|
||||
"id": "ml_auditbeat_hosts_process_events_ecs",
|
||||
"file": "ml_auditbeat_hosts_process_events_ecs.json"
|
||||
}
|
||||
],
|
||||
"visualization": [
|
||||
{
|
||||
"id": "ml_auditbeat_hosts_process_event_rate_by_process_ecs",
|
||||
"file": "ml_auditbeat_hosts_process_event_rate_by_process_ecs.json"
|
||||
},
|
||||
{
|
||||
"id": "ml_auditbeat_hosts_process_event_rate_vis_ecs",
|
||||
"file": "ml_auditbeat_hosts_process_event_rate_vis_ecs.json"
|
||||
},
|
||||
{
|
||||
"id": "ml_auditbeat_hosts_process_occurrence_ecs",
|
||||
"file": "ml_auditbeat_hosts_process_occurrence_ecs.json"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
{
|
||||
"job_id": "JOB_ID",
|
||||
"indexes": [
|
||||
"INDEX_PATTERN_NAME"
|
||||
],
|
||||
"query": {
|
||||
"bool": {
|
||||
"filter": [
|
||||
{ "term": { "event.module": "auditd" } }
|
||||
],
|
||||
"must": {
|
||||
"exists": { "field": "auditd.data.syscall" }
|
||||
},
|
||||
"must_not": {
|
||||
"exists": { "field": "container.runtime" }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
{
|
||||
"job_id": "JOB_ID",
|
||||
"indexes": [
|
||||
"INDEX_PATTERN_NAME"
|
||||
],
|
||||
"query": {
|
||||
"bool": {
|
||||
"filter": [
|
||||
{ "term": { "event.module": "auditd" } }
|
||||
],
|
||||
"must": {
|
||||
"exists": { "field": "auditd.data.syscall" }
|
||||
},
|
||||
"must_not": {
|
||||
"exists": { "field": "container.runtime" }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
{
|
||||
"job_type": "anomaly_detector",
|
||||
"description": "Auditbeat Hosts: Detect unusual increases in process execution rates (ECS)",
|
||||
"groups": ["auditd"],
|
||||
"analysis_config": {
|
||||
"bucket_span": "10m",
|
||||
"detectors": [
|
||||
{
|
||||
"detector_description": "high_non_zero_count partition host.name",
|
||||
"function": "high_non_zero_count",
|
||||
"partition_field_name": "host.name"
|
||||
}
|
||||
],
|
||||
"influencers": [
|
||||
"host.name",
|
||||
"process.executable"
|
||||
]
|
||||
},
|
||||
"analysis_limits": {
|
||||
"model_memory_limit": "256mb"
|
||||
},
|
||||
"data_description": {
|
||||
"time_field": "@timestamp",
|
||||
"time_format": "epoch_ms"
|
||||
},
|
||||
"custom_settings": {
|
||||
"created_by": "ml-module-auditd-hosts",
|
||||
"custom_urls": [
|
||||
{
|
||||
"url_name": "Process rate",
|
||||
"time_range": "1h",
|
||||
"url_value": "kibana#/dashboard/ml_auditbeat_hosts_process_event_rate_ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!f,index:INDEX_PATTERN_ID,key:event.module,negate:!f,params:(query:auditd),type:phrase,value:auditd),query:(match:(event.module:(query:auditd,type:phrase)))),('$state':(store:appState),exists:(field:container.runtime),meta:(alias:!n,disabled:!f,index:INDEX_PATTERN_ID,key:container.runtime,negate:!t,type:exists,value:exists)),('$state':(store:appState),exists:(field:auditd.data.syscall),meta:(alias:!n,disabled:!f,index:INDEX_PATTERN_ID,key:auditd.data.syscall,negate:!f,type:exists,value:exists))),query:(language:lucene,query:\u0027host.name:\u0022$host.name$\u0022\u0027))"
|
||||
},
|
||||
{
|
||||
"url_name": "Raw data",
|
||||
"time_range": "1h",
|
||||
"url_value": "kibana#/discover/ml_auditbeat_hosts_process_events_ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(index:\u0027INDEX_PATTERN_ID\u0027,query:(language:lucene,query:\u0027host.name:\u0022$host.name$\u0022\u0027))"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
{
|
||||
"job_type": "anomaly_detector",
|
||||
"description": "Auditbeat Hosts: Detect rare process executions on hosts (ECS)",
|
||||
"groups": ["auditd"],
|
||||
"analysis_config": {
|
||||
"bucket_span": "10m",
|
||||
"detectors": [
|
||||
{
|
||||
"function": "rare",
|
||||
"by_field_name": "process.executable",
|
||||
"partition_field_name": "host.name"
|
||||
}
|
||||
],
|
||||
"influencers": [
|
||||
"host.name",
|
||||
"process.executable"
|
||||
]
|
||||
},
|
||||
"analysis_limits": {
|
||||
"model_memory_limit": "256mb"
|
||||
},
|
||||
"data_description": {
|
||||
"time_field": "@timestamp",
|
||||
"time_format": "epoch_ms"
|
||||
},
|
||||
"custom_settings": {
|
||||
"created_by": "ml-module-auditd-hosts",
|
||||
"custom_urls": [
|
||||
{
|
||||
"url_name": "Process explorer",
|
||||
"time_range": "1h",
|
||||
"url_value": "kibana#/dashboard/ml_auditbeat_hosts_process_explorer_ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!f,index:INDEX_PATTERN_ID,key:event.module,negate:!f,params:(query:auditd),type:phrase,value:auditd),query:(match:(event.module:(query:auditd,type:phrase)))),('$state':(store:appState),exists:(field:container.runtime),meta:(alias:!n,disabled:!f,index:INDEX_PATTERN_ID,key:container.runtime,negate:!t,type:exists,value:exists)),('$state':(store:appState),exists:(field:auditd.data.syscall),meta:(alias:!n,disabled:!f,index:INDEX_PATTERN_ID,key:auditd.data.syscall,negate:!f,type:exists,value:exists))),query:(language:lucene,query:\u0027host.name:\u0022$host.name$\u0022\u0027))"
|
||||
},
|
||||
{
|
||||
"url_name": "Raw data",
|
||||
"time_range": "1h",
|
||||
"url_value": "kibana#/discover/ml_auditbeat_hosts_process_events_ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(index:\u0027INDEX_PATTERN_ID\u0027,query:(language:lucene,query:\u0027host.name:\u0022$host.name$\u0022 AND process.executable:\u0022$process.executable$\u0022\u0027))"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue