mirror of
https://github.com/elastic/kibana.git
synced 2025-04-24 09:48:58 -04:00
[ML] Fixing module datafeed overrides (#78925)
This commit is contained in:
James Gowdy
GitHub
parent
5f5ef2b344
commit
c4f05c4758
27 changed files with 766 additions and 599 deletions
|
|
@ -11,7 +11,7 @@ export interface ModuleJob {
|
|||
config: Omit<Job, 'job_id'>;
|
||||
}
|
||||
|
||||
export interface ModuleDataFeed {
|
||||
export interface ModuleDatafeed {
|
||||
id: string;
|
||||
config: Omit<Datafeed, 'datafeed_id'>;
|
||||
}
|
||||
|
|
@ -49,7 +49,7 @@ export interface Module {
|
|||
defaultIndexPattern: string;
|
||||
query: any;
|
||||
jobs: ModuleJob[];
|
||||
datafeeds: ModuleDataFeed[];
|
||||
datafeeds: ModuleDatafeed[];
|
||||
kibana: KibanaObjects;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ import { MlInfoResponse } from '../../../common/types/ml_server_info';
|
|||
import {
|
||||
KibanaObjects,
|
||||
KibanaObjectConfig,
|
||||
ModuleDataFeed,
|
||||
ModuleDatafeed,
|
||||
ModuleJob,
|
||||
Module,
|
||||
JobOverride,
|
||||
|
|
@ -283,7 +283,7 @@ export class DataRecognizer {
|
|||
}
|
||||
|
||||
const jobs: ModuleJob[] = [];
|
||||
const datafeeds: ModuleDataFeed[] = [];
|
||||
const datafeeds: ModuleDatafeed[] = [];
|
||||
const kibana: KibanaObjects = {};
|
||||
// load all of the job configs
|
||||
await Promise.all(
|
||||
|
|
@ -710,7 +710,7 @@ export class DataRecognizer {
|
|||
// save the datafeeds.
|
||||
// if any fail (e.g. it already exists), catch the error and mark the result
|
||||
// as success: false
|
||||
async saveDatafeeds(datafeeds: ModuleDataFeed[]) {
|
||||
async saveDatafeeds(datafeeds: ModuleDatafeed[]) {
|
||||
return await Promise.all(
|
||||
datafeeds.map(async (datafeed) => {
|
||||
try {
|
||||
|
|
@ -723,7 +723,7 @@ export class DataRecognizer {
|
|||
);
|
||||
}
|
||||
|
||||
async saveDatafeed(datafeed: ModuleDataFeed) {
|
||||
async saveDatafeed(datafeed: ModuleDatafeed) {
|
||||
return this._asInternalUser.ml.putDatafeed(
|
||||
{
|
||||
datafeed_id: datafeed.id,
|
||||
|
|
@ -734,7 +734,7 @@ export class DataRecognizer {
|
|||
}
|
||||
|
||||
async startDatafeeds(
|
||||
datafeeds: ModuleDataFeed[],
|
||||
datafeeds: ModuleDatafeed[],
|
||||
start?: number,
|
||||
end?: number
|
||||
): Promise<{ [key: string]: DatafeedResponse }> {
|
||||
|
|
@ -746,7 +746,7 @@ export class DataRecognizer {
|
|||
}
|
||||
|
||||
async startDatafeed(
|
||||
datafeed: ModuleDataFeed,
|
||||
datafeed: ModuleDatafeed,
|
||||
start: number | undefined,
|
||||
end: number | undefined
|
||||
): Promise<DatafeedResponse> {
|
||||
|
|
@ -1229,6 +1229,25 @@ export class DataRecognizer {
|
|||
const overrides = Array.isArray(datafeedOverrides) ? datafeedOverrides : [datafeedOverrides];
|
||||
const { datafeeds } = moduleConfig;
|
||||
|
||||
// for some items in the datafeed, we should not merge.
|
||||
// we should instead use the whole override object
|
||||
function overwriteObjects(source: ModuleDatafeed['config'], update: DatafeedOverride) {
|
||||
Object.entries(update).forEach(([key, val]) => {
|
||||
if (typeof val === 'object') {
|
||||
switch (key) {
|
||||
case 'query':
|
||||
case 'aggregations':
|
||||
case 'aggs':
|
||||
case 'script_fields':
|
||||
source[key] = val as any;
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// separate all the overrides.
|
||||
// the overrides which don't contain a datafeed id or a job id will be applied to all jobs in the module
|
||||
const generalOverrides: GeneralDatafeedsOverride[] = [];
|
||||
|
|
@ -1244,6 +1263,7 @@ export class DataRecognizer {
|
|||
generalOverrides.forEach((o) => {
|
||||
datafeeds.forEach(({ config }) => {
|
||||
merge(config, o);
|
||||
overwriteObjects(config, o);
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -1259,6 +1279,7 @@ export class DataRecognizer {
|
|||
delete o.job_id;
|
||||
delete o.datafeed_id;
|
||||
merge(datafeed.config, o);
|
||||
overwriteObjects(datafeed.config, o);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,9 @@
|
|||
{
|
||||
"job_type": "anomaly_detector",
|
||||
"description": "Logs UI: Detects anomalies in count of log entries by category",
|
||||
"groups": ["logs-ui"],
|
||||
"groups": [
|
||||
"logs-ui"
|
||||
],
|
||||
"analysis_config": {
|
||||
"bucket_span": "15m",
|
||||
"categorization_field_name": "message",
|
||||
|
|
@ -14,7 +16,10 @@
|
|||
"use_null": true
|
||||
}
|
||||
],
|
||||
"influencers": ["event.dataset", "mlcategory"],
|
||||
"influencers": [
|
||||
"event.dataset",
|
||||
"mlcategory"
|
||||
],
|
||||
"per_partition_categorization": {
|
||||
"enabled": true,
|
||||
"stop_on_warn": false
|
||||
|
|
|
|||
|
|
@ -1,3 +1,3 @@
|
|||
{
|
||||
"icon": "metricbeatApp"
|
||||
"icon": "metricbeatApp"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,7 +8,12 @@
|
|||
"query": {
|
||||
"bool": {
|
||||
"filter": {
|
||||
"terms" : { "event.dataset" : ["system.cpu", "system.filesystem"]}
|
||||
"terms": {
|
||||
"event.dataset": [
|
||||
"system.cpu",
|
||||
"system.filesystem"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
|
|||
|
|
@ -6,10 +6,14 @@
|
|||
"query": {
|
||||
"bool": {
|
||||
"filter": {
|
||||
"term": { "event.dataset": "system.cpu" }
|
||||
"term": {
|
||||
"event.dataset": "system.cpu"
|
||||
}
|
||||
},
|
||||
"must": {
|
||||
"exists": { "field": "system.cpu.iowait.pct" }
|
||||
"exists": {
|
||||
"field": "system.cpu.iowait.pct"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,16 +1,20 @@
|
|||
{
|
||||
"job_id": "JOB_ID",
|
||||
"indices": [
|
||||
"INDEX_PATTERN_NAME"
|
||||
],
|
||||
"query": {
|
||||
"bool": {
|
||||
"filter": {
|
||||
"term": { "event.dataset": "system.filesystem" }
|
||||
},
|
||||
"must": {
|
||||
"exists": { "field": "system.filesystem.used.pct" }
|
||||
"job_id": "JOB_ID",
|
||||
"indices": [
|
||||
"INDEX_PATTERN_NAME"
|
||||
],
|
||||
"query": {
|
||||
"bool": {
|
||||
"filter": {
|
||||
"term": {
|
||||
"event.dataset": "system.filesystem"
|
||||
}
|
||||
},
|
||||
"must": {
|
||||
"exists": {
|
||||
"field": "system.filesystem.used.pct"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,13 +1,15 @@
|
|||
{
|
||||
"job_id": "JOB_ID",
|
||||
"indices": [
|
||||
"INDEX_PATTERN_NAME"
|
||||
],
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": {
|
||||
"exists": { "field": "event.dataset" }
|
||||
"job_id": "JOB_ID",
|
||||
"indices": [
|
||||
"INDEX_PATTERN_NAME"
|
||||
],
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": {
|
||||
"exists": {
|
||||
"field": "event.dataset"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,54 +1,56 @@
|
|||
{
|
||||
"job_type": "anomaly_detector",
|
||||
"description": "Metricbeat CPU: Detect unusual increases in cpu time spent in iowait (ECS)",
|
||||
"groups": ["metricbeat"],
|
||||
"analysis_config": {
|
||||
"bucket_span": "10m",
|
||||
"detectors": [
|
||||
{
|
||||
"detector_description": "high mean system.cpu.iowait.pct",
|
||||
"function": "high_mean",
|
||||
"field_name": "system.cpu.iowait.pct",
|
||||
"partition_field_name": "host.name",
|
||||
"custom_rules": [
|
||||
{
|
||||
"actions": [
|
||||
"skip_result"
|
||||
],
|
||||
"conditions": [
|
||||
{
|
||||
"applies_to": "actual",
|
||||
"operator": "lt",
|
||||
"value": 0.25
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"influencers": [
|
||||
"host.name"
|
||||
]
|
||||
},
|
||||
"analysis_limits": {
|
||||
"model_memory_limit": "25mb"
|
||||
},
|
||||
"data_description": {
|
||||
"time_field": "@timestamp",
|
||||
"time_format": "epoch_ms"
|
||||
},
|
||||
"custom_settings": {
|
||||
"created_by": "ml-module-metricbeat-system",
|
||||
"custom_urls": [
|
||||
{
|
||||
"url_name": "Host overview",
|
||||
"time_range": "3h",
|
||||
"url_value": "dashboards#/view/79ffd6e0-faa0-11e6-947f-177f697178b8-ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(filters:!(),query:(language:kuery,query:\u0027host.name:\u0022$host.name$\u0022\u0027))"
|
||||
},
|
||||
{
|
||||
"url_name": "Raw data",
|
||||
"url_value": "discover#/?_g=(refreshInterval:(pause:!t,value:0),time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:'event.dataset:\u0022system.cpu\u0022'),sort:!('@timestamp',desc))"
|
||||
}
|
||||
]
|
||||
}
|
||||
"job_type": "anomaly_detector",
|
||||
"description": "Metricbeat CPU: Detect unusual increases in cpu time spent in iowait (ECS)",
|
||||
"groups": [
|
||||
"metricbeat"
|
||||
],
|
||||
"analysis_config": {
|
||||
"bucket_span": "10m",
|
||||
"detectors": [
|
||||
{
|
||||
"detector_description": "high mean system.cpu.iowait.pct",
|
||||
"function": "high_mean",
|
||||
"field_name": "system.cpu.iowait.pct",
|
||||
"partition_field_name": "host.name",
|
||||
"custom_rules": [
|
||||
{
|
||||
"actions": [
|
||||
"skip_result"
|
||||
],
|
||||
"conditions": [
|
||||
{
|
||||
"applies_to": "actual",
|
||||
"operator": "lt",
|
||||
"value": 0.25
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"influencers": [
|
||||
"host.name"
|
||||
]
|
||||
},
|
||||
"analysis_limits": {
|
||||
"model_memory_limit": "25mb"
|
||||
},
|
||||
"data_description": {
|
||||
"time_field": "@timestamp",
|
||||
"time_format": "epoch_ms"
|
||||
},
|
||||
"custom_settings": {
|
||||
"created_by": "ml-module-metricbeat-system",
|
||||
"custom_urls": [
|
||||
{
|
||||
"url_name": "Host overview",
|
||||
"time_range": "3h",
|
||||
"url_value": "dashboards#/view/79ffd6e0-faa0-11e6-947f-177f697178b8-ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(filters:!(),query:(language:kuery,query:\u0027host.name:\u0022$host.name$\u0022\u0027))"
|
||||
},
|
||||
{
|
||||
"url_name": "Raw data",
|
||||
"url_value": "discover#/?_g=(refreshInterval:(pause:!t,value:0),time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:'event.dataset:\u0022system.cpu\u0022'),sort:!('@timestamp',desc))"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,54 +1,56 @@
|
|||
{
|
||||
"job_type": "anomaly_detector",
|
||||
"description": "Metricbeat filesystem: Detect unusual increases in disk utilization (ECS)",
|
||||
"groups": ["metricbeat"],
|
||||
"analysis_config": {
|
||||
"bucket_span": "10m",
|
||||
"detectors": [
|
||||
{
|
||||
"detector_description": "max disk utilization",
|
||||
"function": "max",
|
||||
"field_name": "system.filesystem.used.pct",
|
||||
"partition_field_name": "host.name",
|
||||
"custom_rules": [
|
||||
{
|
||||
"actions": [
|
||||
"skip_result"
|
||||
],
|
||||
"conditions": [
|
||||
{
|
||||
"applies_to": "actual",
|
||||
"operator": "lt",
|
||||
"value": 0.75
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"influencers": [
|
||||
"host.name"
|
||||
]
|
||||
},
|
||||
"analysis_limits": {
|
||||
"model_memory_limit": "25mb"
|
||||
},
|
||||
"data_description": {
|
||||
"time_field": "@timestamp",
|
||||
"time_format": "epoch_ms"
|
||||
},
|
||||
"custom_settings": {
|
||||
"created_by": "ml-module-metricbeat-system",
|
||||
"custom_urls": [
|
||||
{
|
||||
"url_name": "Host overview",
|
||||
"time_range": "3h",
|
||||
"url_value": "dashboards#/view/79ffd6e0-faa0-11e6-947f-177f697178b8-ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(filters:!(),query:(language:kuery,query:\u0027host.name:\u0022$host.name$\u0022\u0027))"
|
||||
},
|
||||
{
|
||||
"url_name": "Raw data",
|
||||
"url_value": "discover#/?_g=(refreshInterval:(pause:!t,value:0),time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:'event.dataset:\u0022system.filesystem\u0022'),sort:!('@timestamp',desc))"
|
||||
}
|
||||
]
|
||||
}
|
||||
"job_type": "anomaly_detector",
|
||||
"description": "Metricbeat filesystem: Detect unusual increases in disk utilization (ECS)",
|
||||
"groups": [
|
||||
"metricbeat"
|
||||
],
|
||||
"analysis_config": {
|
||||
"bucket_span": "10m",
|
||||
"detectors": [
|
||||
{
|
||||
"detector_description": "max disk utilization",
|
||||
"function": "max",
|
||||
"field_name": "system.filesystem.used.pct",
|
||||
"partition_field_name": "host.name",
|
||||
"custom_rules": [
|
||||
{
|
||||
"actions": [
|
||||
"skip_result"
|
||||
],
|
||||
"conditions": [
|
||||
{
|
||||
"applies_to": "actual",
|
||||
"operator": "lt",
|
||||
"value": 0.75
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"influencers": [
|
||||
"host.name"
|
||||
]
|
||||
},
|
||||
"analysis_limits": {
|
||||
"model_memory_limit": "25mb"
|
||||
},
|
||||
"data_description": {
|
||||
"time_field": "@timestamp",
|
||||
"time_format": "epoch_ms"
|
||||
},
|
||||
"custom_settings": {
|
||||
"created_by": "ml-module-metricbeat-system",
|
||||
"custom_urls": [
|
||||
{
|
||||
"url_name": "Host overview",
|
||||
"time_range": "3h",
|
||||
"url_value": "dashboards#/view/79ffd6e0-faa0-11e6-947f-177f697178b8-ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(filters:!(),query:(language:kuery,query:\u0027host.name:\u0022$host.name$\u0022\u0027))"
|
||||
},
|
||||
{
|
||||
"url_name": "Raw data",
|
||||
"url_value": "discover#/?_g=(refreshInterval:(pause:!t,value:0),time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:'event.dataset:\u0022system.filesystem\u0022'),sort:!('@timestamp',desc))"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,34 +1,36 @@
|
|||
{
|
||||
"job_type": "anomaly_detector",
|
||||
"description": "Metricbeat outages: Detect unusual decreases in metricbeat documents (ECS)",
|
||||
"groups": ["metricbeat"],
|
||||
"analysis_config": {
|
||||
"bucket_span": "10m",
|
||||
"detectors": [
|
||||
{
|
||||
"detector_description": "low_count",
|
||||
"function": "low_count",
|
||||
"partition_field_name": "event.dataset"
|
||||
}
|
||||
],
|
||||
"influencers": [
|
||||
"event.dataset"
|
||||
]
|
||||
},
|
||||
"analysis_limits": {
|
||||
"model_memory_limit": "15mb"
|
||||
},
|
||||
"data_description": {
|
||||
"time_field": "@timestamp",
|
||||
"time_format": "epoch_ms"
|
||||
},
|
||||
"custom_settings": {
|
||||
"created_by": "ml-module-metricbeat-system",
|
||||
"custom_urls": [
|
||||
{
|
||||
"url_name": "Raw data",
|
||||
"url_value": "discover#/?_g=(refreshInterval:(pause:!t,value:0),time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:''),sort:!('@timestamp',desc))"
|
||||
}
|
||||
]
|
||||
}
|
||||
"job_type": "anomaly_detector",
|
||||
"description": "Metricbeat outages: Detect unusual decreases in metricbeat documents (ECS)",
|
||||
"groups": [
|
||||
"metricbeat"
|
||||
],
|
||||
"analysis_config": {
|
||||
"bucket_span": "10m",
|
||||
"detectors": [
|
||||
{
|
||||
"detector_description": "low_count",
|
||||
"function": "low_count",
|
||||
"partition_field_name": "event.dataset"
|
||||
}
|
||||
],
|
||||
"influencers": [
|
||||
"event.dataset"
|
||||
]
|
||||
},
|
||||
"analysis_limits": {
|
||||
"model_memory_limit": "15mb"
|
||||
},
|
||||
"data_description": {
|
||||
"time_field": "@timestamp",
|
||||
"time_format": "epoch_ms"
|
||||
},
|
||||
"custom_settings": {
|
||||
"created_by": "ml-module-metricbeat-system",
|
||||
"custom_urls": [
|
||||
{
|
||||
"url_name": "Raw data",
|
||||
"url_value": "discover#/?_g=(refreshInterval:(pause:!t,value:0),time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:''),sort:!('@timestamp',desc))"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,3 @@
|
|||
{
|
||||
"icon": "metricsApp"
|
||||
}
|
||||
"icon": "metricsApp"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,38 +1,38 @@
|
|||
{
|
||||
"id": "metrics_ui_hosts",
|
||||
"title": "Metrics Hosts",
|
||||
"description": "Detect anomalous memory and network behavior on hosts.",
|
||||
"type": "Metricbeat Data",
|
||||
"logoFile": "logo.json",
|
||||
"jobs": [
|
||||
{
|
||||
"id": "hosts_memory_usage",
|
||||
"file": "hosts_memory_usage.json"
|
||||
},
|
||||
{
|
||||
"id": "hosts_network_in",
|
||||
"file": "hosts_network_in.json"
|
||||
},
|
||||
{
|
||||
"id": "hosts_network_out",
|
||||
"file": "hosts_network_out.json"
|
||||
}
|
||||
],
|
||||
"datafeeds": [
|
||||
{
|
||||
"id": "datafeed-hosts_memory_usage",
|
||||
"file": "datafeed_hosts_memory_usage.json",
|
||||
"job_id": "hosts_memory_usage"
|
||||
},
|
||||
{
|
||||
"id": "datafeed-hosts_network_in",
|
||||
"file": "datafeed_hosts_network_in.json",
|
||||
"job_id": "hosts_network_in"
|
||||
},
|
||||
{
|
||||
"id": "datafeed-hosts_network_out",
|
||||
"file": "datafeed_hosts_network_out.json",
|
||||
"job_id": "hosts_network_out"
|
||||
}
|
||||
]
|
||||
}
|
||||
"id": "metrics_ui_hosts",
|
||||
"title": "Metrics Hosts",
|
||||
"description": "Detect anomalous memory and network behavior on hosts.",
|
||||
"type": "Metricbeat Data",
|
||||
"logoFile": "logo.json",
|
||||
"jobs": [
|
||||
{
|
||||
"id": "hosts_memory_usage",
|
||||
"file": "hosts_memory_usage.json"
|
||||
},
|
||||
{
|
||||
"id": "hosts_network_in",
|
||||
"file": "hosts_network_in.json"
|
||||
},
|
||||
{
|
||||
"id": "hosts_network_out",
|
||||
"file": "hosts_network_out.json"
|
||||
}
|
||||
],
|
||||
"datafeeds": [
|
||||
{
|
||||
"id": "datafeed-hosts_memory_usage",
|
||||
"file": "datafeed_hosts_memory_usage.json",
|
||||
"job_id": "hosts_memory_usage"
|
||||
},
|
||||
{
|
||||
"id": "datafeed-hosts_network_in",
|
||||
"file": "datafeed_hosts_network_in.json",
|
||||
"job_id": "hosts_network_in"
|
||||
},
|
||||
{
|
||||
"id": "datafeed-hosts_network_out",
|
||||
"file": "datafeed_hosts_network_out.json",
|
||||
"job_id": "hosts_network_out"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,16 +1,20 @@
|
|||
{
|
||||
"job_id": "JOB_ID",
|
||||
"indices": [
|
||||
"INDEX_PATTERN_NAME"
|
||||
],
|
||||
"indices_options": {
|
||||
"allow_no_indices": true
|
||||
},
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{"exists": {"field": "system.memory"}}
|
||||
]
|
||||
"job_id": "JOB_ID",
|
||||
"indices": [
|
||||
"INDEX_PATTERN_NAME"
|
||||
],
|
||||
"indices_options": {
|
||||
"allow_no_indices": true
|
||||
},
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{
|
||||
"exists": {
|
||||
"field": "system.memory"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,40 +1,65 @@
|
|||
{
|
||||
"job_id": "JOB_ID",
|
||||
"indices": [
|
||||
"INDEX_PATTERN_NAME"
|
||||
],
|
||||
"indices_options": {
|
||||
"allow_no_indices": true
|
||||
},
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{"exists": {"field": "system.network"}}
|
||||
]
|
||||
}
|
||||
},
|
||||
"chunking_config": {
|
||||
"mode": "manual",
|
||||
"time_span": "900s"
|
||||
},
|
||||
"aggregations": {
|
||||
"host.name": {"terms": {"field": "host.name", "size": 100},
|
||||
"aggregations": {
|
||||
"buckets": {
|
||||
"date_histogram": {"field": "@timestamp","fixed_interval": "5m"},
|
||||
"aggregations": {
|
||||
"@timestamp": {"max": {"field": "@timestamp"}},
|
||||
"bytes_in_max": {"max": {"field": "system.network.in.bytes"}},
|
||||
"bytes_in_derivative": {"derivative": {"buckets_path": "bytes_in_max"}},
|
||||
"positive_only":{
|
||||
"bucket_script": {
|
||||
"buckets_path": {"in_derivative": "bytes_in_derivative.value"},
|
||||
"script": "params.in_derivative > 0.0 ? params.in_derivative : 0.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
"job_id": "JOB_ID",
|
||||
"indices": [
|
||||
"INDEX_PATTERN_NAME"
|
||||
],
|
||||
"indices_options": {
|
||||
"allow_no_indices": true
|
||||
},
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{
|
||||
"exists": {
|
||||
"field": "system.network"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"chunking_config": {
|
||||
"mode": "manual",
|
||||
"time_span": "900s"
|
||||
},
|
||||
"aggregations": {
|
||||
"host.name": {
|
||||
"terms": {
|
||||
"field": "host.name",
|
||||
"size": 100
|
||||
},
|
||||
"aggregations": {
|
||||
"buckets": {
|
||||
"date_histogram": {
|
||||
"field": "@timestamp",
|
||||
"fixed_interval": "5m"
|
||||
},
|
||||
"aggregations": {
|
||||
"@timestamp": {
|
||||
"max": {
|
||||
"field": "@timestamp"
|
||||
}
|
||||
},
|
||||
"bytes_in_max": {
|
||||
"max": {
|
||||
"field": "system.network.in.bytes"
|
||||
}
|
||||
},
|
||||
"bytes_in_derivative": {
|
||||
"derivative": {
|
||||
"buckets_path": "bytes_in_max"
|
||||
}
|
||||
},
|
||||
"positive_only": {
|
||||
"bucket_script": {
|
||||
"buckets_path": {
|
||||
"in_derivative": "bytes_in_derivative.value"
|
||||
},
|
||||
"script": "params.in_derivative > 0.0 ? params.in_derivative : 0.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,40 +1,65 @@
|
|||
{
|
||||
"job_id": "JOB_ID",
|
||||
"indices": [
|
||||
"INDEX_PATTERN_NAME"
|
||||
],
|
||||
"indices_options": {
|
||||
"allow_no_indices": true
|
||||
},
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{"exists": {"field": "system.network"}}
|
||||
]
|
||||
}
|
||||
},
|
||||
"chunking_config": {
|
||||
"mode": "manual",
|
||||
"time_span": "900s"
|
||||
},
|
||||
"aggregations": {
|
||||
"host.name": {"terms": {"field": "host.name", "size": 100},
|
||||
"aggregations": {
|
||||
"buckets": {
|
||||
"date_histogram": {"field": "@timestamp","fixed_interval": "5m"},
|
||||
"aggregations": {
|
||||
"@timestamp": {"max": {"field": "@timestamp"}},
|
||||
"bytes_out_max": {"max": {"field": "system.network.out.bytes"}},
|
||||
"bytes_out_derivative": {"derivative": {"buckets_path": "bytes_out_max"}},
|
||||
"positive_only":{
|
||||
"bucket_script": {
|
||||
"buckets_path": {"out_derivative": "bytes_out_derivative.value"},
|
||||
"script": "params.out_derivative > 0.0 ? params.out_derivative : 0.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
"job_id": "JOB_ID",
|
||||
"indices": [
|
||||
"INDEX_PATTERN_NAME"
|
||||
],
|
||||
"indices_options": {
|
||||
"allow_no_indices": true
|
||||
},
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{
|
||||
"exists": {
|
||||
"field": "system.network"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"chunking_config": {
|
||||
"mode": "manual",
|
||||
"time_span": "900s"
|
||||
},
|
||||
"aggregations": {
|
||||
"host.name": {
|
||||
"terms": {
|
||||
"field": "host.name",
|
||||
"size": 100
|
||||
},
|
||||
"aggregations": {
|
||||
"buckets": {
|
||||
"date_histogram": {
|
||||
"field": "@timestamp",
|
||||
"fixed_interval": "5m"
|
||||
},
|
||||
"aggregations": {
|
||||
"@timestamp": {
|
||||
"max": {
|
||||
"field": "@timestamp"
|
||||
}
|
||||
},
|
||||
"bytes_out_max": {
|
||||
"max": {
|
||||
"field": "system.network.out.bytes"
|
||||
}
|
||||
},
|
||||
"bytes_out_derivative": {
|
||||
"derivative": {
|
||||
"buckets_path": "bytes_out_max"
|
||||
}
|
||||
},
|
||||
"positive_only": {
|
||||
"bucket_script": {
|
||||
"buckets_path": {
|
||||
"out_derivative": "bytes_out_derivative.value"
|
||||
},
|
||||
"script": "params.out_derivative > 0.0 ? params.out_derivative : 0.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,50 +1,50 @@
|
|||
{
|
||||
"job_type": "anomaly_detector",
|
||||
"groups": [
|
||||
"hosts",
|
||||
"metrics"
|
||||
],
|
||||
"description": "Metrics: Hosts - Identify unusual spikes in memory usage across hosts.",
|
||||
"analysis_config": {
|
||||
"bucket_span": "15m",
|
||||
"detectors": [
|
||||
{
|
||||
"detector_description": "max('system.memory.actual.used.pct')",
|
||||
"function": "max",
|
||||
"field_name": "system.memory.actual.used.pct",
|
||||
"custom_rules": [
|
||||
{
|
||||
"actions": [
|
||||
"skip_result"
|
||||
],
|
||||
"conditions": [
|
||||
{
|
||||
"applies_to": "actual",
|
||||
"operator": "lt",
|
||||
"value": 0.1
|
||||
}
|
||||
]
|
||||
}
|
||||
"job_type": "anomaly_detector",
|
||||
"groups": [
|
||||
"hosts",
|
||||
"metrics"
|
||||
],
|
||||
"description": "Metrics: Hosts - Identify unusual spikes in memory usage across hosts.",
|
||||
"analysis_config": {
|
||||
"bucket_span": "15m",
|
||||
"detectors": [
|
||||
{
|
||||
"detector_description": "max('system.memory.actual.used.pct')",
|
||||
"function": "max",
|
||||
"field_name": "system.memory.actual.used.pct",
|
||||
"custom_rules": [
|
||||
{
|
||||
"actions": [
|
||||
"skip_result"
|
||||
],
|
||||
"conditions": [
|
||||
{
|
||||
"applies_to": "actual",
|
||||
"operator": "lt",
|
||||
"value": 0.1
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"influencers": [
|
||||
"host.name"
|
||||
]
|
||||
},
|
||||
"data_description": {
|
||||
"time_field": "@timestamp"
|
||||
},
|
||||
"analysis_limits": {
|
||||
"model_memory_limit": "64mb"
|
||||
},
|
||||
"custom_settings": {
|
||||
"created_by": "ml-module-metrics-ui-hosts",
|
||||
"custom_urls": [
|
||||
{
|
||||
"url_name": "Host Metrics",
|
||||
"url_value": "metrics/detail/host/$host.name$?metricTime=(autoReload:!f,refreshInterval:5000,time:(from:%27$earliest$%27,interval:%3E%3D1m,to:%27$latest$%27))"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"influencers": [
|
||||
"host.name"
|
||||
]
|
||||
},
|
||||
"data_description": {
|
||||
"time_field": "@timestamp"
|
||||
},
|
||||
"analysis_limits": {
|
||||
"model_memory_limit": "64mb"
|
||||
},
|
||||
"custom_settings": {
|
||||
"created_by": "ml-module-metrics-ui-hosts",
|
||||
"custom_urls": [
|
||||
{
|
||||
"url_name": "Host Metrics",
|
||||
"url_value": "metrics/detail/host/$host.name$?metricTime=(autoReload:!f,refreshInterval:5000,time:(from:%27$earliest$%27,interval:%3E%3D1m,to:%27$latest$%27))"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,37 +1,37 @@
|
|||
{
|
||||
"job_type": "anomaly_detector",
|
||||
"description": "Metrics: Hosts - Identify unusual spikes in inbound traffic across hosts.",
|
||||
"groups": [
|
||||
"hosts",
|
||||
"metrics"
|
||||
"job_type": "anomaly_detector",
|
||||
"description": "Metrics: Hosts - Identify unusual spikes in inbound traffic across hosts.",
|
||||
"groups": [
|
||||
"hosts",
|
||||
"metrics"
|
||||
],
|
||||
"analysis_config": {
|
||||
"bucket_span": "15m",
|
||||
"detectors": [
|
||||
{
|
||||
"detector_description": "max(bytes_in_derivative)",
|
||||
"function": "max",
|
||||
"field_name": "bytes_in_derivative"
|
||||
}
|
||||
],
|
||||
"analysis_config": {
|
||||
"bucket_span": "15m",
|
||||
"detectors": [
|
||||
{
|
||||
"detector_description": "max(bytes_in_derivative)",
|
||||
"function": "max",
|
||||
"field_name": "bytes_in_derivative"
|
||||
}
|
||||
],
|
||||
"influencers": [
|
||||
"host.name"
|
||||
],
|
||||
"summary_count_field_name": "doc_count"
|
||||
},
|
||||
"data_description": {
|
||||
"time_field": "@timestamp"
|
||||
},
|
||||
"analysis_limits": {
|
||||
"model_memory_limit": "32mb"
|
||||
},
|
||||
"custom_settings": {
|
||||
"created_by": "ml-module-metrics-ui-hosts",
|
||||
"custom_urls": [
|
||||
{
|
||||
"url_name": "Host Metrics",
|
||||
"url_value": "metrics/detail/host/$host.name$?metricTime=(autoReload:!f,refreshInterval:5000,time:(from:%27$earliest$%27,interval:%3E%3D1m,to:%27$latest$%27))"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
"influencers": [
|
||||
"host.name"
|
||||
],
|
||||
"summary_count_field_name": "doc_count"
|
||||
},
|
||||
"data_description": {
|
||||
"time_field": "@timestamp"
|
||||
},
|
||||
"analysis_limits": {
|
||||
"model_memory_limit": "32mb"
|
||||
},
|
||||
"custom_settings": {
|
||||
"created_by": "ml-module-metrics-ui-hosts",
|
||||
"custom_urls": [
|
||||
{
|
||||
"url_name": "Host Metrics",
|
||||
"url_value": "metrics/detail/host/$host.name$?metricTime=(autoReload:!f,refreshInterval:5000,time:(from:%27$earliest$%27,interval:%3E%3D1m,to:%27$latest$%27))"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,37 +1,37 @@
|
|||
{
|
||||
"job_type": "anomaly_detector",
|
||||
"description": "Metrics: Hosts - Identify unusual spikes in outbound traffic across hosts.",
|
||||
"groups": [
|
||||
"hosts",
|
||||
"metrics"
|
||||
"job_type": "anomaly_detector",
|
||||
"description": "Metrics: Hosts - Identify unusual spikes in outbound traffic across hosts.",
|
||||
"groups": [
|
||||
"hosts",
|
||||
"metrics"
|
||||
],
|
||||
"analysis_config": {
|
||||
"bucket_span": "15m",
|
||||
"detectors": [
|
||||
{
|
||||
"detector_description": "max(bytes_out_derivative)",
|
||||
"function": "max",
|
||||
"field_name": "bytes_out_derivative"
|
||||
}
|
||||
],
|
||||
"analysis_config": {
|
||||
"bucket_span": "15m",
|
||||
"detectors": [
|
||||
{
|
||||
"detector_description": "max(bytes_out_derivative)",
|
||||
"function": "max",
|
||||
"field_name": "bytes_out_derivative"
|
||||
}
|
||||
],
|
||||
"influencers": [
|
||||
"host.name"
|
||||
],
|
||||
"summary_count_field_name": "doc_count"
|
||||
},
|
||||
"data_description": {
|
||||
"time_field": "@timestamp"
|
||||
},
|
||||
"analysis_limits": {
|
||||
"model_memory_limit": "32mb"
|
||||
},
|
||||
"custom_settings": {
|
||||
"created_by": "ml-module-metrics-ui-hosts",
|
||||
"custom_urls": [
|
||||
{
|
||||
"url_name": "Host Metrics",
|
||||
"url_value": "metrics/detail/host/$host.name$?metricTime=(autoReload:!f,refreshInterval:5000,time:(from:%27$earliest$%27,interval:%3E%3D1m,to:%27$latest$%27))"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
"influencers": [
|
||||
"host.name"
|
||||
],
|
||||
"summary_count_field_name": "doc_count"
|
||||
},
|
||||
"data_description": {
|
||||
"time_field": "@timestamp"
|
||||
},
|
||||
"analysis_limits": {
|
||||
"model_memory_limit": "32mb"
|
||||
},
|
||||
"custom_settings": {
|
||||
"created_by": "ml-module-metrics-ui-hosts",
|
||||
"custom_urls": [
|
||||
{
|
||||
"url_name": "Host Metrics",
|
||||
"url_value": "metrics/detail/host/$host.name$?metricTime=(autoReload:!f,refreshInterval:5000,time:(from:%27$earliest$%27,interval:%3E%3D1m,to:%27$latest$%27))"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,3 @@
|
|||
{
|
||||
"icon": "metricsApp"
|
||||
}
|
||||
"icon": "metricsApp"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,38 +1,38 @@
|
|||
{
|
||||
"id": "metrics_ui_k8s",
|
||||
"title": "Metrics Kubernetes",
|
||||
"description": "Detect anomalous memory and network behavior on Kubernetes pods.",
|
||||
"type": "Metricbeat Data",
|
||||
"logoFile": "logo.json",
|
||||
"jobs": [
|
||||
{
|
||||
"id": "k8s_memory_usage",
|
||||
"file": "k8s_memory_usage.json"
|
||||
},
|
||||
{
|
||||
"id": "k8s_network_in",
|
||||
"file": "k8s_network_in.json"
|
||||
},
|
||||
{
|
||||
"id": "k8s_network_out",
|
||||
"file": "k8s_network_out.json"
|
||||
}
|
||||
],
|
||||
"datafeeds": [
|
||||
{
|
||||
"id": "datafeed-k8s_memory_usage",
|
||||
"file": "datafeed_k8s_memory_usage.json",
|
||||
"job_id": "k8s_memory_usage"
|
||||
},
|
||||
{
|
||||
"id": "datafeed-k8s_network_in",
|
||||
"file": "datafeed_k8s_network_in.json",
|
||||
"job_id": "k8s_network_in"
|
||||
},
|
||||
{
|
||||
"id": "datafeed-k8s_network_out",
|
||||
"file": "datafeed_k8s_network_out.json",
|
||||
"job_id": "k8s_network_out"
|
||||
}
|
||||
]
|
||||
}
|
||||
"id": "metrics_ui_k8s",
|
||||
"title": "Metrics Kubernetes",
|
||||
"description": "Detect anomalous memory and network behavior on Kubernetes pods.",
|
||||
"type": "Metricbeat Data",
|
||||
"logoFile": "logo.json",
|
||||
"jobs": [
|
||||
{
|
||||
"id": "k8s_memory_usage",
|
||||
"file": "k8s_memory_usage.json"
|
||||
},
|
||||
{
|
||||
"id": "k8s_network_in",
|
||||
"file": "k8s_network_in.json"
|
||||
},
|
||||
{
|
||||
"id": "k8s_network_out",
|
||||
"file": "k8s_network_out.json"
|
||||
}
|
||||
],
|
||||
"datafeeds": [
|
||||
{
|
||||
"id": "datafeed-k8s_memory_usage",
|
||||
"file": "datafeed_k8s_memory_usage.json",
|
||||
"job_id": "k8s_memory_usage"
|
||||
},
|
||||
{
|
||||
"id": "datafeed-k8s_network_in",
|
||||
"file": "datafeed_k8s_network_in.json",
|
||||
"job_id": "k8s_network_in"
|
||||
},
|
||||
{
|
||||
"id": "datafeed-k8s_network_out",
|
||||
"file": "datafeed_k8s_network_out.json",
|
||||
"job_id": "k8s_network_out"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,17 +1,25 @@
|
|||
{
|
||||
"job_id": "JOB_ID",
|
||||
"indices": [
|
||||
"INDEX_PATTERN_NAME"
|
||||
],
|
||||
"indices_options": {
|
||||
"allow_no_indices": true
|
||||
},
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{"exists": {"field": "kubernetes.pod.uid"}},
|
||||
{"exists": {"field": "kubernetes.pod.memory"}}
|
||||
]
|
||||
"job_id": "JOB_ID",
|
||||
"indices": [
|
||||
"INDEX_PATTERN_NAME"
|
||||
],
|
||||
"indices_options": {
|
||||
"allow_no_indices": true
|
||||
},
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{
|
||||
"exists": {
|
||||
"field": "kubernetes.pod.uid"
|
||||
}
|
||||
},
|
||||
{
|
||||
"exists": {
|
||||
"field": "kubernetes.pod.memory"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,44 +1,73 @@
|
|||
{
|
||||
"job_id": "JOB_ID",
|
||||
"indices": [
|
||||
"INDEX_PATTERN_NAME"
|
||||
],
|
||||
"indices_options": {
|
||||
"allow_no_indices": true
|
||||
},
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{"exists": {"field": "kubernetes.pod.network"}}
|
||||
]
|
||||
}
|
||||
},
|
||||
"chunking_config": {
|
||||
"mode": "manual",
|
||||
"time_span": "900s"
|
||||
},
|
||||
"aggregations": {
|
||||
"kubernetes.namespace": {"terms": {"field": "kubernetes.namespace", "size": 25},
|
||||
"aggregations": {
|
||||
"kubernetes.pod.uid": {"terms": {"field": "kubernetes.pod.uid", "size": 100},
|
||||
"aggregations": {
|
||||
"buckets": {
|
||||
"date_histogram": {"field": "@timestamp","fixed_interval": "5m"},
|
||||
"aggregations": {
|
||||
"@timestamp": {"max": {"field": "@timestamp"}},
|
||||
"bytes_in_max": {"max": {"field": "kubernetes.pod.network.rx.bytes"}},
|
||||
"bytes_in_derivative": {"derivative": {"buckets_path": "bytes_in_max"}},
|
||||
"positive_only":{
|
||||
"bucket_script": {
|
||||
"buckets_path": {"in_derivative": "bytes_in_derivative.value"},
|
||||
"script": "params.in_derivative > 0.0 ? params.in_derivative : 0.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
"job_id": "JOB_ID",
|
||||
"indices": [
|
||||
"INDEX_PATTERN_NAME"
|
||||
],
|
||||
"indices_options": {
|
||||
"allow_no_indices": true
|
||||
},
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{
|
||||
"exists": {
|
||||
"field": "kubernetes.pod.network"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"chunking_config": {
|
||||
"mode": "manual",
|
||||
"time_span": "900s"
|
||||
},
|
||||
"aggregations": {
|
||||
"kubernetes.namespace": {
|
||||
"terms": {
|
||||
"field": "kubernetes.namespace",
|
||||
"size": 25
|
||||
},
|
||||
"aggregations": {
|
||||
"kubernetes.pod.uid": {
|
||||
"terms": {
|
||||
"field": "kubernetes.pod.uid",
|
||||
"size": 100
|
||||
},
|
||||
"aggregations": {
|
||||
"buckets": {
|
||||
"date_histogram": {
|
||||
"field": "@timestamp",
|
||||
"fixed_interval": "5m"
|
||||
},
|
||||
"aggregations": {
|
||||
"@timestamp": {
|
||||
"max": {
|
||||
"field": "@timestamp"
|
||||
}
|
||||
},
|
||||
"bytes_in_max": {
|
||||
"max": {
|
||||
"field": "kubernetes.pod.network.rx.bytes"
|
||||
}
|
||||
},
|
||||
"bytes_in_derivative": {
|
||||
"derivative": {
|
||||
"buckets_path": "bytes_in_max"
|
||||
}
|
||||
},
|
||||
"positive_only": {
|
||||
"bucket_script": {
|
||||
"buckets_path": {
|
||||
"in_derivative": "bytes_in_derivative.value"
|
||||
},
|
||||
"script": "params.in_derivative > 0.0 ? params.in_derivative : 0.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,44 +1,73 @@
|
|||
{
|
||||
"job_id": "JOB_ID",
|
||||
"indices": [
|
||||
"INDEX_PATTERN_NAME"
|
||||
],
|
||||
"indices_options": {
|
||||
"allow_no_indices": true
|
||||
},
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{"exists": {"field": "kubernetes.pod.network"}}
|
||||
]
|
||||
}
|
||||
},
|
||||
"chunking_config": {
|
||||
"mode": "manual",
|
||||
"time_span": "900s"
|
||||
},
|
||||
"aggregations": {
|
||||
"kubernetes.namespace": {"terms": {"field": "kubernetes.namespace", "size": 25},
|
||||
"aggregations": {
|
||||
"kubernetes.pod.uid": {"terms": {"field": "kubernetes.pod.uid", "size": 100},
|
||||
"aggregations": {
|
||||
"buckets": {
|
||||
"date_histogram": {"field": "@timestamp","fixed_interval": "5m"},
|
||||
"aggregations": {
|
||||
"@timestamp": {"max": {"field": "@timestamp"}},
|
||||
"bytes_out_max": {"max": {"field": "kubernetes.pod.network.tx.bytes"}},
|
||||
"bytes_out_derivative": {"derivative": {"buckets_path": "bytes_out_max"}},
|
||||
"positive_only":{
|
||||
"bucket_script": {
|
||||
"buckets_path": {"pos_derivative": "bytes_out_derivative.value"},
|
||||
"script": "params.pos_derivative > 0.0 ? params.pos_derivative : 0.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
"job_id": "JOB_ID",
|
||||
"indices": [
|
||||
"INDEX_PATTERN_NAME"
|
||||
],
|
||||
"indices_options": {
|
||||
"allow_no_indices": true
|
||||
},
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{
|
||||
"exists": {
|
||||
"field": "kubernetes.pod.network"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"chunking_config": {
|
||||
"mode": "manual",
|
||||
"time_span": "900s"
|
||||
},
|
||||
"aggregations": {
|
||||
"kubernetes.namespace": {
|
||||
"terms": {
|
||||
"field": "kubernetes.namespace",
|
||||
"size": 25
|
||||
},
|
||||
"aggregations": {
|
||||
"kubernetes.pod.uid": {
|
||||
"terms": {
|
||||
"field": "kubernetes.pod.uid",
|
||||
"size": 100
|
||||
},
|
||||
"aggregations": {
|
||||
"buckets": {
|
||||
"date_histogram": {
|
||||
"field": "@timestamp",
|
||||
"fixed_interval": "5m"
|
||||
},
|
||||
"aggregations": {
|
||||
"@timestamp": {
|
||||
"max": {
|
||||
"field": "@timestamp"
|
||||
}
|
||||
},
|
||||
"bytes_out_max": {
|
||||
"max": {
|
||||
"field": "kubernetes.pod.network.tx.bytes"
|
||||
}
|
||||
},
|
||||
"bytes_out_derivative": {
|
||||
"derivative": {
|
||||
"buckets_path": "bytes_out_max"
|
||||
}
|
||||
},
|
||||
"positive_only": {
|
||||
"bucket_script": {
|
||||
"buckets_path": {
|
||||
"pos_derivative": "bytes_out_derivative.value"
|
||||
},
|
||||
"script": "params.pos_derivative > 0.0 ? params.pos_derivative : 0.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,53 +1,53 @@
|
|||
{
|
||||
"job_type": "anomaly_detector",
|
||||
"groups": [
|
||||
"k8s",
|
||||
"metrics"
|
||||
"job_type": "anomaly_detector",
|
||||
"groups": [
|
||||
"k8s",
|
||||
"metrics"
|
||||
],
|
||||
"description": "Metrics: Kubernetes - Identify unusual spikes in memory usage across Kubernetes pods.",
|
||||
"analysis_config": {
|
||||
"bucket_span": "15m",
|
||||
"detectors": [
|
||||
{
|
||||
"detector_description": "max('kubernetes.pod.memory.usage.node.pct')",
|
||||
"function": "max",
|
||||
"field_name": "kubernetes.pod.memory.usage.node.pct",
|
||||
"partition_field_name": "kubernetes.namespace",
|
||||
"custom_rules": [
|
||||
{
|
||||
"actions": [
|
||||
"skip_result"
|
||||
],
|
||||
"conditions": [
|
||||
{
|
||||
"applies_to": "actual",
|
||||
"operator": "lt",
|
||||
"value": 0.1
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"description": "Metrics: Kubernetes - Identify unusual spikes in memory usage across Kubernetes pods.",
|
||||
"analysis_config": {
|
||||
"bucket_span": "15m",
|
||||
"detectors": [
|
||||
{
|
||||
"detector_description": "max('kubernetes.pod.memory.usage.node.pct')",
|
||||
"function": "max",
|
||||
"field_name": "kubernetes.pod.memory.usage.node.pct",
|
||||
"partition_field_name": "kubernetes.namespace",
|
||||
"custom_rules": [
|
||||
{
|
||||
"actions": [
|
||||
"skip_result"
|
||||
],
|
||||
"conditions": [
|
||||
{
|
||||
"applies_to": "actual",
|
||||
"operator": "lt",
|
||||
"value": 0.1
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"influencers": [
|
||||
"kubernetes.namespace",
|
||||
"kubernetes.node.name",
|
||||
"kubernetes.pod.uid"
|
||||
]
|
||||
},
|
||||
"data_description": {
|
||||
"time_field": "@timestamp"
|
||||
},
|
||||
"analysis_limits": {
|
||||
"model_memory_limit": "64mb"
|
||||
},
|
||||
"custom_settings": {
|
||||
"created_by": "ml-module-metrics-ui-k8s",
|
||||
"custom_urls": [
|
||||
{
|
||||
"url_name": "Pod Metrics",
|
||||
"url_value": "metrics/detail/pod/$kubernetes.pod.uid$?metricTime=(autoReload:!f,refreshInterval:5000,time:(from:%27$earliest$%27,interval:%3E%3D1m,to:%27$latest$%27))"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
"influencers": [
|
||||
"kubernetes.namespace",
|
||||
"kubernetes.node.name",
|
||||
"kubernetes.pod.uid"
|
||||
]
|
||||
},
|
||||
"data_description": {
|
||||
"time_field": "@timestamp"
|
||||
},
|
||||
"analysis_limits": {
|
||||
"model_memory_limit": "64mb"
|
||||
},
|
||||
"custom_settings": {
|
||||
"created_by": "ml-module-metrics-ui-k8s",
|
||||
"custom_urls": [
|
||||
{
|
||||
"url_name": "Pod Metrics",
|
||||
"url_value": "metrics/detail/pod/$kubernetes.pod.uid$?metricTime=(autoReload:!f,refreshInterval:5000,time:(from:%27$earliest$%27,interval:%3E%3D1m,to:%27$latest$%27))"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@
|
|||
"influencers": [
|
||||
"kubernetes.namespace",
|
||||
"kubernetes.pod.uid"
|
||||
],
|
||||
],
|
||||
"summary_count_field_name": "doc_count"
|
||||
},
|
||||
"data_description": {
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@
|
|||
"influencers": [
|
||||
"kubernetes.namespace",
|
||||
"kubernetes.pod.uid"
|
||||
],
|
||||
],
|
||||
"summary_count_field_name": "doc_count"
|
||||
},
|
||||
"data_description": {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue