[ML] APM modules configs for RUM Javascript and NodeJS (#53792)

* [ML] apm modules

* [ML] apm modules

* [ML] update mocha test

* [ML] fix config

* [ML] single line JSON formatting for queries

* [ML] remove an empty path component with a trailing slash

* [ML] change detector descriptions, remove scroll size

* [ML] remove chunking_config from datafeeds

* [ML] fix configs

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
This commit is contained in:
Dima Arnautov 2020-01-10 23:19:26 +01:00 committed by GitHub
parent c87ba85141
commit 51c1a8f805
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
21 changed files with 631 additions and 17 deletions

View file

@ -345,6 +345,98 @@ describe('ML - custom URL utils', () => {
);
});
test('returns expected URL for APM', () => {
const urlConfig = {
url_name: 'APM',
time_range: '2h',
url_value:
'apm#/traces?rangeFrom=$earliest$&rangeTo=$latest$&kuery=trace.id:"$trace.id$" and transaction.name:"$transaction.name$"&_g=()',
};
const testRecords = {
job_id: 'abnormal_trace_durations_nodejs',
result_type: 'record',
probability: 0.025597710862701226,
multi_bucket_impact: 5,
record_score: 13.124152090331723,
initial_record_score: 13.124152090331723,
bucket_span: 900,
detector_index: 0,
is_interim: false,
timestamp: 1573339500000,
by_field_name: 'transaction.name',
by_field_value: 'GET /test-data',
function: 'high_mean',
function_description: 'mean',
typical: [802.0600710562369],
actual: [761.1531339031332],
field_name: 'transaction.duration.us',
influencers: [
{
influencer_field_name: 'transaction.name',
influencer_field_values: ['GET /test-data'],
},
{
influencer_field_name: 'trace.id',
influencer_field_values: [
'000a09d58a428f38550e7e87637733c1',
'0039c771d8bbadf6137767d3aeb89f96',
'01279ed5bb9f4249e3822d16dec7f2f2',
],
},
{
influencer_field_name: 'service.name',
influencer_field_values: ['example-service'],
},
],
'trace.id': [
'000a09d58a428f38550e7e87637733c1',
'0039c771d8bbadf6137767d3aeb89f96',
'01279ed5bb9f4249e3822d16dec7f2f2',
],
'service.name': ['example-service'],
'transaction.name': ['GET /test-data'],
earliest: '2019-11-09T20:45:00.000Z',
latest: '2019-11-10T01:00:00.000Z',
};
expect(getUrlForRecord(urlConfig, testRecords)).toBe(
'apm#/traces?rangeFrom=2019-11-09T20:45:00.000Z&rangeTo=2019-11-10T01:00:00.000Z&kuery=(trace.id:"000a09d58a428f38550e7e87637733c1" OR trace.id:"0039c771d8bbadf6137767d3aeb89f96" OR trace.id:"01279ed5bb9f4249e3822d16dec7f2f2") AND transaction.name:"GET%20%2Ftest-data"&_g=()'
);
});
test('removes an empty path component with a trailing slash', () => {
const urlConfig = {
url_name: 'APM',
time_range: '2h',
url_value:
'apm#/services/$service.name$/transactions?rangeFrom=$earliest$&rangeTo=$latest$&refreshPaused=true&refreshInterval=0&kuery=&transactionType=request',
};
const testRecords = {
job_id: 'decreased_throughput_jsbase',
result_type: 'record',
probability: 8.91350850732573e-9,
multi_bucket_impact: 5,
record_score: 93.63625728951217,
initial_record_score: 93.63625728951217,
bucket_span: 900,
detector_index: 0,
is_interim: false,
timestamp: 1573266600000,
function: 'low_count',
function_description: 'count',
typical: [100615.66506877479],
actual: [25251],
earliest: '2019-11-09T00:30:00.000Z',
latest: '2019-11-09T04:45:00.000Z',
};
expect(getUrlForRecord(urlConfig, testRecords)).toBe(
'apm#/services/transactions?rangeFrom=2019-11-09T00:30:00.000Z&rangeTo=2019-11-09T04:45:00.000Z&refreshPaused=true&refreshInterval=0&kuery=&transactionType=request'
);
});
test('returns expected URL for other type URL', () => {
expect(getUrlForRecord(TEST_OTHER_URL, TEST_RECORD)).toBe(
'http://airlinecodes.info/airline-code-AAL'

View file

@ -97,7 +97,11 @@ export function openCustomUrlWindow(fullUrl: string, urlConfig: UrlConfig) {
// a Kibana Discover or Dashboard page running on the same server as this ML plugin.
function isKibanaUrl(urlConfig: UrlConfig) {
const urlValue = urlConfig.url_value;
return urlValue.startsWith('kibana#/discover') || urlValue.startsWith('kibana#/dashboard');
return (
urlValue.startsWith('kibana#/discover') ||
urlValue.startsWith('kibana#/dashboard') ||
urlValue.startsWith('apm#/')
);
}
/**
@ -136,13 +140,14 @@ function buildKibanaUrl(urlConfig: UrlConfig, record: CustomUrlAnomalyRecordDoc)
commonEscapeCallback
);
return str.replace(/\$([^?&$\'"]+)\$/g, (match, name: string) => {
// Looking for a $token$ with an optional trailing slash
return str.replace(/\$([^?&$\'"]+)\$(\/)?/g, (match, name: string, slash: string = '') => {
// Use lodash get to allow nested JSON fields to be retrieved.
let tokenValue: string | string[] | undefined = get(record, name);
tokenValue = Array.isArray(tokenValue) ? tokenValue[0] : tokenValue;
// If property not found string is not replaced.
return tokenValue === undefined ? match : getResultTokenValue(tokenValue);
// If property not found token is replaced with an empty string.
return tokenValue === undefined ? '' : getResultTokenValue(tokenValue) + slash;
});
};
@ -155,7 +160,7 @@ function buildKibanaUrl(urlConfig: UrlConfig, record: CustomUrlAnomalyRecordDoc)
commonEscapeCallback
);
return str.replace(
/(.+query:')([^']*)('.+)/,
/(.+query:'|.+&kuery=)([^']*)(['&].+)/,
(fullMatch, prefix: string, queryString: string, postfix: string) => {
const [resultPrefix, resultPostfix] = [prefix, postfix].map(replaceSingleTokenValues);
@ -170,28 +175,39 @@ function buildKibanaUrl(urlConfig: UrlConfig, record: CustomUrlAnomalyRecordDoc)
const queryParts: string[] = [];
const joinOperator = ' AND ';
for (let i = 0; i < queryFields.length; i++) {
fieldsLoop: for (let i = 0; i < queryFields.length; i++) {
const field = queryFields[i];
// Use lodash get to allow nested JSON fields to be retrieved.
const tokenValues: string[] | string | null = get(record, field) || null;
let tokenValues: string[] | string | null = get(record, field) || null;
if (tokenValues === null) {
continue;
}
tokenValues = Array.isArray(tokenValues) ? tokenValues : [tokenValues];
// Create a pair `influencerField:value`.
// In cases where there are multiple influencer field values for an anomaly
// combine values with OR operator e.g. `(influencerField:value or influencerField:another_value)`.
let result = (Array.isArray(tokenValues) ? tokenValues : [tokenValues])
.map(value => `${field}:"${getResultTokenValue(value)}"`)
.join(' OR ');
result = tokenValues.length > 1 ? `(${result})` : result;
let result = '';
for (let j = 0; j < tokenValues.length; j++) {
const part = `${j > 0 ? ' OR ' : ''}${field}:"${getResultTokenValue(
tokenValues[j]
)}"`;
// Build up a URL string which is not longer than the allowed length and isn't corrupted by invalid query.
availableCharactersLeft -= result.length - (i === 0 ? 0 : joinOperator.length);
// Build up a URL string which is not longer than the allowed length and isn't corrupted by invalid query.
if (availableCharactersLeft < part.length) {
if (result.length > 0) {
queryParts.push(j > 0 ? `(${result})` : result);
}
break fieldsLoop;
}
if (availableCharactersLeft <= 0) {
break;
} else {
queryParts.push(result);
result += part;
availableCharactersLeft -= result.length;
}
if (result.length > 0) {
queryParts.push(tokenValues.length > 1 ? `(${result})` : result);
}
}

View file

@ -12,6 +12,8 @@ describe('ML - data recognizer', () => {
const moduleIds = [
'apache_ecs',
'apm_jsbase',
'apm_nodejs',
'apm_transaction',
'auditbeat_process_docker_ecs',
'auditbeat_process_hosts_ecs',

View file

@ -0,0 +1,3 @@
{
"icon": "apmApp"
}

View file

@ -0,0 +1,53 @@
{
"id": "apm_jsbase",
"title": "APM: RUM Javascript",
"description": "Detect problematic spans and identify user agents that are potentially causing issues.",
"type": "APM data",
"logoFile": "logo.json",
"defaultIndexPattern": "apm-*",
"query": {
"bool": {
"filter": [{ "term": { "agent.name": "js-base" } }]
}
},
"jobs": [
{
"id": "abnormal_span_durations_jsbase",
"file": "abnormal_span_durations_jsbase.json"
},
{
"id": "anomalous_error_rate_for_user_agents_jsbase",
"file": "anomalous_error_rate_for_user_agents_jsbase.json"
},
{
"id": "decreased_throughput_jsbase",
"file": "decreased_throughput_jsbase.json"
},
{
"id": "high_count_by_user_agent_jsbase",
"file": "high_count_by_user_agent_jsbase.json"
}
],
"datafeeds": [
{
"id": "datafeed-abnormal_span_durations_jsbase",
"file": "datafeed_abnormal_span_durations_jsbase.json",
"job_id": "abnormal_span_durations_jsbase"
},
{
"id": "datafeed-anomalous_error_rate_for_user_agents_jsbase",
"file": "datafeed_anomalous_error_rate_for_user_agents_jsbase.json",
"job_id": "anomalous_error_rate_for_user_agents_jsbase"
},
{
"id": "datafeed-decreased_throughput_jsbase",
"file": "datafeed_decreased_throughput_jsbase.json",
"job_id": "decreased_throughput_jsbase"
},
{
"id": "datafeed-high_count_by_user_agent_jsbase",
"file": "datafeed_high_count_by_user_agent_jsbase.json",
"job_id": "high_count_by_user_agent_jsbase"
}
]
}

View file

@ -0,0 +1,41 @@
{
"job_type": "anomaly_detector",
"groups": [
"apm"
],
"description": "APM JSBase: Looks for spans that are taking longer than usual to process.",
"analysis_config": {
"bucket_span": "15m",
"detectors": [
{
"detector_description": "increased span duration",
"function": "high_mean",
"field_name": "span.duration.us",
"partition_field_name": "span.type"
}
],
"influencers": [
"span.type",
"trace.id",
"span.name",
"service.name"
]
},
"allow_lazy_open": true,
"analysis_limits": {
"model_memory_limit": "128mb"
},
"data_description": {
"time_field": "@timestamp"
},
"custom_settings": {
"created_by": "ml-module-apm-jsbase",
"custom_urls": [
{
"url_name": "APM",
"time_range": "2h",
"url_value": "apm#/traces?rangeFrom=$earliest$&rangeTo=$latest$&kuery=trace.id:\"$trace.id$\"&_g=()"
}
]
}
}

View file

@ -0,0 +1,40 @@
{
"job_type": "anomaly_detector",
"groups": [
"apm"
],
"description": "APM JSBase: Detects user agents that are encountering errors at an above normal rate. This can help detect browser compatibility issues.",
"analysis_config": {
"bucket_span": "15m",
"detectors": [
{
"detector_description": "high error rate for user agent",
"function": "high_non_zero_count",
"partition_field_name": "user_agent.name"
}
],
"influencers": [
"user_agent.name",
"error.exception.message.keyword",
"error.page.url",
"service.name"
]
},
"allow_lazy_open": true,
"analysis_limits": {
"model_memory_limit": "32mb"
},
"data_description": {
"time_field": "@timestamp"
},
"custom_settings": {
"created_by": "ml-module-apm-jsbase",
"custom_urls": [
{
"url_name": "APM",
"time_range": "2h",
"url_value": "apm#/services/$service.name$/errors?rangeFrom=$earliest$&rangeTo=$latest$&refreshPaused=true&refreshInterval=0&kuery=user_agent.name:\"$user_agent.name$\"&_g=()"
}
]
}
}

View file

@ -0,0 +1,15 @@
{
"job_id": "JOB_ID",
"indices": [
"INDEX_PATTERN_NAME"
],
"max_empty_searches": 10,
"query": {
"bool": {
"must": [
{ "bool": { "filter": { "term": { "agent.name": "js-base" } } } },
{ "bool": { "filter": { "term": { "processor.event": "span" } } } }
]
}
}
}

View file

@ -0,0 +1,15 @@
{
"job_id": "JOB_ID",
"indices": [
"INDEX_PATTERN_NAME"
],
"max_empty_searches": 10,
"query": {
"bool": {
"must": [
{ "bool": { "filter": { "term": { "agent.name": "js-base" } } } },
{ "exists": { "field": "user_agent.name" } }
]
}
}
}

View file

@ -0,0 +1,27 @@
{
"job_id": "JOB_ID",
"indices": [
"INDEX_PATTERN_NAME"
],
"max_empty_searches": 10,
"query": {
"bool": {
"filter": { "term": { "agent.name": "js-base" } }
}
},
"aggregations": {
"buckets": {
"date_histogram": {
"field": "@timestamp",
"fixed_interval": "900000ms"
},
"aggregations": {
"@timestamp": {
"max": {
"field": "@timestamp"
}
}
}
}
}
}

View file

@ -0,0 +1,16 @@
{
"job_id": "JOB_ID",
"indices": [
"INDEX_PATTERN_NAME"
],
"max_empty_searches": 10,
"query": {
"bool": {
"must": [
{ "bool": { "filter": { "term": { "agent.name": "js-base" } } } },
{ "bool": { "filter": [{ "exists": { "field": "user_agent.name" } }] } },
{ "bool": { "filter": { "term": { "processor.event": "transaction" } } } }
]
}
}
}

View file

@ -0,0 +1,37 @@
{
"job_type": "anomaly_detector",
"groups": [
"apm"
],
"description": "APM JSBase: Identifies periods during which the application is processing fewer requests than normal.",
"analysis_config": {
"summary_count_field_name": "doc_count",
"bucket_span": "15m",
"detectors": [
{
"detector_description": "low throughput",
"function": "low_count"
}
],
"influencers": [
"service.name"
]
},
"allow_lazy_open": true,
"analysis_limits": {
"model_memory_limit": "10mb"
},
"data_description": {
"time_field": "@timestamp"
},
"custom_settings": {
"created_by": "ml-module-apm-jsbase",
"custom_urls": [
{
"url_name": "APM",
"time_range": "2h",
"url_value": "apm#/services?rangeFrom=$earliest$&rangeTo=$latest$&refreshPaused=true&refreshInterval=0&kuery=&transactionType=request"
}
]
}
}

View file

@ -0,0 +1,38 @@
{
"job_type": "anomaly_detector",
"groups": [
"apm"
],
"description": "APM JSBase: Detects user agents that are making requests at a suspiciously high rate. This is useful in identifying bots.",
"analysis_config": {
"bucket_span": "15m",
"detectors": [
{
"detector_description": "high request rate for user agent",
"function": "high_non_zero_count",
"partition_field_name": "user_agent.name"
}
],
"influencers": [
"user_agent.name",
"service.name"
]
},
"allow_lazy_open": true,
"analysis_limits": {
"model_memory_limit": "32mb"
},
"data_description": {
"time_field": "@timestamp"
},
"custom_settings": {
"created_by": "ml-module-apm-jsbase",
"custom_urls": [
{
"url_name": "APM",
"time_range": "2h",
"url_value": "apm#/services/$service.name$/transactions?rangeFrom=$earliest$&rangeTo=$latest$&refreshPaused=true&refreshInterval=0&kuery=user_agent.name:\"$user_agent.name$\"&_g=()"
}
]
}
}

View file

@ -0,0 +1,3 @@
{
"icon": "apmApp"
}

View file

@ -0,0 +1,42 @@
{
"id": "apm_nodejs",
"title": "APM: NodeJS",
"description": "Detect abnormal traces, anomalous spans, and identify periods of decreased throughput.",
"type": "APM data",
"logoFile": "logo.json",
"defaultIndexPattern": "apm-*",
"query": {
"bool": { "filter": [{ "term": { "agent.name": "nodejs" } }] }
},
"jobs": [
{
"id": "abnormal_span_durations_nodejs",
"file": "abnormal_span_durations_nodejs.json"
},
{
"id": "abnormal_trace_durations_nodejs",
"file": "abnormal_trace_durations_nodejs.json"
},
{
"id": "decreased_throughput_nodejs",
"file": "decreased_throughput_nodejs.json"
}
],
"datafeeds": [
{
"id": "datafeed-abnormal_span_durations_nodejs",
"file": "datafeed_abnormal_span_durations_nodejs.json",
"job_id": "abnormal_span_durations_nodejs"
},
{
"id": "datafeed-abnormal_trace_durations_nodejs",
"file": "datafeed_abnormal_trace_durations_nodejs.json",
"job_id": "abnormal_trace_durations_nodejs"
},
{
"id": "datafeed-decreased_throughput_nodejs",
"file": "datafeed_decreased_throughput_nodejs.json",
"job_id": "decreased_throughput_nodejs"
}
]
}

View file

@ -0,0 +1,41 @@
{
"job_type": "anomaly_detector",
"groups": [
"apm"
],
"description": "APM NodeJS: Looks for spans that are taking longer than usual to process.",
"analysis_config": {
"bucket_span": "15m",
"detectors": [
{
"detector_description": "increased span duration",
"function": "high_mean",
"field_name": "span.duration.us",
"partition_field_name": "span.type"
}
],
"influencers": [
"span.type",
"trace.id",
"span.name",
"service.name"
]
},
"allow_lazy_open": true,
"analysis_limits": {
"model_memory_limit": "128mb"
},
"data_description": {
"time_field": "@timestamp"
},
"custom_settings": {
"created_by": "ml-module-apm-nodejs",
"custom_urls": [
{
"url_name": "APM",
"time_range": "2h",
"url_value": "apm#/traces?rangeFrom=$earliest$&rangeTo=$latest$&kuery=trace.id:\"$trace.id$\"&_g=()"
}
]
}
}

View file

@ -0,0 +1,40 @@
{
"job_type": "anomaly_detector",
"groups": [
"apm"
],
"description": "APM NodeJS: Identifies trace transactions that are processing more slowly than usual.",
"analysis_config": {
"bucket_span": "15m",
"detectors": [
{
"detector_description": "increased trace duration",
"function": "high_mean",
"field_name": "transaction.duration.us",
"by_field_name": "transaction.name"
}
],
"influencers": [
"transaction.name",
"trace.id",
"service.name"
]
},
"allow_lazy_open": true,
"analysis_limits": {
"model_memory_limit": "256mb"
},
"data_description": {
"time_field": "@timestamp"
},
"custom_settings": {
"created_by": "ml-module-apm-nodejs",
"custom_urls": [
{
"url_name": "APM",
"time_range": "2h",
"url_value": "apm#/traces?rangeFrom=$earliest$&rangeTo=$latest$&kuery=trace.id:\"$trace.id$\" and transaction.name:\"$transaction.name$\"&_g=()"
}
]
}
}

View file

@ -0,0 +1,15 @@
{
"job_id": "JOB_ID",
"indices": [
"INDEX_PATTERN_NAME"
],
"max_empty_searches": 10,
"query": {
"bool": {
"must": [
{ "bool": { "filter": { "term": { "agent.name": "nodejs" } } } },
{ "bool": { "filter": { "term": { "processor.event": "span" } } } }
]
}
}
}

View file

@ -0,0 +1,13 @@
{
"job_id": "JOB_ID",
"indices": [
"INDEX_PATTERN_NAME"
],
"max_empty_searches": 10,
"query": {
"bool": {
"must_not": [{ "exists": { "field": "parent.id" } }],
"must": [{ "bool": { "filter": { "term": { "agent.name": "nodejs" } } } }]
}
}
}

View file

@ -0,0 +1,27 @@
{
"job_id": "JOB_ID",
"indices": [
"INDEX_PATTERN_NAME"
],
"max_empty_searches": 10,
"query": {
"bool": {
"filter": { "term": { "agent.name": "nodejs" } }
}
},
"aggregations": {
"buckets": {
"date_histogram": {
"field": "@timestamp",
"fixed_interval": "900000ms"
},
"aggregations": {
"@timestamp": {
"max": {
"field": "@timestamp"
}
}
}
}
}
}

View file

@ -0,0 +1,38 @@
{
"job_type": "anomaly_detector",
"groups": [
"apm"
],
"description": "APM NodeJS: Identifies periods during which the application is processing fewer requests than normal.",
"analysis_config": {
"summary_count_field_name": "doc_count",
"bucket_span": "15m",
"detectors": [
{
"detector_description": "low throughput",
"function": "low_count"
}
],
"influencers": [
"transaction.name",
"service.name"
]
},
"allow_lazy_open": true,
"analysis_limits": {
"model_memory_limit": "10mb"
},
"data_description": {
"time_field": "@timestamp"
},
"custom_settings": {
"created_by": "ml-module-apm-nodejs",
"custom_urls": [
{
"url_name": "APM",
"time_range": "2h",
"url_value": "apm#/services?rangeFrom=$earliest$&rangeTo=$latest$&refreshPaused=true&refreshInterval=0&kuery=&transactionType=request"
}
]
}
}