[ML] Removes APM NodeJS and RUM Javascript modules (#119945) (#120084)

* [ML] Removes APM NodeJS and RUM Javascript modules

* [ML] Delete unnecessary module_apm archive

* [ML] Fix get_module API test

Co-authored-by: Pete Harverson <pete@elastic.co>
This commit is contained in:
Kibana Machine 2021-12-01 08:08:45 -05:00 committed by GitHub
parent a58cad9bae
commit 233643ede4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
24 changed files with 2 additions and 934 deletions

View file

@ -1,3 +0,0 @@
{
"icon": "apmApp"
}

View file

@ -1,53 +0,0 @@
{
"id": "apm_jsbase",
"title": "APM: RUM Javascript",
"description": "Detect problematic spans and identify user agents that are potentially causing issues.",
"type": "APM data",
"logoFile": "logo.json",
"defaultIndexPattern": "apm-*",
"query": {
"bool": {
"filter": [{ "term": { "agent.name": "js-base" } }]
}
},
"jobs": [
{
"id": "abnormal_span_durations_jsbase",
"file": "abnormal_span_durations_jsbase.json"
},
{
"id": "anomalous_error_rate_for_user_agents_jsbase",
"file": "anomalous_error_rate_for_user_agents_jsbase.json"
},
{
"id": "decreased_throughput_jsbase",
"file": "decreased_throughput_jsbase.json"
},
{
"id": "high_count_by_user_agent_jsbase",
"file": "high_count_by_user_agent_jsbase.json"
}
],
"datafeeds": [
{
"id": "datafeed-abnormal_span_durations_jsbase",
"file": "datafeed_abnormal_span_durations_jsbase.json",
"job_id": "abnormal_span_durations_jsbase"
},
{
"id": "datafeed-anomalous_error_rate_for_user_agents_jsbase",
"file": "datafeed_anomalous_error_rate_for_user_agents_jsbase.json",
"job_id": "anomalous_error_rate_for_user_agents_jsbase"
},
{
"id": "datafeed-decreased_throughput_jsbase",
"file": "datafeed_decreased_throughput_jsbase.json",
"job_id": "decreased_throughput_jsbase"
},
{
"id": "datafeed-high_count_by_user_agent_jsbase",
"file": "datafeed_high_count_by_user_agent_jsbase.json",
"job_id": "high_count_by_user_agent_jsbase"
}
]
}

View file

@ -1,41 +0,0 @@
{
"job_type": "anomaly_detector",
"groups": [
"apm"
],
"description": "APM JSBase: Looks for spans that are taking longer than usual to process.",
"analysis_config": {
"bucket_span": "15m",
"detectors": [
{
"detector_description": "increased span duration",
"function": "high_mean",
"field_name": "span.duration.us",
"partition_field_name": "span.type"
}
],
"influencers": [
"span.type",
"trace.id",
"span.name",
"service.name"
]
},
"allow_lazy_open": true,
"analysis_limits": {
"model_memory_limit": "128mb"
},
"data_description": {
"time_field": "@timestamp"
},
"custom_settings": {
"created_by": "ml-module-apm-jsbase",
"custom_urls": [
{
"url_name": "APM",
"time_range": "2h",
"url_value": "apm#/traces?rangeFrom=$earliest$&rangeTo=$latest$&kuery=trace.id:\"$trace.id$\"&_g=()"
}
]
}
}

View file

@ -1,40 +0,0 @@
{
"job_type": "anomaly_detector",
"groups": [
"apm"
],
"description": "APM JSBase: Detects user agents that are encountering errors at an above normal rate. This can help detect browser compatibility issues.",
"analysis_config": {
"bucket_span": "15m",
"detectors": [
{
"detector_description": "high error rate for user agent",
"function": "high_non_zero_count",
"partition_field_name": "user_agent.name"
}
],
"influencers": [
"user_agent.name",
"error.exception.message.keyword",
"error.page.url",
"service.name"
]
},
"allow_lazy_open": true,
"analysis_limits": {
"model_memory_limit": "32mb"
},
"data_description": {
"time_field": "@timestamp"
},
"custom_settings": {
"created_by": "ml-module-apm-jsbase",
"custom_urls": [
{
"url_name": "APM",
"time_range": "2h",
"url_value": "apm#/services/$service.name$/errors?rangeFrom=$earliest$&rangeTo=$latest$&refreshPaused=true&refreshInterval=0&kuery=user_agent.name:\"$user_agent.name$\"&_g=()"
}
]
}
}

View file

@ -1,15 +0,0 @@
{
"job_id": "JOB_ID",
"indices": [
"INDEX_PATTERN_NAME"
],
"max_empty_searches": 10,
"query": {
"bool": {
"must": [
{ "bool": { "filter": { "term": { "agent.name": "js-base" } } } },
{ "bool": { "filter": { "term": { "processor.event": "span" } } } }
]
}
}
}

View file

@ -1,15 +0,0 @@
{
"job_id": "JOB_ID",
"indices": [
"INDEX_PATTERN_NAME"
],
"max_empty_searches": 10,
"query": {
"bool": {
"must": [
{ "bool": { "filter": { "term": { "agent.name": "js-base" } } } },
{ "exists": { "field": "user_agent.name" } }
]
}
}
}

View file

@ -1,27 +0,0 @@
{
"job_id": "JOB_ID",
"indices": [
"INDEX_PATTERN_NAME"
],
"max_empty_searches": 10,
"query": {
"bool": {
"filter": { "term": { "agent.name": "js-base" } }
}
},
"aggregations": {
"buckets": {
"date_histogram": {
"field": "@timestamp",
"fixed_interval": "900000ms"
},
"aggregations": {
"@timestamp": {
"max": {
"field": "@timestamp"
}
}
}
}
}
}

View file

@ -1,16 +0,0 @@
{
"job_id": "JOB_ID",
"indices": [
"INDEX_PATTERN_NAME"
],
"max_empty_searches": 10,
"query": {
"bool": {
"must": [
{ "bool": { "filter": { "term": { "agent.name": "js-base" } } } },
{ "bool": { "filter": [{ "exists": { "field": "user_agent.name" } }] } },
{ "bool": { "filter": { "term": { "processor.event": "transaction" } } } }
]
}
}
}

View file

@ -1,37 +0,0 @@
{
"job_type": "anomaly_detector",
"groups": [
"apm"
],
"description": "APM JSBase: Identifies periods during which the application is processing fewer requests than normal.",
"analysis_config": {
"summary_count_field_name": "doc_count",
"bucket_span": "15m",
"detectors": [
{
"detector_description": "low throughput",
"function": "low_count"
}
],
"influencers": [
"service.name"
]
},
"allow_lazy_open": true,
"analysis_limits": {
"model_memory_limit": "10mb"
},
"data_description": {
"time_field": "@timestamp"
},
"custom_settings": {
"created_by": "ml-module-apm-jsbase",
"custom_urls": [
{
"url_name": "APM",
"time_range": "2h",
"url_value": "apm#/services?rangeFrom=$earliest$&rangeTo=$latest$&refreshPaused=true&refreshInterval=0&kuery=&transactionType=request"
}
]
}
}

View file

@ -1,38 +0,0 @@
{
"job_type": "anomaly_detector",
"groups": [
"apm"
],
"description": "APM JSBase: Detects user agents that are making requests at a suspiciously high rate. This is useful in identifying bots.",
"analysis_config": {
"bucket_span": "15m",
"detectors": [
{
"detector_description": "high request rate for user agent",
"function": "high_non_zero_count",
"partition_field_name": "user_agent.name"
}
],
"influencers": [
"user_agent.name",
"service.name"
]
},
"allow_lazy_open": true,
"analysis_limits": {
"model_memory_limit": "32mb"
},
"data_description": {
"time_field": "@timestamp"
},
"custom_settings": {
"created_by": "ml-module-apm-jsbase",
"custom_urls": [
{
"url_name": "APM",
"time_range": "2h",
"url_value": "apm#/services/$service.name$/transactions?rangeFrom=$earliest$&rangeTo=$latest$&refreshPaused=true&refreshInterval=0&kuery=user_agent.name:\"$user_agent.name$\"&_g=()"
}
]
}
}

View file

@ -1,3 +0,0 @@
{
"icon": "apmApp"
}

View file

@ -1,42 +0,0 @@
{
"id": "apm_nodejs",
"title": "APM: NodeJS",
"description": "Detect abnormal traces, anomalous spans, and identify periods of decreased throughput.",
"type": "APM data",
"logoFile": "logo.json",
"defaultIndexPattern": "apm-*",
"query": {
"bool": { "filter": [{ "term": { "agent.name": "nodejs" } }] }
},
"jobs": [
{
"id": "abnormal_span_durations_nodejs",
"file": "abnormal_span_durations_nodejs.json"
},
{
"id": "abnormal_trace_durations_nodejs",
"file": "abnormal_trace_durations_nodejs.json"
},
{
"id": "decreased_throughput_nodejs",
"file": "decreased_throughput_nodejs.json"
}
],
"datafeeds": [
{
"id": "datafeed-abnormal_span_durations_nodejs",
"file": "datafeed_abnormal_span_durations_nodejs.json",
"job_id": "abnormal_span_durations_nodejs"
},
{
"id": "datafeed-abnormal_trace_durations_nodejs",
"file": "datafeed_abnormal_trace_durations_nodejs.json",
"job_id": "abnormal_trace_durations_nodejs"
},
{
"id": "datafeed-decreased_throughput_nodejs",
"file": "datafeed_decreased_throughput_nodejs.json",
"job_id": "decreased_throughput_nodejs"
}
]
}

View file

@ -1,41 +0,0 @@
{
"job_type": "anomaly_detector",
"groups": [
"apm"
],
"description": "APM NodeJS: Looks for spans that are taking longer than usual to process.",
"analysis_config": {
"bucket_span": "15m",
"detectors": [
{
"detector_description": "increased span duration",
"function": "high_mean",
"field_name": "span.duration.us",
"partition_field_name": "span.type"
}
],
"influencers": [
"span.type",
"trace.id",
"span.name",
"service.name"
]
},
"allow_lazy_open": true,
"analysis_limits": {
"model_memory_limit": "128mb"
},
"data_description": {
"time_field": "@timestamp"
},
"custom_settings": {
"created_by": "ml-module-apm-nodejs",
"custom_urls": [
{
"url_name": "APM",
"time_range": "2h",
"url_value": "apm#/traces?rangeFrom=$earliest$&rangeTo=$latest$&kuery=trace.id:\"$trace.id$\"&_g=()"
}
]
}
}

View file

@ -1,40 +0,0 @@
{
"job_type": "anomaly_detector",
"groups": [
"apm"
],
"description": "APM NodeJS: Identifies trace transactions that are processing more slowly than usual.",
"analysis_config": {
"bucket_span": "15m",
"detectors": [
{
"detector_description": "increased trace duration",
"function": "high_mean",
"field_name": "transaction.duration.us",
"by_field_name": "transaction.name"
}
],
"influencers": [
"transaction.name",
"trace.id",
"service.name"
]
},
"allow_lazy_open": true,
"analysis_limits": {
"model_memory_limit": "256mb"
},
"data_description": {
"time_field": "@timestamp"
},
"custom_settings": {
"created_by": "ml-module-apm-nodejs",
"custom_urls": [
{
"url_name": "APM",
"time_range": "2h",
"url_value": "apm#/traces?rangeFrom=$earliest$&rangeTo=$latest$&kuery=trace.id:\"$trace.id$\" and transaction.name:\"$transaction.name$\"&_g=()"
}
]
}
}

View file

@ -1,15 +0,0 @@
{
"job_id": "JOB_ID",
"indices": [
"INDEX_PATTERN_NAME"
],
"max_empty_searches": 10,
"query": {
"bool": {
"must": [
{ "bool": { "filter": { "term": { "agent.name": "nodejs" } } } },
{ "bool": { "filter": { "term": { "processor.event": "span" } } } }
]
}
}
}

View file

@ -1,13 +0,0 @@
{
"job_id": "JOB_ID",
"indices": [
"INDEX_PATTERN_NAME"
],
"max_empty_searches": 10,
"query": {
"bool": {
"must_not": [{ "exists": { "field": "parent.id" } }],
"must": [{ "bool": { "filter": { "term": { "agent.name": "nodejs" } } } }]
}
}
}

View file

@ -1,27 +0,0 @@
{
"job_id": "JOB_ID",
"indices": [
"INDEX_PATTERN_NAME"
],
"max_empty_searches": 10,
"query": {
"bool": {
"filter": { "term": { "agent.name": "nodejs" } }
}
},
"aggregations": {
"buckets": {
"date_histogram": {
"field": "@timestamp",
"fixed_interval": "900000ms"
},
"aggregations": {
"@timestamp": {
"max": {
"field": "@timestamp"
}
}
}
}
}
}

View file

@ -1,38 +0,0 @@
{
"job_type": "anomaly_detector",
"groups": [
"apm"
],
"description": "APM NodeJS: Identifies periods during which the application is processing fewer requests than normal.",
"analysis_config": {
"summary_count_field_name": "doc_count",
"bucket_span": "15m",
"detectors": [
{
"detector_description": "low throughput",
"function": "low_count"
}
],
"influencers": [
"transaction.name",
"service.name"
]
},
"allow_lazy_open": true,
"analysis_limits": {
"model_memory_limit": "10mb"
},
"data_description": {
"time_field": "@timestamp"
},
"custom_settings": {
"created_by": "ml-module-apm-nodejs",
"custom_urls": [
{
"url_name": "APM",
"time_range": "2h",
"url_value": "apm#/services?rangeFrom=$earliest$&rangeTo=$latest$&refreshPaused=true&refreshInterval=0&kuery=&transactionType=request"
}
]
}
}

View file

@ -25,7 +25,7 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) {
await ml.testResources.deleteIndexPatternByTitle('ft_module_apache');
await ml.testResources.deleteIndexPatternByTitle('ft_module_auditbeat');
await ml.testResources.deleteIndexPatternByTitle('ft_module_apm');
await ml.testResources.deleteIndexPatternByTitle('ft_module_apm_transaction');
await ml.testResources.deleteIndexPatternByTitle('ft_module_heartbeat');
await ml.testResources.deleteIndexPatternByTitle('ft_module_logs');
await ml.testResources.deleteIndexPatternByTitle('ft_module_nginx');
@ -47,7 +47,7 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) {
await esArchiver.unload('x-pack/test/functional/es_archives/ml/categorization_small');
await esArchiver.unload('x-pack/test/functional/es_archives/ml/module_apache');
await esArchiver.unload('x-pack/test/functional/es_archives/ml/module_auditbeat');
await esArchiver.unload('x-pack/test/functional/es_archives/ml/module_apm');
await esArchiver.unload('x-pack/test/functional/es_archives/ml/module_apm_transaction');
await esArchiver.unload('x-pack/test/functional/es_archives/ml/module_heartbeat');
await esArchiver.unload('x-pack/test/functional/es_archives/ml/module_logs');
await esArchiver.unload('x-pack/test/functional/es_archives/ml/module_nginx');

View file

@ -16,8 +16,6 @@ import { isPopulatedObject } from '../../../../../plugins/ml/common/util/object_
const moduleIds = [
'apache_data_stream',
'apache_ecs',
'apm_jsbase',
'apm_nodejs',
'apm_transaction',
'auditbeat_process_docker_ecs',
'auditbeat_process_hosts_ecs',

View file

@ -37,16 +37,6 @@ export default ({ getService }: FtrProviderContext) => {
moduleIds: ['apache_ecs'],
},
},
{
testTitleSuffix: 'for apm dataset',
sourceDataArchive: 'x-pack/test/functional/es_archives/ml/module_apm',
indexPattern: 'ft_module_apm',
user: USER.ML_POWERUSER,
expected: {
responseCode: 200,
moduleIds: ['apm_jsbase', 'apm_nodejs'],
},
},
{
testTitleSuffix: 'for logs dataset',
sourceDataArchive: 'x-pack/test/functional/es_archives/ml/module_logs',

View file

@ -150,43 +150,6 @@ export default ({ getService }: FtrProviderContext) => {
dashboards: ['ml_http_access_explorer_ecs'] as string[],
},
},
{
testTitleSuffix:
'for apm_nodejs with prefix, startDatafeed true and estimateModelMemory true',
sourceDataArchive: 'x-pack/test/functional/es_archives/ml/module_apm',
indexPattern: { name: 'ft_module_apm', timeField: '@timestamp' },
module: 'apm_nodejs',
user: USER.ML_POWERUSER,
requestBody: {
prefix: 'pf4_',
indexPatternName: 'ft_module_apm',
startDatafeed: true,
end: Date.now(),
},
expected: {
responseCode: 200,
jobs: [
{
jobId: 'pf4_abnormal_span_durations_nodejs',
jobState: JOB_STATE.CLOSED,
datafeedState: DATAFEED_STATE.STOPPED,
},
{
jobId: 'pf4_abnormal_trace_durations_nodejs',
jobState: JOB_STATE.CLOSED,
datafeedState: DATAFEED_STATE.STOPPED,
},
{
jobId: 'pf4_decreased_throughput_nodejs',
jobState: JOB_STATE.CLOSED,
datafeedState: DATAFEED_STATE.STOPPED,
},
],
searches: [] as string[],
visualizations: [] as string[],
dashboards: [] as string[],
},
},
{
testTitleSuffix:
'for apm_transaction with prefix, startDatafeed true and estimateModelMemory true',

View file

@ -1,379 +0,0 @@
{
"type": "index",
"value": {
"aliases": {
},
"index": "ft_module_apm",
"mappings": {
"properties": {
"@timestamp": {
"type": "date"
},
"agent": {
"dynamic": "false",
"properties": {
"ephemeral_id": {
"ignore_above": 1024,
"type": "keyword"
},
"hostname": {
"ignore_above": 1024,
"type": "keyword"
},
"id": {
"ignore_above": 1024,
"type": "keyword"
},
"name": {
"ignore_above": 1024,
"type": "keyword"
},
"type": {
"ignore_above": 1024,
"type": "keyword"
},
"version": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"error": {
"dynamic": "false",
"properties": {
"code": {
"ignore_above": 1024,
"type": "keyword"
},
"culprit": {
"ignore_above": 1024,
"type": "keyword"
},
"exception": {
"properties": {
"code": {
"ignore_above": 1024,
"type": "keyword"
},
"handled": {
"type": "boolean"
},
"message": {
"norms": false,
"type": "text"
},
"module": {
"ignore_above": 1024,
"type": "keyword"
},
"type": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"grouping_key": {
"ignore_above": 1024,
"type": "keyword"
},
"id": {
"ignore_above": 1024,
"type": "keyword"
},
"log": {
"properties": {
"level": {
"ignore_above": 1024,
"type": "keyword"
},
"logger_name": {
"ignore_above": 1024,
"type": "keyword"
},
"message": {
"norms": false,
"type": "text"
},
"param_message": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"message": {
"norms": false,
"type": "text"
},
"type": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"parent": {
"dynamic": "false",
"properties": {
"id": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"processor": {
"properties": {
"event": {
"ignore_above": 1024,
"type": "keyword"
},
"name": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"service": {
"dynamic": "false",
"properties": {
"environment": {
"ignore_above": 1024,
"type": "keyword"
},
"ephemeral_id": {
"ignore_above": 1024,
"type": "keyword"
},
"framework": {
"properties": {
"name": {
"ignore_above": 1024,
"type": "keyword"
},
"version": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"id": {
"ignore_above": 1024,
"type": "keyword"
},
"language": {
"properties": {
"name": {
"ignore_above": 1024,
"type": "keyword"
},
"version": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"name": {
"ignore_above": 1024,
"type": "keyword"
},
"runtime": {
"properties": {
"name": {
"ignore_above": 1024,
"type": "keyword"
},
"version": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"state": {
"ignore_above": 1024,
"type": "keyword"
},
"type": {
"ignore_above": 1024,
"type": "keyword"
},
"version": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"span": {
"dynamic": "false",
"properties": {
"action": {
"ignore_above": 1024,
"type": "keyword"
},
"duration": {
"properties": {
"us": {
"type": "long"
}
}
},
"id": {
"ignore_above": 1024,
"type": "keyword"
},
"name": {
"ignore_above": 1024,
"type": "keyword"
},
"start": {
"properties": {
"us": {
"type": "long"
}
}
},
"subtype": {
"ignore_above": 1024,
"type": "keyword"
},
"sync": {
"type": "boolean"
},
"type": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"trace": {
"dynamic": "false",
"properties": {
"id": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"transaction": {
"dynamic": "false",
"properties": {
"duration": {
"properties": {
"us": {
"type": "long"
}
}
},
"id": {
"ignore_above": 1024,
"type": "keyword"
},
"marks": {
"dynamic": "true",
"properties": {
"*": {
"properties": {
"*": {
"dynamic": "true",
"type": "object"
}
}
}
}
},
"name": {
"fields": {
"text": {
"norms": false,
"type": "text"
}
},
"ignore_above": 1024,
"type": "keyword"
},
"result": {
"ignore_above": 1024,
"type": "keyword"
},
"sampled": {
"type": "boolean"
},
"span_count": {
"properties": {
"dropped": {
"type": "long"
}
}
},
"type": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"user_agent": {
"dynamic": "false",
"properties": {
"device": {
"properties": {
"name": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"name": {
"ignore_above": 1024,
"type": "keyword"
},
"original": {
"fields": {
"text": {
"norms": false,
"type": "text"
}
},
"ignore_above": 1024,
"type": "keyword"
},
"os": {
"properties": {
"family": {
"ignore_above": 1024,
"type": "keyword"
},
"full": {
"ignore_above": 1024,
"type": "keyword"
},
"kernel": {
"ignore_above": 1024,
"type": "keyword"
},
"name": {
"ignore_above": 1024,
"type": "keyword"
},
"platform": {
"ignore_above": 1024,
"type": "keyword"
},
"version": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"version": {
"ignore_above": 1024,
"type": "keyword"
}
}
}
}
},
"settings": {
"index": {
"number_of_replicas": "1",
"number_of_shards": "1"
}
}
}
}