github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 09:19:04 -04:00
Code Issues Projects Releases Packages Wiki Activity

[APM] Revert multi-metric ML job. (#114961)

Browse source
This commit is contained in:
Dario Gieselaar 2021-10-15 09:33:06 +02:00 • committed by GitHub
parent 8aeaa5a2ff
commit 008421f170
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 77 additions and 212 deletions
Download patch file Download diff file Expand all files Collapse all files

2
x-pack/plugins/apm/common/anomaly_detection.ts
View file

@ -33,8 +33,6 @@ export function getSeverityColor(score: number) {
return mlGetSeverityColor(score);
}
export const ML_TRANSACTION_LATENCY_DETECTOR_INDEX = 0;
export const ML_ERRORS = {
INVALID_LICENSE: i18n.translate(
'xpack.apm.anomaly_detection.error.invalid_license',

25
x-pack/plugins/apm/common/utils/apm_ml_anomaly_query.ts
View file

@ -1,25 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export function apmMlAnomalyQuery(detectorIndex: 0 | 1 | 2) {
return [
{
bool: {
filter: [
{
terms: {
result_type: ['model_plot', 'record'],
},
},
{
term: { detector_index: detectorIndex },
},
],
},
},
];
}

22
x-pack/plugins/apm/server/lib/anomaly_detection/create_anomaly_detection_jobs.ts
View file

@ -5,21 +5,21 @@
* 2.0.
*/
import Boom from '@hapi/boom';
import { Logger } from 'kibana/server';
import { snakeCase } from 'lodash';
import moment from 'moment';
import uuid from 'uuid/v4';
import { snakeCase } from 'lodash';
import Boom from '@hapi/boom';
import moment from 'moment';
import { ML_ERRORS } from '../../../common/anomaly_detection';
import {
METRICSET_NAME,
PROCESSOR_EVENT,
} from '../../../common/elasticsearch_fieldnames';
import { ProcessorEvent } from '../../../common/processor_event';
import { environmentQuery } from '../../../common/utils/environment_query';
import { withApmSpan } from '../../utils/with_apm_span';
import { Setup } from '../helpers/setup_request';
import {
TRANSACTION_DURATION,
PROCESSOR_EVENT,
} from '../../../common/elasticsearch_fieldnames';
import { APM_ML_JOB_GROUP, ML_MODULE_ID_APM_TRANSACTION } from './constants';
import { withApmSpan } from '../../utils/with_apm_span';
import { getAnomalyDetectionJobs } from './get_anomaly_detection_jobs';
export async function createAnomalyDetectionJobs(
@ -92,8 +92,8 @@ async function createAnomalyDetectionJob({
query: {
bool: {
filter: [
{ term: { [PROCESSOR_EVENT]: ProcessorEvent.metric } },
{ term: { [METRICSET_NAME]: 'transaction' } },
{ term: { [PROCESSOR_EVENT]: ProcessorEvent.transaction } },
{ exists: { field: TRANSACTION_DURATION } },
...environmentQuery(environment),
],
},
@ -105,7 +105,7 @@ async function createAnomalyDetectionJob({
job_tags: {
environment,
// identifies this as an APM ML job & facilitates future migrations
apm_ml_version: 3,
apm_ml_version: 2,
},
},
},

9
x-pack/plugins/apm/server/lib/service_map/get_service_anomalies.ts
View file

@ -11,11 +11,7 @@ import { estypes } from '@elastic/elasticsearch';
import { ESSearchResponse } from '../../../../../../src/core/types/elasticsearch';
import { MlPluginSetup } from '../../../../ml/server';
import { PromiseReturnType } from '../../../../observability/typings/common';
import {
getSeverity,
ML_ERRORS,
ML_TRANSACTION_LATENCY_DETECTOR_INDEX,
} from '../../../common/anomaly_detection';
import { getSeverity, ML_ERRORS } from '../../../common/anomaly_detection';
import { ENVIRONMENT_ALL } from '../../../common/environment_filter_values';
import { getServiceHealthStatus } from '../../../common/service_health_status';
import {
@ -26,7 +22,6 @@ import { rangeQuery } from '../../../../observability/server';
import { withApmSpan } from '../../utils/with_apm_span';
import { getMlJobsWithAPMGroup } from '../anomaly_detection/get_ml_jobs_with_apm_group';
import { Setup } from '../helpers/setup_request';
import { apmMlAnomalyQuery } from '../../../common/utils/apm_ml_anomaly_query';
export const DEFAULT_ANOMALIES: ServiceAnomaliesResponse = {
mlJobIds: [],
@ -61,7 +56,7 @@ export async function getServiceAnomalies({
query: {
bool: {
filter: [
...apmMlAnomalyQuery(ML_TRANSACTION_LATENCY_DETECTOR_INDEX),
{ terms: { result_type: ['model_plot', 'record'] } },
...rangeQuery(
Math.min(end - 30 * 60 * 1000, start),
end,

4
x-pack/plugins/apm/server/lib/transactions/get_anomaly_data/fetcher.ts
View file

@ -12,8 +12,6 @@ import { rangeQuery } from '../../../../../observability/server';
import { asMutableArray } from '../../../../common/utils/as_mutable_array';
import { withApmSpan } from '../../../utils/with_apm_span';
import { Setup } from '../../helpers/setup_request';
import { apmMlAnomalyQuery } from '../../../../common/utils/apm_ml_anomaly_query';
import { ML_TRANSACTION_LATENCY_DETECTOR_INDEX } from '../../../../common/anomaly_detection';
export type ESResponse = Exclude<
PromiseReturnType<typeof anomalySeriesFetcher>,
@ -42,7 +40,7 @@ export function anomalySeriesFetcher({
query: {
bool: {
filter: [
...apmMlAnomalyQuery(ML_TRANSACTION_LATENCY_DETECTOR_INDEX),
{ terms: { result_type: ['model_plot', 'record'] } },
{ term: { partition_field_value: serviceName } },
{ term: { by_field_value: transactionType } },
...rangeQuery(start, end, 'timestamp'),

18
x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/manifest.json
View file

@ -1,29 +1,29 @@
{
"id": "apm_transaction",
"title": "APM",
"description": "Detect anomalies in transactions from your APM services for metric data.",
"description": "Detect anomalies in transactions from your APM services.",
"type": "Transaction data",
"logoFile": "logo.json",
"defaultIndexPattern": "apm-*-metric,metrics-apm*",
"defaultIndexPattern": "apm-*-transaction",
"query": {
"bool": {
"filter": [
{ "term": { "processor.event": "metric" } },
{ "term": { "metricset.name": "transaction" } }
{ "term": { "processor.event": "transaction" } },
{ "exists": { "field": "transaction.duration" } }
]
}
},
"jobs": [
{
"id": "apm_metrics",
"file": "apm_metrics.json"
"id": "high_mean_transaction_duration",
"file": "high_mean_transaction_duration.json"
}
],
"datafeeds": [
{
"id": "datafeed-apm_metrics",
"file": "datafeed_apm_metrics.json",
"job_id": "apm_metrics"
"id": "datafeed-high_mean_transaction_duration",
"file": "datafeed_high_mean_transaction_duration.json",
"job_id": "high_mean_transaction_duration"
}
]
}

53
x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/ml/apm_metrics.json
View file

@ -1,53 +0,0 @@
{
"job_type": "anomaly_detector",
"groups": [
"apm"
],
"description": "Detects anomalies in transaction duration, throughput and error percentage for metric data.",
"analysis_config": {
"bucket_span": "15m",
"summary_count_field_name" : "doc_count",
"detectors" : [
{
"detector_description" : "high duration by transaction type for an APM service",
"function" : "high_mean",
"field_name" : "transaction_duration",
"by_field_name" : "transaction.type",
"partition_field_name" : "service.name"
},
{
"detector_description" : "transactions per minute for an APM service",
"function" : "mean",
"field_name" : "transactions_per_min",
"by_field_name" : "transaction.type",
"partition_field_name" : "service.name"
},
{
"detector_description" : "percent failed for an APM service",
"function" : "high_mean",
"field_name" : "transaction_failure_percentage",
"by_field_name" : "transaction.type",
"partition_field_name" : "service.name"
}
],
"influencers" : [
"transaction.type",
"service.name"
]
},
"analysis_limits": {
"model_memory_limit": "32mb"
},
"data_description": {
"time_field" : "@timestamp",
"time_format" : "epoch_ms"
},
"model_plot_config": {
"enabled" : true,
"annotations_enabled" : true
},
"results_index_name" : "custom-apm",
"custom_settings": {
"created_by": "ml-module-apm-transaction"
}
}

95
x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/ml/datafeed_apm_metrics.json
View file

@ -1,95 +0,0 @@
{
"job_id": "JOB_ID",
"indices": [
"INDEX_PATTERN_NAME"
],
"chunking_config" : {
"mode" : "off"
},
"query": {
"bool": {
"filter": [
{ "term": { "processor.event": "metric" } },
{ "term": { "metricset.name": "transaction" } }
]
}
},
"aggregations" : {
"buckets" : {
"composite" : {
"size" : 5000,
"sources" : [
{
"date" : {
"date_histogram" : {
"field" : "@timestamp",
"fixed_interval" : "90s"
}
}
},
{
"transaction.type" : {
"terms" : {
"field" : "transaction.type"
}
}
},
{
"service.name" : {
"terms" : {
"field" : "service.name"
}
}
}
]
},
"aggs" : {
"@timestamp" : {
"max" : {
"field" : "@timestamp"
}
},
"transactions_per_min" : {
"rate" : {
"unit" : "minute"
}
},
"transaction_duration" : {
"avg" : {
"field" : "transaction.duration.histogram"
}
},
"error_count" : {
"filter" : {
"term" : {
"event.outcome" : "failure"
}
},
"aggs" : {
"actual_error_count" : {
"value_count" : {
"field" : "event.outcome"
}
}
}
},
"success_count" : {
"filter" : {
"term" : {
"event.outcome" : "success"
}
}
},
"transaction_failure_percentage" : {
"bucket_script" : {
"buckets_path" : {
"failure_count" : "error_count>_count",
"success_count" : "success_count>_count"
},
"script" : "if ((params.failure_count + params.success_count)==0){return 0;}else{return params.failure_count/(params.failure_count + params.success_count);}"
}
}
}
}
}
}

14
x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/ml/datafeed_high_mean_transaction_duration.json Normal file
View file

@ -0,0 +1,14 @@
{
"job_id": "JOB_ID",
"indices": [
"INDEX_PATTERN_NAME"
],
"query": {
"bool": {
"filter": [
{ "term": { "processor.event": "transaction" } },
{ "exists": { "field": "transaction.duration.us" } }
]
}
}
}

35
x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/ml/high_mean_transaction_duration.json Normal file
View file

@ -0,0 +1,35 @@
{
"job_type": "anomaly_detector",
"groups": [
"apm"
],
"description": "Detect transaction duration anomalies across transaction types for your APM services.",
"analysis_config": {
"bucket_span": "15m",
"detectors": [
{
"detector_description": "high duration by transaction type for an APM service",
"function": "high_mean",
"field_name": "transaction.duration.us",
"by_field_name": "transaction.type",
"partition_field_name": "service.name"
}
],
"influencers": [
"transaction.type",
"service.name"
]
},
"analysis_limits": {
"model_memory_limit": "32mb"
},
"data_description": {
"time_field": "@timestamp"
},
"model_plot_config": {
"enabled": true
},
"custom_settings": {
"created_by": "ml-module-apm-transaction"
}
}

2
x-pack/test/api_integration/apis/ml/modules/recognize_module.ts
View file

@ -44,7 +44,7 @@ export default ({ getService }: FtrProviderContext) => {
user: USER.ML_POWERUSER,
expected: {
responseCode: 200,
moduleIds: ['apm_jsbase', 'apm_nodejs'],
moduleIds: ['apm_jsbase', 'apm_transaction', 'apm_nodejs'],
},
},
{

10
x-pack/test/api_integration/apis/ml/modules/setup_module.ts
View file

@ -187,11 +187,9 @@ export default ({ getService }: FtrProviderContext) => {
dashboards: [] as string[],
},
},
// Set startDatafeed and estimateModelMemory to false for the APM transaction test
// until there is a new data set available with metric data.
{
testTitleSuffix:
'for apm_transaction with prefix, startDatafeed false and estimateModelMemory false',
'for apm_transaction with prefix, startDatafeed true and estimateModelMemory true',
sourceDataArchive: 'x-pack/test/functional/es_archives/ml/module_apm',
indexPattern: { name: 'ft_module_apm', timeField: '@timestamp' },
module: 'apm_transaction',
@ -199,14 +197,14 @@ export default ({ getService }: FtrProviderContext) => {
requestBody: {
prefix: 'pf5_',
indexPatternName: 'ft_module_apm',
startDatafeed: false,
estimateModelMemory: false,
startDatafeed: true,
end: Date.now(),
},
expected: {
responseCode: 200,
jobs: [
{
jobId: 'pf5_apm_metrics',
jobId: 'pf5_high_mean_transaction_duration',
jobState: JOB_STATE.CLOSED,
datafeedState: DATAFEED_STATE.STOPPED,
},