[APM] Revert multi-metric ML job. (#114961)

x-pack/plugins/apm/common/anomaly_detection.ts

@@ -33,8 +33,6 @@ export function getSeverityColor(score: number) {
   return mlGetSeverityColor(score);
 }
 
-export const ML_TRANSACTION_LATENCY_DETECTOR_INDEX = 0;
-
 export const ML_ERRORS = {
   INVALID_LICENSE: i18n.translate(
     'xpack.apm.anomaly_detection.error.invalid_license',

x-pack/plugins/apm/common/utils/apm_ml_anomaly_query.ts

@@ -1,25 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-export function apmMlAnomalyQuery(detectorIndex: 0 | 1 | 2) {
-  return [
-    {
-      bool: {
-        filter: [
-          {
-            terms: {
-              result_type: ['model_plot', 'record'],
-            },
-          },
-          {
-            term: { detector_index: detectorIndex },
-          },
-        ],
-      },
-    },
-  ];
-}

x-pack/plugins/apm/server/lib/anomaly_detection/create_anomaly_detection_jobs.ts

@@ -5,21 +5,21 @@
  * 2.0.
  */
 
-import Boom from '@hapi/boom';
 import { Logger } from 'kibana/server';
-import { snakeCase } from 'lodash';
-import moment from 'moment';
 import uuid from 'uuid/v4';
+import { snakeCase } from 'lodash';
+import Boom from '@hapi/boom';
+import moment from 'moment';
 import { ML_ERRORS } from '../../../common/anomaly_detection';
-import {
-  METRICSET_NAME,
-  PROCESSOR_EVENT,
-} from '../../../common/elasticsearch_fieldnames';
 import { ProcessorEvent } from '../../../common/processor_event';
 import { environmentQuery } from '../../../common/utils/environment_query';
-import { withApmSpan } from '../../utils/with_apm_span';
 import { Setup } from '../helpers/setup_request';
+import {
+  TRANSACTION_DURATION,
+  PROCESSOR_EVENT,
+} from '../../../common/elasticsearch_fieldnames';
 import { APM_ML_JOB_GROUP, ML_MODULE_ID_APM_TRANSACTION } from './constants';
+import { withApmSpan } from '../../utils/with_apm_span';
 import { getAnomalyDetectionJobs } from './get_anomaly_detection_jobs';
 
 export async function createAnomalyDetectionJobs(
@@ -92,8 +92,8 @@ async function createAnomalyDetectionJob({
       query: {
         bool: {
           filter: [
-            { term: { [PROCESSOR_EVENT]: ProcessorEvent.metric } },
+            { term: { [PROCESSOR_EVENT]: ProcessorEvent.transaction } },
-            { term: { [METRICSET_NAME]: 'transaction' } },
+            { exists: { field: TRANSACTION_DURATION } },
             ...environmentQuery(environment),
           ],
         },
@@ -105,7 +105,7 @@ async function createAnomalyDetectionJob({
             job_tags: {
               environment,
               // identifies this as an APM ML job & facilitates future migrations
-              apm_ml_version: 3,
+              apm_ml_version: 2,
             },
           },
         },

x-pack/plugins/apm/server/lib/service_map/get_service_anomalies.ts

@@ -11,11 +11,7 @@ import { estypes } from '@elastic/elasticsearch';
 import { ESSearchResponse } from '../../../../../../src/core/types/elasticsearch';
 import { MlPluginSetup } from '../../../../ml/server';
 import { PromiseReturnType } from '../../../../observability/typings/common';
-import {
+import { getSeverity, ML_ERRORS } from '../../../common/anomaly_detection';
-  getSeverity,
-  ML_ERRORS,
-  ML_TRANSACTION_LATENCY_DETECTOR_INDEX,
-} from '../../../common/anomaly_detection';
 import { ENVIRONMENT_ALL } from '../../../common/environment_filter_values';
 import { getServiceHealthStatus } from '../../../common/service_health_status';
 import {
@@ -26,7 +22,6 @@ import { rangeQuery } from '../../../../observability/server';
 import { withApmSpan } from '../../utils/with_apm_span';
 import { getMlJobsWithAPMGroup } from '../anomaly_detection/get_ml_jobs_with_apm_group';
 import { Setup } from '../helpers/setup_request';
-import { apmMlAnomalyQuery } from '../../../common/utils/apm_ml_anomaly_query';
 
 export const DEFAULT_ANOMALIES: ServiceAnomaliesResponse = {
   mlJobIds: [],
@@ -61,7 +56,7 @@ export async function getServiceAnomalies({
         query: {
           bool: {
             filter: [
-              ...apmMlAnomalyQuery(ML_TRANSACTION_LATENCY_DETECTOR_INDEX),
+              { terms: { result_type: ['model_plot', 'record'] } },
               ...rangeQuery(
                 Math.min(end - 30 * 60 * 1000, start),
                 end,

x-pack/plugins/apm/server/lib/transactions/get_anomaly_data/fetcher.ts

@@ -12,8 +12,6 @@ import { rangeQuery } from '../../../../../observability/server';
 import { asMutableArray } from '../../../../common/utils/as_mutable_array';
 import { withApmSpan } from '../../../utils/with_apm_span';
 import { Setup } from '../../helpers/setup_request';
-import { apmMlAnomalyQuery } from '../../../../common/utils/apm_ml_anomaly_query';
-import { ML_TRANSACTION_LATENCY_DETECTOR_INDEX } from '../../../../common/anomaly_detection';
 
 export type ESResponse = Exclude<
   PromiseReturnType<typeof anomalySeriesFetcher>,
@@ -42,7 +40,7 @@ export function anomalySeriesFetcher({
         query: {
           bool: {
             filter: [
-              ...apmMlAnomalyQuery(ML_TRANSACTION_LATENCY_DETECTOR_INDEX),
+              { terms: { result_type: ['model_plot', 'record'] } },
               { term: { partition_field_value: serviceName } },
               { term: { by_field_value: transactionType } },
               ...rangeQuery(start, end, 'timestamp'),

x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/manifest.json

@@ -1,29 +1,29 @@
 {
   "id": "apm_transaction",
   "title": "APM",
-  "description": "Detect anomalies in transactions from your APM services for metric data.",
+  "description": "Detect anomalies in transactions from your APM services.",
   "type": "Transaction data",
   "logoFile": "logo.json",
-  "defaultIndexPattern": "apm-*-metric,metrics-apm*",
+  "defaultIndexPattern": "apm-*-transaction",
   "query": {
     "bool": {
       "filter": [
-        { "term": { "processor.event": "metric" } },
+        { "term": { "processor.event": "transaction" } },
-        { "term": { "metricset.name": "transaction" } }
+        { "exists": { "field": "transaction.duration" } }
       ]
     }
   },
   "jobs": [
     {
-      "id": "apm_metrics",
+      "id": "high_mean_transaction_duration",
-      "file": "apm_metrics.json"
+      "file": "high_mean_transaction_duration.json"
     }
   ],
   "datafeeds": [
     {
-      "id": "datafeed-apm_metrics",
+      "id": "datafeed-high_mean_transaction_duration",
-      "file": "datafeed_apm_metrics.json",
+      "file": "datafeed_high_mean_transaction_duration.json",
-      "job_id": "apm_metrics"
+      "job_id": "high_mean_transaction_duration"
     }
   ]
 }

x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/ml/apm_metrics.json

@@ -1,53 +0,0 @@
-{
-  "job_type": "anomaly_detector",
-  "groups": [
-    "apm"
-  ],
-  "description": "Detects anomalies in transaction duration, throughput and error percentage for metric data.",
-  "analysis_config": {
-    "bucket_span": "15m",
-    "summary_count_field_name" : "doc_count",
-    "detectors" : [
-      {
-        "detector_description" : "high duration by transaction type for an APM service",
-        "function" : "high_mean",
-        "field_name" : "transaction_duration",
-        "by_field_name" : "transaction.type",
-        "partition_field_name" : "service.name"
-      },
-      {
-        "detector_description" : "transactions per minute for an APM service",
-        "function" : "mean",
-        "field_name" : "transactions_per_min",
-        "by_field_name" : "transaction.type",
-        "partition_field_name" : "service.name"
-      },
-      {
-        "detector_description" : "percent failed for an APM service",
-        "function" : "high_mean",
-        "field_name" : "transaction_failure_percentage",
-        "by_field_name" : "transaction.type",
-        "partition_field_name" : "service.name"
-      }
-    ],
-    "influencers" : [
-      "transaction.type",
-      "service.name"
-    ]
-  },
-  "analysis_limits": {
-    "model_memory_limit": "32mb"
-  },
-  "data_description": {
-    "time_field" : "@timestamp",
-    "time_format" : "epoch_ms"
-  },
-  "model_plot_config": {
-    "enabled" : true,
-    "annotations_enabled" : true
-  },
-  "results_index_name" : "custom-apm",
-  "custom_settings": {
-    "created_by": "ml-module-apm-transaction"
-  }
-}

x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/ml/datafeed_apm_metrics.json

@@ -1,95 +0,0 @@
-{
-  "job_id": "JOB_ID",
-  "indices": [
-    "INDEX_PATTERN_NAME"
-  ],
-  "chunking_config" : {
-    "mode" : "off"
-  },
-  "query": {
-    "bool": {
-      "filter": [
-        { "term": { "processor.event": "metric" } },
-        { "term": { "metricset.name": "transaction" } }
-      ]
-    }
-  },
-  "aggregations" : {
-    "buckets" : {
-      "composite" : {
-        "size" : 5000,
-        "sources" : [
-          {
-            "date" : {
-              "date_histogram" : {
-                "field" : "@timestamp",
-                "fixed_interval" : "90s"
-              }
-            }
-          },
-          {
-            "transaction.type" : {
-              "terms" : {
-                "field" : "transaction.type"
-              }
-            }
-          },
-          {
-            "service.name" : {
-              "terms" : {
-                "field" : "service.name"
-              }
-            }
-          }
-        ]
-      },
-      "aggs" : {
-        "@timestamp" : {
-          "max" : {
-            "field" : "@timestamp"
-          }
-        },
-        "transactions_per_min" : {
-          "rate" : {
-            "unit" : "minute"
-          }
-        },
-        "transaction_duration" : {
-          "avg" : {
-            "field" : "transaction.duration.histogram"
-          }
-        },
-        "error_count" : {
-          "filter" : {
-            "term" : {
-              "event.outcome" : "failure"
-            }
-          },
-          "aggs" : {
-            "actual_error_count" : {
-              "value_count" : {
-                "field" : "event.outcome"
-              }
-            }
-          }
-        },
-        "success_count" : {
-          "filter" : {
-            "term" : {
-              "event.outcome" : "success"
-            }
-          }
-        },
-        "transaction_failure_percentage" : {
-          "bucket_script" : {
-            "buckets_path" : {
-              "failure_count" : "error_count>_count",
-              "success_count" : "success_count>_count"
-            },
-            "script" : "if ((params.failure_count + params.success_count)==0){return 0;}else{return params.failure_count/(params.failure_count + params.success_count);}"
-          }
-        }
-      }
-    }
-  }
-}

x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/ml/datafeed_high_mean_transaction_duration.json

@@ -0,0 +1,14 @@
+{
+  "job_id": "JOB_ID",
+  "indices": [
+    "INDEX_PATTERN_NAME"
+  ],
+  "query": {
+    "bool": {
+      "filter": [
+        { "term": { "processor.event": "transaction" } },
+        { "exists": { "field": "transaction.duration.us" } }
+      ]
+    }
+  }
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/ml/high_mean_transaction_duration.json

@@ -0,0 +1,35 @@
+{
+  "job_type": "anomaly_detector",
+  "groups": [
+    "apm"
+  ],
+  "description": "Detect transaction duration anomalies across transaction types for your APM services.",
+  "analysis_config": {
+    "bucket_span": "15m",
+    "detectors": [
+      {
+        "detector_description": "high duration by transaction type for an APM service",
+        "function": "high_mean",
+        "field_name": "transaction.duration.us",
+        "by_field_name": "transaction.type",
+        "partition_field_name": "service.name"
+      }
+    ],
+    "influencers": [
+      "transaction.type",
+      "service.name"
+    ]
+  },
+  "analysis_limits": {
+    "model_memory_limit": "32mb"
+  },
+  "data_description": {
+    "time_field": "@timestamp"
+  },
+  "model_plot_config": {
+    "enabled": true
+  },
+  "custom_settings": {
+    "created_by": "ml-module-apm-transaction"
+  }
+}

x-pack/test/api_integration/apis/ml/modules/recognize_module.ts

@@ -44,7 +44,7 @@ export default ({ getService }: FtrProviderContext) => {
       user: USER.ML_POWERUSER,
       expected: {
         responseCode: 200,
-        moduleIds: ['apm_jsbase', 'apm_nodejs'],
+        moduleIds: ['apm_jsbase', 'apm_transaction', 'apm_nodejs'],
       },
     },
     {

x-pack/test/api_integration/apis/ml/modules/setup_module.ts

@@ -187,11 +187,9 @@ export default ({ getService }: FtrProviderContext) => {
         dashboards: [] as string[],
       },
     },
-    // Set startDatafeed and estimateModelMemory to false for the APM transaction test
-    // until there is a new data set available with metric data.
     {
       testTitleSuffix:
-        'for apm_transaction with prefix, startDatafeed false and estimateModelMemory false',
+        'for apm_transaction with prefix, startDatafeed true and estimateModelMemory true',
       sourceDataArchive: 'x-pack/test/functional/es_archives/ml/module_apm',
       indexPattern: { name: 'ft_module_apm', timeField: '@timestamp' },
       module: 'apm_transaction',
@@ -199,14 +197,14 @@ export default ({ getService }: FtrProviderContext) => {
       requestBody: {
         prefix: 'pf5_',
         indexPatternName: 'ft_module_apm',
-        startDatafeed: false,
+        startDatafeed: true,
-        estimateModelMemory: false,
+        end: Date.now(),
       },
       expected: {
         responseCode: 200,
         jobs: [
           {
-            jobId: 'pf5_apm_metrics',
+            jobId: 'pf5_high_mean_transaction_duration',
             jobState: JOB_STATE.CLOSED,
             datafeedState: DATAFEED_STATE.STOPPED,
           },