[ML] AIOps: Add performance journey for log rate analysis. (#178017)

## Summary

Part of #173301.

Add a performance journey for AIOps Log Rate Analysis.

To run the performance journey locally, run:

```
node scripts/run_performance.js --journey-path x-pack/performance/journeys/aiops_log_rate_analysis.ts
```

Review notes:

- The small dataset used isn't set up using `esArchiver`, because it's
just 18 docs we're reusing a file from AIOps integration tests und using
bulk ingest to create the index. The data view necessary to populate the
UI is created with an `kibanaServer.request` call.
- Because of the above, the usual `es/kbnArchiver` cleanup cannot kick
in. To support manual cleanup, I added an `afterSteps` option similar to
the existing `beforeSteps` option. This allows us to delete the index
and data view during teardown.
- Kibana operations team triggered by `.buildkite/ftr_configs.yml`.


Journey showing up in APM:

<img width="1638" alt="image"
src="14f130f5-c125-4390-90d0-96002bc916f3">

### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [x] This was checked for breaking API changes and was [labeled
appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)

.buildkite/ftr_configs.yml

@@ -447,6 +447,7 @@ enabled:
   - x-pack/test_serverless/functional/test_suites/security/common_configs/config.group4.ts
   - x-pack/test_serverless/functional/test_suites/security/common_configs/config.group5.ts
   - x-pack/test_serverless/functional/test_suites/security/common_configs/config.group6.ts
+  - x-pack/performance/journeys_e2e/aiops_log_rate_analysis.ts
   - x-pack/performance/journeys_e2e/ecommerce_dashboard.ts
   - x-pack/performance/journeys_e2e/ecommerce_dashboard_map_only.ts
   - x-pack/performance/journeys_e2e/flight_dashboard.ts

.github/CODEOWNERS

@@ -15,6 +15,7 @@ src/plugins/ai_assistant_management/observability @elastic/obs-knowledge-team
 src/plugins/ai_assistant_management/selection @elastic/obs-knowledge-team
 x-pack/packages/ml/aiops_components @elastic/ml-ui
 x-pack/plugins/aiops @elastic/ml-ui
+x-pack/packages/ml/aiops_test_utils @elastic/ml-ui
 x-pack/packages/ml/aiops_utils @elastic/ml-ui
 x-pack/test/alerting_api_integration/packages/helpers @elastic/response-ops
 x-pack/test/alerting_api_integration/common/plugins/alerts @elastic/response-ops

package.json

@@ -143,6 +143,7 @@
     "@kbn/ai-assistant-management-plugin": "link:src/plugins/ai_assistant_management/selection",
     "@kbn/aiops-components": "link:x-pack/packages/ml/aiops_components",
     "@kbn/aiops-plugin": "link:x-pack/plugins/aiops",
+    "@kbn/aiops-test-utils": "link:x-pack/packages/ml/aiops_test_utils",
     "@kbn/aiops-utils": "link:x-pack/packages/ml/aiops_utils",
     "@kbn/alerting-api-integration-test-plugin": "link:x-pack/test/alerting_api_integration/common/plugins/alerts",
     "@kbn/alerting-example-plugin": "link:x-pack/examples/alerting_example",

tsconfig.base.json

@@ -24,6 +24,8 @@
       "@kbn/aiops-components/*": ["x-pack/packages/ml/aiops_components/*"],
       "@kbn/aiops-plugin": ["x-pack/plugins/aiops"],
       "@kbn/aiops-plugin/*": ["x-pack/plugins/aiops/*"],
+      "@kbn/aiops-test-utils": ["x-pack/packages/ml/aiops_test_utils"],
+      "@kbn/aiops-test-utils/*": ["x-pack/packages/ml/aiops_test_utils/*"],
       "@kbn/aiops-utils": ["x-pack/packages/ml/aiops_utils"],
       "@kbn/aiops-utils/*": ["x-pack/packages/ml/aiops_utils/*"],
       "@kbn/alerting-api-integration-helpers": ["x-pack/test/alerting_api_integration/packages/helpers"],

x-pack/packages/ml/aiops_test_utils/README.md

@@ -0,0 +1,3 @@
+# @kbn/aiops-test-utils
+
+Package containing test utility files used as part of ML team's AIOps features across jest unit tests, functional and API integration tests as well as performance journeys.

x-pack/packages/ml/aiops_test_utils/artificial_logs/filtered_frequent_item_sets.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import type { ItemSet } from '../../types';
+import type { ItemSet } from '@kbn/aiops-utils/types';
 
 export const filteredFrequentItemSets: ItemSet[] = [
   {

x-pack/packages/ml/aiops_test_utils/artificial_logs/frequent_item_sets.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import type { ItemSet } from '../../types';
+import type { ItemSet } from '@kbn/aiops-utils/types';
 
 export const frequentItemSets: ItemSet[] = [
   {

x-pack/packages/ml/aiops_test_utils/jest.config.js

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+module.exports = {
+  preset: '@kbn/test',
+  rootDir: '../../../..',
+  roots: ['<rootDir>/x-pack/packages/ml/aiops_test_utils'],
+};

x-pack/packages/ml/aiops_test_utils/kibana.jsonc

@@ -0,0 +1,5 @@
+{
+  "type": "shared-common",
+  "id": "@kbn/aiops-test-utils",
+  "owner": "@elastic/ml-ui"
+}

x-pack/packages/ml/aiops_test_utils/package.json

@@ -0,0 +1,6 @@
+{
+  "name": "@kbn/aiops-test-utils",
+  "private": true,
+  "version": "1.0.0",
+  "license": "Elastic License 2.0"
+}

x-pack/packages/ml/aiops_test_utils/tsconfig.json

@@ -0,0 +1,22 @@
+{
+  "extends": "../../../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "target/types",
+    "types": [
+      "jest",
+      "node",
+      "react"
+    ]
+  },
+  "include": [
+    "**/*.ts",
+    "**/*.tsx",
+  ],
+  "exclude": [
+    "target/**/*"
+  ],
+  "kbn_references": [
+    "@kbn/ml-agg-utils",
+    "@kbn/aiops-utils",
+  ]
+}

x-pack/packages/ml/aiops_utils/tsconfig.json

@@ -15,6 +15,7 @@
   ],
   "kbn_references": [
     "@kbn/ml-is-populated-object",
+    "@kbn/ml-agg-utils",
   ],
   "exclude": [
     "target/**/*",

x-pack/performance/journeys_e2e/aiops_log_rate_analysis.ts

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { Journey } from '@kbn/journeys';
+import { subj } from '@kbn/test-subj-selector';
+
+import dataView from '../kbn_archives/large_arrays_data_view.json';
+
+export const journey = new Journey({
+  kbnArchives: ['x-pack/performance/kbn_archives/large_arrays_data_view'],
+  esArchives: ['x-pack/test/functional/es_archives/large_arrays'],
+})
+  .step('Go to AIOps Log Rate Analysis', async ({ page, kbnUrl }) => {
+    // Navigate to Log Rate Analysis with a prepopulated url state that runs
+    // the analysis directly on page load without any necessary user interactions.
+    await page.goto(
+      kbnUrl.get(
+        `app/ml/aiops/log_rate_analysis?index=${dataView.id}&_g=%28refreshInterval%3A%28pause%3A%21t%2Cvalue%3A60000%29%2Ctime%3A%28from%3A%272019-07-01T15%3A35%3A38.700Z%27%2Cto%3A%272019-07-05T15%3A35%3A38.700Z%27%29%29&_a=%28logRateAnalysis%3A%28filters%3A%21%28%29%2CsearchQuery%3A%28match_all%3A%28%29%29%2CsearchQueryLanguage%3Akuery%2CsearchString%3A%27%27%2Cwp%3A%28bMax%3A1562198400000%2CbMin%3A1562097600000%2CdMax%3A1562270400000%2CdMin%3A1562234400000%29%29%29`
+      )
+    );
+
+    // Wait for the AIOps Log Rate Analysis page wrapper to load
+    await page.waitForSelector(subj('aiopsLogRateAnalysisPage'));
+  })
+  .step('Run AIOps Log Rate Analysis', async ({ page, kbnUrl }) => {
+    // Wait for the analysis to complete with extended timeout, this one tracks a known issue with slow frequent_item_sets performance.
+    await page.waitForSelector(subj('aiopsAnalysisComplete'), { timeout: 120000 });
+  });

x-pack/performance/kbn_archives/large_arrays_data_view.json

@@ -0,0 +1,22 @@
+{
+  "attributes": {
+    "allowHidden": false,
+    "fieldAttrs": "{}",
+    "fieldFormatMap": "{}",
+    "fields": "[]",
+    "name": "large_arrays",
+    "runtimeFieldMap": "{}",
+    "sourceFilters": "[]",
+    "timeFieldName": "@timestamp",
+    "title": "large_arrays"
+  },
+  "coreMigrationVersion": "8.8.0",
+  "created_at": "2024-03-12T15:53:19.579Z",
+  "id": "69dc58dc-80f7-4c46-8fa1-eb29b0794c52",
+  "managed": false,
+  "references": [],
+  "type": "index-pattern",
+  "typeMigrationVersion": "8.0.0",
+  "updated_at": "2024-03-12T15:53:19.579Z",
+  "version": "WzEwLDFd"
+}

x-pack/performance/tsconfig.json

@@ -2,9 +2,15 @@
   "extends": "../../tsconfig.base.json",
   "compilerOptions": {
     "outDir": "target/types",
-    "types": ["node", "jest"]
+    "types": [
+      "node",
+      "jest"
+    ],
   },
-  "include": ["**/*.ts"],
+  "include": [
+    "**/*.ts",
+    "kbn_archives/large_arrays_data_view.json"
+  ],
   "exclude": [
     "target/**/*",
   ],

x-pack/plugins/aiops/common/api/stream_reducer.test.ts

@@ -5,8 +5,8 @@
  * 2.0.
  */
 
-import { significantTerms } from '../__mocks__/artificial_logs/significant_terms';
-import { finalSignificantItemGroups } from '../__mocks__/artificial_logs/final_significant_item_groups';
+import { significantTerms } from '@kbn/aiops-test-utils/artificial_logs/significant_terms';
+import { finalSignificantItemGroups } from '@kbn/aiops-test-utils/artificial_logs/final_significant_item_groups';
 
 import {
   addSignificantItemsAction,

x-pack/plugins/aiops/public/components/log_categorization/show_flyout.tsx

@@ -34,7 +34,8 @@ export async function showCategorizeFlyout(
   originatingApp: string,
   additionalFilter?: CategorizationAdditionalFilter
 ): Promise<void> {
-  const { http, theme, overlays, application, notifications, uiSettings, i18n } = coreStart;
+  const { analytics, http, theme, overlays, application, notifications, uiSettings, i18n } =
+    coreStart;
 
   return new Promise(async (resolve, reject) => {
     try {
@@ -44,6 +45,7 @@ export async function showCategorizeFlyout(
       };
 
       const appDependencies: AiopsAppDependencies = {
+        analytics,
         notifications,
         uiSettings,
         http,

x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_results.tsx

@@ -6,7 +6,7 @@
  */
 
 import type { FC } from 'react';
-import React, { useEffect, useMemo, useState } from 'react';
+import React, { useEffect, useMemo, useRef, useState } from 'react';
 import { isEqual, uniq } from 'lodash';
 import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 
@@ -22,6 +22,7 @@ import {
   EuiText,
 } from '@elastic/eui';
 
+import { reportPerformanceMetricEvent } from '@kbn/ebt-tools';
 import type { DataView } from '@kbn/data-views-plugin/public';
 import { ProgressControls } from '@kbn/aiops-components';
 import { useFetchStream } from '@kbn/ml-response-stream/client';
@@ -137,7 +138,11 @@ export const LogRateAnalysisResults: FC<LogRateAnalysisResultsProps> = ({
   onAnalysisCompleted,
   embeddingOrigin,
 }) => {
-  const { http } = useAiopsAppContext();
+  const { analytics, http } = useAiopsAppContext();
+
+  // Store the performance metric's start time using a ref
+  // to be able to track it across rerenders.
+  const analysisStartTime = useRef<number | undefined>(window.performance.now());
 
   const { clearAllRowState } = useLogRateAnalysisResultsTableRowContext();
 
@@ -232,8 +237,11 @@ export const LogRateAnalysisResults: FC<LogRateAnalysisResultsProps> = ({
           remainingFieldCandidates,
           significantItems: data.significantItems as AiopsLogRateAnalysisSchemaSignificantItem[],
         });
-      } else {
+      } else if (loaded > 0) {
+        // Reset all overrides.
         setOverrides(undefined);
+
+        // If provided call the `onAnalysisCompleted` callback with the analysis results.
         if (onAnalysisCompleted) {
           onAnalysisCompleted({
             analysisType,
@@ -241,6 +249,19 @@ export const LogRateAnalysisResults: FC<LogRateAnalysisResultsProps> = ({
             significantItemsGroups: data.significantItemsGroups,
           });
         }
+
+        // Track performance metric
+        if (analysisStartTime.current !== undefined) {
+          const analysisDuration = window.performance.now() - analysisStartTime.current;
+
+          // Set this to undefined so reporting the metric gets triggered only once.
+          analysisStartTime.current = undefined;
+
+          reportPerformanceMetricEvent(analytics, {
+            eventName: 'aiopsLogRateAnalysisCompleted',
+            duration: analysisDuration,
+          });
+        }
       }
     }
     // eslint-disable-next-line react-hooks/exhaustive-deps

x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/get_group_table_items.test.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { finalSignificantItemGroups } from '../../../common/__mocks__/artificial_logs/final_significant_item_groups';
+import { finalSignificantItemGroups } from '@kbn/aiops-test-utils/artificial_logs/final_significant_item_groups';
 
 import { getGroupTableItems } from './get_group_table_items';
 

x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/get_table_item_as_kql.test.ts

@@ -5,8 +5,8 @@
  * 2.0.
  */
 
-import { finalSignificantItemGroups } from '../../../common/__mocks__/artificial_logs/final_significant_item_groups';
-import { significantTerms } from '../../../common/__mocks__/artificial_logs/significant_terms';
+import { finalSignificantItemGroups } from '@kbn/aiops-test-utils/artificial_logs/final_significant_item_groups';
+import { significantTerms } from '@kbn/aiops-test-utils/artificial_logs/significant_terms';
 
 import { getGroupTableItems } from './get_group_table_items';
 import { getTableItemAsKQL } from './get_table_item_as_kql';

x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/use_copy_to_clipboard_action.test.tsx

@@ -12,8 +12,8 @@ import { renderHook } from '@testing-library/react-hooks';
 
 import type { SignificantItem } from '@kbn/ml-agg-utils';
 
-import { finalSignificantItemGroups } from '../../../common/__mocks__/artificial_logs/final_significant_item_groups';
-import { significantTerms } from '../../../common/__mocks__/artificial_logs/significant_terms';
+import { finalSignificantItemGroups } from '@kbn/aiops-test-utils/artificial_logs/final_significant_item_groups';
+import { significantTerms } from '@kbn/aiops-test-utils/artificial_logs/significant_terms';
 
 import { getGroupTableItems } from './get_group_table_items';
 import { useCopyToClipboardAction } from './use_copy_to_clipboard_action';

x-pack/plugins/aiops/public/hooks/use_aiops_app_context.ts

@@ -14,6 +14,7 @@ import type { ChartsPluginStart } from '@kbn/charts-plugin/public';
 import type { FieldFormatsStart } from '@kbn/field-formats-plugin/public';
 import type { SharePluginStart } from '@kbn/share-plugin/public';
 import type {
+  AnalyticsServiceStart,
   CoreSetup,
   CoreStart,
   ExecutionContextStart,
@@ -39,6 +40,10 @@ import type { UiActionsStart } from '@kbn/ui-actions-plugin/public';
  * AIOps App Dependencies to be provided via React context.
  */
 export interface AiopsAppDependencies {
+  /**
+   * Used for telemetry/performance metrics.
+   */
+  analytics: AnalyticsServiceStart;
   /**
    * Used to check capabilities for links to other plugins.
    * `application.currentAppId$` is used to close the log pattern analysis flyout

x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_frequent_item_sets.test.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { significantTerms } from '../../../../common/__mocks__/artificial_logs/significant_terms';
+import { significantTerms } from '@kbn/aiops-test-utils/artificial_logs/significant_terms';
 
 import { getShouldClauses, getFrequentItemSetsAggFields } from './fetch_frequent_item_sets';
 

x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_frequent_item_sets.ts

@@ -14,13 +14,13 @@ import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
 import type { Logger } from '@kbn/logging';
 import { type SignificantItem } from '@kbn/ml-agg-utils';
 import { createRandomSamplerWrapper } from '@kbn/ml-random-sampler-utils';
-
-import { RANDOM_SAMPLER_SEED, LOG_RATE_ANALYSIS_SETTINGS } from '../../../../common/constants';
 import type {
   SignificantItemDuplicateGroup,
   ItemSet,
   FetchFrequentItemSetsResponse,
-} from '../../../../common/types';
+} from '@kbn/aiops-utils/types';
+
+import { RANDOM_SAMPLER_SEED, LOG_RATE_ANALYSIS_SETTINGS } from '../../../../common/constants';
 
 interface FrequentItemSetsAggregation extends estypes.AggregationsSamplerAggregation {
   fi: {

x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_terms_2_categories_counts.ts

@@ -12,10 +12,10 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
 import type { Logger } from '@kbn/logging';
 import type { FieldValuePair, SignificantItem } from '@kbn/ml-agg-utils';
+import type { FetchFrequentItemSetsResponse, ItemSet } from '@kbn/aiops-utils/types';
 import { isPopulatedObject } from '@kbn/ml-is-populated-object';
 
 import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis/schema';
-import type { FetchFrequentItemSetsResponse, ItemSet } from '../../../../common/types';
 import { getCategoryQuery } from '../../../../common/api/log_categorization/get_category_query';
 import type { Category } from '../../../../common/api/log_categorization/types';
 import { LOG_RATE_ANALYSIS_SETTINGS } from '../../../../common/constants';

x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_field_value_pair_counts.test.ts

@@ -5,10 +5,10 @@
  * 2.0.
  */
 
-import { significantItemGroups } from '../../../../common/__mocks__/farequote/significant_item_groups';
-import { fields } from '../../../../common/__mocks__/artificial_logs/fields';
-import { filteredFrequentItemSets } from '../../../../common/__mocks__/artificial_logs/filtered_frequent_item_sets';
-import { significantTerms } from '../../../../common/__mocks__/artificial_logs/significant_terms';
+import { significantItemGroups } from '@kbn/aiops-test-utils/farequote/significant_item_groups';
+import { fields } from '@kbn/aiops-test-utils/artificial_logs/fields';
+import { filteredFrequentItemSets } from '@kbn/aiops-test-utils/artificial_logs/filtered_frequent_item_sets';
+import { significantTerms } from '@kbn/aiops-test-utils/artificial_logs/significant_terms';
 
 import { getFieldValuePairCounts } from './get_field_value_pair_counts';
 import { getSimpleHierarchicalTree } from './get_simple_hierarchical_tree';

x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_field_value_pair_counts.ts

@@ -6,8 +6,7 @@
  */
 
 import type { SignificantItemGroup } from '@kbn/ml-agg-utils';
-
-import type { FieldValuePairCounts } from '../../../../common/types';
+import type { FieldValuePairCounts } from '@kbn/aiops-utils/types';
 
 /**
  * Get a nested record of field/value pairs with counts

x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_group_filter.test.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { finalSignificantItemGroups } from '../../../../common/__mocks__/artificial_logs/final_significant_item_groups';
+import { finalSignificantItemGroups } from '@kbn/aiops-test-utils/artificial_logs/final_significant_item_groups';
 
 import { getGroupFilter } from './get_group_filter';
 

x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_groups_with_readded_duplicates.test.ts

@@ -5,8 +5,8 @@
  * 2.0.
  */
 
-import { significantItemGroups } from '../../../../common/__mocks__/artificial_logs/significant_item_groups';
-import { significantTerms } from '../../../../common/__mocks__/artificial_logs/significant_terms';
+import { significantItemGroups } from '@kbn/aiops-test-utils/artificial_logs/significant_item_groups';
+import { significantTerms } from '@kbn/aiops-test-utils/artificial_logs/significant_terms';
 
 import { duplicateIdentifier } from './duplicate_identifier';
 import { getGroupsWithReaddedDuplicates } from './get_groups_with_readded_duplicates';

x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_groups_with_readded_duplicates.ts

@@ -8,8 +8,7 @@
 import { uniqWith, isEqual } from 'lodash';
 
 import type { SignificantItemGroup } from '@kbn/ml-agg-utils';
-
-import type { SignificantItemDuplicateGroup } from '../../../../common/types';
+import type { SignificantItemDuplicateGroup } from '@kbn/aiops-utils/types';
 
 export function getGroupsWithReaddedDuplicates(
   groups: SignificantItemGroup[],

x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_marked_duplicates.test.ts

@@ -5,10 +5,10 @@
  * 2.0.
  */
 
-import { significantItemGroups } from '../../../../common/__mocks__/farequote/significant_item_groups';
-import { fields } from '../../../../common/__mocks__/artificial_logs/fields';
-import { filteredFrequentItemSets } from '../../../../common/__mocks__/artificial_logs/filtered_frequent_item_sets';
-import { significantTerms } from '../../../../common/__mocks__/artificial_logs/significant_terms';
+import { significantItemGroups } from '@kbn/aiops-test-utils/farequote/significant_item_groups';
+import { fields } from '@kbn/aiops-test-utils/artificial_logs/fields';
+import { filteredFrequentItemSets } from '@kbn/aiops-test-utils/artificial_logs/filtered_frequent_item_sets';
+import { significantTerms } from '@kbn/aiops-test-utils/artificial_logs/significant_terms';
 
 import { getFieldValuePairCounts } from './get_field_value_pair_counts';
 import { getMarkedDuplicates } from './get_marked_duplicates';

x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_marked_duplicates.ts

@@ -6,8 +6,7 @@
  */
 
 import type { SignificantItemGroup } from '@kbn/ml-agg-utils';
-
-import type { FieldValuePairCounts } from '../../../../common/types';
+import type { FieldValuePairCounts } from '@kbn/aiops-utils/types';
 
 /**
  * Analyse duplicate field/value pairs in groups.

x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_missing_significant_items.test.ts

@@ -5,8 +5,8 @@
  * 2.0.
  */
 
-import { significantItemGroups } from '../../../../common/__mocks__/artificial_logs/significant_item_groups';
-import { significantTerms } from '../../../../common/__mocks__/artificial_logs/significant_terms';
+import { significantItemGroups } from '@kbn/aiops-test-utils/artificial_logs/significant_item_groups';
+import { significantTerms } from '@kbn/aiops-test-utils/artificial_logs/significant_terms';
 
 import { duplicateIdentifier } from './duplicate_identifier';
 import { getGroupsWithReaddedDuplicates } from './get_groups_with_readded_duplicates';

x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_significant_item_groups.test.ts

@@ -7,10 +7,10 @@
 
 import { orderBy } from 'lodash';
 
-import { fields } from '../../../../common/__mocks__/artificial_logs/fields';
-import { frequentItemSets } from '../../../../common/__mocks__/artificial_logs/frequent_item_sets';
-import { significantTerms } from '../../../../common/__mocks__/artificial_logs/significant_terms';
-import { finalSignificantItemGroups } from '../../../../common/__mocks__/artificial_logs/final_significant_item_groups';
+import { fields } from '@kbn/aiops-test-utils/artificial_logs/fields';
+import { frequentItemSets } from '@kbn/aiops-test-utils/artificial_logs/frequent_item_sets';
+import { significantTerms } from '@kbn/aiops-test-utils/artificial_logs/significant_terms';
+import { finalSignificantItemGroups } from '@kbn/aiops-test-utils/artificial_logs/final_significant_item_groups';
 
 import { getSignificantItemGroups } from './get_significant_item_groups';
 

x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_significant_item_groups.ts

@@ -8,6 +8,7 @@
 import { uniqBy } from 'lodash';
 
 import type { SignificantItem, SignificantItemGroup } from '@kbn/ml-agg-utils';
+import type { ItemSet } from '@kbn/aiops-utils/types';
 
 import { duplicateIdentifier } from './duplicate_identifier';
 import { groupDuplicates } from './fetch_frequent_item_sets';
@@ -17,7 +18,6 @@ import { getSimpleHierarchicalTree } from './get_simple_hierarchical_tree';
 import { getSimpleHierarchicalTreeLeaves } from './get_simple_hierarchical_tree_leaves';
 import { getMissingSignificantItems } from './get_missing_significant_items';
 import { transformSignificantItemToGroup } from './transform_significant_item_to_group';
-import type { ItemSet } from '../../../../common/types';
 
 export function getSignificantItemGroups(
   itemsets: ItemSet[],

x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_simple_hierarchical_tree.test.ts

@@ -5,9 +5,9 @@
  * 2.0.
  */
 
-import { fields } from '../../../../common/__mocks__/artificial_logs/fields';
-import { filteredFrequentItemSets } from '../../../../common/__mocks__/artificial_logs/filtered_frequent_item_sets';
-import { significantTerms } from '../../../../common/__mocks__/artificial_logs/significant_terms';
+import { fields } from '@kbn/aiops-test-utils/artificial_logs/fields';
+import { filteredFrequentItemSets } from '@kbn/aiops-test-utils/artificial_logs/filtered_frequent_item_sets';
+import { significantTerms } from '@kbn/aiops-test-utils/artificial_logs/significant_terms';
 
 import { getSimpleHierarchicalTree } from './get_simple_hierarchical_tree';
 

x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_simple_hierarchical_tree.ts

@@ -6,8 +6,7 @@
  */
 
 import type { SignificantItem } from '@kbn/ml-agg-utils';
-
-import type { ItemSet, SimpleHierarchicalTreeNode } from '../../../../common/types';
+import type { ItemSet, SimpleHierarchicalTreeNode } from '@kbn/aiops-utils/types';
 
 import { getValueCounts } from './get_value_counts';
 import { getValuesDescending } from './get_values_descending';

x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_simple_hierarchical_tree_leaves.test.ts

@@ -5,9 +5,9 @@
  * 2.0.
  */
 
-import { fields } from '../../../../common/__mocks__/artificial_logs/fields';
-import { filteredFrequentItemSets } from '../../../../common/__mocks__/artificial_logs/filtered_frequent_item_sets';
-import { significantTerms } from '../../../../common/__mocks__/artificial_logs/significant_terms';
+import { fields } from '@kbn/aiops-test-utils/artificial_logs/fields';
+import { filteredFrequentItemSets } from '@kbn/aiops-test-utils/artificial_logs/filtered_frequent_item_sets';
+import { significantTerms } from '@kbn/aiops-test-utils/artificial_logs/significant_terms';
 
 import { getSimpleHierarchicalTree } from './get_simple_hierarchical_tree';
 import { getSimpleHierarchicalTreeLeaves } from './get_simple_hierarchical_tree_leaves';

x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_simple_hierarchical_tree_leaves.ts

@@ -8,8 +8,7 @@
 import { orderBy } from 'lodash';
 import type { SignificantItemGroup } from '@kbn/ml-agg-utils';
 import { stringHash } from '@kbn/ml-string-hash';
-
-import type { SimpleHierarchicalTreeNode } from '../../../../common/types';
+import type { SimpleHierarchicalTreeNode } from '@kbn/aiops-utils/types';
 
 /**
  * Get leaves from hierarchical tree.

x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_value_counts.test.ts

@@ -5,7 +5,8 @@
  * 2.0.
  */
 
-import { frequentItemSets } from '../../../../common/__mocks__/artificial_logs/frequent_item_sets';
+import { frequentItemSets } from '@kbn/aiops-test-utils/artificial_logs/frequent_item_sets';
+
 import { getValueCounts } from './get_value_counts';
 
 describe('getValueCounts', () => {

x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_value_counts.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import type { ItemSet } from '../../../../common/types';
+import type { ItemSet } from '@kbn/aiops-utils/types';
 
 export function getValueCounts(df: ItemSet[], field: string) {
   return df.reduce<Record<string, number>>((p, c) => {

x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_values_descending.test.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { frequentItemSets } from '../../../../common/__mocks__/artificial_logs/frequent_item_sets';
+import { frequentItemSets } from '@kbn/aiops-test-utils/artificial_logs/frequent_item_sets';
 import { getValuesDescending } from './get_values_descending';
 
 describe('getValuesDescending', () => {

x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_values_descending.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import type { ItemSet } from '../../../../common/types';
+import type { ItemSet } from '@kbn/aiops-utils/types';
 
 import { getValueCounts } from './get_value_counts';
 

x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/transform_significant_item_to_group.test.ts

@@ -5,8 +5,8 @@
  * 2.0.
  */
 
-import { significantItemGroups } from '../../../../common/__mocks__/artificial_logs/significant_item_groups';
-import { significantTerms } from '../../../../common/__mocks__/artificial_logs/significant_terms';
+import { significantItemGroups } from '@kbn/aiops-test-utils/artificial_logs/significant_item_groups';
+import { significantTerms } from '@kbn/aiops-test-utils/artificial_logs/significant_terms';
 
 import { duplicateIdentifier } from './duplicate_identifier';
 import { getGroupsWithReaddedDuplicates } from './get_groups_with_readded_duplicates';

x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/transform_significant_item_to_group.ts

@@ -7,8 +7,7 @@
 
 import { stringHash } from '@kbn/ml-string-hash';
 import type { SignificantItem, SignificantItemGroup } from '@kbn/ml-agg-utils';
-
-import type { SignificantItemDuplicateGroup } from '../../../../common/types';
+import type { SignificantItemDuplicateGroup } from '@kbn/aiops-utils/types';
 
 export function transformSignificantItemToGroup(
   significantItem: SignificantItem,

x-pack/plugins/aiops/tsconfig.json

@@ -68,6 +68,8 @@
     "@kbn/ml-ui-actions",
     "@kbn/core-http-server",
     "@kbn/core-ui-settings-browser-mocks",
+    "@kbn/ebt-tools",
+    "@kbn/aiops-test-utils",
   ],
   "exclude": [
     "target/**/*",

x-pack/plugins/ml/public/application/aiops/change_point_detection.tsx

@@ -51,6 +51,7 @@ export const ChangePointDetectionPage: FC = () => {
           showFrozenDataTierChoice={showNodeInfo}
           appDependencies={{
             ...pick(services, [
+              'analytics',
               'application',
               'cases',
               'charts',

x-pack/plugins/ml/public/application/aiops/log_categorization.tsx

@@ -45,6 +45,7 @@ export const LogCategorizationPage: FC = () => {
           savedSearch={savedSearch}
           showFrozenDataTierChoice={showNodeInfo}
           appDependencies={pick(services, [
+            'analytics',
             'application',
             'charts',
             'data',

x-pack/plugins/ml/public/application/aiops/log_rate_analysis.tsx

@@ -39,6 +39,7 @@ export const LogRateAnalysisPage: FC = () => {
           savedSearch={savedSearch}
           showFrozenDataTierChoice={showNodeInfo}
           appDependencies={pick(services, [
+            'analytics',
             'application',
             'charts',
             'data',

x-pack/plugins/observability_solution/infra/public/alerting/log_threshold/components/alert_details_app_section/components/log_rate_analysis.tsx

@@ -263,6 +263,7 @@ export const LogRateAnalysis: FC<AlertDetailsLogRateAnalysisSectionProps> = ({ r
             barHighlightColorOverride={colorTransformer(Color.color1)}
             onAnalysisCompleted={onAnalysisCompleted}
             appDependencies={pick(services, [
+              'analytics',
               'application',
               'data',
               'executionContext',

x-pack/plugins/observability_solution/observability/public/components/custom_threshold/components/alert_details_app_section/log_rate_analysis.tsx

@@ -202,6 +202,7 @@ export function LogRateAnalysis({
             barHighlightColorOverride={colorTransformer(Color.color1)}
             onAnalysisCompleted={onAnalysisCompleted}
             appDependencies={pick(services, [
+              'analytics',
               'application',
               'data',
               'executionContext',

x-pack/test/api_integration/apis/aiops/test_data.ts

@@ -8,12 +8,12 @@
 // We're using the mocks for jest unit tests as expected data in the integration tests here.
 // This makes sure should the assertions for the integration tests need to be updated,
 // that also the jest unit tests use mocks that are not outdated.
-import { significantTerms as artificialLogSignificantTerms } from '@kbn/aiops-plugin/common/__mocks__/artificial_logs/significant_terms';
-import { significantLogPatterns as artificialLogSignificantLogPatterns } from '@kbn/aiops-plugin/common/__mocks__/artificial_logs/significant_log_patterns';
-import { finalSignificantItemGroups as artificialLogsSignificantItemGroups } from '@kbn/aiops-plugin/common/__mocks__/artificial_logs/final_significant_item_groups';
-import { finalSignificantItemGroupsTextfield as artificialLogsSignificantItemGroupsTextfield } from '@kbn/aiops-plugin/common/__mocks__/artificial_logs/final_significant_item_groups_textfield';
-import { topTerms } from '@kbn/aiops-plugin/common/__mocks__/artificial_logs/top_terms';
-import { topTermsGroups } from '@kbn/aiops-plugin/common/__mocks__/artificial_logs/top_terms_groups';
+import { significantTerms as artificialLogSignificantTerms } from '@kbn/aiops-test-utils/artificial_logs/significant_terms';
+import { significantLogPatterns as artificialLogSignificantLogPatterns } from '@kbn/aiops-test-utils/artificial_logs/significant_log_patterns';
+import { finalSignificantItemGroups as artificialLogsSignificantItemGroups } from '@kbn/aiops-test-utils/artificial_logs/final_significant_item_groups';
+import { finalSignificantItemGroupsTextfield as artificialLogsSignificantItemGroupsTextfield } from '@kbn/aiops-test-utils/artificial_logs/final_significant_item_groups_textfield';
+import { topTerms } from '@kbn/aiops-test-utils/artificial_logs/top_terms';
+import { topTermsGroups } from '@kbn/aiops-test-utils/artificial_logs/top_terms_groups';
 import type {
   AiopsLogRateAnalysisSchema,
   AiopsLogRateAnalysisApiVersion as ApiVersion,
@@ -21,7 +21,7 @@ import type {
 import {
   frequentItemSetsLargeArraysGroups,
   frequentItemSetsLargeArraysSignificantItems,
-} from '../../../functional/apps/aiops/log_rate_analysis/test_data/__mocks__/frequent_item_sets_large_arrays';
+} from '@kbn/aiops-test-utils/frequent_item_sets_large_arrays';
 
 import type { TestData } from './types';
 

x-pack/test/functional/es_archives/large_arrays/mappings.json

@@ -0,0 +1,23 @@
+{
+  "type": "index",
+  "value": {
+    "index": "large_arrays",
+    "mappings": {
+      "properties": {
+        "@timestamp": {
+          "type": "date"
+        },
+        "items": {
+          "type": "keyword"
+        }
+      }
+    },
+    "settings": {
+      "index": {
+        "auto_expand_replicas": "0-1",
+        "number_of_replicas": "0",
+        "number_of_shards": "1"
+      }
+    }
+  }
+}

x-pack/test/functional/services/aiops/log_rate_analysis_data_generator.ts

@@ -11,8 +11,6 @@ import { LOG_RATE_ANALYSIS_TYPE } from '@kbn/aiops-utils';
 
 import { FtrProviderContext } from '../../ftr_provider_context';
 
-import { frequentItemSetsLargeArraysSource } from '../../apps/aiops/log_rate_analysis/test_data/__mocks__/frequent_item_sets_large_arrays';
-
 const LOG_RATE_ANALYSYS_DATA_GENERATOR = {
   KIBANA_SAMPLE_DATA_LOGS: 'kibana_sample_data_logs',
   FAREQUOTE_WITH_SPIKE: 'farequote_with_spike',
@@ -324,44 +322,7 @@ export function LogRateAnalysisDataGeneratorProvider({ getService }: FtrProvider
           break;
 
         case 'large_arrays':
-          try {
-            await es.indices.delete({
-              index: dataGenerator,
-              ignore_unavailable: true,
-            });
-          } catch (e) {
-            log.info(`Could not delete index '${dataGenerator}' in before() callback`);
-          }
-
-          // Create index with mapping
-          await es.indices.create({
-            index: dataGenerator,
-            mappings: {
-              properties: {
-                items: { type: 'keyword' },
-                '@timestamp': { type: 'date' },
-              },
-            },
-          });
-
-          interface DocWithArray {
-            '@timestamp': number;
-            items: string[];
-          }
-
-          await es.bulk({
-            refresh: 'wait_for',
-            body: frequentItemSetsLargeArraysSource.reduce((docs, items) => {
-              if (docs === undefined) return [];
-              docs.push({ index: { _index: dataGenerator } });
-              docs.push({
-                '@timestamp': 1562254538700,
-                items,
-              });
-              return docs;
-            }, [] as estypes.BulkRequest<DocWithArray, DocWithArray>['body']),
-          });
-
+          await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/large_arrays');
           break;
 
         default:
@@ -389,7 +350,6 @@ export function LogRateAnalysisDataGeneratorProvider({ getService }: FtrProvider
         case 'artificial_logs_with_spike_textfield_zerodocsfallback':
         case 'artificial_logs_with_dip_zerodocsfallback':
         case 'artificial_logs_with_dip_textfield_zerodocsfallback':
-        case 'large_arrays':
           try {
             await es.indices.delete({
               index: dataGenerator,
@@ -399,6 +359,10 @@ export function LogRateAnalysisDataGeneratorProvider({ getService }: FtrProvider
           }
           break;
 
+        case 'large_arrays':
+          await esArchiver.unload('x-pack/test/functional/es_archives/large_arrays');
+          break;
+
         default:
           log.error(`Unsupported data generator '${dataGenerator}`);
       }

x-pack/test/tsconfig.json

@@ -167,6 +167,7 @@
     "@kbn/typed-react-router-config",
     "@kbn/ftr-common-functional-ui-services",
     "@kbn/infra-forge",
+    "@kbn/aiops-test-utils",
     "@kbn/observability-ai-assistant-app-plugin",
   ]
 }

yarn.lock

@@ -3097,6 +3097,10 @@
   version "0.0.0"
   uid ""
 
+"@kbn/aiops-test-utils@link:x-pack/packages/ml/aiops_test_utils":
+  version "0.0.0"
+  uid ""
+
 "@kbn/aiops-utils@link:x-pack/packages/ml/aiops_utils":
   version "0.0.0"
   uid ""