mirror of
https://github.com/elastic/kibana.git
synced 2025-04-23 09:19:04 -04:00
[ML] AIOps: Tweak function API for fetchTopCategories/fetchTopTerms (#189863)
## Summary Follow up to #187669. Part of #187684. Fixes #176387. (Ran the flaky test runner on AIOps functional tests) - Fixes the `size: 0` option to be properly nested for `createCategoryRequest()`. - Changes the arguments structure for `fetchTopCategories` and `fetchTopTerms` from individual arguments to an options object to be more in line with the other functions used for log rate analysis. - Adds jest unit test for `fetchTopCategories` and `fetchTopTerms`. ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [x] [Flaky Test Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was used on any tests changed - [x] This was checked for breaking API changes and was [labeled appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)
This commit is contained in:
Walter Rafelsberger
GitHub
parent
d3d5a7c7fe
commit
0a01fdd591
13 changed files with 1873 additions and 41 deletions
|
|
@ -112,11 +112,11 @@ export function createCategoryRequest(
|
|||
return {
|
||||
params: {
|
||||
index,
|
||||
size: 0,
|
||||
body: {
|
||||
query,
|
||||
aggs: wrap(aggs),
|
||||
...(isPopulatedObject(runtimeMappings) ? { runtime_mappings: runtimeMappings } : {}),
|
||||
size: 0,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
|
|
|||
|
|
@ -0,0 +1,37 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
export const topCategoriesResultMock = [
|
||||
{
|
||||
bg_count: 0,
|
||||
doc_count: 1642,
|
||||
fieldName: 'message',
|
||||
fieldValue:
|
||||
'71.231.222.196 - - [2018-08-13T05:04:08.731Z] "GET /kibana/kibana-6.3.2-windows-x86_64.zip HTTP/1.1" 200 15139 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:6.0a1) Gecko/20110421 Firefox/6.0a1"',
|
||||
key: 'GET HTTP/1.1 Mozilla/5.0 X11 Linux x86_64 rv Gecko/20110421 Firefox/6.0a1',
|
||||
normalizedScore: 0,
|
||||
pValue: 1,
|
||||
score: 0,
|
||||
total_bg_count: 0,
|
||||
total_doc_count: 0,
|
||||
type: 'log_pattern',
|
||||
},
|
||||
{
|
||||
bg_count: 0,
|
||||
doc_count: 1488,
|
||||
fieldName: 'message',
|
||||
fieldValue:
|
||||
'7.210.210.41 - - [2018-08-13T04:20:49.558Z] "GET /elasticsearch/elasticsearch-6.3.2.deb HTTP/1.1" 404 6699 "-" "Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.50 Safari/534.24"',
|
||||
key: 'GET HTTP/1.1 Mozilla/5.0 X11 Linux i686 AppleWebKit/534.24 KHTML like Gecko Chrome/11.0.696.50 Safari/534.24',
|
||||
normalizedScore: 0,
|
||||
pValue: 1,
|
||||
score: 0,
|
||||
total_bg_count: 0,
|
||||
total_doc_count: 0,
|
||||
type: 'log_pattern',
|
||||
},
|
||||
];
|
||||
|
|
@ -0,0 +1,75 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
export const topCategoriesSearchResponseMock = {
|
||||
took: 98,
|
||||
responses: [
|
||||
{
|
||||
took: 98,
|
||||
timed_out: false,
|
||||
_shards: { total: 1, successful: 1, skipped: 0, failed: 0 },
|
||||
hits: { total: { value: 4413, relation: 'eq' }, max_score: null, hits: [] },
|
||||
aggregations: {
|
||||
categories: {
|
||||
buckets: [
|
||||
{
|
||||
doc_count: 1642,
|
||||
key: 'GET HTTP/1.1 Mozilla/5.0 X11 Linux x86_64 rv Gecko/20110421 Firefox/6.0a1',
|
||||
regex:
|
||||
'.*?GET.+?HTTP/1\\.1.+?Mozilla/5\\.0.+?X11.+?Linux.+?x86_64.+?rv.+?Gecko/20110421.+?Firefox/6\\.0a1.*?',
|
||||
max_matching_length: 233,
|
||||
examples: {
|
||||
hits: {
|
||||
total: { value: 1642, relation: 'eq' },
|
||||
max_score: null,
|
||||
hits: [
|
||||
{
|
||||
_index: '.ds-kibana_sample_data_logs-2024.07.08-000001',
|
||||
_id: 'zpkLk5AB4oRN3GwDmOW1',
|
||||
_score: null,
|
||||
_source: {
|
||||
message:
|
||||
'71.231.222.196 - - [2018-08-13T05:04:08.731Z] "GET /kibana/kibana-6.3.2-windows-x86_64.zip HTTP/1.1" 200 15139 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:6.0a1) Gecko/20110421 Firefox/6.0a1"',
|
||||
},
|
||||
sort: [1721624648731],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
doc_count: 1488,
|
||||
key: 'GET HTTP/1.1 Mozilla/5.0 X11 Linux i686 AppleWebKit/534.24 KHTML like Gecko Chrome/11.0.696.50 Safari/534.24',
|
||||
regex:
|
||||
'.*?GET.+?HTTP/1\\.1.+?Mozilla/5\\.0.+?X11.+?Linux.+?i686.+?AppleWebKit/534\\.24.+?KHTML.+?like.+?Gecko.+?Chrome/11\\.0\\.696\\.50.+?Safari/534\\.24.*?',
|
||||
max_matching_length: 266,
|
||||
examples: {
|
||||
hits: {
|
||||
total: { value: 1488, relation: 'eq' },
|
||||
max_score: null,
|
||||
hits: [
|
||||
{
|
||||
_index: '.ds-kibana_sample_data_logs-2024.07.08-000001',
|
||||
_id: 'VpkLk5AB4oRN3GwDmOW1',
|
||||
_score: null,
|
||||
_source: {
|
||||
message:
|
||||
'7.210.210.41 - - [2018-08-13T04:20:49.558Z] "GET /elasticsearch/elasticsearch-6.3.2.deb HTTP/1.1" 404 6699 "-" "Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.50 Safari/534.24"',
|
||||
},
|
||||
sort: [1721622049558],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
status: 200,
|
||||
},
|
||||
],
|
||||
};
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -0,0 +1,232 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
export const topTermsSearchResponseMock = {
|
||||
took: 8,
|
||||
timed_out: false,
|
||||
_shards: { total: 1, successful: 1, skipped: 0, failed: 0 },
|
||||
hits: { total: { value: 329, relation: 'eq' }, max_score: null, hits: [] },
|
||||
aggregations: {
|
||||
top_terms_0: {
|
||||
doc_count_error_upper_bound: 0,
|
||||
sum_other_doc_count: 0,
|
||||
buckets: [
|
||||
{
|
||||
key: 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.50 Safari/534.24',
|
||||
doc_count: 179,
|
||||
},
|
||||
{
|
||||
key: 'Mozilla/5.0 (X11; Linux x86_64; rv:6.0a1) Gecko/20110421 Firefox/6.0a1',
|
||||
doc_count: 87,
|
||||
},
|
||||
{
|
||||
key: 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)',
|
||||
doc_count: 63,
|
||||
},
|
||||
],
|
||||
},
|
||||
top_terms_1: {
|
||||
doc_count_error_upper_bound: 0,
|
||||
sum_other_doc_count: 207,
|
||||
buckets: [
|
||||
{ key: '30.156.16.164', doc_count: 100 },
|
||||
{ key: '107.152.89.90', doc_count: 3 },
|
||||
{ key: '112.106.69.227', doc_count: 3 },
|
||||
{ key: '160.20.100.193', doc_count: 3 },
|
||||
{ key: '186.153.168.71', doc_count: 3 },
|
||||
{ key: '16.241.165.21', doc_count: 2 },
|
||||
{ key: '20.129.3.8', doc_count: 2 },
|
||||
{ key: '24.42.142.201', doc_count: 2 },
|
||||
{ key: '43.86.71.5', doc_count: 2 },
|
||||
{ key: '50.184.59.162', doc_count: 2 },
|
||||
],
|
||||
},
|
||||
top_terms_2: {
|
||||
doc_count_error_upper_bound: 0,
|
||||
sum_other_doc_count: 0,
|
||||
buckets: [{ key: 'sample_web_logs', doc_count: 329 }],
|
||||
},
|
||||
top_terms_3: {
|
||||
doc_count_error_upper_bound: 0,
|
||||
sum_other_doc_count: 0,
|
||||
buckets: [
|
||||
{ key: '', doc_count: 196 },
|
||||
{ key: 'gz', doc_count: 42 },
|
||||
{ key: 'zip', doc_count: 28 },
|
||||
{ key: 'css', doc_count: 27 },
|
||||
{ key: 'deb', doc_count: 26 },
|
||||
{ key: 'rpm', doc_count: 10 },
|
||||
],
|
||||
},
|
||||
top_terms_4: {
|
||||
doc_count_error_upper_bound: 0,
|
||||
sum_other_doc_count: 95,
|
||||
buckets: [
|
||||
{ key: 'IN', doc_count: 135 },
|
||||
{ key: 'CN', doc_count: 38 },
|
||||
{ key: 'US', doc_count: 18 },
|
||||
{ key: 'ID', doc_count: 10 },
|
||||
{ key: 'BD', doc_count: 7 },
|
||||
{ key: 'BR', doc_count: 7 },
|
||||
{ key: 'NG', doc_count: 7 },
|
||||
{ key: 'AR', doc_count: 4 },
|
||||
{ key: 'DE', doc_count: 4 },
|
||||
{ key: 'ET', doc_count: 4 },
|
||||
],
|
||||
},
|
||||
top_terms_5: {
|
||||
doc_count_error_upper_bound: 0,
|
||||
sum_other_doc_count: 0,
|
||||
buckets: [{ key: 'US', doc_count: 329 }],
|
||||
},
|
||||
top_terms_6: {
|
||||
doc_count_error_upper_bound: 0,
|
||||
sum_other_doc_count: 95,
|
||||
buckets: [
|
||||
{ key: 'US:IN', doc_count: 135 },
|
||||
{ key: 'US:CN', doc_count: 38 },
|
||||
{ key: 'US:US', doc_count: 18 },
|
||||
{ key: 'US:ID', doc_count: 10 },
|
||||
{ key: 'US:BD', doc_count: 7 },
|
||||
{ key: 'US:BR', doc_count: 7 },
|
||||
{ key: 'US:NG', doc_count: 7 },
|
||||
{ key: 'US:AR', doc_count: 4 },
|
||||
{ key: 'US:DE', doc_count: 4 },
|
||||
{ key: 'US:ET', doc_count: 4 },
|
||||
],
|
||||
},
|
||||
top_terms_7: {
|
||||
doc_count_error_upper_bound: 0,
|
||||
sum_other_doc_count: 0,
|
||||
buckets: [
|
||||
{ key: 'elastic-elastic-elastic.org', doc_count: 112 },
|
||||
{ key: 'artifacts.elastic.co', doc_count: 106 },
|
||||
{ key: 'www.elastic.co', doc_count: 84 },
|
||||
{ key: 'cdn.elastic-elastic-elastic.org', doc_count: 27 },
|
||||
],
|
||||
},
|
||||
top_terms_8: {
|
||||
doc_count_error_upper_bound: 0,
|
||||
sum_other_doc_count: 0,
|
||||
buckets: [{ key: 'kibana_sample_data_logs', doc_count: 329 }],
|
||||
},
|
||||
top_terms_9: {
|
||||
doc_count_error_upper_bound: 0,
|
||||
sum_other_doc_count: 206,
|
||||
buckets: [
|
||||
{ key: '30.156.16.163', doc_count: 101 },
|
||||
{ key: '107.152.89.90', doc_count: 3 },
|
||||
{ key: '112.106.69.227', doc_count: 3 },
|
||||
{ key: '160.20.100.193', doc_count: 3 },
|
||||
{ key: '186.153.168.71', doc_count: 3 },
|
||||
{ key: '16.241.165.21', doc_count: 2 },
|
||||
{ key: '20.129.3.8', doc_count: 2 },
|
||||
{ key: '24.42.142.201', doc_count: 2 },
|
||||
{ key: '43.86.71.5', doc_count: 2 },
|
||||
{ key: '50.184.59.162', doc_count: 2 },
|
||||
],
|
||||
},
|
||||
top_terms_10: {
|
||||
doc_count_error_upper_bound: 0,
|
||||
sum_other_doc_count: 0,
|
||||
buckets: [
|
||||
{ key: 'win xp', doc_count: 148 },
|
||||
{ key: 'osx', doc_count: 50 },
|
||||
{ key: 'ios', doc_count: 44 },
|
||||
{ key: 'win 7', doc_count: 44 },
|
||||
{ key: 'win 8', doc_count: 43 },
|
||||
],
|
||||
},
|
||||
top_terms_11: {
|
||||
doc_count_error_upper_bound: 0,
|
||||
sum_other_doc_count: 210,
|
||||
buckets: [
|
||||
{ key: 'http://www.elastic-elastic-elastic.com/success/timothy-l-kopra', doc_count: 101 },
|
||||
{ key: 'http://facebook.com/success/daniel-barry', doc_count: 2 },
|
||||
{ key: 'http://facebook.com/success/mark-kelly', doc_count: 2 },
|
||||
{ key: 'http://facebook.com/success/pavel-popovich', doc_count: 2 },
|
||||
{ key: 'http://facebook.com/success/scott-altman', doc_count: 2 },
|
||||
{ key: 'http://twitter.com/success/dafydd-williams', doc_count: 2 },
|
||||
{ key: 'http://twitter.com/success/valentin-lebedev', doc_count: 2 },
|
||||
{ key: 'http://twitter.com/success/viktor-m-afanasyev', doc_count: 2 },
|
||||
{ key: 'http://twitter.com/success/y-ng-l-w-i', doc_count: 2 },
|
||||
{
|
||||
key: 'http://www.elastic-elastic-elastic.com/success/georgi-dobrovolski',
|
||||
doc_count: 2,
|
||||
},
|
||||
],
|
||||
},
|
||||
top_terms_12: {
|
||||
doc_count_error_upper_bound: 0,
|
||||
sum_other_doc_count: 214,
|
||||
buckets: [
|
||||
{ key: '/apm', doc_count: 19 },
|
||||
{ key: '/beats', doc_count: 13 },
|
||||
{ key: '/beats/filebeat', doc_count: 11 },
|
||||
{ key: '/elasticsearch/elasticsearch-6.3.2.tar.gz', doc_count: 11 },
|
||||
{ key: '/kibana/kibana-6.3.2-darwin-x86_64.tar.gz', doc_count: 11 },
|
||||
{ key: '/', doc_count: 10 },
|
||||
{ key: '/apm-server/apm-server-6.3.2-windows-x86.zip', doc_count: 10 },
|
||||
{ key: '/beats/metricbeat', doc_count: 10 },
|
||||
{ key: '/beats/metricbeat/metricbeat-6.3.2-i686.rpm', doc_count: 10 },
|
||||
{ key: '/elasticsearch/elasticsearch-6.3.2.deb', doc_count: 10 },
|
||||
],
|
||||
},
|
||||
top_terms_13: {
|
||||
doc_count_error_upper_bound: 0,
|
||||
sum_other_doc_count: 0,
|
||||
buckets: [
|
||||
{ key: '200', doc_count: 210 },
|
||||
{ key: '404', doc_count: 110 },
|
||||
{ key: '503', doc_count: 9 },
|
||||
],
|
||||
},
|
||||
top_terms_14: {
|
||||
doc_count_error_upper_bound: 0,
|
||||
sum_other_doc_count: 0,
|
||||
buckets: [
|
||||
{ key: 'success', doc_count: 295 },
|
||||
{ key: 'info', doc_count: 279 },
|
||||
{ key: 'security', doc_count: 37 },
|
||||
{ key: 'warning', doc_count: 23 },
|
||||
{ key: 'login', doc_count: 13 },
|
||||
{ key: 'error', doc_count: 11 },
|
||||
],
|
||||
},
|
||||
top_terms_15: {
|
||||
doc_count_error_upper_bound: 0,
|
||||
sum_other_doc_count: 217,
|
||||
buckets: [
|
||||
{ key: 'https://www.elastic.co/downloads/apm', doc_count: 16 },
|
||||
{ key: 'https://www.elastic.co/downloads/beats', doc_count: 13 },
|
||||
{
|
||||
key: 'https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.3.2.tar.gz',
|
||||
doc_count: 11,
|
||||
},
|
||||
{
|
||||
key: 'https://artifacts.elastic.co/downloads/kibana/kibana-6.3.2-darwin-x86_64.tar.gz',
|
||||
doc_count: 11,
|
||||
},
|
||||
{ key: 'https://www.elastic.co/downloads/beats/filebeat', doc_count: 11 },
|
||||
{
|
||||
key: 'https://artifacts.elastic.co/downloads/apm-server/apm-server-6.3.2-windows-x86.zip',
|
||||
doc_count: 10,
|
||||
},
|
||||
{
|
||||
key: 'https://artifacts.elastic.co/downloads/beats/metricbeat/metricbeat-6.3.2-i686.rpm',
|
||||
doc_count: 10,
|
||||
},
|
||||
{
|
||||
key: 'https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.3.2.deb',
|
||||
doc_count: 10,
|
||||
},
|
||||
{ key: 'https://www.elastic.co/downloads/beats/metricbeat', doc_count: 10 },
|
||||
{ key: 'https://www.elastic.co/downloads/enterprise', doc_count: 10 },
|
||||
],
|
||||
},
|
||||
},
|
||||
};
|
||||
|
|
@ -67,7 +67,6 @@ describe('getCategoryRequest', () => {
|
|||
// time range filter whatsoever, for example for start/end (0,50).
|
||||
expect(query).toEqual({
|
||||
index: 'the-index',
|
||||
size: 0,
|
||||
body: {
|
||||
query: {
|
||||
bool: {
|
||||
|
|
@ -117,6 +116,7 @@ describe('getCategoryRequest', () => {
|
|||
},
|
||||
},
|
||||
},
|
||||
size: 0,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -84,6 +84,7 @@ export const getCategoryRequest = (
|
|||
wrap,
|
||||
undefined,
|
||||
undefined,
|
||||
false,
|
||||
false
|
||||
);
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,91 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { paramsMock } from './__mocks__/params_match_all';
|
||||
|
||||
import type { ElasticsearchClient } from '@kbn/core/server';
|
||||
import type { Logger } from '@kbn/logging';
|
||||
|
||||
import { topCategoriesSearchResponseMock } from './__mocks__/top_categories_search_response';
|
||||
import { topCategoriesResultMock } from './__mocks__/top_categories_result';
|
||||
import { fetchTopCategories } from './fetch_top_categories';
|
||||
|
||||
const esClientMock = {
|
||||
msearch: jest.fn().mockImplementation(() => topCategoriesSearchResponseMock),
|
||||
} as unknown as ElasticsearchClient;
|
||||
|
||||
const loggerMock = {} as unknown as Logger;
|
||||
|
||||
describe('fetchTopCategories', () => {
|
||||
afterEach(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
test('should fetch top categories successfully', async () => {
|
||||
const abortSignal = new AbortController().signal;
|
||||
|
||||
const result = await fetchTopCategories({
|
||||
esClient: esClientMock,
|
||||
logger: loggerMock,
|
||||
emitError: jest.fn(),
|
||||
abortSignal,
|
||||
arguments: { ...paramsMock, fieldNames: ['message'] },
|
||||
});
|
||||
expect(result).toEqual(topCategoriesResultMock);
|
||||
expect(esClientMock.msearch).toHaveBeenCalledWith(
|
||||
{
|
||||
searches: [
|
||||
{ index: 'the-index' },
|
||||
{
|
||||
aggs: {
|
||||
categories: {
|
||||
aggs: {
|
||||
examples: {
|
||||
top_hits: { _source: 'message', size: 4, sort: ['the-time-field-name'] },
|
||||
},
|
||||
},
|
||||
categorize_text: { field: 'message', size: 1000 },
|
||||
},
|
||||
},
|
||||
query: {
|
||||
bool: {
|
||||
filter: [
|
||||
{
|
||||
bool: {
|
||||
should: [
|
||||
{
|
||||
range: {
|
||||
'the-time-field-name': {
|
||||
format: 'epoch_millis',
|
||||
gte: 10,
|
||||
lte: 20,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
range: {
|
||||
'the-time-field-name': {
|
||||
format: 'epoch_millis',
|
||||
gte: 30,
|
||||
lte: 40,
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
size: 0,
|
||||
},
|
||||
],
|
||||
},
|
||||
{ maxRetries: 0, signal: abortSignal }
|
||||
);
|
||||
});
|
||||
});
|
||||
|
|
@ -7,24 +7,21 @@
|
|||
|
||||
import { uniq } from 'lodash';
|
||||
|
||||
import type { ElasticsearchClient } from '@kbn/core/server';
|
||||
import type { Logger } from '@kbn/logging';
|
||||
import { type SignificantItem, SIGNIFICANT_ITEM_TYPE } from '@kbn/ml-agg-utils';
|
||||
|
||||
import type { AiopsLogRateAnalysisSchema } from '../api/schema';
|
||||
|
||||
import { fetchCategories } from './fetch_categories';
|
||||
import type { FetchTopOptions } from './fetch_top_types';
|
||||
|
||||
export const fetchTopCategories = async (
|
||||
esClient: ElasticsearchClient,
|
||||
params: AiopsLogRateAnalysisSchema,
|
||||
fieldNames: string[],
|
||||
logger: Logger,
|
||||
export const fetchTopCategories = async ({
|
||||
esClient,
|
||||
abortSignal,
|
||||
emitError,
|
||||
logger,
|
||||
arguments: args,
|
||||
}: FetchTopOptions) => {
|
||||
// The default value of 1 means no sampling will be used
|
||||
sampleProbability: number = 1,
|
||||
emitError: (m: string) => void,
|
||||
abortSignal?: AbortSignal
|
||||
) => {
|
||||
const { fieldNames, sampleProbability = 1, ...params } = args;
|
||||
|
||||
const categoriesOverall = await fetchCategories(
|
||||
esClient,
|
||||
params,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,62 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { paramsMock } from './__mocks__/params_match_all';
|
||||
|
||||
import type { ElasticsearchClient } from '@kbn/core/server';
|
||||
import type { Logger } from '@kbn/logging';
|
||||
|
||||
import { topTermsSearchResponseMock } from './__mocks__/top_terms_search_response';
|
||||
import { topTermsResult } from './__mocks__/top_terms_result';
|
||||
import { fetchTopTerms } from './fetch_top_terms';
|
||||
|
||||
const esClientMock = {
|
||||
search: jest.fn().mockImplementation(() => topTermsSearchResponseMock),
|
||||
} as unknown as ElasticsearchClient;
|
||||
|
||||
const loggerMock = {} as unknown as Logger;
|
||||
|
||||
describe('fetchTopTerms', () => {
|
||||
afterEach(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
test('should fetch top terms successfully', async () => {
|
||||
const abortSignal = new AbortController().signal;
|
||||
|
||||
const result = await fetchTopTerms({
|
||||
esClient: esClientMock,
|
||||
logger: loggerMock,
|
||||
emitError: jest.fn(),
|
||||
abortSignal,
|
||||
arguments: {
|
||||
...paramsMock,
|
||||
fieldNames: [
|
||||
'agent.keyword',
|
||||
'clientip',
|
||||
'event.dataset',
|
||||
'extension.keyword',
|
||||
'geo.dest',
|
||||
'geo.src',
|
||||
'geo.srcdest',
|
||||
'host.keyword',
|
||||
'index.keyword',
|
||||
'ip',
|
||||
'machine.os.keyword',
|
||||
'referer',
|
||||
'request.keyword',
|
||||
'response.keyword',
|
||||
'tags.keyword',
|
||||
'url.keyword',
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
expect(esClientMock.search).toHaveBeenCalledTimes(1);
|
||||
expect(result).toEqual(topTermsResult);
|
||||
});
|
||||
});
|
||||
|
|
@ -5,10 +5,9 @@
|
|||
* 2.0.
|
||||
*/
|
||||
import { uniqBy } from 'lodash';
|
||||
import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
|
||||
import type { ElasticsearchClient } from '@kbn/core/server';
|
||||
|
||||
import type { Logger } from '@kbn/logging';
|
||||
import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
|
||||
|
||||
import { type SignificantItem, SIGNIFICANT_ITEM_TYPE } from '@kbn/ml-agg-utils';
|
||||
import {
|
||||
createRandomSamplerWrapper,
|
||||
|
|
@ -21,6 +20,7 @@ import { LOG_RATE_ANALYSIS_SETTINGS, RANDOM_SAMPLER_SEED } from '../constants';
|
|||
|
||||
import { getQueryWithParams } from './get_query_with_params';
|
||||
import { getRequestBase } from './get_request_base';
|
||||
import type { FetchTopOptions } from './fetch_top_types';
|
||||
|
||||
// TODO Consolidate with duplicate `fetchDurationFieldCandidates` in
|
||||
// `x-pack/plugins/observability_solution/apm/server/routes/correlations/queries/fetch_failed_events_correlation_p_values.ts`
|
||||
|
|
@ -87,16 +87,16 @@ interface Aggs extends estypes.AggregationsLongTermsAggregate {
|
|||
buckets: estypes.AggregationsLongTermsBucket[];
|
||||
}
|
||||
|
||||
export const fetchTopTerms = async (
|
||||
esClient: ElasticsearchClient,
|
||||
params: AiopsLogRateAnalysisSchema,
|
||||
fieldNames: string[],
|
||||
logger: Logger,
|
||||
export const fetchTopTerms = async ({
|
||||
esClient,
|
||||
abortSignal,
|
||||
emitError,
|
||||
logger,
|
||||
arguments: args,
|
||||
}: FetchTopOptions): Promise<SignificantItem[]> => {
|
||||
// The default value of 1 means no sampling will be used
|
||||
sampleProbability: number = 1,
|
||||
emitError: (m: string) => void,
|
||||
abortSignal?: AbortSignal
|
||||
): Promise<SignificantItem[]> => {
|
||||
const { fieldNames, sampleProbability = 1, ...params } = args;
|
||||
|
||||
const randomSamplerWrapper = createRandomSamplerWrapper({
|
||||
probability: sampleProbability,
|
||||
seed: RANDOM_SAMPLER_SEED,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,22 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { ElasticsearchClient } from '@kbn/core/server';
|
||||
import type { Logger } from '@kbn/logging';
|
||||
|
||||
import type { AiopsLogRateAnalysisSchema } from '../api/schema';
|
||||
|
||||
export interface FetchTopOptions {
|
||||
esClient: ElasticsearchClient;
|
||||
logger: Logger;
|
||||
emitError: (m: string) => void;
|
||||
abortSignal?: AbortSignal;
|
||||
arguments: AiopsLogRateAnalysisSchema & {
|
||||
fieldNames: string[];
|
||||
sampleProbability?: number;
|
||||
};
|
||||
}
|
||||
|
|
@ -133,15 +133,17 @@ export const topItemsHandlerFactory =
|
|||
let fetchedTopTerms: Awaited<ReturnType<typeof fetchTopTerms>>;
|
||||
|
||||
try {
|
||||
fetchedTopTerms = await fetchTopTerms(
|
||||
fetchedTopTerms = await fetchTopTerms({
|
||||
esClient,
|
||||
requestBody,
|
||||
fieldNames,
|
||||
logger,
|
||||
stateHandler.sampleProbability(),
|
||||
responseStream.pushError,
|
||||
abortSignal
|
||||
);
|
||||
emitError: responseStream.pushError,
|
||||
abortSignal,
|
||||
arguments: {
|
||||
...requestBody,
|
||||
fieldNames,
|
||||
sampleProbability: stateHandler.sampleProbability(),
|
||||
},
|
||||
});
|
||||
} catch (e) {
|
||||
if (!isRequestAbortedError(e)) {
|
||||
logger.error(
|
||||
|
|
@ -183,15 +185,17 @@ export const topItemsHandlerFactory =
|
|||
} else if (isTextFieldCandidates(payload)) {
|
||||
const { textFieldCandidates: fieldNames } = payload;
|
||||
|
||||
const topCategoriesForField = await await fetchTopCategories(
|
||||
const topCategoriesForField = await fetchTopCategories({
|
||||
esClient,
|
||||
requestBody,
|
||||
fieldNames,
|
||||
logger,
|
||||
stateHandler.sampleProbability(),
|
||||
responseStream.pushError,
|
||||
abortSignal
|
||||
);
|
||||
emitError: responseStream.pushError,
|
||||
abortSignal,
|
||||
arguments: {
|
||||
...requestBody,
|
||||
fieldNames,
|
||||
sampleProbability: stateHandler.sampleProbability(),
|
||||
},
|
||||
});
|
||||
|
||||
if (topCategoriesForField.length > 0) {
|
||||
topCategories.push(...topCategoriesForField);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue