github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 01:38:56 -04:00
Code Issues Projects Releases Packages Wiki Activity

[ML] Adding pattern analysis to anomaly action menu (#169400)

Browse source 
Adds the ability to run pattern analysis on the index used for an
anomaly detection job.
The pattern analysis is run for the whole time range selected in the
page's time picker, but also adds a sub aggregation to focus on the
bucket time range for the selected anomaly.
This allows the user to view either patterns found in the bucket or the
full time range.
If partition field has been used in the detector, this is also added as
a filter in the sub agg, so the patterns shown are only ones where the
doc also matches the partition field value.

A sub agg was used rather than just running the whole analysis on the
bucket to ensure we get a good analysis and good patterns. The more data
we see, the more accurate the patterns.
This way we can find all of the patterns and then find which of those
patterns match the bucket.

The pattern analysis action item is added to the menu if the data view
has a `message` or `error.message` field.


3e8295a0-5c7e-4ba6-b260-13c158d32a29

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
James Gowdy 2023-12-05 15:37:00 +00:00 • committed by GitHub
parent b692493c6f
commit 004632ffc3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
20 changed files with 507 additions and 195 deletions
Download patch file Download diff file Expand all files Collapse all files

21
x-pack/plugins/aiops/common/api/log_categorization/create_categorize_query.ts
View file

@ -12,8 +12,7 @@ import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/type
export function createCategorizeQuery(
queryIn: QueryDslQueryContainer,
timeField: string,
from: number | undefined,
to: number | undefined
timeRange: { from: number; to: number } | undefined
) {
const query = cloneDeep(queryIn);
@ -34,15 +33,17 @@ export function createCategorizeQuery(
delete query.multi_match;
}
(query.bool.must as QueryDslQueryContainer[]).push({
range: {
[timeField]: {
gte: from,
lte: to,
format: 'epoch_millis',
if (timeRange !== undefined) {
(query.bool.must as QueryDslQueryContainer[]).push({
range: {
[timeField]: {
gte: timeRange.from,
lte: timeRange.to,
format: 'epoch_millis',
},
},
},
});
});
}
return query;
}

47
x-pack/plugins/aiops/common/api/log_categorization/create_category_request.ts
View file

@ -17,17 +17,23 @@ import { createCategorizeQuery } from './create_categorize_query';
const CATEGORY_LIMIT = 1000;
const EXAMPLE_LIMIT = 1;
export interface CategorizationAdditionalFilter {
from: number;
to: number;
field?: { name: string; value: string };
}
export function createCategoryRequest(
index: string,
field: string,
timeField: string,
from: number | undefined,
to: number | undefined,
timeRange: { from: number; to: number } | undefined,
queryIn: QueryDslQueryContainer,
wrap: ReturnType<typeof createRandomSamplerWrapper>['wrap'],
intervalMs?: number
intervalMs?: number,
additionalFilter?: CategorizationAdditionalFilter
) {
const query = createCategorizeQuery(queryIn, timeField, from, to);
const query = createCategorizeQuery(queryIn, timeField, timeRange);
const aggs = {
categories: {
categorize_text: {
@ -36,7 +42,7 @@ export function createCategoryRequest(
categorization_analyzer: categorizationAnalyzer,
},
aggs: {
hit: {
examples: {
top_hits: {
size: EXAMPLE_LIMIT,
sort: [timeField],
@ -53,6 +59,37 @@ export function createCategoryRequest(
},
}
: {}),
...(additionalFilter
? {
sub_time_range: {
date_range: {
field: timeField,
format: 'epoch_millis',
ranges: [{ from: additionalFilter.from, to: additionalFilter.to }],
},
aggs: {
examples: {
top_hits: {
size: EXAMPLE_LIMIT,
sort: [timeField],
_source: field,
},
},
...(additionalFilter.field
? {
sub_field: {
filter: {
term: {
[additionalFilter.field.name]: additionalFilter.field.value,
},
},
},
}
: {}),
},
},
}
: {}),
},
},
};

3
x-pack/plugins/aiops/common/api/log_categorization/get_category_query.ts
View file

@ -5,6 +5,7 @@
* 2.0.
*/
import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import type { Category } from './types';
export const QUERY_MODE = {
@ -17,7 +18,7 @@ export const getCategoryQuery = (
field: string,
categories: Category[],
mode: QueryMode = QUERY_MODE.INCLUDE
) => ({
): Record<string, estypes.QueryDslBoolQuery> => ({
bool: {
[mode]: categories.map(({ key: query }) => ({
match: {

14
x-pack/plugins/aiops/common/api/log_categorization/process_category_results.ts
View file

@ -11,14 +11,13 @@ import { estypes } from '@elastic/elasticsearch';
import { createRandomSamplerWrapper } from '@kbn/ml-random-sampler-utils';
import type { Category, CategoriesAgg, CatResponse, SparkLinesPerCategory } from './types';
import type { Category, CategoriesAgg, CatResponse } from './types';
export function processCategoryResults(
result: CatResponse,
field: string,
unwrap: ReturnType<typeof createRandomSamplerWrapper>['unwrap']
) {
const sparkLinesPerCategory: SparkLinesPerCategory = {};
const { aggregations } = result.rawResponse;
if (aggregations === undefined) {
throw new Error('processCategoryResults failed, did not return aggregations.');
@ -30,7 +29,7 @@ export function processCategoryResults(
) as CategoriesAgg;
const categories: Category[] = buckets.map((b) => {
sparkLinesPerCategory[b.key] =
const sparkline =
b.sparkline === undefined
? {}
: b.sparkline.buckets.reduce<Record<number, number>>((acc2, cur2) => {
@ -41,11 +40,16 @@ export function processCategoryResults(
return {
key: b.key,
count: b.doc_count,
examples: b.hit.hits.hits.map((h) => get(h._source, field)),
examples: b.examples.hits.hits.map((h) => get(h._source, field)),
sparkline,
subTimeRangeCount: b.sub_time_range?.buckets[0].doc_count ?? undefined,
subFieldCount: b.sub_time_range?.buckets[0].sub_field?.doc_count ?? undefined,
subFieldExamples:
b.sub_time_range?.buckets[0].examples.hits.hits.map((h) => get(h._source, field)) ??
undefined,
};
});
return {
categories,
sparkLinesPerCategory,
};
}

27
x-pack/plugins/aiops/common/api/log_categorization/types.ts
View file

@ -10,8 +10,15 @@ import { estypes } from '@elastic/elasticsearch';
export interface Category {
key: string;
count: number;
subTimeRangeCount?: number;
subFieldCount?: number;
subFieldExamples?: string[];
examples: string[];
sparkline?: Array<{ doc_count: number; key: number; key_as_string: string }>;
sparkline?: Record<number, number>;
}
interface CategoryExamples {
hits: { hits: Array<{ _source: { message: string } }> };
}
export interface CategoriesAgg {
@ -19,10 +26,24 @@ export interface CategoriesAgg {
buckets: Array<{
key: string;
doc_count: number;
hit: { hits: { hits: Array<{ _source: { message: string } }> } };
examples: CategoryExamples;
sparkline: {
buckets: Array<{ key_as_string: string; key: number; doc_count: number }>;
};
sub_time_range?: {
buckets: Array<{
key: number;
doc_count: number;
to: number;
to_as_string: string;
from: number;
from_as_string: string;
sub_field?: {
doc_count: number;
};
examples: CategoryExamples;
}>;
};
}>;
};
}
@ -34,5 +55,3 @@ interface CategoriesSampleAgg {
export interface CatResponse {
rawResponse: estypes.SearchResponseBody<unknown, CategoriesAgg | CategoriesSampleAgg>;
}
export type SparkLinesPerCategory = Record<string, Record<number, number>>;

4
x-pack/plugins/aiops/public/components/log_categorization/categorize_field_actions.ts
View file

@ -24,7 +24,7 @@ export const createCategorizeFieldAction = (coreStart: CoreStart, plugins: Aiops
return field.esTypes?.includes('text') === true;
},
execute: async (context: CategorizeFieldContext) => {
const { field, dataView, originatingApp } = context;
showCategorizeFlyout(field, dataView, coreStart, plugins, originatingApp);
const { field, dataView, originatingApp, additionalFilter } = context;
showCategorizeFlyout(field, dataView, coreStart, plugins, originatingApp, additionalFilter);
},
});

116
x-pack/plugins/aiops/public/components/log_categorization/category_table/category_table.tsx
View file

@ -24,17 +24,20 @@ import { DataViewField } from '@kbn/data-views-plugin/common';
import { Filter } from '@kbn/es-query';
import { useTableState } from '@kbn/ml-in-memory-table';
import type {
Category,
SparkLinesPerCategory,
} from '../../../../common/api/log_categorization/types';
import moment from 'moment';
import type { CategorizationAdditionalFilter } from '../../../../common/api/log_categorization/create_category_request';
import {
type QueryMode,
QUERY_MODE,
} from '../../../../common/api/log_categorization/get_category_query';
import type { Category } from '../../../../common/api/log_categorization/types';
import { useEuiTheme } from '../../../hooks/use_eui_theme';
import type { LogCategorizationAppState } from '../../../application/url_state/log_pattern_analysis';
import { MiniHistogram } from '../../mini_histogram';
import { useDiscoverLinks, createFilter, QueryMode, QUERY_MODE } from '../use_discover_links';
import { useDiscoverLinks, createFilter } from '../use_discover_links';
import type { EventRate } from '../use_categorize_request';
import { getLabels } from './labels';
@ -42,7 +45,6 @@ import { TableHeader } from './table_header';
interface Props {
categories: Category[];
sparkLines: SparkLinesPerCategory;
eventRate: EventRate;
dataViewId: string;
selectedField: DataViewField | string | undefined;
@ -55,11 +57,12 @@ interface Props {
onAddFilter?: (values: Filter, alias?: string) => void;
onClose?: () => void;
enableRowActions?: boolean;
additionalFilter?: CategorizationAdditionalFilter;
navigateToDiscover?: boolean;
}
export const CategoryTable: FC<Props> = ({
categories,
sparkLines,
eventRate,
dataViewId,
selectedField,
@ -72,6 +75,8 @@ export const CategoryTable: FC<Props> = ({
onAddFilter,
onClose = () => {},
enableRowActions = true,
additionalFilter,
navigateToDiscover = true,
}) => {
const euiTheme = useEuiTheme();
const primaryBackgroundColor = useEuiBackgroundColor('primary');
@ -79,16 +84,21 @@ export const CategoryTable: FC<Props> = ({
const [selectedCategories, setSelectedCategories] = useState<Category[]>([]);
const { onTableChange, pagination, sorting } = useTableState<Category>(categories ?? [], 'key');
const labels = useMemo(
() => getLabels(onAddFilter !== undefined && onClose !== undefined),
[onAddFilter, onClose]
);
const labels = useMemo(() => {
const isFlyout = onAddFilter !== undefined && onClose !== undefined;
return getLabels(isFlyout && navigateToDiscover === false);
}, [navigateToDiscover, onAddFilter, onClose]);
const showSparkline = useMemo(() => {
return categories.some((category) => category.sparkline !== undefined);
}, [categories]);
const openInDiscover = (mode: QueryMode, category?: Category) => {
if (
onAddFilter !== undefined &&
selectedField !== undefined &&
typeof selectedField !== 'string'
typeof selectedField !== 'string' &&
navigateToDiscover === false
) {
onAddFilter(
createFilter('', selectedField.name, selectedCategories, mode, category),
@ -98,7 +108,14 @@ export const CategoryTable: FC<Props> = ({
return;
}
const timefilterActiveBounds = timefilter.getActiveBounds();
const timefilterActiveBounds =
additionalFilter !== undefined
? {
min: moment(additionalFilter.from),
max: moment(additionalFilter.to),
}
: timefilter.getActiveBounds();
if (timefilterActiveBounds === undefined || selectedField === undefined) {
return;
}
@ -110,7 +127,8 @@ export const CategoryTable: FC<Props> = ({
aiopsListState,
timefilterActiveBounds,
mode,
category
category,
additionalFilter?.field
);
};
@ -123,40 +141,6 @@ export const CategoryTable: FC<Props> = ({
sortable: true,
width: '80px',
},
{
field: 'count',
name: i18n.translate('xpack.aiops.logCategorization.column.logRate', {
defaultMessage: 'Log rate',
}),
sortable: false,
width: '100px',
render: (_, { key }) => {
const sparkLine = sparkLines[key];
if (sparkLine === undefined) {
return null;
}
const histogram = eventRate.map(({ key: catKey, docCount }) => {
const term = sparkLine[catKey] ?? 0;
const newTerm = term > docCount ? docCount : term;
const adjustedDocCount = docCount - newTerm;
return {
doc_count_overall: adjustedDocCount,
doc_count_significant_item: newTerm,
key: catKey,
key_as_string: `${catKey}`,
};
});
return (
<MiniHistogram
chartData={histogram}
isLoading={categories === null && histogram === undefined}
label={''}
/>
);
},
},
{
field: 'examples',
name: i18n.translate('xpack.aiops.logCategorization.column.examples', {
@ -202,6 +186,42 @@ export const CategoryTable: FC<Props> = ({
},
] as Array<EuiBasicTableColumn<Category>>;
if (showSparkline === true) {
columns.splice(1, 0, {
field: 'sparkline',
name: i18n.translate('xpack.aiops.logCategorization.column.logRate', {
defaultMessage: 'Log rate',
}),
sortable: false,
width: '100px',
render: (sparkline: Category['sparkline']) => {
if (sparkline === undefined) {
return null;
}
const histogram = eventRate.map(({ key: catKey, docCount }) => {
const term = sparkline[catKey] ?? 0;
const newTerm = term > docCount ? docCount : term;
const adjustedDocCount = docCount - newTerm;
return {
doc_count_overall: adjustedDocCount,
doc_count_significant_item: newTerm,
key: catKey,
key_as_string: `${catKey}`,
};
});
return (
<MiniHistogram
chartData={histogram}
isLoading={categories === null && histogram === undefined}
label={''}
/>
);
},
});
}
const selectionValue: EuiTableSelectionType<Category> | undefined = {
selectable: () => true,
onSelectionChange: (selectedItems) => setSelectedCategories(selectedItems),

5
x-pack/plugins/aiops/public/components/log_categorization/category_table/table_header.tsx
View file

@ -8,9 +8,12 @@
import React, { FC } from 'react';
import { EuiFlexGroup, EuiFlexItem, EuiText, EuiButtonEmpty } from '@elastic/eui';
import { FormattedMessage } from '@kbn/i18n-react';
import {
type QueryMode,
QUERY_MODE,
} from '../../../../common/api/log_categorization/get_category_query';
import { useEuiTheme } from '../../../hooks/use_eui_theme';
import { getLabels } from './labels';
import { QueryMode, QUERY_MODE } from '../use_discover_links';
interface Props {
categoriesCount: number;

17
x-pack/plugins/aiops/public/components/log_categorization/document_count_chart.tsx
View file

@ -10,7 +10,7 @@ import React, { FC, useMemo } from 'react';
import { i18n } from '@kbn/i18n';
import { DocumentCountChart as DocumentCountChartRoot } from '@kbn/aiops-components';
import type { Category, SparkLinesPerCategory } from '../../../common/api/log_categorization/types';
import type { Category } from '../../../common/api/log_categorization/types';
import { useAiopsAppContext } from '../../hooks/use_aiops_app_context';
import { DocumentCountStats } from '../../get_document_stats';
@ -24,13 +24,11 @@ interface Props {
pinnedCategory: Category | null;
selectedCategory: Category | null;
eventRate: EventRate;
sparkLines: SparkLinesPerCategory;
documentCountStats?: DocumentCountStats;
}
export const DocumentCountChart: FC<Props> = ({
eventRate,
sparkLines,
totalCount,
pinnedCategory,
selectedCategory,
@ -48,29 +46,26 @@ export const DocumentCountChart: FC<Props> = ({
const category = selectedCategory ?? pinnedCategory ?? null;
return eventRate.map(({ key, docCount }) => {
let value = docCount;
if (category && sparkLines[category.key] && sparkLines[category.key][key]) {
const val = sparkLines[category.key][key];
if (category && category.sparkline && category.sparkline[key]) {
const val = category.sparkline[key];
value = val > docCount ? 0 : docCount - val;
}
return { time: key, value };
});
}, [eventRate, pinnedCategory, selectedCategory, sparkLines]);
}, [eventRate, pinnedCategory, selectedCategory]);
const chartPointsSplit = useMemo(() => {
const category = selectedCategory ?? pinnedCategory ?? null;
return category !== null
? eventRate.map(({ key, docCount }) => {
const val =
sparkLines && sparkLines[category.key] && sparkLines[category.key][key]
? sparkLines[category.key][key]
: 0;
const val = category.sparkline && category.sparkline[key] ? category.sparkline[key] : 0;
const value = val > docCount ? docCount : val;
return { time: key, value };
})
: undefined;
}, [eventRate, pinnedCategory, selectedCategory, sparkLines]);
}, [eventRate, pinnedCategory, selectedCategory]);
if (documentCountStats?.interval === undefined) {
return null;

206
x-pack/plugins/aiops/public/components/log_categorization/log_categorization_for_flyout.tsx
View file

@ -13,6 +13,11 @@ import {
EuiFlexGroup,
EuiFlexItem,
useEuiTheme,
EuiTabs,
EuiTab,
EuiSpacer,
EuiToolTip,
EuiIcon,
} from '@elastic/eui';
import type { SavedSearch } from '@kbn/saved-search-plugin/public';
@ -22,9 +27,10 @@ import { FormattedMessage } from '@kbn/i18n-react';
import { buildEmptyFilter, Filter } from '@kbn/es-query';
import { usePageUrlState } from '@kbn/ml-url-state';
import type { FieldValidationResults } from '@kbn/ml-category-validator';
import type { CategorizationAdditionalFilter } from '../../../common/api/log_categorization/create_category_request';
import { AIOPS_TELEMETRY_ID } from '../../../common/constants';
import type { Category, SparkLinesPerCategory } from '../../../common/api/log_categorization/types';
import type { Category } from '../../../common/api/log_categorization/types';
import {
type LogCategorizationPageUrlState,
@ -46,6 +52,11 @@ import { useValidateFieldRequest } from './use_validate_category_field';
import { FieldValidationCallout } from './category_validation_callout';
import { CreateCategorizationJobButton } from './create_categorization_job';
enum SELECTED_TAB {
BUCKET,
FULL_TIME_RANGE,
}
export interface LogCategorizationPageProps {
dataView: DataView;
savedSearch: SavedSearch | null;
@ -53,6 +64,7 @@ export interface LogCategorizationPageProps {
onClose: () => void;
/** Identifier to indicate the plugin utilizing the component */
embeddingOrigin: string;
additionalFilter?: CategorizationAdditionalFilter;
}
const BAR_TARGET = 20;
@ -63,6 +75,7 @@ export const LogCategorizationFlyout: FC<LogCategorizationPageProps> = ({
selectedField,
onClose,
embeddingOrigin,
additionalFilter,
}) => {
const {
notifications: { toasts },
@ -96,11 +109,13 @@ export const LogCategorizationFlyout: FC<LogCategorizationPageProps> = ({
const [pinnedCategory, setPinnedCategory] = useState<Category | null>(null);
const [data, setData] = useState<{
categories: Category[];
sparkLines: SparkLinesPerCategory;
categoriesInBucket: Category[] | null;
} | null>(null);
const [fieldValidationResult, setFieldValidationResult] = useState<FieldValidationResults | null>(
null
);
const [showTabs, setShowTabs] = useState<boolean>(false);
const [selectedTab, setSelectedTab] = useState<SELECTED_TAB>(SELECTED_TAB.FULL_TIME_RANGE);
const cancelRequest = useCallback(() => {
cancelValidationRequest();
@ -138,7 +153,12 @@ export const LogCategorizationFlyout: FC<LogCategorizationPageProps> = ({
const { getIndexPattern, timeFieldName: timeField } = dataView;
const index = getIndexPattern();
if (selectedField === undefined || timeField === undefined) {
if (
selectedField === undefined ||
timeField === undefined ||
earliest === undefined ||
latest === undefined
) {
return;
}
@ -148,34 +168,52 @@ export const LogCategorizationFlyout: FC<LogCategorizationPageProps> = ({
setData(null);
setFieldValidationResult(null);
const timeRange = {
from: earliest,
to: latest,
};
try {
const [validationResult, categorizationResult] = await Promise.all([
runValidateFieldRequest(
index,
selectedField.name,
timeField,
earliest,
latest,
searchQuery,
{ [AIOPS_TELEMETRY_ID.AIOPS_ANALYSIS_RUN_ORIGIN]: embeddingOrigin }
),
runValidateFieldRequest(index, selectedField.name, timeField, timeRange, searchQuery, {
[AIOPS_TELEMETRY_ID.AIOPS_ANALYSIS_RUN_ORIGIN]: embeddingOrigin,
}),
runCategorizeRequest(
index,
selectedField.name,
timeField,
earliest,
latest,
timeRange,
searchQuery,
intervalMs
intervalMs,
additionalFilter
),
]);
if (mounted.current === true) {
setFieldValidationResult(validationResult);
const { categories } = categorizationResult;
const hasBucketCategories = categories.some((c) => c.subTimeRangeCount !== undefined);
let categoriesInBucket: any | null = null;
if (additionalFilter !== undefined) {
categoriesInBucket = categorizationResult.categories
.map((category) => ({
...category,
count: category.subFieldCount ?? category.subTimeRangeCount!,
examples: category.subFieldExamples!,
sparkline: undefined,
}))
.filter((category) => category.count > 0)
.sort((a, b) => b.count - a.count);
}
setData({
categories: categorizationResult.categories,
sparkLines: categorizationResult.sparkLinesPerCategory,
categories,
categoriesInBucket,
});
setShowTabs(hasBucketCategories);
setSelectedTab(SELECTED_TAB.BUCKET);
}
} catch (error) {
toasts.addError(error, {
@ -191,15 +229,16 @@ export const LogCategorizationFlyout: FC<LogCategorizationPageProps> = ({
}, [
dataView,
selectedField,
cancelRequest,
runValidateFieldRequest,
earliest,
latest,
cancelRequest,
runValidateFieldRequest,
searchQuery,
embeddingOrigin,
runCategorizeRequest,
intervalMs,
additionalFilter,
toasts,
embeddingOrigin,
]);
const onAddFilter = useCallback(
@ -237,6 +276,8 @@ export const LogCategorizationFlyout: FC<LogCategorizationPageProps> = ({
randomSampler,
]);
const infoIconCss = { marginTop: euiTheme.size.m, marginLeft: euiTheme.size.xxs };
return (
<>
<EuiFlyoutHeader hasBorder>
@ -262,13 +303,15 @@ export const LogCategorizationFlyout: FC<LogCategorizationPageProps> = ({
</EuiFlexGroup>
</EuiFlyoutHeader>
<EuiFlyoutBody data-test-subj="mlJobSelectorFlyoutBody">
<CreateCategorizationJobButton
dataView={dataView}
field={selectedField}
query={searchQuery}
earliest={earliest}
latest={latest}
/>
{showTabs === false && loading === false ? (
<CreateCategorizationJobButton
dataView={dataView}
field={selectedField}
query={searchQuery}
earliest={earliest}
latest={latest}
/>
) : null}
<FieldValidationCallout validationResults={fieldValidationResult} />
{loading === true ? <LoadingCategorization onClose={onClose} /> : null}
<InformationText
@ -278,22 +321,99 @@ export const LogCategorizationFlyout: FC<LogCategorizationPageProps> = ({
fieldSelected={selectedField !== null}
/>
{loading === false && data !== null && data.categories.length > 0 ? (
<CategoryTable
categories={data.categories}
aiopsListState={stateFromUrl}
dataViewId={dataView.id!}
eventRate={eventRate}
sparkLines={data.sparkLines}
selectedField={selectedField}
pinnedCategory={pinnedCategory}
setPinnedCategory={setPinnedCategory}
selectedCategory={selectedCategory}
setSelectedCategory={setSelectedCategory}
timefilter={timefilter}
onAddFilter={onAddFilter}
onClose={onClose}
enableRowActions={false}
/>
<>
{showTabs ? (
<>
<EuiTabs>
<EuiTab
isSelected={selectedTab === SELECTED_TAB.BUCKET}
onClick={() => setSelectedTab(SELECTED_TAB.BUCKET)}
>
<EuiToolTip
content={i18n.translate('xpack.aiops.logCategorization.tabs.bucket.tooltip', {
defaultMessage: 'Patterns that occur in the anomalous bucket.',
})}
>
<EuiFlexGroup gutterSize="none">
<EuiFlexItem>
<FormattedMessage
id="xpack.aiops.logCategorization.tabs.bucket"
defaultMessage="Bucket"
/>
</EuiFlexItem>
<EuiFlexItem grow={false} css={infoIconCss}>
<EuiIcon
size="s"
color="subdued"
type="questionInCircle"
className="eui-alignTop"
/>
</EuiFlexItem>
</EuiFlexGroup>
</EuiToolTip>
</EuiTab>
<EuiTab
isSelected={selectedTab === SELECTED_TAB.FULL_TIME_RANGE}
onClick={() => setSelectedTab(SELECTED_TAB.FULL_TIME_RANGE)}
>
<EuiToolTip
content={i18n.translate(
'xpack.aiops.logCategorization.tabs.fullTimeRange.tooltip',
{
defaultMessage:
'Patterns that occur in the time range selected for the page.',
}
)}
>
<EuiFlexGroup gutterSize="none">
<EuiFlexItem>
<FormattedMessage
id="xpack.aiops.logCategorization.tabs.fullTimeRange"
defaultMessage="Full time range"
/>
</EuiFlexItem>
<EuiFlexItem grow={false} css={infoIconCss}>
<EuiIcon
size="s"
color="subdued"
type="questionInCircle"
className="eui-alignTop"
/>
</EuiFlexItem>
</EuiFlexGroup>
</EuiToolTip>
</EuiTab>
</EuiTabs>
<EuiSpacer size="s" />
</>
) : null}
<CategoryTable
categories={
selectedTab === SELECTED_TAB.BUCKET && data.categoriesInBucket !== null
? data.categoriesInBucket
: data.categories
}
aiopsListState={stateFromUrl}
dataViewId={dataView.id!}
eventRate={eventRate}
selectedField={selectedField}
pinnedCategory={pinnedCategory}
setPinnedCategory={setPinnedCategory}
selectedCategory={selectedCategory}
setSelectedCategory={setSelectedCategory}
timefilter={timefilter}
onAddFilter={onAddFilter}
onClose={onClose}
enableRowActions={false}
additionalFilter={
selectedTab === SELECTED_TAB.BUCKET && additionalFilter !== undefined
? additionalFilter
: undefined
}
navigateToDiscover={additionalFilter !== undefined}
/>
</>
) : null}
</EuiFlyoutBody>
</>

30
x-pack/plugins/aiops/public/components/log_categorization/log_categorization_page.tsx
View file

@ -28,7 +28,7 @@ import type { FieldValidationResults } from '@kbn/ml-category-validator';
import type { SearchQueryLanguage } from '@kbn/ml-query-utils';
import { AIOPS_TELEMETRY_ID } from '../../../common/constants';
import type { Category, SparkLinesPerCategory } from '../../../common/api/log_categorization/types';
import type { Category } from '../../../common/api/log_categorization/types';
import { useDataSource } from '../../hooks/use_data_source';
import { useData } from '../../hooks/use_data';
@ -86,7 +86,6 @@ export const LogCategorizationPage: FC<LogCategorizationPageProps> = ({ embeddin
const [pinnedCategory, setPinnedCategory] = useState<Category | null>(null);
const [data, setData] = useState<{
categories: Category[];
sparkLines: SparkLinesPerCategory;
} | null>(null);
const [fieldValidationResult, setFieldValidationResult] = useState<FieldValidationResults | null>(
null
@ -205,34 +204,35 @@ export const LogCategorizationPage: FC<LogCategorizationPageProps> = ({ embeddin
const { getIndexPattern, timeFieldName: timeField } = dataView;
const index = getIndexPattern();
if (selectedField === undefined || timeField === undefined) {
if (
selectedField === undefined ||
timeField === undefined ||
earliest === undefined ||
latest === undefined
) {
setLoading(false);
return;
}
cancelRequest();
const timeRange = {
from: earliest,
to: latest,
};
try {
const [validationResult, categorizationResult] = await Promise.all([
runValidateFieldRequest(index, selectedField, timeField, earliest, latest, searchQuery, {
runValidateFieldRequest(index, selectedField, timeField, timeRange, searchQuery, {
[AIOPS_TELEMETRY_ID.AIOPS_ANALYSIS_RUN_ORIGIN]: embeddingOrigin,
}),
runCategorizeRequest(
index,
selectedField,
timeField,
earliest,
latest,
searchQuery,
intervalMs
),
runCategorizeRequest(index, selectedField, timeField, timeRange, searchQuery, intervalMs),
]);
setFieldValidationResult(validationResult);
setData({
categories: categorizationResult.categories,
sparkLines: categorizationResult.sparkLinesPerCategory,
});
} catch (error) {
toasts.addError(error, {
@ -355,7 +355,6 @@ export const LogCategorizationPage: FC<LogCategorizationPageProps> = ({ embeddin
eventRate={eventRate}
pinnedCategory={pinnedCategory}
selectedCategory={selectedCategory}
sparkLines={data?.sparkLines ?? {}}
totalCount={totalCount}
documentCountStats={documentStats.documentCountStats}
/>
@ -380,7 +379,6 @@ export const LogCategorizationPage: FC<LogCategorizationPageProps> = ({ embeddin
aiopsListState={stateFromUrl}
dataViewId={dataView.id!}
eventRate={eventRate}
sparkLines={data.sparkLines}
selectedField={selectedField}
pinnedCategory={pinnedCategory}
setPinnedCategory={setPinnedCategory}

7
x-pack/plugins/aiops/public/components/log_categorization/show_flyout.tsx
View file

@ -18,9 +18,10 @@ import type { DataViewField, DataView } from '@kbn/data-views-plugin/common';
import { UI_SETTINGS } from '@kbn/data-plugin/public';
import { DatePickerContextProvider, type DatePickerDependencies } from '@kbn/ml-date-picker';
import { StorageContextProvider } from '@kbn/ml-local-storage';
import type { CategorizationAdditionalFilter } from '../../../common/api/log_categorization/create_category_request';
import type { AiopsPluginStartDeps } from '../../types';
import { AiopsAppContext, type AiopsAppDependencies } from '../../hooks/use_aiops_app_context';
import { LogCategorizationFlyout } from './log_categorization_for_flyout';
import { AiopsAppContext, type AiopsAppDependencies } from '../../hooks/use_aiops_app_context';
import { AIOPS_STORAGE_KEYS } from '../../types/storage';
const localStorage = new Storage(window.localStorage);
@ -30,7 +31,8 @@ export async function showCategorizeFlyout(
dataView: DataView,
coreStart: CoreStart,
plugins: AiopsPluginStartDeps,
originatingApp: string
originatingApp: string,
additionalFilter?: CategorizationAdditionalFilter
): Promise<void> {
const { http, theme, overlays, application, notifications, uiSettings, i18n } = coreStart;
@ -72,6 +74,7 @@ export async function showCategorizeFlyout(
selectedField={field}
onClose={onFlyoutClose}
embeddingOrigin={originatingApp}
additionalFilter={additionalFilter}
/>
</StorageContextProvider>
</DatePickerContextProvider>

32
x-pack/plugins/aiops/public/components/log_categorization/use_categorize_request.ts
View file

@ -12,13 +12,12 @@ import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/type
import { isRunningResponse } from '@kbn/data-plugin/public';
import { useStorage } from '@kbn/ml-local-storage';
import { createCategoryRequest } from '../../../common/api/log_categorization/create_category_request';
import {
type CategorizationAdditionalFilter,
createCategoryRequest,
} from '../../../common/api/log_categorization/create_category_request';
import { processCategoryResults } from '../../../common/api/log_categorization/process_category_results';
import type {
Category,
CatResponse,
SparkLinesPerCategory,
} from '../../../common/api/log_categorization/types';
import type { Category, CatResponse } from '../../../common/api/log_categorization/types';
import { useAiopsAppContext } from '../../hooks/use_aiops_app_context';
import {
@ -68,17 +67,26 @@ export function useCategorizeRequest() {
index: string,
field: string,
timeField: string,
from: number | undefined,
to: number | undefined,
timeRange: { from: number; to: number },
query: QueryDslQueryContainer,
intervalMs?: number
): Promise<{ categories: Category[]; sparkLinesPerCategory: SparkLinesPerCategory }> => {
intervalMs?: number,
additionalFilter?: CategorizationAdditionalFilter
): Promise<{ categories: Category[] }> => {
const { wrap, unwrap } = randomSampler.createRandomSamplerWrapper();
return new Promise((resolve, reject) => {
data.search
.search<ReturnType<typeof createCategoryRequest>, CatResponse>(
createCategoryRequest(index, field, timeField, from, to, query, wrap, intervalMs),
createCategoryRequest(
index,
field,
timeField,
timeRange,
query,
wrap,
intervalMs,
additionalFilter
),
{ abortSignal: abortController.current.signal }
)
.subscribe({
@ -93,7 +101,7 @@ export function useCategorizeRequest() {
},
error: (error) => {
if (error.name === 'AbortError') {
return resolve({ categories: [], sparkLinesPerCategory: {} });
return resolve({ categories: [] });
}
reject(error);
},

32
x-pack/plugins/aiops/public/components/log_categorization/use_discover_links.ts
View file

@ -12,18 +12,15 @@ import type { TimeRangeBounds } from '@kbn/data-plugin/common';
import { i18n } from '@kbn/i18n';
import type { Filter } from '@kbn/es-query';
import { getCategoryQuery } from '../../../common/api/log_categorization/get_category_query';
import {
getCategoryQuery,
type QueryMode,
} from '../../../common/api/log_categorization/get_category_query';
import type { Category } from '../../../common/api/log_categorization/types';
import type { AiOpsIndexBasedAppState } from '../../application/url_state/common';
import { useAiopsAppContext } from '../../hooks/use_aiops_app_context';
export const QUERY_MODE = {
INCLUDE: 'should',
EXCLUDE: 'must_not',
} as const;
export type QueryMode = typeof QUERY_MODE[keyof typeof QUERY_MODE];
export function useDiscoverLinks() {
const {
http: { basePath },
@ -36,7 +33,8 @@ export function useDiscoverLinks() {
aiopsListState: Required<AiOpsIndexBasedAppState>,
timefilterActiveBounds: TimeRangeBounds,
mode: QueryMode,
category?: Category
category?: Category,
additionalField?: { name: string; value: string }
) => {
const _g = rison.encode({
time: {
@ -46,7 +44,10 @@ export function useDiscoverLinks() {
});
const _a = rison.encode({
filters: [...aiopsListState.filters, createFilter(index, field, selection, mode, category)],
filters: [
...aiopsListState.filters,
createFilter(index, field, selection, mode, category, additionalField),
],
index,
interval: 'auto',
query: {
@ -70,11 +71,20 @@ export function createFilter(
field: string,
selection: Category[],
mode: QueryMode,
category?: Category
category?: Category,
additionalField?: { name: string; value: string }
): Filter {
const selectedRows = category === undefined ? selection : [category];
const query = getCategoryQuery(field, selectedRows, mode);
if (additionalField !== undefined) {
query.bool.must = [
{
term: { [additionalField.name]: additionalField.value },
},
];
}
return {
query: getCategoryQuery(field, selectedRows, mode),
query,
meta: {
alias: i18n.translate('xpack.aiops.logCategorization.filterAliasLabel', {
defaultMessage: 'Categorization - {field}',

9
x-pack/plugins/aiops/public/components/log_categorization/use_validate_category_field.ts
View file

@ -27,12 +27,11 @@ export function useValidateFieldRequest() {
index: string,
field: string,
timeField: string,
start: number | undefined,
end: number | undefined,
timeRange: { from: number; to: number },
queryIn: QueryDslQueryContainer,
headers?: HttpFetchOptions['headers']
) => {
const query = createCategorizeQuery(queryIn, timeField, start, end);
const query = createCategorizeQuery(queryIn, timeField, timeRange);
const resp = await http.post<FieldValidationResults>(
AIOPS_API_ENDPOINT.CATEGORIZATION_FIELD_VALIDATION,
{
@ -42,8 +41,8 @@ export function useValidateFieldRequest() {
size: 5,
field,
timeField,
start,
end,
start: timeRange.from,
end: timeRange.to,
// only text fields are supported in pattern analysis,
// and it is not possible to create a text runtime field
// so runtimeMappings are not needed

2
x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_categories.test.ts
View file

@ -144,7 +144,7 @@ describe('getCategoryRequest', () => {
size: 1000,
},
aggs: {
hit: {
examples: {
top_hits: { size: 1, sort: ['the-time-field-name'], _source: 'the-field-name' },
},
},

15
x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_categories.ts
View file

@ -18,11 +18,7 @@ import {
import { RANDOM_SAMPLER_SEED } from '../../../../common/constants';
import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis/schema';
import { createCategoryRequest } from '../../../../common/api/log_categorization/create_category_request';
import type {
Category,
CategoriesAgg,
SparkLinesPerCategory,
} from '../../../../common/api/log_categorization/types';
import type { Category, CategoriesAgg } from '../../../../common/api/log_categorization/types';
import { isRequestAbortedError } from '../../../lib/is_request_aborted_error';
@ -79,7 +75,6 @@ export const getCategoryRequest = (
fieldName,
timeFieldName,
undefined,
undefined,
query,
wrap
);
@ -94,7 +89,6 @@ export const getCategoryRequest = (
export interface FetchCategoriesResponse {
categories: Category[];
sparkLinesPerCategory: SparkLinesPerCategory;
}
export const fetchCategories = async (
@ -155,7 +149,6 @@ export const fetchCategories = async (
continue;
}
const sparkLinesPerCategory: SparkLinesPerCategory = {};
const {
categories: { buckets },
} = randomSamplerWrapper.unwrap(
@ -163,7 +156,7 @@ export const fetchCategories = async (
) as CategoriesAgg;
const categories: Category[] = buckets.map((b) => {
sparkLinesPerCategory[b.key] =
const sparkline =
b.sparkline === undefined
? {}
: b.sparkline.buckets.reduce<Record<number, number>>((acc2, cur2) => {
@ -174,12 +167,12 @@ export const fetchCategories = async (
return {
key: b.key,
count: b.doc_count,
examples: b.hit.hits.hits.map((h) => get(h._source, fieldName)),
examples: b.examples.hits.hits.map((h) => get(h._source, fieldName)),
sparkline,
};
});
result.push({
categories,
sparkLinesPerCategory,
});
}

88
x-pack/plugins/ml/public/application/components/anomalies_table/links_menu.tsx
View file

@ -28,18 +28,21 @@ import {
type MlCustomUrlAnomalyRecordDoc,
type MlKibanaUrlConfig,
type MlAnomaliesTableRecord,
MLCATEGORY,
} from '@kbn/ml-anomaly-utils';
import { formatHumanReadableDateTimeSeconds, timeFormatter } from '@kbn/ml-date-utils';
import { SEARCH_QUERY_LANGUAGE } from '@kbn/ml-query-utils';
import type { DataView, DataViewField } from '@kbn/data-views-plugin/common';
import { CATEGORIZE_FIELD_TRIGGER } from '@kbn/ml-ui-actions';
import { PLUGIN_ID } from '../../../../common/constants/app';
import { mlJobService } from '../../services/job_service';
import { getDataViewIdFromName } from '../../util/index_utils';
import { findMessageField, getDataViewIdFromName } from '../../util/index_utils';
import { getInitialAnomaliesLayers, getInitialSourceIndexFieldLayers } from '../../../maps/util';
import { parseInterval } from '../../../../common/util/parse_interval';
import { ml } from '../../services/ml_api_service';
import { escapeKueryForFieldValuePair, replaceStringTokens } from '../../util/string_utils';
import { getUrlForRecord, openCustomUrlWindow } from '../../util/custom_url_utils';
import { ML_APP_LOCATOR, ML_PAGES } from '../../../../common/constants/locator';
// @ts-ignore
import {
escapeDoubleQuotes,
getDateFormatTz,
@ -48,7 +51,6 @@ import {
import { usePermissionCheck } from '../../capabilities/check_capabilities';
import type { TimeRangeBounds } from '../../util/time_buckets';
import { useMlKibana } from '../../contexts/kibana';
// @ts-ignore
import { getFieldTypeFromMapping } from '../../services/mapping_service';
import { getQueryStringForInfluencers } from './get_query_string_for_influencers';
import { getFiltersForDSLQuery } from '../../../../common/util/job_utils';
@ -68,13 +70,18 @@ export const LinksMenuUI = (props: LinksMenuProps) => {
const [openInDiscoverUrl, setOpenInDiscoverUrl] = useState<string | undefined>();
const [discoverUrlError, setDiscoverUrlError] = useState<string | undefined>();
const [messageField, setMessageField] = useState<{
dataView: DataView;
field: DataViewField;
} | null>(null);
const isCategorizationAnomalyRecord = isCategorizationAnomaly(props.anomaly);
const closePopover = props.onItemClick;
const kibana = useMlKibana();
const {
services: { data, share, application },
services: { data, share, application, uiActions },
} = kibana;
const job = useMemo(() => {
@ -214,6 +221,22 @@ export const LinksMenuUI = (props: LinksMenuProps) => {
return dataViewId;
};
(async () => {
const index = job.datafeed_config.indices[0];
const dataView = (await data.dataViews.find(index)).find(
(dv) => dv.getIndexPattern() === index
);
if (dataView === undefined) {
return;
}
const field = findMessageField(dataView);
if (field !== null) {
setMessageField(field);
}
})();
const generateDiscoverUrl = async () => {
const interval = props.interval;
@ -780,6 +803,43 @@ export const LinksMenuUI = (props: LinksMenuProps) => {
</EuiContextMenuItem>
);
}
if (messageField !== null) {
items.push(
<EuiContextMenuItem
key="create_rule"
icon="machineLearningApp"
onClick={() => {
closePopover();
const additionalField = getAdditionalField(anomaly);
uiActions.getTrigger(CATEGORIZE_FIELD_TRIGGER).exec({
dataView: messageField.dataView,
field: messageField.field,
originatingApp: PLUGIN_ID,
additionalFilter: {
from: anomaly.source.timestamp,
to: anomaly.source.timestamp + anomaly.source.bucket_span * 1000,
...(additionalField !== null
? {
field: {
name: additionalField.name,
value: additionalField.value,
},
}
: {}),
},
});
}}
data-test-subj="mlAnomaliesListRowActionPatternAnalysisButton"
>
<FormattedMessage
id="xpack.ml.anomaliesTable.linksMenu.patternAnalysisLabel"
defaultMessage="Run pattern analysis"
/>
</EuiContextMenuItem>
);
}
return items;
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [
@ -830,3 +890,23 @@ export const LinksMenu: FC<Omit<LinksMenuProps, 'onItemClick'>> = (props) => {
</div>
);
};
function getAdditionalField(anomaly: MlAnomaliesTableRecord) {
if (anomaly.entityName === undefined || anomaly.entityValue === undefined) {
return null;
}
if (anomaly.entityName === MLCATEGORY) {
if (
anomaly.source.partition_field_name === undefined ||
anomaly.source.partition_field_value === undefined
) {
return null;
}
return {
name: anomaly.source.partition_field_name,
value: anomaly.source.partition_field_value,
};
}
return { name: anomaly.entityName, value: anomaly.entityValue };
}

2
x-pack/plugins/ml/public/application/contexts/kibana/kibana_context.ts
View file

@ -9,7 +9,7 @@ import type { DataPublicPluginStart } from '@kbn/data-plugin/public';
import type { DataViewsPublicPluginStart } from '@kbn/data-views-plugin/public';
import type { CoreStart } from '@kbn/core/public';
import type { UsageCollectionSetup } from '@kbn/usage-collection-plugin/public';
import { useKibana, KibanaReactContextValue } from '@kbn/kibana-react-plugin/public';
import { useKibana, type KibanaReactContextValue } from '@kbn/kibana-react-plugin/public';
import type { SecurityPluginStart } from '@kbn/security-plugin/public';
import type { LicenseManagementUIPluginSetup } from '@kbn/license-management-plugin/public';
import type { SharePluginStart } from '@kbn/share-plugin/public';

25
x-pack/plugins/ml/public/application/util/index_utils.ts
View file

@ -6,10 +6,9 @@
*/
import { i18n } from '@kbn/i18n';
import type { DataView } from '@kbn/data-views-plugin/public';
import type { SavedSearch, SavedSearchPublicPluginStart } from '@kbn/saved-search-plugin/public';
import type { Query, Filter } from '@kbn/es-query';
import type { DataViewsContract } from '@kbn/data-views-plugin/common';
import type { DataView, DataViewField, DataViewsContract } from '@kbn/data-views-plugin/common';
import type { Job } from '../../../common/types/anomaly_detection_jobs';
import { getToastNotifications, getDataViews } from './dependency_cache';
@ -132,3 +131,25 @@ export function timeBasedIndexCheck(dataView: DataView, showNotification = false
export function isCcsIndexPattern(indexPattern: string) {
return indexPattern.includes(':');
}
export function findMessageField(
dataView: DataView
): { dataView: DataView; field: DataViewField } | null {
const foundFields: Record<string, DataViewField | null> = { message: null, errorMessage: null };
for (const f of dataView.fields) {
if (f.name === 'message' && f.toSpec().esTypes?.includes('text')) {
foundFields.message = f;
} else if (f.name === 'error.message' && f.toSpec().esTypes?.includes('text')) {
foundFields.errorMessage = f;
}
}
if (foundFields.message !== null) {
return { dataView, field: foundFields.message };
} else if (foundFields.errorMessage !== null) {
return { dataView, field: foundFields.errorMessage };
}
return null;
}