github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 17:28:26 -04:00
Code Issues Projects Releases Packages Wiki Activity

[ML] Explain Log Rate Spikes: Add mini histograms to grouped results table. (#141065)

Browse source 
- Adds mini histograms to grouped results table.
- Fixes row expansion issue where expanded row could show up under wrong row.
This commit is contained in:
Walter Rafelsberger 2022-09-20 21:23:12 +02:00 • committed by GitHub
parent 7d21c5f10b
commit 1598523b4f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 197 additions and 33 deletions
Download patch file Download diff file Expand all files Collapse all files

1
x-pack/packages/ml/agg_utils/index.ts
View file

@ -16,6 +16,7 @@ export type {
AggCardinality,
ChangePoint,
ChangePointGroup,
ChangePointGroupHistogram,
ChangePointHistogram,
ChangePointHistogramItem,
HistogramField,

10
x-pack/packages/ml/agg_utils/src/types.ts
View file

@ -87,6 +87,14 @@ export interface ChangePointHistogram extends FieldValuePair {
histogram: ChangePointHistogramItem[];
}
/**
* Change point histogram data for a group of field/value pairs.
*/
export interface ChangePointGroupHistogram {
id: string;
histogram: ChangePointHistogramItem[];
}
interface ChangePointGroupItem extends FieldValuePair {
duplicate?: boolean;
}
@ -95,7 +103,9 @@ interface ChangePointGroupItem extends FieldValuePair {
* Tree leaves
*/
export interface ChangePointGroup {
id: string;
group: ChangePointGroupItem[];
docCount: number;
pValue: number | null;
histogram?: ChangePointHistogramItem[];
}

23
x-pack/plugins/aiops/common/api/explain_log_rate_spikes/actions.ts
View file

@ -5,12 +5,18 @@
* 2.0.
*/
import type { ChangePoint, ChangePointHistogram, ChangePointGroup } from '@kbn/ml-agg-utils';
import type {
ChangePoint,
ChangePointHistogram,
ChangePointGroup,
ChangePointGroupHistogram,
} from '@kbn/ml-agg-utils';
export const API_ACTION_NAME = {
ADD_CHANGE_POINTS: 'add_change_points',
ADD_CHANGE_POINTS_HISTOGRAM: 'add_change_points_histogram',
ADD_CHANGE_POINTS_GROUP: 'add_change_point_group',
ADD_CHANGE_POINTS_GROUP_HISTOGRAM: 'add_change_point_group_histogram',
ADD_ERROR: 'add_error',
RESET: 'reset',
UPDATE_LOADING_STATE: 'update_loading_state',
@ -57,6 +63,20 @@ export function addChangePointsGroupAction(payload: ApiActionAddChangePointsGrou
};
}
interface ApiActionAddChangePointsGroupHistogram {
type: typeof API_ACTION_NAME.ADD_CHANGE_POINTS_GROUP_HISTOGRAM;
payload: ChangePointGroupHistogram[];
}
export function addChangePointsGroupHistogramAction(
payload: ApiActionAddChangePointsGroupHistogram['payload']
): ApiActionAddChangePointsGroupHistogram {
return {
type: API_ACTION_NAME.ADD_CHANGE_POINTS_GROUP_HISTOGRAM,
payload,
};
}
interface ApiActionAddError {
type: typeof API_ACTION_NAME.ADD_ERROR;
payload: string;
@ -99,6 +119,7 @@ export type AiopsExplainLogRateSpikesApiAction =
| ApiActionAddChangePoints
| ApiActionAddChangePointsGroup
| ApiActionAddChangePointsHistogram
| ApiActionAddChangePointsGroupHistogram
| ApiActionAddError
| ApiActionReset
| ApiActionUpdateLoadingState;

1
x-pack/plugins/aiops/common/api/explain_log_rate_spikes/index.ts
View file

@ -8,6 +8,7 @@
export {
addChangePointsAction,
addChangePointsGroupAction,
addChangePointsGroupHistogramAction,
addChangePointsHistogramAction,
addErrorAction,
resetAction,

9
x-pack/plugins/aiops/common/api/stream_reducer.ts
View file

@ -51,6 +51,15 @@ export function streamReducer(
return { ...state, changePoints };
case API_ACTION_NAME.ADD_CHANGE_POINTS_GROUP:
return { ...state, changePointsGroups: action.payload };
case API_ACTION_NAME.ADD_CHANGE_POINTS_GROUP_HISTOGRAM:
const changePointsGroups = state.changePointsGroups.map((cpg) => {
const cpHistogram = action.payload.find((h) => h.id === cpg.id);
if (cpHistogram) {
cpg.histogram = cpHistogram.histogram;
}
return cpg;
});
return { ...state, changePointsGroups };
case API_ACTION_NAME.ADD_ERROR:
return { ...state, errors: [...state.errors, action.payload] };
case API_ACTION_NAME.RESET:

5
x-pack/plugins/aiops/public/components/explain_log_rate_spikes/explain_log_rate_spikes_analysis.tsx
View file

@ -127,7 +127,7 @@ export const ExplainLogRateSpikesAnalysis: FC<ExplainLogRateSpikesAnalysisProps>
}, []);
const groupTableItems = useMemo(() => {
const tableItems = data.changePointsGroups.map(({ group, docCount, pValue }, index) => {
const tableItems = data.changePointsGroups.map(({ id, group, docCount, histogram, pValue }) => {
const sortedGroup = group.sort((a, b) =>
a.fieldName > b.fieldName ? 1 : b.fieldName > a.fieldName ? -1 : 0
);
@ -144,11 +144,12 @@ export const ExplainLogRateSpikesAnalysis: FC<ExplainLogRateSpikesAnalysisProps>
});
return {
id: index,
id,
docCount,
pValue,
group: dedupedGroup,
repeatedValues,
histogram,
};
});

50
x-pack/plugins/aiops/public/components/spike_analysis_table/spike_analysis_table_groups.tsx
View file

@ -24,7 +24,11 @@ import {
import { i18n } from '@kbn/i18n';
import { FormattedMessage } from '@kbn/i18n-react';
import type { ChangePoint } from '@kbn/ml-agg-utils';
import { useEuiTheme } from '../../hooks/use_eui_theme';
import { MiniHistogram } from '../mini_histogram';
import { SpikeAnalysisTable } from './spike_analysis_table';
const NARROW_COLUMN_WIDTH = '120px';
@ -36,11 +40,12 @@ const DEFAULT_SORT_FIELD = 'pValue';
const DEFAULT_SORT_DIRECTION = 'asc';
interface GroupTableItem {
id: number;
id: string;
docCount: number;
pValue: number | null;
group: Record<string, any>;
repeatedValues: Record<string, any>;
histogram: ChangePoint['histogram'];
}
interface SpikeAnalysisTableProps {
@ -196,6 +201,39 @@ export const SpikeAnalysisGroupsTable: FC<SpikeAnalysisTableProps> = ({
sortable: false,
textOnly: true,
},
{
'data-test-subj': 'aiopsSpikeAnalysisGroupsTableColumnLogRate',
width: NARROW_COLUMN_WIDTH,
field: 'pValue',
name: (
<EuiToolTip
position="top"
content={i18n.translate(
'xpack.aiops.explainLogRateSpikes.spikeAnalysisTableGroups.logRateColumnTooltip',
{
defaultMessage:
'A visual representation of the impact of the field on the message rate difference',
}
)}
>
<>
<FormattedMessage
id="xpack.aiops.explainLogRateSpikes.spikeAnalysisTableGroups.logRateLabel"
defaultMessage="Log rate"
/>
<EuiIcon size="s" color="subdued" type="questionInCircle" className="eui-alignTop" />
</>
</EuiToolTip>
),
render: (_, { histogram, id }) => (
<MiniHistogram
chartData={histogram}
isLoading={loading && histogram === undefined}
label="Group x"
/>
),
sortable: false,
},
{
'data-test-subj': 'aiopsSpikeAnalysisGroupsTableColumnPValue',
width: NARROW_COLUMN_WIDTH,
@ -226,9 +264,12 @@ export const SpikeAnalysisGroupsTable: FC<SpikeAnalysisTableProps> = ({
{
'data-test-subj': 'aiopsSpikeAnalysisGroupsTableColumnDocCount',
field: 'docCount',
name: i18n.translate('xpack.aiops.correlations.spikeAnalysisTableGroups.docCountLabel', {
defaultMessage: 'Doc count',
}),
name: i18n.translate(
'xpack.aiops.explainLogRateSpikes.spikeAnalysisTableGroups.docCountLabel',
{
defaultMessage: 'Doc count',
}
),
sortable: true,
width: '20%',
},
@ -281,6 +322,7 @@ export const SpikeAnalysisGroupsTable: FC<SpikeAnalysisTableProps> = ({
compressed
columns={columns}
items={pageOfItems}
itemId="id"
itemIdToExpandedRowMap={itemIdToExpandedRowMap}
onChange={onChange}
pagination={pagination}

118
x-pack/plugins/aiops/server/routes/explain_log_rate_spikes.ts
View file

@ -18,10 +18,12 @@ import type { DataRequestHandlerContext } from '@kbn/data-plugin/server';
import { streamFactory } from '@kbn/aiops-utils';
import type { ChangePoint, NumericChartData, NumericHistogramField } from '@kbn/ml-agg-utils';
import { fetchHistogramsForFields } from '@kbn/ml-agg-utils';
import { stringHash } from '@kbn/ml-string-hash';
import {
addChangePointsAction,
addChangePointsGroupAction,
addChangePointsGroupHistogramAction,
addChangePointsHistogramAction,
aiopsExplainLogRateSpikesSchema,
addErrorAction,
@ -216,6 +218,21 @@ export const defineExplainLogRateSpikesRoute = (
return;
}
const histogramFields: [NumericHistogramField] = [
{ fieldName: request.body.timeFieldName, type: KBN_FIELD_TYPES.DATE },
];
const [overallTimeSeries] = (await fetchHistogramsForFields(
client,
request.body.index,
{ match_all: {} },
// fields
histogramFields,
// samplerShardSize
-1,
undefined
)) as [NumericChartData];
if (groupingEnabled) {
// To optimize the `frequent_items` query, we identify duplicate change points by count attributes.
// Note this is a compromise and not 100% accurate because there could be change points that
@ -325,27 +342,40 @@ export const defineExplainLogRateSpikesRoute = (
});
changePointGroups.push(
...missingChangePoints.map((cp) => {
...missingChangePoints.map(({ fieldName, fieldValue, doc_count: docCount, pValue }) => {
const duplicates = groupedChangePoints.find((d) =>
d.group.some(
(dg) => dg.fieldName === cp.fieldName && dg.fieldValue === cp.fieldValue
)
d.group.some((dg) => dg.fieldName === fieldName && dg.fieldValue === fieldValue)
);
if (duplicates !== undefined) {
return {
id: `${stringHash(
JSON.stringify(
duplicates.group.map((d) => ({
fieldName: d.fieldName,
fieldValue: d.fieldValue,
}))
)
)}`,
group: duplicates.group.map((d) => ({
fieldName: d.fieldName,
fieldValue: d.fieldValue,
duplicate: false,
})),
docCount: cp.doc_count,
pValue: cp.pValue,
docCount,
pValue,
};
} else {
return {
group: [{ fieldName: cp.fieldName, fieldValue: cp.fieldValue, duplicate: false }],
docCount: cp.doc_count,
pValue: cp.pValue,
id: `${stringHash(JSON.stringify({ fieldName, fieldValue }))}`,
group: [
{
fieldName,
fieldValue,
duplicate: false,
},
],
docCount,
pValue,
};
}
})
@ -358,23 +388,63 @@ export const defineExplainLogRateSpikesRoute = (
if (maxItems > 1) {
push(addChangePointsGroupAction(changePointGroups));
}
if (changePointGroups) {
await asyncForEach(changePointGroups, async (cpg, index) => {
const histogramQuery = {
bool: {
filter: cpg.group.map((d) => ({
term: { [d.fieldName]: d.fieldValue },
})),
},
};
const [cpgTimeSeries] = (await fetchHistogramsForFields(
client,
request.body.index,
histogramQuery,
// fields
[
{
fieldName: request.body.timeFieldName,
type: KBN_FIELD_TYPES.DATE,
interval: overallTimeSeries.interval,
min: overallTimeSeries.stats[0],
max: overallTimeSeries.stats[1],
},
],
// samplerShardSize
-1,
undefined
)) as [NumericChartData];
const histogram =
overallTimeSeries.data.map((o, i) => {
const current = cpgTimeSeries.data.find(
(d1) => d1.key_as_string === o.key_as_string
) ?? {
doc_count: 0,
};
return {
key: o.key,
key_as_string: o.key_as_string ?? '',
doc_count_change_point: current.doc_count,
doc_count_overall: Math.max(0, o.doc_count - current.doc_count),
};
}) ?? [];
push(
addChangePointsGroupHistogramAction([
{
id: cpg.id,
histogram,
},
])
);
});
}
}
const histogramFields: [NumericHistogramField] = [
{ fieldName: request.body.timeFieldName, type: KBN_FIELD_TYPES.DATE },
];
const [overallTimeSeries] = (await fetchHistogramsForFields(
client,
request.body.index,
{ match_all: {} },
// fields
histogramFields,
// samplerShardSize
-1,
undefined
)) as [NumericChartData];
// time series filtered by fields
if (changePoints) {
await asyncForEach(changePoints, async (cp, index) => {

4
x-pack/plugins/aiops/server/routes/queries/get_simple_hierarchical_tree.test.ts
View file

@ -11,6 +11,7 @@ import { getFieldValuePairCounts, markDuplicates } from './get_simple_hierarchic
const changePointGroups: ChangePointGroup[] = [
{
id: 'group-1',
group: [
{
fieldName: 'custom_field.keyword',
@ -25,6 +26,7 @@ const changePointGroups: ChangePointGroup[] = [
pValue: 0.01,
},
{
id: 'group-2',
group: [
{
fieldName: 'custom_field.keyword',
@ -64,6 +66,7 @@ describe('get_simple_hierarchical_tree', () => {
expect(markedDuplicates).toEqual([
{
id: 'group-1',
group: [
{
fieldName: 'custom_field.keyword',
@ -80,6 +83,7 @@ describe('get_simple_hierarchical_tree', () => {
pValue: 0.01,
},
{
id: 'group-2',
group: [
{
fieldName: 'custom_field.keyword',

9
x-pack/plugins/aiops/server/routes/queries/get_simple_hierarchical_tree.ts
View file

@ -8,6 +8,7 @@
// import { omit, uniq } from 'lodash';
import type { ChangePointGroup, FieldValuePair } from '@kbn/ml-agg-utils';
import { stringHash } from '@kbn/ml-string-hash';
import type { ItemsetResult } from './fetch_frequent_items';
@ -230,9 +231,13 @@ export function getSimpleHierarchicalTreeLeaves(
leaves: ChangePointGroup[],
level = 1
) {
// console.log(`${'-'.repeat(level)} ${tree.name} ${tree.children.length}`);
if (tree.children.length === 0) {
leaves.push({ group: tree.set, docCount: tree.docCount, pValue: tree.pValue });
leaves.push({
id: `${stringHash(JSON.stringify(tree.set))}`,
group: tree.set,
docCount: tree.docCount,
pValue: tree.pValue,
});
} else {
for (const child of tree.children) {
const newLeaves = getSimpleHierarchicalTreeLeaves(child, [], level + 1);