mirror of
https://github.com/elastic/kibana.git
synced 2025-04-24 09:48:58 -04:00
[Logs UI] Add dataset-specific categorization warnings (#75351)
This adds dataset-specific categorization warnings for the categorization module. The warnings are displayed in call-outs on the relevant tabs as well as the job setup screens if a prior job with warnings exists. To that end this also changes the categorization job configuration to enable the partitioned categorization mode. Co-authored-by: Alejandro Fernández Gómez <antarticonorte@gmail.com>
This commit is contained in:
Felix Stürmer
GitHub
parent
5ff0c00529
commit
3f2e9f7705
38 changed files with 1027 additions and 273 deletions
|
|
@ -23,7 +23,7 @@ export const storybookAliases = {
|
|||
codeeditor: 'src/plugins/kibana_react/public/code_editor/scripts/storybook.ts',
|
||||
dashboard_enhanced: 'x-pack/plugins/dashboard_enhanced/scripts/storybook.js',
|
||||
embeddable: 'src/plugins/embeddable/scripts/storybook.js',
|
||||
infra: 'x-pack/legacy/plugins/infra/scripts/storybook.js',
|
||||
infra: 'x-pack/plugins/infra/scripts/storybook.js',
|
||||
security_solution: 'x-pack/plugins/security_solution/scripts/storybook.js',
|
||||
ui_actions_enhanced: 'x-pack/plugins/ui_actions_enhanced/scripts/storybook.js',
|
||||
observability: 'x-pack/plugins/observability/scripts/storybook.js',
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
|
||||
export * from './log_entry_categories';
|
||||
export * from './log_entry_category_datasets';
|
||||
export * from './log_entry_category_datasets_stats';
|
||||
export * from './log_entry_category_examples';
|
||||
export * from './log_entry_rate';
|
||||
export * from './log_entry_examples';
|
||||
|
|
|
|||
|
|
@ -0,0 +1,72 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import * as rt from 'io-ts';
|
||||
|
||||
import { timeRangeRT, routeTimingMetadataRT } from '../../shared';
|
||||
|
||||
export const LOG_ANALYSIS_GET_LATEST_LOG_ENTRY_CATEGORY_DATASETS_STATS_PATH =
|
||||
'/api/infra/log_analysis/results/latest_log_entry_category_datasets_stats';
|
||||
|
||||
const categorizerStatusRT = rt.keyof({
|
||||
ok: null,
|
||||
warn: null,
|
||||
});
|
||||
|
||||
export type CategorizerStatus = rt.TypeOf<typeof categorizerStatusRT>;
|
||||
|
||||
/**
|
||||
* request
|
||||
*/
|
||||
|
||||
export const getLatestLogEntryCategoryDatasetsStatsRequestPayloadRT = rt.type({
|
||||
data: rt.type({
|
||||
// the ids of the categorization jobs
|
||||
jobIds: rt.array(rt.string),
|
||||
// the time range to fetch the category datasets stats for
|
||||
timeRange: timeRangeRT,
|
||||
// the categorizer statuses to include stats for, empty means all
|
||||
includeCategorizerStatuses: rt.array(categorizerStatusRT),
|
||||
}),
|
||||
});
|
||||
|
||||
export type GetLatestLogEntryCategoryDatasetsStatsRequestPayload = rt.TypeOf<
|
||||
typeof getLatestLogEntryCategoryDatasetsStatsRequestPayloadRT
|
||||
>;
|
||||
|
||||
/**
|
||||
* response
|
||||
*/
|
||||
|
||||
const logEntryCategoriesDatasetStatsRT = rt.type({
|
||||
categorization_status: categorizerStatusRT,
|
||||
categorized_doc_count: rt.number,
|
||||
dataset: rt.string,
|
||||
dead_category_count: rt.number,
|
||||
failed_category_count: rt.number,
|
||||
frequent_category_count: rt.number,
|
||||
job_id: rt.string,
|
||||
log_time: rt.number,
|
||||
rare_category_count: rt.number,
|
||||
total_category_count: rt.number,
|
||||
});
|
||||
|
||||
export type LogEntryCategoriesDatasetStats = rt.TypeOf<typeof logEntryCategoriesDatasetStatsRT>;
|
||||
|
||||
export const getLatestLogEntryCategoryDatasetsStatsSuccessResponsePayloadRT = rt.intersection([
|
||||
rt.type({
|
||||
data: rt.type({
|
||||
datasetStats: rt.array(logEntryCategoriesDatasetStatsRT),
|
||||
}),
|
||||
}),
|
||||
rt.partial({
|
||||
timing: routeTimingMetadataRT,
|
||||
}),
|
||||
]);
|
||||
|
||||
export type GetLatestLogEntryCategoryDatasetsStatsSuccessResponsePayload = rt.TypeOf<
|
||||
typeof getLatestLogEntryCategoryDatasetsStatsSuccessResponsePayloadRT
|
||||
>;
|
||||
|
|
@ -5,6 +5,7 @@
|
|||
*/
|
||||
|
||||
export * from './log_analysis';
|
||||
export * from './log_analysis_quality';
|
||||
export * from './log_analysis_results';
|
||||
export * from './log_entry_rate_analysis';
|
||||
export * from './log_entry_categories_analysis';
|
||||
|
|
|
|||
|
|
@ -0,0 +1,42 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
interface ManyCategoriesWarningReason {
|
||||
type: 'manyCategories';
|
||||
categoriesDocumentRatio: number;
|
||||
}
|
||||
interface ManyDeadCategoriesWarningReason {
|
||||
type: 'manyDeadCategories';
|
||||
deadCategoriesRatio: number;
|
||||
}
|
||||
interface ManyRareCategoriesWarningReason {
|
||||
type: 'manyRareCategories';
|
||||
rareCategoriesRatio: number;
|
||||
}
|
||||
interface NoFrequentCategoriesWarningReason {
|
||||
type: 'noFrequentCategories';
|
||||
}
|
||||
interface SingleCategoryWarningReason {
|
||||
type: 'singleCategory';
|
||||
}
|
||||
|
||||
export type CategoryQualityWarningReason =
|
||||
| ManyCategoriesWarningReason
|
||||
| ManyDeadCategoriesWarningReason
|
||||
| ManyRareCategoriesWarningReason
|
||||
| NoFrequentCategoriesWarningReason
|
||||
| SingleCategoryWarningReason;
|
||||
|
||||
export type CategoryQualityWarningReasonType = CategoryQualityWarningReason['type'];
|
||||
|
||||
export interface CategoryQualityWarning {
|
||||
type: 'categoryQualityWarning';
|
||||
jobId: string;
|
||||
dataset: string;
|
||||
reasons: CategoryQualityWarningReason[];
|
||||
}
|
||||
|
||||
export type QualityWarning = CategoryQualityWarning;
|
||||
|
|
@ -31,6 +31,7 @@ export const JobConfigurationOutdatedCallout: React.FC<{
|
|||
values={{
|
||||
moduleName,
|
||||
}}
|
||||
tagName="p"
|
||||
/>
|
||||
</RecreateJobCallout>
|
||||
);
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ export const JobDefinitionOutdatedCallout: React.FC<{
|
|||
values={{
|
||||
moduleName,
|
||||
}}
|
||||
tagName="p"
|
||||
/>
|
||||
</RecreateJobCallout>
|
||||
);
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
*/
|
||||
|
||||
import React from 'react';
|
||||
import { QualityWarning } from '../../../containers/logs/log_analysis/log_analysis_module_types';
|
||||
import { QualityWarning } from '../../../../common/log_analysis';
|
||||
import { LogAnalysisJobProblemIndicator } from './log_analysis_job_problem_indicator';
|
||||
import { CategoryQualityWarnings } from './quality_warning_notices';
|
||||
|
||||
|
|
@ -41,6 +41,10 @@ export const CategoryJobNoticesSection: React.FC<{
|
|||
onRecreateMlJobForReconfiguration={onRecreateMlJobForReconfiguration}
|
||||
onRecreateMlJobForUpdate={onRecreateMlJobForUpdate}
|
||||
/>
|
||||
<CategoryQualityWarnings qualityWarnings={qualityWarnings} />
|
||||
<CategoryQualityWarnings
|
||||
hasSetupCapabilities={hasSetupCapabilities}
|
||||
qualityWarnings={qualityWarnings}
|
||||
onRecreateMlJob={onRecreateMlJobForReconfiguration}
|
||||
/>
|
||||
</>
|
||||
);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,68 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import { action } from '@storybook/addon-actions';
|
||||
import { storiesOf } from '@storybook/react';
|
||||
import React from 'react';
|
||||
import { EuiThemeProvider } from '../../../../../observability/public';
|
||||
import { QualityWarning } from '../../../../common/log_analysis';
|
||||
import { CategoryQualityWarnings } from './quality_warning_notices';
|
||||
|
||||
storiesOf('infra/logAnalysis/CategoryQualityWarnings', module)
|
||||
.addDecorator((renderStory) => <EuiThemeProvider>{renderStory()}</EuiThemeProvider>)
|
||||
.add('Partitioned warnings', () => {
|
||||
return (
|
||||
<CategoryQualityWarnings
|
||||
hasSetupCapabilities={true}
|
||||
onRecreateMlJob={action('on-recreate-ml-job')}
|
||||
qualityWarnings={partitionedQualityWarnings}
|
||||
/>
|
||||
);
|
||||
})
|
||||
.add('Unpartitioned warnings', () => {
|
||||
return (
|
||||
<CategoryQualityWarnings
|
||||
hasSetupCapabilities={true}
|
||||
onRecreateMlJob={action('on-recreate-ml-job')}
|
||||
qualityWarnings={unpartitionedQualityWarnings}
|
||||
/>
|
||||
);
|
||||
});
|
||||
|
||||
const partitionedQualityWarnings: QualityWarning[] = [
|
||||
{
|
||||
type: 'categoryQualityWarning',
|
||||
jobId: 'theMlJobId',
|
||||
dataset: 'first.dataset',
|
||||
reasons: [
|
||||
{ type: 'singleCategory' },
|
||||
{ type: 'manyRareCategories', rareCategoriesRatio: 0.95 },
|
||||
{ type: 'manyCategories', categoriesDocumentRatio: 0.7 },
|
||||
],
|
||||
},
|
||||
{
|
||||
type: 'categoryQualityWarning',
|
||||
jobId: 'theMlJobId',
|
||||
dataset: 'second.dataset',
|
||||
reasons: [
|
||||
{ type: 'noFrequentCategories' },
|
||||
{ type: 'manyDeadCategories', deadCategoriesRatio: 0.7 },
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
const unpartitionedQualityWarnings: QualityWarning[] = [
|
||||
{
|
||||
type: 'categoryQualityWarning',
|
||||
jobId: 'theMlJobId',
|
||||
dataset: '',
|
||||
reasons: [
|
||||
{ type: 'singleCategory' },
|
||||
{ type: 'manyRareCategories', rareCategoriesRatio: 0.95 },
|
||||
{ type: 'manyCategories', categoriesDocumentRatio: 0.7 },
|
||||
],
|
||||
},
|
||||
];
|
||||
|
|
@ -4,43 +4,89 @@
|
|||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import { EuiCallOut } from '@elastic/eui';
|
||||
import {
|
||||
EuiAccordion,
|
||||
EuiDescriptionList,
|
||||
EuiDescriptionListDescription,
|
||||
EuiDescriptionListTitle,
|
||||
EuiSpacer,
|
||||
htmlIdGenerator,
|
||||
} from '@elastic/eui';
|
||||
import { i18n } from '@kbn/i18n';
|
||||
import { FormattedMessage } from '@kbn/i18n/react';
|
||||
import React from 'react';
|
||||
import type {
|
||||
import groupBy from 'lodash/groupBy';
|
||||
import React, { Fragment, useState } from 'react';
|
||||
import { euiStyled } from '../../../../../observability/public';
|
||||
import {
|
||||
CategoryQualityWarning,
|
||||
CategoryQualityWarningReason,
|
||||
QualityWarning,
|
||||
} from '../../../containers/logs/log_analysis/log_analysis_module_types';
|
||||
getFriendlyNameForPartitionId,
|
||||
} from '../../../../common/log_analysis';
|
||||
import { RecreateJobCallout } from './recreate_job_callout';
|
||||
|
||||
export const CategoryQualityWarnings: React.FC<{ qualityWarnings: QualityWarning[] }> = ({
|
||||
qualityWarnings,
|
||||
}) => (
|
||||
<>
|
||||
{qualityWarnings.map((qualityWarning, qualityWarningIndex) => (
|
||||
<EuiCallOut
|
||||
key={`${qualityWarningIndex}`}
|
||||
title={categoryQualityWarningCalloutTitle}
|
||||
color="warning"
|
||||
iconType="alert"
|
||||
>
|
||||
<p>
|
||||
export const CategoryQualityWarnings: React.FC<{
|
||||
hasSetupCapabilities: boolean;
|
||||
onRecreateMlJob: () => void;
|
||||
qualityWarnings: CategoryQualityWarning[];
|
||||
}> = ({ hasSetupCapabilities, onRecreateMlJob, qualityWarnings }) => {
|
||||
const [detailAccordionId] = useState(htmlIdGenerator()());
|
||||
|
||||
const categoryQualityWarningsByJob = groupBy(qualityWarnings, 'jobId');
|
||||
|
||||
return (
|
||||
<>
|
||||
{Object.entries(categoryQualityWarningsByJob).map(([jobId, qualityWarningsForJob]) => (
|
||||
<RecreateJobCallout
|
||||
hasSetupCapabilities={hasSetupCapabilities}
|
||||
key={`quality-warnings-callout-${jobId}`}
|
||||
onRecreateMlJob={onRecreateMlJob}
|
||||
title={categoryQualityWarningCalloutTitle}
|
||||
>
|
||||
<FormattedMessage
|
||||
id="xpack.infra.logs.logEntryCategories.categoryQualityWarningCalloutMessage"
|
||||
defaultMessage="While analyzing the log messages we've detected some problems which might indicate a reduced quality of the categorization results."
|
||||
defaultMessage="While analyzing the log messages we've detected some problems which might indicate a reduced quality of the categorization results. Consider excluding the respective datasets from the analysis."
|
||||
tagName="p"
|
||||
/>
|
||||
</p>
|
||||
<ul>
|
||||
{qualityWarning.reasons.map((reason, reasonIndex) => (
|
||||
<li key={`${reasonIndex}`}>
|
||||
<CategoryQualityWarningReasonDescription reason={reason} />
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
</EuiCallOut>
|
||||
))}
|
||||
</>
|
||||
);
|
||||
<EuiAccordion
|
||||
id={detailAccordionId}
|
||||
buttonContent={
|
||||
<FormattedMessage
|
||||
id="xpack.infra.logs.logEntryCategories.categoryQualityWarningDetailsAccordionButtonLabel"
|
||||
defaultMessage="Details"
|
||||
/>
|
||||
}
|
||||
paddingSize="m"
|
||||
>
|
||||
<EuiDescriptionList>
|
||||
{qualityWarningsForJob.flatMap((qualityWarning) => (
|
||||
<Fragment key={`item-${getFriendlyNameForPartitionId(qualityWarning.dataset)}`}>
|
||||
<EuiDescriptionListTitle data-test-subj={`title-${qualityWarning.dataset}`}>
|
||||
{getFriendlyNameForPartitionId(qualityWarning.dataset)}
|
||||
</EuiDescriptionListTitle>
|
||||
{qualityWarning.reasons.map((reason) => (
|
||||
<QualityWarningReasonDescription
|
||||
key={`description-${reason.type}-${qualityWarning.dataset}`}
|
||||
data-test-subj={`description-${reason.type}-${qualityWarning.dataset}`}
|
||||
>
|
||||
<CategoryQualityWarningReasonDescription reason={reason} />
|
||||
</QualityWarningReasonDescription>
|
||||
))}
|
||||
</Fragment>
|
||||
))}
|
||||
</EuiDescriptionList>
|
||||
</EuiAccordion>
|
||||
<EuiSpacer size="l" />
|
||||
</RecreateJobCallout>
|
||||
))}
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
const QualityWarningReasonDescription = euiStyled(EuiDescriptionListDescription)`
|
||||
display: list-item;
|
||||
list-style-type: disc;
|
||||
margin-left: ${(props) => props.theme.eui.paddingSizes.m};
|
||||
`;
|
||||
|
||||
const categoryQualityWarningCalloutTitle = i18n.translate(
|
||||
'xpack.infra.logs.logEntryCategories.categoryQUalityWarningCalloutTitle',
|
||||
|
|
@ -49,7 +95,7 @@ const categoryQualityWarningCalloutTitle = i18n.translate(
|
|||
}
|
||||
);
|
||||
|
||||
const CategoryQualityWarningReasonDescription: React.FC<{
|
||||
export const CategoryQualityWarningReasonDescription: React.FC<{
|
||||
reason: CategoryQualityWarningReason;
|
||||
}> = ({ reason }) => {
|
||||
switch (reason.type) {
|
||||
|
|
@ -57,7 +103,7 @@ const CategoryQualityWarningReasonDescription: React.FC<{
|
|||
return (
|
||||
<FormattedMessage
|
||||
id="xpack.infra.logs.logEntryCategories.singleCategoryWarningReasonDescription"
|
||||
defaultMessage="The analysis couldn't extract more than a single category from the log message."
|
||||
defaultMessage="The analysis couldn't extract more than a single category from the log messages."
|
||||
/>
|
||||
);
|
||||
case 'manyRareCategories':
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ export const RecreateJobCallout: React.FC<{
|
|||
title?: React.ReactNode;
|
||||
}> = ({ children, hasSetupCapabilities, onRecreateMlJob, title }) => (
|
||||
<EuiCallOut color="warning" iconType="alert" title={title}>
|
||||
<p>{children}</p>
|
||||
{children}
|
||||
<RecreateJobButton
|
||||
color="warning"
|
||||
hasSetupCapabilities={hasSetupCapabilities}
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import { EuiDescribedFormGroup, EuiFormRow } from '@elastic/eui';
|
|||
import { i18n } from '@kbn/i18n';
|
||||
import { FormattedMessage } from '@kbn/i18n/react';
|
||||
import React, { useCallback } from 'react';
|
||||
import { QualityWarning } from '../../../../../common/log_analysis';
|
||||
import { LoadingOverlayWrapper } from '../../../loading_overlay_wrapper';
|
||||
import { IndexSetupRow } from './index_setup_row';
|
||||
import { AvailableIndex, ValidationIndicesError } from './validation';
|
||||
|
|
@ -17,12 +18,14 @@ export const AnalysisSetupIndicesForm: React.FunctionComponent<{
|
|||
indices: AvailableIndex[];
|
||||
isValidating: boolean;
|
||||
onChangeSelectedIndices: (selectedIndices: AvailableIndex[]) => void;
|
||||
previousQualityWarnings?: QualityWarning[];
|
||||
validationErrors?: ValidationIndicesError[];
|
||||
}> = ({
|
||||
disabled = false,
|
||||
indices,
|
||||
isValidating,
|
||||
onChangeSelectedIndices,
|
||||
previousQualityWarnings = [],
|
||||
validationErrors = [],
|
||||
}) => {
|
||||
const changeIsIndexSelected = useCallback(
|
||||
|
|
@ -81,6 +84,7 @@ export const AnalysisSetupIndicesForm: React.FunctionComponent<{
|
|||
key={index.name}
|
||||
onChangeIsSelected={changeIsIndexSelected}
|
||||
onChangeDatasetFilter={changeDatasetFilter}
|
||||
previousQualityWarnings={previousQualityWarnings}
|
||||
/>
|
||||
))}
|
||||
</>
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@
|
|||
import {
|
||||
EuiFilterButton,
|
||||
EuiFilterGroup,
|
||||
EuiIconTip,
|
||||
EuiPopover,
|
||||
EuiPopoverTitle,
|
||||
EuiSelectable,
|
||||
|
|
@ -14,11 +15,15 @@ import {
|
|||
} from '@elastic/eui';
|
||||
import { FormattedMessage } from '@kbn/i18n/react';
|
||||
import React, { useCallback, useMemo } from 'react';
|
||||
import { DatasetFilter } from '../../../../../common/log_analysis';
|
||||
import { DatasetFilter, QualityWarning } from '../../../../../common/log_analysis';
|
||||
import { useVisibilityState } from '../../../../utils/use_visibility_state';
|
||||
import { CategoryQualityWarningReasonDescription } from '../../log_analysis_job_status/quality_warning_notices';
|
||||
|
||||
export const IndexSetupDatasetFilter: React.FC<{
|
||||
availableDatasets: string[];
|
||||
availableDatasets: Array<{
|
||||
dataset: string;
|
||||
warnings: QualityWarning[];
|
||||
}>;
|
||||
datasetFilter: DatasetFilter;
|
||||
isDisabled?: boolean;
|
||||
onChangeDatasetFilter: (datasetFilter: DatasetFilter) => void;
|
||||
|
|
@ -40,12 +45,13 @@ export const IndexSetupDatasetFilter: React.FC<{
|
|||
[onChangeDatasetFilter]
|
||||
);
|
||||
|
||||
const selectableOptions: EuiSelectableOption[] = useMemo(
|
||||
const selectableOptions = useMemo<EuiSelectableOption[]>(
|
||||
() =>
|
||||
availableDatasets.map((datasetName) => ({
|
||||
label: datasetName,
|
||||
availableDatasets.map(({ dataset, warnings }) => ({
|
||||
label: dataset,
|
||||
append: warnings.length > 0 ? <DatasetWarningMarker warnings={warnings} /> : null,
|
||||
checked:
|
||||
datasetFilter.type === 'includeSome' && datasetFilter.datasets.includes(datasetName)
|
||||
datasetFilter.type === 'includeSome' && datasetFilter.datasets.includes(dataset)
|
||||
? 'on'
|
||||
: undefined,
|
||||
})),
|
||||
|
|
@ -86,3 +92,15 @@ export const IndexSetupDatasetFilter: React.FC<{
|
|||
</EuiFilterGroup>
|
||||
);
|
||||
};
|
||||
|
||||
const DatasetWarningMarker: React.FC<{ warnings: QualityWarning[] }> = ({ warnings }) => {
|
||||
const warningDescriptions = warnings.flatMap((warning) =>
|
||||
warning.type === 'categoryQualityWarning'
|
||||
? warning.reasons.map((reason) => (
|
||||
<CategoryQualityWarningReasonDescription key={reason.type} reason={reason} />
|
||||
))
|
||||
: []
|
||||
);
|
||||
|
||||
return <EuiIconTip content={warningDescriptions} type="alert" color="warning" />;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -4,10 +4,10 @@
|
|||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import { EuiCheckbox, EuiCode, EuiFlexGroup, EuiFlexItem, EuiIcon, EuiToolTip } from '@elastic/eui';
|
||||
import { EuiCheckbox, EuiCode, EuiFlexGroup, EuiFlexItem, EuiIconTip } from '@elastic/eui';
|
||||
import { FormattedMessage } from '@kbn/i18n/react';
|
||||
import React, { useCallback } from 'react';
|
||||
import { DatasetFilter } from '../../../../../common/log_analysis';
|
||||
import React, { useCallback, useMemo } from 'react';
|
||||
import { DatasetFilter, QualityWarning } from '../../../../../common/log_analysis';
|
||||
import { IndexSetupDatasetFilter } from './index_setup_dataset_filter';
|
||||
import { AvailableIndex, ValidationUIError } from './validation';
|
||||
|
||||
|
|
@ -16,7 +16,14 @@ export const IndexSetupRow: React.FC<{
|
|||
isDisabled: boolean;
|
||||
onChangeDatasetFilter: (indexName: string, datasetFilter: DatasetFilter) => void;
|
||||
onChangeIsSelected: (indexName: string, isSelected: boolean) => void;
|
||||
}> = ({ index, isDisabled, onChangeDatasetFilter, onChangeIsSelected }) => {
|
||||
previousQualityWarnings: QualityWarning[];
|
||||
}> = ({
|
||||
index,
|
||||
isDisabled,
|
||||
onChangeDatasetFilter,
|
||||
onChangeIsSelected,
|
||||
previousQualityWarnings,
|
||||
}) => {
|
||||
const changeIsSelected = useCallback(
|
||||
(event: React.ChangeEvent<HTMLInputElement>) => {
|
||||
onChangeIsSelected(index.name, event.currentTarget.checked);
|
||||
|
|
@ -29,6 +36,29 @@ export const IndexSetupRow: React.FC<{
|
|||
[index.name, onChangeDatasetFilter]
|
||||
);
|
||||
|
||||
const datasets = useMemo(
|
||||
() =>
|
||||
index.validity === 'valid'
|
||||
? index.availableDatasets.map((availableDataset) => ({
|
||||
dataset: availableDataset,
|
||||
warnings: previousQualityWarnings.filter(({ dataset }) => dataset === availableDataset),
|
||||
}))
|
||||
: [],
|
||||
[index, previousQualityWarnings]
|
||||
);
|
||||
|
||||
const datasetIndependentQualityWarnings = useMemo(
|
||||
() => previousQualityWarnings.filter(({ dataset }) => dataset === ''),
|
||||
[previousQualityWarnings]
|
||||
);
|
||||
|
||||
const hasWarnings = useMemo(
|
||||
() =>
|
||||
datasetIndependentQualityWarnings.length > 0 ||
|
||||
datasets.some(({ warnings }) => warnings.length > 0),
|
||||
[datasetIndependentQualityWarnings, datasets]
|
||||
);
|
||||
|
||||
const isSelected = index.validity === 'valid' && index.isSelected;
|
||||
|
||||
return (
|
||||
|
|
@ -37,7 +67,23 @@ export const IndexSetupRow: React.FC<{
|
|||
<EuiCheckbox
|
||||
key={index.name}
|
||||
id={index.name}
|
||||
label={<EuiCode>{index.name}</EuiCode>}
|
||||
label={
|
||||
<>
|
||||
<EuiCode>{index.name}</EuiCode>{' '}
|
||||
{index.validity === 'valid' && hasWarnings ? (
|
||||
<EuiIconTip
|
||||
content={
|
||||
<FormattedMessage
|
||||
id="xpack.infra.logs.analsysisSetup.indexQualityWarningTooltipMessage"
|
||||
defaultMessage="While analyzing the log messages from these indices we've detected some problems which might indicate a reduced quality of the results. Consider excluding these indices or problematic datasets from the analysis."
|
||||
/>
|
||||
}
|
||||
type="alert"
|
||||
color="warning"
|
||||
/>
|
||||
) : null}
|
||||
</>
|
||||
}
|
||||
onChange={changeIsSelected}
|
||||
checked={isSelected}
|
||||
disabled={isDisabled || index.validity === 'invalid'}
|
||||
|
|
@ -45,12 +91,10 @@ export const IndexSetupRow: React.FC<{
|
|||
</EuiFlexItem>
|
||||
<EuiFlexItem grow={false}>
|
||||
{index.validity === 'invalid' ? (
|
||||
<EuiToolTip content={formatValidationError(index.errors)}>
|
||||
<EuiIcon type="alert" color="danger" />
|
||||
</EuiToolTip>
|
||||
<EuiIconTip content={formatValidationError(index.errors)} type="alert" color="danger" />
|
||||
) : index.validity === 'valid' ? (
|
||||
<IndexSetupDatasetFilter
|
||||
availableDatasets={index.availableDatasets}
|
||||
availableDatasets={datasets}
|
||||
datasetFilter={index.datasetFilter}
|
||||
isDisabled={!isSelected || isDisabled}
|
||||
onChangeDatasetFilter={changeDatasetFilter}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,104 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import { actions } from '@storybook/addon-actions';
|
||||
import { storiesOf } from '@storybook/react';
|
||||
import React from 'react';
|
||||
import { EuiThemeProvider } from '../../../../../../observability/public';
|
||||
import { InitialConfigurationStep } from './initial_configuration_step';
|
||||
|
||||
storiesOf('infra/logAnalysis/SetupInitialConfigurationStep', module)
|
||||
.addDecorator((renderStory) => (
|
||||
<EuiThemeProvider>
|
||||
<div style={{ maxWidth: 800 }}>{renderStory()}</div>
|
||||
</EuiThemeProvider>
|
||||
))
|
||||
.add('Reconfiguration with partitioned warnings', () => {
|
||||
return (
|
||||
<InitialConfigurationStep
|
||||
{...storyActions}
|
||||
startTime={Date.now()}
|
||||
endTime={undefined}
|
||||
isValidating={false}
|
||||
setupStatus={{ type: 'required' }}
|
||||
validatedIndices={[
|
||||
{
|
||||
name: 'index-1-*',
|
||||
validity: 'valid',
|
||||
isSelected: true,
|
||||
datasetFilter: { type: 'includeAll' },
|
||||
availableDatasets: ['first', 'second', 'third'],
|
||||
},
|
||||
{
|
||||
name: 'index-2-*',
|
||||
validity: 'invalid',
|
||||
errors: [{ index: 'index-2-*', error: 'INDEX_NOT_FOUND' }],
|
||||
},
|
||||
]}
|
||||
previousQualityWarnings={[
|
||||
{
|
||||
type: 'categoryQualityWarning',
|
||||
jobId: 'job-1',
|
||||
dataset: 'second',
|
||||
reasons: [
|
||||
{ type: 'noFrequentCategories' },
|
||||
{ type: 'manyDeadCategories', deadCategoriesRatio: 0.9 },
|
||||
],
|
||||
},
|
||||
{
|
||||
type: 'categoryQualityWarning',
|
||||
jobId: 'job-1',
|
||||
dataset: 'third',
|
||||
reasons: [{ type: 'singleCategory' }],
|
||||
},
|
||||
]}
|
||||
/>
|
||||
);
|
||||
})
|
||||
.add('Reconfiguration with unpartitioned warnings', () => {
|
||||
return (
|
||||
<InitialConfigurationStep
|
||||
{...storyActions}
|
||||
startTime={Date.now()}
|
||||
endTime={undefined}
|
||||
isValidating={false}
|
||||
setupStatus={{ type: 'required' }}
|
||||
validatedIndices={[
|
||||
{
|
||||
name: 'index-1-*',
|
||||
validity: 'valid',
|
||||
isSelected: true,
|
||||
datasetFilter: { type: 'includeAll' },
|
||||
availableDatasets: ['first', 'second', 'third'],
|
||||
},
|
||||
{
|
||||
name: 'index-2-*',
|
||||
validity: 'invalid',
|
||||
errors: [{ index: 'index-2-*', error: 'INDEX_NOT_FOUND' }],
|
||||
},
|
||||
]}
|
||||
previousQualityWarnings={[
|
||||
{
|
||||
type: 'categoryQualityWarning',
|
||||
jobId: 'job-1',
|
||||
dataset: '',
|
||||
reasons: [
|
||||
{ type: 'noFrequentCategories' },
|
||||
{ type: 'manyDeadCategories', deadCategoriesRatio: 0.9 },
|
||||
],
|
||||
},
|
||||
{
|
||||
type: 'categoryQualityWarning',
|
||||
jobId: 'job-1',
|
||||
dataset: '',
|
||||
reasons: [{ type: 'singleCategory' }],
|
||||
},
|
||||
]}
|
||||
/>
|
||||
);
|
||||
});
|
||||
|
||||
const storyActions = actions('setStartTime', 'setEndTime', 'setValidatedIndices');
|
||||
|
|
@ -9,7 +9,7 @@ import { EuiContainedStepProps } from '@elastic/eui/src/components/steps/steps';
|
|||
import { i18n } from '@kbn/i18n';
|
||||
import { FormattedMessage } from '@kbn/i18n/react';
|
||||
import React, { useMemo } from 'react';
|
||||
import { SetupStatus } from '../../../../../common/log_analysis';
|
||||
import { QualityWarning, SetupStatus } from '../../../../../common/log_analysis';
|
||||
import { AnalysisSetupIndicesForm } from './analysis_setup_indices_form';
|
||||
import { AnalysisSetupTimerangeForm } from './analysis_setup_timerange_form';
|
||||
import {
|
||||
|
|
@ -31,6 +31,7 @@ interface InitialConfigurationStepProps {
|
|||
setupStatus: SetupStatus;
|
||||
setValidatedIndices: (selectedIndices: AvailableIndex[]) => void;
|
||||
validationErrors?: ValidationUIError[];
|
||||
previousQualityWarnings?: QualityWarning[];
|
||||
}
|
||||
|
||||
export const createInitialConfigurationStep = (
|
||||
|
|
@ -50,6 +51,7 @@ export const InitialConfigurationStep: React.FunctionComponent<InitialConfigurat
|
|||
setupStatus,
|
||||
setValidatedIndices,
|
||||
validationErrors = [],
|
||||
previousQualityWarnings = [],
|
||||
}: InitialConfigurationStepProps) => {
|
||||
const disabled = useMemo(() => !editableFormStatus.includes(setupStatus.type), [setupStatus]);
|
||||
|
||||
|
|
@ -75,6 +77,7 @@ export const InitialConfigurationStep: React.FunctionComponent<InitialConfigurat
|
|||
indices={validatedIndices}
|
||||
isValidating={isValidating}
|
||||
onChangeSelectedIndices={setValidatedIndices}
|
||||
previousQualityWarnings={previousQualityWarnings}
|
||||
validationErrors={indexValidationErrors}
|
||||
/>
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
|
||||
import { EuiSpacer, EuiSteps, EuiText, EuiTitle } from '@elastic/eui';
|
||||
import React, { useCallback, useMemo } from 'react';
|
||||
import { useMount } from 'react-use';
|
||||
import { useLogEntryCategoriesSetup } from '../../../../containers/logs/log_analysis/modules/log_entry_categories';
|
||||
import { createInitialConfigurationStep } from '../initial_configuration_step';
|
||||
import { createProcessStep } from '../process_step';
|
||||
|
|
@ -14,8 +15,10 @@ export const LogEntryCategoriesSetupView: React.FC<{
|
|||
onClose: () => void;
|
||||
}> = ({ onClose }) => {
|
||||
const {
|
||||
categoryQualityWarnings,
|
||||
cleanUpAndSetUp,
|
||||
endTime,
|
||||
fetchJobStatus,
|
||||
isValidating,
|
||||
lastSetupErrorMessages,
|
||||
moduleDescriptor,
|
||||
|
|
@ -30,6 +33,10 @@ export const LogEntryCategoriesSetupView: React.FC<{
|
|||
viewResults,
|
||||
} = useLogEntryCategoriesSetup();
|
||||
|
||||
useMount(() => {
|
||||
fetchJobStatus();
|
||||
});
|
||||
|
||||
const viewResultsAndClose = useCallback(() => {
|
||||
viewResults();
|
||||
onClose();
|
||||
|
|
@ -47,6 +54,7 @@ export const LogEntryCategoriesSetupView: React.FC<{
|
|||
setupStatus,
|
||||
setValidatedIndices,
|
||||
validationErrors,
|
||||
previousQualityWarnings: categoryQualityWarnings,
|
||||
}),
|
||||
createProcessStep({
|
||||
cleanUpAndSetUp,
|
||||
|
|
@ -58,6 +66,7 @@ export const LogEntryCategoriesSetupView: React.FC<{
|
|||
}),
|
||||
],
|
||||
[
|
||||
categoryQualityWarnings,
|
||||
cleanUpAndSetUp,
|
||||
endTime,
|
||||
isValidating,
|
||||
|
|
|
|||
|
|
@ -15,14 +15,16 @@ import {
|
|||
} from '@elastic/eui';
|
||||
import { FormattedMessage } from '@kbn/i18n/react';
|
||||
import React from 'react';
|
||||
import { LogEntryRateSetupView } from './log_entry_rate_setup_view';
|
||||
import { LogEntryCategoriesSetupView } from './log_entry_categories_setup_view';
|
||||
import { LogEntryRateSetupView } from './log_entry_rate_setup_view';
|
||||
import { LogAnalysisModuleList } from './module_list';
|
||||
import { useLogAnalysisSetupFlyoutStateContext } from './setup_flyout_state';
|
||||
import { ModuleId, moduleIds, useLogAnalysisSetupFlyoutStateContext } from './setup_flyout_state';
|
||||
|
||||
const FLYOUT_HEADING_ID = 'logAnalysisSetupFlyoutHeading';
|
||||
|
||||
export const LogAnalysisSetupFlyout: React.FC = () => {
|
||||
export const LogAnalysisSetupFlyout: React.FC<{
|
||||
allowedModules?: ModuleId[];
|
||||
}> = ({ allowedModules = moduleIds }) => {
|
||||
const {
|
||||
closeFlyout,
|
||||
flyoutView,
|
||||
|
|
@ -49,32 +51,58 @@ export const LogAnalysisSetupFlyout: React.FC = () => {
|
|||
<EuiFlyoutBody>
|
||||
{flyoutView.view === 'moduleList' ? (
|
||||
<LogAnalysisModuleList onViewModuleSetup={showModuleSetup} />
|
||||
) : flyoutView.view === 'moduleSetup' && flyoutView.module === 'logs_ui_analysis' ? (
|
||||
<LogAnalysisSetupFlyoutSubPage onViewModuleList={showModuleList}>
|
||||
<LogEntryRateSetupView onClose={closeFlyout} />
|
||||
</LogAnalysisSetupFlyoutSubPage>
|
||||
) : flyoutView.view === 'moduleSetup' && flyoutView.module === 'logs_ui_categories' ? (
|
||||
<LogAnalysisSetupFlyoutSubPage onViewModuleList={showModuleList}>
|
||||
<LogEntryCategoriesSetupView onClose={closeFlyout} />
|
||||
</LogAnalysisSetupFlyoutSubPage>
|
||||
) : flyoutView.view === 'moduleSetup' && allowedModules.includes(flyoutView.module) ? (
|
||||
<ModuleSetupView
|
||||
moduleId={flyoutView.module}
|
||||
onClose={closeFlyout}
|
||||
onViewModuleList={allowedModules.length > 1 ? showModuleList : undefined}
|
||||
/>
|
||||
) : null}
|
||||
</EuiFlyoutBody>
|
||||
</EuiFlyout>
|
||||
);
|
||||
};
|
||||
|
||||
const ModuleSetupView: React.FC<{
|
||||
moduleId: ModuleId;
|
||||
onClose: () => void;
|
||||
onViewModuleList?: () => void;
|
||||
}> = ({ moduleId, onClose, onViewModuleList }) => {
|
||||
switch (moduleId) {
|
||||
case 'logs_ui_analysis':
|
||||
return (
|
||||
<LogAnalysisSetupFlyoutSubPage onViewModuleList={onViewModuleList}>
|
||||
<LogEntryRateSetupView onClose={onClose} />
|
||||
</LogAnalysisSetupFlyoutSubPage>
|
||||
);
|
||||
case 'logs_ui_categories':
|
||||
return (
|
||||
<LogAnalysisSetupFlyoutSubPage onViewModuleList={onViewModuleList}>
|
||||
<LogEntryCategoriesSetupView onClose={onClose} />
|
||||
</LogAnalysisSetupFlyoutSubPage>
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
const LogAnalysisSetupFlyoutSubPage: React.FC<{
|
||||
onViewModuleList: () => void;
|
||||
onViewModuleList?: () => void;
|
||||
}> = ({ children, onViewModuleList }) => (
|
||||
<EuiFlexGroup alignItems="flexStart" direction="column" gutterSize="none">
|
||||
<EuiFlexItem grow={false}>
|
||||
<EuiButtonEmpty flush="left" iconSide="left" iconType="arrowLeft" onClick={onViewModuleList}>
|
||||
<FormattedMessage
|
||||
id="xpack.infra.logs.analysis.setupFlyoutGotoListButtonLabel"
|
||||
defaultMessage="All Machine Learning jobs"
|
||||
/>
|
||||
</EuiButtonEmpty>
|
||||
</EuiFlexItem>
|
||||
{onViewModuleList ? (
|
||||
<EuiFlexItem grow={false}>
|
||||
<EuiButtonEmpty
|
||||
flush="left"
|
||||
iconSide="left"
|
||||
iconType="arrowLeft"
|
||||
onClick={onViewModuleList}
|
||||
>
|
||||
<FormattedMessage
|
||||
id="xpack.infra.logs.analysis.setupFlyoutGotoListButtonLabel"
|
||||
defaultMessage="All Machine Learning jobs"
|
||||
/>
|
||||
</EuiButtonEmpty>
|
||||
</EuiFlexItem>
|
||||
) : null}
|
||||
<EuiFlexItem>{children}</EuiFlexItem>
|
||||
</EuiFlexGroup>
|
||||
);
|
||||
|
|
|
|||
|
|
@ -9,6 +9,8 @@ import { useState, useCallback } from 'react';
|
|||
|
||||
export type ModuleId = 'logs_ui_analysis' | 'logs_ui_categories';
|
||||
|
||||
export const moduleIds = ['logs_ui_analysis', 'logs_ui_categories'] as const;
|
||||
|
||||
type FlyoutView =
|
||||
| { view: 'hidden' }
|
||||
| { view: 'moduleList' }
|
||||
|
|
|
|||
|
|
@ -0,0 +1,47 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import { HttpHandler } from 'src/core/public';
|
||||
import {
|
||||
CategorizerStatus,
|
||||
getLatestLogEntryCategoryDatasetsStatsRequestPayloadRT,
|
||||
getLatestLogEntryCategoryDatasetsStatsSuccessResponsePayloadRT,
|
||||
LogEntryCategoriesDatasetStats,
|
||||
LOG_ANALYSIS_GET_LATEST_LOG_ENTRY_CATEGORY_DATASETS_STATS_PATH,
|
||||
} from '../../../../../common/http_api';
|
||||
import { decodeOrThrow } from '../../../../../common/runtime_types';
|
||||
|
||||
export { LogEntryCategoriesDatasetStats };
|
||||
|
||||
export const callGetLatestCategoriesDatasetsStatsAPI = async (
|
||||
{
|
||||
jobIds,
|
||||
startTime,
|
||||
endTime,
|
||||
includeCategorizerStatuses,
|
||||
}: {
|
||||
jobIds: string[];
|
||||
startTime: number;
|
||||
endTime: number;
|
||||
includeCategorizerStatuses: CategorizerStatus[];
|
||||
},
|
||||
fetch: HttpHandler
|
||||
) => {
|
||||
const response = await fetch(LOG_ANALYSIS_GET_LATEST_LOG_ENTRY_CATEGORY_DATASETS_STATS_PATH, {
|
||||
method: 'POST',
|
||||
body: JSON.stringify(
|
||||
getLatestLogEntryCategoryDatasetsStatsRequestPayloadRT.encode({
|
||||
data: {
|
||||
jobIds,
|
||||
timeRange: { startTime, endTime },
|
||||
includeCategorizerStatuses,
|
||||
},
|
||||
})
|
||||
),
|
||||
});
|
||||
|
||||
return decodeOrThrow(getLatestLogEntryCategoryDatasetsStatsSuccessResponsePayloadRT)(response);
|
||||
};
|
||||
|
|
@ -54,6 +54,17 @@ const jobStateRT = rt.keyof({
|
|||
opening: null,
|
||||
});
|
||||
|
||||
const jobAnalysisConfigRT = rt.partial({
|
||||
per_partition_categorization: rt.intersection([
|
||||
rt.type({
|
||||
enabled: rt.boolean,
|
||||
}),
|
||||
rt.partial({
|
||||
stop_on_warn: rt.boolean,
|
||||
}),
|
||||
]),
|
||||
});
|
||||
|
||||
const jobCategorizationStatusRT = rt.keyof({
|
||||
ok: null,
|
||||
warn: null,
|
||||
|
|
@ -64,6 +75,7 @@ const jobModelSizeStatsRT = rt.type({
|
|||
categorized_doc_count: rt.number,
|
||||
dead_category_count: rt.number,
|
||||
frequent_category_count: rt.number,
|
||||
log_time: rt.number,
|
||||
rare_category_count: rt.number,
|
||||
total_category_count: rt.number,
|
||||
});
|
||||
|
|
@ -79,6 +91,8 @@ export const jobSummaryRT = rt.intersection([
|
|||
datafeedIndices: rt.array(rt.string),
|
||||
datafeedState: datafeedStateRT,
|
||||
fullJob: rt.partial({
|
||||
analysis_config: jobAnalysisConfigRT,
|
||||
create_time: rt.number,
|
||||
custom_settings: jobCustomSettingsRT,
|
||||
finished_time: rt.number,
|
||||
model_size_stats: jobModelSizeStatsRT,
|
||||
|
|
|
|||
|
|
@ -50,43 +50,3 @@ export interface ModuleSourceConfiguration {
|
|||
spaceId: string;
|
||||
timestampField: string;
|
||||
}
|
||||
|
||||
interface ManyCategoriesWarningReason {
|
||||
type: 'manyCategories';
|
||||
categoriesDocumentRatio: number;
|
||||
}
|
||||
|
||||
interface ManyDeadCategoriesWarningReason {
|
||||
type: 'manyDeadCategories';
|
||||
deadCategoriesRatio: number;
|
||||
}
|
||||
|
||||
interface ManyRareCategoriesWarningReason {
|
||||
type: 'manyRareCategories';
|
||||
rareCategoriesRatio: number;
|
||||
}
|
||||
|
||||
interface NoFrequentCategoriesWarningReason {
|
||||
type: 'noFrequentCategories';
|
||||
}
|
||||
|
||||
interface SingleCategoryWarningReason {
|
||||
type: 'singleCategory';
|
||||
}
|
||||
|
||||
export type CategoryQualityWarningReason =
|
||||
| ManyCategoriesWarningReason
|
||||
| ManyDeadCategoriesWarningReason
|
||||
| ManyRareCategoriesWarningReason
|
||||
| NoFrequentCategoriesWarningReason
|
||||
| SingleCategoryWarningReason;
|
||||
|
||||
export type CategoryQualityWarningReasonType = CategoryQualityWarningReason['type'];
|
||||
|
||||
export interface CategoryQualityWarning {
|
||||
type: 'categoryQualityWarning';
|
||||
jobId: string;
|
||||
reasons: CategoryQualityWarningReason[];
|
||||
}
|
||||
|
||||
export type QualityWarning = CategoryQualityWarning;
|
||||
|
|
|
|||
|
|
@ -4,43 +4,124 @@
|
|||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import { useMemo } from 'react';
|
||||
import { useMemo, useState } from 'react';
|
||||
import { useDeepCompareEffect } from 'react-use';
|
||||
import {
|
||||
JobModelSizeStats,
|
||||
JobSummary,
|
||||
QualityWarning,
|
||||
CategoryQualityWarningReason,
|
||||
} from '../../log_analysis_module_types';
|
||||
QualityWarning,
|
||||
} from '../../../../../../common/log_analysis';
|
||||
import { useKibanaContextForPlugin } from '../../../../../hooks/use_kibana';
|
||||
import { useTrackedPromise } from '../../../../../utils/use_tracked_promise';
|
||||
import {
|
||||
callGetLatestCategoriesDatasetsStatsAPI,
|
||||
LogEntryCategoriesDatasetStats,
|
||||
} from '../../api/get_latest_categories_datasets_stats';
|
||||
import { JobModelSizeStats, JobSummary } from '../../log_analysis_module_types';
|
||||
|
||||
export const useLogEntryCategoriesQuality = ({ jobSummaries }: { jobSummaries: JobSummary[] }) => {
|
||||
const {
|
||||
services: {
|
||||
http: { fetch },
|
||||
},
|
||||
} = useKibanaContextForPlugin();
|
||||
|
||||
const [lastestWarnedDatasetsStats, setLatestWarnedDatasetsStats] = useState<
|
||||
LogEntryCategoriesDatasetStats[]
|
||||
>([]);
|
||||
|
||||
const jobSummariesWithCategoryWarnings = useMemo(
|
||||
() => jobSummaries.filter(isJobWithCategoryWarnings),
|
||||
[jobSummaries]
|
||||
);
|
||||
|
||||
const jobSummariesWithPartitionedCategoryWarnings = useMemo(
|
||||
() => jobSummariesWithCategoryWarnings.filter(isJobWithPartitionedCategories),
|
||||
[jobSummariesWithCategoryWarnings]
|
||||
);
|
||||
|
||||
const [fetchLatestWarnedDatasetsStatsRequest, fetchLatestWarnedDatasetsStats] = useTrackedPromise(
|
||||
{
|
||||
cancelPreviousOn: 'creation',
|
||||
createPromise: (
|
||||
statsIntervals: Array<{ jobId: string; startTime: number; endTime: number }>
|
||||
) =>
|
||||
Promise.all(
|
||||
statsIntervals.map(({ jobId, startTime, endTime }) =>
|
||||
callGetLatestCategoriesDatasetsStatsAPI(
|
||||
{ jobIds: [jobId], startTime, endTime, includeCategorizerStatuses: ['warn'] },
|
||||
fetch
|
||||
)
|
||||
)
|
||||
),
|
||||
onResolve: (results) => {
|
||||
setLatestWarnedDatasetsStats(results.flatMap(({ data: { datasetStats } }) => datasetStats));
|
||||
},
|
||||
},
|
||||
[]
|
||||
);
|
||||
|
||||
useDeepCompareEffect(() => {
|
||||
fetchLatestWarnedDatasetsStats(
|
||||
jobSummariesWithPartitionedCategoryWarnings.map((jobSummary) => ({
|
||||
jobId: jobSummary.id,
|
||||
startTime: jobSummary.fullJob?.create_time ?? 0,
|
||||
endTime: jobSummary.fullJob?.model_size_stats?.log_time ?? Date.now(),
|
||||
}))
|
||||
);
|
||||
}, [jobSummariesWithPartitionedCategoryWarnings]);
|
||||
|
||||
const categoryQualityWarnings: QualityWarning[] = useMemo(
|
||||
() =>
|
||||
jobSummaries
|
||||
.filter(
|
||||
(jobSummary) => jobSummary.fullJob?.model_size_stats?.categorization_status === 'warn'
|
||||
)
|
||||
() => [
|
||||
...jobSummariesWithCategoryWarnings
|
||||
.filter((jobSummary) => !isJobWithPartitionedCategories(jobSummary))
|
||||
.map((jobSummary) => ({
|
||||
type: 'categoryQualityWarning',
|
||||
type: 'categoryQualityWarning' as const,
|
||||
jobId: jobSummary.id,
|
||||
dataset: '',
|
||||
reasons: jobSummary.fullJob?.model_size_stats
|
||||
? getCategoryQualityWarningReasons(jobSummary.fullJob.model_size_stats)
|
||||
: [],
|
||||
})),
|
||||
[jobSummaries]
|
||||
...lastestWarnedDatasetsStats.map((datasetStats) => ({
|
||||
type: 'categoryQualityWarning' as const,
|
||||
jobId: datasetStats.job_id,
|
||||
dataset: datasetStats.dataset,
|
||||
reasons: getCategoryQualityWarningReasons(datasetStats),
|
||||
})),
|
||||
],
|
||||
[jobSummariesWithCategoryWarnings, lastestWarnedDatasetsStats]
|
||||
);
|
||||
|
||||
return {
|
||||
categoryQualityWarnings,
|
||||
lastLatestWarnedDatasetsStatsRequestErrors:
|
||||
fetchLatestWarnedDatasetsStatsRequest.state === 'rejected'
|
||||
? fetchLatestWarnedDatasetsStatsRequest.value
|
||||
: null,
|
||||
isLoadingCategoryQualityWarnings: fetchLatestWarnedDatasetsStatsRequest.state === 'pending',
|
||||
};
|
||||
};
|
||||
|
||||
const isJobWithCategoryWarnings = (jobSummary: JobSummary) =>
|
||||
jobSummary.fullJob?.model_size_stats?.categorization_status === 'warn';
|
||||
|
||||
const isJobWithPartitionedCategories = (jobSummary: JobSummary) =>
|
||||
jobSummary.fullJob?.analysis_config?.per_partition_categorization ?? false;
|
||||
|
||||
const getCategoryQualityWarningReasons = ({
|
||||
categorized_doc_count: categorizedDocCount,
|
||||
dead_category_count: deadCategoryCount,
|
||||
frequent_category_count: frequentCategoryCount,
|
||||
rare_category_count: rareCategoryCount,
|
||||
total_category_count: totalCategoryCount,
|
||||
}: JobModelSizeStats): CategoryQualityWarningReason[] => {
|
||||
}: Pick<
|
||||
JobModelSizeStats,
|
||||
| 'categorized_doc_count'
|
||||
| 'dead_category_count'
|
||||
| 'frequent_category_count'
|
||||
| 'rare_category_count'
|
||||
| 'total_category_count'
|
||||
>): CategoryQualityWarningReason[] => {
|
||||
const rareCategoriesRatio = rareCategoryCount / totalCategoryCount;
|
||||
const categoriesDocumentRatio = totalCategoryCount / categorizedDocCount;
|
||||
const deadCategoriesRatio = deadCategoryCount / totalCategoryCount;
|
||||
|
|
|
|||
|
|
@ -9,7 +9,9 @@ import { useLogEntryCategoriesModuleContext } from './use_log_entry_categories_m
|
|||
|
||||
export const useLogEntryCategoriesSetup = () => {
|
||||
const {
|
||||
categoryQualityWarnings,
|
||||
cleanUpAndSetUpModule,
|
||||
fetchJobStatus,
|
||||
lastSetupErrorMessages,
|
||||
moduleDescriptor,
|
||||
setUpModule,
|
||||
|
|
@ -37,8 +39,10 @@ export const useLogEntryCategoriesSetup = () => {
|
|||
});
|
||||
|
||||
return {
|
||||
categoryQualityWarnings,
|
||||
cleanUpAndSetUp,
|
||||
endTime,
|
||||
fetchJobStatus,
|
||||
isValidating,
|
||||
lastSetupErrorMessages,
|
||||
moduleDescriptor,
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
*/
|
||||
|
||||
import { i18n } from '@kbn/i18n';
|
||||
import React, { useCallback, useEffect, useState } from 'react';
|
||||
import React, { useCallback, useEffect } from 'react';
|
||||
import { isJobStatusWithResults } from '../../../../common/log_analysis';
|
||||
import { LoadingPage } from '../../../components/loading_page';
|
||||
import {
|
||||
|
|
@ -14,6 +14,10 @@ import {
|
|||
MissingSetupPrivilegesPrompt,
|
||||
SubscriptionSplashContent,
|
||||
} from '../../../components/logging/log_analysis_setup';
|
||||
import {
|
||||
LogAnalysisSetupFlyout,
|
||||
useLogAnalysisSetupFlyoutStateContext,
|
||||
} from '../../../components/logging/log_analysis_setup/setup_flyout';
|
||||
import { SourceErrorPage } from '../../../components/source_error_page';
|
||||
import { SourceLoadingPage } from '../../../components/source_loading_page';
|
||||
import { useLogAnalysisCapabilitiesContext } from '../../../containers/logs/log_analysis';
|
||||
|
|
@ -21,7 +25,6 @@ import { useLogEntryCategoriesModuleContext } from '../../../containers/logs/log
|
|||
import { useLogSourceContext } from '../../../containers/logs/log_source';
|
||||
import { LogEntryCategoriesResultsContent } from './page_results_content';
|
||||
import { LogEntryCategoriesSetupContent } from './page_setup_content';
|
||||
import { LogEntryCategoriesSetupFlyout } from './setup_flyout';
|
||||
|
||||
export const LogEntryCategoriesPageContent = () => {
|
||||
const {
|
||||
|
|
@ -40,9 +43,10 @@ export const LogEntryCategoriesPageContent = () => {
|
|||
|
||||
const { fetchJobStatus, setupStatus, jobStatus } = useLogEntryCategoriesModuleContext();
|
||||
|
||||
const [isFlyoutOpen, setIsFlyoutOpen] = useState<boolean>(false);
|
||||
const openFlyout = useCallback(() => setIsFlyoutOpen(true), []);
|
||||
const closeFlyout = useCallback(() => setIsFlyoutOpen(false), []);
|
||||
const { showModuleSetup } = useLogAnalysisSetupFlyoutStateContext();
|
||||
const showCategoriesModuleSetup = useCallback(() => showModuleSetup('logs_ui_categories'), [
|
||||
showModuleSetup,
|
||||
]);
|
||||
|
||||
useEffect(() => {
|
||||
if (hasLogAnalysisReadCapabilities) {
|
||||
|
|
@ -71,8 +75,8 @@ export const LogEntryCategoriesPageContent = () => {
|
|||
} else if (isJobStatusWithResults(jobStatus['log-entry-categories-count'])) {
|
||||
return (
|
||||
<>
|
||||
<LogEntryCategoriesResultsContent onOpenSetup={openFlyout} />
|
||||
<LogEntryCategoriesSetupFlyout isOpen={isFlyoutOpen} onClose={closeFlyout} />
|
||||
<LogEntryCategoriesResultsContent onOpenSetup={showCategoriesModuleSetup} />
|
||||
<LogAnalysisSetupFlyout allowedModules={allowedSetupModules} />
|
||||
</>
|
||||
);
|
||||
} else if (!hasLogAnalysisSetupCapabilities) {
|
||||
|
|
@ -80,9 +84,11 @@ export const LogEntryCategoriesPageContent = () => {
|
|||
} else {
|
||||
return (
|
||||
<>
|
||||
<LogEntryCategoriesSetupContent onOpenSetup={openFlyout} />
|
||||
<LogEntryCategoriesSetupFlyout isOpen={isFlyoutOpen} onClose={closeFlyout} />
|
||||
<LogEntryCategoriesSetupContent onOpenSetup={showCategoriesModuleSetup} />
|
||||
<LogAnalysisSetupFlyout allowedModules={allowedSetupModules} />
|
||||
</>
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
const allowedSetupModules = ['logs_ui_categories' as const];
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@
|
|||
*/
|
||||
|
||||
import React from 'react';
|
||||
import { LogAnalysisSetupFlyoutStateProvider } from '../../../components/logging/log_analysis_setup/setup_flyout';
|
||||
import { LogEntryCategoriesModuleProvider } from '../../../containers/logs/log_analysis/modules/log_entry_categories';
|
||||
import { useLogSourceContext } from '../../../containers/logs/log_source';
|
||||
import { useActiveKibanaSpace } from '../../../hooks/use_kibana_space';
|
||||
|
|
@ -27,7 +28,7 @@ export const LogEntryCategoriesPageProviders: React.FunctionComponent = ({ child
|
|||
spaceId={space.id}
|
||||
timestampField={sourceConfiguration.configuration.fields.timestamp}
|
||||
>
|
||||
{children}
|
||||
<LogAnalysisSetupFlyoutStateProvider>{children}</LogAnalysisSetupFlyoutStateProvider>
|
||||
</LogEntryCategoriesModuleProvider>
|
||||
);
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,128 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import {
|
||||
EuiFlyout,
|
||||
EuiFlyoutBody,
|
||||
EuiFlyoutHeader,
|
||||
EuiSpacer,
|
||||
EuiSteps,
|
||||
EuiText,
|
||||
EuiTitle,
|
||||
} from '@elastic/eui';
|
||||
import { FormattedMessage } from '@kbn/i18n/react';
|
||||
import React, { useCallback, useMemo } from 'react';
|
||||
import {
|
||||
createInitialConfigurationStep,
|
||||
createProcessStep,
|
||||
} from '../../../components/logging/log_analysis_setup';
|
||||
import { useLogEntryCategoriesSetup } from '../../../containers/logs/log_analysis/modules/log_entry_categories';
|
||||
|
||||
interface LogEntryCategoriesSetupFlyoutProps {
|
||||
isOpen: boolean;
|
||||
onClose: () => void;
|
||||
}
|
||||
|
||||
export const LogEntryCategoriesSetupFlyout: React.FC<LogEntryCategoriesSetupFlyoutProps> = ({
|
||||
isOpen,
|
||||
onClose,
|
||||
}) => {
|
||||
const {
|
||||
cleanUpAndSetUp,
|
||||
endTime,
|
||||
isValidating,
|
||||
lastSetupErrorMessages,
|
||||
setEndTime,
|
||||
setStartTime,
|
||||
setValidatedIndices,
|
||||
setUp,
|
||||
setupStatus,
|
||||
startTime,
|
||||
validatedIndices,
|
||||
validationErrors,
|
||||
viewResults,
|
||||
} = useLogEntryCategoriesSetup();
|
||||
|
||||
const viewResultsAndClose = useCallback(() => {
|
||||
viewResults();
|
||||
onClose();
|
||||
}, [viewResults, onClose]);
|
||||
|
||||
const steps = useMemo(
|
||||
() => [
|
||||
createInitialConfigurationStep({
|
||||
setStartTime,
|
||||
setEndTime,
|
||||
startTime,
|
||||
endTime,
|
||||
isValidating,
|
||||
validatedIndices,
|
||||
setupStatus,
|
||||
setValidatedIndices,
|
||||
validationErrors,
|
||||
}),
|
||||
createProcessStep({
|
||||
cleanUpAndSetUp,
|
||||
errorMessages: lastSetupErrorMessages,
|
||||
isConfigurationValid: validationErrors.length <= 0 && !isValidating,
|
||||
setUp,
|
||||
setupStatus,
|
||||
viewResults: viewResultsAndClose,
|
||||
}),
|
||||
],
|
||||
[
|
||||
cleanUpAndSetUp,
|
||||
endTime,
|
||||
isValidating,
|
||||
lastSetupErrorMessages,
|
||||
setEndTime,
|
||||
setStartTime,
|
||||
setUp,
|
||||
setValidatedIndices,
|
||||
setupStatus,
|
||||
startTime,
|
||||
validatedIndices,
|
||||
validationErrors,
|
||||
viewResultsAndClose,
|
||||
]
|
||||
);
|
||||
|
||||
if (!isOpen) {
|
||||
return null;
|
||||
}
|
||||
return (
|
||||
<EuiFlyout onClose={onClose}>
|
||||
<EuiFlyoutHeader hasBorder>
|
||||
<EuiTitle size="s">
|
||||
<h3>
|
||||
<FormattedMessage
|
||||
id="xpack.infra.logs.setupFlyout.setupFlyoutTitle"
|
||||
defaultMessage="Anomaly detection with Machine Learning"
|
||||
/>
|
||||
</h3>
|
||||
</EuiTitle>
|
||||
</EuiFlyoutHeader>
|
||||
<EuiFlyoutBody>
|
||||
<EuiTitle size="s">
|
||||
<h3>
|
||||
<FormattedMessage
|
||||
id="xpack.infra.logs.setupFlyout.logCategoriesTitle"
|
||||
defaultMessage="Log categories"
|
||||
/>
|
||||
</h3>
|
||||
</EuiTitle>
|
||||
<EuiText size="s">
|
||||
<FormattedMessage
|
||||
id="xpack.infra.logs.setupFlyout.logCategoriesDescription"
|
||||
defaultMessage="Use Machine Learning to automatically categorize log messages."
|
||||
/>
|
||||
</EuiText>
|
||||
<EuiSpacer />
|
||||
<EuiSteps steps={steps} />
|
||||
</EuiFlyoutBody>
|
||||
</EuiFlyout>
|
||||
);
|
||||
};
|
||||
|
|
@ -13,6 +13,7 @@ import { InfraBackendLibs } from './lib/infra_types';
|
|||
import {
|
||||
initGetLogEntryCategoriesRoute,
|
||||
initGetLogEntryCategoryDatasetsRoute,
|
||||
initGetLogEntryCategoryDatasetsStatsRoute,
|
||||
initGetLogEntryCategoryExamplesRoute,
|
||||
initGetLogEntryRateRoute,
|
||||
initGetLogEntryExamplesRoute,
|
||||
|
|
@ -54,6 +55,7 @@ export const initInfraServer = (libs: InfraBackendLibs) => {
|
|||
initIpToHostName(libs);
|
||||
initGetLogEntryCategoriesRoute(libs);
|
||||
initGetLogEntryCategoryDatasetsRoute(libs);
|
||||
initGetLogEntryCategoryDatasetsStatsRoute(libs);
|
||||
initGetLogEntryCategoryExamplesRoute(libs);
|
||||
initGetLogEntryRateRoute(libs);
|
||||
initGetLogEntryAnomaliesRoute(libs);
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ export async function fetchMlJob(mlAnomalyDetectors: MlAnomalyDetectors, jobId:
|
|||
};
|
||||
}
|
||||
|
||||
const COMPOSITE_AGGREGATION_BATCH_SIZE = 1000;
|
||||
export const COMPOSITE_AGGREGATION_BATCH_SIZE = 1000;
|
||||
|
||||
// Finds datasets related to ML job ids
|
||||
export async function getLogEntryDatasets(
|
||||
|
|
|
|||
|
|
@ -6,5 +6,6 @@
|
|||
|
||||
export * from './errors';
|
||||
export * from './log_entry_categories_analysis';
|
||||
export * from './log_entry_categories_datasets_stats';
|
||||
export * from './log_entry_rate_analysis';
|
||||
export * from './log_entry_anomalies';
|
||||
|
|
|
|||
|
|
@ -0,0 +1,94 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import { startTracingSpan } from '../../../common/performance_tracing';
|
||||
import { decodeOrThrow } from '../../../common/runtime_types';
|
||||
import type { MlAnomalyDetectors, MlSystem } from '../../types';
|
||||
import { COMPOSITE_AGGREGATION_BATCH_SIZE } from './common';
|
||||
import {
|
||||
CompositeDatasetKey,
|
||||
createLatestLogEntryCategoriesDatasetsStatsQuery,
|
||||
latestLogEntryCategoriesDatasetsStatsResponseRT,
|
||||
LogEntryCategoryDatasetStatsBucket,
|
||||
} from './queries/latest_log_entry_categories_datasets_stats';
|
||||
|
||||
export async function getLatestLogEntriesCategoriesDatasetsStats(
|
||||
context: {
|
||||
infra: {
|
||||
mlAnomalyDetectors: MlAnomalyDetectors;
|
||||
mlSystem: MlSystem;
|
||||
};
|
||||
},
|
||||
jobIds: string[],
|
||||
startTime: number,
|
||||
endTime: number,
|
||||
includeCategorizerStatuses: Array<'ok' | 'warn'> = []
|
||||
) {
|
||||
const finalizeLogEntryCategoriesDatasetsStats = startTracingSpan('get categories datasets stats');
|
||||
|
||||
let latestLogEntryCategoriesDatasetsStatsBuckets: LogEntryCategoryDatasetStatsBucket[] = [];
|
||||
let afterLatestBatchKey: CompositeDatasetKey | undefined;
|
||||
|
||||
while (true) {
|
||||
const latestLogEntryCategoriesDatasetsStatsResponse = await context.infra.mlSystem.mlAnomalySearch(
|
||||
createLatestLogEntryCategoriesDatasetsStatsQuery(
|
||||
jobIds,
|
||||
startTime,
|
||||
endTime,
|
||||
COMPOSITE_AGGREGATION_BATCH_SIZE,
|
||||
afterLatestBatchKey
|
||||
)
|
||||
);
|
||||
|
||||
const { after_key: afterKey, buckets: latestBatchBuckets = [] } =
|
||||
decodeOrThrow(latestLogEntryCategoriesDatasetsStatsResponseRT)(
|
||||
latestLogEntryCategoriesDatasetsStatsResponse
|
||||
).aggregations?.dataset_composite_terms ?? {};
|
||||
|
||||
const latestIncludedBatchBuckets =
|
||||
includeCategorizerStatuses.length > 0
|
||||
? latestBatchBuckets.filter((bucket) =>
|
||||
bucket.categorizer_stats_top_hits.hits.hits.some((hit) =>
|
||||
includeCategorizerStatuses.includes(hit._source.categorization_status)
|
||||
)
|
||||
)
|
||||
: latestBatchBuckets;
|
||||
|
||||
latestLogEntryCategoriesDatasetsStatsBuckets = [
|
||||
...latestLogEntryCategoriesDatasetsStatsBuckets,
|
||||
...latestIncludedBatchBuckets,
|
||||
];
|
||||
|
||||
afterLatestBatchKey = afterKey;
|
||||
if (afterKey == null || latestBatchBuckets.length < COMPOSITE_AGGREGATION_BATCH_SIZE) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
const logEntryCategoriesDatasetsStatsSpan = finalizeLogEntryCategoriesDatasetsStats();
|
||||
|
||||
return {
|
||||
data: latestLogEntryCategoriesDatasetsStatsBuckets.map((bucket) => {
|
||||
const latestHitSource = bucket.categorizer_stats_top_hits.hits.hits[0]._source;
|
||||
|
||||
return {
|
||||
categorization_status: latestHitSource.categorization_status,
|
||||
categorized_doc_count: latestHitSource.categorized_doc_count,
|
||||
dataset: bucket.key.dataset ?? '',
|
||||
dead_category_count: latestHitSource.dead_category_count,
|
||||
failed_category_count: latestHitSource.failed_category_count,
|
||||
frequent_category_count: latestHitSource.frequent_category_count,
|
||||
job_id: latestHitSource.job_id,
|
||||
log_time: latestHitSource.log_time,
|
||||
rare_category_count: latestHitSource.rare_category_count,
|
||||
total_category_count: latestHitSource.total_category_count,
|
||||
};
|
||||
}),
|
||||
timing: {
|
||||
spans: [logEntryCategoriesDatasetsStatsSpan],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
|
@ -40,7 +40,20 @@ export const createTimeRangeFilters = (startTime: number, endTime: number) => [
|
|||
},
|
||||
];
|
||||
|
||||
export const createResultTypeFilters = (resultTypes: Array<'model_plot' | 'record'>) => [
|
||||
export const createLogTimeRangeFilters = (startTime: number, endTime: number) => [
|
||||
{
|
||||
range: {
|
||||
log_time: {
|
||||
gte: startTime,
|
||||
lte: endTime,
|
||||
},
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
export const createResultTypeFilters = (
|
||||
resultTypes: Array<'categorizer_stats' | 'model_plot' | 'record'>
|
||||
) => [
|
||||
{
|
||||
terms: {
|
||||
result_type: resultTypes,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,133 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import * as rt from 'io-ts';
|
||||
import { commonSearchSuccessResponseFieldsRT } from '../../../utils/elasticsearch_runtime_types';
|
||||
import {
|
||||
createJobIdsFilters,
|
||||
createResultTypeFilters,
|
||||
defaultRequestParameters,
|
||||
createLogTimeRangeFilters,
|
||||
} from './common';
|
||||
|
||||
export const createLatestLogEntryCategoriesDatasetsStatsQuery = (
|
||||
logEntryCategoriesJobIds: string[],
|
||||
startTime: number,
|
||||
endTime: number,
|
||||
size: number,
|
||||
afterKey?: CompositeDatasetKey
|
||||
) => ({
|
||||
...defaultRequestParameters,
|
||||
body: {
|
||||
query: {
|
||||
bool: {
|
||||
filter: [
|
||||
...createJobIdsFilters(logEntryCategoriesJobIds),
|
||||
...createResultTypeFilters(['categorizer_stats']),
|
||||
...createLogTimeRangeFilters(startTime, endTime),
|
||||
],
|
||||
},
|
||||
},
|
||||
aggregations: {
|
||||
dataset_composite_terms: {
|
||||
composite: {
|
||||
after: afterKey,
|
||||
size,
|
||||
sources: [
|
||||
{
|
||||
dataset: {
|
||||
terms: {
|
||||
field: 'partition_field_value',
|
||||
missing_bucket: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
aggs: {
|
||||
categorizer_stats_top_hits: {
|
||||
top_hits: {
|
||||
size: 1,
|
||||
sort: [
|
||||
{
|
||||
log_time: 'desc',
|
||||
},
|
||||
],
|
||||
_source: [
|
||||
'categorization_status',
|
||||
'categorized_doc_count',
|
||||
'dead_category_count',
|
||||
'failed_category_count',
|
||||
'frequent_category_count',
|
||||
'job_id',
|
||||
'log_time',
|
||||
'rare_category_count',
|
||||
'total_category_count',
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
size: 0,
|
||||
});
|
||||
|
||||
export const logEntryCategoryStatusRT = rt.keyof({
|
||||
ok: null,
|
||||
warn: null,
|
||||
});
|
||||
|
||||
export const logEntryCategorizerStatsHitRT = rt.type({
|
||||
_source: rt.type({
|
||||
categorization_status: logEntryCategoryStatusRT,
|
||||
categorized_doc_count: rt.number,
|
||||
dead_category_count: rt.number,
|
||||
failed_category_count: rt.number,
|
||||
frequent_category_count: rt.number,
|
||||
job_id: rt.string,
|
||||
log_time: rt.number,
|
||||
rare_category_count: rt.number,
|
||||
total_category_count: rt.number,
|
||||
}),
|
||||
});
|
||||
|
||||
export type LogEntryCategorizerStatsHit = rt.TypeOf<typeof logEntryCategorizerStatsHitRT>;
|
||||
|
||||
const compositeDatasetKeyRT = rt.type({
|
||||
dataset: rt.union([rt.string, rt.null]),
|
||||
});
|
||||
|
||||
export type CompositeDatasetKey = rt.TypeOf<typeof compositeDatasetKeyRT>;
|
||||
|
||||
const logEntryCategoryDatasetStatsBucketRT = rt.type({
|
||||
key: compositeDatasetKeyRT,
|
||||
categorizer_stats_top_hits: rt.type({
|
||||
hits: rt.type({
|
||||
hits: rt.array(logEntryCategorizerStatsHitRT),
|
||||
}),
|
||||
}),
|
||||
});
|
||||
|
||||
export type LogEntryCategoryDatasetStatsBucket = rt.TypeOf<
|
||||
typeof logEntryCategoryDatasetStatsBucketRT
|
||||
>;
|
||||
|
||||
export const latestLogEntryCategoriesDatasetsStatsResponseRT = rt.intersection([
|
||||
commonSearchSuccessResponseFieldsRT,
|
||||
rt.partial({
|
||||
aggregations: rt.type({
|
||||
dataset_composite_terms: rt.type({
|
||||
after_key: compositeDatasetKeyRT,
|
||||
buckets: rt.array(logEntryCategoryDatasetStatsBucketRT),
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
]);
|
||||
|
||||
export type LatestLogEntryCategoriesDatasetsStatsResponse = rt.TypeOf<
|
||||
typeof latestLogEntryCategoriesDatasetsStatsResponseRT
|
||||
>;
|
||||
|
|
@ -6,6 +6,7 @@
|
|||
|
||||
export * from './log_entry_categories';
|
||||
export * from './log_entry_category_datasets';
|
||||
export * from './log_entry_category_datasets_stats';
|
||||
export * from './log_entry_category_examples';
|
||||
export * from './log_entry_rate';
|
||||
export * from './log_entry_examples';
|
||||
|
|
|
|||
|
|
@ -0,0 +1,79 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import Boom from 'boom';
|
||||
import {
|
||||
getLatestLogEntryCategoryDatasetsStatsRequestPayloadRT,
|
||||
getLatestLogEntryCategoryDatasetsStatsSuccessResponsePayloadRT,
|
||||
LOG_ANALYSIS_GET_LATEST_LOG_ENTRY_CATEGORY_DATASETS_STATS_PATH,
|
||||
} from '../../../../common/http_api/log_analysis';
|
||||
import { createValidationFunction } from '../../../../common/runtime_types';
|
||||
import type { InfraBackendLibs } from '../../../lib/infra_types';
|
||||
import { getLatestLogEntriesCategoriesDatasetsStats } from '../../../lib/log_analysis';
|
||||
import { isMlPrivilegesError } from '../../../lib/log_analysis/errors';
|
||||
import { assertHasInfraMlPlugins } from '../../../utils/request_context';
|
||||
|
||||
export const initGetLogEntryCategoryDatasetsStatsRoute = ({ framework }: InfraBackendLibs) => {
|
||||
framework.registerRoute(
|
||||
{
|
||||
method: 'post',
|
||||
path: LOG_ANALYSIS_GET_LATEST_LOG_ENTRY_CATEGORY_DATASETS_STATS_PATH,
|
||||
validate: {
|
||||
body: createValidationFunction(getLatestLogEntryCategoryDatasetsStatsRequestPayloadRT),
|
||||
},
|
||||
},
|
||||
framework.router.handleLegacyErrors(async (requestContext, request, response) => {
|
||||
const {
|
||||
data: {
|
||||
jobIds,
|
||||
timeRange: { startTime, endTime },
|
||||
includeCategorizerStatuses,
|
||||
},
|
||||
} = request.body;
|
||||
|
||||
try {
|
||||
assertHasInfraMlPlugins(requestContext);
|
||||
|
||||
const { data: datasetStats, timing } = await getLatestLogEntriesCategoriesDatasetsStats(
|
||||
requestContext,
|
||||
jobIds,
|
||||
startTime,
|
||||
endTime,
|
||||
includeCategorizerStatuses
|
||||
);
|
||||
|
||||
return response.ok({
|
||||
body: getLatestLogEntryCategoryDatasetsStatsSuccessResponsePayloadRT.encode({
|
||||
data: {
|
||||
datasetStats,
|
||||
},
|
||||
timing,
|
||||
}),
|
||||
});
|
||||
} catch (error) {
|
||||
if (Boom.isBoom(error)) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
if (isMlPrivilegesError(error)) {
|
||||
return response.customError({
|
||||
statusCode: 403,
|
||||
body: {
|
||||
message: error.message,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return response.customError({
|
||||
statusCode: error.statusCode ?? 500,
|
||||
body: {
|
||||
message: error.message ?? 'An unexpected error occurred',
|
||||
},
|
||||
});
|
||||
}
|
||||
})
|
||||
);
|
||||
};
|
||||
|
|
@ -14,7 +14,11 @@
|
|||
"use_null": true
|
||||
}
|
||||
],
|
||||
"influencers": ["event.dataset", "mlcategory"]
|
||||
"influencers": ["event.dataset", "mlcategory"],
|
||||
"per_partition_categorization": {
|
||||
"enabled": true,
|
||||
"stop_on_warn": false
|
||||
}
|
||||
},
|
||||
"analysis_limits": {
|
||||
"model_memory_limit": "100mb",
|
||||
|
|
@ -29,6 +33,6 @@
|
|||
},
|
||||
"custom_settings": {
|
||||
"created_by": "ml-module-logs-ui-categories",
|
||||
"job_revision": 0
|
||||
"job_revision": 1
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8486,9 +8486,6 @@
|
|||
"xpack.infra.logs.search.searchInLogsAriaLabel": "検索",
|
||||
"xpack.infra.logs.search.searchInLogsPlaceholder": "検索",
|
||||
"xpack.infra.logs.searchResultTooltip": "{bucketCount, plural, one {# 件のハイライトされたエントリー} other {# 件のハイライトされたエントリー}}",
|
||||
"xpack.infra.logs.setupFlyout.logCategoriesDescription": "機械学習を使用して、ログメッセージを自動的に分類します。",
|
||||
"xpack.infra.logs.setupFlyout.logCategoriesTitle": "ログカテゴリー",
|
||||
"xpack.infra.logs.setupFlyout.setupFlyoutTitle": "機械学習を使用した異常検知",
|
||||
"xpack.infra.logs.showingEntriesFromTimestamp": "{timestamp} 以降のエントリーを表示中",
|
||||
"xpack.infra.logs.showingEntriesUntilTimestamp": "{timestamp} までのエントリーを表示中",
|
||||
"xpack.infra.logs.startStreamingButtonLabel": "ライブストリーム",
|
||||
|
|
|
|||
|
|
@ -8491,9 +8491,6 @@
|
|||
"xpack.infra.logs.search.searchInLogsAriaLabel": "搜索",
|
||||
"xpack.infra.logs.search.searchInLogsPlaceholder": "搜索",
|
||||
"xpack.infra.logs.searchResultTooltip": "{bucketCount, plural, one {# 个高亮条目} other {# 个高亮条目}}",
|
||||
"xpack.infra.logs.setupFlyout.logCategoriesDescription": "使用 Machine Learning 自动归类日志消息。",
|
||||
"xpack.infra.logs.setupFlyout.logCategoriesTitle": "日志类别",
|
||||
"xpack.infra.logs.setupFlyout.setupFlyoutTitle": "通过 Machine Learning 检测异常",
|
||||
"xpack.infra.logs.showingEntriesFromTimestamp": "正在显示自 {timestamp} 起的条目",
|
||||
"xpack.infra.logs.showingEntriesUntilTimestamp": "正在显示截止于 {timestamp} 的条目",
|
||||
"xpack.infra.logs.startStreamingButtonLabel": "实时流式传输",
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue