[ML] Inference models management (#74978)

* [ML] init tabs

* [ML] init inference API service in UI

* [ML] server-side routes

* [ML] basic table

* [ML] support deletion

* [ML] delete multiple models

* [ML] WIP expanded row

* [ML] fix types

* [ML] expanded row

* [ML] fix types

* [ML] fix i18n id

* [ML] change server-side permission check

* [ML] refactor types

* [ML] show success toast on model deletion, fix models counter

* [ML] update expanded row

* [ML] pipelines stats

* [ML] use refresh observable

* [ML] endpoint to fetch associated pipelines

* [ML] update the endpoint to fetch associated pipelines

* [ML] show pipelines definition in expanded row

* [ML] change stats layout

* [ML] fix headers

* [ML] change breadcrumb title

* [ML] fetch models config with pipelines

* [ML] change default size to 1000

* [ML] fix collections keys, fix double fetch on initial page load

* [ML] adjust models deletion text

* [ML] fix DFA jobs on the management page

* [ML] small tabs in expanded row

* [ML] fix headers text

* [ML] fix models fetching without pipelines get permissions

* [ML] stats rendering as a description list

* [ML] fix i18n id

* [ML] remove an extra copyright comment, add selectable messages

* [ML] update stats on refresh
This commit is contained in:
Dima Arnautov 2020-08-19 16:22:26 +02:00 committed by GitHub
parent 774c04a2d5
commit 8f7d213944
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
28 changed files with 1679 additions and 97 deletions

View file

@ -5,7 +5,77 @@
*/
import { CustomHttpResponseOptions, ResponseError } from 'kibana/server';
export interface DeleteDataFrameAnalyticsWithIndexStatus {
success: boolean;
error?: CustomHttpResponseOptions<ResponseError>;
}
export type IndexName = string;
export type DataFrameAnalyticsId = string;
export interface OutlierAnalysis {
[key: string]: {};
outlier_detection: {};
}
interface Regression {
dependent_variable: string;
training_percent?: number;
num_top_feature_importance_values?: number;
prediction_field_name?: string;
}
interface Classification {
dependent_variable: string;
training_percent?: number;
num_top_classes?: string;
num_top_feature_importance_values?: number;
prediction_field_name?: string;
}
export interface RegressionAnalysis {
[key: string]: Regression;
regression: Regression;
}
export interface ClassificationAnalysis {
[key: string]: Classification;
classification: Classification;
}
interface GenericAnalysis {
[key: string]: Record<string, any>;
}
export type AnalysisConfig =
| OutlierAnalysis
| RegressionAnalysis
| ClassificationAnalysis
| GenericAnalysis;
export interface DataFrameAnalyticsConfig {
id: DataFrameAnalyticsId;
description?: string;
dest: {
index: IndexName;
results_field: string;
};
source: {
index: IndexName | IndexName[];
query?: any;
};
analysis: AnalysisConfig;
analyzed_fields: {
includes: string[];
excludes: string[];
};
model_memory_limit: string;
max_num_threads?: number;
create_time: number;
version: string;
allow_lazy_start?: boolean;
}

View file

@ -0,0 +1,81 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { DataFrameAnalyticsConfig } from './data_frame_analytics';
export interface IngestStats {
count: number;
time_in_millis: number;
current: number;
failed: number;
}
export interface TrainedModelStat {
model_id: string;
pipeline_count: number;
inference_stats?: {
failure_count: number;
inference_count: number;
cache_miss_count: number;
missing_all_fields_count: number;
timestamp: number;
};
ingest?: {
total: IngestStats;
pipelines: Record<
string,
IngestStats & {
processors: Array<
Record<
string,
{
// TODO use type from ingest_pipelines plugin
type: string;
stats: IngestStats;
}
>
>;
}
>;
};
}
export interface ModelConfigResponse {
created_by: string;
create_time: string;
default_field_map: Record<string, string>;
estimated_heap_memory_usage_bytes: number;
estimated_operations: number;
license_level: string;
metadata?:
| {
analytics_config: DataFrameAnalyticsConfig;
input: any;
}
| Record<string, any>;
model_id: string;
tags: string;
version: string;
inference_config?: Record<string, any>;
pipelines?: Record<string, PipelineDefinition> | null;
}
export interface PipelineDefinition {
processors?: Array<Record<string, any>>;
description?: string;
}
export interface ModelPipelines {
model_id: string;
pipelines: Record<string, PipelineDefinition>;
}
/**
* Get inference response from the ES endpoint
*/
export interface InferenceConfigResponse {
trained_model_configs: ModelConfigResponse[];
}

View file

@ -4,4 +4,4 @@
* you may not use this file except in compliance with the Elastic License.
*/
export { StatsBar, AnalyticStatsBarStats, JobStatsBarStats } from './stats_bar';
export { StatsBar, AnalyticStatsBarStats, JobStatsBarStats, ModelsBarStats } from './stats_bar';

View file

@ -23,7 +23,11 @@ export interface AnalyticStatsBarStats extends Stats {
stopped: StatsBarStat;
}
export type StatsBarStats = JobStatsBarStats | AnalyticStatsBarStats;
export interface ModelsBarStats {
total: StatsBarStat;
}
export type StatsBarStats = JobStatsBarStats | AnalyticStatsBarStats | ModelsBarStats;
type StatsKey = keyof StatsBarStats;
interface StatsBarProps {

View file

@ -5,18 +5,21 @@
*/
import { useEffect } from 'react';
import { BehaviorSubject } from 'rxjs';
import { filter, distinctUntilChanged } from 'rxjs/operators';
import { Subscription } from 'rxjs';
import { BehaviorSubject, Subscription } from 'rxjs';
import { distinctUntilChanged, filter } from 'rxjs/operators';
import { cloneDeep } from 'lodash';
import { ml } from '../../services/ml_api_service';
import { Dictionary } from '../../../../common/types/common';
import { getErrorMessage } from '../../../../common/util/errors';
import { SavedSearchQuery } from '../../contexts/ml';
import {
AnalysisConfig,
ClassificationAnalysis,
OutlierAnalysis,
RegressionAnalysis,
} from '../../../../common/types/data_frame_analytics';
export type IndexName = string;
export type IndexPattern = string;
export type DataFrameAnalyticsId = string;
export enum ANALYSIS_CONFIG_TYPE {
OUTLIER_DETECTION = 'outlier_detection',
@ -46,34 +49,6 @@ export enum OUTLIER_ANALYSIS_METHOD {
DISTANCE_KNN = 'distance_knn',
}
interface OutlierAnalysis {
[key: string]: {};
outlier_detection: {};
}
interface Regression {
dependent_variable: string;
training_percent?: number;
num_top_feature_importance_values?: number;
prediction_field_name?: string;
}
export interface RegressionAnalysis {
[key: string]: Regression;
regression: Regression;
}
interface Classification {
dependent_variable: string;
training_percent?: number;
num_top_classes?: string;
num_top_feature_importance_values?: number;
prediction_field_name?: string;
}
export interface ClassificationAnalysis {
[key: string]: Classification;
classification: Classification;
}
export interface LoadExploreDataArg {
filterByIsTraining?: boolean;
searchQuery: SavedSearchQuery;
@ -165,22 +140,12 @@ export interface ClassificationEvaluateResponse {
};
}
interface GenericAnalysis {
[key: string]: Record<string, any>;
}
interface LoadEvaluateResult {
success: boolean;
eval: RegressionEvaluateResponse | ClassificationEvaluateResponse | null;
error: string | null;
}
type AnalysisConfig =
| OutlierAnalysis
| RegressionAnalysis
| ClassificationAnalysis
| GenericAnalysis;
export const getAnalysisType = (analysis: AnalysisConfig): string => {
const keys = Object.keys(analysis);
@ -342,29 +307,6 @@ export interface UpdateDataFrameAnalyticsConfig {
max_num_threads?: number;
}
export interface DataFrameAnalyticsConfig {
id: DataFrameAnalyticsId;
description?: string;
dest: {
index: IndexName;
results_field: string;
};
source: {
index: IndexName | IndexName[];
query?: any;
};
analysis: AnalysisConfig;
analyzed_fields: {
includes: string[];
excludes: string[];
};
model_memory_limit: string;
max_num_threads?: number;
create_time: number;
version: string;
allow_lazy_start?: boolean;
}
export enum REFRESH_ANALYTICS_LIST_STATE {
ERROR = 'error',
IDLE = 'idle',
@ -379,7 +321,8 @@ export const useRefreshAnalyticsList = (
callback: {
isLoading?(d: boolean): void;
onRefresh?(): void;
} = {}
} = {},
isManagementTable = false
) => {
useEffect(() => {
const distinct$ = refreshAnalyticsList$.pipe(distinctUntilChanged());
@ -387,13 +330,17 @@ export const useRefreshAnalyticsList = (
const subscriptions: Subscription[] = [];
if (typeof callback.onRefresh === 'function') {
// initial call to refresh
callback.onRefresh();
// required in order to fetch the DFA jobs on the management page
if (isManagementTable) callback.onRefresh();
subscriptions.push(
distinct$
.pipe(filter((state) => state === REFRESH_ANALYTICS_LIST_STATE.REFRESH))
.subscribe(() => typeof callback.onRefresh === 'function' && callback.onRefresh())
.subscribe(() => {
if (typeof callback.onRefresh === 'function') {
callback.onRefresh();
}
})
);
}
@ -410,7 +357,7 @@ export const useRefreshAnalyticsList = (
return () => {
subscriptions.map((sub) => sub.unsubscribe());
};
}, []);
}, [callback.onRefresh]);
return {
refresh: () => {

View file

@ -13,13 +13,13 @@ import {
isClassificationAnalysis,
isOutlierAnalysis,
isRegressionAnalysis,
DataFrameAnalyticsConfig,
} from './analytics';
import { Field } from '../../../../common/types/fields';
import { ES_FIELD_TYPES, KBN_FIELD_TYPES } from '../../../../../../../src/plugins/data/public';
import { newJobCapsService } from '../../services/new_job_capabilities_service';
import { FEATURE_IMPORTANCE, FEATURE_INFLUENCE, OUTLIER_SCORE, TOP_CLASSES } from './constants';
import { DataFrameAnalyticsConfig } from '../../../../common/types/data_frame_analytics';
export type EsId = string;
export type EsDocSource = Record<string, any>;

View file

@ -12,7 +12,8 @@ import { ml } from '../../services/ml_api_service';
import { isKeywordAndTextType } from '../common/fields';
import { SavedSearchQuery } from '../../contexts/ml';
import { DataFrameAnalyticsConfig, INDEX_STATUS } from './analytics';
import { INDEX_STATUS } from './analytics';
import { DataFrameAnalyticsConfig } from '../../../../common/types/data_frame_analytics';
export const getIndexData = async (
jobConfig: DataFrameAnalyticsConfig | undefined,

View file

@ -11,10 +11,7 @@ export {
isOutlierAnalysis,
refreshAnalyticsList$,
useRefreshAnalyticsList,
DataFrameAnalyticsId,
DataFrameAnalyticsConfig,
UpdateDataFrameAnalyticsConfig,
IndexName,
IndexPattern,
REFRESH_ANALYTICS_LIST_STATE,
ANALYSIS_CONFIG_TYPE,
@ -45,3 +42,6 @@ export { getIndexData } from './get_index_data';
export { getIndexFields } from './get_index_fields';
export { useResultsViewConfig } from './use_results_view_config';
export { DataFrameAnalyticsConfig } from '../../../../common/types/data_frame_analytics';
export { DataFrameAnalyticsId } from '../../../../common/types/data_frame_analytics';
export { IndexName } from '../../../../common/types/data_frame_analytics';

View file

@ -23,10 +23,10 @@ import { FormattedMessage } from '@kbn/i18n/react';
import { useMlContext } from '../../../contexts/ml';
import { newJobCapsService } from '../../../services/new_job_capabilities_service';
import { ml } from '../../../services/ml_api_service';
import { DataFrameAnalyticsId } from '../../common/analytics';
import { useCreateAnalyticsForm } from '../analytics_management/hooks/use_create_analytics_form';
import { CreateAnalyticsAdvancedEditor } from './components/create_analytics_advanced_editor';
import { AdvancedStep, ConfigurationStep, CreateStep, DetailsStep } from './components';
import { DataFrameAnalyticsId } from '../../../../../common/types/data_frame_analytics';
export enum ANALYTICS_STEPS {
CONFIGURATION,

View file

@ -120,10 +120,13 @@ export const DataFrameAnalyticsList: FC<Props> = ({
}, [selectedIdFromUrlInitialized, analytics]);
// Subscribe to the refresh observable to trigger reloading the analytics list.
useRefreshAnalyticsList({
isLoading: setIsLoading,
onRefresh: () => getAnalytics(true),
});
useRefreshAnalyticsList(
{
isLoading: setIsLoading,
onRefresh: () => getAnalytics(true),
},
isManagementTable
);
const { columns, modals } = useColumns(
expandedRowItemIds,
@ -271,6 +274,7 @@ export const DataFrameAnalyticsList: FC<Props> = ({
return (
<>
{modals}
<EuiSpacer size="m" />
<EuiFlexGroup justifyContent="spaceBetween">
<EuiFlexItem grow={false}>
{analyticsStats && (

View file

@ -0,0 +1,60 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import React, { FC, useCallback, useMemo } from 'react';
import { i18n } from '@kbn/i18n';
import { EuiTab, EuiTabs } from '@elastic/eui';
import { useNavigateToPath } from '../../../../../contexts/kibana';
interface Tab {
id: string;
name: string;
path: string;
}
export const AnalyticsNavigationBar: FC<{ selectedTabId?: string }> = ({ selectedTabId }) => {
const navigateToPath = useNavigateToPath();
const tabs = useMemo(
() => [
{
id: 'data_frame_analytics',
name: i18n.translate('xpack.ml.dataframe.jobsTabLabel', {
defaultMessage: 'Jobs',
}),
path: '/data_frame_analytics',
},
{
id: 'models',
name: i18n.translate('xpack.ml.dataframe.modelsTabLabel', {
defaultMessage: 'Models',
}),
path: '/data_frame_analytics/models',
},
],
[]
);
const onTabClick = useCallback(async (tab: Tab) => {
await navigateToPath(tab.path);
}, []);
return (
<EuiTabs>
{tabs.map((tab) => {
return (
<EuiTab
key={`tab-${tab.id}`}
isSelected={tab.id === selectedTabId}
onClick={onTabClick.bind(null, tab)}
>
{tab.name}
</EuiTab>
);
})}
</EuiTabs>
);
};

View file

@ -0,0 +1,7 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
export * from './analytics_navigation_bar';

View file

@ -0,0 +1,93 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import React, { FC } from 'react';
import { FormattedMessage } from '@kbn/i18n/react';
import {
EuiOverlayMask,
EuiModal,
EuiModalHeader,
EuiModalHeaderTitle,
EuiModalBody,
EuiModalFooter,
EuiButtonEmpty,
EuiButton,
EuiSpacer,
EuiCallOut,
} from '@elastic/eui';
import { ModelItemFull } from './models_list';
interface DeleteModelsModalProps {
models: ModelItemFull[];
onClose: (deletionApproved?: boolean) => void;
}
export const DeleteModelsModal: FC<DeleteModelsModalProps> = ({ models, onClose }) => {
const modelsWithPipelines = models
.filter((model) => !!model.pipelines)
.map((model) => model.model_id);
return (
<EuiOverlayMask>
<EuiModal onClose={onClose.bind(null, false)} initialFocus="[name=cancelModelDeletion]">
<EuiModalHeader>
<EuiModalHeaderTitle>
<FormattedMessage
id="xpack.ml.inference.modelsList.deleteModal.header"
defaultMessage="Delete {modelsCount, plural, one {{modelId}} other {# models}}"
values={{
modelId: models[0].model_id,
modelsCount: models.length,
}}
/>
</EuiModalHeaderTitle>
</EuiModalHeader>
<EuiModalBody>
<FormattedMessage
id="xpack.ml.inference.modelsList.deleteModal.warningMessage"
defaultMessage="Are you sure you want to delete {modelsCount, plural, one{this model} other {these models}}?"
values={{ modelsCount: models.length }}
/>
<EuiSpacer size="m" />
{modelsWithPipelines.length > 0 && (
<EuiCallOut
data-test-subj="modelsWithPipelinesWarning"
color={'danger'}
iconType={'alert'}
size="s"
>
<FormattedMessage
id="xpack.ml.inference.modelsList.deleteModal.modelsWithPipelinesWarningMessage"
defaultMessage="{modelsWithPipelinesCount, plural, one{Model} other {Models}} {modelsWithPipelines} {modelsWithPipelinesCount, plural, one{has} other {have}} associated pipelines!"
values={{
modelsWithPipelinesCount: modelsWithPipelines.length,
modelsWithPipelines: modelsWithPipelines.join(', '),
}}
/>
</EuiCallOut>
)}
</EuiModalBody>
<EuiModalFooter>
<EuiButtonEmpty onClick={onClose.bind(null, false)} name="cancelModelDeletion">
<FormattedMessage
id="xpack.ml.inference.modelsList.deleteModal.cancelButtonLabel"
defaultMessage="Cancel"
/>
</EuiButtonEmpty>
<EuiButton onClick={onClose.bind(null, true)} fill color="danger">
<FormattedMessage
id="xpack.ml.inference.modelsList.deleteModal.deleteButtonLabel"
defaultMessage="Delete"
/>
</EuiButton>
</EuiModalFooter>
</EuiModal>
</EuiOverlayMask>
);
};

View file

@ -0,0 +1,370 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import React, { FC, Fragment } from 'react';
import { FormattedMessage } from '@kbn/i18n/react';
import {
EuiDescriptionList,
EuiPanel,
EuiSpacer,
EuiTabbedContent,
EuiTitle,
EuiNotificationBadge,
EuiFlexGrid,
EuiFlexItem,
EuiCodeBlock,
EuiText,
EuiHorizontalRule,
EuiFlexGroup,
EuiTextColor,
} from '@elastic/eui';
// @ts-ignore
import { formatDate } from '@elastic/eui/lib/services/format';
import { ModelItemFull } from './models_list';
import { TIME_FORMAT } from '../../../../../../../common/constants/time_format';
interface ExpandedRowProps {
item: ModelItemFull;
}
export const ExpandedRow: FC<ExpandedRowProps> = ({ item }) => {
const {
inference_config: inferenceConfig,
stats,
metadata,
tags,
version,
// eslint-disable-next-line @typescript-eslint/naming-convention
estimated_operations,
// eslint-disable-next-line @typescript-eslint/naming-convention
estimated_heap_memory_usage_bytes,
// eslint-disable-next-line @typescript-eslint/naming-convention
default_field_map,
// eslint-disable-next-line @typescript-eslint/naming-convention
license_level,
pipelines,
} = item;
const details = {
tags,
version,
estimated_operations,
estimated_heap_memory_usage_bytes,
default_field_map,
license_level,
};
function formatToListItems(items: Record<string, any>) {
return Object.entries(items)
.map(([title, value]) => {
if (title.includes('timestamp')) {
value = formatDate(value, TIME_FORMAT);
}
return { title, description: typeof value === 'object' ? JSON.stringify(value) : value };
})
.filter(({ description }) => {
return description !== undefined;
});
}
const tabs = [
{
id: 'details',
name: (
<FormattedMessage
id="xpack.ml.inference.modelsList.expandedRow.detailsTabLabel"
defaultMessage="Details"
/>
),
content: (
<>
<EuiSpacer size={'m'} />
<EuiFlexGrid columns={2} gutterSize={'m'}>
<EuiFlexItem>
<EuiPanel>
<EuiTitle size={'xs'}>
<h5>
<FormattedMessage
id="xpack.ml.inference.modelsList.expandedRow.detailsTitle"
defaultMessage="Details"
/>
</h5>
</EuiTitle>
<EuiSpacer size={'m'} />
<EuiDescriptionList
compressed={true}
type="column"
listItems={formatToListItems(details)}
/>
</EuiPanel>
</EuiFlexItem>
</EuiFlexGrid>
</>
),
},
...(inferenceConfig
? [
{
id: 'config',
name: (
<FormattedMessage
id="xpack.ml.inference.modelsList.expandedRow.configTabLabel"
defaultMessage="Config"
/>
),
content: (
<>
<EuiSpacer size={'m'} />
<EuiFlexGrid columns={2} gutterSize={'m'}>
<EuiFlexItem>
<EuiPanel>
<EuiTitle size={'xs'}>
<h5>
<FormattedMessage
id="xpack.ml.inference.modelsList.expandedRow.inferenceConfigTitle"
defaultMessage="Inference configuration"
/>
</h5>
</EuiTitle>
<EuiSpacer size={'m'} />
<EuiDescriptionList
compressed={true}
type="column"
listItems={formatToListItems(
inferenceConfig[Object.keys(inferenceConfig)[0]]
)}
/>
</EuiPanel>
</EuiFlexItem>
{metadata?.analytics_config && (
<EuiFlexItem>
<EuiPanel>
<EuiTitle size={'xs'}>
<h5>
<FormattedMessage
id="xpack.ml.inference.modelsList.expandedRow.analyticsConfigTitle"
defaultMessage="Analytics configuration"
/>
</h5>
</EuiTitle>
<EuiSpacer size={'m'} />
<EuiDescriptionList
compressed={true}
type="column"
listItems={formatToListItems(metadata.analytics_config)}
/>
</EuiPanel>
</EuiFlexItem>
)}
</EuiFlexGrid>
</>
),
},
]
: []),
{
id: 'stats',
name: (
<FormattedMessage
id="xpack.ml.inference.modelsList.expandedRow.statsTabLabel"
defaultMessage="Stats"
/>
),
content: (
<>
<EuiSpacer size={'m'} />
<EuiFlexGrid columns={2}>
{stats.inference_stats && (
<EuiFlexItem>
<EuiPanel>
<EuiTitle size={'xs'}>
<h5>
<FormattedMessage
id="xpack.ml.inference.modelsList.expandedRow.inferenceStatsTitle"
defaultMessage="Inference stats"
/>
</h5>
</EuiTitle>
<EuiSpacer size={'m'} />
<EuiDescriptionList
compressed={true}
type="column"
listItems={formatToListItems(stats.inference_stats)}
/>
</EuiPanel>
</EuiFlexItem>
)}
{stats.ingest?.total && (
<EuiFlexItem>
<EuiPanel style={{ maxHeight: '400px', overflow: 'auto' }}>
<EuiTitle size={'xs'}>
<h5>
<FormattedMessage
id="xpack.ml.inference.modelsList.expandedRow.ingestStatsTitle"
defaultMessage="Ingest stats"
/>
</h5>
</EuiTitle>
<EuiSpacer size={'m'} />
<EuiDescriptionList
compressed={true}
type="column"
listItems={formatToListItems(stats.ingest.total)}
/>
{stats.ingest?.pipelines && (
<>
<EuiSpacer size={'m'} />
<EuiTitle size={'xs'}>
<h5>
<FormattedMessage
id="xpack.ml.inference.modelsList.expandedRow.byPipelineTitle"
defaultMessage="By pipeline"
/>
</h5>
</EuiTitle>
<EuiSpacer size={'s'} />
{Object.entries(stats.ingest.pipelines).map(
([pipelineName, { processors, ...pipelineStats }], i) => {
return (
<Fragment key={pipelineName}>
<EuiFlexGroup>
<EuiFlexItem grow={false}>
<EuiTitle size={'xs'}>
<EuiTextColor color="subdued">
<h5>
{i + 1}. {pipelineName}
</h5>
</EuiTextColor>
</EuiTitle>
</EuiFlexItem>
<EuiFlexItem>
<EuiHorizontalRule size={'full'} margin={'s'} />
</EuiFlexItem>
</EuiFlexGroup>
<EuiSpacer size={'m'} />
<EuiDescriptionList
compressed={true}
type="column"
listItems={formatToListItems(pipelineStats)}
/>
<EuiSpacer size={'m'} />
<EuiTitle size={'xxs'}>
<h6>
<FormattedMessage
id="xpack.ml.inference.modelsList.expandedRow.byProcessorTitle"
defaultMessage="By processor"
/>
</h6>
</EuiTitle>
<EuiSpacer size={'s'} />
<>
{processors.map((processor) => {
const name = Object.keys(processor)[0];
const { stats: processorStats } = processor[name];
return (
<Fragment key={name}>
<EuiFlexGroup>
<EuiFlexItem grow={false}>
<EuiTitle size={'xxs'}>
<EuiTextColor color="subdued">
<h6>{name}</h6>
</EuiTextColor>
</EuiTitle>
</EuiFlexItem>
<EuiFlexItem>
<EuiHorizontalRule size={'full'} margin={'s'} />
</EuiFlexItem>
</EuiFlexGroup>
<EuiSpacer size={'m'} />
<EuiDescriptionList
compressed={true}
type="column"
listItems={formatToListItems(processorStats)}
/>
</Fragment>
);
})}
</>
</Fragment>
);
}
)}
</>
)}
</EuiPanel>
</EuiFlexItem>
)}
</EuiFlexGrid>
</>
),
},
...(pipelines && Object.keys(pipelines).length > 0
? [
{
id: 'pipelines',
name: (
<>
<FormattedMessage
id="xpack.ml.inference.modelsList.expandedRow.pipelinesTabLabel"
defaultMessage="Pipelines"
/>{' '}
<EuiNotificationBadge>{stats.pipeline_count}</EuiNotificationBadge>
</>
),
content: (
<>
<EuiSpacer size={'m'} />
<EuiFlexGrid columns={2} gutterSize={'m'}>
{Object.entries(pipelines).map(([pipelineName, { processors, description }]) => {
return (
<EuiFlexItem key={pipelineName}>
<EuiPanel>
<EuiTitle size={'xs'}>
<h5>{pipelineName}</h5>
</EuiTitle>
{description && <EuiText>{description}</EuiText>}
<EuiSpacer size={'m'} />
<EuiTitle size={'xxs'}>
<h6>
<FormattedMessage
id="xpack.ml.inference.modelsList.expandedRow.processorsTitle"
defaultMessage="Processors"
/>
</h6>
</EuiTitle>
<EuiCodeBlock
language="painless"
fontSize="m"
paddingSize="m"
overflowHeight={300}
isCopyable
>
{JSON.stringify(processors, null, 2)}
</EuiCodeBlock>
</EuiPanel>
</EuiFlexItem>
);
})}
</EuiFlexGrid>
</>
),
},
]
: []),
];
return (
<EuiTabbedContent
size="s"
style={{ width: '100%' }}
tabs={tabs}
initialSelectedTab={tabs[0]}
autoFocus="selected"
onTabClick={(tab) => {}}
/>
);
};

View file

@ -0,0 +1,13 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
export * from './models_list';
export enum ModelsTableToConfigMapping {
id = 'model_id',
createdAt = 'create_time',
type = 'type',
}

View file

@ -0,0 +1,509 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import React, { FC, useState, useCallback, useMemo } from 'react';
import { i18n } from '@kbn/i18n';
import { FormattedMessage } from '@kbn/i18n/react';
import {
Direction,
EuiFlexGroup,
EuiFlexItem,
EuiInMemoryTable,
EuiTitle,
EuiButton,
EuiSearchBarProps,
EuiSpacer,
EuiButtonIcon,
EuiBadge,
} from '@elastic/eui';
// @ts-ignore
import { formatDate } from '@elastic/eui/lib/services/format';
import { EuiBasicTableColumn } from '@elastic/eui/src/components/basic_table/basic_table';
import { EuiTableSelectionType } from '@elastic/eui/src/components/basic_table/table_types';
import { Action } from '@elastic/eui/src/components/basic_table/action_types';
import { StatsBar, ModelsBarStats } from '../../../../../components/stats_bar';
import { useInferenceApiService } from '../../../../../services/ml_api_service/inference';
import { ModelsTableToConfigMapping } from './index';
import { TIME_FORMAT } from '../../../../../../../common/constants/time_format';
import { DeleteModelsModal } from './delete_models_modal';
import { useMlKibana, useNotifications } from '../../../../../contexts/kibana';
import { ExpandedRow } from './expanded_row';
import { getResultsUrl } from '../analytics_list/common';
import {
ModelConfigResponse,
ModelPipelines,
TrainedModelStat,
} from '../../../../../../../common/types/inference';
import {
REFRESH_ANALYTICS_LIST_STATE,
refreshAnalyticsList$,
useRefreshAnalyticsList,
} from '../../../../common';
type Stats = Omit<TrainedModelStat, 'model_id'>;
export type ModelItem = ModelConfigResponse & {
type?: string;
stats?: Stats;
pipelines?: ModelPipelines['pipelines'] | null;
};
export type ModelItemFull = Required<ModelItem>;
export const ModelsList: FC = () => {
const {
services: {
application: { navigateToUrl, capabilities },
},
} = useMlKibana();
const canDeleteDataFrameAnalytics = capabilities.ml.canDeleteDataFrameAnalytics as boolean;
const inferenceApiService = useInferenceApiService();
const { toasts } = useNotifications();
const [searchQueryText, setSearchQueryText] = useState('');
const [pageIndex, setPageIndex] = useState(0);
const [pageSize, setPageSize] = useState(10);
const [sortField, setSortField] = useState<string>(ModelsTableToConfigMapping.id);
const [sortDirection, setSortDirection] = useState<Direction>('asc');
const [isLoading, setIsLoading] = useState(false);
const [items, setItems] = useState<ModelItem[]>([]);
const [selectedModels, setSelectedModels] = useState<ModelItem[]>([]);
const [modelsToDelete, setModelsToDelete] = useState<ModelItemFull[]>([]);
const [itemIdToExpandedRowMap, setItemIdToExpandedRowMap] = useState<Record<string, JSX.Element>>(
{}
);
/**
* Fetches inference trained models.
*/
const fetchData = useCallback(async () => {
try {
const response = await inferenceApiService.getInferenceModel(undefined, {
with_pipelines: true,
size: 1000,
});
const newItems = [];
const expandedItemsToRefresh = [];
for (const model of response) {
const tableItem = {
...model,
...(typeof model.inference_config === 'object'
? { type: Object.keys(model.inference_config)[0] }
: {}),
};
newItems.push(tableItem);
if (itemIdToExpandedRowMap[model.model_id]) {
expandedItemsToRefresh.push(tableItem);
}
}
setItems(newItems);
if (expandedItemsToRefresh.length > 0) {
await fetchModelsStats(expandedItemsToRefresh);
setItemIdToExpandedRowMap(
expandedItemsToRefresh.reduce((acc, item) => {
acc[item.model_id] = <ExpandedRow item={item as ModelItemFull} />;
return acc;
}, {} as Record<string, JSX.Element>)
);
}
} catch (error) {
toasts.addError(new Error(error.body?.message), {
title: i18n.translate('xpack.ml.inference.modelsList.fetchFailedErrorMessage', {
defaultMessage: 'Models fetch failed',
}),
});
}
setIsLoading(false);
refreshAnalyticsList$.next(REFRESH_ANALYTICS_LIST_STATE.IDLE);
}, [itemIdToExpandedRowMap]);
// Subscribe to the refresh observable to trigger reloading the model list.
useRefreshAnalyticsList({
isLoading: setIsLoading,
onRefresh: fetchData,
});
const modelsStats: ModelsBarStats = useMemo(() => {
return {
total: {
show: true,
value: items.length,
label: i18n.translate('xpack.ml.inference.modelsList.totalAmountLabel', {
defaultMessage: 'Total inference trained models',
}),
},
};
}, [items]);
/**
* Fetches models stats and update the original object
*/
const fetchModelsStats = useCallback(async (models: ModelItem[]) => {
const modelIdsToFetch = models.map((model) => model.model_id);
try {
const {
trained_model_stats: modelsStatsResponse,
} = await inferenceApiService.getInferenceModelStats(modelIdsToFetch);
for (const { model_id: id, ...stats } of modelsStatsResponse) {
const model = models.find((m) => m.model_id === id);
model!.stats = stats;
}
return true;
} catch (error) {
toasts.addError(new Error(error.body.message), {
title: i18n.translate('xpack.ml.inference.modelsList.fetchModelStatsErrorMessage', {
defaultMessage: 'Fetch model stats failed',
}),
});
}
}, []);
/**
* Unique inference types from models
*/
const inferenceTypesOptions = useMemo(() => {
const result = items.reduce((acc, item) => {
const type = item.inference_config && Object.keys(item.inference_config)[0];
if (type) {
acc.add(type);
}
return acc;
}, new Set<string>());
return [...result].map((v) => ({
value: v,
name: v,
}));
}, [items]);
async function prepareModelsForDeletion(models: ModelItem[]) {
// Fetch model stats to check associated pipelines
if (await fetchModelsStats(models)) {
setModelsToDelete(models as ModelItemFull[]);
} else {
toasts.addDanger(
i18n.translate('xpack.ml.inference.modelsList.unableToDeleteModelsErrorMessage', {
defaultMessage: 'Unable to delete models',
})
);
}
}
/**
* Deletes the models marked for deletion.
*/
async function deleteModels() {
const modelsToDeleteIds = modelsToDelete.map((model) => model.model_id);
try {
await Promise.all(
modelsToDeleteIds.map((modelId) => inferenceApiService.deleteInferenceModel(modelId))
);
setItems(
items.filter(
(model) => !modelsToDelete.some((toDelete) => toDelete.model_id === model.model_id)
)
);
toasts.addSuccess(
i18n.translate('xpack.ml.inference.modelsList.successfullyDeletedMessage', {
defaultMessage:
'{modelsCount, plural, one {Model {modelsToDeleteIds}} other {# models}} {modelsCount, plural, one {has} other {have}} been successfully deleted',
values: {
modelsCount: modelsToDeleteIds.length,
modelsToDeleteIds: modelsToDeleteIds.join(', '),
},
})
);
} catch (error) {
toasts.addError(new Error(error?.body?.message), {
title: i18n.translate('xpack.ml.inference.modelsList.fetchDeletionErrorMessage', {
defaultMessage: '{modelsCount, plural, one {Model} other {Models}} deletion failed',
values: {
modelsCount: modelsToDeleteIds.length,
},
}),
});
}
}
/**
* Table actions
*/
const actions: Array<Action<ModelItem>> = [
{
name: i18n.translate('xpack.ml.inference.modelsList.viewTrainingDataActionLabel', {
defaultMessage: 'View training data',
}),
description: i18n.translate('xpack.ml.inference.modelsList.viewTrainingDataActionLabel', {
defaultMessage: 'View training data',
}),
icon: 'list',
type: 'icon',
available: (item) => item.metadata?.analytics_config?.id,
onClick: async (item) => {
await navigateToUrl(
getResultsUrl(
item.metadata?.analytics_config.id,
Object.keys(item.metadata?.analytics_config.analysis)[0]
)
);
},
isPrimary: true,
},
{
name: i18n.translate('xpack.ml.inference.modelsList.deleteModelActionLabel', {
defaultMessage: 'Delete model',
}),
description: i18n.translate('xpack.ml.inference.modelsList.deleteModelActionLabel', {
defaultMessage: 'Delete model',
}),
icon: 'trash',
type: 'icon',
color: 'danger',
isPrimary: false,
onClick: async (model) => {
await prepareModelsForDeletion([model]);
},
available: (item) => canDeleteDataFrameAnalytics,
enabled: (item) => {
// TODO check for permissions to delete ingest pipelines.
// ATM undefined means pipelines fetch failed server-side.
return !item.pipelines;
},
},
];
const toggleDetails = async (item: ModelItem) => {
const itemIdToExpandedRowMapValues = { ...itemIdToExpandedRowMap };
if (itemIdToExpandedRowMapValues[item.model_id]) {
delete itemIdToExpandedRowMapValues[item.model_id];
} else {
await fetchModelsStats([item]);
itemIdToExpandedRowMapValues[item.model_id] = <ExpandedRow item={item as ModelItemFull} />;
}
setItemIdToExpandedRowMap(itemIdToExpandedRowMapValues);
};
const columns: Array<EuiBasicTableColumn<ModelItem>> = [
{
align: 'left',
width: '40px',
isExpander: true,
render: (item: ModelItem) => (
<EuiButtonIcon
onClick={toggleDetails.bind(null, item)}
aria-label={
itemIdToExpandedRowMap[item.model_id]
? i18n.translate('xpack.ml.inference.modelsList.collapseRow', {
defaultMessage: 'Collapse',
})
: i18n.translate('xpack.ml.inference.modelsList.expandRow', {
defaultMessage: 'Expand',
})
}
iconType={itemIdToExpandedRowMap[item.model_id] ? 'arrowUp' : 'arrowDown'}
/>
),
},
{
field: ModelsTableToConfigMapping.id,
name: i18n.translate('xpack.ml.inference.modelsList.modelIdHeader', {
defaultMessage: 'ID',
}),
sortable: true,
truncateText: true,
},
{
field: ModelsTableToConfigMapping.type,
name: i18n.translate('xpack.ml.inference.modelsList.typeHeader', {
defaultMessage: 'Type',
}),
sortable: true,
align: 'left',
render: (type: string) => <EuiBadge color="hollow">{type}</EuiBadge>,
},
{
field: ModelsTableToConfigMapping.createdAt,
name: i18n.translate('xpack.ml.inference.modelsList.createdAtHeader', {
defaultMessage: 'Created at',
}),
dataType: 'date',
render: (date: string) => formatDate(date, TIME_FORMAT),
sortable: true,
},
{
name: i18n.translate('xpack.ml.inference.modelsList.actionsHeader', {
defaultMessage: 'Actions',
}),
actions,
},
];
const pagination = {
initialPageIndex: pageIndex,
initialPageSize: pageSize,
totalItemCount: items.length,
pageSizeOptions: [10, 20, 50],
hidePerPageOptions: false,
};
const sorting = {
sort: {
field: sortField,
direction: sortDirection,
},
};
const search: EuiSearchBarProps = {
query: searchQueryText,
onChange: (searchChange) => {
if (searchChange.error !== null) {
return false;
}
setSearchQueryText(searchChange.queryText);
return true;
},
box: {
incremental: true,
},
...(inferenceTypesOptions && inferenceTypesOptions.length > 0
? {
filters: [
{
type: 'field_value_selection',
field: 'type',
name: i18n.translate('xpack.ml.dataframe.analyticsList.typeFilter', {
defaultMessage: 'Type',
}),
multiSelect: 'or',
options: inferenceTypesOptions,
},
],
}
: {}),
...(selectedModels.length > 0
? {
toolsLeft: (
<EuiFlexGroup justifyContent="spaceBetween" alignItems="center">
<EuiFlexItem grow={false}>
<EuiTitle size="s">
<h5>
<FormattedMessage
id="xpack.ml.inference.modelsList.selectedModelsMessage"
defaultMessage="{modelsCount, plural, one{# model} other {# models}} selected"
values={{ modelsCount: selectedModels.length }}
/>
</h5>
</EuiTitle>
</EuiFlexItem>
<EuiFlexItem>
<EuiButton
color="danger"
onClick={prepareModelsForDeletion.bind(null, selectedModels)}
>
<FormattedMessage
id="xpack.ml.inference.modelsList.deleteModelsButtonLabel"
defaultMessage="Delete"
/>
</EuiButton>
</EuiFlexItem>
</EuiFlexGroup>
),
}
: {}),
};
const onTableChange: EuiInMemoryTable<ModelItem>['onTableChange'] = ({
page = { index: 0, size: 10 },
sort = { field: ModelsTableToConfigMapping.id, direction: 'asc' },
}) => {
const { index, size } = page;
setPageIndex(index);
setPageSize(size);
const { field, direction } = sort;
setSortField(field);
setSortDirection(direction);
};
const isSelectionAllowed = canDeleteDataFrameAnalytics;
const selection: EuiTableSelectionType<ModelItem> | undefined = isSelectionAllowed
? {
selectableMessage: (selectable, item) => {
return selectable
? i18n.translate('xpack.ml.inference.modelsList.selectableMessage', {
defaultMessage: 'Select a model',
})
: i18n.translate('xpack.ml.inference.modelsList.disableSelectableMessage', {
defaultMessage: 'Model has associated pipelines',
});
},
selectable: (item) => !item.pipelines,
onSelectionChange: (selectedItems) => {
setSelectedModels(selectedItems);
},
}
: undefined;
return (
<>
<EuiSpacer size="m" />
<EuiFlexGroup justifyContent="spaceBetween">
{modelsStats && (
<EuiFlexItem grow={false}>
<StatsBar stats={modelsStats} dataTestSub={'mlInferenceModelsStatsBar'} />
</EuiFlexItem>
)}
</EuiFlexGroup>
<EuiSpacer size="m" />
<div data-test-subj="mlModelsTableContainer">
<EuiInMemoryTable
allowNeutralSort={false}
columns={columns}
hasActions={true}
isExpandable={true}
itemIdToExpandedRowMap={itemIdToExpandedRowMap}
isSelectable={false}
items={items}
itemId={ModelsTableToConfigMapping.id}
loading={isLoading}
onTableChange={onTableChange}
pagination={pagination}
sorting={sorting}
search={search}
selection={selection}
rowProps={(item) => ({
'data-test-subj': `mlModelsTableRow row-${item.model_id}`,
})}
/>
</div>
{modelsToDelete.length > 0 && (
<DeleteModelsModal
onClose={async (deletionApproved) => {
if (deletionApproved) {
await deleteModels();
}
setModelsToDelete([]);
}}
models={modelsToDelete}
/>
)}
</>
);
};

View file

@ -8,13 +8,12 @@ import { DeepPartial, DeepReadonly } from '../../../../../../../common/types/com
import { checkPermission } from '../../../../../capabilities/check_capabilities';
import { mlNodesAvailable } from '../../../../../ml_nodes_check';
import {
DataFrameAnalyticsId,
DataFrameAnalyticsConfig,
ANALYSIS_CONFIG_TYPE,
defaultSearchQuery,
} from '../../../../common/analytics';
import { ANALYSIS_CONFIG_TYPE, defaultSearchQuery } from '../../../../common/analytics';
import { CloneDataFrameAnalyticsConfig } from '../../components/action_clone';
import {
DataFrameAnalyticsConfig,
DataFrameAnalyticsId,
} from '../../../../../../../common/types/data_frame_analytics';
export enum DEFAULT_MODEL_MEMORY_LIMIT {
regression = '100mb',

View file

@ -4,7 +4,7 @@
* you may not use this file except in compliance with the Elastic License.
*/
import React, { FC, Fragment, useState } from 'react';
import React, { FC, Fragment, useMemo, useState } from 'react';
import { FormattedMessage } from '@kbn/i18n/react';
import { i18n } from '@kbn/i18n';
@ -21,6 +21,7 @@ import {
EuiTitle,
} from '@elastic/eui';
import { useLocation } from 'react-router-dom';
import { NavigationMenu } from '../../../components/navigation_menu';
import { DatePickerWrapper } from '../../../components/navigation_menu/date_picker_wrapper';
import { DataFrameAnalyticsList } from './components/analytics_list';
@ -28,12 +29,17 @@ import { useRefreshInterval } from './components/analytics_list/use_refresh_inte
import { RefreshAnalyticsListButton } from './components/refresh_analytics_list_button';
import { NodeAvailableWarning } from '../../../components/node_available_warning';
import { UpgradeWarning } from '../../../components/upgrade';
import { AnalyticsNavigationBar } from './components/analytics_navigation_bar';
import { ModelsList } from './components/models_management';
export const Page: FC = () => {
const [blockRefresh, setBlockRefresh] = useState(false);
useRefreshInterval(setBlockRefresh);
const location = useLocation();
const selectedTabId = useMemo(() => location.pathname.split('/').pop(), [location]);
return (
<Fragment>
<NavigationMenu tabId="data_frame_analytics" />
@ -45,7 +51,7 @@ export const Page: FC = () => {
<h1>
<FormattedMessage
id="xpack.ml.dataframe.analyticsList.title"
defaultMessage="Data frame analytics jobs"
defaultMessage="Data frame analytics"
/>
<span>&nbsp;</span>
<EuiBetaBadge
@ -81,7 +87,12 @@ export const Page: FC = () => {
<UpgradeWarning />
<EuiPageContent>
<DataFrameAnalyticsList blockRefresh={blockRefresh} />
<AnalyticsNavigationBar selectedTabId={selectedTabId} />
{selectedTabId === 'data_frame_analytics' && (
<DataFrameAnalyticsList blockRefresh={blockRefresh} />
)}
{selectedTabId === 'models' && <ModelsList />}
</EuiPageContent>
</EuiPageBody>
</EuiPage>

View file

@ -7,3 +7,4 @@
export * from './analytics_jobs_list';
export * from './analytics_job_exploration';
export * from './analytics_job_creation';
export * from './models_list';

View file

@ -0,0 +1,40 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import React, { FC } from 'react';
import { i18n } from '@kbn/i18n';
import { NavigateToPath } from '../../../contexts/kibana';
import { MlRoute, PageLoader, PageProps } from '../../router';
import { useResolver } from '../../use_resolver';
import { basicResolvers } from '../../resolvers';
import { Page } from '../../../data_frame_analytics/pages/analytics_management';
import { getBreadcrumbWithUrlForApp } from '../../breadcrumbs';
export const modelsListRouteFactory = (navigateToPath: NavigateToPath): MlRoute => ({
path: '/data_frame_analytics/models',
render: (props, deps) => <PageWrapper {...props} deps={deps} />,
breadcrumbs: [
getBreadcrumbWithUrlForApp('ML_BREADCRUMB', navigateToPath),
getBreadcrumbWithUrlForApp('DATA_FRAME_ANALYTICS_BREADCRUMB', navigateToPath),
{
text: i18n.translate('xpack.ml.dataFrameAnalyticsBreadcrumbs.modelsListLabel', {
defaultMessage: 'Model Management',
}),
href: '',
},
],
});
const PageWrapper: FC<PageProps> = ({ location, deps }) => {
const { context } = useResolver('', undefined, deps.config, basicResolvers(deps));
return (
<PageLoader context={context}>
<Page />
</PageLoader>
);
};

View file

@ -0,0 +1,135 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { useMemo } from 'react';
import { HttpFetchQuery } from 'kibana/public';
import { HttpService } from '../http_service';
import { basePath } from './index';
import { useMlKibana } from '../../contexts/kibana';
import {
ModelConfigResponse,
ModelPipelines,
TrainedModelStat,
} from '../../../../common/types/inference';
export interface InferenceQueryParams {
decompress_definition?: boolean;
from?: number;
include_model_definition?: boolean;
size?: number;
tags?: string;
// Custom kibana endpoint query params
with_pipelines?: boolean;
}
export interface InferenceStatsQueryParams {
from?: number;
size?: number;
}
export interface IngestStats {
count: number;
time_in_millis: number;
current: number;
failed: number;
}
export interface InferenceStatsResponse {
count: number;
trained_model_stats: TrainedModelStat[];
}
/**
* Service with APIs calls to perform inference operations.
* @param httpService
*/
export function inferenceApiProvider(httpService: HttpService) {
const apiBasePath = basePath();
return {
/**
* Fetches configuration information for a trained inference model.
*
* @param modelId - Model ID, collection of Model IDs or Model ID pattern.
* Fetches all In case nothing is provided.
* @param params - Optional query params
*/
getInferenceModel(modelId?: string | string[], params?: InferenceQueryParams) {
let model = modelId ?? '';
if (Array.isArray(modelId)) {
model = modelId.join(',');
}
return httpService.http<ModelConfigResponse[]>({
path: `${apiBasePath}/inference${model && `/${model}`}`,
method: 'GET',
...(params ? { query: params as HttpFetchQuery } : {}),
});
},
/**
* Fetches usage information for trained inference models.
*
* @param modelId - Model ID, collection of Model IDs or Model ID pattern.
* Fetches all In case nothing is provided.
* @param params - Optional query params
*/
getInferenceModelStats(modelId?: string | string[], params?: InferenceStatsQueryParams) {
let model = modelId ?? '_all';
if (Array.isArray(modelId)) {
model = modelId.join(',');
}
return httpService.http<InferenceStatsResponse>({
path: `${apiBasePath}/inference/${model}/_stats`,
method: 'GET',
});
},
/**
* Fetches pipelines associated with provided models
*
* @param modelId - Model ID, collection of Model IDs.
*/
getInferenceModelPipelines(modelId: string | string[]) {
let model = modelId;
if (Array.isArray(modelId)) {
model = modelId.join(',');
}
return httpService.http<ModelPipelines[]>({
path: `${apiBasePath}/inference/${model}/pipelines`,
method: 'GET',
});
},
/**
* Deletes an existing trained inference model.
*
* @param modelId - Model ID
*/
deleteInferenceModel(modelId: string) {
return httpService.http<any>({
path: `${apiBasePath}/inference/${modelId}`,
method: 'DELETE',
});
},
};
}
type InferenceApiService = ReturnType<typeof inferenceApiProvider>;
/**
* Hooks for accessing {@link InferenceApiService} in React components.
*/
export function useInferenceApiService(): InferenceApiService {
const {
services: {
mlServices: { httpService },
},
} = useMlKibana();
return useMemo(() => inferenceApiProvider(httpService), [httpService]);
}

View file

@ -8,7 +8,9 @@ import { boomify, isBoom } from 'boom';
import { ResponseError, CustomHttpResponseOptions } from 'kibana/server';
export function wrapError(error: any): CustomHttpResponseOptions<ResponseError> {
const boom = isBoom(error) ? error : boomify(error, { statusCode: error.status });
const boom = isBoom(error)
? error
: boomify(error, { statusCode: error.status ?? error.statusCode });
const statusCode = boom.output.statusCode;
return {
body: {

View file

@ -5,3 +5,4 @@
*/
export { analyticsAuditMessagesProvider } from './analytics_audit_messages';
export { modelsProvider } from './models_provider';

View file

@ -0,0 +1,42 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { IScopedClusterClient } from 'kibana/server';
import { PipelineDefinition } from '../../../common/types/inference';
export function modelsProvider(client: IScopedClusterClient) {
return {
/**
* Retrieves the map of model ids and associated pipelines.
* @param modelIds
*/
async getModelsPipelines(modelIds: string[]) {
const modelIdsMap = new Map<string, Record<string, PipelineDefinition> | null>(
modelIds.map((id: string) => [id, null])
);
const { body } = await client.asCurrentUser.ingest.getPipeline();
for (const [pipelineName, pipelineDefinition] of Object.entries(body)) {
const { processors } = pipelineDefinition as { processors: Array<Record<string, any>> };
for (const processor of processors) {
const id = processor.inference?.model_id;
if (modelIdsMap.has(id)) {
const obj = modelIdsMap.get(id);
if (obj === null) {
modelIdsMap.set(id, { [pipelineName]: pipelineDefinition });
} else {
obj![pipelineName] = pipelineDefinition;
}
}
}
}
return modelIdsMap;
},
};
}

View file

@ -48,6 +48,7 @@ import { createSharedServices, SharedServices } from './shared_services';
import { getPluginPrivileges } from '../common/types/capabilities';
import { setupCapabilitiesSwitcher } from './lib/capabilities';
import { registerKibanaSettings } from './lib/register_settings';
import { inferenceRoutes } from './routes/inference';
declare module 'kibana/server' {
interface RequestHandlerContext {
@ -172,6 +173,8 @@ export class MlServerPlugin implements Plugin<MlPluginSetup, MlPluginStart, Plug
initMlServerLog({ log: this.log });
initMlTelemetry(coreSetup, plugins.usageCollection);
inferenceRoutes(routeInit);
return {
...createSharedServices(this.mlLicense, plugins.spaces, plugins.cloud, resolveMlCapabilities),
mlClient,

View file

@ -0,0 +1,163 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { RouteInitialization } from '../types';
import { wrapError } from '../client/error_wrapper';
import {
getInferenceQuerySchema,
modelIdSchema,
optionalModelIdSchema,
} from './schemas/inference_schema';
import { modelsProvider } from '../models/data_frame_analytics';
import { InferenceConfigResponse } from '../../common/types/inference';
export function inferenceRoutes({ router, mlLicense }: RouteInitialization) {
/**
* @apiGroup Inference
*
* @api {get} /api/ml/inference/:modelId Get info of a trained inference model
* @apiName GetInferenceModel
* @apiDescription Retrieves configuration information for a trained inference model.
*/
router.get(
{
path: '/api/ml/inference/{modelId?}',
validate: {
params: optionalModelIdSchema,
query: getInferenceQuerySchema,
},
options: {
tags: ['access:ml:canGetDataFrameAnalytics'],
},
},
mlLicense.fullLicenseAPIGuard(async ({ client, request, response }) => {
try {
const { modelId } = request.params;
const { with_pipelines: withPipelines, ...query } = request.query;
const { body } = await client.asInternalUser.ml.getTrainedModels<InferenceConfigResponse>({
size: 1000,
...query,
...(modelId ? { model_id: modelId } : {}),
});
const result = body.trained_model_configs;
try {
if (withPipelines) {
const pipelinesResponse = await modelsProvider(client).getModelsPipelines(
result.map(({ model_id: id }: { model_id: string }) => id)
);
for (const model of result) {
model.pipelines = pipelinesResponse.get(model.model_id)!;
}
}
} catch (e) {
// the user might not have required permissions to fetch pipelines
// eslint-disable-next-line no-console
console.log(e);
}
return response.ok({
body: result,
});
} catch (e) {
return response.customError(wrapError(e));
}
})
);
/**
* @apiGroup Inference
*
* @api {get} /api/ml/inference/:modelId/_stats Get stats of a trained inference model
* @apiName GetInferenceModelStats
* @apiDescription Retrieves usage information for trained inference models.
*/
router.get(
{
path: '/api/ml/inference/{modelId}/_stats',
validate: {
params: modelIdSchema,
},
options: {
tags: ['access:ml:canGetDataFrameAnalytics'],
},
},
mlLicense.fullLicenseAPIGuard(async ({ client, request, response }) => {
try {
const { modelId } = request.params;
const { body } = await client.asInternalUser.ml.getTrainedModelsStats({
...(modelId ? { model_id: modelId } : {}),
});
return response.ok({
body,
});
} catch (e) {
return response.customError(wrapError(e));
}
})
);
/**
* @apiGroup Inference
*
* @api {get} /api/ml/inference/:modelId/pipelines Get model pipelines
* @apiName GetModelPipelines
* @apiDescription Retrieves pipelines associated with a model
*/
router.get(
{
path: '/api/ml/inference/{modelId}/pipelines',
validate: {
params: modelIdSchema,
},
options: {
tags: ['access:ml:canGetDataFrameAnalytics'],
},
},
mlLicense.fullLicenseAPIGuard(async ({ client, request, response }) => {
try {
const { modelId } = request.params;
const result = await modelsProvider(client).getModelsPipelines(modelId.split(','));
return response.ok({
body: [...result].map(([id, pipelines]) => ({ model_id: id, pipelines })),
});
} catch (e) {
return response.customError(wrapError(e));
}
})
);
/**
* @apiGroup Inference
*
* @api {delete} /api/ml/inference/:modelId Get stats of a trained inference model
* @apiName DeleteInferenceModel
* @apiDescription Deletes an existing trained inference model that is currently not referenced by an ingest pipeline.
*/
router.delete(
{
path: '/api/ml/inference/{modelId}',
validate: {
params: modelIdSchema,
},
options: {
tags: ['access:ml:canDeleteDataFrameAnalytics'],
},
},
mlLicense.fullLicenseAPIGuard(async ({ client, request, response }) => {
try {
const { modelId } = request.params;
const { body } = await client.asInternalUser.ml.deleteTrainedModel({
model_id: modelId,
});
return response.ok({
body,
});
} catch (e) {
return response.customError(wrapError(e));
}
})
);
}

View file

@ -0,0 +1,26 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { schema } from '@kbn/config-schema';
export const modelIdSchema = schema.object({
/**
* Model ID
*/
modelId: schema.string(),
});
export const optionalModelIdSchema = schema.object({
/**
* Model ID
*/
modelId: schema.maybe(schema.string()),
});
export const getInferenceQuerySchema = schema.object({
size: schema.maybe(schema.string()),
with_pipelines: schema.maybe(schema.string()),
});

View file

@ -5,14 +5,14 @@
*/
import expect from '@kbn/expect';
import { DataFrameAnalyticsConfig } from '../../../../plugins/ml/public/application/data_frame_analytics/common';
import {
ClassificationAnalysis,
RegressionAnalysis,
} from '../../../../plugins/ml/public/application/data_frame_analytics/common/analytics';
import { FtrProviderContext } from '../../ftr_provider_context';
import { MlCommon } from './common';
import { MlApi } from './api';
import {
ClassificationAnalysis,
RegressionAnalysis,
} from '../../../../plugins/ml/common/types/data_frame_analytics';
enum ANALYSIS_CONFIG_TYPE {
OUTLIER_DETECTION = 'outlier_detection',