github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 01:13:23 -04:00
Code Issues Projects Releases Packages Wiki Activity

[ML] Nodes overview for the Model Management page (#115772)

Browse source 
* [ML] trained models tab

* [ML] wip nodes list

* [ML] add types

* [ML] add types

* [ML] node expanded row

* [ML] wip show memory usage

* [ML] refactor, use model_memory_limit for dfa jobs

* [ML] fix refresh button

* [ML] add process memory overhead

* [ML] trained models memory overview

* [ML] add jvm size, remove node props from the response

* [ML] fix tab name

* [ML] custom colors for the bar chart

* [ML] sub jvm size

* [ML] updates for the model list

* [ML] apply native process overhead

* [ML]add adjusted_total_in_bytes

* [ML] start and stop deployment

* [ML] fix default sorting

* [ML] fix types issues

* [ML] fix const

* [ML] remove unused i18n strings

* [ML] fix lint

* [ML] extra custom URLs test

* [ML] update tests for model provider

* [ML] add node routing state info

* [ML] fix functional tests

* [ML] update for es response

* [ML] GetTrainedModelDeploymentStats

* [ML] add deployment stats

* [ML] add spacer

* [ML] disable stop allocation for models with pipelines

* [ML] fix type

* [ML] add beta label

* [ML] move beta label

* [ML] rename model_size prop

* [ML] update tooltip header

* [ML] update text

* [ML] remove ts ignore

* [ML] update types

* remove commented code

* replace toast notification service

* remove ts-ignore

* remove empty panel

* add comments, update test subjects

* fix ts error

* update comment

* fix applying memory overhead

* Revert "fix applying memory overhead"

This reverts commit 0cf38fbead.

* fix type, remove ts-ignore

* add todo comment
This commit is contained in:
Dima Arnautov 2021-10-26 19:39:37 +02:00 • committed by GitHub
parent c200c44347
commit 605e9e2d3d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
50 changed files with 2459 additions and 135 deletions
Download patch file Download diff file Expand all files Collapse all files

3
x-pack/plugins/ml/common/constants/locator.ts
View file

@ -13,7 +13,8 @@ export const ML_PAGES = {
SINGLE_METRIC_VIEWER: 'timeseriesexplorer',
DATA_FRAME_ANALYTICS_JOBS_MANAGE: 'data_frame_analytics',
DATA_FRAME_ANALYTICS_CREATE_JOB: 'data_frame_analytics/new_job',
DATA_FRAME_ANALYTICS_MODELS_MANAGE: 'data_frame_analytics/models',
TRAINED_MODELS_MANAGE: 'trained_models',
TRAINED_MODELS_NODES: 'trained_models/nodes',
DATA_FRAME_ANALYTICS_EXPLORATION: 'data_frame_analytics/exploration',
DATA_FRAME_ANALYTICS_MAP: 'data_frame_analytics/map',
/**

12
x-pack/plugins/ml/common/types/locator.ts
View file

@ -184,6 +184,10 @@ export interface DataFrameAnalyticsQueryState {
globalState?: MlCommonGlobalState;
}
export interface TrainedModelsQueryState {
modelId?: string;
}
export type DataFrameAnalyticsUrlState = MLPageState<
| typeof ML_PAGES.DATA_FRAME_ANALYTICS_JOBS_MANAGE
| typeof ML_PAGES.DATA_FRAME_ANALYTICS_MAP
@ -250,8 +254,14 @@ export type MlLocatorState =
| DataFrameAnalyticsExplorationUrlState
| CalendarEditUrlState
| FilterEditUrlState
| MlGenericUrlState;
| MlGenericUrlState
| TrainedModelsUrlState;
export type MlLocatorParams = MlLocatorState & SerializableRecord;
export type MlLocator = LocatorPublic<MlLocatorParams>;
export type TrainedModelsUrlState = MLPageState<
typeof ML_PAGES.TRAINED_MODELS_MANAGE,
TrainedModelsQueryState | undefined
>;

81
x-pack/plugins/ml/common/types/trained_models.ts
View file

@ -44,6 +44,7 @@ export interface TrainedModelStat {
}
>;
};
deployment_stats?: Omit<TrainedModelDeploymentStatsResponse, 'model_id'>;
}
type TreeNode = object;
@ -95,6 +96,7 @@ export interface TrainedModelConfigResponse {
model_aliases?: string[];
} & Record<string, unknown>;
model_id: string;
model_type: 'tree_ensemble' | 'pytorch' | 'lang_ident';
tags: string[];
version: string;
inference_config?: Record<string, any>;
@ -117,3 +119,82 @@ export interface ModelPipelines {
export interface InferenceConfigResponse {
trained_model_configs: TrainedModelConfigResponse[];
}
export interface TrainedModelDeploymentStatsResponse {
model_id: string;
model_size_bytes: number;
inference_threads: number;
model_threads: number;
state: string;
allocation_status: { target_allocation_count: number; state: string; allocation_count: number };
nodes: Array<{
node: Record<
string,
{
transport_address: string;
roles: string[];
name: string;
attributes: {
'ml.machine_memory': string;
'xpack.installed': string;
'ml.max_open_jobs': string;
'ml.max_jvm_size': string;
};
ephemeral_id: string;
}
>;
inference_count: number;
routing_state: { routing_state: string };
average_inference_time_ms: number;
last_access: number;
}>;
}
export interface NodeDeploymentStatsResponse {
id: string;
name: string;
transport_address: string;
attributes: Record<string, string>;
roles: string[];
allocated_models: Array<{
inference_threads: number;
allocation_status: {
target_allocation_count: number;
state: string;
allocation_count: number;
};
model_id: string;
state: string;
model_threads: number;
model_size_bytes: number;
}>;
memory_overview: {
machine_memory: {
/** Total machine memory in bytes */
total: number;
jvm: number;
};
/** Open anomaly detection jobs + hardcoded overhead */
anomaly_detection: {
/** Total size in bytes */
total: number;
};
/** DFA jobs currently in training + hardcoded overhead */
dfa_training: {
total: number;
};
/** Allocated trained models */
trained_models: {
total: number;
by_model: Array<{
model_id: string;
model_size: number;
}>;
};
};
}
export interface NodesOverviewResponse {
count: number;
nodes: NodeDeploymentStatsResponse[];
}

7
x-pack/plugins/ml/public/application/app.tsx
View file

@ -27,7 +27,11 @@ import { MlRouter } from './routing';
import { mlApiServicesProvider } from './services/ml_api_service';
import { HttpService } from './services/http_service';
import { ML_APP_LOCATOR, ML_PAGES } from '../../common/constants/locator';
export type MlDependencies = Omit<MlSetupDependencies, 'share' | 'indexPatternManagement'> &
export type MlDependencies = Omit<
MlSetupDependencies,
'share' | 'indexPatternManagement' | 'fieldFormats'
> &
MlStartDependencies;
interface AppProps {
@ -84,6 +88,7 @@ const App: FC<AppProps> = ({ coreStart, deps, appMountParams }) => {
triggersActionsUi: deps.triggersActionsUi,
dataVisualizer: deps.dataVisualizer,
usageCollection: deps.usageCollection,
fieldFormats: deps.fieldFormats,
...coreStart,
};

31
x-pack/plugins/ml/public/application/components/navigation_menu/main_tabs.tsx
View file

@ -7,7 +7,7 @@
import React, { FC, useState, useEffect } from 'react';
import { EuiPageHeader } from '@elastic/eui';
import { EuiPageHeader, EuiBetaBadge } from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import { TabId } from './navigation_menu';
import { useMlKibana, useMlLocator, useNavigateToPath } from '../../contexts/kibana';
@ -20,6 +20,7 @@ export interface Tab {
id: TabId;
name: any;
disabled: boolean;
betaTag?: JSX.Element;
}
interface Props {
@ -50,6 +51,27 @@ function getTabs(disableLinks: boolean): Tab[] {
}),
disabled: disableLinks,
},
{
id: 'trained_models',
name: i18n.translate('xpack.ml.navMenu.trainedModelsTabLinkText', {
defaultMessage: 'Model Management',
}),
disabled: disableLinks,
betaTag: (
<EuiBetaBadge
label={i18n.translate('xpack.ml.navMenu.trainedModelsTabBetaLabel', {
defaultMessage: 'Experimental',
})}
size="m"
color="hollow"
iconType="beaker"
tooltipContent={i18n.translate('xpack.ml.navMenu.trainedModelsTabBetaTooltipContent', {
defaultMessage:
"Model Management is an experimental feature and subject to change. We'd love to hear your feedback.",
})}
/>
),
},
{
id: 'datavisualizer',
name: i18n.translate('xpack.ml.navMenu.dataVisualizerTabLinkText', {
@ -93,6 +115,12 @@ const TAB_DATA: Record<TabId, TabData> = {
defaultMessage: 'Data Frame Analytics',
}),
},
trained_models: {
testSubject: 'mlMainTab modelManagement',
name: i18n.translate('xpack.ml.trainedModelsTabLabel', {
defaultMessage: 'Trained Models',
}),
},
datavisualizer: {
testSubject: 'mlMainTab dataVisualizer',
name: i18n.translate('xpack.ml.dataVisualizerTabLabel', {
@ -173,6 +201,7 @@ export const MainTabs: FC<Props> = ({ tabId, disableLinks }) => {
},
'data-test-subj': testSubject + (id === selectedTabId ? ' selected' : ''),
isSelected: id === selectedTabId,
append: tab.betaTag,
};
})}
/>

1
x-pack/plugins/ml/public/application/components/navigation_menu/navigation_menu.tsx
View file

@ -15,6 +15,7 @@ export type TabId =
| 'access-denied'
| 'anomaly_detection'
| 'data_frame_analytics'
| 'trained_models'
| 'datavisualizer'
| 'overview'
| 'settings';

2
x-pack/plugins/ml/public/application/contexts/kibana/kibana_context.ts
View file

@ -21,6 +21,7 @@ import type { EmbeddableStart } from '../../../../../../../src/plugins/embeddabl
import type { MapsStartApi } from '../../../../../maps/public';
import type { DataVisualizerPluginStart } from '../../../../../data_visualizer/public';
import type { TriggersAndActionsUIPublicPluginStart } from '../../../../../triggers_actions_ui/public';
import type { FieldFormatsRegistry } from '../../../../../../../src/plugins/field_formats/common';
interface StartPlugins {
data: DataPublicPluginStart;
@ -32,6 +33,7 @@ interface StartPlugins {
triggersActionsUi?: TriggersAndActionsUIPublicPluginStart;
dataVisualizer?: DataVisualizerPluginStart;
usageCollection?: UsageCollectionSetup;
fieldFormats: FieldFormatsRegistry;
}
export type StartServices = CoreStart &
StartPlugins & {

17
x-pack/plugins/ml/public/application/contexts/kibana/use_field_formatter.ts Normal file
View file

@ -0,0 +1,17 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { useMlKibana } from './kibana_context';
export function useFieldFormatter(fieldType: 'bytes') {
const {
services: { fieldFormats },
} = useMlKibana();
const fieldFormatter = fieldFormats.deserialize({ id: fieldType });
return fieldFormatter.convert.bind(fieldFormatter);
}

8
x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/analytics_navigation_bar/analytics_navigation_bar.tsx
View file

@ -33,14 +33,6 @@ export const AnalyticsNavigationBar: FC<{
path: '/data_frame_analytics',
testSubj: 'mlAnalyticsJobsTab',
},
{
id: 'models',
name: i18n.translate('xpack.ml.dataframe.modelsTabLabel', {
defaultMessage: 'Models',
}),
path: '/data_frame_analytics/models',
testSubj: 'mlTrainedModelsTab',
},
];
if (jobId !== undefined || modelId !== undefined) {
navTabs.push({

2
x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/page.tsx
View file

@ -31,7 +31,6 @@ import { NodeAvailableWarning } from '../../../components/node_available_warning
import { SavedObjectsWarning } from '../../../components/saved_objects_warning';
import { UpgradeWarning } from '../../../components/upgrade';
import { AnalyticsNavigationBar } from './components/analytics_navigation_bar';
import { ModelsList } from './components/models_management';
import { JobMap } from '../job_map';
import { usePageUrlState } from '../../../util/url_state';
import { ListingPageUrlState } from '../../../../../common/types/common';
@ -125,7 +124,6 @@ export const Page: FC = () => {
updatePageState={setDfaPageState}
/>
)}
{selectedTabId === 'models' && <ModelsList />}
</EuiPageContent>
</EuiPageBody>
</EuiPage>

8
x-pack/plugins/ml/public/application/routing/breadcrumbs.ts
View file

@ -41,6 +41,13 @@ export const DATA_FRAME_ANALYTICS_BREADCRUMB: ChromeBreadcrumb = Object.freeze({
href: '/data_frame_analytics',
});
export const TRAINED_MODELS: ChromeBreadcrumb = Object.freeze({
text: i18n.translate('xpack.ml.trainedModelsLabel', {
defaultMessage: 'Trained Models',
}),
href: '/trained_models',
});
export const DATA_VISUALIZER_BREADCRUMB: ChromeBreadcrumb = Object.freeze({
text: i18n.translate('xpack.ml.datavisualizerBreadcrumbLabel', {
defaultMessage: 'Data Visualizer',
@ -74,6 +81,7 @@ const breadcrumbs = {
SETTINGS_BREADCRUMB,
ANOMALY_DETECTION_BREADCRUMB,
DATA_FRAME_ANALYTICS_BREADCRUMB,
TRAINED_MODELS,
DATA_VISUALIZER_BREADCRUMB,
CREATE_JOB_BREADCRUMB,
CALENDAR_MANAGEMENT_BREADCRUMB,

1
x-pack/plugins/ml/public/application/routing/routes/data_frame_analytics/index.ts
View file

@ -8,5 +8,4 @@
export * from './analytics_jobs_list';
export * from './analytics_job_exploration';
export * from './analytics_job_creation';
export * from './models_list';
export * from './analytics_map';

1
x-pack/plugins/ml/public/application/routing/routes/index.ts
View file

@ -14,3 +14,4 @@ export * from './data_frame_analytics';
export { timeSeriesExplorerRouteFactory } from './timeseriesexplorer';
export * from './explorer';
export * from './access_denied';
export * from './trained_models';

9
x-pack/plugins/ml/public/application/routing/routes/trained_models/index.ts Normal file
View file

@ -0,0 +1,9 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export * from './models_list';
export * from './nodes_list';

8
x-pack/plugins/ml/public/application/routing/routes/data_frame_analytics/models_list.tsx → x-pack/plugins/ml/public/application/routing/routes/trained_models/models_list.tsx
View file

@ -13,20 +13,20 @@ import { NavigateToPath } from '../../../contexts/kibana';
import { MlRoute, PageLoader, PageProps } from '../../router';
import { useResolver } from '../../use_resolver';
import { basicResolvers } from '../../resolvers';
import { Page } from '../../../data_frame_analytics/pages/analytics_management';
import { getBreadcrumbWithUrlForApp } from '../../breadcrumbs';
import { Page } from '../../../trained_models';
export const modelsListRouteFactory = (
navigateToPath: NavigateToPath,
basePath: string
): MlRoute => ({
path: '/data_frame_analytics/models',
path: '/trained_models',
render: (props, deps) => <PageWrapper {...props} deps={deps} />,
breadcrumbs: [
getBreadcrumbWithUrlForApp('ML_BREADCRUMB', navigateToPath, basePath),
getBreadcrumbWithUrlForApp('DATA_FRAME_ANALYTICS_BREADCRUMB', navigateToPath, basePath),
getBreadcrumbWithUrlForApp('TRAINED_MODELS', navigateToPath, basePath),
{
text: i18n.translate('xpack.ml.dataFrameAnalyticsBreadcrumbs.modelsListLabel', {
text: i18n.translate('xpack.ml.trainedModelsBreadcrumbs.modelsListLabel', {
defaultMessage: 'Model Management',
}),
href: '',

44
x-pack/plugins/ml/public/application/routing/routes/trained_models/nodes_list.tsx Normal file
View file

@ -0,0 +1,44 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import React, { FC } from 'react';
import { i18n } from '@kbn/i18n';
import { NavigateToPath } from '../../../contexts/kibana';
import { MlRoute, PageLoader, PageProps } from '../../router';
import { useResolver } from '../../use_resolver';
import { basicResolvers } from '../../resolvers';
import { getBreadcrumbWithUrlForApp } from '../../breadcrumbs';
import { Page } from '../../../trained_models';
export const nodesListRouteFactory = (
navigateToPath: NavigateToPath,
basePath: string
): MlRoute => ({
path: '/trained_models/nodes',
render: (props, deps) => <PageWrapper {...props} deps={deps} />,
breadcrumbs: [
getBreadcrumbWithUrlForApp('ML_BREADCRUMB', navigateToPath, basePath),
getBreadcrumbWithUrlForApp('TRAINED_MODELS', navigateToPath, basePath),
{
text: i18n.translate('xpack.ml.trainedModelsBreadcrumbs.nodesListLabel', {
defaultMessage: 'Nodes Overview',
}),
href: '',
},
],
});
const PageWrapper: FC<PageProps> = ({ location, deps }) => {
const { context } = useResolver(undefined, undefined, deps.config, basicResolvers(deps));
return (
<PageLoader context={context}>
<Page />
</PageLoader>
);
};

40
x-pack/plugins/ml/public/application/services/ml_api_service/trained_models.ts
View file

@ -14,6 +14,8 @@ import {
TrainedModelConfigResponse,
ModelPipelines,
TrainedModelStat,
NodesOverviewResponse,
TrainedModelDeploymentStatsResponse,
} from '../../../../common/types/trained_models';
export interface InferenceQueryParams {
@ -114,11 +116,47 @@ export function trainedModelsApiProvider(httpService: HttpService) {
* @param modelId - Model ID
*/
deleteTrainedModel(modelId: string) {
return httpService.http<any>({
return httpService.http<{ acknowledge: boolean }>({
path: `${apiBasePath}/trained_models/${modelId}`,
method: 'DELETE',
});
},
getTrainedModelDeploymentStats(modelId?: string | string[]) {
let model = modelId ?? '*';
if (Array.isArray(modelId)) {
model = modelId.join(',');
}
return httpService.http<{
count: number;
deployment_stats: TrainedModelDeploymentStatsResponse[];
}>({
path: `${apiBasePath}/trained_models/${model}/deployment/_stats`,
method: 'GET',
});
},
getTrainedModelsNodesOverview() {
return httpService.http<NodesOverviewResponse>({
path: `${apiBasePath}/trained_models/nodes_overview`,
method: 'GET',
});
},
startModelAllocation(modelId: string) {
return httpService.http<{ acknowledge: boolean }>({
path: `${apiBasePath}/trained_models/${modelId}/deployment/_start`,
method: 'POST',
});
},
stopModelAllocation(modelId: string) {
return httpService.http<{ acknowledge: boolean }>({
path: `${apiBasePath}/trained_models/${modelId}/deployment/_stop`,
method: 'POST',
});
},
};
}

8
x-pack/plugins/ml/public/application/trained_models/index.ts Normal file
View file

@ -0,0 +1,8 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export { Page } from './page';

2
x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/models_management/delete_models_modal.tsx → x-pack/plugins/ml/public/application/trained_models/models_management/delete_models_modal.tsx
View file

@ -6,7 +6,6 @@
*/
import React, { FC } from 'react';
import { FormattedMessage } from '@kbn/i18n/react';
import {
EuiModal,
EuiModalHeader,
@ -17,6 +16,7 @@ import {
EuiButton,
EuiCallOut,
} from '@elastic/eui';
import { FormattedMessage } from '@kbn/i18n/react';
import { ModelItemFull } from './models_list';
interface DeleteModelsModalProps {

111
x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/models_management/expanded_row.tsx → x-pack/plugins/ml/public/application/trained_models/models_management/expanded_row.tsx
View file

@ -6,30 +6,30 @@
*/
import React, { FC, Fragment } from 'react';
import { FormattedMessage } from '@kbn/i18n/react';
import {
EuiBadge,
EuiButtonEmpty,
EuiCodeBlock,
EuiDescriptionList,
EuiFlexGrid,
EuiFlexGroup,
EuiFlexItem,
EuiHorizontalRule,
EuiNotificationBadge,
EuiPanel,
EuiSpacer,
EuiTabbedContent,
EuiTitle,
EuiNotificationBadge,
EuiFlexGrid,
EuiFlexItem,
EuiCodeBlock,
EuiText,
EuiHorizontalRule,
EuiFlexGroup,
EuiTextColor,
EuiButtonEmpty,
EuiBadge,
EuiTitle,
} from '@elastic/eui';
import { EuiDescriptionListProps } from '@elastic/eui/src/components/description_list/description_list';
import { FormattedMessage } from '@kbn/i18n/react';
import { ModelItemFull } from './models_list';
import { useMlKibana } from '../../../../../contexts/kibana';
import { timeFormatter } from '../../../../../../../common/util/date_utils';
import { isDefined } from '../../../../../../../common/types/guards';
import { isPopulatedObject } from '../../../../../../../common';
import { useMlKibana } from '../../contexts/kibana';
import { timeFormatter } from '../../../../common/util/date_utils';
import { isDefined } from '../../../../common/types/guards';
import { isPopulatedObject } from '../../../../common';
interface ExpandedRowProps {
item: ModelItemFull;
@ -52,6 +52,38 @@ const formatterDictionary: Record<string, (value: any) => JSX.Element | string |
timestamp: timeFormatter,
};
export function formatToListItems(
items: Record<string, unknown> | object
): EuiDescriptionListProps['listItems'] {
return Object.entries(items)
.filter(([, value]) => isDefined(value))
.map(([title, value]) => {
if (title in formatterDictionary) {
return {
title,
description: formatterDictionary[title](value),
};
}
return {
title,
description:
typeof value === 'object' ? (
<EuiCodeBlock
language="json"
fontSize="s"
paddingSize="s"
overflowHeight={300}
isCopyable={false}
>
{JSON.stringify(value, null, 2)}
</EuiCodeBlock>
) : (
value.toString()
),
};
});
}
export const ExpandedRow: FC<ExpandedRowProps> = ({ item }) => {
const {
inference_config: inferenceConfig,
@ -83,36 +115,6 @@ export const ExpandedRow: FC<ExpandedRowProps> = ({ item }) => {
license_level,
};
function formatToListItems(items: Record<string, any>): EuiDescriptionListProps['listItems'] {
return Object.entries(items)
.filter(([, value]) => isDefined(value))
.map(([title, value]) => {
if (title in formatterDictionary) {
return {
title,
description: formatterDictionary[title](value),
};
}
return {
title,
description:
typeof value === 'object' ? (
<EuiCodeBlock
language="json"
fontSize="s"
paddingSize="s"
overflowHeight={300}
isCopyable={false}
>
{JSON.stringify(value, null, 2)}
</EuiCodeBlock>
) : (
value.toString()
),
};
});
}
const {
services: { share },
} = useMlKibana();
@ -243,6 +245,27 @@ export const ExpandedRow: FC<ExpandedRowProps> = ({ item }) => {
content: (
<>
<EuiSpacer size={'m'} />
{stats.deployment_stats && (
<>
<EuiPanel>
<EuiTitle size={'xs'}>
<h5>
<FormattedMessage
id="xpack.ml.trainedModels.modelsList.expandedRow.deploymentStatsTitle"
defaultMessage="Deployment stats"
/>
</h5>
</EuiTitle>
<EuiSpacer size={'m'} />
<EuiDescriptionList
compressed={true}
type="column"
listItems={formatToListItems(stats.deployment_stats)}
/>
</EuiPanel>
<EuiSpacer size={'m'} />
</>
)}
<EuiFlexGrid columns={2}>
{stats.inference_stats && (
<EuiFlexItem>

1
x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/models_management/index.ts → x-pack/plugins/ml/public/application/trained_models/models_management/index.ts
View file

@ -12,4 +12,5 @@ export const ModelsTableToConfigMapping = {
description: 'description',
createdAt: 'create_time',
type: 'type',
modelType: 'model_type',
} as const;

193
x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/models_management/models_list.tsx → x-pack/plugins/ml/public/application/trained_models/models_management/models_list.tsx
View file

@ -6,8 +6,7 @@
*/
import React, { FC, useState, useCallback, useMemo } from 'react';
import { i18n } from '@kbn/i18n';
import { FormattedMessage } from '@kbn/i18n/react';
import { groupBy } from 'lodash';
import {
EuiInMemoryTable,
EuiFlexGroup,
@ -21,40 +20,37 @@ import {
EuiSearchBarProps,
} from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import { FormattedMessage } from '@kbn/i18n/react';
import { EuiBasicTableColumn } from '@elastic/eui/src/components/basic_table/basic_table';
import { EuiTableSelectionType } from '@elastic/eui/src/components/basic_table/table_types';
import { Action } from '@elastic/eui/src/components/basic_table/action_types';
import { StatsBar, ModelsBarStats } from '../../../../../components/stats_bar';
import { useTrainedModelsApiService } from '../../../../../services/ml_api_service/trained_models';
import { ModelsTableToConfigMapping } from './index';
import { DeleteModelsModal } from './delete_models_modal';
import {
useMlKibana,
useMlLocator,
useNavigateToPath,
useNotifications,
} from '../../../../../contexts/kibana';
import { ExpandedRow } from './expanded_row';
import {
TrainedModelConfigResponse,
ModelPipelines,
TrainedModelStat,
} from '../../../../../../../common/types/trained_models';
import {
getAnalysisType,
REFRESH_ANALYTICS_LIST_STATE,
refreshAnalyticsList$,
useRefreshAnalyticsList,
} from '../../../../common';
import { ML_PAGES } from '../../../../../../../common/constants/locator';
import { DataFrameAnalysisConfigType } from '../../../../../../../common/types/data_frame_analytics';
import { timeFormatter } from '../../../../../../../common/util/date_utils';
import { isPopulatedObject } from '../../../../../../../common';
import { ListingPageUrlState } from '../../../../../../../common/types/common';
import { usePageUrlState } from '../../../../../util/url_state';
import { BUILT_IN_MODEL_TAG } from '../../../../../../../common/constants/data_frame_analytics';
import { useTableSettings } from '../analytics_list/use_table_settings';
} from '../../data_frame_analytics/common';
import { ModelsTableToConfigMapping } from './index';
import { ModelsBarStats, StatsBar } from '../../components/stats_bar';
import { useMlKibana, useMlLocator, useNavigateToPath } from '../../contexts/kibana';
import { useTrainedModelsApiService } from '../../services/ml_api_service/trained_models';
import {
ModelPipelines,
TrainedModelConfigResponse,
TrainedModelStat,
} from '../../../../common/types/trained_models';
import { BUILT_IN_MODEL_TAG } from '../../../../common/constants/data_frame_analytics';
import { DataFrameAnalysisConfigType } from '../../../../common/types/data_frame_analytics';
import { DeleteModelsModal } from './delete_models_modal';
import { ML_PAGES } from '../../../../common/constants/locator';
import { ListingPageUrlState } from '../../../../common/types/common';
import { usePageUrlState } from '../../util/url_state';
import { ExpandedRow } from './expanded_row';
import { isPopulatedObject } from '../../../../common';
import { timeFormatter } from '../../../../common/util/date_utils';
import { useTableSettings } from '../../data_frame_analytics/pages/analytics_management/components/analytics_list/use_table_settings';
import { useToastNotificationService } from '../../services/toast_notification_service';
type Stats = Omit<TrainedModelStat, 'model_id'>;
@ -87,7 +83,7 @@ export const ModelsList: FC = () => {
const urlLocator = useMlLocator()!;
const [pageState, updatePageState] = usePageUrlState(
ML_PAGES.DATA_FRAME_ANALYTICS_MODELS_MANAGE,
ML_PAGES.TRAINED_MODELS_MANAGE,
getDefaultModelsListState()
);
@ -96,7 +92,9 @@ export const ModelsList: FC = () => {
const canDeleteDataFrameAnalytics = capabilities.ml.canDeleteDataFrameAnalytics as boolean;
const trainedModelsApiService = useTrainedModelsApiService();
const { toasts } = useNotifications();
const { displayErrorToast, displayDangerToast, displaySuccessToast } =
useToastNotificationService();
const [isLoading, setIsLoading] = useState(false);
const [items, setItems] = useState<ModelItem[]>([]);
@ -133,6 +131,7 @@ export const ModelsList: FC = () => {
...(typeof model.inference_config === 'object'
? {
type: [
model.model_type,
...Object.keys(model.inference_config),
...(isBuiltInModel(model) ? [BUILT_IN_MODEL_TYPE] : []),
],
@ -159,11 +158,12 @@ export const ModelsList: FC = () => {
);
}
} catch (error) {
toasts.addError(new Error(error.body?.message), {
title: i18n.translate('xpack.ml.trainedModels.modelsList.fetchFailedErrorMessage', {
displayErrorToast(
error,
i18n.translate('xpack.ml.trainedModels.modelsList.fetchFailedErrorMessage', {
defaultMessage: 'Models fetch failed',
}),
});
})
);
}
setIsLoading(false);
refreshAnalyticsList$.next(REFRESH_ANALYTICS_LIST_STATE.IDLE);
@ -191,23 +191,39 @@ export const ModelsList: FC = () => {
* Fetches models stats and update the original object
*/
const fetchModelsStats = useCallback(async (models: ModelItem[]) => {
const modelIdsToFetch = models.map((model) => model.model_id);
const { true: pytorchModels } = groupBy(models, (m) => m.model_type === 'pytorch');
try {
const { trained_model_stats: modelsStatsResponse } =
await trainedModelsApiService.getTrainedModelStats(modelIdsToFetch);
if (models) {
const { trained_model_stats: modelsStatsResponse } =
await trainedModelsApiService.getTrainedModelStats(models.map((m) => m.model_id));
for (const { model_id: id, ...stats } of modelsStatsResponse) {
const model = models.find((m) => m.model_id === id);
model!.stats = stats;
for (const { model_id: id, ...stats } of modelsStatsResponse) {
const model = models.find((m) => m.model_id === id);
model!.stats = stats;
}
}
if (pytorchModels) {
const { deployment_stats: deploymentStatsResponse } =
await trainedModelsApiService.getTrainedModelDeploymentStats(
pytorchModels.map((m) => m.model_id)
);
for (const { model_id: id, ...stats } of deploymentStatsResponse) {
const model = models.find((m) => m.model_id === id);
model!.stats!.deployment_stats = stats;
}
}
return true;
} catch (error) {
toasts.addError(new Error(error.body.message), {
title: i18n.translate('xpack.ml.trainedModels.modelsList.fetchModelStatsErrorMessage', {
displayErrorToast(
error,
i18n.translate('xpack.ml.trainedModels.modelsList.fetchModelStatsErrorMessage', {
defaultMessage: 'Fetch model stats failed',
}),
});
})
);
}
}, []);
@ -220,6 +236,7 @@ export const ModelsList: FC = () => {
if (type) {
acc.add(type);
}
acc.add(item.model_type);
return acc;
}, new Set<string>());
return [...result].map((v) => ({
@ -233,7 +250,7 @@ export const ModelsList: FC = () => {
if (await fetchModelsStats(models)) {
setModelsToDelete(models as ModelItemFull[]);
} else {
toasts.addDanger(
displayDangerToast(
i18n.translate('xpack.ml.trainedModels.modelsList.unableToDeleteModelsErrorMessage', {
defaultMessage: 'Unable to delete models',
})
@ -256,7 +273,7 @@ export const ModelsList: FC = () => {
(model) => !modelsToDelete.some((toDelete) => toDelete.model_id === model.model_id)
)
);
toasts.addSuccess(
displaySuccessToast(
i18n.translate('xpack.ml.trainedModels.modelsList.successfullyDeletedMessage', {
defaultMessage:
'{modelsCount, plural, one {Model {modelsToDeleteIds}} other {# models}} {modelsCount, plural, one {has} other {have}} been successfully deleted',
@ -267,14 +284,15 @@ export const ModelsList: FC = () => {
})
);
} catch (error) {
toasts.addError(new Error(error?.body?.message), {
title: i18n.translate('xpack.ml.trainedModels.modelsList.fetchDeletionErrorMessage', {
displayErrorToast(
error,
i18n.translate('xpack.ml.trainedModels.modelsList.fetchDeletionErrorMessage', {
defaultMessage: '{modelsCount, plural, one {Model} other {Models}} deletion failed',
values: {
modelsCount: modelsToDeleteIds.length,
},
}),
});
})
);
}
}
@ -336,6 +354,77 @@ export const ModelsList: FC = () => {
await navigateToPath(path, false);
},
},
{
name: i18n.translate('xpack.ml.inference.modelsList.startModelAllocationActionLabel', {
defaultMessage: 'Start allocation',
}),
description: i18n.translate('xpack.ml.inference.modelsList.startModelAllocationActionLabel', {
defaultMessage: 'Start allocation',
}),
icon: 'download',
type: 'icon',
isPrimary: true,
available: (item) => item.model_type === 'pytorch',
onClick: async (item) => {
try {
await trainedModelsApiService.startModelAllocation(item.model_id);
displaySuccessToast(
i18n.translate('xpack.ml.trainedModels.modelsList.startSuccess', {
defaultMessage: 'Deployment for "{modelId}" has been started successfully.',
values: {
modelId: item.model_id,
},
})
);
} catch (e) {
displayErrorToast(
e,
i18n.translate('xpack.ml.trainedModels.modelsList.startFailed', {
defaultMessage: 'Failed to start "{modelId}"',
values: {
modelId: item.model_id,
},
})
);
}
},
},
{
name: i18n.translate('xpack.ml.inference.modelsList.stopModelAllocationActionLabel', {
defaultMessage: 'Stop allocation',
}),
description: i18n.translate('xpack.ml.inference.modelsList.stopModelAllocationActionLabel', {
defaultMessage: 'Stop allocation',
}),
icon: 'stop',
type: 'icon',
isPrimary: true,
available: (item) => item.model_type === 'pytorch',
enabled: (item) => !isPopulatedObject(item.pipelines),
onClick: async (item) => {
try {
await trainedModelsApiService.stopModelAllocation(item.model_id);
displaySuccessToast(
i18n.translate('xpack.ml.trainedModels.modelsList.stopSuccess', {
defaultMessage: 'Deployment for "{modelId}" has been stopped successfully.',
values: {
modelId: item.model_id,
},
})
);
} catch (e) {
displayErrorToast(
e,
i18n.translate('xpack.ml.trainedModels.modelsList.stopFailed', {
defaultMessage: 'Failed to stop "{modelId}"',
values: {
modelId: item.model_id,
},
})
);
}
},
},
{
name: i18n.translate('xpack.ml.trainedModels.modelsList.deleteModelActionLabel', {
defaultMessage: 'Delete model',
@ -399,7 +488,7 @@ export const ModelsList: FC = () => {
defaultMessage: 'ID',
}),
sortable: true,
truncateText: true,
truncateText: false,
'data-test-subj': 'mlModelsTableColumnId',
},
{
@ -409,7 +498,7 @@ export const ModelsList: FC = () => {
defaultMessage: 'Description',
}),
sortable: false,
truncateText: true,
truncateText: false,
'data-test-subj': 'mlModelsTableColumnDescription',
},
{

69
x-pack/plugins/ml/public/application/trained_models/navigation_bar.tsx Normal file
View file

@ -0,0 +1,69 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import React, { FC, useCallback, useMemo } from 'react';
import { i18n } from '@kbn/i18n';
import { EuiTab, EuiTabs } from '@elastic/eui';
import { useNavigateToPath } from '../contexts/kibana';
interface Tab {
id: string;
name: string;
path: string;
}
export const TrainedModelsNavigationBar: FC<{
selectedTabId?: string;
}> = ({ selectedTabId }) => {
const navigateToPath = useNavigateToPath();
const tabs = useMemo(() => {
const navTabs = [
{
id: 'trained_models',
name: i18n.translate('xpack.ml.trainedModels.modelsTabLabel', {
defaultMessage: 'Models',
}),
path: '/trained_models',
testSubj: 'mlTrainedModelsTab',
},
{
id: 'nodes',
name: i18n.translate('xpack.ml.trainedModels.nodesTabLabel', {
defaultMessage: 'Nodes',
}),
path: '/trained_models/nodes',
testSubj: 'mlNodesOverviewTab',
},
];
return navTabs;
}, []);
const onTabClick = useCallback(
async (tab: Tab) => {
await navigateToPath(tab.path, true);
},
[navigateToPath]
);
return (
<EuiTabs>
{tabs.map((tab) => {
return (
<EuiTab
key={`tab-${tab.id}`}
isSelected={tab.id === selectedTabId}
onClick={onTabClick.bind(null, tab)}
data-test-subj={tab.testSubj}
>
{tab.name}
</EuiTab>
);
})}
</EuiTabs>
);
};

123
x-pack/plugins/ml/public/application/trained_models/nodes_overview/expanded_row.tsx Normal file
View file

@ -0,0 +1,123 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import React, { FC } from 'react';
import {
EuiDescriptionList,
EuiFlexGrid,
EuiFlexGroup,
EuiFlexItem,
EuiHorizontalRule,
EuiPanel,
EuiSpacer,
EuiTextColor,
EuiTitle,
} from '@elastic/eui';
import { FormattedMessage } from '@kbn/i18n/react';
import { NodeItemWithStats } from './nodes_list';
import { formatToListItems } from '../models_management/expanded_row';
interface ExpandedRowProps {
item: NodeItemWithStats;
}
export const ExpandedRow: FC<ExpandedRowProps> = ({ item }) => {
const {
allocated_models: allocatedModels,
attributes,
memory_overview: memoryOverview,
...details
} = item;
return (
<>
<EuiSpacer size={'m'} />
<EuiFlexGrid columns={2} gutterSize={'m'}>
<EuiFlexItem>
<EuiPanel>
<EuiTitle size={'xs'}>
<h5>
<FormattedMessage
id="xpack.ml.trainedModels.nodesList.expandedRow.detailsTitle"
defaultMessage="Details"
/>
</h5>
</EuiTitle>
<EuiSpacer size={'m'} />
<EuiDescriptionList
compressed={true}
type="column"
listItems={formatToListItems(details)}
/>
</EuiPanel>
<EuiSpacer size={'m'} />
</EuiFlexItem>
<EuiFlexItem>
<EuiPanel>
<EuiTitle size={'xs'}>
<h5>
<FormattedMessage
id="xpack.ml.trainedModels.nodesList.expandedRow.attributesTitle"
defaultMessage="Attributes"
/>
</h5>
</EuiTitle>
<EuiSpacer size={'m'} />
<EuiDescriptionList
compressed={true}
type="column"
listItems={formatToListItems(attributes)}
/>
</EuiPanel>
<EuiSpacer size={'m'} />
<EuiPanel>
<EuiTitle size={'xs'}>
<h5>
<FormattedMessage
id="xpack.ml.trainedModels.nodesList.expandedRow.allocatedModelsTitle"
defaultMessage="Allocated models"
/>
</h5>
</EuiTitle>
<EuiSpacer size={'m'} />
{allocatedModels.map(({ model_id: modelId, ...rest }) => {
return (
<>
<EuiFlexGroup>
<EuiFlexItem grow={false}>
<EuiTitle size="xxs">
<EuiTextColor color="subdued">
<h5>{modelId}</h5>
</EuiTextColor>
</EuiTitle>
</EuiFlexItem>
<EuiFlexItem>
<EuiHorizontalRule size={'full'} margin={'s'} />
</EuiFlexItem>
</EuiFlexGroup>
<EuiDescriptionList
compressed={true}
type="column"
listItems={formatToListItems(rest)}
/>
<EuiSpacer size={'s'} />
</>
);
})}
</EuiPanel>
</EuiFlexItem>
</EuiFlexGrid>
</>
);
};

8
x-pack/plugins/ml/public/application/trained_models/nodes_overview/index.ts Normal file
View file

@ -0,0 +1,8 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export { NodesList } from './nodes_list';

140
x-pack/plugins/ml/public/application/trained_models/nodes_overview/memory_preview_chart.tsx Normal file
View file

@ -0,0 +1,140 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { i18n } from '@kbn/i18n';
import React, { FC, useMemo } from 'react';
import {
Chart,
Settings,
BarSeries,
ScaleType,
Axis,
Position,
SeriesColorAccessor,
} from '@elastic/charts';
import { euiPaletteGray } from '@elastic/eui';
import { NodeDeploymentStatsResponse } from '../../../../common/types/trained_models';
import { useFieldFormatter } from '../../contexts/kibana/use_field_formatter';
import { useCurrentEuiTheme } from '../../components/color_range_legend';
interface MemoryPreviewChartProps {
memoryOverview: NodeDeploymentStatsResponse['memory_overview'];
}
export const MemoryPreviewChart: FC<MemoryPreviewChartProps> = ({ memoryOverview }) => {
const bytesFormatter = useFieldFormatter('bytes');
const { euiTheme } = useCurrentEuiTheme();
const groups = useMemo(
() => ({
jvm: {
name: i18n.translate('xpack.ml.trainedModels.nodesList.jvmHeapSIze', {
defaultMessage: 'JVM heap size',
}),
colour: euiTheme.euiColorVis1,
},
trained_models: {
name: i18n.translate('xpack.ml.trainedModels.nodesList.modelsMemoryUsage', {
defaultMessage: 'Trained models',
}),
colour: euiTheme.euiColorVis2,
},
anomaly_detection: {
name: i18n.translate('xpack.ml.trainedModels.nodesList.adMemoryUsage', {
defaultMessage: 'Anomaly detection jobs',
}),
colour: euiTheme.euiColorVis6,
},
dfa_training: {
name: i18n.translate('xpack.ml.trainedModels.nodesList.dfaMemoryUsage', {
defaultMessage: 'Data frame analytics jobs',
}),
colour: euiTheme.euiColorVis4,
},
available: {
name: i18n.translate('xpack.ml.trainedModels.nodesList.availableMemory', {
defaultMessage: 'Estimated available memory',
}),
colour: euiPaletteGray(5)[0],
},
}),
[]
);
const chartData = [
{
x: 0,
y: memoryOverview.machine_memory.jvm,
g: groups.jvm.name,
},
{
x: 0,
y: memoryOverview.trained_models.total,
g: groups.trained_models.name,
},
{
x: 0,
y: memoryOverview.anomaly_detection.total,
g: groups.anomaly_detection.name,
},
{
x: 0,
y: memoryOverview.dfa_training.total,
g: groups.dfa_training.name,
},
{
x: 0,
y:
memoryOverview.machine_memory.total -
memoryOverview.machine_memory.jvm -
memoryOverview.trained_models.total -
memoryOverview.dfa_training.total -
memoryOverview.anomaly_detection.total,
g: groups.available.name,
},
];
const barSeriesColorAccessor: SeriesColorAccessor = ({ specId, yAccessor, splitAccessors }) => {
const group = splitAccessors.get('g');
return Object.values(groups).find((v) => v.name === group)!.colour;
};
return (
<Chart size={['100%', 50]}>
<Settings
rotation={90}
tooltip={{
headerFormatter: ({ value }) =>
i18n.translate('xpack.ml.trainedModels.nodesList.memoryBreakdown', {
defaultMessage: 'Approximate memory breakdown based on the node info',
}),
}}
/>
<Axis
id="ml_memory"
position={Position.Bottom}
hide
tickFormat={(d: number) => bytesFormatter(d)}
/>
<BarSeries
id="bars"
xScaleType={ScaleType.Linear}
yScaleType={ScaleType.Linear}
xAccessor="x"
yAccessors={['y']}
splitSeriesAccessors={['g']}
stackAccessors={['x']}
data={chartData}
color={barSeriesColorAccessor}
/>
</Chart>
);
};

205
x-pack/plugins/ml/public/application/trained_models/nodes_overview/nodes_list.tsx Normal file
View file

@ -0,0 +1,205 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import React, { FC, useCallback, useMemo, useState } from 'react';
import {
EuiButtonIcon,
EuiFlexGroup,
EuiFlexItem,
EuiInMemoryTable,
EuiSearchBarProps,
EuiSpacer,
} from '@elastic/eui';
import { EuiBasicTableColumn } from '@elastic/eui/src/components/basic_table/basic_table';
import { i18n } from '@kbn/i18n';
import { ModelsBarStats, StatsBar } from '../../components/stats_bar';
import { NodeDeploymentStatsResponse } from '../../../../common/types/trained_models';
import { usePageUrlState } from '../../util/url_state';
import { ML_PAGES } from '../../../../common/constants/locator';
import { useTrainedModelsApiService } from '../../services/ml_api_service/trained_models';
import { useTableSettings } from '../../data_frame_analytics/pages/analytics_management/components/analytics_list/use_table_settings';
import { ExpandedRow } from './expanded_row';
import {
REFRESH_ANALYTICS_LIST_STATE,
refreshAnalyticsList$,
useRefreshAnalyticsList,
} from '../../data_frame_analytics/common';
import { MemoryPreviewChart } from './memory_preview_chart';
import { useFieldFormatter } from '../../contexts/kibana/use_field_formatter';
import { ListingPageUrlState } from '../../../../common/types/common';
export type NodeItem = NodeDeploymentStatsResponse;
export interface NodeItemWithStats extends NodeItem {
stats: any;
}
export const getDefaultNodesListState = (): ListingPageUrlState => ({
pageIndex: 0,
pageSize: 10,
sortField: 'name',
sortDirection: 'asc',
});
export const NodesList: FC = () => {
const trainedModelsApiService = useTrainedModelsApiService();
const bytesFormatter = useFieldFormatter('bytes');
const [items, setItems] = useState<NodeItem[]>([]);
const [isLoading, setIsLoading] = useState(false);
const [itemIdToExpandedRowMap, setItemIdToExpandedRowMap] = useState<Record<string, JSX.Element>>(
{}
);
const [pageState, updatePageState] = usePageUrlState(
ML_PAGES.TRAINED_MODELS_NODES,
getDefaultNodesListState()
);
const searchQueryText = pageState.queryText ?? '';
const fetchNodesData = useCallback(async () => {
const nodesResponse = await trainedModelsApiService.getTrainedModelsNodesOverview();
setItems(nodesResponse.nodes);
setIsLoading(false);
refreshAnalyticsList$.next(REFRESH_ANALYTICS_LIST_STATE.IDLE);
}, []);
const toggleDetails = (item: NodeItem) => {
const itemIdToExpandedRowMapValues = { ...itemIdToExpandedRowMap };
if (itemIdToExpandedRowMapValues[item.id]) {
delete itemIdToExpandedRowMapValues[item.id];
} else {
itemIdToExpandedRowMapValues[item.id] = <ExpandedRow item={item as NodeItemWithStats} />;
}
setItemIdToExpandedRowMap(itemIdToExpandedRowMapValues);
};
const columns: Array<EuiBasicTableColumn<NodeItem>> = [
{
align: 'left',
width: '40px',
isExpander: true,
render: (item: NodeItem) => (
<EuiButtonIcon
onClick={toggleDetails.bind(null, item)}
aria-label={
itemIdToExpandedRowMap[item.id]
? i18n.translate('xpack.ml.trainedModels.nodesList.collapseRow', {
defaultMessage: 'Collapse',
})
: i18n.translate('xpack.ml.trainedModels.nodesList.expandRow', {
defaultMessage: 'Expand',
})
}
iconType={itemIdToExpandedRowMap[item.id] ? 'arrowUp' : 'arrowDown'}
/>
),
'data-test-subj': 'mlNodesTableRowDetailsToggle',
},
{
field: 'name',
name: i18n.translate('xpack.ml.trainedModels.nodesList.nodeNameHeader', {
defaultMessage: 'Name',
}),
sortable: true,
truncateText: true,
'data-test-subj': 'mlNodesTableColumnName',
},
{
name: i18n.translate('xpack.ml.trainedModels.nodesList.nodeTotalMemoryHeader', {
defaultMessage: 'Total memory',
}),
width: '200px',
truncateText: true,
'data-test-subj': 'mlNodesTableColumnTotalMemory',
render: (v: NodeItem) => {
return bytesFormatter(v.attributes['ml.machine_memory']);
},
},
{
name: i18n.translate('xpack.ml.trainedModels.nodesList.nodeMemoryUsageHeader', {
defaultMessage: 'Memory usage',
}),
truncateText: true,
'data-test-subj': 'mlNodesTableColumnMemoryUsage',
render: (v: NodeItem) => {
return <MemoryPreviewChart memoryOverview={v.memory_overview} />;
},
},
];
const nodesStats: ModelsBarStats = useMemo(() => {
return {
total: {
show: true,
value: items.length,
label: i18n.translate('xpack.ml.trainedModels.nodesList.totalAmountLabel', {
defaultMessage: 'Total machine learning nodes',
}),
},
};
}, [items]);
const { onTableChange, pagination, sorting } = useTableSettings<NodeItem>(
items,
pageState,
updatePageState
);
const search: EuiSearchBarProps = {
query: searchQueryText,
onChange: (searchChange) => {
if (searchChange.error !== null) {
return false;
}
updatePageState({ queryText: searchChange.queryText, pageIndex: 0 });
return true;
},
box: {
incremental: true,
},
};
// Subscribe to the refresh observable to trigger reloading the model list.
useRefreshAnalyticsList({
isLoading: setIsLoading,
onRefresh: fetchNodesData,
});
return (
<>
<EuiSpacer size="m" />
<EuiFlexGroup justifyContent="spaceBetween">
{nodesStats && (
<EuiFlexItem grow={false}>
<StatsBar stats={nodesStats} dataTestSub={'mlTrainedModelsNodesStatsBar'} />
</EuiFlexItem>
)}
</EuiFlexGroup>
<EuiSpacer size="m" />
<div data-test-subj="mlNodesTableContainer">
<EuiInMemoryTable<NodeItem>
allowNeutralSort={false}
columns={columns}
hasActions={true}
isExpandable={true}
itemIdToExpandedRowMap={itemIdToExpandedRowMap}
isSelectable={false}
items={items}
itemId={'id'}
loading={isLoading}
search={search}
rowProps={(item) => ({
'data-test-subj': `mlNodesTableRow row-${item.id}`,
})}
pagination={pagination}
onTableChange={onTableChange}
sorting={sorting}
data-test-subj={isLoading ? 'mlNodesTable loading' : 'mlNodesTable loaded'}
/>
</div>
</>
);
};

77
x-pack/plugins/ml/public/application/trained_models/page.tsx Normal file
View file

@ -0,0 +1,77 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import React, { FC, Fragment, useMemo } from 'react';
import { FormattedMessage } from '@kbn/i18n/react';
import {
EuiFlexGroup,
EuiFlexItem,
EuiPage,
EuiPageBody,
EuiPageContent,
EuiPageHeader,
EuiPageHeaderSection,
EuiTitle,
} from '@elastic/eui';
import { useLocation } from 'react-router-dom';
import { NavigationMenu } from '../components/navigation_menu';
import { ModelsList } from './models_management';
import { TrainedModelsNavigationBar } from './navigation_bar';
import { RefreshAnalyticsListButton } from '../data_frame_analytics/pages/analytics_management/components/refresh_analytics_list_button';
import { DatePickerWrapper } from '../components/navigation_menu/date_picker_wrapper';
import { useRefreshAnalyticsList } from '../data_frame_analytics/common';
import { useRefreshInterval } from '../data_frame_analytics/pages/analytics_management/components/analytics_list/use_refresh_interval';
import { NodesList } from './nodes_overview';
export const Page: FC = () => {
useRefreshInterval(() => {});
useRefreshAnalyticsList({ isLoading: () => {} });
const location = useLocation();
const selectedTabId = useMemo(() => location.pathname.split('/').pop(), [location]);
return (
<Fragment>
<NavigationMenu tabId="trained_models" />
<EuiPage data-test-subj="mlPageModelManagement">
<EuiPageBody>
<EuiPageHeader>
<EuiPageHeaderSection>
<EuiTitle>
<h1>
<FormattedMessage
id="xpack.ml.trainedModels.title"
defaultMessage="Trained Models"
/>
</h1>
</EuiTitle>
</EuiPageHeaderSection>
<EuiPageHeaderSection>
<EuiFlexGroup alignItems="center" gutterSize="s">
<EuiFlexItem grow={false}>
<RefreshAnalyticsListButton />
</EuiFlexItem>
<EuiFlexItem grow={false}>
<DatePickerWrapper />
</EuiFlexItem>
</EuiFlexGroup>
</EuiPageHeaderSection>
</EuiPageHeader>
<EuiPageContent>
<TrainedModelsNavigationBar selectedTabId={selectedTabId} />
{selectedTabId === 'trained_models' ? <ModelsList /> : null}
{selectedTabId === 'nodes' ? <NodesList /> : null}
</EuiPageContent>
</EuiPageBody>
</EuiPage>
</Fragment>
);
};

39
x-pack/plugins/ml/public/application/util/custom_url_utils.test.ts
View file

@ -585,6 +585,45 @@ describe('ML - custom URL utils', () => {
'http://airlinecodes.info/airline-code-AAL'
);
});
test('returns expected URL with preserving custom filter', () => {
const urlWithCustomFilter: UrlConfig = {
url_name: 'URL with a custom filter',
url_value: `discover#/?_g=(time:(from:'$earliest$',mode:absolute,to:'$latest$'))&_a=(filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!f,key:subSystem.keyword,negate:!f,params:(query:JDBC),type:phrase),query:(match_phrase:(subSystem.keyword:JDBC)))),index:'eap_wls_server_12c*,*:eap_wls_server_12c*',query:(language:kuery,query:'wlscluster.keyword:"$wlscluster.keyword$"'))`,
};
const testRecords = {
job_id: 'farequote',
result_type: 'record',
probability: 6.533287347648861e-45,
record_score: 93.84475,
initial_record_score: 94.867922946384,
bucket_span: 300,
detector_index: 0,
is_interim: false,
timestamp: 1486656600000,
partition_field_name: 'wlscluster.keyword',
partition_field_value: 'AAL',
function: 'mean',
function_description: 'mean',
typical: [99.2329899996025],
actual: [274.7279901504516],
field_name: 'wlscluster.keyword',
influencers: [
{
influencer_field_name: 'wlscluster.keyword',
influencer_field_values: ['AAL'],
},
],
'wlscluster.keyword': ['AAL'],
earliest: '2019-02-01T16:00:00.000Z',
latest: '2019-02-01T18:59:59.999Z',
};
expect(getUrlForRecord(urlWithCustomFilter, testRecords)).toBe(
`discover#/?_g=(time:(from:'2019-02-01T16:00:00.000Z',mode:absolute,to:'2019-02-01T18:59:59.999Z'))&_a=(filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!f,key:subSystem.keyword,negate:!f,params:(query:JDBC),type:phrase),query:(match_phrase:(subSystem.keyword:JDBC)))),index:'eap_wls_server_12c*,*:eap_wls_server_12c*',query:(language:kuery,query:'wlscluster.keyword:\"AAL\"'))`
);
});
});
describe('isValidLabel', () => {

16
x-pack/plugins/ml/public/locator/formatters/trained_models.ts Normal file
View file

@ -0,0 +1,16 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { TrainedModelsUrlState } from '../../../common/types/locator';
import { ML_PAGES } from '../../../common/constants/locator';
export function formatTrainedModelsManagementUrl(
appBasePath: string,
mlUrlGeneratorState: TrainedModelsUrlState['pageState']
): string {
return `${appBasePath}/${ML_PAGES.TRAINED_MODELS_MANAGE}`;
}

4
x-pack/plugins/ml/public/locator/ml_locator.ts
View file

@ -26,6 +26,7 @@ import {
formatEditCalendarUrl,
formatEditFilterUrl,
} from './formatters';
import { formatTrainedModelsManagementUrl } from './formatters/trained_models';
export { MlLocatorParams, MlLocator };
@ -66,6 +67,9 @@ export class MlLocatorDefinition implements LocatorDefinition<MlLocatorParams> {
case ML_PAGES.DATA_FRAME_ANALYTICS_EXPLORATION:
path = formatDataFrameAnalyticsExplorationUrl('', params.pageState);
break;
case ML_PAGES.TRAINED_MODELS_MANAGE:
path = formatTrainedModelsManagementUrl('', params.pageState);
break;
case ML_PAGES.ANOMALY_DETECTION_CREATE_JOB:
case ML_PAGES.ANOMALY_DETECTION_CREATE_JOB_ADVANCED:
case ML_PAGES.DATA_VISUALIZER:

7
x-pack/plugins/ml/public/plugin.ts
View file

@ -46,6 +46,10 @@ import type { DataVisualizerPluginStart } from '../../data_visualizer/public';
import type { PluginSetupContract as AlertingSetup } from '../../alerting/public';
import { registerManagementSection } from './application/management';
import type { UsageCollectionSetup } from '../../../../src/plugins/usage_collection/public';
import type {
FieldFormatsSetup,
FieldFormatsStart,
} from '../../../../src/plugins/field_formats/public';
export interface MlStartDependencies {
data: DataPublicPluginStart;
@ -57,6 +61,7 @@ export interface MlStartDependencies {
maps?: MapsStartApi;
triggersActionsUi?: TriggersAndActionsUIPublicPluginStart;
dataVisualizer: DataVisualizerPluginStart;
fieldFormats: FieldFormatsStart;
}
export interface MlSetupDependencies {
@ -72,6 +77,7 @@ export interface MlSetupDependencies {
triggersActionsUi?: TriggersAndActionsUIPublicPluginSetup;
alerting?: AlertingSetup;
usageCollection?: UsageCollectionSetup;
fieldFormats: FieldFormatsSetup;
}
export type MlCoreSetup = CoreSetup<MlStartDependencies, MlPluginStart>;
@ -116,6 +122,7 @@ export class MlPlugin implements Plugin<MlPluginSetup, MlPluginStart> {
triggersActionsUi: pluginsStart.triggersActionsUi,
dataVisualizer: pluginsStart.dataVisualizer,
usageCollection: pluginsSetup.usageCollection,
fieldFormats: pluginsStart.fieldFormats,
},
params
);

2
x-pack/plugins/ml/public/register_helper/register_search_links/search_deep_links.ts
View file

@ -38,7 +38,7 @@ const DATA_FRAME_ANALYTICS_DEEP_LINK: AppDeepLink = {
title: i18n.translate('xpack.ml.deepLink.trainedModels', {
defaultMessage: 'Trained Models',
}),
path: `/${ML_PAGES.DATA_FRAME_ANALYTICS_MODELS_MANAGE}`,
path: `/${ML_PAGES.TRAINED_MODELS_MANAGE}`,
},
],
};

21
x-pack/plugins/ml/server/lib/ml_client/ml_client.ts
View file

@ -380,6 +380,27 @@ export function getMlClient(
async getTrainedModelsStats(...p: Parameters<MlClient['getTrainedModelsStats']>) {
return mlClient.getTrainedModelsStats(...p);
},
// TODO update when the new elasticsearch-js client is available
async getTrainedModelsDeploymentStats(...p: Parameters<MlClient['getTrainedModelsStats']>) {
return client.asCurrentUser.transport.request({
method: 'GET',
path: `/_ml/trained_models/${p[0]?.model_id ?? '*'}/deployment/_stats`,
});
},
// TODO update when the new elasticsearch-js client is available
async startTrainedModelDeployment(...p: Parameters<MlClient['deleteTrainedModel']>) {
return client.asCurrentUser.transport.request({
method: 'POST',
path: `/_ml/trained_models/${p[0].model_id}/deployment/_start`,
});
},
// TODO update when the new elasticsearch-js client is available
async stopTrainedModelDeployment(...p: Parameters<MlClient['deleteTrainedModel']>) {
return client.asCurrentUser.transport.request({
method: 'POST',
path: `/_ml/trained_models/${p[0].model_id}/deployment/_stop`,
});
},
async info(...p: Parameters<MlClient['info']>) {
return mlClient.info(...p);
},

13
x-pack/plugins/ml/server/lib/ml_client/types.ts
View file

@ -7,11 +7,24 @@
import { ElasticsearchClient } from 'kibana/server';
import { searchProvider } from './search';
import { TrainedModelDeploymentStatsResponse } from '../../../common/types/trained_models';
type OrigMlClient = ElasticsearchClient['ml'];
export interface MlClient extends OrigMlClient {
anomalySearch: ReturnType<typeof searchProvider>['anomalySearch'];
// TODO remove when the new elasticsearch-js client is available
getTrainedModelsDeploymentStats: (options?: { model_id?: string }) => Promise<{
body: { count: number; deployment_stats: TrainedModelDeploymentStatsResponse[] };
}>;
// TODO remove when the new elasticsearch-js client is available
startTrainedModelDeployment: (options: { model_id: string }) => Promise<{
body: { acknowledge: boolean };
}>;
// TODO remove when the new elasticsearch-js client is available
stopTrainedModelDeployment: (options: { model_id: string }) => Promise<{
body: { acknowledge: boolean };
}>;
}
export type MlClientParams =

357
x-pack/plugins/ml/server/models/data_frame_analytics/__mocks__/mock_deployment_response.json Normal file
View file

@ -0,0 +1,357 @@
{
"count" : 4,
"deployment_stats" : [
{
"model_id" : "distilbert-base-uncased-finetuned-sst-2-english",
"model_size_bytes" : 267386880,
"inference_threads" : 1,
"model_threads" : 1,
"state" : "started",
"allocation_status" : {
"allocation_count" : 2,
"target_allocation_count" : 3,
"state" : "started"
},
"nodes" : [
{
"node" : {
"3qIoLFnbSi-DwVrYioUCdw" : {
"name" : "node3",
"ephemeral_id" : "WeA49KLuRPmJM_ulLx0ANg",
"transport_address" : "10.142.0.2:9353",
"attributes" : {
"ml.machine_memory" : "15599742976",
"xpack.installed" : "true",
"ml.max_jvm_size" : "1073741824"
},
"roles" : [
"data",
"ingest",
"master",
"ml",
"transform"
]
}
},
"routing_state" : {
"routing_state" : "started"
},
"inference_count" : 0,
"average_inference_time_ms" : 0.0
},
{
"node" : {
"DpCy7SOBQla3pu0Dq-tnYw" : {
"name" : "node2",
"ephemeral_id" : "17qcsXsNTYqbJ6uwSvdl9g",
"transport_address" : "10.142.0.2:9352",
"attributes" : {
"ml.machine_memory" : "15599742976",
"xpack.installed" : "true",
"ml.max_jvm_size" : "1073741824"
},
"roles" : [
"data",
"master",
"ml",
"transform"
]
}
},
"routing_state" : {
"routing_state" : "failed",
"reason" : "The object cannot be set twice!"
}
},
{
"node" : {
"pt7s6lKHQJaP4QHKtU-Q0Q" : {
"name" : "node1",
"ephemeral_id" : "nMJBE9WSRQSWotk0zDPi_Q",
"transport_address" : "10.142.0.2:9351",
"attributes" : {
"ml.machine_memory" : "15599742976",
"xpack.installed" : "true",
"ml.max_jvm_size" : "1073741824"
},
"roles" : [
"data",
"master",
"ml"
]
}
},
"routing_state" : {
"routing_state" : "started"
},
"inference_count" : 0,
"average_inference_time_ms" : 0.0
}
]
},
{
"model_id" : "elastic__distilbert-base-cased-finetuned-conll03-english",
"model_size_bytes" : 260947500,
"inference_threads" : 1,
"model_threads" : 1,
"state" : "started",
"allocation_status" : {
"allocation_count" : 2,
"target_allocation_count" : 3,
"state" : "started"
},
"nodes" : [
{
"node" : {
"3qIoLFnbSi-DwVrYioUCdw" : {
"name" : "node3",
"ephemeral_id" : "WeA49KLuRPmJM_ulLx0ANg",
"transport_address" : "10.142.0.2:9353",
"attributes" : {
"ml.machine_memory" : "15599742976",
"xpack.installed" : "true",
"ml.max_jvm_size" : "1073741824"
},
"roles" : [
"data",
"ingest",
"master",
"ml",
"transform"
]
}
},
"routing_state" : {
"routing_state" : "started"
},
"inference_count" : 0,
"average_inference_time_ms" : 0.0
},
{
"node" : {
"DpCy7SOBQla3pu0Dq-tnYw" : {
"name" : "node2",
"ephemeral_id" : "17qcsXsNTYqbJ6uwSvdl9g",
"transport_address" : "10.142.0.2:9352",
"attributes" : {
"ml.machine_memory" : "15599742976",
"xpack.installed" : "true",
"ml.max_jvm_size" : "1073741824"
},
"roles" : [
"data",
"master",
"ml",
"transform"
]
}
},
"routing_state" : {
"routing_state" : "failed",
"reason" : "The object cannot be set twice!"
}
},
{
"node" : {
"pt7s6lKHQJaP4QHKtU-Q0Q" : {
"name" : "node1",
"ephemeral_id" : "nMJBE9WSRQSWotk0zDPi_Q",
"transport_address" : "10.142.0.2:9351",
"attributes" : {
"ml.machine_memory" : "15599742976",
"xpack.installed" : "true",
"ml.max_jvm_size" : "1073741824"
},
"roles" : [
"data",
"master",
"ml"
]
}
},
"routing_state" : {
"routing_state" : "started"
},
"inference_count" : 0,
"average_inference_time_ms" : 0.0
}
]
},
{
"model_id" : "sentence-transformers__msmarco-minilm-l-12-v3",
"model_size_bytes" : 133378867,
"inference_threads" : 1,
"model_threads" : 1,
"state" : "started",
"allocation_status" : {
"allocation_count" : 2,
"target_allocation_count" : 3,
"state" : "started"
},
"nodes" : [
{
"node" : {
"3qIoLFnbSi-DwVrYioUCdw" : {
"name" : "node3",
"ephemeral_id" : "WeA49KLuRPmJM_ulLx0ANg",
"transport_address" : "10.142.0.2:9353",
"attributes" : {
"ml.machine_memory" : "15599742976",
"xpack.installed" : "true",
"ml.max_jvm_size" : "1073741824"
},
"roles" : [
"data",
"ingest",
"master",
"ml",
"transform"
]
}
},
"routing_state" : {
"routing_state" : "started"
},
"inference_count" : 0,
"average_inference_time_ms" : 0.0
},
{
"node" : {
"DpCy7SOBQla3pu0Dq-tnYw" : {
"name" : "node2",
"ephemeral_id" : "17qcsXsNTYqbJ6uwSvdl9g",
"transport_address" : "10.142.0.2:9352",
"attributes" : {
"ml.machine_memory" : "15599742976",
"xpack.installed" : "true",
"ml.max_jvm_size" : "1073741824"
},
"roles" : [
"data",
"master",
"ml",
"transform"
]
}
},
"routing_state" : {
"routing_state" : "failed",
"reason" : "The object cannot be set twice!"
}
},
{
"node" : {
"pt7s6lKHQJaP4QHKtU-Q0Q" : {
"name" : "node1",
"ephemeral_id" : "nMJBE9WSRQSWotk0zDPi_Q",
"transport_address" : "10.142.0.2:9351",
"attributes" : {
"ml.machine_memory" : "15599742976",
"xpack.installed" : "true",
"ml.max_jvm_size" : "1073741824"
},
"roles" : [
"data",
"master",
"ml"
]
}
},
"routing_state" : {
"routing_state" : "started"
},
"inference_count" : 0,
"average_inference_time_ms" : 0.0
}
]
},
{
"model_id" : "typeform__mobilebert-uncased-mnli",
"model_size_bytes" : 100139008,
"inference_threads" : 1,
"model_threads" : 1,
"state" : "started",
"allocation_status" : {
"allocation_count" : 2,
"target_allocation_count" : 3,
"state" : "started"
},
"nodes" : [
{
"node" : {
"3qIoLFnbSi-DwVrYioUCdw" : {
"name" : "node3",
"ephemeral_id" : "WeA49KLuRPmJM_ulLx0ANg",
"transport_address" : "10.142.0.2:9353",
"attributes" : {
"ml.machine_memory" : "15599742976",
"xpack.installed" : "true",
"ml.max_jvm_size" : "1073741824"
},
"roles" : [
"data",
"ingest",
"master",
"ml",
"transform"
]
}
},
"routing_state" : {
"routing_state" : "started"
},
"inference_count" : 0,
"average_inference_time_ms" : 0.0
},
{
"node" : {
"DpCy7SOBQla3pu0Dq-tnYw" : {
"name" : "node2",
"ephemeral_id" : "17qcsXsNTYqbJ6uwSvdl9g",
"transport_address" : "10.142.0.2:9352",
"attributes" : {
"ml.machine_memory" : "15599742976",
"xpack.installed" : "true",
"ml.max_jvm_size" : "1073741824"
},
"roles" : [
"data",
"master",
"ml",
"transform"
]
}
},
"routing_state" : {
"routing_state" : "failed",
"reason" : "The object cannot be set twice!"
}
},
{
"node" : {
"pt7s6lKHQJaP4QHKtU-Q0Q" : {
"name" : "node1",
"ephemeral_id" : "nMJBE9WSRQSWotk0zDPi_Q",
"transport_address" : "10.142.0.2:9351",
"attributes" : {
"ml.machine_memory" : "15599742976",
"xpack.installed" : "true",
"ml.max_jvm_size" : "1073741824"
},
"roles" : [
"data",
"master",
"ml"
]
}
},
"routing_state" : {
"routing_state" : "started"
},
"inference_count" : 0,
"average_inference_time_ms" : 0.0
}
]
}
]
}

503
x-pack/plugins/ml/server/models/data_frame_analytics/model_provider.test.ts Normal file
View file

@ -0,0 +1,503 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { ModelService, modelsProvider } from './models_provider';
import { IScopedClusterClient } from 'kibana/server';
import { MlClient } from '../../lib/ml_client';
import mockResponse from './__mocks__/mock_deployment_response.json';
import { MemoryOverviewService } from '../memory_overview/memory_overview_service';
describe('Model service', () => {
const client = {
asCurrentUser: {
nodes: {
stats: jest.fn(() => {
return Promise.resolve({
body: {
_nodes: {
total: 3,
successful: 3,
failed: 0,
},
cluster_name: 'test_cluster',
nodes: {
'3qIoLFnbSi-DwVrYioUCdw': {
timestamp: 1635167166946,
name: 'node3',
transport_address: '10.10.10.2:9353',
host: '10.10.10.2',
ip: '10.10.10.2:9353',
roles: ['data', 'ingest', 'master', 'ml', 'transform'],
attributes: {
'ml.machine_memory': '15599742976',
'xpack.installed': 'true',
'ml.max_jvm_size': '1073741824',
},
os: {
mem: {
total_in_bytes: 15599742976,
adjusted_total_in_bytes: 15599742976,
free_in_bytes: 376324096,
used_in_bytes: 15223418880,
free_percent: 2,
used_percent: 98,
},
},
},
'DpCy7SOBQla3pu0Dq-tnYw': {
timestamp: 1635167166946,
name: 'node2',
transport_address: '10.10.10.2:9352',
host: '10.10.10.2',
ip: '10.10.10.2:9352',
roles: ['data', 'master', 'ml', 'transform'],
attributes: {
'ml.machine_memory': '15599742976',
'xpack.installed': 'true',
'ml.max_jvm_size': '1073741824',
},
os: {
timestamp: 1635167166959,
mem: {
total_in_bytes: 15599742976,
adjusted_total_in_bytes: 15599742976,
free_in_bytes: 376324096,
used_in_bytes: 15223418880,
free_percent: 2,
used_percent: 98,
},
},
},
'pt7s6lKHQJaP4QHKtU-Q0Q': {
timestamp: 1635167166945,
name: 'node1',
transport_address: '10.10.10.2:9351',
host: '10.10.10.2',
ip: '10.10.10.2:9351',
roles: ['data', 'master', 'ml'],
attributes: {
'ml.machine_memory': '15599742976',
'xpack.installed': 'true',
'ml.max_jvm_size': '1073741824',
},
os: {
timestamp: 1635167166959,
mem: {
total_in_bytes: 15599742976,
adjusted_total_in_bytes: 15599742976,
free_in_bytes: 376324096,
used_in_bytes: 15223418880,
free_percent: 2,
used_percent: 98,
},
},
},
},
},
});
}),
},
},
} as unknown as jest.Mocked<IScopedClusterClient>;
const mlClient = {
getTrainedModelsDeploymentStats: jest.fn(() => {
return Promise.resolve({ body: mockResponse });
}),
} as unknown as jest.Mocked<MlClient>;
const memoryOverviewService = {
getDFAMemoryOverview: jest.fn(() => {
return Promise.resolve([{ job_id: '', node_id: '', model_size: 32165465 }]);
}),
getAnomalyDetectionMemoryOverview: jest.fn(() => {
return Promise.resolve([{ job_id: '', node_id: '', model_size: 32165465 }]);
}),
} as unknown as jest.Mocked<MemoryOverviewService>;
let service: ModelService;
beforeEach(() => {
service = modelsProvider(client, mlClient, memoryOverviewService);
});
afterEach(() => {});
it('extract nodes list correctly', async () => {
expect(await service.getNodesOverview()).toEqual({
count: 3,
nodes: [
{
name: 'node3',
allocated_models: [
{
allocation_status: {
allocation_count: 2,
state: 'started',
target_allocation_count: 3,
},
inference_threads: 1,
model_id: 'distilbert-base-uncased-finetuned-sst-2-english',
model_size_bytes: 267386880,
model_threads: 1,
state: 'started',
node: {
average_inference_time_ms: 0,
inference_count: 0,
routing_state: {
routing_state: 'started',
},
},
},
{
allocation_status: {
allocation_count: 2,
state: 'started',
target_allocation_count: 3,
},
inference_threads: 1,
model_id: 'elastic__distilbert-base-cased-finetuned-conll03-english',
model_size_bytes: 260947500,
model_threads: 1,
state: 'started',
node: {
average_inference_time_ms: 0,
inference_count: 0,
routing_state: {
routing_state: 'started',
},
},
},
{
allocation_status: {
allocation_count: 2,
state: 'started',
target_allocation_count: 3,
},
inference_threads: 1,
model_id: 'sentence-transformers__msmarco-minilm-l-12-v3',
model_size_bytes: 133378867,
model_threads: 1,
state: 'started',
node: {
average_inference_time_ms: 0,
inference_count: 0,
routing_state: {
routing_state: 'started',
},
},
},
{
allocation_status: {
allocation_count: 2,
state: 'started',
target_allocation_count: 3,
},
inference_threads: 1,
model_id: 'typeform__mobilebert-uncased-mnli',
model_size_bytes: 100139008,
model_threads: 1,
state: 'started',
node: {
average_inference_time_ms: 0,
inference_count: 0,
routing_state: {
routing_state: 'started',
},
},
},
],
attributes: {
'ml.machine_memory': '15599742976',
'ml.max_jvm_size': '1073741824',
'xpack.installed': 'true',
},
host: '10.10.10.2',
id: '3qIoLFnbSi-DwVrYioUCdw',
ip: '10.10.10.2:9353',
memory_overview: {
anomaly_detection: {
total: 0,
},
dfa_training: {
total: 0,
},
machine_memory: {
jvm: 1073741824,
total: 15599742976,
},
trained_models: {
by_model: [
{
model_id: 'distilbert-base-uncased-finetuned-sst-2-english',
model_size: 267386880,
},
{
model_id: 'elastic__distilbert-base-cased-finetuned-conll03-english',
model_size: 260947500,
},
{
model_id: 'sentence-transformers__msmarco-minilm-l-12-v3',
model_size: 133378867,
},
{
model_id: 'typeform__mobilebert-uncased-mnli',
model_size: 100139008,
},
],
total: 793309535,
},
},
roles: ['data', 'ingest', 'master', 'ml', 'transform'],
transport_address: '10.10.10.2:9353',
},
{
name: 'node2',
allocated_models: [
{
allocation_status: {
allocation_count: 2,
state: 'started',
target_allocation_count: 3,
},
inference_threads: 1,
model_id: 'distilbert-base-uncased-finetuned-sst-2-english',
model_size_bytes: 267386880,
model_threads: 1,
state: 'started',
node: {
routing_state: {
reason: 'The object cannot be set twice!',
routing_state: 'failed',
},
},
},
{
allocation_status: {
allocation_count: 2,
state: 'started',
target_allocation_count: 3,
},
inference_threads: 1,
model_id: 'elastic__distilbert-base-cased-finetuned-conll03-english',
model_size_bytes: 260947500,
model_threads: 1,
state: 'started',
node: {
routing_state: {
reason: 'The object cannot be set twice!',
routing_state: 'failed',
},
},
},
{
allocation_status: {
allocation_count: 2,
state: 'started',
target_allocation_count: 3,
},
inference_threads: 1,
model_id: 'sentence-transformers__msmarco-minilm-l-12-v3',
model_size_bytes: 133378867,
model_threads: 1,
state: 'started',
node: {
routing_state: {
reason: 'The object cannot be set twice!',
routing_state: 'failed',
},
},
},
{
allocation_status: {
allocation_count: 2,
state: 'started',
target_allocation_count: 3,
},
inference_threads: 1,
model_id: 'typeform__mobilebert-uncased-mnli',
model_size_bytes: 100139008,
model_threads: 1,
state: 'started',
node: {
routing_state: {
reason: 'The object cannot be set twice!',
routing_state: 'failed',
},
},
},
],
attributes: {
'ml.machine_memory': '15599742976',
'ml.max_jvm_size': '1073741824',
'xpack.installed': 'true',
},
host: '10.10.10.2',
id: 'DpCy7SOBQla3pu0Dq-tnYw',
ip: '10.10.10.2:9352',
memory_overview: {
anomaly_detection: {
total: 0,
},
dfa_training: {
total: 0,
},
machine_memory: {
jvm: 1073741824,
total: 15599742976,
},
trained_models: {
by_model: [
{
model_id: 'distilbert-base-uncased-finetuned-sst-2-english',
model_size: 267386880,
},
{
model_id: 'elastic__distilbert-base-cased-finetuned-conll03-english',
model_size: 260947500,
},
{
model_id: 'sentence-transformers__msmarco-minilm-l-12-v3',
model_size: 133378867,
},
{
model_id: 'typeform__mobilebert-uncased-mnli',
model_size: 100139008,
},
],
total: 793309535,
},
},
roles: ['data', 'master', 'ml', 'transform'],
transport_address: '10.10.10.2:9352',
},
{
allocated_models: [
{
allocation_status: {
allocation_count: 2,
state: 'started',
target_allocation_count: 3,
},
inference_threads: 1,
model_id: 'distilbert-base-uncased-finetuned-sst-2-english',
model_size_bytes: 267386880,
model_threads: 1,
state: 'started',
node: {
average_inference_time_ms: 0,
inference_count: 0,
routing_state: {
routing_state: 'started',
},
},
},
{
allocation_status: {
allocation_count: 2,
state: 'started',
target_allocation_count: 3,
},
inference_threads: 1,
model_id: 'elastic__distilbert-base-cased-finetuned-conll03-english',
model_size_bytes: 260947500,
model_threads: 1,
state: 'started',
node: {
average_inference_time_ms: 0,
inference_count: 0,
routing_state: {
routing_state: 'started',
},
},
},
{
allocation_status: {
allocation_count: 2,
state: 'started',
target_allocation_count: 3,
},
inference_threads: 1,
model_id: 'sentence-transformers__msmarco-minilm-l-12-v3',
model_size_bytes: 133378867,
model_threads: 1,
state: 'started',
node: {
average_inference_time_ms: 0,
inference_count: 0,
routing_state: {
routing_state: 'started',
},
},
},
{
allocation_status: {
allocation_count: 2,
state: 'started',
target_allocation_count: 3,
},
inference_threads: 1,
model_id: 'typeform__mobilebert-uncased-mnli',
model_size_bytes: 100139008,
model_threads: 1,
state: 'started',
node: {
average_inference_time_ms: 0,
inference_count: 0,
routing_state: {
routing_state: 'started',
},
},
},
],
attributes: {
'ml.machine_memory': '15599742976',
'ml.max_jvm_size': '1073741824',
'xpack.installed': 'true',
},
host: '10.10.10.2',
id: 'pt7s6lKHQJaP4QHKtU-Q0Q',
ip: '10.10.10.2:9351',
memory_overview: {
anomaly_detection: {
total: 0,
},
dfa_training: {
total: 0,
},
machine_memory: {
jvm: 1073741824,
total: 15599742976,
},
trained_models: {
by_model: [
{
model_id: 'distilbert-base-uncased-finetuned-sst-2-english',
model_size: 267386880,
},
{
model_id: 'elastic__distilbert-base-cased-finetuned-conll03-english',
model_size: 260947500,
},
{
model_id: 'sentence-transformers__msmarco-minilm-l-12-v3',
model_size: 133378867,
},
{
model_id: 'typeform__mobilebert-uncased-mnli',
model_size: 100139008,
},
],
total: 793309535,
},
},
name: 'node1',
roles: ['data', 'master', 'ml'],
transport_address: '10.10.10.2:9351',
},
],
});
});
});

135
x-pack/plugins/ml/server/models/data_frame_analytics/models_provider.ts
View file

@ -5,10 +5,39 @@
* 2.0.
*/
import { IScopedClusterClient } from 'kibana/server';
import { PipelineDefinition } from '../../../common/types/trained_models';
import type { IScopedClusterClient } from 'kibana/server';
import { sumBy, pick } from 'lodash';
import { NodesInfoNodeInfo } from '@elastic/elasticsearch/api/types';
import type {
NodeDeploymentStatsResponse,
PipelineDefinition,
NodesOverviewResponse,
} from '../../../common/types/trained_models';
import type { MlClient } from '../../lib/ml_client';
import {
MemoryOverviewService,
NATIVE_EXECUTABLE_CODE_OVERHEAD,
} from '../memory_overview/memory_overview_service';
export function modelsProvider(client: IScopedClusterClient) {
export type ModelService = ReturnType<typeof modelsProvider>;
const NODE_FIELDS = [
'attributes',
'name',
'roles',
'ip',
'host',
'transport_address',
'version',
] as const;
export type RequiredNodeFields = Pick<NodesInfoNodeInfo, typeof NODE_FIELDS[number]>;
export function modelsProvider(
client: IScopedClusterClient,
mlClient: MlClient,
memoryOverviewService?: MemoryOverviewService
) {
return {
/**
* Retrieves the map of model ids and aliases with associated pipelines.
@ -39,5 +68,105 @@ export function modelsProvider(client: IScopedClusterClient) {
return modelIdsMap;
},
/**
* Provides the ML nodes overview with allocated models.
*/
async getNodesOverview(): Promise<NodesOverviewResponse> {
if (!memoryOverviewService) {
throw new Error('Memory overview service is not provided');
}
const { body: deploymentStats } = await mlClient.getTrainedModelsDeploymentStats();
const {
body: { nodes: clusterNodes },
} = await client.asCurrentUser.nodes.stats();
const mlNodes = Object.entries(clusterNodes).filter(([id, node]) =>
node.roles.includes('ml')
);
const adMemoryReport = await memoryOverviewService.getAnomalyDetectionMemoryOverview();
const dfaMemoryReport = await memoryOverviewService.getDFAMemoryOverview();
const nodeDeploymentStatsResponses: NodeDeploymentStatsResponse[] = mlNodes.map(
([nodeId, node]) => {
const nodeFields = pick(node, NODE_FIELDS) as RequiredNodeFields;
const allocatedModels = deploymentStats.deployment_stats
.filter((v) => v.nodes.some((n) => Object.keys(n.node)[0] === nodeId))
.map(({ nodes, ...rest }) => {
const { node: tempNode, ...nodeRest } = nodes.find(
(v) => Object.keys(v.node)[0] === nodeId
)!;
return {
...rest,
node: nodeRest,
};
});
const modelsMemoryUsage = allocatedModels.map((v) => {
return {
model_id: v.model_id,
model_size: v.model_size_bytes,
};
});
const memoryRes = {
adTotalMemory: sumBy(
adMemoryReport.filter((ad) => ad.node_id === nodeId),
'model_size'
),
dfaTotalMemory: sumBy(
dfaMemoryReport.filter((dfa) => dfa.node_id === nodeId),
'model_size'
),
trainedModelsTotalMemory: sumBy(modelsMemoryUsage, 'model_size'),
};
for (const key of Object.keys(memoryRes)) {
if (memoryRes[key as keyof typeof memoryRes] > 0) {
/**
* The amount of memory needed to load the ML native code shared libraries. The assumption is that the first
* ML job to run on a given node will do this, and then subsequent ML jobs on the same node will reuse the
* same already-loaded code.
*/
memoryRes[key as keyof typeof memoryRes] += NATIVE_EXECUTABLE_CODE_OVERHEAD;
break;
}
}
return {
id: nodeId,
...nodeFields,
allocated_models: allocatedModels,
memory_overview: {
machine_memory: {
// TODO remove ts-ignore when elasticsearch client is updated
// @ts-ignore
total: Number(node.os?.mem.adjusted_total_in_bytes ?? node.os?.mem.total_in_bytes),
jvm: Number(node.attributes['ml.max_jvm_size']),
},
anomaly_detection: {
total: memoryRes.adTotalMemory,
},
dfa_training: {
total: memoryRes.dfaTotalMemory,
},
trained_models: {
total: memoryRes.trainedModelsTotalMemory,
by_model: modelsMemoryUsage,
},
},
};
}
);
return {
count: nodeDeploymentStatsResponses.length,
nodes: nodeDeploymentStatsResponses,
};
},
};
}

8
x-pack/plugins/ml/server/models/memory_overview/index.ts Normal file
View file

@ -0,0 +1,8 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export { memoryOverviewServiceProvider } from './memory_overview_service';

90
x-pack/plugins/ml/server/models/memory_overview/memory_overview_service.ts Normal file
View file

@ -0,0 +1,90 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import numeral from '@elastic/numeral';
import { keyBy } from 'lodash';
import { MlClient } from '../../lib/ml_client';
export type MemoryOverviewService = ReturnType<typeof memoryOverviewServiceProvider>;
export interface MlJobMemoryOverview {
job_id: string;
node_id: string;
model_size: number;
}
const MB = Math.pow(2, 20);
const AD_PROCESS_MEMORY_OVERHEAD = 10 * MB;
const DFA_PROCESS_MEMORY_OVERHEAD = 5 * MB;
export const NATIVE_EXECUTABLE_CODE_OVERHEAD = 30 * MB;
/**
* Provides a service for memory overview across ML.
* @param mlClient
*/
export function memoryOverviewServiceProvider(mlClient: MlClient) {
return {
/**
* Retrieves memory consumed my started DFA jobs.
*/
async getDFAMemoryOverview(): Promise<MlJobMemoryOverview[]> {
const {
body: { data_frame_analytics: dfaStats },
} = await mlClient.getDataFrameAnalyticsStats();
const dfaMemoryReport = dfaStats
.filter((dfa) => dfa.state === 'started')
.map((dfa) => {
return {
node_id: dfa.node?.id,
job_id: dfa.id,
};
}) as MlJobMemoryOverview[];
if (dfaMemoryReport.length === 0) {
return [];
}
const dfaMemoryKeyByJobId = keyBy(dfaMemoryReport, 'job_id');
const {
body: { data_frame_analytics: startedDfaJobs },
} = await mlClient.getDataFrameAnalytics({
id: dfaMemoryReport.map((v) => v.job_id).join(','),
});
startedDfaJobs.forEach((dfa) => {
dfaMemoryKeyByJobId[dfa.id].model_size =
numeral(
dfa.model_memory_limit?.toUpperCase()
// @ts-ignore
).value() + DFA_PROCESS_MEMORY_OVERHEAD;
});
return dfaMemoryReport;
},
/**
* Retrieves memory consumed by opened Anomaly Detection jobs.
*/
async getAnomalyDetectionMemoryOverview(): Promise<MlJobMemoryOverview[]> {
const {
body: { jobs: jobsStats },
} = await mlClient.getJobStats();
return jobsStats
.filter((v) => v.state === 'opened')
.map((jobStats) => {
return {
node_id: jobStats.node.id,
model_size: jobStats.model_size_stats.model_bytes + AD_PROCESS_MEMORY_OVERHEAD,
job_id: jobStats.job_id,
};
});
},
};
}

6
x-pack/plugins/ml/server/routes/apidoc.json
View file

@ -123,7 +123,7 @@
"GetJobAuditMessages",
"GetAllJobAuditMessages",
"ClearJobAuditMessages",
"JobValidation",
"EstimateBucketSpan",
"CalculateModelMemoryLimit",
@ -160,7 +160,11 @@
"TrainedModels",
"GetTrainedModel",
"GetTrainedModelStats",
"GetTrainedModelDeploymentStats",
"GetTrainedModelsNodesOverview",
"GetTrainedModelPipelines",
"StartTrainedModelDeployment",
"StopTrainedModelDeployment",
"DeleteTrainedModel",
"Alerting",

139
x-pack/plugins/ml/server/routes/trained_models.ts
View file

@ -14,6 +14,7 @@ import {
} from './schemas/inference_schema';
import { modelsProvider } from '../models/data_frame_analytics';
import { TrainedModelConfigResponse } from '../../common/types/trained_models';
import { memoryOverviewServiceProvider } from '../models/memory_overview';
export function trainedModelsRoutes({ router, routeGuard }: RouteInitialization) {
/**
@ -44,6 +45,8 @@ export function trainedModelsRoutes({ router, routeGuard }: RouteInitialization)
...query,
...(modelId ? { model_id: modelId } : {}),
});
// model_type is missing
// @ts-ignore
const result = body.trained_model_configs as TrainedModelConfigResponse[];
try {
if (withPipelines) {
@ -57,7 +60,7 @@ export function trainedModelsRoutes({ router, routeGuard }: RouteInitialization)
)
);
const pipelinesResponse = await modelsProvider(client).getModelsPipelines(
const pipelinesResponse = await modelsProvider(client, mlClient).getModelsPipelines(
modelIdsAndAliases
);
for (const model of result) {
@ -136,10 +139,12 @@ export function trainedModelsRoutes({ router, routeGuard }: RouteInitialization)
tags: ['access:ml:canGetDataFrameAnalytics'],
},
},
routeGuard.fullLicenseAPIGuard(async ({ client, request, response }) => {
routeGuard.fullLicenseAPIGuard(async ({ client, request, mlClient, response }) => {
try {
const { modelId } = request.params;
const result = await modelsProvider(client).getModelsPipelines(modelId.split(','));
const result = await modelsProvider(client, mlClient).getModelsPipelines(
modelId.split(',')
);
return response.ok({
body: [...result].map(([id, pipelines]) => ({ model_id: id, pipelines })),
});
@ -180,4 +185,132 @@ export function trainedModelsRoutes({ router, routeGuard }: RouteInitialization)
}
})
);
/**
* @apiGroup TrainedModels
*
* @api {get} /api/ml/trained_models/nodes_overview Get node overview about the models allocation
* @apiName GetTrainedModelsNodesOverview
* @apiDescription Retrieves the list of ML nodes with memory breakdown and allocated models info
*/
router.get(
{
path: '/api/ml/trained_models/nodes_overview',
validate: {},
options: {
tags: ['access:ml:canGetDataFrameAnalytics'],
},
},
routeGuard.fullLicenseAPIGuard(async ({ client, mlClient, request, response }) => {
try {
const memoryOverviewService = memoryOverviewServiceProvider(mlClient);
const result = await modelsProvider(
client,
mlClient,
memoryOverviewService
).getNodesOverview();
return response.ok({
body: result,
});
} catch (e) {
return response.customError(wrapError(e));
}
})
);
/**
* @apiGroup TrainedModels
*
* @api {post} /api/ml/trained_models/:modelId/deployment/_start Start trained model deployment
* @apiName StartTrainedModelDeployment
* @apiDescription Starts trained model deployment.
*/
router.post(
{
path: '/api/ml/trained_models/{modelId}/deployment/_start',
validate: {
params: modelIdSchema,
},
options: {
tags: ['access:ml:canGetDataFrameAnalytics'],
},
},
routeGuard.fullLicenseAPIGuard(async ({ mlClient, request, response }) => {
try {
const { modelId } = request.params;
const { body } = await mlClient.startTrainedModelDeployment({
model_id: modelId,
});
return response.ok({
body,
});
} catch (e) {
return response.customError(wrapError(e));
}
})
);
/**
* @apiGroup TrainedModels
*
* @api {post} /api/ml/trained_models/:modelId/deployment/_stop Stop trained model deployment
* @apiName StopTrainedModelDeployment
* @apiDescription Stops trained model deployment.
*/
router.post(
{
path: '/api/ml/trained_models/{modelId}/deployment/_stop',
validate: {
params: modelIdSchema,
},
options: {
tags: ['access:ml:canGetDataFrameAnalytics'],
},
},
routeGuard.fullLicenseAPIGuard(async ({ mlClient, request, response }) => {
try {
const { modelId } = request.params;
const { body } = await mlClient.stopTrainedModelDeployment({
model_id: modelId,
});
return response.ok({
body,
});
} catch (e) {
return response.customError(wrapError(e));
}
})
);
/**
* @apiGroup TrainedModels
*
* @api {get} /api/ml/trained_models/:modelId/deployment/_stats Get trained model deployment stats
* @apiName GetTrainedModelDeploymentStats
* @apiDescription Gets trained model deployment stats.
*/
router.get(
{
path: '/api/ml/trained_models/{modelId}/deployment/_stats',
validate: {
params: modelIdSchema,
},
options: {
tags: ['access:ml:canGetDataFrameAnalytics'],
},
},
routeGuard.fullLicenseAPIGuard(async ({ mlClient, request, response }) => {
try {
const { modelId } = request.params;
const { body } = await mlClient.getTrainedModelsDeploymentStats({
model_id: modelId,
});
return response.ok({
body,
});
} catch (e) {
return response.customError(wrapError(e));
}
})
);
}

2
x-pack/plugins/translations/translations/ja-JP.json
View file

@ -15837,14 +15837,12 @@
"xpack.ml.dataframe.analyticsMap.modelIdTitle": "学習済みモデル ID {modelId} のマップ",
"xpack.ml.dataframe.jobsTabLabel": "ジョブ",
"xpack.ml.dataframe.mapTabLabel": "マップ",
"xpack.ml.dataframe.modelsTabLabel": "モデル",
"xpack.ml.dataframe.stepDetailsForm.destinationIndexInvalidErrorLink": "インデックス名の制限に関する詳細。",
"xpack.ml.dataFrameAnalyticsBreadcrumbs.analyticsMapLabel": "分析マップ",
"xpack.ml.dataFrameAnalyticsBreadcrumbs.dataFrameExplorationLabel": "探索",
"xpack.ml.dataFrameAnalyticsBreadcrumbs.dataFrameListLabel": "ジョブ管理",
"xpack.ml.dataFrameAnalyticsBreadcrumbs.dataFrameManagementLabel": "データフレーム分析",
"xpack.ml.dataFrameAnalyticsBreadcrumbs.indexLabel": "インデックス",
"xpack.ml.dataFrameAnalyticsBreadcrumbs.modelsListLabel": "モデル管理",
"xpack.ml.dataFrameAnalyticsLabel": "データフレーム分析",
"xpack.ml.dataFrameAnalyticsTabLabel": "データフレーム分析",
"xpack.ml.dataGrid.CcsWarningCalloutBody": "インデックスパターンのデータの取得中に問題が発生しました。ソースプレビューとクラスター横断検索を組み合わせることは、バージョン7.10以上ではサポートされていません。変換を構成して作成することはできます。",

2
x-pack/plugins/translations/translations/zh-CN.json
View file

@ -16039,7 +16039,6 @@
"xpack.ml.dataframe.analyticsMap.modelIdTitle": "已训练模型 ID {modelId} 的地图",
"xpack.ml.dataframe.jobsTabLabel": "作业",
"xpack.ml.dataframe.mapTabLabel": "地图",
"xpack.ml.dataframe.modelsTabLabel": "模型",
"xpack.ml.dataframe.stepCreateForm.createDataFrameAnalyticsSuccessMessage": "数据帧分析 {jobId} 创建请求已确认。",
"xpack.ml.dataframe.stepDetailsForm.destinationIndexInvalidErrorLink": "详细了解索引名称限制。",
"xpack.ml.dataFrameAnalyticsBreadcrumbs.analyticsMapLabel": "分析地图",
@ -16047,7 +16046,6 @@
"xpack.ml.dataFrameAnalyticsBreadcrumbs.dataFrameListLabel": "作业管理",
"xpack.ml.dataFrameAnalyticsBreadcrumbs.dataFrameManagementLabel": "数据帧分析",
"xpack.ml.dataFrameAnalyticsBreadcrumbs.indexLabel": "索引",
"xpack.ml.dataFrameAnalyticsBreadcrumbs.modelsListLabel": "模型管理",
"xpack.ml.dataFrameAnalyticsLabel": "数据帧分析",
"xpack.ml.dataFrameAnalyticsTabLabel": "数据帧分析",
"xpack.ml.dataGrid.CcsWarningCalloutBody": "检索索引模式的数据时有问题。源预览和跨集群搜索仅在 7.10 及以上版本上受支持。可能需要配置和创建转换。",

1
x-pack/test/alerting_api_integration/spaces_only/tests/alerting/transform_rule_types/index.ts
View file

@ -10,7 +10,6 @@ import { FtrProviderContext } from '../../../../common/ftr_provider_context';
// eslint-disable-next-line import/no-default-export
export default function alertingTests({ loadTestFile }: FtrProviderContext) {
describe('transform alert rule types', function () {
this.tags('dima');
loadTestFile(require.resolve('./transform_health'));
});
}

1
x-pack/test/functional/apps/ml/data_frame_analytics/index.ts
View file

@ -16,6 +16,5 @@ export default function ({ loadTestFile }: FtrProviderContext) {
loadTestFile(require.resolve('./classification_creation'));
loadTestFile(require.resolve('./cloning'));
loadTestFile(require.resolve('./feature_importance'));
loadTestFile(require.resolve('./trained_models'));
});
}

1
x-pack/test/functional/apps/ml/index.ts
View file

@ -50,6 +50,7 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) {
loadTestFile(require.resolve('./anomaly_detection'));
loadTestFile(require.resolve('./data_visualizer'));
loadTestFile(require.resolve('./data_frame_analytics'));
loadTestFile(require.resolve('./model_management'));
});
describe('', function () {

16
x-pack/test/functional/apps/ml/model_management/index.ts Normal file
View file

@ -0,0 +1,16 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ loadTestFile }: FtrProviderContext) {
describe('model management', function () {
this.tags(['mlqa', 'skipFirefox']);
loadTestFile(require.resolve('./model_list'));
});
}

6
x-pack/test/functional/apps/ml/data_frame_analytics/trained_models.ts → x-pack/test/functional/apps/ml/model_management/model_list.ts
View file

@ -27,19 +27,19 @@ export default function ({ getService }: FtrProviderContext) {
const builtInModelData = {
modelId: 'lang_ident_model_1',
description: 'Model used for identifying language from arbitrary input text.',
modelTypes: ['classification', 'built-in'],
modelTypes: ['classification', 'built-in', 'lang_ident'],
};
const modelWithPipelineData = {
modelId: 'dfa_classification_model_n_0',
description: '',
modelTypes: ['classification'],
modelTypes: ['classification', 'tree_ensemble'],
};
const modelWithoutPipelineData = {
modelId: 'dfa_regression_model_n_0',
description: '',
modelTypes: ['regression'],
modelTypes: ['regression', 'tree_ensemble'],
};
it('renders trained models list', async () => {

13
x-pack/test/functional/services/ml/navigation.ts
View file

@ -130,13 +130,24 @@ export function MachineLearningNavigationProvider({
await this.navigateToArea('~mlMainTab & ~dataFrameAnalytics', 'mlPageDataFrameAnalytics');
},
async navigateToModelManagement() {
await this.navigateToArea('~mlMainTab & ~modelManagement', 'mlPageModelManagement');
},
async navigateToTrainedModels() {
await this.navigateToMl();
await this.navigateToDataFrameAnalytics();
await this.navigateToModelManagement();
await testSubjects.click('mlTrainedModelsTab');
await testSubjects.existOrFail('mlModelsTableContainer');
},
async navigateToModelManagementNodeList() {
await this.navigateToMl();
await this.navigateToModelManagement();
await testSubjects.click('mlNodesOverviewTab');
await testSubjects.existOrFail('mlNodesTableContainer');
},
async navigateToDataVisualizer() {
await this.navigateToArea('~mlMainTab & ~dataVisualizer', 'mlPageDataVisualizerSelector');
},