[ML] Support E5 model download from the list view, mark not supported/not optimized models (#189372)

## Summary - Adds the "Show all" switch at the top of the page (off by default) that responsible for hiding unsupported/not optimised model versions - Adds a download action for E5 models - Removes the toast notification on successful start of a model download - Adds a warning icon with an appropriate tooltip text if model version is not supported by the cluster architecture or not optimised for it <img width="1401" alt="image" src="https://github.com/user-attachments/assets/30748315-3018-46e2-91dd-fd5128b0695e"> ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios
2025-04-23 09:19:04 -04:00 · 2024-08-07 08:43:18 +02:00 · 2024-08-07 08:43:18 +02:00 · 514db549d2
commit 514db549d2
parent 199d0f60f9
10 changed files with 85 additions and 48 deletions
--- a/x-pack/packages/ml/trained_models_utils/src/constants/trained_models.ts
+++ b/x-pack/packages/ml/trained_models_utils/src/constants/trained_models.ts
@ -58,7 +58,10 @@ export const BUILT_IN_MODEL_TAG = 'prepackaged';

 export const ELASTIC_MODEL_TAG = 'elastic';

-export const ELASTIC_MODEL_DEFINITIONS: Record<string, ModelDefinition> = Object.freeze({
+export const ELASTIC_MODEL_DEFINITIONS: Record<
+  string,
+  Omit<ModelDefinition, 'supported'>
+> = Object.freeze({
  [ELSER_ID_V1]: {
    modelName: 'elser',
    hidden: true,
@ -156,6 +159,8 @@ export interface ModelDefinition {
  default?: boolean;
  /** Indicates if model version is recommended for deployment based on the cluster configuration */
  recommended?: boolean;
+  /** Indicates if model version is supported by the cluster */
+  supported: boolean;
  hidden?: boolean;
  /** Software license of a model, e.g. MIT */
  license?: string;
--- a/x-pack/plugins/ml/common/types/common.ts
+++ b/x-pack/plugins/ml/common/types/common.ts
@ -41,6 +41,7 @@ export interface ListingPageUrlState {
  sortField: string;
  sortDirection: string;
  queryText?: string;
+  showAll?: boolean;
 }

 export type AppPageState<T> = {
--- a/x-pack/plugins/ml/public/application/model_management/model_actions.tsx
+++ b/x-pack/plugins/ml/public/application/model_management/model_actions.tsx
@ -15,10 +15,7 @@ import {
  DEPLOYMENT_STATE,
  TRAINED_MODEL_TYPE,
 } from '@kbn/ml-trained-models-utils';
-import {
-  ELASTIC_MODEL_TAG,
-  MODEL_STATE,
-} from '@kbn/ml-trained-models-utils/src/constants/trained_models';
+import { MODEL_STATE } from '@kbn/ml-trained-models-utils/src/constants/trained_models';
 import {
  getAnalysisType,
  type DataFrameAnalysisConfigType,
@ -409,10 +406,7 @@ export function useModelActions({
        icon: 'download',
        type: 'icon',
        isPrimary: true,
-        available: (item) =>
-          canCreateTrainedModels &&
-          item.tags.includes(ELASTIC_MODEL_TAG) &&
-          item.state === MODEL_STATE.NOT_DOWNLOADED,
+        available: (item) => canCreateTrainedModels && item.state === MODEL_STATE.NOT_DOWNLOADED,
        enabled: (item) => !isLoading,
        onClick: async (item) => {
          onModelDownloadRequest(item.model_id);
--- a/x-pack/plugins/ml/public/application/model_management/models_list.tsx
+++ b/x-pack/plugins/ml/public/application/model_management/models_list.tsx
@ -17,13 +17,15 @@ import {
  EuiFlexGroup,
  EuiFlexItem,
  EuiHealth,
+  EuiIcon,
  EuiInMemoryTable,
  EuiLink,
-  type EuiSearchBarProps,
+  EuiProgress,
  EuiSpacer,
+  EuiSwitch,
  EuiTitle,
  EuiToolTip,
-  EuiProgress,
+  type EuiSearchBarProps,
 } from '@elastic/eui';
 import { groupBy, isEmpty } from 'lodash';
 import { i18n } from '@kbn/i18n';
@ -94,6 +96,7 @@ export type ModelItem = TrainedModelConfigResponse & {
   */
  stateDescription?: string;
  recommended?: boolean;
+  supported: boolean;
  /**
   * Model name, e.g. elser
   */
@ -129,6 +132,7 @@ export const getDefaultModelsListState = (): ListingPageUrlState => ({
  pageSize: 10,
  sortField: modelIdColumnName,
  sortDirection: 'asc',
+  showAll: false,
 });

 interface Props {
@ -286,9 +290,13 @@ export const ModelsList: FC<Props> = ({
        );
        const forDownload = await trainedModelsApiService.getTrainedModelDownloads();
        const notDownloaded: ModelItem[] = forDownload
-          .filter(({ model_id: modelId, hidden, recommended }) => {
-            if (recommended && idMap.has(modelId)) {
-              idMap.get(modelId)!.recommended = true;
+          .filter(({ model_id: modelId, hidden, recommended, supported }) => {
+            if (idMap.has(modelId)) {
+              const model = idMap.get(modelId)!;
+              if (recommended) {
+                model.recommended = true;
+              }
+              model.supported = supported;
            }
            return !idMap.has(modelId) && !hidden;
          })
@ -306,6 +314,7 @@ export const ModelsList: FC<Props> = ({
              arch: modelDefinition.arch,
              softwareLicense: modelDefinition.license,
              licenseUrl: modelDefinition.licenseUrl,
+              supported: modelDefinition.supported,
            } as ModelItem;
          });
        resultItems = [...resultItems, ...notDownloaded];
@ -530,12 +539,6 @@ export const ModelsList: FC<Props> = ({
      try {
        setIsLoading(true);
        await trainedModelsApiService.installElasticTrainedModelConfig(modelId);
-        displaySuccessToast(
-          i18n.translate('xpack.ml.trainedModels.modelsList.downloadSuccess', {
-            defaultMessage: '"{modelId}" model download has been started successfully.',
-            values: { modelId },
-          })
-        );
        // Need to fetch model state updates
        await fetchModelsData();
      } catch (e) {
@ -549,7 +552,7 @@ export const ModelsList: FC<Props> = ({
        setIsLoading(true);
      }
    },
-    [displayErrorToast, displaySuccessToast, fetchModelsData, trainedModelsApiService]
+    [displayErrorToast, fetchModelsData, trainedModelsApiService]
  );

  /**
@ -633,26 +636,28 @@ export const ModelsList: FC<Props> = ({
      }),
      truncateText: false,
      'data-test-subj': 'mlModelsTableColumnDescription',
-      render: ({ description, recommended }: ModelItem) => {
+      render: ({ description, recommended, tags, supported }: ModelItem) => {
        if (!description) return null;
        const descriptionText = description.replace('(Tech Preview)', '');
-        return recommended ? (
-          <EuiToolTip
-            content={
-              <FormattedMessage
-                id="xpack.ml.trainedModels.modelsList.recommendedDownloadContent"
-                defaultMessage="Recommended model version for your cluster's hardware configuration"
-              />
-            }
-          >
+
+        const tooltipContent =
+          supported === false ? (
+            <FormattedMessage
+              id="xpack.ml.trainedModels.modelsList.notSupportedDownloadContent"
+              defaultMessage="Model version is not supported by your cluster's hardware configuration"
+            />
+          ) : recommended === false ? (
+            <FormattedMessage
+              id="xpack.ml.trainedModels.modelsList.notRecommendedDownloadContent"
+              defaultMessage="Model version is not optimized for your cluster's hardware configuration"
+            />
+          ) : null;
+
+        return tooltipContent ? (
+          <EuiToolTip content={tooltipContent}>
            <>
              {descriptionText}&nbsp;
-              <b>
-                <FormattedMessage
-                  id="xpack.ml.trainedModels.modelsList.recommendedDownloadLabel"
-                  defaultMessage="(Recommended)"
-                />
-              </b>
+              <EuiIcon type={'warning'} color="warning" />
            </>
          </EuiToolTip>
        ) : (
@ -861,6 +866,14 @@ export const ModelsList: FC<Props> = ({
  const isElserCalloutVisible =
    !isElserCalloutDismissed && items.findIndex((i) => i.model_id === ELSER_ID_V1) >= 0;

+  const tableItems = useMemo(() => {
+    if (pageState.showAll) {
+      return items;
+    } else {
+      return items.filter((item) => item.supported !== false);
+    }
+  }, [items, pageState.showAll]);
+
  if (!isInitialized) return null;

  return (
@ -868,8 +881,24 @@ export const ModelsList: FC<Props> = ({
      <SavedObjectsWarning onCloseFlyout={fetchModelsData} forceRefresh={isLoading} />
      <EuiFlexGroup justifyContent="spaceBetween">
        {modelsStats ? (
-          <EuiFlexItem grow={false}>
-            <StatsBar stats={modelsStats} dataTestSub={'mlInferenceModelsStatsBar'} />
+          <EuiFlexItem>
+            <EuiFlexGroup alignItems="center">
+              <EuiFlexItem grow={false}>
+                <StatsBar stats={modelsStats} dataTestSub={'mlInferenceModelsStatsBar'} />
+              </EuiFlexItem>
+              <EuiFlexItem grow={false}>
+                <EuiSwitch
+                  label={
+                    <FormattedMessage
+                      id="xpack.ml.trainedModels.modelsList.showAllLabel"
+                      defaultMessage="Show all"
+                    />
+                  }
+                  checked={!!pageState.showAll}
+                  onChange={(e) => updatePageState({ showAll: e.target.checked })}
+                />
+              </EuiFlexItem>
+            </EuiFlexGroup>
          </EuiFlexItem>
        ) : null}
        <EuiFlexItem grow={false}>
@ -894,7 +923,7 @@ export const ModelsList: FC<Props> = ({
          allowNeutralSort={false}
          columns={columns}
          itemIdToExpandedRowMap={itemIdToExpandedRowMap}
-          items={items}
+          items={tableItems}
          itemId={ModelsTableToConfigMapping.id}
          loading={isLoading}
          search={search}
--- a/x-pack/plugins/ml/server/models/model_management/model_provider.test.ts
+++ b/x-pack/plugins/ml/server/models/model_management/model_provider.test.ts
@ -58,6 +58,7 @@ describe('modelsProvider', () => {
          config: { input: { field_names: ['text_field'] } },
          description: 'Elastic Learned Sparse EncodeR v1 (Tech Preview)',
          hidden: true,
+          supported: false,
          model_id: '.elser_model_1',
          version: 1,
          modelName: 'elser',
@ -66,6 +67,7 @@ describe('modelsProvider', () => {
        {
          config: { input: { field_names: ['text_field'] } },
          default: true,
+          supported: true,
          description: 'Elastic Learned Sparse EncodeR v2',
          model_id: '.elser_model_2',
          version: 2,
@ -79,6 +81,7 @@ describe('modelsProvider', () => {
          model_id: '.elser_model_2_linux-x86_64',
          os: 'Linux',
          recommended: true,
+          supported: true,
          version: 2,
          modelName: 'elser',
          type: ['elastic', 'pytorch', 'text_expansion'],
@ -88,6 +91,7 @@ describe('modelsProvider', () => {
          description: 'E5 (EmbEddings from bidirEctional Encoder rEpresentations)',
          model_id: '.multilingual-e5-small',
          default: true,
+          supported: true,
          version: 1,
          modelName: 'e5',
          license: 'MIT',
@ -102,6 +106,7 @@ describe('modelsProvider', () => {
          model_id: '.multilingual-e5-small_linux-x86_64',
          os: 'Linux',
          recommended: true,
+          supported: true,
          version: 1,
          modelName: 'e5',
          license: 'MIT',
@ -140,6 +145,7 @@ describe('modelsProvider', () => {
          config: { input: { field_names: ['text_field'] } },
          description: 'Elastic Learned Sparse EncodeR v1 (Tech Preview)',
          hidden: true,
+          supported: false,
          model_id: '.elser_model_1',
          version: 1,
          modelName: 'elser',
@ -148,6 +154,7 @@ describe('modelsProvider', () => {
        {
          config: { input: { field_names: ['text_field'] } },
          recommended: true,
+          supported: true,
          description: 'Elastic Learned Sparse EncodeR v2',
          model_id: '.elser_model_2',
          version: 2,
@ -163,12 +170,14 @@ describe('modelsProvider', () => {
          version: 2,
          modelName: 'elser',
          type: ['elastic', 'pytorch', 'text_expansion'],
+          supported: false,
        },
        {
          config: { input: { field_names: ['text_field'] } },
          description: 'E5 (EmbEddings from bidirEctional Encoder rEpresentations)',
          model_id: '.multilingual-e5-small',
          recommended: true,
+          supported: true,
          version: 1,
          modelName: 'e5',
          type: ['pytorch', 'text_embedding'],
@ -182,6 +191,7 @@ describe('modelsProvider', () => {
            'E5 (EmbEddings from bidirEctional Encoder rEpresentations), optimized for linux-x86_64',
          model_id: '.multilingual-e5-small_linux-x86_64',
          os: 'Linux',
+          supported: false,
          version: 1,
          modelName: 'e5',
          type: ['pytorch', 'text_embedding'],
--- a/x-pack/plugins/ml/server/models/model_management/models_provider.ts
+++ b/x-pack/plugins/ml/server/models/model_management/models_provider.ts
@ -476,6 +476,7 @@ export class ModelsProvider {
      const modelDefinitionResponse = {
        ...def,
        ...(recommended ? { recommended } : {}),
+        supported: !!def.default || recommended,
        model_id: modelId,
      };

--- a/x-pack/plugins/translations/translations/fr-FR.json
+++ b/x-pack/plugins/translations/translations/fr-FR.json
@ -28304,7 +28304,6 @@
    "xpack.ml.trainedModels.modelsList.disableSelectableMessage": "Le modèle a des pipelines associés",
    "xpack.ml.trainedModels.modelsList.downloadFailed": "Échec du téléchargement de \"{modelId}\"",
    "xpack.ml.trainedModels.modelsList.downloadStatusCheckErrorMessage": "Échec de la vérification du statut du téléchargement",
-    "xpack.ml.trainedModels.modelsList.downloadSuccess": "Le téléchargement du modèle \"{modelId}\" a bien été démarré.",
    "xpack.ml.trainedModels.modelsList.e5Title": "E5 (EmbEddings from bidirEctional Encoder rEpresentations)",
    "xpack.ml.trainedModels.modelsList.e5v1Description": "E5 (EmbEddings from bidirEctional Encoder rEpresentations)",
    "xpack.ml.trainedModels.modelsList.e5v1x86Description": "E5 (EmbEddings from bidirEctional Encoder rEpresentations), optimisé for linux-x86_64",
@ -28362,7 +28361,6 @@
    "xpack.ml.trainedModels.modelsList.pipelines.processorStats.timePerDocHeader": "Temps par document",
    "xpack.ml.trainedModels.modelsList.pipelines.processorStats.typeHeader": "Type de processeur",
    "xpack.ml.trainedModels.modelsList.recommendedDownloadContent": "Version du modèle recommandée pour la configuration matérielle de votre cluster",
-    "xpack.ml.trainedModels.modelsList.recommendedDownloadLabel": "(Recommandée)",
    "xpack.ml.trainedModels.modelsList.selectableMessage": "Sélectionner un modèle",
    "xpack.ml.trainedModels.modelsList.selectedModelsMessage": "{modelsCount, plural, one{# modèle sélectionné} other {# modèles sélectionnés}}",
    "xpack.ml.trainedModels.modelsList.startDeployment.cancelButton": "Annuler",
--- a/x-pack/plugins/translations/translations/ja-JP.json
+++ b/x-pack/plugins/translations/translations/ja-JP.json
@ -28209,7 +28209,6 @@
    "xpack.ml.trainedModels.modelsList.disableSelectableMessage": "モデルにはパイプラインが関連付けられています",
    "xpack.ml.trainedModels.modelsList.downloadFailed": "\"{modelId}\"をダウンロードできませんでした",
    "xpack.ml.trainedModels.modelsList.downloadStatusCheckErrorMessage": "ダウンロードステータスを確認できませんでした",
-    "xpack.ml.trainedModels.modelsList.downloadSuccess": "\"{modelId}\"モデルのダウンロードが正常に開始しました。",
    "xpack.ml.trainedModels.modelsList.e5Title": "E5（bidirEctional Encoder rEpresentationsからのEmbEddings）",
    "xpack.ml.trainedModels.modelsList.e5v1Description": "E5（bidirEctional Encoder rEpresentationsからのEmbEddings）",
    "xpack.ml.trainedModels.modelsList.e5v1x86Description": "E5（bidirEctional Encoder rEpresentationsからのEmbEddings）、inux-x86_64向けに最適化",
@ -28267,7 +28266,6 @@
    "xpack.ml.trainedModels.modelsList.pipelines.processorStats.timePerDocHeader": "ドキュメントごとの時間",
    "xpack.ml.trainedModels.modelsList.pipelines.processorStats.typeHeader": "プロセッサータイプ",
    "xpack.ml.trainedModels.modelsList.recommendedDownloadContent": "クラスターのハードウェア構成に応じた推奨モデルバージョン",
-    "xpack.ml.trainedModels.modelsList.recommendedDownloadLabel": "（推奨）",
    "xpack.ml.trainedModels.modelsList.selectableMessage": "モデルを選択",
    "xpack.ml.trainedModels.modelsList.selectedModelsMessage": "{modelsCount, plural, other  {#個のモデル}}が選択されました",
    "xpack.ml.trainedModels.modelsList.startDeployment.cancelButton": "キャンセル",
--- a/x-pack/plugins/translations/translations/zh-CN.json
+++ b/x-pack/plugins/translations/translations/zh-CN.json
@ -28344,7 +28344,6 @@
    "xpack.ml.trainedModels.modelsList.disableSelectableMessage": "模型有关联的管道",
    "xpack.ml.trainedModels.modelsList.downloadFailed": "无法下载“{modelId}”",
    "xpack.ml.trainedModels.modelsList.downloadStatusCheckErrorMessage": "无法检查下载状态",
-    "xpack.ml.trainedModels.modelsList.downloadSuccess": "已成功启动“{modelId}”模型下载。",
    "xpack.ml.trainedModels.modelsList.e5Title": "E5 (EmbEddings from bidirEctional Encoder rEpresentations)",
    "xpack.ml.trainedModels.modelsList.e5v1Description": "E5 (EmbEddings from bidirEctional Encoder rEpresentations)",
    "xpack.ml.trainedModels.modelsList.e5v1x86Description": "针对 linux-x86_64 进行了优化的 E5 (EmbEddings from bidirEctional Encoder rEpresentations)",
@ -28402,7 +28401,6 @@
    "xpack.ml.trainedModels.modelsList.pipelines.processorStats.timePerDocHeader": "每个文档的时间",
    "xpack.ml.trainedModels.modelsList.pipelines.processorStats.typeHeader": "处理器类型",
    "xpack.ml.trainedModels.modelsList.recommendedDownloadContent": "为您集群的硬件配置推荐的模型版本",
-    "xpack.ml.trainedModels.modelsList.recommendedDownloadLabel": "（推荐）",
    "xpack.ml.trainedModels.modelsList.selectableMessage": "选择模型",
    "xpack.ml.trainedModels.modelsList.selectedModelsMessage": "{modelsCount, plural, other {# 个模型}}已选择",
    "xpack.ml.trainedModels.modelsList.startDeployment.cancelButton": "取消",
--- a/x-pack/test/api_integration/apis/ml/trained_models/model_downloads.ts
+++ b/x-pack/test/api_integration/apis/ml/trained_models/model_downloads.ts
@ -51,6 +51,7 @@ export default ({ getService }: FtrProviderContext) => {
        {
          modelName: 'elser',
          hidden: true,
+          supported: false,
          version: 1,
          config: {
            input: {
@ -72,6 +73,7 @@ export default ({ getService }: FtrProviderContext) => {
          description: 'Elastic Learned Sparse EncodeR v2',
          type: ['elastic', 'pytorch', 'text_expansion'],
          model_id: '.elser_model_2',
+          supported: true,
          ...(isIntelBased ? { default: true } : { recommended: true }),
        },
        {
@ -87,7 +89,7 @@ export default ({ getService }: FtrProviderContext) => {
          description: 'Elastic Learned Sparse EncodeR v2, optimized for linux-x86_64',
          type: ['elastic', 'pytorch', 'text_expansion'],
          model_id: '.elser_model_2_linux-x86_64',
-          ...(isIntelBased ? { recommended: true } : {}),
+          ...(isIntelBased ? { recommended: true, supported: true } : { supported: false }),
        },
        {
          modelName: 'e5',
@ -102,6 +104,7 @@ export default ({ getService }: FtrProviderContext) => {
          licenseUrl: 'https://huggingface.co/elastic/multilingual-e5-small',
          type: ['pytorch', 'text_embedding'],
          model_id: '.multilingual-e5-small',
+          supported: true,
          ...(isIntelBased ? { default: true } : { recommended: true }),
        },
        {
@ -120,7 +123,7 @@ export default ({ getService }: FtrProviderContext) => {
          licenseUrl: 'https://huggingface.co/elastic/multilingual-e5-small_linux-x86_64',
          type: ['pytorch', 'text_embedding'],
          model_id: '.multilingual-e5-small_linux-x86_64',
-          ...(isIntelBased ? { recommended: true } : {}),
+          ...(isIntelBased ? { recommended: true, supported: true } : { supported: false }),
        },
      ]);
    });