Add semantic_text to index mapping (#179575)

In this PR, we added the following items.

- Add a semantic_text field type
- Allow the users to add semantic_text to index mapping
- Allow the user to select a text field as reference field
- Allow the user to select inference_id for semantic_text field

Please be aware that currently, we won't be able to save the mapping
using the 'Save mappings' button because the 'semantic_text'
functionality doesn't support 'inference_id'. However, there is ongoing
parallel work in a GitHub
[branch](https://github.com/elastic/elasticsearch/tree/feature/semantic-text)
to enable 'inference_id' in 'semantic_text' for Elasticsearch.
This commit is contained in:
Saikat Sarkar 2024-04-04 10:03:10 -06:00 committed by GitHub
parent 7d80e4f689
commit fff6bcffde
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
31 changed files with 651 additions and 163 deletions

View file

@ -37,6 +37,7 @@ export enum KNOWN_FIELD_TYPES {
POINT = 'point',
SHAPE = 'shape',
SPARSE_VECTOR = 'sparse_vector',
SEMANTIC_TEXT = 'semantic_text',
STRING = 'string',
TEXT = 'text',
VERSION = 'version',

View file

@ -122,6 +122,10 @@ export function getFieldTypeDescription(type?: string) {
return i18n.translate('fieldUtils.fieldNameDescription.sparseVectorField', {
defaultMessage: 'Records sparse vectors of float values.',
});
case KNOWN_FIELD_TYPES.SEMANTIC_TEXT:
return i18n.translate('fieldUtils.fieldNameDescription.semanticTextField', {
defaultMessage: 'References model id used for text embeddings.',
});
case KNOWN_FIELD_TYPES.STRING:
return i18n.translate('fieldUtils.fieldNameDescription.stringField', {
defaultMessage: 'Full text such as the body of an email or a product description.',

View file

@ -127,6 +127,10 @@ export function getFieldTypeName(type?: string) {
return i18n.translate('fieldUtils.fieldNameIcons.sparseVectorFieldAriaLabel', {
defaultMessage: 'Sparse vector',
});
case KNOWN_FIELD_TYPES.SEMANTIC_TEXT:
return i18n.translate('fieldUtils.fieldNameIcons.semanticTextFieldAriaLabel', {
defaultMessage: 'Semantic text',
});
case KNOWN_FIELD_TYPES.STRING:
return i18n.translate('fieldUtils.fieldNameIcons.stringFieldAriaLabel', {
defaultMessage: 'String',

View file

@ -33,6 +33,7 @@ export interface FieldIconProps extends Omit<EuiTokenProps, 'iconType'> {
| 'point'
| 'shape'
| 'sparse_vector'
| 'semantic_text'
| 'string'
| string
| 'nested'

View file

@ -19,6 +19,7 @@ export {
type ModelDefinition,
type ModelDefinitionResponse,
type ElserVersion,
type InferenceAPIConfigResponse,
type GetModelDownloadConfigOptions,
type ElasticCuratedModelName,
ELSER_ID_V1,

View file

@ -177,3 +177,37 @@ export type ElserVersion = 1 | 2;
export interface GetModelDownloadConfigOptions {
version?: ElserVersion;
}
export type InferenceServiceSettings =
| {
service: 'elser';
service_settings: {
num_allocations: number;
num_threads: number;
model_id: string;
};
}
| {
service: 'openai';
service_settings: {
api_key: string;
organization_id: string;
url: string;
};
}
| {
service: 'hugging_face';
service_settings: {
api_key: string;
url: string;
};
};
export type InferenceAPIConfigResponse = {
// Refers to a deployment id
model_id: string;
task_type: 'sparse_embedding' | 'text_embedding';
task_settings: {
model?: string;
};
} & InferenceServiceSettings;

View file

@ -577,6 +577,44 @@ describe('<IndexDetailsPage />', () => {
JSON.stringify({ mappings: mockIndexMappingResponse }, null, 2)
);
});
it('can add a semantic_text field and can save mappings', async () => {
const mockIndexMappingResponseForSemanticText: any = {
...testIndexMappings.mappings,
properties: {
...testIndexMappings.mappings.properties,
sem: {
type: 'semantic_text',
inference_id: 'my-elser',
},
},
};
httpRequestsMockHelpers.setLoadIndexMappingResponse(testIndexName, {
mappings: mockIndexMappingResponseForSemanticText,
});
await testBed.actions.mappings.addNewMappingFieldNameAndType([
{ name: 'sem', type: 'semantic_text' },
]);
await testBed.actions.mappings.clickSaveMappingsButton();
// add field button is available again
expect(testBed.exists('indexDetailsMappingsAddField')).toBe(true);
expect(testBed.find('semField-datatype').props()['data-type-value']).toBe('semantic_text');
expect(httpSetup.get).toHaveBeenCalledTimes(5);
expect(httpSetup.get).toHaveBeenLastCalledWith(
`${API_BASE_PATH}/mapping/${testIndexName}`,
requestOptions
);
// refresh mappings and page re-renders
expect(testBed.exists('indexDetailsMappingsAddField')).toBe(true);
expect(testBed.actions.mappings.isSearchBarDisabled()).toBe(false);
const treeViewContent = testBed.actions.mappings.getTreeViewContent('semField');
expect(treeViewContent).toContain('sem');
await testBed.actions.mappings.clickToggleViewButton();
const jsonContent = testBed.actions.mappings.getCodeBlockContent();
expect(jsonContent).toEqual(
JSON.stringify({ mappings: mockIndexMappingResponseForSemanticText }, null, 2)
);
});
it('there is a callout with error message when save mappings fail', async () => {
const error = {
statusCode: 400,

View file

@ -6,27 +6,9 @@
"id": "indexManagement",
"server": true,
"browser": true,
"configPath": [
"xpack",
"index_management"
],
"requiredPlugins": [
"home",
"management",
"features",
"share"
],
"optionalPlugins": [
"security",
"usageCollection",
"fleet",
"cloud",
"console"
],
"requiredBundles": [
"kibanaReact",
"esUiShared",
"runtimeFields"
]
"configPath": ["xpack", "index_management"],
"requiredPlugins": ["home", "management", "features", "share"],
"optionalPlugins": ["security", "usageCollection", "fleet", "cloud", "ml", "console"],
"requiredBundles": ["kibanaReact", "esUiShared", "runtimeFields"]
}
}

View file

@ -95,6 +95,13 @@ export const getApi = (
});
}
async function getInferenceModels() {
return sendRequest({
path: `${apiBasePath}/inference/all`,
method: 'get',
});
}
async function postDataStreamRollover(name: string) {
return sendRequest<ComponentTemplateDatastreams>({
path: `${apiBasePath}/data_streams/${encodeURIComponent(name)}/rollover`,
@ -119,5 +126,6 @@ export const getApi = (
getComponentTemplateDatastreams,
postDataStreamRollover,
postDataStreamMappingsFromTemplate,
getInferenceModels,
};
};

View file

@ -20,9 +20,18 @@ interface Props {
searchResultComponent?: React.ReactElement;
onCancelAddingNewFields?: () => void;
isAddingFields?: boolean;
isSemanticTextEnabled?: boolean;
indexName?: string;
}
export const DocumentFields = React.memo(
({ searchComponent, searchResultComponent, onCancelAddingNewFields, isAddingFields }: Props) => {
({
searchComponent,
searchResultComponent,
onCancelAddingNewFields,
isAddingFields,
isSemanticTextEnabled,
indexName,
}: Props) => {
const { fields, documentFields } = useMappingsState();
const dispatch = useDispatch();
const { addContent: addContentToGlobalFlyout, removeContent: removeContentFromGlobalFlyout } =
@ -44,6 +53,8 @@ export const DocumentFields = React.memo(
<DocumentFieldsTreeEditor
onCancelAddingNewFields={onCancelAddingNewFields}
isAddingFields={isAddingFields}
isSemanticTextEnabled={isSemanticTextEnabled}
indexName={indexName}
/>
);

View file

@ -0,0 +1,98 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { EuiCallOut, EuiFlexGroup, EuiFlexItem } from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import React, { useEffect, useState } from 'react';
import type { InferenceAPIConfigResponse } from '@kbn/ml-trained-models-utils';
import { useComponentTemplatesContext } from '../../../../component_templates/component_templates_context';
import { getFieldConfig } from '../../../lib';
import { Form, SuperSelectField, UseField, useForm } from '../../../shared_imports';
import { SuperSelectOption } from '../../../types';
interface Props {
onChange(value: string): void;
'data-test-subj'?: string;
}
interface InferenceModel {
data: InferenceAPIConfigResponse[];
}
export const InferenceIdSelects = ({ onChange, 'data-test-subj': dataTestSubj }: Props) => {
const { form } = useForm({ defaultValue: { main: 'elser_model_2' } });
const { subscribe } = form;
const { api } = useComponentTemplatesContext();
const [inferenceModels, setInferenceModels] = useState<InferenceModel>({ data: [] });
const fieldConfigModelId = getFieldConfig('inference_id');
const defaultInferenceIds: SuperSelectOption[] = [
{ value: 'elser_model_2', inputDisplay: 'elser_model_2' },
{ value: 'e5', inputDisplay: 'e5' },
];
const inferenceIdOptionsFromModels: SuperSelectOption[] =
inferenceModels?.data?.map((model: InferenceAPIConfigResponse) => ({
value: model.model_id,
inputDisplay: model.model_id,
})) || [];
const inferenceIdOptions: SuperSelectOption[] = [
...defaultInferenceIds,
...inferenceIdOptionsFromModels,
];
useEffect(() => {
const fetchInferenceModels = async () => {
const models = await api.getInferenceModels();
setInferenceModels(models);
};
fetchInferenceModels();
}, [api]);
useEffect(() => {
const subscription = subscribe((updateData) => {
const formData = updateData.data.internal;
const value = formData.main;
onChange(value);
});
return subscription.unsubscribe;
}, [subscribe, onChange]);
return (
<Form form={form}>
<EuiFlexGroup>
<EuiFlexItem grow={false}>
<UseField path="main" config={fieldConfigModelId}>
{(field) => (
<SuperSelectField
field={field}
euiFieldProps={{ options: inferenceIdOptions }}
data-test-subj={dataTestSubj}
/>
)}
</UseField>
</EuiFlexItem>
<EuiFlexItem grow={true}>
<EuiCallOut
size="s"
color="warning"
title={i18n.translate(
'xpack.idxMgmt.mappingsEditor.parameters.noReferenceModelStartWarningMessage',
{
defaultMessage:
'The referenced model for this inference endpoint will be started when adding this field.',
}
)}
/>
</EuiFlexItem>
</EuiFlexGroup>
</Form>
);
};

View file

@ -0,0 +1,64 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import React, { useEffect } from 'react';
import { useLoadIndexMappings } from '../../../../../services';
import { getFieldConfig } from '../../../lib';
import { Form, SuperSelectField, UseField, useForm } from '../../../shared_imports';
import { SuperSelectOption } from '../../../types';
interface Props {
onChange(value: string): void;
'data-test-subj'?: string;
indexName?: string;
}
export const ReferenceFieldSelects = ({
onChange,
'data-test-subj': dataTestSubj,
indexName,
}: Props) => {
const { form } = useForm();
const { subscribe } = form;
const { data } = useLoadIndexMappings(indexName ?? '');
const referenceFieldOptions: SuperSelectOption[] = [];
if (data && data.mappings && data.mappings.properties) {
Object.keys(data.mappings.properties).forEach((key) => {
const field = data.mappings.properties[key];
if (field.type === 'text') {
referenceFieldOptions.push({ value: key, inputDisplay: key });
}
});
}
const fieldConfigReferenceField = getFieldConfig('reference_field');
useEffect(() => {
const subscription = subscribe((updateData) => {
const formData = updateData.data.internal;
const value = formData.main;
onChange(value);
});
return subscription.unsubscribe;
}, [subscribe, onChange]);
return (
<Form form={form}>
<UseField path="main" config={fieldConfigReferenceField}>
{(field) => (
<SuperSelectField
field={field}
euiFieldProps={{ options: referenceFieldOptions }}
data-test-subj={dataTestSubj}
/>
)}
</UseField>
</Form>
);
};

View file

@ -5,7 +5,7 @@
* 2.0.
*/
import React from 'react';
import React, { useMemo } from 'react';
import { EuiFormRow, EuiComboBox, EuiText, EuiLink } from '@elastic/eui';
import { i18n } from '@kbn/i18n';
@ -23,60 +23,82 @@ interface Props {
isRootLevelField: boolean;
isMultiField?: boolean | null;
showDocLink?: boolean;
isSemanticTextEnabled?: boolean;
}
export const TypeParameter = ({ isMultiField, isRootLevelField, showDocLink = false }: Props) => (
<UseField<ComboBoxOption[]> path="type" config={getFieldConfig<ComboBoxOption[]>('type')}>
{(typeField) => {
const error = typeField.getErrorsMessages();
const isInvalid = error ? Boolean(error.length) : false;
export const TypeParameter = ({
isMultiField,
isRootLevelField,
showDocLink = false,
isSemanticTextEnabled = false,
}: Props) => {
const fieldTypeOptions = useMemo(() => {
let options = isMultiField
? filterTypesForMultiField(FIELD_TYPES_OPTIONS)
: isRootLevelField
? FIELD_TYPES_OPTIONS
: filterTypesForNonRootFields(FIELD_TYPES_OPTIONS);
let docLink = null;
if (showDocLink && typeField.value.length > 0) {
const selectedType = typeField.value[0].value as DataType;
docLink = documentationService.getTypeDocLink(selectedType);
}
if (!isSemanticTextEnabled) {
options = options.filter((option) => option.value !== 'semantic_text');
}
return (
<EuiFormRow
label={typeField.label}
error={error}
isInvalid={isInvalid}
helpText={
docLink ? (
<EuiText size="xs">
<EuiLink href={docLink} target="_blank">
{i18n.translate('xpack.idxMgmt.mappingsEditor.typeField.documentationLinkLabel', {
defaultMessage: '{typeName} documentation',
values: {
typeName:
typeField.value && typeField.value[0] ? typeField.value[0].label : '',
},
})}
</EuiLink>
</EuiText>
) : null
}
>
<EuiComboBox
placeholder={i18n.translate('xpack.idxMgmt.mappingsEditor.typeField.placeholderLabel', {
defaultMessage: 'Select a type',
})}
singleSelection={{ asPlainText: true }}
options={
isMultiField
? filterTypesForMultiField(FIELD_TYPES_OPTIONS)
: isRootLevelField
? FIELD_TYPES_OPTIONS
: filterTypesForNonRootFields(FIELD_TYPES_OPTIONS)
return options;
}, [isMultiField, isRootLevelField, isSemanticTextEnabled]);
return (
<UseField<ComboBoxOption[]> path="type" config={getFieldConfig<ComboBoxOption[]>('type')}>
{(typeField) => {
const error = typeField.getErrorsMessages();
const isInvalid = error ? Boolean(error.length) : false;
let docLink = null;
if (showDocLink && typeField.value.length > 0) {
const selectedType = typeField.value[0].value as DataType;
docLink = documentationService.getTypeDocLink(selectedType);
}
return (
<EuiFormRow
label={typeField.label}
error={error}
isInvalid={isInvalid}
helpText={
docLink ? (
<EuiText size="xs">
<EuiLink href={docLink} target="_blank">
{i18n.translate(
'xpack.idxMgmt.mappingsEditor.typeField.documentationLinkLabel',
{
defaultMessage: '{typeName} documentation',
values: {
typeName:
typeField.value && typeField.value[0] ? typeField.value[0].label : '',
},
}
)}
</EuiLink>
</EuiText>
) : null
}
selectedOptions={typeField.value}
onChange={typeField.setValue}
isClearable={false}
data-test-subj="fieldType"
/>
</EuiFormRow>
);
}}
</UseField>
);
>
<EuiComboBox
placeholder={i18n.translate(
'xpack.idxMgmt.mappingsEditor.typeField.placeholderLabel',
{
defaultMessage: 'Select a type',
}
)}
singleSelection={{ asPlainText: true }}
options={fieldTypeOptions}
selectedOptions={typeField.value}
onChange={typeField.setValue}
isClearable={false}
data-test-subj="fieldType"
/>
</EuiFormRow>
);
}}
</UseField>
);
};

View file

@ -5,26 +5,35 @@
* 2.0.
*/
import React, { useEffect } from 'react';
import classNames from 'classnames';
import React, { useEffect, useState } from 'react';
import { i18n } from '@kbn/i18n';
import {
EuiButtonEmpty,
EuiButton,
EuiButtonEmpty,
EuiFlexGroup,
EuiFlexItem,
EuiOutsideClickDetector,
EuiSpacer,
} from '@elastic/eui';
import { FieldWithSemanticTextInfo } from '../../../../types';
import { useForm, Form, FormDataProvider } from '../../../../shared_imports';
import { EUI_SIZE, TYPE_DEFINITION } from '../../../../constants';
import { useDispatch } from '../../../../mappings_state_context';
import { fieldSerializer } from '../../../../lib';
import { Field, NormalizedFields, MainType } from '../../../../types';
import { NameParameter, TypeParameter, SubTypeParameter } from '../../field_parameters';
import { useDispatch } from '../../../../mappings_state_context';
import {
Form,
FormDataProvider,
FormHook,
UseField,
useForm,
useFormData,
} from '../../../../shared_imports';
import { MainType, NormalizedFields } from '../../../../types';
import { NameParameter, SubTypeParameter, TypeParameter } from '../../field_parameters';
import { InferenceIdSelects } from '../../field_parameters/inference_id_selects';
import { ReferenceFieldSelects } from '../../field_parameters/reference_field_selects';
import { FieldBetaBadge } from '../field_beta_badge';
import { getRequiredParametersFormForType } from './required_parameters_forms';
@ -39,8 +48,23 @@ interface Props {
maxNestedDepth?: number;
onCancelAddingNewFields?: () => void;
isAddingFields?: boolean;
isSemanticTextEnabled?: boolean;
indexName?: string;
}
const useFieldEffect = (
form: FormHook,
fieldName: string,
setState: React.Dispatch<React.SetStateAction<string | undefined>>
) => {
const fieldValue = form.getFields()?.[fieldName]?.value as string;
useEffect(() => {
if (fieldValue !== undefined) {
setState(fieldValue);
}
}, [form, fieldValue, setState]);
};
export const CreateField = React.memo(function CreateFieldComponent({
allFields,
isRootLevelField,
@ -50,14 +74,18 @@ export const CreateField = React.memo(function CreateFieldComponent({
maxNestedDepth,
onCancelAddingNewFields,
isAddingFields,
isSemanticTextEnabled,
indexName,
}: Props) {
const dispatch = useDispatch();
const { form } = useForm<Field>({
const { form } = useForm<FieldWithSemanticTextInfo>({
serializer: fieldSerializer,
options: { stripEmptyFields: false },
});
useFormData({ form });
const { subscribe } = form;
useEffect(() => {
@ -76,6 +104,25 @@ export const CreateField = React.memo(function CreateFieldComponent({
}
};
const [referenceFieldComboValue, setReferenceFieldComboValue] = useState<string>();
const [nameValue, setNameValue] = useState<string>();
const [inferenceIdComboValue, setInferenceIdComboValue] = useState<string>();
const [semanticFieldType, setSemanticTextFieldType] = useState<string>();
useFieldEffect(form, 'referenceField', setReferenceFieldComboValue);
useFieldEffect(form, 'inferenceId', setInferenceIdComboValue);
useFieldEffect(form, 'name', setNameValue);
const fieldTypeValue = form.getFields()?.type?.value as Array<{ value: string }>;
useEffect(() => {
if (fieldTypeValue === undefined || fieldTypeValue.length === 0) {
return;
}
setSemanticTextFieldType(
fieldTypeValue[0]?.value === 'semantic_text' ? fieldTypeValue[0].value : undefined
);
}, [form, fieldTypeValue]);
const submitForm = async (e?: React.FormEvent, exitAfter: boolean = false) => {
if (e) {
e.preventDefault();
@ -85,7 +132,11 @@ export const CreateField = React.memo(function CreateFieldComponent({
if (isValid) {
form.reset();
dispatch({ type: 'field.add', value: data });
if (data.type === 'semantic_text') {
dispatch({ type: 'field.addSemanticText', value: data });
} else {
dispatch({ type: 'field.add', value: data });
}
if (exitAfter) {
cancel();
@ -107,17 +158,13 @@ export const CreateField = React.memo(function CreateFieldComponent({
const renderFormFields = () => (
<EuiFlexGroup gutterSize="s">
{/* Field name */}
<EuiFlexItem>
<NameParameter />
</EuiFlexItem>
{/* Field type */}
<EuiFlexItem>
<EuiFlexItem grow={false}>
<TypeParameter
isRootLevelField={isRootLevelField}
isMultiField={isMultiField}
showDocLink
isSemanticTextEnabled={isSemanticTextEnabled}
/>
</EuiFlexItem>
@ -139,9 +186,25 @@ export const CreateField = React.memo(function CreateFieldComponent({
);
}}
</FormDataProvider>
{/* Field reference_field for semantic_text field type */}
<ReferenceFieldCombo indexName={indexName} />
{/* Field name */}
<EuiFlexItem>
<NameParameter />
</EuiFlexItem>
</EuiFlexGroup>
);
const isAddFieldButtonDisabled = (): boolean => {
if (semanticFieldType) {
return !referenceFieldComboValue || !nameValue || !inferenceIdComboValue;
}
return false;
};
const renderFormActions = () => (
<EuiFlexGroup gutterSize="s" justifyContent="flexEnd">
{(isCancelable !== false || isAddingFields) && (
@ -160,6 +223,7 @@ export const CreateField = React.memo(function CreateFieldComponent({
onClick={submitForm}
type="submit"
data-test-subj="addButton"
isDisabled={isAddFieldButtonDisabled()}
>
{isMultiField
? i18n.translate('xpack.idxMgmt.mappingsEditor.createField.addMultiFieldButtonLabel', {
@ -197,10 +261,7 @@ export const CreateField = React.memo(function CreateFieldComponent({
>
<div className="mappingsEditor__createFieldContent">
<EuiFlexGroup gutterSize="s" alignItems="center" justifyContent="spaceBetween">
<EuiFlexItem className="mappingsEditor__createFieldContent__formFields">
{renderFormFields()}
</EuiFlexItem>
<EuiFlexItem grow={false}>{renderFormActions()}</EuiFlexItem>
<EuiFlexItem>{renderFormFields()}</EuiFlexItem>
</EuiFlexGroup>
<FormDataProvider pathsToWatch={['type', 'subType']}>
@ -230,9 +291,49 @@ export const CreateField = React.memo(function CreateFieldComponent({
);
}}
</FormDataProvider>
{/* Field inference_id for semantic_text field type */}
<InferenceIdCombo />
<EuiFlexGroup gutterSize="s" alignItems="center">
<EuiFlexItem grow={true} />
<EuiFlexItem grow={false}>{renderFormActions()}</EuiFlexItem>
</EuiFlexGroup>
</div>
</div>
</Form>
</EuiOutsideClickDetector>
);
});
function ReferenceFieldCombo({ indexName }: { indexName?: string }) {
const [{ type }] = useFormData({ watch: 'type' });
if (type === undefined || type[0]?.value !== 'semantic_text') {
return null;
}
return (
<EuiFlexItem grow={false}>
<UseField path="referenceField">
{(field) => <ReferenceFieldSelects onChange={field.setValue} indexName={indexName} />}
</UseField>
</EuiFlexItem>
);
}
function InferenceIdCombo() {
const [{ type }] = useFormData({ watch: 'type' });
if (type === undefined || type[0]?.value !== 'semantic_text') {
return null;
}
return (
<>
<EuiSpacer />
<UseField path="inferenceId">
{(field) => <InferenceIdSelects onChange={field.setValue} />}
</UseField>
</>
);
}

View file

@ -15,9 +15,16 @@ import { FieldsList, CreateField } from './fields';
interface Props {
onCancelAddingNewFields?: () => void;
isAddingFields?: boolean;
isSemanticTextEnabled?: boolean;
indexName?: string;
}
export const DocumentFieldsTreeEditor = ({ onCancelAddingNewFields, isAddingFields }: Props) => {
export const DocumentFieldsTreeEditor = ({
onCancelAddingNewFields,
isAddingFields,
isSemanticTextEnabled = false,
indexName,
}: Props) => {
const dispatch = useDispatch();
const {
fields: { byId, rootLevelFields },
@ -46,6 +53,8 @@ export const DocumentFieldsTreeEditor = ({ onCancelAddingNewFields, isAddingFiel
isRootLevelField
onCancelAddingNewFields={onCancelAddingNewFields}
isAddingFields={isAddingFields}
isSemanticTextEnabled={isSemanticTextEnabled}
indexName={indexName}
/>
);
};

View file

@ -900,6 +900,23 @@ export const TYPE_DEFINITION: { [key in DataType]: DataTypeDefinition } = {
</p>
),
},
semantic_text: {
label: i18n.translate('xpack.idxMgmt.mappingsEditor.dataType.semanticTextDescription', {
defaultMessage: 'Semantic text',
}),
value: 'semantic_text',
documentation: {
main: 'semantic-text.html',
},
description: () => (
<p>
<FormattedMessage
id="xpack.idxMgmt.mappingsEditor.dataType.semanticTextLongDescription"
defaultMessage="Semantic text fields enable semantic search using text embeddings."
/>
</p>
),
},
point: {
label: i18n.translate('xpack.idxMgmt.mappingsEditor.dataType.pointDescription', {
defaultMessage: 'Point',
@ -1007,6 +1024,7 @@ export const MAIN_TYPES: MainType[] = [
'rank_features',
'search_as_you_type',
'shape',
'semantic_text',
'sparse_vector',
'text',
'token_count',

View file

@ -1041,6 +1041,7 @@ export const PARAMETERS_DEFINITION: { [key in ParameterName]: ParameterDefinitio
},
schema: t.number,
},
dims: {
fieldConfig: {
defaultValue: '',
@ -1067,6 +1068,26 @@ export const PARAMETERS_DEFINITION: { [key in ParameterName]: ParameterDefinitio
},
schema: t.string,
},
reference_field: {
fieldConfig: {
label: i18n.translate('xpack.idxMgmt.mappingsEditor.parameters.referenceFieldLabel', {
defaultMessage: 'Reference field',
}),
helpText: i18n.translate('xpack.idxMgmt.mappingsEditor.parameters.referenceFieldHelpText', {
defaultMessage: 'Reference field for model inference.',
}),
},
schema: t.string,
},
inference_id: {
fieldConfig: {
label: i18n.translate('xpack.idxMgmt.mappingsEditor.parameters.inferenceIdLabel', {
defaultMessage: 'Select an inference endpoint:',
}),
},
schema: t.string,
},
relations: {
fieldConfig: {
defaultValue: [] as any, // Needed for FieldParams typing

View file

@ -5,18 +5,25 @@
* 2.0.
*/
import { Field, NormalizedFields, NormalizedField, State, Action } from './types';
import {
getFieldMeta,
getUniqueId,
shouldDeleteChildFieldsAfterTypeChange,
getAllChildFields,
getMaxNestedDepth,
normalize,
updateFieldsPathAfterFieldNameChange,
searchFields,
} from './lib';
import { PARAMETERS_DEFINITION } from './constants';
import {
getAllChildFields,
getFieldMeta,
getMaxNestedDepth,
getUniqueId,
normalize,
searchFields,
shouldDeleteChildFieldsAfterTypeChange,
updateFieldsPathAfterFieldNameChange,
} from './lib';
import {
Action,
Field,
FieldWithSemanticTextInfo,
NormalizedField,
NormalizedFields,
State,
} from './types';
export const addFieldToState = (field: Field, state: State): State => {
const updatedFields = { ...state.fields };
@ -316,6 +323,28 @@ export const reducer = (state: State, action: Action): State => {
case 'field.add': {
return addFieldToState(action.value, state);
}
case 'field.addSemanticText': {
const addTexFieldWithCopyToActionValue: FieldWithSemanticTextInfo = {
name: action.value.referenceField as string,
type: 'text',
copy_to: [action.value.name],
};
// Add text field to state with copy_to of semantic_text field
let updatedState = addFieldToState(addTexFieldWithCopyToActionValue, state);
const addSemanticTextFieldActionValue: FieldWithSemanticTextInfo = {
name: action.value.name,
inference_id: action.value.inferenceId,
type: 'semantic_text',
};
// Add semantic_text field to state and reset fieldToAddFieldTo
updatedState = addFieldToState(addSemanticTextFieldActionValue, updatedState);
updatedState.documentFields.fieldToAddFieldTo = undefined;
return updatedState;
}
case 'field.remove': {
const field = state.fields.byId[action.value];
const { id, hasChildFields, hasMultiFields } = field;

View file

@ -59,6 +59,7 @@ export type MainType =
| 'shape'
| 'search_as_you_type'
| 'sparse_vector'
| 'semantic_text'
| 'date'
| 'date_nanos'
| 'geo_point'
@ -155,6 +156,8 @@ export type ParameterName =
| 'points_only'
| 'path'
| 'dims'
| 'inference_id'
| 'reference_field'
| 'depth_limit'
| 'relations'
| 'max_shingle_size'
@ -192,6 +195,11 @@ type FieldParams = {
export type Field = FieldBasic & Partial<FieldParams>;
export interface FieldWithSemanticTextInfo extends Field {
referenceField?: string;
inferenceId?: string;
}
export interface FieldMeta {
childFieldsName: ChildFieldName | undefined;
canHaveChildFields: boolean;

View file

@ -8,6 +8,7 @@
import { FormHook, OnFormUpdateArg, RuntimeField } from '../shared_imports';
import {
Field,
FieldWithSemanticTextInfo,
NormalizedFields,
NormalizedRuntimeField,
NormalizedRuntimeFields,
@ -107,6 +108,10 @@ export type Action =
| { type: 'templates.save'; value: MappingsTemplates }
| { type: 'fieldForm.update'; value: OnFormUpdateArg<any> }
| { type: 'field.add'; value: Field }
| {
type: 'field.addSemanticText';
value: FieldWithSemanticTextInfo;
}
| { type: 'field.remove'; value: string }
| { type: 'field.edit'; value: Field }
| { type: 'field.toggleExpand'; value: { fieldId: string; isExpanded?: boolean } }

View file

@ -70,7 +70,15 @@ export const DetailsPageMappingsContent: FunctionComponent<{
showAboutMappings: boolean;
jsonData: any;
refetchMapping: () => void;
}> = ({ index, data, jsonData, refetchMapping, showAboutMappings }) => {
isSemanticTextEnabled?: boolean;
}> = ({
index,
data,
jsonData,
refetchMapping,
showAboutMappings,
isSemanticTextEnabled = false,
}) => {
const {
services: { extensionsService },
core: { getUrlForApp },
@ -481,9 +489,15 @@ export const DetailsPageMappingsContent: FunctionComponent<{
<DocumentFields
onCancelAddingNewFields={onCancelAddingNewFields}
isAddingFields={isAddingFields}
isSemanticTextEnabled={isSemanticTextEnabled}
indexName={indexName}
/>
) : (
<DocumentFields isAddingFields={isAddingFields} />
<DocumentFields
isAddingFields={isAddingFields}
isSemanticTextEnabled={isSemanticTextEnabled}
indexName={indexName}
/>
)}
</EuiPanel>
</EuiAccordion>

View file

@ -0,0 +1,14 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { RouteDependencies } from '../../../types';
import { registerGetAllRoute } from './register_get_route';
export function registerInferenceModelRoutes(dependencies: RouteDependencies) {
registerGetAllRoute(dependencies);
}

View file

@ -0,0 +1,39 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { InferenceAPIConfigResponse } from '@kbn/ml-trained-models-utils';
import { addBasePath } from '..';
import { RouteDependencies } from '../../../types';
export function registerGetAllRoute({ router, lib: { handleEsError } }: RouteDependencies) {
// Get all inference models
router.get(
{
path: addBasePath('/inference/all'),
validate: {},
},
async (context, request, response) => {
const { client } = (await context.core).elasticsearch;
// TODO: Use the client's built-in function rather than the transport when it's available
try {
const { models } = await client.asCurrentUser.transport.request<{
models: InferenceAPIConfigResponse[];
}>({
method: 'GET',
path: `/_inference/_all`,
});
return response.ok({
body: models,
});
} catch (error) {
return handleEsError({ error, response });
}
}
);
}

View file

@ -7,15 +7,16 @@
import { RouteDependencies } from '../types';
import { registerComponentTemplateRoutes } from './api/component_templates';
import { registerDataStreamRoutes } from './api/data_streams';
import { registerEnrichPoliciesRoute } from './api/enrich_policies';
import { registerIndicesRoutes } from './api/indices';
import { registerTemplateRoutes } from './api/templates';
import { registerInferenceModelRoutes } from './api/inference_models';
import { registerIndexMappingRoutes } from './api/mapping/register_index_mapping_route';
import { registerNodesRoute } from './api/nodes';
import { registerSettingsRoutes } from './api/settings';
import { registerStatsRoute } from './api/stats';
import { registerComponentTemplateRoutes } from './api/component_templates';
import { registerNodesRoute } from './api/nodes';
import { registerEnrichPoliciesRoute } from './api/enrich_policies';
import { registerIndexMappingRoutes } from './api/mapping/register_index_mapping_route';
import { registerTemplateRoutes } from './api/templates';
export class ApiRoutes {
setup(dependencies: RouteDependencies) {
@ -25,6 +26,7 @@ export class ApiRoutes {
registerSettingsRoutes(dependencies);
registerIndexMappingRoutes(dependencies);
registerComponentTemplateRoutes(dependencies);
registerInferenceModelRoutes(dependencies);
registerNodesRoute(dependencies);
registerEnrichPoliciesRoute(dependencies);

View file

@ -1,7 +1,7 @@
{
"extends": "../../../tsconfig.base.json",
"compilerOptions": {
"outDir": "target/types",
"outDir": "target/types"
},
"include": [
"__jest__/**/*",
@ -9,7 +9,7 @@
"public/**/*",
"server/**/*",
"test/**/*",
"../../../typings/**/*",
"../../../typings/**/*"
],
"kbn_references": [
"@kbn/core",
@ -24,6 +24,7 @@
"@kbn/runtime-fields-plugin",
"@kbn/test-jest-helpers",
"@kbn/i18n",
"@kbn/ml-trained-models-utils",
"@kbn/analytics",
"@kbn/utility-types",
"@kbn/i18n-react",
@ -45,9 +46,7 @@
"@kbn/code-editor",
"@kbn/monaco",
"@kbn/console-plugin",
"@kbn/shared-ux-utility",
"@kbn/shared-ux-utility"
],
"exclude": [
"target/**/*",
]
"exclude": ["target/**/*"]
}

View file

@ -12,6 +12,7 @@ import type {
TotalFeatureImportance,
} from '@kbn/ml-data-frame-analytics-utils';
import type { IndexName, IndicesIndexState } from '@elastic/elasticsearch/lib/api/types';
import type { InferenceAPIConfigResponse } from '@kbn/ml-trained-models-utils';
import type { XOR } from './common';
import type { MlSavedObjectType } from './saved_objects';
@ -128,40 +129,6 @@ export interface PipelineDefinition {
description?: string;
}
export type InferenceServiceSettings =
| {
service: 'elser';
service_settings: {
num_allocations: number;
num_threads: number;
model_id: string;
};
}
| {
service: 'openai';
service_settings: {
api_key: string;
organization_id: string;
url: string;
};
}
| {
service: 'hugging_face';
service_settings: {
api_key: string;
url: string;
};
};
export type InferenceAPIConfigResponse = {
// Refers to a deployment id
model_id: string;
task_type: 'sparse_embedding' | 'text_embedding';
task_settings: {
model?: string;
};
} & InferenceServiceSettings;
export interface ModelPipelines {
model_id: string;
pipelines: Record<string, PipelineDefinition>;

View file

@ -16,7 +16,7 @@ import {
EuiTitle,
} from '@elastic/eui';
import { FormattedMessage } from '@kbn/i18n-react';
import type { InferenceAPIConfigResponse } from '../../../common/types/trained_models';
import type { InferenceAPIConfigResponse } from '@kbn/ml-trained-models-utils';
export interface InferenceAPITabProps {
inferenceApis: InferenceAPIConfigResponse[];

View file

@ -10,7 +10,7 @@ import { groupBy } from 'lodash';
import { schema } from '@kbn/config-schema';
import type { ErrorType } from '@kbn/ml-error-utils';
import type { CloudSetup } from '@kbn/cloud-plugin/server';
import type { ElserVersion } from '@kbn/ml-trained-models-utils';
import type { ElserVersion, InferenceAPIConfigResponse } from '@kbn/ml-trained-models-utils';
import { isDefined } from '@kbn/ml-is-defined';
import type { IScopedClusterClient } from '@kbn/core-elasticsearch-server';
import { type MlFeatures, ML_INTERNAL_BASE_PATH } from '../../common/constants/app';
@ -31,10 +31,7 @@ import {
createIngestPipelineSchema,
modelDownloadsQuery,
} from './schemas/inference_schema';
import type {
InferenceAPIConfigResponse,
PipelineDefinition,
} from '../../common/types/trained_models';
import type { PipelineDefinition } from '../../common/types/trained_models';
import { type TrainedModelConfigResponse } from '../../common/types/trained_models';
import { mlLog } from '../lib/log';
import { forceQuerySchema } from './schemas/anomaly_detectors_schema';

View file

@ -20583,7 +20583,6 @@
"xpack.idxMgmt.mappingsEditor.tokenCount.nullValueFieldDescription": "Accepte une valeur numérique du même type que le champ qui remplace une valeur nulle explicite.",
"xpack.idxMgmt.mappingsEditor.tokenCountRequired.analyzerFieldLabel": "Analyseur d'index",
"xpack.idxMgmt.mappingsEditor.typeField.placeholderLabel": "Sélectionner un type",
"xpack.idxMgmt.mappingsEditor.typeFieldLabel": "Type du champ",
"xpack.idxMgmt.mappingsEditor.updateField.confirmationModal.confirmDescription": "Confirmer le changement de type",
"xpack.idxMgmt.mappingsEditor.updateField.confirmationModal.title": "Confirmez le changement du type \"{fieldName}\" en \"{fieldType}\".",
"xpack.idxMgmt.mappingsEditor.useNormsFieldDescription": "Tenez compte de la longueur du champ lors de la notation des requêtes. Les normes nécessitent une mémoire importante. Elles ne sont pas obligatoires pour les champs utilisés uniquement pour le filtrage ou les agrégations.",

View file

@ -20560,7 +20560,6 @@
"xpack.idxMgmt.mappingsEditor.tokenCount.nullValueFieldDescription": "明確なnull値の代わりに使用される、フィールドと同じタイプの数値を受け入れます。",
"xpack.idxMgmt.mappingsEditor.tokenCountRequired.analyzerFieldLabel": "インデックスアナライザー",
"xpack.idxMgmt.mappingsEditor.typeField.placeholderLabel": "タイプを選択",
"xpack.idxMgmt.mappingsEditor.typeFieldLabel": "フィールド型",
"xpack.idxMgmt.mappingsEditor.updateField.confirmationModal.confirmDescription": "型の変更を確認",
"xpack.idxMgmt.mappingsEditor.updateField.confirmationModal.title": "'{fieldName}'型から'{fieldType}'への変更を確認",
"xpack.idxMgmt.mappingsEditor.useNormsFieldDescription": "クエリをスコアリングするときにフィールドの長さを考慮します。Normsには大量のメモリが必要です。フィルタリングまたは集約専用のフィールドは必要ありません。",

View file

@ -20589,7 +20589,6 @@
"xpack.idxMgmt.mappingsEditor.tokenCount.nullValueFieldDescription": "接受类型与替换任何显式 null 值的字段相同的数值。",
"xpack.idxMgmt.mappingsEditor.tokenCountRequired.analyzerFieldLabel": "索引分析器",
"xpack.idxMgmt.mappingsEditor.typeField.placeholderLabel": "选择类型",
"xpack.idxMgmt.mappingsEditor.typeFieldLabel": "字段类型",
"xpack.idxMgmt.mappingsEditor.updateField.confirmationModal.confirmDescription": "确认类型更改",
"xpack.idxMgmt.mappingsEditor.updateField.confirmationModal.title": "确认将“{fieldName}”类型更改为“{fieldType}”。",
"xpack.idxMgmt.mappingsEditor.useNormsFieldDescription": "对查询评分时解释字段长度。Norms 需要很大的内存,对于仅用于筛选或聚合的字段,其不是必需的。",