[8.x] [ML] File upload adding deployment initialization step (#198446) (#199741)

# Backport

This will backport the following commits from `main` to `8.x`:
- [[ML] File upload adding deployment initialization step
(#198446)](https://github.com/elastic/kibana/pull/198446)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"James
Gowdy","email":"jgowdy@elastic.co"},"sourceCommit":{"committedDate":"2024-11-12T10:02:26Z","message":"[ML]
File upload adding deployment initialization step (#198446)\n\nFixes
https://github.com/elastic/kibana/issues/196696\r\n\r\nWhen adding a
semantic text field, we now have an additional step in the\r\nfile
uploading process which calls inference for the selected
inference\r\nendpoint.\r\nThe response of the inference call is ignored
and a poll is started to\r\ncheck to see of the model has been deployed
by check to see if\r\n`num_allocations > 0`\r\nAny errors returned from
the inference call will stop the upload, unless\r\nthey are timeout
errors which are
ignored.\r\n\r\n\r\nhttps://github.com/user-attachments/assets/382ce565-3b4b-47a3-a081-d79c15aa462f","sha":"fa6d8ee9e0f6fcaa0280fbf5ab60217f9f28279f","branchLabelMapping":{"^v9.0.0$":"main","^v8.17.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:enhancement",":ml","Feature:File
and Index Data Viz","Feature:File
Upload","v9.0.0","backport:version","v8.17.0"],"title":"[ML] File upload
adding deployment initialization
step","number":198446,"url":"https://github.com/elastic/kibana/pull/198446","mergeCommit":{"message":"[ML]
File upload adding deployment initialization step (#198446)\n\nFixes
https://github.com/elastic/kibana/issues/196696\r\n\r\nWhen adding a
semantic text field, we now have an additional step in the\r\nfile
uploading process which calls inference for the selected
inference\r\nendpoint.\r\nThe response of the inference call is ignored
and a poll is started to\r\ncheck to see of the model has been deployed
by check to see if\r\n`num_allocations > 0`\r\nAny errors returned from
the inference call will stop the upload, unless\r\nthey are timeout
errors which are
ignored.\r\n\r\n\r\nhttps://github.com/user-attachments/assets/382ce565-3b4b-47a3-a081-d79c15aa462f","sha":"fa6d8ee9e0f6fcaa0280fbf5ab60217f9f28279f"}},"sourceBranch":"main","suggestedTargetBranches":["8.x"],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/198446","number":198446,"mergeCommit":{"message":"[ML]
File upload adding deployment initialization step (#198446)\n\nFixes
https://github.com/elastic/kibana/issues/196696\r\n\r\nWhen adding a
semantic text field, we now have an additional step in the\r\nfile
uploading process which calls inference for the selected
inference\r\nendpoint.\r\nThe response of the inference call is ignored
and a poll is started to\r\ncheck to see of the model has been deployed
by check to see if\r\n`num_allocations > 0`\r\nAny errors returned from
the inference call will stop the upload, unless\r\nthey are timeout
errors which are
ignored.\r\n\r\n\r\nhttps://github.com/user-attachments/assets/382ce565-3b4b-47a3-a081-d79c15aa462f","sha":"fa6d8ee9e0f6fcaa0280fbf5ab60217f9f28279f"}},{"branch":"8.x","label":"v8.17.0","branchLabelMappingKey":"^v8.17.0$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->

Co-authored-by: James Gowdy <jgowdy@elastic.co>
This commit is contained in:
Kibana Machine 2024-11-12 22:48:26 +11:00 committed by GitHub
parent 2633bed588
commit 04a6dd94a6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 300 additions and 67 deletions

View file

@ -360,9 +360,6 @@ export class FileDataVisualizerView extends Component {
fileName={fileName}
fileContents={fileContents}
data={data}
dataViewsContract={this.props.dataViewsContract}
dataStart={this.props.dataStart}
fileUpload={this.props.fileUpload}
getAdditionalLinks={this.props.getAdditionalLinks}
resultLinks={this.props.resultLinks}
capabilities={this.props.capabilities}

View file

@ -31,6 +31,8 @@ export interface Statuses {
createDataView: boolean;
createPipeline: boolean;
permissionCheckStatus: IMPORT_STATUS;
initializeDeployment: boolean;
initializeDeploymentStatus: IMPORT_STATUS;
}
export const ImportProgress: FC<{ statuses: Statuses }> = ({ statuses }) => {
@ -45,6 +47,8 @@ export const ImportProgress: FC<{ statuses: Statuses }> = ({ statuses }) => {
uploadStatus,
createDataView,
createPipeline,
initializeDeployment,
initializeDeploymentStatus,
} = statuses;
let statusInfo = null;
@ -58,28 +62,38 @@ export const ImportProgress: FC<{ statuses: Statuses }> = ({ statuses }) => {
) {
completedStep = 0;
}
if (
readStatus === IMPORT_STATUS.COMPLETE &&
indexCreatedStatus === IMPORT_STATUS.INCOMPLETE &&
ingestPipelineCreatedStatus === IMPORT_STATUS.INCOMPLETE
initializeDeployment === true &&
initializeDeploymentStatus === IMPORT_STATUS.INCOMPLETE
) {
completedStep = 1;
}
if (indexCreatedStatus === IMPORT_STATUS.COMPLETE) {
if (
readStatus === IMPORT_STATUS.COMPLETE &&
(initializeDeployment === false || initializeDeploymentStatus === IMPORT_STATUS.COMPLETE) &&
indexCreatedStatus === IMPORT_STATUS.INCOMPLETE &&
ingestPipelineCreatedStatus === IMPORT_STATUS.INCOMPLETE
) {
completedStep = 2;
}
if (indexCreatedStatus === IMPORT_STATUS.COMPLETE) {
completedStep = 3;
}
if (
ingestPipelineCreatedStatus === IMPORT_STATUS.COMPLETE ||
(createPipeline === false && indexCreatedStatus === IMPORT_STATUS.COMPLETE)
) {
completedStep = 3;
}
if (uploadStatus === IMPORT_STATUS.COMPLETE) {
completedStep = 4;
}
if (dataViewCreatedStatus === IMPORT_STATUS.COMPLETE) {
if (uploadStatus === IMPORT_STATUS.COMPLETE) {
completedStep = 5;
}
if (dataViewCreatedStatus === IMPORT_STATUS.COMPLETE) {
completedStep = 6;
}
let processFileTitle = i18n.translate(
'xpack.dataVisualizer.file.importProgress.processFileTitle',
@ -87,6 +101,12 @@ export const ImportProgress: FC<{ statuses: Statuses }> = ({ statuses }) => {
defaultMessage: 'Process file',
}
);
let initializeDeploymentTitle = i18n.translate(
'xpack.dataVisualizer.file.importProgress.initializeDeploymentTitle',
{
defaultMessage: 'Initialize model deployment',
}
);
let createIndexTitle = i18n.translate(
'xpack.dataVisualizer.file.importProgress.createIndexTitle',
{
@ -146,13 +166,43 @@ export const ImportProgress: FC<{ statuses: Statuses }> = ({ statuses }) => {
</p>
);
}
if (completedStep >= 1) {
if (initializeDeployment) {
if (completedStep >= 1) {
processFileTitle = i18n.translate(
'xpack.dataVisualizer.file.importProgress.fileProcessedTitle',
{
defaultMessage: 'File processed',
}
);
initializeDeploymentTitle = i18n.translate(
'xpack.dataVisualizer.file.importProgress.initializingDeploymentTitle',
{
defaultMessage: 'Initializing model deployment',
}
);
statusInfo = (
<p>
<FormattedMessage
id="xpack.dataVisualizer.file.importProgress.processingImportedFileDescription"
defaultMessage="Initializing model deployment"
/>
</p>
);
}
}
if (completedStep >= 2) {
processFileTitle = i18n.translate(
'xpack.dataVisualizer.file.importProgress.fileProcessedTitle',
{
defaultMessage: 'File processed',
}
);
initializeDeploymentTitle = i18n.translate(
'xpack.dataVisualizer.file.importProgress.deploymentInitializedTitle',
{
defaultMessage: 'Model deployed',
}
);
createIndexTitle = i18n.translate(
'xpack.dataVisualizer.file.importProgress.creatingIndexTitle',
{
@ -162,7 +212,7 @@ export const ImportProgress: FC<{ statuses: Statuses }> = ({ statuses }) => {
statusInfo =
createPipeline === true ? creatingIndexAndIngestPipelineStatus : creatingIndexStatus;
}
if (completedStep >= 2) {
if (completedStep >= 3) {
createIndexTitle = i18n.translate(
'xpack.dataVisualizer.file.importProgress.indexCreatedTitle',
{
@ -178,7 +228,7 @@ export const ImportProgress: FC<{ statuses: Statuses }> = ({ statuses }) => {
statusInfo =
createPipeline === true ? creatingIndexAndIngestPipelineStatus : creatingIndexStatus;
}
if (completedStep >= 3) {
if (completedStep >= 4) {
createIngestPipelineTitle = i18n.translate(
'xpack.dataVisualizer.file.importProgress.ingestPipelineCreatedTitle',
{
@ -193,7 +243,7 @@ export const ImportProgress: FC<{ statuses: Statuses }> = ({ statuses }) => {
);
statusInfo = <UploadFunctionProgress progress={uploadProgress} />;
}
if (completedStep >= 4) {
if (completedStep >= 5) {
uploadingDataTitle = i18n.translate(
'xpack.dataVisualizer.file.importProgress.dataUploadedTitle',
{
@ -219,7 +269,7 @@ export const ImportProgress: FC<{ statuses: Statuses }> = ({ statuses }) => {
statusInfo = null;
}
}
if (completedStep >= 5) {
if (completedStep >= 6) {
createDataViewTitle = i18n.translate(
'xpack.dataVisualizer.file.importProgress.dataViewCreatedTitle',
{
@ -239,44 +289,58 @@ export const ImportProgress: FC<{ statuses: Statuses }> = ({ statuses }) => {
: 'selected') as EuiStepStatus,
onClick: () => {},
},
{
title: createIndexTitle,
status: (indexCreatedStatus !== IMPORT_STATUS.INCOMPLETE // Show failure/completed states first
? indexCreatedStatus
: completedStep === 1 // Then show selected/incomplete states
? 'selected'
: 'incomplete') as EuiStepStatus,
onClick: () => {},
},
{
title: uploadingDataTitle,
status: (uploadStatus !== IMPORT_STATUS.INCOMPLETE // Show failure/completed states first
? uploadStatus
: completedStep === 3 // Then show selected/incomplete states
? 'selected'
: 'incomplete') as EuiStepStatus,
onClick: () => {},
},
];
if (createPipeline === true) {
steps.splice(2, 0, {
title: createIngestPipelineTitle,
status: (ingestPipelineCreatedStatus !== IMPORT_STATUS.INCOMPLETE // Show failure/completed states first
? ingestPipelineCreatedStatus
: completedStep === 2 // Then show selected/incomplete states
if (initializeDeployment === true) {
steps.push({
title: initializeDeploymentTitle,
status: (initializeDeploymentStatus !== IMPORT_STATUS.INCOMPLETE // Show failure/completed states first
? initializeDeploymentStatus
: completedStep === 1 // Then show selected/incomplete states
? 'selected'
: 'incomplete') as EuiStepStatus,
onClick: () => {},
});
}
steps.push({
title: createIndexTitle,
status: (indexCreatedStatus !== IMPORT_STATUS.INCOMPLETE // Show failure/completed states first
? indexCreatedStatus
: completedStep === 2 // Then show selected/incomplete states
? 'selected'
: 'incomplete') as EuiStepStatus,
onClick: () => {},
});
if (createPipeline === true) {
steps.push({
title: createIngestPipelineTitle,
status: (ingestPipelineCreatedStatus !== IMPORT_STATUS.INCOMPLETE // Show failure/completed states first
? ingestPipelineCreatedStatus
: completedStep === 3 // Then show selected/incomplete states
? 'selected'
: 'incomplete') as EuiStepStatus,
onClick: () => {},
});
}
steps.push({
title: uploadingDataTitle,
status: (uploadStatus !== IMPORT_STATUS.INCOMPLETE // Show failure/completed states first
? uploadStatus
: completedStep === 4 // Then show selected/incomplete states
? 'selected'
: 'incomplete') as EuiStepStatus,
onClick: () => {},
});
if (createDataView === true) {
steps.push({
title: createDataViewTitle,
status: (dataViewCreatedStatus !== IMPORT_STATUS.INCOMPLETE // Show failure/completed states first
? dataViewCreatedStatus
: completedStep === 4 // Then show selected/incomplete states
: completedStep === 5 // Then show selected/incomplete states
? 'selected'
: 'incomplete') as EuiStepStatus,
onClick: () => {},
@ -284,21 +348,21 @@ export const ImportProgress: FC<{ statuses: Statuses }> = ({ statuses }) => {
}
return (
<React.Fragment>
<>
<EuiStepsHorizontal steps={steps} style={{ backgroundColor: 'transparent' }} />
{statusInfo && (
<React.Fragment>
<>
<EuiSpacer size="m" />
{statusInfo}
</React.Fragment>
</>
)}
</React.Fragment>
</>
);
};
const UploadFunctionProgress: FC<{ progress: number }> = ({ progress }) => {
return (
<React.Fragment>
<>
<p>
<FormattedMessage
id="xpack.dataVisualizer.file.importProgress.uploadingDataDescription"
@ -306,11 +370,11 @@ const UploadFunctionProgress: FC<{ progress: number }> = ({ progress }) => {
/>
</p>
{progress < 100 && (
<React.Fragment>
<>
<EuiSpacer size="s" />
<EuiProgress value={progress} max={100} color="primary" size="s" />
</React.Fragment>
</>
)}
</React.Fragment>
</>
);
};

View file

@ -0,0 +1,76 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { InferenceInferenceEndpointInfo } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import type { HttpSetup } from '@kbn/core/public';
const POLL_INTERVAL = 5; // seconds
export class AutoDeploy {
private inferError: Error | null = null;
constructor(private readonly http: HttpSetup, private readonly inferenceId: string) {}
public async deploy() {
this.inferError = null;
if (await this.isDeployed()) {
return;
}
this.infer().catch((e) => {
// ignore timeout errors
// The deployment may take a long time
// we'll know when it's ready from polling the inference endpoints
// looking for num_allocations
const status = e.response?.status;
if (status === 408 || status === 504 || status === 502) {
return;
}
this.inferError = e;
});
await this.pollIsDeployed();
}
private async infer() {
return this.http.fetch<InferenceInferenceEndpointInfo[]>(
`/internal/data_visualizer/inference/${this.inferenceId}`,
{
method: 'POST',
version: '1',
body: JSON.stringify({ input: '' }),
}
);
}
private async isDeployed() {
const inferenceEndpoints = await this.http.fetch<InferenceInferenceEndpointInfo[]>(
'/internal/data_visualizer/inference_endpoints',
{
method: 'GET',
version: '1',
}
);
return inferenceEndpoints.some((endpoint) => {
return (
endpoint.inference_id === this.inferenceId && endpoint.service_settings.num_allocations > 0
);
});
}
private async pollIsDeployed() {
while (true) {
if (this.inferError !== null) {
throw this.inferError;
}
const isDeployed = await this.isDeployed();
if (isDeployed) {
// break out of the loop once we have a successful deployment
return;
}
await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL * 1000));
}
}
}

View file

@ -12,13 +12,17 @@ import type {
} from '@kbn/file-upload-plugin/common/types';
import type { FileUploadStartApi } from '@kbn/file-upload-plugin/public/api';
import { i18n } from '@kbn/i18n';
import type { HttpSetup } from '@kbn/core/public';
import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import { IMPORT_STATUS } from '../import_progress/import_progress';
import { AutoDeploy } from './auto_deploy';
interface Props {
data: ArrayBuffer;
results: FindFileStructureResponse;
dataViewsContract: DataViewsServicePublic;
fileUpload: FileUploadStartApi;
http: HttpSetup;
}
interface Config {
@ -33,7 +37,7 @@ interface Config {
}
export async function importData(props: Props, config: Config, setState: (state: unknown) => void) {
const { data, results, dataViewsContract, fileUpload } = props;
const { data, results, dataViewsContract, fileUpload, http } = props;
const {
index,
dataView,
@ -76,14 +80,6 @@ export async function importData(props: Props, config: Config, setState: (state:
return;
}
setState({
importing: true,
imported: false,
reading: true,
initialized: true,
permissionCheckStatus: IMPORT_STATUS.COMPLETE,
});
let success = true;
let settings = {};
@ -122,7 +118,15 @@ export async function importData(props: Props, config: Config, setState: (state:
errors.push(`${parseError} ${error.message}`);
}
const inferenceId = getInferenceId(mappings);
setState({
importing: true,
imported: false,
reading: true,
initialized: true,
permissionCheckStatus: IMPORT_STATUS.COMPLETE,
initializeDeployment: inferenceId !== null,
parseJSONStatus: getSuccess(success),
});
@ -147,6 +151,32 @@ export async function importData(props: Props, config: Config, setState: (state:
return;
}
if (inferenceId) {
// Initialize deployment
const autoDeploy = new AutoDeploy(http, inferenceId);
try {
await autoDeploy.deploy();
setState({
initializeDeploymentStatus: IMPORT_STATUS.COMPLETE,
});
} catch (error) {
success = false;
const deployError = i18n.translate('xpack.dataVisualizer.file.importView.deployModelError', {
defaultMessage: 'Error deploying trained model:',
});
errors.push(`${deployError} ${error.message}`);
setState({
initializeDeploymentStatus: IMPORT_STATUS.FAILED,
errors,
});
}
}
if (success === false) {
return;
}
const initializeImportResp = await importer.initializeImport(index, settings, mappings, pipeline);
const timeFieldName = importer.getTimeField();
@ -245,3 +275,12 @@ async function createKibanaDataView(
function getSuccess(success: boolean) {
return success ? IMPORT_STATUS.COMPLETE : IMPORT_STATUS.FAILED;
}
function getInferenceId(mappings: MappingTypeMapping) {
for (const value of Object.values(mappings.properties ?? {})) {
if (value.type === 'semantic_text') {
return value.inference_id;
}
}
return null;
}

View file

@ -21,6 +21,7 @@ import {
} from '@elastic/eui';
import { debounce } from 'lodash';
import { context } from '@kbn/kibana-react-plugin/public';
import { ResultsLinks } from '../../../common/components/results_links';
import { FilebeatConfigFlyout } from '../../../common/components/filebeat_config_flyout';
import { ImportProgress, IMPORT_STATUS } from '../import_progress';
@ -76,14 +77,17 @@ const DEFAULT_STATE = {
combinedFields: [],
importer: undefined,
createPipeline: true,
initializeDeployment: false,
initializeDeploymentStatus: IMPORT_STATUS.INCOMPLETE,
};
export class ImportView extends Component {
static contextType = context;
constructor(props) {
super(props);
this.state = getDefaultState(DEFAULT_STATE, this.props.results, this.props.capabilities);
this.dataViewsContract = props.dataViewsContract;
}
componentDidMount() {
@ -98,7 +102,12 @@ export class ImportView extends Component {
};
clickImport = () => {
const { data, results, dataViewsContract, fileUpload } = this.props;
const { data, results } = this.props;
const {
data: { dataViews: dataViewsContract },
fileUpload,
http,
} = this.context.services;
const {
index,
dataView,
@ -110,12 +119,13 @@ export class ImportView extends Component {
} = this.state;
const createPipeline = pipelineString !== '';
this.setState({
createPipeline,
});
importData(
{ data, results, dataViewsContract, fileUpload },
{ data, results, dataViewsContract, fileUpload, http },
{
index,
dataView,
@ -150,7 +160,7 @@ export class ImportView extends Component {
return;
}
const exists = await this.props.fileUpload.checkIndexExists(index);
const exists = await this.context.services.fileUpload.checkIndexExists(index);
const indexNameError = exists ? (
<FormattedMessage
id="xpack.dataVisualizer.file.importView.indexNameAlreadyExistsErrorMessage"
@ -231,7 +241,7 @@ export class ImportView extends Component {
async loadDataViewNames() {
try {
const dataViewNames = await this.dataViewsContract.getTitles();
const dataViewNames = await this.context.services.data.dataViews.getTitles();
this.setState({ dataViewNames });
} catch (error) {
console.error('failed to load data views', error);
@ -271,6 +281,8 @@ export class ImportView extends Component {
combinedFields,
importer,
createPipeline,
initializeDeployment,
initializeDeploymentStatus,
} = this.state;
const statuses = {
@ -285,6 +297,8 @@ export class ImportView extends Component {
uploadStatus,
createDataView,
createPipeline,
initializeDeployment,
initializeDeploymentStatus,
};
const disableImport =
@ -391,7 +405,7 @@ export class ImportView extends Component {
this.props.results.format !== FILE_FORMATS.TIKA && (
<DocCountChart
statuses={statuses}
dataStart={this.props.dataStart}
dataStart={this.context.services.data}
importer={importer}
/>
)}

View file

@ -44,7 +44,6 @@ export const FileDataVisualizer: FC<Props> = ({ getAdditionalLinks, resultLinks
<KibanaContextProvider services={{ ...services }}>
<CloudContext>
<FileDataVisualizerView
dataViewsContract={data.dataViews}
dataStart={data}
http={coreStart.http}
fileUpload={fileUpload}

View file

@ -96,9 +96,7 @@ export function routes(coreSetup: CoreSetup<StartDeps, unknown>, logger: Logger)
const filteredInferenceEndpoints = endpoints.filter((endpoint) => {
return (
(endpoint.task_type === 'sparse_embedding' ||
endpoint.task_type === 'text_embedding') &&
endpoint.service_settings.num_allocations >= 0
endpoint.task_type === 'sparse_embedding' || endpoint.task_type === 'text_embedding'
);
});
@ -108,4 +106,50 @@ export function routes(coreSetup: CoreSetup<StartDeps, unknown>, logger: Logger)
}
}
);
/**
* @apiGroup DataVisualizer
*
* @api {get} /internal/data_visualizer/inference/{inferenceId} Runs inference on a given inference endpoint with the provided input
* @apiName inference
* @apiDescription Runs inference on a given inference endpoint with the provided input.
*/
router.versioned
.post({
path: '/internal/data_visualizer/inference/{inferenceId}',
access: 'internal',
options: {
tags: ['access:fileUpload:analyzeFile'],
},
})
.addVersion(
{
version: '1',
validate: {
request: {
params: schema.object({
inferenceId: schema.string(),
}),
body: schema.object({
input: schema.string(),
}),
},
},
},
async (context, request, response) => {
try {
const inferenceId = request.params.inferenceId;
const input = request.body.input;
const esClient = (await context.core).elasticsearch.client;
const body = await esClient.asCurrentUser.inference.inference({
inference_id: inferenceId,
input,
});
return response.ok({ body });
} catch (e) {
return response.customError(wrapError(e));
}
}
);
}