mirror of
https://github.com/elastic/kibana.git
synced 2025-04-23 09:19:04 -04:00
Use default elser deployment for product documentation (#204760)
## Summary Fix https://github.com/elastic/kibana/issues/204559 Use the default ELSER deployment (`.elser-2-elasticsearch`) for the product documentation semantic_text fields instead of maintaining our own custom deployment. --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Pierre Gayvallet
GitHub
parent
ad3b9880c7
commit
015911d2bb
17 changed files with 46 additions and 262 deletions
|
|
@ -104,3 +104,4 @@ export {
|
|||
isSupportedConnector,
|
||||
type InferenceConnector,
|
||||
} from './src/connectors';
|
||||
export { defaultInferenceEndpoints } from './src/inference_endpoints';
|
||||
|
|
|
|||
|
|
@ -5,6 +5,10 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
export { waitUntilModelDeployed } from './wait_until_model_deployed';
|
||||
export { getModelInstallStatus } from './get_model_install_status';
|
||||
export { installElser } from './install_elser';
|
||||
/**
|
||||
* Constants for all default (preconfigured) inference endpoints.
|
||||
*/
|
||||
export const defaultInferenceEndpoints = {
|
||||
ELSER: '.elser-2-elasticsearch',
|
||||
MULTILINGUAL_E5_SMALL: '.multilingual-e5-small-elasticsearch',
|
||||
};
|
||||
|
|
@ -5,10 +5,11 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import { defaultInferenceEndpoints } from '@kbn/inference-common';
|
||||
|
||||
export const productDocInstallStatusSavedObjectTypeName = 'product-doc-install-status';
|
||||
|
||||
/**
|
||||
* The id of the inference endpoint we're creating for our product doc indices.
|
||||
* Could be replaced with the default elser 2 endpoint once the default endpoint feature is available.
|
||||
*/
|
||||
export const internalElserInferenceId = 'kibana-internal-elser2';
|
||||
export const internalElserInferenceId = defaultInferenceEndpoints.ELSER;
|
||||
|
|
|
|||
|
|
@ -21,7 +21,6 @@ import {
|
|||
} from './types';
|
||||
import { productDocInstallStatusSavedObjectType } from './saved_objects';
|
||||
import { PackageInstaller } from './services/package_installer';
|
||||
import { InferenceEndpointManager } from './services/inference_endpoint';
|
||||
import { ProductDocInstallClient } from './services/doc_install_status';
|
||||
import { DocumentationManager } from './services/doc_manager';
|
||||
import { SearchService } from './services/search';
|
||||
|
|
@ -79,15 +78,9 @@ export class ProductDocBasePlugin
|
|||
);
|
||||
const productDocClient = new ProductDocInstallClient({ soClient });
|
||||
|
||||
const endpointManager = new InferenceEndpointManager({
|
||||
esClient: core.elasticsearch.client.asInternalUser,
|
||||
logger: this.logger.get('endpoint-manager'),
|
||||
});
|
||||
|
||||
const packageInstaller = new PackageInstaller({
|
||||
esClient: core.elasticsearch.client.asInternalUser,
|
||||
productDocClient,
|
||||
endpointManager,
|
||||
kibanaVersion: this.context.env.packageInfo.version,
|
||||
artifactsFolder: Path.join(getDataPath(), 'ai-kb-artifacts'),
|
||||
artifactRepositoryUrl: this.context.config.get().artifactRepositoryUrl,
|
||||
|
|
|
|||
|
|
@ -1,58 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { loggerMock, type MockedLogger } from '@kbn/logging-mocks';
|
||||
import { elasticsearchServiceMock } from '@kbn/core/server/mocks';
|
||||
import { InferenceEndpointManager } from './endpoint_manager';
|
||||
|
||||
jest.mock('./utils');
|
||||
import { installElser, getModelInstallStatus, waitUntilModelDeployed } from './utils';
|
||||
const installElserMock = installElser as jest.MockedFn<typeof installElser>;
|
||||
const getModelInstallStatusMock = getModelInstallStatus as jest.MockedFn<
|
||||
typeof getModelInstallStatus
|
||||
>;
|
||||
const waitUntilModelDeployedMock = waitUntilModelDeployed as jest.MockedFn<
|
||||
typeof waitUntilModelDeployed
|
||||
>;
|
||||
|
||||
describe('InferenceEndpointManager', () => {
|
||||
let logger: MockedLogger;
|
||||
let esClient: ReturnType<typeof elasticsearchServiceMock.createElasticsearchClient>;
|
||||
let endpointManager: InferenceEndpointManager;
|
||||
|
||||
beforeEach(() => {
|
||||
logger = loggerMock.create();
|
||||
esClient = elasticsearchServiceMock.createElasticsearchClient();
|
||||
|
||||
endpointManager = new InferenceEndpointManager({ esClient, logger });
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
installElserMock.mockReset();
|
||||
getModelInstallStatusMock.mockReset();
|
||||
waitUntilModelDeployedMock.mockReset();
|
||||
});
|
||||
|
||||
describe('#ensureInternalElserInstalled', () => {
|
||||
it('installs ELSER if not already installed', async () => {
|
||||
getModelInstallStatusMock.mockResolvedValue({ installed: true });
|
||||
|
||||
await endpointManager.ensureInternalElserInstalled();
|
||||
|
||||
expect(installElserMock).not.toHaveBeenCalled();
|
||||
expect(waitUntilModelDeployedMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
it('does not install ELSER if already present', async () => {
|
||||
getModelInstallStatusMock.mockResolvedValue({ installed: false });
|
||||
|
||||
await endpointManager.ensureInternalElserInstalled();
|
||||
|
||||
expect(installElserMock).toHaveBeenCalledTimes(1);
|
||||
expect(waitUntilModelDeployedMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -1,41 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { ElasticsearchClient, Logger } from '@kbn/core/server';
|
||||
import { internalElserInferenceId } from '../../../common/consts';
|
||||
import { installElser, getModelInstallStatus, waitUntilModelDeployed } from './utils';
|
||||
|
||||
export class InferenceEndpointManager {
|
||||
private readonly log: Logger;
|
||||
private readonly esClient: ElasticsearchClient;
|
||||
|
||||
constructor({ logger, esClient }: { logger: Logger; esClient: ElasticsearchClient }) {
|
||||
this.log = logger;
|
||||
this.esClient = esClient;
|
||||
}
|
||||
|
||||
async ensureInternalElserInstalled() {
|
||||
const { installed } = await getModelInstallStatus({
|
||||
inferenceId: internalElserInferenceId,
|
||||
client: this.esClient,
|
||||
log: this.log,
|
||||
});
|
||||
if (!installed) {
|
||||
await installElser({
|
||||
inferenceId: internalElserInferenceId,
|
||||
client: this.esClient,
|
||||
log: this.log,
|
||||
});
|
||||
}
|
||||
|
||||
await waitUntilModelDeployed({
|
||||
modelId: internalElserInferenceId,
|
||||
client: this.esClient,
|
||||
log: this.log,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
export { InferenceEndpointManager } from './endpoint_manager';
|
||||
|
|
@ -1,20 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { InferenceEndpointManager } from './endpoint_manager';
|
||||
|
||||
export type InferenceEndpointManagerMock = jest.Mocked<InferenceEndpointManager>;
|
||||
|
||||
const createMock = (): InferenceEndpointManagerMock => {
|
||||
return {
|
||||
ensureInternalElserInstalled: jest.fn(),
|
||||
} as unknown as InferenceEndpointManagerMock;
|
||||
};
|
||||
|
||||
export const inferenceManagerMock = {
|
||||
create: createMock,
|
||||
};
|
||||
|
|
@ -1,34 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { InferenceTaskType } from '@elastic/elasticsearch/lib/api/types';
|
||||
import type { ElasticsearchClient, Logger } from '@kbn/core/server';
|
||||
|
||||
export const getModelInstallStatus = async ({
|
||||
inferenceId,
|
||||
taskType = 'sparse_embedding',
|
||||
client,
|
||||
}: {
|
||||
inferenceId: string;
|
||||
taskType?: InferenceTaskType;
|
||||
client: ElasticsearchClient;
|
||||
log: Logger;
|
||||
}) => {
|
||||
const getInferenceRes = await client.inference.get(
|
||||
{
|
||||
task_type: taskType,
|
||||
inference_id: inferenceId,
|
||||
},
|
||||
{ ignore: [404] }
|
||||
);
|
||||
|
||||
const installed = (getInferenceRes.endpoints ?? []).some(
|
||||
(endpoint) => endpoint.inference_id === inferenceId
|
||||
);
|
||||
|
||||
return { installed };
|
||||
};
|
||||
|
|
@ -1,35 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { ElasticsearchClient, Logger } from '@kbn/core/server';
|
||||
|
||||
export const installElser = async ({
|
||||
inferenceId,
|
||||
client,
|
||||
log,
|
||||
}: {
|
||||
inferenceId: string;
|
||||
client: ElasticsearchClient;
|
||||
log: Logger;
|
||||
}) => {
|
||||
await client.inference.put(
|
||||
{
|
||||
task_type: 'sparse_embedding',
|
||||
inference_id: inferenceId,
|
||||
inference_config: {
|
||||
service: 'elasticsearch',
|
||||
service_settings: {
|
||||
adaptive_allocations: { enabled: true },
|
||||
num_threads: 1,
|
||||
model_id: '.elser_model_2',
|
||||
},
|
||||
task_settings: {},
|
||||
},
|
||||
},
|
||||
{ requestTimeout: 5 * 60 * 1000 }
|
||||
);
|
||||
};
|
||||
|
|
@ -1,39 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { ElasticsearchClient, Logger } from '@kbn/core/server';
|
||||
|
||||
export const waitUntilModelDeployed = async ({
|
||||
modelId,
|
||||
client,
|
||||
log,
|
||||
maxRetries = 20,
|
||||
delay = 2000,
|
||||
}: {
|
||||
modelId: string;
|
||||
client: ElasticsearchClient;
|
||||
log: Logger;
|
||||
maxRetries?: number;
|
||||
delay?: number;
|
||||
}) => {
|
||||
for (let i = 0; i < maxRetries; i++) {
|
||||
const statsRes = await client.ml.getTrainedModelsStats({
|
||||
model_id: modelId,
|
||||
});
|
||||
const deploymentStats = statsRes.trained_model_stats[0]?.deployment_stats;
|
||||
if (!deploymentStats || deploymentStats.nodes.length === 0) {
|
||||
log.debug(`ML model [${modelId}] was not deployed - attempt ${i + 1} of ${maxRetries}`);
|
||||
await sleep(delay);
|
||||
continue;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
throw new Error(`Timeout waiting for ML model ${modelId} to be deployed`);
|
||||
};
|
||||
|
||||
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
|
@ -24,6 +24,7 @@ jest.doMock('./steps', () => {
|
|||
export const downloadToDiskMock = jest.fn();
|
||||
export const openZipArchiveMock = jest.fn();
|
||||
export const loadMappingFileMock = jest.fn();
|
||||
export const ensureDefaultElserDeployedMock = jest.fn();
|
||||
|
||||
jest.doMock('./utils', () => {
|
||||
const actual = jest.requireActual('./utils');
|
||||
|
|
@ -32,5 +33,6 @@ jest.doMock('./utils', () => {
|
|||
downloadToDisk: downloadToDiskMock,
|
||||
openZipArchive: openZipArchiveMock,
|
||||
loadMappingFile: loadMappingFileMock,
|
||||
ensureDefaultElserDeployed: ensureDefaultElserDeployedMock,
|
||||
};
|
||||
});
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ import {
|
|||
openZipArchiveMock,
|
||||
validateArtifactArchiveMock,
|
||||
fetchArtifactVersionsMock,
|
||||
ensureDefaultElserDeployedMock,
|
||||
} from './package_installer.test.mocks';
|
||||
|
||||
import {
|
||||
|
|
@ -24,7 +25,6 @@ import {
|
|||
import { elasticsearchServiceMock } from '@kbn/core/server/mocks';
|
||||
import { loggerMock, type MockedLogger } from '@kbn/logging-mocks';
|
||||
import { installClientMock } from '../doc_install_status/service.mock';
|
||||
import { inferenceManagerMock } from '../inference_endpoint/service.mock';
|
||||
import type { ProductInstallState } from '../../../common/install_status';
|
||||
import { PackageInstaller } from './package_installer';
|
||||
|
||||
|
|
@ -40,7 +40,6 @@ describe('PackageInstaller', () => {
|
|||
let logger: MockedLogger;
|
||||
let esClient: ReturnType<typeof elasticsearchServiceMock.createElasticsearchClient>;
|
||||
let productDocClient: ReturnType<typeof installClientMock.create>;
|
||||
let endpointManager: ReturnType<typeof inferenceManagerMock.create>;
|
||||
|
||||
let packageInstaller: PackageInstaller;
|
||||
|
||||
|
|
@ -48,13 +47,11 @@ describe('PackageInstaller', () => {
|
|||
logger = loggerMock.create();
|
||||
esClient = elasticsearchServiceMock.createElasticsearchClient();
|
||||
productDocClient = installClientMock.create();
|
||||
endpointManager = inferenceManagerMock.create();
|
||||
packageInstaller = new PackageInstaller({
|
||||
artifactsFolder,
|
||||
logger,
|
||||
esClient,
|
||||
productDocClient,
|
||||
endpointManager,
|
||||
artifactRepositoryUrl,
|
||||
kibanaVersion,
|
||||
});
|
||||
|
|
@ -68,6 +65,7 @@ describe('PackageInstaller', () => {
|
|||
openZipArchiveMock.mockReset();
|
||||
validateArtifactArchiveMock.mockReset();
|
||||
fetchArtifactVersionsMock.mockReset();
|
||||
ensureDefaultElserDeployedMock.mockReset();
|
||||
});
|
||||
|
||||
describe('installPackage', () => {
|
||||
|
|
@ -87,7 +85,7 @@ describe('PackageInstaller', () => {
|
|||
productVersion: '8.16',
|
||||
});
|
||||
const indexName = getProductDocIndexName('kibana');
|
||||
expect(endpointManager.ensureInternalElserInstalled).toHaveBeenCalledTimes(1);
|
||||
expect(ensureDefaultElserDeployedMock).toHaveBeenCalledTimes(1);
|
||||
|
||||
expect(downloadToDiskMock).toHaveBeenCalledTimes(1);
|
||||
expect(downloadToDiskMock).toHaveBeenCalledWith(
|
||||
|
|
@ -128,9 +126,7 @@ describe('PackageInstaller', () => {
|
|||
it('executes the steps in the right order', async () => {
|
||||
await packageInstaller.installPackage({ productName: 'kibana', productVersion: '8.16' });
|
||||
|
||||
expect(callOrder(endpointManager.ensureInternalElserInstalled)).toBeLessThan(
|
||||
callOrder(downloadToDiskMock)
|
||||
);
|
||||
expect(callOrder(ensureDefaultElserDeployedMock)).toBeLessThan(callOrder(downloadToDiskMock));
|
||||
expect(callOrder(downloadToDiskMock)).toBeLessThan(callOrder(openZipArchiveMock));
|
||||
expect(callOrder(openZipArchiveMock)).toBeLessThan(callOrder(loadMappingFileMock));
|
||||
expect(callOrder(loadMappingFileMock)).toBeLessThan(callOrder(createIndexMock));
|
||||
|
|
|
|||
|
|
@ -14,8 +14,13 @@ import {
|
|||
type ProductName,
|
||||
} from '@kbn/product-doc-common';
|
||||
import type { ProductDocInstallClient } from '../doc_install_status';
|
||||
import type { InferenceEndpointManager } from '../inference_endpoint';
|
||||
import { downloadToDisk, openZipArchive, loadMappingFile, type ZipArchive } from './utils';
|
||||
import {
|
||||
downloadToDisk,
|
||||
openZipArchive,
|
||||
loadMappingFile,
|
||||
ensureDefaultElserDeployed,
|
||||
type ZipArchive,
|
||||
} from './utils';
|
||||
import { majorMinor, latestVersion } from './utils/semver';
|
||||
import {
|
||||
validateArtifactArchive,
|
||||
|
|
@ -29,7 +34,6 @@ interface PackageInstallerOpts {
|
|||
logger: Logger;
|
||||
esClient: ElasticsearchClient;
|
||||
productDocClient: ProductDocInstallClient;
|
||||
endpointManager: InferenceEndpointManager;
|
||||
artifactRepositoryUrl: string;
|
||||
kibanaVersion: string;
|
||||
}
|
||||
|
|
@ -39,7 +43,6 @@ export class PackageInstaller {
|
|||
private readonly artifactsFolder: string;
|
||||
private readonly esClient: ElasticsearchClient;
|
||||
private readonly productDocClient: ProductDocInstallClient;
|
||||
private readonly endpointManager: InferenceEndpointManager;
|
||||
private readonly artifactRepositoryUrl: string;
|
||||
private readonly currentVersion: string;
|
||||
|
||||
|
|
@ -48,14 +51,12 @@ export class PackageInstaller {
|
|||
logger,
|
||||
esClient,
|
||||
productDocClient,
|
||||
endpointManager,
|
||||
artifactRepositoryUrl,
|
||||
kibanaVersion,
|
||||
}: PackageInstallerOpts) {
|
||||
this.esClient = esClient;
|
||||
this.productDocClient = productDocClient;
|
||||
this.artifactsFolder = artifactsFolder;
|
||||
this.endpointManager = endpointManager;
|
||||
this.artifactRepositoryUrl = artifactRepositoryUrl;
|
||||
this.currentVersion = majorMinor(kibanaVersion);
|
||||
this.log = logger;
|
||||
|
|
@ -144,7 +145,7 @@ export class PackageInstaller {
|
|||
productVersion,
|
||||
});
|
||||
|
||||
await this.endpointManager.ensureInternalElserInstalled();
|
||||
await ensureDefaultElserDeployed({ client: this.esClient });
|
||||
|
||||
const artifactFileName = getArtifactName({ productName, productVersion });
|
||||
const artifactUrl = `${this.artifactRepositoryUrl}/${artifactFileName}`;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,19 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { ElasticsearchClient } from '@kbn/core/server';
|
||||
import { defaultInferenceEndpoints } from '@kbn/inference-common';
|
||||
|
||||
export const ensureDefaultElserDeployed = async ({ client }: { client: ElasticsearchClient }) => {
|
||||
await client.inference.inference(
|
||||
{
|
||||
inference_id: defaultInferenceEndpoints.ELSER,
|
||||
input: 'I just want to call the API to force the model to download and allocate',
|
||||
},
|
||||
{ requestTimeout: 10 * 60 * 1000 }
|
||||
);
|
||||
};
|
||||
|
|
@ -8,3 +8,4 @@
|
|||
export { downloadToDisk } from './download';
|
||||
export { openZipArchive, type ZipArchive } from './zip_archive';
|
||||
export { loadManifestFile, loadMappingFile } from './archive_accessors';
|
||||
export { ensureDefaultElserDeployed } from './ensure_default_elser_deployed';
|
||||
|
|
|
|||
|
|
@ -25,5 +25,6 @@
|
|||
"@kbn/logging-mocks",
|
||||
"@kbn/licensing-plugin",
|
||||
"@kbn/task-manager-plugin",
|
||||
"@kbn/inference-common",
|
||||
]
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue