mirror of
https://github.com/elastic/kibana.git
synced 2025-04-23 17:28:26 -04:00
[LLM tasks] Add product documentation retrieval task (#194379)
## Summary Close https://github.com/elastic/kibana/issues/193473 Close https://github.com/elastic/kibana/issues/193474 This PR utilize the documentation packages that are build via the tool introduced by https://github.com/elastic/kibana/pull/193847, allowing to install them in Kibana and expose documentation retrieval as an LLM task that AI assistants (or other consumers) can call. Users can now decide to install the Elastic documentation from the assistant's config screen, which will expose a new tool for the assistant, `retrieve_documentation` (only implemented for the o11y assistant in the current PR, shall be done for security as a follow up). For more information, please refer to the self-review. ## General architecture <img width="1118" alt="Screenshot 2024-10-17 at 09 22 32" src="https://github.com/user-attachments/assets/3df8c30a-9ccc-49ab-92ce-c204b96d6fc4"> ## What this PR does Adds two plugin: - `productDocBase`: contains all the logic related to product documentation installation, status, and search. This is meant to be a "low level" components only responsible for this specific part. - `llmTasks`: an higher level plugin that will contain various LLM tasks to be used by assistants and genAI consumers. The intent is not to have a single place to put all llm tasks, but more to have a default place where we can introduce new tasks from. (fwiw, the `nlToEsql` task will probably be moved to that plugin). - Add a `retrieve_documentation` tool registration for the o11y assistant - Add a component on the o11y assistant configuration page to install the product doc (wiring the feature to the o11y assistant was done for testing purposes mostly, any addition / changes / enhancement should be done by the owning team - either in this PR or as a follow-up) ## What is NOT included in this PR: - Wire product base feature to the security assistant (should be done by the owning team as a follow-up) - installation - utilization as tool - FTR tests: this is somewhat blocked by the same things we need to figure out for https://github.com/elastic/kibana-team/issues/1271 ## Screenshots ### Installation from o11y assistant configuration page <img width="1476" alt="Screenshot 2024-10-17 at 09 41 24" src="https://github.com/user-attachments/assets/31daa585-9fb2-400a-a2d1-5917a262367a"> ### Example of output #### Without product documentation installed <img width="739" alt="Screenshot 2024-10-10 at 09 59 41" src="https://github.com/user-attachments/assets/993fb216-6c9a-433f-bf44-f6e383d20d9d"> #### With product documentation installed <img width="718" alt="Screenshot 2024-10-10 at 09 55 38" src="https://github.com/user-attachments/assets/805ea4ca-8bc9-4355-a434-0ba81f8228a9"> --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Alex Szabo <alex.szabo@elastic.co> Co-authored-by: Matthias Wilhelm <matthias.wilhelm@elastic.co> Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
This commit is contained in:
Pierre Gayvallet
GitHub
parent
8352b86f59
commit
455c781c6d
150 changed files with 5662 additions and 64 deletions
3
.github/CODEOWNERS
vendored
3
.github/CODEOWNERS
vendored
|
|
@ -768,6 +768,7 @@ x-pack/examples/triggers_actions_ui_example @elastic/response-ops
|
|||
x-pack/examples/ui_actions_enhanced_examples @elastic/appex-sharedux
|
||||
x-pack/packages/ai-infra/inference-common @elastic/appex-ai-infra
|
||||
x-pack/packages/ai-infra/product-doc-artifact-builder @elastic/appex-ai-infra
|
||||
x-pack/packages/ai-infra/product-doc-common @elastic/appex-ai-infra
|
||||
x-pack/packages/index-lifecycle-management/index_lifecycle_management_common_shared @elastic/kibana-management
|
||||
x-pack/packages/index-management/index_management_shared_types @elastic/kibana-management
|
||||
x-pack/packages/kbn-ai-assistant @elastic/search-kibana
|
||||
|
|
@ -857,6 +858,8 @@ x-pack/packages/security/role_management_model @elastic/kibana-security
|
|||
x-pack/packages/security/ui_components @elastic/kibana-security
|
||||
x-pack/performance @elastic/appex-qa
|
||||
x-pack/plugins/actions @elastic/response-ops
|
||||
x-pack/plugins/ai_infra/llm_tasks @elastic/appex-ai-infra
|
||||
x-pack/plugins/ai_infra/product_doc_base @elastic/appex-ai-infra
|
||||
x-pack/plugins/aiops @elastic/ml-ui
|
||||
x-pack/plugins/alerting @elastic/response-ops
|
||||
x-pack/plugins/banners @elastic/appex-sharedux
|
||||
|
|
|
|||
|
|
@ -690,6 +690,10 @@ the infrastructure monitoring use-case within Kibana.
|
|||
using the CURL scripts in the scripts folder.
|
||||
|
||||
|
||||
|{kib-repo}blob/{branch}/x-pack/plugins/ai_infra/llm_tasks/README.md[llmTasks]
|
||||
|This plugin contains various LLM tasks.
|
||||
|
||||
|
||||
|{kib-repo}blob/{branch}/x-pack/plugins/observability_solution/logs_data_access/README.md[logsDataAccess]
|
||||
|Exposes services to access logs data.
|
||||
|
||||
|
|
@ -767,6 +771,10 @@ Elastic.
|
|||
|This plugin helps users learn how to use the Painless scripting language.
|
||||
|
||||
|
||||
|{kib-repo}blob/{branch}/x-pack/plugins/ai_infra/product_doc_base/README.md[productDocBase]
|
||||
|This plugin contains the product documentation base service.
|
||||
|
||||
|
||||
|{kib-repo}blob/{branch}/x-pack/plugins/observability_solution/profiling/README.md[profiling]
|
||||
|Universal Profiling provides fleet-wide, whole-system, continuous profiling with zero instrumentation. Get a comprehensive understanding of what lines of code are consuming compute resources throughout your entire fleet by visualizing your data in Kibana using the flamegraph, stacktraces, and top functions views.
|
||||
|
||||
|
|
|
|||
|
|
@ -148,6 +148,9 @@ Refer to the corresponding {es} logs for potential write errors.
|
|||
| `success` | Creating trained model.
|
||||
| `failure` | Failed to create trained model.
|
||||
|
||||
.1+| `product_documentation_create`
|
||||
| `unknown` | User requested to install the product documentation for use in AI Assistants.
|
||||
|
||||
3+a|
|
||||
====== Type: change
|
||||
|
||||
|
|
@ -334,6 +337,9 @@ Refer to the corresponding {es} logs for potential write errors.
|
|||
| `success` | Updating trained model deployment.
|
||||
| `failure` | Failed to update trained model deployment.
|
||||
|
||||
.1+| `product_documentation_update`
|
||||
| `unknown` | User requested to update the product documentation for use in AI Assistants.
|
||||
|
||||
3+a|
|
||||
====== Type: deletion
|
||||
|
||||
|
|
@ -425,6 +431,9 @@ Refer to the corresponding {es} logs for potential write errors.
|
|||
| `success` | Deleting trained model.
|
||||
| `failure` | Failed to delete trained model.
|
||||
|
||||
.1+| `product_documentation_delete`
|
||||
| `unknown` | User requested to delete the product documentation for use in AI Assistants.
|
||||
|
||||
3+a|
|
||||
====== Type: access
|
||||
|
||||
|
|
|
|||
|
|
@ -617,6 +617,7 @@
|
|||
"@kbn/licensing-plugin": "link:x-pack/plugins/licensing",
|
||||
"@kbn/links-plugin": "link:src/plugins/links",
|
||||
"@kbn/lists-plugin": "link:x-pack/plugins/lists",
|
||||
"@kbn/llm-tasks-plugin": "link:x-pack/plugins/ai_infra/llm_tasks",
|
||||
"@kbn/locator-examples-plugin": "link:examples/locator_examples",
|
||||
"@kbn/locator-explorer-plugin": "link:examples/locator_explorer",
|
||||
"@kbn/logging": "link:packages/kbn-logging",
|
||||
|
|
@ -721,6 +722,8 @@
|
|||
"@kbn/presentation-panel-plugin": "link:src/plugins/presentation_panel",
|
||||
"@kbn/presentation-publishing": "link:packages/presentation/presentation_publishing",
|
||||
"@kbn/presentation-util-plugin": "link:src/plugins/presentation_util",
|
||||
"@kbn/product-doc-base-plugin": "link:x-pack/plugins/ai_infra/product_doc_base",
|
||||
"@kbn/product-doc-common": "link:x-pack/packages/ai-infra/product-doc-common",
|
||||
"@kbn/profiling-data-access-plugin": "link:x-pack/plugins/observability_solution/profiling_data_access",
|
||||
"@kbn/profiling-plugin": "link:x-pack/plugins/observability_solution/profiling",
|
||||
"@kbn/profiling-utils": "link:packages/kbn-profiling-utils",
|
||||
|
|
|
|||
|
|
@ -855,6 +855,13 @@
|
|||
"policy-settings-protection-updates-note": [
|
||||
"note"
|
||||
],
|
||||
"product-doc-install-status": [
|
||||
"index_name",
|
||||
"installation_status",
|
||||
"last_installation_date",
|
||||
"product_name",
|
||||
"product_version"
|
||||
],
|
||||
"query": [
|
||||
"description",
|
||||
"title",
|
||||
|
|
|
|||
|
|
@ -2841,6 +2841,26 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"product-doc-install-status": {
|
||||
"dynamic": false,
|
||||
"properties": {
|
||||
"index_name": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"installation_status": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"last_installation_date": {
|
||||
"type": "date"
|
||||
},
|
||||
"product_name": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"product_version": {
|
||||
"type": "keyword"
|
||||
}
|
||||
}
|
||||
},
|
||||
"query": {
|
||||
"dynamic": false,
|
||||
"properties": {
|
||||
|
|
|
|||
|
|
@ -124,6 +124,7 @@ pageLoadAssetSize:
|
|||
painlessLab: 179748
|
||||
presentationPanel: 55463
|
||||
presentationUtil: 58834
|
||||
productDocBase: 22500
|
||||
profiling: 36694
|
||||
remoteClusters: 51327
|
||||
reporting: 58600
|
||||
|
|
|
|||
|
|
@ -145,6 +145,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
|
|||
"osquery-pack-asset": "cd140bc2e4b092e93692b587bf6e38051ef94c75",
|
||||
"osquery-saved-query": "6095e288750aa3164dfe186c74bc5195c2bf2bd4",
|
||||
"policy-settings-protection-updates-note": "33924bb246f9e5bcb876109cc83e3c7a28308352",
|
||||
"product-doc-install-status": "ca6e96840228e4cc2f11bae24a0797f4f7238c8c",
|
||||
"query": "501bece68f26fe561286a488eabb1a8ab12f1137",
|
||||
"risk-engine-configuration": "bab237d09c2e7189dddddcb1b28f19af69755efb",
|
||||
"rules-settings": "ba57ef1881b3dcbf48fbfb28902d8f74442190b2",
|
||||
|
|
|
|||
|
|
@ -115,6 +115,7 @@ const previouslyRegisteredTypes = [
|
|||
'osquery-usage-metric',
|
||||
'osquery-manager-usage-metric',
|
||||
'policy-settings-protection-updates-note',
|
||||
'product-doc-install-status',
|
||||
'query',
|
||||
'rules-settings',
|
||||
'sample-data-telemetry',
|
||||
|
|
|
|||
|
|
@ -1148,6 +1148,8 @@
|
|||
"@kbn/lint-ts-projects-cli/*": ["packages/kbn-lint-ts-projects-cli/*"],
|
||||
"@kbn/lists-plugin": ["x-pack/plugins/lists"],
|
||||
"@kbn/lists-plugin/*": ["x-pack/plugins/lists/*"],
|
||||
"@kbn/llm-tasks-plugin": ["x-pack/plugins/ai_infra/llm_tasks"],
|
||||
"@kbn/llm-tasks-plugin/*": ["x-pack/plugins/ai_infra/llm_tasks/*"],
|
||||
"@kbn/locator-examples-plugin": ["examples/locator_examples"],
|
||||
"@kbn/locator-examples-plugin/*": ["examples/locator_examples/*"],
|
||||
"@kbn/locator-explorer-plugin": ["examples/locator_explorer"],
|
||||
|
|
@ -1390,6 +1392,10 @@
|
|||
"@kbn/presentation-util-plugin/*": ["src/plugins/presentation_util/*"],
|
||||
"@kbn/product-doc-artifact-builder": ["x-pack/packages/ai-infra/product-doc-artifact-builder"],
|
||||
"@kbn/product-doc-artifact-builder/*": ["x-pack/packages/ai-infra/product-doc-artifact-builder/*"],
|
||||
"@kbn/product-doc-base-plugin": ["x-pack/plugins/ai_infra/product_doc_base"],
|
||||
"@kbn/product-doc-base-plugin/*": ["x-pack/plugins/ai_infra/product_doc_base/*"],
|
||||
"@kbn/product-doc-common": ["x-pack/packages/ai-infra/product-doc-common"],
|
||||
"@kbn/product-doc-common/*": ["x-pack/packages/ai-infra/product-doc-common/*"],
|
||||
"@kbn/profiling-data-access-plugin": ["x-pack/plugins/observability_solution/profiling_data_access"],
|
||||
"@kbn/profiling-data-access-plugin/*": ["x-pack/plugins/observability_solution/profiling_data_access/*"],
|
||||
"@kbn/profiling-plugin": ["x-pack/plugins/observability_solution/profiling"],
|
||||
|
|
|
|||
|
|
@ -1,3 +1,49 @@
|
|||
# @kbn/product-doc-artifact-builder
|
||||
|
||||
Script to build the knowledge base artifacts
|
||||
Script to build the knowledge base artifacts.
|
||||
|
||||
## How to run
|
||||
|
||||
```
|
||||
node scripts/build_product_doc_artifacts.js --stack-version {version} --product-name {product}
|
||||
```
|
||||
|
||||
### parameters
|
||||
|
||||
#### `stack-version`:
|
||||
|
||||
the stack version to generate the artifacts for.
|
||||
|
||||
#### `product-name`:
|
||||
|
||||
(multi-value) the list of products to generate artifacts for.
|
||||
|
||||
possible values:
|
||||
- "kibana"
|
||||
- "elasticsearch"
|
||||
- "observability"
|
||||
- "security"
|
||||
|
||||
#### `target-folder`:
|
||||
|
||||
The folder to generate the artifacts in.
|
||||
|
||||
Defaults to `{REPO_ROOT}/build-kb-artifacts`.
|
||||
|
||||
#### `build-folder`:
|
||||
|
||||
The folder to use for temporary files.
|
||||
|
||||
Defaults to `{REPO_ROOT}/build/temp-kb-artifacts`
|
||||
|
||||
#### Cluster infos
|
||||
|
||||
- params for the source cluster:
|
||||
`sourceClusterUrl` / env.KIBANA_SOURCE_CLUSTER_URL
|
||||
`sourceClusterUsername` / env.KIBANA_SOURCE_CLUSTER_USERNAME
|
||||
`sourceClusterPassword` / env.KIBANA_SOURCE_CLUSTER_PASSWORD
|
||||
|
||||
- params for the embedding cluster:
|
||||
`embeddingClusterUrl` / env.KIBANA_EMBEDDING_CLUSTER_URL
|
||||
`embeddingClusterUsername` / env.KIBANA_EMBEDDING_CLUSTER_USERNAME
|
||||
`embeddingClusterPassword` / env.KIBANA_EMBEDDING_CLUSTER_PASSWORD
|
||||
|
|
@ -5,17 +5,13 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
export interface ArtifactManifest {
|
||||
formatVersion: string;
|
||||
productName: string;
|
||||
productVersion: string;
|
||||
}
|
||||
import type { ArtifactManifest, ProductName } from '@kbn/product-doc-common';
|
||||
|
||||
export const getArtifactManifest = ({
|
||||
productName,
|
||||
stackVersion,
|
||||
}: {
|
||||
productName: string;
|
||||
productName: ProductName;
|
||||
stackVersion: string;
|
||||
}): ArtifactManifest => {
|
||||
return {
|
||||
|
|
|
|||
|
|
@ -21,10 +21,7 @@ export const getArtifactMappings = (inferenceEndpoint: string): MappingTypeMappi
|
|||
slug: { type: 'keyword' },
|
||||
url: { type: 'keyword' },
|
||||
version: { type: 'version' },
|
||||
ai_subtitle: {
|
||||
type: 'semantic_text',
|
||||
inference_id: inferenceEndpoint,
|
||||
},
|
||||
ai_subtitle: { type: 'text' },
|
||||
ai_summary: {
|
||||
type: 'semantic_text',
|
||||
inference_id: inferenceEndpoint,
|
||||
|
|
|
|||
|
|
@ -5,7 +5,34 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
/**
|
||||
* The allowed product names, as found in the source's cluster
|
||||
*/
|
||||
export const sourceProductNames = ['Kibana', 'Elasticsearch', 'Security', 'Observability'];
|
||||
import type { ProductName } from '@kbn/product-doc-common';
|
||||
|
||||
const productNameToSourceNamesMap: Record<ProductName, string[]> = {
|
||||
kibana: ['Kibana'],
|
||||
elasticsearch: ['Elasticsearch'],
|
||||
security: ['Security'],
|
||||
observability: ['Observability'],
|
||||
};
|
||||
|
||||
const sourceNameToProductName = Object.entries(productNameToSourceNamesMap).reduce<
|
||||
Record<string, ProductName>
|
||||
>((map, [productName, sourceNames]) => {
|
||||
sourceNames.forEach((sourceName) => {
|
||||
map[sourceName] = productName as ProductName;
|
||||
});
|
||||
return map;
|
||||
}, {});
|
||||
|
||||
export const getSourceNamesFromProductName = (productName: ProductName): string[] => {
|
||||
if (!productNameToSourceNamesMap[productName]) {
|
||||
throw new Error(`Unknown product name: ${productName}`);
|
||||
}
|
||||
return productNameToSourceNamesMap[productName];
|
||||
};
|
||||
|
||||
export const getProductNameFromSource = (source: string): ProductName => {
|
||||
if (!sourceNameToProductName[source]) {
|
||||
throw new Error(`Unknown source name: ${source}`);
|
||||
}
|
||||
return sourceNameToProductName[source];
|
||||
};
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@
|
|||
import Path from 'path';
|
||||
import { Client } from '@elastic/elasticsearch';
|
||||
import { ToolingLog } from '@kbn/tooling-log';
|
||||
import type { ProductName } from '@kbn/product-doc-common';
|
||||
import {
|
||||
// checkConnectivity,
|
||||
createTargetIndex,
|
||||
|
|
@ -18,6 +19,7 @@ import {
|
|||
createArtifact,
|
||||
cleanupFolders,
|
||||
deleteIndex,
|
||||
processDocuments,
|
||||
} from './tasks';
|
||||
import type { TaskConfig } from './types';
|
||||
|
||||
|
|
@ -93,7 +95,7 @@ const buildArtifact = async ({
|
|||
sourceClient,
|
||||
log,
|
||||
}: {
|
||||
productName: string;
|
||||
productName: ProductName;
|
||||
stackVersion: string;
|
||||
buildFolder: string;
|
||||
targetFolder: string;
|
||||
|
|
@ -105,7 +107,7 @@ const buildArtifact = async ({
|
|||
|
||||
const targetIndex = getTargetIndexName({ productName, stackVersion });
|
||||
|
||||
const documents = await extractDocumentation({
|
||||
let documents = await extractDocumentation({
|
||||
client: sourceClient,
|
||||
index: 'search-docs-1',
|
||||
log,
|
||||
|
|
@ -113,6 +115,8 @@ const buildArtifact = async ({
|
|||
stackVersion,
|
||||
});
|
||||
|
||||
documents = await processDocuments({ documents, log });
|
||||
|
||||
await createTargetIndex({
|
||||
client: embeddingClient,
|
||||
indexName: targetIndex,
|
||||
|
|
|
|||
|
|
@ -6,19 +6,19 @@
|
|||
*/
|
||||
|
||||
import Path from 'path';
|
||||
import { REPO_ROOT } from '@kbn/repo-info';
|
||||
import yargs from 'yargs';
|
||||
import { REPO_ROOT } from '@kbn/repo-info';
|
||||
import { DocumentationProduct } from '@kbn/product-doc-common';
|
||||
import type { TaskConfig } from './types';
|
||||
import { buildArtifacts } from './build_artifacts';
|
||||
import { sourceProductNames } from './artifact/product_name';
|
||||
|
||||
function options(y: yargs.Argv) {
|
||||
return y
|
||||
.option('productName', {
|
||||
describe: 'name of products to generate documentation for',
|
||||
array: true,
|
||||
choices: sourceProductNames,
|
||||
default: ['Kibana'],
|
||||
choices: Object.values(DocumentationProduct),
|
||||
default: [DocumentationProduct.kibana],
|
||||
})
|
||||
.option('stackVersion', {
|
||||
describe: 'The stack version to generate documentation for',
|
||||
|
|
|
|||
|
|
@ -8,9 +8,9 @@
|
|||
import Path from 'path';
|
||||
import AdmZip from 'adm-zip';
|
||||
import type { ToolingLog } from '@kbn/tooling-log';
|
||||
import { getArtifactName, type ProductName } from '@kbn/product-doc-common';
|
||||
import { getArtifactMappings } from '../artifact/mappings';
|
||||
import { getArtifactManifest } from '../artifact/manifest';
|
||||
import { getArtifactName } from '../artifact/artifact_name';
|
||||
|
||||
export const createArtifact = async ({
|
||||
productName,
|
||||
|
|
@ -21,7 +21,7 @@ export const createArtifact = async ({
|
|||
}: {
|
||||
buildFolder: string;
|
||||
targetFolder: string;
|
||||
productName: string;
|
||||
productName: ProductName;
|
||||
stackVersion: string;
|
||||
log: ToolingLog;
|
||||
}) => {
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import Fs from 'fs/promises';
|
|||
import type { Client } from '@elastic/elasticsearch';
|
||||
import type { ToolingLog } from '@kbn/tooling-log';
|
||||
|
||||
const fileSizeLimit = 250_000;
|
||||
const fileSizeLimit = 500_000;
|
||||
|
||||
export const createChunkFiles = async ({
|
||||
index,
|
||||
|
|
|
|||
|
|
@ -21,10 +21,7 @@ const mappings: MappingTypeMapping = {
|
|||
slug: { type: 'keyword' },
|
||||
url: { type: 'keyword' },
|
||||
version: { type: 'version' },
|
||||
ai_subtitle: {
|
||||
type: 'semantic_text',
|
||||
inference_id: 'kibana-elser2',
|
||||
},
|
||||
ai_subtitle: { type: 'text' },
|
||||
ai_summary: {
|
||||
type: 'semantic_text',
|
||||
inference_id: 'kibana-elser2',
|
||||
|
|
|
|||
|
|
@ -8,6 +8,8 @@
|
|||
import type { Client } from '@elastic/elasticsearch';
|
||||
import type { SearchHit } from '@elastic/elasticsearch/lib/api/types';
|
||||
import type { ToolingLog } from '@kbn/tooling-log';
|
||||
import type { ProductName } from '@kbn/product-doc-common';
|
||||
import { getSourceNamesFromProductName, getProductNameFromSource } from '../artifact/product_name';
|
||||
|
||||
/** the list of fields to import from the source cluster */
|
||||
const fields = [
|
||||
|
|
@ -27,7 +29,7 @@ const fields = [
|
|||
export interface ExtractedDocument {
|
||||
content_title: string;
|
||||
content_body: string;
|
||||
product_name: string;
|
||||
product_name: ProductName;
|
||||
root_type: string;
|
||||
slug: string;
|
||||
url: string;
|
||||
|
|
@ -43,7 +45,7 @@ const convertHit = (hit: SearchHit<any>): ExtractedDocument => {
|
|||
return {
|
||||
content_title: source.content_title,
|
||||
content_body: source.content_body,
|
||||
product_name: source.product_name,
|
||||
product_name: getProductNameFromSource(source.product_name),
|
||||
root_type: 'documentation',
|
||||
slug: source.slug,
|
||||
url: source.url,
|
||||
|
|
@ -65,7 +67,7 @@ export const extractDocumentation = async ({
|
|||
client: Client;
|
||||
index: string;
|
||||
stackVersion: string;
|
||||
productName: string;
|
||||
productName: ProductName;
|
||||
log: ToolingLog;
|
||||
}) => {
|
||||
log.info(`Starting to extract documents from source cluster`);
|
||||
|
|
@ -76,7 +78,7 @@ export const extractDocumentation = async ({
|
|||
query: {
|
||||
bool: {
|
||||
must: [
|
||||
{ term: { product_name: productName } },
|
||||
{ terms: { product_name: getSourceNamesFromProductName(productName) } },
|
||||
{ term: { version: stackVersion } },
|
||||
{ exists: { field: 'ai_fields.ai_summary' } },
|
||||
],
|
||||
|
|
|
|||
|
|
@ -10,8 +10,8 @@ export { indexDocuments } from './index_documents';
|
|||
export { createTargetIndex } from './create_index';
|
||||
export { installElser } from './install_elser';
|
||||
export { createChunkFiles } from './create_chunk_files';
|
||||
export { performSemanticSearch } from './perform_semantic_search';
|
||||
export { checkConnectivity } from './check_connectivity';
|
||||
export { createArtifact } from './create_artifact';
|
||||
export { cleanupFolders } from './cleanup_folders';
|
||||
export { deleteIndex } from './delete_index';
|
||||
export { processDocuments } from './process_documents';
|
||||
|
|
|
|||
|
|
@ -0,0 +1,59 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { uniqBy } from 'lodash';
|
||||
import { encode } from 'gpt-tokenizer';
|
||||
import type { ToolingLog } from '@kbn/tooling-log';
|
||||
import type { ExtractedDocument } from './extract_documentation';
|
||||
|
||||
export const processDocuments = async ({
|
||||
documents,
|
||||
log,
|
||||
}: {
|
||||
documents: ExtractedDocument[];
|
||||
log: ToolingLog;
|
||||
}): Promise<ExtractedDocument[]> => {
|
||||
log.info('Starting processing documents.');
|
||||
const initialCount = documents.length;
|
||||
documents = removeDuplicates(documents);
|
||||
const noDupCount = documents.length;
|
||||
log.info(`Removed ${initialCount - noDupCount} duplicates`);
|
||||
documents.forEach(processDocument);
|
||||
documents = filterEmptyDocs(documents);
|
||||
log.info(`Removed ${noDupCount - documents.length} empty documents`);
|
||||
log.info('Done processing documents.');
|
||||
return documents;
|
||||
};
|
||||
|
||||
const removeDuplicates = (documents: ExtractedDocument[]): ExtractedDocument[] => {
|
||||
return uniqBy(documents, (doc) => doc.slug);
|
||||
};
|
||||
|
||||
/**
|
||||
* Filter "this content has moved" or "deleted pages" type of documents, just based on token count.
|
||||
*/
|
||||
const filterEmptyDocs = (documents: ExtractedDocument[]): ExtractedDocument[] => {
|
||||
return documents.filter((doc) => {
|
||||
const tokenCount = encode(doc.content_body).length;
|
||||
if (tokenCount < 100) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
};
|
||||
|
||||
const processDocument = (document: ExtractedDocument) => {
|
||||
document.content_body = document.content_body
|
||||
// remove those "edit" button text that got embedded into titles.
|
||||
.replaceAll(/([a-zA-Z])edit\n/g, (match) => {
|
||||
return `${match[0]}\n`;
|
||||
})
|
||||
// limit to 2 consecutive carriage return
|
||||
.replaceAll(/\n\n+/g, '\n\n');
|
||||
|
||||
return document;
|
||||
};
|
||||
|
|
@ -5,8 +5,10 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { ProductName } from '@kbn/product-doc-common';
|
||||
|
||||
export interface TaskConfig {
|
||||
productNames: string[];
|
||||
productNames: ProductName[];
|
||||
stackVersion: string;
|
||||
buildFolder: string;
|
||||
targetFolder: string;
|
||||
|
|
|
|||
|
|
@ -16,5 +16,6 @@
|
|||
"kbn_references": [
|
||||
"@kbn/tooling-log",
|
||||
"@kbn/repo-info",
|
||||
"@kbn/product-doc-common",
|
||||
]
|
||||
}
|
||||
|
|
|
|||
3
x-pack/packages/ai-infra/product-doc-common/README.md
Normal file
3
x-pack/packages/ai-infra/product-doc-common/README.md
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
# @kbn/product-doc-common
|
||||
|
||||
Common types and utilities for the product documentation feature.
|
||||
17
x-pack/packages/ai-infra/product-doc-common/index.ts
Normal file
17
x-pack/packages/ai-infra/product-doc-common/index.ts
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
export { getArtifactName, parseArtifactName } from './src/artifact';
|
||||
export { type ArtifactManifest } from './src/manifest';
|
||||
export { DocumentationProduct, type ProductName } from './src/product';
|
||||
export { isArtifactContentFilePath } from './src/artifact_content';
|
||||
export {
|
||||
productDocIndexPrefix,
|
||||
productDocIndexPattern,
|
||||
getProductDocIndexName,
|
||||
} from './src/indices';
|
||||
export type { ProductDocumentationAttributes } from './src/documents';
|
||||
12
x-pack/packages/ai-infra/product-doc-common/jest.config.js
Normal file
12
x-pack/packages/ai-infra/product-doc-common/jest.config.js
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
module.exports = {
|
||||
preset: '@kbn/test/jest_node',
|
||||
rootDir: '../../../..',
|
||||
roots: ['<rootDir>/x-pack/packages/ai-infra/product-doc-common'],
|
||||
};
|
||||
5
x-pack/packages/ai-infra/product-doc-common/kibana.jsonc
Normal file
5
x-pack/packages/ai-infra/product-doc-common/kibana.jsonc
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"type": "shared-common",
|
||||
"id": "@kbn/product-doc-common",
|
||||
"owner": "@elastic/appex-ai-infra"
|
||||
}
|
||||
6
x-pack/packages/ai-infra/product-doc-common/package.json
Normal file
6
x-pack/packages/ai-infra/product-doc-common/package.json
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
{
|
||||
"name": "@kbn/product-doc-common",
|
||||
"private": true,
|
||||
"version": "1.0.0",
|
||||
"license": "Elastic License 2.0"
|
||||
}
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { getArtifactName, parseArtifactName } from './artifact';
|
||||
|
||||
describe('getArtifactName', () => {
|
||||
it('builds the name based on the provided product name and version', () => {
|
||||
expect(
|
||||
getArtifactName({
|
||||
productName: 'kibana',
|
||||
productVersion: '8.16',
|
||||
})
|
||||
).toEqual('kb-product-doc-kibana-8.16.zip');
|
||||
});
|
||||
|
||||
it('excludes the extension when excludeExtension is true', () => {
|
||||
expect(
|
||||
getArtifactName({
|
||||
productName: 'elasticsearch',
|
||||
productVersion: '8.17',
|
||||
excludeExtension: true,
|
||||
})
|
||||
).toEqual('kb-product-doc-elasticsearch-8.17');
|
||||
});
|
||||
|
||||
it('generates a lowercase name', () => {
|
||||
expect(
|
||||
getArtifactName({
|
||||
// @ts-expect-error testing
|
||||
productName: 'ElasticSearch',
|
||||
productVersion: '8.17',
|
||||
excludeExtension: true,
|
||||
})
|
||||
).toEqual('kb-product-doc-elasticsearch-8.17');
|
||||
});
|
||||
});
|
||||
|
||||
describe('parseArtifactName', () => {
|
||||
it('parses an artifact name with extension', () => {
|
||||
expect(parseArtifactName('kb-product-doc-kibana-8.16.zip')).toEqual({
|
||||
productName: 'kibana',
|
||||
productVersion: '8.16',
|
||||
});
|
||||
});
|
||||
|
||||
it('parses an artifact name without extension', () => {
|
||||
expect(parseArtifactName('kb-product-doc-security-8.17')).toEqual({
|
||||
productName: 'security',
|
||||
productVersion: '8.17',
|
||||
});
|
||||
});
|
||||
|
||||
it('returns undefined if the provided string does not match the artifact name pattern', () => {
|
||||
expect(parseArtifactName('some-wrong-name')).toEqual(undefined);
|
||||
});
|
||||
|
||||
it('returns undefined if the provided string is not strictly lowercase', () => {
|
||||
expect(parseArtifactName('kb-product-doc-Security-8.17')).toEqual(undefined);
|
||||
});
|
||||
});
|
||||
39
x-pack/packages/ai-infra/product-doc-common/src/artifact.ts
Normal file
39
x-pack/packages/ai-infra/product-doc-common/src/artifact.ts
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { type ProductName, DocumentationProduct } from './product';
|
||||
|
||||
// kb-product-doc-elasticsearch-8.15.zip
|
||||
const artifactNameRegexp = /^kb-product-doc-([a-z]+)-([0-9]+\.[0-9]+)(\.zip)?$/;
|
||||
const allowedProductNames: ProductName[] = Object.values(DocumentationProduct);
|
||||
|
||||
export const getArtifactName = ({
|
||||
productName,
|
||||
productVersion,
|
||||
excludeExtension = false,
|
||||
}: {
|
||||
productName: ProductName;
|
||||
productVersion: string;
|
||||
excludeExtension?: boolean;
|
||||
}): string => {
|
||||
const ext = excludeExtension ? '' : '.zip';
|
||||
return `kb-product-doc-${productName}-${productVersion}${ext}`.toLowerCase();
|
||||
};
|
||||
|
||||
export const parseArtifactName = (artifactName: string) => {
|
||||
const match = artifactNameRegexp.exec(artifactName);
|
||||
if (match) {
|
||||
const productName = match[1].toLowerCase() as ProductName;
|
||||
const productVersion = match[2].toLowerCase();
|
||||
if (allowedProductNames.includes(productName)) {
|
||||
return {
|
||||
productName,
|
||||
productVersion,
|
||||
};
|
||||
}
|
||||
}
|
||||
};
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { isArtifactContentFilePath } from './artifact_content';
|
||||
|
||||
describe('isArtifactContentFilePath', () => {
|
||||
it('returns true for filenames matching the pattern', () => {
|
||||
expect(isArtifactContentFilePath('content/content-0.ndjson')).toEqual(true);
|
||||
expect(isArtifactContentFilePath('content/content-007.ndjson')).toEqual(true);
|
||||
expect(isArtifactContentFilePath('content/content-9042.ndjson')).toEqual(true);
|
||||
});
|
||||
|
||||
it('returns false for filenames not matching the pattern', () => {
|
||||
expect(isArtifactContentFilePath('content-0.ndjson')).toEqual(false);
|
||||
expect(isArtifactContentFilePath('content/content-0')).toEqual(false);
|
||||
expect(isArtifactContentFilePath('content/content.ndjson')).toEqual(false);
|
||||
expect(isArtifactContentFilePath('content/content-9042.json')).toEqual(false);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
const contentFileRegexp = /^content\/content-[0-9]+\.ndjson$/;
|
||||
|
||||
export const isArtifactContentFilePath = (path: string): boolean => {
|
||||
return contentFileRegexp.test(path);
|
||||
};
|
||||
31
x-pack/packages/ai-infra/product-doc-common/src/documents.ts
Normal file
31
x-pack/packages/ai-infra/product-doc-common/src/documents.ts
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { ProductName } from './product';
|
||||
|
||||
// don't need to define the other props
|
||||
interface SemanticTextField {
|
||||
text: string;
|
||||
}
|
||||
|
||||
interface SemanticTextArrayField {
|
||||
text: string[];
|
||||
}
|
||||
|
||||
export interface ProductDocumentationAttributes {
|
||||
content_title: string;
|
||||
content_body: SemanticTextField;
|
||||
product_name: ProductName;
|
||||
root_type: string;
|
||||
slug: string;
|
||||
url: string;
|
||||
version: string;
|
||||
ai_subtitle: string;
|
||||
ai_summary: SemanticTextField;
|
||||
ai_questions_answered: SemanticTextArrayField;
|
||||
ai_tags: string[];
|
||||
}
|
||||
15
x-pack/packages/ai-infra/product-doc-common/src/indices.ts
Normal file
15
x-pack/packages/ai-infra/product-doc-common/src/indices.ts
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { ProductName } from './product';
|
||||
|
||||
export const productDocIndexPrefix = '.kibana-ai-product-doc';
|
||||
export const productDocIndexPattern = `${productDocIndexPrefix}-*`;
|
||||
|
||||
export const getProductDocIndexName = (productName: ProductName): string => {
|
||||
return `${productDocIndexPrefix}-${productName.toLowerCase()}`;
|
||||
};
|
||||
|
|
@ -5,12 +5,10 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
export const getArtifactName = ({
|
||||
productName,
|
||||
productVersion,
|
||||
}: {
|
||||
productName: string;
|
||||
import type { ProductName } from './product';
|
||||
|
||||
export interface ArtifactManifest {
|
||||
formatVersion: string;
|
||||
productName: ProductName;
|
||||
productVersion: string;
|
||||
}): string => {
|
||||
return `kibana-kb-${productName}-${productVersion}.zip`.toLowerCase();
|
||||
};
|
||||
}
|
||||
15
x-pack/packages/ai-infra/product-doc-common/src/product.ts
Normal file
15
x-pack/packages/ai-infra/product-doc-common/src/product.ts
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
export enum DocumentationProduct {
|
||||
kibana = 'kibana',
|
||||
elasticsearch = 'elasticsearch',
|
||||
observability = 'observability',
|
||||
security = 'security',
|
||||
}
|
||||
|
||||
export type ProductName = keyof typeof DocumentationProduct;
|
||||
17
x-pack/packages/ai-infra/product-doc-common/tsconfig.json
Normal file
17
x-pack/packages/ai-infra/product-doc-common/tsconfig.json
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
{
|
||||
"extends": "../../../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"outDir": "target/types",
|
||||
"types": [
|
||||
"jest",
|
||||
"node"
|
||||
]
|
||||
},
|
||||
"include": [
|
||||
"**/*.ts",
|
||||
],
|
||||
"exclude": [
|
||||
"target/**/*"
|
||||
],
|
||||
"kbn_references": []
|
||||
}
|
||||
45
x-pack/plugins/ai_infra/llm_tasks/README.md
Normal file
45
x-pack/plugins/ai_infra/llm_tasks/README.md
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
# LLM Tasks plugin
|
||||
|
||||
This plugin contains various LLM tasks.
|
||||
|
||||
## Retrieve documentation
|
||||
|
||||
This task allows to retrieve documents from our Elastic product documentation.
|
||||
|
||||
The task depends on the `product-doc-base` plugin, as this dependency is used
|
||||
to install and manage the product documentation.
|
||||
|
||||
### Checking if the task is available
|
||||
|
||||
A `retrieveDocumentationAvailable` API is exposed from the start contract, that
|
||||
should be used to assert that the `retrieve_doc` task can be used in the current
|
||||
context.
|
||||
|
||||
That API receive the inbound request as parameter.
|
||||
|
||||
Example:
|
||||
```ts
|
||||
if (await llmTasksStart.retrieveDocumentationAvailable({ request })) {
|
||||
// task is available
|
||||
} else {
|
||||
// task is not available
|
||||
}
|
||||
```
|
||||
|
||||
### Executing the task
|
||||
|
||||
The task is executed as an API of the plugin's start contract, and can be invoked
|
||||
as any other lifecycle API would.
|
||||
|
||||
Example:
|
||||
```ts
|
||||
const result = await llmTasksStart.retrieveDocumentation({
|
||||
searchTerm: "How to create a space in Kibana?",
|
||||
request,
|
||||
connectorId: 'my-connector-id',
|
||||
});
|
||||
|
||||
const { success, documents } = result;
|
||||
```
|
||||
|
||||
The exhaustive list of options for the task is available on the `RetrieveDocumentationParams` type's TS doc.
|
||||
19
x-pack/plugins/ai_infra/llm_tasks/jest.config.js
Normal file
19
x-pack/plugins/ai_infra/llm_tasks/jest.config.js
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
module.exports = {
|
||||
preset: '@kbn/test',
|
||||
rootDir: '../../../..',
|
||||
roots: ['<rootDir>/x-pack/plugins/ai_infra/llm_tasks/server'],
|
||||
setupFiles: [],
|
||||
collectCoverage: true,
|
||||
collectCoverageFrom: [
|
||||
'<rootDir>/x-pack/plugins/ai_infra/llm_tasks/{public,server,common}/**/*.{js,ts,tsx}',
|
||||
],
|
||||
|
||||
coverageReporters: ['html'],
|
||||
};
|
||||
15
x-pack/plugins/ai_infra/llm_tasks/kibana.jsonc
Normal file
15
x-pack/plugins/ai_infra/llm_tasks/kibana.jsonc
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
{
|
||||
"type": "plugin",
|
||||
"id": "@kbn/llm-tasks-plugin",
|
||||
"owner": "@elastic/appex-ai-infra",
|
||||
"plugin": {
|
||||
"id": "llmTasks",
|
||||
"server": true,
|
||||
"browser": false,
|
||||
"configPath": ["xpack", "llmTasks"],
|
||||
"requiredPlugins": ["inference", "productDocBase"],
|
||||
"requiredBundles": [],
|
||||
"optionalPlugins": [],
|
||||
"extraPublicDirs": []
|
||||
}
|
||||
}
|
||||
18
x-pack/plugins/ai_infra/llm_tasks/server/config.ts
Normal file
18
x-pack/plugins/ai_infra/llm_tasks/server/config.ts
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { schema, type TypeOf } from '@kbn/config-schema';
|
||||
import type { PluginConfigDescriptor } from '@kbn/core/server';
|
||||
|
||||
const configSchema = schema.object({});
|
||||
|
||||
export const config: PluginConfigDescriptor<LlmTasksConfig> = {
|
||||
schema: configSchema,
|
||||
exposeToBrowser: {},
|
||||
};
|
||||
|
||||
export type LlmTasksConfig = TypeOf<typeof configSchema>;
|
||||
28
x-pack/plugins/ai_infra/llm_tasks/server/index.ts
Normal file
28
x-pack/plugins/ai_infra/llm_tasks/server/index.ts
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { PluginInitializer, PluginInitializerContext } from '@kbn/core/server';
|
||||
import type { LlmTasksConfig } from './config';
|
||||
import type {
|
||||
LlmTasksPluginSetup,
|
||||
LlmTasksPluginStart,
|
||||
PluginSetupDependencies,
|
||||
PluginStartDependencies,
|
||||
} from './types';
|
||||
import { LlmTasksPlugin } from './plugin';
|
||||
|
||||
export { config } from './config';
|
||||
|
||||
export type { LlmTasksPluginSetup, LlmTasksPluginStart };
|
||||
|
||||
export const plugin: PluginInitializer<
|
||||
LlmTasksPluginSetup,
|
||||
LlmTasksPluginStart,
|
||||
PluginSetupDependencies,
|
||||
PluginStartDependencies
|
||||
> = async (pluginInitializerContext: PluginInitializerContext<LlmTasksConfig>) =>
|
||||
new LlmTasksPlugin(pluginInitializerContext);
|
||||
56
x-pack/plugins/ai_infra/llm_tasks/server/plugin.ts
Normal file
56
x-pack/plugins/ai_infra/llm_tasks/server/plugin.ts
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { Logger } from '@kbn/logging';
|
||||
import type { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '@kbn/core/server';
|
||||
import type { LlmTasksConfig } from './config';
|
||||
import type {
|
||||
LlmTasksPluginSetup,
|
||||
LlmTasksPluginStart,
|
||||
PluginSetupDependencies,
|
||||
PluginStartDependencies,
|
||||
} from './types';
|
||||
import { retrieveDocumentation } from './tasks';
|
||||
|
||||
export class LlmTasksPlugin
|
||||
implements
|
||||
Plugin<
|
||||
LlmTasksPluginSetup,
|
||||
LlmTasksPluginStart,
|
||||
PluginSetupDependencies,
|
||||
PluginStartDependencies
|
||||
>
|
||||
{
|
||||
private logger: Logger;
|
||||
|
||||
constructor(context: PluginInitializerContext<LlmTasksConfig>) {
|
||||
this.logger = context.logger.get();
|
||||
}
|
||||
setup(
|
||||
coreSetup: CoreSetup<PluginStartDependencies, LlmTasksPluginStart>,
|
||||
setupDependencies: PluginSetupDependencies
|
||||
): LlmTasksPluginSetup {
|
||||
return {};
|
||||
}
|
||||
|
||||
start(core: CoreStart, startDependencies: PluginStartDependencies): LlmTasksPluginStart {
|
||||
const { inference, productDocBase } = startDependencies;
|
||||
return {
|
||||
retrieveDocumentationAvailable: async () => {
|
||||
const docBaseStatus = await startDependencies.productDocBase.management.getStatus();
|
||||
return docBaseStatus.status === 'installed';
|
||||
},
|
||||
retrieveDocumentation: (options) => {
|
||||
return retrieveDocumentation({
|
||||
outputAPI: inference.getClient({ request: options.request }).output,
|
||||
searchDocAPI: productDocBase.search,
|
||||
logger: this.logger.get('tasks.retrieve-documentation'),
|
||||
})(options);
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
8
x-pack/plugins/ai_infra/llm_tasks/server/tasks/index.ts
Normal file
8
x-pack/plugins/ai_infra/llm_tasks/server/tasks/index.ts
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
export { retrieveDocumentation } from './retrieve_documentation';
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
export { retrieveDocumentation } from './retrieve_documentation';
|
||||
export type {
|
||||
RetrieveDocumentationAPI,
|
||||
RetrieveDocumentationResult,
|
||||
RetrieveDocumentationParams,
|
||||
} from './types';
|
||||
|
|
@ -0,0 +1,182 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { httpServerMock } from '@kbn/core/server/mocks';
|
||||
import { loggerMock, type MockedLogger } from '@kbn/logging-mocks';
|
||||
import type { DocSearchResult } from '@kbn/product-doc-base-plugin/server/services/search';
|
||||
|
||||
import { retrieveDocumentation } from './retrieve_documentation';
|
||||
import { truncate, count as countTokens } from '../../utils/tokens';
|
||||
jest.mock('../../utils/tokens');
|
||||
const truncateMock = truncate as jest.MockedFn<typeof truncate>;
|
||||
const countTokensMock = countTokens as jest.MockedFn<typeof countTokens>;
|
||||
|
||||
import { summarizeDocument } from './summarize_document';
|
||||
jest.mock('./summarize_document');
|
||||
const summarizeDocumentMock = summarizeDocument as jest.MockedFn<typeof summarizeDocument>;
|
||||
|
||||
describe('retrieveDocumentation', () => {
|
||||
let logger: MockedLogger;
|
||||
let request: ReturnType<typeof httpServerMock.createKibanaRequest>;
|
||||
let outputAPI: jest.Mock;
|
||||
let searchDocAPI: jest.Mock;
|
||||
let retrieve: ReturnType<typeof retrieveDocumentation>;
|
||||
|
||||
const createResult = (parts: Partial<DocSearchResult> = {}): DocSearchResult => {
|
||||
return {
|
||||
title: 'title',
|
||||
content: 'content',
|
||||
url: 'url',
|
||||
productName: 'kibana',
|
||||
...parts,
|
||||
};
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
logger = loggerMock.create();
|
||||
request = httpServerMock.createKibanaRequest();
|
||||
outputAPI = jest.fn();
|
||||
searchDocAPI = jest.fn();
|
||||
retrieve = retrieveDocumentation({ logger, searchDocAPI, outputAPI });
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
summarizeDocumentMock.mockReset();
|
||||
truncateMock.mockReset();
|
||||
countTokensMock.mockReset();
|
||||
});
|
||||
|
||||
it('calls the search API with the right parameters', async () => {
|
||||
searchDocAPI.mockResolvedValue({ results: [] });
|
||||
|
||||
const result = await retrieve({
|
||||
searchTerm: 'What is Kibana?',
|
||||
products: ['kibana'],
|
||||
request,
|
||||
max: 5,
|
||||
connectorId: '.my-connector',
|
||||
functionCalling: 'simulated',
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
success: true,
|
||||
documents: [],
|
||||
});
|
||||
|
||||
expect(searchDocAPI).toHaveBeenCalledTimes(1);
|
||||
expect(searchDocAPI).toHaveBeenCalledWith({
|
||||
query: 'What is Kibana?',
|
||||
products: ['kibana'],
|
||||
max: 5,
|
||||
});
|
||||
});
|
||||
|
||||
it('reduces the document length using the truncate strategy', async () => {
|
||||
searchDocAPI.mockResolvedValue({
|
||||
results: [
|
||||
createResult({ content: 'content-1' }),
|
||||
createResult({ content: 'content-2' }),
|
||||
createResult({ content: 'content-3' }),
|
||||
],
|
||||
});
|
||||
|
||||
countTokensMock.mockImplementation((text) => {
|
||||
if (text === 'content-2') {
|
||||
return 150;
|
||||
} else {
|
||||
return 50;
|
||||
}
|
||||
});
|
||||
truncateMock.mockReturnValue('truncated');
|
||||
|
||||
const result = await retrieve({
|
||||
searchTerm: 'What is Kibana?',
|
||||
request,
|
||||
connectorId: '.my-connector',
|
||||
maxDocumentTokens: 100,
|
||||
tokenReductionStrategy: 'truncate',
|
||||
});
|
||||
|
||||
expect(result.documents.length).toEqual(3);
|
||||
expect(result.documents[0].content).toEqual('content-1');
|
||||
expect(result.documents[1].content).toEqual('truncated');
|
||||
expect(result.documents[2].content).toEqual('content-3');
|
||||
|
||||
expect(truncateMock).toHaveBeenCalledTimes(1);
|
||||
expect(truncateMock).toHaveBeenCalledWith('content-2', 100);
|
||||
});
|
||||
|
||||
it('reduces the document length using the summarize strategy', async () => {
|
||||
searchDocAPI.mockResolvedValue({
|
||||
results: [
|
||||
createResult({ content: 'content-1' }),
|
||||
createResult({ content: 'content-2' }),
|
||||
createResult({ content: 'content-3' }),
|
||||
],
|
||||
});
|
||||
|
||||
countTokensMock.mockImplementation((text) => {
|
||||
if (text === 'content-2') {
|
||||
return 50;
|
||||
} else {
|
||||
return 150;
|
||||
}
|
||||
});
|
||||
truncateMock.mockImplementation((text) => text);
|
||||
|
||||
summarizeDocumentMock.mockImplementation(({ documentContent }) => {
|
||||
return Promise.resolve({ summary: `${documentContent}-summarized` });
|
||||
});
|
||||
|
||||
const result = await retrieve({
|
||||
searchTerm: 'What is Kibana?',
|
||||
request,
|
||||
connectorId: '.my-connector',
|
||||
maxDocumentTokens: 100,
|
||||
tokenReductionStrategy: 'summarize',
|
||||
});
|
||||
|
||||
expect(result.documents.length).toEqual(3);
|
||||
expect(result.documents[0].content).toEqual('content-1-summarized');
|
||||
expect(result.documents[1].content).toEqual('content-2');
|
||||
expect(result.documents[2].content).toEqual('content-3-summarized');
|
||||
|
||||
expect(truncateMock).toHaveBeenCalledTimes(2);
|
||||
expect(truncateMock).toHaveBeenCalledWith('content-1-summarized', 100);
|
||||
expect(truncateMock).toHaveBeenCalledWith('content-3-summarized', 100);
|
||||
});
|
||||
|
||||
it('logs an error and return an empty list of docs in case of error', async () => {
|
||||
searchDocAPI.mockResolvedValue({
|
||||
results: [createResult({ content: 'content-1' })],
|
||||
});
|
||||
countTokensMock.mockImplementation(() => {
|
||||
return 150;
|
||||
});
|
||||
summarizeDocumentMock.mockImplementation(() => {
|
||||
throw new Error('woups');
|
||||
});
|
||||
|
||||
const result = await retrieve({
|
||||
searchTerm: 'What is Kibana?',
|
||||
request,
|
||||
connectorId: '.my-connector',
|
||||
maxDocumentTokens: 100,
|
||||
tokenReductionStrategy: 'summarize',
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
success: false,
|
||||
documents: [],
|
||||
});
|
||||
|
||||
expect(logger.error).toHaveBeenCalledTimes(1);
|
||||
expect(logger.error).toHaveBeenCalledWith(
|
||||
expect.stringContaining('Error retrieving documentation')
|
||||
);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,88 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { Logger } from '@kbn/logging';
|
||||
import type { OutputAPI } from '@kbn/inference-common';
|
||||
import type { ProductDocSearchAPI } from '@kbn/product-doc-base-plugin/server';
|
||||
import { truncate, count as countTokens } from '../../utils/tokens';
|
||||
import type { RetrieveDocumentationAPI } from './types';
|
||||
import { summarizeDocument } from './summarize_document';
|
||||
|
||||
const MAX_DOCUMENTS_DEFAULT = 3;
|
||||
const MAX_TOKENS_DEFAULT = 1000;
|
||||
|
||||
export const retrieveDocumentation =
|
||||
({
|
||||
outputAPI,
|
||||
searchDocAPI,
|
||||
logger: log,
|
||||
}: {
|
||||
outputAPI: OutputAPI;
|
||||
searchDocAPI: ProductDocSearchAPI;
|
||||
logger: Logger;
|
||||
}): RetrieveDocumentationAPI =>
|
||||
async ({
|
||||
searchTerm,
|
||||
connectorId,
|
||||
products,
|
||||
functionCalling,
|
||||
max = MAX_DOCUMENTS_DEFAULT,
|
||||
maxDocumentTokens = MAX_TOKENS_DEFAULT,
|
||||
tokenReductionStrategy = 'summarize',
|
||||
}) => {
|
||||
try {
|
||||
const { results } = await searchDocAPI({ query: searchTerm, products, max });
|
||||
|
||||
log.debug(`searching with term=[${searchTerm}] returned ${results.length} documents`);
|
||||
|
||||
const processedDocuments = await Promise.all(
|
||||
results.map(async (document) => {
|
||||
const tokenCount = countTokens(document.content);
|
||||
const docHasTooManyTokens = tokenCount >= maxDocumentTokens;
|
||||
log.debug(
|
||||
`processing doc [${document.url}] - tokens : [${tokenCount}] - tooManyTokens: [${docHasTooManyTokens}]`
|
||||
);
|
||||
|
||||
let content = document.content;
|
||||
if (docHasTooManyTokens) {
|
||||
if (tokenReductionStrategy === 'summarize') {
|
||||
const extractResponse = await summarizeDocument({
|
||||
searchTerm,
|
||||
documentContent: document.content,
|
||||
outputAPI,
|
||||
connectorId,
|
||||
functionCalling,
|
||||
});
|
||||
content = truncate(extractResponse.summary, maxDocumentTokens);
|
||||
} else {
|
||||
content = truncate(document.content, maxDocumentTokens);
|
||||
}
|
||||
}
|
||||
|
||||
log.debug(`done processing document [${document.url}]`);
|
||||
return {
|
||||
title: document.title,
|
||||
url: document.url,
|
||||
content,
|
||||
};
|
||||
})
|
||||
);
|
||||
|
||||
log.debug(() => {
|
||||
const docsAsJson = JSON.stringify(processedDocuments);
|
||||
return `searching with term=[${searchTerm}] - results: ${docsAsJson}`;
|
||||
});
|
||||
|
||||
return {
|
||||
success: true,
|
||||
documents: processedDocuments.filter((doc) => doc.content.length > 0),
|
||||
};
|
||||
} catch (e) {
|
||||
log.error(`Error retrieving documentation: ${e.message}. Returning empty results.`);
|
||||
return { success: false, documents: [] };
|
||||
}
|
||||
};
|
||||
|
|
@ -0,0 +1,67 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { ToolSchema, FunctionCallingMode, OutputAPI } from '@kbn/inference-common';
|
||||
|
||||
const summarizeDocumentSchema = {
|
||||
type: 'object',
|
||||
properties: {
|
||||
useful: {
|
||||
type: 'boolean',
|
||||
description: `Whether the provided document has any useful information related to the user's query.`,
|
||||
},
|
||||
summary: {
|
||||
type: 'string',
|
||||
description: `The condensed version of the document that can be used to answer the question. Can be empty.`,
|
||||
},
|
||||
},
|
||||
required: ['useful'],
|
||||
} as const satisfies ToolSchema;
|
||||
|
||||
interface SummarizeDocumentResponse {
|
||||
summary: string;
|
||||
}
|
||||
|
||||
export const summarizeDocument = async ({
|
||||
searchTerm,
|
||||
documentContent,
|
||||
connectorId,
|
||||
outputAPI,
|
||||
functionCalling,
|
||||
}: {
|
||||
searchTerm: string;
|
||||
documentContent: string;
|
||||
outputAPI: OutputAPI;
|
||||
connectorId: string;
|
||||
functionCalling?: FunctionCallingMode;
|
||||
}): Promise<SummarizeDocumentResponse> => {
|
||||
const result = await outputAPI({
|
||||
id: 'summarize_document',
|
||||
connectorId,
|
||||
functionCalling,
|
||||
system: `You are an helpful Elastic assistant, and your current task is to help answer the user's question.
|
||||
|
||||
Given a question and a document, please provide a condensed version of the document that can be used to answer the question.
|
||||
- Limit the length of the output to 500 words.
|
||||
- Try to include all relevant information that could be used to answer the question. If this
|
||||
can't be done within the 500 words limit, then only include the most relevant information related to the question.
|
||||
- If you think the document isn't relevant at all to answer the question, just return an empty text`,
|
||||
input: `
|
||||
## User question
|
||||
|
||||
${searchTerm}
|
||||
|
||||
## Document
|
||||
|
||||
${documentContent}
|
||||
`,
|
||||
schema: summarizeDocumentSchema,
|
||||
});
|
||||
return {
|
||||
summary: result.output.summary ?? '',
|
||||
};
|
||||
};
|
||||
|
|
@ -0,0 +1,72 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { KibanaRequest } from '@kbn/core/server';
|
||||
import type { FunctionCallingMode } from '@kbn/inference-common';
|
||||
import type { ProductName } from '@kbn/product-doc-common';
|
||||
|
||||
/**
|
||||
* Parameters for {@link RetrieveDocumentationAPI}
|
||||
*/
|
||||
export interface RetrieveDocumentationParams {
|
||||
/**
|
||||
* The search term to perform semantic text with.
|
||||
* E.g. "What is Kibana Lens?"
|
||||
*/
|
||||
searchTerm: string;
|
||||
/**
|
||||
* Maximum number of documents to return.
|
||||
* Defaults to 3.
|
||||
*/
|
||||
max?: number;
|
||||
/**
|
||||
* Optional list of products to restrict the search to.
|
||||
*/
|
||||
products?: ProductName[];
|
||||
/**
|
||||
* The maximum number of tokens to return *per document*.
|
||||
* Documents exceeding this limit will go through token reduction.
|
||||
*
|
||||
* Defaults to `1000`.
|
||||
*/
|
||||
maxDocumentTokens?: number;
|
||||
/**
|
||||
* The token reduction strategy to apply for documents exceeding max token count.
|
||||
* - truncate: Will keep the N first tokens
|
||||
* - summarize: Will call the LLM asking to generate a contextualized summary of the document
|
||||
*
|
||||
* Overall, `summarize` is way more efficient, but significantly slower, given that an additional
|
||||
* LLM call will be performed.
|
||||
*
|
||||
* Defaults to `summarize`
|
||||
*/
|
||||
tokenReductionStrategy?: 'truncate' | 'summarize';
|
||||
/**
|
||||
* The request that initiated the task.
|
||||
*/
|
||||
request: KibanaRequest;
|
||||
/**
|
||||
* Id of the LLM connector to use for the task.
|
||||
*/
|
||||
connectorId: string;
|
||||
functionCalling?: FunctionCallingMode;
|
||||
}
|
||||
|
||||
export interface RetrievedDocument {
|
||||
title: string;
|
||||
url: string;
|
||||
content: string;
|
||||
}
|
||||
|
||||
export interface RetrieveDocumentationResult {
|
||||
success: boolean;
|
||||
documents: RetrievedDocument[];
|
||||
}
|
||||
|
||||
export type RetrieveDocumentationAPI = (
|
||||
options: RetrieveDocumentationParams
|
||||
) => Promise<RetrieveDocumentationResult>;
|
||||
42
x-pack/plugins/ai_infra/llm_tasks/server/types.ts
Normal file
42
x-pack/plugins/ai_infra/llm_tasks/server/types.ts
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { InferenceServerStart } from '@kbn/inference-plugin/server';
|
||||
import type { ProductDocBaseStartContract } from '@kbn/product-doc-base-plugin/server';
|
||||
import type { RetrieveDocumentationAPI } from './tasks/retrieve_documentation';
|
||||
|
||||
/* eslint-disable @typescript-eslint/no-empty-interface*/
|
||||
|
||||
export interface PluginSetupDependencies {}
|
||||
|
||||
export interface PluginStartDependencies {
|
||||
inference: InferenceServerStart;
|
||||
productDocBase: ProductDocBaseStartContract;
|
||||
}
|
||||
|
||||
/**
|
||||
* Describes public llmTasks plugin contract returned at the `setup` stage.
|
||||
*/
|
||||
export interface LlmTasksPluginSetup {}
|
||||
|
||||
/**
|
||||
* Describes public llmTasks plugin contract returned at the `start` stage.
|
||||
*/
|
||||
export interface LlmTasksPluginStart {
|
||||
/**
|
||||
* Checks if all prerequisites to use the `retrieveDocumentation` task
|
||||
* are respected. Can be used to check if the task can be registered
|
||||
* as LLM tool for example.
|
||||
*/
|
||||
retrieveDocumentationAvailable: () => Promise<boolean>;
|
||||
/**
|
||||
* Perform the `retrieveDocumentation` task.
|
||||
*
|
||||
* @see RetrieveDocumentationAPI
|
||||
*/
|
||||
retrieveDocumentation: RetrieveDocumentationAPI;
|
||||
}
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { count, truncate } from './tokens';
|
||||
|
||||
describe('count', () => {
|
||||
it('returns the token count of a given text', () => {
|
||||
expect(count('some short sentence')).toBeGreaterThan(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('truncate', () => {
|
||||
it('truncates text that exceed the specified maximum token count', () => {
|
||||
const text = 'some sentence that is likely longer than 5 tokens.';
|
||||
const output = truncate(text, 5);
|
||||
expect(output.length).toBeLessThan(text.length);
|
||||
});
|
||||
it('keeps text with a smaller amount of tokens unchanged', () => {
|
||||
const text = 'some sentence that is likely less than 100 tokens.';
|
||||
const output = truncate(text, 100);
|
||||
expect(output.length).toEqual(text.length);
|
||||
});
|
||||
});
|
||||
21
x-pack/plugins/ai_infra/llm_tasks/server/utils/tokens.ts
Normal file
21
x-pack/plugins/ai_infra/llm_tasks/server/utils/tokens.ts
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { encode, decode } from 'gpt-tokenizer';
|
||||
|
||||
export const count = (text: string): number => {
|
||||
return encode(text).length;
|
||||
};
|
||||
|
||||
export const truncate = (text: string, maxTokens: number): string => {
|
||||
const encoded = encode(text);
|
||||
if (encoded.length > maxTokens) {
|
||||
const truncated = encoded.slice(0, maxTokens);
|
||||
return decode(truncated);
|
||||
}
|
||||
return text;
|
||||
};
|
||||
27
x-pack/plugins/ai_infra/llm_tasks/tsconfig.json
Normal file
27
x-pack/plugins/ai_infra/llm_tasks/tsconfig.json
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
{
|
||||
"extends": "../../../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"outDir": "target/types"
|
||||
},
|
||||
"include": [
|
||||
"../../../../typings/**/*",
|
||||
"common/**/*",
|
||||
"public/**/*",
|
||||
"typings/**/*",
|
||||
"public/**/*.json",
|
||||
"server/**/*",
|
||||
"scripts/**/*",
|
||||
".storybook/**/*"
|
||||
],
|
||||
"exclude": ["target/**/*", ".storybook/**/*.js"],
|
||||
"kbn_references": [
|
||||
"@kbn/core",
|
||||
"@kbn/logging",
|
||||
"@kbn/config-schema",
|
||||
"@kbn/product-doc-common",
|
||||
"@kbn/inference-plugin",
|
||||
"@kbn/product-doc-base-plugin",
|
||||
"@kbn/logging-mocks",
|
||||
"@kbn/inference-common",
|
||||
]
|
||||
}
|
||||
3
x-pack/plugins/ai_infra/product_doc_base/README.md
Normal file
3
x-pack/plugins/ai_infra/product_doc_base/README.md
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
# Product documentation base plugin
|
||||
|
||||
This plugin contains the product documentation base service.
|
||||
14
x-pack/plugins/ai_infra/product_doc_base/common/consts.ts
Normal file
14
x-pack/plugins/ai_infra/product_doc_base/common/consts.ts
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
export const productDocInstallStatusSavedObjectTypeName = 'product-doc-install-status';
|
||||
|
||||
/**
|
||||
* The id of the inference endpoint we're creating for our product doc indices.
|
||||
* Could be replaced with the default elser 2 endpoint once the default endpoint feature is available.
|
||||
*/
|
||||
export const internalElserInferenceId = 'kibana-internal-elser2';
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { ProductName } from '@kbn/product-doc-common';
|
||||
import type { ProductInstallState, InstallationStatus } from '../install_status';
|
||||
|
||||
export const INSTALLATION_STATUS_API_PATH = '/internal/product_doc_base/status';
|
||||
export const INSTALL_ALL_API_PATH = '/internal/product_doc_base/install';
|
||||
export const UNINSTALL_ALL_API_PATH = '/internal/product_doc_base/uninstall';
|
||||
|
||||
export interface InstallationStatusResponse {
|
||||
overall: InstallationStatus;
|
||||
perProducts: Record<ProductName, ProductInstallState>;
|
||||
}
|
||||
|
||||
export interface PerformInstallResponse {
|
||||
installed: boolean;
|
||||
}
|
||||
|
||||
export interface UninstallResponse {
|
||||
success: boolean;
|
||||
}
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { ProductName } from '@kbn/product-doc-common';
|
||||
|
||||
export type InstallationStatus = 'installed' | 'uninstalled' | 'installing' | 'error';
|
||||
|
||||
/**
|
||||
* DTO representation of the product doc install status SO
|
||||
*/
|
||||
export interface ProductDocInstallStatus {
|
||||
id: string;
|
||||
productName: ProductName;
|
||||
productVersion: string;
|
||||
installationStatus: InstallationStatus;
|
||||
lastInstallationDate: Date | undefined;
|
||||
lastInstallationFailureReason: string | undefined;
|
||||
indexName?: string;
|
||||
}
|
||||
|
||||
export interface ProductInstallState {
|
||||
status: InstallationStatus;
|
||||
version?: string;
|
||||
}
|
||||
23
x-pack/plugins/ai_infra/product_doc_base/jest.config.js
Normal file
23
x-pack/plugins/ai_infra/product_doc_base/jest.config.js
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
module.exports = {
|
||||
preset: '@kbn/test',
|
||||
rootDir: '../../../..',
|
||||
roots: [
|
||||
'<rootDir>/x-pack/plugins/ai_infra/product_doc_base/public',
|
||||
'<rootDir>/x-pack/plugins/ai_infra/product_doc_base/server',
|
||||
'<rootDir>/x-pack/plugins/ai_infra/product_doc_base/common',
|
||||
],
|
||||
setupFiles: [],
|
||||
collectCoverage: true,
|
||||
collectCoverageFrom: [
|
||||
'<rootDir>/x-pack/plugins/ai_infra/product_doc_base/{public,server,common}/**/*.{js,ts,tsx}',
|
||||
],
|
||||
|
||||
coverageReporters: ['html'],
|
||||
};
|
||||
15
x-pack/plugins/ai_infra/product_doc_base/kibana.jsonc
Normal file
15
x-pack/plugins/ai_infra/product_doc_base/kibana.jsonc
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
{
|
||||
"type": "plugin",
|
||||
"id": "@kbn/product-doc-base-plugin",
|
||||
"owner": "@elastic/appex-ai-infra",
|
||||
"plugin": {
|
||||
"id": "productDocBase",
|
||||
"server": true,
|
||||
"browser": true,
|
||||
"configPath": ["xpack", "productDocBase"],
|
||||
"requiredPlugins": ["licensing", "taskManager"],
|
||||
"requiredBundles": [],
|
||||
"optionalPlugins": [],
|
||||
"extraPublicDirs": []
|
||||
}
|
||||
}
|
||||
26
x-pack/plugins/ai_infra/product_doc_base/public/index.ts
Normal file
26
x-pack/plugins/ai_infra/product_doc_base/public/index.ts
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { PluginInitializer, PluginInitializerContext } from '@kbn/core/public';
|
||||
import { ProductDocBasePlugin } from './plugin';
|
||||
import type {
|
||||
ProductDocBasePluginSetup,
|
||||
ProductDocBasePluginStart,
|
||||
PluginSetupDependencies,
|
||||
PluginStartDependencies,
|
||||
PublicPluginConfig,
|
||||
} from './types';
|
||||
|
||||
export type { ProductDocBasePluginSetup, ProductDocBasePluginStart };
|
||||
|
||||
export const plugin: PluginInitializer<
|
||||
ProductDocBasePluginSetup,
|
||||
ProductDocBasePluginStart,
|
||||
PluginSetupDependencies,
|
||||
PluginStartDependencies
|
||||
> = (pluginInitializerContext: PluginInitializerContext<PublicPluginConfig>) =>
|
||||
new ProductDocBasePlugin(pluginInitializerContext);
|
||||
51
x-pack/plugins/ai_infra/product_doc_base/public/plugin.tsx
Normal file
51
x-pack/plugins/ai_infra/product_doc_base/public/plugin.tsx
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '@kbn/core/public';
|
||||
import type { Logger } from '@kbn/logging';
|
||||
import type {
|
||||
PublicPluginConfig,
|
||||
ProductDocBasePluginSetup,
|
||||
ProductDocBasePluginStart,
|
||||
PluginSetupDependencies,
|
||||
PluginStartDependencies,
|
||||
} from './types';
|
||||
import { InstallationService } from './services/installation';
|
||||
|
||||
export class ProductDocBasePlugin
|
||||
implements
|
||||
Plugin<
|
||||
ProductDocBasePluginSetup,
|
||||
ProductDocBasePluginStart,
|
||||
PluginSetupDependencies,
|
||||
PluginStartDependencies
|
||||
>
|
||||
{
|
||||
logger: Logger;
|
||||
|
||||
constructor(context: PluginInitializerContext<PublicPluginConfig>) {
|
||||
this.logger = context.logger.get();
|
||||
}
|
||||
setup(
|
||||
coreSetup: CoreSetup<PluginStartDependencies, ProductDocBasePluginStart>,
|
||||
pluginsSetup: PluginSetupDependencies
|
||||
): ProductDocBasePluginSetup {
|
||||
return {};
|
||||
}
|
||||
|
||||
start(coreStart: CoreStart, pluginsStart: PluginStartDependencies): ProductDocBasePluginStart {
|
||||
const installationService = new InstallationService({ http: coreStart.http });
|
||||
|
||||
return {
|
||||
installation: {
|
||||
getStatus: () => installationService.getInstallationStatus(),
|
||||
install: () => installationService.install(),
|
||||
uninstall: () => installationService.uninstall(),
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
export { InstallationService } from './installation_service';
|
||||
export type { InstallationAPI } from './types';
|
||||
|
|
@ -0,0 +1,79 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { httpServiceMock } from '@kbn/core/public/mocks';
|
||||
import { InstallationService } from './installation_service';
|
||||
import {
|
||||
INSTALLATION_STATUS_API_PATH,
|
||||
INSTALL_ALL_API_PATH,
|
||||
UNINSTALL_ALL_API_PATH,
|
||||
} from '../../../common/http_api/installation';
|
||||
|
||||
describe('InstallationService', () => {
|
||||
let http: ReturnType<typeof httpServiceMock.createSetupContract>;
|
||||
let service: InstallationService;
|
||||
|
||||
beforeEach(() => {
|
||||
http = httpServiceMock.createSetupContract();
|
||||
service = new InstallationService({ http });
|
||||
});
|
||||
|
||||
describe('#getInstallationStatus', () => {
|
||||
it('calls the endpoint with the right parameters', async () => {
|
||||
await service.getInstallationStatus();
|
||||
expect(http.get).toHaveBeenCalledTimes(1);
|
||||
expect(http.get).toHaveBeenCalledWith(INSTALLATION_STATUS_API_PATH);
|
||||
});
|
||||
it('returns the value from the server', async () => {
|
||||
const expected = { stubbed: true };
|
||||
http.get.mockResolvedValue(expected);
|
||||
|
||||
const response = await service.getInstallationStatus();
|
||||
expect(response).toEqual(expected);
|
||||
});
|
||||
});
|
||||
describe('#install', () => {
|
||||
beforeEach(() => {
|
||||
http.post.mockResolvedValue({ installed: true });
|
||||
});
|
||||
|
||||
it('calls the endpoint with the right parameters', async () => {
|
||||
await service.install();
|
||||
expect(http.post).toHaveBeenCalledTimes(1);
|
||||
expect(http.post).toHaveBeenCalledWith(INSTALL_ALL_API_PATH);
|
||||
});
|
||||
it('returns the value from the server', async () => {
|
||||
const expected = { installed: true };
|
||||
http.post.mockResolvedValue(expected);
|
||||
|
||||
const response = await service.install();
|
||||
expect(response).toEqual(expected);
|
||||
});
|
||||
it('throws when the server returns installed: false', async () => {
|
||||
const expected = { installed: false };
|
||||
http.post.mockResolvedValue(expected);
|
||||
|
||||
await expect(service.install()).rejects.toThrowErrorMatchingInlineSnapshot(
|
||||
`"Installation did not complete successfully"`
|
||||
);
|
||||
});
|
||||
});
|
||||
describe('#uninstall', () => {
|
||||
it('calls the endpoint with the right parameters', async () => {
|
||||
await service.uninstall();
|
||||
expect(http.post).toHaveBeenCalledTimes(1);
|
||||
expect(http.post).toHaveBeenCalledWith(UNINSTALL_ALL_API_PATH);
|
||||
});
|
||||
it('returns the value from the server', async () => {
|
||||
const expected = { stubbed: true };
|
||||
http.post.mockResolvedValue(expected);
|
||||
|
||||
const response = await service.uninstall();
|
||||
expect(response).toEqual(expected);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { HttpSetup } from '@kbn/core-http-browser';
|
||||
import {
|
||||
INSTALLATION_STATUS_API_PATH,
|
||||
INSTALL_ALL_API_PATH,
|
||||
UNINSTALL_ALL_API_PATH,
|
||||
InstallationStatusResponse,
|
||||
PerformInstallResponse,
|
||||
UninstallResponse,
|
||||
} from '../../../common/http_api/installation';
|
||||
|
||||
export class InstallationService {
|
||||
private readonly http: HttpSetup;
|
||||
|
||||
constructor({ http }: { http: HttpSetup }) {
|
||||
this.http = http;
|
||||
}
|
||||
|
||||
async getInstallationStatus(): Promise<InstallationStatusResponse> {
|
||||
return await this.http.get<InstallationStatusResponse>(INSTALLATION_STATUS_API_PATH);
|
||||
}
|
||||
|
||||
async install(): Promise<PerformInstallResponse> {
|
||||
const response = await this.http.post<PerformInstallResponse>(INSTALL_ALL_API_PATH);
|
||||
if (!response.installed) {
|
||||
throw new Error('Installation did not complete successfully');
|
||||
}
|
||||
return response;
|
||||
}
|
||||
|
||||
async uninstall(): Promise<UninstallResponse> {
|
||||
return await this.http.post<UninstallResponse>(UNINSTALL_ALL_API_PATH);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type {
|
||||
InstallationStatusResponse,
|
||||
PerformInstallResponse,
|
||||
UninstallResponse,
|
||||
} from '../../../common/http_api/installation';
|
||||
|
||||
export interface InstallationAPI {
|
||||
getStatus(): Promise<InstallationStatusResponse>;
|
||||
install(): Promise<PerformInstallResponse>;
|
||||
uninstall(): Promise<UninstallResponse>;
|
||||
}
|
||||
22
x-pack/plugins/ai_infra/product_doc_base/public/types.ts
Normal file
22
x-pack/plugins/ai_infra/product_doc_base/public/types.ts
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { InstallationAPI } from './services/installation';
|
||||
|
||||
/* eslint-disable @typescript-eslint/no-empty-interface*/
|
||||
|
||||
export interface PublicPluginConfig {}
|
||||
|
||||
export interface PluginSetupDependencies {}
|
||||
|
||||
export interface PluginStartDependencies {}
|
||||
|
||||
export interface ProductDocBasePluginSetup {}
|
||||
|
||||
export interface ProductDocBasePluginStart {
|
||||
installation: InstallationAPI;
|
||||
}
|
||||
22
x-pack/plugins/ai_infra/product_doc_base/server/config.ts
Normal file
22
x-pack/plugins/ai_infra/product_doc_base/server/config.ts
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { schema, type TypeOf } from '@kbn/config-schema';
|
||||
import type { PluginConfigDescriptor } from '@kbn/core/server';
|
||||
|
||||
const configSchema = schema.object({
|
||||
artifactRepositoryUrl: schema.string({
|
||||
defaultValue: 'https://kibana-knowledge-base-artifacts.elastic.co',
|
||||
}),
|
||||
});
|
||||
|
||||
export const config: PluginConfigDescriptor<ProductDocBaseConfig> = {
|
||||
schema: configSchema,
|
||||
exposeToBrowser: {},
|
||||
};
|
||||
|
||||
export type ProductDocBaseConfig = TypeOf<typeof configSchema>;
|
||||
29
x-pack/plugins/ai_infra/product_doc_base/server/index.ts
Normal file
29
x-pack/plugins/ai_infra/product_doc_base/server/index.ts
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { PluginInitializer, PluginInitializerContext } from '@kbn/core/server';
|
||||
import type { ProductDocBaseConfig } from './config';
|
||||
import type {
|
||||
ProductDocBaseSetupContract,
|
||||
ProductDocBaseStartContract,
|
||||
ProductDocBaseSetupDependencies,
|
||||
ProductDocBaseStartDependencies,
|
||||
} from './types';
|
||||
import { ProductDocBasePlugin } from './plugin';
|
||||
|
||||
export { config } from './config';
|
||||
|
||||
export type { ProductDocBaseSetupContract, ProductDocBaseStartContract };
|
||||
export type { SearchApi as ProductDocSearchAPI } from './services/search/types';
|
||||
|
||||
export const plugin: PluginInitializer<
|
||||
ProductDocBaseSetupContract,
|
||||
ProductDocBaseStartContract,
|
||||
ProductDocBaseSetupDependencies,
|
||||
ProductDocBaseStartDependencies
|
||||
> = async (pluginInitializerContext: PluginInitializerContext<ProductDocBaseConfig>) =>
|
||||
new ProductDocBasePlugin(pluginInitializerContext);
|
||||
|
|
@ -0,0 +1,96 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { coreMock } from '@kbn/core/server/mocks';
|
||||
import { licensingMock } from '@kbn/licensing-plugin/server/mocks';
|
||||
import { taskManagerMock } from '@kbn/task-manager-plugin/server/mocks';
|
||||
import { productDocInstallStatusSavedObjectTypeName } from '../common/consts';
|
||||
import { ProductDocBasePlugin } from './plugin';
|
||||
import { ProductDocBaseSetupDependencies, ProductDocBaseStartDependencies } from './types';
|
||||
|
||||
jest.mock('./services/package_installer');
|
||||
jest.mock('./services/search');
|
||||
jest.mock('./services/doc_install_status');
|
||||
jest.mock('./routes');
|
||||
jest.mock('./tasks');
|
||||
import { registerRoutes } from './routes';
|
||||
import { PackageInstaller } from './services/package_installer';
|
||||
import { registerTaskDefinitions, scheduleEnsureUpToDateTask } from './tasks';
|
||||
|
||||
const PackageInstallMock = PackageInstaller as jest.Mock;
|
||||
|
||||
describe('ProductDocBasePlugin', () => {
|
||||
let initContext: ReturnType<typeof coreMock.createPluginInitializerContext>;
|
||||
let plugin: ProductDocBasePlugin;
|
||||
let pluginSetupDeps: ProductDocBaseSetupDependencies;
|
||||
let pluginStartDeps: ProductDocBaseStartDependencies;
|
||||
|
||||
beforeEach(() => {
|
||||
initContext = coreMock.createPluginInitializerContext();
|
||||
plugin = new ProductDocBasePlugin(initContext);
|
||||
pluginSetupDeps = {
|
||||
taskManager: taskManagerMock.createSetup(),
|
||||
};
|
||||
pluginStartDeps = {
|
||||
licensing: licensingMock.createStart(),
|
||||
taskManager: taskManagerMock.createStart(),
|
||||
};
|
||||
|
||||
PackageInstallMock.mockReturnValue({ ensureUpToDate: jest.fn().mockResolvedValue({}) });
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
(scheduleEnsureUpToDateTask as jest.Mock).mockReset();
|
||||
});
|
||||
|
||||
describe('#setup', () => {
|
||||
it('register the routes', () => {
|
||||
plugin.setup(coreMock.createSetup(), pluginSetupDeps);
|
||||
|
||||
expect(registerRoutes).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
it('register the product-doc SO type', () => {
|
||||
const coreSetup = coreMock.createSetup();
|
||||
plugin.setup(coreSetup, pluginSetupDeps);
|
||||
|
||||
expect(coreSetup.savedObjects.registerType).toHaveBeenCalledTimes(1);
|
||||
expect(coreSetup.savedObjects.registerType).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
name: productDocInstallStatusSavedObjectTypeName,
|
||||
})
|
||||
);
|
||||
});
|
||||
it('register the task definitions', () => {
|
||||
plugin.setup(coreMock.createSetup(), pluginSetupDeps);
|
||||
|
||||
expect(registerTaskDefinitions).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
});
|
||||
|
||||
describe('#start', () => {
|
||||
it('returns a contract with the expected shape', () => {
|
||||
plugin.setup(coreMock.createSetup(), pluginSetupDeps);
|
||||
const startContract = plugin.start(coreMock.createStart(), pluginStartDeps);
|
||||
expect(startContract).toEqual({
|
||||
management: {
|
||||
getStatus: expect.any(Function),
|
||||
install: expect.any(Function),
|
||||
uninstall: expect.any(Function),
|
||||
update: expect.any(Function),
|
||||
},
|
||||
search: expect.any(Function),
|
||||
});
|
||||
});
|
||||
|
||||
it('schedules the update task', () => {
|
||||
plugin.setup(coreMock.createSetup(), pluginSetupDeps);
|
||||
plugin.start(coreMock.createStart(), pluginStartDeps);
|
||||
|
||||
expect(scheduleEnsureUpToDateTask).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
});
|
||||
133
x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts
Normal file
133
x-pack/plugins/ai_infra/product_doc_base/server/plugin.ts
Normal file
|
|
@ -0,0 +1,133 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import Path from 'path';
|
||||
import type { Logger } from '@kbn/logging';
|
||||
import { getDataPath } from '@kbn/utils';
|
||||
import type { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '@kbn/core/server';
|
||||
import { SavedObjectsClient } from '@kbn/core/server';
|
||||
import { productDocInstallStatusSavedObjectTypeName } from '../common/consts';
|
||||
import type { ProductDocBaseConfig } from './config';
|
||||
import {
|
||||
ProductDocBaseSetupContract,
|
||||
ProductDocBaseStartContract,
|
||||
ProductDocBaseSetupDependencies,
|
||||
ProductDocBaseStartDependencies,
|
||||
InternalServices,
|
||||
} from './types';
|
||||
import { productDocInstallStatusSavedObjectType } from './saved_objects';
|
||||
import { PackageInstaller } from './services/package_installer';
|
||||
import { InferenceEndpointManager } from './services/inference_endpoint';
|
||||
import { ProductDocInstallClient } from './services/doc_install_status';
|
||||
import { DocumentationManager } from './services/doc_manager';
|
||||
import { SearchService } from './services/search';
|
||||
import { registerRoutes } from './routes';
|
||||
import { registerTaskDefinitions } from './tasks';
|
||||
|
||||
export class ProductDocBasePlugin
|
||||
implements
|
||||
Plugin<
|
||||
ProductDocBaseSetupContract,
|
||||
ProductDocBaseStartContract,
|
||||
ProductDocBaseSetupDependencies,
|
||||
ProductDocBaseStartDependencies
|
||||
>
|
||||
{
|
||||
private logger: Logger;
|
||||
private internalServices?: InternalServices;
|
||||
|
||||
constructor(private readonly context: PluginInitializerContext<ProductDocBaseConfig>) {
|
||||
this.logger = context.logger.get();
|
||||
}
|
||||
setup(
|
||||
coreSetup: CoreSetup<ProductDocBaseStartDependencies, ProductDocBaseStartContract>,
|
||||
{ taskManager }: ProductDocBaseSetupDependencies
|
||||
): ProductDocBaseSetupContract {
|
||||
const getServices = () => {
|
||||
if (!this.internalServices) {
|
||||
throw new Error('getServices called before #start');
|
||||
}
|
||||
return this.internalServices;
|
||||
};
|
||||
|
||||
coreSetup.savedObjects.registerType(productDocInstallStatusSavedObjectType);
|
||||
|
||||
registerTaskDefinitions({
|
||||
taskManager,
|
||||
getServices,
|
||||
});
|
||||
|
||||
const router = coreSetup.http.createRouter();
|
||||
registerRoutes({
|
||||
router,
|
||||
getServices,
|
||||
});
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
start(
|
||||
core: CoreStart,
|
||||
{ licensing, taskManager }: ProductDocBaseStartDependencies
|
||||
): ProductDocBaseStartContract {
|
||||
const soClient = new SavedObjectsClient(
|
||||
core.savedObjects.createInternalRepository([productDocInstallStatusSavedObjectTypeName])
|
||||
);
|
||||
const productDocClient = new ProductDocInstallClient({ soClient });
|
||||
|
||||
const endpointManager = new InferenceEndpointManager({
|
||||
esClient: core.elasticsearch.client.asInternalUser,
|
||||
logger: this.logger.get('endpoint-manager'),
|
||||
});
|
||||
|
||||
const packageInstaller = new PackageInstaller({
|
||||
esClient: core.elasticsearch.client.asInternalUser,
|
||||
productDocClient,
|
||||
endpointManager,
|
||||
kibanaVersion: this.context.env.packageInfo.version,
|
||||
artifactsFolder: Path.join(getDataPath(), 'ai-kb-artifacts'),
|
||||
artifactRepositoryUrl: this.context.config.get().artifactRepositoryUrl,
|
||||
logger: this.logger.get('package-installer'),
|
||||
});
|
||||
|
||||
const searchService = new SearchService({
|
||||
esClient: core.elasticsearch.client.asInternalUser,
|
||||
logger: this.logger.get('search-service'),
|
||||
});
|
||||
|
||||
const documentationManager = new DocumentationManager({
|
||||
logger: this.logger.get('doc-manager'),
|
||||
docInstallClient: productDocClient,
|
||||
licensing,
|
||||
taskManager,
|
||||
auditService: core.security.audit,
|
||||
});
|
||||
|
||||
this.internalServices = {
|
||||
logger: this.logger,
|
||||
packageInstaller,
|
||||
installClient: productDocClient,
|
||||
documentationManager,
|
||||
licensing,
|
||||
taskManager,
|
||||
};
|
||||
|
||||
documentationManager.update().catch((err) => {
|
||||
this.logger.error(`Error scheduling product documentation update task: ${err.message}`);
|
||||
});
|
||||
|
||||
return {
|
||||
management: {
|
||||
install: documentationManager.install.bind(documentationManager),
|
||||
update: documentationManager.update.bind(documentationManager),
|
||||
uninstall: documentationManager.uninstall.bind(documentationManager),
|
||||
getStatus: documentationManager.getStatus.bind(documentationManager),
|
||||
},
|
||||
search: searchService.search.bind(searchService),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { IRouter } from '@kbn/core/server';
|
||||
import { registerInstallationRoutes } from './installation';
|
||||
import type { InternalServices } from '../types';
|
||||
|
||||
export const registerRoutes = ({
|
||||
router,
|
||||
getServices,
|
||||
}: {
|
||||
router: IRouter;
|
||||
getServices: () => InternalServices;
|
||||
}) => {
|
||||
registerInstallationRoutes({ getServices, router });
|
||||
};
|
||||
|
|
@ -0,0 +1,115 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { IRouter } from '@kbn/core/server';
|
||||
import {
|
||||
INSTALLATION_STATUS_API_PATH,
|
||||
INSTALL_ALL_API_PATH,
|
||||
UNINSTALL_ALL_API_PATH,
|
||||
InstallationStatusResponse,
|
||||
PerformInstallResponse,
|
||||
UninstallResponse,
|
||||
} from '../../common/http_api/installation';
|
||||
import type { InternalServices } from '../types';
|
||||
|
||||
export const registerInstallationRoutes = ({
|
||||
router,
|
||||
getServices,
|
||||
}: {
|
||||
router: IRouter;
|
||||
getServices: () => InternalServices;
|
||||
}) => {
|
||||
router.get(
|
||||
{
|
||||
path: INSTALLATION_STATUS_API_PATH,
|
||||
validate: false,
|
||||
options: {
|
||||
access: 'internal',
|
||||
security: {
|
||||
authz: {
|
||||
requiredPrivileges: ['manage_llm_product_doc'],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
async (ctx, req, res) => {
|
||||
const { installClient, documentationManager } = getServices();
|
||||
const installStatus = await installClient.getInstallationStatus();
|
||||
const { status: overallStatus } = await documentationManager.getStatus();
|
||||
|
||||
return res.ok<InstallationStatusResponse>({
|
||||
body: {
|
||||
perProducts: installStatus,
|
||||
overall: overallStatus,
|
||||
},
|
||||
});
|
||||
}
|
||||
);
|
||||
|
||||
router.post(
|
||||
{
|
||||
path: INSTALL_ALL_API_PATH,
|
||||
validate: false,
|
||||
options: {
|
||||
access: 'internal',
|
||||
security: {
|
||||
authz: {
|
||||
requiredPrivileges: ['manage_llm_product_doc'],
|
||||
},
|
||||
},
|
||||
timeout: { idleSocket: 20 * 60 * 1000 }, // install can take time.
|
||||
},
|
||||
},
|
||||
async (ctx, req, res) => {
|
||||
const { documentationManager } = getServices();
|
||||
|
||||
await documentationManager.install({
|
||||
request: req,
|
||||
force: false,
|
||||
wait: true,
|
||||
});
|
||||
|
||||
// check status after installation in case of failure
|
||||
const { status } = await documentationManager.getStatus();
|
||||
|
||||
return res.ok<PerformInstallResponse>({
|
||||
body: {
|
||||
installed: status === 'installed',
|
||||
},
|
||||
});
|
||||
}
|
||||
);
|
||||
|
||||
router.post(
|
||||
{
|
||||
path: UNINSTALL_ALL_API_PATH,
|
||||
validate: false,
|
||||
options: {
|
||||
access: 'internal',
|
||||
security: {
|
||||
authz: {
|
||||
requiredPrivileges: ['manage_llm_product_doc'],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
async (ctx, req, res) => {
|
||||
const { documentationManager } = getServices();
|
||||
|
||||
await documentationManager.uninstall({
|
||||
request: req,
|
||||
wait: true,
|
||||
});
|
||||
|
||||
return res.ok<UninstallResponse>({
|
||||
body: {
|
||||
success: true,
|
||||
},
|
||||
});
|
||||
}
|
||||
);
|
||||
};
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
export {
|
||||
productDocInstallStatusSavedObjectType,
|
||||
type ProductDocInstallStatusAttributes,
|
||||
} from './product_doc_install';
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { SavedObjectsType } from '@kbn/core/server';
|
||||
import type { ProductName } from '@kbn/product-doc-common';
|
||||
import { productDocInstallStatusSavedObjectTypeName } from '../../common/consts';
|
||||
import type { InstallationStatus } from '../../common/install_status';
|
||||
|
||||
/**
|
||||
* Interface describing the raw attributes of the product doc install SO type.
|
||||
* Contains more fields than the mappings, which only list
|
||||
* indexed fields.
|
||||
*/
|
||||
export interface ProductDocInstallStatusAttributes {
|
||||
product_name: ProductName;
|
||||
product_version: string;
|
||||
installation_status: InstallationStatus;
|
||||
last_installation_date?: number;
|
||||
last_installation_failure_reason?: string;
|
||||
index_name?: string;
|
||||
}
|
||||
|
||||
export const productDocInstallStatusSavedObjectType: SavedObjectsType<ProductDocInstallStatusAttributes> =
|
||||
{
|
||||
name: productDocInstallStatusSavedObjectTypeName,
|
||||
hidden: true,
|
||||
namespaceType: 'agnostic',
|
||||
mappings: {
|
||||
dynamic: false,
|
||||
properties: {
|
||||
product_name: { type: 'keyword' },
|
||||
product_version: { type: 'keyword' },
|
||||
installation_status: { type: 'keyword' },
|
||||
last_installation_date: { type: 'date' },
|
||||
index_name: { type: 'keyword' },
|
||||
},
|
||||
},
|
||||
management: {
|
||||
importableAndExportable: false,
|
||||
},
|
||||
modelVersions: {},
|
||||
};
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
export { ProductDocInstallClient } from './product_doc_install_service';
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { SavedObject } from '@kbn/core/server';
|
||||
import type { ProductDocInstallStatusAttributes } from '../../saved_objects';
|
||||
import { soToModel } from './model_conversion';
|
||||
|
||||
const createObj = (
|
||||
attrs: ProductDocInstallStatusAttributes
|
||||
): SavedObject<ProductDocInstallStatusAttributes> => {
|
||||
return {
|
||||
id: 'some-id',
|
||||
type: 'product-doc-install-status',
|
||||
attributes: attrs,
|
||||
references: [],
|
||||
};
|
||||
};
|
||||
|
||||
describe('soToModel', () => {
|
||||
it('converts the SO to the expected shape', () => {
|
||||
const input = createObj({
|
||||
product_name: 'kibana',
|
||||
product_version: '8.16',
|
||||
installation_status: 'installed',
|
||||
last_installation_date: 9000,
|
||||
index_name: '.kibana',
|
||||
});
|
||||
|
||||
const output = soToModel(input);
|
||||
|
||||
expect(output).toEqual({
|
||||
id: 'some-id',
|
||||
productName: 'kibana',
|
||||
productVersion: '8.16',
|
||||
indexName: '.kibana',
|
||||
installationStatus: 'installed',
|
||||
lastInstallationDate: expect.any(Date),
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { SavedObject } from '@kbn/core/server';
|
||||
import type { ProductDocInstallStatus } from '../../../common/install_status';
|
||||
import type { ProductDocInstallStatusAttributes } from '../../saved_objects';
|
||||
|
||||
export const soToModel = (
|
||||
so: SavedObject<ProductDocInstallStatusAttributes>
|
||||
): ProductDocInstallStatus => {
|
||||
return {
|
||||
id: so.id,
|
||||
productName: so.attributes.product_name,
|
||||
productVersion: so.attributes.product_version,
|
||||
installationStatus: so.attributes.installation_status,
|
||||
indexName: so.attributes.index_name,
|
||||
lastInstallationDate: so.attributes.last_installation_date
|
||||
? new Date(so.attributes.last_installation_date)
|
||||
: undefined,
|
||||
lastInstallationFailureReason: so.attributes.last_installation_failure_reason,
|
||||
};
|
||||
};
|
||||
|
|
@ -0,0 +1,65 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { SavedObjectsFindResult } from '@kbn/core/server';
|
||||
import { DocumentationProduct } from '@kbn/product-doc-common';
|
||||
import type { ProductDocInstallStatusAttributes as TypeAttributes } from '../../saved_objects';
|
||||
import { savedObjectsClientMock } from '@kbn/core/server/mocks';
|
||||
import { ProductDocInstallClient } from './product_doc_install_service';
|
||||
|
||||
const createObj = (attrs: TypeAttributes): SavedObjectsFindResult<TypeAttributes> => {
|
||||
return {
|
||||
id: attrs.product_name,
|
||||
type: 'type',
|
||||
references: [],
|
||||
attributes: attrs,
|
||||
score: 42,
|
||||
};
|
||||
};
|
||||
|
||||
describe('ProductDocInstallClient', () => {
|
||||
let soClient: ReturnType<typeof savedObjectsClientMock.create>;
|
||||
let service: ProductDocInstallClient;
|
||||
|
||||
beforeEach(() => {
|
||||
soClient = savedObjectsClientMock.create();
|
||||
service = new ProductDocInstallClient({ soClient });
|
||||
});
|
||||
|
||||
describe('getInstallationStatus', () => {
|
||||
it('returns the installation status based on existing entries', async () => {
|
||||
soClient.find.mockResolvedValue({
|
||||
saved_objects: [
|
||||
createObj({
|
||||
product_name: 'kibana',
|
||||
product_version: '8.15',
|
||||
installation_status: 'installed',
|
||||
}),
|
||||
createObj({
|
||||
product_name: 'elasticsearch',
|
||||
product_version: '8.15',
|
||||
installation_status: 'installing',
|
||||
}),
|
||||
],
|
||||
total: 2,
|
||||
per_page: 100,
|
||||
page: 1,
|
||||
});
|
||||
|
||||
const installStatus = await service.getInstallationStatus();
|
||||
|
||||
expect(Object.keys(installStatus).sort()).toEqual(Object.keys(DocumentationProduct).sort());
|
||||
expect(installStatus.kibana).toEqual({
|
||||
status: 'installed',
|
||||
version: '8.15',
|
||||
});
|
||||
expect(installStatus.security).toEqual({
|
||||
status: 'uninstalled',
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,89 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { SavedObjectsClientContract } from '@kbn/core/server';
|
||||
import { SavedObjectsErrorHelpers } from '@kbn/core-saved-objects-server';
|
||||
import { ProductName, DocumentationProduct } from '@kbn/product-doc-common';
|
||||
import type { ProductInstallState } from '../../../common/install_status';
|
||||
import { productDocInstallStatusSavedObjectTypeName as typeName } from '../../../common/consts';
|
||||
import type { ProductDocInstallStatusAttributes as TypeAttributes } from '../../saved_objects';
|
||||
|
||||
export class ProductDocInstallClient {
|
||||
private soClient: SavedObjectsClientContract;
|
||||
|
||||
constructor({ soClient }: { soClient: SavedObjectsClientContract }) {
|
||||
this.soClient = soClient;
|
||||
}
|
||||
|
||||
async getInstallationStatus(): Promise<Record<ProductName, ProductInstallState>> {
|
||||
const response = await this.soClient.find<TypeAttributes>({
|
||||
type: typeName,
|
||||
perPage: 100,
|
||||
});
|
||||
|
||||
const installStatus = Object.values(DocumentationProduct).reduce((memo, product) => {
|
||||
memo[product] = { status: 'uninstalled' };
|
||||
return memo;
|
||||
}, {} as Record<ProductName, ProductInstallState>);
|
||||
|
||||
response.saved_objects.forEach(({ attributes }) => {
|
||||
installStatus[attributes.product_name as ProductName] = {
|
||||
status: attributes.installation_status,
|
||||
version: attributes.product_version,
|
||||
};
|
||||
});
|
||||
|
||||
return installStatus;
|
||||
}
|
||||
|
||||
async setInstallationStarted(fields: { productName: ProductName; productVersion: string }) {
|
||||
const { productName, productVersion } = fields;
|
||||
const objectId = getObjectIdFromProductName(productName);
|
||||
const attributes = {
|
||||
product_name: productName,
|
||||
product_version: productVersion,
|
||||
installation_status: 'installing' as const,
|
||||
last_installation_failure_reason: '',
|
||||
};
|
||||
await this.soClient.update<TypeAttributes>(typeName, objectId, attributes, {
|
||||
upsert: attributes,
|
||||
});
|
||||
}
|
||||
|
||||
async setInstallationSuccessful(productName: ProductName, indexName: string) {
|
||||
const objectId = getObjectIdFromProductName(productName);
|
||||
await this.soClient.update<TypeAttributes>(typeName, objectId, {
|
||||
installation_status: 'installed',
|
||||
index_name: indexName,
|
||||
});
|
||||
}
|
||||
|
||||
async setInstallationFailed(productName: ProductName, failureReason: string) {
|
||||
const objectId = getObjectIdFromProductName(productName);
|
||||
await this.soClient.update<TypeAttributes>(typeName, objectId, {
|
||||
installation_status: 'error',
|
||||
last_installation_failure_reason: failureReason,
|
||||
});
|
||||
}
|
||||
|
||||
async setUninstalled(productName: ProductName) {
|
||||
const objectId = getObjectIdFromProductName(productName);
|
||||
try {
|
||||
await this.soClient.update<TypeAttributes>(typeName, objectId, {
|
||||
installation_status: 'uninstalled',
|
||||
last_installation_failure_reason: '',
|
||||
});
|
||||
} catch (e) {
|
||||
if (!SavedObjectsErrorHelpers.isNotFoundError(e)) {
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const getObjectIdFromProductName = (productName: ProductName) =>
|
||||
`kb-product-doc-${productName}-status`.toLowerCase();
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { ProductDocInstallClient } from './product_doc_install_service';
|
||||
|
||||
export type InstallClientMock = jest.Mocked<ProductDocInstallClient>;
|
||||
|
||||
const createInstallClientMock = (): InstallClientMock => {
|
||||
return {
|
||||
getInstallationStatus: jest.fn(),
|
||||
setInstallationStarted: jest.fn(),
|
||||
setInstallationSuccessful: jest.fn(),
|
||||
setInstallationFailed: jest.fn(),
|
||||
setUninstalled: jest.fn(),
|
||||
} as unknown as InstallClientMock;
|
||||
};
|
||||
|
||||
export const installClientMock = {
|
||||
create: createInstallClientMock,
|
||||
};
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { ILicense } from '@kbn/licensing-plugin/server';
|
||||
|
||||
export const checkLicense = (license: ILicense): boolean => {
|
||||
const result = license.check('elastic documentation', 'enterprise');
|
||||
return result.state === 'valid';
|
||||
};
|
||||
|
|
@ -0,0 +1,247 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { loggerMock, MockedLogger } from '@kbn/logging-mocks';
|
||||
import { securityServiceMock, httpServerMock } from '@kbn/core/server/mocks';
|
||||
import { taskManagerMock } from '@kbn/task-manager-plugin/server/mocks';
|
||||
import { licensingMock } from '@kbn/licensing-plugin/server/mocks';
|
||||
import type { ProductDocInstallClient } from '../doc_install_status';
|
||||
import { DocumentationManager } from './doc_manager';
|
||||
|
||||
jest.mock('../../tasks');
|
||||
import {
|
||||
scheduleInstallAllTask,
|
||||
scheduleUninstallAllTask,
|
||||
scheduleEnsureUpToDateTask,
|
||||
getTaskStatus,
|
||||
waitUntilTaskCompleted,
|
||||
} from '../../tasks';
|
||||
|
||||
const scheduleInstallAllTaskMock = scheduleInstallAllTask as jest.MockedFn<
|
||||
typeof scheduleInstallAllTask
|
||||
>;
|
||||
const scheduleUninstallAllTaskMock = scheduleUninstallAllTask as jest.MockedFn<
|
||||
typeof scheduleUninstallAllTask
|
||||
>;
|
||||
const scheduleEnsureUpToDateTaskMock = scheduleEnsureUpToDateTask as jest.MockedFn<
|
||||
typeof scheduleEnsureUpToDateTask
|
||||
>;
|
||||
const waitUntilTaskCompletedMock = waitUntilTaskCompleted as jest.MockedFn<
|
||||
typeof waitUntilTaskCompleted
|
||||
>;
|
||||
const getTaskStatusMock = getTaskStatus as jest.MockedFn<typeof getTaskStatus>;
|
||||
|
||||
describe('DocumentationManager', () => {
|
||||
let logger: MockedLogger;
|
||||
let taskManager: ReturnType<typeof taskManagerMock.createStart>;
|
||||
let licensing: ReturnType<typeof licensingMock.createStart>;
|
||||
let auditService: ReturnType<typeof securityServiceMock.createStart>['audit'];
|
||||
let docInstallClient: jest.Mocked<ProductDocInstallClient>;
|
||||
|
||||
let docManager: DocumentationManager;
|
||||
|
||||
beforeEach(() => {
|
||||
logger = loggerMock.create();
|
||||
taskManager = taskManagerMock.createStart();
|
||||
licensing = licensingMock.createStart();
|
||||
auditService = securityServiceMock.createStart().audit;
|
||||
|
||||
docInstallClient = {
|
||||
getInstallationStatus: jest.fn(),
|
||||
} as unknown as jest.Mocked<ProductDocInstallClient>;
|
||||
|
||||
docManager = new DocumentationManager({
|
||||
logger,
|
||||
taskManager,
|
||||
licensing,
|
||||
auditService,
|
||||
docInstallClient,
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
scheduleInstallAllTaskMock.mockReset();
|
||||
scheduleUninstallAllTaskMock.mockReset();
|
||||
scheduleEnsureUpToDateTaskMock.mockReset();
|
||||
waitUntilTaskCompletedMock.mockReset();
|
||||
getTaskStatusMock.mockReset();
|
||||
});
|
||||
|
||||
describe('#install', () => {
|
||||
beforeEach(() => {
|
||||
licensing.getLicense.mockResolvedValue(
|
||||
licensingMock.createLicense({ license: { type: 'enterprise' } })
|
||||
);
|
||||
|
||||
getTaskStatusMock.mockResolvedValue('not_scheduled');
|
||||
|
||||
docInstallClient.getInstallationStatus.mockResolvedValue({
|
||||
kibana: { status: 'uninstalled' },
|
||||
} as Awaited<ReturnType<ProductDocInstallClient['getInstallationStatus']>>);
|
||||
});
|
||||
|
||||
it('calls `scheduleInstallAllTask`', async () => {
|
||||
await docManager.install({});
|
||||
|
||||
expect(scheduleInstallAllTaskMock).toHaveBeenCalledTimes(1);
|
||||
expect(scheduleInstallAllTaskMock).toHaveBeenCalledWith({
|
||||
taskManager,
|
||||
logger,
|
||||
});
|
||||
|
||||
expect(waitUntilTaskCompletedMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('calls waitUntilTaskCompleted if wait=true', async () => {
|
||||
await docManager.install({ wait: true });
|
||||
|
||||
expect(scheduleInstallAllTaskMock).toHaveBeenCalledTimes(1);
|
||||
expect(waitUntilTaskCompletedMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('does not call scheduleInstallAllTask if already installed and not force', async () => {
|
||||
docInstallClient.getInstallationStatus.mockResolvedValue({
|
||||
kibana: { status: 'installed' },
|
||||
} as Awaited<ReturnType<ProductDocInstallClient['getInstallationStatus']>>);
|
||||
|
||||
await docManager.install({ wait: true });
|
||||
|
||||
expect(scheduleInstallAllTaskMock).not.toHaveBeenCalled();
|
||||
expect(waitUntilTaskCompletedMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('records an audit log when request is provided', async () => {
|
||||
const request = httpServerMock.createKibanaRequest();
|
||||
|
||||
const auditLog = auditService.withoutRequest;
|
||||
auditService.asScoped = jest.fn(() => auditLog);
|
||||
|
||||
await docManager.install({ force: false, wait: false, request });
|
||||
|
||||
expect(auditLog.log).toHaveBeenCalledTimes(1);
|
||||
expect(auditLog.log).toHaveBeenCalledWith({
|
||||
message: expect.any(String),
|
||||
event: {
|
||||
action: 'product_documentation_create',
|
||||
category: ['database'],
|
||||
type: ['creation'],
|
||||
outcome: 'unknown',
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('throws an error if license level is not sufficient', async () => {
|
||||
licensing.getLicense.mockResolvedValue(
|
||||
licensingMock.createLicense({ license: { type: 'basic' } })
|
||||
);
|
||||
|
||||
await expect(
|
||||
docManager.install({ force: false, wait: false })
|
||||
).rejects.toThrowErrorMatchingInlineSnapshot(
|
||||
`"Elastic documentation requires an enterprise license"`
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('#update', () => {
|
||||
beforeEach(() => {
|
||||
getTaskStatusMock.mockResolvedValue('not_scheduled');
|
||||
|
||||
docInstallClient.getInstallationStatus.mockResolvedValue({
|
||||
kibana: { status: 'uninstalled' },
|
||||
} as Awaited<ReturnType<ProductDocInstallClient['getInstallationStatus']>>);
|
||||
});
|
||||
|
||||
it('calls `scheduleEnsureUpToDateTask`', async () => {
|
||||
await docManager.update({});
|
||||
|
||||
expect(scheduleEnsureUpToDateTaskMock).toHaveBeenCalledTimes(1);
|
||||
expect(scheduleEnsureUpToDateTaskMock).toHaveBeenCalledWith({
|
||||
taskManager,
|
||||
logger,
|
||||
});
|
||||
|
||||
expect(waitUntilTaskCompletedMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('calls waitUntilTaskCompleted if wait=true', async () => {
|
||||
await docManager.update({ wait: true });
|
||||
|
||||
expect(scheduleEnsureUpToDateTaskMock).toHaveBeenCalledTimes(1);
|
||||
expect(waitUntilTaskCompletedMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('records an audit log when request is provided', async () => {
|
||||
const request = httpServerMock.createKibanaRequest();
|
||||
|
||||
const auditLog = auditService.withoutRequest;
|
||||
auditService.asScoped = jest.fn(() => auditLog);
|
||||
|
||||
await docManager.update({ wait: false, request });
|
||||
|
||||
expect(auditLog.log).toHaveBeenCalledTimes(1);
|
||||
expect(auditLog.log).toHaveBeenCalledWith({
|
||||
message: expect.any(String),
|
||||
event: {
|
||||
action: 'product_documentation_update',
|
||||
category: ['database'],
|
||||
type: ['change'],
|
||||
outcome: 'unknown',
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('#uninstall', () => {
|
||||
beforeEach(() => {
|
||||
getTaskStatusMock.mockResolvedValue('not_scheduled');
|
||||
|
||||
docInstallClient.getInstallationStatus.mockResolvedValue({
|
||||
kibana: { status: 'uninstalled' },
|
||||
} as Awaited<ReturnType<ProductDocInstallClient['getInstallationStatus']>>);
|
||||
});
|
||||
|
||||
it('calls `scheduleUninstallAllTask`', async () => {
|
||||
await docManager.uninstall({});
|
||||
|
||||
expect(scheduleUninstallAllTaskMock).toHaveBeenCalledTimes(1);
|
||||
expect(scheduleUninstallAllTaskMock).toHaveBeenCalledWith({
|
||||
taskManager,
|
||||
logger,
|
||||
});
|
||||
|
||||
expect(waitUntilTaskCompletedMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('calls waitUntilTaskCompleted if wait=true', async () => {
|
||||
await docManager.uninstall({ wait: true });
|
||||
|
||||
expect(scheduleUninstallAllTaskMock).toHaveBeenCalledTimes(1);
|
||||
expect(waitUntilTaskCompletedMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('records an audit log when request is provided', async () => {
|
||||
const request = httpServerMock.createKibanaRequest();
|
||||
|
||||
const auditLog = auditService.withoutRequest;
|
||||
auditService.asScoped = jest.fn(() => auditLog);
|
||||
|
||||
await docManager.uninstall({ wait: false, request });
|
||||
|
||||
expect(auditLog.log).toHaveBeenCalledTimes(1);
|
||||
expect(auditLog.log).toHaveBeenCalledWith({
|
||||
message: expect.any(String),
|
||||
event: {
|
||||
action: 'product_documentation_delete',
|
||||
category: ['database'],
|
||||
type: ['deletion'],
|
||||
outcome: 'unknown',
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,204 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { Logger } from '@kbn/logging';
|
||||
import type { CoreAuditService } from '@kbn/core/server';
|
||||
import { type TaskManagerStartContract, TaskStatus } from '@kbn/task-manager-plugin/server';
|
||||
import type { LicensingPluginStart } from '@kbn/licensing-plugin/server';
|
||||
import type { InstallationStatus } from '../../../common/install_status';
|
||||
import type { ProductDocInstallClient } from '../doc_install_status';
|
||||
import {
|
||||
INSTALL_ALL_TASK_ID,
|
||||
scheduleInstallAllTask,
|
||||
scheduleUninstallAllTask,
|
||||
scheduleEnsureUpToDateTask,
|
||||
getTaskStatus,
|
||||
waitUntilTaskCompleted,
|
||||
} from '../../tasks';
|
||||
import { checkLicense } from './check_license';
|
||||
import type {
|
||||
DocumentationManagerAPI,
|
||||
DocGetStatusResponse,
|
||||
DocInstallOptions,
|
||||
DocUninstallOptions,
|
||||
DocUpdateOptions,
|
||||
} from './types';
|
||||
|
||||
const TEN_MIN_IN_MS = 10 * 60 * 1000;
|
||||
|
||||
/**
|
||||
* High-level installation service, handling product documentation
|
||||
* installation as unary operations, abstracting away the fact
|
||||
* that documentation is composed of multiple entities.
|
||||
*/
|
||||
export class DocumentationManager implements DocumentationManagerAPI {
|
||||
private logger: Logger;
|
||||
private taskManager: TaskManagerStartContract;
|
||||
private licensing: LicensingPluginStart;
|
||||
private docInstallClient: ProductDocInstallClient;
|
||||
private auditService: CoreAuditService;
|
||||
|
||||
constructor({
|
||||
logger,
|
||||
taskManager,
|
||||
licensing,
|
||||
docInstallClient,
|
||||
auditService,
|
||||
}: {
|
||||
logger: Logger;
|
||||
taskManager: TaskManagerStartContract;
|
||||
licensing: LicensingPluginStart;
|
||||
docInstallClient: ProductDocInstallClient;
|
||||
auditService: CoreAuditService;
|
||||
}) {
|
||||
this.logger = logger;
|
||||
this.taskManager = taskManager;
|
||||
this.licensing = licensing;
|
||||
this.docInstallClient = docInstallClient;
|
||||
this.auditService = auditService;
|
||||
}
|
||||
|
||||
async install(options: DocInstallOptions = {}): Promise<void> {
|
||||
const { request, force = false, wait = false } = options;
|
||||
|
||||
const { status } = await this.getStatus();
|
||||
if (!force && status === 'installed') {
|
||||
return;
|
||||
}
|
||||
|
||||
const license = await this.licensing.getLicense();
|
||||
if (!checkLicense(license)) {
|
||||
throw new Error('Elastic documentation requires an enterprise license');
|
||||
}
|
||||
|
||||
const taskId = await scheduleInstallAllTask({
|
||||
taskManager: this.taskManager,
|
||||
logger: this.logger,
|
||||
});
|
||||
|
||||
if (request) {
|
||||
this.auditService.asScoped(request).log({
|
||||
message: `User is requesting installation of product documentation for AI Assistants. Task ID=[${taskId}]`,
|
||||
event: {
|
||||
action: 'product_documentation_create',
|
||||
category: ['database'],
|
||||
type: ['creation'],
|
||||
outcome: 'unknown',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
if (wait) {
|
||||
await waitUntilTaskCompleted({
|
||||
taskManager: this.taskManager,
|
||||
taskId,
|
||||
timeout: TEN_MIN_IN_MS,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async update(options: DocUpdateOptions = {}): Promise<void> {
|
||||
const { request, wait = false } = options;
|
||||
|
||||
const taskId = await scheduleEnsureUpToDateTask({
|
||||
taskManager: this.taskManager,
|
||||
logger: this.logger,
|
||||
});
|
||||
|
||||
if (request) {
|
||||
this.auditService.asScoped(request).log({
|
||||
message: `User is requesting update of product documentation for AI Assistants. Task ID=[${taskId}]`,
|
||||
event: {
|
||||
action: 'product_documentation_update',
|
||||
category: ['database'],
|
||||
type: ['change'],
|
||||
outcome: 'unknown',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
if (wait) {
|
||||
await waitUntilTaskCompleted({
|
||||
taskManager: this.taskManager,
|
||||
taskId,
|
||||
timeout: TEN_MIN_IN_MS,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async uninstall(options: DocUninstallOptions = {}): Promise<void> {
|
||||
const { request, wait = false } = options;
|
||||
|
||||
const taskId = await scheduleUninstallAllTask({
|
||||
taskManager: this.taskManager,
|
||||
logger: this.logger,
|
||||
});
|
||||
|
||||
if (request) {
|
||||
this.auditService.asScoped(request).log({
|
||||
message: `User is requesting deletion of product documentation for AI Assistants. Task ID=[${taskId}]`,
|
||||
event: {
|
||||
action: 'product_documentation_delete',
|
||||
category: ['database'],
|
||||
type: ['deletion'],
|
||||
outcome: 'unknown',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
if (wait) {
|
||||
await waitUntilTaskCompleted({
|
||||
taskManager: this.taskManager,
|
||||
taskId,
|
||||
timeout: TEN_MIN_IN_MS,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async getStatus(): Promise<DocGetStatusResponse> {
|
||||
const taskStatus = await getTaskStatus({
|
||||
taskManager: this.taskManager,
|
||||
taskId: INSTALL_ALL_TASK_ID,
|
||||
});
|
||||
if (taskStatus !== 'not_scheduled') {
|
||||
const status = convertTaskStatus(taskStatus);
|
||||
if (status !== 'unknown') {
|
||||
return { status };
|
||||
}
|
||||
}
|
||||
|
||||
const installStatus = await this.docInstallClient.getInstallationStatus();
|
||||
const overallStatus = getOverallStatus(Object.values(installStatus).map((v) => v.status));
|
||||
return { status: overallStatus };
|
||||
}
|
||||
}
|
||||
|
||||
const convertTaskStatus = (taskStatus: TaskStatus): InstallationStatus | 'unknown' => {
|
||||
switch (taskStatus) {
|
||||
case TaskStatus.Idle:
|
||||
case TaskStatus.Claiming:
|
||||
case TaskStatus.Running:
|
||||
return 'installing';
|
||||
case TaskStatus.Failed:
|
||||
return 'error';
|
||||
case TaskStatus.Unrecognized:
|
||||
case TaskStatus.DeadLetter:
|
||||
case TaskStatus.ShouldDelete:
|
||||
default:
|
||||
return 'unknown';
|
||||
}
|
||||
};
|
||||
|
||||
const getOverallStatus = (statuses: InstallationStatus[]): InstallationStatus => {
|
||||
const statusOrder: InstallationStatus[] = ['error', 'installing', 'uninstalled', 'installed'];
|
||||
for (const status of statusOrder) {
|
||||
if (statuses.includes(status)) {
|
||||
return status;
|
||||
}
|
||||
}
|
||||
return 'installed';
|
||||
};
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
export { DocumentationManager } from './doc_manager';
|
||||
export type {
|
||||
DocumentationManagerAPI,
|
||||
DocUninstallOptions,
|
||||
DocInstallOptions,
|
||||
DocUpdateOptions,
|
||||
DocGetStatusResponse,
|
||||
} from './types';
|
||||
|
|
@ -0,0 +1,98 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { KibanaRequest } from '@kbn/core/server';
|
||||
import type { InstallationStatus } from '../../../common/install_status';
|
||||
|
||||
/**
|
||||
* APIs to manage the product documentation.
|
||||
*/
|
||||
export interface DocumentationManagerAPI {
|
||||
/**
|
||||
* Install the product documentation.
|
||||
* By default, will only try to install if not already present.
|
||||
* Can use the `force` option to forcefully reinstall.
|
||||
*/
|
||||
install(options?: DocInstallOptions): Promise<void>;
|
||||
/**
|
||||
* Update the product documentation to the latest version.
|
||||
* No-op if the product documentation is not currently installed.
|
||||
*/
|
||||
update(options?: DocUpdateOptions): Promise<void>;
|
||||
/**
|
||||
* Uninstall the product documentation.
|
||||
* No-op if the product documentation is not currently installed.
|
||||
*/
|
||||
uninstall(options?: DocUninstallOptions): Promise<void>;
|
||||
/**
|
||||
* Returns the overall installation status of the documentation.
|
||||
*/
|
||||
getStatus(): Promise<DocGetStatusResponse>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return type for {@link DocumentationManagerAPI.getStatus}
|
||||
*/
|
||||
export interface DocGetStatusResponse {
|
||||
status: InstallationStatus;
|
||||
}
|
||||
|
||||
/**
|
||||
* Options for {@link DocumentationManagerAPI.install}
|
||||
*/
|
||||
export interface DocInstallOptions {
|
||||
/**
|
||||
* When the operation was requested by a user, the request that initiated it.
|
||||
*
|
||||
* If not provided, the call will be considered as being done on behalf of system.
|
||||
*/
|
||||
request?: KibanaRequest;
|
||||
/**
|
||||
* If true, will reinstall the documentation even if already present.
|
||||
* Defaults to `false`
|
||||
*/
|
||||
force?: boolean;
|
||||
/**
|
||||
* If true, the returned promise will wait until the update task has completed before resolving.
|
||||
* Defaults to `false`
|
||||
*/
|
||||
wait?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Options for {@link DocumentationManagerAPI.uninstall}
|
||||
*/
|
||||
export interface DocUninstallOptions {
|
||||
/**
|
||||
* When the operation was requested by a user, the request that initiated it.
|
||||
*
|
||||
* If not provided, the call will be considered as being done on behalf of system.
|
||||
*/
|
||||
request?: KibanaRequest;
|
||||
/**
|
||||
* If true, the returned promise will wait until the update task has completed before resolving.
|
||||
* Defaults to `false`
|
||||
*/
|
||||
wait?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Options for {@link DocumentationManagerAPI.update}
|
||||
*/
|
||||
export interface DocUpdateOptions {
|
||||
/**
|
||||
* When the operation was requested by a user, the request that initiated it.
|
||||
*
|
||||
* If not provided, the call will be considered as being done on behalf of system.
|
||||
*/
|
||||
request?: KibanaRequest;
|
||||
/**
|
||||
* If true, the returned promise will wait until the update task has completed before resolving.
|
||||
* Defaults to `false`
|
||||
*/
|
||||
wait?: boolean;
|
||||
}
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { loggerMock, type MockedLogger } from '@kbn/logging-mocks';
|
||||
import { elasticsearchServiceMock } from '@kbn/core/server/mocks';
|
||||
import { InferenceEndpointManager } from './endpoint_manager';
|
||||
|
||||
jest.mock('./utils');
|
||||
import { installElser, getModelInstallStatus, waitUntilModelDeployed } from './utils';
|
||||
const installElserMock = installElser as jest.MockedFn<typeof installElser>;
|
||||
const getModelInstallStatusMock = getModelInstallStatus as jest.MockedFn<
|
||||
typeof getModelInstallStatus
|
||||
>;
|
||||
const waitUntilModelDeployedMock = waitUntilModelDeployed as jest.MockedFn<
|
||||
typeof waitUntilModelDeployed
|
||||
>;
|
||||
|
||||
describe('InferenceEndpointManager', () => {
|
||||
let logger: MockedLogger;
|
||||
let esClient: ReturnType<typeof elasticsearchServiceMock.createElasticsearchClient>;
|
||||
let endpointManager: InferenceEndpointManager;
|
||||
|
||||
beforeEach(() => {
|
||||
logger = loggerMock.create();
|
||||
esClient = elasticsearchServiceMock.createElasticsearchClient();
|
||||
|
||||
endpointManager = new InferenceEndpointManager({ esClient, logger });
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
installElserMock.mockReset();
|
||||
getModelInstallStatusMock.mockReset();
|
||||
waitUntilModelDeployedMock.mockReset();
|
||||
});
|
||||
|
||||
describe('#ensureInternalElserInstalled', () => {
|
||||
it('installs ELSER if not already installed', async () => {
|
||||
getModelInstallStatusMock.mockResolvedValue({ installed: true });
|
||||
|
||||
await endpointManager.ensureInternalElserInstalled();
|
||||
|
||||
expect(installElserMock).not.toHaveBeenCalled();
|
||||
expect(waitUntilModelDeployedMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
it('does not install ELSER if already present', async () => {
|
||||
getModelInstallStatusMock.mockResolvedValue({ installed: false });
|
||||
|
||||
await endpointManager.ensureInternalElserInstalled();
|
||||
|
||||
expect(installElserMock).toHaveBeenCalledTimes(1);
|
||||
expect(waitUntilModelDeployedMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { ElasticsearchClient, Logger } from '@kbn/core/server';
|
||||
import { internalElserInferenceId } from '../../../common/consts';
|
||||
import { installElser, getModelInstallStatus, waitUntilModelDeployed } from './utils';
|
||||
|
||||
export class InferenceEndpointManager {
|
||||
private readonly log: Logger;
|
||||
private readonly esClient: ElasticsearchClient;
|
||||
|
||||
constructor({ logger, esClient }: { logger: Logger; esClient: ElasticsearchClient }) {
|
||||
this.log = logger;
|
||||
this.esClient = esClient;
|
||||
}
|
||||
|
||||
async ensureInternalElserInstalled() {
|
||||
const { installed } = await getModelInstallStatus({
|
||||
inferenceId: internalElserInferenceId,
|
||||
client: this.esClient,
|
||||
log: this.log,
|
||||
});
|
||||
if (!installed) {
|
||||
await installElser({
|
||||
inferenceId: internalElserInferenceId,
|
||||
client: this.esClient,
|
||||
log: this.log,
|
||||
});
|
||||
}
|
||||
|
||||
await waitUntilModelDeployed({
|
||||
modelId: internalElserInferenceId,
|
||||
client: this.esClient,
|
||||
log: this.log,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
export { InferenceEndpointManager } from './endpoint_manager';
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { InferenceEndpointManager } from './endpoint_manager';
|
||||
|
||||
export type InferenceEndpointManagerMock = jest.Mocked<InferenceEndpointManager>;
|
||||
|
||||
const createMock = (): InferenceEndpointManagerMock => {
|
||||
return {
|
||||
ensureInternalElserInstalled: jest.fn(),
|
||||
} as unknown as InferenceEndpointManagerMock;
|
||||
};
|
||||
|
||||
export const inferenceManagerMock = {
|
||||
create: createMock,
|
||||
};
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { InferenceTaskType } from '@elastic/elasticsearch/lib/api/types';
|
||||
import type { ElasticsearchClient, Logger } from '@kbn/core/server';
|
||||
|
||||
export const getModelInstallStatus = async ({
|
||||
inferenceId,
|
||||
taskType = 'sparse_embedding',
|
||||
client,
|
||||
}: {
|
||||
inferenceId: string;
|
||||
taskType?: InferenceTaskType;
|
||||
client: ElasticsearchClient;
|
||||
log: Logger;
|
||||
}) => {
|
||||
const getInferenceRes = await client.inference.get(
|
||||
{
|
||||
task_type: taskType,
|
||||
inference_id: inferenceId,
|
||||
},
|
||||
{ ignore: [404] }
|
||||
);
|
||||
|
||||
const installed = (getInferenceRes.endpoints ?? []).some(
|
||||
(endpoint) => endpoint.inference_id === inferenceId
|
||||
);
|
||||
|
||||
return { installed };
|
||||
};
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
export { waitUntilModelDeployed } from './wait_until_model_deployed';
|
||||
export { getModelInstallStatus } from './get_model_install_status';
|
||||
export { installElser } from './install_elser';
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { ElasticsearchClient, Logger } from '@kbn/core/server';
|
||||
|
||||
export const installElser = async ({
|
||||
inferenceId,
|
||||
client,
|
||||
log,
|
||||
}: {
|
||||
inferenceId: string;
|
||||
client: ElasticsearchClient;
|
||||
log: Logger;
|
||||
}) => {
|
||||
await client.inference.put(
|
||||
{
|
||||
task_type: 'sparse_embedding',
|
||||
inference_id: inferenceId,
|
||||
inference_config: {
|
||||
service: 'elasticsearch',
|
||||
service_settings: {
|
||||
num_allocations: 1,
|
||||
num_threads: 1,
|
||||
model_id: '.elser_model_2',
|
||||
},
|
||||
task_settings: {},
|
||||
},
|
||||
},
|
||||
{ requestTimeout: 5 * 60 * 1000 }
|
||||
);
|
||||
};
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { ElasticsearchClient, Logger } from '@kbn/core/server';
|
||||
|
||||
export const waitUntilModelDeployed = async ({
|
||||
modelId,
|
||||
client,
|
||||
log,
|
||||
maxRetries = 20,
|
||||
delay = 2000,
|
||||
}: {
|
||||
modelId: string;
|
||||
client: ElasticsearchClient;
|
||||
log: Logger;
|
||||
maxRetries?: number;
|
||||
delay?: number;
|
||||
}) => {
|
||||
for (let i = 0; i < maxRetries; i++) {
|
||||
const statsRes = await client.ml.getTrainedModelsStats({
|
||||
model_id: modelId,
|
||||
});
|
||||
const deploymentStats = statsRes.trained_model_stats[0]?.deployment_stats;
|
||||
if (!deploymentStats || deploymentStats.nodes.length === 0) {
|
||||
log.debug(`ML model [${modelId}] was not deployed - attempt ${i + 1} of ${maxRetries}`);
|
||||
await sleep(delay);
|
||||
continue;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
throw new Error(`Timeout waiting for ML model ${modelId} to be deployed`);
|
||||
};
|
||||
|
||||
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
export { PackageInstaller } from './package_installer';
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
export const validateArtifactArchiveMock = jest.fn();
|
||||
export const fetchArtifactVersionsMock = jest.fn();
|
||||
export const createIndexMock = jest.fn();
|
||||
export const populateIndexMock = jest.fn();
|
||||
|
||||
jest.doMock('./steps', () => {
|
||||
const actual = jest.requireActual('./steps');
|
||||
return {
|
||||
...actual,
|
||||
validateArtifactArchive: validateArtifactArchiveMock,
|
||||
fetchArtifactVersions: fetchArtifactVersionsMock,
|
||||
createIndex: createIndexMock,
|
||||
populateIndex: populateIndexMock,
|
||||
};
|
||||
});
|
||||
|
||||
export const downloadToDiskMock = jest.fn();
|
||||
export const openZipArchiveMock = jest.fn();
|
||||
export const loadMappingFileMock = jest.fn();
|
||||
|
||||
jest.doMock('./utils', () => {
|
||||
const actual = jest.requireActual('./utils');
|
||||
return {
|
||||
...actual,
|
||||
downloadToDisk: downloadToDiskMock,
|
||||
openZipArchive: openZipArchiveMock,
|
||||
loadMappingFile: loadMappingFileMock,
|
||||
};
|
||||
});
|
||||
|
|
@ -0,0 +1,255 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import {
|
||||
downloadToDiskMock,
|
||||
createIndexMock,
|
||||
populateIndexMock,
|
||||
loadMappingFileMock,
|
||||
openZipArchiveMock,
|
||||
validateArtifactArchiveMock,
|
||||
fetchArtifactVersionsMock,
|
||||
} from './package_installer.test.mocks';
|
||||
|
||||
import {
|
||||
getArtifactName,
|
||||
getProductDocIndexName,
|
||||
DocumentationProduct,
|
||||
ProductName,
|
||||
} from '@kbn/product-doc-common';
|
||||
import { elasticsearchServiceMock } from '@kbn/core/server/mocks';
|
||||
import { loggerMock, type MockedLogger } from '@kbn/logging-mocks';
|
||||
import { installClientMock } from '../doc_install_status/service.mock';
|
||||
import { inferenceManagerMock } from '../inference_endpoint/service.mock';
|
||||
import type { ProductInstallState } from '../../../common/install_status';
|
||||
import { PackageInstaller } from './package_installer';
|
||||
|
||||
const artifactsFolder = '/lost';
|
||||
const artifactRepositoryUrl = 'https://repository.com';
|
||||
const kibanaVersion = '8.16.3';
|
||||
|
||||
const callOrder = (fn: { mock: { invocationCallOrder: number[] } }): number => {
|
||||
return fn.mock.invocationCallOrder[0];
|
||||
};
|
||||
|
||||
describe('PackageInstaller', () => {
|
||||
let logger: MockedLogger;
|
||||
let esClient: ReturnType<typeof elasticsearchServiceMock.createElasticsearchClient>;
|
||||
let productDocClient: ReturnType<typeof installClientMock.create>;
|
||||
let endpointManager: ReturnType<typeof inferenceManagerMock.create>;
|
||||
|
||||
let packageInstaller: PackageInstaller;
|
||||
|
||||
beforeEach(() => {
|
||||
logger = loggerMock.create();
|
||||
esClient = elasticsearchServiceMock.createElasticsearchClient();
|
||||
productDocClient = installClientMock.create();
|
||||
endpointManager = inferenceManagerMock.create();
|
||||
packageInstaller = new PackageInstaller({
|
||||
artifactsFolder,
|
||||
logger,
|
||||
esClient,
|
||||
productDocClient,
|
||||
endpointManager,
|
||||
artifactRepositoryUrl,
|
||||
kibanaVersion,
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
downloadToDiskMock.mockReset();
|
||||
createIndexMock.mockReset();
|
||||
populateIndexMock.mockReset();
|
||||
loadMappingFileMock.mockReset();
|
||||
openZipArchiveMock.mockReset();
|
||||
validateArtifactArchiveMock.mockReset();
|
||||
fetchArtifactVersionsMock.mockReset();
|
||||
});
|
||||
|
||||
describe('installPackage', () => {
|
||||
it('calls the steps with the right parameters', async () => {
|
||||
const zipArchive = {
|
||||
close: jest.fn(),
|
||||
};
|
||||
openZipArchiveMock.mockResolvedValue(zipArchive);
|
||||
|
||||
const mappings = Symbol('mappings');
|
||||
loadMappingFileMock.mockResolvedValue(mappings);
|
||||
|
||||
await packageInstaller.installPackage({ productName: 'kibana', productVersion: '8.16' });
|
||||
|
||||
const artifactName = getArtifactName({
|
||||
productName: 'kibana',
|
||||
productVersion: '8.16',
|
||||
});
|
||||
const indexName = getProductDocIndexName('kibana');
|
||||
expect(endpointManager.ensureInternalElserInstalled).toHaveBeenCalledTimes(1);
|
||||
|
||||
expect(downloadToDiskMock).toHaveBeenCalledTimes(1);
|
||||
expect(downloadToDiskMock).toHaveBeenCalledWith(
|
||||
`${artifactRepositoryUrl}/${artifactName}`,
|
||||
`${artifactsFolder}/${artifactName}`
|
||||
);
|
||||
|
||||
expect(openZipArchiveMock).toHaveBeenCalledTimes(1);
|
||||
expect(openZipArchiveMock).toHaveBeenCalledWith(`${artifactsFolder}/${artifactName}`);
|
||||
|
||||
expect(loadMappingFileMock).toHaveBeenCalledTimes(1);
|
||||
expect(loadMappingFileMock).toHaveBeenCalledWith(zipArchive);
|
||||
|
||||
expect(createIndexMock).toHaveBeenCalledTimes(1);
|
||||
expect(createIndexMock).toHaveBeenCalledWith({
|
||||
indexName,
|
||||
mappings,
|
||||
esClient,
|
||||
log: logger,
|
||||
});
|
||||
|
||||
expect(populateIndexMock).toHaveBeenCalledTimes(1);
|
||||
expect(populateIndexMock).toHaveBeenCalledWith({
|
||||
indexName,
|
||||
archive: zipArchive,
|
||||
esClient,
|
||||
log: logger,
|
||||
});
|
||||
|
||||
expect(productDocClient.setInstallationSuccessful).toHaveBeenCalledTimes(1);
|
||||
expect(productDocClient.setInstallationSuccessful).toHaveBeenCalledWith('kibana', indexName);
|
||||
|
||||
expect(zipArchive.close).toHaveBeenCalledTimes(1);
|
||||
|
||||
expect(productDocClient.setInstallationFailed).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('executes the steps in the right order', async () => {
|
||||
await packageInstaller.installPackage({ productName: 'kibana', productVersion: '8.16' });
|
||||
|
||||
expect(callOrder(endpointManager.ensureInternalElserInstalled)).toBeLessThan(
|
||||
callOrder(downloadToDiskMock)
|
||||
);
|
||||
expect(callOrder(downloadToDiskMock)).toBeLessThan(callOrder(openZipArchiveMock));
|
||||
expect(callOrder(openZipArchiveMock)).toBeLessThan(callOrder(loadMappingFileMock));
|
||||
expect(callOrder(loadMappingFileMock)).toBeLessThan(callOrder(createIndexMock));
|
||||
expect(callOrder(createIndexMock)).toBeLessThan(callOrder(populateIndexMock));
|
||||
expect(callOrder(populateIndexMock)).toBeLessThan(
|
||||
callOrder(productDocClient.setInstallationSuccessful)
|
||||
);
|
||||
});
|
||||
|
||||
it('closes the archive and calls setInstallationFailed if the installation fails', async () => {
|
||||
const zipArchive = {
|
||||
close: jest.fn(),
|
||||
};
|
||||
openZipArchiveMock.mockResolvedValue(zipArchive);
|
||||
|
||||
populateIndexMock.mockImplementation(async () => {
|
||||
throw new Error('something bad');
|
||||
});
|
||||
|
||||
await expect(
|
||||
packageInstaller.installPackage({ productName: 'kibana', productVersion: '8.16' })
|
||||
).rejects.toThrowError();
|
||||
|
||||
expect(productDocClient.setInstallationSuccessful).not.toHaveBeenCalled();
|
||||
|
||||
expect(zipArchive.close).toHaveBeenCalledTimes(1);
|
||||
|
||||
expect(logger.error).toHaveBeenCalledTimes(1);
|
||||
expect(logger.error).toHaveBeenCalledWith(
|
||||
expect.stringContaining('Error during documentation installation')
|
||||
);
|
||||
|
||||
expect(productDocClient.setInstallationFailed).toHaveBeenCalledTimes(1);
|
||||
expect(productDocClient.setInstallationFailed).toHaveBeenCalledWith(
|
||||
'kibana',
|
||||
'something bad'
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('installALl', () => {
|
||||
it('installs all the packages to their latest version', async () => {
|
||||
jest.spyOn(packageInstaller, 'installPackage');
|
||||
|
||||
fetchArtifactVersionsMock.mockResolvedValue({
|
||||
kibana: ['8.15', '8.16'],
|
||||
elasticsearch: ['8.15'],
|
||||
});
|
||||
|
||||
await packageInstaller.installAll({});
|
||||
|
||||
expect(packageInstaller.installPackage).toHaveBeenCalledTimes(2);
|
||||
|
||||
expect(packageInstaller.installPackage).toHaveBeenCalledWith({
|
||||
productName: 'kibana',
|
||||
productVersion: '8.16',
|
||||
});
|
||||
expect(packageInstaller.installPackage).toHaveBeenCalledWith({
|
||||
productName: 'elasticsearch',
|
||||
productVersion: '8.15',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('ensureUpToDate', () => {
|
||||
it('updates the installed packages to the latest version', async () => {
|
||||
fetchArtifactVersionsMock.mockResolvedValue({
|
||||
kibana: ['8.15', '8.16'],
|
||||
security: ['8.15', '8.16'],
|
||||
elasticsearch: ['8.15'],
|
||||
});
|
||||
|
||||
productDocClient.getInstallationStatus.mockResolvedValue({
|
||||
kibana: { status: 'installed', version: '8.15' },
|
||||
security: { status: 'installed', version: '8.16' },
|
||||
elasticsearch: { status: 'uninstalled' },
|
||||
} as Record<ProductName, ProductInstallState>);
|
||||
|
||||
jest.spyOn(packageInstaller, 'installPackage');
|
||||
|
||||
await packageInstaller.ensureUpToDate({});
|
||||
|
||||
expect(packageInstaller.installPackage).toHaveBeenCalledTimes(1);
|
||||
expect(packageInstaller.installPackage).toHaveBeenCalledWith({
|
||||
productName: 'kibana',
|
||||
productVersion: '8.16',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('uninstallPackage', () => {
|
||||
it('performs the uninstall steps', async () => {
|
||||
await packageInstaller.uninstallPackage({ productName: 'kibana' });
|
||||
|
||||
expect(esClient.indices.delete).toHaveBeenCalledTimes(1);
|
||||
expect(esClient.indices.delete).toHaveBeenCalledWith(
|
||||
{
|
||||
index: getProductDocIndexName('kibana'),
|
||||
},
|
||||
expect.objectContaining({ ignore: [404] })
|
||||
);
|
||||
|
||||
expect(productDocClient.setUninstalled).toHaveBeenCalledTimes(1);
|
||||
expect(productDocClient.setUninstalled).toHaveBeenCalledWith('kibana');
|
||||
});
|
||||
});
|
||||
|
||||
describe('uninstallAll', () => {
|
||||
it('calls uninstall for all packages', async () => {
|
||||
jest.spyOn(packageInstaller, 'uninstallPackage');
|
||||
|
||||
await packageInstaller.uninstallAll();
|
||||
|
||||
expect(packageInstaller.uninstallPackage).toHaveBeenCalledTimes(
|
||||
Object.keys(DocumentationProduct).length
|
||||
);
|
||||
Object.values(DocumentationProduct).forEach((productName) => {
|
||||
expect(packageInstaller.uninstallPackage).toHaveBeenCalledWith({ productName });
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,218 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { Logger } from '@kbn/logging';
|
||||
import type { ElasticsearchClient } from '@kbn/core/server';
|
||||
import {
|
||||
getArtifactName,
|
||||
getProductDocIndexName,
|
||||
DocumentationProduct,
|
||||
type ProductName,
|
||||
} from '@kbn/product-doc-common';
|
||||
import type { ProductDocInstallClient } from '../doc_install_status';
|
||||
import type { InferenceEndpointManager } from '../inference_endpoint';
|
||||
import { downloadToDisk, openZipArchive, loadMappingFile, type ZipArchive } from './utils';
|
||||
import { majorMinor, latestVersion } from './utils/semver';
|
||||
import {
|
||||
validateArtifactArchive,
|
||||
fetchArtifactVersions,
|
||||
createIndex,
|
||||
populateIndex,
|
||||
} from './steps';
|
||||
|
||||
interface PackageInstallerOpts {
|
||||
artifactsFolder: string;
|
||||
logger: Logger;
|
||||
esClient: ElasticsearchClient;
|
||||
productDocClient: ProductDocInstallClient;
|
||||
endpointManager: InferenceEndpointManager;
|
||||
artifactRepositoryUrl: string;
|
||||
kibanaVersion: string;
|
||||
}
|
||||
|
||||
export class PackageInstaller {
|
||||
private readonly log: Logger;
|
||||
private readonly artifactsFolder: string;
|
||||
private readonly esClient: ElasticsearchClient;
|
||||
private readonly productDocClient: ProductDocInstallClient;
|
||||
private readonly endpointManager: InferenceEndpointManager;
|
||||
private readonly artifactRepositoryUrl: string;
|
||||
private readonly currentVersion: string;
|
||||
|
||||
constructor({
|
||||
artifactsFolder,
|
||||
logger,
|
||||
esClient,
|
||||
productDocClient,
|
||||
endpointManager,
|
||||
artifactRepositoryUrl,
|
||||
kibanaVersion,
|
||||
}: PackageInstallerOpts) {
|
||||
this.esClient = esClient;
|
||||
this.productDocClient = productDocClient;
|
||||
this.artifactsFolder = artifactsFolder;
|
||||
this.endpointManager = endpointManager;
|
||||
this.artifactRepositoryUrl = artifactRepositoryUrl;
|
||||
this.currentVersion = majorMinor(kibanaVersion);
|
||||
this.log = logger;
|
||||
}
|
||||
|
||||
/**
|
||||
* Make sure that the currently installed doc packages are up to date.
|
||||
* Will not upgrade products that are not already installed
|
||||
*/
|
||||
async ensureUpToDate({}: {}) {
|
||||
const [repositoryVersions, installStatuses] = await Promise.all([
|
||||
fetchArtifactVersions({
|
||||
artifactRepositoryUrl: this.artifactRepositoryUrl,
|
||||
}),
|
||||
this.productDocClient.getInstallationStatus(),
|
||||
]);
|
||||
|
||||
const toUpdate: Array<{
|
||||
productName: ProductName;
|
||||
productVersion: string;
|
||||
}> = [];
|
||||
Object.entries(installStatuses).forEach(([productName, productState]) => {
|
||||
if (productState.status === 'uninstalled') {
|
||||
return;
|
||||
}
|
||||
const availableVersions = repositoryVersions[productName as ProductName];
|
||||
if (!availableVersions || !availableVersions.length) {
|
||||
return;
|
||||
}
|
||||
const selectedVersion = selectVersion(this.currentVersion, availableVersions);
|
||||
if (productState.version !== selectedVersion) {
|
||||
toUpdate.push({
|
||||
productName: productName as ProductName,
|
||||
productVersion: selectedVersion,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
for (const { productName, productVersion } of toUpdate) {
|
||||
await this.installPackage({
|
||||
productName,
|
||||
productVersion,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async installAll({}: {}) {
|
||||
const repositoryVersions = await fetchArtifactVersions({
|
||||
artifactRepositoryUrl: this.artifactRepositoryUrl,
|
||||
});
|
||||
const allProducts = Object.values(DocumentationProduct) as ProductName[];
|
||||
for (const productName of allProducts) {
|
||||
const availableVersions = repositoryVersions[productName];
|
||||
if (!availableVersions || !availableVersions.length) {
|
||||
this.log.warn(`No version found for product [${productName}]`);
|
||||
continue;
|
||||
}
|
||||
const selectedVersion = selectVersion(this.currentVersion, availableVersions);
|
||||
|
||||
await this.installPackage({
|
||||
productName,
|
||||
productVersion: selectedVersion,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async installPackage({
|
||||
productName,
|
||||
productVersion,
|
||||
}: {
|
||||
productName: ProductName;
|
||||
productVersion: string;
|
||||
}) {
|
||||
this.log.info(
|
||||
`Starting installing documentation for product [${productName}] and version [${productVersion}]`
|
||||
);
|
||||
|
||||
productVersion = majorMinor(productVersion);
|
||||
|
||||
await this.uninstallPackage({ productName });
|
||||
|
||||
let zipArchive: ZipArchive | undefined;
|
||||
try {
|
||||
await this.productDocClient.setInstallationStarted({
|
||||
productName,
|
||||
productVersion,
|
||||
});
|
||||
|
||||
await this.endpointManager.ensureInternalElserInstalled();
|
||||
|
||||
const artifactFileName = getArtifactName({ productName, productVersion });
|
||||
const artifactUrl = `${this.artifactRepositoryUrl}/${artifactFileName}`;
|
||||
const artifactPath = `${this.artifactsFolder}/${artifactFileName}`;
|
||||
|
||||
this.log.debug(`Downloading from [${artifactUrl}] to [${artifactPath}]`);
|
||||
await downloadToDisk(artifactUrl, artifactPath);
|
||||
|
||||
zipArchive = await openZipArchive(artifactPath);
|
||||
|
||||
validateArtifactArchive(zipArchive);
|
||||
|
||||
const mappings = await loadMappingFile(zipArchive);
|
||||
|
||||
const indexName = getProductDocIndexName(productName);
|
||||
|
||||
await createIndex({
|
||||
indexName,
|
||||
mappings,
|
||||
esClient: this.esClient,
|
||||
log: this.log,
|
||||
});
|
||||
|
||||
await populateIndex({
|
||||
indexName,
|
||||
archive: zipArchive,
|
||||
esClient: this.esClient,
|
||||
log: this.log,
|
||||
});
|
||||
await this.productDocClient.setInstallationSuccessful(productName, indexName);
|
||||
|
||||
this.log.info(
|
||||
`Documentation installation successful for product [${productName}] and version [${productVersion}]`
|
||||
);
|
||||
} catch (e) {
|
||||
this.log.error(
|
||||
`Error during documentation installation of product [${productName}]/[${productVersion}] : ${e.message}`
|
||||
);
|
||||
|
||||
await this.productDocClient.setInstallationFailed(productName, e.message);
|
||||
throw e;
|
||||
} finally {
|
||||
zipArchive?.close();
|
||||
}
|
||||
}
|
||||
|
||||
async uninstallPackage({ productName }: { productName: ProductName }) {
|
||||
const indexName = getProductDocIndexName(productName);
|
||||
await this.esClient.indices.delete(
|
||||
{
|
||||
index: indexName,
|
||||
},
|
||||
{ ignore: [404] }
|
||||
);
|
||||
|
||||
await this.productDocClient.setUninstalled(productName);
|
||||
}
|
||||
|
||||
async uninstallAll() {
|
||||
const allProducts = Object.values(DocumentationProduct);
|
||||
for (const productName of allProducts) {
|
||||
await this.uninstallPackage({ productName });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const selectVersion = (currentVersion: string, availableVersions: string[]): string => {
|
||||
return availableVersions.includes(currentVersion)
|
||||
? currentVersion
|
||||
: latestVersion(availableVersions);
|
||||
};
|
||||
|
|
@ -0,0 +1,84 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types';
|
||||
import { loggerMock, type MockedLogger } from '@kbn/logging-mocks';
|
||||
import type { ElasticsearchClient } from '@kbn/core/server';
|
||||
import { elasticsearchServiceMock } from '@kbn/core/server/mocks';
|
||||
import { createIndex } from './create_index';
|
||||
import { internalElserInferenceId } from '../../../../common/consts';
|
||||
|
||||
describe('createIndex', () => {
|
||||
let log: MockedLogger;
|
||||
let esClient: ElasticsearchClient;
|
||||
|
||||
beforeEach(() => {
|
||||
log = loggerMock.create();
|
||||
esClient = elasticsearchServiceMock.createElasticsearchClient();
|
||||
});
|
||||
|
||||
it('calls esClient.indices.create with the right parameters', async () => {
|
||||
const mappings: MappingTypeMapping = {
|
||||
properties: {},
|
||||
};
|
||||
const indexName = '.some-index';
|
||||
|
||||
await createIndex({
|
||||
indexName,
|
||||
mappings,
|
||||
log,
|
||||
esClient,
|
||||
});
|
||||
|
||||
expect(esClient.indices.create).toHaveBeenCalledTimes(1);
|
||||
expect(esClient.indices.create).toHaveBeenCalledWith({
|
||||
index: indexName,
|
||||
mappings,
|
||||
settings: {
|
||||
number_of_shards: 1,
|
||||
auto_expand_replicas: '0-1',
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('rewrites the inference_id attribute of semantic_text fields in the mapping', async () => {
|
||||
const mappings: MappingTypeMapping = {
|
||||
properties: {
|
||||
semantic: {
|
||||
type: 'semantic_text',
|
||||
inference_id: '.elser',
|
||||
},
|
||||
bool: {
|
||||
type: 'boolean',
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
await createIndex({
|
||||
indexName: '.some-index',
|
||||
mappings,
|
||||
log,
|
||||
esClient,
|
||||
});
|
||||
|
||||
expect(esClient.indices.create).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
mappings: {
|
||||
properties: {
|
||||
semantic: {
|
||||
type: 'semantic_text',
|
||||
inference_id: internalElserInferenceId,
|
||||
},
|
||||
bool: {
|
||||
type: 'boolean',
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { Logger } from '@kbn/logging';
|
||||
import type { ElasticsearchClient } from '@kbn/core/server';
|
||||
import type { MappingTypeMapping, MappingProperty } from '@elastic/elasticsearch/lib/api/types';
|
||||
import { internalElserInferenceId } from '../../../../common/consts';
|
||||
|
||||
export const createIndex = async ({
|
||||
esClient,
|
||||
indexName,
|
||||
mappings,
|
||||
log,
|
||||
}: {
|
||||
esClient: ElasticsearchClient;
|
||||
indexName: string;
|
||||
mappings: MappingTypeMapping;
|
||||
log: Logger;
|
||||
}) => {
|
||||
log.debug(`Creating index ${indexName}`);
|
||||
|
||||
overrideInferenceId(mappings, internalElserInferenceId);
|
||||
|
||||
await esClient.indices.create({
|
||||
index: indexName,
|
||||
mappings,
|
||||
settings: {
|
||||
number_of_shards: 1,
|
||||
auto_expand_replicas: '0-1',
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
const overrideInferenceId = (mappings: MappingTypeMapping, inferenceId: string) => {
|
||||
const recursiveOverride = (current: MappingTypeMapping | MappingProperty) => {
|
||||
if ('type' in current && current.type === 'semantic_text') {
|
||||
current.inference_id = inferenceId;
|
||||
}
|
||||
if ('properties' in current && current.properties) {
|
||||
for (const prop of Object.values(current.properties)) {
|
||||
recursiveOverride(prop);
|
||||
}
|
||||
}
|
||||
};
|
||||
recursiveOverride(mappings);
|
||||
};
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue