[8.18] Support local file path for xpack.productDocBase.artifactRepositoryUrl (#217046) (#224284)

# Backport This will backport the following commits from `main` to `8.18`: - [Support local file path for `xpack.productDocBase.artifactRepositoryUrl` (#217046)](https://github.com/elastic/kibana/pull/217046)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sorenlouv/backport)
2025-06-27 10:40:07 -04:00 · 2025-06-17 19:29:30 +02:00 · 2025-06-17 19:29:30 +02:00 · 7da29174c7
commit 7da29174c7
parent 36fed063f1
6 changed files with 219 additions and 12 deletions
--- a/docs/settings/ai-assistant-settings.asciidoc
+++ b/docs/settings/ai-assistant-settings.asciidoc
@ -7,14 +7,14 @@

 `xpack.productDocBase.artifactRepositoryUrl`::
 Url of the repository to use to download and install the Elastic product documentation artifacts for the AI assistants.
-Defaults to `https://kibana-knowledge-base-artifacts.elastic.co`
+Supports both HTTP(S) URLs and local file paths (`file://`). Defaults to `https://kibana-knowledge-base-artifacts.elastic.co`

 [[configuring-product-doc-for-airgap]]
 ==== Configuring product documentation for air-gapped environments

 Installing product documentation requires network access to its artifact repository.
-For air-gapped environments, or environments where remote network traffic is blocked or filtered,
-the artifact repository must be manually deployed somewhere accessible by the Kibana deployment.
+In air-gapped environments, or environments where remote network traffic is blocked or filtered,
+ you can use a local artifact repository by specifying the path with the `file://` URI scheme.

 Deploying a custom product documentation repository can be done in 2 ways: using a S3 bucket, or using a CDN.

--- a/x-pack/platform/plugins/shared/ai_infra/product_doc_base/server/services/package_installer/steps/fetch_artifact_versions.test.ts
+++ b/x-pack/platform/plugins/shared/ai_infra/product_doc_base/server/services/package_installer/steps/fetch_artifact_versions.test.ts
@ -5,11 +5,14 @@
 * 2.0.
 */

+import * as fs from 'fs';
 import fetch, { Response } from 'node-fetch';
 import { fetchArtifactVersions } from './fetch_artifact_versions';
 import { getArtifactName, DocumentationProduct, ProductName } from '@kbn/product-doc-common';

 jest.mock('node-fetch');
+jest.mock('fs');
+
 const fetchMock = fetch as jest.MockedFn<typeof fetch>;

 const createResponse = ({
@ -41,6 +44,7 @@ const createResponse = ({
 };

 const artifactRepositoryUrl = 'https://lost.com';
+const localArtifactRepositoryUrl = 'file://usr/local/local_artifacts';

 const expectVersions = (
  versions: Partial<Record<ProductName, string[]>>
@ -58,6 +62,7 @@ const expectVersions = (
 describe('fetchArtifactVersions', () => {
  beforeEach(() => {
    fetchMock.mockReset();
+    jest.clearAllMocks();
  });

  const mockResponse = (responseText: string) => {
@ -67,6 +72,13 @@ describe('fetchArtifactVersions', () => {
    fetchMock.mockResolvedValue(response as Response);
  };

+  const mockFileResponse = (responseText: string) => {
+    const mockData = Buffer.from(responseText);
+    (fs.readFile as unknown as jest.Mock).mockImplementation((path, callback) => {
+      callback(null, mockData);
+    });
+  };
+
  it('calls fetch with the right parameters', async () => {
    mockResponse(createResponse({ artifactNames: [] }));

@ -76,6 +88,56 @@ describe('fetchArtifactVersions', () => {
    expect(fetchMock).toHaveBeenCalledWith(`${artifactRepositoryUrl}?max-keys=1000`);
  });

+  it('parses the local file', async () => {
+    const artifactNames = [
+      getArtifactName({ productName: 'kibana', productVersion: '8.16' }),
+      getArtifactName({ productName: 'elasticsearch', productVersion: '8.16' }),
+    ];
+    mockFileResponse(createResponse({ artifactNames }));
+
+    const result = await fetchArtifactVersions({
+      artifactRepositoryUrl: localArtifactRepositoryUrl,
+    });
+
+    expect(fs.readFile as unknown as jest.Mock).toHaveBeenCalledWith(
+      '/local/local_artifacts/index.xml',
+      expect.any(Function)
+    );
+
+    expect(result).toEqual({
+      elasticsearch: ['8.16'],
+      kibana: ['8.16'],
+      observability: [],
+      security: [],
+    });
+  });
+
+  it('supports win32 env', async () => {
+    const artifactNames = [
+      getArtifactName({ productName: 'kibana', productVersion: '8.16' }),
+      getArtifactName({ productName: 'elasticsearch', productVersion: '8.16' }),
+    ];
+    mockFileResponse(createResponse({ artifactNames }));
+
+    const originalPlatform = process.platform;
+    Object.defineProperty(process, 'platform', {
+      value: 'win32',
+    });
+
+    await fetchArtifactVersions({
+      artifactRepositoryUrl: 'file:///C:/path/local_artifacts',
+    });
+
+    expect(fs.readFile as unknown as jest.Mock).toHaveBeenCalledWith(
+      'C:/path/local_artifacts/index.xml',
+      expect.any(Function)
+    );
+
+    Object.defineProperty(process, 'platform', {
+      value: originalPlatform,
+    });
+  });
+
  it('returns the list of versions from the repository', async () => {
    const artifactNames = [
      getArtifactName({ productName: 'kibana', productVersion: '8.16' }),
--- a/x-pack/platform/plugins/shared/ai_infra/product_doc_base/server/services/package_installer/steps/fetch_artifact_versions.ts
+++ b/x-pack/platform/plugins/shared/ai_infra/product_doc_base/server/services/package_installer/steps/fetch_artifact_versions.ts
@ -5,9 +5,13 @@
 * 2.0.
 */

+import { DocumentationProduct, parseArtifactName, type ProductName } from '@kbn/product-doc-common';
+import * as fs from 'fs';
 import fetch from 'node-fetch';
+import Path from 'path';
+import { URL } from 'url';
 import { parseString } from 'xml2js';
-import { type ProductName, DocumentationProduct, parseArtifactName } from '@kbn/product-doc-common';
+import { resolveLocalArtifactsPath } from '../utils/local_artifacts';

 type ArtifactAvailableVersions = Record<ProductName, string[]>;

@ -16,8 +20,17 @@ export const fetchArtifactVersions = async ({
 }: {
  artifactRepositoryUrl: string;
 }): Promise<ArtifactAvailableVersions> => {
-  const res = await fetch(`${artifactRepositoryUrl}?max-keys=1000`);
-  const xml = await res.text();
+  const parsedUrl = new URL(artifactRepositoryUrl);
+
+  let xml: string;
+  if (parsedUrl.protocol === 'file:') {
+    const file = await fetchLocalFile(parsedUrl);
+    xml = file.toString();
+  } else {
+    const res = await fetch(`${artifactRepositoryUrl}?max-keys=1000`);
+    xml = await res.text();
+  }
+
  return new Promise((resolve, reject) => {
    parseString(xml, (err, result: ListBucketResponse) => {
      if (err) {
@ -50,6 +63,21 @@ export const fetchArtifactVersions = async ({
  });
 };

+function fetchLocalFile(parsedUrl: URL): Promise<Buffer> {
+  return new Promise((resolve, reject) => {
+    const normalizedPath = resolveLocalArtifactsPath(parsedUrl);
+    const xmlFilePath = Path.join(normalizedPath, 'index.xml');
+
+    fs.readFile(xmlFilePath, (err, data) => {
+      if (err) {
+        reject(err);
+      } else {
+        resolve(data);
+      }
+    });
+  });
+}
+
 interface ListBucketResponse {
  ListBucketResult: {
    Name?: string[];
--- a/x-pack/platform/plugins/shared/ai_infra/product_doc_base/server/services/package_installer/utils/download.test.ts
+++ b/x-pack/platform/plugins/shared/ai_infra/product_doc_base/server/services/package_installer/utils/download.test.ts
@ -0,0 +1,86 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { createReadStream } from 'fs';
+import { mkdir } from 'fs/promises';
+import fetch from 'node-fetch';
+import { downloadToDisk } from './download';
+
+jest.mock('fs', () => ({
+  createReadStream: jest.fn().mockReturnValue({
+    on: jest.fn(),
+    pipe: jest.fn(),
+  }),
+  createWriteStream: jest.fn(() => ({
+    on: jest.fn((event, callback) => {
+      if (event === 'finish') {
+        callback();
+      }
+    }),
+    pipe: jest.fn(),
+  })),
+}));
+
+jest.mock('fs/promises', () => ({
+  mkdir: jest.fn(),
+}));
+
+jest.mock('node-fetch', () => jest.fn());
+
+describe('downloadToDisk', () => {
+  const mockFileUrl = 'http://example.com/file.txt';
+  const mockFilePath = '/path/to/file.txt';
+  const mockDirPath = '/path/to';
+  const mockLocalPath = '/local/path/to/file.txt';
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  it('should create the directory if it does not exist', async () => {
+    (fetch as unknown as jest.Mock).mockResolvedValue({
+      body: {
+        pipe: jest.fn(),
+        on: jest.fn(),
+      },
+    });
+
+    await downloadToDisk(mockFileUrl, mockFilePath);
+
+    expect(mkdir).toHaveBeenCalledWith(mockDirPath, { recursive: true });
+  });
+
+  it('should download a file from a remote URL', async () => {
+    const mockResponseBody = {
+      pipe: jest.fn(),
+      on: jest.fn((event, callback) => {}),
+    };
+
+    (fetch as unknown as jest.Mock).mockResolvedValue({
+      body: mockResponseBody,
+    });
+
+    await downloadToDisk(mockFileUrl, mockFilePath);
+
+    expect(fetch).toHaveBeenCalledWith(mockFileUrl);
+  });
+
+  it('should copy a file from a local file URL', async () => {
+    const mockLocalFileUrl = 'file:///local/path/to/file.txt';
+
+    await downloadToDisk(mockLocalFileUrl, mockFilePath);
+
+    expect(createReadStream).toHaveBeenCalledWith(mockLocalPath);
+  });
+
+  it('should handle errors during the download process', async () => {
+    const mockError = new Error('Download failed');
+    (fetch as unknown as jest.Mock).mockRejectedValue(mockError);
+
+    await expect(downloadToDisk(mockFileUrl, mockFilePath)).rejects.toThrow('Download failed');
+  });
+});
--- a/x-pack/platform/plugins/shared/ai_infra/product_doc_base/server/services/package_installer/utils/download.ts
+++ b/x-pack/platform/plugins/shared/ai_infra/product_doc_base/server/services/package_installer/utils/download.ts
@ -5,19 +5,32 @@
 * 2.0.
 */

-import { createWriteStream } from 'fs';
+import { type ReadStream, createReadStream, createWriteStream } from 'fs';
 import { mkdir } from 'fs/promises';
 import Path from 'path';
 import fetch from 'node-fetch';
+import { resolveLocalArtifactsPath } from './local_artifacts';

 export const downloadToDisk = async (fileUrl: string, filePath: string) => {
  const dirPath = Path.dirname(filePath);
  await mkdir(dirPath, { recursive: true });
-  const res = await fetch(fileUrl);
-  const fileStream = createWriteStream(filePath);
+  const writeStream = createWriteStream(filePath);
+  let readStream: ReadStream | NodeJS.ReadableStream;
+
+  const parsedUrl = new URL(fileUrl);
+
+  if (parsedUrl.protocol === 'file:') {
+    const path = resolveLocalArtifactsPath(parsedUrl);
+    readStream = createReadStream(path);
+  } else {
+    const res = await fetch(fileUrl);
+
+    readStream = res.body;
+  }
+
  await new Promise((resolve, reject) => {
-    res.body.pipe(fileStream);
-    res.body.on('error', reject);
-    fileStream.on('finish', resolve);
+    readStream.pipe(writeStream);
+    readStream.on('error', reject);
+    writeStream.on('finish', resolve);
  });
 };
--- a/x-pack/platform/plugins/shared/ai_infra/product_doc_base/server/services/package_installer/utils/local_artifacts.ts
+++ b/x-pack/platform/plugins/shared/ai_infra/product_doc_base/server/services/package_installer/utils/local_artifacts.ts
@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/** Resolve a path to the artifacts folder */
+export function resolveLocalArtifactsPath(parsedUrl: URL): string {
+  if (parsedUrl.protocol !== 'file:') {
+    throw new Error(`Expected file URL, got ${parsedUrl.protocol}`);
+  }
+  const filePath = parsedUrl.pathname;
+  // On Windows, remove leading "/" (e.g., file:///C:/path should be C:/path)
+  const normalizedPath =
+    process.platform === 'win32' && filePath.startsWith('/') ? filePath.substring(1) : filePath;
+  return normalizedPath;
+}