[Fleet] Adds index name patterns for files being delivered to Host/Agents (#158773)

## Summary

- Adds `const`'s for index patterns for files that are delivered to
hosts/agents
- modifies `createFilesClient.toHost()` (exposed in `Plugin.start()`) to
use these new index names
- Adds code to the package install flow to ensure new file delivery
indexes are created
- The files client factory now validates if the integration name is
allowed to use fleet files functionality.


FYI: PR that will add index mappings to ES:
https://github.com/elastic/elasticsearch/pull/96504

x-pack/plugins/fleet/common/constants/file_storage.ts

@@ -5,16 +5,33 @@
  * 2.0.
  */
 
-// File storage indexes supporting endpoint Upload/download
+// File storage indexes supporting file upload from the host to Elastic/Kibana
 // If needing to get an integration specific index name, use the utility functions
 // found in `common/services/file_storage`
 export const FILE_STORAGE_METADATA_INDEX_PATTERN = '.fleet-files-*';
 export const FILE_STORAGE_DATA_INDEX_PATTERN = '.fleet-file-data-*';
 
+// File storage indexes supporting user uplaoded files (via kibana) that will be
+// delivered to the host agent/endpoint
+export const FILE_STORAGE_TO_HOST_METADATA_INDEX_PATTERN = '.fleet-filedelivery-meta-*';
+export const FILE_STORAGE_TO_HOST_DATA_INDEX_PATTERN = '.fleet-filedelivery-data-*';
+
 // which integrations support file upload and the name to use for the file upload index
-export const FILE_STORAGE_INTEGRATION_INDEX_NAMES: Readonly<Record<string, string>> = {
-  elastic_agent: 'agent',
-  endpoint: 'endpoint',
+export const FILE_STORAGE_INTEGRATION_INDEX_NAMES: Readonly<
+  Record<
+    string,
+    {
+      /** name to be used for the index */
+      name: string;
+      /** If integration supports files sent from host to ES/Kibana */
+      fromHost: boolean;
+      /** If integration supports files to be sent to host from kibana */
+      toHost: boolean;
+    }
+  >
+> = {
+  elastic_agent: { name: 'agent', fromHost: true, toHost: false },
+  endpoint: { name: 'endpoint', fromHost: true, toHost: true },
 };
 export const FILE_STORAGE_INTEGRATION_NAMES: Readonly<string[]> = Object.keys(
   FILE_STORAGE_INTEGRATION_INDEX_NAMES

x-pack/plugins/fleet/common/services/file_storage.test.ts

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { getFileDataIndexName, getFileMetadataIndexName } from '..';
+
+describe('File Storage services', () => {
+  describe('File Index Names', () => {
+    it('should generate file metadata index name for files received from host', () => {
+      expect(getFileMetadataIndexName('foo')).toEqual('.fleet-files-foo');
+    });
+
+    it('should generate file data index name for files received from host', () => {
+      expect(getFileDataIndexName('foo')).toEqual('.fleet-file-data-foo');
+    });
+
+    it('should generate file metadata index name for files to be delivered to host', () => {
+      expect(getFileMetadataIndexName('foo', true)).toEqual('.fleet-filedelivery-meta-foo');
+    });
+
+    it('should generate file data index name for files to be delivered to host', () => {
+      expect(getFileDataIndexName('foo', true)).toEqual('.fleet-filedelivery-data-foo');
+    });
+  });
+});

x-pack/plugins/fleet/common/services/file_storage.ts

@@ -5,32 +5,54 @@
  * 2.0.
  */
 
-import { FILE_STORAGE_DATA_INDEX_PATTERN, FILE_STORAGE_METADATA_INDEX_PATTERN } from '../constants';
+import {
+  FILE_STORAGE_DATA_INDEX_PATTERN,
+  FILE_STORAGE_METADATA_INDEX_PATTERN,
+  FILE_STORAGE_TO_HOST_DATA_INDEX_PATTERN,
+  FILE_STORAGE_TO_HOST_METADATA_INDEX_PATTERN,
+} from '../constants';
 
 /**
  * Returns the index name for File Metadata storage for a given integration
  * @param integrationName
+ * @param forHostDelivery
  */
-export const getFileMetadataIndexName = (integrationName: string): string => {
-  if (FILE_STORAGE_METADATA_INDEX_PATTERN.indexOf('*') !== -1) {
-    return FILE_STORAGE_METADATA_INDEX_PATTERN.replace('*', integrationName);
+export const getFileMetadataIndexName = (
+  integrationName: string,
+  /** if set to true, then the index returned will be for files that are being sent to the host */
+  forHostDelivery: boolean = false
+): string => {
+  const metaIndex = forHostDelivery
+    ? FILE_STORAGE_TO_HOST_METADATA_INDEX_PATTERN
+    : FILE_STORAGE_METADATA_INDEX_PATTERN;
+
+  if (metaIndex.indexOf('*') !== -1) {
+    return metaIndex.replace('*', integrationName);
   }
 
   throw new Error(
-    `Unable to define integration file data index. No '*' in index pattern: ${FILE_STORAGE_METADATA_INDEX_PATTERN}`
+    `Unable to define integration file data index. No '*' in index pattern: ${metaIndex}`
   );
 };
 /**
  * Returns the index name for File data (chunks) storage for a given integration
  * @param integrationName
  */
-export const getFileDataIndexName = (integrationName: string): string => {
-  if (FILE_STORAGE_DATA_INDEX_PATTERN.indexOf('*') !== -1) {
-    return FILE_STORAGE_DATA_INDEX_PATTERN.replace('*', integrationName);
+export const getFileDataIndexName = (
+  integrationName: string,
+  /** if set to true, then the index returned will be for files that are being sent to the host */
+  forHostDelivery: boolean = false
+): string => {
+  const dataIndex = forHostDelivery
+    ? FILE_STORAGE_TO_HOST_DATA_INDEX_PATTERN
+    : FILE_STORAGE_DATA_INDEX_PATTERN;
+
+  if (dataIndex.indexOf('*') !== -1) {
+    return dataIndex.replace('*', integrationName);
   }
 
   throw new Error(
-    `Unable to define integration file data index. No '*' in index pattern: ${FILE_STORAGE_DATA_INDEX_PATTERN}`
+    `Unable to define integration file data index. No '*' in index pattern: ${dataIndex}`
   );
 };
 

x-pack/plugins/fleet/server/plugin.ts

@@ -62,14 +62,10 @@ import {
   MESSAGE_SIGNING_KEYS_SAVED_OBJECT_TYPE,
   INTEGRATIONS_PLUGIN_ID,
   UNINSTALL_TOKENS_SAVED_OBJECT_TYPE,
-  getFileMetadataIndexName,
-  getFileDataIndexName,
 } from '../common';
 import { parseExperimentalConfigValue } from '../common/experimental_features';
 
-import { FleetFromHostFilesClient } from './services/files/client_from_host';
-
-import type { FleetFromHostFileClientInterface } from './services/files/types';
+import { getFilesClientFactory } from './services/files/get_files_client_factory';
 
 import type { MessageSigningServiceInterface } from './services/security';
 import {
@@ -130,8 +126,7 @@ import {
   UninstallTokenService,
   type UninstallTokenServiceInterface,
 } from './services/security/uninstall_token_service';
-import type { FleetToHostFileClientInterface } from './services/files/types';
-import { FleetToHostFilesClient } from './services/files/client_to_host';
+import type { FilesClientFactory } from './services/files/types';
 
 export interface FleetSetupDeps {
   security: SecurityPluginSetup;
@@ -231,28 +226,7 @@ export interface FleetStartContract {
    * @param type
    * @param maxSizeBytes
    */
-  createFilesClient: Readonly<{
-    /**
-     * Client to interact with files that will be sent to a host.
-     * @param packageName
-     * @param maxSizeBytes
-     */
-    toHost: (
-      /** The integration package name */
-      packageName: string,
-      /** Max file size allow to be created (in bytes) */
-      maxSizeBytes?: number
-    ) => FleetToHostFileClientInterface;
-
-    /**
-     * Client to interact with files that were sent from the host
-     * @param packageName
-     */
-    fromHost: (
-      /** The integration package name */
-      packageName: string
-    ) => FleetFromHostFileClientInterface;
-  }>;
+  createFilesClient: Readonly<FilesClientFactory>;
 
   messageSigningService: MessageSigningServiceInterface;
   uninstallTokenService: UninstallTokenServiceInterface;
@@ -604,27 +578,12 @@ export class FleetPlugin
       createArtifactsClient(packageName: string) {
         return new FleetArtifactsClient(core.elasticsearch.client.asInternalUser, packageName);
       },
-      createFilesClient: Object.freeze({
-        fromHost: (packageName) => {
-          return new FleetFromHostFilesClient(
-            core.elasticsearch.client.asInternalUser,
-            this.initializerContext.logger.get('fleetFiles', packageName),
-            getFileMetadataIndexName(packageName),
-            getFileDataIndexName(packageName)
-          );
-        },
-
-        toHost: (packageName, maxFileBytes) => {
-          return new FleetToHostFilesClient(
-            core.elasticsearch.client.asInternalUser,
-            this.initializerContext.logger.get('fleetFiles', packageName),
-            // FIXME:PT define once we have new index patterns (defend workflows team issue #6553)
-            getFileMetadataIndexName(packageName),
-            getFileDataIndexName(packageName),
-            maxFileBytes
-          );
-        },
-      }),
+      createFilesClient: Object.freeze(
+        getFilesClientFactory({
+          esClient: core.elasticsearch.client.asInternalUser,
+          logger: this.initializerContext.logger,
+        })
+      ),
       messageSigningService,
       uninstallTokenService,
     };

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts

@@ -471,8 +471,28 @@ export async function ensureFileUploadWriteIndices(opts: {
 
   return Promise.all(
     integrationsWithFileUpload.flatMap((integrationName) => {
-      const indexName = FILE_STORAGE_INTEGRATION_INDEX_NAMES[integrationName];
-      return [ensure(getFileDataIndexName(indexName)), ensure(getFileMetadataIndexName(indexName))];
+      const {
+        name: indexName,
+        fromHost,
+        toHost,
+      } = FILE_STORAGE_INTEGRATION_INDEX_NAMES[integrationName];
+      const indexCreateRequests: Array<Promise<void>> = [];
+
+      if (fromHost) {
+        indexCreateRequests.push(
+          ensure(getFileDataIndexName(indexName)),
+          ensure(getFileMetadataIndexName(indexName))
+        );
+      }
+
+      if (toHost) {
+        indexCreateRequests.push(
+          ensure(getFileDataIndexName(indexName, true)),
+          ensure(getFileMetadataIndexName(indexName, true))
+        );
+      }
+
+      return indexCreateRequests;
     })
   );
 }
@@ -529,6 +549,8 @@ export async function ensureAliasHasWriteIndex(opts: {
   );
 
   if (!existingIndex) {
+    logger.info(`Creating write index [${writeIndexName}], alias [${aliasName}]`);
+
     await retryTransientEsErrors(
       () => esClient.indices.create({ index: writeIndexName, ...body }, { ignore: [404] }),
       {

x-pack/plugins/fleet/server/services/files/client_to_host.test.ts

@@ -23,6 +23,11 @@ import type { DeeplyMockedKeys } from '@kbn/utility-types-jest';
 
 import type { File } from '@kbn/files-plugin/common';
 
+import {
+  FILE_STORAGE_TO_HOST_DATA_INDEX_PATTERN,
+  FILE_STORAGE_TO_HOST_METADATA_INDEX_PATTERN,
+} from '../../../common/constants';
+
 import { getFileDataIndexName, getFileMetadataIndexName } from '../../../common';
 
 import type { HapiReadableStream } from '../..';
@@ -55,8 +60,8 @@ describe('FleetToHostFilesClient', () => {
     return new FleetToHostFilesClient(
       esClientMock,
       loggerMock,
-      getFileMetadataIndexName('foo'),
-      getFileDataIndexName('foo'),
+      getFileMetadataIndexName('foo', true),
+      getFileDataIndexName('foo', true),
       12345
     );
   };
@@ -97,11 +102,19 @@ describe('FleetToHostFilesClient', () => {
 
     esClientMock.search.mockImplementation(async (searchRequest = {}) => {
       // File metadata
-      if ((searchRequest.index as string).startsWith('.fleet-files-')) {
+      if (
+        (searchRequest.index as string).startsWith(
+          FILE_STORAGE_TO_HOST_METADATA_INDEX_PATTERN.replace('*', '')
+        )
+      ) {
         return fleetFilesIndexSearchResponse;
       }
 
-      if ((searchRequest.index as string).startsWith('.fleet-file-data-')) {
+      if (
+        (searchRequest.index as string).startsWith(
+          FILE_STORAGE_TO_HOST_DATA_INDEX_PATTERN.replace('*', '')
+        )
+      ) {
         return fleetFileDataIndexSearchResponse;
       }
 
@@ -117,9 +130,8 @@ describe('FleetToHostFilesClient', () => {
     expect(createEsFileClientMock).toHaveBeenCalledWith({
       elasticsearchClient: esClientMock,
       logger: loggerMock,
-      // FIXME:PT adjust indexes once new index patterns are added to ES
-      metadataIndex: '.fleet-files-foo',
-      blobStorageIndex: '.fleet-file-data-foo',
+      metadataIndex: '.fleet-filedelivery-meta-foo',
+      blobStorageIndex: '.fleet-filedelivery-data-foo',
       maxSizeBytes: 12345,
       indexIsAlias: true,
     });

x-pack/plugins/fleet/server/services/files/get_files_client_factory.test.ts

@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { loggingSystemMock } from '@kbn/core-logging-server-mocks';
+
+import { elasticsearchServiceMock } from '@kbn/core-elasticsearch-server-mocks';
+
+import type { FilesClientFactory } from './types';
+import { FleetFromHostFilesClient } from './client_from_host';
+import { FleetToHostFilesClient } from './client_to_host';
+
+import { getFilesClientFactory } from './get_files_client_factory';
+
+jest.mock('@kbn/files-plugin/server');
+
+describe('getFilesClientFactory()', () => {
+  let clientFactory: FilesClientFactory;
+
+  beforeEach(() => {
+    clientFactory = getFilesClientFactory({
+      esClient: elasticsearchServiceMock.createElasticsearchClient(),
+      logger: loggingSystemMock.create().asLoggerFactory(),
+    });
+  });
+
+  it('should return a client when `fromHost()` is called', () => {
+    expect(clientFactory.fromHost('endpoint')).toBeInstanceOf(FleetFromHostFilesClient);
+  });
+
+  it('should return a client when `toHost()` is called', () => {
+    expect(clientFactory.toHost('endpoint')).toBeInstanceOf(FleetToHostFilesClient);
+  });
+
+  it('should throw an error if `fromHost()` is called, but package name is not authorized', () => {
+    expect(() => clientFactory.fromHost('foo')).toThrow(
+      'Integration name [foo] does not have access to files received from host'
+    );
+  });
+
+  it('should throw an error if `toHost()` is called, but package name is not authorized', () => {
+    expect(() => clientFactory.toHost('foo')).toThrow(
+      'Integration name [foo] does not have access to files for delivery to host'
+    );
+  });
+});

x-pack/plugins/fleet/server/services/files/get_files_client_factory.ts

@@ -0,0 +1,64 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
+import type { LoggerFactory } from '@kbn/core/server';
+
+import { FILE_STORAGE_INTEGRATION_INDEX_NAMES } from '../../../common/constants';
+
+import { getFileDataIndexName, getFileMetadataIndexName } from '../../../common';
+
+import { FleetFilesClientError } from '../../errors';
+
+import { FleetToHostFilesClient } from './client_to_host';
+
+import { FleetFromHostFilesClient } from './client_from_host';
+
+import type { FilesClientFactory } from './types';
+
+interface GetFilesClientFactoryParams {
+  esClient: ElasticsearchClient;
+  logger: LoggerFactory;
+}
+
+export const getFilesClientFactory = ({
+  esClient,
+  logger,
+}: GetFilesClientFactoryParams): FilesClientFactory => {
+  return {
+    fromHost: (packageName) => {
+      if (!FILE_STORAGE_INTEGRATION_INDEX_NAMES[packageName]?.fromHost) {
+        throw new FleetFilesClientError(
+          `Integration name [${packageName}] does not have access to files received from host`
+        );
+      }
+
+      return new FleetFromHostFilesClient(
+        esClient,
+        logger.get('fleetFiles', packageName),
+        getFileMetadataIndexName(packageName),
+        getFileDataIndexName(packageName)
+      );
+    },
+
+    toHost: (packageName, maxFileBytes) => {
+      if (!FILE_STORAGE_INTEGRATION_INDEX_NAMES[packageName]?.toHost) {
+        throw new FleetFilesClientError(
+          `Integration name [${packageName}] does not have access to files for delivery to host`
+        );
+      }
+
+      return new FleetToHostFilesClient(
+        esClient,
+        logger.get('fleetFiles', packageName),
+        getFileMetadataIndexName(packageName, true),
+        getFileDataIndexName(packageName, true),
+        maxFileBytes
+      );
+    },
+  };
+};

x-pack/plugins/fleet/server/services/files/types.ts

@@ -119,3 +119,26 @@ export interface FileCustomMeta {
   target_agents: string[];
   action_id: string;
 }
+
+export interface FilesClientFactory {
+  /**
+   * Client to interact with files that will be sent to a host.
+   * @param packageName
+   * @param maxSizeBytes
+   */
+  toHost: (
+    /** The integration package name */
+    packageName: string,
+    /** Max file size allow to be created (in bytes) */
+    maxSizeBytes?: number
+  ) => FleetToHostFileClientInterface;
+
+  /**
+   * Client to interact with files that were sent from the host
+   * @param packageName
+   */
+  fromHost: (
+    /** The integration package name */
+    packageName: string
+  ) => FleetFromHostFileClientInterface;
+}