[Security Solution][Endpoint] Misc. updates in support of get-file response action (#144948)

## Summary

- Updates the `get-file` action response `outputs` to match latest from
endpoint
- Fix server size `doesFileHanveChunks()` and remove the `.keyword` from
the search field term (index mapping will be setup correctly for these
indexes)
- Updates the names of the File storage indexes
- Sets the `endpointRbacV1Enabled` FF to `true` (enables feature by
default)
- Uses Fleet exposed function utilities to retrieve the indexes for
File's metadata and data chunks

The following Fleet changes were also done

- Created common methods in fleet for retrieving the file metadata and
data indexes using an integration name (should protect us against index
names going forward and avoid having integrations in kibana keep
hard-coded values)
- Removed the .keyword from a few places in the file server service
(still need to test)
- Adjusted both the Fleet and the Security Solution code to use the new
methods for getting the integration specific index names (cc/
@juliaElastic )

x-pack/plugins/fleet/common/constants/file_storage.ts

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+// File storage indexes supporting endpoint Upload/download
+// If needing to get an integration specific index name, use the utility functions
+// found in `common/services/file_storage`
+export const FILE_STORAGE_METADATA_INDEX_PATTERN = '.fleet-files-*';
+export const FILE_STORAGE_DATA_INDEX_PATTERN = '.fleet-file-data-*';

x-pack/plugins/fleet/common/constants/index.ts

@@ -18,6 +18,7 @@ export * from './preconfiguration';
 export * from './download_source';
 export * from './fleet_server_policy_config';
 export * from './authz';
+export * from './file_storage';
 
 // TODO: This is the default `index.max_result_window` ES setting, which dictates
 // the maximum amount of results allowed to be returned from a search. It's possible

x-pack/plugins/fleet/common/index.ts

@@ -65,6 +65,8 @@ export {
   INVALID_NAMESPACE_CHARACTERS,
   // TODO Should probably not be exposed by Fleet
   decodeCloudId,
+  getFileMetadataIndexName,
+  getFileDataIndexName,
 } from './services';
 
 export type { FleetAuthz } from './authz';

x-pack/plugins/fleet/common/services/file_storage.ts

@@ -0,0 +1,65 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FILE_STORAGE_DATA_INDEX_PATTERN, FILE_STORAGE_METADATA_INDEX_PATTERN } from '../constants';
+
+/**
+ * Returns the index name for File Metadata storage for a given integration
+ * @param integrationName
+ */
+export const getFileMetadataIndexName = (integrationName: string): string => {
+  if (FILE_STORAGE_METADATA_INDEX_PATTERN.indexOf('*') !== -1) {
+    return FILE_STORAGE_METADATA_INDEX_PATTERN.replace('*', integrationName);
+  }
+
+  throw new Error(
+    `Unable to define integration file data index. No '*' in index pattern: ${FILE_STORAGE_METADATA_INDEX_PATTERN}`
+  );
+};
+/**
+ * Returns the index name for File data (chunks) storage for a given integration
+ * @param integrationName
+ */
+export const getFileDataIndexName = (integrationName: string): string => {
+  if (FILE_STORAGE_DATA_INDEX_PATTERN.indexOf('*') !== -1) {
+    return FILE_STORAGE_DATA_INDEX_PATTERN.replace('*', integrationName);
+  }
+
+  throw new Error(
+    `Unable to define integration file data index. No '*' in index pattern: ${FILE_STORAGE_DATA_INDEX_PATTERN}`
+  );
+};
+
+/**
+ * Returns back the integration name for a given File Data (chunks) index name.
+ *
+ * @example
+ * // Given a File data index pattern of `.fleet-file-data-*`:
+ *
+ * getIntegrationNameFromFileDataIndexName('.fleet-file-data-agent');
+ * // return 'agent'
+ *
+ * getIntegrationNameFromFileDataIndexName('.fleet-file-data-agent-00001');
+ * // return 'agent'
+ */
+export const getIntegrationNameFromFileDataIndexName = (indexName: string): string => {
+  const integrationNameIndexPosition = FILE_STORAGE_DATA_INDEX_PATTERN.split('-').indexOf('*');
+
+  if (integrationNameIndexPosition === -1) {
+    throw new Error(
+      `Unable to parse index name. No '*' in index pattern: ${FILE_STORAGE_DATA_INDEX_PATTERN}`
+    );
+  }
+
+  const indexPieces = indexName.split('-');
+
+  if (indexPieces[integrationNameIndexPosition]) {
+    return indexPieces[integrationNameIndexPosition];
+  }
+
+  throw new Error(`Index name ${indexName} does not seem to be a File Data storage index`);
+};

x-pack/plugins/fleet/common/services/index.ts

@@ -48,3 +48,5 @@ export {
   getRegistryDataStreamAssetBaseName,
   getComponentTemplateNameForDatastream,
 } from './datastream_es_name';
+
+export * from './file_storage';

x-pack/plugins/fleet/server/constants/fleet_es_assets.ts

@@ -5,6 +5,8 @@
  * 2.0.
  */
 
+import { getFileDataIndexName, getFileMetadataIndexName } from '../../common';
+
 import { getESAssetMetadata } from '../services/epm/elasticsearch/meta';
 
 const meta = getESAssetMetadata();
@@ -195,6 +197,6 @@ on_failure:
       value:
         - 'failed in Fleet agent final_pipeline: {{ _ingest.on_failure_message }}'`;
 
-// File storage indexes supporting endpoint Upload/download
-export const FILE_STORAGE_METADATA_INDEX_PATTERN = '.fleet-*-files';
-export const FILE_STORAGE_DATA_INDEX_PATTERN = '.fleet-*-file-data';
+// Fleet Agent indexes for storing files
+export const FILE_STORAGE_METADATA_AGENT_INDEX = getFileMetadataIndexName('agent');
+export const FILE_STORAGE_DATA_AGENT_INDEX = getFileDataIndexName('agent');

x-pack/plugins/fleet/server/constants/index.ts

@@ -83,3 +83,5 @@ export {
   FLEET_INSTALL_FORMAT_VERSION,
   FLEET_AGENT_POLICIES_SCHEMA_VERSION,
 } from './fleet_es_assets';
+export { FILE_STORAGE_DATA_AGENT_INDEX } from './fleet_es_assets';
+export { FILE_STORAGE_METADATA_AGENT_INDEX } from './fleet_es_assets';

x-pack/plugins/fleet/server/services/agents/uploads.ts

@@ -17,14 +17,15 @@ import type { AgentDiagnostics } from '../../../common/types/models';
 import { appContextService } from '../app_context';
 import {
   AGENT_ACTIONS_INDEX,
-  agentRouteService,
   AGENT_ACTIONS_RESULTS_INDEX,
+  agentRouteService,
 } from '../../../common';
 
-import { SO_SEARCH_LIMIT } from '../../constants';
-
-const FILE_STORAGE_METADATA_AGENT_INDEX = '.fleet-agent-files';
-const FILE_STORAGE_DATA_AGENT_INDEX = '.fleet-agent-file-data';
+import {
+  FILE_STORAGE_DATA_AGENT_INDEX,
+  FILE_STORAGE_METADATA_AGENT_INDEX,
+  SO_SEARCH_LIMIT,
+} from '../../constants';
 
 export async function getAgentUploads(
   esClient: ElasticsearchClient,

x-pack/plugins/fleet/server/services/files/index.test.ts

@@ -8,16 +8,19 @@
 import type { ElasticsearchClientMock } from '@kbn/core/server/mocks';
 import { elasticsearchServiceMock } from '@kbn/core/server/mocks';
 
-import { ES_SEARCH_LIMIT } from '../../../common/constants';
 import {
   FILE_STORAGE_DATA_INDEX_PATTERN,
   FILE_STORAGE_METADATA_INDEX_PATTERN,
-} from '../../constants/fleet_es_assets';
+} from '../../../common/constants/file_storage';
+
+import { getFileDataIndexName, getFileMetadataIndexName } from '../../../common/services';
+
+import { ES_SEARCH_LIMIT } from '../../../common/constants';
 
 import { fileIdsWithoutChunksByIndex, getFilesByStatus, updateFilesStatus } from '.';
 
-const ENDPOINT_FILE_METADATA_INDEX = '.fleet-endpoint-files';
-const ENDPOINT_FILE_INDEX = '.fleet-endpoint-file-data';
+const ENDPOINT_FILE_METADATA_INDEX = getFileMetadataIndexName('endpoint');
+const ENDPOINT_FILE_INDEX = getFileDataIndexName('endpoint');
 
 describe('files service', () => {
   let esClientMock: ElasticsearchClientMock;
@@ -66,7 +69,7 @@ describe('files service', () => {
             size: ES_SEARCH_LIMIT,
             query: {
               term: {
-                'file.Status.keyword': status,
+                'file.Status': status,
               },
             },
             _source: false,
@@ -132,7 +135,7 @@ describe('files service', () => {
                 must: [
                   {
                     terms: {
-                      'bid.keyword': Array.from(files.map((file) => file._id)),
+                      bid: Array.from(files.map((file) => file._id)),
                     },
                   },
                   {
@@ -157,7 +160,7 @@ describe('files service', () => {
 
   describe('#updateFilesStatus()', () => {
     it('calls esClient.updateByQuery with expected values', () => {
-      const FAKE_INTEGRATION_METADATA_INDEX = '.fleet-someintegration-files';
+      const FAKE_INTEGRATION_METADATA_INDEX = getFileMetadataIndexName('someintegration');
       const files = {
         [ENDPOINT_FILE_METADATA_INDEX]: new Set(['delete1', 'delete2']),
         [FAKE_INTEGRATION_METADATA_INDEX]: new Set(['delete2', 'delete3']),

x-pack/plugins/fleet/server/services/files/index.ts

@@ -6,13 +6,19 @@
  */
 
 import type { ElasticsearchClient } from '@kbn/core/server';
-import type { UpdateByQueryResponse, SearchHit } from '@elastic/elasticsearch/lib/api/types';
+import type { SearchHit, UpdateByQueryResponse } from '@elastic/elasticsearch/lib/api/types';
 import type { FileStatus } from '@kbn/files-plugin/common/types';
 
 import {
   FILE_STORAGE_DATA_INDEX_PATTERN,
   FILE_STORAGE_METADATA_INDEX_PATTERN,
-} from '../../constants/fleet_es_assets';
+} from '../../../common/constants';
+
+import {
+  getFileMetadataIndexName,
+  getIntegrationNameFromFileDataIndexName,
+} from '../../../common/services';
+
 import { ES_SEARCH_LIMIT } from '../../../common/constants';
 
 /**
@@ -34,7 +40,7 @@ export async function getFilesByStatus(
         size: ES_SEARCH_LIMIT,
         query: {
           term: {
-            'file.Status.keyword': status,
+            'file.Status': status,
           },
         },
         _source: false,
@@ -82,7 +88,7 @@ export async function fileIdsWithoutChunksByIndex(
             must: [
               {
                 terms: {
-                  'bid.keyword': Array.from(allFileIds),
+                  bid: Array.from(allFileIds),
                 },
               },
               {
@@ -102,8 +108,8 @@ export async function fileIdsWithoutChunksByIndex(
   chunks.hits.hits.forEach((hit) => {
     const fileId = hit._source?.bid;
     if (!fileId) return;
-    const integration = hit._index.split('-')[1];
-    const metadataIndex = `.fleet-${integration}-files`;
+    const integration = getIntegrationNameFromFileDataIndexName(hit._index);
+    const metadataIndex = getFileMetadataIndexName(integration);
     if (noChunkFileIdsByIndex[metadataIndex]?.delete(fileId)) {
       allFileIds.delete(fileId);
     }

x-pack/plugins/fleet/server/tasks/check_deleted_files_task.test.ts

@@ -13,15 +13,17 @@ import type { CoreSetup } from '@kbn/core/server';
 import type { ElasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks';
 import { loggingSystemMock } from '@kbn/core/server/mocks';
 
+import { getFileDataIndexName, getFileMetadataIndexName } from '../../common';
+
 import { createAppContextStartContractMock } from '../mocks';
-import {
-  FILE_STORAGE_DATA_INDEX_PATTERN,
-  FILE_STORAGE_METADATA_INDEX_PATTERN,
-} from '../constants/fleet_es_assets';
+
 import { appContextService } from '../services';
 
 import { CheckDeletedFilesTask, TYPE, VERSION } from './check_deleted_files_task';
 
+const MOCK_FILE_METADATA_INDEX = getFileMetadataIndexName('mock');
+const MOCK_FILE_DATA_INDEX = getFileDataIndexName('mock');
+
 const MOCK_TASK_INSTANCE = {
   id: `${TYPE}:${VERSION}`,
   runAt: new Date(),
@@ -118,12 +120,12 @@ describe('check deleted files task', () => {
             hits: [
               {
                 _id: 'metadata-testid1',
-                _index: FILE_STORAGE_METADATA_INDEX_PATTERN,
+                _index: MOCK_FILE_METADATA_INDEX,
                 _source: { file: { status: 'READY' } },
               },
               {
                 _id: 'metadata-testid2',
-                _index: FILE_STORAGE_METADATA_INDEX_PATTERN,
+                _index: MOCK_FILE_METADATA_INDEX,
                 _source: { file: { status: 'READY' } },
               },
             ],
@@ -147,7 +149,7 @@ describe('check deleted files task', () => {
             hits: [
               {
                 _id: 'data-testid1',
-                _index: FILE_STORAGE_DATA_INDEX_PATTERN,
+                _index: MOCK_FILE_DATA_INDEX,
                 _source: {
                   bid: 'metadata-testid1',
                 },
@@ -160,7 +162,7 @@ describe('check deleted files task', () => {
 
       expect(esClient.updateByQuery).toHaveBeenCalledWith(
         {
-          index: FILE_STORAGE_METADATA_INDEX_PATTERN,
+          index: MOCK_FILE_METADATA_INDEX,
           query: {
             ids: {
               values: ['metadata-testid2'],

x-pack/plugins/security_solution/common/endpoint/constants.ts

@@ -6,6 +6,8 @@
  */
 
 /** endpoint data streams that are used for host isolation  */
+import { getFileDataIndexName, getFileMetadataIndexName } from '@kbn/fleet-plugin/common';
+
 /** for index patterns `.logs-endpoint.actions-* and .logs-endpoint.action.responses-*`*/
 export const ENDPOINT_ACTIONS_DS = '.logs-endpoint.actions';
 export const ENDPOINT_ACTIONS_INDEX = `${ENDPOINT_ACTIONS_DS}-default`;
@@ -42,8 +44,8 @@ export const policyIndexPattern = 'metrics-endpoint.policy-*';
 export const telemetryIndexPattern = 'metrics-endpoint.telemetry-*';
 
 // File storage indexes supporting endpoint Upload/download
-export const FILE_STORAGE_METADATA_INDEX = '.fleet-endpoint-files';
-export const FILE_STORAGE_DATA_INDEX = '.fleet-endpoint-file-data';
+export const FILE_STORAGE_METADATA_INDEX = getFileMetadataIndexName('endpoint');
+export const FILE_STORAGE_DATA_INDEX = getFileDataIndexName('endpoint');
 
 // Endpoint API routes
 export const BASE_ENDPOINT_ROUTE = '/api/endpoint';

x-pack/plugins/security_solution/common/endpoint/data_generators/endpoint_action_generator.ts

@@ -89,9 +89,16 @@ export class EndpointActionGenerator extends BaseDataGenerator {
           type: 'json',
           content: {
             code: 'ra_get-file_success_done',
-            path: '/some/path/bad_file.txt',
-            size: 1234,
             zip_size: 123,
+            contents: [
+              {
+                type: 'file',
+                path: '/some/path/bad_file.txt',
+                size: 1234,
+                file_name: 'bad_file.txt',
+                sha256: '9558c5cb39622e9b3653203e772b129d6c634e7dbd7af1b244352fc1d704601f',
+              },
+            ],
           },
         };
       }

x-pack/plugins/security_solution/common/endpoint/types/actions.ts

@@ -53,9 +53,15 @@ export interface KillProcessActionOutputContent {
 
 export interface ResponseActionGetFileOutputContent {
   code: string;
-  path: string;
-  size: number;
   zip_size: number;
+  /** The contents of the zip file. One entry per file */
+  contents: Array<{
+    path: string;
+    sha256: string;
+    size: number;
+    file_name: string;
+    type: string;
+  }>;
 }
 
 export const ActivityLogItemTypes = {

x-pack/plugins/security_solution/common/experimental_features.ts

@@ -70,7 +70,7 @@ export const allowedExperimentalValues = Object.freeze({
    * Enables endpoint package level rbac for response actions only.
    * if endpointRbacEnabled is enabled, it will take precedence.
    */
-  endpointRbacV1Enabled: false,
+  endpointRbacV1Enabled: true,
 
   /**
    * Enables the Guided Onboarding tour in security

x-pack/plugins/security_solution/public/management/pages/integration_tests/index.test.tsx

@@ -96,7 +96,7 @@ describe('when in the Administration tab', () => {
       });
 
       mockedContext.history.push('/administration/response_actions_history');
-      expect(await render().findByTestId('noIngestPermissions')).toBeTruthy();
+      expect(await render().findByTestId('noPrivilegesPage')).toBeTruthy();
     });
   });
 

x-pack/plugins/security_solution/scripts/endpoint/agent_emulator/services/endpoint_response_actions.ts

@@ -281,15 +281,22 @@ const getOutputDataIfNeeded = (action: ActionDetails): ResponseOutput => {
         output: {
           type: 'json',
           content: {
-            code: 'ra_get-file-success',
-            path: (
-              action as ActionDetails<
-                ResponseActionGetFileOutputContent,
-                ResponseActionGetFileParameters
-              >
-            ).parameters?.path,
-            size: 1234,
+            code: 'ra_get-file_success_done',
             zip_size: 123,
+            contents: [
+              {
+                type: 'file',
+                path: (
+                  action as ActionDetails<
+                    ResponseActionGetFileOutputContent,
+                    ResponseActionGetFileParameters
+                  >
+                ).parameters?.path,
+                size: 1234,
+                file_name: 'bad_file.txt',
+                sha256: '9558c5cb39622e9b3653203e772b129d6c634e7dbd7af1b244352fc1d704601f',
+              },
+            ],
           },
         },
       } as ResponseOutput<ResponseActionGetFileOutputContent>;

x-pack/plugins/security_solution/server/endpoint/services/actions/action_files.ts

@@ -99,8 +99,6 @@ export const getFileInfo = async (
       }
     }
 
-    // TODO: add `ttl` to the return payload by retrieving the value from ILM?
-
     return {
       name,
       id,
@@ -123,7 +121,7 @@ const doesFileHaveChunks = async (
     body: {
       query: {
         term: {
-          'bid.keyword': fileId,
+          bid: fileId,
         },
       },
       // Setting `_source` to false - we don't need the actual document to be returned

x-pack/plugins/security_solution/server/endpoint/services/actions/utils.test.ts

@@ -445,8 +445,15 @@ describe('When using Actions service utilities', () => {
             '456': {
               content: {
                 code: 'ra_get-file_success_done',
-                path: '/some/path/bad_file.txt',
-                size: 1234,
+                contents: [
+                  {
+                    file_name: 'bad_file.txt',
+                    path: '/some/path/bad_file.txt',
+                    sha256: '9558c5cb39622e9b3653203e772b129d6c634e7dbd7af1b244352fc1d704601f',
+                    size: 1234,
+                    type: 'file',
+                  },
+                ],
                 zip_size: 123,
               },
               type: 'json',

x-pack/test/api_integration/apis/security/privileges.ts

@@ -29,7 +29,17 @@ export default function ({ getService }: FtrProviderContext) {
       actions: ['all', 'read', 'minimal_all', 'minimal_read'],
       stackAlerts: ['all', 'read', 'minimal_all', 'minimal_read'],
       ml: ['all', 'read', 'minimal_all', 'minimal_read'],
-      siem: ['all', 'read', 'minimal_all', 'minimal_read'],
+      siem: [
+        'all',
+        'read',
+        'minimal_all',
+        'minimal_read',
+        'actions_log_management_all',
+        'actions_log_management_read',
+        'host_isolation_all',
+        'process_operations_all',
+        'file_operations_all',
+      ],
       uptime: ['all', 'read', 'minimal_all', 'minimal_read'],
       securitySolutionCases: ['all', 'read', 'minimal_all', 'minimal_read', 'cases_delete'],
       infrastructure: ['all', 'read', 'minimal_all', 'minimal_read'],

x-pack/test/api_integration/apis/security/privileges_basic.ts

@@ -96,7 +96,17 @@ export default function ({ getService }: FtrProviderContext) {
             actions: ['all', 'read', 'minimal_all', 'minimal_read'],
             stackAlerts: ['all', 'read', 'minimal_all', 'minimal_read'],
             ml: ['all', 'read', 'minimal_all', 'minimal_read'],
-            siem: ['all', 'read', 'minimal_all', 'minimal_read'],
+            siem: [
+              'actions_log_management_all',
+              'actions_log_management_read',
+              'all',
+              'file_operations_all',
+              'host_isolation_all',
+              'minimal_all',
+              'minimal_read',
+              'process_operations_all',
+              'read',
+            ],
             uptime: ['all', 'read', 'minimal_all', 'minimal_read'],
             securitySolutionCases: ['all', 'read', 'minimal_all', 'minimal_read', 'cases_delete'],
             infrastructure: ['all', 'read', 'minimal_all', 'minimal_read'],

x-pack/test/fleet_api_integration/apis/agents/uploads.ts

@@ -7,6 +7,10 @@
 
 import expect from '@kbn/expect';
 import { AGENT_ACTIONS_INDEX, AGENT_ACTIONS_RESULTS_INDEX } from '@kbn/fleet-plugin/common';
+import {
+  FILE_STORAGE_DATA_AGENT_INDEX,
+  FILE_STORAGE_METADATA_AGENT_INDEX,
+} from '@kbn/fleet-plugin/server/constants';
 import { FtrProviderContext } from '../../../api_integration/ftr_provider_context';
 import { setupFleetAndAgents } from './services';
 import { skipIfNoDockerRegistry } from '../../helpers';
@@ -57,7 +61,7 @@ export default function (providerContext: FtrProviderContext) {
       );
 
       await esClient.update({
-        index: '.fleet-agent-files',
+        index: FILE_STORAGE_METADATA_AGENT_INDEX,
         id: 'file1',
         refresh: true,
         body: {
@@ -102,7 +106,7 @@ export default function (providerContext: FtrProviderContext) {
 
     it('should get agent uploaded file', async () => {
       await esClient.update({
-        index: '.fleet-agent-file-data',
+        index: FILE_STORAGE_DATA_AGENT_INDEX,
         id: 'file1.0',
         refresh: true,
         body: {