[Fleet] enabling diagnostics feature flag and changed query for files to use upload_id (#149575)

## Summary

Closes https://github.com/elastic/kibana/issues/141074

Enabled feature flag and tweaked implementation to find file by
`upload_id` rather than doc id.

How to test:
- Start local kibana, start Fleet Server, enroll Elastic Agent from
local (pull [these
changes](https://github.com/elastic/elastic-agent/pull/1703) )
- Click on Request Diagnostics action on the Agent
- The diagnostics file should appear on Agent Details / Diagnostics tab.
- The action should be completed on Agent activity

<img width="1585" alt="image"
src="https://user-images.githubusercontent.com/90178898/214805187-2b1abe34-ba7e-4612-9fad-7ef1f5942f47.png">
<img width="745" alt="image"
src="https://user-images.githubusercontent.com/90178898/214805997-20fdaa01-e4c5-461c-b395-1b1e43117f8a.png">

The file metadata and binary can be queried from these indices:

```
GET .fleet-files-agent/_search

GET .fleet-file-data-agent/_search
```

Tweaked the implementation so that the pending actions are showing up as
soon as the `.fleet-actions` record is created (it can take several
minutes until the action result is ready)
Plus added a tooltip for error status

<img width="948" alt="image"
src="https://user-images.githubusercontent.com/90178898/214841337-eacbb1fc-4934-4d8b-9d52-8db4502d2493.png">



### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

x-pack/plugins/fleet/common/experimental_features.ts

@@ -15,7 +15,7 @@ export const allowedExperimentalValues = Object.freeze({
   createPackagePolicyMultiPageLayout: true,
   packageVerification: true,
   showDevtoolsRequest: true,
-  diagnosticFileUploadEnabled: false,
+  diagnosticFileUploadEnabled: true,
   experimentalDataStreamSettings: false,
   displayAgentMetrics: true,
   showIntegrationsSubcategories: false,

x-pack/plugins/fleet/common/types/models/agent.ts

@@ -161,7 +161,7 @@ export interface AgentDiagnostics {
   name: string;
   createTime: string;
   filePath: string;
-  status: 'READY' | 'AWAITING_UPLOAD' | 'DELETED' | 'IN_PROGRESS';
+  status: 'READY' | 'AWAITING_UPLOAD' | 'DELETED' | 'IN_PROGRESS' | 'FAILED';
   actionId: string;
 }
 

x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_diagnostics/index.tsx

@@ -6,6 +6,7 @@
  */
 
 import type { EuiTableFieldDataColumnType } from '@elastic/eui';
+import { EuiToolTip } from '@elastic/eui';
 import {
   EuiBasicTable,
   EuiButton,
@@ -131,7 +132,10 @@ export const AgentDiagnosticsTab: React.FunctionComponent<AgentDiagnosticsProps>
           </EuiText>
         ) : (
           <EuiText color="subdued">
-            <EuiIcon type="alert" color="red" /> &nbsp;
+            <EuiToolTip content={`Diagnostics status: ${currentItem?.status}`}>
+              <EuiIcon type="alert" color="red" />
+            </EuiToolTip>
+            &nbsp;
             {currentItem?.name}
           </EuiText>
         );

x-pack/plugins/fleet/server/services/agents/uploads.ts

@@ -34,13 +34,23 @@ export async function getAgentUploads(
   const getFile = async (fileId: string) => {
     if (!fileId) return;
     try {
-      const file = await esClient.get({
+      const fileResponse = await esClient.search({
         index: FILE_STORAGE_METADATA_AGENT_INDEX,
-        id: fileId,
+        query: {
+          bool: {
+            filter: {
+              term: { upload_id: fileId },
+            },
+          },
+        },
       });
+      if (fileResponse.hits.total === 0) {
+        appContextService.getLogger().debug(`No matches for upload_id ${fileId}`);
+        return;
+      }
       return {
-        id: file._id,
-        ...(file._source as any)?.file,
+        id: fileResponse.hits.hits[0]._id,
+        ...(fileResponse.hits.hits[0]._source as any)?.file,
       };
     } catch (err) {
       if (err.statusCode === 404) {
@@ -56,13 +66,13 @@ export async function getAgentUploads(
 
   const results = [];
   for (const action of actions) {
-    const file = await getFile(action.fileId);
+    const file = action.fileId ? await getFile(action.fileId) : undefined;
     const fileName = file?.name ?? `${moment(action.timestamp!).format('YYYY-MM-DD HH:mm:ss')}.zip`;
     const filePath = file ? agentRouteService.getAgentFileDownloadLink(file.id, file.name) : '';
     const result = {
       actionId: action.actionId,
       id: file?.id ?? action.actionId,
-      status: file?.Status ?? 'IN_PROGRESS',
+      status: file?.Status ?? (action.error ? 'FAILED' : 'IN_PROGRESS'),
       name: fileName,
       createTime: action.timestamp!,
       filePath,
@@ -76,7 +86,7 @@ export async function getAgentUploads(
 async function _getRequestDiagnosticsActions(
   esClient: ElasticsearchClient,
   agentId: string
-): Promise<Array<{ actionId: string; timestamp?: string; fileId: string }>> {
+): Promise<Array<{ actionId: string; timestamp?: string; fileId?: string; error?: string }>> {
   const agentActionRes = await esClient.search<any>({
     index: AGENT_ACTIONS_INDEX,
     ignore_unavailable: true,
@@ -99,14 +109,17 @@ async function _getRequestDiagnosticsActions(
     },
   });
 
-  const agentActionIds = agentActionRes.hits.hits.map((hit) => hit._source?.action_id as string);
+  const agentActions = agentActionRes.hits.hits.map((hit) => ({
+    actionId: hit._source?.action_id as string,
+    timestamp: hit._source?.['@timestamp'],
+  }));
 
-  if (agentActionIds.length === 0) {
+  if (agentActions.length === 0) {
     return [];
   }
 
   try {
-    const actionResults = await esClient.search<any>({
+    const actionResultsRes = await esClient.search<any>({
       index: AGENT_ACTIONS_RESULTS_INDEX,
       ignore_unavailable: true,
       size: SO_SEARCH_LIMIT,
@@ -115,7 +128,7 @@ async function _getRequestDiagnosticsActions(
           must: [
             {
               terms: {
-                action_id: agentActionIds,
+                action_id: agentActions.map((action) => action.actionId),
               },
             },
             {
@@ -127,11 +140,21 @@ async function _getRequestDiagnosticsActions(
         },
       },
     });
-    return actionResults.hits.hits.map((hit) => ({
+    const actionResults = actionResultsRes.hits.hits.map((hit) => ({
       actionId: hit._source?.action_id as string,
       timestamp: hit._source?.['@timestamp'],
-      fileId: hit._source?.data?.file_id as string,
+      fileId: hit._source?.data?.upload_id as string,
+      error: hit._source?.error,
     }));
+    return agentActions.map((action) => {
+      const actionResult = actionResults.find((result) => result.actionId === action.actionId);
+      return {
+        actionId: action.actionId,
+        timestamp: actionResult?.timestamp ?? action.timestamp,
+        fileId: actionResult?.fileId,
+        error: actionResult?.error,
+      };
+    });
   } catch (err) {
     if (err.statusCode === 404) {
       // .fleet-actions-results does not yet exist

x-pack/test/fleet_api_integration/apis/agents/uploads.ts

@@ -53,7 +53,7 @@ export default function (providerContext: FtrProviderContext) {
             agent_id: 'agent1',
             '@timestamp': '2022-10-07T12:00:00.000Z',
             data: {
-              file_id: 'file1',
+              upload_id: 'file1',
             },
           },
         },
@@ -67,6 +67,7 @@ export default function (providerContext: FtrProviderContext) {
         body: {
           doc_as_upsert: true,
           doc: {
+            upload_id: 'file1',
             file: {
               ChunkSize: 4194304,
               extension: 'zip',