[8.12] [Fleet] Fix package showing 'Needs authorization' warning even after transform assets were authorized successfully (#176647) (#177236)

# Backport

This will backport the following commits from `main` to `8.12`:
- [[Fleet] Fix package showing 'Needs authorization' warning
even after transform assets were authorized successfully
(#176647)](https://github.com/elastic/kibana/pull/176647)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Quynh Nguyen
(Quinn)","email":"43350163+qn895@users.noreply.github.com"},"sourceCommit":{"committedDate":"2024-02-19T19:12:17Z","message":"[Fleet]
Fix package showing 'Needs authorization' warning even after transform
assets were authorized successfully
(#176647)","sha":"4e10d1c70b30cf1c6d8eec8a87a9badc6ad422cb","branchLabelMapping":{"^v8.14.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["bug","release_note:fix",":ml","Feature:Transforms","Team:Fleet","v8.13.0","v8.12.2","v8.14.0"],"title":"[Fleet]
Fix package showing 'Needs authorization' warning even after transform
assets were authorized
successfully","number":176647,"url":"https://github.com/elastic/kibana/pull/176647","mergeCommit":{"message":"[Fleet]
Fix package showing 'Needs authorization' warning even after transform
assets were authorized successfully
(#176647)","sha":"4e10d1c70b30cf1c6d8eec8a87a9badc6ad422cb"}},"sourceBranch":"main","suggestedTargetBranches":["8.13","8.12"],"targetPullRequestStates":[{"branch":"8.13","label":"v8.13.0","branchLabelMappingKey":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"8.12","label":"v8.12.2","branchLabelMappingKey":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"main","label":"v8.14.0","branchLabelMappingKey":"^v8.14.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/176647","number":176647,"mergeCommit":{"message":"[Fleet]
Fix package showing 'Needs authorization' warning even after transform
assets were authorized successfully
(#176647)","sha":"4e10d1c70b30cf1c6d8eec8a87a9badc6ad422cb"}}]}]
BACKPORT-->

Co-authored-by: Quynh Nguyen (Quinn) <43350163+qn895@users.noreply.github.com>

x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/assets/assets.tsx

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import React, { Fragment, useEffect, useState } from 'react';
+import React, { Fragment, useEffect, useState, useCallback } from 'react';
 import { Redirect } from 'react-router-dom';
 import { FormattedMessage } from '@kbn/i18n-react';
 import { EuiFlexGroup, EuiFlexItem, EuiLink, EuiSpacer, EuiTitle, EuiCallOut } from '@elastic/eui';
@@ -38,10 +38,12 @@ import { AssetsAccordion } from './assets_accordion';
 
 interface AssetsPanelProps {
   packageInfo: PackageInfo;
+  refetchPackageInfo: () => void;
 }
 
-export const AssetsPage = ({ packageInfo }: AssetsPanelProps) => {
+export const AssetsPage = ({ packageInfo, refetchPackageInfo }: AssetsPanelProps) => {
   const { name, version } = packageInfo;
+
   const pkgkey = `${name}-${version}`;
   const { spaces, docLinks } = useStartServices();
   const customAssetsExtension = useUIExtension(packageInfo.name, 'package-detail-assets');
@@ -60,6 +62,12 @@ export const AssetsPage = ({ packageInfo }: AssetsPanelProps) => {
   const [fetchError, setFetchError] = useState<undefined | Error>();
   const [isLoading, setIsLoading] = useState<boolean>(true);
 
+  const forceRefreshAssets = useCallback(() => {
+    if (refetchPackageInfo) {
+      refetchPackageInfo();
+    }
+  }, [refetchPackageInfo]);
+
   useEffect(() => {
     const fetchAssetSavedObjects = async () => {
       if ('installationInfo' in packageInfo) {
@@ -245,6 +253,7 @@ export const AssetsPage = ({ packageInfo }: AssetsPanelProps) => {
       <DeferredAssetsSection
         deferredInstallations={deferredInstallations}
         packageInfo={packageInfo}
+        forceRefreshAssets={forceRefreshAssets}
       />
       <EuiSpacer size="m" />
     </>

x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/assets/deferred_assets_accordion.tsx

@@ -26,11 +26,13 @@ import { DeferredTransformAccordion } from './deferred_transforms_accordion';
 interface Props {
   packageInfo: PackageInfo;
   deferredInstallations: EsAssetReference[];
+  forceRefreshAssets?: () => void;
 }
 
 export const DeferredAssetsSection: FunctionComponent<Props> = ({
   deferredInstallations,
   packageInfo,
+  forceRefreshAssets,
 }) => {
   const authz = useAuthz();
 
@@ -60,6 +62,7 @@ export const DeferredAssetsSection: FunctionComponent<Props> = ({
         packageInfo={packageInfo}
         type={ElasticsearchAssetType.transform}
         deferredInstallations={deferredTransforms}
+        forceRefreshAssets={forceRefreshAssets}
       />
     </>
   );

x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/assets/deferred_transforms_accordion.tsx

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import React, { Fragment, useCallback, useMemo, useState } from 'react';
+import React, { Fragment, useCallback, useState, useMemo } from 'react';
 import type { FunctionComponent, MouseEvent } from 'react';
 
 import {
@@ -42,6 +42,7 @@ interface Props {
   packageInfo: PackageInfo;
   type: ElasticsearchAssetType.transform;
   deferredInstallations: EsAssetReference[];
+  forceRefreshAssets?: () => void;
 }
 
 export const getDeferredAssetDescription = (
@@ -83,6 +84,7 @@ export const DeferredTransformAccordion: FunctionComponent<Props> = ({
   packageInfo,
   type,
   deferredInstallations,
+  forceRefreshAssets,
 }) => {
   const { notifications } = useStartServices();
   const [isLoading, setIsLoading] = useState(false);
@@ -159,6 +161,9 @@ export const DeferredTransformAccordion: FunctionComponent<Props> = ({
               ),
               { toastLifeTimeMs: 1000 }
             );
+            if (forceRefreshAssets) {
+              forceRefreshAssets();
+            }
           }
         }
       } catch (e) {
@@ -171,11 +176,14 @@ export const DeferredTransformAccordion: FunctionComponent<Props> = ({
               }
             ),
           });
+          if (forceRefreshAssets) {
+            forceRefreshAssets();
+          }
         }
       }
       setIsLoading(false);
     },
-    [notifications.toasts, packageInfo.name, packageInfo.version]
+    [notifications.toasts, packageInfo.name, packageInfo.version, forceRefreshAssets]
   );
   if (deferredTransforms.length === 0) return null;
   return (

x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/index.tsx

@@ -776,7 +776,7 @@ export function Detail() {
             <SettingsPage packageInfo={packageInfo} theme$={services.theme.theme$} />
           </Route>
           <Route path={INTEGRATIONS_ROUTING_PATHS.integration_details_assets}>
-            <AssetsPage packageInfo={packageInfo} />
+            <AssetsPage packageInfo={packageInfo} refetchPackageInfo={refetchPackageInfo} />
           </Route>
           <Route path={INTEGRATIONS_ROUTING_PATHS.integration_details_configs}>
             <Configs packageInfo={packageInfo} />

x-pack/plugins/fleet/server/routes/epm/handlers.ts

@@ -412,6 +412,8 @@ export const bulkInstallPackagesFromRegistryHandler: FleetRequestHandler<
   const savedObjectsClient = fleetContext.internalSoClient;
   const esClient = coreContext.elasticsearch.client.asInternalUser;
   const spaceId = fleetContext.spaceId;
+  const user = (await appContextService.getSecurity()?.authc.getCurrentUser(request)) || undefined;
+  const authorizationHeader = HTTPAuthorizationHeader.parseFromRequest(request, user?.username);
 
   const bulkInstalledResponses = await bulkInstallPackages({
     savedObjectsClient,
@@ -420,6 +422,7 @@ export const bulkInstallPackagesFromRegistryHandler: FleetRequestHandler<
     spaceId,
     prerelease: request.query.prerelease,
     force: request.body.force,
+    authorizationHeader,
   });
   const payload = bulkInstalledResponses.map(bulkInstallServiceResponseToHttpEntry);
   const body: BulkInstallPackagesResponse = {

x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts

@@ -447,7 +447,6 @@ const installTransformsAssets = async (
         })
       : // No need to generate api key/secondary auth if all transforms are run as kibana_system user
         undefined;
-
     // delete all previous transform
     await Promise.all([
       deleteTransforms(
@@ -761,7 +760,9 @@ async function handleTransformInstall({
         throw err;
       }
     }
-  } else {
+  }
+
+  if (startTransform === false || transform?.content?.settings?.unattended === true) {
     // if transform was not set to start automatically in yml config,
     // we need to check using _stats if the transform had insufficient permissions
     try {
@@ -773,7 +774,11 @@ async function handleTransformInstall({
           ),
         { logger, additionalResponseStatuses: [400] }
       );
-      if (Array.isArray(transformStats.transforms) && transformStats.transforms.length === 1) {
+      if (
+        transformStats &&
+        Array.isArray(transformStats.transforms) &&
+        transformStats.transforms.length === 1
+      ) {
         const transformHealth = transformStats.transforms[0].health;
         if (
           transformHealth &&

x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/reauthorize.ts

@@ -10,6 +10,8 @@ import type { Logger } from '@kbn/logging';
 import type { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server';
 
 import { sortBy, uniqBy } from 'lodash';
+import { isPopulatedObject } from '@kbn/ml-is-populated-object';
+import type { ErrorResponseBase } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 
 import type { SecondaryAuthorizationHeader } from '../../../../../common/types/models/transform_api_key';
 import { updateEsAssetReferences } from '../../packages/install';
@@ -30,6 +32,9 @@ interface FleetTransformMetadata {
   transformId: string;
 }
 
+const isErrorResponse = (arg: unknown): arg is ErrorResponseBase =>
+  isPopulatedObject(arg, ['error']);
+
 async function reauthorizeAndStartTransform({
   esClient,
   logger,
@@ -68,6 +73,19 @@ async function reauthorizeAndStartTransform({
       () => esClient.transform.startTransform({ transform_id: transformId }, { ignore: [409] }),
       { logger, additionalResponseStatuses: [400] }
     );
+
+    // Transform can already be started even without sufficient permission if 'unattended: true'
+    // So we are just catching that special case to showcase in the UI
+    // If unattended, calling _start will return a successful response, but with the error message in the body
+    if (
+      isErrorResponse(startedTransform) &&
+      startedTransform.status === 409 &&
+      Array.isArray(startedTransform.error?.root_cause) &&
+      startedTransform.error.root_cause[0]?.reason?.includes('already started')
+    ) {
+      return { transformId, success: true, error: null };
+    }
+
     logger.debug(`Started transform: ${transformId}`);
     return { transformId, success: startedTransform.acknowledged, error: null };
   } catch (err) {

x-pack/plugins/fleet/server/services/epm/packages/get_bulk_assets.ts

@@ -37,8 +37,8 @@ export async function getBulkAssets(
         type: obj.type as unknown as ElasticsearchAssetType | KibanaSavedObjectType,
         updatedAt: obj.updated_at,
         attributes: {
-          title: obj.attributes.title,
-          description: obj.attributes.description,
+          title: obj.attributes?.title,
+          description: obj.attributes?.description,
         },
       };
     });

x-pack/plugins/fleet/server/services/epm/packages/remove.ts

@@ -190,24 +190,35 @@ async function deleteAssets(
   // must delete index templates first, or component templates which reference them cannot be deleted
   // must delete ingestPipelines first, or ml models referenced in them cannot be deleted.
   // separate the assets into Index Templates and other assets.
-  type Tuple = [EsAssetReference[], EsAssetReference[], EsAssetReference[]];
-  const [indexTemplatesAndPipelines, indexAssets, otherAssets] = installedEs.reduce<Tuple>(
-    ([indexTemplateAndPipelineTypes, indexAssetTypes, otherAssetTypes], asset) => {
-      if (
-        asset.type === ElasticsearchAssetType.indexTemplate ||
-        asset.type === ElasticsearchAssetType.ingestPipeline
-      ) {
-        indexTemplateAndPipelineTypes.push(asset);
-      } else if (asset.type === ElasticsearchAssetType.index) {
-        indexAssetTypes.push(asset);
-      } else {
-        otherAssetTypes.push(asset);
-      }
+  type Tuple = [EsAssetReference[], EsAssetReference[], EsAssetReference[], EsAssetReference[]];
+  const [indexTemplatesAndPipelines, indexAssets, transformAssets, otherAssets] =
+    installedEs.reduce<Tuple>(
+      (
+        [indexTemplateAndPipelineTypes, indexAssetTypes, transformAssetTypes, otherAssetTypes],
+        asset
+      ) => {
+        if (
+          asset.type === ElasticsearchAssetType.indexTemplate ||
+          asset.type === ElasticsearchAssetType.ingestPipeline
+        ) {
+          indexTemplateAndPipelineTypes.push(asset);
+        } else if (asset.type === ElasticsearchAssetType.index) {
+          indexAssetTypes.push(asset);
+        } else if (asset.type === ElasticsearchAssetType.transform) {
+          transformAssetTypes.push(asset);
+        } else {
+          otherAssetTypes.push(asset);
+        }
 
-      return [indexTemplateAndPipelineTypes, indexAssetTypes, otherAssetTypes];
-    },
-    [[], [], []]
-  );
+        return [
+          indexTemplateAndPipelineTypes,
+          indexAssetTypes,
+          transformAssetTypes,
+          otherAssetTypes,
+        ];
+      },
+      [[], [], [], []]
+    );
 
   try {
     // must first unset any default pipeline associated with any existing indices
@@ -215,7 +226,12 @@ async function deleteAssets(
     await Promise.all(
       indexAssets.map((asset) => updateIndexSettings(esClient, asset.id, { default_pipeline: '' }))
     );
-    // must delete index templates and pipelines first
+
+    // in case transform's destination index contains any pipline,
+    // we should delete the transforms first
+    await Promise.all(deleteESAssets(transformAssets, esClient));
+
+    // then delete index templates and pipelines
     await Promise.all(deleteESAssets(indexTemplatesAndPipelines, esClient));
     // then the other asset types
     await Promise.all([