[Index Management] Support rollover of datastreams from component templates linked to managed/packaged index templates (#187733)

x-pack/plugins/index_management/common/types/component_templates.ts

@@ -49,3 +49,11 @@ export interface ComponentTemplateListItem {
 export interface ComponentTemplateDatastreams {
   data_streams: string[];
 }
+
+export interface ComponentTemplateMeta {
+  managed: boolean;
+  managed_by: string;
+  package: {
+    name: string;
+  };
+}

x-pack/plugins/index_management/public/application/components/component_templates/__jest__/client_integration/component_template_edit.test.tsx

@@ -78,7 +78,6 @@ describe('<ComponentTemplateEdit />', () => {
     template: {
       settings: { number_of_shards: 1 },
     },
-    _kbnMeta: { usedBy: [], isManaged: false },
   };
 
   beforeEach(async () => {
@@ -166,27 +165,30 @@ describe('<ComponentTemplateEdit />', () => {
           }),
         })
       );
-      // Mapping rollout modal should not be opened if the component template is not managed by Fleet
+      // Mapping rollout modal should not be opened if the component template is not managed
       expect(coreStart.overlays.openModal).not.toBeCalled();
     });
   });
 
-  describe('managed by fleet', () => {
+  describe('can rollover linked datastreams', () => {
     const DATASTREAM_NAME = 'logs-test-default';
+    const CUSTOM_COMPONENT_TEMPLATE = 'comp-1@custom';
+    const ENCODED_CUSTOM_COMPONENT_TEMPLATE = encodeURIComponent(CUSTOM_COMPONENT_TEMPLATE);
+
     beforeEach(async () => {
       httpRequestsMockHelpers.setLoadComponentTemplateResponse(
-        COMPONENT_TEMPLATE_TO_EDIT.name,
+        ENCODED_CUSTOM_COMPONENT_TEMPLATE,
         Object.assign({}, COMPONENT_TEMPLATE_TO_EDIT, {
-          _meta: { managed_by: 'fleet' },
+          name: CUSTOM_COMPONENT_TEMPLATE,
         })
       );
 
-      httpRequestsMockHelpers.setGetComponentTemplateDatastream(COMPONENT_TEMPLATE_TO_EDIT.name, {
+      httpRequestsMockHelpers.setGetComponentTemplateDatastream(ENCODED_CUSTOM_COMPONENT_TEMPLATE, {
         data_streams: [DATASTREAM_NAME],
       });
 
       await act(async () => {
-        testBed = await setup(httpSetup);
+        testBed = await setup(httpSetup, '@custom');
       });
 
       testBed.component.update();
@@ -221,7 +223,7 @@ describe('<ComponentTemplateEdit />', () => {
       component.update();
 
       expect(httpSetup.put).toHaveBeenLastCalledWith(
-        `${API_BASE_PATH}/component_templates/${COMPONENT_TEMPLATE_TO_EDIT.name}`,
+        `${API_BASE_PATH}/component_templates/${ENCODED_CUSTOM_COMPONENT_TEMPLATE}`,
         expect.anything()
       );
       expect(httpSetup.post).toHaveBeenLastCalledWith(
@@ -259,7 +261,7 @@ describe('<ComponentTemplateEdit />', () => {
       component.update();
 
       expect(httpSetup.put).toHaveBeenLastCalledWith(
-        `${API_BASE_PATH}/component_templates/${COMPONENT_TEMPLATE_TO_EDIT.name}`,
+        `${API_BASE_PATH}/component_templates/${ENCODED_CUSTOM_COMPONENT_TEMPLATE}`,
         expect.anything()
       );
       expect(httpSetup.post).toHaveBeenLastCalledWith(
@@ -269,5 +271,76 @@ describe('<ComponentTemplateEdit />', () => {
 
       expect(coreStart.overlays.openModal).not.toBeCalled();
     });
+
+    it('should show mappings rollover modal on save if referenced index template is managed and packaged', async () => {
+      httpRequestsMockHelpers.setLoadComponentTemplateResponse(
+        COMPONENT_TEMPLATE_TO_EDIT.name,
+        Object.assign({}, COMPONENT_TEMPLATE_TO_EDIT, {
+          _meta: {},
+        })
+      );
+
+      httpRequestsMockHelpers.setGetComponentTemplateDatastream(COMPONENT_TEMPLATE_TO_EDIT.name, {
+        data_streams: [DATASTREAM_NAME],
+      });
+
+      httpRequestsMockHelpers.setLoadReferencedIndexTemplateMetaResponse(
+        COMPONENT_TEMPLATE_TO_EDIT.name,
+        {
+          package: {
+            name: 'security',
+          },
+          managed_by: 'security',
+          managed: true,
+        }
+      );
+
+      await act(async () => {
+        testBed = await setup(httpSetup);
+      });
+
+      testBed.component.update();
+
+      httpRequestsMockHelpers.setPostDatastreamMappingsFromTemplate(
+        DATASTREAM_NAME,
+        {},
+        { message: 'Bad request', statusCode: 400 }
+      );
+      const { exists, actions, component, form, coreStart } = testBed;
+
+      await act(async () => {
+        form.setInputValue('versionField.input', '1');
+      });
+
+      await act(async () => {
+        actions.clickNextButton();
+      });
+
+      component.update();
+
+      await actions.completeStepSettings();
+      await actions.completeStepMappings();
+      await actions.completeStepAliases();
+
+      // Make sure the list of affected mappings is shown
+      expect(exists('affectedMappingsList')).toBe(true);
+
+      await act(async () => {
+        actions.clickNextButton();
+      });
+
+      component.update();
+
+      expect(httpSetup.put).toHaveBeenLastCalledWith(
+        `${API_BASE_PATH}/component_templates/${COMPONENT_TEMPLATE_TO_EDIT.name}`,
+        expect.anything()
+      );
+      expect(httpSetup.post).toHaveBeenLastCalledWith(
+        `${API_BASE_PATH}/data_streams/${DATASTREAM_NAME}/mappings_from_template`,
+        expect.anything()
+      );
+
+      expect(coreStart.overlays.openModal).toBeCalled();
+    });
   });
 });

x-pack/plugins/index_management/public/application/components/component_templates/__jest__/client_integration/helpers/component_template_form.helpers.ts

@@ -189,4 +189,5 @@ export type ComponentTemplateFormTestSubjects =
   | 'aliasesEditor'
   | 'mappingsEditor'
   | 'settingsEditor'
+  | 'affectedMappingsList'
   | 'versionField.input';

x-pack/plugins/index_management/public/application/components/component_templates/__jest__/client_integration/helpers/http_requests.ts

@@ -76,6 +76,18 @@ const registerHttpRequestMockHelpers = (
     error?: ResponseError
   ) => mockResponse('GET', `${API_BASE_PATH}/component_templates/${templateId}`, response, error);
 
+  const setLoadReferencedIndexTemplateMetaResponse = (
+    templateId: string,
+    response?: HttpResponse,
+    error?: ResponseError
+  ) =>
+    mockResponse(
+      'GET',
+      `${API_BASE_PATH}/component_templates/${templateId}/referenced_index_template_meta`,
+      response,
+      error
+    );
+
   const setDeleteComponentTemplateResponse = (
     templateId: string,
     response?: HttpResponse,
@@ -100,6 +112,7 @@ const registerHttpRequestMockHelpers = (
 
   return {
     setLoadComponentTemplatesResponse,
+    setLoadReferencedIndexTemplateMetaResponse,
     setDeleteComponentTemplateResponse,
     setLoadComponentTemplateResponse,
     setCreateComponentTemplateResponse,

x-pack/plugins/index_management/public/application/components/component_templates/component_template_wizard/component_template_create/component_template_create.tsx

@@ -17,7 +17,6 @@ import { useComponentTemplatesContext } from '../../component_templates_context'
 import { ComponentTemplateForm } from '../component_template_form';
 import { useStepFromQueryString } from '../use_step_from_query_string';
 import { useDatastreamsRollover } from '../component_template_datastreams_rollover/use_datastreams_rollover';
-import { MANAGED_BY_FLEET } from '../../constants';
 
 interface Props {
   /**
@@ -33,13 +32,38 @@ export const ComponentTemplateCreate: React.FunctionComponent<RouteComponentProp
   const [isSaving, setIsSaving] = useState<boolean>(false);
   const [saveError, setSaveError] = useState<any>(null);
   const redirectTo = useRedirectPath(history);
+  const [currentStep, setCurrentStep] = useState<string>('logistics');
+  const [componentName, setComponentName] = useState<string | undefined>();
+  const [canRollover, setCanRollover] = useState<boolean>(false);
 
+  const { api } = useComponentTemplatesContext();
   const { activeStep: defaultActiveStep, updateStep } = useStepFromQueryString(history);
 
   const locationSearchParams = useMemo(() => {
     return new URLSearchParams(history.location.search);
   }, [history.location.search]);
 
+  // Effect for computing if we should allow the user to rollover attached datastreams
+  useEffect(() => {
+    async function computeCanRollover() {
+      // When the current step is not logistics, we have an available component template
+      // name that we can use to query the referenced index template.
+      if (currentStep !== 'logistics') {
+        // If the component template is referenced by an index template that is part of
+        // a package and is managed we can allow the user to roll it over if possible.
+        const { data: refIndexTemplate } = await api.getReferencedIndexTemplateMeta(
+          componentName as string
+        );
+
+        setCanRollover(Boolean(refIndexTemplate?.managed_by && refIndexTemplate?.package));
+      }
+
+      setCanRollover(false);
+    }
+
+    computeCanRollover();
+  }, [api, currentStep, componentName, setCanRollover]);
+
   const defaultValue = useMemo(() => {
     if (sourceComponentTemplate) {
       return sourceComponentTemplate;
@@ -59,7 +83,6 @@ export const ComponentTemplateCreate: React.FunctionComponent<RouteComponentProp
     };
   }, [locationSearchParams, sourceComponentTemplate]);
 
-  const { api } = useComponentTemplatesContext();
   const { showDatastreamRolloverModal } = useDatastreamsRollover();
 
   const onSave = async (componentTemplate: ComponentTemplateDeserialized) => {
@@ -76,7 +99,11 @@ export const ComponentTemplateCreate: React.FunctionComponent<RouteComponentProp
       setSaveError(error);
       return;
     }
-    if (componentTemplate._meta?.managed_by === MANAGED_BY_FLEET) {
+
+    // We only want to allow rolling over linked datastreams for either @custom templates
+    // or when the component template is referenced by an index template that is part of
+    // a package and is managed.
+    if (componentTemplate.name.endsWith('@custom') || canRollover) {
       await showDatastreamRolloverModal(componentTemplate.name);
     }
 
@@ -121,11 +148,15 @@ export const ComponentTemplateCreate: React.FunctionComponent<RouteComponentProp
 
       <ComponentTemplateForm
         defaultActiveWizardSection={defaultActiveStep}
-        onStepChange={updateStep}
+        onStepChange={(step) => {
+          setCurrentStep(step);
+          updateStep(step);
+        }}
         defaultValue={defaultValue}
         onSave={onSave}
         isSaving={isSaving}
         saveError={saveError}
+        setComponentName={setComponentName}
         clearSaveError={clearSaveError}
       />
     </EuiPageSection>

x-pack/plugins/index_management/public/application/components/component_templates/component_template_wizard/component_template_edit/component_template_edit.tsx

@@ -21,7 +21,6 @@ import {
 } from '../../shared_imports';
 import { ComponentTemplateForm } from '../component_template_form';
 import { useRedirectPath } from '../../../../hooks/redirect_path';
-import { MANAGED_BY_FLEET } from '../../constants';
 
 import { useStepFromQueryString } from '../use_step_from_query_string';
 import { useDatastreamsRollover } from '../component_template_datastreams_rollover/use_datastreams_rollover';
@@ -48,6 +47,13 @@ export const ComponentTemplateEdit: React.FunctionComponent<RouteComponentProps<
   const { error, data: componentTemplate, isLoading } = api.useLoadComponentTemplate(decodedName);
   const { data: dataStreamResponse } = api.useLoadComponentTemplatesDatastream(decodedName);
   const dataStreams = useMemo(() => dataStreamResponse?.data_streams ?? [], [dataStreamResponse]);
+  // If the component template is referenced by an index template that is part of
+  // a package and is managed we can allow the user to roll it over if possible.
+  const { data: refIndexTemplate } = api.useLoadReferencedIndexTemplateMeta(decodedName);
+  const canRollover = useMemo(
+    () => Boolean(refIndexTemplate?.managed_by && refIndexTemplate?.package),
+    [refIndexTemplate]
+  );
 
   const { showDatastreamRolloverModal } = useDatastreamsRollover();
 
@@ -68,9 +74,13 @@ export const ComponentTemplateEdit: React.FunctionComponent<RouteComponentProps<
       return;
     }
 
-    if (updatedComponentTemplate._meta?.managed_by === MANAGED_BY_FLEET) {
+    // We only want to allow rolling over linked datastreams for either @custom templates
+    // or when the component template is referenced by an index template that is part of
+    // a package and is managed.
+    if (updatedComponentTemplate.name.endsWith('@custom') || canRollover) {
       await showDatastreamRolloverModal(updatedComponentTemplate.name);
     }
+
     redirectTo({
       pathname: encodeURI(
         `/component_templates/${encodeURIComponent(updatedComponentTemplate.name)}`
@@ -150,6 +160,7 @@ export const ComponentTemplateEdit: React.FunctionComponent<RouteComponentProps<
       <ComponentTemplateForm
         defaultValue={componentTemplate!}
         dataStreams={dataStreams}
+        canRollover={canRollover}
         defaultActiveWizardSection={defaultActiveStep}
         onStepChange={updateStep}
         onSave={onSave}

x-pack/plugins/index_management/public/application/components/component_templates/component_template_wizard/component_template_form/component_template_form.tsx

@@ -38,6 +38,7 @@ export type WizardSection = keyof WizardContent | 'review';
 interface Props {
   onSave: (componentTemplate: ComponentTemplateDeserialized) => void;
   clearSaveError: () => void;
+  setComponentName?: (name: string) => void;
   isSaving: boolean;
   saveError: any;
   defaultValue?: ComponentTemplateDeserialized;
@@ -45,6 +46,7 @@ interface Props {
   defaultActiveWizardSection?: WizardSection;
   onStepChange?: (stepId: string) => void;
   dataStreams?: string[];
+  canRollover?: boolean;
 }
 
 const wizardSections: { [id: string]: { id: WizardSection; label: string } } = {
@@ -90,7 +92,9 @@ export const ComponentTemplateForm = ({
       isManaged: false,
     },
   },
+  setComponentName,
   dataStreams,
+  canRollover,
   isEditing,
   isSaving,
   saveError,
@@ -237,6 +241,16 @@ export const ComponentTemplateForm = ({
       texts={i18nTexts}
       defaultActiveStep={defaultActiveStepIndex}
       onStepChange={onStepChange}
+      onChange={(attrs) => {
+        // Let the parent component know the name of the component template in the
+        // form has changed, so that it can re-compute the canRollover prop.
+        // This is needed for determinating if the user should see a rollover
+        // attached datastreams modal or not.
+        const data = attrs.getData();
+        if (setComponentName) {
+          setComponentName(data.logistics.name);
+        }
+      }}
     >
       <FormWizardStep
         id={wizardSections.logistics.id}
@@ -262,6 +276,7 @@ export const ComponentTemplateForm = ({
         <StepReviewContainer
           getComponentTemplateData={buildComponentTemplateObject(defaultValue)}
           dataStreams={dataStreams}
+          canRollover={canRollover}
         />
       </FormWizardStep>
     </FormWizard>

x-pack/plugins/index_management/public/application/components/component_templates/component_template_wizard/component_template_form/steps/step_review.tsx

@@ -27,7 +27,6 @@ import {
   serializers,
   serializeComponentTemplate,
 } from '../../../shared_imports';
-import { MANAGED_BY_FLEET } from '../../../constants';
 import { getLifecycleValue } from '../../../../../lib/data_streams';
 
 const INFINITE_AS_ICON = true;
@@ -52,10 +51,11 @@ const getDescriptionText = (data: any) => {
 interface Props {
   componentTemplate: ComponentTemplateDeserialized;
   dataStreams?: string[];
+  canRollover?: boolean;
 }
 
 export const StepReview: React.FunctionComponent<Props> = React.memo(
-  ({ dataStreams, componentTemplate }) => {
+  ({ dataStreams, canRollover, componentTemplate }) => {
     const { name } = componentTemplate;
 
     const serializedComponentTemplate = serializeComponentTemplate(
@@ -70,8 +70,8 @@ export const StepReview: React.FunctionComponent<Props> = React.memo(
       version: serializedVersion,
     } = serializedComponentTemplate;
 
-    const isFleetDatastreamsVisible =
+    const areDatastreamsVisible =
-      Boolean(dataStreams?.length) && componentTemplate._meta?.managed_by === MANAGED_BY_FLEET;
+      Boolean(dataStreams?.length) && (componentTemplate.name.endsWith('@custom') || canRollover);
 
     const SummaryTab = () => (
       <div data-test-subj="summaryTab">
@@ -138,8 +138,8 @@ export const StepReview: React.FunctionComponent<Props> = React.memo(
               </EuiDescriptionListDescription>
             </EuiDescriptionList>
           </EuiFlexItem>
-          {isFleetDatastreamsVisible && dataStreams && (
+          {areDatastreamsVisible && dataStreams && (
-            <EuiFlexItem>
+            <EuiFlexItem data-test-subj="affectedMappingsList">
               {/* Datastream mappings */}
               <FormattedMessage
                 id="xpack.idxMgmt.templateForm.stepReview.summaryTab.datastreamsLabel"
@@ -207,7 +207,7 @@ export const StepReview: React.FunctionComponent<Props> = React.memo(
             {request}
           </EuiCodeBlock>
 
-          {isFleetDatastreamsVisible && (
+          {areDatastreamsVisible && (
             <>
               <EuiSpacer size="m" />
               <EuiText>

x-pack/plugins/index_management/public/application/components/component_templates/component_template_wizard/component_template_form/steps/step_review_container.tsx

@@ -14,16 +14,23 @@ import { StepReview } from './step_review';
 interface Props {
   getComponentTemplateData: (wizardContent: WizardContent) => ComponentTemplateDeserialized;
   dataStreams?: string[];
+  canRollover?: boolean;
 }
 
 export const StepReviewContainer = React.memo(
-  ({ getComponentTemplateData, dataStreams }: Props) => {
+  ({ getComponentTemplateData, dataStreams, canRollover }: Props) => {
     const { getData } = Forms.useMultiContentContext<WizardContent>();
 
     const wizardContent = getData();
     // Build the final template object, providing the wizard content data
     const componentTemplate = getComponentTemplateData(wizardContent);
 
-    return <StepReview dataStreams={dataStreams} componentTemplate={componentTemplate} />;
+    return (
+      <StepReview
+        dataStreams={dataStreams}
+        canRollover={canRollover}
+        componentTemplate={componentTemplate}
+      />
+    );
   }
 );

x-pack/plugins/index_management/public/application/components/component_templates/lib/api.ts

@@ -11,6 +11,7 @@ import {
   ComponentTemplateDeserialized,
   ComponentTemplateSerialized,
   ComponentTemplateDatastreams,
+  ComponentTemplateMeta,
 } from '../shared_imports';
 import {
   UIM_COMPONENT_TEMPLATE_DELETE_MANY,
@@ -109,13 +110,33 @@ export const getApi = (
     });
   }
 
+  function useLoadReferencedIndexTemplateMeta(name: string) {
+    return useRequest<ComponentTemplateMeta>({
+      path: `${apiBasePath}/component_templates/${encodeURIComponent(
+        name
+      )}/referenced_index_template_meta`,
+      method: 'get',
+    });
+  }
+
+  async function getReferencedIndexTemplateMeta(name: string) {
+    return sendRequest<ComponentTemplateMeta>({
+      path: `${apiBasePath}/component_templates/${encodeURIComponent(
+        name
+      )}/referenced_index_template_meta`,
+      method: 'get',
+    });
+  }
+
   return {
     useLoadComponentTemplates,
     deleteComponentTemplates,
     useLoadComponentTemplate,
     createComponentTemplate,
     updateComponentTemplate,
+    useLoadReferencedIndexTemplateMeta,
     useLoadComponentTemplatesDatastream,
+    getReferencedIndexTemplateMeta,
     getComponentTemplateDatastreams,
     postDataStreamRollover,
     postDataStreamMappingsFromTemplate,

x-pack/plugins/index_management/public/application/components/component_templates/shared_imports.ts

@@ -68,6 +68,7 @@ export type {
   ComponentTemplateDeserialized,
   ComponentTemplateListItem,
   ComponentTemplateDatastreams,
+  ComponentTemplateMeta,
 } from '../../../../common';
 
 export { serializeComponentTemplate } from '../../../../common/lib';

x-pack/plugins/index_management/server/routes/api/component_templates/index.ts

@@ -12,13 +12,17 @@ import { registerCreateRoute } from './register_create_route';
 import { registerUpdateRoute } from './register_update_route';
 import { registerDeleteRoute } from './register_delete_route';
 import { registerPrivilegesRoute } from './register_privileges_route';
-import { registerGetDatastreams } from './register_datastream_route';
+import {
+  registerGetDatastreams,
+  registerReferencedIndexTemplateMeta,
+} from './register_datastream_route';
 
 export function registerComponentTemplateRoutes(dependencies: RouteDependencies) {
   registerGetAllRoute(dependencies);
   registerCreateRoute(dependencies);
   registerUpdateRoute(dependencies);
   registerGetDatastreams(dependencies);
+  registerReferencedIndexTemplateMeta(dependencies);
   registerDeleteRoute(dependencies);
   registerPrivilegesRoute(dependencies);
 }

x-pack/plugins/index_management/server/routes/api/component_templates/register_datastream_route.ts

@@ -77,3 +77,41 @@ export const registerGetDatastreams = ({
     }
   );
 };
+
+export const registerReferencedIndexTemplateMeta = ({
+  router,
+  lib: { handleEsError },
+}: RouteDependencies): void => {
+  router.get(
+    {
+      path: addBasePath('/component_templates/{name}/referenced_index_template_meta'),
+      validate: {
+        params: paramsSchema,
+      },
+    },
+    async (context, request, response) => {
+      const { client } = (await context.core).elasticsearch;
+      const { name } = request.params;
+
+      try {
+        const { index_templates: indexTemplates } =
+          await client.asCurrentUser.indices.getIndexTemplate();
+        const result = indexTemplates.filter((indexTemplate) =>
+          indexTemplate.index_template?.composed_of?.includes(name)
+        );
+
+        // We should always match against the first result which should yield
+        // the index template we are after.
+        if (result[0]) {
+          return response.ok({
+            body: result[0].index_template._meta,
+          });
+        }
+
+        return response.notFound();
+      } catch (error) {
+        return handleEsError({ error, response });
+      }
+    }
+  );
+};