[Fleet] Show pipelines and mappings editor for input packages (#154077)

## Summary

Closes #153105

Show the pipelines and mappings editor for input packages. This can be
seen when editing an input package integration policy.

Previously this was hidden for input packages and any integration
package which allowed datastream to be configured. I have tweaked this
logic so that we check if the matching index template for a stream
exists, if it does then show the editor.
<img width="407" alt="Screenshot 2023-03-30 at 17 02 26"
src="https://user-images.githubusercontent.com/3315046/228896543-192e5fc5-3dc3-4a0f-a08f-a7ff83555ef3.png">

This allows us to handle an edge case (which will become more common as
input packages are rolled out) when integration package which has since
been upgraded to an input package, for example:
- user installs the custom logs integration package and sets dataset to
helloworld
- the `logs-helloworld` index template is not created as integration
packages only create index templates on install (custom logs creates
`logs-log.log`)
- there is therefore no ingest pipeline `@custom` or `@package`component
template to customize, we do not show the pipeline editor.
- The user upgrades to an input package version of custom logs (coming
in v2.0.0) there still isn't an index template because we do not create
them on upgrade, so we still don't want to show the pipelines and
mappings editor
- Just checking for input package in this scenario would mean we showed
the editor incorrectly

I have opted to show the editors even if the stream is owned by another
package e.g custom logs data is being sent to an nginx stream.


### Automated tests

I have added cypress tests for checking that the mappings pipelines
editors are shown for input packages.

### Manual test cases

Here is version 2.0.0 of logs which is an input package:
[log-2.0.0.zip](11113579/log-2.0.0.zip)
it can be added to a deployment using curl:

```
curl -XPOST -H 'content-type: application/zip' -H 'kbn-xsrf: true' http://localhost:5601/mark/api/fleet/epm/packages -u elastic:changeme --data-binary @/path/to/log-2.0.0.zip
```


- Create a policy for an integration package which does not allow
dataset to be configured (e.g Apache)
    - pipelines & editor should be visible when editing package policy
- Create a policy for an integration package which does allow dataset to
be configured (e.g custom logs 1.6.1)
- pipelines & mappings editor should not be shown if dataset doesn't
match the one installed by the package `logs.log`
- Upgrade custom logs to the input package version (2.0.0)
    - pipelines and mappings editor still should not be shown

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

src/dev/precommit_hook/casing_check_config.js

@@ -29,6 +29,7 @@ export const IGNORE_FILE_GLOBS = [
   'x-pack/plugins/canvas/server/templates/assets/*.{png,jpg,svg}',
   'x-pack/plugins/cases/docs/**/*',
   'x-pack/plugins/monitoring/public/lib/jquery_flot/**/*',
+  'x-pack/plugins/fleet/cypress/packages/*.zip',
   '**/.*',
   '**/__mocks__/**/*',
   'x-pack/docs/**/*',

x-pack/plugins/fleet/cypress/e2e/package_policy_real.cy.ts

@@ -0,0 +1,98 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+  ADD_INTEGRATION_POLICY_BTN,
+  CREATE_PACKAGE_POLICY_SAVE_BTN,
+  INTEGRATION_NAME_LINK,
+  POLICY_EDITOR,
+} from '../screens/integrations';
+import { EXISTING_HOSTS_TAB } from '../screens/fleet';
+import { CONFIRM_MODAL } from '../screens/navigation';
+
+const TEST_PACKAGE = 'input_package-1.0.0';
+const agentPolicyId = 'test-input-package-policy';
+const agentPolicyName = 'Test input package policy';
+const inputPackagePolicyName = 'input-package-policy';
+
+function editPackagePolicyandShowAdvanced() {
+  cy.visit(`/app/integrations/detail/${TEST_PACKAGE}/policies`);
+
+  cy.getBySel(INTEGRATION_NAME_LINK).contains(inputPackagePolicyName).click();
+
+  cy.get('button').contains('Change defaults').click();
+  cy.get('[data-test-subj^="advancedStreamOptionsToggle"]').click();
+}
+describe('Input package create and edit package policy', () => {
+  before(() => {
+    cy.task('installTestPackage', TEST_PACKAGE);
+
+    cy.request({
+      method: 'POST',
+      url: `/api/fleet/agent_policies`,
+      body: {
+        id: agentPolicyId,
+        name: agentPolicyName,
+        description: 'desc',
+        namespace: 'default',
+        monitoring_enabled: [],
+      },
+      headers: { 'kbn-xsrf': 'cypress' },
+    });
+  });
+  after(() => {
+    // delete agent policy
+    cy.request({
+      method: 'POST',
+      url: `/api/fleet/agent_policies/delete`,
+      headers: { 'kbn-xsrf': 'cypress' },
+      body: JSON.stringify({
+        agentPolicyId,
+      }),
+    });
+    cy.task('uninstallTestPackage', TEST_PACKAGE);
+  });
+  it('should successfully create a package policy', () => {
+    cy.visit(`/app/integrations/detail/${TEST_PACKAGE}/overview`);
+    cy.getBySel(ADD_INTEGRATION_POLICY_BTN).click();
+
+    cy.getBySel(POLICY_EDITOR.POLICY_NAME_INPUT).click().clear().type(inputPackagePolicyName);
+    cy.getBySel('multiTextInput-paths')
+      .find('[data-test-subj="multiTextInputRow-0"]')
+      .click()
+      .type('/var/log/test.log');
+
+    cy.getBySel('multiTextInput-tags')
+      .find('[data-test-subj="multiTextInputRow-0"]')
+      .click()
+      .type('tag1');
+
+    cy.getBySel(POLICY_EDITOR.DATASET_SELECT).click().type('testdataset');
+
+    cy.getBySel(EXISTING_HOSTS_TAB).click();
+
+    cy.getBySel(POLICY_EDITOR.AGENT_POLICY_SELECT).click().get(`#${agentPolicyId}`).click();
+    cy.wait(500); // wait for policy id to be set
+    cy.getBySel(CREATE_PACKAGE_POLICY_SAVE_BTN).click();
+
+    cy.getBySel(CONFIRM_MODAL.CANCEL_BUTTON).click();
+  });
+
+  it('should show pipelines editor with link to pipeline', () => {
+    editPackagePolicyandShowAdvanced();
+    cy.getBySel(POLICY_EDITOR.INSPECT_PIPELINES_BTN).click();
+    cy.getBySel(CONFIRM_MODAL.CONFIRM_BUTTON).click();
+    cy.get('body').should('not.contain', 'Pipeline not found');
+    cy.get('body').should('contain', '"managed_by": "fleet"');
+  });
+  it('should show mappings editor with link to create custom template', () => {
+    editPackagePolicyandShowAdvanced();
+    cy.getBySel(POLICY_EDITOR.EDIT_MAPPINGS_BTN).click();
+    cy.getBySel(CONFIRM_MODAL.CONFIRM_BUTTON).click();
+    cy.get('body').should('contain', 'logs-testdataset@custom');
+  });
+});

x-pack/plugins/fleet/cypress/plugins/index.ts

@@ -5,12 +5,42 @@
  * 2.0.
  */
 
+import { promisify } from 'util';
+
+import fs from 'fs';
+
+import fetch from 'node-fetch';
 import { createEsClientForTesting } from '@kbn/test';
 
 const plugin: Cypress.PluginConfig = (on, config) => {
   const client = createEsClientForTesting({
     esUrl: config.env.ELASTICSEARCH_URL,
   });
+
+  async function kibanaFetch(opts: {
+    method: string;
+    path: string;
+    body?: any;
+    contentType?: string;
+  }) {
+    const { method, path, body, contentType } = opts;
+    const Authorization = `Basic ${Buffer.from(
+      `elastic:${config.env.ELASTICSEARCH_PASSWORD}`
+    ).toString('base64')}`;
+
+    const url = `${config.env.KIBANA_URL}${path}`;
+    const res = await fetch(url, {
+      method,
+      headers: {
+        'kbn-xsrf': 'cypress',
+        'Content-Type': contentType || 'application/json',
+        Authorization,
+      },
+      ...(body ? { body } : {}),
+    });
+
+    return res.json();
+  }
   on('task', {
     async insertDoc({ index, doc, id }: { index: string; doc: any; id: string }) {
       return client.create({ id, document: doc, index, refresh: 'wait_for' });
@@ -37,6 +67,23 @@ const plugin: Cypress.PluginConfig = (on, config) => {
         conflicts: 'proceed',
       });
     },
+    async installTestPackage(packageName: string) {
+      const zipPath = require.resolve('../packages/' + packageName + '.zip');
+      const zipContent = await promisify(fs.readFile)(zipPath, 'base64');
+      return kibanaFetch({
+        method: 'POST',
+        path: '/api/fleet/epm/packages',
+        body: Buffer.from(zipContent, 'base64'),
+        contentType: 'application/zip',
+      });
+    },
+
+    async uninstallTestPackage(packageName: string) {
+      return kibanaFetch({
+        method: 'DELETE',
+        path: `/api/fleet/epm/packages/${packageName}`,
+      });
+    },
   });
 };
 

x-pack/plugins/fleet/cypress/screens/integrations.ts

@@ -35,6 +35,14 @@ export const SETTINGS = {
   UNINSTALL_ASSETS_BTN: 'uninstallAssetsButton',
 };
 
+export const POLICY_EDITOR = {
+  POLICY_NAME_INPUT: 'packagePolicyNameInput',
+  DATASET_SELECT: 'datasetComboBox',
+  AGENT_POLICY_SELECT: 'agentPolicySelect',
+  INSPECT_PIPELINES_BTN: 'datastreamInspectPipelineBtn',
+  EDIT_MAPPINGS_BTN: 'datastreamEditMappingsBtn',
+};
+
 export const INTEGRATION_POLICIES_UPGRADE_CHECKBOX = 'epmDetails.upgradePoliciesCheckbox';
 
 export const getIntegrationCard = (integration: string) => `integration-card:epr:${integration}`;

x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/datastream_hooks.tsx

@@ -6,8 +6,9 @@
  */
 
 import { useRouteMatch } from 'react-router-dom';
+import { useQuery } from '@tanstack/react-query';
 
-import { useLink } from '../../../../hooks';
+import { sendRequestForRq, useLink } from '../../../../hooks';
 
 export function usePackagePolicyEditorPageUrl(dataStreamId?: string) {
   const {
@@ -27,3 +28,32 @@ export function usePackagePolicyEditorPageUrl(dataStreamId?: string) {
 
   return `${baseUrl}${dataStreamId ? `?datastreamId=${encodeURIComponent(dataStreamId)}` : ''}`;
 }
+
+export function useIndexTemplateExists(
+  templateName: string,
+  enabled: boolean
+): {
+  exists?: boolean;
+  isLoading: boolean;
+} {
+  const { data, isLoading } = useQuery(
+    ['indexTemplateExists', templateName],
+    () =>
+      sendRequestForRq({
+        path: `/api/index_management/index_templates/${templateName}`,
+        method: 'get',
+      }),
+    { enabled: enabled || !!templateName }
+  );
+
+  if (isLoading) {
+    return {
+      isLoading: true,
+    };
+  }
+
+  return {
+    exists: !!data,
+    isLoading: false,
+  };
+}

x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/datastream_mappings.tsx

@@ -18,7 +18,11 @@ import { usePackagePolicyEditorPageUrl } from './datastream_hooks';
 
 export interface PackagePolicyEditorDatastreamMappingsProps {
   packageInfo: PackageInfo;
-  packageInputStream: { id?: string; data_stream: { dataset: string; type: string } };
+  packageInputStream: {
+    id?: string;
+    data_stream: { dataset: string; type: string };
+  };
+  customDataset?: string;
 }
 
 function useComponentTemplates(dataStream: { dataset: string; type: string }) {
@@ -35,8 +39,10 @@ function useComponentTemplates(dataStream: { dataset: string; type: string }) {
 
 export const PackagePolicyEditorDatastreamMappings: React.FunctionComponent<
   PackagePolicyEditorDatastreamMappingsProps
-> = ({ packageInputStream, packageInfo }) => {
-  const dataStream = packageInputStream.data_stream;
+> = ({ packageInputStream, packageInfo, customDataset }) => {
+  const dataStream = customDataset
+    ? { ...packageInputStream.data_stream, dataset: customDataset }
+    : packageInputStream.data_stream;
   const pageUrl = usePackagePolicyEditorPageUrl(packageInputStream.id);
 
   const { application, docLinks } = useStartServices();

x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/datastream_pipelines.tsx

@@ -31,6 +31,7 @@ import { usePackagePolicyEditorPageUrl } from './datastream_hooks';
 export interface PackagePolicyEditorDatastreamPipelinesProps {
   packageInfo: PackageInfo;
   packageInputStream: { id?: string; data_stream: { dataset: string; type: string } };
+  customDataset?: string;
 }
 
 interface PipelineItem {
@@ -92,8 +93,10 @@ function useDatastreamIngestPipelines(
 
 export const PackagePolicyEditorDatastreamPipelines: React.FunctionComponent<
   PackagePolicyEditorDatastreamPipelinesProps
-> = ({ packageInputStream, packageInfo }) => {
-  const dataStream = packageInputStream.data_stream;
+> = ({ packageInputStream, packageInfo, customDataset }) => {
+  const dataStream = customDataset
+    ? { ...packageInputStream.data_stream, dataset: customDataset }
+    : packageInputStream.data_stream;
   const { application, share, docLinks } = useStartServices();
   const ingestPipelineLocator = share.url.locators.get('INGEST_PIPELINES_APP_LOCATOR');
 

x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/dataset_combo.tsx

@@ -94,6 +94,7 @@ export const DatasetComboBox: React.FC<{
         })}
         isClearable={false}
         isDisabled={isDisabled}
+        data-test-subj="datasetComboBox"
       />
       {valueAsOption && valueAsOption.value.package !== pkgName && (
         <>

x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/multi_text_input.tsx

@@ -25,6 +25,7 @@ interface Props {
   errors?: Array<{ message: string; index?: number }>;
   isInvalid?: boolean;
   isDisabled?: boolean;
+  'data-test-subj'?: string;
 }
 
 interface RowProps {
@@ -69,6 +70,7 @@ const Row: FunctionComponent<RowProps> = ({
           autoFocus={autoFocus}
           disabled={isDisabled}
           onBlur={onBlur}
+          data-test-subj={`multiTextInputRow-${index}`}
         />
       </EuiFlexItem>
       {showDeleteButton && (
@@ -99,6 +101,7 @@ export const MultiTextInput: FunctionComponent<Props> = ({
   isInvalid,
   isDisabled,
   errors,
+  'data-test-subj': dataTestSubj,
 }) => {
   const [autoFocus, setAutoFocus] = useState(false);
   const [rows, setRows] = useState(() => defaultValue(value));
@@ -139,7 +142,7 @@ export const MultiTextInput: FunctionComponent<Props> = ({
 
   return (
     <>
-      <EuiFlexGroup gutterSize="s" direction="column">
+      <EuiFlexGroup gutterSize="s" direction="column" data-test-subj={dataTestSubj}>
         {rows.map((row, idx) => (
           <EuiFlexItem key={idx}>
             <Row

x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_stream.tsx

@@ -25,7 +25,10 @@ import { DATASET_VAR_NAME } from '../../../../../../../../../common/constants';
 
 import { useConfig, useGetDataStreams } from '../../../../../../../../hooks';
 
-import { mapPackageReleaseToIntegrationCardRelease } from '../../../../../../../../../common/services';
+import {
+  getRegistryDataStreamAssetBaseName,
+  mapPackageReleaseToIntegrationCardRelease,
+} from '../../../../../../../../../common/services';
 import type { ExperimentalDataStreamFeature } from '../../../../../../../../../common/types/models/epm';
 
 import type {
@@ -41,6 +44,8 @@ import { isAdvancedVar, validationHasErrors } from '../../../services';
 import { PackagePolicyEditorDatastreamPipelines } from '../../datastream_pipelines';
 import { PackagePolicyEditorDatastreamMappings } from '../../datastream_mappings';
 
+import { useIndexTemplateExists } from '../../datastream_hooks';
+
 import { ExperimentDatastreamSettings } from './experimental_datastream_settings';
 import { PackagePolicyInputVarField } from './package_policy_input_var_field';
 import { useDataStreamId } from './hooks';
@@ -90,12 +95,25 @@ export const PackagePolicyInputStreamConfig = memo<Props>(
       !!packagePolicyInputStream.id &&
       packagePolicyInputStream.id === defaultDataStreamId;
     const isPackagePolicyEdit = !!packagePolicyId;
-    const isInputOnlyPackage = packageInfo.type === 'input';
 
-    const hasDatasetVar = packageInputStream.vars?.some(
-      (varDef) => varDef.name === DATASET_VAR_NAME
-    );
-    const showPipelinesAndMappings = !isInputOnlyPackage && !hasDatasetVar;
+    const customDatasetVar = packagePolicyInputStream.vars?.[DATASET_VAR_NAME];
+    const customDatasetVarValue = customDatasetVar?.value?.dataset || customDatasetVar?.value;
+
+    const { exists: indexTemplateExists, isLoading: isLoadingIndexTemplate } =
+      useIndexTemplateExists(
+        getRegistryDataStreamAssetBaseName({
+          dataset: customDatasetVarValue,
+          type: packageInputStream.data_stream.type,
+        }),
+        isPackagePolicyEdit
+      );
+
+    // only show pipelines and mappings if the matching index template exists
+    // in the legacy case (e.g logs package pre 2.0.0) the index template will not exist
+    // because we allowed dataset to be customized but didnt create a matching index template
+    // for the new dataset.
+    const showPipelinesAndMappings = !isLoadingIndexTemplate && indexTemplateExists;
+
     useEffect(() => {
       if (isDefaultDatastream && containerRef.current) {
         containerRef.current.scrollIntoView();
@@ -249,6 +267,7 @@ export const PackagePolicyInputStreamConfig = memo<Props>(
                         iconType={isShowingAdvanced ? 'arrowDown' : 'arrowRight'}
                         onClick={() => setIsShowingAdvanced(!isShowingAdvanced)}
                         flush="left"
+                        data-test-subj={`advancedStreamOptionsToggle-${packagePolicyInputStream.id}`}
                       >
                         <FormattedMessage
                           id="xpack.fleet.createPackagePolicy.stepConfigure.toggleAdvancedOptionsButtonText"
@@ -302,19 +321,20 @@ export const PackagePolicyInputStreamConfig = memo<Props>(
                         </EuiFlexItem>
                       );
                     })}
-                    {/* Only show datastream pipelines and mappings on edit and not for input packages*/}
                     {isPackagePolicyEdit && showPipelinesAndMappings && (
                       <>
                         <EuiFlexItem>
                           <PackagePolicyEditorDatastreamPipelines
                             packageInputStream={packagePolicyInputStream}
                             packageInfo={packageInfo}
+                            customDataset={customDatasetVarValue}
                           />
                         </EuiFlexItem>
                         <EuiFlexItem>
                           <PackagePolicyEditorDatastreamMappings
                             packageInputStream={packagePolicyInputStream}
                             packageInfo={packageInfo}
+                            customDataset={customDatasetVarValue}
                           />
                         </EuiFlexItem>
                       </>

x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_var_field.tsx

@@ -63,7 +63,7 @@ export const PackagePolicyInputVarField: React.FunctionComponent<{
     const isInvalid = (isDirty || forceShowErrors) && !!varErrors;
     const errors = isInvalid ? varErrors : null;
     const fieldLabel = title || name;
-
+    const fieldTestSelector = fieldLabel.replace(/\s/g, '-').toLowerCase();
     const field = useMemo(() => {
       if (multi) {
         return (
@@ -72,6 +72,7 @@ export const PackagePolicyInputVarField: React.FunctionComponent<{
             onChange={onChange}
             onBlur={() => setIsDirty(true)}
             isDisabled={frozen}
+            data-test-subj={`multiTextInput-${fieldTestSelector}`}
           />
         );
       }
@@ -96,6 +97,7 @@ export const PackagePolicyInputVarField: React.FunctionComponent<{
               onBlur={() => setIsDirty(true)}
               disabled={frozen}
               resize="vertical"
+              data-test-subj={`textAreaInput-${fieldTestSelector}`}
             />
           );
         case 'yaml':
@@ -142,6 +144,7 @@ export const PackagePolicyInputVarField: React.FunctionComponent<{
               onChange={(e) => onChange(e.target.checked)}
               onBlur={() => setIsDirty(true)}
               disabled={frozen}
+              data-test-subj={`switch-${fieldTestSelector}`}
             />
           );
         case 'password':
@@ -153,6 +156,7 @@ export const PackagePolicyInputVarField: React.FunctionComponent<{
               onChange={(e) => onChange(e.target.value)}
               onBlur={() => setIsDirty(true)}
               disabled={frozen}
+              data-test-subj={`passwordInput-${fieldTestSelector}`}
             />
           );
         case 'select':
@@ -177,6 +181,7 @@ export const PackagePolicyInputVarField: React.FunctionComponent<{
                 return onChange(newValue);
               }}
               onBlur={() => setIsDirty(true)}
+              data-test-subj={`select-${fieldTestSelector}`}
             />
           );
         default:
@@ -187,6 +192,7 @@ export const PackagePolicyInputVarField: React.FunctionComponent<{
               onChange={(e) => onChange(e.target.value)}
               onBlur={() => setIsDirty(true)}
               disabled={frozen}
+              data-test-subj={`textInput-${fieldTestSelector}`}
             />
           );
       }
@@ -204,6 +210,7 @@ export const PackagePolicyInputVarField: React.FunctionComponent<{
       isInvalid,
       fieldLabel,
       options,
+      fieldTestSelector,
     ]);
 
     // Boolean cannot be optional by default set to false