[9.0] [Fleet] Fix dataset validation to show errors on upgrade (#219741) (#219842)

# Backport

This will backport the following commits from `main` to `9.0`:
- [[Fleet] Fix dataset validation to show errors on upgrade
(#219741)](https://github.com/elastic/kibana/pull/219741)

<!--- Backport version: 9.6.6 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sorenlouv/backport)

<!--BACKPORT [{"author":{"name":"Nicolas
Chaulet","email":"nicolas.chaulet@elastic.co"},"sourceCommit":{"committedDate":"2025-05-01T11:36:24Z","message":"[Fleet]
Fix dataset validation to show errors on upgrade
(#219741)","sha":"8081bd35eb5e2d2d50e2291a1dbabd0e37307ab7","branchLabelMapping":{"^v9.1.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","Team:Fleet","v9.0.0","backport:prev-minor","backport:version","v9.1.0","v8.19.0","backport:8.18","v8.18.1","v8.18.2"],"title":"[Fleet]
Fix dataset validation to show errors on
upgrade","number":219741,"url":"https://github.com/elastic/kibana/pull/219741","mergeCommit":{"message":"[Fleet]
Fix dataset validation to show errors on upgrade
(#219741)","sha":"8081bd35eb5e2d2d50e2291a1dbabd0e37307ab7"}},"sourceBranch":"main","suggestedTargetBranches":["9.0","8.19","8.18"],"targetPullRequestStates":[{"branch":"9.0","label":"v9.0.0","branchLabelMappingKey":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"main","label":"v9.1.0","branchLabelMappingKey":"^v9.1.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/219741","number":219741,"mergeCommit":{"message":"[Fleet]
Fix dataset validation to show errors on upgrade
(#219741)","sha":"8081bd35eb5e2d2d50e2291a1dbabd0e37307ab7"}},{"branch":"8.19","label":"v8.19.0","branchLabelMappingKey":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"8.18","label":"v8.18.1","branchLabelMappingKey":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

oas_docs/bundle.json

@@ -37295,12 +37295,36 @@
                               "type": "object"
                             },
                             {
-                              "additionalProperties": false,
+                              "additionalProperties": true,
                               "properties": {
+                                "created_at": {
+                                  "type": "string"
+                                },
+                                "created_by": {
+                                  "type": "string"
+                                },
                                 "description": {
                                   "description": "Package policy description",
                                   "type": "string"
                                 },
+                                "elasticsearch": {
+                                  "additionalProperties": true,
+                                  "properties": {
+                                    "privileges": {
+                                      "additionalProperties": true,
+                                      "properties": {
+                                        "cluster": {
+                                          "items": {
+                                            "type": "string"
+                                          },
+                                          "type": "array"
+                                        }
+                                      },
+                                      "type": "object"
+                                    }
+                                  },
+                                  "type": "object"
+                                },
                                 "enabled": {
                                   "type": "boolean"
                                 },
@@ -37332,6 +37356,7 @@
                                   "items": {
                                     "additionalProperties": false,
                                     "properties": {
+                                      "compiled_input": {},
                                       "config": {
                                         "additionalProperties": {
                                           "additionalProperties": false,
@@ -37503,7 +37528,8 @@
                                     "required": [
                                       "type",
                                       "enabled",
-                                      "streams"
+                                      "streams",
+                                      "compiled_input"
                                     ],
                                     "type": "object"
                                   },
@@ -37613,12 +37639,36 @@
                                   },
                                   "type": "array"
                                 },
+                                "revision": {
+                                  "type": "number"
+                                },
+                                "secret_references": {
+                                  "items": {
+                                    "additionalProperties": false,
+                                    "properties": {
+                                      "id": {
+                                        "type": "string"
+                                      }
+                                    },
+                                    "required": [
+                                      "id"
+                                    ],
+                                    "type": "object"
+                                  },
+                                  "type": "array"
+                                },
                                 "supports_agentless": {
                                   "default": false,
                                   "description": "Indicates whether the package policy belongs to an agentless agent policy.",
                                   "nullable": true,
                                   "type": "boolean"
                                 },
+                                "updated_at": {
+                                  "type": "string"
+                                },
+                                "updated_by": {
+                                  "type": "string"
+                                },
                                 "vars": {
                                   "additionalProperties": {
                                     "additionalProperties": false,
@@ -37638,6 +37688,9 @@
                                   },
                                   "description": "Package variable (see integration documentation for more information)",
                                   "type": "object"
+                                },
+                                "version": {
+                                  "type": "string"
                                 }
                               },
                               "required": [

oas_docs/output/kibana.yaml

@@ -37660,12 +37660,28 @@ paths:
                               - updated_by
                               - created_at
                               - created_by
-                          - additionalProperties: false
+                          - additionalProperties: true
                             type: object
                             properties:
+                              created_at:
+                                type: string
+                              created_by:
+                                type: string
                               description:
                                 description: Package policy description
                                 type: string
+                              elasticsearch:
+                                additionalProperties: true
+                                type: object
+                                properties:
+                                  privileges:
+                                    additionalProperties: true
+                                    type: object
+                                    properties:
+                                      cluster:
+                                        items:
+                                          type: string
+                                        type: array
                               enabled:
                                 type: boolean
                               errors:
@@ -37689,6 +37705,7 @@ paths:
                                   additionalProperties: false
                                   type: object
                                   properties:
+                                    compiled_input: {}
                                     config:
                                       additionalProperties:
                                         additionalProperties: false
@@ -37809,6 +37826,7 @@ paths:
                                     - type
                                     - enabled
                                     - streams
+                                    - compiled_input
                                 type: array
                               is_managed:
                                 type: boolean
@@ -37884,11 +37902,27 @@ paths:
                                   description: Agent policy IDs where that package policy will be added
                                   type: string
                                 type: array
+                              revision:
+                                type: number
+                              secret_references:
+                                items:
+                                  additionalProperties: false
+                                  type: object
+                                  properties:
+                                    id:
+                                      type: string
+                                  required:
+                                    - id
+                                type: array
                               supports_agentless:
                                 default: false
                                 description: Indicates whether the package policy belongs to an agentless agent policy.
                                 nullable: true
                                 type: boolean
+                              updated_at:
+                                type: string
+                              updated_by:
+                                type: string
                               vars:
                                 additionalProperties:
                                   additionalProperties: false
@@ -37903,6 +37937,8 @@ paths:
                                     - value
                                 description: Package variable (see integration documentation for more information)
                                 type: object
+                              version:
+                                type: string
                             required:
                               - name
                               - enabled

x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/dataset_component.test.tsx

@@ -13,7 +13,7 @@ import { createFleetTestRendererMock } from '../../../../../../../../mock';
 import { DatasetComponent } from './dataset_component';
 
 describe('DatasetComponent', () => {
-  function render(value = 'generic', datastreams: any = []) {
+  function render(value = 'generic', datastreams: any = [], props?: any) {
     const renderer = createFleetTestRendererMock();
     const mockOnChange = jest.fn();
     const fieldLabel = 'Dataset name';
@@ -29,6 +29,7 @@ describe('DatasetComponent', () => {
         onChange={mockOnChange}
         isDisabled={false}
         fieldLabel={fieldLabel}
+        {...props}
       />
     );
 
@@ -36,22 +37,23 @@ describe('DatasetComponent', () => {
   }
 
   it('should show validation error if dataset is invalid', () => {
-    const { utils } = render();
+    const { utils } = render('generic', [], {
-
+      errors: ['Dataset contains invalid characters'],
-    const inputEl = utils.getByTestId('comboBoxSearchInput');
+      isInvalid: true,
-    fireEvent.change(inputEl, { target: { value: 'generic*' } });
+    });
-    fireEvent.keyDown(inputEl, { key: 'Enter', code: 'Enter' });
 
     utils.getByText('Dataset contains invalid characters');
   });
 
   it('should not show validation error if dataset is valid', () => {
-    const { utils } = render();
+    const { utils, mockOnChange } = render();
 
     const inputEl = utils.getByTestId('comboBoxSearchInput');
     fireEvent.change(inputEl, { target: { value: 'test' } });
     fireEvent.keyDown(inputEl, { key: 'Enter', code: 'Enter' });
 
+    expect(mockOnChange).toBeCalled();
+
     expect(utils.queryByText('Dataset contains invalid characters')).toBeNull();
   });
 

x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/dataset_component.tsx

@@ -23,7 +23,6 @@ import { FormattedMessage } from '@kbn/i18n-react';
 
 import type { DataStream } from '../../../../../../../../../common/types';
 import { GENERIC_DATASET_NAME } from '../../../../../../../../../common/constants';
-import { isValidDataset } from '../../../../../../../../../common';
 
 const FormRow = styled(EuiFormRow)`
   .euiFormRow__label {
@@ -48,7 +47,19 @@ export const DatasetComponent: React.FC<{
   isDisabled?: boolean;
   fieldLabel: string;
   description?: string;
-}> = ({ value, onChange, datastreams, isDisabled, pkgName = '', fieldLabel, description }) => {
+  errors?: string[] | null;
+  isInvalid?: boolean;
+}> = ({
+  value,
+  errors,
+  onChange,
+  datastreams,
+  isDisabled,
+  isInvalid,
+  pkgName = '',
+  fieldLabel,
+  description,
+}) => {
   const datasetOptions =
     datastreams.map((datastream: DataStream) => ({
       label: datastream.dataset,
@@ -67,8 +78,8 @@ export const DatasetComponent: React.FC<{
     };
 
   const [selectedOptions, setSelectedOptions] = useState<Array<{ label: string }>>([defaultOption]);
-  const [isInvalid, setIsInvalid] = useState<boolean>(false);
+
-  const [error, setError] = useState<string | undefined>(undefined);
+  const error = errors ? errors.join(', ') : undefined;
 
   useEffect(() => {
     if (!value || typeof value === 'string') onChange(defaultOption.value as SelectedDataset);
@@ -77,9 +88,7 @@ export const DatasetComponent: React.FC<{
   const onDatasetChange = (newSelectedOptions: Array<{ label: string; value?: DataStream }>) => {
     setSelectedOptions(newSelectedOptions);
     const dataStream = newSelectedOptions[0].value;
-    const { valid, error: dsError } = isValidDataset(newSelectedOptions[0].label, false);
+
-    setIsInvalid(!valid);
-    setError(dsError);
     onChange({
       dataset: newSelectedOptions[0].label,
       package: !dataStream || typeof dataStream === 'string' ? pkgName : dataStream.package,
@@ -96,9 +105,6 @@ export const DatasetComponent: React.FC<{
       value: { dataset: searchValue, package: pkgName },
     };
     setSelectedOptions([newOption]);
-    const { valid, error: dsError } = isValidDataset(searchValue, false);
-    setIsInvalid(!valid);
-    setError(dsError);
     onChange({
       dataset: searchValue,
       package: pkgName,

x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_var_field.tsx

@@ -107,6 +107,8 @@ export const PackagePolicyInputVarField: React.FunctionComponent<InputFieldProps
           datastreams={datastreams}
           value={value}
           onChange={onChange}
+          errors={errors}
+          isInvalid={isInvalid}
           isDisabled={isEditPage}
           fieldLabel={fieldLabel}
           description={description}

x-pack/platform/plugins/shared/fleet/server/types/models/package_policy.ts

@@ -402,20 +402,29 @@ export const OrphanedPackagePoliciesResponseSchema = schema.object({
   total: schema.number(),
 });
 
-export const DryRunPackagePolicySchema = schema.object({
+export const DryRunPackagePolicySchema = PackagePolicySchema.extends(
-  ...PackagePolicyBaseSchema,
+  {
-  id: schema.maybe(schema.string()),
+    id: schema.maybe(schema.string()),
-  force: schema.maybe(schema.boolean()),
+    force: schema.maybe(schema.boolean()),
-  errors: schema.maybe(
+    revision: schema.maybe(schema.number()),
-    schema.arrayOf(
+    updated_at: schema.maybe(schema.string()),
-      schema.object({
+    updated_by: schema.maybe(schema.string()),
-        message: schema.string(),
+    created_at: schema.maybe(schema.string()),
-        key: schema.maybe(schema.string()),
+    created_by: schema.maybe(schema.string()),
-      })
+    errors: schema.maybe(
-    )
+      schema.arrayOf(
-  ),
+        schema.object({
-  missingVars: schema.maybe(schema.arrayOf(schema.string())),
+          message: schema.string(),
-});
+          key: schema.maybe(schema.string()),
+        })
+      )
+    ),
+    missingVars: schema.maybe(schema.arrayOf(schema.string())),
+  },
+  {
+    unknowns: 'allow',
+  }
+);
 
 export const PackagePolicyStatusResponseSchema = schema.object({
   id: schema.string(),