[8.x][Index management] Project level retention support (#193715) (#197414)

# Backport This will backport the following commits from `main` to `8.x`: - https://github.com/elastic/kibana/pull/193715 Note: Created the backport manually because of merge conflicts in `config/serverless.security.yml` Co-authored-by: Ignacio Rivas <rivasign@gmail.com>
2025-04-24 01:38:56 -04:00 · 2024-10-23 15:14:22 +01:00 · 2024-10-23 15:14:22 +01:00 · 47f11ddeb2
commit 47f11ddeb2
parent 722dd5d4be
25 changed files with 422 additions and 57 deletions
--- a/config/serverless.security.yml
+++ b/config/serverless.security.yml
@ -119,6 +119,9 @@ xpack.ml.compatibleModuleType: 'security'
 # Disable the embedded Dev Console
 console.ui.embeddedEnabled: false

+# Enable project level rentention checks in DSL form from Index Management UI
+xpack.index_management.enableProjectLevelRetentionChecks: true
+
 # Experimental Security Solution features

 # This feature is disabled in Serverless until fully performance tested within a Serverless environment
--- a/config/serverless.yml
+++ b/config/serverless.yml
@ -110,6 +110,8 @@ xpack.index_management.editableIndexSettings: limited
 xpack.index_management.enableMappingsSourceFieldSection: false
 # Disable toggle for enabling data retention in DSL form from Index Management UI
 xpack.index_management.enableTogglingDataRetention: false
+# Disable project level rentention checks in DSL form from Index Management UI
+xpack.index_management.enableProjectLevelRetentionChecks: false

 # Disable Manage Processors UI in Ingest Pipelines
 xpack.ingest_pipelines.enableManageProcessors: false
--- a/test/plugin_functional/test_suites/core_plugins/rendering.ts
+++ b/test/plugin_functional/test_suites/core_plugins/rendering.ts
@ -273,6 +273,7 @@ export default function ({ getService }: PluginFunctionalProviderContext) {
        'xpack.index_management.ui.enabled (boolean?)',
        'xpack.infra.sources.default.fields.message (array?)',
        'xpack.index_management.enableTogglingDataRetention (boolean?|never)',
+        'xpack.index_management.enableProjectLevelRetentionChecks (boolean?|never)',
        'xpack.integration_assistant.enableExperimental (array?)',
        /**
         * Feature flags bellow are conditional based on traditional/serverless offering
--- a/x-pack/plugins/index_management/jest/client_integration/helpers/test_subjects.ts
+++ b/x-pack/plugins/index_management/jest/client_integration/helpers/test_subjects.ts
@ -100,6 +100,7 @@ export type TestSubjects =
  | 'configuredByILMWarning'
  | 'show-filters-button'
  | 'filter-option-h'
+  | 'filter-option-d'
  | 'infiniteRetentionPeriod.input'
  | 'saveButton'
  | 'dsIsFullyManagedByILM'
--- a/x-pack/plugins/index_management/jest/client_integration/home/data_streams_project_level_retention.test.ts
+++ b/x-pack/plugins/index_management/jest/client_integration/home/data_streams_project_level_retention.test.ts
@ -0,0 +1,150 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { act } from 'react-dom/test-utils';
+import { createMemoryHistory } from 'history';
+import { notificationServiceMock } from '@kbn/core/public/mocks';
+
+import { breadcrumbService } from '../../../public/application/services/breadcrumbs';
+import { MAX_DATA_RETENTION } from '../../../common/constants';
+import * as fixtures from '../../../test/fixtures';
+import { setupEnvironment } from '../helpers';
+import { notificationService } from '../../../public/application/services/notification';
+
+import {
+  DataStreamsTabTestBed,
+  setup,
+  createDataStreamPayload,
+  createDataStreamBackingIndex,
+  createNonDataStreamIndex,
+} from './data_streams_tab.helpers';
+
+const urlServiceMock = {
+  locators: {
+    get: () => ({
+      getLocation: async () => ({
+        app: '',
+        path: '',
+        state: {},
+      }),
+      getUrl: async ({ policyName }: { policyName: string }) => `/test/${policyName}`,
+      navigate: async () => {},
+      useUrl: () => '',
+    }),
+  },
+};
+
+describe('Data Streams - Project level max retention', () => {
+  const { httpSetup, httpRequestsMockHelpers } = setupEnvironment();
+  let testBed: DataStreamsTabTestBed;
+  jest.spyOn(breadcrumbService, 'setBreadcrumbs');
+
+  const notificationsServiceMock = notificationServiceMock.createStartContract();
+
+  beforeEach(async () => {
+    const {
+      setLoadIndicesResponse,
+      setLoadDataStreamsResponse,
+      setLoadDataStreamResponse,
+      setLoadTemplateResponse,
+      setLoadTemplatesResponse,
+    } = httpRequestsMockHelpers;
+
+    setLoadIndicesResponse([
+      createDataStreamBackingIndex('data-stream-index', 'dataStream1'),
+      createNonDataStreamIndex('non-data-stream-index'),
+    ]);
+
+    const dataStreamForDetailPanel = createDataStreamPayload({
+      name: 'dataStream1',
+      storageSize: '5b',
+      storageSizeBytes: 5,
+      // metering API mock
+      meteringStorageSize: '156kb',
+      meteringStorageSizeBytes: 156000,
+      meteringDocsCount: 10000,
+    });
+
+    setLoadDataStreamsResponse([
+      dataStreamForDetailPanel,
+      createDataStreamPayload({
+        name: 'dataStream2',
+        storageSize: '1kb',
+        storageSizeBytes: 1000,
+        // metering API mock
+        meteringStorageSize: '156kb',
+        meteringStorageSizeBytes: 156000,
+        meteringDocsCount: 10000,
+        lifecycle: {
+          enabled: true,
+          data_retention: '7d',
+          effective_retention: '5d',
+          globalMaxRetention: '20d',
+          retention_determined_by: MAX_DATA_RETENTION,
+        },
+      }),
+    ]);
+
+    setLoadDataStreamResponse(dataStreamForDetailPanel.name, dataStreamForDetailPanel);
+
+    const indexTemplate = fixtures.getTemplate({ name: 'indexTemplate' });
+    setLoadTemplatesResponse({ templates: [indexTemplate], legacyTemplates: [] });
+    setLoadTemplateResponse(indexTemplate.name, indexTemplate);
+
+    notificationService.setup(notificationsServiceMock);
+    testBed = await setup(httpSetup, {
+      history: createMemoryHistory(),
+      services: {
+        notificationService,
+      },
+      config: {
+        enableProjectLevelRetentionChecks: true,
+      },
+    });
+    await act(async () => {
+      testBed.actions.goToDataStreamsList();
+    });
+    testBed.component.update();
+  });
+
+  it('Should show error when retention value is bigger than project level retention', async () => {
+    const { setLoadDataStreamsResponse, setLoadDataStreamResponse } = httpRequestsMockHelpers;
+
+    const ds1 = createDataStreamPayload({
+      name: 'dataStream1',
+      lifecycle: {
+        enabled: true,
+        data_retention: '25d',
+        effective_retention: '25d',
+        retention_determined_by: MAX_DATA_RETENTION,
+        globalMaxRetention: '20d',
+      },
+    });
+
+    setLoadDataStreamsResponse([ds1]);
+    setLoadDataStreamResponse(ds1.name, ds1);
+
+    testBed = await setup(httpSetup, {
+      history: createMemoryHistory(),
+      url: urlServiceMock,
+      config: {
+        enableProjectLevelRetentionChecks: true,
+      },
+    });
+    await act(async () => {
+      testBed.actions.goToDataStreamsList();
+    });
+    testBed.component.update();
+
+    const { actions } = testBed;
+
+    await actions.clickNameAt(0);
+    actions.clickEditDataRetentionButton();
+
+    expect(testBed.form.getErrorsMessages().length).toBeGreaterThan(0);
+  });
+});
--- a/x-pack/plugins/index_management/common/types/data_streams.ts
+++ b/x-pack/plugins/index_management/common/types/data_streams.ts
@ -34,6 +34,7 @@ export type DataStreamIndexFromEs = IndicesDataStreamIndex;
 export type Health = 'green' | 'yellow' | 'red';

 export interface EnhancedDataStreamFromEs extends IndicesDataStream {
+  global_max_retention?: string;
  store_size?: IndicesDataStreamsStatsDataStreamsStatsItem['store_size'];
  store_size_bytes?: IndicesDataStreamsStatsDataStreamsStatsItem['store_size_bytes'];
  maximum_timestamp?: IndicesDataStreamsStatsDataStreamsStatsItem['maximum_timestamp'];
@ -68,6 +69,7 @@ export interface DataStream {
    enabled?: boolean;
    effective_retention?: string;
    retention_determined_by?: string;
+    globalMaxRetention?: string;
  };
 }

--- a/x-pack/plugins/index_management/public/application/app_context.tsx
+++ b/x-pack/plugins/index_management/public/application/app_context.tsx
@ -68,6 +68,7 @@ export interface AppDependencies {
    editableIndexSettings: 'all' | 'limited';
    enableMappingsSourceFieldSection: boolean;
    enableTogglingDataRetention: boolean;
+    enableProjectLevelRetentionChecks: boolean;
    enableSemanticText: boolean;
  };
  history: ScopedHistory;
--- a/x-pack/plugins/index_management/public/application/hooks/use_state_with_localstorage.ts
+++ b/x-pack/plugins/index_management/public/application/hooks/use_state_with_localstorage.ts
@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { Dispatch, SetStateAction, useEffect, useState } from 'react';
+
+function parseJsonOrDefault<Obj>(value: string | null, defaultValue: Obj): Obj {
+  if (!value) {
+    return defaultValue;
+  }
+  try {
+    return JSON.parse(value) as Obj;
+  } catch (e) {
+    return defaultValue;
+  }
+}
+
+export function useStateWithLocalStorage<State>(
+  key: string,
+  defaultState: State
+): [State, Dispatch<SetStateAction<State>>] {
+  const storageState = localStorage.getItem(key);
+  const [state, setState] = useState<State>(parseJsonOrDefault<State>(storageState, defaultState));
+  useEffect(() => {
+    localStorage.setItem(key, JSON.stringify(state));
+  }, [key, state]);
+  return [state, setState];
+}
--- a/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_list.tsx
+++ b/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_list.tsx
@ -12,12 +12,12 @@ import { i18n } from '@kbn/i18n';
 import {
  EuiFlexGroup,
  EuiFlexItem,
-  EuiSwitch,
  EuiText,
-  EuiIconTip,
  EuiSpacer,
  EuiPageSection,
  EuiEmptyPrompt,
+  EuiCallOut,
+  EuiButton,
  EuiLink,
 } from '@elastic/eui';
 import { ScopedHistory } from '@kbn/core/public';
@ -40,8 +40,10 @@ import { documentationService } from '../../../services/documentation';
 import { DataStreamTable } from './data_stream_table';
 import { DataStreamDetailPanel } from './data_stream_detail_panel';
 import { filterDataStreams, isSelectedDataStreamHidden } from '../../../lib/data_streams';
-import { FilterListButton, Filters } from '../components';
+import { Filters } from '../components';
+import { useStateWithLocalStorage } from '../../../hooks/use_state_with_localstorage';

+const SHOW_PROJECT_LEVEL_RETENTION = 'showProjectLevelRetention';
 export type DataStreamFilterName = 'managed' | 'hidden';
 interface MatchParams {
  dataStreamName?: string;
@ -58,8 +60,9 @@ export const DataStreamList: React.FunctionComponent<RouteComponentProps<MatchPa
  const decodedDataStreamName = attemptToURIDecode(dataStreamName);

  const {
+    config: { enableProjectLevelRetentionChecks },
    core: { getUrlForApp, executionContext },
-    plugins: { isFleetEnabled },
+    plugins: { isFleetEnabled, cloud },
  } = useAppContext();

  useExecutionContext(executionContext, {
@ -81,6 +84,9 @@ export const DataStreamList: React.FunctionComponent<RouteComponentProps<MatchPa
    includeStats: isIncludeStatsChecked,
  });

+  const [projectLevelRetentionCallout, setprojectLevelRetentionCallout] =
+    useStateWithLocalStorage<boolean>(SHOW_PROJECT_LEVEL_RETENTION, true);
+
  const [filters, setFilters] = useState<Filters<DataStreamFilterName>>({
    managed: {
      name: i18n.translate('xpack.idxMgmt.dataStreamList.viewManagedLabel', {
@ -125,7 +131,7 @@ export const DataStreamList: React.FunctionComponent<RouteComponentProps<MatchPa
    return (
      <EuiFlexGroup alignItems="center" justifyContent="spaceBetween">
        <EuiFlexItem>
-          <EuiText color="subdued">
+          <EuiText color="subdued" css={{ maxWidth: '80%' }}>
            <FormattedMessage
              id="xpack.idxMgmt.dataStreamList.dataStreamsDescription"
              defaultMessage="Data streams store time-series data across multiple indices and can be created from index templates. {learnMoreLink}"
@ -146,38 +152,16 @@ export const DataStreamList: React.FunctionComponent<RouteComponentProps<MatchPa
          </EuiText>
        </EuiFlexItem>

-        <EuiFlexItem grow={false}>
-          <EuiFlexGroup gutterSize="s">
-            <EuiFlexItem grow={false}>
-              <EuiSwitch
-                label={i18n.translate(
-                  'xpack.idxMgmt.dataStreamListControls.includeStatsSwitchLabel',
-                  {
-                    defaultMessage: 'Include stats',
-                  }
-                )}
-                checked={isIncludeStatsChecked}
-                onChange={(e) => setIsIncludeStatsChecked(e.target.checked)}
-                data-test-subj="includeStatsSwitch"
+        {enableProjectLevelRetentionChecks && (
+          <EuiFlexItem grow={false}>
+            <EuiLink href={cloud?.deploymentUrl} target="_blank">
+              <FormattedMessage
+                id="xpack.idxMgmt.dataStreamList.projectlevelRetention.linkText"
+                defaultMessage="Project data retention"
              />
-            </EuiFlexItem>
-
-            <EuiFlexItem grow={false}>
-              <EuiIconTip
-                content={i18n.translate(
-                  'xpack.idxMgmt.dataStreamListControls.includeStatsSwitchToolTip',
-                  {
-                    defaultMessage: 'Including stats can increase reload times',
-                  }
-                )}
-                position="top"
-              />
-            </EuiFlexItem>
-          </EuiFlexGroup>
-        </EuiFlexItem>
-        <EuiFlexItem grow={false}>
-          <FilterListButton<DataStreamFilterName> filters={filters} onChange={setFilters} />
-        </EuiFlexItem>
+            </EuiLink>
+          </EuiFlexItem>
+        )}
      </EuiFlexGroup>
    );
  };
@ -276,6 +260,37 @@ export const DataStreamList: React.FunctionComponent<RouteComponentProps<MatchPa
    activateHiddenFilter(isSelectedDataStreamHidden(dataStreams!, decodedDataStreamName));
    content = (
      <EuiPageSection paddingSize="none" data-test-subj="dataStreamList">
+        {enableProjectLevelRetentionChecks && projectLevelRetentionCallout && (
+          <>
+            <EuiCallOut
+              onDismiss={() => setprojectLevelRetentionCallout(false)}
+              data-test-subj="projectLevelRetentionCallout"
+              title={i18n.translate(
+                'xpack.idxMgmt.dataStreamList.projectLevelRetentionCallout.titleText',
+                {
+                  defaultMessage:
+                    'You can now configure data stream retention settings for your entire project',
+                }
+              )}
+            >
+              <p>
+                <FormattedMessage
+                  id="xpack.idxMgmt.dataStreamList.projectLevelRetentionCallout.descriptionText"
+                  defaultMessage="Optionally define a maximum and default retention period to manage your compliance and storage size needs."
+                />
+              </p>
+
+              <EuiButton href={cloud?.deploymentUrl} fill data-test-subj="cloudLinkButton">
+                <FormattedMessage
+                  id="xpack.idxMgmt.dataStreamList.projectLevelRetentionCallout.buttonText"
+                  defaultMessage="Get started"
+                />
+              </EuiButton>
+            </EuiCallOut>
+            <EuiSpacer size="m" />
+          </>
+        )}
+
        {renderHeader()}
        <EuiSpacer size="l" />

@ -287,8 +302,11 @@ export const DataStreamList: React.FunctionComponent<RouteComponentProps<MatchPa
          }
          dataStreams={filteredDataStreams}
          reload={reload}
+          viewFilters={filters}
+          onViewFilterChange={setFilters}
          history={history as ScopedHistory}
          includeStats={isIncludeStatsChecked}
+          setIncludeStats={setIsIncludeStatsChecked}
        />
      </EuiPageSection>
    );
--- a/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_table/data_stream_table.tsx
+++ b/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_table/data_stream_table.tsx
@ -16,6 +16,10 @@ import {
  EuiIcon,
  EuiToolTip,
  EuiTextColor,
+  EuiFlexGroup,
+  EuiFlexItem,
+  EuiSwitch,
+  EuiIconTip,
 } from '@elastic/eui';
 import { ScopedHistory } from '@kbn/core/public';
 import { useEuiTablePersist } from '@kbn/shared-ux-table-persist';
@ -32,6 +36,8 @@ import { humanizeTimeStamp } from '../humanize_time_stamp';
 import { DataStreamsBadges } from '../data_stream_badges';
 import { ConditionalWrap } from '../data_stream_detail_panel';
 import { isDataStreamFullyManagedByILM } from '../../../../lib/data_streams';
+import { FilterListButton, Filters } from '../../components';
+import { type DataStreamFilterName } from '../data_stream_list';

 interface TableDataStream extends DataStream {
  isDataStreamFullyManagedByILM: boolean;
@ -42,7 +48,10 @@ interface Props {
  reload: UseRequestResponse['resendRequest'];
  history: ScopedHistory;
  includeStats: boolean;
-  filters?: string;
+  filters: string;
+  viewFilters: Filters<DataStreamFilterName>;
+  onViewFilterChange: (newFilter: Filters<DataStreamFilterName>) => void;
+  setIncludeStats: (includeStats: boolean) => void;
 }

 const INFINITE_AS_ICON = true;
@ -54,6 +63,9 @@ export const DataStreamTable: React.FunctionComponent<Props> = ({
  history,
  filters,
  includeStats,
+  setIncludeStats,
+  onViewFilterChange,
+  viewFilters,
 }) => {
  const [selection, setSelection] = useState<DataStream[]>([]);
  const [dataStreamsToDelete, setDataStreamsToDelete] = useState<string[]>([]);
@ -282,6 +294,34 @@ export const DataStreamTable: React.FunctionComponent<Props> = ({
        </EuiButton>
      ) : undefined,
    toolsRight: [
+      <EuiFlexGroup gutterSize="s">
+        <EuiFlexItem grow={false}>
+          <EuiSwitch
+            label={i18n.translate('xpack.idxMgmt.dataStreamListControls.includeStatsSwitchLabel', {
+              defaultMessage: 'Include stats',
+            })}
+            checked={includeStats}
+            onChange={(e) => setIncludeStats(e.target.checked)}
+            data-test-subj="includeStatsSwitch"
+          />
+        </EuiFlexItem>
+
+        <EuiFlexItem grow={false}>
+          <EuiIconTip
+            content={i18n.translate(
+              'xpack.idxMgmt.dataStreamListControls.includeStatsSwitchToolTip',
+              {
+                defaultMessage: 'Including stats can increase reload times',
+              }
+            )}
+            position="top"
+          />
+        </EuiFlexItem>
+      </EuiFlexGroup>,
+      <FilterListButton<DataStreamFilterName>
+        filters={viewFilters}
+        onChange={onViewFilterChange}
+      />,
      <EuiButton
        color="success"
        iconType="refresh"
--- a/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/edit_data_retention_modal/edit_data_retention_modal.tsx
+++ b/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/edit_data_retention_modal/edit_data_retention_modal.tsx
@ -5,7 +5,7 @@
 * 2.0.
 */

-import React from 'react';
+import React, { useEffect } from 'react';
 import {
  EuiModal,
  EuiModalBody,
@ -53,24 +53,72 @@ interface Props {
  onClose: (data?: { hasUpdatedDataRetention: boolean }) => void;
 }

+const convertToMinutes = (value: string) => {
+  const { size, unit } = splitSizeAndUnits(value);
+  const sizeNum = parseInt(size, 10);
+
+  switch (unit) {
+    case 'd':
+      // days to minutes
+      return sizeNum * 24 * 60;
+    case 'h':
+      // hours to minutes
+      return sizeNum * 60;
+    case 'm':
+      // minutes to minutes
+      return sizeNum;
+    case 's':
+      // seconds to minutes
+      return sizeNum / 60;
+    default:
+      throw new Error(`Unknown unit: ${unit}`);
+  }
+};
+
+const isRetentionBiggerThan = (valueA: string, valueB: string) => {
+  const minutesA = convertToMinutes(valueA);
+  const minutesB = convertToMinutes(valueB);
+
+  return minutesA > minutesB;
+};
+
 const configurationFormSchema: FormSchema = {
  dataRetention: {
    type: FIELD_TYPES.TEXT,
    label: i18n.translate(
      'xpack.idxMgmt.dataStreamsDetailsPanel.editDataRetentionModal.dataRetentionField',
      {
-        defaultMessage: 'Data retention',
+        defaultMessage: 'Data retention period',
      }
    ),
    formatters: [fieldFormatters.toInt],
    validations: [
      {
-        validator: ({ value, formData }) => {
+        validator: ({ value, formData, customData }) => {
          // If infiniteRetentionPeriod is set, we dont need to validate the data retention field
          if (formData.infiniteRetentionPeriod) {
            return undefined;
          }

+          // If project level data retention is enabled, we need to enforce the global max retention
+          const { globalMaxRetention, enableProjectLevelRetentionChecks } = customData.value as any;
+          if (enableProjectLevelRetentionChecks) {
+            const currentValue = `${value}${formData.timeUnit}`;
+            if (globalMaxRetention && isRetentionBiggerThan(currentValue, globalMaxRetention)) {
+              return {
+                message: i18n.translate(
+                  'xpack.idxMgmt.dataStreamsDetailsPanel.editDataRetentionModal.dataRetentionFieldMaxError',
+                  {
+                    defaultMessage:
+                      'Maximum data retention period on this project is {maxRetention} days.',
+                    // Remove the unit from the globalMaxRetention value
+                    values: { maxRetention: globalMaxRetention.slice(0, -1) },
+                  }
+                ),
+              };
+            }
+          }
+
          if (!value) {
            return {
              message: i18n.translate(
@ -107,12 +155,6 @@ const configurationFormSchema: FormSchema = {
  infiniteRetentionPeriod: {
    type: FIELD_TYPES.TOGGLE,
    defaultValue: false,
-    label: i18n.translate(
-      'xpack.idxMgmt.dataStreamsDetailsPanel.editDataRetentionModal.infiniteRetentionPeriodField',
-      {
-        defaultMessage: 'Keep data indefinitely',
-      }
-    ),
  },
  dataRetentionEnabled: {
    type: FIELD_TYPES.TOGGLE,
@ -194,7 +236,7 @@ export const EditDataRetentionModal: React.FunctionComponent<Props> = ({
  const { size, unit } = splitSizeAndUnits(lifecycle?.data_retention as string);
  const {
    services: { notificationService },
-    config: { enableTogglingDataRetention },
+    config: { enableTogglingDataRetention, enableProjectLevelRetentionChecks },
  } = useAppContext();

  const { form } = useForm({
@ -213,6 +255,15 @@ export const EditDataRetentionModal: React.FunctionComponent<Props> = ({
  const [formData] = useFormData({ form });
  const isDirty = useFormIsModified({ form });

+  const formHasErrors = form.getErrors().length > 0;
+  const disableSubmit = formHasErrors || !isDirty || form.isValid === false;
+
+  // Whenever the formData changes, we need to re-validate the dataRetention field
+  // as it depends on the timeUnit field.
+  useEffect(() => {
+    form.validateFields(['dataRetention']);
+  }, [formData, form]);
+
  const onSubmitForm = async () => {
    const { isValid, data } = await form.submit();

@ -268,7 +319,11 @@ export const EditDataRetentionModal: React.FunctionComponent<Props> = ({
  };

  return (
-    <EuiModal onClose={() => onClose()} data-test-subj="editDataRetentionModal">
+    <EuiModal
+      onClose={() => onClose()}
+      data-test-subj="editDataRetentionModal"
+      css={{ minWidth: 450 }}
+    >
      <Form form={form} data-test-subj="editDataRetentionForm">
        <EuiModalHeader>
          <EuiModalHeaderTitle>
@ -292,6 +347,17 @@ export const EditDataRetentionModal: React.FunctionComponent<Props> = ({
            </>
          )}

+          {enableProjectLevelRetentionChecks && lifecycle?.globalMaxRetention && (
+            <>
+              <FormattedMessage
+                id="xpack.idxMgmt.dataStreamsDetailsPanel.editDataRetentionModal.modalTitleText"
+                defaultMessage="Maximum data retention period is {maxRetention} days"
+                values={{ maxRetention: lifecycle?.globalMaxRetention.slice(0, -1) }}
+              />
+              <EuiSpacer />
+            </>
+          )}
+
          {enableTogglingDataRetention && (
            <UseField
              path="dataRetentionEnabled"
@ -303,13 +369,17 @@ export const EditDataRetentionModal: React.FunctionComponent<Props> = ({
          <UseField
            path="dataRetention"
            component={NumericField}
+            validationData={{
+              globalMaxRetention: lifecycle?.globalMaxRetention,
+              enableProjectLevelRetentionChecks,
+            }}
            labelAppend={
              <EuiText size="xs">
                <EuiLink href={documentationService.getUpdateExistingDS()} target="_blank" external>
                  {i18n.translate(
                    'xpack.idxMgmt.dataStreamsDetailsPanel.editDataRetentionModal.learnMoreLinkText',
                    {
-                      defaultMessage: 'How does it work?',
+                      defaultMessage: 'How does this work?',
                    }
                  )}
                </EuiLink>
@ -350,6 +420,14 @@ export const EditDataRetentionModal: React.FunctionComponent<Props> = ({
            path="infiniteRetentionPeriod"
            component={ToggleField}
            data-test-subj="infiniteRetentionPeriod"
+            label={i18n.translate(
+              'xpack.idxMgmt.dataStreamsDetailsPanel.editDataRetentionModal.infiniteRetentionPeriodField',
+              {
+                defaultMessage:
+                  'Keep data {withProjectLevelRetention, plural, one {up to maximum retention period} other {indefinitely}}',
+                values: { withProjectLevelRetention: enableProjectLevelRetentionChecks ? 1 : 0 },
+              }
+            )}
            componentProps={{
              euiFieldProps: {
                disabled: !formData.dataRetentionEnabled && enableTogglingDataRetention,
@ -372,7 +450,7 @@ export const EditDataRetentionModal: React.FunctionComponent<Props> = ({
            fill
            type="submit"
            isLoading={false}
-            disabled={(form.isSubmitted && form.isValid === false) || !isDirty}
+            disabled={disableSubmit}
            data-test-subj="saveButton"
            onClick={onSubmitForm}
          >
--- a/x-pack/plugins/index_management/public/plugin.ts
+++ b/x-pack/plugins/index_management/public/plugin.ts
@ -54,6 +54,7 @@ export class IndexMgmtUIPlugin
    isIndexManagementUiEnabled: boolean;
    enableMappingsSourceFieldSection: boolean;
    enableTogglingDataRetention: boolean;
+    enableProjectLevelRetentionChecks: boolean;
    enableSemanticText: boolean;
  };

@ -72,6 +73,7 @@ export class IndexMgmtUIPlugin
      editableIndexSettings,
      enableMappingsSourceFieldSection,
      enableTogglingDataRetention,
+      enableProjectLevelRetentionChecks,
      dev: { enableSemanticText },
    } = ctx.config.get<ClientConfigType>();
    this.config = {
@ -84,6 +86,7 @@ export class IndexMgmtUIPlugin
      editableIndexSettings: editableIndexSettings ?? 'all',
      enableMappingsSourceFieldSection: enableMappingsSourceFieldSection ?? true,
      enableTogglingDataRetention: enableTogglingDataRetention ?? true,
+      enableProjectLevelRetentionChecks: enableProjectLevelRetentionChecks ?? false,
      enableSemanticText: enableSemanticText ?? true,
    };
  }
--- a/x-pack/plugins/index_management/public/types.ts
+++ b/x-pack/plugins/index_management/public/types.ts
@ -57,6 +57,7 @@ export interface ClientConfigType {
  editableIndexSettings?: 'all' | 'limited';
  enableMappingsSourceFieldSection?: boolean;
  enableTogglingDataRetention?: boolean;
+  enableProjectLevelRetentionChecks?: boolean;
  dev: {
    enableSemanticText?: boolean;
  };
--- a/x-pack/plugins/index_management/server/config.ts
+++ b/x-pack/plugins/index_management/server/config.ts
@ -69,6 +69,11 @@ const schemaLatest = schema.object(
      // We take this approach in order to have a central place (serverless.yml) for serverless config across Kibana
      serverless: schema.boolean({ defaultValue: true }),
    }),
+    enableProjectLevelRetentionChecks: offeringBasedSchema({
+      // Max project level retention checks is enabled in serverless; refer to the serverless.yml file as the source of truth
+      // We take this approach in order to have a central place (serverless.yml) for serverless config across Kibana
+      serverless: schema.boolean({ defaultValue: true }),
+    }),
  },
  { defaultValue: undefined }
 );
@ -87,6 +92,7 @@ const configLatest: PluginConfigDescriptor<IndexManagementConfig> = {
    editableIndexSettings: true,
    enableMappingsSourceFieldSection: true,
    enableTogglingDataRetention: true,
+    enableProjectLevelRetentionChecks: true,
  },
  schema: schemaLatest,
  deprecations: ({ unused }) => [unused('dev.enableIndexDetailsPage', { level: 'warning' })],
--- a/x-pack/plugins/index_management/server/lib/data_stream_serialization.ts
+++ b/x-pack/plugins/index_management/server/lib/data_stream_serialization.ts
@ -26,6 +26,7 @@ export function deserializeDataStream(dataStreamFromEs: EnhancedDataStreamFromEs
    privileges,
    hidden,
    lifecycle,
+    global_max_retention: globalMaxRetention,
    next_generation_managed_by: nextGenerationManagedBy,
  } = dataStreamFromEs;
  const meteringStorageSize =
@ -67,7 +68,10 @@ export function deserializeDataStream(dataStreamFromEs: EnhancedDataStreamFromEs
    _meta,
    privileges,
    hidden,
-    lifecycle,
+    lifecycle: {
+      ...lifecycle,
+      globalMaxRetention,
+    },
    nextGenerationManagedBy,
  };
 }
--- a/x-pack/plugins/index_management/server/plugin.ts
+++ b/x-pack/plugins/index_management/server/plugin.ts
@ -57,6 +57,7 @@ export class IndexMgmtServerPlugin implements Plugin<IndexManagementPluginSetup,
        isLegacyTemplatesEnabled: this.config.enableLegacyTemplates,
        isIndexStatsEnabled: this.config.enableIndexStats ?? true,
        isSizeAndDocCountEnabled: this.config.enableSizeAndDocCount ?? false,
+        enableProjectLevelRetentionChecks: this.config.enableProjectLevelRetentionChecks ?? false,
        isDataStreamStatsEnabled: this.config.enableDataStreamStats,
        enableMappingsSourceFieldSection: this.config.enableMappingsSourceFieldSection,
        enableTogglingDataRetention: this.config.enableTogglingDataRetention,
--- a/x-pack/plugins/index_management/server/routes/api/component_templates/register_privileges_route.test.ts
+++ b/x-pack/plugins/index_management/server/routes/api/component_templates/register_privileges_route.test.ts
@ -52,6 +52,7 @@ describe('GET privileges', () => {
        isDataStreamStatsEnabled: true,
        enableMappingsSourceFieldSection: true,
        enableTogglingDataRetention: true,
+        enableProjectLevelRetentionChecks: false,
      },
      indexDataEnricher: mockedIndexDataEnricher,
      lib: {
@ -124,6 +125,7 @@ describe('GET privileges', () => {
          isDataStreamStatsEnabled: true,
          enableMappingsSourceFieldSection: true,
          enableTogglingDataRetention: true,
+          enableProjectLevelRetentionChecks: false,
        },
        indexDataEnricher: mockedIndexDataEnricher,
        lib: {
--- a/x-pack/plugins/index_management/server/routes/api/data_streams/register_get_route.ts
+++ b/x-pack/plugins/index_management/server/routes/api/data_streams/register_get_route.ts
@ -30,15 +30,18 @@ const enhanceDataStreams = ({
  dataStreamsStats,
  meteringStats,
  dataStreamsPrivileges,
+  globalMaxRetention,
 }: {
  dataStreams: IndicesDataStream[];
  dataStreamsStats?: IndicesDataStreamsStatsDataStreamsStatsItem[];
  meteringStats?: MeteringStats[];
  dataStreamsPrivileges?: SecurityHasPrivilegesResponse;
+  globalMaxRetention?: string;
 }): EnhancedDataStreamFromEs[] => {
  return dataStreams.map((dataStream) => {
    const enhancedDataStream: EnhancedDataStreamFromEs = {
      ...dataStream,
+      ...(globalMaxRetention ? { global_max_retention: globalMaxRetention } : {}),
      privileges: {
        delete_index: dataStreamsPrivileges
          ? dataStreamsPrivileges.index[dataStream.name].delete_index
@ -79,6 +82,12 @@ const getDataStreams = (client: IScopedClusterClient, name = '*') => {
  });
 };

+const getDataStreamLifecycle = (client: IScopedClusterClient, name: string) => {
+  return client.asCurrentUser.indices.getDataLifecycle({
+    name,
+  });
+};
+
 const getDataStreamsStats = (client: IScopedClusterClient, name = '*') => {
  return client.asCurrentUser.indices.dataStreamsStats({
    name,
@ -176,6 +185,10 @@ export function registerGetOneRoute({ router, lib: { handleEsError }, config }:
      try {
        const { data_streams: dataStreams } = await getDataStreams(client, name);

+        const lifecycle = await getDataStreamLifecycle(client, name);
+        // @ts-ignore - TS doesn't know about the `global_retention` property yet
+        const globalMaxRetention = lifecycle?.global_retention?.max_retention;
+
        if (config.isDataStreamStatsEnabled !== false) {
          ({ data_streams: dataStreamsStats } = await getDataStreamsStats(client, name));
        }
@ -196,6 +209,7 @@ export function registerGetOneRoute({ router, lib: { handleEsError }, config }:
            dataStreamsStats,
            meteringStats,
            dataStreamsPrivileges,
+            globalMaxRetention,
          });
          const body = deserializeDataStream(enhancedDataStreams[0]);
          return response.ok({ body });
--- a/x-pack/plugins/index_management/server/routes/api/enrich_policies/register_privileges_route.test.ts
+++ b/x-pack/plugins/index_management/server/routes/api/enrich_policies/register_privileges_route.test.ts
@ -52,6 +52,7 @@ describe('GET privileges', () => {
        isDataStreamStatsEnabled: true,
        enableMappingsSourceFieldSection: true,
        enableTogglingDataRetention: true,
+        enableProjectLevelRetentionChecks: false,
      },
      indexDataEnricher: mockedIndexDataEnricher,
      lib: {
@ -124,6 +125,7 @@ describe('GET privileges', () => {
          isDataStreamStatsEnabled: true,
          enableMappingsSourceFieldSection: true,
          enableTogglingDataRetention: true,
+          enableProjectLevelRetentionChecks: false,
        },
        indexDataEnricher: mockedIndexDataEnricher,
        lib: {
--- a/x-pack/plugins/index_management/server/test/helpers/route_dependencies.ts
+++ b/x-pack/plugins/index_management/server/test/helpers/route_dependencies.ts
@ -18,6 +18,7 @@ export const routeDependencies: Omit<RouteDependencies, 'router'> = {
    isDataStreamStatsEnabled: true,
    enableMappingsSourceFieldSection: true,
    enableTogglingDataRetention: true,
+    enableProjectLevelRetentionChecks: false,
  },
  indexDataEnricher: new IndexDataEnricher(),
  lib: {
--- a/x-pack/plugins/index_management/server/types.ts
+++ b/x-pack/plugins/index_management/server/types.ts
@ -29,6 +29,7 @@ export interface RouteDependencies {
    isDataStreamStatsEnabled: boolean;
    enableMappingsSourceFieldSection: boolean;
    enableTogglingDataRetention: boolean;
+    enableProjectLevelRetentionChecks: boolean;
  };
  indexDataEnricher: IndexDataEnricher;
  lib: {
--- a/x-pack/plugins/translations/translations/fr-FR.json
+++ b/x-pack/plugins/translations/translations/fr-FR.json
@ -21277,9 +21277,7 @@
    "xpack.idxMgmt.dataStreamsDetailsPanel.editDataRetentionModal.fullyManagedByILMButtonLabel": "Stratégie ILM",
    "xpack.idxMgmt.dataStreamsDetailsPanel.editDataRetentionModal.fullyManagedByILMDescription": "Afin de modifier la conservation des données pour ce flux de données, vous devez modifier le {link} associé.",
    "xpack.idxMgmt.dataStreamsDetailsPanel.editDataRetentionModal.fullyManagedByILMTitle": "Ce flux de données et les index associés sont gérés par la stratégie ILM",
-    "xpack.idxMgmt.dataStreamsDetailsPanel.editDataRetentionModal.infiniteRetentionPeriodField": "Conserver indéfiniment les données",
    "xpack.idxMgmt.dataStreamsDetailsPanel.editDataRetentionModal.learnMoreLinkText": "Comment ça fonctionne ?",
-    "xpack.idxMgmt.dataStreamsDetailsPanel.editDataRetentionModal.modalTitleText": "Modifier la conservation des données",
    "xpack.idxMgmt.dataStreamsDetailsPanel.editDataRetentionModal.saveButtonLabel": "Enregistrer",
    "xpack.idxMgmt.dataStreamsDetailsPanel.editDataRetentionModal.someManagedByILMBody": "Un index ou plus sont gérés par une politique ILM ({viewAllIndicesLink}). La mise à niveau de la conservation des données pour ce flux de données n'aura pas d'incidence sur ces index. À la place, vous devrez mettre à niveau la politique {ilmPolicyLink}.",
    "xpack.idxMgmt.dataStreamsDetailsPanel.editDataRetentionModal.someManagedByILMTitle": "Certains index sont gérés par la stratégie ILM",
--- a/x-pack/plugins/translations/translations/ja-JP.json
+++ b/x-pack/plugins/translations/translations/ja-JP.json
@ -21026,9 +21026,7 @@
    "xpack.idxMgmt.dataStreamsDetailsPanel.editDataRetentionModal.fullyManagedByILMButtonLabel": "ILMポリシー",
    "xpack.idxMgmt.dataStreamsDetailsPanel.editDataRetentionModal.fullyManagedByILMDescription": "このデータストリームのデータ保持を編集するには、関連する{link}を編集する必要があります。",
    "xpack.idxMgmt.dataStreamsDetailsPanel.editDataRetentionModal.fullyManagedByILMTitle": "このデータストリームと関連するインデックスはILMによって管理されます。",
-    "xpack.idxMgmt.dataStreamsDetailsPanel.editDataRetentionModal.infiniteRetentionPeriodField": "データを無期限に保持",
    "xpack.idxMgmt.dataStreamsDetailsPanel.editDataRetentionModal.learnMoreLinkText": "仕組み",
-    "xpack.idxMgmt.dataStreamsDetailsPanel.editDataRetentionModal.modalTitleText": "データ保持を編集",
    "xpack.idxMgmt.dataStreamsDetailsPanel.editDataRetentionModal.saveButtonLabel": "保存",
    "xpack.idxMgmt.dataStreamsDetailsPanel.editDataRetentionModal.someManagedByILMBody": "ILMポリシー（{viewAllIndicesLink}）によって1つ以上のインデックスが管理されます。このデータストリームのデータ保持を更新しても、これらのインデックスには影響しません。代わりに、{ilmPolicyLink}ポリシーを更新する必要があります。",
    "xpack.idxMgmt.dataStreamsDetailsPanel.editDataRetentionModal.someManagedByILMTitle": "一部のインデックスはILMによって管理されます。",
--- a/x-pack/plugins/translations/translations/zh-CN.json
+++ b/x-pack/plugins/translations/translations/zh-CN.json
@ -21057,9 +21057,7 @@
    "xpack.idxMgmt.dataStreamsDetailsPanel.editDataRetentionModal.fullyManagedByILMButtonLabel": "ILM 策略",
    "xpack.idxMgmt.dataStreamsDetailsPanel.editDataRetentionModal.fullyManagedByILMDescription": "要编辑此数据流的数据保留，必须编辑其关联 {link}。",
    "xpack.idxMgmt.dataStreamsDetailsPanel.editDataRetentionModal.fullyManagedByILMTitle": "此数据流及其关联索引由 ILM 管理",
-    "xpack.idxMgmt.dataStreamsDetailsPanel.editDataRetentionModal.infiniteRetentionPeriodField": "无期限地保留数据",
    "xpack.idxMgmt.dataStreamsDetailsPanel.editDataRetentionModal.learnMoreLinkText": "工作原理？",
-    "xpack.idxMgmt.dataStreamsDetailsPanel.editDataRetentionModal.modalTitleText": "编辑数据保留",
    "xpack.idxMgmt.dataStreamsDetailsPanel.editDataRetentionModal.saveButtonLabel": "保存",
    "xpack.idxMgmt.dataStreamsDetailsPanel.editDataRetentionModal.someManagedByILMBody": "一个或多个索引由 ILM 策略管理 ({viewAllIndicesLink})。更新此数据流的数据保留不会影响到这些索引。相反，您必须更新 {ilmPolicyLink} 策略。",
    "xpack.idxMgmt.dataStreamsDetailsPanel.editDataRetentionModal.someManagedByILMTitle": "某些索引由 ILM 管理",
--- a/x-pack/test_serverless/functional/test_suites/common/management/index_management/data_streams.ts
+++ b/x-pack/test_serverless/functional/test_suites/common/management/index_management/data_streams.ts
@ -121,6 +121,15 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
      expect(await successToast.getVisibleText()).to.contain('Data retention updated');
    });

+    describe('Project level data retention checks - security solution', () => {
+      this.tags(['skipSvlOblt', 'skipSvlSearch']);
+
+      it('shows project data retention in the datastreams list', async () => {
+        expect(await testSubjects.exists('projectLevelRetentionCallout')).to.be(true);
+        expect(await testSubjects.exists('cloudLinkButton')).to.be(true);
+      });
+    });
+
    it('disabling data retention in serverless is not allowed', async () => {
      // Open details flyout
      await pageObjects.indexManagement.clickDataStreamNameLink(TEST_DS_NAME);