[Discover] Added context aware logic for logs view in discover to show Load More… (#211176)

## Summary

Closes - https://github.com/elastic/kibana/issues/166679

## What's included ?

- The PR adds a feature in Logs View of Observability (to start with) to
hide the regular pagination toolbar from the footer and show Load More
only when the user has scrolled to the bottom of the page.
- The table would always load the items in batches of default set 500 
- This PR also add 2 helper functions `useThrottleFn` and
`useDebounceFn`. Current React help library which KIbana uses called
-`react-use` does not have these and we cannot use Lodash variant of
these. We need such hooks which are React safe. Hence added these 2


## What's pending ?

- [x] Unit tests for the 2 new helper React hooks
- [x] Unit tests for data table footer component
- [x] Unit tests for Profile Resolution
- [x] Functional Serverless Tests
- [x] Functional Stateful Tests


![Feb-14-2025
15-25-18](https://github.com/user-attachments/assets/fa66de6e-b3bd-46b4-a0ed-e30c4209a695)

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Davis McPhee <davismcphee@hotmail.com>
Co-authored-by: Felix Stürmer <weltenwort@users.noreply.github.com>
Co-authored-by: Davis McPhee <davis.mcphee@elastic.co>

.buildkite/ftr_platform_stateful_configs.yml

@@ -81,6 +81,7 @@ enabled:
   - test/functional/apps/discover/group7/config.ts
   - test/functional/apps/discover/group8/config.ts
   - test/functional/apps/discover/context_awareness/config.ts
+  - test/functional/apps/discover/observability/config.ts
   - test/functional/apps/getting_started/config.ts
   - test/functional/apps/home/config.ts
   - test/functional/apps/kibana_overview/config.ts

.github/CODEOWNERS

@@ -1438,6 +1438,10 @@ packages/kbn-monaco/src/esql @elastic/kibana-esql
 /x-pack/test/alerting_api_integration/observability/index.ts @elastic/obs-ux-management-team
 /x-pack/test_serverless/api_integration/test_suites/observability/synthetics @elastic/obs-ux-management-team
 
+# Observability-ui folder level permissions (need to be before individual files inside the folder)
+/x-pack/test_serverless/functional/test_suites/observability @elastic/observability-ui
+/test/functional/apps/discover/observability @elastic/observability-ui
+
 # obs-ux-logs-team
 /x-pack/test/functional/es_archives/observability_logs_explorer @elastic/obs-ux-logs-team
 /x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality @elastic/obs-ux-logs-team
@@ -1463,9 +1467,10 @@ packages/kbn-monaco/src/esql @elastic/kibana-esql
 /x-pack/test_serverless/functional/test_suites/observability/discover @elastic/obs-ux-logs-team @elastic/kibana-data-discovery
 /src/platform/plugins/shared/unified_doc_viewer/public/components/doc_viewer_logs_overview @elastic/obs-ux-logs-team
 /x-pack/test/api_integration/apis/logs_shared @elastic/obs-ux-logs-team
+/test/functional/apps/discover/observability/embeddable @elastic/obs-ux-logs-team
+/test/functional/apps/discover/observability/logs @elastic/obs-ux-logs-team
 
 # Observability-ui
-/x-pack/test_serverless/functional/test_suites/observability @elastic/observability-ui
 /x-pack/test_serverless/api_integration/test_suites/observability/index.ts @elastic/observability-ui
 /x-pack/test/functional_solution_sidenav/tests/observability_sidenav.ts @elastic/observability-ui
 /x-pack/test/functional/page_objects/observability_page.ts @elastic/observability-ui
@@ -1699,6 +1704,7 @@ packages/kbn-monaco/src/esql @elastic/kibana-esql
 /test/functional/page_objects/header_page.ts @elastic/appex-qa
 /test/functional/page_objects/error_page.ts @elastic/appex-qa
 /test/functional/page_objects/common_page.ts @elastic/appex-qa
+/test/functional/page_objects/space_settings.ts @elastic/appex-qa
 /test/functional/jest.config.js @elastic/appex-qa
 /test/functional/ftr_provider_context.ts @elastic/appex-qa
 /test/functional/fixtures/es_archiver/README.md @elastic/appex-qa

src/platform/packages/shared/kbn-react-hooks/index.ts

@@ -9,6 +9,8 @@
 
 export { useBoolean } from './src/use_boolean';
 export { useErrorTextStyle } from './src/use_error_text_style';
+export { useDebounceFn } from './src/use_debounce_fn';
+export { useThrottleFn } from './src/use_throttle_fn';
 export { useAbortController } from './src/use_abort_controller';
 export { useAbortableAsync } from './src/use_abortable_async';
 export type { UseAbortableAsync, AbortableAsyncState } from './src/use_abortable_async';

src/platform/packages/shared/kbn-react-hooks/src/use_debounce_fn/index.ts

@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+export * from './use_debounce_fn';

src/platform/packages/shared/kbn-react-hooks/src/use_debounce_fn/use_debounce_fn.test.ts

@@ -0,0 +1,100 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { renderHook, act } from '@testing-library/react';
+import { useDebounceFn } from '../..';
+
+describe('useDebounceFn hook', () => {
+  jest.useFakeTimers();
+
+  it('should debounce the function call', () => {
+    const fn = jest.fn();
+    const { result } = renderHook(() => useDebounceFn(fn, { wait: 200 }));
+
+    act(() => {
+      result.current.run();
+      result.current.run();
+    });
+
+    expect(fn).not.toBeCalled();
+
+    act(() => {
+      jest.advanceTimersByTime(200);
+    });
+
+    expect(fn).toBeCalledTimes(1);
+  });
+
+  it('should cancel the debounced function call', () => {
+    const fn = jest.fn();
+    const { result } = renderHook(() => useDebounceFn(fn, { wait: 200 }));
+
+    act(() => {
+      result.current.run();
+      result.current.cancel();
+    });
+
+    act(() => {
+      jest.advanceTimersByTime(200);
+    });
+
+    expect(fn).not.toBeCalled();
+  });
+
+  it('should flush the debounced function call', () => {
+    const fn = jest.fn();
+    const { result } = renderHook(() => useDebounceFn(fn, { wait: 200 }));
+
+    act(() => {
+      result.current.run();
+      result.current.flush();
+    });
+
+    expect(fn).toBeCalledTimes(1);
+  });
+
+  it('should handle leading option correctly', () => {
+    const fn = jest.fn();
+    const { result } = renderHook(() => useDebounceFn(fn, { wait: 200, leading: true }));
+
+    act(() => {
+      result.current.run();
+    });
+
+    expect(fn).toBeCalledTimes(1);
+
+    act(() => {
+      jest.advanceTimersByTime(200);
+    });
+
+    act(() => {
+      result.current.run();
+    });
+
+    expect(fn).toBeCalledTimes(2);
+  });
+
+  it('should handle trailing option correctly', () => {
+    const fn = jest.fn();
+    const { result } = renderHook(() => useDebounceFn(fn, { wait: 200, trailing: true }));
+
+    act(() => {
+      result.current.run();
+      result.current.run();
+    });
+
+    expect(fn).not.toBeCalled();
+
+    act(() => {
+      jest.advanceTimersByTime(200);
+    });
+
+    expect(fn).toBeCalledTimes(1);
+  });
+});

src/platform/packages/shared/kbn-react-hooks/src/use_debounce_fn/use_debounce_fn.ts

@@ -0,0 +1,71 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { debounce, isFunction } from 'lodash';
+import { useMemo } from 'react';
+import useLatest from 'react-use/lib/useLatest';
+import useUnmount from 'react-use/lib/useUnmount';
+
+type Noop = (...args: any[]) => any;
+
+export interface DebounceOptions {
+  wait?: number;
+  leading?: boolean;
+  trailing?: boolean;
+  maxWait?: number;
+}
+
+/**
+ * Custom hook that returns a React safe debounced version of the provided function.
+ *
+ * @param {Function} fn - The function to debounce.
+ * @param {Object} [options] - Optional configuration options for debounce.
+ * @param {number} [options.wait=1000] - The number of milliseconds to delay.
+ * @param {boolean} [options.leading=false] - Specify invoking on the leading edge of the timeout.
+ * @param {boolean} [options.trailing=true] - Specify invoking on the trailing edge of the timeout.
+ * @param {number} [options.maxWait] - The maximum time `fn` is allowed to be delayed before it's invoked.
+ *
+ * @returns {Object} - An object containing the debounced function (`run`), and methods to cancel (`cancel`) or flush (`flush`) the debounce.
+ *
+ * @throws {Error} - Throws an error if the provided `fn` is not a function.
+ *
+ * @caveat The debounce does not cancel if `options` or `wait` are changed between calls.
+ */
+
+export function useDebounceFn<T extends Noop>(fn: T, options?: DebounceOptions) {
+  if (!isFunction(fn)) {
+    throw Error(`useDebounceFn expected parameter is a function, got ${typeof fn}`);
+  }
+
+  const fnRef = useLatest(fn);
+
+  const wait = options?.wait ?? 1000;
+
+  const debounced = useMemo(
+    () =>
+      debounce(
+        (...args: Parameters<T>): ReturnType<T> => {
+          return fnRef.current(...args);
+        },
+        wait,
+        options
+      ),
+    [fnRef, options, wait]
+  );
+
+  useUnmount(() => {
+    debounced.cancel();
+  });
+
+  return {
+    run: debounced,
+    cancel: debounced.cancel,
+    flush: debounced.flush,
+  };
+}

src/platform/packages/shared/kbn-react-hooks/src/use_throttle_fn/index.ts

@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+export * from './use_throttle_fn';

src/platform/packages/shared/kbn-react-hooks/src/use_throttle_fn/use_throttle_fn.test.ts

@@ -0,0 +1,109 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { renderHook, act } from '@testing-library/react';
+import { useThrottleFn } from '../..';
+
+describe('useThrottleFn hook', () => {
+  jest.useFakeTimers();
+
+  it('should throttle the function call', () => {
+    const fn = jest.fn();
+    const { result } = renderHook(() => useThrottleFn(fn, { wait: 200, leading: false }));
+
+    act(() => {
+      result.current.run();
+      result.current.run();
+      result.current.run();
+    });
+
+    // Because leading is false, the fn won't be called immediately.
+    expect(fn).toBeCalledTimes(0);
+
+    act(() => {
+      jest.advanceTimersByTime(200);
+    });
+
+    expect(fn).toBeCalledTimes(1);
+
+    act(() => {
+      result.current.run();
+      jest.advanceTimersByTime(200);
+    });
+
+    expect(fn).toBeCalledTimes(2);
+  });
+
+  it('should cancel the throttled function call', () => {
+    const fn = jest.fn();
+    const { result } = renderHook(() => useThrottleFn(fn, { wait: 200, leading: false }));
+
+    act(() => {
+      result.current.run();
+      result.current.cancel();
+    });
+
+    act(() => {
+      jest.advanceTimersByTime(200);
+    });
+
+    expect(fn).not.toBeCalled();
+  });
+
+  it('should flush the throttled function call', () => {
+    const fn = jest.fn();
+    const { result } = renderHook(() => useThrottleFn(fn, { wait: 200 }));
+
+    act(() => {
+      result.current.run();
+      result.current.flush();
+    });
+
+    expect(fn).toBeCalledTimes(1);
+  });
+
+  it('should handle leading option correctly', () => {
+    const fn = jest.fn();
+    const { result } = renderHook(() => useThrottleFn(fn, { wait: 200, leading: true }));
+
+    act(() => {
+      result.current.run();
+    });
+
+    expect(fn).toBeCalledTimes(1);
+
+    act(() => {
+      jest.advanceTimersByTime(200);
+    });
+
+    act(() => {
+      result.current.run();
+    });
+
+    expect(fn).toBeCalledTimes(2);
+  });
+
+  it('should handle trailing option correctly', () => {
+    const fn = jest.fn();
+    const { result } = renderHook(() => useThrottleFn(fn, { wait: 200, trailing: true }));
+
+    act(() => {
+      result.current.run();
+      result.current.run();
+    });
+
+    expect(fn).toBeCalledTimes(1);
+
+    act(() => {
+      jest.advanceTimersByTime(200);
+    });
+
+    expect(fn).toBeCalledTimes(2);
+  });
+});

src/platform/packages/shared/kbn-react-hooks/src/use_throttle_fn/use_throttle_fn.ts

@@ -0,0 +1,68 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { throttle, isFunction } from 'lodash';
+import { useMemo } from 'react';
+import useLatest from 'react-use/lib/useLatest';
+import useUnmount from 'react-use/lib/useUnmount';
+
+type Noop = (...args: any[]) => any;
+
+export interface ThrottleOptions {
+  wait?: number;
+  leading?: boolean;
+  trailing?: boolean;
+}
+
+/**
+ * Custom hook that returns a React safe  throttled version of the provided function.
+ *
+ * @param {Function} fn - The function to throttle.
+ * @param {Object} [options] - Optional configuration options for throttle.
+ * @param {number} [options.wait=1000] - The number of milliseconds to delay.
+ * @param {boolean} [options.leading=true] - Specify invoking on the leading edge of the timeout.
+ * @param {boolean} [options.trailing=false] - Specify invoking on the trailing edge of the timeout.
+ *
+ * @returns {Object} - An object containing the throttled function (`run`), and methods to cancel (`cancel`) or flush (`flush`) the throttle.
+ *
+ * @throws {Error} - Throws an error if the provided `fn` is not a function.
+ *
+ * @caveat The throttle does not cancel if `options` or `wait` are changed between calls.
+ */
+export function useThrottleFn<T extends Noop>(fn: T, options?: ThrottleOptions) {
+  if (!isFunction(fn)) {
+    throw Error(`useThrottleFn expected parameter is a function, got ${typeof fn}`);
+  }
+
+  const fnRef = useLatest(fn);
+
+  const wait = options?.wait ?? 1000;
+
+  const throttled = useMemo(
+    () =>
+      throttle(
+        (...args: Parameters<T>): ReturnType<T> => {
+          return fnRef.current(...args);
+        },
+        wait,
+        options
+      ),
+    [fnRef, options, wait]
+  );
+
+  useUnmount(() => {
+    throttled.cancel();
+  });
+
+  return {
+    run: throttled,
+    cancel: throttled.cancel,
+    flush: throttled.flush,
+  };
+}

src/platform/packages/shared/kbn-unified-data-table/index.ts

@@ -15,7 +15,7 @@ export {
 } from './src/components/row_height_settings';
 export { getDisplayedColumns, SOURCE_COLUMN } from './src/utils/columns';
 export { getTextBasedColumnsMeta } from './src/utils/get_columns_meta';
-export { ROWS_HEIGHT_OPTIONS, DataGridDensity } from './src/constants';
+export { ROWS_HEIGHT_OPTIONS, DataGridDensity, DEFAULT_PAGINATION_MODE } from './src/constants';
 
 export { JSONCodeEditorCommonMemoized } from './src/components/json_code_editor/json_code_editor_common';
 export { SourceDocument } from './src/components/source_document';

src/platform/packages/shared/kbn-unified-data-table/src/components/data_table.tsx

@@ -51,6 +51,7 @@ import { type DataPublicPluginStart } from '@kbn/data-plugin/public';
 import type { DocViewFilterFn } from '@kbn/unified-doc-viewer/types';
 import { AdditionalFieldGroups } from '@kbn/unified-field-list';
 import { useDataGridInTableSearch } from '@kbn/data-grid-in-table-search';
+import { useThrottleFn } from '@kbn/react-hooks';
 import { DATA_GRID_DENSITY_STYLE_MAP, useDataGridDensity } from '../hooks/use_data_grid_density';
 import {
   UnifiedDataTableSettings,
@@ -58,6 +59,7 @@ import {
   DataTableColumnsMeta,
   CustomCellRenderer,
   CustomGridColumnsConfiguration,
+  DataGridPaginationMode,
 } from '../types';
 import { getDisplayedColumns } from '../utils/columns';
 import { convertValueToString } from '../utils/convert_value_to_string';
@@ -79,6 +81,7 @@ import {
   ROWS_HEIGHT_OPTIONS,
   toolbarVisibility as toolbarVisibilityDefaults,
   DataGridDensity,
+  DEFAULT_PAGINATION_MODE,
 } from '../constants';
 import { UnifiedDataTableFooter } from './data_table_footer';
 import { UnifiedDataTableAdditionalDisplaySettings } from './data_table_additional_display_settings';
@@ -225,6 +228,14 @@ export interface UnifiedDataTableProps {
    * Manage pagination control
    */
   isPaginationEnabled?: boolean;
+  /**
+   * Manage pagination mode
+   * @default 'multiPage'
+   * "multiPage" - Regular pagination with numbers and arrows to control the page
+   * "singlePage" - Hides the general pagination bar and shows Load more button at the bottom of the grid
+   * "infinite" - Hides the general pagination bar and loads more data as the user scrolls [Not yet implemented]
+   */
+  paginationMode?: DataGridPaginationMode;
   /**
    * List of used control columns (available: 'openDetails', 'select')
    */
@@ -461,6 +472,7 @@ export const UnifiedDataTable = ({
   sort,
   isSortEnabled = true,
   isPaginationEnabled = true,
+  paginationMode = DEFAULT_PAGINATION_MODE,
   cellActionsTriggerId,
   cellActionsMetadata,
   cellActionsHandling = 'replace',
@@ -512,6 +524,7 @@ export const UnifiedDataTable = ({
   const dataGridRef = useRef<EuiDataGridRefProps>(null);
   const [isFilterActive, setIsFilterActive] = useState(false);
   const [isCompareActive, setIsCompareActive] = useState(false);
+  const [hasScrolledToBottom, setHasScrolledToBottom] = useState(false);
   const displayedColumns = getDisplayedColumns(columns, dataView);
   const defaultColumns = displayedColumns.includes('_source');
   const docMap = useMemo(
@@ -1137,6 +1150,58 @@ export const UnifiedDataTable = ({
     rowLineHeight: rowLineHeightOverride,
   });
 
+  const handleOnScroll = useCallback(
+    (event: { scrollTop: number }) => {
+      setHasScrolledToBottom((prevHasScrolledToBottom) => {
+        if (loadingState !== DataLoadingState.loaded) {
+          return prevHasScrolledToBottom;
+        }
+
+        // We need to manually query the react-window wrapper since EUI doesn't
+        // expose outerRef in virtualizationOptions, but we should request it
+        const outerRef = dataGridWrapper?.querySelector<HTMLElement>('.euiDataGrid__virtualized');
+
+        if (!outerRef) {
+          return prevHasScrolledToBottom;
+        }
+
+        // Account for footer height when it's visible to avoid flickering
+        const scrollBottomMargin = prevHasScrolledToBottom ? 140 : 100;
+        const isScrollable = outerRef.scrollHeight > outerRef.offsetHeight;
+        const isScrolledToBottom =
+          event.scrollTop + outerRef.offsetHeight >= outerRef.scrollHeight - scrollBottomMargin;
+
+        return isScrollable && isScrolledToBottom;
+      });
+    },
+    [dataGridWrapper, loadingState]
+  );
+
+  const { run: throttledHandleOnScroll } = useThrottleFn(handleOnScroll, { wait: 200 });
+
+  useEffect(() => {
+    if (loadingState === DataLoadingState.loadingMore) {
+      setHasScrolledToBottom(false);
+    }
+  }, [loadingState]);
+
+  const virtualizationOptions = useMemo(() => {
+    const options = {
+      onScroll: paginationMode === 'multiPage' ? undefined : throttledHandleOnScroll,
+    };
+
+    // Don't use row "overscan" when showing Summary column since
+    // rendering so much DOM content in each cell impacts performance
+    if (defaultColumns) {
+      return options;
+    }
+
+    return {
+      ...VIRTUALIZATION_OPTIONS,
+      ...options,
+    };
+  }, [defaultColumns, paginationMode, throttledHandleOnScroll]);
+
   const isRenderComplete = loadingState !== DataLoadingState.loading;
 
   if (!rowCount && loadingState === DataLoadingState.loading) {
@@ -1219,7 +1284,7 @@ export const UnifiedDataTable = ({
               data-test-subj="docTable"
               leadingControlColumns={leadingControlColumns}
               onColumnResize={onResize}
-              pagination={paginationObj}
+              pagination={paginationMode === 'multiPage' ? paginationObj : undefined}
               renderCellValue={renderCellValueWithInTableSearchSupport}
               ref={dataGridRef}
               rowCount={rowCount}
@@ -1233,9 +1298,7 @@ export const UnifiedDataTable = ({
               trailingControlColumns={trailingControlColumns}
               cellContext={cellContextWithInTableSearchSupport}
               renderCellPopover={renderCustomPopover}
-              // Don't use row overscan when showing Document column since
-              // rendering so much DOM content in each cell impacts performance
-              virtualizationOptions={defaultColumns ? undefined : VIRTUALIZATION_OPTIONS}
+              virtualizationOptions={virtualizationOptions}
             />
           )}
         </div>
@@ -1253,6 +1316,8 @@ export const UnifiedDataTable = ({
               onFetchMoreRecords={onFetchMoreRecords}
               data={data}
               fieldFormats={fieldFormats}
+              paginationMode={paginationMode}
+              hasScrolledToBottom={paginationMode !== 'multiPage' ? hasScrolledToBottom : true}
             />
           )}
         {searchTitle && (

src/platform/packages/shared/kbn-unified-data-table/src/components/data_table_footer.test.tsx

@@ -13,6 +13,7 @@ import { findTestSubject } from '@elastic/eui/lib/test';
 import { UnifiedDataTableFooter } from './data_table_footer';
 import { servicesMock } from '../../__mocks__/services';
 import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public';
+import { DEFAULT_PAGINATION_MODE } from '../..';
 
 describe('UnifiedDataTableFooter', function () {
   it('should not render anything when not on the last page', async () => {
@@ -26,6 +27,8 @@ describe('UnifiedDataTableFooter', function () {
           rowCount={500}
           data={servicesMock.data}
           fieldFormats={servicesMock.fieldFormats}
+          paginationMode={DEFAULT_PAGINATION_MODE}
+          hasScrolledToBottom={false}
         />
       </KibanaContextProvider>
     );
@@ -42,6 +45,8 @@ describe('UnifiedDataTableFooter', function () {
         rowCount={500}
         data={servicesMock.data}
         fieldFormats={servicesMock.fieldFormats}
+        paginationMode={DEFAULT_PAGINATION_MODE}
+        hasScrolledToBottom={false}
       />
     );
     expect(component.isEmptyRender()).toBe(true);
@@ -57,6 +62,8 @@ describe('UnifiedDataTableFooter', function () {
         rowCount={500}
         data={servicesMock.data}
         fieldFormats={servicesMock.fieldFormats}
+        paginationMode={DEFAULT_PAGINATION_MODE}
+        hasScrolledToBottom={false}
       />
     );
     expect(findTestSubject(component, 'unifiedDataTableFooter').text()).toBe(
@@ -79,6 +86,8 @@ describe('UnifiedDataTableFooter', function () {
         onFetchMoreRecords={mockLoadMore}
         data={servicesMock.data}
         fieldFormats={servicesMock.fieldFormats}
+        paginationMode={DEFAULT_PAGINATION_MODE}
+        hasScrolledToBottom={false}
       />
     );
     expect(findTestSubject(component, 'unifiedDataTableFooter').text()).toBe(
@@ -93,6 +102,50 @@ describe('UnifiedDataTableFooter', function () {
     expect(mockLoadMore).toHaveBeenCalledTimes(1);
   });
 
+  it('should render the load more button where pagination mode is set to singlePage and user has reached the bottom of the page', () => {
+    const mockLoadMore = jest.fn();
+
+    const component = mountWithIntl(
+      <UnifiedDataTableFooter
+        pageCount={5}
+        pageIndex={4}
+        sampleSize={500}
+        totalHits={1000}
+        rowCount={500}
+        isLoadingMore={false}
+        onFetchMoreRecords={mockLoadMore}
+        data={servicesMock.data}
+        fieldFormats={servicesMock.fieldFormats}
+        paginationMode={'singlePage'}
+        hasScrolledToBottom={true}
+      />
+    );
+
+    expect(findTestSubject(component, 'dscGridSampleSizeFetchMoreLink').exists()).toBe(true);
+  });
+
+  it('should not render the load more button where pagination mode is set to singlePage and but the user has not reached the bottom of the page', () => {
+    const mockLoadMore = jest.fn();
+
+    const component = mountWithIntl(
+      <UnifiedDataTableFooter
+        pageCount={5}
+        pageIndex={4}
+        sampleSize={500}
+        totalHits={1000}
+        rowCount={500}
+        isLoadingMore={false}
+        onFetchMoreRecords={mockLoadMore}
+        data={servicesMock.data}
+        fieldFormats={servicesMock.fieldFormats}
+        paginationMode={'singlePage'}
+        hasScrolledToBottom={false}
+      />
+    );
+
+    expect(findTestSubject(component, 'dscGridSampleSizeFetchMoreLink').exists()).toBe(false);
+  });
+
   it('should render a disabled button when loading more', async () => {
     const mockLoadMore = jest.fn();
 
@@ -107,6 +160,8 @@ describe('UnifiedDataTableFooter', function () {
         onFetchMoreRecords={mockLoadMore}
         data={servicesMock.data}
         fieldFormats={servicesMock.fieldFormats}
+        paginationMode={DEFAULT_PAGINATION_MODE}
+        hasScrolledToBottom={false}
       />
     );
     expect(findTestSubject(component, 'unifiedDataTableFooter').text()).toBe(
@@ -132,6 +187,8 @@ describe('UnifiedDataTableFooter', function () {
         rowCount={10000}
         data={servicesMock.data}
         fieldFormats={servicesMock.fieldFormats}
+        paginationMode={DEFAULT_PAGINATION_MODE}
+        hasScrolledToBottom={false}
       />
     );
     expect(findTestSubject(component, 'unifiedDataTableFooter').text()).toBe(

src/platform/packages/shared/kbn-unified-data-table/src/components/data_table_footer.tsx

@@ -16,6 +16,7 @@ import { ES_FIELD_TYPES, KBN_FIELD_TYPES } from '@kbn/field-types';
 import type { DataPublicPluginStart } from '@kbn/data-plugin/public';
 import type { FieldFormatsStart } from '@kbn/field-formats-plugin/public';
 import { MAX_LOADED_GRID_ROWS } from '../constants';
+import { DataGridPaginationMode } from '../..';
 
 export interface UnifiedDataTableFooterProps {
   isLoadingMore?: boolean;
@@ -27,6 +28,8 @@ export interface UnifiedDataTableFooterProps {
   onFetchMoreRecords?: () => void;
   data: DataPublicPluginStart;
   fieldFormats: FieldFormatsStart;
+  paginationMode: DataGridPaginationMode;
+  hasScrolledToBottom: boolean;
 }
 
 export const UnifiedDataTableFooter: FC<PropsWithChildren<UnifiedDataTableFooterProps>> = (
@@ -41,6 +44,8 @@ export const UnifiedDataTableFooter: FC<PropsWithChildren<UnifiedDataTableFooter
     totalHits = 0,
     onFetchMoreRecords,
     data,
+    paginationMode,
+    hasScrolledToBottom,
   } = props;
   const timefilter = data.query.timefilter.timefilter;
   const [refreshInterval, setRefreshInterval] = useState(timefilter.getRefreshInterval());
@@ -60,7 +65,10 @@ export const UnifiedDataTableFooter: FC<PropsWithChildren<UnifiedDataTableFooter
   const { euiTheme } = useEuiTheme();
   const isOnLastPage = pageIndex === pageCount - 1 && rowCount < totalHits;
 
-  if (!isOnLastPage) {
+  if (
+    (paginationMode === 'multiPage' && !isOnLastPage) ||
+    (paginationMode === 'singlePage' && !hasScrolledToBottom && !isLoadingMore)
+  ) {
     return null;
   }
 

src/platform/packages/shared/kbn-unified-data-table/src/constants.ts

@@ -16,6 +16,8 @@ export const SCORE_COLUMN_NAME = '_score';
 export const DEFAULT_ROWS_PER_PAGE = 100;
 export const MAX_LOADED_GRID_ROWS = 10000;
 export const ROWS_PER_PAGE_OPTIONS = [10, 25, 50, DEFAULT_ROWS_PER_PAGE, 250, 500];
+
+export const DEFAULT_PAGINATION_MODE = 'multiPage';
 /**
  * Row height might be a value from -1 to 20
  * A value of -1 automatically adjusts the row height to fit the contents.

src/platform/packages/shared/kbn-unified-data-table/src/types.ts

@@ -57,3 +57,5 @@ export type CustomGridColumnsConfiguration = Record<
   string,
   (props: CustomGridColumnProps) => EuiDataGridColumn
 >;
+
+export type DataGridPaginationMode = 'multiPage' | 'singlePage' | 'infinite';

src/platform/packages/shared/kbn-unified-data-table/tsconfig.json

@@ -45,5 +45,6 @@
     "@kbn/sort-predicates",
     "@kbn/data-grid-in-table-search",
     "@kbn/data-view-utils",
+    "@kbn/react-hooks",
   ]
 }

src/platform/plugins/shared/discover/public/application/main/components/layout/discover_documents.test.tsx

@@ -29,11 +29,16 @@ import { DiscoverCustomizationProvider } from '../../../../customizations';
 import { createCustomizationService } from '../../../../customizations/customization_service';
 import { DiscoverGrid } from '../../../../components/discover_grid';
 import { createDataViewDataSource } from '../../../../../common/data_sources';
+import type { ProfilesManager } from '../../../../context_awareness';
 import { internalStateActions } from '../../state_management/redux';
 
 const customisationService = createCustomizationService();
 
-async function mountComponent(fetchStatus: FetchStatus, hits: EsHitRecord[]) {
+async function mountComponent(
+  fetchStatus: FetchStatus,
+  hits: EsHitRecord[],
+  profilesManager?: ProfilesManager
+) {
   const services = discoverServiceMock;
 
   services.data.query.timefilter.timefilter.getTime = () => {
@@ -72,7 +77,9 @@ async function mountComponent(fetchStatus: FetchStatus, hits: EsHitRecord[]) {
   };
 
   const component = mountWithIntl(
-    <KibanaContextProvider services={services}>
+    <KibanaContextProvider
+      services={{ ...services, profilesManager: profilesManager ?? services.profilesManager }}
+    >
       <DiscoverCustomizationProvider value={customisationService}>
         <DiscoverMainProvider value={stateContainer}>
           <EuiProvider>

src/platform/plugins/shared/discover/public/components/discover_grid/discover_grid.tsx

@@ -9,6 +9,7 @@
 
 import React, { useMemo } from 'react';
 import {
+  DEFAULT_PAGINATION_MODE,
   renderCustomToolbar,
   UnifiedDataTable,
   type UnifiedDataTableProps,
@@ -55,6 +56,13 @@ export const DiscoverGrid: React.FC<DiscoverGridProps> = ({
     query,
   ]);
 
+  const getPaginationConfigAccessor = useProfileAccessor('getPaginationConfig');
+  const paginationModeConfig = useMemo(() => {
+    return getPaginationConfigAccessor(() => ({
+      paginationMode: DEFAULT_PAGINATION_MODE,
+    }))();
+  }, [getPaginationConfigAccessor]);
+
   return (
     <UnifiedDataTable
       showColumnTokens
@@ -65,6 +73,7 @@ export const DiscoverGrid: React.FC<DiscoverGridProps> = ({
       getRowIndicator={getRowIndicator}
       rowAdditionalLeadingControls={rowAdditionalLeadingControls}
       visibleCellActions={3} // this allows to show up to 3 actions on cell hover if available (filter in, filter out, and copy)
+      paginationMode={paginationModeConfig.paginationMode}
       {...props}
     />
   );

src/platform/plugins/shared/discover/public/context_awareness/__mocks__/index.tsx

@@ -100,6 +100,10 @@ export const createContextAwarenessMocks = ({
           },
         },
       ]),
+      getPaginationConfig: jest.fn((prev) => () => ({
+        ...prev(),
+        paginationMode: 'multiPage',
+      })),
     },
     resolve: jest.fn(() => ({
       isMatch: true,

src/platform/plugins/shared/discover/public/context_awareness/profile_providers/example/example_data_source_profile/profile.tsx

@@ -255,6 +255,10 @@ export const createExampleDataSourceProfileProvider = (): DataSourceProfileProvi
           isCompatible: ({ field }) => field.name !== 'message',
         },
       ],
+    getPaginationConfig: (prev) => () => ({
+      ...prev(),
+      paginationMode: 'singlePage',
+    }),
   },
   resolve: (params) => {
     let indexPattern: string | undefined;
@@ -269,7 +273,7 @@ export const createExampleDataSourceProfileProvider = (): DataSourceProfileProvi
       indexPattern = params.dataView.getIndexPattern();
     }
 
-    if (indexPattern !== 'my-example-logs') {
+    if (indexPattern !== 'my-example-logs' && indexPattern !== 'my-example-logs,logstash*') {
       return { isMatch: false };
     }
 

src/platform/plugins/shared/discover/public/context_awareness/profile_providers/observability/logs_data_source_profile/accessors/get_pagination_config.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import type { DataSourceProfileProvider } from '../../../..';
+
+export const getPaginationConfig: DataSourceProfileProvider['profile']['getPaginationConfig'] =
+  (prev) => () => ({
+    ...(prev ? prev() : {}),
+    paginationMode: 'singlePage',
+  });

src/platform/plugins/shared/discover/public/context_awareness/profile_providers/observability/logs_data_source_profile/accessors/index.ts

@@ -11,3 +11,4 @@ export { getRowIndicatorProvider } from './get_row_indicator_provider';
 export { createGetDefaultAppState } from './get_default_app_state';
 export { getCellRenderers } from './get_cell_renderers';
 export { getRowAdditionalLeadingControls } from './get_row_additional_leading_controls';
+export { getPaginationConfig } from './get_pagination_config';

src/platform/plugins/shared/discover/public/context_awareness/profile_providers/observability/logs_data_source_profile/profile.ts

@@ -15,6 +15,7 @@ import {
   getRowIndicatorProvider,
   getRowAdditionalLeadingControls,
   createGetDefaultAppState,
+  getPaginationConfig,
 } from './accessors';
 import { extractIndexPatternFrom } from '../../extract_index_pattern_from';
 import { OBSERVABILITY_ROOT_PROFILE_ID } from '../consts';
@@ -28,6 +29,7 @@ export const createLogsDataSourceProfileProvider = (
     getCellRenderers,
     getRowIndicatorProvider,
     getRowAdditionalLeadingControls,
+    getPaginationConfig,
   },
   resolve: (params) => {
     if (params.rootContext.profileId !== OBSERVABILITY_ROOT_PROFILE_ID) {

src/platform/plugins/shared/discover/public/context_awareness/types.ts

@@ -12,6 +12,7 @@ import type {
   CustomCellRenderer,
   DataGridDensity,
   UnifiedDataTableProps,
+  DataGridPaginationMode,
 } from '@kbn/unified-data-table';
 import type { DocViewsRegistry } from '@kbn/unified-doc-viewer';
 import type { AppMenuRegistry, DataTableRecord } from '@kbn/discover-utils';
@@ -38,6 +39,17 @@ export interface AppMenuExtension {
   appMenuRegistry: (prevRegistry: AppMenuRegistry) => AppMenuRegistry;
 }
 
+/**
+ * Supports extending the Pagination Config in Discover
+ */
+export interface PaginationConfigExtension {
+  /**
+   * Supports customizing the pagination mode in Discover
+   * @returns paginationMode - which mode to use for loading Pagination toolbar
+   */
+  paginationMode: DataGridPaginationMode;
+}
+
 /**
  * Parameters passed to the app menu extension
  */
@@ -321,6 +333,14 @@ export interface Profile {
    */
   getAdditionalCellActions: () => AdditionalCellAction[];
 
+  /**
+   * Allows setting the pagination mode and its configuration
+   * The `getPaginationConfig` extension point currently gives `paginationMode` which can be set to 'multiPage' | 'singlePage' | 'infinite';
+   * Note: This extension point currently only returns `paginationMode` but can be extended to return `pageSize` etc as well.
+   * @returns The pagination mode extension
+   */
+  getPaginationConfig: () => PaginationConfigExtension;
+
   /**
    * Document viewer flyout
    */

test/functional/apps/discover/context_awareness/extensions/_get_pagination_config.ts

@@ -0,0 +1,95 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import kbnRison from '@kbn/rison';
+import type { FtrProviderContext } from '../../ftr_provider_context';
+
+export default function ({ getService, getPageObjects }: FtrProviderContext) {
+  const { common, discover } = getPageObjects(['common', 'discover']);
+  const testSubjects = getService('testSubjects');
+  const dataViews = getService('dataViews');
+  const dataGrid = getService('dataGrid');
+  const esArchiver = getService('esArchiver');
+  const kibanaServer = getService('kibanaServer');
+
+  const resetTimeFrame = {
+    from: '2024-06-10T14:00:00.000Z',
+    to: '2024-06-10T16:30:00.000Z',
+  };
+
+  const currentTimeFrame = {
+    from: '2015-09-20T01:00:00.000Z',
+    to: '2015-09-24T16:30:00.000Z',
+  };
+
+  describe('extension getPaginationConfig', () => {
+    before(async () => {
+      // To load more than 500 records
+      await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/logstash_functional');
+      await kibanaServer.uiSettings.update({
+        'timepicker:timeDefaults': `{ "from": "${currentTimeFrame.from}", "to": "${currentTimeFrame.to}"}`,
+      });
+    });
+
+    after(async () => {
+      await esArchiver.unload('test/functional/fixtures/es_archiver/logstash_functional');
+      await kibanaServer.uiSettings.update({
+        'timepicker:timeDefaults': `{ "from": "${resetTimeFrame.from}", "to": "${resetTimeFrame.to}"}`,
+      });
+    });
+
+    describe('ES|QL mode', () => {
+      it('should render without pagination using a single page', async () => {
+        const state = kbnRison.encode({
+          dataSource: { type: 'esql' },
+          query: { esql: 'from logstash* | sort @timestamp desc' },
+        });
+        await common.navigateToActualUrl('discover', `?_a=${state}`, {
+          ensureCurrentUrl: false,
+        });
+        await discover.waitUntilSearchingHasFinished();
+
+        // In ESQL Mode, pagination is disabled
+        await testSubjects.missingOrFail('tablePaginationPopoverButton');
+        await testSubjects.missingOrFail('pagination-button-previous');
+        await testSubjects.missingOrFail('pagination-button-next');
+      });
+    });
+
+    describe('data view mode', () => {
+      it('should render single page pagination without page numbers', async () => {
+        await common.navigateToActualUrl('discover', undefined, {
+          ensureCurrentUrl: false,
+        });
+        await dataViews.switchTo('my-example-logs,logstash*');
+        await discover.waitUntilSearchingHasFinished();
+        await testSubjects.missingOrFail('tablePaginationPopoverButton');
+        await testSubjects.missingOrFail('pagination-button-previous');
+        await testSubjects.missingOrFail('pagination-button-next');
+      });
+
+      it('should render default pagination with page numbers', async () => {
+        await common.navigateToActualUrl('discover', undefined, {
+          ensureCurrentUrl: false,
+        });
+        await dataViews.createFromSearchBar({
+          name: 'lo', // Must be anything but log/logs, since pagination is disabled for log sources
+          adHoc: true,
+          hasTimeField: true,
+        });
+        await discover.waitUntilSearchingHasFinished();
+
+        await testSubjects.existOrFail('tablePaginationPopoverButton');
+        await testSubjects.existOrFail('pagination-button-previous');
+        await testSubjects.existOrFail('pagination-button-next');
+        await dataGrid.checkCurrentRowsPerPageToBe(100);
+      });
+    });
+  });
+}

test/functional/apps/discover/context_awareness/index.ts

@@ -47,5 +47,6 @@ export default function ({ getService, getPageObjects, loadTestFile }: FtrProvid
     loadTestFile(require.resolve('./extensions/_get_app_menu'));
     loadTestFile(require.resolve('./extensions/_get_render_app_wrapper'));
     loadTestFile(require.resolve('./extensions/_get_default_ad_hoc_data_views'));
+    loadTestFile(require.resolve('./extensions/_get_pagination_config'));
   });
 }

test/functional/apps/discover/observability/config.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { FtrConfigProviderContext } from '@kbn/test';
+
+export default async function ({ readConfigFile }: FtrConfigProviderContext) {
+  const functionalConfig = await readConfigFile(require.resolve('../../../config.base.js'));
+  const baseConfig = functionalConfig.getAll();
+
+  return {
+    ...baseConfig,
+    testFiles: [require.resolve('.')],
+  };
+}

test/functional/apps/discover/observability/embeddable/_saved_search_embeddable.ts

@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import expect from '@kbn/expect';
+import { FtrProviderContext } from '../../ftr_provider_context';
+
+export default function ({ getService, getPageObjects }: FtrProviderContext) {
+  const dataGrid = getService('dataGrid');
+  const dashboardAddPanel = getService('dashboardAddPanel');
+  const esArchiver = getService('esArchiver');
+  const kibanaServer = getService('kibanaServer');
+  const testSubjects = getService('testSubjects');
+  const { dashboard, header } = getPageObjects(['dashboard', 'header']);
+
+  describe('discover/observability saved search embeddable', () => {
+    before(async () => {
+      await kibanaServer.savedObjects.cleanStandardList();
+      await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/dashboard/current/data');
+      await kibanaServer.importExport.load(
+        'test/functional/fixtures/kbn_archiver/dashboard/current/kibana'
+      );
+      await dashboard.navigateToApp();
+      await dashboard.gotoDashboardLandingPage();
+      await dashboard.clickNewDashboard();
+    });
+
+    after(async () => {
+      await esArchiver.unload('test/functional/fixtures/es_archiver/dashboard/current/data');
+      await kibanaServer.savedObjects.cleanStandardList();
+    });
+
+    const addSearchEmbeddableToDashboard = async () => {
+      await dashboardAddPanel.addSavedSearch('Rendering-Test:-saved-search');
+      await header.waitUntilLoadingHasFinished();
+      await dashboard.waitForRenderComplete();
+      const rows = await dataGrid.getDocTableRows();
+      expect(rows.length).to.be.above(0);
+    };
+
+    it('should render content with singlePage pagination mode', async () => {
+      await addSearchEmbeddableToDashboard();
+
+      // Pagination toolbar should not be visible
+      await testSubjects.missingOrFail('tablePaginationPopoverButton');
+      await testSubjects.missingOrFail('pagination-button-previous');
+      await testSubjects.missingOrFail('pagination-button-next');
+
+      // Scroll to the bottom of the page
+      await dataGrid.scrollTo(500, 1500); // Ugly hack to add 1500 to the current scroll position due to virtualized table adding unexpected ordered rows
+
+      // Check the pagination footer loads
+      await testSubjects.existOrFail('unifiedDataTableFooter');
+    });
+  });
+}

test/functional/apps/discover/observability/index.ts

@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { FtrProviderContext } from '../ftr_provider_context';
+
+export default function ({ getService, getPageObjects, loadTestFile }: FtrProviderContext) {
+  const esArchiver = getService('esArchiver');
+  const kibanaServer = getService('kibanaServer');
+  const { common, spaceSettings } = getPageObjects(['common', 'spaceSettings']);
+
+  const from = 'Sep 22, 2015 @ 00:00:00.000';
+  const to = 'Sep 23, 2015 @ 00:00:00.000';
+
+  describe('discover/observability', () => {
+    before(async () => {
+      await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/logstash_functional');
+      await kibanaServer.uiSettings.replace({
+        defaultIndex: '0bf35f60-3dc9-11e8-8660-4d65aa086b3c',
+      });
+      await common.setTime({
+        from,
+        to,
+      });
+      await spaceSettings.switchSpaceSolutionType({
+        spaceName: 'default',
+        solution: 'oblt',
+      });
+    });
+
+    after(async () => {
+      await spaceSettings.switchSpaceSolutionType({
+        spaceName: 'default',
+        solution: 'classic',
+      });
+      await common.unsetTime();
+      await esArchiver.unload('test/functional/fixtures/es_archiver/logstash_functional');
+    });
+
+    loadTestFile(require.resolve('./embeddable/_saved_search_embeddable'));
+    loadTestFile(require.resolve('./logs/_get_pagination_config'));
+  });
+}

test/functional/apps/discover/observability/logs/_get_pagination_config.ts

@@ -0,0 +1,125 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import kbnRison from '@kbn/rison';
+import { FtrProviderContext } from '../../ftr_provider_context';
+
+export default function ({ getService, getPageObjects }: FtrProviderContext) {
+  const PageObjects = getPageObjects(['common', 'discover']);
+  const testSubjects = getService('testSubjects');
+  const dataViews = getService('dataViews');
+  const dataGrid = getService('dataGrid');
+  const esArchiver = getService('esArchiver');
+  const kibanaServer = getService('kibanaServer');
+
+  const resetTimeFrame = {
+    from: '2024-06-10T14:00:00.000Z',
+    to: '2024-06-10T16:30:00.000Z',
+  };
+
+  const currentTimeFrame = {
+    from: '2015-09-20T01:00:00.000Z',
+    to: '2015-09-24T16:30:00.000Z',
+  };
+
+  describe('extension getPaginationConfig', () => {
+    before(async () => {
+      // To load more than 500 records
+      await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/logstash_functional');
+      await kibanaServer.uiSettings.update({
+        'timepicker:timeDefaults': `{ "from": "${currentTimeFrame.from}", "to": "${currentTimeFrame.to}"}`,
+      });
+    });
+
+    after(async () => {
+      await esArchiver.unload('test/functional/fixtures/es_archiver/logstash_functional');
+      await kibanaServer.uiSettings.update({
+        'timepicker:timeDefaults': `{ "from": "${resetTimeFrame.from}", "to": "${resetTimeFrame.to}"}`,
+      });
+    });
+
+    describe('ES|QL mode', () => {
+      it('should render without pagination using a single page', async () => {
+        const limit = 200;
+        const state = kbnRison.encode({
+          dataSource: { type: 'esql' },
+          query: { esql: `from logstash* | sort @timestamp desc | limit ${limit}` },
+        });
+        await PageObjects.common.navigateToActualUrl('discover', `?_a=${state}`, {
+          ensureCurrentUrl: false,
+        });
+        await PageObjects.discover.waitUntilSearchingHasFinished();
+
+        await dataGrid.scrollTo(300);
+
+        await PageObjects.discover.waitUntilSearchingHasFinished();
+        // In ESQL Mode, pagination is disabled
+        await testSubjects.missingOrFail('tablePaginationPopoverButton');
+        await testSubjects.missingOrFail('pagination-button-previous');
+        await testSubjects.missingOrFail('pagination-button-next');
+      });
+    });
+
+    describe('data view mode', () => {
+      it('should render default pagination with page numbers', async () => {
+        await PageObjects.common.navigateToActualUrl('discover', undefined, {
+          ensureCurrentUrl: false,
+        });
+        await dataViews.createFromSearchBar({
+          name: 'lo', // Must be anything but log/logs, since pagination is disabled for log sources
+          adHoc: true,
+          hasTimeField: true,
+        });
+
+        await PageObjects.discover.waitUntilSearchingHasFinished();
+
+        await testSubjects.existOrFail('tablePaginationPopoverButton');
+        await testSubjects.existOrFail('pagination-button-previous');
+        await testSubjects.existOrFail('pagination-button-next');
+        await dataGrid.checkCurrentRowsPerPageToBe(100);
+      });
+
+      it('should render single page pagination without page numbers', async () => {
+        const defaultPageLimit = 500;
+        await PageObjects.common.navigateToActualUrl('discover', undefined, {
+          ensureCurrentUrl: false,
+        });
+        await dataViews.createFromSearchBar({
+          name: 'logs',
+          adHoc: true,
+          hasTimeField: true,
+        });
+        await PageObjects.discover.waitUntilSearchingHasFinished();
+
+        await testSubjects.missingOrFail('tablePaginationPopoverButton');
+        await testSubjects.missingOrFail('pagination-button-previous');
+        await testSubjects.missingOrFail('pagination-button-next');
+
+        // Now scroll to bottom to load footer
+        await dataGrid.scrollTo(defaultPageLimit);
+        await PageObjects.discover.waitUntilSearchingHasFinished();
+
+        await testSubjects.existOrFail('unifiedDataTableFooter');
+        await testSubjects.existOrFail('dscGridSampleSizeFetchMoreLink');
+
+        // Clicking on Load more should fetch more data and hide the footer
+        const loadMoreButton = await testSubjects.find('dscGridSampleSizeFetchMoreLink');
+        await loadMoreButton.click();
+
+        await PageObjects.discover.waitUntilSearchingHasFinished();
+
+        // Scroll needs to be triggered to hide the footer
+        await dataGrid.scrollTo(defaultPageLimit + 10);
+
+        await testSubjects.missingOrFail('unifiedDataTableFooter');
+        await testSubjects.missingOrFail('dscGridSampleSizeFetchMoreLink');
+      });
+    });
+  });
+}

test/functional/page_objects/index.ts

@@ -39,6 +39,7 @@ import { FilesManagementPageObject } from './files_management';
 import { AnnotationEditorPageObject } from './annotation_library_editor_page';
 import { SolutionNavigationProvider } from './solution_navigation';
 import { EmbeddedConsoleProvider } from './embedded_console';
+import { SpaceSettingsPageObject } from './space_settings';
 
 export const pageObjects = {
   annotationEditor: AnnotationEditorPageObject,
@@ -73,6 +74,7 @@ export const pageObjects = {
   unifiedSearch: UnifiedSearchPageObject,
   unifiedFieldList: UnifiedFieldListPageObject,
   filesManagement: FilesManagementPageObject,
+  spaceSettings: SpaceSettingsPageObject,
 };
 
 export { SolutionNavigationProvider } from './solution_navigation';

test/functional/page_objects/space_settings.ts

@@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { FtrService } from '../ftr_provider_context';
+
+export class SpaceSettingsPageObject extends FtrService {
+  private readonly testSubjects = this.ctx.getService('testSubjects');
+  private readonly common = this.ctx.getPageObject('common');
+
+  public async navigateTo(spaceName: string = 'default') {
+    await this.common.navigateToUrl('management', `kibana/spaces/edit/${spaceName}`, {
+      shouldUseHashForSubUrl: false,
+    });
+  }
+
+  public async switchSpace(spaceName: string) {
+    await this.testSubjects.click('spacesNavSelector');
+    await this.testSubjects.click(`${spaceName}-selectableSpaceItem`);
+  }
+
+  public async switchSpaceSolutionType({
+    spaceName = 'default',
+    solution = 'oblt',
+  }: {
+    spaceName?: string;
+    solution?: 'security' | 'oblt' | 'search' | 'classic';
+  }) {
+    const solutionSpecificTestSubjectMap = {
+      search: 'solutionViewEsOption',
+      oblt: 'solutionViewObltOption',
+      security: 'solutionViewSecurityOption',
+      classic: 'solutionViewClassicOption',
+    };
+
+    await this.navigateTo(spaceName);
+
+    await this.testSubjects.click('solutionViewSelect');
+    await this.testSubjects.click(solutionSpecificTestSubjectMap[solution]);
+    await this.testSubjects.click('save-space-button');
+    await this.testSubjects.click('confirmModalConfirmButton');
+
+    await this.common.navigateToUrl('discover');
+  }
+}

test/functional/services/data_grid.ts

@@ -982,4 +982,36 @@ export class DataGridService extends FtrService {
   public async getInTableSearchCellMatchesCount(rowIndex: number, columnName: string) {
     return (await this.getInTableSearchCellMatchElements(rowIndex, columnName)).length;
   }
+
+  public async scrollTo(rowCount: number, finalScrollIncrement: number = 100) {
+    const dataGridTargetIndex = rowCount - 1; // 0-based index
+    let lastRowIndex = -1;
+
+    while (true) {
+      const rows = await this.find.allByCssSelector('.euiDataGridRow');
+      const lastRow = rows[rows.length - 1];
+      const currentLastRowIndex = parseInt(
+        (await lastRow.getAttribute('data-grid-row-index')) as string,
+        10
+      );
+
+      const container = await this.find.byCssSelector('.euiDataGrid__virtualized');
+
+      if (currentLastRowIndex === lastRowIndex) {
+        break; // Exit if no further scrolling is possible
+      }
+
+      if (currentLastRowIndex >= dataGridTargetIndex) {
+        await this.browser.execute(`arguments[0].scrollTop += ${finalScrollIncrement}`, container); // Final scroll increment
+        break; // Exit if reached the target index
+      }
+
+      lastRowIndex = currentLastRowIndex;
+
+      await this.browser.execute('arguments[0].scrollTop += 500', container); // Increase scroll increment
+
+      // Delay to make sure content is loaded
+      await new Promise((resolve) => setTimeout(resolve, 100));
+    }
+  }
 }

x-pack/test_serverless/functional/test_suites/common/discover/context_awareness/extensions/_get_pagination_config.ts

@@ -0,0 +1,99 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import kbnRison from '@kbn/rison';
+import type { FtrProviderContext } from '../../../../../ftr_provider_context';
+
+export default function ({ getService, getPageObjects }: FtrProviderContext) {
+  const { common, discover, svlCommonPage } = getPageObjects([
+    'common',
+    'discover',
+    'svlCommonPage',
+  ]);
+  const testSubjects = getService('testSubjects');
+  const dataViews = getService('dataViews');
+  const dataGrid = getService('dataGrid');
+  const esArchiver = getService('esArchiver');
+  const kibanaServer = getService('kibanaServer');
+
+  const resetTimeFrame = {
+    from: '2024-06-10T14:00:00.000Z',
+    to: '2024-06-10T16:30:00.000Z',
+  };
+
+  const currentTimeFrame = {
+    from: '2015-09-20T01:00:00.000Z',
+    to: '2015-09-24T16:30:00.000Z',
+  };
+
+  describe('extension getPaginationConfig', () => {
+    before(async () => {
+      await svlCommonPage.loginAsAdmin();
+      // To load more than 500 records
+      await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/logstash_functional');
+      await kibanaServer.uiSettings.update({
+        'timepicker:timeDefaults': `{ "from": "${currentTimeFrame.from}", "to": "${currentTimeFrame.to}"}`,
+      });
+    });
+
+    after(async () => {
+      await esArchiver.unload('test/functional/fixtures/es_archiver/logstash_functional');
+      await kibanaServer.uiSettings.update({
+        'timepicker:timeDefaults': `{ "from": "${resetTimeFrame.from}", "to": "${resetTimeFrame.to}"}`,
+      });
+    });
+
+    describe('ES|QL mode', () => {
+      it('should render without pagination using a single page', async () => {
+        const state = kbnRison.encode({
+          dataSource: { type: 'esql' },
+          query: { esql: 'from logstash* | sort @timestamp desc' },
+        });
+        await common.navigateToActualUrl('discover', `?_a=${state}`, {
+          ensureCurrentUrl: false,
+        });
+        await discover.waitUntilSearchingHasFinished();
+
+        // In ESQL Mode, pagination is disabled
+        await testSubjects.missingOrFail('tablePaginationPopoverButton');
+        await testSubjects.missingOrFail('pagination-button-previous');
+        await testSubjects.missingOrFail('pagination-button-next');
+      });
+    });
+
+    describe('data view mode', () => {
+      it('should render single page pagination without page numbers', async () => {
+        await common.navigateToActualUrl('discover', undefined, {
+          ensureCurrentUrl: false,
+        });
+        await dataViews.switchTo('my-example-logs,logstash*');
+        await discover.waitUntilSearchingHasFinished();
+        await testSubjects.missingOrFail('tablePaginationPopoverButton');
+        await testSubjects.missingOrFail('pagination-button-previous');
+        await testSubjects.missingOrFail('pagination-button-next');
+      });
+
+      it('should render default pagination with page numbers', async () => {
+        await common.navigateToActualUrl('discover', undefined, {
+          ensureCurrentUrl: false,
+        });
+
+        await dataViews.createFromSearchBar({
+          name: 'lo', // Must be anything but log/logs, since pagination is disabled for log sources
+          adHoc: true,
+          hasTimeField: true,
+        });
+
+        await discover.waitUntilSearchingHasFinished();
+        await testSubjects.existOrFail('tablePaginationPopoverButton');
+        await testSubjects.existOrFail('pagination-button-previous');
+        await testSubjects.existOrFail('pagination-button-next');
+        await dataGrid.checkCurrentRowsPerPageToBe(100);
+      });
+    });
+  });
+}

x-pack/test_serverless/functional/test_suites/common/discover/context_awareness/index.ts

@@ -45,5 +45,6 @@ export default function ({ getService, getPageObjects, loadTestFile }: FtrProvid
     loadTestFile(require.resolve('./extensions/_get_app_menu'));
     loadTestFile(require.resolve('./extensions/_get_render_app_wrapper'));
     loadTestFile(require.resolve('./extensions/_get_default_ad_hoc_data_views'));
+    loadTestFile(require.resolve('./extensions/_get_pagination_config'));
   });
 }

x-pack/test_serverless/functional/test_suites/common/discover/embeddable/_saved_search_embeddable.ts

@@ -17,6 +17,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
   const esArchiver = getService('esArchiver');
   const kibanaServer = getService('kibanaServer');
   const testSubjects = getService('testSubjects');
+  const dataViews = getService('dataViews');
   const { common, svlCommonPage, dashboard, header, discover } = getPageObjects([
     'common',
     'svlCommonPage',
@@ -25,6 +26,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
     'discover',
   ]);
 
+  const nonLogsSavedSearchName = 'Rendering-Test:-saved-search-non-logs';
+
   describe('discover saved search embeddable', () => {
     before(async () => {
       await browser.setWindowSize(1300, 800);
@@ -42,6 +45,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
         from: 'Sep 22, 2015 @ 00:00:00.000',
         to: 'Sep 23, 2015 @ 00:00:00.000',
       });
+      await createNonLogsSavedSearch();
     });
 
     after(async () => {
@@ -58,8 +62,24 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
       await dashboard.clickNewDashboard();
     });
 
-    const addSearchEmbeddableToDashboard = async () => {
-      await dashboardAddPanel.addSavedSearch('Rendering-Test:-saved-search');
+    const createNonLogsSavedSearch = async () => {
+      await common.navigateToActualUrl('discover', undefined, {
+        ensureCurrentUrl: false,
+      });
+      await dataViews.createFromSearchBar({
+        name: 'lo', // Must be anything but log/logs, since pagination is disabled for log sources
+        adHoc: true,
+        hasTimeField: true,
+      });
+      await discover.waitUntilSearchingHasFinished();
+
+      await discover.saveSearch(nonLogsSavedSearchName);
+    };
+
+    const addSearchEmbeddableToDashboard = async (
+      saveSearchName: string = 'Rendering-Test:-saved-search'
+    ) => {
+      await dashboardAddPanel.addSavedSearch(saveSearchName);
       await header.waitUntilLoadingHasFinished();
       await dashboard.waitForRenderComplete();
       const rows = await dataGrid.getDocTableRows();
@@ -74,7 +94,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
 
     it('can save a search embeddable with a defined rows per page number', async function () {
       const dashboardName = 'Dashboard with a Paginated Saved Search';
-      await addSearchEmbeddableToDashboard();
+      await addSearchEmbeddableToDashboard(nonLogsSavedSearchName);
       await dataGrid.checkCurrentRowsPerPageToBe(100);
 
       await dashboard.saveDashboard(dashboardName, {

x-pack/test_serverless/functional/test_suites/common/discover/group6/_unsaved_changes_badge.ts

@@ -25,6 +25,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
     'timePicker',
     'unifiedFieldList',
   ]);
+  const dataViews = getService('dataViews');
   const security = getService('security');
   const defaultSettings = {
     defaultIndex: 'logstash-*',
@@ -65,6 +66,11 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
     });
 
     it('should show the badge only after changes to a persisted saved search', async () => {
+      await dataViews.createFromSearchBar({
+        name: 'lo', // Must be anything but log/logs, since pagination is disabled for log sources
+        adHoc: true,
+        hasTimeField: true,
+      });
       await PageObjects.discover.saveSearch(SAVED_SEARCH_NAME);
       await PageObjects.discover.waitUntilSearchingHasFinished();
 

x-pack/test_serverless/functional/test_suites/observability/discover/context_awareness/_get_pagination_config.ts

@@ -0,0 +1,101 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import kbnRison from '@kbn/rison';
+import type { FtrProviderContext } from '../../../../ftr_provider_context';
+
+export default function ({ getService, getPageObjects }: FtrProviderContext) {
+  const PageObjects = getPageObjects(['common', 'discover', 'svlCommonPage']);
+  const testSubjects = getService('testSubjects');
+  const dataViews = getService('dataViews');
+  const dataGrid = getService('dataGrid');
+  const esArchiver = getService('esArchiver');
+  const kibanaServer = getService('kibanaServer');
+
+  const resetTimeFrame = {
+    from: '2024-06-10T14:00:00.000Z',
+    to: '2024-06-10T16:30:00.000Z',
+  };
+
+  const currentTimeFrame = {
+    from: '2015-09-20T01:00:00.000Z',
+    to: '2015-09-24T16:30:00.000Z',
+  };
+
+  describe('extension getPaginationConfig', () => {
+    before(async () => {
+      await PageObjects.svlCommonPage.loginAsViewer();
+      // To load more than 500 records
+      await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/logstash_functional');
+      await kibanaServer.uiSettings.update({
+        'timepicker:timeDefaults': `{ "from": "${currentTimeFrame.from}", "to": "${currentTimeFrame.to}"}`,
+      });
+    });
+
+    after(async () => {
+      await esArchiver.unload('test/functional/fixtures/es_archiver/logstash_functional');
+      await kibanaServer.uiSettings.update({
+        'timepicker:timeDefaults': `{ "from": "${resetTimeFrame.from}", "to": "${resetTimeFrame.to}"}`,
+      });
+    });
+
+    describe('ES|QL mode', () => {
+      it('should render without pagination using a single page', async () => {
+        const limit = 200;
+        const state = kbnRison.encode({
+          dataSource: { type: 'esql' },
+          query: { esql: `from logstash* | sort @timestamp desc | limit ${limit}` },
+        });
+        await PageObjects.common.navigateToActualUrl('discover', `?_a=${state}`, {
+          ensureCurrentUrl: false,
+        });
+        await PageObjects.discover.waitUntilSearchingHasFinished();
+
+        await dataGrid.scrollTo(300);
+
+        await PageObjects.discover.waitUntilSearchingHasFinished();
+        // In ESQL Mode, pagination is disabled
+        await testSubjects.missingOrFail('tablePaginationPopoverButton');
+        await testSubjects.missingOrFail('pagination-button-previous');
+        await testSubjects.missingOrFail('pagination-button-next');
+      });
+    });
+
+    describe('data view mode', () => {
+      it('should render default pagination with page numbers', async () => {
+        const defaultPageLimit = 500;
+        await PageObjects.common.navigateToActualUrl('discover', undefined, {
+          ensureCurrentUrl: false,
+        });
+        await dataViews.switchTo('my-example-logs,logstash*');
+        await PageObjects.discover.waitUntilSearchingHasFinished();
+
+        await dataGrid.scrollTo(defaultPageLimit);
+
+        await PageObjects.discover.waitUntilSearchingHasFinished();
+        await testSubjects.missingOrFail('tablePaginationPopoverButton');
+        await testSubjects.missingOrFail('pagination-button-previous');
+        await testSubjects.missingOrFail('pagination-button-next');
+
+        await testSubjects.existOrFail('unifiedDataTableFooter');
+        await testSubjects.existOrFail('dscGridSampleSizeFetchMoreLink');
+
+        // Clicking on Load more should fetch more data and hide the footer
+        const loadMoreButton = await testSubjects.find('dscGridSampleSizeFetchMoreLink');
+        await loadMoreButton.click();
+
+        await PageObjects.discover.waitUntilSearchingHasFinished();
+
+        // Scroll needs to be triggered to hide the footer
+        await dataGrid.scrollTo(defaultPageLimit + 10);
+
+        await testSubjects.missingOrFail('unifiedDataTableFooter');
+        await testSubjects.missingOrFail('dscGridSampleSizeFetchMoreLink');
+      });
+    });
+  });
+}

x-pack/test_serverless/functional/test_suites/observability/discover/context_awareness/index.ts

@@ -39,5 +39,6 @@ export default function ({ getService, getPageObjects, loadTestFile }: FtrProvid
     loadTestFile(require.resolve('./_get_doc_viewer'));
     loadTestFile(require.resolve('./_get_cell_renderers'));
     loadTestFile(require.resolve('./_get_app_menu'));
+    loadTestFile(require.resolve('./_get_pagination_config'));
   });
 }