[Security Solution] Add Discover Data View picker to Timeline (#184928)

## Summary

Add new `Dataview picker` component and some initial redux setup to feed
it with data.
Dont expect this to work just like the original timeline sourcerer does
just yet.

### Checklist

Delete any items that are not applicable to this PR.

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

### Testing
Do `localStorage.setItem('EXPERIMENTAL_SOURCERER_ENABLED', true)` in the
browser console, reload the page,
then open new timeline.

You should see the new dataview picker (colored in red temporarily),
that should allow data view switching.

Known issues: dataview editor is showing behind the picker (to be fixed
in subsequent PR).

src/plugins/data_view_editor/public/open_editor.tsx

@@ -85,6 +85,10 @@ export const getEditorOpener =
         {
           hideCloseButton: true,
           size: 'l',
+          maskProps: {
+            // EUI TODO: This z-index override of EuiOverlayMask is a workaround, and ideally should be resolved with a cleaner UI/UX flow long-term
+            style: 'z-index: 1003', // we need this flyout to be above the timeline flyout (which has a z-index of 1002)
+          },
         }
       );
 

x-pack/plugins/security_solution/public/app/home/template_wrapper/index.tsx

@@ -12,6 +12,7 @@ import { IS_DRAGGING_CLASS_NAME } from '@kbn/securitysolution-t-grid';
 import { KibanaPageTemplate } from '@kbn/shared-ux-page-kibana-template';
 import type { KibanaPageTemplateProps } from '@kbn/shared-ux-page-kibana-template';
 import { ExpandableFlyoutProvider } from '@kbn/expandable-flyout';
+import { DataViewPickerProvider } from '../../../sourcerer/experimental/containers/dataview_picker_provider';
 import { AttackDiscoveryTour } from '../../../attack_discovery/tour';
 import { URL_PARAM_KEY } from '../../../common/hooks/use_url_state';
 import { SecuritySolutionFlyout, TimelineFlyout } from '../../../flyout';
@@ -103,10 +104,12 @@ export const SecuritySolutionTemplateWrapper: React.FC<SecuritySolutionTemplateW
               component="div"
               grow={true}
             >
-              <ExpandableFlyoutProvider urlKey={isPreview ? undefined : URL_PARAM_KEY.flyout}>
-                {children}
-                <SecuritySolutionFlyout />
-              </ExpandableFlyoutProvider>
+              <DataViewPickerProvider>
+                <ExpandableFlyoutProvider urlKey={isPreview ? undefined : URL_PARAM_KEY.flyout}>
+                  {children}
+                  <SecuritySolutionFlyout />
+                </ExpandableFlyoutProvider>
+              </DataViewPickerProvider>
 
               {didMount && <AttackDiscoveryTour />}
             </KibanaPageTemplate.Section>

x-pack/plugins/security_solution/public/common/mock/global_state.ts

@@ -47,6 +47,7 @@ import { initialGroupingState } from '../store/grouping/reducer';
 import type { SourcererState } from '../../sourcerer/store';
 import { EMPTY_RESOLVER } from '../../resolver/store/helpers';
 import { getMockDiscoverInTimelineState } from './mock_discover_state';
+import { initialState as dataViewPickerInitialState } from '../../sourcerer/experimental/redux/reducer';
 
 const mockFieldMap: DataViewSpec['fields'] = Object.fromEntries(
   mockIndexFields.map((field) => [field.name, field])
@@ -501,6 +502,7 @@ export const mockGlobalState: State = {
    */
   management: mockManagementState as ManagementState,
   discover: getMockDiscoverInTimelineState(),
+  dataViewPicker: dataViewPickerInitialState,
   notes: {
     ids: ['1'],
     entities: {

x-pack/plugins/security_solution/public/common/store/reducer.test.tsx

@@ -13,6 +13,7 @@ import { useSourcererDataView } from '../../sourcerer/containers';
 import { renderHook } from '@testing-library/react-hooks';
 import { initialGroupingState } from './grouping/reducer';
 import { initialAnalyzerState } from '../../resolver/store/helpers';
+import { initialState as dataViewPickerInitialState } from '../../sourcerer/experimental/redux/reducer';
 import { initialNotesState } from '../../notes/store/notes.slice';
 
 jest.mock('../hooks/use_selector');
@@ -71,6 +72,7 @@ describe('createInitialState', () => {
       {
         analyzer: initialAnalyzerState,
       },
+      dataViewPickerInitialState,
       initialNotesState
     );
 
@@ -110,7 +112,7 @@ describe('createInitialState', () => {
         {
           analyzer: initialAnalyzerState,
         },
-
+        dataViewPickerInitialState,
         initialNotesState
       );
       const { result } = renderHook(() => useSourcererDataView(), {

x-pack/plugins/security_solution/public/common/store/reducer.ts

@@ -35,6 +35,10 @@ import type { GroupState } from './grouping/types';
 import { analyzerReducer } from '../../resolver/store/reducer';
 import { securitySolutionDiscoverReducer } from './discover/reducer';
 import type { AnalyzerState } from '../../resolver/types';
+import {
+  type DataviewPickerState,
+  reducer as dataviewPickerReducer,
+} from '../../sourcerer/experimental/redux/reducer';
 import type { NotesState } from '../../notes/store/notes.slice';
 import { notesReducer } from '../../notes/store/notes.slice';
 
@@ -69,6 +73,7 @@ export const createInitialState = (
   dataTableState: DataTableState,
   groupsState: GroupState,
   analyzerState: AnalyzerState,
+  dataviewPickerState: DataviewPickerState,
   notesState: NotesState
 ): State => {
   const initialPatterns = {
@@ -131,6 +136,7 @@ export const createInitialState = (
       internal: undefined,
       savedSearch: undefined,
     },
+    dataViewPicker: dataviewPickerState,
     notes: notesState,
   };
 
@@ -150,6 +156,7 @@ export const createReducer: (
     sourcerer: sourcererReducer,
     globalUrlParam: globalUrlParamReducer,
     dataTable: dataTableReducer,
+    dataViewPicker: dataviewPickerReducer,
     groups: groupsReducer,
     analyzer: analyzerReducer,
     discover: securitySolutionDiscoverReducer,

x-pack/plugins/security_solution/public/common/store/store.ts

@@ -55,6 +55,11 @@ import { dataAccessLayerFactory } from '../../resolver/data_access_layer/factory
 import { sourcererActions } from '../../sourcerer/store';
 import { createMiddlewares } from './middlewares';
 import { addNewTimeline } from '../../timelines/store/helpers';
+import {
+  reducer as dataViewPickerReducer,
+  initialState as dataViewPickerState,
+} from '../../sourcerer/experimental/redux/reducer';
+import { listenerMiddleware } from '../../sourcerer/experimental/redux/listeners';
 import { initialNotesState } from '../../notes/store/notes.slice';
 
 let store: Store<State, Action> | null = null;
@@ -171,6 +176,7 @@ export const createStoreFactory = async (
     dataTableInitialState,
     groupsInitialState,
     analyzerInitialState,
+    dataViewPickerState,
     initialNotesState
   );
 
@@ -178,12 +184,14 @@ export const createStoreFactory = async (
     ...subPlugins.explore.store.reducer,
     timeline: timelineReducer,
     ...subPlugins.management.store.reducer,
+    dataViewPicker: dataViewPickerReducer,
   };
 
   return createStore(initialState, rootReducer, coreStart, storage, [
     ...(subPlugins.management.store.middleware ?? []),
     ...(subPlugins.explore.store.middleware ?? []),
     ...[resolverMiddlewareFactory(dataAccessLayerFactory(coreStart)) ?? []],
+    listenerMiddleware.middleware,
   ]);
 };
 

x-pack/plugins/security_solution/public/common/store/types.ts

@@ -25,6 +25,7 @@ import type { GlobalUrlParam } from './global_url_param';
 import type { GroupState } from './grouping/types';
 import type { SecuritySolutionDiscoverState } from './discover/model';
 import type { AnalyzerState } from '../../resolver/types';
+import { type DataviewPickerState } from '../../sourcerer/experimental/redux/reducer';
 import type { NotesState } from '../../notes/store/notes.slice';
 
 export type State = HostsPluginState &
@@ -38,6 +39,7 @@ export type State = HostsPluginState &
     sourcerer: SourcererState;
     globalUrlParam: GlobalUrlParam;
     discover: SecuritySolutionDiscoverState;
+    dataViewPicker: DataviewPickerState;
   } & DataTableState &
   GroupState &
   AnalyzerState & { notes: NotesState };

x-pack/plugins/security_solution/public/sourcerer/experimental/components/dataview_picker/index.test.tsx

@@ -0,0 +1,94 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React from 'react';
+import { render, screen, fireEvent } from '@testing-library/react';
+import '@testing-library/jest-dom';
+import { useKibana } from '../../../../common/lib/kibana/kibana_react';
+import { useDispatch, useSelector } from 'react-redux';
+import { selectDataView } from '../../redux/actions';
+import { DataViewPicker } from '.';
+
+// Mock the required hooks and dependencies
+jest.mock('../../../../common/lib/kibana/kibana_react', () => ({
+  useKibana: jest.fn(),
+}));
+
+jest.mock('react-redux', () => ({
+  useDispatch: jest.fn(),
+  useSelector: jest.fn(),
+}));
+
+jest.mock('../../redux/actions', () => ({
+  selectDataView: jest.fn(),
+}));
+
+jest.mock('@kbn/unified-search-plugin/public', () => ({
+  DataViewPicker: jest.fn((props) => (
+    <div>
+      <div>{props.trigger.label}</div>
+      <button
+        type="button"
+        onClick={() => props.onChangeDataView('new-id')}
+      >{`Change DataView`}</button>
+      <button type="button" onClick={props.onAddField}>
+        {`Add Field`}
+      </button>
+      <button type="button" onClick={props.onDataViewCreated}>
+        {`Create New DataView`}
+      </button>
+    </div>
+  )),
+}));
+
+describe('DataViewPicker', () => {
+  const mockDispatch = jest.fn();
+  const mockDataViewEditor = {
+    openEditor: jest.fn(),
+  };
+  const mockDataViewFieldEditor = {
+    openEditor: jest.fn(),
+  };
+  const mockData = {
+    dataViews: {
+      get: jest.fn().mockResolvedValue({}),
+    },
+  };
+
+  beforeEach(() => {
+    (useDispatch as jest.Mock).mockReturnValue(mockDispatch);
+    (useKibana as jest.Mock).mockReturnValue({
+      services: {
+        dataViewEditor: mockDataViewEditor,
+        data: mockData,
+        dataViewFieldEditor: mockDataViewFieldEditor,
+      },
+    });
+    (useSelector as jest.Mock).mockReturnValue({ dataViewId: 'test-id' });
+  });
+
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+
+  test('renders the DataviewPicker component', () => {
+    render(<DataViewPicker />);
+    expect(screen.getByText('Dataview')).toBeInTheDocument();
+  });
+
+  test('calls dispatch on data view change', () => {
+    render(<DataViewPicker />);
+    fireEvent.click(screen.getByText('Change DataView'));
+    expect(mockDispatch).toHaveBeenCalledWith(selectDataView('new-id'));
+  });
+
+  test('opens data view editor when creating a new data view', () => {
+    render(<DataViewPicker />);
+    fireEvent.click(screen.getByText('Create New DataView'));
+    expect(mockDataViewEditor.openEditor).toHaveBeenCalled();
+  });
+});

x-pack/plugins/security_solution/public/sourcerer/experimental/components/dataview_picker/index.tsx

@@ -0,0 +1,99 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { DataViewPicker as USDataViewPicker } from '@kbn/unified-search-plugin/public';
+import React, { useCallback, useRef, useMemo, memo } from 'react';
+import { useDispatch, useSelector } from 'react-redux';
+
+import { useKibana } from '../../../../common/lib/kibana/kibana_react';
+import { DEFAULT_SECURITY_SOLUTION_DATA_VIEW_ID } from '../../constants';
+import { selectDataView } from '../../redux/actions';
+import { sourcererAdapterSelector } from '../../redux/selectors';
+
+const TRIGGER_CONFIG = {
+  label: 'Dataview',
+  color: 'danger',
+  title: 'Experimental data view picker',
+  iconType: 'beaker',
+} as const;
+
+export const DataViewPicker = memo(() => {
+  const dispatch = useDispatch();
+
+  const {
+    services: { dataViewEditor, data, dataViewFieldEditor },
+  } = useKibana();
+
+  const closeDataViewEditor = useRef<() => void | undefined>();
+  const closeFieldEditor = useRef<() => void | undefined>();
+
+  // TODO: should this be implemented like that? If yes, we need to source dataView somehow or implement the same thing based on the existing state value.
+  // const canEditDataView =
+  // Boolean(dataViewEditor?.userPermissions.editDataView()) || !dataView.isPersisted();
+  const canEditDataView = true;
+
+  const { dataViewId } = useSelector(sourcererAdapterSelector);
+
+  const createNewDataView = useCallback(() => {
+    closeDataViewEditor.current = dataViewEditor.openEditor({
+      // eslint-disable-next-line no-console
+      onSave: () => console.log('new data view saved'),
+      allowAdHocDataView: true,
+    });
+  }, [dataViewEditor]);
+
+  const onFieldEdited = useCallback(() => {}, []);
+
+  const editField = useMemo(() => {
+    if (!canEditDataView) {
+      return;
+    }
+    return async (fieldName?: string, _uiAction: 'edit' | 'add' = 'edit') => {
+      if (!dataViewId) {
+        return;
+      }
+
+      const dataViewInstance = await data.dataViews.get(dataViewId);
+      closeFieldEditor.current = dataViewFieldEditor.openEditor({
+        ctx: {
+          dataView: dataViewInstance,
+        },
+        fieldName,
+        onSave: async () => {
+          onFieldEdited();
+        },
+      });
+    };
+  }, [canEditDataView, dataViewId, data.dataViews, dataViewFieldEditor, onFieldEdited]);
+
+  const addField = useMemo(
+    () => (canEditDataView && editField ? () => editField(undefined, 'add') : undefined),
+    [editField, canEditDataView]
+  );
+
+  const handleChangeDataView = useCallback(
+    (id: string) => {
+      dispatch(selectDataView(id));
+    },
+    [dispatch]
+  );
+
+  const handleEditDataView = useCallback(() => {}, []);
+
+  return (
+    <USDataViewPicker
+      currentDataViewId={dataViewId || DEFAULT_SECURITY_SOLUTION_DATA_VIEW_ID}
+      trigger={TRIGGER_CONFIG}
+      onChangeDataView={handleChangeDataView}
+      onEditDataView={handleEditDataView}
+      onAddField={addField}
+      onDataViewCreated={createNewDataView}
+    />
+  );
+});
+
+DataViewPicker.displayName = 'DataviewPicker';

x-pack/plugins/security_solution/public/sourcerer/experimental/components/dataview_picker/readme.md

@@ -0,0 +1,3 @@
+# Dataview Picker
+
+A replacement for the Sourcerer component, based on the Discover implementation.

x-pack/plugins/security_solution/public/sourcerer/experimental/constants.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const DEFAULT_SECURITY_SOLUTION_DATA_VIEW_ID = 'security-solution-default';

x-pack/plugins/security_solution/public/sourcerer/experimental/containers/dataview_picker_provider.test.tsx

@@ -0,0 +1,81 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React from 'react';
+import { render } from '@testing-library/react';
+import '@testing-library/jest-dom';
+import { useDispatch } from 'react-redux';
+import { useKibana } from '../../../common/lib/kibana';
+import { DataViewPickerProvider } from './dataview_picker_provider';
+import {
+  startAppListening,
+  listenerMiddleware,
+  createChangeDataviewListener,
+  createInitDataviewListener,
+} from '../redux/listeners';
+import { init } from '../redux/actions';
+import { DEFAULT_SECURITY_SOLUTION_DATA_VIEW_ID } from '../constants';
+import type { DataViewsServicePublic } from '@kbn/data-views-plugin/public';
+
+jest.mock('../../../common/lib/kibana', () => ({
+  useKibana: jest.fn(),
+}));
+
+jest.mock('react-redux', () => ({
+  useDispatch: jest.fn(),
+}));
+
+jest.mock('../redux/listeners', () => ({
+  listenerMiddleware: {
+    clearListeners: jest.fn(),
+  },
+  startAppListening: jest.fn(),
+  createChangeDataviewListener: jest.fn(),
+  createInitDataviewListener: jest.fn(),
+}));
+
+describe('DataviewPickerProvider', () => {
+  const mockDispatch = jest.fn();
+  const mockServices = {
+    dataViews: {} as unknown as DataViewsServicePublic,
+  };
+
+  beforeEach(() => {
+    (useDispatch as jest.Mock).mockReturnValue(mockDispatch);
+    (useKibana as jest.Mock).mockReturnValue({ services: mockServices });
+  });
+
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+
+  test('starts listeners and dispatches init action on mount', () => {
+    render(
+      <DataViewPickerProvider>
+        <div>{`Test Child`}</div>
+      </DataViewPickerProvider>
+    );
+
+    expect(startAppListening).toHaveBeenCalledWith(createInitDataviewListener({}));
+    expect(startAppListening).toHaveBeenCalledWith(
+      createChangeDataviewListener({ dataViewsService: mockServices.dataViews })
+    );
+    expect(mockDispatch).toHaveBeenCalledWith(init(DEFAULT_SECURITY_SOLUTION_DATA_VIEW_ID));
+  });
+
+  test('clears listeners on unmount', () => {
+    const { unmount } = render(
+      <DataViewPickerProvider>
+        <div>{`Test Child`}</div>
+      </DataViewPickerProvider>
+    );
+
+    unmount();
+
+    expect(listenerMiddleware.clearListeners).toHaveBeenCalled();
+  });
+});

x-pack/plugins/security_solution/public/sourcerer/experimental/containers/dataview_picker_provider.tsx

@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React, { memo, useEffect, type FC, type PropsWithChildren } from 'react';
+import { useDispatch } from 'react-redux';
+
+import { useKibana } from '../../../common/lib/kibana';
+import {
+  createChangeDataviewListener,
+  createInitDataviewListener,
+  listenerMiddleware,
+  startAppListening,
+} from '../redux/listeners';
+import { init } from '../redux/actions';
+import { DEFAULT_SECURITY_SOLUTION_DATA_VIEW_ID } from '../constants';
+
+// NOTE: this can be spawned multiple times, eq. when you need something like a separate data view picker for a subsection of the app -
+// for example, in the timeline.
+export const DataViewPickerProvider: FC<PropsWithChildren<{}>> = memo(({ children }) => {
+  const { services } = useKibana();
+
+  const dispatch = useDispatch();
+
+  useEffect(() => {
+    // NOTE: the goal here is to move all side effects and business logic to Redux,
+    // so that we only do presentation layer things on React side - for performance reasons and
+    // to make the state easier to predict.
+    // see: https://redux-toolkit.js.org/api/createListenerMiddleware#overview
+    startAppListening(createInitDataviewListener({}));
+    startAppListening(createChangeDataviewListener({ dataViewsService: services.dataViews }));
+
+    // NOTE: this can be dispatched at any point, with any data view id
+    dispatch(init(DEFAULT_SECURITY_SOLUTION_DATA_VIEW_ID));
+
+    // NOTE: Clear existing listeners when services change for some reason (they should not)
+    return () => listenerMiddleware.clearListeners();
+  }, [services, dispatch]);
+
+  return <>{children}</>;
+});
+
+DataViewPickerProvider.displayName = 'DataviewPickerProvider';

x-pack/plugins/security_solution/public/sourcerer/experimental/is_enabled.ts

@@ -11,6 +11,5 @@
  * - display the experimental component instead of the stable one
  * - use experimental data views hook instead of the stable one
  */
-export const IS_EXPERIMENTAL_SOURCERER_ENABLED = !!window.localStorage.getItem(
-  'EXPERIMENTAL_SOURCERER_ENABLED'
-);
+export const isExperimentalSourcererEnabled = () =>
+  !!window.localStorage.getItem('EXPERIMENTAL_SOURCERER_ENABLED');

x-pack/plugins/security_solution/public/sourcerer/experimental/readme.md

@@ -15,4 +15,7 @@ using the same Kibana instance deployed locally (for now).
 
 ## Architecture
 
-TODO
+- Redux based 
+- Limited use of useEffect or stateful hooks - in favor of thunks and redux middleware (supporting request cancellation and caching)
+- Allows multiple instances of the picker - just wrap the subsection of the app with its own DataviewPickerProvider
+- Data exposed back to Security Solution is memoized with `reselect` for performance

x-pack/plugins/security_solution/public/sourcerer/experimental/redux/actions.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { type DataViewSpec } from '@kbn/data-views-plugin/common';
+import { createAction } from '@reduxjs/toolkit';
+
+type DataViewId = string;
+
+export const init = createAction<DataViewId>('init');
+export const selectDataView = createAction<DataViewId>('changeDataView');
+export const setDataViewData = createAction<DataViewSpec>('setDataView');
+export const setPatternList = createAction<string[]>('setPatternList');

x-pack/plugins/security_solution/public/sourcerer/experimental/redux/listeners.test.ts

@@ -0,0 +1,101 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+  init,
+  selectDataView,
+  setDataViewData as setDataViewSpec,
+  setPatternList,
+} from './actions';
+import { createInitDataviewListener, createChangeDataviewListener } from './listeners';
+import { isExperimentalSourcererEnabled } from '../is_enabled';
+import type { DataViewsServicePublic } from '@kbn/data-views-plugin/public';
+import { type ListenerEffectAPI } from '@reduxjs/toolkit';
+import type { AppDispatch } from './listeners';
+import { type State } from '../../../common/store/types';
+
+jest.mock('../is_enabled', () => ({
+  isExperimentalSourcererEnabled: jest.fn().mockReturnValue(true),
+}));
+
+type ListenerApi = ListenerEffectAPI<State, AppDispatch>;
+
+describe('Listeners', () => {
+  describe('createInitDataviewListener', () => {
+    let listenerOptions: ReturnType<typeof createInitDataviewListener>;
+    let listenerApi: ListenerApi;
+
+    beforeEach(() => {
+      listenerOptions = createInitDataviewListener({});
+      listenerApi = {
+        dispatch: jest.fn(),
+        getState: jest.fn(() => ({ dataViewPicker: { state: 'pristine' } })),
+      } as unknown as ListenerApi;
+    });
+
+    afterEach(() => {
+      jest.clearAllMocks();
+    });
+
+    test('does not dispatch if experimental feature is disabled', async () => {
+      jest.mocked(isExperimentalSourcererEnabled).mockReturnValue(false);
+
+      await listenerOptions.effect(init('test-view'), listenerApi);
+      expect(listenerApi.dispatch).not.toHaveBeenCalled();
+    });
+
+    test('does not dispatch if state is not pristine', async () => {
+      jest.mocked(isExperimentalSourcererEnabled).mockReturnValue(true);
+      listenerApi.getState = jest.fn(() => ({
+        dataViewPicker: { state: 'not_pristine' },
+      })) as unknown as ListenerApi['getState'];
+
+      await listenerOptions.effect(init('test-view'), listenerApi);
+      expect(listenerApi.dispatch).not.toHaveBeenCalled();
+    });
+
+    test('dispatches selectDataView action if state is pristine and experimental feature is enabled', async () => {
+      jest.mocked(isExperimentalSourcererEnabled).mockReturnValue(true);
+      await listenerOptions.effect(init('test-id'), listenerApi);
+      expect(listenerApi.dispatch).toHaveBeenCalledWith(selectDataView('test-id'));
+    });
+  });
+
+  describe('createChangeDataviewListener', () => {
+    let listenerOptions: ReturnType<typeof createChangeDataviewListener>;
+    let listenerApi: ListenerApi;
+    let dataViewsServiceMock: DataViewsServicePublic;
+
+    beforeEach(() => {
+      dataViewsServiceMock = {
+        get: jest.fn(async () => ({
+          toSpec: jest.fn(() => ({ id: 'test_spec' })),
+          getIndexPattern: jest.fn(() => 'index_pattern'),
+        })),
+        getExistingIndices: jest.fn(async () => ['pattern1', 'pattern2']),
+      } as unknown as DataViewsServicePublic;
+
+      listenerOptions = createChangeDataviewListener({ dataViewsService: dataViewsServiceMock });
+      listenerApi = {
+        dispatch: jest.fn(),
+      } as unknown as ListenerApi;
+    });
+
+    afterEach(() => {
+      jest.clearAllMocks();
+    });
+
+    test('fetches data view and dispatches setDataViewSpec and setPatternList actions', async () => {
+      await listenerOptions.effect(selectDataView('test_id'), listenerApi);
+
+      expect(dataViewsServiceMock.get).toHaveBeenCalledWith('test_id', true, false);
+      expect(listenerApi.dispatch).toHaveBeenCalledWith(setDataViewSpec({ id: 'test_spec' }));
+      expect(dataViewsServiceMock.getExistingIndices).toHaveBeenCalledWith(['index_pattern']);
+      expect(listenerApi.dispatch).toHaveBeenCalledWith(setPatternList(['pattern1', 'pattern2']));
+    });
+  });
+});

x-pack/plugins/security_solution/public/sourcerer/experimental/redux/listeners.ts

@@ -0,0 +1,102 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { DataViewsServicePublic } from '@kbn/data-views-plugin/public';
+import {
+  createListenerMiddleware,
+  type ActionCreator,
+  type ListenerEffectAPI,
+} from '@reduxjs/toolkit';
+import type { ListenerPredicate } from '@reduxjs/toolkit/dist/listenerMiddleware/types';
+import type { Action, Store } from 'redux';
+
+import { ensurePatternFormat } from '../../../../common/utils/sourcerer';
+import { isExperimentalSourcererEnabled } from '../is_enabled';
+import {
+  init,
+  selectDataView,
+  setDataViewData as setDataViewSpec,
+  setPatternList,
+} from './actions';
+import { type State } from '../../../common/store/types';
+
+export type AppDispatch = Store<State, Action>['dispatch'];
+
+export type DatapickerActions = ReturnType<typeof selectDataView>;
+
+// NOTE: types below exist because we are using redux-toolkit version lower than 2.x
+// in v2, there are TS helpers that make it easy to setup overrides that are necessary here.
+export interface ListenerOptions {
+  // Match with a function accepting action and state. This is broken in v1.x,
+  // the predicate is always required
+  predicate?: ListenerPredicate<DatapickerActions, State>;
+  // Match action by type
+  type?: string;
+  // Exact action type match based on the RTK action creator
+  actionCreator?: ActionCreator<DatapickerActions>;
+  // An effect to call
+  effect: (action: DatapickerActions, api: ListenerEffectAPI<State, AppDispatch>) => Promise<void>;
+}
+
+/**
+ * This is the proposed way of handling side effects within sourcerer code. We will no longer rely on useEffect for doing things like
+ * enriching the store with data fetched asynchronously in response to user doing something.
+ * Thunks are also considered for simpler flows but this has the advantage of cancellation support through `listnerApi` below.
+ */
+
+export type ListenerCreator<TDependencies> = (
+  // Only specify a subset of required services here, so that it is easier to mock and therefore test the listener
+  dependencies: TDependencies
+) => ListenerOptions;
+
+// NOTE: this should only be executed once in the application lifecycle, to LAZILY setup the component data
+export const createInitDataviewListener: ListenerCreator<{}> = (): ListenerOptions => {
+  return {
+    actionCreator: init,
+    effect: async (action, listenerApi) => {
+      // WARN:  Skip the init call if the experimental implementation is disabled
+      if (!isExperimentalSourcererEnabled()) {
+        return;
+      }
+      // NOTE: We should only run this once, when particular sourcerer instance is in pristine state (not touched by the user)
+      if (listenerApi.getState().dataViewPicker.state !== 'pristine') {
+        return;
+      }
+
+      // NOTE: dispatch the regular change listener
+      listenerApi.dispatch(selectDataView(action.payload));
+    },
+  };
+};
+
+// NOTE: this listener is executed whenever user decides to select dataview from the picker
+export const createChangeDataviewListener: ListenerCreator<{
+  dataViewsService: DataViewsServicePublic;
+}> = ({ dataViewsService }): ListenerOptions => {
+  return {
+    actionCreator: selectDataView,
+    effect: async (action, listenerApi) => {
+      const dataViewId = action.payload;
+      const refreshFields = false;
+
+      const dataView = await dataViewsService.get(dataViewId, true, refreshFields);
+      const dataViewData = dataView.toSpec();
+      listenerApi.dispatch(setDataViewSpec(dataViewData));
+
+      const defaultPatternsList = ensurePatternFormat(dataView.getIndexPattern().split(','));
+      const patternList = await dataViewsService.getExistingIndices(defaultPatternsList);
+      listenerApi.dispatch(setPatternList(patternList));
+    },
+  };
+};
+
+export const listenerMiddleware = createListenerMiddleware();
+
+// NOTE: register side effect listeners
+export const startAppListening = listenerMiddleware.startListening as unknown as (
+  options: ListenerOptions
+) => void;

x-pack/plugins/security_solution/public/sourcerer/experimental/redux/reducer.ts

@@ -0,0 +1,55 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { DataViewSpec } from '@kbn/data-views-plugin/common';
+import { createReducer } from '@reduxjs/toolkit';
+
+import { DEFAULT_SECURITY_SOLUTION_DATA_VIEW_ID } from '../constants';
+
+import { selectDataView, setDataViewData, setPatternList } from './actions';
+
+export interface SelectedDataViewState {
+  dataView: DataViewSpec;
+  patternList: string[];
+  /**
+   * There are several states the picker can be in internally:
+   * - pristine - not initialized yet
+   * - loading
+   * - error - some kind of a problem during data init
+   * - ready - ready to provide index information to the client
+   */
+  state: 'pristine' | 'loading' | 'error' | 'ready';
+}
+
+export const initialDataView: DataViewSpec = {
+  id: DEFAULT_SECURITY_SOLUTION_DATA_VIEW_ID,
+  title: '',
+  fields: {},
+};
+
+export const initialState: SelectedDataViewState = {
+  dataView: initialDataView,
+  state: 'pristine',
+  patternList: [],
+};
+
+export const reducer = createReducer(initialState, (builder) => {
+  builder.addCase(selectDataView, (state) => {
+    state.state = 'loading';
+  });
+
+  builder.addCase(setDataViewData, (state, action) => {
+    state.dataView = action.payload;
+  });
+
+  builder.addCase(setPatternList, (state, action) => {
+    state.patternList = action.payload;
+    state.state = 'ready';
+  });
+});
+
+export type DataviewPickerState = ReturnType<typeof reducer>;

x-pack/plugins/security_solution/public/sourcerer/experimental/redux/selectors.ts

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { createSelector } from '@reduxjs/toolkit';
+import type { SelectedDataView } from '../../store/model';
+import { type State } from '../../../common/store/types';
+
+/**
+ * Compatibility layer / adapter for legacy selector consumers.
+ * It is used in useSecuritySolutionDataView hook as alternative data source (behind a flag)
+ */
+export const sourcererAdapterSelector = createSelector(
+  [(state: State) => state.dataViewPicker],
+  (dataViewPicker): SelectedDataView => {
+    return {
+      loading: dataViewPicker.state === 'loading',
+      dataViewId: dataViewPicker.dataView.id || '',
+      patternList: dataViewPicker.patternList,
+      indicesExist: true,
+      browserFields: {},
+      activePatterns: dataViewPicker.patternList,
+      runtimeMappings: {},
+      selectedPatterns: dataViewPicker.patternList,
+      indexPattern: { fields: [], title: dataViewPicker.dataView.title || '' },
+      sourcererDataView: {},
+    };
+  }
+);

x-pack/plugins/security_solution/public/sourcerer/experimental/use_unstable_security_solution_data_view.ts

@@ -5,12 +5,16 @@
  * 2.0.
  */
 
+import { useMemo } from 'react';
+import { useSelector } from 'react-redux';
+
 import { type SourcererScopeName, type SelectedDataView } from '../store/model';
 
-import { IS_EXPERIMENTAL_SOURCERER_ENABLED } from './is_enabled';
+import { isExperimentalSourcererEnabled } from './is_enabled';
+import { sourcererAdapterSelector } from './redux/selectors';
 
 /**
- * FOR INTERNAL USE ONLY
+ * WARN: FOR INTERNAL USE ONLY
  * This hook provides data for experimental Sourcerer replacement in Security Solution.
  * Do not use in client code as the API will change frequently.
  * It will be extended in the future, covering more and more functionality from the current sourcerer.
@@ -19,6 +23,16 @@ export const useUnstableSecuritySolutionDataView = (
   _scopeId: SourcererScopeName,
   fallbackDataView: SelectedDataView
 ): SelectedDataView => {
-  // TODO: extend the fallback state with values computed using new logic
-  return IS_EXPERIMENTAL_SOURCERER_ENABLED ? fallbackDataView : fallbackDataView;
+  const dataView: SelectedDataView = useSelector(sourcererAdapterSelector);
+
+  const dataViewWithFallbacks: SelectedDataView = useMemo(() => {
+    return {
+      ...dataView,
+      // NOTE: temporary values sourced from the fallback. Will be replaced in the near future.
+      browserFields: fallbackDataView.browserFields,
+      sourcererDataView: fallbackDataView.sourcererDataView,
+    };
+  }, [dataView, fallbackDataView.browserFields, fallbackDataView.sourcererDataView]);
+
+  return isExperimentalSourcererEnabled() ? dataViewWithFallbacks : fallbackDataView;
 };

x-pack/plugins/security_solution/public/timelines/components/timeline/query_bar/eql/index.tsx

@@ -83,14 +83,17 @@ export const EqlQueryBarTimeline = memo(({ timelineId }: { timelineId: string })
     selectedPatterns,
   } = useSourcererDataView(SourcererScopeName.timeline);
 
-  const initialState = {
-    ...defaultValues,
-    index: selectedPatterns.sort(),
-    eqlQueryBar: {
-      ...defaultValues.eqlQueryBar,
-      query: { query: optionsSelected.query ?? '', language: 'eql' },
-    },
-  };
+  const initialState = useMemo(
+    () => ({
+      ...defaultValues,
+      index: [...selectedPatterns].sort(),
+      eqlQueryBar: {
+        ...defaultValues.eqlQueryBar,
+        query: { query: optionsSelected.query ?? '', language: 'eql' },
+      },
+    }),
+    [optionsSelected.query, selectedPatterns]
+  );
 
   const { form } = useForm<TimelineEqlQueryBar>({
     defaultValue: initialState,
@@ -144,7 +147,7 @@ export const EqlQueryBarTimeline = memo(({ timelineId }: { timelineId: string })
 
   useEffect(() => {
     const { index: indexField } = getFields();
-    const newIndexValue = selectedPatterns.sort();
+    const newIndexValue = [...selectedPatterns].sort();
     const indexFieldValue = (indexField.value as string[]).sort();
     if (!isEqual(indexFieldValue, newIndexValue)) {
       indexField.setValue(newIndexValue);

x-pack/plugins/security_solution/public/timelines/components/timeline/search_or_filter/search_or_filter.tsx

@@ -11,6 +11,8 @@ import styled from 'styled-components';
 import type { Filter } from '@kbn/es-query';
 
 import type { FilterManager } from '@kbn/data-plugin/public';
+import { DataViewPicker } from '../../../../sourcerer/experimental/components/dataview_picker';
+import { isExperimentalSourcererEnabled } from '../../../../sourcerer/experimental/is_enabled';
 import { TimelineType } from '../../../../../common/api/timeline';
 import { InputsModelId } from '../../../../common/store/inputs/constants';
 import type { KqlMode } from '../../../store/model';
@@ -103,6 +105,12 @@ export const SearchOrFilter = React.memo<Props>(
       [isDataProviderEmpty, isDataProviderVisible]
     );
 
+    const dataviewPicker = isExperimentalSourcererEnabled() ? (
+      <DataViewPicker />
+    ) : (
+      <Sourcerer scope={SourcererScopeName.timeline} />
+    );
+
     return (
       <>
         <SearchOrFilterContainer>
@@ -113,7 +121,7 @@ export const SearchOrFilter = React.memo<Props>(
             responsive={false}
           >
             <EuiFlexItem grow={false} id={TIMELINE_TOUR_CONFIG_ANCHORS.DATA_VIEW}>
-              <Sourcerer scope={SourcererScopeName.timeline} />
+              {dataviewPicker}
             </EuiFlexItem>
             <EuiFlexItem data-test-subj="timeline-search-or-filter-search-container" grow={1}>
               <QueryBarTimeline