[Security Solution]: on ESC key close the timeline flyout instead of the timeline modal (#224352)

## Summary

Fixes: https://github.com/elastic/kibana/issues/190761

I've added a `handleKeyDown` to capture the `ESC` key press behavior.

1. First `ESC` finds any .euiFlyout and close it with`closeFlyout()`
from `useExpandableFlyoutApi()`, then stops.

2. Next `ESC` (when no flyout) clicks `openToggleRef.current`, toggling
the timeline modal closed.


https://github.com/user-attachments/assets/0e42f9e9-2694-429d-8a5a-df86aa671809

### Checklist

Check the PR satisfies following conditions. 

Reviewers should verify this PR satisfies this list as well.

- [ ] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

---------

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

x-pack/solutions/security/plugins/security_solution/public/flyout/document_details/shared/hooks/use_which_flyout.test.tsx

@@ -7,8 +7,13 @@
 
 import type { RenderHookResult } from '@testing-library/react';
 import { renderHook } from '@testing-library/react';
-import { useWhichFlyout } from './use_which_flyout';
+import {
+  isSecuritySolutionFlyoutOpen,
+  isTimelineFlyoutOpen,
+  useWhichFlyout,
+} from './use_which_flyout';
 import { Flyouts } from '../constants/flyouts';
+import { URL_PARAM_KEY } from '../../../../common/hooks/use_url_state';
 
 describe('useWhichFlyout', () => {
   let hookResult: RenderHookResult<string | null, unknown>;
@@ -93,6 +98,62 @@ describe('useWhichFlyout', () => {
     });
   });
 
+  describe('isSecuritySolutionFlyoutOpen', () => {
+    it('returns false when no flyout param is present', () => {
+      const params = new URLSearchParams();
+      expect(isSecuritySolutionFlyoutOpen(params)).toBe(false);
+    });
+
+    it('returns false when flyout param value is "()"', () => {
+      const params = new URLSearchParams();
+      params.set(URL_PARAM_KEY.flyout, '()');
+      expect(isSecuritySolutionFlyoutOpen(params)).toBe(false);
+    });
+
+    it('returns false when flyout param value is "(preview:!())"', () => {
+      const params = new URLSearchParams();
+      params.set(URL_PARAM_KEY.flyout, '(preview:!())');
+      expect(isSecuritySolutionFlyoutOpen(params)).toBe(false);
+    });
+
+    it('returns true when flyout param has any other non-empty value', () => {
+      const params = new URLSearchParams();
+      params.set(URL_PARAM_KEY.flyout, '(id:123)');
+      expect(isSecuritySolutionFlyoutOpen(params)).toBe(true);
+
+      params.set(URL_PARAM_KEY.flyout, 'foo');
+      expect(isSecuritySolutionFlyoutOpen(params)).toBe(true);
+    });
+  });
+
+  describe('isTimelineFlyoutOpen', () => {
+    it('returns false when no timelineFlyout param is present', () => {
+      const params = new URLSearchParams();
+      expect(isTimelineFlyoutOpen(params)).toBe(false);
+    });
+
+    it('returns false when timelineFlyout value is "()"', () => {
+      const params = new URLSearchParams();
+      params.set(URL_PARAM_KEY.timelineFlyout, '()');
+      expect(isTimelineFlyoutOpen(params)).toBe(false);
+    });
+
+    it('returns false when timelineFlyout value is "(preview:!())"', () => {
+      const params = new URLSearchParams();
+      params.set(URL_PARAM_KEY.timelineFlyout, '(preview:!())');
+      expect(isTimelineFlyoutOpen(params)).toBe(false);
+    });
+
+    it('returns true when timelineFlyout has any other non-empty value', () => {
+      const params = new URLSearchParams();
+      params.set(URL_PARAM_KEY.timelineFlyout, '(preview:!(456))');
+      expect(isTimelineFlyoutOpen(params)).toBe(true);
+
+      params.set(URL_PARAM_KEY.timelineFlyout, 'bar');
+      expect(isTimelineFlyoutOpen(params)).toBe(true);
+    });
+  });
+
   describe('Timeline flyout open', () => {
     it('should return Timeline flyout if flyout and timelineFlyout are in the url', () => {
       Object.defineProperty(window, 'location', {

x-pack/solutions/security/plugins/security_solution/public/flyout/document_details/shared/hooks/use_which_flyout.ts

@@ -8,6 +8,27 @@
 import { Flyouts } from '../constants/flyouts';
 import { URL_PARAM_KEY } from '../../../../common/hooks/use_url_state';
 
+/**
+ * Returns true if the SecuritySolution flyout is open.
+ */
+export const isSecuritySolutionFlyoutOpen = (query: URLSearchParams): boolean => {
+  const queryHasSecuritySolutionFlyout = query.has(URL_PARAM_KEY.flyout);
+  const securitySolutionFlyoutHasValue =
+    query.get(URL_PARAM_KEY.flyout) !== '()' && query.get(URL_PARAM_KEY.flyout) !== '(preview:!())';
+  return queryHasSecuritySolutionFlyout && securitySolutionFlyoutHasValue;
+};
+
+/**
+ * Returns true if the Timeline flyout is open.
+ */
+export const isTimelineFlyoutOpen = (query: URLSearchParams): boolean => {
+  const queryHasTimelineFlyout = query.has(URL_PARAM_KEY.timelineFlyout);
+  const timelineFlyoutHasValue =
+    query.get(URL_PARAM_KEY.timelineFlyout) !== '()' &&
+    query.get(URL_PARAM_KEY.timelineFlyout) !== '(preview:!())';
+  return queryHasTimelineFlyout && timelineFlyoutHasValue;
+};
+
 /**
  * Hook that returns which flyout is the user currently interacting with.
  * If the url contains timelineFlyout parameter and its value is not empty, we know the timeline flyout is rendered.
@@ -16,27 +37,18 @@ import { URL_PARAM_KEY } from '../../../../common/hooks/use_url_state';
 export const useWhichFlyout = (): string | null => {
   const query = new URLSearchParams(window.location.search);
 
-  const queryHasSecuritySolutionFlyout = query.has(URL_PARAM_KEY.flyout);
-  const securitySolutionFlyoutHasValue =
-    query.get(URL_PARAM_KEY.flyout) !== '()' && query.get(URL_PARAM_KEY.flyout) !== '(preview:!())';
-  const isSecuritySolutionFlyoutOpen =
-    queryHasSecuritySolutionFlyout && securitySolutionFlyoutHasValue;
+  const securitySolutionFlyoutOpen = isSecuritySolutionFlyoutOpen(query);
+  const timelineFlyoutOpen = isTimelineFlyoutOpen(query);
 
-  const queryHasTimelineFlyout = query.has(URL_PARAM_KEY.timelineFlyout);
-  const timelineFlyoutHasValue =
-    query.get(URL_PARAM_KEY.timelineFlyout) !== '()' &&
-    query.get(URL_PARAM_KEY.timelineFlyout) !== '(preview:!())';
-  const isTimelineFlyoutOpen = queryHasTimelineFlyout && timelineFlyoutHasValue;
-
-  if (isSecuritySolutionFlyoutOpen && isTimelineFlyoutOpen) {
+  if (securitySolutionFlyoutOpen && timelineFlyoutOpen) {
     return Flyouts.timeline;
   }
 
-  if (isSecuritySolutionFlyoutOpen) {
+  if (securitySolutionFlyoutOpen) {
     return Flyouts.securitySolution;
   }
 
-  if (isTimelineFlyoutOpen) {
+  if (timelineFlyoutOpen) {
     return Flyouts.timeline;
   }
 

x-pack/solutions/security/plugins/security_solution/public/timelines/components/modal/index.tsx

@@ -33,6 +33,9 @@ interface TimelineModalProps {
    * If true the timeline modal will be visible
    */
   visible?: boolean;
+  /**
+   * Ref to the element opening/closing the modal
+   */
   openToggleRef: React.MutableRefObject<null | HTMLAnchorElement | HTMLButtonElement>;
 }
 
@@ -52,7 +55,7 @@ export const TimelineModal = React.memo<TimelineModalProps>(
     });
 
     const sibling: HTMLDivElement | null = useMemo(
-      () => (!visible ? ref?.current : null),
+      () => (!visible ? ref.current : null),
       [visible]
     );
 

x-pack/solutions/security/plugins/security_solution/public/timelines/wrapper/index.tsx

@@ -9,6 +9,7 @@ import { EuiFocusTrap, EuiWindowEvent, keys } from '@elastic/eui';
 import React, { useMemo, useCallback, useRef } from 'react';
 import type { AppLeaveHandler } from '@kbn/core/public';
 import { useDispatch } from 'react-redux';
+import { useExpandableFlyoutApi } from '@kbn/expandable-flyout';
 import { TimelineModal } from '../components/modal';
 import type { TimelineId } from '../../../common/types';
 import { useDeepEqualSelector } from '../../common/hooks/use_selector';
@@ -16,6 +17,7 @@ import { TimelineBottomBar } from '../components/bottom_bar';
 import { getTimelineShowStatusByIdSelector } from '../store/selectors';
 import { useTimelineSavePrompt } from '../../common/hooks/timeline/use_timeline_save_prompt';
 import { timelineActions } from '../store';
+import { isTimelineFlyoutOpen } from '../../flyout/document_details/shared/hooks/use_which_flyout';
 
 interface TimelineWrapperProps {
   /**
@@ -42,15 +44,23 @@ export const TimelineWrapper: React.FC<TimelineWrapperProps> = React.memo(
     const handleClose = useCallback(() => {
       dispatch(timelineActions.showTimeline({ id: timelineId, show: false }));
     }, [dispatch, timelineId]);
+    const { closeFlyout } = useExpandableFlyoutApi();
 
-    // pressing the ESC key closes the timeline portal
+    // pressing the ESC key closes the timeline portal unless a flyout is opened on top of it
     const onKeyDown = useCallback(
       (ev: KeyboardEvent) => {
         if (ev.key === keys.ESCAPE) {
+          const query = new URLSearchParams(window.location.search);
+          const timelineFlyoutOpen = isTimelineFlyoutOpen(query);
+
+          if (timelineFlyoutOpen) {
+            closeFlyout();
+            return;
+          }
           handleClose();
         }
       },
-      [handleClose]
+      [closeFlyout, handleClose]
     );
 
     useTimelineSavePrompt(timelineId, onAppLeave);