[data.search] Server-side background session service (#81099) (#83766)

* [Search] Add request context and asScoped pattern

* Update docs

* Unify interface for getting search client

* [WIP] [data.search] Server-side background session service

* Update examples/search_examples/server/my_strategy.ts

Co-authored-by: Anton Dosov <dosantappdev@gmail.com>

* Review feedback

* Fix checks

* Add tapFirst and additional props for session

* Fix CI

* Fix security search

* Fix test

* Fix test for reals

* Add restore method

* Add code to search examples

* Add restore and search using restored ID

* Fix handling of preference and order of params

* Trim & cleanup

* Fix types

* Review feedback

* Add tests and remove handling of username

* Update docs

* Move utils to server

* Review feedback

* More review feedback

* Regenerate docs

* Review feedback

* Doc changes

Co-authored-by: Anton Dosov <dosantappdev@gmail.com>
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

Co-authored-by: Anton Dosov <dosantappdev@gmail.com>
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.isrestore.md

@@ -0,0 +1,13 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [ISearchOptions](./kibana-plugin-plugins-data-public.isearchoptions.md) &gt; [isRestore](./kibana-plugin-plugins-data-public.isearchoptions.isrestore.md)
+
+## ISearchOptions.isRestore property
+
+Whether the session is restored (i.e. search requests should re-use the stored search IDs, rather than starting from scratch)
+
+<b>Signature:</b>
+
+```typescript
+isRestore?: boolean;
+```

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.isstored.md

@@ -0,0 +1,13 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [ISearchOptions](./kibana-plugin-plugins-data-public.isearchoptions.md) &gt; [isStored](./kibana-plugin-plugins-data-public.isearchoptions.isstored.md)
+
+## ISearchOptions.isStored property
+
+Whether the session is already saved (i.e. sent to background)
+
+<b>Signature:</b>
+
+```typescript
+isStored?: boolean;
+```

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.md

@@ -15,6 +15,8 @@ export interface ISearchOptions
 |  Property | Type | Description |
 |  --- | --- | --- |
 |  [abortSignal](./kibana-plugin-plugins-data-public.isearchoptions.abortsignal.md) | <code>AbortSignal</code> | An <code>AbortSignal</code> that allows the caller of <code>search</code> to abort a search request. |
+|  [isRestore](./kibana-plugin-plugins-data-public.isearchoptions.isrestore.md) | <code>boolean</code> | Whether the session is restored (i.e. search requests should re-use the stored search IDs, rather than starting from scratch) |
+|  [isStored](./kibana-plugin-plugins-data-public.isearchoptions.isstored.md) | <code>boolean</code> | Whether the session is already saved (i.e. sent to background) |
 |  [sessionId](./kibana-plugin-plugins-data-public.isearchoptions.sessionid.md) | <code>string</code> | A session ID, grouping multiple search requests into a single session. |
 |  [strategy](./kibana-plugin-plugins-data-public.isearchoptions.strategy.md) | <code>string</code> | Use this option to force using a specific server side search strategy. Leave empty to use the default strategy. |
 

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isessionservice.delete.md

@@ -0,0 +1,13 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [ISessionService](./kibana-plugin-plugins-data-public.isessionservice.md) &gt; [delete](./kibana-plugin-plugins-data-public.isessionservice.delete.md)
+
+## ISessionService.delete property
+
+Deletes a session
+
+<b>Signature:</b>
+
+```typescript
+delete: (sessionId: string) => Promise<void>;
+```

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isessionservice.find.md

@@ -0,0 +1,13 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [ISessionService](./kibana-plugin-plugins-data-public.isessionservice.md) &gt; [find](./kibana-plugin-plugins-data-public.isessionservice.find.md)
+
+## ISessionService.find property
+
+Gets a list of saved sessions
+
+<b>Signature:</b>
+
+```typescript
+find: (options: SearchSessionFindOptions) => Promise<SavedObjectsFindResponse<BackgroundSessionSavedObjectAttributes>>;
+```

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isessionservice.get.md

@@ -0,0 +1,13 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [ISessionService](./kibana-plugin-plugins-data-public.isessionservice.md) &gt; [get](./kibana-plugin-plugins-data-public.isessionservice.get.md)
+
+## ISessionService.get property
+
+Gets a saved session
+
+<b>Signature:</b>
+
+```typescript
+get: (sessionId: string) => Promise<SavedObject<BackgroundSessionSavedObjectAttributes>>;
+```

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isessionservice.isrestore.md

@@ -0,0 +1,13 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [ISessionService](./kibana-plugin-plugins-data-public.isessionservice.md) &gt; [isRestore](./kibana-plugin-plugins-data-public.isessionservice.isrestore.md)
+
+## ISessionService.isRestore property
+
+Whether the active session is restored (i.e. reusing previous search IDs)
+
+<b>Signature:</b>
+
+```typescript
+isRestore: () => boolean;
+```

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isessionservice.isstored.md

@@ -0,0 +1,13 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [ISessionService](./kibana-plugin-plugins-data-public.isessionservice.md) &gt; [isStored](./kibana-plugin-plugins-data-public.isessionservice.isstored.md)
+
+## ISessionService.isStored property
+
+Whether the active session is already saved (i.e. sent to background)
+
+<b>Signature:</b>
+
+```typescript
+isStored: () => boolean;
+```

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isessionservice.md

@@ -15,8 +15,15 @@ export interface ISessionService
 |  Property | Type | Description |
 |  --- | --- | --- |
 |  [clear](./kibana-plugin-plugins-data-public.isessionservice.clear.md) | <code>() =&gt; void</code> | Clears the active session. |
+|  [delete](./kibana-plugin-plugins-data-public.isessionservice.delete.md) | <code>(sessionId: string) =&gt; Promise&lt;void&gt;</code> | Deletes a session |
+|  [find](./kibana-plugin-plugins-data-public.isessionservice.find.md) | <code>(options: SearchSessionFindOptions) =&gt; Promise&lt;SavedObjectsFindResponse&lt;BackgroundSessionSavedObjectAttributes&gt;&gt;</code> | Gets a list of saved sessions |
+|  [get](./kibana-plugin-plugins-data-public.isessionservice.get.md) | <code>(sessionId: string) =&gt; Promise&lt;SavedObject&lt;BackgroundSessionSavedObjectAttributes&gt;&gt;</code> | Gets a saved session |
 |  [getSession$](./kibana-plugin-plugins-data-public.isessionservice.getsession_.md) | <code>() =&gt; Observable&lt;string &#124; undefined&gt;</code> | Returns the observable that emits an update every time the session ID changes |
 |  [getSessionId](./kibana-plugin-plugins-data-public.isessionservice.getsessionid.md) | <code>() =&gt; string &#124; undefined</code> | Returns the active session ID |
-|  [restore](./kibana-plugin-plugins-data-public.isessionservice.restore.md) | <code>(sessionId: string) =&gt; void</code> | Restores existing session |
+|  [isRestore](./kibana-plugin-plugins-data-public.isessionservice.isrestore.md) | <code>() =&gt; boolean</code> | Whether the active session is restored (i.e. reusing previous search IDs) |
+|  [isStored](./kibana-plugin-plugins-data-public.isessionservice.isstored.md) | <code>() =&gt; boolean</code> | Whether the active session is already saved (i.e. sent to background) |
+|  [restore](./kibana-plugin-plugins-data-public.isessionservice.restore.md) | <code>(sessionId: string) =&gt; Promise&lt;SavedObject&lt;BackgroundSessionSavedObjectAttributes&gt;&gt;</code> | Restores existing session |
+|  [save](./kibana-plugin-plugins-data-public.isessionservice.save.md) | <code>(name: string, url: string) =&gt; Promise&lt;SavedObject&lt;BackgroundSessionSavedObjectAttributes&gt;&gt;</code> | Saves a session |
 |  [start](./kibana-plugin-plugins-data-public.isessionservice.start.md) | <code>() =&gt; string</code> | Starts a new session |
+|  [update](./kibana-plugin-plugins-data-public.isessionservice.update.md) | <code>(sessionId: string, attributes: Partial&lt;BackgroundSessionSavedObjectAttributes&gt;) =&gt; Promise&lt;any&gt;</code> | Updates a session |
 

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isessionservice.restore.md

@@ -9,5 +9,5 @@ Restores existing session
 <b>Signature:</b>
 
 ```typescript
-restore: (sessionId: string) => void;
+restore: (sessionId: string) => Promise<SavedObject<BackgroundSessionSavedObjectAttributes>>;
 ```

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isessionservice.save.md

@@ -0,0 +1,13 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [ISessionService](./kibana-plugin-plugins-data-public.isessionservice.md) &gt; [save](./kibana-plugin-plugins-data-public.isessionservice.save.md)
+
+## ISessionService.save property
+
+Saves a session
+
+<b>Signature:</b>
+
+```typescript
+save: (name: string, url: string) => Promise<SavedObject<BackgroundSessionSavedObjectAttributes>>;
+```

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isessionservice.update.md

@@ -0,0 +1,13 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [ISessionService](./kibana-plugin-plugins-data-public.isessionservice.md) &gt; [update](./kibana-plugin-plugins-data-public.isessionservice.update.md)
+
+## ISessionService.update property
+
+Updates a session
+
+<b>Signature:</b>
+
+```typescript
+update: (sessionId: string, attributes: Partial<BackgroundSessionSavedObjectAttributes>) => Promise<any>;
+```

docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.isrestore.md

@@ -0,0 +1,13 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-server](./kibana-plugin-plugins-data-server.md) &gt; [ISearchOptions](./kibana-plugin-plugins-data-server.isearchoptions.md) &gt; [isRestore](./kibana-plugin-plugins-data-server.isearchoptions.isrestore.md)
+
+## ISearchOptions.isRestore property
+
+Whether the session is restored (i.e. search requests should re-use the stored search IDs, rather than starting from scratch)
+
+<b>Signature:</b>
+
+```typescript
+isRestore?: boolean;
+```

docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.isstored.md

@@ -0,0 +1,13 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-server](./kibana-plugin-plugins-data-server.md) &gt; [ISearchOptions](./kibana-plugin-plugins-data-server.isearchoptions.md) &gt; [isStored](./kibana-plugin-plugins-data-server.isearchoptions.isstored.md)
+
+## ISearchOptions.isStored property
+
+Whether the session is already saved (i.e. sent to background)
+
+<b>Signature:</b>
+
+```typescript
+isStored?: boolean;
+```

docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.md

@@ -15,6 +15,8 @@ export interface ISearchOptions
 |  Property | Type | Description |
 |  --- | --- | --- |
 |  [abortSignal](./kibana-plugin-plugins-data-server.isearchoptions.abortsignal.md) | <code>AbortSignal</code> | An <code>AbortSignal</code> that allows the caller of <code>search</code> to abort a search request. |
+|  [isRestore](./kibana-plugin-plugins-data-server.isearchoptions.isrestore.md) | <code>boolean</code> | Whether the session is restored (i.e. search requests should re-use the stored search IDs, rather than starting from scratch) |
+|  [isStored](./kibana-plugin-plugins-data-server.isearchoptions.isstored.md) | <code>boolean</code> | Whether the session is already saved (i.e. sent to background) |
 |  [sessionId](./kibana-plugin-plugins-data-server.isearchoptions.sessionid.md) | <code>string</code> | A session ID, grouping multiple search requests into a single session. |
 |  [strategy](./kibana-plugin-plugins-data-server.isearchoptions.strategy.md) | <code>string</code> | Use this option to force using a specific server side search strategy. Leave empty to use the default strategy. |
 

examples/search_examples/kibana.json

@@ -4,7 +4,7 @@
   "kibanaVersion": "kibana",
   "server": true,
   "ui": true,
-  "requiredPlugins": ["navigation", "data", "developerExamples"],
+  "requiredPlugins": ["navigation", "data", "developerExamples", "kibanaUtils"],
   "optionalPlugins": [],
   "requiredBundles": []
 }

src/plugins/data/common/search/session/index.ts

@@ -17,4 +17,5 @@
  * under the License.
  */
 
+export * from './status';
 export * from './types';

src/plugins/data/common/search/session/mocks.ts

@@ -27,5 +27,12 @@ export function getSessionServiceMock(): jest.Mocked<ISessionService> {
     restore: jest.fn(),
     getSessionId: jest.fn(),
     getSession$: jest.fn(() => new BehaviorSubject(undefined).asObservable()),
+    isStored: jest.fn(),
+    isRestore: jest.fn(),
+    save: jest.fn(),
+    get: jest.fn(),
+    find: jest.fn(),
+    update: jest.fn(),
+    delete: jest.fn(),
   };
 }

src/plugins/data/common/search/session/status.ts

@@ -0,0 +1,26 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export enum BackgroundSessionStatus {
+  IN_PROGRESS = 'in_progress',
+  ERROR = 'error',
+  COMPLETE = 'complete',
+  CANCELLED = 'cancelled',
+  EXPIRED = 'expired',
+}

src/plugins/data/common/search/session/types.ts

@@ -18,6 +18,7 @@
  */
 
 import { Observable } from 'rxjs';
+import type { SavedObject, SavedObjectsFindResponse } from 'kibana/server';
 
 export interface ISessionService {
   /**
@@ -30,6 +31,17 @@ export interface ISessionService {
    * @returns `Observable`
    */
   getSession$: () => Observable<string | undefined>;
+
+  /**
+   * Whether the active session is already saved (i.e. sent to background)
+   */
+  isStored: () => boolean;
+
+  /**
+   * Whether the active session is restored (i.e. reusing previous search IDs)
+   */
+  isRestore: () => boolean;
+
   /**
    * Starts a new session
    */
@@ -38,10 +50,58 @@ export interface ISessionService {
   /**
    * Restores existing session
    */
-  restore: (sessionId: string) => void;
+  restore: (sessionId: string) => Promise<SavedObject<BackgroundSessionSavedObjectAttributes>>;
 
   /**
    * Clears the active session.
    */
   clear: () => void;
+
+  /**
+   * Saves a session
+   */
+  save: (name: string, url: string) => Promise<SavedObject<BackgroundSessionSavedObjectAttributes>>;
+
+  /**
+   * Gets a saved session
+   */
+  get: (sessionId: string) => Promise<SavedObject<BackgroundSessionSavedObjectAttributes>>;
+
+  /**
+   * Gets a list of saved sessions
+   */
+  find: (
+    options: SearchSessionFindOptions
+  ) => Promise<SavedObjectsFindResponse<BackgroundSessionSavedObjectAttributes>>;
+
+  /**
+   * Updates a session
+   */
+  update: (
+    sessionId: string,
+    attributes: Partial<BackgroundSessionSavedObjectAttributes>
+  ) => Promise<any>;
+
+  /**
+   * Deletes a session
+   */
+  delete: (sessionId: string) => Promise<void>;
+}
+
+export interface BackgroundSessionSavedObjectAttributes {
+  name: string;
+  created: string;
+  expires: string;
+  status: string;
+  initialState: Record<string, unknown>;
+  restoreState: Record<string, unknown>;
+  idMapping: Record<string, string>;
+}
+
+export interface SearchSessionFindOptions {
+  page?: number;
+  perPage?: number;
+  sortField?: string;
+  sortOrder?: string;
+  filter?: string;
 }

src/plugins/data/common/search/types.ts

@@ -92,4 +92,15 @@ export interface ISearchOptions {
    * A session ID, grouping multiple search requests into a single session.
    */
   sessionId?: string;
+
+  /**
+   * Whether the session is already saved (i.e. sent to background)
+   */
+  isStored?: boolean;
+
+  /**
+   * Whether the session is restored (i.e. search requests should re-use the stored search IDs,
+   * rather than starting from scratch)
+   */
+  isRestore?: boolean;
 }

src/plugins/data/common/utils/index.ts

@@ -19,3 +19,4 @@
 
 /** @internal */
 export { shortenDottedString } from './shorten_dotted_string';
+export { tapFirst } from './tap_first';

src/plugins/data/common/utils/tap_first.test.ts

@@ -0,0 +1,30 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { of } from 'rxjs';
+import { tapFirst } from './tap_first';
+
+describe('tapFirst', () => {
+  it('should tap the first and only the first', () => {
+    const fn = jest.fn();
+    of(1, 2, 3).pipe(tapFirst(fn)).subscribe();
+    expect(fn).toBeCalledTimes(1);
+    expect(fn).lastCalledWith(1);
+  });
+});

src/plugins/data/common/utils/tap_first.ts

@@ -0,0 +1,31 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { pipe } from 'rxjs';
+import { tap } from 'rxjs/operators';
+
+export function tapFirst<T>(next: (x: T) => void) {
+  let isFirst = true;
+  return pipe(
+    tap<T>((x: T) => {
+      if (isFirst) next(x);
+      isFirst = false;
+    })
+  );
+}

src/plugins/data/public/public.api.md

@@ -70,10 +70,12 @@ import { RequestAdapter } from 'src/plugins/inspector/common';
 import { RequestStatistics as RequestStatistics_2 } from 'src/plugins/inspector/common';
 import { Required } from '@kbn/utility-types';
 import * as Rx from 'rxjs';
-import { SavedObject } from 'src/core/server';
-import { SavedObject as SavedObject_2 } from 'src/core/public';
+import { SavedObject } from 'kibana/server';
+import { SavedObject as SavedObject_2 } from 'src/core/server';
+import { SavedObject as SavedObject_3 } from 'src/core/public';
 import { SavedObjectReference } from 'src/core/types';
 import { SavedObjectsClientContract } from 'src/core/public';
+import { SavedObjectsFindResponse } from 'kibana/server';
 import { Search } from '@elastic/elasticsearch/api/requestParams';
 import { SearchResponse } from 'elasticsearch';
 import { SerializedFieldFormat as SerializedFieldFormat_2 } from 'src/plugins/expressions/common';
@@ -1395,7 +1397,7 @@ export class IndexPatternsService {
     // Warning: (ae-forgotten-export) The symbol "IndexPatternSavedObjectAttrs" needs to be exported by the entry point index.d.ts
     //
     // (undocumented)
-    getCache: () => Promise<SavedObject<IndexPatternSavedObjectAttrs>[] | null | undefined>;
+    getCache: () => Promise<SavedObject_2<IndexPatternSavedObjectAttrs>[] | null | undefined>;
     getDefault: () => Promise<IndexPattern | null>;
     getFieldsForIndexPattern: (indexPattern: IndexPattern | IndexPatternSpec, options?: GetFieldsOptions | undefined) => Promise<any>;
     // Warning: (ae-forgotten-export) The symbol "GetFieldsOptions" needs to be exported by the entry point index.d.ts
@@ -1409,7 +1411,7 @@ export class IndexPatternsService {
     // (undocumented)
     migrate(indexPattern: IndexPattern, newTitle: string): Promise<this>;
     refreshFields: (indexPattern: IndexPattern) => Promise<void>;
-    savedObjectToSpec: (savedObject: SavedObject<IndexPatternAttributes>) => IndexPatternSpec;
+    savedObjectToSpec: (savedObject: SavedObject_2<IndexPatternAttributes>) => IndexPatternSpec;
     setDefault: (id: string, force?: boolean) => Promise<void>;
     updateSavedObject(indexPattern: IndexPattern, saveAttempts?: number, ignoreErrors?: boolean): Promise<void | Error>;
 }
@@ -1454,6 +1456,8 @@ export type ISearchGeneric = <SearchStrategyRequest extends IKibanaSearchRequest
 // @public (undocumented)
 export interface ISearchOptions {
     abortSignal?: AbortSignal;
+    isRestore?: boolean;
+    isStored?: boolean;
     sessionId?: string;
     strategy?: string;
 }
@@ -1506,10 +1510,19 @@ export const isErrorResponse: (response?: IKibanaSearchResponse<any> | undefined
 // @public (undocumented)
 export interface ISessionService {
     clear: () => void;
+    delete: (sessionId: string) => Promise<void>;
+    // Warning: (ae-forgotten-export) The symbol "SearchSessionFindOptions" needs to be exported by the entry point index.d.ts
+    find: (options: SearchSessionFindOptions) => Promise<SavedObjectsFindResponse<BackgroundSessionSavedObjectAttributes>>;
+    get: (sessionId: string) => Promise<SavedObject<BackgroundSessionSavedObjectAttributes>>;
     getSession$: () => Observable<string | undefined>;
     getSessionId: () => string | undefined;
-    restore: (sessionId: string) => void;
+    isRestore: () => boolean;
+    isStored: () => boolean;
+    // Warning: (ae-forgotten-export) The symbol "BackgroundSessionSavedObjectAttributes" needs to be exported by the entry point index.d.ts
+    restore: (sessionId: string) => Promise<SavedObject<BackgroundSessionSavedObjectAttributes>>;
+    save: (name: string, url: string) => Promise<SavedObject<BackgroundSessionSavedObjectAttributes>>;
     start: () => string;
+    update: (sessionId: string, attributes: Partial<BackgroundSessionSavedObjectAttributes>) => Promise<any>;
 }
 
 // Warning: (ae-missing-release-tag) "isFilter" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
@@ -2077,7 +2090,7 @@ export class SearchInterceptor {
     // @internal
     protected pendingCount$: BehaviorSubject<number>;
     // @internal (undocumented)
-    protected runSearch(request: IKibanaSearchRequest, signal: AbortSignal, strategy?: string): Promise<IKibanaSearchResponse>;
+    protected runSearch(request: IKibanaSearchRequest, options?: ISearchOptions): Promise<IKibanaSearchResponse>;
     search(request: IKibanaSearchRequest, options?: ISearchOptions): Observable<IKibanaSearchResponse>;
     // @internal (undocumented)
     protected setupAbortSignal({ abortSignal, timeout, }: {

src/plugins/data/public/search/search_interceptor.ts

@@ -126,18 +126,25 @@ export class SearchInterceptor {
    */
   protected runSearch(
     request: IKibanaSearchRequest,
-    signal: AbortSignal,
-    strategy?: string
+    options?: ISearchOptions
   ): Promise<IKibanaSearchResponse> {
     const { id, ...searchRequest } = request;
-    const path = trimEnd(`/internal/search/${strategy || ES_SEARCH_STRATEGY}/${id || ''}`, '/');
-    const body = JSON.stringify(searchRequest);
+    const path = trimEnd(
+      `/internal/search/${options?.strategy ?? ES_SEARCH_STRATEGY}/${id ?? ''}`,
+      '/'
+    );
+    const body = JSON.stringify({
+      sessionId: options?.sessionId,
+      isStored: options?.isStored,
+      isRestore: options?.isRestore,
+      ...searchRequest,
+    });
 
     return this.deps.http.fetch({
       method: 'POST',
       path,
       body,
-      signal,
+      signal: options?.abortSignal,
     });
   }
 
@@ -235,7 +242,7 @@ export class SearchInterceptor {
         abortSignal: options?.abortSignal,
       });
       this.pendingCount$.next(this.pendingCount$.getValue() + 1);
-      return from(this.runSearch(request, combinedSignal, options?.strategy)).pipe(
+      return from(this.runSearch(request, { ...options, abortSignal: combinedSignal })).pipe(
         catchError((e: Error) => {
           return throwError(this.handleSearchError(e, request, timeoutSignal, options));
         }),

src/plugins/data/public/search/session_service.ts

@@ -19,9 +19,13 @@
 
 import uuid from 'uuid';
 import { BehaviorSubject, Subscription } from 'rxjs';
-import { PluginInitializerContext, StartServicesAccessor } from 'kibana/public';
+import { HttpStart, PluginInitializerContext, StartServicesAccessor } from 'kibana/public';
 import { ConfigSchema } from '../../config';
-import { ISessionService } from '../../common/search';
+import {
+  ISessionService,
+  BackgroundSessionSavedObjectAttributes,
+  SearchSessionFindOptions,
+} from '../../common';
 
 export class SessionService implements ISessionService {
   private session$ = new BehaviorSubject<string | undefined>(undefined);
@@ -30,6 +34,18 @@ export class SessionService implements ISessionService {
   }
   private appChangeSubscription$?: Subscription;
   private curApp?: string;
+  private http!: HttpStart;
+
+  /**
+   * Has the session already been stored (i.e. "sent to background")?
+   */
+  private _isStored: boolean = false;
+
+  /**
+   * Is this session a restored session (have these requests already been made, and we're just
+   * looking to re-use the previous search IDs)?
+   */
+  private _isRestore: boolean = false;
 
   constructor(
     initializerContext: PluginInitializerContext<ConfigSchema>,
@@ -39,6 +55,8 @@ export class SessionService implements ISessionService {
       Make sure that apps don't leave sessions open.
      */
     getStartServices().then(([coreStart]) => {
+      this.http = coreStart.http;
+
       this.appChangeSubscription$ = coreStart.application.currentAppId$.subscribe((appName) => {
         if (this.sessionId) {
           const message = `Application '${this.curApp}' had an open session while navigating`;
@@ -69,16 +87,63 @@ export class SessionService implements ISessionService {
     return this.session$.asObservable();
   }
 
+  public isStored() {
+    return this._isStored;
+  }
+
+  public isRestore() {
+    return this._isRestore;
+  }
+
   public start() {
+    this._isStored = false;
+    this._isRestore = false;
     this.session$.next(uuid.v4());
     return this.sessionId!;
   }
 
   public restore(sessionId: string) {
+    this._isStored = true;
+    this._isRestore = true;
     this.session$.next(sessionId);
+    return this.http.get(`/internal/session/${encodeURIComponent(sessionId)}`);
   }
 
   public clear() {
+    this._isStored = false;
+    this._isRestore = false;
     this.session$.next(undefined);
   }
+
+  public async save(name: string, url: string) {
+    const response = await this.http.post(`/internal/session`, {
+      body: JSON.stringify({
+        name,
+        url,
+        sessionId: this.sessionId,
+      }),
+    });
+    this._isStored = true;
+    return response;
+  }
+
+  public get(sessionId: string) {
+    return this.http.get(`/internal/session/${encodeURIComponent(sessionId)}`);
+  }
+
+  public find(options: SearchSessionFindOptions) {
+    return this.http.post(`/internal/session`, {
+      body: JSON.stringify(options),
+    });
+  }
+
+  public update(sessionId: string, attributes: Partial<BackgroundSessionSavedObjectAttributes>) {
+    return this.http.put(`/internal/session/${encodeURIComponent(sessionId)}`, {
+      body: JSON.stringify(attributes),
+    });
+  }
+
+  public delete(sessionId: string) {
+    return this.http.delete(`/internal/session/${encodeURIComponent(sessionId)}`);
+  }
 }

src/plugins/data/server/saved_objects/background_session.ts

@@ -0,0 +1,56 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { SavedObjectsType } from 'kibana/server';
+
+export const BACKGROUND_SESSION_TYPE = 'background-session';
+
+export const backgroundSessionMapping: SavedObjectsType = {
+  name: BACKGROUND_SESSION_TYPE,
+  namespaceType: 'single',
+  hidden: true,
+  mappings: {
+    properties: {
+      name: {
+        type: 'keyword',
+      },
+      created: {
+        type: 'date',
+      },
+      expires: {
+        type: 'date',
+      },
+      status: {
+        type: 'keyword',
+      },
+      initialState: {
+        type: 'object',
+        enabled: false,
+      },
+      restoreState: {
+        type: 'object',
+        enabled: false,
+      },
+      idMapping: {
+        type: 'object',
+        enabled: false,
+      },
+    },
+  },
+};

src/plugins/data/server/saved_objects/index.ts

@@ -20,3 +20,4 @@ export { querySavedObjectType } from './query';
 export { indexPatternSavedObjectType } from './index_patterns';
 export { kqlTelemetry } from './kql_telemetry';
 export { searchTelemetry } from './search_telemetry';
+export { BACKGROUND_SESSION_TYPE, backgroundSessionMapping } from './background_session';

src/plugins/data/server/search/mocks.ts

@@ -17,6 +17,8 @@
  * under the License.
  */
 
+import type { RequestHandlerContext } from 'src/core/server';
+import { coreMock } from '../../../../core/server/mocks';
 import { ISearchSetup, ISearchStart } from './types';
 import { searchAggsSetupMock, searchAggsStartMock } from './aggs/mocks';
 import { searchSourceMock } from './search_source/mocks';
@@ -40,3 +42,22 @@ export function createSearchStartMock(): jest.Mocked<ISearchStart> {
     searchSource: searchSourceMock.createStartContract(),
   };
 }
+
+export function createSearchRequestHandlerContext(): jest.Mocked<RequestHandlerContext> {
+  return {
+    core: coreMock.createRequestHandlerContext(),
+    search: {
+      search: jest.fn(),
+      cancel: jest.fn(),
+      session: {
+        save: jest.fn(),
+        get: jest.fn(),
+        find: jest.fn(),
+        update: jest.fn(),
+        delete: jest.fn(),
+        trackId: jest.fn(),
+        getId: jest.fn(),
+      },
+    },
+  };
+}

src/plugins/data/server/search/routes/search.ts

@@ -35,11 +35,18 @@ export function registerSearchRoute(router: IRouter): void {
 
         query: schema.object({}, { unknowns: 'allow' }),
 
-        body: schema.object({}, { unknowns: 'allow' }),
+        body: schema.object(
+          {
+            sessionId: schema.maybe(schema.string()),
+            isStored: schema.maybe(schema.boolean()),
+            isRestore: schema.maybe(schema.boolean()),
+          },
+          { unknowns: 'allow' }
+        ),
       },
     },
     async (context, request, res) => {
-      const searchRequest = request.body;
+      const { sessionId, isStored, isRestore, ...searchRequest } = request.body;
       const { strategy, id } = request.params;
       const abortSignal = getRequestAbortedSignal(request.events.aborted$);
 
@@ -50,6 +57,9 @@ export function registerSearchRoute(router: IRouter): void {
             {
               abortSignal,
               strategy,
+              sessionId,
+              isStored,
+              isRestore,
             }
           )
           .pipe(first())

src/plugins/data/server/search/routes/session.test.ts

@@ -0,0 +1,119 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import type { MockedKeys } from '@kbn/utility-types/jest';
+import type { CoreSetup, RequestHandlerContext } from 'kibana/server';
+import type { DataPluginStart } from '../../plugin';
+import { coreMock, httpServerMock } from '../../../../../../src/core/server/mocks';
+import { createSearchRequestHandlerContext } from '../mocks';
+import { registerSessionRoutes } from './session';
+
+describe('registerSessionRoutes', () => {
+  let mockCoreSetup: MockedKeys<CoreSetup<{}, DataPluginStart>>;
+  let mockContext: jest.Mocked<RequestHandlerContext>;
+
+  beforeEach(() => {
+    mockCoreSetup = coreMock.createSetup();
+    mockContext = createSearchRequestHandlerContext();
+    registerSessionRoutes(mockCoreSetup.http.createRouter());
+  });
+
+  it('save calls session.save with sessionId and attributes', async () => {
+    const sessionId = 'd7170a35-7e2c-48d6-8dec-9a056721b489';
+    const name = 'my saved background search session';
+    const body = { sessionId, name };
+
+    const mockRequest = httpServerMock.createKibanaRequest({ body });
+    const mockResponse = httpServerMock.createResponseFactory();
+
+    const mockRouter = mockCoreSetup.http.createRouter.mock.results[0].value;
+    const [[, saveHandler]] = mockRouter.post.mock.calls;
+
+    saveHandler(mockContext, mockRequest, mockResponse);
+
+    expect(mockContext.search!.session.save).toHaveBeenCalledWith(sessionId, { name });
+  });
+
+  it('get calls session.get with sessionId', async () => {
+    const id = 'd7170a35-7e2c-48d6-8dec-9a056721b489';
+    const params = { id };
+
+    const mockRequest = httpServerMock.createKibanaRequest({ params });
+    const mockResponse = httpServerMock.createResponseFactory();
+
+    const mockRouter = mockCoreSetup.http.createRouter.mock.results[0].value;
+    const [[, getHandler]] = mockRouter.get.mock.calls;
+
+    getHandler(mockContext, mockRequest, mockResponse);
+
+    expect(mockContext.search!.session.get).toHaveBeenCalledWith(id);
+  });
+
+  it('find calls session.find with options', async () => {
+    const page = 1;
+    const perPage = 5;
+    const sortField = 'my_field';
+    const sortOrder = 'desc';
+    const filter = 'foo: bar';
+    const body = { page, perPage, sortField, sortOrder, filter };
+
+    const mockRequest = httpServerMock.createKibanaRequest({ body });
+    const mockResponse = httpServerMock.createResponseFactory();
+
+    const mockRouter = mockCoreSetup.http.createRouter.mock.results[0].value;
+    const [, [, findHandler]] = mockRouter.post.mock.calls;
+
+    findHandler(mockContext, mockRequest, mockResponse);
+
+    expect(mockContext.search!.session.find).toHaveBeenCalledWith(body);
+  });
+
+  it('update calls session.update with id and attributes', async () => {
+    const id = 'd7170a35-7e2c-48d6-8dec-9a056721b489';
+    const name = 'my saved background search session';
+    const expires = new Date().toISOString();
+    const params = { id };
+    const body = { name, expires };
+
+    const mockRequest = httpServerMock.createKibanaRequest({ params, body });
+    const mockResponse = httpServerMock.createResponseFactory();
+
+    const mockRouter = mockCoreSetup.http.createRouter.mock.results[0].value;
+    const [[, updateHandler]] = mockRouter.put.mock.calls;
+
+    updateHandler(mockContext, mockRequest, mockResponse);
+
+    expect(mockContext.search!.session.update).toHaveBeenCalledWith(id, body);
+  });
+
+  it('delete calls session.delete with id', async () => {
+    const id = 'd7170a35-7e2c-48d6-8dec-9a056721b489';
+    const params = { id };
+
+    const mockRequest = httpServerMock.createKibanaRequest({ params });
+    const mockResponse = httpServerMock.createResponseFactory();
+
+    const mockRouter = mockCoreSetup.http.createRouter.mock.results[0].value;
+    const [[, deleteHandler]] = mockRouter.delete.mock.calls;
+
+    deleteHandler(mockContext, mockRequest, mockResponse);
+
+    expect(mockContext.search!.session.delete).toHaveBeenCalledWith(id);
+  });
+});

src/plugins/data/server/search/routes/session.ts

@@ -0,0 +1,201 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { schema } from '@kbn/config-schema';
+import { IRouter } from 'src/core/server';
+
+export function registerSessionRoutes(router: IRouter): void {
+  router.post(
+    {
+      path: '/internal/session',
+      validate: {
+        body: schema.object({
+          sessionId: schema.string(),
+          name: schema.string(),
+          expires: schema.maybe(schema.string()),
+          initialState: schema.maybe(schema.object({}, { unknowns: 'allow' })),
+          restoreState: schema.maybe(schema.object({}, { unknowns: 'allow' })),
+        }),
+      },
+    },
+    async (context, request, res) => {
+      const { sessionId, name, expires, initialState, restoreState } = request.body;
+
+      try {
+        const response = await context.search!.session.save(sessionId, {
+          name,
+          expires,
+          initialState,
+          restoreState,
+        });
+
+        return res.ok({
+          body: response,
+        });
+      } catch (err) {
+        return res.customError({
+          statusCode: err.statusCode || 500,
+          body: {
+            message: err.message,
+            attributes: {
+              error: err.body?.error || err.message,
+            },
+          },
+        });
+      }
+    }
+  );
+
+  router.get(
+    {
+      path: '/internal/session/{id}',
+      validate: {
+        params: schema.object({
+          id: schema.string(),
+        }),
+      },
+    },
+    async (context, request, res) => {
+      const { id } = request.params;
+      try {
+        const response = await context.search!.session.get(id);
+
+        return res.ok({
+          body: response,
+        });
+      } catch (err) {
+        return res.customError({
+          statusCode: err.statusCode || 500,
+          body: {
+            message: err.message,
+            attributes: {
+              error: err.body?.error || err.message,
+            },
+          },
+        });
+      }
+    }
+  );
+
+  router.post(
+    {
+      path: '/internal/session/_find',
+      validate: {
+        body: schema.object({
+          page: schema.maybe(schema.number()),
+          perPage: schema.maybe(schema.number()),
+          sortField: schema.maybe(schema.string()),
+          sortOrder: schema.maybe(schema.string()),
+          filter: schema.maybe(schema.string()),
+        }),
+      },
+    },
+    async (context, request, res) => {
+      const { page, perPage, sortField, sortOrder, filter } = request.body;
+      try {
+        const response = await context.search!.session.find({
+          page,
+          perPage,
+          sortField,
+          sortOrder,
+          filter,
+        });
+
+        return res.ok({
+          body: response,
+        });
+      } catch (err) {
+        return res.customError({
+          statusCode: err.statusCode || 500,
+          body: {
+            message: err.message,
+            attributes: {
+              error: err.body?.error || err.message,
+            },
+          },
+        });
+      }
+    }
+  );
+
+  router.delete(
+    {
+      path: '/internal/session/{id}',
+      validate: {
+        params: schema.object({
+          id: schema.string(),
+        }),
+      },
+    },
+    async (context, request, res) => {
+      const { id } = request.params;
+      try {
+        await context.search!.session.delete(id);
+
+        return res.ok();
+      } catch (err) {
+        return res.customError({
+          statusCode: err.statusCode || 500,
+          body: {
+            message: err.message,
+            attributes: {
+              error: err.body?.error || err.message,
+            },
+          },
+        });
+      }
+    }
+  );
+
+  router.put(
+    {
+      path: '/internal/session/{id}',
+      validate: {
+        params: schema.object({
+          id: schema.string(),
+        }),
+        body: schema.object({
+          name: schema.maybe(schema.string()),
+          expires: schema.maybe(schema.string()),
+        }),
+      },
+    },
+    async (context, request, res) => {
+      const { id } = request.params;
+      const { name, expires } = request.body;
+      try {
+        const response = await context.search!.session.update(id, { name, expires });
+
+        return res.ok({
+          body: response,
+        });
+      } catch (err) {
+        return res.customError({
+          statusCode: err.statusCode || 500,
+          body: {
+            message: err.message,
+            attributes: {
+              error: err.body?.error || err.message,
+            },
+          },
+        });
+      }
+    }
+  );
+}

src/plugins/data/server/search/search_service.ts

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-import { BehaviorSubject, Observable } from 'rxjs';
+import { BehaviorSubject, from, Observable } from 'rxjs';
 import { pick } from 'lodash';
 import {
   CoreSetup,
@@ -29,7 +29,7 @@ import {
   SharedGlobalConfig,
   StartServicesAccessor,
 } from 'src/core/server';
-import { first } from 'rxjs/operators';
+import { first, switchMap } from 'rxjs/operators';
 import { ExpressionsServerSetup } from 'src/plugins/expressions/server';
 import {
   ISearchSetup,
@@ -49,7 +49,7 @@ import { DataPluginStart } from '../plugin';
 import { UsageCollectionSetup } from '../../../usage_collection/server';
 import { registerUsageCollector } from './collectors/register';
 import { usageProvider } from './collectors/usage';
-import { searchTelemetry } from '../saved_objects';
+import { BACKGROUND_SESSION_TYPE, searchTelemetry } from '../saved_objects';
 import {
   IEsSearchRequest,
   IEsSearchResponse,
@@ -70,10 +70,14 @@ import {
 } from '../../common/search/aggs/buckets/shard_delay';
 import { aggShardDelay } from '../../common/search/aggs/buckets/shard_delay_fn';
 import { ConfigSchema } from '../../config';
+import { BackgroundSessionService, ISearchSessionClient } from './session';
+import { registerSessionRoutes } from './routes/session';
+import { backgroundSessionMapping } from '../saved_objects';
+import { tapFirst } from '../../common/utils';
 
 declare module 'src/core/server' {
   interface RequestHandlerContext {
-    search?: ISearchClient;
+    search?: ISearchClient & { session: ISearchSessionClient };
   }
 }
 
@@ -102,6 +106,7 @@ export class SearchService implements Plugin<ISearchSetup, ISearchStart> {
   private readonly searchSourceService = new SearchSourceService();
   private defaultSearchStrategyName: string = ES_SEARCH_STRATEGY;
   private searchStrategies: StrategyMap = {};
+  private sessionService: BackgroundSessionService = new BackgroundSessionService();
 
   constructor(
     private initializerContext: PluginInitializerContext<ConfigSchema>,
@@ -121,12 +126,17 @@ export class SearchService implements Plugin<ISearchSetup, ISearchStart> {
     };
     registerSearchRoute(router);
     registerMsearchRoute(router, routeDependencies);
+    registerSessionRoutes(router);
 
     core.http.registerRouteHandlerContext('search', async (context, request) => {
       const [coreStart] = await core.getStartServices();
-      return this.asScopedProvider(coreStart)(request);
+      const search = this.asScopedProvider(coreStart)(request);
+      const session = this.sessionService.asScopedProvider(coreStart)(request);
+      return { ...search, session };
     });
 
+    core.savedObjects.registerType(backgroundSessionMapping);
+
     this.registerSearchStrategy(
       ES_SEARCH_STRATEGY,
       esSearchStrategyProvider(
@@ -223,6 +233,7 @@ export class SearchService implements Plugin<ISearchSetup, ISearchStart> {
 
   public stop() {
     this.aggsService.stop();
+    this.sessionService.stop();
   }
 
   private registerSearchStrategy = <
@@ -248,7 +259,24 @@ export class SearchService implements Plugin<ISearchSetup, ISearchStart> {
       options.strategy
     );
 
-    return strategy.search(searchRequest, options, deps);
+    // If this is a restored background search session, look up the ID using the provided sessionId
+    const getSearchRequest = async () =>
+      !options.isRestore || searchRequest.id
+        ? searchRequest
+        : {
+            ...searchRequest,
+            id: await this.sessionService.getId(searchRequest, options, deps),
+          };
+
+    return from(getSearchRequest()).pipe(
+      switchMap((request) => strategy.search(request, options, deps)),
+      tapFirst((response) => {
+        if (searchRequest.id || !options.sessionId || !response.id || options.isRestore) return;
+        this.sessionService.trackId(searchRequest, response.id, options, {
+          savedObjectsClient: deps.savedObjectsClient,
+        });
+      })
+    );
   };
 
   private cancel = (id: string, options: ISearchOptions, deps: SearchStrategyDependencies) => {
@@ -273,7 +301,9 @@ export class SearchService implements Plugin<ISearchSetup, ISearchStart> {
 
   private asScopedProvider = ({ elasticsearch, savedObjects, uiSettings }: CoreStart) => {
     return (request: KibanaRequest): ISearchClient => {
-      const savedObjectsClient = savedObjects.getScopedClient(request);
+      const savedObjectsClient = savedObjects.getScopedClient(request, {
+        includedHiddenTypes: [BACKGROUND_SESSION_TYPE],
+      });
       const deps = {
         savedObjectsClient,
         esClient: elasticsearch.client.asScoped(request),

src/plugins/data/server/search/session/index.ts

@@ -0,0 +1,20 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export { BackgroundSessionService, ISearchSessionClient } from './session_service';

src/plugins/data/server/search/session/session_service.test.ts

@@ -0,0 +1,233 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import type { SavedObject, SavedObjectsClientContract } from 'kibana/server';
+import { savedObjectsClientMock } from '../../../../../core/server/mocks';
+import { BackgroundSessionStatus } from '../../../common';
+import { BACKGROUND_SESSION_TYPE } from '../../saved_objects';
+import { BackgroundSessionService } from './session_service';
+import { createRequestHash } from './utils';
+
+describe('BackgroundSessionService', () => {
+  let savedObjectsClient: jest.Mocked<SavedObjectsClientContract>;
+  let service: BackgroundSessionService;
+
+  const mockSavedObject: SavedObject = {
+    id: 'd7170a35-7e2c-48d6-8dec-9a056721b489',
+    type: BACKGROUND_SESSION_TYPE,
+    attributes: {
+      name: 'my_name',
+      idMapping: {},
+    },
+    references: [],
+  };
+
+  beforeEach(() => {
+    savedObjectsClient = savedObjectsClientMock.create();
+    service = new BackgroundSessionService();
+  });
+
+  it('save throws if `name` is not provided', () => {
+    const sessionId = 'd7170a35-7e2c-48d6-8dec-9a056721b489';
+
+    expect(() => service.save(sessionId, {}, { savedObjectsClient })).rejects.toMatchInlineSnapshot(
+      `[Error: Name is required]`
+    );
+  });
+
+  it('get calls saved objects client', async () => {
+    savedObjectsClient.get.mockResolvedValue(mockSavedObject);
+
+    const sessionId = 'd7170a35-7e2c-48d6-8dec-9a056721b489';
+    const response = await service.get(sessionId, { savedObjectsClient });
+
+    expect(response).toBe(mockSavedObject);
+    expect(savedObjectsClient.get).toHaveBeenCalledWith(BACKGROUND_SESSION_TYPE, sessionId);
+  });
+
+  it('find calls saved objects client', async () => {
+    const mockFindSavedObject = {
+      ...mockSavedObject,
+      score: 1,
+    };
+    const mockResponse = {
+      saved_objects: [mockFindSavedObject],
+      total: 1,
+      per_page: 1,
+      page: 0,
+    };
+    savedObjectsClient.find.mockResolvedValue(mockResponse);
+
+    const options = { page: 0, perPage: 5 };
+    const response = await service.find(options, { savedObjectsClient });
+
+    expect(response).toBe(mockResponse);
+    expect(savedObjectsClient.find).toHaveBeenCalledWith({
+      ...options,
+      type: BACKGROUND_SESSION_TYPE,
+    });
+  });
+
+  it('update calls saved objects client', async () => {
+    const mockUpdateSavedObject = {
+      ...mockSavedObject,
+      attributes: {},
+    };
+    savedObjectsClient.update.mockResolvedValue(mockUpdateSavedObject);
+
+    const sessionId = 'd7170a35-7e2c-48d6-8dec-9a056721b489';
+    const attributes = { name: 'new_name' };
+    const response = await service.update(sessionId, attributes, { savedObjectsClient });
+
+    expect(response).toBe(mockUpdateSavedObject);
+    expect(savedObjectsClient.update).toHaveBeenCalledWith(
+      BACKGROUND_SESSION_TYPE,
+      sessionId,
+      attributes
+    );
+  });
+
+  it('delete calls saved objects client', async () => {
+    savedObjectsClient.delete.mockResolvedValue({});
+
+    const sessionId = 'd7170a35-7e2c-48d6-8dec-9a056721b489';
+    const response = await service.delete(sessionId, { savedObjectsClient });
+
+    expect(response).toEqual({});
+    expect(savedObjectsClient.delete).toHaveBeenCalledWith(BACKGROUND_SESSION_TYPE, sessionId);
+  });
+
+  describe('trackId', () => {
+    it('stores hash in memory when `isStored` is `false` for when `save` is called', async () => {
+      const searchRequest = { params: {} };
+      const requestHash = createRequestHash(searchRequest.params);
+      const searchId = 'FnpFYlBpeXdCUTMyZXhCLTc1TWFKX0EbdDFDTzJzTE1Sck9PVTBIcW1iU05CZzo4MDA0';
+      const sessionId = 'd7170a35-7e2c-48d6-8dec-9a056721b489';
+      const isStored = false;
+      const name = 'my saved background search session';
+      const created = new Date().toISOString();
+      const expires = new Date().toISOString();
+
+      await service.trackId(
+        searchRequest,
+        searchId,
+        { sessionId, isStored },
+        { savedObjectsClient }
+      );
+
+      expect(savedObjectsClient.update).not.toHaveBeenCalled();
+
+      await service.save(sessionId, { name, created, expires }, { savedObjectsClient });
+
+      expect(savedObjectsClient.create).toHaveBeenCalledWith(
+        BACKGROUND_SESSION_TYPE,
+        {
+          name,
+          created,
+          expires,
+          initialState: {},
+          restoreState: {},
+          status: BackgroundSessionStatus.IN_PROGRESS,
+          idMapping: { [requestHash]: searchId },
+        },
+        { id: sessionId }
+      );
+    });
+
+    it('updates saved object when `isStored` is `true`', async () => {
+      const searchRequest = { params: {} };
+      const requestHash = createRequestHash(searchRequest.params);
+      const searchId = 'FnpFYlBpeXdCUTMyZXhCLTc1TWFKX0EbdDFDTzJzTE1Sck9PVTBIcW1iU05CZzo4MDA0';
+      const sessionId = 'd7170a35-7e2c-48d6-8dec-9a056721b489';
+      const isStored = true;
+
+      await service.trackId(
+        searchRequest,
+        searchId,
+        { sessionId, isStored },
+        { savedObjectsClient }
+      );
+
+      expect(savedObjectsClient.update).toHaveBeenCalledWith(BACKGROUND_SESSION_TYPE, sessionId, {
+        idMapping: { [requestHash]: searchId },
+      });
+    });
+  });
+
+  describe('getId', () => {
+    it('throws if `sessionId` is not provided', () => {
+      const searchRequest = { params: {} };
+
+      expect(() =>
+        service.getId(searchRequest, {}, { savedObjectsClient })
+      ).rejects.toMatchInlineSnapshot(`[Error: Session ID is required]`);
+    });
+
+    it('throws if there is not a saved object', () => {
+      const searchRequest = { params: {} };
+      const sessionId = 'd7170a35-7e2c-48d6-8dec-9a056721b489';
+
+      expect(() =>
+        service.getId(searchRequest, { sessionId, isStored: false }, { savedObjectsClient })
+      ).rejects.toMatchInlineSnapshot(
+        `[Error: Cannot get search ID from a session that is not stored]`
+      );
+    });
+
+    it('throws if not restoring a saved session', () => {
+      const searchRequest = { params: {} };
+      const sessionId = 'd7170a35-7e2c-48d6-8dec-9a056721b489';
+
+      expect(() =>
+        service.getId(
+          searchRequest,
+          { sessionId, isStored: true, isRestore: false },
+          { savedObjectsClient }
+        )
+      ).rejects.toMatchInlineSnapshot(
+        `[Error: Get search ID is only supported when restoring a session]`
+      );
+    });
+
+    it('returns the search ID from the saved object ID mapping', async () => {
+      const searchRequest = { params: {} };
+      const requestHash = createRequestHash(searchRequest.params);
+      const searchId = 'FnpFYlBpeXdCUTMyZXhCLTc1TWFKX0EbdDFDTzJzTE1Sck9PVTBIcW1iU05CZzo4MDA0';
+      const sessionId = 'd7170a35-7e2c-48d6-8dec-9a056721b489';
+      const mockSession = {
+        id: 'd7170a35-7e2c-48d6-8dec-9a056721b489',
+        type: BACKGROUND_SESSION_TYPE,
+        attributes: {
+          name: 'my_name',
+          idMapping: { [requestHash]: searchId },
+        },
+        references: [],
+      };
+      savedObjectsClient.get.mockResolvedValue(mockSession);
+
+      const id = await service.getId(
+        searchRequest,
+        { sessionId, isStored: true, isRestore: true },
+        { savedObjectsClient }
+      );
+
+      expect(id).toBe(searchId);
+    });
+  });
+});

src/plugins/data/server/search/session/session_service.ts

@@ -0,0 +1,204 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { CoreStart, KibanaRequest, SavedObjectsClientContract } from 'kibana/server';
+import {
+  BackgroundSessionSavedObjectAttributes,
+  IKibanaSearchRequest,
+  ISearchOptions,
+  SearchSessionFindOptions,
+  BackgroundSessionStatus,
+} from '../../../common';
+import { BACKGROUND_SESSION_TYPE } from '../../saved_objects';
+import { createRequestHash } from './utils';
+
+const DEFAULT_EXPIRATION = 7 * 24 * 60 * 60 * 1000;
+
+export interface BackgroundSessionDependencies {
+  savedObjectsClient: SavedObjectsClientContract;
+}
+
+export type ISearchSessionClient = ReturnType<
+  ReturnType<BackgroundSessionService['asScopedProvider']>
+>;
+
+export class BackgroundSessionService {
+  /**
+   * Map of sessionId to { [requestHash]: searchId }
+   * @private
+   */
+  private sessionSearchMap = new Map<string, Map<string, string>>();
+
+  constructor() {}
+
+  public setup = () => {};
+
+  public start = (core: CoreStart) => {
+    return {
+      asScoped: this.asScopedProvider(core),
+    };
+  };
+
+  public stop = () => {
+    this.sessionSearchMap.clear();
+  };
+
+  // TODO: Generate the `userId` from the realm type/realm name/username
+  public save = async (
+    sessionId: string,
+    {
+      name,
+      created = new Date().toISOString(),
+      expires = new Date(Date.now() + DEFAULT_EXPIRATION).toISOString(),
+      status = BackgroundSessionStatus.IN_PROGRESS,
+      initialState = {},
+      restoreState = {},
+    }: Partial<BackgroundSessionSavedObjectAttributes>,
+    { savedObjectsClient }: BackgroundSessionDependencies
+  ) => {
+    if (!name) throw new Error('Name is required');
+
+    // Get the mapping of request hash/search ID for this session
+    const searchMap = this.sessionSearchMap.get(sessionId) ?? new Map<string, string>();
+    const idMapping = Object.fromEntries(searchMap.entries());
+    const attributes = { name, created, expires, status, initialState, restoreState, idMapping };
+    const session = await savedObjectsClient.create<BackgroundSessionSavedObjectAttributes>(
+      BACKGROUND_SESSION_TYPE,
+      attributes,
+      { id: sessionId }
+    );
+
+    // Clear out the entries for this session ID so they don't get saved next time
+    this.sessionSearchMap.delete(sessionId);
+
+    return session;
+  };
+
+  // TODO: Throw an error if this session doesn't belong to this user
+  public get = (sessionId: string, { savedObjectsClient }: BackgroundSessionDependencies) => {
+    return savedObjectsClient.get<BackgroundSessionSavedObjectAttributes>(
+      BACKGROUND_SESSION_TYPE,
+      sessionId
+    );
+  };
+
+  // TODO: Throw an error if this session doesn't belong to this user
+  public find = (
+    options: SearchSessionFindOptions,
+    { savedObjectsClient }: BackgroundSessionDependencies
+  ) => {
+    return savedObjectsClient.find<BackgroundSessionSavedObjectAttributes>({
+      ...options,
+      type: BACKGROUND_SESSION_TYPE,
+    });
+  };
+
+  // TODO: Throw an error if this session doesn't belong to this user
+  public update = (
+    sessionId: string,
+    attributes: Partial<BackgroundSessionSavedObjectAttributes>,
+    { savedObjectsClient }: BackgroundSessionDependencies
+  ) => {
+    return savedObjectsClient.update<BackgroundSessionSavedObjectAttributes>(
+      BACKGROUND_SESSION_TYPE,
+      sessionId,
+      attributes
+    );
+  };
+
+  // TODO: Throw an error if this session doesn't belong to this user
+  public delete = (sessionId: string, { savedObjectsClient }: BackgroundSessionDependencies) => {
+    return savedObjectsClient.delete(BACKGROUND_SESSION_TYPE, sessionId);
+  };
+
+  /**
+   * Tracks the given search request/search ID in the saved session (if it exists). Otherwise, just
+   * store it in memory until a saved session exists.
+   * @internal
+   */
+  public trackId = async (
+    searchRequest: IKibanaSearchRequest,
+    searchId: string,
+    { sessionId, isStored }: ISearchOptions,
+    deps: BackgroundSessionDependencies
+  ) => {
+    if (!sessionId || !searchId) return;
+    const requestHash = createRequestHash(searchRequest.params);
+
+    // If there is already a saved object for this session, update it to include this request/ID.
+    // Otherwise, just update the in-memory mapping for this session for when the session is saved.
+    if (isStored) {
+      const attributes = { idMapping: { [requestHash]: searchId } };
+      await this.update(sessionId, attributes, deps);
+    } else {
+      const map = this.sessionSearchMap.get(sessionId) ?? new Map<string, string>();
+      map.set(requestHash, searchId);
+      this.sessionSearchMap.set(sessionId, map);
+    }
+  };
+
+  /**
+   * Look up an existing search ID that matches the given request in the given session so that the
+   * request can continue rather than restart.
+   * @internal
+   */
+  public getId = async (
+    searchRequest: IKibanaSearchRequest,
+    { sessionId, isStored, isRestore }: ISearchOptions,
+    deps: BackgroundSessionDependencies
+  ) => {
+    if (!sessionId) {
+      throw new Error('Session ID is required');
+    } else if (!isStored) {
+      throw new Error('Cannot get search ID from a session that is not stored');
+    } else if (!isRestore) {
+      throw new Error('Get search ID is only supported when restoring a session');
+    }
+
+    const session = await this.get(sessionId, deps);
+    const requestHash = createRequestHash(searchRequest.params);
+    if (!session.attributes.idMapping.hasOwnProperty(requestHash)) {
+      throw new Error('No search ID in this session matching the given search request');
+    }
+
+    return session.attributes.idMapping[requestHash];
+  };
+
+  public asScopedProvider = ({ savedObjects }: CoreStart) => {
+    return (request: KibanaRequest) => {
+      const savedObjectsClient = savedObjects.getScopedClient(request, {
+        includedHiddenTypes: [BACKGROUND_SESSION_TYPE],
+      });
+      const deps = { savedObjectsClient };
+      return {
+        save: (sessionId: string, attributes: Partial<BackgroundSessionSavedObjectAttributes>) =>
+          this.save(sessionId, attributes, deps),
+        get: (sessionId: string) => this.get(sessionId, deps),
+        find: (options: SearchSessionFindOptions) => this.find(options, deps),
+        update: (sessionId: string, attributes: Partial<BackgroundSessionSavedObjectAttributes>) =>
+          this.update(sessionId, attributes, deps),
+        delete: (sessionId: string) => this.delete(sessionId, deps),
+        trackId: (searchRequest: IKibanaSearchRequest, searchId: string, options: ISearchOptions) =>
+          this.trackId(searchRequest, searchId, options, deps),
+        getId: (searchRequest: IKibanaSearchRequest, options: ISearchOptions) =>
+          this.getId(searchRequest, options, deps),
+      };
+    };
+  };
+}

src/plugins/data/server/search/session/utils.test.ts

@@ -0,0 +1,37 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { createRequestHash } from './utils';
+
+describe('data/search/session utils', () => {
+  describe('createRequestHash', () => {
+    it('ignores `preference`', () => {
+      const request = {
+        foo: 'bar',
+      };
+
+      const withPreference = {
+        ...request,
+        preference: 1234,
+      };
+
+      expect(createRequestHash(request)).toEqual(createRequestHash(withPreference));
+    });
+  });
+});

src/plugins/data/server/search/session/utils.ts

@@ -0,0 +1,30 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { createHash } from 'crypto';
+
+/**
+ * Generate the hash for this request so that, in the future, this hash can be used to look up
+ * existing search IDs for this request. Ignores the `preference` parameter since it generally won't
+ * match from one request to another identical request.
+ */
+export function createRequestHash(keys: Record<any, any>) {
+  const { preference, ...params } = keys;
+  return createHash(`sha256`).update(JSON.stringify(params)).digest('hex');
+}

src/plugins/data/server/server.api.md

@@ -757,6 +757,8 @@ export class IndexPatternsService implements Plugin_3<void, IndexPatternsService
 // @public (undocumented)
 export interface ISearchOptions {
     abortSignal?: AbortSignal;
+    isRestore?: boolean;
+    isStored?: boolean;
     sessionId?: string;
     strategy?: string;
 }

src/plugins/embeddable/public/public.api.md

@@ -63,11 +63,13 @@ import { RecursiveReadonly } from '@kbn/utility-types';
 import { RequestAdapter as RequestAdapter_2 } from 'src/plugins/inspector/common';
 import { Required } from '@kbn/utility-types';
 import * as Rx from 'rxjs';
-import { SavedObject as SavedObject_2 } from 'src/core/server';
+import { SavedObject as SavedObject_2 } from 'kibana/server';
+import { SavedObject as SavedObject_3 } from 'src/core/server';
 import { SavedObjectAttributes } from 'kibana/server';
 import { SavedObjectAttributes as SavedObjectAttributes_2 } from 'src/core/public';
 import { SavedObjectAttributes as SavedObjectAttributes_3 } from 'kibana/public';
 import { SavedObjectsClientContract as SavedObjectsClientContract_3 } from 'src/core/public';
+import { SavedObjectsFindResponse as SavedObjectsFindResponse_2 } from 'kibana/server';
 import { Search } from '@elastic/elasticsearch/api/requestParams';
 import { SearchResponse } from 'elasticsearch';
 import { SerializedFieldFormat as SerializedFieldFormat_2 } from 'src/plugins/expressions/common';

x-pack/plugins/data_enhanced/public/search/search_interceptor.ts

@@ -76,8 +76,12 @@ export class EnhancedSearchInterceptor extends SearchInterceptor {
     this.pendingCount$.next(this.pendingCount$.getValue() + 1);
 
     return doPartialSearch<IEsSearchResponse>(
-      () => this.runSearch(request, combinedSignal, strategy),
-      (requestId) => this.runSearch({ ...request, id: requestId }, combinedSignal, strategy),
+      () => this.runSearch(request, { ...options, strategy, abortSignal: combinedSignal }),
+      (requestId) =>
+        this.runSearch(
+          { ...request, id: requestId },
+          { ...options, strategy, abortSignal: combinedSignal }
+        ),
       (r) => !r.isRunning,
       (response) => response.id,
       id,

x-pack/plugins/data_enhanced/server/search/es_search_strategy.ts

@@ -57,6 +57,7 @@ export const enhancedEsSearchStrategyProvider = (
           utils.toSnakeCase({
             ...(await getDefaultSearchParams(uiSettingsClient)),
             batchedReduceSize: 64,
+            keepOnCompletion: !!options.sessionId, // Always return an ID, even if the request completes quickly
             ...asyncOptions,
             ...request.params,
           })