[Search Sessions] Replace search session constants with kibana.yml configs (#88023)

* Replace search session constants with kibana.yml configs

* fix test

* jest

* rename feature flag

* @lukasolson code review

* Code review - config retreival

* jest fix

rfcs/text/0013_search_sessions.md

@@ -285,7 +285,6 @@ interface ISessionService {
     * @param url TODO: is the URL provided here? How?
     * @returns The stored `SearchSessionAttributes` object
     * @throws Throws an error in OSS.
-    * @internal (Consumers should use searchInterceptor.sendToBackground())
     */
   store: (
     request: KibanaRequest,

x-pack/plugins/data_enhanced/config.ts

@@ -8,8 +8,13 @@ import { schema, TypeOf } from '@kbn/config-schema';
 
 export const configSchema = schema.object({
   search: schema.object({
-    sendToBackground: schema.object({
+    sessions: schema.object({
       enabled: schema.boolean({ defaultValue: false }),
+      pageSize: schema.number({ defaultValue: 10000 }),
+      trackingInterval: schema.duration({ defaultValue: '10s' }),
+      inMemTimeout: schema.duration({ defaultValue: '1m' }),
+      maxUpdateRetries: schema.number({ defaultValue: 3 }),
+      defaultExpiration: schema.duration({ defaultValue: '7d' }),
     }),
   }),
 });

x-pack/plugins/data_enhanced/public/plugin.ts

@@ -62,7 +62,7 @@ export class DataEnhancedPlugin
   public start(core: CoreStart, plugins: DataEnhancedStartDependencies) {
     setAutocompleteService(plugins.data.autocomplete);
 
-    if (this.initializerContext.config.get().search.sendToBackground.enabled) {
+    if (this.initializerContext.config.get().search.sessions.enabled) {
       core.chrome.setBreadcrumbsAppendExtension({
         content: toMountPoint(
           React.createElement(

x-pack/plugins/data_enhanced/server/plugin.ts

@@ -61,7 +61,10 @@ export class EnhancedDataServerPlugin
       eqlSearchStrategyProvider(this.logger)
     );
 
-    this.sessionService = new SearchSessionService(this.logger);
+    this.sessionService = new SearchSessionService(
+      this.logger,
+      this.initializerContext.config.create()
+    );
 
     deps.data.__enhance({
       search: {
@@ -81,7 +84,6 @@ export class EnhancedDataServerPlugin
   public start(core: CoreStart, { taskManager }: StartDependencies) {
     this.sessionService.start(core, {
       taskManager,
-      config$: this.initializerContext.config.create(),
     });
   }
 

x-pack/plugins/data_enhanced/server/search/session/constants.ts

@@ -1,11 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-export const INMEM_MAX_SESSIONS = 10000;
-export const DEFAULT_EXPIRATION = 7 * 24 * 60 * 60 * 1000;
-export const INMEM_TRACKING_INTERVAL = 10 * 1000;
-export const INMEM_TRACKING_TIMEOUT_SEC = 60;
-export const MAX_UPDATE_RETRIES = 3;

x-pack/plugins/data_enhanced/server/search/session/monitoring_task.ts

@@ -4,6 +4,9 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
+import { Observable } from 'rxjs';
+import { first } from 'rxjs/operators';
+import { Duration } from 'moment';
 import {
   TaskManagerSetupContract,
   TaskManagerStartContract,
@@ -12,15 +15,22 @@ import {
 import { checkRunningSessions } from './check_running_sessions';
 import { CoreSetup, SavedObjectsClient, Logger } from '../../../../../../src/core/server';
 import { SEARCH_SESSION_TYPE } from '../../saved_objects';
+import { ConfigSchema } from '../../../config';
 
 export const SEARCH_SESSIONS_TASK_TYPE = 'bg_monitor';
 export const SEARCH_SESSIONS_TASK_ID = `data_enhanced_${SEARCH_SESSIONS_TASK_TYPE}`;
-export const MONITOR_INTERVAL = 15; // in seconds
 
-function searchSessionRunner(core: CoreSetup, logger: Logger) {
+interface SearchSessionTaskDeps {
+  taskManager: TaskManagerSetupContract;
+  logger: Logger;
+  config$: Observable<ConfigSchema>;
+}
+
+function searchSessionRunner(core: CoreSetup, { logger, config$ }: SearchSessionTaskDeps) {
   return ({ taskInstance }: RunContext) => {
     return {
       async run() {
+        const config = await config$.pipe(first()).toPromise();
         const [coreStart] = await core.getStartServices();
         const internalRepo = coreStart.savedObjects.createInternalRepository([SEARCH_SESSION_TYPE]);
         const internalSavedObjectsClient = new SavedObjectsClient(internalRepo);
@@ -31,7 +41,7 @@ function searchSessionRunner(core: CoreSetup, logger: Logger) {
         );
 
         return {
-          runAt: new Date(Date.now() + MONITOR_INTERVAL * 1000),
+          runAt: new Date(Date.now() + config.search.sessions.trackingInterval.asMilliseconds()),
           state: {},
         };
       },
@@ -39,22 +49,19 @@ function searchSessionRunner(core: CoreSetup, logger: Logger) {
   };
 }
 
-export function registerSearchSessionsTask(
-  core: CoreSetup,
-  taskManager: TaskManagerSetupContract,
-  logger: Logger
-) {
-  taskManager.registerTaskDefinitions({
+export function registerSearchSessionsTask(core: CoreSetup, deps: SearchSessionTaskDeps) {
+  deps.taskManager.registerTaskDefinitions({
     [SEARCH_SESSIONS_TASK_TYPE]: {
       title: 'Search Sessions Monitor',
-      createTaskRunner: searchSessionRunner(core, logger),
+      createTaskRunner: searchSessionRunner(core, deps),
     },
   });
 }
 
 export async function scheduleSearchSessionsTasks(
   taskManager: TaskManagerStartContract,
-  logger: Logger
+  logger: Logger,
+  trackingInterval: Duration
 ) {
   await taskManager.removeIfExists(SEARCH_SESSIONS_TASK_ID);
 
@@ -63,7 +70,7 @@ export async function scheduleSearchSessionsTasks(
       id: SEARCH_SESSIONS_TASK_ID,
       taskType: SEARCH_SESSIONS_TASK_TYPE,
       schedule: {
-        interval: `${MONITOR_INTERVAL}s`,
+        interval: `${trackingInterval.asSeconds()}s`,
       },
       state: {},
       params: {},

x-pack/plugins/data_enhanced/server/search/session/session_service.test.ts

@@ -17,9 +17,11 @@ import { coreMock } from 'src/core/server/mocks';
 import { ConfigSchema } from '../../../config';
 // @ts-ignore
 import { taskManagerMock } from '../../../../task_manager/server/mocks';
-import { INMEM_TRACKING_INTERVAL, MAX_UPDATE_RETRIES } from './constants';
 import { SearchStatus } from './types';
 
+const INMEM_TRACKING_INTERVAL = 10000;
+const MAX_UPDATE_RETRIES = 3;
+
 const flushPromises = () => new Promise((resolve) => setImmediate(resolve));
 
 describe('SearchSessionService', () => {
@@ -103,12 +105,36 @@ describe('SearchSessionService', () => {
 
   beforeEach(async () => {
     savedObjectsClient = savedObjectsClientMock.create();
+    const config$ = new BehaviorSubject<ConfigSchema>({
+      search: {
+        sessions: {
+          enabled: true,
+          pageSize: 10000,
+          inMemTimeout: moment.duration(1, 'm'),
+          maxUpdateRetries: 3,
+          defaultExpiration: moment.duration(7, 'd'),
+          trackingInterval: moment.duration(10, 's'),
+        },
+      },
+    });
     const mockLogger: any = {
       debug: jest.fn(),
       warn: jest.fn(),
       error: jest.fn(),
     };
-    service = new SearchSessionService(mockLogger);
+    service = new SearchSessionService(mockLogger, config$);
+    const coreStart = coreMock.createStart();
+    const mockTaskManager = taskManagerMock.createStart();
+    jest.useFakeTimers();
+    await flushPromises();
+    await service.start(coreStart, {
+      taskManager: mockTaskManager,
+    });
+  });
+
+  afterEach(() => {
+    service.stop();
+    jest.useRealTimers();
   });
 
   it('search throws if `name` is not provided', () => {
@@ -411,28 +437,6 @@ describe('SearchSessionService', () => {
   });
 
   describe('Monitor', () => {
-    beforeEach(async () => {
-      jest.useFakeTimers();
-      const config$ = new BehaviorSubject<ConfigSchema>({
-        search: {
-          sendToBackground: {
-            enabled: true,
-          },
-        },
-      });
-      const mockTaskManager = taskManagerMock.createStart();
-      await service.start(coreMock.createStart(), {
-        config$,
-        taskManager: mockTaskManager,
-      });
-      await flushPromises();
-    });
-
-    afterEach(() => {
-      jest.useRealTimers();
-      service.stop();
-    });
-
     it('schedules the next iteration', async () => {
       const findSpy = jest.fn().mockResolvedValue({ saved_objects: [] });
       createMockInternalSavedObjectClient(findSpy);

x-pack/plugins/data_enhanced/server/search/session/session_service.ts

@@ -44,13 +44,6 @@ import { SEARCH_SESSION_TYPE } from '../../saved_objects';
 import { createRequestHash } from './utils';
 import { ConfigSchema } from '../../../config';
 import { registerSearchSessionsTask, scheduleSearchSessionsTasks } from './monitoring_task';
-import {
-  DEFAULT_EXPIRATION,
-  INMEM_MAX_SESSIONS,
-  INMEM_TRACKING_INTERVAL,
-  INMEM_TRACKING_TIMEOUT_SEC,
-  MAX_UPDATE_RETRIES,
-} from './constants';
 import { SearchStatus } from './types';
 
 export interface SearchSessionDependencies {
@@ -69,8 +62,10 @@ interface SetupDependencies {
 
 interface StartDependencies {
   taskManager: TaskManagerStartContract;
-  config$: Observable<ConfigSchema>;
 }
+
+type SearchSessionsConfig = ConfigSchema['search']['sessions'];
+
 export class SearchSessionService implements ISessionService {
   /**
    * Map of sessionId to { [requestHash]: searchId }
@@ -79,14 +74,24 @@ export class SearchSessionService implements ISessionService {
   private sessionSearchMap = new Map<string, SessionInfo>();
   private internalSavedObjectsClient!: SavedObjectsClientContract;
   private monitorTimer!: NodeJS.Timeout;
+  private config!: SearchSessionsConfig;
 
-  constructor(private readonly logger: Logger) {}
+  constructor(
+    private readonly logger: Logger,
+    private readonly config$: Observable<ConfigSchema>
+  ) {}
 
   public setup(core: CoreSetup, deps: SetupDependencies) {
-    registerSearchSessionsTask(core, deps.taskManager, this.logger);
+    registerSearchSessionsTask(core, {
+      config$: this.config$,
+      taskManager: deps.taskManager,
+      logger: this.logger,
+    });
   }
 
   public async start(core: CoreStart, deps: StartDependencies) {
+    const configPromise = await this.config$.pipe(first()).toPromise();
+    this.config = (await configPromise).search.sessions;
     return this.setupMonitoring(core, deps);
   }
 
@@ -96,9 +101,8 @@ export class SearchSessionService implements ISessionService {
   }
 
   private setupMonitoring = async (core: CoreStart, deps: StartDependencies) => {
-    const config = await deps.config$.pipe(first()).toPromise();
-    if (config.search.sendToBackground.enabled) {
-      scheduleSearchSessionsTasks(deps.taskManager, this.logger);
+    if (this.config.enabled) {
+      scheduleSearchSessionsTasks(deps.taskManager, this.logger, this.config.trackingInterval);
       this.logger.debug(`setupMonitoring | Enabling monitoring`);
       const internalRepo = core.savedObjects.createInternalRepository([SEARCH_SESSION_TYPE]);
       this.internalSavedObjectsClient = new SavedObjectsClient(internalRepo);
@@ -129,7 +133,7 @@ export class SearchSessionService implements ISessionService {
   private async getAllMappedSavedObjects() {
     const filter = this.sessionIdsAsFilters(Array.from(this.sessionSearchMap.keys()));
     const res = await this.internalSavedObjectsClient.find<SearchSessionSavedObjectAttributes>({
-      perPage: INMEM_MAX_SESSIONS, // If there are more sessions in memory, they will be synced when some items are cleared out.
+      perPage: this.config.pageSize, // If there are more sessions in memory, they will be synced when some items are cleared out.
       type: SEARCH_SESSION_TYPE,
       filter,
       namespaces: ['*'],
@@ -138,17 +142,17 @@ export class SearchSessionService implements ISessionService {
     return res.saved_objects;
   }
 
-  private clearSessions = () => {
+  private clearSessions = async () => {
     const curTime = moment();
 
     this.sessionSearchMap.forEach((sessionInfo, sessionId) => {
       if (
-        moment.duration(curTime.diff(sessionInfo.insertTime)).asSeconds() >
-        INMEM_TRACKING_TIMEOUT_SEC
+        moment.duration(curTime.diff(sessionInfo.insertTime)).asMilliseconds() >
+        this.config.inMemTimeout.asMilliseconds()
       ) {
         this.logger.debug(`clearSessions | Deleting expired session ${sessionId}`);
         this.sessionSearchMap.delete(sessionId);
-      } else if (sessionInfo.retryCount >= MAX_UPDATE_RETRIES) {
+      } else if (sessionInfo.retryCount >= this.config.maxUpdateRetries) {
         this.logger.warn(`clearSessions | Deleting failed session ${sessionId}`);
         this.sessionSearchMap.delete(sessionId);
       }
@@ -192,7 +196,7 @@ export class SearchSessionService implements ISessionService {
       } finally {
         this.monitorMappedIds();
       }
-    }, INMEM_TRACKING_INTERVAL);
+    }, this.config.trackingInterval.asMilliseconds());
   }
 
   private async updateAllSavedObjects(
@@ -256,7 +260,7 @@ export class SearchSessionService implements ISessionService {
       name,
       appId,
       created = new Date().toISOString(),
-      expires = new Date(Date.now() + DEFAULT_EXPIRATION).toISOString(),
+      expires = new Date(Date.now() + this.config.defaultExpiration.asMilliseconds()).toISOString(),
       status = SearchSessionStatus.IN_PROGRESS,
       urlGeneratorId,
       initialState = {},

x-pack/test/api_integration/config.ts

@@ -30,7 +30,7 @@ export async function getApiIntegrationConfig({ readConfigFile }: FtrConfigProvi
         '--telemetry.optIn=true',
         '--xpack.fleet.enabled=true',
         '--xpack.fleet.agents.pollingRequestTimeout=5000', // 5 seconds
-        '--xpack.data_enhanced.search.sendToBackground.enabled=true', // enable WIP send to background UI
+        '--xpack.data_enhanced.search.sessions.enabled=true', // enable WIP send to background UI
       ],
     },
     esTestCluster: {

x-pack/test/send_search_to_background_integration/config.ts

@@ -29,7 +29,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
       ...xpackFunctionalConfig.get('kbnTestServer'),
       serverArgs: [
         ...xpackFunctionalConfig.get('kbnTestServer.serverArgs'),
-        '--xpack.data_enhanced.search.sendToBackground.enabled=true', // enable WIP send to background UI
+        '--xpack.data_enhanced.search.sessions.enabled=true', // enable WIP send to background UI
       ],
     },
     services: {