[LockManager] Expose as package (#219220)

Expose LockManager as package to make it easier to consume from other
plugins

cc @nchaulet

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Viduni Wickramarachchi <viduni.ushanka@gmail.com>

.github/CODEOWNERS

@@ -70,6 +70,7 @@ packages/kbn-json-ast @elastic/kibana-operations
 packages/kbn-kibana-manifest-schema @elastic/kibana-operations
 packages/kbn-lint-packages-cli @elastic/kibana-operations
 packages/kbn-lint-ts-projects-cli @elastic/kibana-operations
+packages/kbn-lock-manager @elastic/obs-ai-assistant
 packages/kbn-managed-vscode-config @elastic/kibana-operations
 packages/kbn-managed-vscode-config-cli @elastic/kibana-operations
 packages/kbn-manifest @elastic/kibana-core

package.json

@@ -633,6 +633,7 @@
     "@kbn/llm-tasks-plugin": "link:x-pack/platform/plugins/shared/ai_infra/llm_tasks",
     "@kbn/locator-examples-plugin": "link:examples/locator_examples",
     "@kbn/locator-explorer-plugin": "link:examples/locator_explorer",
+    "@kbn/lock-manager": "link:packages/kbn-lock-manager",
     "@kbn/logging": "link:src/platform/packages/shared/kbn-logging",
     "@kbn/logging-mocks": "link:src/platform/packages/shared/kbn-logging-mocks",
     "@kbn/logs-data-access-plugin": "link:x-pack/platform/plugins/shared/logs_data_access",

packages/kbn-lock-manager/README.md

@@ -0,0 +1,54 @@
+# Kibana Lock Manager
+
+A simple, distributed lock manager built on top of Elasticsearch.  
+Ensures that only one process at a time can hold a named lock, with automatic lease renewal and token fencing for safe release.
+
+# API Documentation
+
+## `withLock<T>(lockId, callback, options)`
+
+Acquires a lock and executes the provided callback. If the lock is already held by another process, the method will throw a `LockAcquisitionError` and the callback will not be executed. When the callback returns the lock is released.
+
+### Parameters
+
+- **`lockId`** (`string`): Unique identifier for the lock
+
+- **`callback`** (`() => Promise<T>`): Asynchronous function to execute once the lock is acquired. This function will be executed only if the lock acquisition succeeds.
+
+- **`options`** (`object`, optional): Additional configuration options.
+  - **`metadata`** (`Record<string, any>`, optional): Custom metadata to store with the lock. 
+
+## Example
+
+```ts
+import { LockManagerService, LockAcquisitionError } from '@kbn/lock-manager';
+
+
+async function reIndexWithLock() {
+  // Attempt to acquire "my_lock"; if successful, runs the callback.
+  const lmService = new LockManagerService(coreSetup, logger);
+  return lmService.withLock('my_lock', async () => {
+    // …perform your exclusive operation here…
+  });
+}
+
+reIndexWithLock().catch((err) => {
+  if (err instanceof LockAcquisitionError) {
+    logger.debug('Re-index already in progress, skipping.');
+    return;
+  }
+  logger.error(`Failed to re-index: ${err.message}`);
+});
+```
+
+## How It Works
+**Atomic Acquire**
+Performs one atomic Elasticsearch update that creates a new lock or renews an existing one - so if multiple processes race for the same lock, only one succeeds.
+
+**TTL-Based Lease**
+Each lock has a short, fixed lifespan (default 30s) and will automatically expire if not renewed. While the callback is executing, the lock will automatically extend the TTL to keep the lock active. This safeguards against deadlocks because if a Kibana node crashes after having obtained a lock it will automatically be released after 30 seconds.
+
+Note: If Kibana node crashes, another process could acquire the same lock and start that task again when the lock automatically expires. To prevent your operation from running multiple times, include an application-level check (for example, querying Elasticsearch or your own status flag) to verify the operation isn’t already in progress before proceeding.
+
+**Token Fencing**
+Each lock operation carries a unique token. Only the process with the matching token can extend or release the lock, preventing stale holders from interfering.

packages/kbn-lock-manager/index.ts

@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+export { LockAcquisitionError } from './src/lock_manager_client';
+export { LockManagerService } from './src/lock_manager_service';

packages/kbn-lock-manager/jest.config.js

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+module.exports = {
+  preset: '@kbn/test/jest_node',
+  rootDir: '../..',
+  roots: ['<rootDir>/packages/kbn-lock-manager'],
+};

packages/kbn-lock-manager/kibana.jsonc

@@ -0,0 +1,6 @@
+{
+  "type": "shared-server",
+  "id": "@kbn/lock-manager",
+  "owner": ["@elastic/obs-ai-assistant"],  
+  "devOnly": false
+}

packages/kbn-lock-manager/package.json

@@ -0,0 +1,6 @@
+{
+  "name": "@kbn/lock-manager",
+  "private": true,
+  "version": "1.0.0",
+  "license": "Elastic License 2.0 OR AGPL-3.0-only OR SSPL-1.0"
+}

packages/kbn-lock-manager/src/lock_manager_client.ts

@@ -1,8 +1,10 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
 // eslint-disable-next-line max-classes-per-file
@@ -13,7 +15,7 @@ import prettyMilliseconds from 'pretty-ms';
 import { once } from 'lodash';
 import { duration } from 'moment';
 import { ElasticsearchClient } from '@kbn/core/server';
-import { LOCKS_CONCRETE_INDEX_NAME, setuplockManagerIndex } from './setup_lock_manager_index';
+import { LOCKS_CONCRETE_INDEX_NAME, setupLockManagerIndex } from './setup_lock_manager_index';
 
 export type LockId = string;
 export interface LockDocument {
@@ -38,9 +40,9 @@ export interface AcquireOptions {
 
 // The index assets should only be set up once
 // For testing purposes, we need to be able to set it up every time
-let runSetupIndexAssetOnce = once(setuplockManagerIndex);
+let runSetupIndexAssetOnce = once(setupLockManagerIndex);
 export function runSetupIndexAssetEveryTime() {
-  runSetupIndexAssetOnce = setuplockManagerIndex;
+  runSetupIndexAssetOnce = setupLockManagerIndex;
 }
 
 export class LockManager {

packages/kbn-lock-manager/src/lock_manager_service.ts

@@ -1,8 +1,10 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
 import { CoreSetup, Logger } from '@kbn/core/server';

packages/kbn-lock-manager/src/setup_lock_manager_index.ts

@@ -1,8 +1,10 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
 import { errors } from '@elastic/elasticsearch';
@@ -10,6 +12,7 @@ import { ElasticsearchClient, Logger } from '@kbn/core/server';
 import { IndicesGetMappingResponse } from '@elastic/elasticsearch/lib/api/types';
 
 const LOCKS_INDEX_ALIAS = '.kibana_locks';
+const INDEX_PATTERN = `${LOCKS_INDEX_ALIAS}*`;
 export const LOCKS_CONCRETE_INDEX_NAME = `${LOCKS_INDEX_ALIAS}-000001`;
 export const LOCKS_COMPONENT_TEMPLATE_NAME = `${LOCKS_INDEX_ALIAS}-component`;
 export const LOCKS_INDEX_TEMPLATE_NAME = `${LOCKS_INDEX_ALIAS}-index-template`;
@@ -52,8 +55,6 @@ export async function ensureTemplatesAndIndexCreated(
   esClient: ElasticsearchClient,
   logger: Logger
 ): Promise<void> {
-  const INDEX_PATTERN = `${LOCKS_INDEX_ALIAS}*`;
-
   await esClient.cluster.putComponentTemplate({
     name: LOCKS_COMPONENT_TEMPLATE_NAME,
     template: {
@@ -87,11 +88,25 @@ export async function ensureTemplatesAndIndexCreated(
   });
   logger.info(`Index template ${LOCKS_INDEX_TEMPLATE_NAME} created or updated successfully.`);
 
-  await esClient.indices.create({ index: LOCKS_CONCRETE_INDEX_NAME }, { ignore: [400] });
-  logger.info(`Index ${LOCKS_CONCRETE_INDEX_NAME} created or updated successfully.`);
+  try {
+    await esClient.indices.create({ index: LOCKS_CONCRETE_INDEX_NAME });
+    logger.info(`Index ${LOCKS_CONCRETE_INDEX_NAME} created successfully.`);
+  } catch (error) {
+    const isIndexAlreadyExistsError =
+      error instanceof errors.ResponseError &&
+      error.body.error.type === 'resource_already_exists_exception';
+
+    if (isIndexAlreadyExistsError) {
+      logger.debug(`Index ${LOCKS_CONCRETE_INDEX_NAME} already exists. Skipping creation.`);
+      return;
+    }
+
+    logger.error(`Unable to create index ${LOCKS_CONCRETE_INDEX_NAME}: ${error.message}`);
+    throw error;
+  }
 }
 
-export async function setuplockManagerIndex(esClient: ElasticsearchClient, logger: Logger) {
+export async function setupLockManagerIndex(esClient: ElasticsearchClient, logger: Logger) {
   await removeLockIndexWithIncorrectMappings(esClient, logger); // TODO: should be removed in the future (after 9.1). See https://github.com/elastic/kibana/issues/218944
   await ensureTemplatesAndIndexCreated(esClient, logger);
 }

packages/kbn-lock-manager/tsconfig.json

@@ -0,0 +1,20 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "target/types",
+    "types": [
+      "jest",
+      "node"
+    ]
+  },
+  "include": [
+    "**/*.ts"
+  ],
+  "exclude": [
+    "target/**/*"
+  ],
+  "kbn_references": [
+    "@kbn/logging",
+    "@kbn/core",
+  ]
+}

tsconfig.base.json

@@ -1184,6 +1184,8 @@
       "@kbn/locator-examples-plugin/*": ["examples/locator_examples/*"],
       "@kbn/locator-explorer-plugin": ["examples/locator_explorer"],
       "@kbn/locator-explorer-plugin/*": ["examples/locator_explorer/*"],
+      "@kbn/lock-manager": ["packages/kbn-lock-manager"],
+      "@kbn/lock-manager/*": ["packages/kbn-lock-manager/*"],
       "@kbn/logging": ["src/platform/packages/shared/kbn-logging"],
       "@kbn/logging/*": ["src/platform/packages/shared/kbn-logging/*"],
       "@kbn/logging-mocks": ["src/platform/packages/shared/kbn-logging-mocks"],

x-pack/platform/plugins/shared/observability_ai_assistant/server/service/knowledge_base_service/index.ts

@@ -10,6 +10,7 @@ import type { CoreSetup, ElasticsearchClient, IUiSettingsClient } from '@kbn/cor
 import type { Logger } from '@kbn/logging';
 import { orderBy } from 'lodash';
 import { encode } from 'gpt-tokenizer';
+import { LockAcquisitionError } from '@kbn/lock-manager';
 import { resourceNames } from '..';
 import {
   Instruction,
@@ -34,7 +35,6 @@ import {
   isSemanticTextUnsupportedError,
   reIndexKnowledgeBaseWithLock,
 } from './reindex_knowledge_base';
-import { LockAcquisitionError } from '../distributed_lock_manager/lock_manager_client';
 
 interface Dependencies {
   core: CoreSetup<ObservabilityAIAssistantPluginStartDependencies>;

x-pack/platform/plugins/shared/observability_ai_assistant/server/service/knowledge_base_service/reindex_knowledge_base.ts

@@ -9,9 +9,9 @@ import { errors as EsErrors } from '@elastic/elasticsearch';
 import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
 import { Logger } from '@kbn/logging';
 import { CoreSetup } from '@kbn/core/server';
+import { LockManagerService } from '@kbn/lock-manager';
 import { resourceNames } from '..';
 import { createKbConcreteIndex } from '../startup_migrations/create_or_update_index_assets';
-import { LockManagerService } from '../distributed_lock_manager/lock_manager_service';
 import { ObservabilityAIAssistantPluginStartDependencies } from '../../types';
 
 export const KB_REINDEXING_LOCK_ID = 'observability_ai_assistant:kb_reindexing';

x-pack/platform/plugins/shared/observability_ai_assistant/server/service/startup_migrations/populate_missing_semantic_text_field_migration.ts

@@ -10,14 +10,13 @@ import pLimit from 'p-limit';
 import type { CoreSetup, Logger } from '@kbn/core/server';
 import { uniq } from 'lodash';
 import pRetry from 'p-retry';
+import { LockAcquisitionError, LockManagerService } from '@kbn/lock-manager';
 import { KnowledgeBaseEntry } from '../../../common';
 import { resourceNames } from '..';
 import { waitForKbModel } from '../inference_endpoint';
 import { ObservabilityAIAssistantPluginStartDependencies } from '../../types';
 import { ObservabilityAIAssistantConfig } from '../../config';
 import { reIndexKnowledgeBaseWithLock } from '../knowledge_base_service/reindex_knowledge_base';
-import { LockManagerService } from '../distributed_lock_manager/lock_manager_service';
-import { LockAcquisitionError } from '../distributed_lock_manager/lock_manager_client';
 
 const PLUGIN_STARTUP_LOCK_ID = 'observability_ai_assistant:startup_migrations';
 

x-pack/platform/plugins/shared/observability_ai_assistant/tsconfig.json

@@ -53,7 +53,8 @@
     "@kbn/ai-assistant-icon",
     "@kbn/core-http-browser",
     "@kbn/sse-utils",
-    "@kbn/core-security-server"
+    "@kbn/core-security-server",
+    "@kbn/lock-manager"
   ],
   "exclude": ["target/**/*"]
 }

x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/distributed_lock_manager/distributed_lock_manager.spec.ts

@@ -8,23 +8,25 @@
 import expect from '@kbn/expect';
 import { v4 as uuid } from 'uuid';
 import prettyMilliseconds from 'pretty-ms';
-import {
-  LockId,
-  LockManager,
-  LockDocument,
-  withLock,
-  runSetupIndexAssetEveryTime,
-} from '@kbn/observability-ai-assistant-plugin/server/service/distributed_lock_manager/lock_manager_client';
 import nock from 'nock';
 import { Client } from '@elastic/elasticsearch';
 import { times } from 'lodash';
 import { ToolingLog } from '@kbn/tooling-log';
 import pRetry from 'p-retry';
+import {
+  LockId,
+  LockManager,
+  LockDocument,
+  withLock,
+  runSetupIndexAssetEveryTime,
+} from '@kbn/lock-manager/src/lock_manager_client';
 import {
   LOCKS_COMPONENT_TEMPLATE_NAME,
   LOCKS_CONCRETE_INDEX_NAME,
   LOCKS_INDEX_TEMPLATE_NAME,
-} from '@kbn/observability-ai-assistant-plugin/server/service/distributed_lock_manager/setup_lock_manager_index';
+  setupLockManagerIndex,
+} from '@kbn/lock-manager/src/setup_lock_manager_index';
+
 import type { DeploymentAgnosticFtrProviderContext } from '../../../../ftr_provider_context';
 import { getLoggerMock } from '../utils/logger';
 import { dateAsTimestamp, durationAsMs, sleep } from '../utils/time';
@@ -726,6 +728,40 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon
           expect(settingsAfter?.uuid).to.be(settingsBefore?.uuid);
         });
       });
+
+      describe('when setting up index assets', () => {
+        beforeEach(async () => {
+          await deleteLockIndexAssets(es, log);
+        });
+
+        it('can run in parallel', async () => {
+          try {
+            await Promise.all([
+              setupLockManagerIndex(es, logger),
+              setupLockManagerIndex(es, logger),
+              setupLockManagerIndex(es, logger),
+            ]);
+          } catch (error) {
+            expect().fail(`Parallel setup should not throw but got error: ${error.message}`);
+          }
+
+          const indexExists = await es.indices.exists({ index: LOCKS_CONCRETE_INDEX_NAME });
+          expect(indexExists).to.be(true);
+        });
+
+        it('can run in sequence', async () => {
+          try {
+            await setupLockManagerIndex(es, logger);
+            await setupLockManagerIndex(es, logger);
+            await setupLockManagerIndex(es, logger);
+          } catch (error) {
+            expect().fail(`Sequential setup should not throw but got error: ${error.message}`);
+          }
+
+          const indexExists = await es.indices.exists({ index: LOCKS_CONCRETE_INDEX_NAME });
+          expect(indexExists).to.be(true);
+        });
+      });
     });
   });
 }

x-pack/test/tsconfig.json

@@ -188,5 +188,6 @@
     "@kbn/aiops-change-point-detection",
     "@kbn/es-errors",
     "@kbn/content-packs-schema",
+    "@kbn/lock-manager",
   ]
 }

yarn.lock

@@ -6058,6 +6058,10 @@
   version "0.0.0"
   uid ""
 
+"@kbn/lock-manager@link:packages/kbn-lock-manager":
+  version "0.0.0"
+  uid ""
+
 "@kbn/logging-mocks@link:src/platform/packages/shared/kbn-logging-mocks":
   version "0.0.0"
   uid ""