[LockManager] Add isLockAcquisitionError helper (#220067)

- Adds a small helper method for handling errors of type
`LockAcquisitionError`.
- Flips retry logic for `populateMissingSemanticTextFieldWithLock ` so
we retry all errors except `LockAcquisitionError`

packages/kbn-lock-manager/index.ts

@@ -7,5 +7,5 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-export { LockAcquisitionError } from './src/lock_manager_client';
+export { LockAcquisitionError, isLockAcquisitionError } from './src/lock_manager_client';
 export { LockManagerService } from './src/lock_manager_service';

packages/kbn-lock-manager/src/lock_manager_client.ts

@@ -269,6 +269,10 @@ export async function getLock({
   return lockManager.get();
 }
 
+export function isLockAcquisitionError(error: unknown): error is LockAcquisitionError {
+  return error instanceof LockAcquisitionError;
+}
+
 export async function withLock<T>(
   {
     esClient,

x-pack/platform/plugins/shared/observability_ai_assistant/server/service/client/index.ts

@@ -31,8 +31,7 @@ import { v4 } from 'uuid';
 import type { AssistantScope } from '@kbn/ai-assistant-common';
 import type { InferenceClient } from '@kbn/inference-plugin/server';
 import { ChatCompleteResponse, FunctionCallingMode, ToolChoiceType } from '@kbn/inference-common';
-
-import { LockAcquisitionError } from '@kbn/lock-manager';
+import { isLockAcquisitionError } from '@kbn/lock-manager';
 import { resourceNames } from '..';
 import {
   ChatCompletionChunkEvent,
@@ -730,8 +729,7 @@ export class ObservabilityAIAssistantClient {
         });
       })
       .catch((e) => {
-        const isLockAcquisitionError = e instanceof LockAcquisitionError;
-        if (isLockAcquisitionError) {
+        if (isLockAcquisitionError(e)) {
           logger.info(e.message);
         } else {
           logger.error(

x-pack/platform/plugins/shared/observability_ai_assistant/server/service/knowledge_base_service/index.ts

@@ -10,7 +10,7 @@ import type { CoreSetup, ElasticsearchClient, IUiSettingsClient } from '@kbn/cor
 import type { Logger } from '@kbn/logging';
 import { orderBy } from 'lodash';
 import { encode } from 'gpt-tokenizer';
-import { LockAcquisitionError } from '@kbn/lock-manager';
+import { isLockAcquisitionError } from '@kbn/lock-manager';
 import { resourceNames } from '..';
 import {
   Instruction,
@@ -449,7 +449,7 @@ export class KnowledgeBaseService {
           esClient: this.dependencies.esClient,
           inferenceId,
         }).catch((e) => {
-          if (error instanceof LockAcquisitionError) {
+          if (isLockAcquisitionError(e)) {
             this.dependencies.logger.info(`Re-indexing operation is already in progress`);
             return;
           }

x-pack/platform/plugins/shared/observability_ai_assistant/server/service/startup_migrations/run_startup_migrations.ts

@@ -9,7 +9,7 @@ import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
 import type { CoreSetup, Logger } from '@kbn/core/server';
 import pRetry from 'p-retry';
 import { errors } from '@elastic/elasticsearch';
-import { LockAcquisitionError, LockManagerService } from '@kbn/lock-manager';
+import { LockManagerService, isLockAcquisitionError } from '@kbn/lock-manager';
 import { resourceNames } from '..';
 import { ObservabilityAIAssistantPluginStartDependencies } from '../../types';
 import { ObservabilityAIAssistantConfig } from '../../config';
@@ -66,8 +66,10 @@ export async function runStartupMigrations({
           retries: 5,
           minTimeout: 10_000,
           onFailedAttempt: async (error) => {
-            const isLockAcquisitionError = error instanceof LockAcquisitionError;
-            if (!isLockAcquisitionError) {
+            // if the error is a LockAcquisitionError the operation is already in progress and we should not retry
+            // for other errors we should retry
+            // throwing the error will cause pRetry to abort all retries
+            if (isLockAcquisitionError(error)) {
               throw error;
             }
           },
@@ -75,10 +77,11 @@ export async function runStartupMigrations({
       );
     })
     .catch((error) => {
-      const isLockAcquisitionError = error instanceof LockAcquisitionError;
-      if (!isLockAcquisitionError) {
+      // we should propogate the error if it is not a LockAcquisitionError
+      if (!isLockAcquisitionError(error)) {
         throw error;
       }
+      logger.info('Startup migrations are already in progress. Aborting startup migrations');
     });
 }