[8.12] [Obs AI Assistant] More lenient parsing of suggestion scores (#177898) (#178178)

# Backport

This will backport the following commits from `main` to `8.12`:
- [[Obs AI Assistant] More lenient parsing of suggestion scores
(#177898)](https://github.com/elastic/kibana/pull/177898)

<!--- Backport version: 8.9.8 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Dario
Gieselaar","email":"dario.gieselaar@elastic.co"},"sourceCommit":{"committedDate":"2024-03-04T08:39:26Z","message":"[Obs
AI Assistant] More lenient parsing of suggestion scores
(#177898)\n\nCloses #177855. Also adds the scores to `data` for easier
debugging.","sha":"0fd880be2ba63da78505465a999f3102f1ded714","branchLabelMapping":{"^v8.14.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:fix","v8.13.0","v8.14.0","v8.12.3"],"number":177898,"url":"https://github.com/elastic/kibana/pull/177898","mergeCommit":{"message":"[Obs
AI Assistant] More lenient parsing of suggestion scores
(#177898)\n\nCloses #177855. Also adds the scores to `data` for easier
debugging.","sha":"0fd880be2ba63da78505465a999f3102f1ded714"}},"sourceBranch":"main","suggestedTargetBranches":["8.12"],"targetPullRequestStates":[{"branch":"8.13","label":"v8.13.0","labelRegex":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"url":"https://github.com/elastic/kibana/pull/178152","number":178152,"state":"OPEN"},{"branch":"main","label":"v8.14.0","labelRegex":"^v8.14.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/177898","number":177898,"mergeCommit":{"message":"[Obs
AI Assistant] More lenient parsing of suggestion scores
(#177898)\n\nCloses #177855. Also adds the scores to `data` for easier
debugging.","sha":"0fd880be2ba63da78505465a999f3102f1ded714"}},{"branch":"8.12","label":"v8.12.3","labelRegex":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->

x-pack/plugins/observability_ai_assistant/server/functions/parse_suggestion_scores.test.ts

@@ -0,0 +1,69 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import dedent from 'dedent';
+import { parseSuggestionScores } from './parse_suggestion_scores';
+
+describe('parseSuggestionScores', () => {
+  it('parses newlines as separators', () => {
+    expect(
+      parseSuggestionScores(
+        dedent(
+          `0,1
+      2,7
+      3,10`
+        )
+      )
+    ).toEqual([
+      {
+        index: 0,
+        score: 1,
+      },
+      {
+        index: 2,
+        score: 7,
+      },
+      {
+        index: 3,
+        score: 10,
+      },
+    ]);
+  });
+
+  it('parses semi-colons as separators', () => {
+    expect(parseSuggestionScores(`0,1;2,7;3,10`)).toEqual([
+      {
+        index: 0,
+        score: 1,
+      },
+      {
+        index: 2,
+        score: 7,
+      },
+      {
+        index: 3,
+        score: 10,
+      },
+    ]);
+  });
+
+  it('parses spaces as separators', () => {
+    expect(parseSuggestionScores(`0,1 2,7 3,10`)).toEqual([
+      {
+        index: 0,
+        score: 1,
+      },
+      {
+        index: 2,
+        score: 7,
+      },
+      {
+        index: 3,
+        score: 10,
+      },
+    ]);
+  });
+});

x-pack/plugins/observability_ai_assistant/server/functions/parse_suggestion_scores.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export function parseSuggestionScores(scoresAsString: string) {
+  // make sure that spaces, semi-colons etc work as separators as well
+  const scores = scoresAsString
+    .replace(/[^0-9,]/g, ' ')
+    .trim()
+    .split(/\s+/)
+    .map((pair) => {
+      const [index, score] = pair.split(',').map((str) => parseInt(str, 10));
+
+      return {
+        index,
+        score,
+      };
+    });
+
+  return scores;
+}

x-pack/plugins/observability_ai_assistant/server/functions/recall.ts

@@ -17,6 +17,7 @@ import { concatenateOpenAiChunks } from '../../common/utils/concatenate_openai_c
 import { processOpenAiStream } from '../../common/utils/process_openai_stream';
 import type { ObservabilityAIAssistantClient } from '../service/client';
 import { streamIntoObservable } from '../service/util/stream_into_observable';
+import { parseSuggestionScores } from './parse_suggestion_scores';
 
 export function registerRecallFunction({
   client,
@@ -106,13 +107,7 @@ export function registerRecallFunction({
 
       resources.logger.debug(JSON.stringify(suggestions, null, 2));
 
-      if (suggestions.length === 0) {
-        return {
-          content: [] as unknown as Serializable,
-        };
-      }
-
-      const relevantDocuments = await scoreSuggestions({
+      const { relevantDocuments, scores } = await scoreSuggestions({
         suggestions,
         queries: queriesOrUserPrompt,
         messages,
@@ -121,11 +116,21 @@ export function registerRecallFunction({
         signal,
       });
 
+      if (scores.length === 0) {
+        return {
+          content: { learnings: relevantDocuments as unknown as Serializable },
+          data: {
+            scores,
+            suggestions,
+          },
+        };
+      }
+
       resources.logger.debug(`Received ${relevantDocuments.length} relevant documents`);
       resources.logger.debug(JSON.stringify(relevantDocuments, null, 2));
 
       return {
-        content: relevantDocuments as unknown as Serializable,
+        content: { learnings: relevantDocuments as unknown as Serializable },
       };
     }
   );
@@ -243,17 +248,16 @@ async function scoreSuggestions({
     scoreFunctionRequest.message.function_call.arguments
   );
 
-  const scores = scoresAsString.split('\n').map((line) => {
-    const [index, score] = line
-      .split(',')
-      .map((value) => value.trim())
-      .map(Number);
-
-    return { id: suggestions[index].id, score };
+  const scores = parseSuggestionScores(scoresAsString).map(({ index, score }) => {
+    return {
+      id: suggestions[index].id,
+      score,
+    };
   });
 
   if (scores.length === 0) {
-    return [];
+    // seemingly invalid or no scores, return all
+    return { relevantDocuments: suggestions, scores: [] };
   }
 
   const suggestionIds = suggestions.map((document) => document.id);
@@ -269,5 +273,5 @@ async function scoreSuggestions({
     relevantDocumentIds.includes(suggestion.id)
   );
 
-  return relevantDocuments;
+  return { relevantDocuments, scores };
 }