Remove is_correction and confidence attributes from kb entry (#222814)

## Summary

Closes https://github.com/elastic/kibana/issues/222555
Remove `confidence` and `is_correction` attributes from knowledge base
entry since they are no longer used.

### Checklist

Check the PR satisfies following conditions. 

Reviewers should verify this PR satisfies this list as well.

- [x] The PR description includes the appropriate Release Notes section,
and the correct `release_note:*` label is applied per the
[guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)

---------

Co-authored-by: Søren Louv-Jansen <sorenlouv@gmail.com>

x-pack/platform/plugins/private/observability_ai_assistant_management/public/hooks/use_import_knowledge_base_entries.ts

@@ -27,10 +27,7 @@ export function useImportKnowledgeBaseEntries() {
     ServerError,
     {
       entries: Array<
-        Omit<
-          KnowledgeBaseEntry,
-          '@timestamp' | 'confidence' | 'is_correction' | 'public' | 'labels'
-        > & { title: string }
+        Omit<KnowledgeBaseEntry, '@timestamp' | 'public' | 'labels'> & { title: string }
       >;
     }
   >(

x-pack/platform/plugins/private/observability_ai_assistant_management/public/routes/components/knowledge_base_edit_manual_entry_flyout.tsx

@@ -62,8 +62,6 @@ export function KnowledgeBaseEditManualEntryFlyout({
         text: newEntryText,
         public: isPublic,
         role: KnowledgeBaseEntryRole.UserEntry,
-        confidence: 'high',
-        is_correction: false,
         labels: {},
       },
     });

x-pack/platform/plugins/private/observability_ai_assistant_management/public/routes/components/knowledge_base_tab.test.tsx

@@ -206,8 +206,6 @@ describe('KnowledgeBaseTab', () => {
             public: false,
             text: 'bar',
             role: 'user_entry',
-            confidence: 'high',
-            is_correction: false,
             labels: expect.any(Object),
           },
         });

x-pack/platform/plugins/shared/observability_ai_assistant/common/types.ts

@@ -99,8 +99,6 @@ export interface KnowledgeBaseEntry {
   id: string;
   title?: string;
   text: string;
-  confidence: 'low' | 'medium' | 'high';
-  is_correction: boolean;
   type?: 'user_instruction' | 'contextual';
   public: boolean;
   labels?: Record<string, string>;
@@ -108,6 +106,8 @@ export interface KnowledgeBaseEntry {
   user?: {
     name: string;
   };
+  confidence?: 'low' | 'medium' | 'high'; // deprecated
+  is_correction?: boolean; // deprecated
 }
 
 export interface Instruction {

x-pack/platform/plugins/shared/observability_ai_assistant/server/functions/summarize.ts

@@ -20,7 +20,6 @@ export function registerSummarizationFunction({
     {
       name: SUMMARIZE_FUNCTION_NAME,
       description: `Use this function to store facts in the knowledge database if the user requests it.
-        You can score the learnings with a confidence metric, whether it is a correction on a previous learning.
         An embedding will be created that you can recall later with a semantic search.
         When you create this summarisation, make sure you craft it in a way that can be recalled with a semantic
         search later, and that it would have answered the user's original request.`,
@@ -39,34 +38,16 @@ export function registerSummarizationFunction({
             description:
               "A human-readable summary of what you have learned, described in such a way that you can recall it later with semantic search, and that it would have answered the user's original request.",
           },
-          is_correction: {
-            type: 'boolean',
-            description: 'Whether this is a correction for a previous learning.',
-          },
-          confidence: {
-            type: 'string',
-            description: 'How confident you are about this being a correct and useful learning',
-            enum: ['low' as const, 'medium' as const, 'high' as const],
-          },
           public: {
             type: 'boolean',
             description:
               'Whether this information is specific to the user, or generally applicable to any user of the product',
           },
         },
-        required: [
-          'title' as const,
-          'text' as const,
-          'is_correction' as const,
-          'confidence' as const,
-          'public' as const,
-        ],
+        required: ['title' as const, 'text' as const, 'public' as const],
       },
     },
-    async (
-      { arguments: { title, text, is_correction: isCorrection, confidence, public: isPublic } },
-      signal
-    ) => {
+    async ({ arguments: { title, text, public: isPublic } }, signal) => {
       const id = v4();
       resources.logger.debug(`Creating new knowledge base entry with id: ${id}`);
 
@@ -78,8 +59,6 @@ export function registerSummarizationFunction({
             text,
             public: isPublic,
             role: KnowledgeBaseEntryRole.AssistantSummarization,
-            confidence,
-            is_correction: isCorrection,
             labels: {},
           },
           // signal,

x-pack/platform/plugins/shared/observability_ai_assistant/server/routes/functions/route.ts

@@ -167,8 +167,6 @@ const functionSummariseRoute = createObservabilityAIAssistantServerRoute({
     body: t.type({
       title: t.string,
       text: nonEmptyStringRt,
-      confidence: t.union([t.literal('low'), t.literal('medium'), t.literal('high')]),
-      is_correction: toBooleanRt,
       public: toBooleanRt,
       labels: t.record(t.string, t.string),
     }),
@@ -181,21 +179,12 @@ const functionSummariseRoute = createObservabilityAIAssistantServerRoute({
   handler: async (resources): Promise<void> => {
     const client = await resources.service.getClient({ request: resources.request });
 
-    const {
-      title,
-      confidence,
-      is_correction: isCorrection,
-      text,
-      public: isPublic,
-      labels,
-    } = resources.params.body;
+    const { title, text, public: isPublic, labels } = resources.params.body;
 
     return client.addKnowledgeBaseEntry({
       entry: {
         title,
-        confidence,
         id: v4(),
-        is_correction: isCorrection,
         text,
         public: isPublic,
         labels,

x-pack/platform/plugins/shared/observability_ai_assistant/server/routes/knowledge_base/route.ts

@@ -215,8 +215,6 @@ const knowledgeBaseEntryRt = t.intersection([
     text: nonEmptyStringRt,
   }),
   t.partial({
-    confidence: t.union([t.literal('low'), t.literal('medium'), t.literal('high')]),
-    is_correction: toBooleanRt,
     public: toBooleanRt,
     labels: t.record(t.string, t.string),
     role: t.union([
@@ -243,8 +241,6 @@ const saveKnowledgeBaseEntry = createObservabilityAIAssistantServerRoute({
     const entry = resources.params.body;
     return client.addKnowledgeBaseEntry({
       entry: {
-        confidence: 'high',
-        is_correction: false,
         public: true,
         labels: {},
         role: KnowledgeBaseEntryRole.UserEntry,
@@ -294,8 +290,6 @@ const importKnowledgeBaseEntries = createObservabilityAIAssistantServerRoute({
     }
 
     const entries = resources.params.body.entries.map((entry) => ({
-      confidence: 'high' as const,
-      is_correction: false,
       public: true,
       labels: {},
       role: KnowledgeBaseEntryRole.UserEntry,

x-pack/platform/plugins/shared/observability_ai_assistant/server/service/client/index.ts

@@ -785,10 +785,7 @@ export class ObservabilityAIAssistantClient {
   addUserInstruction = async ({
     entry,
   }: {
-    entry: Omit<
-      KnowledgeBaseEntry,
-      '@timestamp' | 'confidence' | 'is_correction' | 'type' | 'role'
-    >;
+    entry: Omit<KnowledgeBaseEntry, '@timestamp' | 'type' | 'role'>;
   }): Promise<void> => {
     // for now we want to limit the number of user instructions to 1 per user
     // if a user instruction already exists for the user, we get the id and update it
@@ -815,8 +812,6 @@ export class ObservabilityAIAssistantClient {
       user: this.dependencies.user,
       entry: {
         ...entry,
-        confidence: 'high',
-        is_correction: false,
         type: KnowledgeBaseType.UserInstruction,
         labels: {},
         role: KnowledgeBaseEntryRole.UserEntry,

x-pack/platform/plugins/shared/observability_ai_assistant/server/service/index_assets/templates/kb_component_template.ts

@@ -66,8 +66,9 @@ export function getComponentTemplate(inferenceId: string) {
         'ml.tokens': {
           type: 'rank_features',
         },
-        confidence: keyword,
+        confidence: keyword, // deprecated but kept for backwards compatibility
         is_correction: {
+          // deprecated but kept for backwards compatibility
           type: 'boolean',
         },
         public: {

x-pack/platform/plugins/shared/observability_ai_assistant/server/service/knowledge_base_service/index.ts

@@ -47,7 +47,6 @@ export interface RecalledEntry {
   title?: string;
   text: string;
   esScore: number | null;
-  is_correction?: boolean;
   labels?: Record<string, string>;
 }
 
@@ -70,7 +69,7 @@ export class KnowledgeBaseService {
     user?: { name: string };
   }): Promise<RecalledEntry[]> {
     const response = await this.dependencies.esClient.asInternalUser.search<
-      Pick<KnowledgeBaseEntry, 'text' | 'is_correction' | 'labels' | 'title'> & { doc_id?: string }
+      Pick<KnowledgeBaseEntry, 'text' | 'labels' | 'title'> & { doc_id?: string }
     >({
       index: [resourceNames.writeIndexAlias.kb],
       query: {
@@ -96,13 +95,12 @@ export class KnowledgeBaseService {
       },
       size: 20,
       _source: {
-        includes: ['text', 'is_correction', 'labels', 'doc_id', 'title'],
+        includes: ['text', 'labels', 'doc_id', 'title'],
       },
     });
 
     return response.hits.hits.map((hit) => ({
       text: hit._source?.text!,
-      is_correction: hit._source?.is_correction,
       labels: hit._source?.labels,
       title: hit._source?.title ?? hit._source?.doc_id, // use `doc_id` as fallback title for backwards compatibility
       esScore: hit._score!,
@@ -284,19 +282,7 @@ export class KnowledgeBaseService {
             : [{ [String(sortBy)]: { order: sortDirection } }],
         size: 500,
         _source: {
-          includes: [
-            'title',
-            'doc_id',
-            'text',
-            'is_correction',
-            'labels',
-            'confidence',
-            'public',
-            '@timestamp',
-            'role',
-            'user.name',
-            'type',
-          ],
+          excludes: ['confidence', 'is_correction'], // fields deprecated in https://github.com/elastic/kibana/pull/222814
         },
       });
 

x-pack/platform/plugins/shared/observability_ai_assistant/server/service/knowledge_base_service/recall_from_search_connectors.ts

@@ -113,7 +113,6 @@ async function recallFromSemanticTextConnectors({
   const results = response.hits.hits.map((hit) => ({
     text: JSON.stringify(hit._source),
     esScore: hit._score!,
-    is_correction: false,
     id: hit._id!,
   }));
 
@@ -199,7 +198,6 @@ async function recallFromLegacyConnectors({
   const results = response.hits.hits.map((hit) => ({
     text: JSON.stringify(hit._source),
     esScore: hit._score!,
-    is_correction: false,
     id: hit._id!,
   }));
 

x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/complete/functions/summarize.spec.ts

@@ -50,8 +50,6 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon
           arguments: JSON.stringify({
             title: 'My Title',
             text: 'Hello world',
-            is_correction: false,
-            confidence: 'high',
             public: false,
           }),
         },

x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/knowledge_base/knowledge_base_basic_operations.spec.ts

@@ -6,10 +6,17 @@
  */
 
 import expect from '@kbn/expect';
-import { type KnowledgeBaseEntry } from '@kbn/observability-ai-assistant-plugin/common';
+import {
+  type KnowledgeBaseEntry,
+  KnowledgeBaseEntryRole,
+} from '@kbn/observability-ai-assistant-plugin/common';
 import { orderBy, size, toPairs } from 'lodash';
 import type { DeploymentAgnosticFtrProviderContext } from '../../../../ftr_provider_context';
-import { clearKnowledgeBase, getKnowledgeBaseEntriesFromEs } from '../utils/knowledge_base';
+import {
+  clearKnowledgeBase,
+  getKnowledgeBaseEntriesFromEs,
+  addKnowledgeBaseEntryToEs,
+} from '../utils/knowledge_base';
 import {
   teardownTinyElserModelAndInferenceEndpoint,
   deployTinyElserAndSetupKb,
@@ -44,6 +51,14 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon
     return res.body.entries;
   }
 
+  async function saveEntry(knowledgeBaseEntry: { title: string; text: string; id: string }) {
+    const { status } = await observabilityAIAssistantAPIClient.editor({
+      endpoint: 'POST /internal/observability_ai_assistant/kb/entries/save',
+      params: { body: knowledgeBaseEntry },
+    });
+    expect(status).to.be(200);
+  }
+
   describe('Knowledge base: Basic operations', function () {
     before(async () => {
       await deployTinyElserAndSetupKb(getService);
@@ -62,11 +77,7 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon
       };
 
       before(async () => {
-        const { status } = await observabilityAIAssistantAPIClient.editor({
-          endpoint: 'POST /internal/observability_ai_assistant/kb/entries/save',
-          params: { body: knowledgeBaseEntry },
-        });
-        expect(status).to.be(200);
+        await saveEntry(knowledgeBaseEntry);
       });
 
       it('can retrieve the entry', async () => {
@@ -77,6 +88,33 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon
         expect(entry.text).to.equal(knowledgeBaseEntry.text);
       });
 
+      it('does not retrieve deprecated properties', async () => {
+        await clearKnowledgeBase(es);
+        await addKnowledgeBaseEntryToEs(es, {
+          confidence: 'high' as const,
+          '@timestamp': new Date().toISOString(),
+          role: KnowledgeBaseEntryRole.UserEntry,
+          is_correction: true,
+          public: false,
+          labels: {},
+          ...knowledgeBaseEntry,
+        });
+
+        const hits = await getKnowledgeBaseEntriesFromEs(es);
+        const hitSource = hits[0]._source;
+        expect(hitSource).to.have.property('is_correction');
+        expect(hitSource).to.have.property('confidence');
+
+        const entries = await getEntries();
+        const entry = entries[0];
+        expect(entry).not.to.have.property('confidence');
+        expect(entry).not.to.have.property('is_correction');
+
+        await clearKnowledgeBase(es);
+
+        await saveEntry(knowledgeBaseEntry);
+      });
+
       it('generates sparse embeddings', async () => {
         const hits = await getKnowledgeBaseEntriesFromEs(es);
         const embeddings =

x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/utils/knowledge_base.ts

@@ -269,6 +269,16 @@ export async function getKnowledgeBaseEntriesFromEs(es: Client) {
   return res.hits.hits;
 }
 
+export async function addKnowledgeBaseEntryToEs(es: Client, entry: KnowledgeBaseEntry) {
+  const result = await es.index({
+    index: resourceNames.writeIndexAlias.kb,
+    document: entry,
+    refresh: true,
+  });
+
+  return result;
+}
+
 export function getKnowledgeBaseEntriesFromApi({
   observabilityAIAssistantAPIClient,
   query = '',

x-pack/test/observability_ai_assistant_functional/tests/knowledge_base_management/index.spec.ts

@@ -44,8 +44,6 @@ export default function ApiTest({ getService, getPageObjects }: FtrProviderConte
         body: {
           title: 'Favourite color',
           text,
-          confidence: 'high',
-          is_correction: false,
           public: false,
           labels: {},
         },