[Obs AI Assistant] Attempt to resolve flaky knowledge based user instructions test (#196026)

Closes https://github.com/elastic/kibana/issues/192222

## Summary

### Problem
The "when creating private and public user instructions" test has been
marked as flaky and has been skipped.
Based on the error recorded in the ticket, 2 possible scenarios could be

- **Race Conditions**: When multiple instructions are created
asynchronously, the instructions might not be assigned to the right user
or role. Data could be overwritten.
- **Data Fetching Issues**: The API might return inconsistent data if
the knowledge base is not properly cleared between tests, or if the
instructions are not properly isolated per user.

### Solution
When running the test locally, the actual output and expected outcome
are the same, therefore the test passes. The flaky test runner didn't
output anything meaningful either.

However, in order to resolve any missing entries, the before hook was
updated to retry adding only the missing entries. Hopefully, this will
help resolve the flakiness.

### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated to match the most common scenarios
- [x] [Flaky Test
Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was
used on any tests changed
This commit is contained in:
Viduni Wickramarachchi 2024-10-31 15:17:08 -04:00 committed by GitHub
parent 287c1ec62e
commit a7a09f798f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194

View file

@ -10,6 +10,7 @@ import { kbnTestConfig } from '@kbn/test';
import { sortBy } from 'lodash';
import { Message, MessageRole } from '@kbn/observability-ai-assistant-plugin/common';
import { CONTEXT_FUNCTION_NAME } from '@kbn/observability-ai-assistant-plugin/server/functions/context';
import { Instruction } from '@kbn/observability-ai-assistant-plugin/common/types';
import { FtrProviderContext } from '../../common/ftr_provider_context';
import {
clearConversations,
@ -51,8 +52,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
await clearConversations(es);
});
// FLAKY: https://github.com/elastic/kibana/issues/192222
describe.skip('when creating private and public user instructions', () => {
describe('when creating private and public user instructions', () => {
before(async () => {
await clearKnowledgeBase(es);
@ -75,6 +75,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
},
].map(async ({ username, isPublic }) => {
const visibility = isPublic ? 'Public' : 'Private';
await getScopedApiClientForUsername(username)({
endpoint: 'PUT /internal/observability_ai_assistant/kb/user_instructions',
params: {
@ -94,9 +95,12 @@ export default function ApiTest({ getService }: FtrProviderContext) {
const res = await observabilityAIAssistantAPIClient.editorUser({
endpoint: 'GET /internal/observability_ai_assistant/kb/user_instructions',
});
const instructions = res.body.userInstructions;
const sortByDocId = (data: any) => sortBy(data, 'doc_id');
const sortByDocId = (data: Array<Instruction & { public?: boolean }>) =>
sortBy(data, 'doc_id');
expect(sortByDocId(instructions)).to.eql(
sortByDocId([
{
@ -124,7 +128,9 @@ export default function ApiTest({ getService }: FtrProviderContext) {
});
const instructions = res.body.userInstructions;
const sortByDocId = (data: any) => sortBy(data, 'doc_id');
const sortByDocId = (data: Array<Instruction & { public?: boolean }>) =>
sortBy(data, 'doc_id');
expect(sortByDocId(instructions)).to.eql(
sortByDocId([
{