github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 17:59:23 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Obs AI Assistant] Add retry statements as an attempt to resolve flaky tests (#200022)

Browse source 
Closes https://github.com/elastic/kibana/issues/192222

## Summary

### Problem
The test appears to be flaky, potentially because the entries are not
available at the time of retrieval. This cannot be reproduced locally or
via the flaky test runner. (more details
[here](https://github.com/elastic/kibana/pull/196026#issuecomment-2409056856))

### Solution
Add a retry when fetching the instructions and check whether the number
of instructions returned by the API endpoint is the same number of
instructions expected.

### Checklist

- [x] [Flaky Test
Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was
used on any tests changed
This commit is contained in:
Viduni Wickramarachchi 2024-11-14 11:09:42 -05:00 • committed by GitHub
parent 50f0016cd7
commit 53c05a33e7
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 59 additions and 51 deletions
Download patch file Download diff file Expand all files Collapse all files

110
x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts
View file

@ -30,6 +30,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
const es = getService('es');
const ml = getService('ml');
const log = getService('log');
const retry = getService('retry');
const getScopedApiClientForUsername = getService('getScopedApiClientForUsername');
describe('Knowledge base user instructions', () => {
@ -94,62 +95,69 @@ export default function ApiTest({ getService }: FtrProviderContext) {
});
it('"editor" can retrieve their own private instructions and the public instruction', async () => {
const res = await observabilityAIAssistantAPIClient.editor({
endpoint: 'GET /internal/observability_ai_assistant/kb/user_instructions',
await retry.try(async () => {
const res = await observabilityAIAssistantAPIClient.editor({
endpoint: 'GET /internal/observability_ai_assistant/kb/user_instructions',
});
const instructions = res.body.userInstructions;
expect(instructions).to.have.length(3);
const sortById = (data: Array<Instruction & { public?: boolean }>) => sortBy(data, 'id');
expect(sortById(instructions)).to.eql(
sortById([
{
id: 'private-doc-from-editor',
public: false,
text: 'Private user instruction from "editor"',
},
{
id: 'public-doc-from-editor',
public: true,
text: 'Public user instruction from "editor"',
},
{
id: 'public-doc-from-secondary_editor',
public: true,
text: 'Public user instruction from "secondary_editor"',
},
])
);
});
const instructions = res.body.userInstructions;
const sortById = (data: Array<Instruction & { public?: boolean }>) => sortBy(data, 'id');
expect(sortById(instructions)).to.eql(
sortById([
{
id: 'private-doc-from-editor',
public: false,
text: 'Private user instruction from "editor"',
},
{
id: 'public-doc-from-editor',
public: true,
text: 'Public user instruction from "editor"',
},
{
id: 'public-doc-from-secondary_editor',
public: true,
text: 'Public user instruction from "secondary_editor"',
},
])
);
});
it('"secondaryEditor" can retrieve their own private instructions and the public instruction', async () => {
const res = await observabilityAIAssistantAPIClient.secondaryEditor({
endpoint: 'GET /internal/observability_ai_assistant/kb/user_instructions',
await retry.try(async () => {
const res = await observabilityAIAssistantAPIClient.secondaryEditor({
endpoint: 'GET /internal/observability_ai_assistant/kb/user_instructions',
});
const instructions = res.body.userInstructions;
expect(instructions).to.have.length(3);
const sortById = (data: Array<Instruction & { public?: boolean }>) => sortBy(data, 'id');
expect(sortById(instructions)).to.eql(
sortById([
{
id: 'public-doc-from-editor',
public: true,
text: 'Public user instruction from "editor"',
},
{
id: 'public-doc-from-secondary_editor',
public: true,
text: 'Public user instruction from "secondary_editor"',
},
{
id: 'private-doc-from-secondary_editor',
public: false,
text: 'Private user instruction from "secondary_editor"',
},
])
);
});
const instructions = res.body.userInstructions;
const sortById = (data: Array<Instruction & { public?: boolean }>) => sortBy(data, 'id');
expect(sortById(instructions)).to.eql(
sortById([
{
id: 'public-doc-from-editor',
public: true,
text: 'Public user instruction from "editor"',
},
{
id: 'public-doc-from-secondary_editor',
public: true,
text: 'Public user instruction from "secondary_editor"',
},
{
id: 'private-doc-from-secondary_editor',
public: false,
text: 'Private user instruction from "secondary_editor"',
},
])
);
});
});