github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 17:59:23 -04:00
Code Issues Projects Releases Packages Wiki Activity

Fixes flaky backfill tests (#181868)

Browse source 
Resolves https://github.com/elastic/kibana/issues/181778
Resolves https://github.com/elastic/kibana/issues/181861
Resolves https://github.com/elastic/kibana/issues/181862
Resolves https://github.com/elastic/kibana/issues/181875
Resolves https://github.com/elastic/kibana/issues/181840
Resolves https://github.com/elastic/kibana/issues/181893
Resolves https://github.com/elastic/kibana/issues/182029
Resolves https://github.com/elastic/kibana/issues/181907



## Summary

The schedule request tries to schedule backfills for multiple rule
types. The test was testing for a 403 Forbidden message containing a
specific rule type ID (the first one in the schedule request) but in
practice, the rule type that the authorization check runs on is
non-deterministic so could be any of the rule type IDs in the schedule
request. Changed the test to match the error pattern instead.

For the `GET` and `FIND` tests, we're getting 404s when we're expecting
200s so it seems like the backfill runs are completing before the get &
find request are issued. I increased the schedule for the backfill runs
so the jobs don't finish and get deleted before the tests run. I also
changed the pattern for matching `status` to allow for something other
than `pending` status (for backfill schedules that complete before the
GET request is made)

Flaky test runner runs: 
-
https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/5804
-
https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/5806

---------

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Ying Mao 2024-04-30 13:42:47 -04:00 • committed by GitHub
parent 76d0683e2c
commit 717f5d6910
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 107 additions and 130 deletions
Download patch file Download diff file Expand all files Collapse all files

15
x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/backfill/delete.ts
View file

@ -6,6 +6,7 @@
*/
import expect from '@kbn/expect';
import { asyncForEach } from '@kbn/std';
import { GetResponse } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import { UserAtSpaceScenarios } from '../../../../scenarios';
import {
@ -23,9 +24,18 @@ export default function deleteBackfillTests({ getService }: FtrProviderContext)
const supertestWithoutAuth = getService('supertestWithoutAuth');
describe('delete backfill', () => {
let backfillIds: Array<{ id: string; spaceId: string }> = [];
const objectRemover = new ObjectRemover(supertest);
after(() => objectRemover.removeAll());
afterEach(async () => {
asyncForEach(backfillIds, async ({ id, spaceId }: { id: string; spaceId: string }) => {
await supertest
.delete(`${getUrlPrefix(spaceId)}/internal/alerting/rules/backfill/${id}`)
.set('kbn-xsrf', 'foo');
});
backfillIds = [];
await objectRemover.removeAll();
});
function getRule(overwrites = {}) {
return getTestRuleData({
@ -89,6 +99,8 @@ export default function deleteBackfillTests({ getService }: FtrProviderContext)
expect(scheduleResult.length).to.eql(2);
const backfillId1 = scheduleResult[0].id;
const backfillId2 = scheduleResult[1].id;
backfillIds.push({ id: backfillId1, spaceId: apiOptions.spaceId });
backfillIds.push({ id: backfillId2, spaceId: apiOptions.spaceId });
// ensure backfills exist
await supertest
@ -271,6 +283,7 @@ export default function deleteBackfillTests({ getService }: FtrProviderContext)
const scheduleResult = scheduleResponse.body;
expect(scheduleResult.length).to.eql(1);
const backfillId = scheduleResult[0].id;
backfillIds.push({ id: backfillId, spaceId: apiOptions.spaceId });
// delete backfill as current user
const response = await supertestWithoutAuth

84
x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/backfill/find.ts
View file

@ -48,42 +48,49 @@ export default function findBackfillTests({ getService }: FtrProviderContext) {
expect(data.duration).to.eql('12h');
expect(data.enabled).to.eql(true);
expect(data.start).to.eql('2023-10-19T12:00:00.000Z');
expect(data.end).to.eql('2023-10-20T12:00:00.000Z');
expect(data.end).to.eql('2023-10-25T12:00:00.000Z');
expect(data.status).to.eql('pending');
expect(data.space_id).to.eql(spaceId);
expect(typeof data.created_at).to.be('string');
testExpectedRule(data, ruleId, false);
expect(data.schedule).to.eql([
{
run_at: '2023-10-20T00:00:00.000Z',
status: 'pending',
interval: '12h',
},
{
run_at: '2023-10-20T12:00:00.000Z',
status: 'pending',
interval: '12h',
},
]);
expect(data.schedule.length).to.eql(12);
data.schedule.forEach((sched: any) => {
expect(sched.interval).to.eql('12h');
expect(sched.status).to.match(/complete|pending|running|error|timeout/);
});
expect(data.schedule[0].run_at).to.eql('2023-10-20T00:00:00.000Z');
expect(data.schedule[1].run_at).to.eql('2023-10-20T12:00:00.000Z');
expect(data.schedule[2].run_at).to.eql('2023-10-21T00:00:00.000Z');
expect(data.schedule[3].run_at).to.eql('2023-10-21T12:00:00.000Z');
expect(data.schedule[4].run_at).to.eql('2023-10-22T00:00:00.000Z');
expect(data.schedule[5].run_at).to.eql('2023-10-22T12:00:00.000Z');
expect(data.schedule[6].run_at).to.eql('2023-10-23T00:00:00.000Z');
expect(data.schedule[7].run_at).to.eql('2023-10-23T12:00:00.000Z');
expect(data.schedule[8].run_at).to.eql('2023-10-24T00:00:00.000Z');
expect(data.schedule[9].run_at).to.eql('2023-10-24T12:00:00.000Z');
expect(data.schedule[10].run_at).to.eql('2023-10-25T00:00:00.000Z');
expect(data.schedule[11].run_at).to.eql('2023-10-25T12:00:00.000Z');
}
function testExpectedBackfill2(data: any, id: string, ruleId: string, spaceId: string) {
expect(data.id).to.eql(id);
expect(data.duration).to.eql('12h');
expect(data.enabled).to.eql(true);
expect(data.start).to.eql('2023-10-20T11:00:00.000Z');
expect(data.end).to.eql('2023-10-20T23:00:00.000Z');
expect(data.start).to.eql('2023-10-21T12:00:00.000Z');
expect(data.end).to.eql('2023-10-23T12:00:00.000Z');
expect(data.status).to.eql('pending');
expect(data.space_id).to.eql(spaceId);
expect(typeof data.created_at).to.be('string');
testExpectedRule(data, ruleId, false);
expect(data.schedule).to.eql([
{
run_at: '2023-10-20T23:00:00.000Z',
status: 'pending',
interval: '12h',
},
]);
expect(data.schedule.length).to.eql(4);
data.schedule.forEach((sched: any) => {
expect(sched.interval).to.eql('12h');
expect(sched.status).to.match(/complete|pending|running|error|timeout/);
});
expect(data.schedule[0].run_at).to.eql('2023-10-22T00:00:00.000Z');
expect(data.schedule[1].run_at).to.eql('2023-10-22T12:00:00.000Z');
expect(data.schedule[2].run_at).to.eql('2023-10-23T00:00:00.000Z');
expect(data.schedule[3].run_at).to.eql('2023-10-23T12:00:00.000Z');
}
function testExpectedRule(result: any, ruleId: string | undefined, isSO: boolean) {
@ -129,8 +136,7 @@ export default function findBackfillTests({ getService }: FtrProviderContext) {
for (const scenario of UserAtSpaceScenarios) {
const { user, space } = scenario;
// FLAKY: https://github.com/elastic/kibana/issues/181862
describe.skip(scenario.id, () => {
describe(scenario.id, () => {
const apiOptions = {
spaceId: space.id,
username: user.username,
@ -162,9 +168,13 @@ export default function findBackfillTests({ getService }: FtrProviderContext) {
{
rule_id: ruleId1,
start: '2023-10-19T12:00:00.000Z',
end: '2023-10-20T12:00:00.000Z',
end: '2023-10-25T12:00:00.000Z',
},
{
rule_id: ruleId2,
start: '2023-10-21T12:00:00.000Z',
end: '2023-10-23T12:00:00.000Z',
},
{ rule_id: ruleId2, start: '2023-10-20T11:00:00.000Z' },
]);
const scheduleResult = scheduleResponse.body;
@ -245,7 +255,7 @@ export default function findBackfillTests({ getService }: FtrProviderContext) {
.post(
`${getUrlPrefix(
apiOptions.spaceId
)}/internal/alerting/rules/backfill/_find?start=2023-10-21T08:00:00.000Z`
)}/internal/alerting/rules/backfill/_find?start=2023-10-22T08:00:00.000Z`
)
.set('kbn-xsrf', 'foo')
.auth(apiOptions.username, apiOptions.password);
@ -255,7 +265,7 @@ export default function findBackfillTests({ getService }: FtrProviderContext) {
.post(
`${getUrlPrefix(
apiOptions.spaceId
)}/internal/alerting/rules/backfill/_find?end=2023-10-21T00:00:00.000Z`
)}/internal/alerting/rules/backfill/_find?end=2023-10-26T00:00:00.000Z`
)
.set('kbn-xsrf', 'foo')
.auth(apiOptions.username, apiOptions.password);
@ -265,7 +275,7 @@ export default function findBackfillTests({ getService }: FtrProviderContext) {
.post(
`${getUrlPrefix(
apiOptions.spaceId
)}/internal/alerting/rules/backfill/_find?end=2023-10-20T18:00:00.000Z`
)}/internal/alerting/rules/backfill/_find?end=2023-10-24T18:00:00.000Z`
)
.set('kbn-xsrf', 'foo')
.auth(apiOptions.username, apiOptions.password);
@ -285,7 +295,7 @@ export default function findBackfillTests({ getService }: FtrProviderContext) {
.post(
`${getUrlPrefix(
apiOptions.spaceId
)}/internal/alerting/rules/backfill/_find?start=2023-10-19T00:00:00.000Z&end=2023-10-21T00:00:00.000Z`
)}/internal/alerting/rules/backfill/_find?start=2023-10-19T00:00:00.000Z&end=2023-10-26T00:00:00.000Z`
)
.set('kbn-xsrf', 'foo')
.auth(apiOptions.username, apiOptions.password);
@ -295,7 +305,7 @@ export default function findBackfillTests({ getService }: FtrProviderContext) {
.post(
`${getUrlPrefix(
apiOptions.spaceId
)}/internal/alerting/rules/backfill/_find?start=2023-10-19T00:00:00.000Z&end=2023-10-20T13:00:00.000Z`
)}/internal/alerting/rules/backfill/_find?start=2023-10-21T00:00:00.000Z&end=2023-10-23T13:00:00.000Z`
)
.set('kbn-xsrf', 'foo')
.auth(apiOptions.username, apiOptions.password);
@ -502,10 +512,10 @@ export default function findBackfillTests({ getService }: FtrProviderContext) {
expect(resultFindWithEndOneRuleResponse.per_page).to.eql(10);
expect(resultFindWithEndOneRuleResponse.total).to.eql(1);
expect(resultFindWithEndOneRuleResponse.data.length).to.eql(1);
testExpectedBackfill1(
testExpectedBackfill2(
resultFindWithEndOneRuleResponse.data[0],
backfillId1,
ruleId1,
backfillId2,
ruleId2,
space.id
);
@ -540,10 +550,10 @@ export default function findBackfillTests({ getService }: FtrProviderContext) {
expect(resultFindWithStartAndEndOneRuleResponse.per_page).to.eql(10);
expect(resultFindWithStartAndEndOneRuleResponse.total).to.eql(1);
expect(resultFindWithStartAndEndOneRuleResponse.data.length).to.eql(1);
testExpectedBackfill1(
testExpectedBackfill2(
resultFindWithStartAndEndOneRuleResponse.data[0],
backfillId1,
ruleId1,
backfillId2,
ruleId2,
space.id
);

112
x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/backfill/get.ts
View file

@ -86,10 +86,7 @@ export default function getBackfillTests({ getService }: FtrProviderContext) {
for (const scenario of UserAtSpaceScenarios) {
const { user, space } = scenario;
// FLAKY: https://github.com/elastic/kibana/issues/181840
// FLAKY: https://github.com/elastic/kibana/issues/181861
// FLAKY: https://github.com/elastic/kibana/issues/181875
describe.skip(scenario.id, () => {
describe(scenario.id, () => {
const apiOptions = {
spaceId: space.id,
username: user.username,
@ -123,7 +120,11 @@ export default function getBackfillTests({ getService }: FtrProviderContext) {
start: '2023-10-19T12:00:00.000Z',
end: '2023-10-25T12:00:00.000Z',
},
{ rule_id: ruleId2, start: '2023-10-19T12:00:00.000Z' },
{
rule_id: ruleId2,
start: '2023-10-19T12:00:00.000Z',
end: '2023-10-23T12:00:00.000Z',
},
]);
const scheduleResult = scheduleResponse.body;
@ -188,85 +189,46 @@ export default function getBackfillTests({ getService }: FtrProviderContext) {
expect(getResponse1.body.space_id).to.eql(space.id);
expect(typeof getResponse1.body.created_at).to.be('string');
testExpectedRule(getResponse1.body, ruleId1, false);
expect(getResponse1.body.schedule).to.eql([
{
run_at: '2023-10-20T00:00:00.000Z',
status: 'pending',
interval: '12h',
},
{
run_at: '2023-10-20T12:00:00.000Z',
status: 'pending',
interval: '12h',
},
{
run_at: '2023-10-21T00:00:00.000Z',
status: 'pending',
interval: '12h',
},
{
run_at: '2023-10-21T12:00:00.000Z',
status: 'pending',
interval: '12h',
},
{
run_at: '2023-10-22T00:00:00.000Z',
status: 'pending',
interval: '12h',
},
{
run_at: '2023-10-22T12:00:00.000Z',
status: 'pending',
interval: '12h',
},
{
run_at: '2023-10-23T00:00:00.000Z',
status: 'pending',
interval: '12h',
},
{
run_at: '2023-10-23T12:00:00.000Z',
status: 'pending',
interval: '12h',
},
{
run_at: '2023-10-24T00:00:00.000Z',
status: 'pending',
interval: '12h',
},
{
run_at: '2023-10-24T12:00:00.000Z',
status: 'pending',
interval: '12h',
},
{
run_at: '2023-10-25T00:00:00.000Z',
status: 'pending',
interval: '12h',
},
{
run_at: '2023-10-25T12:00:00.000Z',
status: 'pending',
interval: '12h',
},
]);
expect(getResponse1.body.schedule.length).to.eql(12);
getResponse1.body.schedule.forEach((sched: any) => {
expect(sched.interval).to.eql('12h');
expect(sched.status).to.match(/complete|pending|running|error|timeout/);
});
expect(getResponse1.body.schedule[0].run_at).to.eql('2023-10-20T00:00:00.000Z');
expect(getResponse1.body.schedule[1].run_at).to.eql('2023-10-20T12:00:00.000Z');
expect(getResponse1.body.schedule[2].run_at).to.eql('2023-10-21T00:00:00.000Z');
expect(getResponse1.body.schedule[3].run_at).to.eql('2023-10-21T12:00:00.000Z');
expect(getResponse1.body.schedule[4].run_at).to.eql('2023-10-22T00:00:00.000Z');
expect(getResponse1.body.schedule[5].run_at).to.eql('2023-10-22T12:00:00.000Z');
expect(getResponse1.body.schedule[6].run_at).to.eql('2023-10-23T00:00:00.000Z');
expect(getResponse1.body.schedule[7].run_at).to.eql('2023-10-23T12:00:00.000Z');
expect(getResponse1.body.schedule[8].run_at).to.eql('2023-10-24T00:00:00.000Z');
expect(getResponse1.body.schedule[9].run_at).to.eql('2023-10-24T12:00:00.000Z');
expect(getResponse1.body.schedule[10].run_at).to.eql('2023-10-25T00:00:00.000Z');
expect(getResponse1.body.schedule[11].run_at).to.eql('2023-10-25T12:00:00.000Z');
expect(getResponse2.body.id).to.eql(backfillId2);
expect(getResponse2.body.duration).to.eql('12h');
expect(getResponse2.body.enabled).to.eql(true);
expect(getResponse2.body.start).to.eql('2023-10-19T12:00:00.000Z');
expect(getResponse2.body.end).to.eql('2023-10-20T00:00:00.000Z');
expect(getResponse2.body.end).to.eql('2023-10-23T12:00:00.000Z');
expect(getResponse2.body.status).to.eql('pending');
expect(getResponse2.body.space_id).to.eql(space.id);
expect(typeof getResponse2.body.created_at).to.be('string');
testExpectedRule(getResponse2.body, ruleId2, false);
expect(getResponse2.body.schedule).to.eql([
{
run_at: '2023-10-20T00:00:00.000Z',
status: 'pending',
interval: '12h',
},
]);
expect(getResponse2.body.schedule.length).to.eql(8);
getResponse2.body.schedule.forEach((sched: any) => {
expect(sched.interval).to.eql('12h');
expect(sched.status).to.match(/complete|pending|running|error|timeout/);
});
expect(getResponse2.body.schedule[0].run_at).to.eql('2023-10-20T00:00:00.000Z');
expect(getResponse2.body.schedule[1].run_at).to.eql('2023-10-20T12:00:00.000Z');
expect(getResponse2.body.schedule[2].run_at).to.eql('2023-10-21T00:00:00.000Z');
expect(getResponse2.body.schedule[3].run_at).to.eql('2023-10-21T12:00:00.000Z');
expect(getResponse2.body.schedule[4].run_at).to.eql('2023-10-22T00:00:00.000Z');
expect(getResponse2.body.schedule[5].run_at).to.eql('2023-10-22T12:00:00.000Z');
expect(getResponse2.body.schedule[6].run_at).to.eql('2023-10-23T00:00:00.000Z');
expect(getResponse2.body.schedule[7].run_at).to.eql('2023-10-23T12:00:00.000Z');
break;
default:
throw new Error(`Scenario untested: ${JSON.stringify(scenario)}`);

12
x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/backfill/schedule.ts
View file

@ -27,8 +27,7 @@ export default function scheduleBackfillTests({ getService }: FtrProviderContext
const es = getService('es');
const supertestWithoutAuth = getService('supertestWithoutAuth');
// Failing: See https://github.com/elastic/kibana/issues/181778
describe.skip('schedule backfill', () => {
describe('schedule backfill', () => {
let backfillIds: Array<{ id: string; spaceId: string }> = [];
const objectRemover = new ObjectRemover(supertest);
@ -955,11 +954,10 @@ export default function scheduleBackfillTests({ getService }: FtrProviderContext
// User has read privileges in this space
case 'global_read at space1':
expect(response.statusCode).to.eql(403);
expect(response.body).to.eql({
error: 'Forbidden',
message: `Unauthorized by "alertsFixture" to scheduleBackfill "test.patternFiringAutoRecoverFalse" rule`,
statusCode: 403,
});
expect(response.body.error).to.eql('Forbidden');
expect(response.body.message).to.match(
/Unauthorized by "alertsFixture" to scheduleBackfill "[^"]+" rule/
);
break;
// User doesn't have access to actions but that doesn't matter for backfill jobs
case 'space_1_all_alerts_none_actions at space1':

14
x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/backfill/task_runner.ts
View file

@ -422,7 +422,6 @@ export default function createBackfillTaskRunnerTests({ getService }: FtrProvide
{
rule_id: ruleId,
start: '2023-10-19T12:00:00.000Z',
end: '2023-10-20T12:00:00.000Z',
},
])
.expect(200);
@ -430,18 +429,13 @@ export default function createBackfillTaskRunnerTests({ getService }: FtrProvide
const scheduleResult = response2.body;
expect(scheduleResult.length).to.eql(1);
expect(scheduleResult[0].schedule.length).to.eql(2);
expect(scheduleResult[0].schedule.length).to.eql(1);
expect(scheduleResult[0].schedule).to.eql([
{
interval: '12h',
run_at: '2023-10-20T00:00:00.000Z',
status: 'pending',
},
{
interval: '12h',
run_at: '2023-10-20T12:00:00.000Z',
status: 'pending',
},
]);
const backfillId = scheduleResult[0].id;
@ -462,8 +456,8 @@ export default function createBackfillTaskRunnerTests({ getService }: FtrProvide
backfillId,
spaceId,
new Map([
['execute-timeout', { equal: 2 }],
['execute-backfill', { equal: 2 }],
['execute-timeout', { equal: 1 }],
['execute-backfill', { equal: 1 }],
])
);
@ -615,7 +609,7 @@ export default function createBackfillTaskRunnerTests({ getService }: FtrProvide
spaceId,
});
// get the execute-timeout and execute-backfill events
// get the execute-backfill events
const events: IValidatedEvent[] = await waitForEventLogDocs(
backfillId,
spaceId,