github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 17:28:26 -04:00
Code Issues Projects Releases Packages Wiki Activity

[ResponseOps][Actions] Make actions retry when encountering failures (#143224)

Browse source 
* Updating maxAttempts to be 3

* Setting attempts to 3 in one place

* Fixing test failure

* Setting retry to true

* Fixing tests

* Fixing tests

* Updating a missed test

* Updating test instead of deleting

* Adding constant
This commit is contained in:
doakalexi 2022-10-20 13:55:50 -04:00 • committed by GitHub
parent e42fe559c1
commit b76caca9ef
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
15 changed files with 102 additions and 32 deletions
Download patch file Download diff file Expand all files Collapse all files

2
x-pack/plugins/actions/server/action_type_registry.test.ts
View file

@ -70,7 +70,7 @@ describe('register()', () => {
"actions:my-action-type": Object {
"createTaskRunner": [Function],
"getRetry": [Function],
"maxAttempts": 1,
"maxAttempts": 3,
"title": "My action type",
},
},

4
x-pack/plugins/actions/server/action_type_registry.ts
View file

@ -25,6 +25,8 @@ import {
ActionTypeParams,
} from './types';
export const MAX_ATTEMPTS: number = 3;
export interface ActionTypeRegistryOpts {
licensing: LicensingPluginSetup;
taskManager: TaskManagerSetupContract;
@ -151,7 +153,7 @@ export class ActionTypeRegistry {
this.taskManager.registerTaskDefinitions({
[`actions:${actionType.id}`]: {
title: actionType.name,
maxAttempts: actionType.maxAttempts || 1,
maxAttempts: actionType.maxAttempts || MAX_ATTEMPTS,
getRetry(attempts: number, error: unknown) {
if (error instanceof ExecutorError) {
return error.retry == null ? false : error.retry;

2
x-pack/plugins/actions/server/lib/action_executor.test.ts
View file

@ -540,7 +540,7 @@ test('logs a warning and error when alert executor throws an error', async () =>
executorMock.mockRejectedValue(err);
await actionExecutor.execute(executeParams);
expect(loggerMock.warn).toBeCalledWith(
'action execution failure: test:1: action-1: an error occurred while running the action: this action execution is intended to fail'
'action execution failure: test:1: action-1: an error occurred while running the action: this action execution is intended to fail; retry: true'
);
expect(loggerMock.error).toBeCalledWith(err, {
error: { stack_trace: 'foo error\n stack 1\n stack 2\n stack 3' },

2
x-pack/plugins/actions/server/lib/action_executor.ts
View file

@ -242,7 +242,7 @@ export class ActionExecutor {
message: 'an error occurred while running the action',
serviceMessage: err.message,
error: err,
retry: false,
retry: true,
};
}
}

3
x-pack/plugins/actions/server/plugin.ts
View file

@ -43,7 +43,7 @@ import {
import { ActionsConfig, getValidatedConfig } from './config';
import { resolveCustomHosts } from './lib/custom_host_settings';
import { ActionsClient } from './actions_client';
import { ActionTypeRegistry } from './action_type_registry';
import { ActionTypeRegistry, MAX_ATTEMPTS } from './action_type_registry';
import {
createExecutionEnqueuerFunction,
createEphemeralExecutionEnqueuerFunction,
@ -351,6 +351,7 @@ export class ActionsPlugin implements Plugin<PluginSetupContract, PluginStartCon
actionType: ActionType<Config, Secrets, Params, ExecutorResultData>
) => {
ensureSufficientLicense(actionType);
actionType.maxAttempts = actionType.maxAttempts ?? MAX_ATTEMPTS;
actionTypeRegistry.register(actionType);
},
registerSubActionConnectorType: <

14
x-pack/plugins/stack_connectors/server/connector_types/stack/pagerduty/index.ts
View file

@ -223,6 +223,20 @@ async function executor(
};
}
if (response == null) {
const message = i18n.translate(
'xpack.stackConnectors.pagerduty.unexpectedNullResponseErrorMessage',
{
defaultMessage: 'unexpected null response from pagerduty',
}
);
return {
status: 'error',
actionId,
message,
};
}
logger.debug(`response posting pagerduty event: ${response.status}`);
if (response.status === 202) {

15
x-pack/plugins/stack_connectors/server/connector_types/stack/teams/index.ts
View file

@ -136,6 +136,10 @@ async function teamsExecutor(
})
);
if (result == null) {
return errorResultUnexpectedNullResponse(actionId);
}
if (isOk(result)) {
const {
value: { status, statusText, data: responseData, headers: responseHeaders },
@ -206,6 +210,17 @@ function errorResultInvalid(
};
}
function errorResultUnexpectedNullResponse(actionId: string): ConnectorTypeExecutorResult<void> {
const message = i18n.translate('xpack.stackConnectors.teams.unexpectedNullResponseErrorMessage', {
defaultMessage: 'unexpected null response from Microsoft Teams',
});
return {
status: 'error',
actionId,
message,
};
}
function retryResult(actionId: string, message: string): ConnectorTypeExecutorResult<void> {
const errMessage = i18n.translate(
'xpack.stackConnectors.teams.errorPostingRetryLaterErrorMessage',

18
x-pack/plugins/stack_connectors/server/connector_types/stack/webhook/index.ts
View file

@ -184,6 +184,10 @@ export async function executor(
})
);
if (result == null) {
return errorResultUnexpectedNullResponse(actionId);
}
if (isOk(result)) {
const {
value: { status, statusText },
@ -280,6 +284,20 @@ function errorResultUnexpectedError(actionId: string): ConnectorTypeExecutorResu
};
}
function errorResultUnexpectedNullResponse(actionId: string): ConnectorTypeExecutorResult<void> {
const message = i18n.translate(
'xpack.stackConnectors.webhook.unexpectedNullResponseErrorMessage',
{
defaultMessage: 'unexpected null response from webhook',
}
);
return {
status: 'error',
actionId,
message,
};
}
function retryResult(actionId: string, serviceMessage: string): ConnectorTypeExecutorResult<void> {
const errMessage = i18n.translate(
'xpack.stackConnectors.webhook.invalidResponseRetryLaterErrorMessage',

14
x-pack/plugins/stack_connectors/server/connector_types/stack/xmatters/index.ts
View file

@ -274,6 +274,20 @@ export async function executor(
};
}
if (result == null) {
const message = i18n.translate(
'xpack.stackConnectors.xmatters.unexpectedNullResponseErrorMessage',
{
defaultMessage: 'unexpected null response from xmatters',
}
);
return {
status: 'error',
actionId,
message,
};
}
if (result.status >= 200 && result.status < 300) {
const { status, statusText } = result;
logger.debug(`Response from xMatters action "${actionId}": [HTTP ${status}] ${statusText}`);

12
x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/connector_types/cases/cases_webhook.ts
View file

@ -456,7 +456,7 @@ export default function casesWebhookTest({ getService }: FtrProviderContext) {
expect(resp.body).to.eql({
connector_id: simulatedActionId,
status: 'error',
retry: false,
retry: true,
message: 'an error occurred while running the action',
service_message:
'[Action][Webhook - Case Management]: Unable to create case. Error: JSON Error: Create case JSON body must be valid JSON. ',
@ -486,7 +486,7 @@ export default function casesWebhookTest({ getService }: FtrProviderContext) {
expect(resp.body).to.eql({
connector_id: simulatedActionId,
status: 'error',
retry: false,
retry: true,
message: 'an error occurred while running the action',
service_message:
'[Action][Webhook - Case Management]: Unable to update case with id 12345. Error: JSON Error: Update case JSON body must be valid JSON. ',
@ -553,7 +553,7 @@ export default function casesWebhookTest({ getService }: FtrProviderContext) {
expect(resp.body).to.eql({
connector_id: simulatedActionId,
status: 'error',
retry: false,
retry: true,
message: 'an error occurred while running the action',
service_message:
'[Action][Webhook - Case Management]: Unable to create comment at case with id 123. Error: JSON Error: Create comment JSON body must be valid JSON. ',
@ -620,7 +620,7 @@ export default function casesWebhookTest({ getService }: FtrProviderContext) {
expect(resp.body).to.eql({
connector_id: simulatedActionId,
status: 'error',
retry: false,
retry: true,
message: 'an error occurred while running the action',
service_message:
'[Action][Webhook - Case Management]: Unable to create case. Error: Invalid Create case URL: Error: Invalid protocol. ',
@ -650,7 +650,7 @@ export default function casesWebhookTest({ getService }: FtrProviderContext) {
expect(resp.body).to.eql({
connector_id: simulatedActionId,
status: 'error',
retry: false,
retry: true,
message: 'an error occurred while running the action',
service_message:
'[Action][Webhook - Case Management]: Unable to update case with id 12345. Error: Invalid Update case URL: Error: Invalid URL. ',
@ -717,7 +717,7 @@ export default function casesWebhookTest({ getService }: FtrProviderContext) {
expect(resp.body).to.eql({
connector_id: simulatedActionId,
status: 'error',
retry: false,
retry: true,
message: 'an error occurred while running the action',
service_message:
'[Action][Webhook - Case Management]: Unable to create comment at case with id 123. Error: Invalid Create comment URL: Error: Invalid URL. ',

18
x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/connector_types/stack/opsgenie.ts
View file

@ -181,7 +181,7 @@ export default function opsgenieTest({ getService }: FtrProviderContext) {
expect(body).to.eql({
connector_id: opsgenieActionId,
status: 'error',
retry: false,
retry: true,
message: 'an error occurred while running the action',
service_message: `Sub action "invalidAction" is not registered. Connector id: ${opsgenieActionId}. Connector name: Opsgenie. Connector type: .opsgenie`,
});
@ -199,7 +199,7 @@ export default function opsgenieTest({ getService }: FtrProviderContext) {
expect(body).to.eql({
connector_id: opsgenieActionId,
status: 'error',
retry: false,
retry: true,
message: 'an error occurred while running the action',
service_message:
'Request validation failed (Error: [message]: expected value of type [string] but got [undefined])',
@ -218,7 +218,7 @@ export default function opsgenieTest({ getService }: FtrProviderContext) {
expect(body).to.eql({
connector_id: opsgenieActionId,
status: 'error',
retry: false,
retry: true,
message: 'an error occurred while running the action',
service_message:
'Request validation failed (Error: [alias]: expected value of type [string] but got [undefined])',
@ -250,7 +250,7 @@ export default function opsgenieTest({ getService }: FtrProviderContext) {
expect(body).to.eql({
connector_id: opsgenieActionId,
status: 'error',
retry: false,
retry: true,
message: 'an error occurred while running the action',
service_message:
'Request validation failed (Error: [responders.0]: types that failed validation:\n- [responders.0.0.type]: types that failed validation:\n - [responders.0.type.0]: expected value to equal [team]\n - [responders.0.type.1]: expected value to equal [user]\n - [responders.0.type.2]: expected value to equal [escalation]\n - [responders.0.type.3]: expected value to equal [schedule]\n- [responders.0.1.id]: expected value of type [string] but got [undefined])',
@ -279,7 +279,7 @@ export default function opsgenieTest({ getService }: FtrProviderContext) {
expect(body).to.eql({
connector_id: opsgenieActionId,
status: 'error',
retry: false,
retry: true,
message: 'an error occurred while running the action',
service_message:
'Request validation failed (Error: [responders.0]: types that failed validation:\n- [responders.0.0.name]: expected value of type [string] but got [undefined]\n- [responders.0.1.id]: expected value of type [string] but got [undefined])',
@ -381,7 +381,7 @@ export default function opsgenieTest({ getService }: FtrProviderContext) {
expect(body).to.eql({
connector_id: opsgenieActionId,
status: 'error',
retry: false,
retry: true,
message: 'an error occurred while running the action',
service_message:
'Request validation failed (Error: [visibleTo.0]: types that failed validation:\n- [visibleTo.0.0.type]: expected value to equal [team]\n- [visibleTo.0.1.id]: expected value of type [string] but got [undefined]\n- [visibleTo.0.2.id]: expected value of type [string] but got [undefined]\n- [visibleTo.0.3.username]: expected value of type [string] but got [undefined])',
@ -445,7 +445,7 @@ export default function opsgenieTest({ getService }: FtrProviderContext) {
expect(body).to.eql({
connector_id: opsgenieActionId,
status: 'error',
retry: false,
retry: true,
message: 'an error occurred while running the action',
service_message:
'Request validation failed (Error: [details.bananas]: expected value of type [string] but got [number])',
@ -680,7 +680,7 @@ export default function opsgenieTest({ getService }: FtrProviderContext) {
expect(body).to.eql({
status: 'error',
message: 'an error occurred while running the action',
retry: false,
retry: true,
connector_id: opsgenieActionId,
service_message: 'Status code: undefined. Message: Message: failed',
});
@ -702,7 +702,7 @@ export default function opsgenieTest({ getService }: FtrProviderContext) {
expect(body).to.eql({
status: 'error',
message: 'an error occurred while running the action',
retry: false,
retry: true,
connector_id: opsgenieActionId,
service_message: 'Status code: undefined. Message: Message: failed',
});

10
x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/sub_action_framework/index.ts
View file

@ -166,7 +166,7 @@ export default function createActionTests({ getService }: FtrProviderContext) {
expect(execRes.body).to.eql({
status: 'error',
message: 'an error occurred while running the action',
retry: false,
retry: true,
connector_id: res.body.id,
service_message:
'Request validation failed (Error: [id]: expected value of type [string] but got [undefined])',
@ -245,7 +245,7 @@ export default function createActionTests({ getService }: FtrProviderContext) {
expect(execRes.body).to.eql({
status: 'error',
message: 'an error occurred while running the action',
retry: false,
retry: true,
connector_id: res.body.id,
service_message: `Sub action \"notRegistered\" is not registered. Connector id: ${res.body.id}. Connector name: Test: Sub action connector. Connector type: .test-sub-action-connector`,
});
@ -265,7 +265,7 @@ export default function createActionTests({ getService }: FtrProviderContext) {
expect(execRes.body).to.eql({
status: 'error',
message: 'an error occurred while running the action',
retry: false,
retry: true,
connector_id: res.body.id,
service_message: `Method \"notAFunction\" does not exists in service. Sub action: \"notAFunction\". Connector id: ${res.body.id}. Connector name: Test: Sub action connector. Connector type: .test-sub-action-connector`,
});
@ -285,7 +285,7 @@ export default function createActionTests({ getService }: FtrProviderContext) {
expect(execRes.body).to.eql({
status: 'error',
message: 'an error occurred while running the action',
retry: false,
retry: true,
connector_id: res.body.id,
service_message: `Method \"notExist\" does not exists in service. Sub action: \"notExist\". Connector id: ${res.body.id}. Connector name: Test: Sub action connector. Connector type: .test-sub-action-connector`,
});
@ -308,7 +308,7 @@ export default function createActionTests({ getService }: FtrProviderContext) {
expect(execRes.body).to.eql({
status: 'error',
message: 'an error occurred while running the action',
retry: false,
retry: true,
connector_id: res.body.id,
service_message: 'You should register at least one subAction for your connector type',
});

12
x-pack/test/alerting_api_integration/spaces_only/tests/actions/enqueue.ts
View file

@ -70,7 +70,7 @@ export default function ({ getService }: FtrProviderContext) {
await esTestIndexTool.waitForDocs('action:test.index-record', reference, 1);
});
it('should cleanup task after a failure', async () => {
it('should retry task after a failure', async () => {
const testStart = new Date().toISOString();
const { body: createdAction } = await supertest
.post(`${getUrlPrefix(Spaces.space1.id)}/api/actions/connector`)
@ -85,6 +85,7 @@ export default function ({ getService }: FtrProviderContext) {
objectRemover.add(Spaces.space1.id, createdAction.id, 'action', 'actions');
const reference = `actions-enqueue-2:${Spaces.space1.id}:${createdAction.id}`;
let runAt: number;
await supertest
.post(
`${getUrlPrefix(Spaces.space1.id)}/api/alerts_fixture/${createdAction.id}/enqueue_action`
@ -96,7 +97,10 @@ export default function ({ getService }: FtrProviderContext) {
index: ES_TEST_INDEX_NAME,
},
})
.expect(204);
.expect(204)
.then(() => {
runAt = Date.now();
});
await esTestIndexTool.waitForDocs('action:test.failing', reference, 1);
await retry.try(async () => {
@ -123,7 +127,9 @@ export default function ({ getService }: FtrProviderContext) {
},
},
});
expect((searchResult.hits.total as estypes.SearchTotalHits).value).to.eql(0);
const hit = searchResult.hits.hits as Array<estypes.SearchHit<any>>;
expect(Date.parse(hit[0]._source.task.runAt)).to.greaterThan(runAt);
expect(Date.parse(hit[0]._source.task.attempts)).to.greaterThan(1);
});
});

6
x-pack/test/alerting_api_integration/spaces_only/tests/actions/execute.ts
View file

@ -133,7 +133,7 @@ export default function ({ getService }: FtrProviderContext) {
status: 'error',
message: 'an error occurred while running the action',
service_message: `expected failure for ${ES_TEST_INDEX_NAME} ${reference}`,
retry: false,
retry: true,
});
await validateEventLog({
@ -142,7 +142,7 @@ export default function ({ getService }: FtrProviderContext) {
actionTypeId: 'test.failing',
outcome: 'failure',
message: `action execution failure: test.failing:${createdAction.id}: failing action`,
errorMessage: `an error occurred while running the action: expected failure for .kibana-alerting-test-data actions-failure-1:space1`,
errorMessage: `an error occurred while running the action: expected failure for .kibana-alerting-test-data actions-failure-1:space1; retry: true`,
});
});
@ -327,7 +327,7 @@ export default function ({ getService }: FtrProviderContext) {
status: 'error',
message: 'an error occurred while running the action',
serviceMessage: `expected failure for ${ES_TEST_INDEX_NAME} ${reference}`,
retry: false,
retry: true,
});
});
});

2
x-pack/test/alerting_api_integration/spaces_only/tests/alerting/get_action_error_log.ts
View file

@ -115,7 +115,7 @@ export default function createGetActionErrorLogTests({ getService }: FtrProvider
for (const errors of response.body.errors) {
expect(errors.type).to.equal('actions');
expect(errors.message).to.equal(
`action execution failure: test.throw:${createdConnector.id}: connector that throws - an error occurred while running the action: this action is intended to fail`
`action execution failure: test.throw:${createdConnector.id}: connector that throws - an error occurred while running the action: this action is intended to fail; retry: true`
);
}
});