github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 17:28:26 -04:00
Code Issues Projects Releases Packages Wiki Activity

[E&C][ES Query] adds runtime mappings and fields support to the ES Query ruletype (#138427)

Browse source 
This PR adds Runtime Fields support to the ES Query Rule Type when using the DSL Query mode.
This commit is contained in:
Gidi Meir Morris 2022-08-22 13:17:46 +01:00 • committed by GitHub
parent ba8a267050
commit 502dc0a4d0
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 738 additions and 100 deletions
Download patch file Download diff file Expand all files Collapse all files

39
docs/user/alerting/rule-types/es-query.asciidoc
View file

@ -1,5 +1,5 @@
[role="xpack"]
[[rule-type-es-query]]
[role="xpack"]
=== {es} query
The {es} query rule type runs a user-configured query, compares the number of
@ -27,8 +27,8 @@ the *time window*.
Size:: Specifies the number of documents to pass to the configured actions when
the threshold condition is met.
{es} query:: Specifies the ES DSL query. The number of documents that
match this query is evaluated against the threshold condition. Only the `query`
field is used, other DSL fields are not considered.
match this query is evaluated against the threshold condition. Only the `query`, `fields` and `runtime_mappings`
fields are used, other DSL fields are not considered.
Threshold:: Defines a threshold value and a comparison operator (`is above`,
`is above or equals`, `is below`, `is below or equals`, or `is between`). The
number of documents that match the specified query is compared to this
@ -74,7 +74,38 @@ over these hits to get values from the ES documents into your actions.
+
[role="screenshot"]
image::images/rule-types-es-query-example-action-variable.png[Iterate over hits using Mustache template syntax]
+
The documents returned by `context.hits` include the {ref}/mapping-source-field.html[`_source`] field.
If the {es} query search API's {ref}/search-fields.html#search-fields-param[`fields`] parameter is used, documents will also return the `fields` field,
which can be used to access any runtime fields defined by the {ref}/runtime-search-request.html[`runtime_mappings`] parameter as the following example shows:
+
--
[source]
--------------------------------------------------
{{#context.hits}}
timestamp: {{_source.@timestamp}}
day of the week: {{fields.day_of_week}} <1>
{{/context.hits}}
--------------------------------------------------
// NOTCONSOLE
<1> The `fields` parameter here is used to access the `day_of_week` runtime field.
--
+
As the {ref}/search-fields.html#search-fields-response[`fields`] response always returns an array of values for each field,
the https://mustache.github.io/[Mustache] template array syntax is used to iterate over these values in your actions as the following example shows:
+
--
[source]
--------------------------------------------------
{{#context.hits}}
Labels:
{{#fields.labels}}
- {{.}}
{{/fields.labels}}
{{/context.hits}}
--------------------------------------------------
// NOTCONSOLE
--
[float]
==== Test your query

57
x-pack/plugins/alerting/server/saved_objects/migrations.ts
View file

@ -7,7 +7,7 @@
import { isRuleType, ruleTypeMappings } from '@kbn/securitysolution-rules';
import { isString } from 'lodash/fp';
import { omit } from 'lodash';
import { omit, pick } from 'lodash';
import moment from 'moment-timezone';
import { gte } from 'semver';
import {
@ -40,7 +40,8 @@ interface AlertLogMeta extends LogMeta {
}
type AlertMigration = (
doc: SavedObjectUnsanitizedDoc<RawRule>
doc: SavedObjectUnsanitizedDoc<RawRule>,
context: SavedObjectMigrationContext
) => SavedObjectUnsanitizedDoc<RawRule>;
function createEsoMigration(
@ -169,6 +170,12 @@ export function getMigrations(
pipeMigrations(addSearchType, removeInternalTags, convertSnoozes)
);
const migrationRules850 = createEsoMigration(
encryptedSavedObjects,
(doc): doc is SavedObjectUnsanitizedDoc<RawRule> => isEsQueryRuleType(doc),
pipeMigrations(stripOutRuntimeFieldsInOldESQuery)
);
return mergeSavedObjectMigrationMaps(
{
'7.10.0': executeMigrationWithErrorHandling(migrationWhenRBACWasIntroduced, '7.10.0'),
@ -182,6 +189,7 @@ export function getMigrations(
'8.0.1': executeMigrationWithErrorHandling(migrationRules801, '8.0.1'),
'8.2.0': executeMigrationWithErrorHandling(migrationRules820, '8.2.0'),
'8.3.0': executeMigrationWithErrorHandling(migrationRules830, '8.3.0'),
'8.5.0': executeMigrationWithErrorHandling(migrationRules850, '8.5.0'),
},
getSearchSourceMigrations(encryptedSavedObjects, searchSourceMigrations)
);
@ -750,6 +758,47 @@ function addSecuritySolutionAADRuleTypeTags(
: doc;
}
function stripOutRuntimeFieldsInOldESQuery(
doc: SavedObjectUnsanitizedDoc<RawRule>,
context: SavedObjectMigrationContext
): SavedObjectUnsanitizedDoc<RawRule> {
const isESDSLrule =
isEsQueryRuleType(doc) && !isSerializedSearchSource(doc.attributes.params.searchConfiguration);
if (isESDSLrule) {
try {
const parsedQuery = JSON.parse(doc.attributes.params.esQuery as string);
// parsing and restringifying will cause us to lose the formatting so we only do so if this rule has
// fields other than `query` which is the only valid field at this stage
const hasFieldsOtherThanQuery = Object.keys(parsedQuery).some((key) => key !== 'query');
return hasFieldsOtherThanQuery
? {
...doc,
attributes: {
...doc.attributes,
params: {
...doc.attributes.params,
esQuery: JSON.stringify(pick(parsedQuery, 'query'), null, 4),
},
},
}
: doc;
} catch (err) {
// Instead of failing the upgrade when an unparsable rule is encountered, we log that the rule caouldn't be migrated and
// as a result legacy parameters might cause the rule to behave differently if it is, in fact, still running at all
context.log.error<AlertLogMeta>(
`unable to migrate and remove legacy runtime fields in rule ${doc.id} due to invalid query: "${doc.attributes.params.esQuery}" - query must be JSON`,
{
migrations: {
alertDocument: doc,
},
}
);
}
}
return doc;
}
function addThreatIndicatorPathToThreatMatchRules(
doc: SavedObjectUnsanitizedDoc<RawRule>
): SavedObjectUnsanitizedDoc<RawRule> {
@ -957,8 +1006,8 @@ function removeInternalTags(
}
function pipeMigrations(...migrations: AlertMigration[]): AlertMigration {
return (doc: SavedObjectUnsanitizedDoc<RawRule>) =>
migrations.reduce((migratedDoc, nextMigration) => nextMigration(migratedDoc), doc);
return (doc: SavedObjectUnsanitizedDoc<RawRule>, context: SavedObjectMigrationContext) =>
migrations.reduce((migratedDoc, nextMigration) => nextMigration(migratedDoc, context), doc);
}
function mapSearchSourceMigrationFunc(

7
x-pack/plugins/stack_alerts/common/build_sorted_events_query.ts
View file

@ -21,6 +21,8 @@ export interface BuildSortedEventsQuery extends BuildSortedEventsQueryOpts {
sortOrder?: 'asc' | 'desc';
searchAfterSortId: string | number | undefined;
timeField: string;
fields?: string[];
runtime_mappings?: unknown;
}
export const buildSortedEventsQuery = ({
@ -35,6 +37,9 @@ export const buildSortedEventsQuery = ({
timeField,
// eslint-disable-next-line @typescript-eslint/naming-convention
track_total_hits,
fields,
// eslint-disable-next-line @typescript-eslint/naming-convention
runtime_mappings,
}: BuildSortedEventsQuery): ESSearchRequest => {
const sortField = timeField;
const docFields = [timeField].map((tstamp) => ({
@ -82,6 +87,8 @@ export const buildSortedEventsQuery = ({
},
],
},
...(runtime_mappings ? { runtime_mappings } : {}),
...(fields ? { fields } : {}),
};
if (searchAfterSortId) {

21
x-pack/plugins/stack_alerts/server/alert_types/es_query/lib/fetch_es_query.ts
View file

@ -26,13 +26,18 @@ export async function fetchEsQuery(
) {
const { scopedClusterClient, logger } = services;
const esClient = scopedClusterClient.asCurrentUser;
const { parsedQuery, dateStart, dateEnd } = getSearchParams(params);
const {
// eslint-disable-next-line @typescript-eslint/naming-convention
parsedQuery: { query, fields, runtime_mappings },
dateStart,
dateEnd,
} = getSearchParams(params);
const filter = timestamp
? {
bool: {
filter: [
parsedQuery.query,
query,
{
bool: {
must_not: [
@ -56,9 +61,9 @@ export async function fetchEsQuery(
],
},
}
: parsedQuery.query;
: query;
const query = buildSortedEventsQuery({
const sortedQuery = buildSortedEventsQuery({
index: params.index,
from: dateStart,
to: dateEnd,
@ -68,11 +73,15 @@ export async function fetchEsQuery(
searchAfterSortId: undefined,
timeField: params.timeField,
track_total_hits: true,
fields,
runtime_mappings,
});
logger.debug(`es query rule ${ES_QUERY_ID}:${ruleId} "${name}" query - ${JSON.stringify(query)}`);
logger.debug(
`es query rule ${ES_QUERY_ID}:${ruleId} "${name}" query - ${JSON.stringify(sortedQuery)}`
);
const { body: searchResult } = await esClient.search(query, { meta: true });
const { body: searchResult } = await esClient.search(sortedQuery, { meta: true });
logger.debug(
` es query rule ${ES_QUERY_ID}:${ruleId} "${name}" result - ${JSON.stringify(searchResult)}`

109
x-pack/test/alerting_api_integration/spaces_only/tests/alerting/builtin_alert_types/es_query/common.ts Normal file
View file

@ -0,0 +1,109 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { FtrProviderContext } from '../../../../../common/ftr_provider_context';
import { Spaces } from '../../../../scenarios';
import {
ESTestIndexTool,
ES_TEST_INDEX_NAME,
getUrlPrefix,
ObjectRemover,
} from '../../../../../common/lib';
import { createEsDocuments } from '../lib/create_test_data';
export const RULE_TYPE_ID = '.es-query';
export const CONNECTOR_TYPE_ID = '.index';
export const ES_TEST_INDEX_SOURCE = 'builtin-rule:es-query';
export const ES_TEST_INDEX_REFERENCE = '-na-';
export const ES_TEST_OUTPUT_INDEX_NAME = `${ES_TEST_INDEX_NAME}-output`;
export const ES_TEST_DATA_STREAM_NAME = 'test-data-stream';
export const RULE_INTERVALS_TO_WRITE = 5;
export const RULE_INTERVAL_SECONDS = 4;
export const RULE_INTERVAL_MILLIS = RULE_INTERVAL_SECONDS * 1000;
export const ES_GROUPS_TO_WRITE = 3;
export async function createConnector(
supertest: any,
objectRemover: ObjectRemover,
index: string
): Promise<string> {
const { body: createdConnector } = await supertest
.post(`${getUrlPrefix(Spaces.space1.id)}/api/actions/connector`)
.set('kbn-xsrf', 'foo')
.send({
name: 'index action for es query FT',
connector_type_id: CONNECTOR_TYPE_ID,
config: {
index,
},
secrets: {},
})
.expect(200);
const connectorId = createdConnector.id;
objectRemover.add(Spaces.space1.id, connectorId, 'connector', 'actions');
return connectorId;
}
export interface CreateRuleParams {
name: string;
size: number;
thresholdComparator: string;
threshold: number[];
timeWindowSize?: number;
esQuery?: string;
timeField?: string;
searchConfiguration?: unknown;
searchType?: 'searchSource';
notifyWhen?: string;
indexName?: string;
}
export function getRuleServices(getService: FtrProviderContext['getService']) {
const retry = getService('retry');
const es = getService('es');
const esTestIndexTool = new ESTestIndexTool(es, retry);
const esTestIndexToolOutput = new ESTestIndexTool(es, retry, ES_TEST_OUTPUT_INDEX_NAME);
const esTestIndexToolDataStream = new ESTestIndexTool(es, retry, ES_TEST_DATA_STREAM_NAME);
async function createEsDocumentsInGroups(
groups: number,
endDate: string,
indexTool: ESTestIndexTool = esTestIndexTool,
indexName: string = ES_TEST_INDEX_NAME
) {
await createEsDocuments(
es,
indexTool,
endDate,
RULE_INTERVALS_TO_WRITE,
RULE_INTERVAL_MILLIS,
groups,
indexName
);
}
async function waitForDocs(count: number): Promise<any[]> {
return await esTestIndexToolOutput.waitForDocs(
ES_TEST_INDEX_SOURCE,
ES_TEST_INDEX_REFERENCE,
count
);
}
return {
retry,
es,
esTestIndexTool,
esTestIndexToolOutput,
esTestIndexToolDataStream,
createEsDocumentsInGroups,
waitForDocs,
};
}

279
x-pack/test/alerting_api_integration/spaces_only/tests/alerting/builtin_alert_types/es_query/query_dsl_only.ts Normal file
View file

@ -0,0 +1,279 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import expect from '@kbn/expect';
import { Spaces } from '../../../../scenarios';
import { FtrProviderContext } from '../../../../../common/ftr_provider_context';
import { ES_TEST_INDEX_NAME, getUrlPrefix, ObjectRemover } from '../../../../../common/lib';
import { createDataStream, deleteDataStream } from '../lib/create_test_data';
import {
createConnector,
CreateRuleParams,
ES_GROUPS_TO_WRITE,
ES_TEST_DATA_STREAM_NAME,
ES_TEST_INDEX_REFERENCE,
ES_TEST_INDEX_SOURCE,
ES_TEST_OUTPUT_INDEX_NAME,
getRuleServices,
RULE_INTERVALS_TO_WRITE,
RULE_INTERVAL_MILLIS,
RULE_INTERVAL_SECONDS,
RULE_TYPE_ID,
} from './common';
// eslint-disable-next-line import/no-default-export
export default function ruleTests({ getService }: FtrProviderContext) {
const supertest = getService('supertest');
const { es, esTestIndexTool, esTestIndexToolOutput, createEsDocumentsInGroups, waitForDocs } =
getRuleServices(getService);
describe('rule', async () => {
let endDate: string;
let connectorId: string;
const objectRemover = new ObjectRemover(supertest);
beforeEach(async () => {
await esTestIndexTool.destroy();
await esTestIndexTool.setup();
await esTestIndexToolOutput.destroy();
await esTestIndexToolOutput.setup();
connectorId = await createConnector(supertest, objectRemover, ES_TEST_OUTPUT_INDEX_NAME);
// write documents in the future, figure out the end date
const endDateMillis = Date.now() + (RULE_INTERVALS_TO_WRITE - 1) * RULE_INTERVAL_MILLIS;
endDate = new Date(endDateMillis).toISOString();
await createDataStream(es, ES_TEST_DATA_STREAM_NAME);
});
afterEach(async () => {
await objectRemover.removeAll();
await esTestIndexTool.destroy();
await esTestIndexToolOutput.destroy();
await deleteDataStream(es, ES_TEST_DATA_STREAM_NAME);
});
it(`runs correctly: runtime fields for esQuery search type`, async () => {
// write documents from now to the future end date in groups
await createEsDocumentsInGroups(ES_GROUPS_TO_WRITE, endDate);
await createRule({
name: 'always fire',
esQuery: `
{
"runtime_mappings": {
"testedValueSquared": {
"type": "long",
"script": {
"source": "emit(doc['testedValue'].value * doc['testedValue'].value);"
}
},
"evenOrOdd": {
"type": "keyword",
"script": {
"source": "emit(doc['testedValue'].value % 2 == 0 ? 'even' : 'odd');"
}
}
},
"fields": ["testedValueSquared", "evenOrOdd"],
"query": {
"match_all": { }
}
}`.replace(`"`, `\"`),
size: 100,
thresholdComparator: '>',
threshold: [-1],
});
const docs = await waitForDocs(2);
for (let i = 0; i < docs.length; i++) {
const doc = docs[i];
const { name, title } = doc._source.params;
expect(name).to.be('always fire');
expect(title).to.be(`rule 'always fire' matched query`);
const hits = JSON.parse(doc._source.hits);
expect(hits).not.to.be.empty();
hits.forEach((hit: any) => {
expect(hit.fields).not.to.be.empty();
expect(hit.fields.testedValueSquared).not.to.be.empty();
// fields returns as an array of values
hit.fields.testedValueSquared.forEach((testedValueSquared: number) => {
expect(hit._source.testedValue * hit._source.testedValue).to.be(testedValueSquared);
});
hit.fields.evenOrOdd.forEach((evenOrOdd: string) => {
expect(hit._source.testedValue % 2 === 0 ? 'even' : 'odd').to.be(evenOrOdd);
});
});
}
});
it(`runs correctly: fetches wildcard fields in esQuery search type`, async () => {
// write documents from now to the future end date in groups
await createEsDocumentsInGroups(ES_GROUPS_TO_WRITE, endDate);
await createRule({
name: 'always fire',
esQuery: `
{
"fields": ["*"],
"query": {
"match_all": { }
}
}`.replace(`"`, `\"`),
size: 100,
thresholdComparator: '>',
threshold: [-1],
});
const docs = await waitForDocs(2);
for (let i = 0; i < docs.length; i++) {
const doc = docs[i];
const { name, title } = doc._source.params;
expect(name).to.be('always fire');
expect(title).to.be(`rule 'always fire' matched query`);
const hits = JSON.parse(doc._source.hits);
expect(hits).not.to.be.empty();
hits.forEach((hit: any) => {
expect(hit.fields).not.to.be.empty();
expect(Object.keys(hit.fields).sort()).to.eql(Object.keys(hit._source).sort());
});
}
});
it(`runs correctly: fetches field formatting in esQuery search type`, async () => {
const reIsNumeric = /^\d+$/;
// write documents from now to the future end date in groups
await createEsDocumentsInGroups(ES_GROUPS_TO_WRITE, endDate);
await createRule({
name: 'always fire',
esQuery: `
{
"fields": [
{
"field": "@timestamp",
"format": "epoch_millis"
}
],
"query": {
"match_all": { }
}
}`.replace(`"`, `\"`),
size: 100,
thresholdComparator: '>',
threshold: [-1],
});
const docs = await waitForDocs(2);
for (let i = 0; i < docs.length; i++) {
const doc = docs[i];
const { name, title } = doc._source.params;
expect(name).to.be('always fire');
expect(title).to.be(`rule 'always fire' matched query`);
const hits = JSON.parse(doc._source.hits);
expect(hits).not.to.be.empty();
hits.forEach((hit: any) => {
expect(hit.fields).not.to.be.empty();
hit.fields['@timestamp'].forEach((timestamp: string) => {
expect(reIsNumeric.test(timestamp)).to.be(true);
});
});
}
});
async function createRule(params: CreateRuleParams): Promise<string> {
const action = {
id: connectorId,
group: 'query matched',
params: {
documents: [
{
source: ES_TEST_INDEX_SOURCE,
reference: ES_TEST_INDEX_REFERENCE,
params: {
name: '{{{rule.name}}}',
value: '{{{context.value}}}',
title: '{{{context.title}}}',
message: '{{{context.message}}}',
},
// wrap in brackets
hits: '[{{context.hits}}]',
date: '{{{context.date}}}',
previousTimestamp: '{{{state.latestTimestamp}}}',
},
],
},
};
const recoveryAction = {
id: connectorId,
group: 'recovered',
params: {
documents: [
{
source: ES_TEST_INDEX_SOURCE,
reference: ES_TEST_INDEX_REFERENCE,
params: {
name: '{{{rule.name}}}',
value: '{{{context.value}}}',
title: '{{{context.title}}}',
message: '{{{context.message}}}',
},
// wrap in brackets
hits: '[{{context.hits}}]',
date: '{{{context.date}}}',
},
],
},
};
const ruleParams =
params.searchType === 'searchSource'
? {
searchConfiguration: params.searchConfiguration,
}
: {
index: [params.indexName || ES_TEST_INDEX_NAME],
timeField: params.timeField || 'date',
esQuery: params.esQuery,
};
const { body: createdRule } = await supertest
.post(`${getUrlPrefix(Spaces.space1.id)}/api/alerting/rule`)
.set('kbn-xsrf', 'foo')
.send({
name: params.name,
consumer: 'alerts',
enabled: true,
rule_type_id: RULE_TYPE_ID,
schedule: { interval: `${RULE_INTERVAL_SECONDS}s` },
actions: [action, recoveryAction],
notify_when: params.notifyWhen || 'onActiveAlert',
params: {
size: params.size,
timeWindowSize: params.timeWindowSize || RULE_INTERVAL_SECONDS * 5,
timeWindowUnit: 's',
thresholdComparator: params.thresholdComparator,
threshold: params.threshold,
searchType: params.searchType,
...ruleParams,
},
})
.expect(200);
const ruleId = createdRule.id;
objectRemover.add(Spaces.space1.id, ruleId, 'rule', 'alerting');
return ruleId;
}
});
}

115
x-pack/test/alerting_api_integration/spaces_only/tests/alerting/builtin_alert_types/es_query/rule.ts
View file

@ -9,35 +9,35 @@ import expect from '@kbn/expect';
import { Spaces } from '../../../../scenarios';
import { FtrProviderContext } from '../../../../../common/ftr_provider_context';
import { ES_TEST_INDEX_NAME, getUrlPrefix, ObjectRemover } from '../../../../../common/lib';
import {
ESTestIndexTool,
ES_TEST_INDEX_NAME,
getUrlPrefix,
ObjectRemover,
} from '../../../../../common/lib';
import { createEsDocuments, createDataStream, deleteDataStream } from '../lib/create_test_data';
const RULE_TYPE_ID = '.es-query';
const CONNECTOR_TYPE_ID = '.index';
const ES_TEST_INDEX_SOURCE = 'builtin-rule:es-query';
const ES_TEST_INDEX_REFERENCE = '-na-';
const ES_TEST_OUTPUT_INDEX_NAME = `${ES_TEST_INDEX_NAME}-output`;
const ES_TEST_DATA_STREAM_NAME = 'test-data-stream';
const RULE_INTERVALS_TO_WRITE = 5;
const RULE_INTERVAL_SECONDS = 4;
const RULE_INTERVAL_MILLIS = RULE_INTERVAL_SECONDS * 1000;
const ES_GROUPS_TO_WRITE = 3;
createConnector,
CreateRuleParams,
ES_GROUPS_TO_WRITE,
ES_TEST_DATA_STREAM_NAME,
ES_TEST_INDEX_REFERENCE,
ES_TEST_INDEX_SOURCE,
ES_TEST_OUTPUT_INDEX_NAME,
getRuleServices,
RULE_INTERVALS_TO_WRITE,
RULE_INTERVAL_MILLIS,
RULE_INTERVAL_SECONDS,
RULE_TYPE_ID,
} from './common';
import { createDataStream, deleteDataStream } from '../lib/create_test_data';
// eslint-disable-next-line import/no-default-export
export default function ruleTests({ getService }: FtrProviderContext) {
const supertest = getService('supertest');
const retry = getService('retry');
const indexPatterns = getService('indexPatterns');
const es = getService('es');
const esTestIndexTool = new ESTestIndexTool(es, retry);
const esTestIndexToolOutput = new ESTestIndexTool(es, retry, ES_TEST_OUTPUT_INDEX_NAME);
const esTestIndexToolDataStream = new ESTestIndexTool(es, retry, ES_TEST_DATA_STREAM_NAME);
const {
es,
esTestIndexTool,
esTestIndexToolOutput,
esTestIndexToolDataStream,
createEsDocumentsInGroups,
waitForDocs,
} = getRuleServices(getService);
describe('rule', async () => {
let endDate: string;
@ -51,7 +51,7 @@ export default function ruleTests({ getService }: FtrProviderContext) {
await esTestIndexToolOutput.destroy();
await esTestIndexToolOutput.setup();
connectorId = await createConnector(supertest, objectRemover);
connectorId = await createConnector(supertest, objectRemover, ES_TEST_OUTPUT_INDEX_NAME);
// write documents in the future, figure out the end date
const endDateMillis = Date.now() + (RULE_INTERVALS_TO_WRITE - 1) * RULE_INTERVAL_MILLIS;
@ -130,7 +130,7 @@ export default function ruleTests({ getService }: FtrProviderContext) {
].forEach(([searchType, initData]) =>
it(`runs correctly: threshold on hit count < > for ${searchType} search type`, async () => {
// write documents from now to the future end date in groups
await createEsDocumentsInGroups(ES_GROUPS_TO_WRITE);
await createEsDocumentsInGroups(ES_GROUPS_TO_WRITE, endDate);
await initData();
const docs = await waitForDocs(2);
@ -222,7 +222,7 @@ export default function ruleTests({ getService }: FtrProviderContext) {
].forEach(([searchType, initData]) =>
it(`runs correctly: use epoch millis - threshold on hit count < > for ${searchType} search type`, async () => {
// write documents from now to the future end date in groups
await createEsDocumentsInGroups(ES_GROUPS_TO_WRITE);
await createEsDocumentsInGroups(ES_GROUPS_TO_WRITE, endDate);
await initData();
const docs = await waitForDocs(2);
@ -333,7 +333,7 @@ export default function ruleTests({ getService }: FtrProviderContext) {
].forEach(([searchType, initData]) =>
it(`runs correctly with query: threshold on hit count < > for ${searchType}`, async () => {
// write documents from now to the future end date in groups
await createEsDocumentsInGroups(ES_GROUPS_TO_WRITE);
await createEsDocumentsInGroups(ES_GROUPS_TO_WRITE, endDate);
await initData();
const docs = await waitForDocs(1);
@ -471,7 +471,7 @@ export default function ruleTests({ getService }: FtrProviderContext) {
// delay to let rule run once before adding data
await new Promise((resolve) => setTimeout(resolve, 3000));
await createEsDocumentsInGroups(1);
await createEsDocumentsInGroups(1, endDate);
const docs = await waitForDocs(2);
const activeDoc = docs[0];
@ -573,6 +573,7 @@ export default function ruleTests({ getService }: FtrProviderContext) {
// write documents from now to the future end date in groups
await createEsDocumentsInGroups(
ES_GROUPS_TO_WRITE,
endDate,
esTestIndexToolDataStream,
ES_TEST_DATA_STREAM_NAME
);
@ -602,44 +603,6 @@ export default function ruleTests({ getService }: FtrProviderContext) {
})
);
async function createEsDocumentsInGroups(
groups: number,
indexTool: ESTestIndexTool = esTestIndexTool,
indexName: string = ES_TEST_INDEX_NAME
) {
await createEsDocuments(
es,
indexTool,
endDate,
RULE_INTERVALS_TO_WRITE,
RULE_INTERVAL_MILLIS,
groups,
indexName
);
}
async function waitForDocs(count: number): Promise<any[]> {
return await esTestIndexToolOutput.waitForDocs(
ES_TEST_INDEX_SOURCE,
ES_TEST_INDEX_REFERENCE,
count
);
}
interface CreateRuleParams {
name: string;
size: number;
thresholdComparator: string;
threshold: number[];
timeWindowSize?: number;
esQuery?: string;
timeField?: string;
searchConfiguration?: unknown;
searchType?: 'searchSource';
notifyWhen?: string;
indexName?: string;
}
async function createRule(params: CreateRuleParams): Promise<string> {
const action = {
id: connectorId,
@ -725,23 +688,3 @@ export default function ruleTests({ getService }: FtrProviderContext) {
}
});
}
async function createConnector(supertest: any, objectRemover: ObjectRemover): Promise<string> {
const { body: createdConnector } = await supertest
.post(`${getUrlPrefix(Spaces.space1.id)}/api/actions/connector`)
.set('kbn-xsrf', 'foo')
.send({
name: 'index action for es query FT',
connector_type_id: CONNECTOR_TYPE_ID,
config: {
index: ES_TEST_OUTPUT_INDEX_NAME,
},
secrets: {},
})
.expect(200);
const connectorId = createdConnector.id;
objectRemover.add(Spaces.space1.id, connectorId, 'connector', 'actions');
return connectorId;
}

58
x-pack/test/alerting_api_integration/spaces_only/tests/alerting/migrations.ts
View file

@ -443,5 +443,63 @@ export default function createGetTests({ getService }: FtrProviderContext) {
expect(response.statusCode).to.equal(200);
expect(response.body._source?.alert?.tags).to.eql(['test-tag-1', 'foo-tag']);
});
it('8.5.0 removes runtime and field params from older ES Query rules', async () => {
const response = await es.get<{
alert: {
params: {
esQuery: string;
};
};
}>(
{
index: '.kibana',
id: 'alert:c8b39c29-d860-43b6-8817-b8058d80ddbc',
},
{ meta: true }
);
expect(response.statusCode).to.eql(200);
expect(response.body._source?.alert?.params?.esQuery).to.eql(
JSON.stringify({ query: { match_all: {} } }, null, 4)
);
});
it('8.5.0 doesnt reformat ES Query rules that dot have a runetime field on them', async () => {
const response = await es.get<{
alert: {
params: {
esQuery: string;
};
};
}>(
{
index: '.kibana',
id: 'alert:62c62b7f-8bf3-4104-a064-6247b7bda44f',
},
{ meta: true }
);
expect(response.statusCode).to.eql(200);
expect(response.body._source?.alert?.params?.esQuery).to.eql(
'{\n\t"query":\n{\n\t"match_all":\n\t{}\n}\n}'
);
});
it('8.5.0 doesnt fail upgrade when an ES Query rule is not parsable', async () => {
const response = await es.get<{
alert: {
params: {
esQuery: string;
};
};
}>(
{
index: '.kibana',
id: 'alert:f0d13f4d-35ae-4554-897a-6392e97bb84c',
},
{ meta: true }
);
expect(response.statusCode).to.eql(200);
expect(response.body._source?.alert?.params?.esQuery).to.eql('{"query":}');
});
});
}

153
x-pack/test/functional/es_archives/alerts/data.json
View file

@ -1041,3 +1041,156 @@
}
}
}
{
"type": "doc",
"value": {
"id": "alert:c8b39c29-d860-43b6-8817-b8058d80ddbc",
"index": ".kibana_1",
"source": {
"alert": {
"name": "Old ESQuery with Runtimefield",
"alertTypeId": ".es-query",
"consumer": "alerts",
"params": {
"esQuery": "{\n \"runtime_mappings\": {\n \"kebabAsSnake\": {\n \"type\": \"keyword\",\n \"script\": {\n \"source\": \"\\\"\\\"\\\"\\\"\"\n }\n }\n },\n \"fields\": [\"kebabAsSnake\"],\n \"query\":{\n \"match_all\" : {}\n }}",
"size": 100,
"timeWindowSize": 5,
"timeWindowUnit": "m",
"threshold": [
1000
],
"thresholdComparator": ">",
"index": [
"kibana_sample_data_ecommerce"
],
"timeField": "order_date"
},
"schedule": {
"interval": "1m"
},
"enabled": true,
"actions": [
],
"throttle": null,
"apiKeyOwner": null,
"createdBy" : "elastic",
"updatedBy" : "elastic",
"createdAt": "2022-03-26T16:04:50.698Z",
"muteAll": false,
"mutedInstanceIds": [],
"scheduledTaskId": "c8b39c29-d860-43b6-8817-b8058d80ddbc",
"tags": []
},
"type": "alert",
"updated_at": "2022-03-26T16:05:55.957Z",
"migrationVersion": {
"alert": "8.0.1"
},
"references": [
]
}
}
}
{
"type": "doc",
"value": {
"id": "alert:62c62b7f-8bf3-4104-a064-6247b7bda44f",
"index": ".kibana_1",
"source": {
"alert": {
"name": "Old ESQuery without a Runtimefield but custom formatting",
"alertTypeId": ".es-query",
"consumer": "alerts",
"params": {
"esQuery": "{\n\t\"query\":\n{\n\t\"match_all\":\n\t{}\n}\n}",
"size": 100,
"timeWindowSize": 5,
"timeWindowUnit": "m",
"threshold": [
1000
],
"thresholdComparator": ">",
"index": [
"kibana_sample_data_ecommerce"
],
"timeField": "order_date"
},
"schedule": {
"interval": "1m"
},
"enabled": true,
"actions": [
],
"throttle": null,
"apiKeyOwner": null,
"createdBy" : "elastic",
"updatedBy" : "elastic",
"createdAt": "2022-03-26T16:04:50.698Z",
"muteAll": false,
"mutedInstanceIds": [],
"scheduledTaskId": "62c62b7f-8bf3-4104-a064-6247b7bda44f",
"tags": []
},
"type": "alert",
"updated_at": "2022-03-26T16:05:55.957Z",
"migrationVersion": {
"alert": "8.0.1"
},
"references": [
]
}
}
}
{
"type": "doc",
"value": {
"id": "alert:f0d13f4d-35ae-4554-897a-6392e97bb84c",
"index": ".kibana_1",
"source": {
"alert": {
"name": "Old ESQuery with unparsable query",
"alertTypeId": ".es-query",
"consumer": "alerts",
"params": {
"esQuery": "{\"query\":}",
"size": 100,
"timeWindowSize": 5,
"timeWindowUnit": "m",
"threshold": [
1000
],
"thresholdComparator": ">",
"index": [
"kibana_sample_data_ecommerce"
],
"timeField": "order_date"
},
"schedule": {
"interval": "1m"
},
"enabled": true,
"actions": [
],
"throttle": null,
"apiKeyOwner": null,
"createdBy" : "elastic",
"updatedBy" : "elastic",
"createdAt": "2022-03-26T16:04:50.698Z",
"muteAll": false,
"mutedInstanceIds": [],
"scheduledTaskId": "f0d13f4d-35ae-4554-897a-6392e97bb84c",
"tags": []
},
"type": "alert",
"updated_at": "2022-03-26T16:05:55.957Z",
"migrationVersion": {
"alert": "8.0.1"
},
"references": [
]
}
}
}