github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 09:48:58 -04:00
Code Issues Projects Releases Packages Wiki Activity

[ResponseOps] [Alerting] Index threshold alert UI does not fill index picker with data streams (#137584)

Browse source 
* Adding data streams to the index picker

* Adding tests

* Adding es query rule tests

* Using promise.all to query in parallel

* Updating combine function

* Fixing test failures

* Reverting parallel changes

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
doakalexi 2022-08-08 13:33:34 -04:00 • committed by GitHub
parent 762962dcf8
commit 7139155c66
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 262 additions and 19 deletions
Download patch file Download diff file Expand all files Collapse all files

27
x-pack/plugins/triggers_actions_ui/server/data/routes/indices.ts
View file

@ -66,15 +66,24 @@ export function createIndicesRoute(logger: Logger, router: IRouter, baseRoute: s
logger.warn(`route ${path} error getting indices from pattern "${pattern}": ${err.message}`);
}
const result = { indices: uniqueCombined(aliases, indices, MAX_INDICES) };
let dataStreams: string[] = [];
try {
dataStreams = await getDataStreamsFromPattern(esClient, pattern);
} catch (err) {
logger.warn(
`route ${path} error getting data streams from pattern "${pattern}": ${err.message}`
);
}
const result = { indices: uniqueCombined(aliases, indices, dataStreams, MAX_INDICES) };
logger.debug(`route ${path} response: ${JSON.stringify(result)}`);
return res.ok({ body: result });
}
}
function uniqueCombined(list1: string[], list2: string[], limit: number) {
const set = new Set(list1.concat(list2));
function uniqueCombined(list1: string[], list2: string[], list3: string[], limit: number) {
const set = new Set(list1.concat(list2).concat(list3));
const result = Array.from(set);
result.sort((string1, string2) => string1.localeCompare(string2));
return result.slice(0, limit);
@ -139,6 +148,18 @@ async function getAliasesFromPattern(
return result;
}
async function getDataStreamsFromPattern(
esClient: ElasticsearchClient,
pattern: string
): Promise<string[]> {
const params = {
name: pattern,
};
const { data_streams: response } = await esClient.indices.getDataStream(params);
return response.map((r) => r.name);
}
interface IndiciesAggregation {
indices: {
buckets: Array<{ key: string }>;

119
x-pack/test/alerting_api_integration/spaces_only/tests/alerting/builtin_alert_types/es_query/rule.ts
View file

@ -15,13 +15,14 @@ import {
getUrlPrefix,
ObjectRemover,
} from '../../../../../common/lib';
import { createEsDocuments } from '../lib/create_test_data';
import { createEsDocuments, createDataStream, deleteDataStream } from '../lib/create_test_data';
const RULE_TYPE_ID = '.es-query';
const CONNECTOR_TYPE_ID = '.index';
const ES_TEST_INDEX_SOURCE = 'builtin-rule:es-query';
const ES_TEST_INDEX_REFERENCE = '-na-';
const ES_TEST_OUTPUT_INDEX_NAME = `${ES_TEST_INDEX_NAME}-output`;
const ES_TEST_DATA_STREAM_NAME = 'test-data-stream';
const RULE_INTERVALS_TO_WRITE = 5;
const RULE_INTERVAL_SECONDS = 4;
@ -36,6 +37,7 @@ export default function ruleTests({ getService }: FtrProviderContext) {
const es = getService('es');
const esTestIndexTool = new ESTestIndexTool(es, retry);
const esTestIndexToolOutput = new ESTestIndexTool(es, retry, ES_TEST_OUTPUT_INDEX_NAME);
const esTestIndexToolDataStream = new ESTestIndexTool(es, retry, ES_TEST_DATA_STREAM_NAME);
describe('rule', async () => {
let endDate: string;
@ -54,12 +56,15 @@ export default function ruleTests({ getService }: FtrProviderContext) {
// write documents in the future, figure out the end date
const endDateMillis = Date.now() + (RULE_INTERVALS_TO_WRITE - 1) * RULE_INTERVAL_MILLIS;
endDate = new Date(endDateMillis).toISOString();
await createDataStream(es, ES_TEST_DATA_STREAM_NAME);
});
afterEach(async () => {
await objectRemover.removeAll();
await esTestIndexTool.destroy();
await esTestIndexToolOutput.destroy();
await deleteDataStream(es, ES_TEST_DATA_STREAM_NAME);
});
[
@ -499,14 +504,117 @@ export default function ruleTests({ getService }: FtrProviderContext) {
})
);
async function createEsDocumentsInGroups(groups: number) {
[
[
'esQuery',
async () => {
await createRule({
name: 'never fire',
esQuery: `{\n \"query\":{\n \"match_all\" : {}\n }\n}`,
size: 100,
thresholdComparator: '<',
threshold: [0],
indexName: ES_TEST_DATA_STREAM_NAME,
timeField: '@timestamp',
});
await createRule({
name: 'always fire',
esQuery: `{\n \"query\":{\n \"match_all\" : {}\n }\n}`,
size: 100,
thresholdComparator: '>',
threshold: [-1],
indexName: ES_TEST_DATA_STREAM_NAME,
timeField: '@timestamp',
});
},
] as const,
[
'searchSource',
async () => {
const esTestDataView = await indexPatterns.create(
{ title: ES_TEST_DATA_STREAM_NAME, timeFieldName: 'date' },
{ override: true },
getUrlPrefix(Spaces.space1.id)
);
await createRule({
name: 'never fire',
size: 100,
thresholdComparator: '<',
threshold: [0],
searchType: 'searchSource',
searchConfiguration: {
query: {
query: '',
language: 'kuery',
},
index: esTestDataView.id,
filter: [],
},
});
await createRule({
name: 'always fire',
size: 100,
thresholdComparator: '>',
threshold: [-1],
searchType: 'searchSource',
searchConfiguration: {
query: {
query: '',
language: 'kuery',
},
index: esTestDataView.id,
filter: [],
},
});
},
] as const,
].forEach(([searchType, initData]) =>
it(`runs correctly over a data stream: threshold on hit count < > for ${searchType} search type`, async () => {
// write documents from now to the future end date in groups
await createEsDocumentsInGroups(
ES_GROUPS_TO_WRITE,
esTestIndexToolDataStream,
ES_TEST_DATA_STREAM_NAME
);
await initData();
const docs = await waitForDocs(2);
for (let i = 0; i < docs.length; i++) {
const doc = docs[i];
const { previousTimestamp, hits } = doc._source;
const { name, title, message } = doc._source.params;
expect(name).to.be('always fire');
expect(title).to.be(`rule 'always fire' matched query`);
const messagePattern =
/rule 'always fire' is active:\n\n- Value: \d+\n- Conditions Met: Number of matching documents is greater than -1 over 20s\n- Timestamp: \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z/;
expect(message).to.match(messagePattern);
expect(hits).not.to.be.empty();
// during the first execution, the latestTimestamp value should be empty
// since this rule always fires, the latestTimestamp value should be updated each execution
if (!i) {
expect(previousTimestamp).to.be.empty();
} else {
expect(previousTimestamp).not.to.be.empty();
}
}
})
);
async function createEsDocumentsInGroups(
groups: number,
indexTool: ESTestIndexTool = esTestIndexTool,
indexName: string = ES_TEST_INDEX_NAME
) {
await createEsDocuments(
es,
esTestIndexTool,
indexTool,
endDate,
RULE_INTERVALS_TO_WRITE,
RULE_INTERVAL_MILLIS,
groups
groups,
indexName
);
}
@ -529,6 +637,7 @@ export default function ruleTests({ getService }: FtrProviderContext) {
searchConfiguration?: unknown;
searchType?: 'searchSource';
notifyWhen?: string;
indexName?: string;
}
async function createRule(params: CreateRuleParams): Promise<string> {
@ -581,7 +690,7 @@ export default function ruleTests({ getService }: FtrProviderContext) {
searchConfiguration: params.searchConfiguration,
}
: {
index: [ES_TEST_INDEX_NAME],
index: [params.indexName || ES_TEST_INDEX_NAME],
timeField: params.timeField || 'date',
esQuery: params.esQuery,
};

56
x-pack/test/alerting_api_integration/spaces_only/tests/alerting/builtin_alert_types/index_threshold/alert.ts
View file

@ -16,12 +16,14 @@ import {
ObjectRemover,
} from '../../../../../common/lib';
import { createEsDocuments } from './create_test_data';
import { createDataStream, deleteDataStream } from '../lib/create_test_data';
const RULE_TYPE_ID = '.index-threshold';
const CONNECTOR_TYPE_ID = '.index';
const ES_TEST_INDEX_SOURCE = 'builtin-alert:index-threshold';
const ES_TEST_INDEX_REFERENCE = '-na-';
const ES_TEST_OUTPUT_INDEX_NAME = `${ES_TEST_INDEX_NAME}-output`;
const ES_TEST_DATA_STREAM_NAME = 'test-data-stream';
const RULE_INTERVALS_TO_WRITE = 5;
const RULE_INTERVAL_SECONDS = 3;
@ -55,12 +57,15 @@ export default function ruleTests({ getService }: FtrProviderContext) {
// write documents from now to the future end date in 3 groups
await createEsDocumentsInGroups(3);
await createDataStream(es, ES_TEST_DATA_STREAM_NAME);
});
afterEach(async () => {
await objectRemover.removeAll();
await esTestIndexTool.destroy();
await esTestIndexToolOutput.destroy();
await deleteDataStream(es, ES_TEST_DATA_STREAM_NAME);
});
// The tests below create two alerts, one that will fire, one that will
@ -354,14 +359,58 @@ export default function ruleTests({ getService }: FtrProviderContext) {
);
});
async function createEsDocumentsInGroups(groups: number) {
it('runs correctly over a data stream: count all < >', async () => {
await createRule({
name: 'never fire',
aggType: 'count',
groupBy: 'all',
thresholdComparator: '<',
threshold: [0],
indexName: ES_TEST_DATA_STREAM_NAME,
timeField: '@timestamp',
});
await createRule({
name: 'always fire',
aggType: 'count',
groupBy: 'all',
thresholdComparator: '>',
threshold: [0],
indexName: ES_TEST_DATA_STREAM_NAME,
timeField: '@timestamp',
});
await createEsDocumentsInGroups(1, ES_TEST_DATA_STREAM_NAME);
const docs = await waitForDocs(2);
for (const doc of docs) {
const { group } = doc._source;
const { name, title, message } = doc._source.params;
expect(name).to.be('always fire');
expect(group).to.be('all documents');
// we'll check title and message in this test, but not subsequent ones
expect(title).to.be('alert always fire group all documents met threshold');
const messagePattern =
/alert 'always fire' is active for group \'all documents\':\n\n- Value: \d+\n- Conditions Met: count is greater than 0 over 15s\n- Timestamp: \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z/;
expect(message).to.match(messagePattern);
}
});
async function createEsDocumentsInGroups(
groups: number,
indexName: string = ES_TEST_INDEX_NAME
) {
await createEsDocuments(
es,
esTestIndexTool,
endDate,
RULE_INTERVALS_TO_WRITE,
RULE_INTERVAL_MILLIS,
groups
groups,
indexName
);
}
@ -385,6 +434,7 @@ export default function ruleTests({ getService }: FtrProviderContext) {
thresholdComparator: string;
threshold: number[];
notifyWhen?: string;
indexName?: string;
}
async function createRule(params: CreateRuleParams): Promise<string> {
@ -445,7 +495,7 @@ export default function ruleTests({ getService }: FtrProviderContext) {
actions: [action, recoveryAction],
notify_when: params.notifyWhen || 'onActiveAlert',
params: {
index: ES_TEST_INDEX_NAME,
index: params.indexName || ES_TEST_INDEX_NAME,
timeField: params.timeField || 'date',
aggType: params.aggType,
aggField: params.aggField,

12
x-pack/test/alerting_api_integration/spaces_only/tests/alerting/builtin_alert_types/index_threshold/create_test_data.ts
View file

@ -29,7 +29,8 @@ export async function createEsDocuments(
endDate: string = END_DATE,
intervals: number = 1,
intervalMillis: number = 1000,
groups: number = 2
groups: number = 2,
indexName: string = ES_TEST_INDEX_NAME
) {
const endDateMillis = Date.parse(endDate) - intervalMillis / 2;
@ -42,7 +43,7 @@ export async function createEsDocuments(
// don't need await on these, wait at the end of the function
times(groups, (group) => {
promises.push(createEsDocument(es, date, testedValue + group, `group-${group}`));
promises.push(createEsDocument(es, date, testedValue + group, `group-${group}`, indexName));
});
});
await Promise.all(promises);
@ -55,7 +56,8 @@ async function createEsDocument(
es: Client,
epochMillis: number,
testedValue: number,
group: string
group: string,
indexName: string
) {
const document = {
source: DOCUMENT_SOURCE,
@ -64,12 +66,14 @@ async function createEsDocument(
date_epoch_millis: epochMillis,
testedValue,
group,
'@timestamp': new Date(epochMillis).toISOString(),
};
const response = await es.index({
id: uuid(),
index: ES_TEST_INDEX_NAME,
index: indexName,
refresh: 'wait_for',
op_type: 'create',
body: document,
});
// console.log(`writing document to ${ES_TEST_INDEX_NAME}:`, JSON.stringify(document, null, 4));

10
x-pack/test/alerting_api_integration/spaces_only/tests/alerting/builtin_alert_types/index_threshold/indices_endpoint.ts
View file

@ -11,6 +11,7 @@ import { Spaces } from '../../../../scenarios';
import { FtrProviderContext } from '../../../../../common/ftr_provider_context';
import { ESTestIndexTool, ES_TEST_INDEX_NAME, getUrlPrefix } from '../../../../../common/lib';
import { createEsDocuments } from './create_test_data';
import { createDataStream, deleteDataStream } from '../lib/create_test_data';
const API_URI = 'api/triggers_actions_ui/data/_indices';
@ -20,15 +21,18 @@ export default function indicesEndpointTests({ getService }: FtrProviderContext)
const retry = getService('retry');
const es = getService('es');
const esTestIndexTool = new ESTestIndexTool(es, retry);
const ES_TEST_DATA_STREAM_NAME = 'test-data-stream';
describe('indices endpoint', () => {
before(async () => {
await esTestIndexTool.destroy();
await esTestIndexTool.setup();
await createEsDocuments(es, esTestIndexTool);
await createDataStream(es, ES_TEST_DATA_STREAM_NAME);
});
after(async () => {
await deleteDataStream(es, ES_TEST_DATA_STREAM_NAME);
await esTestIndexTool.destroy();
});
@ -112,6 +116,12 @@ export default function indicesEndpointTests({ getService }: FtrProviderContext)
const result = await runQueryExpect({ pattern: '*a:b,c:d*' }, 200);
expect(result.indices.length).to.be(0);
});
it('should handle data streams', async () => {
const result = await runQueryExpect({ pattern: ES_TEST_DATA_STREAM_NAME }, 200);
expect(result.indices).to.be.an('array');
expect(result.indices.includes(ES_TEST_DATA_STREAM_NAME)).to.be(true);
});
});
async function runQueryExpect(requestBody: any, status: number): Promise<any> {

57
x-pack/test/alerting_api_integration/spaces_only/tests/alerting/builtin_alert_types/lib/create_test_data.ts
View file

@ -21,7 +21,8 @@ export async function createEsDocuments(
endDate: string = END_DATE,
intervals: number = 1,
intervalMillis: number = 1000,
groups: number = 2
groups: number = 2,
indexName: string = ES_TEST_INDEX_NAME
) {
const endDateMillis = Date.parse(endDate) - intervalMillis / 2;
@ -32,7 +33,7 @@ export async function createEsDocuments(
// don't need await on these, wait at the end of the function
times(groups, () => {
promises.push(createEsDocument(es, date, testedValue++));
promises.push(createEsDocument(es, date, testedValue++, indexName));
});
});
await Promise.all(promises);
@ -41,19 +42,26 @@ export async function createEsDocuments(
await esTestIndexTool.waitForDocs(DOCUMENT_SOURCE, DOCUMENT_REFERENCE, totalDocuments);
}
async function createEsDocument(es: Client, epochMillis: number, testedValue: number) {
async function createEsDocument(
es: Client,
epochMillis: number,
testedValue: number,
indexName: string
) {
const document = {
source: DOCUMENT_SOURCE,
reference: DOCUMENT_REFERENCE,
date: new Date(epochMillis).toISOString(),
date_epoch_millis: epochMillis,
testedValue,
'@timestamp': new Date(epochMillis).toISOString(),
};
const response = await es.index({
id: uuid(),
index: ES_TEST_INDEX_NAME,
index: indexName,
refresh: 'wait_for',
op_type: 'create',
body: document,
});
@ -61,3 +69,44 @@ async function createEsDocument(es: Client, epochMillis: number, testedValue: nu
throw new Error(`document not created: ${JSON.stringify(response)}`);
}
}
export async function createDataStream(es: Client, name: string) {
// A data stream requires an index template before it can be created.
await es.indices.putIndexTemplate({
name,
body: {
index_patterns: [name + '*'],
template: {
mappings: {
properties: {
'@timestamp': {
type: 'date',
},
source: {
type: 'keyword',
},
reference: {
type: 'keyword',
},
params: {
enabled: false,
type: 'object',
},
},
},
},
data_stream: {},
},
});
await es.indices.createDataStream({ name });
}
async function deleteComposableIndexTemplate(es: Client, name: string) {
await es.indices.deleteIndexTemplate({ name });
}
export async function deleteDataStream(es: Client, name: string) {
await es.indices.deleteDataStream({ name });
await deleteComposableIndexTemplate(es, name);
}