mirror of
https://github.com/elastic/kibana.git
synced 2025-04-23 01:13:23 -04:00
[Stack Monitoring] support ecs format in fetch_nodes_from_cluster (#153783)
## Summary Closes https://github.com/elastic/kibana/issues/147793 Closes https://github.com/elastic/kibana/issues/148448 The `nodes_changed` alert was not looking at the correct property path to extract node information from ecs documents. Also fixes a case where only one document is returned instead of two. ## Testing - Run a multi-node env (see [howto for local one](https://github.com/elastic/kibana/blob/main/x-pack/plugins/monitoring/dev_docs/how_to/running_components_from_source.md#multi-node-configuration); I used a cloud env) - Collect elasticsearch metrics with metricbeat (use `scope: cluster` for cloud) - Enable default rules - Change node configuration - Alert is triggered --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Kevin Lacabane
GitHub
parent
4f0e458954
commit
9ae374955d
2 changed files with 210 additions and 9 deletions
|
|
@ -30,7 +30,7 @@ describe('fetchNodesFromClusterStats', () => {
|
|||
},
|
||||
];
|
||||
|
||||
const esRes = {
|
||||
const legacyRes = {
|
||||
aggregations: {
|
||||
clusters: {
|
||||
buckets: [
|
||||
|
|
@ -123,10 +123,109 @@ describe('fetchNodesFromClusterStats', () => {
|
|||
},
|
||||
};
|
||||
|
||||
it('fetch stats', async () => {
|
||||
const ecsRes = {
|
||||
aggregations: {
|
||||
clusters: {
|
||||
buckets: [
|
||||
{
|
||||
key: 'NG2d5jHiSBGPE6HLlUN2Bg',
|
||||
doc_count: 4,
|
||||
top: {
|
||||
hits: {
|
||||
total: { value: 4, relation: 'eq' },
|
||||
max_score: null,
|
||||
hits: [
|
||||
{
|
||||
_index: '.ds-.monitoring-es-8-mb-2023.03.27-000001',
|
||||
_id: 'CUJ6I4cBwUW49K58n-b9',
|
||||
_score: null,
|
||||
_source: {
|
||||
elasticsearch: {
|
||||
cluster: {
|
||||
stats: {
|
||||
state: {
|
||||
nodes: {
|
||||
'LjJ9FhDATIq9uh1kAa-XPA': {
|
||||
name: 'instance-0000000000',
|
||||
ephemeral_id: '3ryJEBWZS1e3x-_K_Yt-ww',
|
||||
transport_address: '127.0.0.1:9300',
|
||||
external_id: 'instance-0000000000',
|
||||
attributes: {
|
||||
logical_availability_zone: 'zone-0',
|
||||
'xpack.installed': 'true',
|
||||
data: 'hot',
|
||||
region: 'unknown-region',
|
||||
availability_zone: 'us-central1-a',
|
||||
},
|
||||
roles: [
|
||||
'data_content',
|
||||
'data_hot',
|
||||
'ingest',
|
||||
'master',
|
||||
'remote_cluster_client',
|
||||
'transform',
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
sort: [1679927450602],
|
||||
},
|
||||
{
|
||||
_index: '.ds-.monitoring-es-8-mb-2023.03.27-000001',
|
||||
_id: '6kJ6I4cBwUW49K58KuXP',
|
||||
_score: null,
|
||||
_source: {
|
||||
elasticsearch: {
|
||||
cluster: {
|
||||
stats: {
|
||||
state: {
|
||||
nodes: {
|
||||
'LjJ9FhDATIq9uh1kAa-XPA': {
|
||||
name: 'instance-0000000000',
|
||||
ephemeral_id: '3ryJEBWZS1e3x-_K_Yt-ww',
|
||||
transport_address: '127.0.0.1:9300',
|
||||
external_id: 'instance-0000000000',
|
||||
attributes: {
|
||||
logical_availability_zone: 'zone-0',
|
||||
'xpack.installed': 'true',
|
||||
data: 'hot',
|
||||
region: 'unknown-region',
|
||||
availability_zone: 'us-central1-a',
|
||||
},
|
||||
roles: [
|
||||
'data_content',
|
||||
'data_hot',
|
||||
'ingest',
|
||||
'master',
|
||||
'remote_cluster_client',
|
||||
'transform',
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
sort: [1679927420602],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
it('fetch legacy stats', async () => {
|
||||
esClient.search.mockResponse(
|
||||
// @ts-expect-error not full response interface
|
||||
esRes
|
||||
legacyRes
|
||||
);
|
||||
const result = await fetchNodesFromClusterStats(esClient, clusters);
|
||||
expect(result).toEqual([
|
||||
|
|
@ -150,11 +249,38 @@ describe('fetchNodesFromClusterStats', () => {
|
|||
]);
|
||||
});
|
||||
|
||||
it('fetch ecs stats', async () => {
|
||||
esClient.search.mockResponse(
|
||||
// @ts-expect-error not full response interface
|
||||
ecsRes
|
||||
);
|
||||
const result = await fetchNodesFromClusterStats(esClient, clusters);
|
||||
expect(result).toEqual([
|
||||
{
|
||||
clusterUuid: 'NG2d5jHiSBGPE6HLlUN2Bg',
|
||||
recentNodes: [
|
||||
{
|
||||
nodeUuid: 'LjJ9FhDATIq9uh1kAa-XPA',
|
||||
nodeEphemeralId: '3ryJEBWZS1e3x-_K_Yt-ww',
|
||||
nodeName: 'instance-0000000000',
|
||||
},
|
||||
],
|
||||
priorNodes: [
|
||||
{
|
||||
nodeUuid: 'LjJ9FhDATIq9uh1kAa-XPA',
|
||||
nodeEphemeralId: '3ryJEBWZS1e3x-_K_Yt-ww',
|
||||
nodeName: 'instance-0000000000',
|
||||
},
|
||||
],
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('should call ES with correct query', async () => {
|
||||
let params = null;
|
||||
esClient.search.mockImplementation((...args) => {
|
||||
params = args[0];
|
||||
return Promise.resolve(esRes as any);
|
||||
return Promise.resolve(legacyRes as any);
|
||||
});
|
||||
await fetchNodesFromClusterStats(esClient, clusters);
|
||||
expect(params).toStrictEqual({
|
||||
|
|
@ -193,7 +319,7 @@ describe('fetchNodesFromClusterStats', () => {
|
|||
top_hits: {
|
||||
sort: [{ timestamp: { order: 'desc', unmapped_type: 'long' } }],
|
||||
_source: {
|
||||
includes: ['cluster_state.nodes', 'elasticsearch.cluster.stats.nodes'],
|
||||
includes: ['cluster_state.nodes', 'elasticsearch.cluster.stats.state.nodes'],
|
||||
},
|
||||
size: 2,
|
||||
},
|
||||
|
|
@ -210,7 +336,7 @@ describe('fetchNodesFromClusterStats', () => {
|
|||
let params = null;
|
||||
esClient.search.mockImplementation((...args) => {
|
||||
params = args[0];
|
||||
return Promise.resolve(esRes as any);
|
||||
return Promise.resolve(legacyRes as any);
|
||||
});
|
||||
await fetchNodesFromClusterStats(esClient, clusters);
|
||||
// @ts-ignore
|
||||
|
|
@ -218,4 +344,74 @@ describe('fetchNodesFromClusterStats', () => {
|
|||
'.monitoring-es-*,metrics-elasticsearch.stack_monitoring.cluster_stats-*'
|
||||
);
|
||||
});
|
||||
|
||||
it('ignores buckets with only one document', async () => {
|
||||
const singleHitRes = {
|
||||
aggregations: {
|
||||
clusters: {
|
||||
buckets: [
|
||||
{
|
||||
key: 'NG2d5jHiSBGPE6HLlUN2Bg',
|
||||
doc_count: 1,
|
||||
top: {
|
||||
hits: {
|
||||
total: { value: 1, relation: 'eq' },
|
||||
max_score: null,
|
||||
hits: [
|
||||
{
|
||||
_index: '.ds-.monitoring-es-8-mb-2023.03.27-000001',
|
||||
_id: 'CUJ6I4cBwUW49K58n-b9',
|
||||
_score: null,
|
||||
_source: {
|
||||
elasticsearch: {
|
||||
cluster: {
|
||||
stats: {
|
||||
state: {
|
||||
nodes: {
|
||||
'LjJ9FhDATIq9uh1kAa-XPA': {
|
||||
name: 'instance-0000000000',
|
||||
ephemeral_id: '3ryJEBWZS1e3x-_K_Yt-ww',
|
||||
transport_address: '127.0.0.1:9300',
|
||||
external_id: 'instance-0000000000',
|
||||
attributes: {
|
||||
logical_availability_zone: 'zone-0',
|
||||
'xpack.installed': 'true',
|
||||
data: 'hot',
|
||||
region: 'unknown-region',
|
||||
availability_zone: 'us-central1-a',
|
||||
},
|
||||
roles: [
|
||||
'data_content',
|
||||
'data_hot',
|
||||
'ingest',
|
||||
'master',
|
||||
'remote_cluster_client',
|
||||
'transform',
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
sort: [1679927450602],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
esClient.search.mockResponse(
|
||||
// @ts-expect-error not full response interface
|
||||
singleHitRes
|
||||
);
|
||||
|
||||
const result = await fetchNodesFromClusterStats(esClient, clusters);
|
||||
expect(result).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -87,7 +87,7 @@ export async function fetchNodesFromClusterStats(
|
|||
},
|
||||
],
|
||||
_source: {
|
||||
includes: ['cluster_state.nodes', 'elasticsearch.cluster.stats.nodes'],
|
||||
includes: ['cluster_state.nodes', 'elasticsearch.cluster.stats.state.nodes'],
|
||||
},
|
||||
size: 2,
|
||||
},
|
||||
|
|
@ -117,14 +117,19 @@ export async function fetchNodesFromClusterStats(
|
|||
for (const clusterBucket of clusterBuckets) {
|
||||
const clusterUuid = clusterBucket.key;
|
||||
const hits = clusterBucket.top.hits.hits;
|
||||
if (hits.length < 2) {
|
||||
continue;
|
||||
}
|
||||
const indexName = hits[0]._index;
|
||||
nodes.push({
|
||||
clusterUuid,
|
||||
recentNodes: formatNode(
|
||||
hits[0]._source.cluster_state?.nodes || hits[0]._source.elasticsearch.cluster.stats.nodes
|
||||
hits[0]._source.cluster_state?.nodes ||
|
||||
hits[0]._source.elasticsearch.cluster.stats.state.nodes
|
||||
),
|
||||
priorNodes: formatNode(
|
||||
hits[1]._source.cluster_state?.nodes || hits[1]._source.elasticsearch.cluster.stats.nodes
|
||||
hits[1]._source.cluster_state?.nodes ||
|
||||
hits[1]._source.elasticsearch.cluster.stats.state.nodes
|
||||
),
|
||||
ccs: indexName.includes(':') ? indexName.split(':')[0] : undefined,
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue