github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 09:48:58 -04:00
Code Issues Projects Releases Packages Wiki Activity

Move to modified version of original status count for speed (#41210) (#41325)

Browse source 
This attempts to speed up the now slow snapshot count @justinkambic noticed. It uses a different approach form #41203, it's actually a modified version of the old formula. It should be faster. It achieves this executing a query that's very friendly to composite aggs, even if it returns more data than strictly necessary.

This also fixes a bug in the original implementation, where the total would be equivalent to the filtered value, rather than the pre-filtered value. This made little sense since you'd just see the total twice.
This commit is contained in:
Andrew Cholakian 2019-07-17 04:08:22 -05:00 • committed by GitHub
parent 18479b0a68
commit 722ce1af60
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 73 additions and 89 deletions
Download patch file Download diff file Expand all files Collapse all files

2
x-pack/legacy/plugins/uptime/server/graphql/monitors/resolvers.ts
View file

@ -109,7 +109,7 @@ export const createMonitorsResolvers: CreateUMGraphQLResolvers = (
{ dateRangeStart, dateRangeEnd, filters },
{ req }
): Promise<Snapshot> {
const counts = await libs.monitorStates.getSummaryCount(
const counts = await libs.monitors.getSnapshotCount(
req,
dateRangeStart,
dateRangeEnd,

8
x-pack/legacy/plugins/uptime/server/lib/adapters/monitor_states/adapter_types.ts
View file

@ -4,7 +4,7 @@
* you may not use this file except in compliance with the Elastic License.
*/
import { MonitorSummary, SnapshotCount, StatesIndexStatus } from '../../../../common/graphql/types';
import { MonitorSummary, StatesIndexStatus } from '../../../../common/graphql/types';
export interface UMMonitorStatesAdapter {
getMonitorStates(
@ -20,11 +20,5 @@ export interface UMMonitorStatesAdapter {
dateRangeEnd: string,
filters?: string | null
): Promise<MonitorSummary[]>;
getSummaryCount(
request: any,
dateRangeStart: string,
dateRangeEnd: string,
filters?: string | null
): Promise<SnapshotCount>;
statesIndexExists(request: any): Promise<StatesIndexStatus>;
}

44
x-pack/legacy/plugins/uptime/server/lib/adapters/monitor_states/elasticsearch_monitor_states_adapter.ts
View file

@ -11,7 +11,6 @@ import {
MonitorSummary,
SummaryHistogram,
Check,
SnapshotCount,
StatesIndexStatus,
} from '../../../../common/graphql/types';
import { INDEX_NAMES, LEGACY_STATES_QUERY_SIZE } from '../../../../common/constants';
@ -581,49 +580,6 @@ export class ElasticsearchMonitorStatesAdapter implements UMMonitorStatesAdapter
}, {});
}
public async getSummaryCount(
request: any,
dateRangeStart: string,
dateRangeEnd: string,
filters?: string | null
): Promise<SnapshotCount> {
// TODO: adapt this to the states index in future release
// const { count } = await this.database.count(request, { index: 'heartbeat-states-8.0.0' });
// return { count };
const count: SnapshotCount = {
up: 0,
down: 0,
mixed: 0,
total: 0,
};
let searchAfter: any | null = null;
do {
const { afterKey, result, statusFilter } = await this.runLegacyMonitorStatesQuery(
request,
dateRangeStart,
dateRangeEnd,
filters,
searchAfter
);
searchAfter = afterKey;
this.getMonitorBuckets(result, statusFilter).reduce((acc: SnapshotCount, monitor: any) => {
const status = get<string | undefined>(monitor, 'state.value.monitor.status', undefined);
if (status === 'up') {
acc.up++;
} else if (status === 'down') {
acc.down++;
} else if (status === 'mixed') {
acc.mixed++;
}
acc.total++;
return acc;
}, count);
} while (searchAfter !== null);
return count;
}
public async statesIndexExists(request: any): Promise<StatesIndexStatus> {
// TODO: adapt this to the states index in future release
const {

10
x-pack/legacy/plugins/uptime/server/lib/adapters/monitor_states/memory_monitor_states_adapter.ts
View file

@ -5,7 +5,7 @@
*/
import { UMMonitorStatesAdapter } from './adapter_types';
import { MonitorSummary, SnapshotCount, StatesIndexStatus } from '../../../../common/graphql/types';
import { MonitorSummary, StatesIndexStatus } from '../../../../common/graphql/types';
/**
* This class will be implemented for server-side tests.
@ -28,14 +28,6 @@ export class UMMemoryMonitorStatesAdapter implements UMMonitorStatesAdapter {
): Promise<MonitorSummary[]> {
throw new Error('Method not implemented.');
}
public async getSummaryCount(
request: any,
dateRangeStart: string,
dateRangeEnd: string,
filters?: string | null | undefined
): Promise<SnapshotCount> {
throw new Error('Method not implemented.');
}
public async statesIndexExists(request: any): Promise<StatesIndexStatus> {
throw new Error('Method not implemented.');
}

83
x-pack/legacy/plugins/uptime/server/lib/adapters/monitors/elasticsearch_monitors_adapter.ts
View file

@ -4,7 +4,7 @@
* you may not use this file except in compliance with the Elastic License.
*/
import { get, set } from 'lodash';
import { get, set, reduce } from 'lodash';
import { INDEX_NAMES } from '../../../../common/constants';
import {
ErrorListItem,
@ -197,11 +197,10 @@ export class ElasticsearchMonitorsAdapter implements UMMonitorsAdapter {
aggs: {
latest: {
top_hits: {
sort: [
{
'@timestamp': { order: 'desc' },
},
],
sort: [{ '@timestamp': { order: 'desc' } }],
_source: {
includes: ['summary.*', 'monitor.id', '@timestamp', 'observer.geo.name'],
},
size: 1,
},
},
@ -211,10 +210,16 @@ export class ElasticsearchMonitorsAdapter implements UMMonitorsAdapter {
},
};
let up: number = 0;
let down: number = 0;
let searchAfter: any = null;
const summaryByIdLocation: {
// ID
[key: string]: {
// Location
[key: string]: { up: number; down: number; timestamp: number };
};
} = {};
do {
if (searchAfter) {
set(params, 'body.aggs.ids.composite.after', searchAfter);
@ -225,20 +230,66 @@ export class ElasticsearchMonitorsAdapter implements UMMonitorsAdapter {
idBuckets.forEach(bucket => {
// We only get the latest doc
const status = get(bucket, 'latest.hits.hits[0]._source.monitor.status', null);
if (!statusFilter || (statusFilter && statusFilter === status)) {
if (status === 'up') {
up++;
} else {
down++;
}
const source: any = get(bucket, 'latest.hits.hits[0]._source');
const {
summary: { up, down },
monitor: { id },
} = source;
const timestamp = get(source, '@timestamp', 0);
const location = get(source, 'observer.geo.name', '');
let idSummary = summaryByIdLocation[id];
if (!idSummary) {
idSummary = {};
summaryByIdLocation[id] = idSummary;
}
const locationSummary = idSummary[location];
if (!locationSummary || locationSummary.timestamp < timestamp) {
idSummary[location] = { timestamp, up, down };
}
});
searchAfter = get(queryResult, 'aggregations.ids.after_key');
} while (searchAfter);
return { up, down, total: up + down };
let up: number = 0;
let mixed: number = 0;
let down: number = 0;
for (const id in summaryByIdLocation) {
if (!summaryByIdLocation.hasOwnProperty(id)) {
continue;
}
const locationInfo = summaryByIdLocation[id];
const { up: locationUp, down: locationDown } = reduce(
locationInfo,
(acc, value, key) => {
acc.up += value.up;
acc.down += value.down;
return acc;
},
{ up: 0, down: 0 }
);
if (locationDown === 0) {
up++;
} else if (locationUp > 0) {
mixed++;
} else {
down++;
}
}
const result: any = { up, down, mixed, total: up + down + mixed };
if (statusFilter) {
for (const status in result) {
if (status !== 'total' && status !== statusFilter) {
result[status] = 0;
}
}
}
return result;
}
/**

11
x-pack/legacy/plugins/uptime/server/lib/domains/monitor_states.ts
View file

@ -5,7 +5,7 @@
*/
import { UMMonitorStatesAdapter } from '../adapters/monitor_states';
import { MonitorSummary, SnapshotCount, StatesIndexStatus } from '../../../common/graphql/types';
import { MonitorSummary, StatesIndexStatus } from '../../../common/graphql/types';
export class UMMonitorStatesDomain {
constructor(private readonly adapter: UMMonitorStatesAdapter, libs: {}) {
@ -22,15 +22,6 @@ export class UMMonitorStatesDomain {
return this.adapter.getMonitorStates(request, pageIndex, pageSize, sortField, sortDirection);
}
public async getSummaryCount(
request: any,
dateRangeStart: string,
dateRangeEnd: string,
filters?: string | null
): Promise<SnapshotCount> {
return this.adapter.getSummaryCount(request, dateRangeStart, dateRangeEnd, filters);
}
public async statesIndexExists(request: any): Promise<StatesIndexStatus> {
return this.adapter.statesIndexExists(request);
}

2
x-pack/test/api_integration/apis/uptime/graphql/fixtures/snapshot_filtered_by_down.json
View file

@ -1,5 +1,5 @@
{
"snapshot": {
"counts": { "down": 2, "mixed": 0, "up": 0, "total": 2 }
"counts": { "down": 2, "mixed": 0, "up": 0, "total": 10 }
}
}

2
x-pack/test/api_integration/apis/uptime/graphql/fixtures/snapshot_filtered_by_up.json
View file

@ -1,5 +1,5 @@
{
"snapshot": {
"counts": { "down": 0, "mixed": 0, "up": 8, "total": 8 }
"counts": { "down": 0, "mixed": 0, "up": 8, "total": 10 }
}
}