github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 09:48:58 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Usage Collection] Use PIT for collecting UI/Usage Counters & AppUsage (#135689)

Browse source 
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Alejandro Fernández Haro 2022-07-06 10:01:52 +02:00 • committed by GitHub
parent 113391618f
commit 4d301fdf4d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 93 additions and 69 deletions
Download patch file Download diff file Expand all files Collapse all files

28
src/plugins/kibana_usage_collection/server/collectors/application_usage/fetch_all_saved_objects.ts Normal file
View file

@ -0,0 +1,28 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import type {
ISavedObjectsRepository,
SavedObjectsCreatePointInTimeFinderOptions,
SavedObjectsFindResult,
} from '@kbn/core/server';
export async function fetchAllSavedObjects<T>(
soRepository: ISavedObjectsRepository,
findOptions: SavedObjectsCreatePointInTimeFinderOptions
): Promise<Array<SavedObjectsFindResult<T>>> {
const finder = soRepository.createPointInTimeFinder<T>({ ...findOptions, perPage: 1000 });
const allSavedObjects: Array<SavedObjectsFindResult<T>> = [];
for await (const { saved_objects: savedObjects } of finder.find()) {
allSavedObjects.push(...savedObjects);
}
return allSavedObjects;
}

12
src/plugins/kibana_usage_collection/server/collectors/application_usage/rollups/total.ts
View file

@ -16,6 +16,7 @@ import {
SAVED_OBJECTS_TOTAL_TYPE,
} from '../saved_objects_types';
import { serializeKey } from './utils';
import { fetchAllSavedObjects } from '../fetch_all_saved_objects';
/**
* Moves all the daily documents into aggregated "total" documents as we don't care about any granularity after 90 days
@ -28,16 +29,11 @@ export async function rollTotals(logger: Logger, savedObjectsClient?: ISavedObje
}
try {
const [
{ saved_objects: rawApplicationUsageTotals },
{ saved_objects: rawApplicationUsageDaily },
] = await Promise.all([
savedObjectsClient.find<ApplicationUsageTotal>({
perPage: 10000,
const [rawApplicationUsageTotals, rawApplicationUsageDaily] = await Promise.all([
fetchAllSavedObjects<ApplicationUsageTotal>(savedObjectsClient, {
type: SAVED_OBJECTS_TOTAL_TYPE,
}),
savedObjectsClient.find<ApplicationUsageDaily>({
perPage: 10000,
fetchAllSavedObjects<ApplicationUsageDaily>(savedObjectsClient, {
type: SAVED_OBJECTS_DAILY_TYPE,
filter: `${SAVED_OBJECTS_DAILY_TYPE}.attributes.timestamp < now-90d`,
}),

22
src/plugins/kibana_usage_collection/server/collectors/application_usage/telemetry_application_usage_collector.ts
View file

@ -8,12 +8,12 @@
import moment from 'moment';
import { type Observable, takeUntil, timer } from 'rxjs';
import { ISavedObjectsRepository, Logger, SavedObjectsServiceSetup } from '@kbn/core/server';
import { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server';
import type { ISavedObjectsRepository, Logger, SavedObjectsServiceSetup } from '@kbn/core/server';
import type { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server';
import { MAIN_APP_DEFAULT_VIEW_ID } from '@kbn/usage-collection-plugin/common/constants';
import {
ApplicationUsageDaily,
ApplicationUsageTotal,
type ApplicationUsageDaily,
type ApplicationUsageTotal,
registerMappings,
SAVED_OBJECTS_DAILY_TYPE,
SAVED_OBJECTS_TOTAL_TYPE,
@ -21,7 +21,8 @@ import {
import { applicationUsageSchema } from './schema';
import { rollTotals, serializeKey } from './rollups';
import { ROLL_TOTAL_INDICES_INTERVAL, ROLL_INDICES_START } from './constants';
import { ApplicationUsageTelemetryReport, ApplicationUsageViews } from './types';
import type { ApplicationUsageTelemetryReport, ApplicationUsageViews } from './types';
import { fetchAllSavedObjects } from './fetch_all_saved_objects';
export const transformByApplicationViews = (
report: ApplicationUsageViews
@ -67,17 +68,12 @@ export function registerApplicationUsageCollector(
if (typeof savedObjectsClient === 'undefined') {
return;
}
const [
{ saved_objects: rawApplicationUsageTotals },
{ saved_objects: rawApplicationUsageDaily },
] = await Promise.all([
savedObjectsClient.find<ApplicationUsageTotal>({
const [rawApplicationUsageTotals, rawApplicationUsageDaily] = await Promise.all([
fetchAllSavedObjects<ApplicationUsageTotal>(savedObjectsClient, {
type: SAVED_OBJECTS_TOTAL_TYPE,
perPage: 10000, // We only have 44 apps for now. This limit is OK.
}),
savedObjectsClient.find<ApplicationUsageDaily>({
fetchAllSavedObjects<ApplicationUsageDaily>(savedObjectsClient, {
type: SAVED_OBJECTS_DAILY_TYPE,
perPage: 10000, // We can have up to 44 apps * 91 days = 4004 docs. This limit is OK
}),
]);

42
src/plugins/kibana_usage_collection/server/collectors/ui_counters/register_ui_counters_collector.ts
View file

@ -57,29 +57,29 @@ export function transformRawUsageCounterObject(
}
export async function fetchUiCounters({ soClient }: CollectorFetchContext) {
const { saved_objects: rawUsageCounters } =
await soClient.find<UsageCountersSavedObjectAttributes>({
type: USAGE_COUNTERS_SAVED_OBJECT_TYPE,
fields: ['count', 'counterName', 'counterType', 'domainId'],
filter: `${USAGE_COUNTERS_SAVED_OBJECT_TYPE}.attributes.domainId: uiCounter`,
perPage: 10000,
});
const finder = soClient.createPointInTimeFinder<UsageCountersSavedObjectAttributes>({
type: USAGE_COUNTERS_SAVED_OBJECT_TYPE,
fields: ['count', 'counterName', 'counterType', 'domainId'],
filter: `${USAGE_COUNTERS_SAVED_OBJECT_TYPE}.attributes.domainId: uiCounter`,
perPage: 1000,
});
return {
dailyEvents: Object.values(
rawUsageCounters.reduce((acc, raw) => {
try {
const event = transformRawUsageCounterObject(raw);
if (event) {
acc[raw.id] = event;
}
} catch (_) {
// swallow error; allows sending successfully transformed objects.
const dailyEvents: UiCounterEvent[] = [];
for await (const { saved_objects: rawUsageCounters } of finder.find()) {
rawUsageCounters.forEach((raw) => {
try {
const event = transformRawUsageCounterObject(raw);
if (event) {
dailyEvents.push(event);
}
return acc;
}, {} as Record<string, UiCounterEvent>)
),
};
} catch (_) {
// swallow error; allows sending successfully transformed objects.
}
});
}
return { dailyEvents };
}
export function registerUiCountersUsageCollector(usageCollection: UsageCollectionSetup) {

30
src/plugins/kibana_usage_collection/server/collectors/ui_metric/telemetry_ui_metric_collector.ts
View file

@ -52,26 +52,28 @@ export function registerUiMetricUsageCollector(
return;
}
const { saved_objects: rawUiMetrics } = await savedObjectsClient.find<UIMetricsSavedObjects>({
const finder = savedObjectsClient.createPointInTimeFinder<UIMetricsSavedObjects>({
type: 'ui-metric',
fields: ['count'],
perPage: 10000,
perPage: 1000,
});
const uiMetricsByAppName = rawUiMetrics.reduce((accum, rawUiMetric) => {
const {
id,
attributes: { count },
} = rawUiMetric;
const uiMetricsByAppName: UIMetricUsage = {};
const [appName, ...metricType] = id.split(':');
for await (const { saved_objects: rawUiMetrics } of finder.find()) {
rawUiMetrics.forEach((rawUiMetric) => {
const {
id,
attributes: { count },
} = rawUiMetric;
const pair = { key: metricType.join(':'), value: count };
return {
...accum,
[appName]: [...(accum[appName] || []), pair],
};
}, {} as UIMetricUsage);
const [appName, ...metricType] = id.split(':');
const pair = { key: metricType.join(':'), value: count };
uiMetricsByAppName[appName] = [...(uiMetricsByAppName[appName] || []), pair];
});
}
return uiMetricsByAppName;
},

28
src/plugins/kibana_usage_collection/server/collectors/usage_counters/register_usage_counters_collector.ts
View file

@ -86,27 +86,29 @@ export function registerUsageCountersUsageCollector(usageCollection: UsageCollec
},
},
fetch: async ({ soClient }: CollectorFetchContext) => {
const { saved_objects: rawUsageCounters } =
await soClient.find<UsageCountersSavedObjectAttributes>({
type: USAGE_COUNTERS_SAVED_OBJECT_TYPE,
fields: ['count', 'counterName', 'counterType', 'domainId'],
filter: `NOT ${USAGE_COUNTERS_SAVED_OBJECT_TYPE}.attributes.domainId: uiCounter`,
perPage: 10000,
});
const finder = soClient.createPointInTimeFinder<UsageCountersSavedObjectAttributes>({
type: USAGE_COUNTERS_SAVED_OBJECT_TYPE,
fields: ['count', 'counterName', 'counterType', 'domainId'],
filter: `NOT ${USAGE_COUNTERS_SAVED_OBJECT_TYPE}.attributes.domainId: uiCounter`,
perPage: 1000,
});
return {
dailyEvents: rawUsageCounters.reduce((acc, rawUsageCounter) => {
const dailyEvents: UsageCounterEvent[] = [];
for await (const { saved_objects: rawUsageCounters } of finder.find()) {
rawUsageCounters.forEach((rawUsageCounter) => {
try {
const event = transformRawCounter(rawUsageCounter);
if (event) {
acc.push(event);
dailyEvents.push(event);
}
} catch (_) {
// swallow error; allows sending successfully transformed objects.
}
return acc;
}, [] as UsageCounterEvent[]),
};
});
}
return { dailyEvents };
},
isReady: () => true,
});