github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 17:59:23 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Usage Collection] remove daily rollups for ui counters and application usage (#130794)

Browse source 
Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Ahmad Bamieh 2022-05-05 15:32:00 +03:00 • committed by GitHub
parent 0f3d63b1aa
commit 6dc4f7b3cf
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
21 changed files with 71 additions and 1005 deletions
Download patch file Download diff file Expand all files Collapse all files

30
src/core/server/saved_objects/migrations/__snapshots__/migrations_state_action_machine.test.ts.snap generated
View file

@ -139,6 +139,11 @@ Object {
"type": "tsvb-validation-telemetry",
},
},
Object {
"term": Object {
"type": "ui-counter",
},
},
Object {
"bool": Object {
"must": Array [
@ -305,6 +310,11 @@ Object {
"type": "tsvb-validation-telemetry",
},
},
Object {
"term": Object {
"type": "ui-counter",
},
},
Object {
"bool": Object {
"must": Array [
@ -475,6 +485,11 @@ Object {
"type": "tsvb-validation-telemetry",
},
},
Object {
"term": Object {
"type": "ui-counter",
},
},
Object {
"bool": Object {
"must": Array [
@ -649,6 +664,11 @@ Object {
"type": "tsvb-validation-telemetry",
},
},
Object {
"term": Object {
"type": "ui-counter",
},
},
Object {
"bool": Object {
"must": Array [
@ -860,6 +880,11 @@ Object {
"type": "tsvb-validation-telemetry",
},
},
Object {
"term": Object {
"type": "ui-counter",
},
},
Object {
"bool": Object {
"must": Array [
@ -1037,6 +1062,11 @@ Object {
"type": "tsvb-validation-telemetry",
},
},
Object {
"term": Object {
"type": "ui-counter",
},
},
Object {
"bool": Object {
"must": Array [

2
src/core/server/saved_objects/migrations/core/unused_types.ts
View file

@ -33,6 +33,8 @@ export const REMOVED_TYPES: string[] = [
'siem-detection-engine-rule-status',
// Was removed in 7.16
'timelion-sheet',
// Removed in 8.3 https://github.com/elastic/kibana/issues/127745
'ui-counter',
].sort();
// When migrating from the outdated index we use a read query which excludes

7
src/plugins/kibana_usage_collection/server/collectors/application_usage/README.md
View file

@ -120,10 +120,9 @@ This collection occurs by default for every application registered via the menti
In order to keep the count of the events, this collector uses 3 Saved Objects:
1. `application_usage_transactional`: It stores each individually reported event. Grouped by `timestamp` and `appId`. The reason for having these documents instead of editing `application_usage_daily` documents on very report is to provide faster response to the requests to `/api/ui_counters/_report` (creating new documents instead of finding and editing existing ones) and to avoid conflicts when multiple users reach to the API concurrently.
2. `application_usage_daily`: Periodically, documents from `application_usage_transactional` are aggregated to daily summaries and deleted. Also grouped by `timestamp` and `appId` for the main view concatenated with `viewId` for other views.
3. `application_usage_totals`: It stores the sum of all the events older than 90 days old, grouped by `appId` for the main view concatenated with `viewId` for other views.
1. `application_usage_transactional`: It stores each individually reported event. Grouped by `timestamp` and `appId`.
2. `application_usage_totals`: It stores the sum of all the events older than 90 days old, grouped by `appId` for the main view concatenated with `viewId` for other views.
All the types use the shared fields `appId: 'keyword'`, `viewId: 'keyword'`, `numberOfClicks: 'long'` and `minutesOnScreen: 'float'`, but they are currently not added in the mappings because we don't use them for search purposes, and we need to be thoughtful with the number of mapped fields in the SavedObjects index ([#43673](https://github.com/elastic/kibana/issues/43673)). `application_usage_transactional` and `application_usage_daily` also store `timestamp: { type: 'date' }`.
All the types use the shared fields `appId: 'keyword'`, `viewId: 'keyword'`, `numberOfClicks: 'long'` and `minutesOnScreen: 'float'`, but they are currently not added in the mappings because we don't use them for search purposes, and we need to be thoughtful with the number of mapped fields in the SavedObjects index ([#43673](https://github.com/elastic/kibana/issues/43673)). The SO type `application_usage_transactional` also stores `timestamp: { type: 'date' }`.
Rollups uses `appId` in the savedObject id for the default view. For other views `viewId` is concatenated. This keeps backwards compatiblity with previously stored documents on the clusters without requiring any form of migration.

5
src/plugins/kibana_usage_collection/server/collectors/application_usage/constants.ts
View file

@ -11,11 +11,6 @@
*/
export const ROLL_TOTAL_INDICES_INTERVAL = 24 * 60 * 60 * 1000;
/**
* Roll daily indices every 24h
*/
export const ROLL_DAILY_INDICES_INTERVAL = 24 * 60 * 60 * 1000;
/**
* Start rolling indices after 5 minutes up
*/

1
src/plugins/kibana_usage_collection/server/collectors/application_usage/index.ts
View file

@ -7,4 +7,3 @@
*/
export { registerApplicationUsageCollector } from './telemetry_application_usage_collector';
export { rollDailyData as migrateTransactionalDocs } from './rollups';

203
src/plugins/kibana_usage_collection/server/collectors/application_usage/rollups/daily.test.ts
View file

@ -1,203 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { savedObjectsRepositoryMock, loggingSystemMock } from '@kbn/core/server/mocks';
import { SavedObjectsErrorHelpers } from '@kbn/core/server';
import { SAVED_OBJECTS_DAILY_TYPE, SAVED_OBJECTS_TRANSACTIONAL_TYPE } from '../saved_objects_types';
import { rollDailyData } from './daily';
describe('rollDailyData', () => {
const logger = loggingSystemMock.createLogger();
test('returns false if no savedObjectsClient initialised yet', async () => {
await expect(rollDailyData(logger, undefined)).resolves.toBe(false);
});
test('handle empty results', async () => {
const savedObjectClient = savedObjectsRepositoryMock.create();
savedObjectClient.find.mockImplementation(async ({ type, page = 1, perPage = 10 }) => {
switch (type) {
case SAVED_OBJECTS_TRANSACTIONAL_TYPE:
return { saved_objects: [], total: 0, page, per_page: perPage };
default:
throw new Error(`Unexpected type [${type}]`);
}
});
await expect(rollDailyData(logger, savedObjectClient)).resolves.toBe(true);
expect(savedObjectClient.get).not.toBeCalled();
expect(savedObjectClient.bulkCreate).not.toBeCalled();
expect(savedObjectClient.delete).not.toBeCalled();
});
test('migrate some docs', async () => {
const savedObjectClient = savedObjectsRepositoryMock.create();
let timesCalled = 0;
savedObjectClient.find.mockImplementation(async ({ type, page = 1, perPage = 10 }) => {
switch (type) {
case SAVED_OBJECTS_TRANSACTIONAL_TYPE:
if (timesCalled++ > 0) {
return { saved_objects: [], total: 0, page, per_page: perPage };
}
return {
saved_objects: [
{
id: 'test-id-1',
type,
score: 0,
references: [],
attributes: {
appId: 'appId',
timestamp: '2020-01-01T10:31:00.000Z',
minutesOnScreen: 0.5,
numberOfClicks: 1,
},
},
{
id: 'test-id-2',
type,
score: 0,
references: [],
attributes: {
appId: 'appId',
timestamp: '2020-01-01T11:31:00.000Z',
minutesOnScreen: 2.5,
numberOfClicks: 2,
},
},
{
id: 'test-id-3',
type,
score: 0,
references: [],
attributes: {
appId: 'appId',
viewId: 'appId_viewId',
timestamp: '2020-01-01T11:31:00.000Z',
minutesOnScreen: 1,
numberOfClicks: 5,
},
},
],
total: 3,
page,
per_page: perPage,
};
default:
throw new Error(`Unexpected type [${type}]`);
}
});
savedObjectClient.get.mockImplementation(async (type, id) => {
throw SavedObjectsErrorHelpers.createGenericNotFoundError(type, id);
});
await expect(rollDailyData(logger, savedObjectClient)).resolves.toBe(true);
expect(savedObjectClient.get).toHaveBeenCalledTimes(2);
expect(savedObjectClient.get).toHaveBeenNthCalledWith(
1,
SAVED_OBJECTS_DAILY_TYPE,
'appId:2020-01-01'
);
expect(savedObjectClient.get).toHaveBeenNthCalledWith(
2,
SAVED_OBJECTS_DAILY_TYPE,
'appId:2020-01-01:appId_viewId'
);
expect(savedObjectClient.bulkCreate).toHaveBeenCalledTimes(1);
expect(savedObjectClient.bulkCreate).toHaveBeenCalledWith(
[
{
type: SAVED_OBJECTS_DAILY_TYPE,
id: 'appId:2020-01-01',
attributes: {
appId: 'appId',
viewId: undefined,
timestamp: '2020-01-01T00:00:00.000Z',
minutesOnScreen: 3.0,
numberOfClicks: 3,
},
},
{
type: SAVED_OBJECTS_DAILY_TYPE,
id: 'appId:2020-01-01:appId_viewId',
attributes: {
appId: 'appId',
viewId: 'appId_viewId',
timestamp: '2020-01-01T00:00:00.000Z',
minutesOnScreen: 1.0,
numberOfClicks: 5,
},
},
],
{ overwrite: true }
);
expect(savedObjectClient.delete).toHaveBeenCalledTimes(3);
expect(savedObjectClient.delete).toHaveBeenNthCalledWith(
1,
SAVED_OBJECTS_TRANSACTIONAL_TYPE,
'test-id-1'
);
expect(savedObjectClient.delete).toHaveBeenNthCalledWith(
2,
SAVED_OBJECTS_TRANSACTIONAL_TYPE,
'test-id-2'
);
expect(savedObjectClient.delete).toHaveBeenNthCalledWith(
3,
SAVED_OBJECTS_TRANSACTIONAL_TYPE,
'test-id-3'
);
});
test('error getting the daily document', async () => {
const savedObjectClient = savedObjectsRepositoryMock.create();
let timesCalled = 0;
savedObjectClient.find.mockImplementation(async ({ type, page = 1, perPage = 10 }) => {
switch (type) {
case SAVED_OBJECTS_TRANSACTIONAL_TYPE:
if (timesCalled++ > 0) {
return { saved_objects: [], total: 0, page, per_page: perPage };
}
return {
saved_objects: [
{
id: 'test-id-1',
type,
score: 0,
references: [],
attributes: {
appId: 'appId',
timestamp: '2020-01-01T10:31:00.000Z',
minutesOnScreen: 0.5,
numberOfClicks: 1,
},
},
],
total: 1,
page,
per_page: perPage,
};
default:
throw new Error(`Unexpected type [${type}]`);
}
});
savedObjectClient.get.mockImplementation(async (type, id) => {
throw new Error('Something went terribly wrong');
});
await expect(rollDailyData(logger, savedObjectClient)).resolves.toBe(false);
expect(savedObjectClient.get).toHaveBeenCalledTimes(1);
expect(savedObjectClient.get).toHaveBeenCalledWith(
SAVED_OBJECTS_DAILY_TYPE,
'appId:2020-01-01'
);
expect(savedObjectClient.bulkCreate).toHaveBeenCalledTimes(0);
expect(savedObjectClient.delete).toHaveBeenCalledTimes(0);
});
});

143
src/plugins/kibana_usage_collection/server/collectors/application_usage/rollups/daily.ts
View file

@ -1,143 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import moment from 'moment';
import type { Logger } from '@kbn/logging';
import { ISavedObjectsRepository, SavedObject, SavedObjectsErrorHelpers } from '@kbn/core/server';
import { getDailyId } from '@kbn/usage-collection-plugin/common/application_usage';
import {
ApplicationUsageDaily,
ApplicationUsageTransactional,
SAVED_OBJECTS_DAILY_TYPE,
SAVED_OBJECTS_TRANSACTIONAL_TYPE,
} from '../saved_objects_types';
/**
* For Rolling the daily data, we only care about the stored attributes and the version (to avoid overwriting via concurrent requests)
*/
type ApplicationUsageDailyWithVersion = Pick<
SavedObject<ApplicationUsageDaily>,
'version' | 'attributes'
>;
/**
* Aggregates all the transactional events into daily aggregates
* @param logger
* @param savedObjectsClient
*/
export async function rollDailyData(
logger: Logger,
savedObjectsClient?: ISavedObjectsRepository
): Promise<boolean> {
if (!savedObjectsClient) {
return false;
}
try {
let toCreate: Map<string, ApplicationUsageDailyWithVersion>;
do {
toCreate = new Map();
const { saved_objects: rawApplicationUsageTransactional } =
await savedObjectsClient.find<ApplicationUsageTransactional>({
type: SAVED_OBJECTS_TRANSACTIONAL_TYPE,
perPage: 1000, // Process 1000 at a time as a compromise of speed and overload
});
for (const doc of rawApplicationUsageTransactional) {
const {
attributes: { appId, viewId, minutesOnScreen, numberOfClicks, timestamp },
} = doc;
const dayId = moment(timestamp).format('YYYY-MM-DD');
const dailyId = getDailyId({ dayId, appId, viewId });
const existingDoc =
toCreate.get(dailyId) ||
(await getDailyDoc(savedObjectsClient, dailyId, appId, viewId, dayId));
toCreate.set(dailyId, {
...existingDoc,
attributes: {
...existingDoc.attributes,
minutesOnScreen: existingDoc.attributes.minutesOnScreen + minutesOnScreen,
numberOfClicks: existingDoc.attributes.numberOfClicks + numberOfClicks,
},
});
}
if (toCreate.size > 0) {
await savedObjectsClient.bulkCreate(
[...toCreate.entries()].map(([id, { attributes, version }]) => ({
type: SAVED_OBJECTS_DAILY_TYPE,
id,
attributes,
version, // Providing version to ensure via conflict matching that only 1 Kibana instance (or interval) is taking care of the updates
})),
{ overwrite: true }
);
const promiseStatuses = await Promise.allSettled(
rawApplicationUsageTransactional.map(
({ id }) => savedObjectsClient.delete(SAVED_OBJECTS_TRANSACTIONAL_TYPE, id) // There is no bulkDelete :(
)
);
const rejectedPromises = promiseStatuses.filter(
(settledResult): settledResult is PromiseRejectedResult =>
settledResult.status === 'rejected'
);
if (rejectedPromises.length > 0) {
throw new Error(
`Failed to delete some items in ${SAVED_OBJECTS_TRANSACTIONAL_TYPE}: ${JSON.stringify(
rejectedPromises.map(({ reason }) => reason)
)}`
);
}
}
} while (toCreate.size > 0);
return true;
} catch (err) {
logger.debug(`Failed to rollup transactional to daily entries`);
logger.debug(err);
return false;
}
}
/**
* Gets daily doc from the SavedObjects repository. Creates a new one if not found
* @param savedObjectsClient
* @param id The ID of the document to retrieve (typically, `${appId}:${dayId}`)
* @param appId The application ID
* @param viewId The application view ID
* @param dayId The date of the document in the format YYYY-MM-DD
*/
async function getDailyDoc(
savedObjectsClient: ISavedObjectsRepository,
id: string,
appId: string,
viewId: string,
dayId: string
): Promise<ApplicationUsageDailyWithVersion> {
try {
const { attributes, version } = await savedObjectsClient.get<ApplicationUsageDaily>(
SAVED_OBJECTS_DAILY_TYPE,
id
);
return { attributes, version };
} catch (err) {
if (SavedObjectsErrorHelpers.isNotFoundError(err)) {
return {
attributes: {
appId,
viewId,
// Concatenating the day in YYYY-MM-DD form to T00:00:00Z to reduce the TZ effects
timestamp: moment(`${moment(dayId).format('YYYY-MM-DD')}T00:00:00Z`).toISOString(),
minutesOnScreen: 0,
numberOfClicks: 0,
},
};
}
throw err;
}
}

1
src/plugins/kibana_usage_collection/server/collectors/application_usage/rollups/index.ts
View file

@ -6,6 +6,5 @@
* Side Public License, v 1.
*/
export { rollDailyData } from './daily';
export { rollTotals } from './total';
export { serializeKey } from './utils';

19
src/plugins/kibana_usage_collection/server/collectors/application_usage/telemetry_application_usage_collector.ts
View file

@ -19,12 +19,8 @@ import {
SAVED_OBJECTS_TOTAL_TYPE,
} from './saved_objects_types';
import { applicationUsageSchema } from './schema';
import { rollTotals, rollDailyData, serializeKey } from './rollups';
import {
ROLL_TOTAL_INDICES_INTERVAL,
ROLL_DAILY_INDICES_INTERVAL,
ROLL_INDICES_START,
} from './constants';
import { rollTotals, serializeKey } from './rollups';
import { ROLL_TOTAL_INDICES_INTERVAL, ROLL_INDICES_START } from './constants';
import { ApplicationUsageTelemetryReport, ApplicationUsageViews } from './types';
export const transformByApplicationViews = (
@ -60,17 +56,6 @@ export function registerApplicationUsageCollector(
rollTotals(logger, getSavedObjectsClient())
);
const dailyRollingSub = timer(ROLL_INDICES_START, ROLL_DAILY_INDICES_INTERVAL).subscribe(
async () => {
const success = await rollDailyData(logger, getSavedObjectsClient());
// we only need to roll the transactional documents once to assure BWC
// once we rolling succeeds, we can stop.
if (success) {
dailyRollingSub.unsubscribe();
}
}
);
const collector = usageCollection.makeUsageCollector<ApplicationUsageTelemetryReport | undefined>(
{
type: 'application_usage',

6
src/plugins/kibana_usage_collection/server/collectors/index.ts
View file

@ -19,11 +19,7 @@ export { registerCspCollector } from './csp';
export { registerCoreUsageCollector } from './core';
export { registerLocalizationUsageCollector } from './localization';
export { registerConfigUsageCollector } from './config_usage';
export {
registerUiCountersUsageCollector,
registerUiCounterSavedObjectType,
registerUiCountersRollups,
} from './ui_counters';
export { registerUiCountersUsageCollector } from './ui_counters';
export {
registerUsageCountersRollups,
registerUsageCountersUsageCollector,

51
src/plugins/kibana_usage_collection/server/collectors/ui_counters/__fixtures__/ui_counter_saved_objects.ts
View file

@ -1,51 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import type { UICounterSavedObject } from '../ui_counter_saved_object_type';
export const rawUiCounters: UICounterSavedObject[] = [
{
type: 'ui-counter',
id: 'Kibana_home:23102020:click:different_type',
attributes: {
count: 1,
},
references: [],
updated_at: '2020-11-24T11:27:57.067Z',
version: 'WzI5NDRd',
},
{
type: 'ui-counter',
id: 'Kibana_home:25102020:loaded:intersecting_event',
attributes: {
count: 1,
},
references: [],
updated_at: '2020-10-25T11:27:57.067Z',
version: 'WzI5NDRd',
},
{
type: 'ui-counter',
id: 'Kibana_home:23102020:loaded:intersecting_event',
attributes: {
count: 3,
},
references: [],
updated_at: '2020-10-23T11:27:57.067Z',
version: 'WzI5NDRd',
},
{
type: 'ui-counter',
id: 'Kibana_home:24112020:click:only_reported_in_ui_counters',
attributes: {
count: 1,
},
references: [],
updated_at: '2020-11-24T11:27:57.067Z',
version: 'WzI5NDRd',
},
];

2
src/plugins/kibana_usage_collection/server/collectors/ui_counters/index.ts
View file

@ -7,5 +7,3 @@
*/
export { registerUiCountersUsageCollector } from './register_ui_counters_collector';
export { registerUiCounterSavedObjectType } from './ui_counter_saved_object_type';
export { registerUiCountersRollups } from './rollups';

98
src/plugins/kibana_usage_collection/server/collectors/ui_counters/register_ui_counters_collector.test.ts
View file

@ -6,16 +6,9 @@
* Side Public License, v 1.
*/
import {
transformRawUiCounterObject,
transformRawUsageCounterObject,
createFetchUiCounters,
} from './register_ui_counters_collector';
import { BehaviorSubject } from 'rxjs';
import { rawUiCounters } from './__fixtures__/ui_counter_saved_objects';
import { transformRawUsageCounterObject, fetchUiCounters } from './register_ui_counters_collector';
import { rawUsageCounters } from './__fixtures__/usage_counter_saved_objects';
import { savedObjectsClientMock } from '@kbn/core/server/mocks';
import { UI_COUNTER_SAVED_OBJECT_TYPE } from './ui_counter_saved_object_type';
import { USAGE_COUNTERS_SAVED_OBJECT_TYPE } from '@kbn/usage-collection-plugin/server';
describe('transformRawUsageCounterObject', () => {
@ -63,57 +56,13 @@ describe('transformRawUsageCounterObject', () => {
});
});
describe('transformRawUiCounterObject', () => {
it('transforms ui counters savedObject raw entries', () => {
const result = rawUiCounters.map(transformRawUiCounterObject);
expect(result).toMatchInlineSnapshot(`
Array [
Object {
"appName": "Kibana_home",
"counterType": "click",
"eventName": "different_type",
"fromTimestamp": "2020-11-24T00:00:00Z",
"lastUpdatedAt": "2020-11-24T11:27:57.067Z",
"total": 1,
},
Object {
"appName": "Kibana_home",
"counterType": "loaded",
"eventName": "intersecting_event",
"fromTimestamp": "2020-10-25T00:00:00Z",
"lastUpdatedAt": "2020-10-25T11:27:57.067Z",
"total": 1,
},
Object {
"appName": "Kibana_home",
"counterType": "loaded",
"eventName": "intersecting_event",
"fromTimestamp": "2020-10-23T00:00:00Z",
"lastUpdatedAt": "2020-10-23T11:27:57.067Z",
"total": 3,
},
Object {
"appName": "Kibana_home",
"counterType": "click",
"eventName": "only_reported_in_ui_counters",
"fromTimestamp": "2020-11-24T00:00:00Z",
"lastUpdatedAt": "2020-11-24T11:27:57.067Z",
"total": 1,
},
]
`);
});
});
describe('createFetchUiCounters', () => {
let stopUsingUiCounterIndicies$: BehaviorSubject<boolean>;
describe('fetchUiCounters', () => {
const soClientMock = savedObjectsClientMock.create();
beforeEach(() => {
jest.clearAllMocks();
stopUsingUiCounterIndicies$ = new BehaviorSubject<boolean>(false);
});
it('does not query ui_counters saved objects if stopUsingUiCounterIndicies$ is complete', async () => {
it('returns saved objects only from usage_counters saved objects', async () => {
// @ts-expect-error incomplete mock implementation
soClientMock.find.mockImplementation(async ({ type }) => {
switch (type) {
@ -124,35 +73,11 @@ describe('createFetchUiCounters', () => {
}
});
stopUsingUiCounterIndicies$.complete();
// @ts-expect-error incomplete mock implementation
const { dailyEvents } = await createFetchUiCounters(stopUsingUiCounterIndicies$)({
const { dailyEvents } = await fetchUiCounters({
soClient: soClientMock,
});
const transforemdUsageCounters = rawUsageCounters.map(transformRawUsageCounterObject);
expect(soClientMock.find).toBeCalledTimes(1);
expect(dailyEvents).toEqual(transforemdUsageCounters.filter(Boolean));
});
it('merges saved objects from both ui_counters and usage_counters saved objects', async () => {
// @ts-expect-error incomplete mock implementation
soClientMock.find.mockImplementation(async ({ type }) => {
switch (type) {
case UI_COUNTER_SAVED_OBJECT_TYPE:
return { saved_objects: rawUiCounters };
case USAGE_COUNTERS_SAVED_OBJECT_TYPE:
return { saved_objects: rawUsageCounters };
default:
throw new Error(`unexpected type ${type}`);
}
});
// @ts-expect-error incomplete mock implementation
const { dailyEvents } = await createFetchUiCounters(stopUsingUiCounterIndicies$)({
soClient: soClientMock,
});
expect(dailyEvents).toHaveLength(7);
expect(dailyEvents).toHaveLength(4);
const intersectingEntry = dailyEvents.find(
({ eventName, fromTimestamp }) =>
eventName === 'intersecting_event' && fromTimestamp === '2020-10-23T00:00:00Z'
@ -179,16 +104,7 @@ describe('createFetchUiCounters', () => {
expect(invalidCountEntry).toBe(undefined);
expect(nonUiCountersEntry).toBe(undefined);
expect(zeroCountEntry).toBe(undefined);
expect(onlyFromUICountersEntry).toMatchInlineSnapshot(`
Object {
"appName": "Kibana_home",
"counterType": "click",
"eventName": "only_reported_in_ui_counters",
"fromTimestamp": "2020-11-24T00:00:00Z",
"lastUpdatedAt": "2020-11-24T11:27:57.067Z",
"total": 1,
}
`);
expect(onlyFromUICountersEntry).toBe(undefined);
expect(onlyFromUsageCountersEntry).toMatchInlineSnapshot(`
Object {
"appName": "myApp",
@ -206,7 +122,7 @@ describe('createFetchUiCounters', () => {
"eventName": "intersecting_event",
"fromTimestamp": "2020-10-23T00:00:00Z",
"lastUpdatedAt": "2020-10-23T11:27:57.067Z",
"total": 63,
"total": 60,
}
`);
});

134
src/plugins/kibana_usage_collection/server/collectors/ui_counters/register_ui_counters_collector.ts
View file

@ -7,8 +7,6 @@
*/
import moment from 'moment';
import { mergeWith } from 'lodash';
import type { Subject } from 'rxjs';
import {
CollectorFetchContext,
@ -16,18 +14,9 @@ import {
USAGE_COUNTERS_SAVED_OBJECT_TYPE,
UsageCountersSavedObject,
UsageCountersSavedObjectAttributes,
serializeCounterKey,
} from '@kbn/usage-collection-plugin/server';
import {
deserializeUiCounterName,
serializeUiCounterName,
} from '@kbn/usage-collection-plugin/common/ui_counters';
import {
UICounterSavedObject,
UICounterSavedObjectAttributes,
UI_COUNTER_SAVED_OBJECT_TYPE,
} from './ui_counter_saved_object_type';
import { deserializeUiCounterName } from '@kbn/usage-collection-plugin/common/ui_counters';
interface UiCounterEvent {
appName: string;
@ -42,32 +31,6 @@ export interface UiCountersUsage {
dailyEvents: UiCounterEvent[];
}
export function transformRawUiCounterObject(
rawUiCounter: UICounterSavedObject
): UiCounterEvent | undefined {
const {
id,
attributes: { count },
updated_at: lastUpdatedAt,
} = rawUiCounter;
if (typeof count !== 'number' || count < 1) {
return;
}
const [appName, , counterType, ...restId] = id.split(':');
const eventName = restId.join(':');
const fromTimestamp = moment(lastUpdatedAt).utc().startOf('day').format();
return {
appName,
eventName,
lastUpdatedAt,
fromTimestamp,
counterType,
total: count,
};
}
export function transformRawUsageCounterObject(
rawUsageCounter: UsageCountersSavedObject
): UiCounterEvent | undefined {
@ -93,80 +56,33 @@ export function transformRawUsageCounterObject(
};
}
export const createFetchUiCounters = (stopUsingUiCounterIndicies$: Subject<void>) =>
async function fetchUiCounters({ soClient }: CollectorFetchContext) {
const { saved_objects: rawUsageCounters } =
await soClient.find<UsageCountersSavedObjectAttributes>({
type: USAGE_COUNTERS_SAVED_OBJECT_TYPE,
fields: ['count', 'counterName', 'counterType', 'domainId'],
filter: `${USAGE_COUNTERS_SAVED_OBJECT_TYPE}.attributes.domainId: uiCounter`,
perPage: 10000,
});
export async function fetchUiCounters({ soClient }: CollectorFetchContext) {
const { saved_objects: rawUsageCounters } =
await soClient.find<UsageCountersSavedObjectAttributes>({
type: USAGE_COUNTERS_SAVED_OBJECT_TYPE,
fields: ['count', 'counterName', 'counterType', 'domainId'],
filter: `${USAGE_COUNTERS_SAVED_OBJECT_TYPE}.attributes.domainId: uiCounter`,
perPage: 10000,
});
const skipFetchingUiCounters = stopUsingUiCounterIndicies$.isStopped;
const result =
skipFetchingUiCounters ||
(await soClient.find<UICounterSavedObjectAttributes>({
type: UI_COUNTER_SAVED_OBJECT_TYPE,
fields: ['count'],
perPage: 10000,
}));
const rawUiCounters = typeof result === 'object' ? result.saved_objects : [];
const dailyEventsFromUiCounters = rawUiCounters.reduce((acc, raw) => {
try {
const event = transformRawUiCounterObject(raw);
if (event) {
const { appName, eventName, counterType } = event;
const key = serializeCounterKey({
domainId: 'uiCounter',
counterName: serializeUiCounterName({ appName, eventName }),
counterType,
date: event.lastUpdatedAt,
});
acc[key] = event;
return {
dailyEvents: Object.values(
rawUsageCounters.reduce((acc, raw) => {
try {
const event = transformRawUsageCounterObject(raw);
if (event) {
acc[raw.id] = event;
}
} catch (_) {
// swallow error; allows sending successfully transformed objects.
}
} catch (_) {
// swallow error; allows sending successfully transformed objects.
}
return acc;
}, {} as Record<string, UiCounterEvent>);
const dailyEventsFromUsageCounters = rawUsageCounters.reduce((acc, raw) => {
try {
const event = transformRawUsageCounterObject(raw);
if (event) {
acc[raw.id] = event;
}
} catch (_) {
// swallow error; allows sending successfully transformed objects.
}
return acc;
}, {} as Record<string, UiCounterEvent>);
const mergedDailyCounters = mergeWith(
dailyEventsFromUsageCounters,
dailyEventsFromUiCounters,
(value: UiCounterEvent | undefined, srcValue: UiCounterEvent): UiCounterEvent => {
if (!value) {
return srcValue;
}
return {
...srcValue,
total: srcValue.total + value.total,
};
}
);
return { dailyEvents: Object.values(mergedDailyCounters) };
return acc;
}, {} as Record<string, UiCounterEvent>)
),
};
}
export function registerUiCountersUsageCollector(
usageCollection: UsageCollectionSetup,
stopUsingUiCounterIndicies$: Subject<void>
) {
export function registerUiCountersUsageCollector(usageCollection: UsageCollectionSetup) {
const collector = usageCollection.makeUsageCollector<UiCountersUsage>({
type: 'ui_counters',
schema: {
@ -197,7 +113,7 @@ export function registerUiCountersUsageCollector(
},
},
},
fetch: createFetchUiCounters(stopUsingUiCounterIndicies$),
fetch: fetchUiCounters,
isReady: () => true,
});

22
src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/constants.ts
View file

@ -1,22 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
/**
* Roll indices every 24h
*/
export const ROLL_INDICES_INTERVAL = 24 * 60 * 60 * 1000;
/**
* Start rolling indices after 5 minutes up
*/
export const ROLL_INDICES_START = 5 * 60 * 1000;
/**
* Number of days to keep the UI counters saved object documents
*/
export const UI_COUNTERS_KEEP_DOCS_FOR_DAYS = 3;

9
src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/index.ts
View file

@ -1,9 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
export { registerUiCountersRollups } from './register_rollups';

25
src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/register_rollups.ts
View file

@ -1,25 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { Subject, timer } from 'rxjs';
import { takeUntil } from 'rxjs/operators';
import { Logger, ISavedObjectsRepository } from '@kbn/core/server';
import { ROLL_INDICES_INTERVAL, ROLL_INDICES_START } from './constants';
import { rollUiCounterIndices } from './rollups';
export function registerUiCountersRollups(
logger: Logger,
stopRollingUiCounterIndicies$: Subject<void>,
getSavedObjectsClient: () => ISavedObjectsRepository | undefined
) {
timer(ROLL_INDICES_START, ROLL_INDICES_INTERVAL)
.pipe(takeUntil(stopRollingUiCounterIndicies$))
.subscribe(() =>
rollUiCounterIndices(logger, stopRollingUiCounterIndicies$, getSavedObjectsClient())
);
}

192
src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/rollups.test.ts
View file

@ -1,192 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import moment from 'moment';
import * as Rx from 'rxjs';
import { isSavedObjectOlderThan, rollUiCounterIndices } from './rollups';
import { savedObjectsRepositoryMock, loggingSystemMock } from '@kbn/core/server/mocks';
import { SavedObjectsFindResult } from '@kbn/core/server';
import {
UICounterSavedObjectAttributes,
UI_COUNTER_SAVED_OBJECT_TYPE,
} from '../ui_counter_saved_object_type';
import { UI_COUNTERS_KEEP_DOCS_FOR_DAYS } from './constants';
const createMockSavedObjectDoc = (updatedAt: moment.Moment, id: string) =>
({
id,
type: 'ui-counter',
attributes: {
count: 3,
},
references: [],
updated_at: updatedAt.format(),
version: 'WzI5LDFd',
score: 0,
} as SavedObjectsFindResult<UICounterSavedObjectAttributes>);
describe('isSavedObjectOlderThan', () => {
it(`returns true if doc is older than x days`, () => {
const numberOfDays = 1;
const startDate = moment().format();
const doc = createMockSavedObjectDoc(moment().subtract(2, 'days'), 'some-id');
const result = isSavedObjectOlderThan({
numberOfDays,
startDate,
doc,
});
expect(result).toBe(true);
});
it(`returns false if doc is exactly x days old`, () => {
const numberOfDays = 1;
const startDate = moment().format();
const doc = createMockSavedObjectDoc(moment().subtract(1, 'days'), 'some-id');
const result = isSavedObjectOlderThan({
numberOfDays,
startDate,
doc,
});
expect(result).toBe(false);
});
it(`returns false if doc is younger than x days`, () => {
const numberOfDays = 2;
const startDate = moment().format();
const doc = createMockSavedObjectDoc(moment().subtract(1, 'days'), 'some-id');
const result = isSavedObjectOlderThan({
numberOfDays,
startDate,
doc,
});
expect(result).toBe(false);
});
});
describe('rollUiCounterIndices', () => {
let logger: ReturnType<typeof loggingSystemMock.createLogger>;
let savedObjectClient: ReturnType<typeof savedObjectsRepositoryMock.create>;
let stopUsingUiCounterIndicies$: Rx.Subject<void>;
beforeEach(() => {
logger = loggingSystemMock.createLogger();
savedObjectClient = savedObjectsRepositoryMock.create();
stopUsingUiCounterIndicies$ = new Rx.Subject();
});
it('returns undefined if no savedObjectsClient initialised yet', async () => {
await expect(
rollUiCounterIndices(logger, stopUsingUiCounterIndicies$, undefined)
).resolves.toBe(undefined);
expect(logger.warn).toHaveBeenCalledTimes(0);
});
it('does not delete any documents on empty saved objects', async () => {
savedObjectClient.find.mockImplementation(async ({ type, page = 1, perPage = 10 }) => {
switch (type) {
case UI_COUNTER_SAVED_OBJECT_TYPE:
return { saved_objects: [], total: 0, page, per_page: perPage };
default:
throw new Error(`Unexpected type [${type}]`);
}
});
await expect(
rollUiCounterIndices(logger, stopUsingUiCounterIndicies$, savedObjectClient)
).resolves.toEqual([]);
expect(savedObjectClient.find).toBeCalled();
expect(savedObjectClient.delete).not.toBeCalled();
expect(logger.warn).toHaveBeenCalledTimes(0);
});
it('calls Subject complete() on empty saved objects', async () => {
savedObjectClient.find.mockImplementation(async ({ type, page = 1, perPage = 10 }) => {
switch (type) {
case UI_COUNTER_SAVED_OBJECT_TYPE:
return { saved_objects: [], total: 0, page, per_page: perPage };
default:
throw new Error(`Unexpected type [${type}]`);
}
});
await expect(
rollUiCounterIndices(logger, stopUsingUiCounterIndicies$, savedObjectClient)
).resolves.toEqual([]);
expect(stopUsingUiCounterIndicies$.isStopped).toBe(true);
});
it(`deletes documents older than ${UI_COUNTERS_KEEP_DOCS_FOR_DAYS} days`, async () => {
const mockSavedObjects = [
createMockSavedObjectDoc(moment().subtract(5, 'days'), 'doc-id-1'),
createMockSavedObjectDoc(moment().subtract(1, 'days'), 'doc-id-2'),
createMockSavedObjectDoc(moment().subtract(6, 'days'), 'doc-id-3'),
];
savedObjectClient.find.mockImplementation(async ({ type, page = 1, perPage = 10 }) => {
switch (type) {
case UI_COUNTER_SAVED_OBJECT_TYPE:
return { saved_objects: mockSavedObjects, total: 0, page, per_page: perPage };
default:
throw new Error(`Unexpected type [${type}]`);
}
});
await expect(
rollUiCounterIndices(logger, stopUsingUiCounterIndicies$, savedObjectClient)
).resolves.toHaveLength(2);
expect(savedObjectClient.find).toBeCalled();
expect(savedObjectClient.delete).toHaveBeenCalledTimes(2);
expect(savedObjectClient.delete).toHaveBeenNthCalledWith(
1,
UI_COUNTER_SAVED_OBJECT_TYPE,
'doc-id-1'
);
expect(savedObjectClient.delete).toHaveBeenNthCalledWith(
2,
UI_COUNTER_SAVED_OBJECT_TYPE,
'doc-id-3'
);
expect(logger.warn).toHaveBeenCalledTimes(0);
});
it(`logs warnings on savedObject.find failure`, async () => {
savedObjectClient.find.mockImplementation(async () => {
throw new Error(`Expected error!`);
});
await expect(
rollUiCounterIndices(logger, stopUsingUiCounterIndicies$, savedObjectClient)
).resolves.toEqual(undefined);
expect(savedObjectClient.find).toBeCalled();
expect(savedObjectClient.delete).not.toBeCalled();
expect(logger.warn).toHaveBeenCalledTimes(2);
});
it(`logs warnings on savedObject.delete failure`, async () => {
const mockSavedObjects = [createMockSavedObjectDoc(moment().subtract(5, 'days'), 'doc-id-1')];
savedObjectClient.find.mockImplementation(async ({ type, page = 1, perPage = 10 }) => {
switch (type) {
case UI_COUNTER_SAVED_OBJECT_TYPE:
return { saved_objects: mockSavedObjects, total: 0, page, per_page: perPage };
default:
throw new Error(`Unexpected type [${type}]`);
}
});
savedObjectClient.delete.mockImplementation(async () => {
throw new Error(`Expected error!`);
});
await expect(
rollUiCounterIndices(logger, stopUsingUiCounterIndicies$, savedObjectClient)
).resolves.toEqual(undefined);
expect(savedObjectClient.find).toBeCalled();
expect(savedObjectClient.delete).toHaveBeenCalledTimes(1);
expect(savedObjectClient.delete).toHaveBeenNthCalledWith(
1,
UI_COUNTER_SAVED_OBJECT_TYPE,
'doc-id-1'
);
expect(logger.warn).toHaveBeenCalledTimes(2);
});
});

90
src/plugins/kibana_usage_collection/server/collectors/ui_counters/rollups/rollups.ts
View file

@ -1,90 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { ISavedObjectsRepository, Logger } from '@kbn/core/server';
import moment from 'moment';
import type { Subject } from 'rxjs';
import { UI_COUNTERS_KEEP_DOCS_FOR_DAYS } from './constants';
import {
UICounterSavedObject,
UI_COUNTER_SAVED_OBJECT_TYPE,
} from '../ui_counter_saved_object_type';
export function isSavedObjectOlderThan({
numberOfDays,
startDate,
doc,
}: {
numberOfDays: number;
startDate: moment.Moment | string | number;
doc: Pick<UICounterSavedObject, 'updated_at'>;
}): boolean {
const { updated_at: updatedAt } = doc;
const today = moment(startDate).startOf('day');
const updateDay = moment(updatedAt).startOf('day');
const diffInDays = today.diff(updateDay, 'days');
if (diffInDays > numberOfDays) {
return true;
}
return false;
}
export async function rollUiCounterIndices(
logger: Logger,
stopUsingUiCounterIndicies$: Subject<void>,
savedObjectsClient?: ISavedObjectsRepository
) {
if (!savedObjectsClient) {
return;
}
const now = moment();
try {
const { saved_objects: rawUiCounterDocs } = await savedObjectsClient.find<UICounterSavedObject>(
{
type: UI_COUNTER_SAVED_OBJECT_TYPE,
perPage: 1000, // Process 1000 at a time as a compromise of speed and overload
}
);
if (rawUiCounterDocs.length === 0) {
/**
* @deprecated 7.13 to be removed in 8.0.0
* Stop triggering rollups when we've rolled up all documents.
*
* This Saved Object registry is no longer used.
* Migration from one SO registry to another is not yet supported.
* In a future release we can remove this piece of code and
* migrate any docs to the Usage Counters Saved object.
*
* @removeBy 8.0.0
*/
stopUsingUiCounterIndicies$.complete();
}
const docsToDelete = rawUiCounterDocs.filter((doc) =>
isSavedObjectOlderThan({
numberOfDays: UI_COUNTERS_KEEP_DOCS_FOR_DAYS,
startDate: now,
doc,
})
);
return await Promise.all(
docsToDelete.map(({ id }) => savedObjectsClient.delete(UI_COUNTER_SAVED_OBJECT_TYPE, id))
);
} catch (err) {
logger.warn(`Failed to rollup UI Counters saved objects.`);
logger.warn(err);
}
}

30
src/plugins/kibana_usage_collection/server/collectors/ui_counters/ui_counter_saved_object_type.ts
View file

@ -1,30 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { SavedObject, SavedObjectAttributes, SavedObjectsServiceSetup } from '@kbn/core/server';
export interface UICounterSavedObjectAttributes extends SavedObjectAttributes {
count: number;
}
export type UICounterSavedObject = SavedObject<UICounterSavedObjectAttributes>;
export const UI_COUNTER_SAVED_OBJECT_TYPE = 'ui-counter';
export function registerUiCounterSavedObjectType(savedObjectsSetup: SavedObjectsServiceSetup) {
savedObjectsSetup.registerType({
name: UI_COUNTER_SAVED_OBJECT_TYPE,
hidden: false,
namespaceType: 'agnostic',
mappings: {
properties: {
count: { type: 'integer' },
},
},
});
}

6
src/plugins/kibana_usage_collection/server/plugin.ts
View file

@ -37,8 +37,6 @@ import {
registerCoreUsageCollector,
registerLocalizationUsageCollector,
registerUiCountersUsageCollector,
registerUiCounterSavedObjectType,
registerUiCountersRollups,
registerConfigUsageCollector,
registerUsageCountersRollups,
registerUsageCountersUsageCollector,
@ -125,9 +123,7 @@ export class KibanaUsageCollectionPlugin implements Plugin {
const getUiSettingsClient = () => this.uiSettingsClient;
const getCoreUsageDataService = () => this.coreUsageData!;
registerUiCounterSavedObjectType(coreSetup.savedObjects);
registerUiCountersRollups(this.logger.get('ui-counters'), pluginStop$, getSavedObjectsClient);
registerUiCountersUsageCollector(usageCollection, pluginStop$);
registerUiCountersUsageCollector(usageCollection);
registerUsageCountersRollups(this.logger.get('usage-counters-rollup'), getSavedObjectsClient);
registerUsageCountersUsageCollector(usageCollection);