[Usage Collection] Usage collection add saved objects client to collector fetch context (#80554)

This commit is contained in:
Christiane (Tina) Heiligers 2020-10-14 16:26:49 -07:00 committed by GitHub
parent 66b2976656
commit 9afd63f56d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
22 changed files with 193 additions and 70 deletions

View file

@ -20,6 +20,7 @@ import { EnvironmentMode } from '@kbn/config';
import { ErrorToastOptions } from 'src/core/public/notifications'; import { ErrorToastOptions } from 'src/core/public/notifications';
import { ExpressionAstFunction } from 'src/plugins/expressions/common'; import { ExpressionAstFunction } from 'src/plugins/expressions/common';
import { ExpressionsServerSetup } from 'src/plugins/expressions/server'; import { ExpressionsServerSetup } from 'src/plugins/expressions/server';
import { ISavedObjectsRepository } from 'kibana/server';
import { ISearchOptions as ISearchOptions_2 } from 'src/plugins/data/public'; import { ISearchOptions as ISearchOptions_2 } from 'src/plugins/data/public';
import { ISearchSource } from 'src/plugins/data/public'; import { ISearchSource } from 'src/plugins/data/public';
import { KibanaRequest } from 'src/core/server'; import { KibanaRequest } from 'src/core/server';

View file

@ -35,6 +35,7 @@ import {
Logger, Logger,
IClusterClient, IClusterClient,
UiSettingsServiceStart, UiSettingsServiceStart,
SavedObjectsServiceStart,
} from '../../../core/server'; } from '../../../core/server';
import { registerRoutes } from './routes'; import { registerRoutes } from './routes';
import { registerCollection } from './telemetry_collection'; import { registerCollection } from './telemetry_collection';
@ -88,6 +89,7 @@ export class TelemetryPlugin implements Plugin<TelemetryPluginSetup, TelemetryPl
private readonly oldUiSettingsHandled$ = new AsyncSubject(); private readonly oldUiSettingsHandled$ = new AsyncSubject();
private savedObjectsClient?: ISavedObjectsRepository; private savedObjectsClient?: ISavedObjectsRepository;
private elasticsearchClient?: IClusterClient; private elasticsearchClient?: IClusterClient;
private savedObjectsService?: SavedObjectsServiceStart;
constructor(initializerContext: PluginInitializerContext<TelemetryConfigType>) { constructor(initializerContext: PluginInitializerContext<TelemetryConfigType>) {
this.logger = initializerContext.logger.get(); this.logger = initializerContext.logger.get();
@ -110,7 +112,8 @@ export class TelemetryPlugin implements Plugin<TelemetryPluginSetup, TelemetryPl
registerCollection( registerCollection(
telemetryCollectionManager, telemetryCollectionManager,
elasticsearch.legacy.client, elasticsearch.legacy.client,
() => this.elasticsearchClient () => this.elasticsearchClient,
() => this.savedObjectsService
); );
const router = http.createRouter(); const router = http.createRouter();
@ -139,6 +142,7 @@ export class TelemetryPlugin implements Plugin<TelemetryPluginSetup, TelemetryPl
const savedObjectsInternalRepository = savedObjects.createInternalRepository(); const savedObjectsInternalRepository = savedObjects.createInternalRepository();
this.savedObjectsClient = savedObjectsInternalRepository; this.savedObjectsClient = savedObjectsInternalRepository;
this.elasticsearchClient = elasticsearch.client; this.elasticsearchClient = elasticsearch.client;
this.savedObjectsService = savedObjects;
// Not catching nor awaiting these promises because they should never reject // Not catching nor awaiting these promises because they should never reject
this.handleOldUiSettings(uiSettings); this.handleOldUiSettings(uiSettings);

View file

@ -19,7 +19,11 @@
import { omit } from 'lodash'; import { omit } from 'lodash';
import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; import { UsageCollectionSetup } from 'src/plugins/usage_collection/server';
import { LegacyAPICaller } from 'kibana/server'; import {
ISavedObjectsRepository,
LegacyAPICaller,
SavedObjectsClientContract,
} from 'kibana/server';
import { StatsCollectionContext } from 'src/plugins/telemetry_collection_manager/server'; import { StatsCollectionContext } from 'src/plugins/telemetry_collection_manager/server';
import { ElasticsearchClient } from 'src/core/server'; import { ElasticsearchClient } from 'src/core/server';
@ -84,8 +88,9 @@ export function handleKibanaStats(
export async function getKibana( export async function getKibana(
usageCollection: UsageCollectionSetup, usageCollection: UsageCollectionSetup,
callWithInternalUser: LegacyAPICaller, callWithInternalUser: LegacyAPICaller,
asInternalUser: ElasticsearchClient asInternalUser: ElasticsearchClient,
soClient: SavedObjectsClientContract | ISavedObjectsRepository
): Promise<KibanaUsageStats> { ): Promise<KibanaUsageStats> {
const usage = await usageCollection.bulkFetch(callWithInternalUser, asInternalUser); const usage = await usageCollection.bulkFetch(callWithInternalUser, asInternalUser, soClient);
return usageCollection.toObject(usage); return usageCollection.toObject(usage);
} }

View file

@ -20,7 +20,10 @@
import { merge, omit } from 'lodash'; import { merge, omit } from 'lodash';
import { getLocalStats, handleLocalStats } from './get_local_stats'; import { getLocalStats, handleLocalStats } from './get_local_stats';
import { usageCollectionPluginMock } from '../../../usage_collection/server/mocks'; import {
usageCollectionPluginMock,
createCollectorFetchContextMock,
} from '../../../usage_collection/server/mocks';
import { elasticsearchServiceMock } from '../../../../../src/core/server/mocks'; import { elasticsearchServiceMock } from '../../../../../src/core/server/mocks';
function mockUsageCollection(kibanaUsage = {}) { function mockUsageCollection(kibanaUsage = {}) {
@ -79,6 +82,16 @@ function mockGetLocalStats(clusterInfo: any, clusterStats: any) {
return esClient; return esClient;
} }
function mockStatsCollectionConfig(clusterInfo: any, clusterStats: any, kibana: {}) {
return {
...createCollectorFetchContextMock(),
esClient: mockGetLocalStats(clusterInfo, clusterStats),
usageCollection: mockUsageCollection(kibana),
start: '',
end: '',
};
}
describe('get_local_stats', () => { describe('get_local_stats', () => {
const clusterUuid = 'abc123'; const clusterUuid = 'abc123';
const clusterName = 'my-cool-cluster'; const clusterName = 'my-cool-cluster';
@ -224,12 +237,10 @@ describe('get_local_stats', () => {
describe('getLocalStats', () => { describe('getLocalStats', () => {
it('returns expected object with kibana data', async () => { it('returns expected object with kibana data', async () => {
const callCluster = jest.fn(); const statsCollectionConfig = mockStatsCollectionConfig(clusterInfo, clusterStats, kibana);
const usageCollection = mockUsageCollection(kibana);
const esClient = mockGetLocalStats(clusterInfo, clusterStats);
const response = await getLocalStats( const response = await getLocalStats(
[{ clusterUuid: 'abc123' }], [{ clusterUuid: 'abc123' }],
{ callCluster, usageCollection, esClient, start: '', end: '' }, { ...statsCollectionConfig },
context context
); );
const result = response[0]; const result = response[0];
@ -244,14 +255,8 @@ describe('get_local_stats', () => {
}); });
it('returns an empty array when no cluster uuid is provided', async () => { it('returns an empty array when no cluster uuid is provided', async () => {
const callCluster = jest.fn(); const statsCollectionConfig = mockStatsCollectionConfig(clusterInfo, clusterStats, kibana);
const usageCollection = mockUsageCollection(kibana); const response = await getLocalStats([], { ...statsCollectionConfig }, context);
const esClient = mockGetLocalStats(clusterInfo, clusterStats);
const response = await getLocalStats(
[],
{ callCluster, usageCollection, esClient, start: '', end: '' },
context
);
expect(response).toBeDefined(); expect(response).toBeDefined();
expect(response.length).toEqual(0); expect(response.length).toEqual(0);
}); });

View file

@ -68,10 +68,10 @@ export type TelemetryLocalStats = ReturnType<typeof handleLocalStats>;
*/ */
export const getLocalStats: StatsGetter<{}, TelemetryLocalStats> = async ( export const getLocalStats: StatsGetter<{}, TelemetryLocalStats> = async (
clustersDetails, // array of cluster uuid's clustersDetails, // array of cluster uuid's
config, // contains the new esClient already scoped contains usageCollection, callCluster, esClient, start, end config, // contains the new esClient already scoped contains usageCollection, callCluster, esClient, start, end and the saved objects client scoped to the request or the internal repository
context // StatsCollectionContext contains logger and version (string) context // StatsCollectionContext contains logger and version (string)
) => { ) => {
const { callCluster, usageCollection, esClient } = config; const { callCluster, usageCollection, esClient, soClient } = config;
return await Promise.all( return await Promise.all(
clustersDetails.map(async (clustersDetail) => { clustersDetails.map(async (clustersDetail) => {
@ -79,7 +79,7 @@ export const getLocalStats: StatsGetter<{}, TelemetryLocalStats> = async (
getClusterInfo(esClient), // cluster info getClusterInfo(esClient), // cluster info
getClusterStats(esClient), // cluster stats (not to be confused with cluster _state_) getClusterStats(esClient), // cluster stats (not to be confused with cluster _state_)
getNodesUsage(esClient), // nodes_usage info getNodesUsage(esClient), // nodes_usage info
getKibana(usageCollection, callCluster, esClient), getKibana(usageCollection, callCluster, esClient, soClient),
getDataTelemetry(esClient), getDataTelemetry(esClient),
]); ]);
return handleLocalStats( return handleLocalStats(

View file

@ -36,7 +36,7 @@
* under the License. * under the License.
*/ */
import { ILegacyClusterClient } from 'kibana/server'; import { ILegacyClusterClient, SavedObjectsServiceStart } from 'kibana/server';
import { TelemetryCollectionManagerPluginSetup } from 'src/plugins/telemetry_collection_manager/server'; import { TelemetryCollectionManagerPluginSetup } from 'src/plugins/telemetry_collection_manager/server';
import { IClusterClient } from '../../../../../src/core/server'; import { IClusterClient } from '../../../../../src/core/server';
import { getLocalStats } from './get_local_stats'; import { getLocalStats } from './get_local_stats';
@ -46,11 +46,13 @@ import { getLocalLicense } from './get_local_license';
export function registerCollection( export function registerCollection(
telemetryCollectionManager: TelemetryCollectionManagerPluginSetup, telemetryCollectionManager: TelemetryCollectionManagerPluginSetup,
esCluster: ILegacyClusterClient, esCluster: ILegacyClusterClient,
esClientGetter: () => IClusterClient | undefined esClientGetter: () => IClusterClient | undefined,
soServiceGetter: () => SavedObjectsServiceStart | undefined
) { ) {
telemetryCollectionManager.setCollection({ telemetryCollectionManager.setCollection({
esCluster, esCluster,
esClientGetter, esClientGetter,
soServiceGetter,
title: 'local', title: 'local',
priority: 0, priority: 0,
statsGetter: getLocalStats, statsGetter: getLocalStats,

View file

@ -25,6 +25,7 @@ import {
Plugin, Plugin,
Logger, Logger,
IClusterClient, IClusterClient,
SavedObjectsServiceStart,
} from '../../../core/server'; } from '../../../core/server';
import { import {
@ -90,6 +91,7 @@ export class TelemetryCollectionManagerPlugin
priority, priority,
esCluster, esCluster,
esClientGetter, esClientGetter,
soServiceGetter,
statsGetter, statsGetter,
clusterDetailsGetter, clusterDetailsGetter,
licenseGetter, licenseGetter,
@ -112,6 +114,9 @@ export class TelemetryCollectionManagerPlugin
if (!esClientGetter) { if (!esClientGetter) {
throw Error('esClientGetter method not set.'); throw Error('esClientGetter method not set.');
} }
if (!soServiceGetter) {
throw Error('soServiceGetter method not set.');
}
if (!clusterDetailsGetter) { if (!clusterDetailsGetter) {
throw Error('Cluster UUIds method is not set.'); throw Error('Cluster UUIds method is not set.');
} }
@ -126,6 +131,7 @@ export class TelemetryCollectionManagerPlugin
esCluster, esCluster,
title, title,
esClientGetter, esClientGetter,
soServiceGetter,
}); });
this.usageGetterMethodPriority = priority; this.usageGetterMethodPriority = priority;
} }
@ -135,6 +141,7 @@ export class TelemetryCollectionManagerPlugin
config: StatsGetterConfig, config: StatsGetterConfig,
collection: Collection, collection: Collection,
collectionEsClient: IClusterClient, collectionEsClient: IClusterClient,
collectionSoService: SavedObjectsServiceStart,
usageCollection: UsageCollectionSetup usageCollection: UsageCollectionSetup
): StatsCollectionConfig { ): StatsCollectionConfig {
const { start, end, request } = config; const { start, end, request } = config;
@ -146,7 +153,11 @@ export class TelemetryCollectionManagerPlugin
const esClient = config.unencrypted const esClient = config.unencrypted
? collectionEsClient.asScoped(config.request).asCurrentUser ? collectionEsClient.asScoped(config.request).asCurrentUser
: collectionEsClient.asInternalUser; : collectionEsClient.asInternalUser;
return { callCluster, start, end, usageCollection, esClient }; // Scope the saved objects client appropriately and pass to the stats collection config
const soClient = config.unencrypted
? collectionSoService.getScopedClient(config.request)
: collectionSoService.createInternalRepository();
return { callCluster, start, end, usageCollection, esClient, soClient };
} }
private async getOptInStats(optInStatus: boolean, config: StatsGetterConfig) { private async getOptInStats(optInStatus: boolean, config: StatsGetterConfig) {
@ -156,11 +167,13 @@ export class TelemetryCollectionManagerPlugin
for (const collection of this.collections) { for (const collection of this.collections) {
// first fetch the client and make sure it's not undefined. // first fetch the client and make sure it's not undefined.
const collectionEsClient = collection.esClientGetter(); const collectionEsClient = collection.esClientGetter();
if (collectionEsClient !== undefined) { const collectionSoService = collection.soServiceGetter();
if (collectionEsClient !== undefined && collectionSoService !== undefined) {
const statsCollectionConfig = this.getStatsCollectionConfig( const statsCollectionConfig = this.getStatsCollectionConfig(
config, config,
collection, collection,
collectionEsClient, collectionEsClient,
collectionSoService,
this.usageCollection this.usageCollection
); );
@ -215,11 +228,13 @@ export class TelemetryCollectionManagerPlugin
} }
for (const collection of this.collections) { for (const collection of this.collections) {
const collectionEsClient = collection.esClientGetter(); const collectionEsClient = collection.esClientGetter();
if (collectionEsClient !== undefined) { const collectionSavedObjectsService = collection.soServiceGetter();
if (collectionEsClient !== undefined && collectionSavedObjectsService !== undefined) {
const statsCollectionConfig = this.getStatsCollectionConfig( const statsCollectionConfig = this.getStatsCollectionConfig(
config, config,
collection, collection,
collectionEsClient, collectionEsClient,
collectionSavedObjectsService,
this.usageCollection this.usageCollection
); );
try { try {

View file

@ -23,6 +23,9 @@ import {
KibanaRequest, KibanaRequest,
ILegacyClusterClient, ILegacyClusterClient,
IClusterClient, IClusterClient,
SavedObjectsServiceStart,
SavedObjectsClientContract,
ISavedObjectsRepository,
} from 'kibana/server'; } from 'kibana/server';
import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; import { UsageCollectionSetup } from 'src/plugins/usage_collection/server';
import { ElasticsearchClient } from '../../../../src/core/server'; import { ElasticsearchClient } from '../../../../src/core/server';
@ -77,6 +80,7 @@ export interface StatsCollectionConfig {
start: string | number; start: string | number;
end: string | number; end: string | number;
esClient: ElasticsearchClient; esClient: ElasticsearchClient;
soClient: SavedObjectsClientContract | ISavedObjectsRepository;
} }
export interface BasicStatsPayload { export interface BasicStatsPayload {
@ -141,6 +145,7 @@ export interface CollectionConfig<
priority: number; priority: number;
esCluster: ILegacyClusterClient; esCluster: ILegacyClusterClient;
esClientGetter: () => IClusterClient | undefined; // --> by now we know that the client getter will return the IClusterClient but we assure that through a code check esClientGetter: () => IClusterClient | undefined; // --> by now we know that the client getter will return the IClusterClient but we assure that through a code check
soServiceGetter: () => SavedObjectsServiceStart | undefined; // --> by now we know that the service getter will return the SavedObjectsServiceStart but we assure that through a code check
statsGetter: StatsGetter<CustomContext, T>; statsGetter: StatsGetter<CustomContext, T>;
clusterDetailsGetter: ClusterDetailsGetter<CustomContext>; clusterDetailsGetter: ClusterDetailsGetter<CustomContext>;
licenseGetter: LicenseGetter<CustomContext>; licenseGetter: LicenseGetter<CustomContext>;
@ -157,5 +162,6 @@ export interface Collection<
clusterDetailsGetter: ClusterDetailsGetter<CustomContext>; clusterDetailsGetter: ClusterDetailsGetter<CustomContext>;
esCluster: ILegacyClusterClient; esCluster: ILegacyClusterClient;
esClientGetter: () => IClusterClient | undefined; // the collection could still return undefined for the es client getter. esClientGetter: () => IClusterClient | undefined; // the collection could still return undefined for the es client getter.
soServiceGetter: () => SavedObjectsServiceStart | undefined; // the collection could still return undefined for the Saved Objects Service getter.
title: string; title: string;
} }

View file

@ -64,7 +64,7 @@ All you need to provide is a `type` for organizing your fields, `schema` field t
}, },
fetch: async (collectorFetchContext: CollectorFetchContext) => { fetch: async (collectorFetchContext: CollectorFetchContext) => {
// query ES and get some data // query ES or saved objects and get some data
// summarize the data into a model // summarize the data into a model
// return the modeled object that includes whatever you want to track // return the modeled object that includes whatever you want to track
@ -85,9 +85,11 @@ Some background:
- `MY_USAGE_TYPE` can be any string. It usually matches the plugin name. As a safety mechanism, we double check there are no duplicates at the moment of registering the collector. - `MY_USAGE_TYPE` can be any string. It usually matches the plugin name. As a safety mechanism, we double check there are no duplicates at the moment of registering the collector.
- The `fetch` method needs to support multiple contexts in which it is called. For example, when stats are pulled from a Kibana Metricbeat module, the Beat calls Kibana's stats API to invoke usage collection. - The `fetch` method needs to support multiple contexts in which it is called. For example, when stats are pulled from a Kibana Metricbeat module, the Beat calls Kibana's stats API to invoke usage collection.
In this case, the `fetch` method is called as a result of an HTTP API request and `callCluster` wraps `callWithRequest` or `esClient` wraps `asCurrentUser`, where the request headers are expected to have read privilege on the entire `.kibana' index. In this case, the `fetch` method is called as a result of an HTTP API request and `callCluster` wraps `callWithRequest` or `esClient` wraps `asCurrentUser`, where the request headers are expected to have read privilege on the entire `.kibana' index. The `fetch` method also exposes the saved objects client that will have the correct scope when the collectors' `fetch` method is called.
Note: there will be many cases where you won't need to use the `callCluster` (or `esClient`) function that gets passed in to your `fetch` method at all. Your feature might have an accumulating value in server memory, or read something from the OS, or use other clients like a custom SavedObjects client. In that case it's up to the plugin to initialize those clients like the example below: Note: there will be many cases where you won't need to use the `callCluster`, `esClient` or `soClient` function that gets passed in to your `fetch` method at all. Your feature might have an accumulating value in server memory, or read something from the OS.
In the case of using a custom SavedObjects client, it is up to the plugin to initialize the client to save the data and it is strongly recommended to scope that client to the `kibana_system` user.
```ts ```ts
// server/plugin.ts // server/plugin.ts
@ -98,7 +100,7 @@ class Plugin {
private savedObjectsRepository?: ISavedObjectsRepository; private savedObjectsRepository?: ISavedObjectsRepository;
public setup(core: CoreSetup, plugins: { usageCollection?: UsageCollectionSetup }) { public setup(core: CoreSetup, plugins: { usageCollection?: UsageCollectionSetup }) {
registerMyPluginUsageCollector(() => this.savedObjectsRepository, plugins.usageCollection); registerMyPluginUsageCollector(plugins.usageCollection);
} }
public start(core: CoreStart) { public start(core: CoreStart) {

View file

@ -17,7 +17,13 @@
* under the License. * under the License.
*/ */
import { Logger, LegacyAPICaller, ElasticsearchClient } from 'kibana/server'; import {
Logger,
LegacyAPICaller,
ElasticsearchClient,
ISavedObjectsRepository,
SavedObjectsClientContract,
} from 'kibana/server';
export type CollectorFormatForBulkUpload<T, U> = (result: T) => { type: string; payload: U }; export type CollectorFormatForBulkUpload<T, U> = (result: T) => { type: string; payload: U };
@ -56,7 +62,14 @@ export interface CollectorFetchContext {
* - When building the telemetry data payload to report to the remote cluster, the requests are scoped to the `kibana` internal user * - When building the telemetry data payload to report to the remote cluster, the requests are scoped to the `kibana` internal user
*/ */
esClient: ElasticsearchClient; esClient: ElasticsearchClient;
/**
* Request-scoped Saved Objects client:
* - When users are requesting a sample of data, it is scoped to their role to avoid exposing data they should't read
* - When building the telemetry data payload to report to the remote cluster, the requests are scoped to the `kibana` internal user
*/
soClient: SavedObjectsClientContract | ISavedObjectsRepository;
} }
export interface CollectorOptions<T = unknown, U = T> { export interface CollectorOptions<T = unknown, U = T> {
type: string; type: string;
init?: Function; init?: Function;

View file

@ -21,7 +21,11 @@ import { noop } from 'lodash';
import { Collector } from './collector'; import { Collector } from './collector';
import { CollectorSet } from './collector_set'; import { CollectorSet } from './collector_set';
import { UsageCollector } from './usage_collector'; import { UsageCollector } from './usage_collector';
import { elasticsearchServiceMock, loggingSystemMock } from '../../../../core/server/mocks'; import {
elasticsearchServiceMock,
loggingSystemMock,
savedObjectsRepositoryMock,
} from '../../../../core/server/mocks';
const logger = loggingSystemMock.createLogger(); const logger = loggingSystemMock.createLogger();
@ -40,9 +44,9 @@ describe('CollectorSet', () => {
loggerSpies.debug.mockRestore(); loggerSpies.debug.mockRestore();
loggerSpies.warn.mockRestore(); loggerSpies.warn.mockRestore();
}); });
const mockCallCluster = jest.fn().mockResolvedValue({ passTest: 1000 }); const mockCallCluster = jest.fn().mockResolvedValue({ passTest: 1000 });
const mockEsClient = elasticsearchServiceMock.createClusterClient().asInternalUser; const mockEsClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
const mockSoClient = savedObjectsRepositoryMock.create();
it('should throw an error if non-Collector type of object is registered', () => { it('should throw an error if non-Collector type of object is registered', () => {
const collectors = new CollectorSet({ logger }); const collectors = new CollectorSet({ logger });
@ -88,7 +92,7 @@ describe('CollectorSet', () => {
}) })
); );
const result = await collectors.bulkFetch(mockCallCluster, mockEsClient); const result = await collectors.bulkFetch(mockCallCluster, mockEsClient, mockSoClient);
expect(loggerSpies.debug).toHaveBeenCalledTimes(1); expect(loggerSpies.debug).toHaveBeenCalledTimes(1);
expect(loggerSpies.debug).toHaveBeenCalledWith( expect(loggerSpies.debug).toHaveBeenCalledWith(
'Fetching data from MY_TEST_COLLECTOR collector' 'Fetching data from MY_TEST_COLLECTOR collector'
@ -113,7 +117,7 @@ describe('CollectorSet', () => {
let result; let result;
try { try {
result = await collectors.bulkFetch(mockCallCluster, mockEsClient); result = await collectors.bulkFetch(mockCallCluster, mockEsClient, mockSoClient);
} catch (err) { } catch (err) {
// Do nothing // Do nothing
} }
@ -131,7 +135,7 @@ describe('CollectorSet', () => {
}) })
); );
const result = await collectors.bulkFetch(mockCallCluster, mockEsClient); const result = await collectors.bulkFetch(mockCallCluster, mockEsClient, mockSoClient);
expect(result).toStrictEqual([ expect(result).toStrictEqual([
{ {
type: 'MY_TEST_COLLECTOR', type: 'MY_TEST_COLLECTOR',
@ -149,7 +153,7 @@ describe('CollectorSet', () => {
} as any) } as any)
); );
const result = await collectors.bulkFetch(mockCallCluster, mockEsClient); const result = await collectors.bulkFetch(mockCallCluster, mockEsClient, mockSoClient);
expect(result).toStrictEqual([ expect(result).toStrictEqual([
{ {
type: 'MY_TEST_COLLECTOR', type: 'MY_TEST_COLLECTOR',
@ -172,7 +176,7 @@ describe('CollectorSet', () => {
}) })
); );
const result = await collectors.bulkFetch(mockCallCluster, mockEsClient); const result = await collectors.bulkFetch(mockCallCluster, mockEsClient, mockSoClient);
expect(result).toStrictEqual([ expect(result).toStrictEqual([
{ {
type: 'MY_TEST_COLLECTOR', type: 'MY_TEST_COLLECTOR',

View file

@ -18,7 +18,13 @@
*/ */
import { snakeCase } from 'lodash'; import { snakeCase } from 'lodash';
import { Logger, LegacyAPICaller, ElasticsearchClient } from 'kibana/server'; import {
Logger,
LegacyAPICaller,
ElasticsearchClient,
ISavedObjectsRepository,
SavedObjectsClientContract,
} from 'kibana/server';
import { Collector, CollectorOptions } from './collector'; import { Collector, CollectorOptions } from './collector';
import { UsageCollector } from './usage_collector'; import { UsageCollector } from './usage_collector';
@ -125,6 +131,7 @@ export class CollectorSet {
public bulkFetch = async ( public bulkFetch = async (
callCluster: LegacyAPICaller, callCluster: LegacyAPICaller,
esClient: ElasticsearchClient, esClient: ElasticsearchClient,
soClient: SavedObjectsClientContract | ISavedObjectsRepository,
collectors: Map<string, Collector<any, any>> = this.collectors collectors: Map<string, Collector<any, any>> = this.collectors
) => { ) => {
const responses = await Promise.all( const responses = await Promise.all(
@ -133,7 +140,7 @@ export class CollectorSet {
try { try {
return { return {
type: collector.type, type: collector.type,
result: await collector.fetch({ callCluster, esClient }), result: await collector.fetch({ callCluster, esClient, soClient }),
}; };
} catch (err) { } catch (err) {
this.logger.warn(err); this.logger.warn(err);
@ -155,9 +162,18 @@ export class CollectorSet {
return this.makeCollectorSetFromArray(filtered); return this.makeCollectorSetFromArray(filtered);
}; };
public bulkFetchUsage = async (callCluster: LegacyAPICaller, esClient: ElasticsearchClient) => { public bulkFetchUsage = async (
callCluster: LegacyAPICaller,
esClient: ElasticsearchClient,
savedObjectsClient: SavedObjectsClientContract | ISavedObjectsRepository
) => {
const usageCollectors = this.getFilteredCollectorSet((c) => c instanceof UsageCollector); const usageCollectors = this.getFilteredCollectorSet((c) => c instanceof UsageCollector);
return await this.bulkFetch(callCluster, esClient, usageCollectors.collectors); return await this.bulkFetch(
callCluster,
esClient,
savedObjectsClient,
usageCollectors.collectors
);
}; };
// convert an array of fetched stats results into key/object // convert an array of fetched stats results into key/object

View file

@ -26,8 +26,10 @@ import { first } from 'rxjs/operators';
import { import {
ElasticsearchClient, ElasticsearchClient,
IRouter, IRouter,
ISavedObjectsRepository,
LegacyAPICaller, LegacyAPICaller,
MetricsServiceSetup, MetricsServiceSetup,
SavedObjectsClientContract,
ServiceStatus, ServiceStatus,
ServiceStatusLevels, ServiceStatusLevels,
} from '../../../../../core/server'; } from '../../../../../core/server';
@ -64,9 +66,10 @@ export function registerStatsRoute({
}) { }) {
const getUsage = async ( const getUsage = async (
callCluster: LegacyAPICaller, callCluster: LegacyAPICaller,
esClient: ElasticsearchClient esClient: ElasticsearchClient,
savedObjectsClient: SavedObjectsClientContract | ISavedObjectsRepository
): Promise<any> => { ): Promise<any> => {
const usage = await collectorSet.bulkFetchUsage(callCluster, esClient); const usage = await collectorSet.bulkFetchUsage(callCluster, esClient, savedObjectsClient);
return collectorSet.toObject(usage); return collectorSet.toObject(usage);
}; };
@ -101,6 +104,7 @@ export function registerStatsRoute({
if (isExtended) { if (isExtended) {
const callCluster = context.core.elasticsearch.legacy.client.callAsCurrentUser; const callCluster = context.core.elasticsearch.legacy.client.callAsCurrentUser;
const esClient = context.core.elasticsearch.client.asCurrentUser; const esClient = context.core.elasticsearch.client.asCurrentUser;
const savedObjectsClient = context.core.savedObjects.client;
if (shouldGetUsage) { if (shouldGetUsage) {
const collectorsReady = await collectorSet.areAllCollectorsReady(); const collectorsReady = await collectorSet.areAllCollectorsReady();
@ -109,7 +113,9 @@ export function registerStatsRoute({
} }
} }
const usagePromise = shouldGetUsage ? getUsage(callCluster, esClient) : Promise.resolve({}); const usagePromise = shouldGetUsage
? getUsage(callCluster, esClient, savedObjectsClient)
: Promise.resolve({});
const [usage, clusterUuid] = await Promise.all([usagePromise, getClusterUuid(callCluster)]); const [usage, clusterUuid] = await Promise.all([usagePromise, getClusterUuid(callCluster)]);
let modifiedUsage = usage; let modifiedUsage = usage;

View file

@ -17,7 +17,10 @@
* under the License. * under the License.
*/ */
import { elasticsearchServiceMock } from '../../../../src/core/server/mocks'; import {
elasticsearchServiceMock,
savedObjectsRepositoryMock,
} from '../../../../src/core/server/mocks';
import { CollectorOptions } from './collector/collector'; import { CollectorOptions } from './collector/collector';
import { UsageCollectionSetup, CollectorFetchContext } from './index'; import { UsageCollectionSetup, CollectorFetchContext } from './index';
@ -52,6 +55,7 @@ export function createCollectorFetchContextMock(): jest.Mocked<CollectorFetchCon
const collectorFetchClientsMock: jest.Mocked<CollectorFetchContext> = { const collectorFetchClientsMock: jest.Mocked<CollectorFetchContext> = {
callCluster: elasticsearchServiceMock.createLegacyClusterClient().callAsInternalUser, callCluster: elasticsearchServiceMock.createLegacyClusterClient().callAsInternalUser,
esClient: elasticsearchServiceMock.createClusterClient().asInternalUser, esClient: elasticsearchServiceMock.createClusterClient().asInternalUser,
soClient: savedObjectsRepositoryMock.create(),
}; };
return collectorFetchClientsMock; return collectorFetchClientsMock;
} }

View file

@ -21,6 +21,7 @@ import {
CustomHttpResponseOptions, CustomHttpResponseOptions,
ResponseError, ResponseError,
IClusterClient, IClusterClient,
SavedObjectsServiceStart,
} from 'kibana/server'; } from 'kibana/server';
import { DEFAULT_APP_CATEGORIES } from '../../../../src/core/server'; import { DEFAULT_APP_CATEGORIES } from '../../../../src/core/server';
import { import {
@ -76,6 +77,7 @@ export class Plugin {
private legacyShimDependencies = {} as LegacyShimDependencies; private legacyShimDependencies = {} as LegacyShimDependencies;
private bulkUploader: IBulkUploader = {} as IBulkUploader; private bulkUploader: IBulkUploader = {} as IBulkUploader;
private telemetryElasticsearchClient: IClusterClient | undefined; private telemetryElasticsearchClient: IClusterClient | undefined;
private telemetrySavedObjectsService: SavedObjectsServiceStart | undefined;
constructor(initializerContext: PluginInitializerContext) { constructor(initializerContext: PluginInitializerContext) {
this.initializerContext = initializerContext; this.initializerContext = initializerContext;
@ -145,14 +147,15 @@ export class Plugin {
// Initialize telemetry // Initialize telemetry
if (plugins.telemetryCollectionManager) { if (plugins.telemetryCollectionManager) {
registerMonitoringCollection( registerMonitoringCollection({
plugins.telemetryCollectionManager, telemetryCollectionManager: plugins.telemetryCollectionManager,
this.cluster, esCluster: this.cluster,
() => this.telemetryElasticsearchClient, esClientGetter: () => this.telemetryElasticsearchClient,
{ soServiceGetter: () => this.telemetrySavedObjectsService,
customContext: {
maxBucketSize: config.ui.max_bucket_size, maxBucketSize: config.ui.max_bucket_size,
} },
); });
} }
// Register collector objects for stats to show up in the APIs // Register collector objects for stats to show up in the APIs
@ -249,12 +252,15 @@ export class Plugin {
}; };
} }
start({ elasticsearch }: CoreStart) { start({ elasticsearch, savedObjects }: CoreStart) {
// TODO: For the telemetry plugin to work, we need to provide the new ES client. // TODO: For the telemetry plugin to work, we need to provide the new ES client.
// The new client should be inititalized with a similar config to `this.cluster` but, since we're not using // The new client should be inititalized with a similar config to `this.cluster` but, since we're not using
// the new client in Monitoring Telemetry collection yet, setting the local client allos progress for now. // the new client in Monitoring Telemetry collection yet, setting the local client allows progress for now.
// The usage collector `fetch` method has been refactored to accept a `collectorFetchContext` object,
// exposing both es clients and the saved objects client.
// We will update the client in a follow up PR. // We will update the client in a follow up PR.
this.telemetryElasticsearchClient = elasticsearch.client; this.telemetryElasticsearchClient = elasticsearch.client;
this.telemetrySavedObjectsService = savedObjects;
} }
stop() { stop() {

View file

@ -16,6 +16,7 @@ describe('get_all_stats', () => {
const end = 1; const end = 1;
const callCluster = sinon.stub(); const callCluster = sinon.stub();
const esClient = sinon.stub(); const esClient = sinon.stub();
const soClient = sinon.stub();
const esClusters = [ const esClusters = [
{ cluster_uuid: 'a' }, { cluster_uuid: 'a' },
@ -178,6 +179,7 @@ describe('get_all_stats', () => {
{ {
callCluster: callCluster as any, callCluster: callCluster as any,
esClient: esClient as any, esClient: esClient as any,
soClient: soClient as any,
usageCollection: {} as any, usageCollection: {} as any,
start, start,
end, end,
@ -204,6 +206,7 @@ describe('get_all_stats', () => {
{ {
callCluster: callCluster as any, callCluster: callCluster as any,
esClient: esClient as any, esClient: esClient as any,
soClient: soClient as any,
usageCollection: {} as any, usageCollection: {} as any,
start, start,
end, end,

View file

@ -28,7 +28,7 @@ export interface CustomContext {
*/ */
export const getAllStats: StatsGetter<CustomContext> = async ( export const getAllStats: StatsGetter<CustomContext> = async (
clustersDetails, clustersDetails,
{ callCluster, start, end, esClient }, { callCluster, start, end, esClient, soClient },
{ maxBucketSize } { maxBucketSize }
) => { ) => {
const clusterUuids = clustersDetails.map((clusterDetails) => clusterDetails.clusterUuid); const clusterUuids = clustersDetails.map((clusterDetails) => clusterDetails.clusterUuid);

View file

@ -5,7 +5,7 @@
*/ */
import sinon from 'sinon'; import sinon from 'sinon';
import { elasticsearchServiceMock } from 'src/core/server/mocks'; import { elasticsearchServiceMock, savedObjectsRepositoryMock } from 'src/core/server/mocks';
import { import {
getClusterUuids, getClusterUuids,
fetchClusterUuids, fetchClusterUuids,
@ -15,6 +15,7 @@ import {
describe('get_cluster_uuids', () => { describe('get_cluster_uuids', () => {
const callCluster = sinon.stub(); const callCluster = sinon.stub();
const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
const soClient = savedObjectsRepositoryMock.create();
const response = { const response = {
aggregations: { aggregations: {
cluster_uuids: { cluster_uuids: {
@ -32,9 +33,12 @@ describe('get_cluster_uuids', () => {
it('returns cluster UUIDs', async () => { it('returns cluster UUIDs', async () => {
callCluster.withArgs('search').returns(Promise.resolve(response)); callCluster.withArgs('search').returns(Promise.resolve(response));
expect( expect(
await getClusterUuids({ callCluster, esClient, start, end, usageCollection: {} as any }, { await getClusterUuids(
maxBucketSize: 1, { callCluster, esClient, soClient, start, end, usageCollection: {} as any },
} as any) {
maxBucketSize: 1,
} as any
)
).toStrictEqual(expectedUuids); ).toStrictEqual(expectedUuids);
}); });
}); });
@ -43,9 +47,12 @@ describe('get_cluster_uuids', () => {
it('searches for clusters', async () => { it('searches for clusters', async () => {
callCluster.returns(Promise.resolve(response)); callCluster.returns(Promise.resolve(response));
expect( expect(
await fetchClusterUuids({ callCluster, esClient, start, end, usageCollection: {} as any }, { await fetchClusterUuids(
maxBucketSize: 1, { callCluster, esClient, soClient, start, end, usageCollection: {} as any },
} as any) {
maxBucketSize: 1,
} as any
)
).toStrictEqual(response); ).toStrictEqual(response);
}); });
}); });

View file

@ -4,21 +4,33 @@
* you may not use this file except in compliance with the Elastic License. * you may not use this file except in compliance with the Elastic License.
*/ */
import { ILegacyCustomClusterClient, IClusterClient } from 'kibana/server'; import {
ILegacyCustomClusterClient,
IClusterClient,
SavedObjectsServiceStart,
} from 'kibana/server';
import { TelemetryCollectionManagerPluginSetup } from 'src/plugins/telemetry_collection_manager/server'; import { TelemetryCollectionManagerPluginSetup } from 'src/plugins/telemetry_collection_manager/server';
import { getAllStats, CustomContext } from './get_all_stats'; import { getAllStats, CustomContext } from './get_all_stats';
import { getClusterUuids } from './get_cluster_uuids'; import { getClusterUuids } from './get_cluster_uuids';
import { getLicenses } from './get_licenses'; import { getLicenses } from './get_licenses';
export function registerMonitoringCollection( export function registerMonitoringCollection({
telemetryCollectionManager: TelemetryCollectionManagerPluginSetup, telemetryCollectionManager,
esCluster: ILegacyCustomClusterClient, esCluster,
esClientGetter: () => IClusterClient | undefined, esClientGetter,
customContext: CustomContext soServiceGetter,
) { customContext,
}: {
telemetryCollectionManager: TelemetryCollectionManagerPluginSetup;
esCluster: ILegacyCustomClusterClient;
esClientGetter: () => IClusterClient | undefined;
soServiceGetter: () => SavedObjectsServiceStart | undefined;
customContext: CustomContext;
}) {
telemetryCollectionManager.setCollection({ telemetryCollectionManager.setCollection({
esCluster, esCluster,
esClientGetter, esClientGetter,
soServiceGetter,
title: 'monitoring', title: 'monitoring',
priority: 2, priority: 2,
statsGetter: getAllStats, statsGetter: getAllStats,

View file

@ -10,6 +10,7 @@ import {
CoreStart, CoreStart,
Plugin, Plugin,
IClusterClient, IClusterClient,
SavedObjectsServiceStart,
} from 'kibana/server'; } from 'kibana/server';
import { TelemetryCollectionManagerPluginSetup } from 'src/plugins/telemetry_collection_manager/server'; import { TelemetryCollectionManagerPluginSetup } from 'src/plugins/telemetry_collection_manager/server';
import { getClusterUuids, getLocalLicense } from '../../../../src/plugins/telemetry/server'; import { getClusterUuids, getLocalLicense } from '../../../../src/plugins/telemetry/server';
@ -21,12 +22,14 @@ interface TelemetryCollectionXpackDepsSetup {
export class TelemetryCollectionXpackPlugin implements Plugin { export class TelemetryCollectionXpackPlugin implements Plugin {
private elasticsearchClient?: IClusterClient; private elasticsearchClient?: IClusterClient;
private savedObjectsService?: SavedObjectsServiceStart;
constructor(initializerContext: PluginInitializerContext) {} constructor(initializerContext: PluginInitializerContext) {}
public setup(core: CoreSetup, { telemetryCollectionManager }: TelemetryCollectionXpackDepsSetup) { public setup(core: CoreSetup, { telemetryCollectionManager }: TelemetryCollectionXpackDepsSetup) {
telemetryCollectionManager.setCollection({ telemetryCollectionManager.setCollection({
esCluster: core.elasticsearch.legacy.client, esCluster: core.elasticsearch.legacy.client,
esClientGetter: () => this.elasticsearchClient, esClientGetter: () => this.elasticsearchClient,
soServiceGetter: () => this.savedObjectsService,
title: 'local_xpack', title: 'local_xpack',
priority: 1, priority: 1,
statsGetter: getStatsWithXpack, statsGetter: getStatsWithXpack,
@ -37,5 +40,6 @@ export class TelemetryCollectionXpackPlugin implements Plugin {
public start(core: CoreStart) { public start(core: CoreStart) {
this.elasticsearchClient = core.elasticsearch.client; this.elasticsearchClient = core.elasticsearch.client;
this.savedObjectsService = core.savedObjects;
} }
} }

View file

@ -13,7 +13,11 @@ import {
ServiceStatus, ServiceStatus,
ServiceStatusLevels, ServiceStatusLevels,
} from '../../../../../src/core/server'; } from '../../../../../src/core/server';
import { contextServiceMock, elasticsearchServiceMock } from '../../../../../src/core/server/mocks'; import {
contextServiceMock,
elasticsearchServiceMock,
savedObjectsServiceMock,
} from '../../../../../src/core/server/mocks';
import { createHttpServer } from '../../../../../src/core/server/test_utils'; import { createHttpServer } from '../../../../../src/core/server/test_utils';
import { registerSettingsRoute } from './settings'; import { registerSettingsRoute } from './settings';
@ -42,6 +46,9 @@ describe('/api/settings', () => {
asCurrentUser: elasticsearchServiceMock.createScopedClusterClient().asCurrentUser, asCurrentUser: elasticsearchServiceMock.createScopedClusterClient().asCurrentUser,
}, },
}, },
savedObjects: {
client: savedObjectsServiceMock.create(),
},
}, },
}), }),
}); });

View file

@ -45,6 +45,7 @@ export function registerSettingsRoute({
const collectorFetchContext = { const collectorFetchContext = {
callCluster: callAsCurrentUser, callCluster: callAsCurrentUser,
esClient: context.core.elasticsearch.client.asCurrentUser, esClient: context.core.elasticsearch.client.asCurrentUser,
soClient: context.core.savedObjects.client,
}; };
const settingsCollector = usageCollection.getCollectorByType(KIBANA_SETTINGS_TYPE) as const settingsCollector = usageCollection.getCollectorByType(KIBANA_SETTINGS_TYPE) as