github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 09:19:04 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Usage Collection] [schema] Support spreads + canvas definition (#78481)

Browse source
This commit is contained in:
Alejandro Fernández Haro 2020-09-29 12:42:13 +01:00 • committed by GitHub
parent a3b3a4d2f3
commit 406c47af46
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 458 additions and 56 deletions
Download patch file Download diff file Expand all files Collapse all files

69
packages/kbn-telemetry-tools/src/tools/__fixture__/parsed_schema_defined_with_spreads_collector.ts Normal file
View file

@ -0,0 +1,69 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { SyntaxKind } from 'typescript';
import { ParsedUsageCollection } from '../ts_parser';
export const parsedSchemaDefinedWithSpreadsCollector: ParsedUsageCollection = [
'src/fixtures/telemetry_collectors/schema_defined_with_spreads_collector.ts',
{
collectorName: 'schema_defined_with_spreads',
schema: {
value: {
flat: {
type: 'keyword',
},
my_str: {
type: 'text',
},
my_objects: {
total: {
type: 'number',
},
type: {
type: 'boolean',
},
},
},
},
fetch: {
typeName: 'Usage',
typeDescriptor: {
flat: {
kind: SyntaxKind.StringKeyword,
type: 'StringKeyword',
},
my_str: {
kind: SyntaxKind.StringKeyword,
type: 'StringKeyword',
},
my_objects: {
total: {
kind: SyntaxKind.NumberKeyword,
type: 'NumberKeyword',
},
type: {
kind: SyntaxKind.BooleanKeyword,
type: 'BooleanKeyword',
},
},
},
},
},
];

47
packages/kbn-telemetry-tools/src/tools/__snapshots__/extract_collectors.test.ts.snap generated
View file

@ -142,6 +142,53 @@ Array [
},
},
],
Array [
"src/fixtures/telemetry_collectors/schema_defined_with_spreads_collector.ts",
Object {
"collectorName": "schema_defined_with_spreads",
"fetch": Object {
"typeDescriptor": Object {
"flat": Object {
"kind": 146,
"type": "StringKeyword",
},
"my_objects": Object {
"total": Object {
"kind": 143,
"type": "NumberKeyword",
},
"type": Object {
"kind": 131,
"type": "BooleanKeyword",
},
},
"my_str": Object {
"kind": 146,
"type": "StringKeyword",
},
},
"typeName": "Usage",
},
"schema": Object {
"value": Object {
"flat": Object {
"type": "keyword",
},
"my_objects": Object {
"total": Object {
"type": "number",
},
"type": Object {
"type": "boolean",
},
},
"my_str": Object {
"type": "text",
},
},
},
},
],
Array [
"src/fixtures/telemetry_collectors/working_collector.ts",
Object {

2
packages/kbn-telemetry-tools/src/tools/extract_collectors.test.ts
View file

@ -34,7 +34,7 @@ describe('extractCollectors', () => {
const programPaths = await getProgramPaths(configs[0]);
const results = [...extractCollectors(programPaths, tsConfig)];
expect(results).toHaveLength(7);
expect(results).toHaveLength(8);
expect(results).toMatchSnapshot();
});
});

7
packages/kbn-telemetry-tools/src/tools/ts_parser.test.ts
View file

@ -25,6 +25,7 @@ import { parsedNestedCollector } from './__fixture__/parsed_nested_collector';
import { parsedExternallyDefinedCollector } from './__fixture__/parsed_externally_defined_collector';
import { parsedImportedUsageInterface } from './__fixture__/parsed_imported_usage_interface';
import { parsedImportedSchemaCollector } from './__fixture__/parsed_imported_schema';
import { parsedSchemaDefinedWithSpreadsCollector } from './__fixture__/parsed_schema_defined_with_spreads_collector';
export function loadFixtureProgram(fixtureName: string) {
const fixturePath = path.resolve(
@ -62,6 +63,12 @@ describe('parseUsageCollection', () => {
expect(result).toEqual([parsedWorkingCollector]);
});
it('parses collector with schema defined as union of spreads', () => {
const { program, sourceFile } = loadFixtureProgram('schema_defined_with_spreads_collector');
const result = [...parseUsageCollection(sourceFile, program)];
expect(result).toEqual([parsedSchemaDefinedWithSpreadsCollector]);
});
it('parses nested collectors', () => {
const { program, sourceFile } = loadFixtureProgram('nested_collector');
const result = [...parseUsageCollection(sourceFile, program)];

103
packages/kbn-telemetry-tools/src/tools/utils.ts
View file

@ -100,42 +100,55 @@ export function getIdentifierDeclaration(node: ts.Node) {
return getIdentifierDeclarationFromSource(node, source);
}
export function getVariableValue(node: ts.Node): string | Record<string, any> {
export function getVariableValue(node: ts.Node, program: ts.Program): string | Record<string, any> {
if (ts.isStringLiteral(node) || ts.isNumericLiteral(node)) {
return node.text;
}
if (ts.isObjectLiteralExpression(node)) {
return serializeObject(node);
return serializeObject(node, program);
}
if (ts.isIdentifier(node)) {
const declaration = getIdentifierDeclaration(node);
if (ts.isVariableDeclaration(declaration) && declaration.initializer) {
return getVariableValue(declaration.initializer);
return getVariableValue(declaration.initializer, program);
} else {
// Go fetch it in another file
return getIdentifierValue(node, node, program, { chaseImport: true });
}
// TODO: If this is another imported value from another file, we'll need to go fetch it like in getPropertyValue
}
throw Error(`Unsuppored Node: cannot get value of node (${node.getText()}) of kind ${node.kind}`);
if (ts.isSpreadAssignment(node)) {
return getVariableValue(node.expression, program);
}
throw Error(
`Unsupported Node: cannot get value of node (${node.getText()}) of kind ${node.kind}`
);
}
export function serializeObject(node: ts.Node) {
export function serializeObject(node: ts.Node, program: ts.Program) {
if (!ts.isObjectLiteralExpression(node)) {
throw new Error(`Expecting Object literal Expression got ${node.getText()}`);
}
const value: Record<string, any> = {};
let value: Record<string, any> = {};
for (const property of node.properties) {
const propertyName = property.name?.getText();
const val = ts.isPropertyAssignment(property)
? getVariableValue(property.initializer, program)
: getVariableValue(property, program);
if (typeof propertyName === 'undefined') {
throw new Error(`Unable to get property name ${property.getText()}`);
}
const cleanPropertyName = propertyName.replace(/["']/g, '');
if (ts.isPropertyAssignment(property)) {
value[cleanPropertyName] = getVariableValue(property.initializer);
if (typeof val === 'object') {
value = { ...value, ...val };
} else {
throw new Error(`Unable to get property name ${property.getText()}`);
}
} else {
value[cleanPropertyName] = getVariableValue(property);
const cleanPropertyName = propertyName.replace(/["']/g, '');
value[cleanPropertyName] = val;
}
}
@ -155,45 +168,53 @@ export function getResolvedModuleSourceFile(
return resolvedModuleSourceFile;
}
export function getIdentifierValue(
node: ts.Node,
initializer: ts.Identifier,
program: ts.Program,
config: Optional<{ chaseImport: boolean }> = {}
) {
const { chaseImport = false } = config;
const identifierName = initializer.getText();
const declaration = getIdentifierDeclaration(initializer);
if (ts.isImportSpecifier(declaration)) {
if (!chaseImport) {
throw new Error(
`Value of node ${identifierName} is imported from another file. Chasing imports is not allowed.`
);
}
const importedModuleName = getModuleSpecifier(declaration);
const source = node.getSourceFile();
const declarationSource = getResolvedModuleSourceFile(source, program, importedModuleName);
const declarationNode = getIdentifierDeclarationFromSource(initializer, declarationSource);
if (!ts.isVariableDeclaration(declarationNode)) {
throw new Error(`Expected ${identifierName} to be variable declaration.`);
}
if (!declarationNode.initializer) {
throw new Error(`Expected ${identifierName} to be initialized.`);
}
const serializedObject = serializeObject(declarationNode.initializer, program);
return serializedObject;
}
return getVariableValue(declaration, program);
}
export function getPropertyValue(
node: ts.Node,
program: ts.Program,
config: Optional<{ chaseImport: boolean }> = {}
) {
const { chaseImport = false } = config;
if (ts.isPropertyAssignment(node)) {
const { initializer } = node;
if (ts.isIdentifier(initializer)) {
const identifierName = initializer.getText();
const declaration = getIdentifierDeclaration(initializer);
if (ts.isImportSpecifier(declaration)) {
if (!chaseImport) {
throw new Error(
`Value of node ${identifierName} is imported from another file. Chasing imports is not allowed.`
);
}
const importedModuleName = getModuleSpecifier(declaration);
const source = node.getSourceFile();
const declarationSource = getResolvedModuleSourceFile(source, program, importedModuleName);
const declarationNode = getIdentifierDeclarationFromSource(initializer, declarationSource);
if (!ts.isVariableDeclaration(declarationNode)) {
throw new Error(`Expected ${identifierName} to be variable declaration.`);
}
if (!declarationNode.initializer) {
throw new Error(`Expected ${identifierName} to be initialized.`);
}
const serializedObject = serializeObject(declarationNode.initializer);
return serializedObject;
}
return getVariableValue(declaration);
return getIdentifierValue(node, initializer, program, config);
}
return getVariableValue(initializer);
return getVariableValue(initializer, program);
}
}

3
src/fixtures/telemetry_collectors/indexed_interface_with_not_matching_schema.ts
View file

@ -41,8 +41,9 @@ export const myCollector = makeUsageCollector<Usage>({
return { something: { count_2: 2 } };
},
schema: {
// @ts-expect-error Intentionally missing count_2
something: {
count_1: { type: 'long' }, // Intentionally missing count_2
count_1: { type: 'long' },
},
},
});

77
src/fixtures/telemetry_collectors/schema_defined_with_spreads_collector.ts Normal file
View file

@ -0,0 +1,77 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { CollectorSet, MakeSchemaFrom } from '../../plugins/usage_collection/server/collector';
import { loggerMock } from '../../core/server/logging/logger.mock';
const { makeUsageCollector } = new CollectorSet({
logger: loggerMock.create(),
maximumWaitTimeForAllCollectorsInS: 0,
});
interface MyObject {
total: number;
type: boolean;
}
interface Usage {
flat?: string;
my_str?: string;
my_objects: MyObject;
}
const SOME_NUMBER: number = 123;
const someSchema: MakeSchemaFrom<Pick<Usage, 'flat' | 'my_str'>> = {
flat: {
type: 'keyword',
},
my_str: {
type: 'text',
},
};
const someOtherSchema: MakeSchemaFrom<Pick<Usage, 'my_objects'>> = {
my_objects: {
total: {
type: 'number',
},
type: { type: 'boolean' },
},
};
export const myCollector = makeUsageCollector<Usage>({
type: 'schema_defined_with_spreads',
isReady: () => true,
fetch() {
const testString = '123';
return {
flat: 'hello',
my_str: testString,
my_objects: {
total: SOME_NUMBER,
type: true,
},
};
},
schema: {
...someSchema,
...someOtherSchema,
},
});

2
src/plugins/telemetry/server/collectors/usage/schema.ts
View file

@ -31,7 +31,7 @@ const licenseSchema: MakeSchemaFrom<LicenseUsage> = {
max_resource_units: { type: 'long' },
};
export const staticTelemetrySchema: MakeSchemaFrom<Required<StaticTelemetryUsage>> = {
export const staticTelemetrySchema: MakeSchemaFrom<StaticTelemetryUsage> = {
ece: {
kb_uuid: { type: 'keyword' },
es_uuid: { type: 'keyword' },

7
src/plugins/usage_collection/server/collector/collector.ts
View file

@ -38,16 +38,17 @@ export type RecursiveMakeSchemaFrom<U> = U extends object
? MakeSchemaFrom<U>
: { type: AllowedSchemaTypes };
// Using Required to enforce all optional keys in the object
export type MakeSchemaFrom<Base> = {
[Key in keyof Base]: Base[Key] extends Array<infer U>
[Key in keyof Required<Base>]: Required<Base>[Key] extends Array<infer U>
? { type: 'array'; items: RecursiveMakeSchemaFrom<U> }
: RecursiveMakeSchemaFrom<Base[Key]>;
: RecursiveMakeSchemaFrom<Required<Base>[Key]>;
};
export interface CollectorOptions<T = unknown, U = T> {
type: string;
init?: Function;
schema?: MakeSchemaFrom<Required<T>>; // Using Required to enforce all optional keys in the object
schema?: MakeSchemaFrom<T>;
fetch: (callCluster: LegacyAPICaller, esClient?: ElasticsearchClient) => Promise<T> | T;
/*
* A hook for allowing the fetched data payload to be organized into a typed

1
x-pack/.telemetryrc.json
View file

@ -5,7 +5,6 @@
"plugins/actions/server/usage/actions_usage_collector.ts",
"plugins/alerts/server/usage/alerts_usage_collector.ts",
"plugins/apm/server/lib/apm_telemetry/index.ts",
"plugins/canvas/server/collectors/collector.ts",
"plugins/infra/server/usage/usage_collector.ts",
"plugins/reporting/server/usage/reporting_usage_collector.ts",
"plugins/maps/server/maps_telemetry/collectors/register.ts"

18
x-pack/plugins/canvas/server/collectors/collector.ts
View file

@ -5,11 +5,16 @@
*/
import { UsageCollectionSetup } from 'src/plugins/usage_collection/server';
import { LegacyAPICaller } from 'kibana/server';
import { TelemetryCollector } from '../../types';
import { workpadCollector } from './workpad_collector';
import { customElementCollector } from './custom_element_collector';
import { workpadCollector, workpadSchema, WorkpadTelemetry } from './workpad_collector';
import {
customElementCollector,
CustomElementTelemetry,
customElementSchema,
} from './custom_element_collector';
type CanvasUsage = WorkpadTelemetry & CustomElementTelemetry;
const collectors: TelemetryCollector[] = [workpadCollector, customElementCollector];
@ -29,18 +34,19 @@ export function registerCanvasUsageCollector(
return;
}
const canvasCollector = usageCollection.makeUsageCollector({
const canvasCollector = usageCollection.makeUsageCollector<CanvasUsage>({
type: 'canvas',
isReady: () => true,
fetch: async (callCluster: LegacyAPICaller) => {
fetch: async (callCluster) => {
const collectorResults = await Promise.all(
collectors.map((collector) => collector(kibanaIndex, callCluster))
);
return collectorResults.reduce((reduction, usage) => {
return { ...reduction, ...usage };
}, {});
}, {}) as CanvasUsage; // We need the casting because `TelemetryCollector` claims it returns `Record<string, any>`
},
schema: { ...workpadSchema, ...customElementSchema },
});
usageCollection.registerCollector(canvasCollector);

15
x-pack/plugins/canvas/server/collectors/custom_element_collector.ts
View file

@ -6,6 +6,7 @@
import { SearchParams } from 'elasticsearch';
import { get } from 'lodash';
import { MakeSchemaFrom } from 'src/plugins/usage_collection/server';
import { collectFns } from './collector_helpers';
import {
TelemetryCollector,
@ -19,7 +20,7 @@ interface CustomElementSearch {
[CUSTOM_ELEMENT_TYPE]: TelemetryCustomElementDocument;
}
interface CustomElementTelemetry {
export interface CustomElementTelemetry {
custom_elements?: {
count: number;
elements: {
@ -31,6 +32,18 @@ interface CustomElementTelemetry {
};
}
export const customElementSchema: MakeSchemaFrom<CustomElementTelemetry> = {
custom_elements: {
count: { type: 'long' },
elements: {
min: { type: 'long' },
max: { type: 'long' },
avg: { type: 'float' },
},
functions_in_use: { type: 'array', items: { type: 'keyword' } },
},
};
function isCustomElement(maybeCustomElement: any): maybeCustomElement is TelemetryCustomElement {
return (
maybeCustomElement !== null &&

40
x-pack/plugins/canvas/server/collectors/workpad_collector.ts
View file

@ -6,6 +6,7 @@
import { SearchParams } from 'elasticsearch';
import { sum as arraySum, min as arrayMin, max as arrayMax, get } from 'lodash';
import { MakeSchemaFrom } from 'src/plugins/usage_collection/server';
import { CANVAS_TYPE } from '../../common/lib/constants';
import { collectFns } from './collector_helpers';
import { TelemetryCollector, CanvasWorkpad } from '../../types';
@ -15,7 +16,7 @@ interface WorkpadSearch {
[CANVAS_TYPE]: CanvasWorkpad;
}
interface WorkpadTelemetry {
export interface WorkpadTelemetry {
workpads?: {
total: number;
};
@ -54,6 +55,43 @@ interface WorkpadTelemetry {
};
}
export const workpadSchema: MakeSchemaFrom<WorkpadTelemetry> = {
workpads: { total: { type: 'long' } },
pages: {
total: { type: 'long' },
per_workpad: {
avg: { type: 'float' },
min: { type: 'long' },
max: { type: 'long' },
},
},
elements: {
total: { type: 'long' },
per_page: {
avg: { type: 'float' },
min: { type: 'long' },
max: { type: 'long' },
},
},
functions: {
total: { type: 'long' },
in_use: { type: 'array', items: { type: 'keyword' } },
per_element: {
avg: { type: 'float' },
min: { type: 'long' },
max: { type: 'long' },
},
},
variables: {
total: { type: 'long' },
per_workpad: {
avg: { type: 'float' },
min: { type: 'long' },
max: { type: 'long' },
},
},
};
/**
Gather statistic about the given workpads
@param workpadDocs a collection of workpad documents

123
x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json
View file

@ -1,5 +1,128 @@
{
"properties": {
"canvas": {
"properties": {
"workpads": {
"properties": {
"total": {
"type": "long"
}
}
},
"pages": {
"properties": {
"total": {
"type": "long"
},
"per_workpad": {
"properties": {
"avg": {
"type": "float"
},
"min": {
"type": "long"
},
"max": {
"type": "long"
}
}
}
}
},
"elements": {
"properties": {
"total": {
"type": "long"
},
"per_page": {
"properties": {
"avg": {
"type": "float"
},
"min": {
"type": "long"
},
"max": {
"type": "long"
}
}
}
}
},
"functions": {
"properties": {
"total": {
"type": "long"
},
"in_use": {
"type": "array",
"items": {
"type": "keyword"
}
},
"per_element": {
"properties": {
"avg": {
"type": "float"
},
"min": {
"type": "long"
},
"max": {
"type": "long"
}
}
}
}
},
"variables": {
"properties": {
"total": {
"type": "long"
},
"per_workpad": {
"properties": {
"avg": {
"type": "float"
},
"min": {
"type": "long"
},
"max": {
"type": "long"
}
}
}
}
},
"custom_elements": {
"properties": {
"count": {
"type": "long"
},
"elements": {
"properties": {
"min": {
"type": "long"
},
"max": {
"type": "long"
},
"avg": {
"type": "float"
}
}
},
"functions_in_use": {
"type": "array",
"items": {
"type": "keyword"
}
}
}
}
}
},
"cloud": {
"properties": {
"isCloudEnabled": {