github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 01:38:56 -04:00
Code Issues Projects Releases Packages Wiki Activity

[8.x] [Sustainable Architecture] Telemetry schemas (#201760) (#202042)

Browse source 
# Backport

This will backport the following commits from `main` to `8.x`:
- [[Sustainable Architecture] Telemetry schemas
(#201760)](https://github.com/elastic/kibana/pull/201760)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Alejandro Fernández
Haro","email":"alejandro.haro@elastic.co"},"sourceCommit":{"committedDate":"2024-11-27T17:18:09Z","message":"[Sustainable
Architecture] Telemetry schemas (#201760)\n\n## Summary\n\nSince we are
moving code around, the schema extractor now needs to adapt\nto the new
directories.\n\nThis PR adds new per-solution schemas to the
`.telemetryrc.json` files\nand adapts FTRs accordingly.\n\nRelated
https://github.com/elastic/kibana/pull/201653\n\n### Checklist\n\n-
[x]\n[Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html)\nwas
added for features that require explanation or tutorials\n- [x] [Unit or
functional\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\nwere
updated or added to match the most common
scenarios","sha":"dac87ef6fec2ff2291ea72ffe5ec00588265500a","branchLabelMapping":{"^v9.0.0$":"main","^v8.18.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["Team:Core","Feature:Telemetry","technical
debt","release_note:skip","v9.0.0","backport:prev-minor"],"title":"[Sustainable
Architecture] Telemetry
schemas","number":201760,"url":"https://github.com/elastic/kibana/pull/201760","mergeCommit":{"message":"[Sustainable
Architecture] Telemetry schemas (#201760)\n\n## Summary\n\nSince we are
moving code around, the schema extractor now needs to adapt\nto the new
directories.\n\nThis PR adds new per-solution schemas to the
`.telemetryrc.json` files\nand adapts FTRs accordingly.\n\nRelated
https://github.com/elastic/kibana/pull/201653\n\n### Checklist\n\n-
[x]\n[Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html)\nwas
added for features that require explanation or tutorials\n- [x] [Unit or
functional\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\nwere
updated or added to match the most common
scenarios","sha":"dac87ef6fec2ff2291ea72ffe5ec00588265500a"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/201760","number":201760,"mergeCommit":{"message":"[Sustainable
Architecture] Telemetry schemas (#201760)\n\n## Summary\n\nSince we are
moving code around, the schema extractor now needs to adapt\nto the new
directories.\n\nThis PR adds new per-solution schemas to the
`.telemetryrc.json` files\nand adapts FTRs accordingly.\n\nRelated
https://github.com/elastic/kibana/pull/201653\n\n### Checklist\n\n-
[x]\n[Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html)\nwas
added for features that require explanation or tutorials\n- [x] [Unit or
functional\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\nwere
updated or added to match the most common
scenarios","sha":"dac87ef6fec2ff2291ea72ffe5ec00588265500a"}}]}]
BACKPORT-->

Co-authored-by: Alejandro Fernández Haro <alejandro.haro@elastic.co>
This commit is contained in:
Kibana Machine 2024-11-28 06:09:08 +11:00 • committed by GitHub
parent 42839cacb7
commit 86e6ff0352
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
15 changed files with 102 additions and 25 deletions
Download patch file Download diff file Expand all files Collapse all files

5
.telemetryrc.json
View file

@ -10,5 +10,10 @@
"output": "src/plugins/telemetry/schema/kbn_packages.json",
"root": "packages/",
"exclude": []
},
{
"output": "src/plugins/telemetry/schema/oss_platform.json",
"root": "src/platform/",
"exclude": []
}
]

3
packages/kbn-telemetry-tools/src/tools/extract_collectors.ts
View file

@ -37,7 +37,8 @@ export async function getProgramPaths({
);
if (filePaths.length === 0) {
throw Error(`No files found in ${root}`);
return []; // Temporarily accept empty directories while https://github.com/elastic/kibana-team/issues/1066 is completed
// throw Error(`No files found in ${root}`);
}
const fullPaths = filePaths

6
src/plugins/telemetry/schema/README.md
View file

@ -2,16 +2,18 @@
This list of `.json` files describes the format of the payloads sent to the Remote Telemetry Service. All the files should follow the schema convention as defined in the `usage_collection` plugin and `@kbn/telemetry-tools`, with the addition of the type `pass_through`. This additional `type` indicates Kibana sends the payload as-is from the output of an external ES query.
There are currently 2 files:
There are currently 4 files:
- `oss_root.json`: Defines the schema for the payload from the root keys.
Manually maintained for now because the frequency it changes should be pretty low.
- `oss_plugins.json`: The schema for the content that will be nested in `stack_stats.kibana.plugins`.
It is automatically generated by `@kbn/telemetry-tools` based on the `schema` property provided by all the registered Usage Collectors via the `usageCollection.makeUsageCollector` API.
More details in the [Schema field](../../usage_collection/README.md#schema-field) chapter in the UsageCollection's docs.
- `kbn_packages.json`: Same as `oss_plugins` but for collectors defined in `<ROOT_DIR>/packages/*`.
- `oss_platform.json`: Same as `oss_plugins` but for collectors defined in `<ROOT_DIR>/src/platform/*`.
NOTE: Despite its similarities to ES mappings, the intention of these files is not to define any index mappings. They should be considered as a tool to understand the format of the payload that will be sent when reporting telemetry to the Remote Service.
## Testing
Functional tests are defined at `test/api_integration/apis/telemetry/telemetry_local.ts`. They merge both files, and validates the actual output of the telemetry endpoint against the final schema.
Functional tests are defined at `x-pack/test/api_integration/apis/telemetry/telemetry_local.ts`. They merge all files and [the x-pack counterparts](../../../../x-pack/plugins/telemetry_collection_xpack/schema), and validates the actual output of the telemetry endpoint against the final schema.

3
src/plugins/telemetry/schema/oss_platform.json Normal file
View file

@ -0,0 +1,3 @@
{
"properties": {}
}

4
src/plugins/telemetry/tsconfig.json
View file

@ -9,9 +9,7 @@
"server/**/**/*",
"common/**/*",
"../../../typings/**/*",
"schema/oss_plugins.json",
"schema/oss_root.json",
"schema/kbn_packages.json"
"schema/*.json",
],
"kbn_references": [
"@kbn/core",

20
x-pack/.telemetryrc.json
View file

@ -10,5 +10,25 @@
"output": "plugins/telemetry_collection_xpack/schema/xpack_monitoring.json",
"root": "plugins/monitoring/server/telemetry_collection/",
"exclude": []
},
{
"output": "plugins/telemetry_collection_xpack/schema/xpack_platform.json",
"root": "platform/",
"exclude": []
},
{
"output": "plugins/telemetry_collection_xpack/schema/xpack_observability.json",
"root": "solutions/observability",
"exclude": []
},
{
"output": "plugins/telemetry_collection_xpack/schema/xpack_search.json",
"root": "solutions/search",
"exclude": []
},
{
"output": "plugins/telemetry_collection_xpack/schema/xpack_security.json",
"root": "solutions/security",
"exclude": []
}
]

8
x-pack/plugins/telemetry_collection_xpack/schema/README.md
View file

@ -2,17 +2,21 @@
This is an extension of the [OSS Telemetry Schemas](../../../../src/plugins/telemetry/schema) to add the X-Pack-related data. The payloads described in these `.json` files must be merged to the OSS ones to get the structure of the full payload sent to the Remote Telemetry Service. All the files follow the schema convention as defined in the `usage_collection` plugin and `@kbn/telemetry-tools`.
There are currently 2 files:
There are currently 7 files:
- `xpack_root.json`: Defines the extra fields x-pack reports over the OSS payload defined in the `oss_root.json`.
Manually maintained for now because the frequency it changes is expected to be pretty low.
- `xpack_plugins.json`: The X-Pack related schema for the content that will be nested in `stack_stats.kibana.plugins`.
It is automatically generated by `@kbn/telemetry-tools` based on the `schema` property provided by all the registered Usage Collectors via the `usageCollection.makeUsageCollector` API.
More details in the [Schema field](../../usage_collection/README.md#schema-field) chapter in the UsageCollection's docs.
- `xpack_platform.json`: Same as `xpack_plugins.json` but for collectors defined in `<ROOT_DIR>/x-pack/platform/*`.
- `xpack_observability.json`: Same as `xpack_plugins.json` but for collectors defined in `<ROOT_DIR>/x-pack/solutions/observability/*`.
- `xpack_search.json`: Same as `xpack_plugins.json` but for collectors defined in `<ROOT_DIR>/x-pack/solutions/search/*`.
- `xpack_security.json`: Same as `xpack_plugins.json` but for collectors defined in `<ROOT_DIR>/x-pack/solutions/security/*`.
- `xpack_monitoring.json`: It declares the payload sent by the monitoring-sourced telemetry. The actual schema for the payload is declared under `properties.monitoringTelemetry.properties.stats.items`, but due to the general behaviour in the `@kbn/telemetry-tools`, it gets nested down in that path.
NOTE: Despite its similarities to ES mappings, the intention of these files is not to define any index mappings. They should be considered as a tool to understand the format of the payload that will be sent when reporting telemetry to the Remote Service.
## Testing
Functional tests are defined at `x-pack/test/api_integration/apis/telemetry/telemetry_local.ts`. They merge both files (+ the OSS definitions), and validates the actual output of the telemetry endpoint against the final schema.
Functional tests are defined at `x-pack/test/api_integration/apis/telemetry/telemetry_local.ts`. They merge these files (+ the [OSS definitions](../../../../src/plugins/telemetry/schema)), and validates the actual output of the telemetry endpoint against the final schema.

3
x-pack/plugins/telemetry_collection_xpack/schema/xpack_observability.json Normal file
View file

@ -0,0 +1,3 @@
{
"properties": {}
}

3
x-pack/plugins/telemetry_collection_xpack/schema/xpack_platform.json Normal file
View file

@ -0,0 +1,3 @@
{
"properties": {}
}

3
x-pack/plugins/telemetry_collection_xpack/schema/xpack_search.json Normal file
View file

@ -0,0 +1,3 @@
{
"properties": {}
}

3
x-pack/plugins/telemetry_collection_xpack/schema/xpack_security.json Normal file
View file

@ -0,0 +1,3 @@
{
"properties": {}
}

4
x-pack/plugins/telemetry_collection_xpack/tsconfig.json
View file

@ -8,9 +8,7 @@
"common/**/*",
"server/**/*",
"../../../typings/*",
"schema/xpack_monitoring.json",
"schema/xpack_plugins.json",
"schema/xpack_root.json",
"schema/*.json",
],
"kbn_references": [
"@kbn/core",

22
x-pack/test/api_integration/apis/telemetry/telemetry.ts
View file

@ -14,8 +14,13 @@ import ossRootTelemetrySchema from '@kbn/telemetry-plugin/schema/oss_root.json';
import xpackRootTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_root.json';
import monitoringRootTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_monitoring.json';
import ossPluginsTelemetrySchema from '@kbn/telemetry-plugin/schema/oss_plugins.json';
import ossPlatformTelemetrySchema from '@kbn/telemetry-plugin/schema/oss_platform.json';
import ossPackagesTelemetrySchema from '@kbn/telemetry-plugin/schema/kbn_packages.json';
import xpackPluginsTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_plugins.json';
import xpackPlatformTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_platform.json';
import xpackObservabilityTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_observability.json';
import xpackSearchTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_search.json';
import xpackSecurityTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_security.json';
import type { UnencryptedTelemetryPayload } from '@kbn/telemetry-plugin/common/types';
import type {
UsageStatsPayload,
@ -161,10 +166,17 @@ export default function ({ getService }: FtrProviderContext) {
// It's nested because of the way it's collected and declared
monitoringRootTelemetrySchema.properties.monitoringTelemetry.properties.stats.items
);
const plugins = deepmerge(
deepmerge(ossPluginsTelemetrySchema, ossPackagesTelemetrySchema),
xpackPluginsTelemetrySchema
);
const plugins = [
ossPluginsTelemetrySchema,
ossPackagesTelemetrySchema,
ossPlatformTelemetrySchema,
xpackPluginsTelemetrySchema,
xpackPlatformTelemetrySchema,
xpackObservabilityTelemetrySchema,
xpackSearchTelemetrySchema,
xpackSecurityTelemetrySchema,
].reduce((acc, schema) => deepmerge(acc, schema));
try {
assertTelemetryPayload({ root, plugins }, localXPack);
@ -172,7 +184,7 @@ export default function ({ getService }: FtrProviderContext) {
assertTelemetryPayload({ root: monitoringRoot, plugins }, stats);
});
} catch (err) {
err.message = `The telemetry schemas in 'x-pack/plugins/telemetry_collection_xpack/schema/' are out-of-date, please update it as required: ${err.message}`;
err.message = `The telemetry schemas in are out-of-date. Please define the schema of your collector and run "node scripts/telemetry_check --fix" to update them: ${err.message}`;
throw err;
}
});

21
x-pack/test/api_integration/apis/telemetry/telemetry_local.ts
View file

@ -9,9 +9,14 @@ import expect from '@kbn/expect';
import deepmerge from 'deepmerge';
import ossRootTelemetrySchema from '@kbn/telemetry-plugin/schema/oss_root.json';
import ossPluginsTelemetrySchema from '@kbn/telemetry-plugin/schema/oss_plugins.json';
import ossPlatformTelemetrySchema from '@kbn/telemetry-plugin/schema/oss_platform.json';
import ossPackagesTelemetrySchema from '@kbn/telemetry-plugin/schema/kbn_packages.json';
import xpackRootTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_root.json';
import xpackPluginsTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_plugins.json';
import xpackPlatformTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_platform.json';
import xpackObservabilityTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_observability.json';
import xpackSearchTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_search.json';
import xpackSecurityTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_security.json';
import { assertTelemetryPayload } from '@kbn/telemetry-tools';
import {
ELASTIC_HTTP_VERSION_HEADER,
@ -57,15 +62,21 @@ export default function ({ getService }: FtrProviderContext) {
it('should pass the schema validation', () => {
const root = deepmerge(ossRootTelemetrySchema, xpackRootTelemetrySchema);
const plugins = deepmerge(
deepmerge(ossPluginsTelemetrySchema, ossPackagesTelemetrySchema),
xpackPluginsTelemetrySchema
);
const plugins = [
ossPluginsTelemetrySchema,
ossPackagesTelemetrySchema,
ossPlatformTelemetrySchema,
xpackPluginsTelemetrySchema,
xpackPlatformTelemetrySchema,
xpackObservabilityTelemetrySchema,
xpackSearchTelemetrySchema,
xpackSecurityTelemetrySchema,
].reduce((acc, schema) => deepmerge(acc, schema));
try {
assertTelemetryPayload({ root, plugins }, stats);
} catch (err) {
err.message = `The telemetry schemas in 'x-pack/plugins/telemetry_collection_xpack/schema/' are out-of-date, please update it as required: ${err.message}`;
err.message = `The telemetry schemas in are out-of-date. Please define the schema of your collector and run "node scripts/telemetry_check --fix" to update them: ${err.message}`;
throw err;
}
});

19
x-pack/test_serverless/api_integration/test_suites/common/telemetry/snapshot_telemetry.ts
View file

@ -11,7 +11,12 @@ import ossRootTelemetrySchema from '@kbn/telemetry-plugin/schema/oss_root.json';
import xpackRootTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_root.json';
import ossPluginsTelemetrySchema from '@kbn/telemetry-plugin/schema/oss_plugins.json';
import ossPackagesTelemetrySchema from '@kbn/telemetry-plugin/schema/kbn_packages.json';
import ossPlatformTelemetrySchema from '@kbn/telemetry-plugin/schema/oss_platform.json';
import xpackPluginsTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_plugins.json';
import xpackPlatformTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_platform.json';
import xpackObservabilityTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_observability.json';
import xpackSearchTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_search.json';
import xpackSecurityTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_security.json';
import { assertTelemetryPayload } from '@kbn/telemetry-tools';
import type { UsageStatsPayloadTestFriendly } from '@kbn/test-suites-xpack/api_integration/services/usage_api';
import type { RoleCredentials } from '../../../../shared/services';
@ -42,10 +47,16 @@ export default function ({ getService }: FtrProviderContext) {
it('should pass the schema validation (ensures BWC with Classic offering)', () => {
const root = deepmerge(ossRootTelemetrySchema, xpackRootTelemetrySchema);
const plugins = deepmerge(
deepmerge(ossPluginsTelemetrySchema, ossPackagesTelemetrySchema),
xpackPluginsTelemetrySchema
);
const plugins = [
ossPluginsTelemetrySchema,
ossPackagesTelemetrySchema,
ossPlatformTelemetrySchema,
xpackPluginsTelemetrySchema,
xpackPlatformTelemetrySchema,
xpackObservabilityTelemetrySchema,
xpackSearchTelemetrySchema,
xpackSecurityTelemetrySchema,
].reduce((acc, schema) => deepmerge(acc, schema));
try {
assertTelemetryPayload({ root, plugins }, stats);