[Serverless Telemetry] Add serverless label to inform of the project type (#159549)

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

config/serverless.es.yml

@@ -13,3 +13,6 @@ uiSettings.overrides.defaultRoute: /app/elasticsearch
 
 ## Set the dev project switcher current type
 xpack.serverless.plugin.developer.projectSwitcher.currentType: 'search'
+
+# Specify in telemetry the project type
+telemetry.labels.serverless: search

config/serverless.oblt.yml

@@ -27,3 +27,6 @@ xpack.apm.serverlessOnboarding: true
 xpack.fleet.packages:
   - name: apm
     version: latest
+
+# Specify in telemetry the project type
+telemetry.labels.serverless: observability

config/serverless.security.yml

@@ -19,3 +19,6 @@ uiSettings.overrides.defaultRoute: /app/security/get_started
 
 ## Set the dev project switcher current type
 xpack.serverless.plugin.developer.projectSwitcher.currentType: 'security'
+
+# Specify in telemetry the project type
+telemetry.labels.serverless: security

packages/kbn-telemetry-tools/index.ts

@@ -8,3 +8,4 @@
 
 export { runTelemetryCheck } from './src/cli/run_telemetry_check';
 export { runTelemetryExtract } from './src/cli/run_telemetry_extract';
+export { assertTelemetryPayload } from './src/schema_ftr_validations';

packages/kbn-telemetry-tools/src/schema_ftr_validations/index.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export { assertTelemetryPayload } from './schema_to_config_schema';

packages/kbn-telemetry-tools/src/schema_ftr_validations/schema_to_config_schema.test.ts

@@ -6,10 +6,6 @@
  * Side Public License, v 1.
  */
 
-/*
- * It's a JS file because we cannot use Jest types in here because of a clash in the `expect` types
- */
-
 import { assertTelemetryPayload } from './schema_to_config_schema';
 
 describe(`assertTelemetryPayload`, () => {
@@ -158,10 +154,8 @@ describe(`assertTelemetryPayload`, () => {
         {
           root: {
             properties: {
-              im_only_passing_through_data: {
-                type: 'pass_through',
-                properties: {},
-              },
+              // @ts-expect-error: TS doesn't allow pass_through with properties, but it may occur during the tests in runtime, so we want to validate this test case.
+              im_only_passing_through_data: { type: 'pass_through', properties: {} },
             },
           },
           plugins: { properties: {} },

packages/kbn-telemetry-tools/tsconfig.json

@@ -18,6 +18,8 @@
     "@kbn/repo-info",
     "@kbn/import-resolver",
     "@kbn/usage-collection-plugin",
+    "@kbn/config-schema",
+    "@kbn/safer-lodash-set",
   ],
   "exclude": [
     "target/**/*",

src/plugins/telemetry/schema/oss_plugins.json

@@ -9628,6 +9628,12 @@
         },
         "last_reported": {
           "type": "long"
+        },
+        "labels": {
+          "type": "pass_through",
+          "_meta": {
+            "description": "Custom labels added to the telemetry.labels config in the kibana.yml"
+          }
         }
       }
     },

src/plugins/telemetry/server/collectors/telemetry_plugin/telemetry_plugin_collector.ts

@@ -10,13 +10,16 @@ import { Observable, firstValueFrom } from 'rxjs';
 import { ISavedObjectsRepository } from '@kbn/core/server';
 import { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server';
 import { getTelemetrySavedObject, TelemetrySavedObject } from '../../saved_objects';
-import { TelemetryConfigType } from '../../config';
+import type { TelemetryConfigType } from '../../config';
 import { getTelemetryOptIn, getTelemetrySendUsageFrom } from '../../telemetry_config';
 
 export interface TelemetryUsageStats {
   opt_in_status?: boolean | null;
   usage_fetcher?: 'browser' | 'server';
   last_reported?: number;
+  // Not using TelemetryConfigLabels because the @kbn/telemetry-tools goes crazy with the types in @kbn/config-schema
+  // For telemetry purposes, we are OK with it being unknown. It's already validated in '../../config/telemetry_labels.ts'
+  labels: Record<string, unknown>;
 }
 
 export interface TelemetryPluginUsageCollectorOptions {
@@ -31,7 +34,12 @@ export function createCollectorFetch({
   getSavedObjectsClient,
 }: TelemetryPluginUsageCollectorOptions) {
   return async function fetchUsageStats(): Promise<TelemetryUsageStats> {
-    const { sendUsageFrom, allowChangingOptInStatus, optIn = null } = await firstValueFrom(config$);
+    const {
+      sendUsageFrom,
+      allowChangingOptInStatus,
+      optIn = null,
+      labels,
+    } = await firstValueFrom(config$);
     const configTelemetrySendUsageFrom = sendUsageFrom;
     const configTelemetryOptIn = optIn;
 
@@ -56,6 +64,7 @@ export function createCollectorFetch({
         telemetrySavedObject,
         configTelemetrySendUsageFrom,
       }),
+      labels,
     };
   };
 }
@@ -72,6 +81,12 @@ export function registerTelemetryPluginUsageCollector(
       opt_in_status: { type: 'boolean' },
       usage_fetcher: { type: 'keyword' },
       last_reported: { type: 'long' },
+      labels: {
+        type: 'pass_through',
+        _meta: {
+          description: 'Custom labels added to the telemetry.labels config in the kibana.yml',
+        },
+      },
     },
   });
 

src/plugins/telemetry/server/config/telemetry_labels.ts

@@ -27,6 +27,13 @@ export const labelsSchema = schema.object(
     testBuildId: schema.maybe(schema.string()),
     testJobId: schema.maybe(schema.string()),
     ciBuildName: schema.maybe(schema.string()),
+    /**
+     * The serverless project type.
+     * Flagging it as maybe because these settings should never affect how Kibana runs.
+     */
+    serverless: schema.maybe(
+      schema.conditional(schema.contextRef('serverless'), true, schema.string(), schema.never())
+    ),
   },
   { defaultValue: {} }
 );

src/plugins/telemetry/server/routes/telemetry_usage_stats.test.ts

@@ -99,9 +99,28 @@ describe('registerTelemetryUsageStatsRoutes', () => {
       });
     });
 
+    it('calls getStats when unencrypted is set to true, there is security, but it is not enabled on ES', async () => {
+      const securityStartMock = securityMock.createStart();
+      getSecurity.mockImplementationOnce(() => {
+        securityStartMock.authz.mode.useRbacForRequest.mockReturnValue(false);
+        return securityStartMock;
+      });
+      registerTelemetryUsageStatsRoutes(mockRouter, telemetryCollectionManager, true, getSecurity);
+      await runRequest(mockRouter, {
+        refreshCache: false,
+        unencrypted: true,
+      });
+      expect(telemetryCollectionManager.getStats).toBeCalledWith({
+        unencrypted: true,
+        refreshCache: true,
+      });
+      expect(securityStartMock.authz.checkPrivilegesWithRequest).not.toHaveBeenCalled();
+    });
+
     it('returns 403 when the user does not have enough permissions to request unencrypted telemetry', async () => {
       const getSecurityMock = jest.fn().mockImplementation(() => {
         const securityStartMock = securityMock.createStart();
+        securityStartMock.authz.mode.useRbacForRequest.mockReturnValue(true);
         securityStartMock.authz.checkPrivilegesWithRequest.mockReturnValue({
           globally: () => ({ hasAllRequested: false }),
         });

src/plugins/telemetry/server/routes/telemetry_usage_stats.ts

@@ -46,7 +46,8 @@ export function registerTelemetryUsageStatsRoutes(
       }
 
       const security = getSecurity();
-      if (security && unencrypted) {
+      // We need to check useRbacForRequest to figure out if ES has security enabled before making the privileges check
+      if (security && unencrypted && security.authz.mode.useRbacForRequest(req)) {
         // Normally we would use `options: { tags: ['access:decryptedTelemetry'] }` in the route definition to check authorization for an
         // API action, however, we want to check this conditionally based on the `unencrypted` parameter. In this case we need to use the
         // security API directly to check privileges for this action. Note that the 'decryptedTelemetry' API privilege string is only

test/api_integration/apis/telemetry/utils/index.ts

@@ -7,4 +7,3 @@
  */
 
 export { flatKeys } from './flat_keys';
-export { assertTelemetryPayload } from './schema_to_config_schema';

test/plugin_functional/test_suites/core_plugins/rendering.ts

@@ -154,6 +154,7 @@ export default function ({ getService }: PluginFunctionalProviderContext) {
         'telemetry.labels.testBuildId (string)',
         'telemetry.labels.testJobId (string)',
         'telemetry.labels.ciBuildName (string)',
+        'telemetry.labels.serverless (any)',
         'telemetry.hidePrivacyStatement (boolean)',
         'telemetry.optIn (boolean)',
         'telemetry.sendUsageFrom (alternatives)',

test/tsconfig.json

@@ -62,7 +62,6 @@
     "@kbn/screenshot-mode-plugin",
     "@kbn/dev-utils",
     "@kbn/analytics-client",
-    "@kbn/safer-lodash-set",
     "@kbn/utility-types",
     "@kbn/dev-proc-runner",
     "@kbn/enterprise-search-plugin",

x-pack/test/api_integration/apis/telemetry/telemetry.ts

@@ -20,7 +20,7 @@ import type {
   UsageStatsPayload,
   CacheDetails,
 } from '@kbn/telemetry-collection-manager-plugin/server/types';
-import { assertTelemetryPayload } from '../../../../../test/api_integration/apis/telemetry/utils';
+import { assertTelemetryPayload } from '@kbn/telemetry-tools';
 import basicClusterFixture from './fixtures/basiccluster.json';
 import multiClusterFixture from './fixtures/multicluster.json';
 import type { SecurityService } from '../../../../../test/common/services/security/security';

x-pack/test/api_integration/apis/telemetry/telemetry_local.ts

@@ -11,10 +11,8 @@ import ossRootTelemetrySchema from '@kbn/telemetry-plugin/schema/oss_root.json';
 import ossPluginsTelemetrySchema from '@kbn/telemetry-plugin/schema/oss_plugins.json';
 import xpackRootTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_root.json';
 import xpackPluginsTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_plugins.json';
-import {
-  assertTelemetryPayload,
-  flatKeys,
-} from '../../../../../test/api_integration/apis/telemetry/utils';
+import { assertTelemetryPayload } from '@kbn/telemetry-tools';
+import { flatKeys } from '../../../../../test/api_integration/apis/telemetry/utils';
 import type { FtrProviderContext } from '../../ftr_provider_context';
 
 const disableCollection = {

x-pack/test/tsconfig.json

@@ -128,6 +128,7 @@
     "@kbn/core-saved-objects-common",
     "@kbn/core-http-common",
     "@kbn/slo-schema",
-    "@kbn/lens-plugin"
+    "@kbn/lens-plugin",
+    "@kbn/telemetry-tools"
   ]
 }

x-pack/test_serverless/api_integration/test_suites/observability/index.ts

@@ -10,5 +10,6 @@ import { FtrProviderContext } from '../../ftr_provider_context';
 export default function ({ loadTestFile }: FtrProviderContext) {
   describe('serverless observability API', function () {
     loadTestFile(require.resolve('./security_users'));
+    loadTestFile(require.resolve('./snapshot_telemetry'));
   });
 }

x-pack/test_serverless/api_integration/test_suites/observability/snapshot_telemetry.ts

@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from 'expect';
+import deepmerge from 'deepmerge';
+import ossRootTelemetrySchema from '@kbn/telemetry-plugin/schema/oss_root.json';
+import xpackRootTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_root.json';
+import ossPluginsTelemetrySchema from '@kbn/telemetry-plugin/schema/oss_plugins.json';
+import xpackPluginsTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_plugins.json';
+import { assertTelemetryPayload } from '@kbn/telemetry-tools';
+import { FtrProviderContext } from '../../ftr_provider_context';
+import type { UsageStatsPayloadTestFriendly } from '../../../../test/api_integration/services/usage_api';
+
+export default function ({ getService }: FtrProviderContext) {
+  const usageApi = getService('usageAPI');
+
+  describe('Snapshot telemetry', function () {
+    let stats: UsageStatsPayloadTestFriendly;
+
+    before(async () => {
+      const [unencryptedPayload] = await usageApi.getTelemetryStats({ unencrypted: true });
+      stats = unencryptedPayload.stats;
+    });
+
+    it('should pass the schema validation (ensures BWC with Classic offering)', () => {
+      const root = deepmerge(ossRootTelemetrySchema, xpackRootTelemetrySchema);
+      const plugins = deepmerge(ossPluginsTelemetrySchema, xpackPluginsTelemetrySchema);
+
+      try {
+        assertTelemetryPayload({ root, plugins }, stats);
+      } catch (err) {
+        err.message = `The telemetry schemas in are out-of-date. Please define the schema of your collector and run "node scripts/telemetry_check --fix" to update them: ${err.message}`;
+        throw err;
+      }
+    });
+
+    it('includes the serverless info in the body', async () => {
+      const [unencryptedPayload] = await usageApi.getTelemetryStats({ unencrypted: true });
+
+      expect(
+        unencryptedPayload.stats.stack_stats.kibana?.plugins?.telemetry?.labels?.serverless
+      ).toBe('observability');
+    });
+  });
+}

x-pack/test_serverless/api_integration/test_suites/search/index.ts

@@ -10,5 +10,6 @@ import { FtrProviderContext } from '../../ftr_provider_context';
 export default function ({ loadTestFile }: FtrProviderContext) {
   describe('serverless search API', function () {
     loadTestFile(require.resolve('./security_users'));
+    loadTestFile(require.resolve('./snapshot_telemetry'));
   });
 }

x-pack/test_serverless/api_integration/test_suites/search/snapshot_telemetry.ts

@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from 'expect';
+import deepmerge from 'deepmerge';
+import { assertTelemetryPayload } from '@kbn/telemetry-tools';
+import ossRootTelemetrySchema from '@kbn/telemetry-plugin/schema/oss_root.json';
+import ossPluginsTelemetrySchema from '@kbn/telemetry-plugin/schema/oss_plugins.json';
+import xpackRootTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_root.json';
+import xpackPluginsTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_plugins.json';
+import type { FtrProviderContext } from '../../ftr_provider_context';
+import type { UsageStatsPayloadTestFriendly } from '../../../../test/api_integration/services/usage_api';
+
+export default function ({ getService }: FtrProviderContext) {
+  const usageApi = getService('usageAPI');
+
+  describe('Snapshot telemetry', function () {
+    let stats: UsageStatsPayloadTestFriendly;
+
+    before(async () => {
+      const [unencryptedPayload] = await usageApi.getTelemetryStats({ unencrypted: true });
+      stats = unencryptedPayload.stats;
+    });
+
+    it('should pass the schema validation (ensures BWC with Classic offering)', () => {
+      const root = deepmerge(ossRootTelemetrySchema, xpackRootTelemetrySchema);
+      const plugins = deepmerge(ossPluginsTelemetrySchema, xpackPluginsTelemetrySchema);
+
+      try {
+        assertTelemetryPayload({ root, plugins }, stats);
+      } catch (err) {
+        err.message = `The telemetry schemas in are out-of-date. Please define the schema of your collector and run "node scripts/telemetry_check --fix" to update them: ${err.message}`;
+        throw err;
+      }
+    });
+
+    it('includes the serverless info in the body', async () => {
+      expect(stats.stack_stats.kibana?.plugins?.telemetry?.labels?.serverless).toBe('search');
+    });
+  });
+}

x-pack/test_serverless/api_integration/test_suites/security/index.ts

@@ -10,5 +10,6 @@ import { FtrProviderContext } from '../../ftr_provider_context';
 export default function ({ loadTestFile }: FtrProviderContext) {
   describe('serverless security API', function () {
     loadTestFile(require.resolve('./security_users'));
+    loadTestFile(require.resolve('./snapshot_telemetry'));
   });
 }

x-pack/test_serverless/api_integration/test_suites/security/snapshot_telemetry.ts

@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from 'expect';
+import deepmerge from 'deepmerge';
+import ossRootTelemetrySchema from '@kbn/telemetry-plugin/schema/oss_root.json';
+import xpackRootTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_root.json';
+import ossPluginsTelemetrySchema from '@kbn/telemetry-plugin/schema/oss_plugins.json';
+import xpackPluginsTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_plugins.json';
+import { assertTelemetryPayload } from '@kbn/telemetry-tools';
+import { FtrProviderContext } from '../../ftr_provider_context';
+import type { UsageStatsPayloadTestFriendly } from '../../../../test/api_integration/services/usage_api';
+
+export default function ({ getService }: FtrProviderContext) {
+  const usageApi = getService('usageAPI');
+
+  describe('Snapshot telemetry', function () {
+    let stats: UsageStatsPayloadTestFriendly;
+
+    before(async () => {
+      const [unencryptedPayload] = await usageApi.getTelemetryStats({ unencrypted: true });
+      stats = unencryptedPayload.stats;
+    });
+
+    it('should pass the schema validation (ensures BWC with Classic offering)', () => {
+      const root = deepmerge(ossRootTelemetrySchema, xpackRootTelemetrySchema);
+      const plugins = deepmerge(ossPluginsTelemetrySchema, xpackPluginsTelemetrySchema);
+
+      try {
+        assertTelemetryPayload({ root, plugins }, stats);
+      } catch (err) {
+        err.message = `The telemetry schemas in are out-of-date. Please define the schema of your collector and run "node scripts/telemetry_check --fix" to update them: ${err.message}`;
+        throw err;
+      }
+    });
+
+    it('includes the serverless info in the body', async () => {
+      expect(stats.stack_stats.kibana?.plugins?.telemetry?.labels?.serverless).toBe('security');
+    });
+  });
+}

x-pack/test_serverless/tsconfig.json

@@ -23,5 +23,8 @@
     "@kbn/repo-info",
     "@kbn/cypress-config",
     "@kbn/dev-proc-runner",
+    "@kbn/telemetry-plugin",
+    "@kbn/telemetry-collection-xpack-plugin",
+    "@kbn/telemetry-tools",
   ]
 }