[Usage Collection] [schema] apm (#79000)

packages/kbn-telemetry-tools/src/tools/serializer.test.ts

@@ -138,4 +138,22 @@ describe('getDescriptor', () => {
       },
     });
   });
+
+  it('serializes RecordWithKnownProps', () => {
+    const usageInterface = usageInterfaces.get('RecordWithKnownProps')!;
+    const descriptor = getDescriptor(usageInterface, tsProgram);
+    expect(descriptor).toEqual({
+      prop1: { kind: ts.SyntaxKind.NumberKeyword, type: 'NumberKeyword' },
+      prop2: { kind: ts.SyntaxKind.NumberKeyword, type: 'NumberKeyword' },
+    });
+  });
+
+  it('serializes IndexedAccessType', () => {
+    const usageInterface = usageInterfaces.get('IndexedAccessType')!;
+    const descriptor = getDescriptor(usageInterface, tsProgram);
+    expect(descriptor).toEqual({
+      prop1: { kind: ts.SyntaxKind.StringKeyword, type: 'StringKeyword' },
+      prop2: { kind: ts.SyntaxKind.StringKeyword, type: 'StringKeyword' },
+    });
+  });
 });

packages/kbn-telemetry-tools/src/tools/serializer.ts

@@ -18,7 +18,7 @@
  */
 
 import * as ts from 'typescript';
-import { uniqBy } from 'lodash';
+import { uniqBy, pick } from 'lodash';
 import {
   getResolvedModuleSourceFile,
   getIdentifierDeclarationFromSource,
@@ -95,7 +95,16 @@ export function getConstraints(node: ts.Node, program: ts.Program): any {
     return node.literal.text;
   }
 
-  throw Error(`Unsupported constraint`);
+  if (ts.isImportSpecifier(node)) {
+    const source = node.getSourceFile();
+    const importedModuleName = getModuleSpecifier(node);
+
+    const declarationSource = getResolvedModuleSourceFile(source, program, importedModuleName);
+    const declarationNode = getIdentifierDeclarationFromSource(node.name, declarationSource);
+    return getConstraints(declarationNode, program);
+  }
+
+  throw Error(`Unsupported constraint of kind ${node.kind} [${ts.SyntaxKind[node.kind]}]`);
 }
 
 export function getDescriptor(node: ts.Node, program: ts.Program): Descriptor | DescriptorValue {
@@ -157,9 +166,25 @@ export function getDescriptor(node: ts.Node, program: ts.Program): Descriptor |
       return { kind: TelemetryKinds.Date, type: 'Date' };
     }
     // Support `Record<string, SOMETHING>`
-    if (symbolName === 'Record' && node.typeArguments![0].kind === ts.SyntaxKind.StringKeyword) {
-      return { '@@INDEX@@': getDescriptor(node.typeArguments![1], program) };
+    if (symbolName === 'Record') {
+      const descriptor = getDescriptor(node.typeArguments![1], program);
+      if (node.typeArguments![0].kind === ts.SyntaxKind.StringKeyword) {
+        return { '@@INDEX@@': descriptor };
+      }
+      const constraints = getConstraints(node.typeArguments![0], program);
+      const constraintsArray = Array.isArray(constraints) ? constraints : [constraints];
+      if (typeof constraintsArray[0] === 'string') {
+        return constraintsArray.reduce((acc, c) => ({ ...acc, [c]: descriptor }), {});
+      }
     }
+
+    // Support `Pick<SOMETHING, 'prop1' | 'prop2'>`
+    if (symbolName === 'Pick') {
+      const parentDescriptor = getDescriptor(node.typeArguments![0], program);
+      const pickPropNames = getConstraints(node.typeArguments![1], program);
+      return pick(parentDescriptor, pickPropNames);
+    }
+
     const declaration = (symbol?.getDeclarations() || [])[0];
     if (declaration) {
       return getDescriptor(declaration, program);

packages/kbn-telemetry-tools/src/tools/utils.ts

@@ -78,14 +78,14 @@ export function getIdentifierDeclarationFromSource(node: ts.Node, source: ts.Sou
   const identifierName = node.getText();
   const identifierDefinition: ts.Node = (source as any).locals.get(identifierName);
   if (!identifierDefinition) {
-    throw new Error(`Unable to fine identifier in source ${identifierName}`);
+    throw new Error(`Unable to find identifier in source ${identifierName}`);
   }
   const declarations = (identifierDefinition as any).declarations as ts.Node[];
 
   const latestDeclaration: ts.Node | false | undefined =
     Array.isArray(declarations) && declarations[declarations.length - 1];
   if (!latestDeclaration) {
-    throw new Error(`Unable to fine declaration for identifier ${identifierName}`);
+    throw new Error(`Unable to find declaration for identifier ${identifierName}`);
   }
 
   return latestDeclaration;

src/fixtures/telemetry_collectors/constants.ts

@@ -66,3 +66,7 @@ export interface MappedTypes {
     [key in 'prop3']: number;
   };
 }
+
+export type RecordWithKnownProps = Record<MappedTypeProps, number>;
+
+export type IndexedAccessType = Pick<WithUnion, 'prop1' | 'prop2'>;

x-pack/.telemetryrc.json

@@ -1,7 +1,5 @@
 {
   "output": "plugins/telemetry_collection_xpack/schema/xpack_plugins.json",
   "root": "plugins/",
-  "exclude": [
-    "plugins/apm/server/lib/apm_telemetry/index.ts"
-  ]
+  "exclude": []
 }

x-pack/plugins/apm/common/apm_telemetry.ts

@@ -4,260 +4,39 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 import { produce } from 'immer';
-import { AGENT_NAMES } from './agent_name';
+// eslint-disable-next-line @kbn/eslint/no-restricted-paths
+import { apmSchema } from '../server/lib/apm_telemetry/schema';
+
+function schemaToMapping(schemaLeaf: any): any {
+  // convert "array" definition to mapping
+  if (schemaLeaf.type === 'array') {
+    return schemaToMapping(schemaLeaf.items);
+  }
+
+  if (typeof schemaLeaf.type === 'string') {
+    return schemaLeaf;
+  }
+
+  return Object.entries<any>(schemaLeaf).reduce((acc, [key, value]) => {
+    const propMapping = schemaToMapping(value);
+
+    return {
+      ...acc,
+      [key]:
+        typeof propMapping.type === 'string'
+          ? propMapping
+          : { properties: propMapping },
+    };
+  }, {});
+}
 
 /**
- * Generate an object containing the mapping used for APM telemetry. Can be used
- * with the `upload-telemetry-data` script or to update the mapping in the
- * telemetry repository.
- *
- * This function breaks things up to make the mapping easier to understand.
+ * Generate an object containing the mapping used for APM telemetry based on the schema specified
+ * in the usage collector. Can be used with the `upload-telemetry-data` script or to update the
+ * mapping in the telemetry repository.
  */
 export function getApmTelemetryMapping() {
-  const keyword = {
-    type: 'keyword',
-    ignore_above: 1024,
-  };
-
-  const float = {
-    type: 'float',
-  };
-
-  const long = {
-    type: 'long',
-  };
-
-  const allProperties = {
-    properties: {
-      all: long,
-    },
-  };
-
-  const oneDayProperties = {
-    properties: {
-      '1d': long,
-    },
-  };
-
-  const oneDayAllProperties = {
-    properties: {
-      '1d': long,
-      all: long,
-    },
-  };
-
-  const msProperties = {
-    properties: {
-      ms: long,
-    },
-  };
-
-  const tookProperties = {
-    properties: {
-      took: msProperties,
-    },
-  };
-
-  const compositeNameVersionProperties = {
-    properties: {
-      composite: keyword,
-      name: keyword,
-      version: keyword,
-    },
-  };
-
-  const agentProperties = {
-    properties: { version: keyword },
-  };
-
-  const serviceProperties = {
-    properties: {
-      framework: compositeNameVersionProperties,
-      language: compositeNameVersionProperties,
-      runtime: compositeNameVersionProperties,
-    },
-  };
-
-  const aggregatedTransactionsProperties = {
-    properties: {
-      expected_metric_document_count: long,
-      transaction_count: long,
-      ratio: float,
-    },
-  };
-
-  return {
-    properties: {
-      agents: {
-        properties: AGENT_NAMES.reduce<Record<string, any>>(
-          (previousValue, currentValue) => {
-            previousValue[currentValue] = {
-              properties: {
-                agent: agentProperties,
-                service: serviceProperties,
-              },
-            };
-
-            return previousValue;
-          },
-          {}
-        ),
-      },
-      aggregated_transactions: {
-        properties: {
-          current_implementation: aggregatedTransactionsProperties,
-          no_observer_name: aggregatedTransactionsProperties,
-          with_country: aggregatedTransactionsProperties,
-        },
-      },
-      environments: {
-        properties: {
-          services_without_environment: long,
-          services_with_multiple_environments: long,
-          top_enviroments: keyword,
-        },
-      },
-      cloud: {
-        properties: {
-          availability_zone: keyword,
-          provider: keyword,
-          region: keyword,
-        },
-      },
-      counts: {
-        properties: {
-          agent_configuration: allProperties,
-          error: oneDayAllProperties,
-          max_error_groups_per_service: oneDayProperties,
-          max_transaction_groups_per_service: oneDayProperties,
-          metric: oneDayAllProperties,
-          onboarding: oneDayAllProperties,
-          services: oneDayProperties,
-          sourcemap: oneDayAllProperties,
-          span: oneDayAllProperties,
-          traces: oneDayProperties,
-          transaction: oneDayAllProperties,
-        },
-      },
-      cardinality: {
-        properties: {
-          client: {
-            properties: {
-              geo: {
-                properties: {
-                  country_iso_code: { properties: { rum: oneDayProperties } },
-                },
-              },
-            },
-          },
-          user_agent: {
-            properties: {
-              original: {
-                properties: {
-                  all_agents: oneDayProperties,
-                  rum: oneDayProperties,
-                },
-              },
-            },
-          },
-          transaction: {
-            properties: {
-              name: {
-                properties: {
-                  all_agents: oneDayProperties,
-                  rum: oneDayProperties,
-                },
-              },
-            },
-          },
-        },
-      },
-      has_any_services: {
-        type: 'boolean',
-      },
-      indices: {
-        properties: {
-          all: {
-            properties: {
-              total: {
-                properties: {
-                  docs: {
-                    properties: {
-                      count: long,
-                    },
-                  },
-                  store: {
-                    properties: {
-                      size_in_bytes: long,
-                    },
-                  },
-                },
-              },
-            },
-          },
-          shards: {
-            properties: {
-              total: long,
-            },
-          },
-        },
-      },
-      integrations: {
-        properties: {
-          ml: {
-            properties: {
-              all_jobs_count: long,
-            },
-          },
-        },
-      },
-      retainment: {
-        properties: {
-          error: msProperties,
-          metric: msProperties,
-          onboarding: msProperties,
-          span: msProperties,
-          transaction: msProperties,
-        },
-      },
-      services_per_agent: {
-        properties: AGENT_NAMES.reduce<Record<string, any>>(
-          (previousValue, currentValue) => {
-            previousValue[currentValue] = { ...long, null_value: 0 };
-            return previousValue;
-          },
-          {}
-        ),
-      },
-      tasks: {
-        properties: {
-          aggregated_transactions: tookProperties,
-          agent_configuration: tookProperties,
-          agents: tookProperties,
-          cardinality: tookProperties,
-          cloud: tookProperties,
-          environments: tookProperties,
-          groupings: tookProperties,
-          indices_stats: tookProperties,
-          integrations: tookProperties,
-          processor_events: tookProperties,
-          services: tookProperties,
-          versions: tookProperties,
-        },
-      },
-      version: {
-        properties: {
-          apm_server: {
-            properties: {
-              major: long,
-              minor: long,
-              patch: long,
-            },
-          },
-        },
-      },
-    },
-  };
+  return { properties: schemaToMapping(apmSchema) };
 }
 
 /**

x-pack/plugins/apm/server/lib/apm_telemetry/index.ts

@@ -6,7 +6,6 @@
 import { Observable } from 'rxjs';
 import { take } from 'rxjs/operators';
 import { UsageCollectionSetup } from 'src/plugins/usage_collection/server';
-import { DeepRequired } from 'utility-types';
 import {
   CoreSetup,
   Logger,
@@ -21,14 +20,14 @@ import {
   APM_TELEMETRY_SAVED_OBJECT_ID,
   APM_TELEMETRY_SAVED_OBJECT_TYPE,
 } from '../../../common/apm_saved_object_constants';
-import { getApmTelemetryMapping } from '../../../common/apm_telemetry';
 import { getInternalSavedObjectsClient } from '../helpers/get_internal_saved_objects_client';
 import { getApmIndices } from '../settings/apm_indices/get_apm_indices';
 import {
   collectDataTelemetry,
   CollectTelemetryParams,
 } from './collect_data_telemetry';
-import { APMDataTelemetry } from './types';
+import { APMUsage } from './types';
+import { apmSchema } from './schema';
 
 const APM_TELEMETRY_TASK_NAME = 'apm-telemetry-task';
 
@@ -107,9 +106,9 @@ export async function createApmTelemetry({
     );
   };
 
-  const collector = usageCollector.makeUsageCollector({
+  const collector = usageCollector.makeUsageCollector<APMUsage | {}>({
     type: 'apm',
-    schema: getApmTelemetryMapping(),
+    schema: apmSchema,
     fetch: async () => {
       try {
         const { kibanaVersion: storedKibanaVersion, ...data } = (
@@ -117,9 +116,7 @@ export async function createApmTelemetry({
             APM_TELEMETRY_SAVED_OBJECT_TYPE,
             APM_TELEMETRY_SAVED_OBJECT_ID
           )
-        ).attributes as { kibanaVersion: string } & DeepRequired<
-          APMDataTelemetry
-        >;
+        ).attributes as { kibanaVersion: string } & APMUsage;
 
         return data;
       } catch (err) {

x-pack/plugins/apm/server/lib/apm_telemetry/schema.ts

@@ -0,0 +1,206 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { MakeSchemaFrom } from 'src/plugins/usage_collection/server';
+import {
+  AggregatedTransactionsCounts,
+  APMUsage,
+  TimeframeMap,
+  TimeframeMap1d,
+  TimeframeMapAll,
+} from './types';
+import { AgentName } from '../../../typings/es_schemas/ui/fields/agent';
+
+const long: { type: 'long' } = { type: 'long' };
+
+const aggregatedTransactionCountSchema: MakeSchemaFrom<AggregatedTransactionsCounts> = {
+  expected_metric_document_count: long,
+  transaction_count: long,
+};
+
+const timeframeMap1dSchema: MakeSchemaFrom<TimeframeMap1d> = {
+  '1d': long,
+};
+
+const timeframeMapAllSchema: MakeSchemaFrom<TimeframeMapAll> = {
+  all: long,
+};
+
+const timeframeMapSchema: MakeSchemaFrom<TimeframeMap> = {
+  ...timeframeMap1dSchema,
+  ...timeframeMapAllSchema,
+};
+
+const agentSchema: MakeSchemaFrom<APMUsage>['agents'][AgentName] = {
+  agent: {
+    version: { type: 'array', items: { type: 'keyword' } },
+  },
+  service: {
+    framework: {
+      name: { type: 'array', items: { type: 'keyword' } },
+      version: { type: 'array', items: { type: 'keyword' } },
+      composite: { type: 'array', items: { type: 'keyword' } },
+    },
+    language: {
+      name: { type: 'array', items: { type: 'keyword' } },
+      version: { type: 'array', items: { type: 'keyword' } },
+      composite: { type: 'array', items: { type: 'keyword' } },
+    },
+    runtime: {
+      name: { type: 'array', items: { type: 'keyword' } },
+      version: { type: 'array', items: { type: 'keyword' } },
+      composite: { type: 'array', items: { type: 'keyword' } },
+    },
+  },
+};
+
+const apmPerAgentSchema: Pick<
+  MakeSchemaFrom<APMUsage>,
+  'services_per_agent' | 'agents'
+> = {
+  // services_per_agent: AGENT_NAMES.reduce(
+  //   (acc, name) => ({ ...acc, [name]: long }),
+  //   {} as Record<AgentName, typeof long>
+  // ),
+  // agents: AGENT_NAMES.reduce(
+  //   (acc, name) => ({ ...acc, [name]: agentSchema }),
+  //   {} as Record<AgentName, typeof agentSchema>
+  // ),
+  // TODO: Find a way for `@kbn/telemetry-tools` to understand and evaluate expressions.
+  //  In the meanwhile, we'll have to maintain these lists up to date (TS will remind us to update)
+  services_per_agent: {
+    dotnet: long,
+    go: long,
+    java: long,
+    'js-base': long,
+    nodejs: long,
+    python: long,
+    ruby: long,
+    'rum-js': long,
+    otlp: long,
+    'opentelemetry/cpp': long,
+    'opentelemetry/dotnet': long,
+    'opentelemetry/erlang': long,
+    'opentelemetry/go': long,
+    'opentelemetry/java': long,
+    'opentelemetry/nodejs': long,
+    'opentelemetry/php': long,
+    'opentelemetry/python': long,
+    'opentelemetry/ruby': long,
+    'opentelemetry/webjs': long,
+  },
+  agents: {
+    dotnet: agentSchema,
+    go: agentSchema,
+    java: agentSchema,
+    'js-base': agentSchema,
+    nodejs: agentSchema,
+    python: agentSchema,
+    ruby: agentSchema,
+    'rum-js': agentSchema,
+    otlp: agentSchema,
+    'opentelemetry/cpp': agentSchema,
+    'opentelemetry/dotnet': agentSchema,
+    'opentelemetry/erlang': agentSchema,
+    'opentelemetry/go': agentSchema,
+    'opentelemetry/java': agentSchema,
+    'opentelemetry/nodejs': agentSchema,
+    'opentelemetry/php': agentSchema,
+    'opentelemetry/python': agentSchema,
+    'opentelemetry/ruby': agentSchema,
+    'opentelemetry/webjs': agentSchema,
+  },
+};
+
+export const apmSchema: MakeSchemaFrom<APMUsage> = {
+  ...apmPerAgentSchema,
+  has_any_services: { type: 'boolean' },
+  version: {
+    apm_server: {
+      major: long,
+      minor: long,
+      patch: long,
+    },
+  },
+  environments: {
+    services_without_environments: long,
+    services_with_multiple_environments: long,
+    top_environments: { type: 'array', items: { type: 'keyword' } },
+  },
+  aggregated_transactions: {
+    current_implementation: aggregatedTransactionCountSchema,
+    no_observer_name: aggregatedTransactionCountSchema,
+    no_rum: aggregatedTransactionCountSchema,
+    no_rum_no_observer_name: aggregatedTransactionCountSchema,
+    only_rum: aggregatedTransactionCountSchema,
+    only_rum_no_observer_name: aggregatedTransactionCountSchema,
+  },
+  cloud: {
+    availability_zone: { type: 'array', items: { type: 'keyword' } },
+    provider: { type: 'array', items: { type: 'keyword' } },
+    region: { type: 'array', items: { type: 'keyword' } },
+  },
+  counts: {
+    transaction: timeframeMapSchema,
+    span: timeframeMapSchema,
+    error: timeframeMapSchema,
+    metric: timeframeMapSchema,
+    sourcemap: timeframeMapSchema,
+    onboarding: timeframeMapSchema,
+    agent_configuration: timeframeMapAllSchema,
+    max_transaction_groups_per_service: timeframeMapSchema,
+    max_error_groups_per_service: timeframeMapSchema,
+    traces: timeframeMapSchema,
+    services: timeframeMapSchema,
+  },
+  cardinality: {
+    client: { geo: { country_iso_code: { rum: timeframeMap1dSchema } } },
+    user_agent: {
+      original: {
+        all_agents: timeframeMap1dSchema,
+        rum: timeframeMap1dSchema,
+      },
+    },
+    transaction: {
+      name: {
+        all_agents: timeframeMap1dSchema,
+        rum: timeframeMap1dSchema,
+      },
+    },
+  },
+  retainment: {
+    span: { ms: long },
+    transaction: { ms: long },
+    error: { ms: long },
+    metric: { ms: long },
+    sourcemap: { ms: long },
+    onboarding: { ms: long },
+  },
+  integrations: { ml: { all_jobs_count: long } },
+
+  indices: {
+    shards: { total: long },
+    all: {
+      total: {
+        docs: { count: long },
+        store: { size_in_bytes: long },
+      },
+    },
+  },
+  tasks: {
+    aggregated_transactions: { took: { ms: long } },
+    cloud: { took: { ms: long } },
+    processor_events: { took: { ms: long } },
+    agent_configuration: { took: { ms: long } },
+    services: { took: { ms: long } },
+    versions: { took: { ms: long } },
+    groupings: { took: { ms: long } },
+    integrations: { took: { ms: long } },
+    agents: { took: { ms: long } },
+    indices_stats: { took: { ms: long } },
+    cardinality: { took: { ms: long } },
+  },
+};

x-pack/plugins/apm/server/lib/apm_telemetry/types.ts

@@ -20,7 +20,7 @@ export interface AggregatedTransactionsCounts {
   transaction_count: number;
 }
 
-export type APMDataTelemetry = DeepPartial<{
+export interface APMUsage {
   has_any_services: boolean;
   services_per_agent: Record<AgentName, number>;
   version: {
@@ -139,6 +139,8 @@ export type APMDataTelemetry = DeepPartial<{
     | 'cardinality',
     { took: { ms: number } }
   >;
-}>;
+}
+
+export type APMDataTelemetry = DeepPartial<APMUsage>;
 
 export type APMTelemetry = APMDataTelemetry;