Fix agent config indicator when applied through fleet integration (#131820)

* Fix agent config indicator when applied through fleet integration

* Add synthrace scenario

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

packages/elastic-apm-synthtrace/src/index.ts

@@ -9,6 +9,7 @@
 export { timerange } from './lib/timerange';
 export { apm } from './lib/apm';
 export { stackMonitoring } from './lib/stack_monitoring';
+export { observer } from './lib/agent_config';
 export { cleanWriteTargets } from './lib/utils/clean_write_targets';
 export { createLogger, LogLevel } from './lib/utils/create_logger';
 

packages/elastic-apm-synthtrace/src/lib/agent_config/agent_config.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { AgentConfigFields } from './agent_config_fields';
+import { Metricset } from '../apm/metricset';
+
+export class AgentConfig extends Metricset<AgentConfigFields> {
+  constructor() {
+    super({
+      'metricset.name': 'agent_config',
+      agent_config_applied: 1,
+    });
+  }
+
+  etag(etag: string) {
+    this.fields['labels.etag'] = etag;
+    return this;
+  }
+}

packages/elastic-apm-synthtrace/src/lib/agent_config/agent_config_fields.ts

@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { ApmFields } from '../apm/apm_fields';
+
+export type AgentConfigFields = Pick<
+  ApmFields,
+  | '@timestamp'
+  | 'processor.event'
+  | 'processor.name'
+  | 'metricset.name'
+  | 'observer'
+  | 'ecs.version'
+  | 'event.ingested'
+> &
+  Partial<{
+    'labels.etag': string;
+    agent_config_applied: number;
+    'event.agent_id_status': string;
+  }>;

packages/elastic-apm-synthtrace/src/lib/agent_config/index.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export { observer } from './observer';

packages/elastic-apm-synthtrace/src/lib/agent_config/observer.ts

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { AgentConfigFields } from './agent_config_fields';
+import { AgentConfig } from './agent_config';
+import { Entity } from '../entity';
+
+export class Observer extends Entity<AgentConfigFields> {
+  agentConfig() {
+    return new AgentConfig();
+  }
+}
+
+export function observer() {
+  return new Observer({});
+}

packages/elastic-apm-synthtrace/src/lib/apm/instance.ts

@@ -45,7 +45,7 @@ export class Instance extends Entity<ApmFields> {
   }
 
   appMetrics(metrics: ApmApplicationMetricFields) {
-    return new Metricset({
+    return new Metricset<ApmFields>({
       ...this.fields,
       'metricset.name': 'app',
       ...metrics,

packages/elastic-apm-synthtrace/src/lib/apm/metricset.ts

@@ -7,10 +7,10 @@
  */
 
 import { Serializable } from '../serializable';
-import { ApmFields } from './apm_fields';
+import { Fields } from '../entity';
 
-export class Metricset extends Serializable<ApmFields> {
-  constructor(fields: ApmFields) {
+export class Metricset<TFields extends Fields> extends Serializable<TFields> {
+  constructor(fields: TFields) {
     super({
       'processor.event': 'metric',
       'processor.name': 'metric',

packages/elastic-apm-synthtrace/src/lib/stream_processor.ts

@@ -211,7 +211,9 @@ export class StreamProcessor<TFields extends Fields = ApmFields> {
     const eventType = d.processor.event as keyof ApmElasticsearchOutputWriteTargets;
     let dataStream = writeTargets[eventType];
     if (eventType === 'metric') {
-      if (!d.service?.name) {
+      if (d.metricset?.name === 'agent_config') {
+        dataStream = 'metrics-apm.internal-default';
+      } else if (!d.service?.name) {
         dataStream = 'metrics-apm.app-default';
       } else {
         if (!d.transaction && !d.span) {

packages/elastic-apm-synthtrace/src/scripts/examples/04_agent_config.ts

@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { observer, timerange } from '../..';
+import { Scenario } from '../scenario';
+import { getLogger } from '../utils/get_common_services';
+import { RunOptions } from '../utils/parse_run_cli_flags';
+import { AgentConfigFields } from '../../lib/agent_config/agent_config_fields';
+
+const scenario: Scenario<AgentConfigFields> = async (runOptions: RunOptions) => {
+  const logger = getLogger(runOptions);
+
+  return {
+    generate: ({ from, to }) => {
+      const agentConfig = observer().agentConfig();
+
+      const range = timerange(from, to);
+      return range
+        .interval('30s')
+        .rate(1)
+        .generator((timestamp) => {
+          const events = logger.perf('generating_agent_config_events', () => {
+            return agentConfig.etag('test-etag').timestamp(timestamp);
+          });
+          return events;
+        });
+    },
+  };
+};
+
+export default scenario;

x-pack/plugins/apm/common/agent_configuration/configuration_types.d.ts

@@ -15,6 +15,6 @@ export type AgentConfigurationIntake = t.TypeOf<
 export type AgentConfiguration = {
   '@timestamp': number;
   applied_by_agent?: boolean;
-  etag?: string;
+  etag: string;
   agent_name?: string;
 } & AgentConfigurationIntake;

x-pack/plugins/apm/server/routes/settings/agent_configuration/convert_settings_to_string.ts

@@ -13,17 +13,27 @@ import { AgentConfiguration } from '../../../../common/agent_configuration/confi
 export function convertConfigSettingsToString(
   hit: SearchHit<AgentConfiguration>
 ) {
-  const config = hit._source;
+  const { settings } = hit._source;
 
-  if (config.settings?.transaction_sample_rate) {
-    config.settings.transaction_sample_rate =
-      config.settings.transaction_sample_rate.toString();
-  }
+  const convertedConfigSettings = {
+    ...settings,
+    ...(settings?.transaction_sample_rate
+      ? {
+          transaction_sample_rate: settings.transaction_sample_rate.toString(),
+        }
+      : {}),
+    ...(settings?.transaction_max_spans
+      ? {
+          transaction_max_spans: settings.transaction_max_spans.toString(),
+        }
+      : {}),
+  };
 
-  if (config.settings?.transaction_max_spans) {
-    config.settings.transaction_max_spans =
-      config.settings.transaction_max_spans.toString();
-  }
-
-  return hit;
+  return {
+    ...hit,
+    _source: {
+      ...hit._source,
+      settings: convertedConfigSettings,
+    },
+  };
 }

x-pack/plugins/apm/server/routes/settings/agent_configuration/find_exact_configuration.ts

@@ -13,6 +13,7 @@ import {
 } from '../../../../common/elasticsearch_fieldnames';
 import { Setup } from '../../../lib/helpers/setup_request';
 import { convertConfigSettingsToString } from './convert_settings_to_string';
+import { getConfigsAppliedToAgentsThroughFleet } from './get_config_applied_to_agent_through_fleet';
 
 export async function findExactConfiguration({
   service,
@@ -40,16 +41,27 @@ export async function findExactConfiguration({
     },
   };
 
-  const resp = await internalClient.search<AgentConfiguration, typeof params>(
-    'find_exact_agent_configuration',
-    params
-  );
+  const [agentConfig, configsAppliedToAgentsThroughFleet] = await Promise.all([
+    internalClient.search<AgentConfiguration, typeof params>(
+      'find_exact_agent_configuration',
+      params
+    ),
+    getConfigsAppliedToAgentsThroughFleet({ setup }),
+  ]);
 
-  const hit = resp.hits.hits[0] as SearchHit<AgentConfiguration> | undefined;
+  const hit = agentConfig.hits.hits[0] as
+    | SearchHit<AgentConfiguration>
+    | undefined;
 
   if (!hit) {
     return;
   }
 
-  return convertConfigSettingsToString(hit);
+  return {
+    id: hit._id,
+    ...convertConfigSettingsToString(hit)._source,
+    applied_by_agent:
+      hit._source.applied_by_agent ||
+      configsAppliedToAgentsThroughFleet.hasOwnProperty(hit._source.etag),
+  };
 }

x-pack/plugins/apm/server/routes/settings/agent_configuration/get_config_applied_to_agent_through_fleet.ts

@@ -0,0 +1,60 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { termQuery, rangeQuery } from '@kbn/observability-plugin/server';
+import datemath from '@kbn/datemath';
+import { METRICSET_NAME } from '../../../../common/elasticsearch_fieldnames';
+import { Setup } from '../../../lib/helpers/setup_request';
+
+export async function getConfigsAppliedToAgentsThroughFleet({
+  setup,
+}: {
+  setup: Setup;
+}) {
+  const { internalClient, indices } = setup;
+
+  const params = {
+    index: indices.metric,
+    size: 0,
+    body: {
+      query: {
+        bool: {
+          filter: [
+            ...termQuery(METRICSET_NAME, 'agent_config'),
+            ...rangeQuery(
+              datemath.parse('now-15m')!.valueOf(),
+              datemath.parse('now')!.valueOf()
+            ),
+          ],
+        },
+      },
+      aggs: {
+        config_by_etag: {
+          terms: {
+            field: 'labels.etag',
+            size: 200,
+          },
+        },
+      },
+    },
+  };
+
+  const response = await internalClient.search(
+    'get_config_applied_to_agent_through_fleet',
+    params
+  );
+
+  return (
+    response.aggregations?.config_by_etag.buckets.reduce(
+      (configsAppliedToAgentsThroughFleet, bucket) => {
+        configsAppliedToAgentsThroughFleet[bucket.key as string] = true;
+        return configsAppliedToAgentsThroughFleet;
+      },
+      {} as Record<string, true>
+    ) ?? {}
+  );
+}

x-pack/plugins/apm/server/routes/settings/agent_configuration/list_configurations.ts

@@ -8,6 +8,7 @@
 import { Setup } from '../../../lib/helpers/setup_request';
 import { AgentConfiguration } from '../../../../common/agent_configuration/configuration_types';
 import { convertConfigSettingsToString } from './convert_settings_to_string';
+import { getConfigsAppliedToAgentsThroughFleet } from './get_config_applied_to_agent_through_fleet';
 
 export async function listConfigurations({ setup }: { setup: Setup }) {
   const { internalClient, indices } = setup;
@@ -17,12 +18,22 @@ export async function listConfigurations({ setup }: { setup: Setup }) {
     size: 200,
   };
 
-  const resp = await internalClient.search<AgentConfiguration>(
-    'list_agent_configuration',
-    params
-  );
+  const [agentConfigs, configsAppliedToAgentsThroughFleet] = await Promise.all([
+    internalClient.search<AgentConfiguration>(
+      'list_agent_configuration',
+      params
+    ),
+    getConfigsAppliedToAgentsThroughFleet({ setup }),
+  ]);
 
-  return resp.hits.hits
+  return agentConfigs.hits.hits
     .map(convertConfigSettingsToString)
-    .map((hit) => hit._source);
+    .map((hit) => {
+      return {
+        ...hit._source,
+        applied_by_agent:
+          hit._source.applied_by_agent ||
+          configsAppliedToAgentsThroughFleet.hasOwnProperty(hit._source.etag),
+      };
+    });
 }

x-pack/plugins/apm/server/routes/settings/agent_configuration/route.ts

@@ -38,7 +38,9 @@ const agentConfigurationRoute = createApmServerRoute({
     >;
   }> => {
     const setup = await setupRequest(resources);
+
     const configurations = await listConfigurations({ setup });
+
     return { configurations };
   },
 });
@@ -71,7 +73,7 @@ const getSingleAgentConfigurationRoute = createApmServerRoute({
       throw Boom.notFound();
     }
 
-    return config._source;
+    return config;
   },
 });
 
@@ -102,11 +104,11 @@ const deleteAgentConfigurationRoute = createApmServerRoute({
     }
 
     logger.info(
-      `Deleting config ${service.name}/${service.environment} (${config._id})`
+      `Deleting config ${service.name}/${service.environment} (${config.id})`
     );
 
     const deleteConfigurationResult = await deleteConfiguration({
-      configurationId: config._id,
+      configurationId: config.id,
       setup,
     });
 
@@ -162,7 +164,7 @@ const createOrUpdateAgentConfigurationRoute = createApmServerRoute({
     );
 
     await createOrUpdateConfiguration({
-      configurationId: config?._id,
+      configurationId: config?.id,
       configurationIntake: body,
       setup,
     });

x-pack/test/apm_api_integration/tests/settings/agent_configuration/add_agent_config_metrics.ts

@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { timerange, observer } from '@elastic/apm-synthtrace';
+import type { ApmSynthtraceEsClient } from '@elastic/apm-synthtrace';
+
+export async function addAgentConfigMetrics({
+  synthtraceEsClient,
+  start,
+  end,
+  etag,
+}: {
+  synthtraceEsClient: ApmSynthtraceEsClient;
+  start: number;
+  end: number;
+  etag?: string;
+}) {
+  const agentConfig = observer().agentConfig();
+
+  const agentConfigEvents = [
+    timerange(start, end)
+      .interval('1m')
+      .rate(1)
+      .generator((timestamp) => agentConfig.etag(etag ?? 'test-etag').timestamp(timestamp)),
+  ];
+
+  await synthtraceEsClient.index(agentConfigEvents);
+}

x-pack/test/apm_api_integration/tests/settings/agent_configuration/agent_configuration.spec.ts

@@ -11,14 +11,17 @@ import expect from '@kbn/expect';
 import { omit, orderBy } from 'lodash';
 import { AgentConfigurationIntake } from '@kbn/apm-plugin/common/agent_configuration/configuration_types';
 import { AgentConfigSearchParams } from '@kbn/apm-plugin/server/routes/settings/agent_configuration/route';
-
-import { FtrProviderContext } from '../../common/ftr_provider_context';
+import { APIReturnType } from '@kbn/apm-plugin/public/services/rest/create_call_apm_api';
+import moment from 'moment';
+import { FtrProviderContext } from '../../../common/ftr_provider_context';
+import { addAgentConfigMetrics } from './add_agent_config_metrics';
 
 export default function agentConfigurationTests({ getService }: FtrProviderContext) {
   const registry = getService('registry');
   const apmApiClient = getService('apmApiClient');
 
   const log = getService('log');
+  const synthtraceEsClient = getService('synthtraceEsClient');
 
   const archiveName = 'apm_8.0.0';
 
@@ -77,6 +80,18 @@ export default function agentConfigurationTests({ getService }: FtrProviderConte
     });
   }
 
+  function findExactConfiguration(name: string, environment: string) {
+    return apmApiClient.readUser({
+      endpoint: 'GET /api/apm/settings/agent-configuration/view',
+      params: {
+        query: {
+          name,
+          environment,
+        },
+      },
+    });
+  }
+
   registry.when(
     'agent configuration when no data is loaded',
     { config: 'basic', archives: [] },
@@ -297,7 +312,7 @@ export default function agentConfigurationTests({ getService }: FtrProviderConte
           service: { name: 'myservice', environment: 'production' },
           settings: { transaction_sample_rate: '0.9' },
         };
-        let etag: string | undefined;
+        let etag: string;
 
         before(async () => {
           log.debug('creating agent configuration');
@@ -371,6 +386,74 @@ export default function agentConfigurationTests({ getService }: FtrProviderConte
     }
   );
 
+  registry.when(
+    'Agent configurations through fleet',
+    { config: 'basic', archives: ['apm_mappings_only_8.0.0'] },
+    () => {
+      const name = 'myservice';
+      const environment = 'development';
+      const testConfig = {
+        service: { name, environment },
+        settings: { transaction_sample_rate: '0.9' },
+      };
+
+      let agentConfiguration:
+        | APIReturnType<'GET /api/apm/settings/agent-configuration/view'>
+        | undefined;
+
+      before(async () => {
+        log.debug('creating agent configuration');
+        await createConfiguration(testConfig);
+        const { body } = await findExactConfiguration(name, environment);
+        agentConfiguration = body;
+      });
+
+      after(async () => {
+        await deleteConfiguration(testConfig);
+      });
+
+      it(`should have 'applied_by_agent=false' when there are no agent config metrics for this etag`, async () => {
+        expect(agentConfiguration?.applied_by_agent).to.be(false);
+      });
+
+      describe('when there are agent config metrics for this etag', () => {
+        before(async () => {
+          const start = new Date().getTime();
+          const end = moment(start).add(15, 'minutes').valueOf();
+
+          await addAgentConfigMetrics({
+            synthtraceEsClient,
+            start,
+            end,
+            etag: agentConfiguration?.etag,
+          });
+        });
+
+        after(() => synthtraceEsClient.clean());
+
+        it(`should have 'applied_by_agent=true' when getting a config from all configurations`, async () => {
+          const {
+            body: { configurations },
+          } = await getAllConfigurations();
+
+          const updatedConfig = configurations.find(
+            (x) => x.service.name === name && x.service.environment === environment
+          );
+
+          expect(updatedConfig?.applied_by_agent).to.be(true);
+        });
+
+        it(`should have 'applied_by_agent=true' when getting a single config`, async () => {
+          const {
+            body: { applied_by_agent: appliedByAgent },
+          } = await findExactConfiguration(name, environment);
+
+          expect(appliedByAgent).to.be(true);
+        });
+      });
+    }
+  );
+
   registry.when(
     'agent configuration when data is loaded',
     { config: 'basic', archives: [archiveName] },