[8.18] [APM] Audit log level apm (#215543) (#215883)

# Backport

This will backport the following commits from `main` to `8.18`:
- [[APM] Audit log level apm
(#215543)](https://github.com/elastic/kibana/pull/215543)

<!--- Backport version: 9.6.6 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sorenlouv/backport)

<!--BACKPORT
[{"author":{"name":"Katerina","email":"aikaterini.patticha@elastic.co"},"sourceCommit":{"committedDate":"2025-03-25T14:05:28Z","message":"[APM]
Audit log level apm (#215543)\n\ncloses
https://github.com/elastic/observability-dev/issues/4411\n\n##
Summary\n\nAccording
to\nhttps://docs.elastic.dev/kibana-dev-docs/services/logging#log-level\n\n-
audit log level and update it wherever possible \n- merge error logs
into one for better analysis \n- log only server errors for HTTP
requests \n\n###
Checklist","sha":"d975d1960b0a04aaea4542c9c1d4c28238aa6c0d","branchLabelMapping":{"^v9.1.0$":"main","^v8.19.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","v9.0.0","backport:prev-minor","Team:obs-ux-infra_services","Team:obs-ux-management","v8.18.0","v9.1.0"],"title":"[APM]
Audit log level
apm","number":215543,"url":"https://github.com/elastic/kibana/pull/215543","mergeCommit":{"message":"[APM]
Audit log level apm (#215543)\n\ncloses
https://github.com/elastic/observability-dev/issues/4411\n\n##
Summary\n\nAccording
to\nhttps://docs.elastic.dev/kibana-dev-docs/services/logging#log-level\n\n-
audit log level and update it wherever possible \n- merge error logs
into one for better analysis \n- log only server errors for HTTP
requests \n\n###
Checklist","sha":"d975d1960b0a04aaea4542c9c1d4c28238aa6c0d"}},"sourceBranch":"main","suggestedTargetBranches":["9.0","8.18"],"targetPullRequestStates":[{"branch":"9.0","label":"v9.0.0","branchLabelMappingKey":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"8.18","label":"v8.18.0","branchLabelMappingKey":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"main","label":"v9.1.0","branchLabelMappingKey":"^v9.1.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/215543","number":215543,"mergeCommit":{"message":"[APM]
Audit log level apm (#215543)\n\ncloses
https://github.com/elastic/observability-dev/issues/4411\n\n##
Summary\n\nAccording
to\nhttps://docs.elastic.dev/kibana-dev-docs/services/logging#log-level\n\n-
audit log level and update it wherever possible \n- merge error logs
into one for better analysis \n- log only server errors for HTTP
requests \n\n###
Checklist","sha":"d975d1960b0a04aaea4542c9c1d4c28238aa6c0d"}}]}]
BACKPORT-->

Co-authored-by: Katerina <aikaterini.patticha@elastic.co>

x-pack/solutions/observability/plugins/apm/server/lib/anomaly_detection/create_anomaly_detection_jobs.ts

@@ -49,7 +49,7 @@ export async function createAnomalyDetectionJobs({
   }
 
   return withApmSpan('create_anomaly_detection_jobs', async () => {
-    logger.info(`Creating ML anomaly detection jobs for environments: [${uniqueMlJobEnvs}].`);
+    logger.debug(`Creating ML anomaly detection jobs for environments: [${uniqueMlJobEnvs}].`);
     const apmMetricIndex = indices.metric;
     const responses = [];
     const failedJobs = [];

x-pack/solutions/observability/plugins/apm/server/lib/anomaly_detection/get_anomaly_timeseries.ts

@@ -194,7 +194,7 @@ export async function getAnomalyTimeseries({
       const job = maybe(jobsById[jobId]);
 
       if (!job) {
-        logger.warn(`Could not find job for id ${jobId}`);
+        logger.debug(`Could not find job for id ${jobId}`);
         return undefined;
       }
 

x-pack/solutions/observability/plugins/apm/server/plugin.ts

@@ -249,20 +249,17 @@ export class APMPlugin
 
     // create .apm-agent-configuration index without blocking start lifecycle
     createApmAgentConfigurationIndex({ client, logger }).catch((e) => {
-      logger.error('Failed to create .apm-agent-configuration index');
-      logger.error(e);
+      logger.debug(`Failed to create .apm-agent-configuration index: ${e.message}`);
     });
 
     // create .apm-custom-link index without blocking start lifecycle
     createApmCustomLinkIndex({ client, logger }).catch((e) => {
-      logger.error('Failed to create .apm-custom-link index');
-      logger.error(e);
+      logger.debug(`Failed to create .apm-custom-link index: ${e.message}`);
     });
 
     // create .apm-source-map index without blocking start lifecycle
     createApmSourceMapIndexTemplate({ client, logger }).catch((e) => {
-      logger.error('Failed to create apm-source-map index template');
-      logger.error(e);
+      logger.debug(`Failed to create apm-source-map index template: ${e.message}`);
     });
   }
 

x-pack/solutions/observability/plugins/apm/server/routes/alerts/rule_types/anomaly/register_anomaly_rule_type.ts

@@ -256,7 +256,7 @@ export function registerAnomalyRuleType({
             const job = mlJobs.find((j) => j.jobId === latest.job_id);
 
             if (!job) {
-              logger.warn(`Could not find matching job for job id ${latest.job_id}`);
+              logger.debug(`Could not find matching job for job id ${latest.job_id}`);
               return undefined;
             }
 

x-pack/solutions/observability/plugins/apm/server/routes/apm_routes/register_apm_server_routes.ts

@@ -185,8 +185,6 @@ export function registerRoutes({
 
         return response.ok({ body });
       } catch (error) {
-        logger.error(error);
-
         if (!options.disableTelemetry && telemetryUsageCounter) {
           telemetryUsageCounter.incrementCounter({
             counterName: `${method.toUpperCase()} ${pathname}`,
@@ -213,6 +211,10 @@ export function registerRoutes({
           opts.body.attributes.data = error?.data;
         }
 
+        if (opts.statusCode >= 500) {
+          logger.error(error);
+        }
+
         // capture error with APM node agent
         apm.captureError(error);
 

x-pack/solutions/observability/plugins/apm/server/routes/assistant_functions/get_apm_service_summary/get_anomalies.ts

@@ -148,7 +148,7 @@ export async function getAnomalies({
     const job = maybe(jobsById[jobId]);
 
     if (!job) {
-      logger.warn(`Could not find job for id ${jobId}`);
+      logger.debug(`Could not find job for id ${jobId}`);
       return undefined;
     }
 

x-pack/solutions/observability/plugins/apm/server/routes/assistant_functions/get_apm_service_summary/index.ts

@@ -118,8 +118,7 @@ export async function getApmServiceSummary({
       logger,
       transactionType,
     }).catch((error) => {
-      logger.error('Failed to get anomalies');
-      logger.error(error);
+      logger.warn('Failed to get anomalies', { error });
       return {
         error,
       };

x-pack/solutions/observability/plugins/apm/server/routes/assistant_functions/get_observability_alert_details_context/index.ts

@@ -313,8 +313,7 @@ export const getAlertDetailsContextHandler = (
         try {
           return await dataFetcher();
         } catch (error) {
-          logger.error('Error while fetching observability alert details context');
-          logger.error(error);
+          logger.error('Error while fetching observability alert details context', { error });
           return;
         }
       })

x-pack/solutions/observability/plugins/apm/server/routes/data_view/create_static_data_view.test.ts

@@ -40,6 +40,7 @@ const coreMock = {
 
 const logger = {
   info: jest.fn,
+  debug: jest.fn,
 } as unknown as Logger;
 
 const apmEventClientMock = {

x-pack/solutions/observability/plugins/apm/server/routes/data_view/create_static_data_view.ts

@@ -36,7 +36,7 @@ export async function createOrUpdateStaticDataView({
 }): CreateDataViewResponse {
   const { config } = resources;
   const dataViewId = getStaticDataViewId(spaceId);
-  logger.info(`create static data view ${dataViewId}`);
+  logger.debug(`create static data view ${dataViewId}`);
 
   return withApmSpan('create_static_data_view', async () => {
     // don't auto-create APM data view if it's been disabled via the config

x-pack/solutions/observability/plugins/apm/server/routes/fleet/create_cloud_apm_package_policy.ts

@@ -62,13 +62,13 @@ export async function createCloudApmPackgePolicy({
     fleetPluginStart,
     apmIndices,
   });
-  logger.info(`Fleet migration on Cloud - apmPackagePolicy create start`);
+  logger.debug(`Fleet migration on Cloud - apmPackagePolicy create start`);
   const apmPackagePolicy = await fleetPluginStart.packagePolicyService.create(
     savedObjectsClient,
     esClient,
     mergedAPMPackagePolicy,
     { id: ELASTIC_CLOUD_APM_AGENT_POLICY_ID, force: true, bumpRevision: true }
   );
-  logger.info(`Fleet migration on Cloud - apmPackagePolicy create end`);
+  logger.debug(`Fleet migration on Cloud - apmPackagePolicy create end`);
   return apmPackagePolicy;
 }

x-pack/solutions/observability/plugins/apm/server/routes/fleet/route.ts

@@ -98,7 +98,7 @@ const saveApmServerSchemaRoute = createApmServerRoute({
       { schemaJson: JSON.stringify(schema) },
       { id: APM_SERVER_SCHEMA_SAVED_OBJECT_ID, overwrite: true }
     );
-    logger.info(`Stored apm-server schema.`);
+    logger.debug(`Stored apm-server schema.`);
   },
 });
 

x-pack/solutions/observability/plugins/apm/server/routes/service_map/get_service_map.ts

@@ -124,8 +124,7 @@ export function getServiceMap(
 
       // always catch error to avoid breaking service maps if there is a problem with ML
     ).catch((error) => {
-      logger.warn(`Unable to retrieve anomalies for service maps.`);
-      logger.error(error);
+      logger.debug(`Unable to retrieve anomalies for service maps.`, { error });
       return DEFAULT_ANOMALIES;
     });
 

x-pack/solutions/observability/plugins/apm/server/routes/services/get_services/get_services_items.ts

@@ -90,11 +90,11 @@ export async function getServicesItems({
         apmEventClient,
       }),
       getHealthStatuses({ ...commonParams, mlClient }).catch((err) => {
-        logger.error(err);
+        logger.debug(err);
         return [];
       }),
       getServicesAlerts({ ...commonParams, apmAlertsClient }).catch((err) => {
-        logger.error(err);
+        logger.debug(err);
         return [];
       }),
     ]);

x-pack/solutions/observability/plugins/apm/server/routes/settings/agent_configuration/route.ts

@@ -87,7 +87,7 @@ const getSingleAgentConfigurationRoute = createApmServerRoute({
     });
 
     if (!exactConfig) {
-      logger.info(`Config was not found for ${service.name}/${service.environment}`);
+      logger.debug(`Config was not found for ${service.name}/${service.environment}`);
 
       throw Boom.notFound();
     }
@@ -125,12 +125,12 @@ const deleteAgentConfigurationRoute = createApmServerRoute({
       apmEventClient,
     });
     if (!exactConfig) {
-      logger.info(`Config was not found for ${service.name}/${service.environment}`);
+      logger.debug(`Config was not found for ${service.name}/${service.environment}`);
 
       throw Boom.notFound();
     }
 
-    logger.info(`Deleting config ${service.name}/${service.environment} (${exactConfig.id})`);
+    logger.debug(`Deleting config ${service.name}/${service.environment} (${exactConfig.id})`);
 
     const deleteConfigurationResult = await deleteConfiguration({
       configurationId: exactConfig.id!,
@@ -145,7 +145,7 @@ const deleteAgentConfigurationRoute = createApmServerRoute({
         apmIndices,
         telemetryUsageCounter,
       });
-      logger.info(
+      logger.debug(
         `Updated Fleet integration policy for APM to remove the deleted agent configuration.`
       );
     }
@@ -191,7 +191,7 @@ const createOrUpdateAgentConfigurationRoute = createApmServerRoute({
       );
     }
 
-    logger.info(
+    logger.debug(
       `${exactConfig ? 'Updating' : 'Creating'} config ${body.service.name}/${
         body.service.environment
       }`
@@ -211,7 +211,7 @@ const createOrUpdateAgentConfigurationRoute = createApmServerRoute({
         internalESClient,
         telemetryUsageCounter,
       });
-      logger.info(`Saved latest agent settings to Fleet integration policy for APM.`);
+      logger.debug(`Saved latest agent settings to Fleet integration policy for APM.`);
     }
   },
 });

x-pack/solutions/observability/plugins/apm/server/routes/source_maps/schedule_source_map_migration.ts

@@ -125,8 +125,7 @@ export async function runFleetSourcemapArtifactsMigration({
       internalESClient,
     });
   } catch (e) {
-    logger.error('Failed to migrate APM fleet source map artifacts');
-    logger.error(e);
+    logger.error('Failed to migrate APM fleet source map artifacts', { error: e });
   }
 }
 
@@ -184,7 +183,7 @@ async function paginateArtifacts({
   }
 
   const migratedCount = (page - 1) * PER_PAGE + artifacts.length;
-  logger.info(`Migrating ${migratedCount} of ${total} source maps`);
+  logger.debug(`Migrating ${migratedCount} of ${total} source maps`);
 
   await bulkCreateApmSourceMaps({ artifacts, internalESClient });
 
@@ -199,7 +198,7 @@ async function paginateArtifacts({
       internalESClient,
     });
   } else {
-    logger.info(`Successfully migrated ${total} source maps`);
+    logger.debug(`Successfully migrated ${total} source maps`);
   }
 }
 

x-pack/solutions/observability/plugins/observability/server/utils/create_or_update_index.ts

@@ -60,7 +60,7 @@ export async function createOrUpdateIndex({
       }
     );
   } catch (e) {
-    logger.error(`Could not create index: '${index}'. Error: ${e.message}.`);
+    logger.error(`Could not create index: '${index}'`, { error: e });
   }
 }