[APM] Add log statements for flaky test (#53775) (#56962)

* [APM] Add log statements for flaky test

* Improve logging

* Improve logging

* Log full index on error

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

x-pack/legacy/plugins/apm/server/routes/settings/agent_configuration.ts

@@ -117,10 +117,19 @@ export const createAgentConfigurationRoute = createRoute(() => ({
   },
   handler: async ({ context, request }) => {
     const setup = await setupRequest(context, request);
-    return await createOrUpdateConfiguration({
-      configuration: context.params.body,
+    const configuration = context.params.body;
+
+    // TODO: Remove logger. Only added temporarily to debug flaky test (https://github.com/elastic/kibana/issues/51764)
+    context.logger.info(
+      `Hitting: /api/apm/settings/agent-configuration/new with ${configuration.service.name}/${configuration.service.environment}`
+    );
+    const res = await createOrUpdateConfiguration({
+      configuration,
       setup
     });
+    context.logger.info(`Created agent configuration`);
+
+    return res;
   }
 }));
 
@@ -161,8 +170,14 @@ export const agentConfigurationSearchRoute = createRoute(core => ({
     })
   },
   handler: async ({ context, request }) => {
-    const setup = await setupRequest(context, request);
     const { body } = context.params;
+
+    // TODO: Remove logger. Only added temporarily to debug flaky test (https://github.com/elastic/kibana/issues/51764)
+    context.logger.info(
+      `Hitting: /api/apm/settings/agent-configuration/search for ${body.service.name}/${body.service.environment}`
+    );
+
+    const setup = await setupRequest(context, request);
     const config = await searchConfigurations({
       serviceName: body.service.name,
       environment: body.service.environment,
@@ -176,6 +191,10 @@ export const agentConfigurationSearchRoute = createRoute(core => ({
       throw new Boom('Not found', { statusCode: 404 });
     }
 
+    context.logger.info(
+      `Config was found for ${body.service.name}/${body.service.environment}`
+    );
+
     // update `applied_by_agent` field if etags match
     if (body.etag === config._source.etag && !config._source.applied_by_agent) {
       markAppliedByAgent({ id: config._id, body: config._source, setup });

x-pack/test/api_integration/apis/apm/feature_controls.ts

@@ -4,6 +4,8 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
+/* eslint-disable no-console */
+
 import expect from '@kbn/expect';
 import { FtrProviderContext } from '../../ftr_provider_context';
 
@@ -13,6 +15,7 @@ export default function featureControlsTests({ getService }: FtrProviderContext)
   const security = getService('security');
   const spaces = getService('spaces');
   const log = getService('log');
+  const es = getService('legacyEs');
 
   const start = encodeURIComponent(new Date(Date.now() - 10000).toISOString());
   const end = encodeURIComponent(new Date().toISOString());
@@ -35,6 +38,7 @@ export default function featureControlsTests({ getService }: FtrProviderContext)
     };
     expectForbidden: (result: any) => void;
     expectResponse: (result: any) => void;
+    onExpectationFail?: () => Promise<any>;
   }
   const endpoints: Endpoint[] = [
     {
@@ -139,10 +143,17 @@ export default function featureControlsTests({ getService }: FtrProviderContext)
       },
       expectForbidden: expect404,
       expectResponse: expect200,
+      onExpectationFail: async () => {
+        const res = await es.search({
+          index: '.apm-agent-configuration',
+        });
+
+        console.warn(JSON.stringify(res, null, 2));
+      },
     },
   ];
 
-  const elasticsearchRole = {
+  const elasticsearchPrivileges = {
     indices: [
       { names: ['apm-*'], privileges: ['read', 'view_index_metadata'] },
       { names: ['.apm-agent-configuration'], privileges: ['read', 'write', 'view_index_metadata'] },
@@ -205,8 +216,10 @@ export default function featureControlsTests({ getService }: FtrProviderContext)
     spaceId?: string;
   }) {
     for (const endpoint of endpoints) {
-      log.debug(`hitting ${endpoint.req.url}`);
+      console.log(`Requesting: ${endpoint.req.url}. Expecting: ${expectation}`);
       const result = await executeAsUser(endpoint.req, username, password, spaceId);
+      console.log(`Responded: ${endpoint.req.url}`);
+
       try {
         if (expectation === 'forbidden') {
           endpoint.expectForbidden(result);
@@ -214,6 +227,10 @@ export default function featureControlsTests({ getService }: FtrProviderContext)
           endpoint.expectResponse(result);
         }
       } catch (e) {
+        if (endpoint.onExpectationFail) {
+          await endpoint.onExpectationFail();
+        }
+
         const { statusCode, body, req } = result.response;
         throw new Error(
           `Endpoint: ${req.method} ${req.path}
@@ -229,7 +246,7 @@ export default function featureControlsTests({ getService }: FtrProviderContext)
   describe('apm feature controls', () => {
     let res: any;
     before(async () => {
-      log.debug('creating agent configuration');
+      console.log(`Creating agent configuration`);
       res = await executeAsAdmin({
         method: 'post',
         url: '/api/apm/settings/agent-configuration/new',
@@ -238,10 +255,11 @@ export default function featureControlsTests({ getService }: FtrProviderContext)
           settings: { transaction_sample_rate: 0.5 },
         },
       });
+      console.log(`Agent configuration created`);
     });
 
     after(async () => {
-      log.debug('deleting agent configuration');
+      console.log('deleting agent configuration');
       const configurationId = res.body._id;
       await executeAsAdmin({
         method: 'delete',
@@ -255,7 +273,7 @@ export default function featureControlsTests({ getService }: FtrProviderContext)
       const password = `${username}-password`;
       try {
         await security.role.create(roleName, {
-          elasticsearch: elasticsearchRole,
+          elasticsearch: elasticsearchPrivileges,
         });
 
         await security.user.create(username, {
@@ -277,7 +295,7 @@ export default function featureControlsTests({ getService }: FtrProviderContext)
       const password = `${username}-password`;
       try {
         await security.role.create(roleName, {
-          elasticsearch: elasticsearchRole,
+          elasticsearch: elasticsearchPrivileges,
           kibana: [{ base: ['all'], spaces: ['*'] }],
         });
 
@@ -301,7 +319,7 @@ export default function featureControlsTests({ getService }: FtrProviderContext)
       const password = `${username}-password`;
       try {
         await security.role.create(roleName, {
-          elasticsearch: elasticsearchRole,
+          elasticsearch: elasticsearchPrivileges,
           kibana: [{ feature: { dashboard: ['all'] }, spaces: ['*'] }],
         });
 
@@ -339,7 +357,7 @@ export default function featureControlsTests({ getService }: FtrProviderContext)
           disabledFeatures: [],
         });
         await security.role.create(roleName, {
-          elasticsearch: elasticsearchRole,
+          elasticsearch: elasticsearchPrivileges,
           kibana: [
             { feature: { apm: ['read'] }, spaces: [space1Id] },
             { feature: { dashboard: ['all'] }, spaces: [space2Id] },