[Reporting] Consolidate API Integration Test configs (#66637)

* [Reporting] Consolidate API Integration Test configs

* fix test isolation

* tweak

* import order

* fix ts refactor ish

* fix a test bug

* fix test

* --wip-- [skip ci]

* revision

* undo bad cherrypick

* fix delete reports

* log tweak

* fix default index pattern

* fix the test

* revert

* --wip-- [skip ci]

* unrevert

* harden the deleteAllReportingIndexes function

* fix tests

* move the log.debug line

* fix config path

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

x-pack/scripts/functional_tests.js

@@ -11,8 +11,6 @@ const alwaysImportedTests = [
   require.resolve('../test/functional/config_security_trial.ts'),
 ];
 const onlyNotInCoverageTests = [
-  require.resolve('../test/reporting/configs/chromium_api.js'),
-  require.resolve('../test/reporting/configs/generate_api.js'),
   require.resolve('../test/api_integration/config_security_basic.js'),
   require.resolve('../test/api_integration/config.js'),
   require.resolve('../test/alerting_api_integration/basic/config.ts'),
@@ -50,6 +48,7 @@ const onlyNotInCoverageTests = [
   require.resolve('../test/licensing_plugin/config.public.ts'),
   require.resolve('../test/licensing_plugin/config.legacy.ts'),
   require.resolve('../test/endpoint_api_integration_no_ingest/config.ts'),
+  require.resolve('../test/reporting_api_integration/config.js'),
 ];
 
 require('@kbn/plugin-helpers').babelRegister();

x-pack/test/reporting/.gitignore

@@ -1 +0,0 @@
-functional/reports/session/

x-pack/test/reporting/api/chromium_tests.ts

@@ -1,34 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-import { OSS_DATA_ARCHIVE_PATH, OSS_KIBANA_ARCHIVE_PATH } from './constants';
-import { FtrProviderContext } from '../ftr_provider_context';
-
-// eslint-disable-next-line import/no-default-export
-export default function({ loadTestFile, getService }: FtrProviderContext) {
-  const esArchiver = getService('esArchiver');
-  const kibanaServer = getService('kibanaServer');
-
-  describe('chromium', function() {
-    this.tags('ciGroup2');
-
-    before(async () => {
-      await esArchiver.load(OSS_KIBANA_ARCHIVE_PATH);
-      await esArchiver.load(OSS_DATA_ARCHIVE_PATH);
-
-      await kibanaServer.uiSettings.update({
-        defaultIndex: '0bf35f60-3dc9-11e8-8660-4d65aa086b3c',
-      });
-    });
-
-    after(async () => {
-      await esArchiver.unload(OSS_KIBANA_ARCHIVE_PATH);
-      await esArchiver.unload(OSS_DATA_ARCHIVE_PATH);
-    });
-
-    loadTestFile(require.resolve('./usage'));
-  });
-}

x-pack/test/reporting/configs/chromium_api.js

@@ -1,38 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-import { ReportingAPIProvider } from '../services';
-
-export default async function({ readConfigFile }) {
-  const apiConfig = await readConfigFile(require.resolve('../../api_integration/config.js'));
-  const functionalConfig = await readConfigFile(require.resolve('../../functional/config.js'));
-
-  return {
-    servers: apiConfig.get('servers'),
-    junit: { reportName: 'X-Pack Chromium API Reporting Tests' },
-    testFiles: [require.resolve('../api/chromium_tests')],
-    services: {
-      ...apiConfig.get('services'),
-      reportingAPI: ReportingAPIProvider,
-    },
-    kbnTestServer: {
-      ...apiConfig.get('kbnTestServer'),
-      serverArgs: [
-        // Reporting API tests use functionalConfig instead of apiConfig because they needs a fully working UI. By default, the API config
-        // does not have optimize setting enabled, and Kibana would not have a working UI.
-        ...functionalConfig.get('kbnTestServer.serverArgs'),
-        '--logging.events.log',
-        '["info","warning","error","fatal","optimize","reporting"]',
-        '--xpack.reporting.csv.enablePanelActionDownload=true',
-        '--xpack.reporting.capture.maxAttempts=1',
-        '--xpack.security.session.idleTimeout=3600000',
-        '--xpack.spaces.enabled=false',
-      ],
-    },
-    esArchiver: apiConfig.get('esArchiver'),
-    esTestCluster: apiConfig.get('esTestCluster'),
-  };
-}

x-pack/test/reporting/services/index.ts

@@ -1,15 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-import { ReportingAPIProvider } from './reporting_api';
-import { services as xpackServices } from '../../functional/services';
-
-export const services = {
-  ...xpackServices,
-  reportingAPI: ReportingAPIProvider,
-};
-
-export { ReportingAPIProvider };

x-pack/test/reporting_api_integration/config.js

@@ -6,15 +6,16 @@
 
 import { esTestConfig, kbnTestConfig, kibanaServerTestUser } from '@kbn/test';
 import { format as formatUrl } from 'url';
-import { ReportingAPIProvider } from '../services';
+import { ReportingAPIProvider } from './services';
 
 export default async function({ readConfigFile }) {
-  const apiConfig = await readConfigFile(require.resolve('../../api_integration/config.js'));
+  const apiConfig = await readConfigFile(require.resolve('../api_integration/config'));
+  const functionalConfig = await readConfigFile(require.resolve('../functional/config')); // Reporting API tests need a fully working UI
 
   return {
     servers: apiConfig.get('servers'),
-    junit: { reportName: 'X-Pack Reporting Generate API Integration Tests' },
-    testFiles: [require.resolve('../api/generate')],
+    junit: { reportName: 'X-Pack Reporting API Integration Tests' },
+    testFiles: [require.resolve('./reporting')],
     services: {
       ...apiConfig.get('services'),
       reportingAPI: ReportingAPIProvider,
@@ -22,18 +23,18 @@ export default async function({ readConfigFile }) {
     kbnTestServer: {
       ...apiConfig.get('kbnTestServer'),
       serverArgs: [
-        '--logging.events.log',
-        '["info","warning","error","fatal","optimize","reporting"]',
+        ...functionalConfig.get('kbnTestServer.serverArgs'),
+
         `--elasticsearch.hosts=${formatUrl(esTestConfig.getUrlParts())}`,
         `--elasticsearch.password=${kibanaServerTestUser.password}`,
         `--elasticsearch.username=${kibanaServerTestUser.username}`,
         `--logging.json=false`,
-        `--optimize.enabled=false`,
         `--server.maxPayloadBytes=1679958`,
         `--server.port=${kbnTestConfig.getPort()}`,
-        `--xpack.reporting.csv.enablePanelActionDownload=true`,
+        `--xpack.reporting.capture.maxAttempts=1`,
         `--xpack.reporting.csv.maxSizeBytes=2850`,
         `--xpack.reporting.queue.pollInterval=3000`,
+        `--xpack.security.session.idleTimeout=3600000`,
         `--xpack.spaces.enabled=false`,
       ],
     },

x-pack/test/reporting_api_integration/reporting/constants.ts

@@ -3,13 +3,15 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
+
+import { REPO_ROOT } from '@kbn/dev-utils';
 import path from 'path';
 
 export const OSS_KIBANA_ARCHIVE_PATH = path.resolve(
-  __dirname,
-  '../../../../test/functional/fixtures/es_archiver/dashboard/current/kibana'
+  REPO_ROOT,
+  'test/functional/fixtures/es_archiver/dashboard/current/kibana'
 );
 export const OSS_DATA_ARCHIVE_PATH = path.resolve(
-  __dirname,
-  '../../../../test/functional/fixtures/es_archiver/dashboard/current/data'
+  REPO_ROOT,
+  'test/functional/fixtures/es_archiver/dashboard/current/data'
 );

x-pack/test/reporting_api_integration/reporting/csv_job_params.ts

@@ -6,12 +6,15 @@
 
 import expect from '@kbn/expect';
 import supertest from 'supertest';
-import { JOB_PARAMS_RISON } from './fixtures';
+import { JOB_PARAMS_RISON } from '../fixtures';
+import { FtrProviderContext } from '../ftr_provider_context';
 
 // eslint-disable-next-line import/no-default-export
-export default function({ getService }: { getService: any }) {
+export default function({ getService }: FtrProviderContext) {
   const esArchiver = getService('esArchiver');
   const supertestSvc = getService('supertest');
+  const reportingAPI = getService('reportingAPI');
+
   const generateAPI = {
     getCsvFromParamsInPayload: async (jobParams: object = {}) => {
       return await supertestSvc
@@ -30,11 +33,13 @@ export default function({ getService }: { getService: any }) {
     before(async () => {
       await esArchiver.load('reporting/logs');
       await esArchiver.load('logstash_functional');
-    }); // prettier-ignore
+    });
+
     after(async () => {
       await esArchiver.unload('reporting/logs');
       await esArchiver.unload('logstash_functional');
-    }); // prettier-ignore
+      await reportingAPI.deleteAllReports();
+    });
 
     it('Rejects bogus jobParams', async () => {
       const { status: resStatus, text: resText } = (await generateAPI.getCsvFromParamsInPayload({

x-pack/test/reporting_api_integration/reporting/csv_saved_search.ts

@@ -7,15 +7,16 @@
 import expect from '@kbn/expect';
 import supertest from 'supertest';
 import {
+  CSV_RESULT_DOCVALUE,
   CSV_RESULT_HUGE,
+  CSV_RESULT_NANOS,
   CSV_RESULT_SCRIPTED,
   CSV_RESULT_SCRIPTED_REQUERY,
   CSV_RESULT_SCRIPTED_RESORTED,
   CSV_RESULT_TIMEBASED,
   CSV_RESULT_TIMELESS,
-  CSV_RESULT_NANOS,
-  CSV_RESULT_DOCVALUE,
-} from './fixtures';
+} from '../fixtures';
+import { FtrProviderContext } from '../ftr_provider_context';
 
 interface GenerateOpts {
   timerange?: {
@@ -27,9 +28,11 @@ interface GenerateOpts {
 }
 
 // eslint-disable-next-line import/no-default-export
-export default function({ getService }: { getService: any }) {
+export default function({ getService }: FtrProviderContext) {
   const esArchiver = getService('esArchiver');
   const supertestSvc = getService('supertest');
+  const reportingAPI = getService('reportingAPI');
+
   const generateAPI = {
     getCsvFromSavedSearch: async (
       id: string,
@@ -45,6 +48,10 @@ export default function({ getService }: { getService: any }) {
 
   describe('Generation from Saved Search ID', () => {
     describe('Saved Search Features', () => {
+      after(async () => {
+        await reportingAPI.deleteAllReports();
+      });
+
       it('With filters and timebased data', async () => {
         // load test data that contains a saved search and documents
         await esArchiver.load('reporting/logs');

x-pack/test/reporting_api_integration/reporting/index.ts

@@ -4,13 +4,14 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { FtrProviderContext } from '../../ftr_provider_context';
+import { FtrProviderContext } from '../ftr_provider_context';
 
 // eslint-disable-next-line import/no-default-export
 export default function({ loadTestFile }: FtrProviderContext) {
-  describe('CSV', function() {
+  describe('Reporting APIs', function() {
     this.tags('ciGroup2');
-    loadTestFile(require.resolve('./csv_saved_search'));
     loadTestFile(require.resolve('./csv_job_params'));
+    loadTestFile(require.resolve('./csv_saved_search'));
+    loadTestFile(require.resolve('./usage'));
   });
 }

x-pack/test/reporting_api_integration/reporting/usage.ts

@@ -6,8 +6,9 @@
 
 import expect from '@kbn/expect';
 import { FtrProviderContext } from '../ftr_provider_context';
-import { ReportingUsageStats } from '../services/reporting_api';
-import * as GenerationUrls from './generation_urls';
+import * as GenerationUrls from '../generation_urls';
+import { ReportingUsageStats } from '../services';
+import { OSS_DATA_ARCHIVE_PATH, OSS_KIBANA_ARCHIVE_PATH } from './constants';
 
 interface UsageStats {
   reporting: ReportingUsageStats;
@@ -16,12 +17,29 @@ interface UsageStats {
 // eslint-disable-next-line import/no-default-export
 export default function({ getService }: FtrProviderContext) {
   const esArchiver = getService('esArchiver');
+  const kibanaServer = getService('kibanaServer');
   const reportingAPI = getService('reportingAPI');
   const usageAPI = getService('usageAPI' as any); // NOTE Usage API service is not Typescript
 
   describe('reporting usage', () => {
-    before(() => reportingAPI.deleteAllReportingIndexes());
-    afterEach(() => reportingAPI.deleteAllReportingIndexes());
+    before(async () => {
+      await esArchiver.load(OSS_KIBANA_ARCHIVE_PATH);
+      await esArchiver.load(OSS_DATA_ARCHIVE_PATH);
+
+      await kibanaServer.uiSettings.update({
+        defaultIndex: '0bf35f60-3dc9-11e8-8660-4d65aa086b3c',
+      });
+      await reportingAPI.deleteAllReports();
+    });
+
+    after(async () => {
+      await esArchiver.unload(OSS_KIBANA_ARCHIVE_PATH);
+      await esArchiver.unload(OSS_DATA_ARCHIVE_PATH);
+    });
+
+    afterEach(async () => {
+      await reportingAPI.deleteAllReports();
+    });
 
     describe('initial state', () => {
       let usage: UsageStats;
@@ -98,7 +116,7 @@ export default function({ getService }: FtrProviderContext) {
     });
 
     describe('from new jobs posted', () => {
-      it('csv', async () => {
+      it('should handle csv', async () => {
         await reportingAPI.expectAllJobsToFinishSuccessfully(
           await Promise.all([
             reportingAPI.postJob(GenerationUrls.CSV_DISCOVER_KUERY_AND_FILTER_6_3),
@@ -114,7 +132,7 @@ export default function({ getService }: FtrProviderContext) {
         reportingAPI.expectRecentJobTypeTotalStats(usage, 'printable_pdf', 0);
       });
 
-      it('preserve_layout pdf', async () => {
+      it('should handle preserve_layout pdf', async () => {
         await reportingAPI.expectAllJobsToFinishSuccessfully(
           await Promise.all([
             reportingAPI.postJob(GenerationUrls.PDF_PRESERVE_DASHBOARD_FILTER_6_3),
@@ -131,7 +149,7 @@ export default function({ getService }: FtrProviderContext) {
         reportingAPI.expectRecentJobTypeTotalStats(usage, 'printable_pdf', 2);
       });
 
-      it('print_layout pdf', async () => {
+      it('should handle print_layout pdf', async () => {
         await reportingAPI.expectAllJobsToFinishSuccessfully(
           await Promise.all([
             reportingAPI.postJob(GenerationUrls.PDF_PRINT_DASHBOARD_6_3),

x-pack/test/reporting_api_integration/services.ts

@@ -5,9 +5,12 @@
  */
 
 import expect from '@kbn/expect';
+import * as Rx from 'rxjs';
+import { filter, first, mapTo, switchMap, timeout } from 'rxjs/operators';
 // @ts-ignore no module definition
-import { indexTimestamp } from '../../../legacy/plugins/reporting/server/lib/esqueue/helpers/index_timestamp';
-import { FtrProviderContext } from '../ftr_provider_context';
+import { indexTimestamp } from '../../legacy/plugins/reporting/server/lib/esqueue/helpers/index_timestamp';
+import { services as xpackServices } from '../functional/services';
+import { FtrProviderContext } from './ftr_provider_context';
 
 interface PDFAppCounts {
   app: {
@@ -56,7 +59,13 @@ export function ReportingAPIProvider({ getService }: FtrProviderContext) {
             .get(downloadReportPath)
             .responseType('blob')
             .set('kbn-xsrf', 'xxx')) as any;
-          log.debug(`Report at path ${downloadReportPath} returned code ${response.statusCode}`);
+          if (response.statusCode === 503) {
+            log.debug(`Report at path ${downloadReportPath} is pending`);
+          } else if (response.statusCode === 200) {
+            log.debug(`Report at path ${downloadReportPath} is complete`);
+          } else {
+            log.debug(`Report at path ${downloadReportPath} returned code ${response.statusCode}`);
+          }
           if (response.statusCode !== JOB_IS_PENDING_CODE) {
             clearInterval(intervalId);
             resolve(response.statusCode);
@@ -120,9 +129,25 @@ export function ReportingAPIProvider({ getService }: FtrProviderContext) {
       };
     },
 
-    async deleteAllReportingIndexes() {
-      log.debug('ReportingAPI.deleteAllReportingIndexes');
-      await esSupertest.delete('/.reporting*').expect(200);
+    async deleteAllReports() {
+      log.debug('ReportingAPI.deleteAllReports');
+
+      // ignores 409 errs and keeps retrying
+      const deleted$ = Rx.interval(100).pipe(
+        switchMap(() =>
+          esSupertest
+            .post('/.reporting*/_delete_by_query')
+            .send({ query: { match_all: {} } })
+            .then(({ status }) => status)
+        ),
+        filter(status => status === 200),
+        mapTo(true),
+        first(),
+        timeout(5000)
+      );
+
+      const reportsDeleted = await deleted$.toPromise();
+      expect(reportsDeleted).to.be(true);
     },
 
     expectRecentPdfAppStats(stats: UsageStats, app: string, count: number) {
@@ -158,3 +183,8 @@ export function ReportingAPIProvider({ getService }: FtrProviderContext) {
     },
   };
 }
+
+export const services = {
+  ...xpackServices,
+  reportingAPI: ReportingAPIProvider,
+};