[Reporting] Functional test structure & improvements (#114298) (#115021)

* [Reporting] Functional test structure & improvements

* show the error of the report generation failure in the test failure

* update snapshot

* remove import to non-existent functional app test

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

vars/kibanaPipeline.groovy

@@ -191,7 +191,6 @@ def withGcsArtifactUpload(workerName, closure) {
     'x-pack/test/**/screenshots/diff/*.png',
     'x-pack/test/**/screenshots/failure/*.png',
     'x-pack/test/**/screenshots/session/*.png',
-    'x-pack/test/functional/apps/reporting/reports/session/*.pdf',
     'x-pack/test/functional/failure_debug/html/*.html',
     '.es/**/*.hprof'
   ]

x-pack/plugins/reporting/public/lib/__snapshots__/stream_handler.test.ts.snap

@@ -82,6 +82,7 @@ Array [
       "reactNode": <React.Fragment>
         <EuiCallOut
           color="danger"
+          data-test-errorText="this is the failed report error"
           iconType="alert"
           size="m"
           title="The reporting job failed"

x-pack/plugins/reporting/public/notifier/job_failure.tsx

@@ -35,6 +35,7 @@ export const getFailureToast = (
           })}
           color="danger"
           iconType="alert"
+          data-test-errorText={errorText}
         >
           {errorText}
         </EuiCallOut>

x-pack/test/functional/apps/reporting/README.md

@@ -0,0 +1,18 @@
+## Reporting functional tests
+
+Functional tests on report generation are under the applications that use reporting.
+
+**PDF/PNG Report testing:**
+ - `x-pack/test/functional/apps/canvas/reports.ts`
+ - `x-pack/test/functional/apps/dashboard/reporting/screenshots.ts`
+ - `x-pack/test/functional/apps/lens/lens_reporting.ts`
+ - `x-pack/test/functional/apps/visualize/reporting.ts`
+
+**CSV Report testing:**
+ - `x-pack/test/functional/apps/dashboard/reporting/download_csv.ts`
+ - `x-pack/test/functional/apps/discover/reporting.ts`
+
+Reporting Management app tests are in `functional/apps/reporting_management`.
+
+**Manage reports testing:**
+ - `x-pack/test/functional/apps/reporting_management` 

x-pack/test/functional/apps/reporting/index.ts

@@ -1,14 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import { FtrProviderContext } from '../../ftr_provider_context';
-
-export default function ({ loadTestFile }: FtrProviderContext) {
-  describe('Reporting', function () {
-    loadTestFile(require.resolve('./reporting'));
-  });
-}

x-pack/test/functional/apps/reporting/reporting.ts

@@ -1,55 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import expect from '@kbn/expect';
-import { FtrProviderContext } from '../../ftr_provider_context';
-
-export default function ({ getService, getPageObjects }: FtrProviderContext) {
-  const pageObjects = getPageObjects(['dashboard', 'common', 'reporting']);
-  const es = getService('es');
-  const kibanaServer = getService('kibanaServer');
-
-  const retry = getService('retry');
-
-  describe('Reporting', function () {
-    this.tags(['smoke', 'ciGroup2']);
-    before(async () => {
-      await kibanaServer.importExport.load(
-        'x-pack/test/functional/fixtures/kbn_archiver/packaging'
-      );
-    });
-
-    after(async () => {
-      await kibanaServer.importExport.unload(
-        'x-pack/test/functional/fixtures/kbn_archiver/packaging'
-      );
-      await es.deleteByQuery({
-        index: '.reporting-*',
-        refresh: true,
-        body: { query: { match_all: {} } },
-      });
-    });
-
-    it('downloaded PDF has OK status', async function () {
-      this.timeout(180000);
-
-      await pageObjects.common.navigateToApp('dashboards');
-      await retry.waitFor('dashboard landing page', async () => {
-        return await pageObjects.dashboard.onDashboardLandingPage();
-      });
-      await pageObjects.dashboard.loadSavedDashboard('dashboard');
-      await pageObjects.reporting.openPdfReportingPanel();
-      await pageObjects.reporting.clickGenerateReportButton();
-
-      const url = await pageObjects.reporting.getReportURL(60000);
-      const res = await pageObjects.reporting.getResponse(url);
-
-      expect(res.status).to.equal(200);
-      expect(res.get('content-type')).to.equal('application/pdf');
-    });
-  });
-}

x-pack/test/functional/config.js

@@ -57,7 +57,6 @@ export default async function ({ readConfigFile }) {
       resolve(__dirname, './apps/transform'),
       resolve(__dirname, './apps/reporting_management'),
       resolve(__dirname, './apps/management'),
-      resolve(__dirname, './apps/reporting'),
 
       //This upgrade assistant file needs to be run after the rest of the jobs because of
       //it being destructive.

x-pack/test/functional/page_objects/reporting_page.ts

@@ -19,6 +19,7 @@ export class ReportingPageObject extends FtrService {
   private readonly retry = this.ctx.getService('retry');
   private readonly security = this.ctx.getService('security');
   private readonly testSubjects = this.ctx.getService('testSubjects');
+  private readonly find = this.ctx.getService('find');
   private readonly share = this.ctx.getPageObject('share');
   private readonly timePicker = this.ctx.getPageObject('timePicker');
 
@@ -33,15 +34,21 @@ export class ReportingPageObject extends FtrService {
   async getReportURL(timeout: number) {
     this.log.debug('getReportURL');
 
-    const url = await this.testSubjects.getAttribute(
+    try {
-      'downloadCompletedReportButton',
+      const url = await this.testSubjects.getAttribute(
-      'href',
+        'downloadCompletedReportButton',
-      timeout
+        'href',
-    );
+        timeout
+      );
+      this.log.debug(`getReportURL got url: ${url}`);
 
-    this.log.debug(`getReportURL got url: ${url}`);
+      return url;
-
+    } catch (err) {
-    return url;
+      const errorTextEl = await this.find.byCssSelector('[data-test-errorText]');
+      const errorText = await errorTextEl.getAttribute('data-test-errorText');
+      const newError = new Error(`Test report failed: ${errorText}: ${err}`);
+      throw newError;
+    }
   }
 
   async removeForceSharedItemsContainerSize() {