Upgrade patch reporting issue, add more tests, reorganize files (#130839)

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

test/functional/page_objects/home_page.ts

@@ -93,6 +93,21 @@ export class HomePageObject extends FtrService {
     await this.find.clickByLinkText('Map');
   }
 
+  async launchSampleLogs(id: string) {
+    await this.launchSampleDataSet(id);
+    await this.find.clickByLinkText('Logs');
+  }
+
+  async launchSampleGraph(id: string) {
+    await this.launchSampleDataSet(id);
+    await this.find.clickByLinkText('Graph');
+  }
+
+  async launchSampleML(id: string) {
+    await this.launchSampleDataSet(id);
+    await this.find.clickByLinkText('ML jobs');
+  }
+
   async launchSampleDataSet(id: string) {
     await this.addSampleDataSet(id);
     await this.common.closeToastIfExists();

x-pack/test/functional/page_objects/graph_page.ts

@@ -283,4 +283,9 @@ export class GraphPageObject extends FtrService {
     const el = await this.find.byCssSelector('small.gphLinkSummary__term--2');
     return await el.getVisibleText();
   }
+
+  async getAllGraphNodes() {
+    const el = await this.find.allByCssSelector('.gphNode');
+    return el.length;
+  }
 }

x-pack/test/upgrade/apps/canvas/canvas_smoke_tests.ts

@@ -14,7 +14,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
   const testSubjects = getService('testSubjects');
   const browser = getService('browser');
 
-  describe('canvas smoke tests', function describeIndexTests() {
+  describe('upgrade canvas smoke tests', function describeIndexTests() {
     const spaces = [
       { space: 'default', basePath: '' },
       { space: 'automation', basePath: 's/automation' },
@@ -28,17 +28,17 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
     ];
 
     spaces.forEach(({ space, basePath }) => {
-      describe('space ' + space, () => {
-        beforeEach(async () => {
-          await PageObjects.common.navigateToActualUrl('home', '/tutorial_directory/sampleData', {
-            basePath,
-          });
-          await PageObjects.header.waitUntilLoadingHasFinished();
-        });
-        canvasTests.forEach(({ name, numElements, page }) => {
-          it('renders elements on workpad ' + name + ' page ' + page, async () => {
+      canvasTests.forEach(({ name, numElements, page }) => {
+        describe('space: ' + space, () => {
+          before(async () => {
+            await PageObjects.common.navigateToActualUrl('home', '/tutorial_directory/sampleData', {
+              basePath,
+            });
+            await PageObjects.header.waitUntilLoadingHasFinished();
             await PageObjects.home.launchSampleCanvas(name);
             await PageObjects.header.waitUntilLoadingHasFinished();
+          });
+          it('renders elements on workpad ' + name + ' page ' + page, async () => {
             const currentUrl = await browser.getCurrentUrl();
             const [, hash] = currentUrl.split('#/');
             if (hash.length === 0) {

x-pack/test/upgrade/apps/canvas/index.ts

@@ -5,8 +5,10 @@
  * 2.0.
  */
 
-export default ({ loadTestFile }) => {
+import { FtrProviderContext } from '../../ftr_provider_context';
+
+export default function ({ loadTestFile }: FtrProviderContext) {
   describe('upgrade', function () {
     loadTestFile(require.resolve('./canvas_smoke_tests'));
   });
-};
+}

x-pack/test/upgrade/apps/dashboard/dashboard_smoke_tests.ts

@@ -18,7 +18,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
   const kibanaServer = getService('kibanaServer');
   const browser = getService('browser');
 
-  describe('dashboard smoke tests', function describeIndexTests() {
+  describe('upgrade dashboard smoke tests', function describeIndexTests() {
     const spaces = [
       { space: 'default', basePath: '' },
       { space: 'automation', basePath: 's/automation' },
@@ -31,7 +31,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
     ];
 
     spaces.forEach(({ space, basePath }) => {
-      describe('space ' + space, () => {
+      describe('space: ' + space, () => {
         beforeEach(async () => {
           await PageObjects.common.navigateToActualUrl('home', '/tutorial_directory/sampleData', {
             basePath,

x-pack/test/upgrade/apps/dashboard/index.ts

@@ -5,8 +5,10 @@
  * 2.0.
  */
 
-export default ({ loadTestFile }) => {
+import { FtrProviderContext } from '../../ftr_provider_context';
+
+export default function ({ loadTestFile }: FtrProviderContext) {
   describe('upgrade', function () {
     loadTestFile(require.resolve('./dashboard_smoke_tests'));
   });
-};
+}

x-pack/test/upgrade/apps/discover/discover_smoke_tests.ts

@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import { FtrProviderContext } from '../../ftr_provider_context';
+
+export default function ({ getPageObjects, getService }: FtrProviderContext) {
+  const PageObjects = getPageObjects(['common', 'header', 'home', 'discover', 'timePicker']);
+
+  describe('upgrade discover smoke tests', function describeIndexTests() {
+    const spaces = [
+      { space: 'default', basePath: '' },
+      { space: 'automation', basePath: 's/automation' },
+    ];
+
+    const discoverTests = [
+      { name: 'kibana_sample_data_flights', timefield: true, hits: '' },
+      { name: 'kibana_sample_data_logs', timefield: true, hits: '' },
+      { name: 'kibana_sample_data_ecommerce', timefield: true, hits: '' },
+    ];
+
+    spaces.forEach(({ space, basePath }) => {
+      discoverTests.forEach(({ name, timefield, hits }) => {
+        describe('space: ' + space + ', name: ' + name, () => {
+          before(async () => {
+            await PageObjects.common.navigateToApp('discover', {
+              basePath,
+            });
+            await PageObjects.header.waitUntilLoadingHasFinished();
+            await PageObjects.discover.selectIndexPattern(name);
+            await PageObjects.discover.waitUntilSearchingHasFinished();
+            if (timefield) {
+              await PageObjects.timePicker.setCommonlyUsedTime('Last_24 hours');
+              await PageObjects.discover.waitUntilSearchingHasFinished();
+            }
+          });
+          it('shows hit count greater than zero', async () => {
+            const hitCount = await PageObjects.discover.getHitCount();
+            if (hits === '') {
+              expect(hitCount).to.be.greaterThan(0);
+            } else {
+              expect(hitCount).to.be.equal(hits);
+            }
+          });
+          it('shows table rows not empty', async () => {
+            const tableRows = await PageObjects.discover.getDocTableRows();
+            expect(tableRows.length).to.be.greaterThan(0);
+          });
+        });
+      });
+    });
+  });
+}

x-pack/test/upgrade/apps/discover/index.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrProviderContext } from '../../ftr_provider_context';
+
+export default function ({ loadTestFile }: FtrProviderContext) {
+  describe('upgrade', function () {
+    loadTestFile(require.resolve('./discover_smoke_tests'));
+  });
+}

x-pack/test/upgrade/apps/graph/graph_smoke_tests.ts

@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import { FtrProviderContext } from '../../ftr_provider_context';
+
+export default function ({ getPageObjects, getService }: FtrProviderContext) {
+  const PageObjects = getPageObjects(['common', 'header', 'home', 'graph']);
+
+  describe('upgrade graph smoke tests', function describeIndexTests() {
+    const spaces = [
+      { space: 'default', basePath: '' },
+      { space: 'automation', basePath: 's/automation' },
+    ];
+
+    const graphTests = [
+      { name: 'flights', numNodes: 91 },
+      { name: 'logs', numNodes: 27 },
+      { name: 'ecommerce', numNodes: 12 },
+    ];
+
+    spaces.forEach(({ space, basePath }) => {
+      graphTests.forEach(({ name, numNodes }) => {
+        describe('space: ' + space, () => {
+          before(async () => {
+            await PageObjects.common.navigateToActualUrl('home', '/tutorial_directory/sampleData', {
+              basePath,
+            });
+            await PageObjects.header.waitUntilLoadingHasFinished();
+            await PageObjects.home.launchSampleGraph(name);
+            await PageObjects.header.waitUntilLoadingHasFinished();
+          });
+          it('renders graph for ' + name, async () => {
+            const elements = await PageObjects.graph.getAllGraphNodes();
+            expect(elements).to.be.equal(numNodes);
+          });
+        });
+      });
+    });
+  });
+}

x-pack/test/upgrade/apps/graph/index.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrProviderContext } from '../../ftr_provider_context';
+
+export default function ({ loadTestFile }: FtrProviderContext) {
+  describe('upgrade', function () {
+    loadTestFile(require.resolve('./graph_smoke_tests'));
+  });
+}

x-pack/test/upgrade/apps/logs/index.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrProviderContext } from '../../ftr_provider_context';
+
+export default function ({ loadTestFile }: FtrProviderContext) {
+  describe('upgrade', function () {
+    loadTestFile(require.resolve('./logs_smoke_tests'));
+  });
+}

x-pack/test/upgrade/apps/logs/logs_smoke_tests.ts

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import { FtrProviderContext } from '../../ftr_provider_context';
+
+export default function ({ getPageObjects, getService }: FtrProviderContext) {
+  const PageObjects = getPageObjects(['common', 'header', 'home']);
+  const logsUi = getService('logsUi');
+
+  describe('upgrade logs smoke tests', function describeIndexTests() {
+    const spaces = [
+      { space: 'default', basePath: '' },
+      { space: 'automation', basePath: 's/automation' },
+    ];
+
+    spaces.forEach(({ space, basePath }) => {
+      describe('space: ' + space, () => {
+        before(async () => {
+          await PageObjects.common.navigateToActualUrl('home', '/tutorial_directory/sampleData', {
+            basePath,
+          });
+          await PageObjects.header.waitUntilLoadingHasFinished();
+          await PageObjects.home.launchSampleLogs('logs');
+          await PageObjects.header.waitUntilLoadingHasFinished();
+        });
+
+        it('should show log streams', async () => {
+          const logStreamEntries = await logsUi.logStreamPage.getStreamEntries();
+          expect(logStreamEntries.length).to.be.greaterThan(100);
+        });
+      });
+    });
+  });
+}

x-pack/test/upgrade/apps/maps/index.ts

@@ -5,8 +5,10 @@
  * 2.0.
  */
 
-export default ({ loadTestFile }) => {
+import { FtrProviderContext } from '../../ftr_provider_context';
+
+export default function ({ loadTestFile }: FtrProviderContext) {
   describe('upgrade', function () {
     loadTestFile(require.resolve('./maps_smoke_tests'));
   });
-};
+}

x-pack/test/upgrade/apps/maps/maps_smoke_tests.ts

@@ -82,7 +82,7 @@ export default function ({
   // Only update the baseline images from Jenkins session images after comparing them
   // These tests might fail locally because of scaling factors and resolution.
 
-  describe('maps smoke tests', function describeIndexTests() {
+  describe('upgrade maps smoke tests', function describeIndexTests() {
     const spaces = [
       { space: 'default', basePath: '' },
       { space: 'automation', basePath: 's/automation' },
@@ -101,7 +101,7 @@ export default function ({
     });
 
     spaces.forEach(({ space, basePath }) => {
-      describe('space ' + space + ' ecommerce', () => {
+      describe('space: ' + space + ', name: ecommerce', () => {
         before(async () => {
           await PageObjects.common.navigateToActualUrl('home', '/tutorial_directory/sampleData', {
             basePath,
@@ -129,7 +129,7 @@ export default function ({
           expect(percentDifference.toFixed(3)).to.be.lessThan(0.031);
         });
       });
-      describe('space ' + space + ' flights', () => {
+      describe('space: ' + space + ', name: flights', () => {
         before(async () => {
           await PageObjects.common.navigateToActualUrl('home', '/tutorial_directory/sampleData', {
             basePath,
@@ -153,7 +153,7 @@ export default function ({
           expect(percentDifference.toFixed(3)).to.be.lessThan(0.031);
         });
       });
-      describe('space ' + space + ' web logs', () => {
+      describe('space: ' + space + ', name: web logs', () => {
         before(async () => {
           await PageObjects.common.navigateToActualUrl('home', '/tutorial_directory/sampleData', {
             basePath,

x-pack/test/upgrade/apps/reporting/index.ts

@@ -5,8 +5,10 @@
  * 2.0.
  */
 
-export default ({ loadTestFile }) => {
+import { FtrProviderContext } from '../../ftr_provider_context';
+
+export default function ({ loadTestFile }: FtrProviderContext) {
   describe('upgrade', function () {
     loadTestFile(require.resolve('./reporting_smoke_tests'));
   });
-};
+}

x-pack/test/upgrade/apps/reporting/reporting_smoke_tests.ts

@@ -8,7 +8,7 @@
 import expect from '@kbn/expect';
 import { parse } from 'url';
 import { FtrProviderContext } from '../../ftr_provider_context';
-import { ReportingUsageStats } from '../../reporting_services';
+import { ReportingUsageStats } from '../../services/reporting_upgrade_services';
 
 interface UsageStats {
   reporting: ReportingUsageStats;
@@ -21,6 +21,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
   const browser = getService('browser');
   const PageObjects = getPageObjects(['common', 'header', 'home', 'dashboard', 'share']);
   const testSubjects = getService('testSubjects');
+  const log = getService('log');
 
   const spaces = [
     { space: 'default', basePath: '' },
@@ -39,7 +40,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
     { name: 'ecommerce', type: 'png', link: 'PNG Reports' },
   ];
 
-  describe('reporting smoke tests', () => {
+  describe('upgrade reporting smoke tests', () => {
     let completedReportCount: number;
     let usage: UsageStats;
     describe('initial state', () => {
@@ -53,7 +54,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
       });
     });
     spaces.forEach(({ space, basePath }) => {
-      describe('generate report space ' + space, () => {
+      describe('generate report for space ' + space, () => {
         beforeEach(async () => {
           await PageObjects.common.navigateToActualUrl('home', '/tutorial_directory/sampleData', {
             basePath,
@@ -63,7 +64,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
           completedReportCount = reportingAPI.getCompletedReportCount(usage);
         });
         reportingTests.forEach(({ name, type, link }) => {
-          it('name ' + name + ' type ' + type, async () => {
+          it('name: ' + name + ' type: ' + type, async () => {
             await PageObjects.home.launchSampleDashboard(name);
             await PageObjects.share.openShareMenuItem(link);
             if (type === 'pdf_optimize') {
@@ -89,9 +90,17 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
             const postUrl = await find.byXPath(`//button[descendant::*[text()='Copy POST URL']]`);
             await postUrl.click();
             const url = await browser.getClipboardValue();
-            await reportingAPI.expectAllJobsToFinishSuccessfully([
-              await reportingAPI.postJob(parse(url).pathname + '?' + parse(url).query),
-            ]);
+            // Add try/catch for https://github.com/elastic/elastic-stack-testing/issues/1199
+            // Waiting for job to finish sometimes gets socket hang up error, from what I
+            // observed during debug testing the command does complete.
+            // Checking expected report count will still fail if the job did not finish.
+            try {
+              await reportingAPI.expectAllJobsToFinishSuccessfully([
+                await reportingAPI.postJob(parse(url).pathname + '?' + parse(url).query),
+              ]);
+            } catch (e) {
+              log.debug(`Error waiting for job to finish: ${e}`);
+            }
             usage = (await usageAPI.getUsageStats()) as UsageStats;
             reportingAPI.expectCompletedReportCount(usage, completedReportCount + 1);
           });

x-pack/test/upgrade/config.ts

@@ -6,9 +6,9 @@
  */
 
 import { FtrConfigProviderContext } from '@kbn/test';
-import { pageObjects } from './page_objects';
-import { ReportingAPIProvider } from './reporting_services';
-import { MapsHelper } from './maps_upgrade_services';
+import { pageObjects } from '../functional/page_objects';
+import { ReportingAPIProvider } from './services/reporting_upgrade_services';
+import { MapsHelper } from './services/maps_upgrade_services';
 
 export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const apiConfig = await readConfigFile(require.resolve('../api_integration/config'));
@@ -20,6 +20,9 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
     testFiles: [
       require.resolve('./apps/canvas'),
       require.resolve('./apps/dashboard'),
+      require.resolve('./apps/discover'),
+      require.resolve('./apps/graph'),
+      require.resolve('./apps/logs'),
       require.resolve('./apps/maps'),
       require.resolve('./apps/reporting'),
     ],

x-pack/test/upgrade/ftr_provider_context.d.ts

@@ -7,7 +7,7 @@
 
 import { GenericFtrProviderContext } from '@kbn/test';
 
-import { pageObjects } from './page_objects';
+import { pageObjects } from '../functional/page_objects';
 import { services } from './services';
 
 export type FtrProviderContext = GenericFtrProviderContext<typeof services, typeof pageObjects>;

x-pack/test/upgrade/page_objects.ts

@@ -1,10 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import { pageObjects } from '../functional/page_objects';
-
-export { pageObjects };

x-pack/test/upgrade/services/index.ts

@@ -5,8 +5,8 @@
  * 2.0.
  */
 
-import { services as functionalServices } from '../functional/services';
-import { services as reportingServices } from './reporting_services';
+import { services as functionalServices } from '../../functional/services';
+import { services as reportingServices } from './reporting_upgrade_services';
 import { services as mapsUpgradeServices } from './maps_upgrade_services';
 
 export const services = {

x-pack/test/upgrade/services/maps_upgrade_services.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { FtrProviderContext } from './ftr_provider_context';
+import { FtrProviderContext } from '../ftr_provider_context';
 
 export function MapsHelper({ getPageObjects, getService }: FtrProviderContext) {
   const PageObjects = getPageObjects(['maps']);

x-pack/test/upgrade/services/reporting_upgrade_services.ts

@@ -7,9 +7,9 @@
 
 import expect from '@kbn/expect';
 import { indexTimestamp } from '@kbn/reporting-plugin/server/lib/store/index_timestamp';
-import { services as xpackServices } from '../functional/services';
-import { services as apiIntegrationServices } from '../api_integration/services';
-import { FtrProviderContext } from './ftr_provider_context';
+import { services as xpackServices } from '../../functional/services';
+import { services as apiIntegrationServices } from '../../api_integration/services';
+import { FtrProviderContext } from '../ftr_provider_context';
 
 interface PDFAppCounts {
   app: {
@@ -104,7 +104,7 @@ export function ReportingAPIProvider({ getService }: FtrProviderContext) {
      *
      * @return {Promise<Function>} A function to call to clean up the index alias that was added.
      */
-    async coerceReportsIntoExistingIndex(indexName: string) {
+    async coerceReportsIntoExistingIndex(indexName: string): Promise<Function> {
       log.debug(`ReportingAPI.coerceReportsIntoExistingIndex(${indexName})`);
 
       // Adding an index alias coerces the report to be generated on an existing index which means any new