[Reporting] Baseline capture tests (#113910)

* added page to reporting example app that contains the capture tests

* first version of PNG capture for test A

* added types file to common

* added data-shared-item attr to image, also added capture menu items

* fix image CSS by providing a fixed width and height

* explicitly add layout for print, does not seem to do anything though?

* added magic numbers of image sizes

* added reporting examples test folder

* first version of capture test for generating and comparing PNGs

* added PNG service and PNG baseline fixture

* added pdf-to-img dev dependency

* refactor compare_pngs to accept a buffer

* added comment to interface

* png service -> compare images service

* export image compare service

* added test for pdf export

* clean up log

* minor fixes and added pdf print optimized test

* added pdf and pdf print fixtures

* refactor lib function name

* Update difference thresholds

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

package.json

@@ -764,6 +764,7 @@
     "oboe": "^2.1.4",
     "parse-link-header": "^1.0.1",
     "pbf": "3.2.1",
+    "pdf-to-img": "^1.1.1",
     "pirates": "^4.0.1",
     "pixelmatch": "^5.1.0",
     "postcss": "^7.0.32",

test/functional/services/lib/compare_pngs.ts

@@ -10,26 +10,56 @@ import { parse, join } from 'path';
 import Jimp from 'jimp';
 import { ToolingLog } from '@kbn/dev-utils';
 
+interface PngDescriptor {
+  path: string;
+
+  /**
+   * If a buffer is provided this will avoid the extra step of reading from disk
+   */
+  buffer?: Buffer;
+}
+
+const toDescriptor = (imageInfo: string | PngDescriptor): PngDescriptor => {
+  if (typeof imageInfo === 'string') {
+    return { path: imageInfo };
+  }
+  return {
+    ...imageInfo,
+  };
+};
+
+/**
+ * Override Jimp types that expect to be mapped to either string or buffer even though Jimp
+ * accepts both https://www.npmjs.com/package/jimp#basic-usage.
+ */
+const toJimp = (imageInfo: string | Buffer): Promise<Jimp> => {
+  return (Jimp.read as (value: string | Buffer) => Promise<Jimp>)(imageInfo);
+};
+
 /**
  * Comparing pngs and writing result to provided directory
  *
- * @param sessionPath
- * @param baselinePath
+ * @param session
+ * @param baseline
  * @param diffPath
  * @param sessionDirectory
  * @param log
  * @returns Percent
  */
 export async function comparePngs(
-  sessionPath: string,
-  baselinePath: string,
+  sessionInfo: string | PngDescriptor,
+  baselineInfo: string | PngDescriptor,
   diffPath: string,
   sessionDirectory: string,
   log: ToolingLog
 ) {
-  log.debug(`comparePngs: ${sessionPath} vs ${baselinePath}`);
-  const session = (await Jimp.read(sessionPath)).clone();
-  const baseline = (await Jimp.read(baselinePath)).clone();
+  const sessionDescriptor = toDescriptor(sessionInfo);
+  const baselineDescriptor = toDescriptor(baselineInfo);
+
+  log.debug(`comparePngs: ${sessionDescriptor.path} vs ${baselineDescriptor.path}`);
+
+  const session = (await toJimp(sessionDescriptor.buffer ?? sessionDescriptor.path)).clone();
+  const baseline = (await toJimp(baselineDescriptor.buffer ?? baselineDescriptor.path)).clone();
 
   if (
     session.bitmap.width !== baseline.bitmap.width ||
@@ -63,8 +93,12 @@ export async function comparePngs(
     image.write(diffPath);
 
     // For debugging purposes it'll help to see the resized images and how they compare.
-    session.write(join(sessionDirectory, `${parse(sessionPath).name}-session-resized.png`));
-    baseline.write(join(sessionDirectory, `${parse(baselinePath).name}-baseline-resized.png`));
+    session.write(
+      join(sessionDirectory, `${parse(sessionDescriptor.path).name}-session-resized.png`)
+    );
+    baseline.write(
+      join(sessionDirectory, `${parse(baselineDescriptor.path).name}-baseline-resized.png`)
+    );
   }
   return percent;
 }

x-pack/examples/reporting_example/common/index.ts

@@ -8,6 +8,8 @@
 export const PLUGIN_ID = 'reportingExample';
 export const PLUGIN_NAME = 'reportingExample';
 
+export { MyForwardableState } from './types';
+
 export {
   REPORTING_EXAMPLE_LOCATOR_ID,
   ReportingExampleLocatorDefinition,

x-pack/examples/reporting_example/common/locator.ts

@@ -8,6 +8,7 @@
 import { SerializableRecord } from '@kbn/utility-types';
 import type { LocatorDefinition } from '../../../../src/plugins/share/public';
 import { PLUGIN_ID } from '../common';
+import type { MyForwardableState } from '../public/types';
 
 export const REPORTING_EXAMPLE_LOCATOR_ID = 'REPORTING_EXAMPLE_LOCATOR_ID';
 
@@ -20,10 +21,11 @@ export class ReportingExampleLocatorDefinition implements LocatorDefinition<{}>
     '1.0.0': (state: {}) => ({ ...state, migrated: true }),
   };
 
-  public readonly getLocation = async (params: {}) => {
+  public readonly getLocation = async (params: MyForwardableState) => {
+    const path = Boolean(params.captureTest) ? '/captureTest' : '/';
     return {
       app: PLUGIN_ID,
-      path: '/',
+      path,
       state: params,
     };
   };

x-pack/examples/reporting_example/common/types.ts

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { Ensure, SerializableRecord } from '@kbn/utility-types';
+
+export type MyForwardableState = Ensure<
+  SerializableRecord & { captureTest: 'A' },
+  SerializableRecord
+>;

x-pack/examples/reporting_example/public/application.tsx

@@ -7,23 +7,29 @@
 
 import React from 'react';
 import ReactDOM from 'react-dom';
+import { Router, Route, Switch } from 'react-router-dom';
 import { AppMountParameters, CoreStart } from '../../../../src/core/public';
-import { ReportingExampleApp } from './components/app';
+import { CaptureTest } from './containers/capture_test';
+import { Main } from './containers/main';
+import { ApplicationContextProvider } from './application_context';
 import { SetupDeps, StartDeps, MyForwardableState } from './types';
+import { ROUTES } from './constants';
 
 export const renderApp = (
   coreStart: CoreStart,
   deps: Omit<StartDeps & SetupDeps, 'developerExamples'>,
-  { appBasePath, element }: AppMountParameters, // FIXME: appBasePath is deprecated
+  { appBasePath, element, history }: AppMountParameters, // FIXME: appBasePath is deprecated
   forwardedParams: MyForwardableState
 ) => {
   ReactDOM.render(
-    <ReportingExampleApp
-      basename={appBasePath}
-      {...coreStart}
-      {...deps}
-      forwardedParams={forwardedParams}
-    />,
+    <ApplicationContextProvider forwardedState={forwardedParams}>
+      <Router history={history}>
+        <Switch>
+          <Route path={ROUTES.captureTest} exact render={() => <CaptureTest />} />
+          <Route render={() => <Main basename={appBasePath} {...coreStart} {...deps} />} />
+        </Switch>
+      </Router>
+    </ApplicationContextProvider>,
     element
   );
 

x-pack/examples/reporting_example/public/application_context.tsx

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React, { useContext, createContext, FC } from 'react';
+
+import type { MyForwardableState } from './types';
+
+interface ContextValue {
+  forwardedState?: MyForwardableState;
+}
+
+const ApplicationContext = createContext<undefined | ContextValue>(undefined);
+
+export const ApplicationContextProvider: FC<{ forwardedState: ContextValue['forwardedState'] }> = ({
+  forwardedState,
+  children,
+}) => {
+  return (
+    <ApplicationContext.Provider value={{ forwardedState }}>{children}</ApplicationContext.Provider>
+  );
+};
+
+export const useApplicationContext = (): ContextValue => {
+  const ctx = useContext(ApplicationContext);
+  if (!ctx) {
+    throw new Error('useApplicationContext called outside of ApplicationContext!');
+  }
+  return ctx;
+};

x-pack/examples/reporting_example/public/components/index.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export { TestImageA } from './test_image_a';

x-pack/examples/reporting_example/public/constants.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+// Values based on A4 page size
+export const VIS = {
+  width: 1950 / 2,
+  height: 1200 / 2,
+};
+
+export const ROUTES = {
+  captureTest: '/captureTest',
+  main: '/',
+};

x-pack/examples/reporting_example/public/containers/capture_test.scss

@@ -0,0 +1,10 @@
+.reportingExample {
+  &__captureContainer {
+    display: flex;
+    flex-direction: column;
+    align-items: center;
+
+    margin-top: $euiSizeM;
+    margin-bottom: $euiSizeM;
+  }
+}

x-pack/examples/reporting_example/public/containers/capture_test.tsx

@@ -0,0 +1,91 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { FunctionComponent } from 'react';
+import React from 'react';
+import { useHistory } from 'react-router-dom';
+import { parsePath } from 'history';
+import {
+  EuiTabbedContent,
+  EuiTabbedContentTab,
+  EuiSpacer,
+  EuiButtonEmpty,
+  EuiFlexGroup,
+  EuiFlexItem,
+  EuiPage,
+  EuiPageHeader,
+  EuiPageBody,
+  EuiPageContent,
+  EuiPageContentBody,
+} from '@elastic/eui';
+
+import { TestImageA } from '../components';
+import { useApplicationContext } from '../application_context';
+import { MyForwardableState } from '../types';
+import { ROUTES } from '../constants';
+
+import './capture_test.scss';
+
+const ItemsContainer: FunctionComponent<{ count: string }> = ({ count, children }) => (
+  <div
+    className="reportingExample__captureContainer"
+    data-shared-items-container
+    data-shared-items-count={count}
+  >
+    {children}
+  </div>
+);
+
+const tabs: Array<EuiTabbedContentTab & { id: MyForwardableState['captureTest'] }> = [
+  {
+    id: 'A',
+    name: 'Test A',
+    content: (
+      <ItemsContainer count="4">
+        <TestImageA />
+        <TestImageA />
+        <TestImageA />
+        <TestImageA />
+      </ItemsContainer>
+    ),
+  },
+];
+
+export const CaptureTest: FunctionComponent = () => {
+  const { forwardedState } = useApplicationContext();
+  const tabToRender = forwardedState?.captureTest;
+  const history = useHistory();
+  return (
+    <EuiPage>
+      <EuiPageBody>
+        <EuiPageContent>
+          <EuiPageHeader>
+            <EuiFlexGroup alignItems="center" justifyContent="flexEnd">
+              <EuiFlexItem grow={false}>
+                <EuiButtonEmpty
+                  iconType="arrowLeft"
+                  href={history.createHref(parsePath(ROUTES.main))}
+                >
+                  Back to main
+                </EuiButtonEmpty>
+              </EuiFlexItem>
+            </EuiFlexGroup>
+          </EuiPageHeader>
+          <EuiPageContentBody>
+            <EuiSpacer />
+            <EuiTabbedContent
+              tabs={tabs}
+              initialSelectedTab={
+                tabToRender ? tabs.find((tab) => tab.id === tabToRender) : undefined
+              }
+            />
+          </EuiPageContentBody>
+        </EuiPageContent>
+      </EuiPageBody>
+    </EuiPage>
+  );
+};

x-pack/examples/reporting_example/public/containers/main.tsx

@@ -23,41 +23,42 @@ import {
   EuiTitle,
   EuiCodeBlock,
   EuiSpacer,
+  EuiLink,
+  EuiContextMenuProps,
 } from '@elastic/eui';
 import moment from 'moment';
 import { I18nProvider } from '@kbn/i18n/react';
 import React, { useEffect, useState } from 'react';
-import { BrowserRouter as Router } from 'react-router-dom';
+import { parsePath } from 'history';
+import { BrowserRouter as Router, useHistory } from 'react-router-dom';
 import * as Rx from 'rxjs';
 import { takeWhile } from 'rxjs/operators';
 import { ScreenshotModePluginSetup } from 'src/plugins/screenshot_mode/public';
-import { constants, ReportingStart } from '../../../../../x-pack/plugins/reporting/public';
+import { constants, ReportingStart } from '../../../../plugins/reporting/public';
 import type { JobParamsPDFV2 } from '../../../../plugins/reporting/server/export_types/printable_pdf_v2/types';
 import type { JobParamsPNGV2 } from '../../../../plugins/reporting/server/export_types/png_v2/types';
 
 import { REPORTING_EXAMPLE_LOCATOR_ID } from '../../common';
 
-import { MyForwardableState } from '../types';
+import { useApplicationContext } from '../application_context';
+import { ROUTES } from '../constants';
+import type { MyForwardableState } from '../types';
 
 interface ReportingExampleAppProps {
   basename: string;
   reporting: ReportingStart;
   screenshotMode: ScreenshotModePluginSetup;
-  forwardedParams?: MyForwardableState;
 }
 
 const sourceLogos = ['Beats', 'Cloud', 'Logging', 'Kibana'];
 
-export const ReportingExampleApp = ({
-  basename,
-  reporting,
-  screenshotMode,
-  forwardedParams,
-}: ReportingExampleAppProps) => {
+export const Main = ({ basename, reporting, screenshotMode }: ReportingExampleAppProps) => {
+  const history = useHistory();
+  const { forwardedState } = useApplicationContext();
   useEffect(() => {
     // eslint-disable-next-line no-console
-    console.log('forwardedParams', forwardedParams);
-  }, [forwardedParams]);
+    console.log('forwardedState', forwardedState);
+  }, [forwardedState]);
 
   // Context Menu
   const [isPopoverOpen, setPopover] = useState(false);
@@ -123,12 +124,54 @@ export const ReportingExampleApp = ({
     };
   };
 
-  const panels = [
+  const getCaptureTestPNGJobParams = (): JobParamsPNGV2 => {
+    return {
+      version: '8.0.0',
+      layout: {
+        id: constants.LAYOUT_TYPES.PRESERVE_LAYOUT,
+      },
+      locatorParams: {
+        id: REPORTING_EXAMPLE_LOCATOR_ID,
+        version: '0.5.0',
+        params: { captureTest: 'A' } as MyForwardableState,
+      },
+      objectType: 'develeloperExample',
+      title: 'Reporting Developer Example',
+      browserTimezone: moment.tz.guess(),
+    };
+  };
+
+  const getCaptureTestPDFJobParams = (print: boolean) => (): JobParamsPDFV2 => {
+    return {
+      version: '8.0.0',
+      layout: {
+        id: print ? constants.LAYOUT_TYPES.PRINT : constants.LAYOUT_TYPES.PRESERVE_LAYOUT,
+        dimensions: {
+          // Magic numbers based on height of components not rendered on this screen :(
+          height: 2400,
+          width: 1822,
+        },
+      },
+      locatorParams: [
+        {
+          id: REPORTING_EXAMPLE_LOCATOR_ID,
+          version: '0.5.0',
+          params: { captureTest: 'A' } as MyForwardableState,
+        },
+      ],
+      objectType: 'develeloperExample',
+      title: 'Reporting Developer Example',
+      browserTimezone: moment.tz.guess(),
+    };
+  };
+
+  const panels: EuiContextMenuProps['panels'] = [
     {
       id: 0,
       items: [
         { name: 'PDF Reports', icon: 'document', panel: 1 },
         { name: 'PNG Reports', icon: 'document', panel: 7 },
+        { name: 'Capture test', icon: 'document', panel: 8, 'data-test-subj': 'captureTestPanel' },
       ],
     },
     {
@@ -141,6 +184,31 @@ export const ReportingExampleApp = ({
         { name: 'Canvas Layout Option', icon: 'canvasApp', panel: 3 },
       ],
     },
+    {
+      id: 8,
+      initialFocusedItemIndex: 0,
+      title: 'Capture test',
+      items: [
+        {
+          name: 'Capture test A - PNG',
+          icon: 'document',
+          panel: 9,
+          'data-test-subj': 'captureTestPNG',
+        },
+        {
+          name: 'Capture test A - PDF',
+          icon: 'document',
+          panel: 10,
+          'data-test-subj': 'captureTestPDF',
+        },
+        {
+          name: 'Capture test A - PDF print optimized',
+          icon: 'document',
+          panel: 11,
+          'data-test-subj': 'captureTestPDFPrint',
+        },
+      ],
+    },
     {
       id: 7,
       initialFocusedItemIndex: 0,
@@ -188,6 +256,37 @@ export const ReportingExampleApp = ({
         />
       ),
     },
+    {
+      id: 9,
+      title: 'Test A',
+      content: (
+        <reporting.components.ReportingPanelPNGV2
+          getJobParams={getCaptureTestPNGJobParams}
+          onClose={closePopover}
+        />
+      ),
+    },
+    {
+      id: 10,
+      title: 'Test A',
+      content: (
+        <reporting.components.ReportingPanelPDFV2
+          getJobParams={getCaptureTestPDFJobParams(false)}
+          onClose={closePopover}
+        />
+      ),
+    },
+    {
+      id: 11,
+      title: 'Test A',
+      content: (
+        <reporting.components.ReportingPanelPDFV2
+          layoutOption="print"
+          getJobParams={getCaptureTestPDFJobParams(true)}
+          onClose={closePopover}
+        />
+      ),
+    },
   ];
 
   return (
@@ -207,30 +306,45 @@ export const ReportingExampleApp = ({
                 </EuiTitle>
                 <EuiSpacer />
                 <EuiText>
-                  <EuiPopover
-                    id="contextMenuExample"
-                    button={<EuiButton onClick={onButtonClick}>Share</EuiButton>}
-                    isOpen={isPopoverOpen}
-                    closePopover={closePopover}
-                    panelPaddingSize="none"
-                    anchorPosition="downLeft"
-                  >
-                    <EuiContextMenu initialPanelId={0} panels={panels} />
-                  </EuiPopover>
+                  <EuiFlexGroup alignItems="center" gutterSize="l">
+                    <EuiFlexItem grow={false}>
+                      <EuiPopover
+                        id="contextMenuExample"
+                        button={
+                          <EuiButton data-test-subj="shareButton" onClick={onButtonClick}>
+                            Share
+                          </EuiButton>
+                        }
+                        isOpen={isPopoverOpen}
+                        closePopover={closePopover}
+                        panelPaddingSize="none"
+                        anchorPosition="downLeft"
+                      >
+                        <EuiContextMenu initialPanelId={0} panels={panels} />
+                      </EuiPopover>
+                    </EuiFlexItem>
+                    <EuiFlexItem grow={false}>
+                      <EuiText size="s">
+                        <EuiLink href={history.createHref(parsePath(ROUTES.captureTest))}>
+                          Go to capture test
+                        </EuiLink>
+                      </EuiText>
+                    </EuiFlexItem>
+                  </EuiFlexGroup>
 
                   <EuiHorizontalRule />
 
                   <div data-shared-items-container data-shared-items-count="5">
                     <EuiFlexGroup gutterSize="l">
                       <EuiFlexItem data-shared-item>
-                        {forwardedParams ? (
+                        {forwardedState ? (
                           <>
                             <EuiText>
                               <p>
                                 <strong>Forwarded app state</strong>
                               </p>
                             </EuiText>
-                            <EuiCodeBlock>{JSON.stringify(forwardedParams)}</EuiCodeBlock>
+                            <EuiCodeBlock>{JSON.stringify(forwardedState)}</EuiCodeBlock>
                           </>
                         ) : (
                           <>

x-pack/examples/reporting_example/public/types.ts

@@ -10,6 +10,7 @@ import { ScreenshotModePluginSetup } from 'src/plugins/screenshot_mode/public';
 import { SharePluginSetup } from 'src/plugins/share/public';
 import { DeveloperExamplesSetup } from '../../../../examples/developer_examples/public';
 import { ReportingStart } from '../../../plugins/reporting/public';
+import type { MyForwardableState } from '../common';
 
 // eslint-disable-next-line @typescript-eslint/no-empty-interface
 export interface PluginSetup {}
@@ -26,4 +27,4 @@ export interface StartDeps {
   reporting: ReportingStart;
 }
 
-export type MyForwardableState = Record<string, unknown>;
+export type { MyForwardableState };

x-pack/test/examples/config.ts

@@ -33,7 +33,11 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
       reportName: 'X-Pack Example plugin functional tests',
     },
 
-    testFiles: [require.resolve('./search_examples'), require.resolve('./embedded_lens')],
+    testFiles: [
+      require.resolve('./search_examples'),
+      require.resolve('./embedded_lens'),
+      require.resolve('./reporting_examples'),
+    ],
 
     kbnTestServer: {
       ...xpackFunctionalConfig.get('kbnTestServer'),

x-pack/test/examples/reporting_examples/capture_test.ts

@@ -0,0 +1,91 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import path from 'path';
+import type { FtrProviderContext } from '../../functional/ftr_provider_context';
+
+// eslint-disable-next-line import/no-default-export
+export default function ({ getService, getPageObjects }: FtrProviderContext) {
+  const PageObjects = getPageObjects(['common', 'reporting']);
+  const compareImages = getService('compareImages');
+  const testSubjects = getService('testSubjects');
+
+  const appId = 'reportingExample';
+
+  const fixtures = {
+    baselineAPng: path.resolve(__dirname, 'fixtures/baseline/capture_a.png'),
+    baselineAPdf: path.resolve(__dirname, 'fixtures/baseline/capture_a.pdf'),
+    baselineAPdfPrint: path.resolve(__dirname, 'fixtures/baseline/capture_a_print.pdf'),
+  };
+
+  describe('Captures', () => {
+    it('PNG that matches the baseline', async () => {
+      await PageObjects.common.navigateToApp(appId);
+
+      await (await testSubjects.find('shareButton')).click();
+      await (await testSubjects.find('captureTestPanel')).click();
+      await (await testSubjects.find('captureTestPNG')).click();
+
+      await PageObjects.reporting.clickGenerateReportButton();
+      const url = await PageObjects.reporting.getReportURL(60000);
+      const captureData = await PageObjects.reporting.getRawPdfReportData(url);
+
+      const pngSessionFilePath = await compareImages.writeToSessionFile(
+        'capture_test_baseline_a',
+        captureData
+      );
+
+      expect(
+        await compareImages.checkIfPngsMatch(pngSessionFilePath, fixtures.baselineAPng)
+      ).to.be.lessThan(0.09);
+    });
+
+    it('PDF that matches the baseline', async () => {
+      await PageObjects.common.navigateToApp(appId);
+
+      await (await testSubjects.find('shareButton')).click();
+      await (await testSubjects.find('captureTestPanel')).click();
+      await (await testSubjects.find('captureTestPDF')).click();
+
+      await PageObjects.reporting.clickGenerateReportButton();
+      const url = await PageObjects.reporting.getReportURL(60000);
+      const captureData = await PageObjects.reporting.getRawPdfReportData(url);
+
+      const pdfSessionFilePath = await compareImages.writeToSessionFile(
+        'capture_test_baseline_a',
+        captureData
+      );
+
+      expect(
+        await compareImages.checkIfPdfsMatch(pdfSessionFilePath, fixtures.baselineAPdf)
+      ).to.be.lessThan(0.001);
+    });
+
+    it('print-optimized PDF that matches the baseline', async () => {
+      await PageObjects.common.navigateToApp(appId);
+
+      await (await testSubjects.find('shareButton')).click();
+      await (await testSubjects.find('captureTestPanel')).click();
+      await (await testSubjects.find('captureTestPDFPrint')).click();
+
+      await PageObjects.reporting.checkUsePrintLayout();
+      await PageObjects.reporting.clickGenerateReportButton();
+      const url = await PageObjects.reporting.getReportURL(60000);
+      const captureData = await PageObjects.reporting.getRawPdfReportData(url);
+
+      const pdfSessionFilePath = await compareImages.writeToSessionFile(
+        'capture_test_baseline_a',
+        captureData
+      );
+
+      expect(
+        await compareImages.checkIfPdfsMatch(pdfSessionFilePath, fixtures.baselineAPdfPrint)
+      ).to.be.lessThan(0.001);
+    });
+  });
+}

x-pack/test/examples/reporting_examples/index.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { PluginFunctionalProviderContext } from 'test/plugin_functional/services';
+
+// eslint-disable-next-line import/no-default-export
+export default function ({ loadTestFile }: PluginFunctionalProviderContext) {
+  describe('reporting examples', function () {
+    this.tags('ciGroup13');
+
+    loadTestFile(require.resolve('./capture_test'));
+  });
+}

x-pack/test/functional/services/compare_images.ts

@@ -0,0 +1,149 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import path from 'path';
+import { promises as fs } from 'fs';
+import { pdf as pdfToPng } from 'pdf-to-img';
+import { comparePngs } from '../../../../test/functional/services/lib/compare_pngs';
+import { FtrProviderContext } from '../ftr_provider_context';
+
+export function CompareImagesProvider({ getService }: FtrProviderContext) {
+  const log = getService('log');
+  const config = getService('config');
+
+  const screenshotsDir = config.get('screenshots.directory');
+
+  const writeToSessionFile = async (name: string, rawPdf: Buffer) => {
+    const sessionDirectory = path.resolve(screenshotsDir, 'session');
+    await fs.mkdir(sessionDirectory, { recursive: true });
+    const sessionReportPath = path.resolve(sessionDirectory, `${name}.png`);
+    await fs.writeFile(sessionReportPath, rawPdf);
+    return sessionReportPath;
+  };
+
+  return {
+    writeToSessionFile,
+    async checkIfPngsMatch(
+      actualPngPath: string,
+      baselinePngPath: string,
+      screenshotsDirectory: string = screenshotsDir
+    ) {
+      log.debug(`checkIfPngsMatch: ${baselinePngPath}`);
+      // Copy the pngs into the screenshot session directory, as that's where the generated pngs will automatically be
+      // stored.
+      const sessionDirectoryPath = path.resolve(screenshotsDirectory, 'session');
+      const failureDirectoryPath = path.resolve(screenshotsDirectory, 'failure');
+
+      await fs.mkdir(sessionDirectoryPath, { recursive: true });
+      await fs.mkdir(failureDirectoryPath, { recursive: true });
+
+      const actualPngFileName = path.basename(actualPngPath, '.png');
+      const baselinePngFileName = path.basename(baselinePngPath, '.png');
+
+      const baselineCopyPath = path.resolve(
+        sessionDirectoryPath,
+        `${baselinePngFileName}_baseline.png`
+      );
+      // Don't cause a test failure if the baseline snapshot doesn't exist - we don't have all OS's covered and we
+      // don't want to start causing failures for other devs working on OS's which are lacking snapshots.  We have
+      // mac and linux covered which is better than nothing for now.
+      try {
+        log.debug(`writeFile: ${baselineCopyPath}`);
+        await fs.writeFile(baselineCopyPath, await fs.readFile(baselinePngPath));
+      } catch (error) {
+        throw new Error(`No baseline png found at ${baselinePngPath}`);
+      }
+
+      const actualCopyPath = path.resolve(sessionDirectoryPath, `${actualPngFileName}_actual.png`);
+      log.debug(`writeFile: ${actualCopyPath}`);
+      await fs.writeFile(actualCopyPath, await fs.readFile(actualPngPath));
+
+      let diffTotal = 0;
+
+      const diffPngPath = path.resolve(failureDirectoryPath, `${baselinePngFileName}-${1}.png`);
+      diffTotal += await comparePngs(
+        actualCopyPath,
+        baselineCopyPath,
+        diffPngPath,
+        sessionDirectoryPath,
+        log
+      );
+
+      return diffTotal;
+    },
+    async checkIfPdfsMatch(
+      actualPdfPath: string,
+      baselinePdfPath: string,
+      screenshotsDirectory = screenshotsDir
+    ) {
+      log.debug(`checkIfPdfsMatch: ${actualPdfPath} vs ${baselinePdfPath}`);
+      // Copy the pdfs into the screenshot session directory, as that's where the generated pngs will automatically be
+      // stored.
+      const sessionDirectoryPath = path.resolve(screenshotsDirectory, 'session');
+      const failureDirectoryPath = path.resolve(screenshotsDirectory, 'failure');
+
+      await fs.mkdir(sessionDirectoryPath, { recursive: true });
+      await fs.mkdir(failureDirectoryPath, { recursive: true });
+
+      const actualPdfFileName = path.basename(actualPdfPath, '.pdf');
+      const baselinePdfFileName = path.basename(baselinePdfPath, '.pdf');
+
+      const baselineCopyPath = path.resolve(
+        sessionDirectoryPath,
+        `${baselinePdfFileName}_baseline.pdf`
+      );
+      const actualCopyPath = path.resolve(sessionDirectoryPath, `${actualPdfFileName}_actual.pdf`);
+
+      // Don't cause a test failure if the baseline snapshot doesn't exist - we don't have all OS's covered and we
+      // don't want to start causing failures for other devs working on OS's which are lacking snapshots.  We have
+      // mac and linux covered which is better than nothing for now.
+      try {
+        log.debug(`writeFileSync: ${baselineCopyPath}`);
+        await fs.writeFile(baselineCopyPath, await fs.readFile(baselinePdfPath));
+      } catch (error) {
+        log.error(`No baseline pdf found at ${baselinePdfPath}`);
+        return 0;
+      }
+      log.debug(`writeFileSync: ${actualCopyPath}`);
+      await fs.writeFile(actualCopyPath, await fs.readFile(actualPdfPath));
+
+      const actualPdf = await pdfToPng(actualCopyPath);
+      const baselinePdf = await pdfToPng(baselineCopyPath);
+
+      log.debug(`Checking number of pages`);
+
+      if (actualPdf.length !== baselinePdf.length) {
+        throw new Error(
+          `Expected ${baselinePdf.length} pages but got ${actualPdf.length} in PDFs expected: "${baselineCopyPath}" actual: "${actualCopyPath}".`
+        );
+      }
+
+      let diffTotal = 0;
+      let pageNum = 1;
+
+      for await (const actualPage of actualPdf) {
+        for await (const baselinePage of baselinePdf) {
+          const diffPngPath = path.resolve(
+            failureDirectoryPath,
+            `${baselinePdfFileName}-${pageNum}.png`
+          );
+          diffTotal += await comparePngs(
+            { path: path.resolve(screenshotsDirectory, '_actual.png'), buffer: actualPage },
+            { path: path.resolve(screenshotsDirectory, '_baseline.png'), buffer: baselinePage },
+            diffPngPath,
+            sessionDirectoryPath,
+            log
+          );
+          ++pageNum;
+          break;
+        }
+      }
+
+      return diffTotal;
+    },
+  };
+}

x-pack/test/functional/services/index.ts

@@ -61,6 +61,7 @@ import {
 } from './dashboard';
 import { SearchSessionsService } from './search_sessions';
 import { ObservabilityProvider } from './observability';
+import { CompareImagesProvider } from './compare_images';
 
 // define the name and providers for services that should be
 // available to your tests. If you don't specify anything here
@@ -112,4 +113,5 @@ export const services = {
   reporting: ReportingFunctionalProvider,
   searchSessions: SearchSessionsService,
   observability: ObservabilityProvider,
+  compareImages: CompareImagesProvider,
 };

yarn.lock

@@ -4086,6 +4086,21 @@
   resolved "https://registry.yarnpkg.com/@mapbox/mapbox-gl-supported/-/mapbox-gl-supported-1.5.0.tgz#f60b6a55a5d8e5ee908347d2ce4250b15103dc8e"
   integrity sha512-/PT1P6DNf7vjEEiPkVIRJkvibbqWtqnyGaBz3nfRdcxclNSnSdaLU5tfAgcD7I8Yt5i+L19s406YLl1koLnLbg==
 
+"@mapbox/node-pre-gyp@^1.0.0":
+  version "1.0.5"
+  resolved "https://registry.yarnpkg.com/@mapbox/node-pre-gyp/-/node-pre-gyp-1.0.5.tgz#2a0b32fcb416fb3f2250fd24cb2a81421a4f5950"
+  integrity sha512-4srsKPXWlIxp5Vbqz5uLfBN+du2fJChBoYn/f2h991WLdk7jUvcSk/McVLSv/X+xQIPI8eGD5GjrnygdyHnhPA==
+  dependencies:
+    detect-libc "^1.0.3"
+    https-proxy-agent "^5.0.0"
+    make-dir "^3.1.0"
+    node-fetch "^2.6.1"
+    nopt "^5.0.0"
+    npmlog "^4.1.2"
+    rimraf "^3.0.2"
+    semver "^7.3.4"
+    tar "^6.1.0"
+
 "@mapbox/point-geometry@0.1.0", "@mapbox/point-geometry@^0.1.0", "@mapbox/point-geometry@~0.1.0":
   version "0.1.0"
   resolved "https://registry.yarnpkg.com/@mapbox/point-geometry/-/point-geometry-0.1.0.tgz#8a83f9335c7860effa2eeeca254332aa0aeed8f2"
@@ -10073,6 +10088,15 @@ caniuse-lite@^1.0.0, caniuse-lite@^1.0.30001097, caniuse-lite@^1.0.30001109, can
   resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001261.tgz#96d89813c076ea061209a4e040d8dcf0c66a1d01"
   integrity sha512-vM8D9Uvp7bHIN0fZ2KQ4wnmYFpJo/Etb4Vwsuc+ka0tfGDHvOPrFm6S/7CCNLSOkAUjenT2HnUPESdOIL91FaA==
 
+canvas@2.8.0:
+  version "2.8.0"
+  resolved "https://registry.yarnpkg.com/canvas/-/canvas-2.8.0.tgz#f99ca7f25e6e26686661ffa4fec1239bbef74461"
+  integrity sha512-gLTi17X8WY9Cf5GZ2Yns8T5lfBOcGgFehDFb+JQwDqdOoBOcECS9ZWMEAqMSVcMYwXD659J8NyzjRY/2aE+C2Q==
+  dependencies:
+    "@mapbox/node-pre-gyp" "^1.0.0"
+    nan "^2.14.0"
+    simple-get "^3.0.3"
+
 capture-exit@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/capture-exit/-/capture-exit-2.0.0.tgz#fb953bfaebeb781f62898239dabb426d08a509a4"
@@ -22288,6 +22312,19 @@ pbkdf2@^3.0.3:
     safe-buffer "^5.0.1"
     sha.js "^2.4.8"
 
+pdf-to-img@^1.1.1:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/pdf-to-img/-/pdf-to-img-1.1.1.tgz#1918738477c3cc95a6786877bb1e36de81909400"
+  integrity sha512-e+4BpKSDhU+BZt34yo2P5OAqO0CRRy8xSNGDP7HhpT2FMEo5H7mzNcXdymYKRcj7xIr0eK1gYFhyjpWwHGp46Q==
+  dependencies:
+    canvas "2.8.0"
+    pdfjs-dist "2.9.359"
+
+pdfjs-dist@2.9.359:
+  version "2.9.359"
+  resolved "https://registry.yarnpkg.com/pdfjs-dist/-/pdfjs-dist-2.9.359.tgz#e67bafebf20e50fc41f1a5c189155ad008ac4f81"
+  integrity sha512-P2nYtkacdlZaNNwrBLw1ZyMm0oE2yY/5S/GDCAmMJ7U4+ciL/D0mrlEC/o4HZZc/LNE3w8lEVzBEyVgEQlPVKQ==
+
 pdfkit@>=0.8.1, pdfkit@^0.11.0:
   version "0.11.0"
   resolved "https://registry.yarnpkg.com/pdfkit/-/pdfkit-0.11.0.tgz#9cdb2fc42bd2913587fe3ddf48cc5bbb3c36f7de"
@@ -27519,7 +27556,7 @@ tar@6.1.9:
     mkdirp "^1.0.3"
     yallist "^4.0.0"
 
-tar@^6.0.2, tar@^6.1.11, tar@^6.1.2:
+tar@^6.0.2, tar@^6.1.0, tar@^6.1.11, tar@^6.1.2:
   version "6.1.11"
   resolved "https://registry.yarnpkg.com/tar/-/tar-6.1.11.tgz#6760a38f003afa1b2ffd0ffe9e9abbd0eab3d621"
   integrity sha512-an/KZQzQUkZCkuoAA64hM92X0Urb6VpRhAFllDzz44U2mcD5scmT3zBc4VgVpkugF580+DQn8eAFSyoQt0tznA==