adding runPipeline tests (#27015)

package.json

@@ -82,7 +82,8 @@
       "packages/*",
       "x-pack",
       "x-pack/plugins/*",
-      "test/plugin_functional/plugins/*"
+      "test/plugin_functional/plugins/*",
+      "test/interpreter_functional/plugins/*"
     ],
     "nohoist": [
       "**/@types/*",
@@ -388,6 +389,7 @@
     "normalize-path": "^3.0.0",
     "pixelmatch": "4.0.2",
     "pkg-up": "^2.0.0",
+    "pngjs": "^3.4.0",
     "postcss": "^7.0.5",
     "postcss-url": "^8.0.0",
     "prettier": "^1.14.3",

packages/kbn-pm/src/config.ts

@@ -43,6 +43,7 @@ export function getProjectPaths(rootPath: string, options: IProjectPathOptions)
   // In anyway, have a plugin declaring their own dependencies is the
   // correct and the expect behavior.
   projectPaths.push(resolve(rootPath, 'test/plugin_functional/plugins/*'));
+  projectPaths.push(resolve(rootPath, 'test/interpreter_functional/plugins/*'));
 
   if (!ossOnly) {
     projectPaths.push(resolve(rootPath, 'x-pack'));

scripts/functional_tests.js

@@ -22,4 +22,5 @@ require('@kbn/test').runTestsCli([
   require.resolve('../test/functional/config.js'),
   require.resolve('../test/api_integration/config.js'),
   require.resolve('../test/plugin_functional/config.js'),
+  require.resolve('../test/interpreter_functional/config.js'),
 ]);

src/functional_test_runner/lib/config/schema.ts

@@ -220,6 +220,13 @@ export const schema = Joi.object()
       })
       .default(),
 
+    // settings for the snapshots module
+    snapshots: Joi.object()
+      .keys({
+        directory: Joi.string().default(defaultRelativeToConfigPath('snapshots')),
+      })
+      .default(),
+
     // settings for the failureDebugging module
     failureDebugging: Joi.object()
       .keys({

src/legacy/core_plugins/interpreter/public/renderers/visualization.js

@@ -49,7 +49,7 @@ export const visualization = () => ({
 
     handlers.onDestroy(() => visualizationLoader.destroy());
 
-    await visualizationLoader.render(domNode, handlers.vis, visData, visConfig || handlers.vis.params, uiState, params).then(() => {
+    await visualizationLoader.render(domNode, handlers.vis, visData, handlers.vis.params, uiState, params).then(() => {
       if (handlers.done) handlers.done();
     });
   },

tasks/config/run.js

@@ -208,6 +208,17 @@ module.exports = function (grunt) {
       ],
     },
 
+    interpreterFunctionalTestsRelease: {
+      cmd: process.execPath,
+      args: [
+        'scripts/functional_tests',
+        '--config', 'test/interpreter_functional/config.js',
+        '--bail',
+        '--debug',
+        '--kibana-install-dir', KIBANA_INSTALL_DIR,
+      ],
+    },
+
     pluginFunctionalTestsRelease: {
       cmd: process.execPath,
       args: [

test/functional/services/browser.js

@@ -323,6 +323,39 @@ export async function BrowserProvider({ getService }) {
         }
       }));
     }
+
+    async executeAsync(fn, ...args) {
+      return await driver.executeAsyncScript(fn, ...cloneDeep(args, arg => {
+        if (arg instanceof WebElementWrapper) {
+          return arg._webElement;
+        }
+      }));
+    }
+
+    getScrollTop() {
+      return driver
+        .executeScript('return document.body.scrollTop')
+        .then(scrollSize => parseInt(scrollSize, 10));
+    }
+
+    getScrollLeft() {
+      return driver
+        .executeScript('return document.body.scrollLeft')
+        .then(scrollSize => parseInt(scrollSize, 10));
+    }
+
+    // return promise with REAL scroll position
+    setScrollTop(scrollSize) {
+      return driver
+        .executeScript('document.body.scrollTop = ' + scrollSize)
+        .then(() => this.getScrollTop(driver));
+    }
+
+    setScrollLeft(scrollSize) {
+      return driver
+        .executeScript('document.body.scrollLeft = ' + scrollSize)
+        .then(() => this.getScrollLeft(driver));
+    }
   }
 
   return  new BrowserService();

test/functional/services/index.ts

@@ -54,6 +54,8 @@ import { RenderableProvider } from './renderable';
 // @ts-ignore not TS yet
 import { ScreenshotsProvider } from './screenshots';
 // @ts-ignore not TS yet
+import { SnapshotsProvider } from './snapshots';
+// @ts-ignore not TS yet
 import { TableProvider } from './table';
 // @ts-ignore not TS yet
 import { TestSubjectsProvider } from './test_subjects';
@@ -70,6 +72,7 @@ export const services = {
   testSubjects: TestSubjectsProvider,
   docTable: DocTableProvider,
   screenshots: ScreenshotsProvider,
+  snapshots: SnapshotsProvider,
   dashboardVisualizations: DashboardVisualizationProvider,
   dashboardExpect: DashboardExpectProvider,
   failureDebugging: FailureDebuggingProvider,

test/functional/services/lib/web_element_wrapper/web_element_wrapper.js

@@ -19,6 +19,7 @@
 
 import { scrollIntoViewIfNecessary } from './scroll_into_view_if_necessary';
 import { delay } from 'bluebird';
+import { PNG } from 'pngjs';
 import cheerio from 'cheerio';
 import testSubjSelector from '@kbn/test-subj-selector';
 
@@ -443,4 +444,19 @@ export class WebElementWrapper {
 
     return $;
   }
+
+  /**
+   * Creates the screenshot of the element
+   *
+   * @returns {Promise<void>}
+   */
+  async takeScreenshot() {
+    const screenshot = await this._driver.takeScreenshot();
+    const buffer = Buffer.from(screenshot.toString(), 'base64');
+    const { width, height, x, y } = await this.getPosition();
+    const src = PNG.sync.read(buffer);
+    const dst = new PNG({ width, height });
+    PNG.bitblt(src, dst, x, y, width, height, 0, 0);
+    return PNG.sync.write(dst);
+  }
 }

test/functional/services/screenshots.js

@@ -45,10 +45,10 @@ export async function ScreenshotsProvider({ getService }) {
      * @param updateBaselines {boolean} optional, pass true to update the baseline snapshot.
      * @return {Promise.<number>} Percentage difference between the baseline and the current snapshot.
      */
-    async compareAgainstBaseline(name, updateBaselines) {
+    async compareAgainstBaseline(name, updateBaselines, el) {
       log.debug('compareAgainstBaseline');
       const sessionPath = resolve(SESSION_DIRECTORY, `${name}.png`);
-      await this._take(sessionPath);
+      await this._take(sessionPath, el);
 
       const baselinePath = resolve(BASELINE_DIRECTORY, `${name}.png`);
       const failurePath = resolve(FAILURE_DIRECTORY, `${name}.png`);
@@ -63,21 +63,19 @@ export async function ScreenshotsProvider({ getService }) {
       }
     }
 
-    async take(name) {
-      return await this._take(resolve(SESSION_DIRECTORY, `${name}.png`));
+    async take(name, el) {
+      return await this._take(resolve(SESSION_DIRECTORY, `${name}.png`), el);
     }
 
-    async takeForFailure(name) {
-      await this._take(resolve(FAILURE_DIRECTORY, `${name}.png`));
+    async takeForFailure(name, el) {
+      await this._take(resolve(FAILURE_DIRECTORY, `${name}.png`), el);
     }
 
-    async _take(path) {
+    async _take(path, el) {
       try {
         log.info(`Taking screenshot "${path}"`);
-        const [screenshot] = await Promise.all([
-          browser.takeScreenshot(),
-          fcb(cb => mkdirp(dirname(path), cb)),
-        ]);
+        const screenshot = await (el ? el.takeScreenshot() : browser.takeScreenshot());
+        await fcb(cb => mkdirp(dirname(path), cb));
         await fcb(cb => writeFile(path, screenshot, 'base64', cb));
       } catch (err) {
         log.error('SCREENSHOT FAILED');

test/functional/services/snapshots.js

@@ -0,0 +1,85 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import expect from 'expect.js';
+import { dirname, resolve } from 'path';
+import { writeFile, readFileSync } from 'fs';
+import { fromNode as fcb, promisify } from 'bluebird';
+import mkdirp from 'mkdirp';
+import del from 'del';
+
+const writeFileAsync = promisify(writeFile);
+
+export async function SnapshotsProvider({ getService }) {
+  const log = getService('log');
+  const config = getService('config');
+
+  const SESSION_DIRECTORY = resolve(config.get('snapshots.directory'), 'session');
+  const BASELINE_DIRECTORY = resolve(config.get('snapshots.directory'), 'baseline');
+  await del([SESSION_DIRECTORY]);
+
+  class Snapshots {
+
+    /**
+     *
+     * @param name {string} name of the file to use for comparison
+     * @param value {object} value to compare to baseline.
+     * @param updateBaselines {boolean} optional, pass true to update the baseline snapshot.
+     * @return {Promise.<number>} returns 0 if successful.
+     */
+    async compareAgainstBaseline(name, value, updateBaselines) {
+      log.debug('compareAgainstBaseline');
+      const sessionPath = resolve(SESSION_DIRECTORY, `${name}.json`);
+      await this._take(sessionPath, value);
+
+      const baselinePath = resolve(BASELINE_DIRECTORY, `${name}.json`);
+
+      if (updateBaselines) {
+        await writeFileAsync(baselinePath, readFileSync(sessionPath));
+        return 0;
+      } else {
+        log.debug('comparing');
+        return this._compare(sessionPath, baselinePath);
+      }
+    }
+
+    _compare(sessionPath, baselinePath) {
+      const currentObject = readFileSync(sessionPath, { encoding: 'utf8' });
+      const baselineObject = readFileSync(baselinePath, { encoding: 'utf8' });
+      expect(currentObject).to.eql(baselineObject);
+      return 0;
+    }
+
+    async take(name) {
+      return await this._take(resolve(SESSION_DIRECTORY, `${name}.json`));
+    }
+
+    async _take(path, snapshot) {
+      try {
+        await fcb(cb => mkdirp(dirname(path), cb));
+        await fcb(cb => writeFile(path, JSON.stringify(snapshot), 'utf8', cb));
+      } catch (err) {
+        log.error('SNAPSHOT FAILED');
+        log.error(err);
+      }
+    }
+  }
+
+  return new Snapshots();
+}

test/interpreter_functional/README.md

@@ -0,0 +1,21 @@
+# Interpreter Functional Tests
+
+This folder contains interpreter functional tests.
+
+Add new test suites into the `test_suites` folder and reference them from the
+`config.js` file. These test suites work the same as regular functional test.
+
+## Run the test
+
+To run these tests during development you can use the following commands:
+
+```
+# Start the test server (can continue running)
+node scripts/functional_tests_server.js --config test/interpreter_functional/config.js
+# Start a test run
+node scripts/functional_test_runner.js --config test/interpreter_functional/config.js
+```
+
+# Writing tests
+
+Look into test_suites/run_pipeline/basic.js for examples

test/interpreter_functional/config.js

@@ -0,0 +1,57 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import path from 'path';
+import fs from 'fs';
+
+export default async function ({ readConfigFile }) {
+  const functionalConfig = await readConfigFile(require.resolve('../functional/config'));
+
+  // Find all folders in ./plugins since we treat all them as plugin folder
+  const allFiles = fs.readdirSync(path.resolve(__dirname, 'plugins'));
+  const plugins = allFiles.filter(file => fs.statSync(path.resolve(__dirname, 'plugins', file)).isDirectory());
+
+  return {
+    testFiles: [
+      require.resolve('./test_suites/run_pipeline'),
+    ],
+    services: functionalConfig.get('services'),
+    pageObjects: functionalConfig.get('pageObjects'),
+    servers: functionalConfig.get('servers'),
+    esTestCluster: functionalConfig.get('esTestCluster'),
+    apps: functionalConfig.get('apps'),
+    esArchiver: {
+      directory: path.resolve(__dirname, '../es_archives')
+    },
+    screenshots: functionalConfig.get('screenshots'),
+    snapshots: {
+      directory: path.resolve(__dirname, 'snapshots'),
+    },
+    junit: {
+      reportName: 'Interpreter Functional Tests',
+    },
+    kbnTestServer: {
+      ...functionalConfig.get('kbnTestServer'),
+      serverArgs: [
+        ...functionalConfig.get('kbnTestServer.serverArgs'),
+        ...plugins.map(pluginDir => `--plugin-path=${path.resolve(__dirname, 'plugins', pluginDir)}`),
+      ],
+    },
+  };
+}

test/interpreter_functional/plugins/kbn_tp_run_pipeline/index.js

@@ -0,0 +1,39 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export default function (kibana) {
+  return new kibana.Plugin({
+    uiExports: {
+      app: {
+        title: 'Run Pipeline',
+        description: 'This is a sample plugin to test running pipeline expressions',
+        main: 'plugins/kbn_tp_run_pipeline/app',
+      }
+    },
+
+    init(server) {
+      // The following lines copy over some configuration variables from Kibana
+      // to this plugin. This will be needed when embedding visualizations, so that e.g.
+      // region map is able to get its configuration.
+      server.injectUiAppVars('kbn_tp_run_pipeline', async () => {
+        return await server.getInjectedUiAppVars('kibana');
+      });
+    }
+  });
+}

test/interpreter_functional/plugins/kbn_tp_run_pipeline/package.json

@@ -0,0 +1,14 @@
+{
+  "name": "kbn_tp_run_pipeline",
+  "version": "1.0.0",
+  "kibana": {
+    "version": "kibana",
+    "templateVersion": "1.0.0"
+  },
+  "license": "Apache-2.0",
+  "dependencies": {
+    "@elastic/eui": "9.4.0",
+    "react": "^16.8.0",
+    "react-dom": "^16.8.0"
+  }
+}

test/interpreter_functional/plugins/kbn_tp_run_pipeline/public/app.js

@@ -0,0 +1,80 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import React from 'react';
+import { render, unmountComponentAtNode } from 'react-dom';
+
+import { uiModules } from 'ui/modules';
+import chrome from 'ui/chrome';
+
+import { RequestAdapter } from 'ui/inspector/adapters/request';
+import { DataAdapter } from 'ui/inspector/adapters/data';
+import { runPipeline } from 'ui/visualize/loader/pipeline_helpers';
+import { visualizationLoader } from 'ui/visualize/loader/visualization_loader';
+
+import { registries } from 'plugins/interpreter/registries';
+
+// This is required so some default styles and required scripts/Angular modules are loaded,
+// or the timezone setting is correctly applied.
+import 'ui/autoload/all';
+
+// These are all the required uiExports you need to import in case you want to embed visualizations.
+import 'uiExports/visTypes';
+import 'uiExports/visResponseHandlers';
+import 'uiExports/visRequestHandlers';
+import 'uiExports/visEditorTypes';
+import 'uiExports/visualize';
+import 'uiExports/savedObjectTypes';
+import 'uiExports/fieldFormats';
+import 'uiExports/search';
+
+import { Main } from './components/main';
+
+const app = uiModules.get('apps/kbnRunPipelinePlugin', ['kibana']);
+
+app.config($locationProvider => {
+  $locationProvider.html5Mode({
+    enabled: false,
+    requireBase: false,
+    rewriteLinks: false,
+  });
+});
+app.config(stateManagementConfigProvider =>
+  stateManagementConfigProvider.disable()
+);
+
+function RootController($scope, $element) {
+  const domNode = $element[0];
+
+  // render react to DOM
+  render(<Main
+    RequestAdapter={RequestAdapter}
+    DataAdapter={DataAdapter}
+    runPipeline={runPipeline}
+    registries={registries}
+    visualizationLoader={visualizationLoader}
+  />, domNode);
+
+  // unmount react on controller destroy
+  $scope.$on('$destroy', () => {
+    unmountComponentAtNode(domNode);
+  });
+}
+
+chrome.setRootController('kbnRunPipelinePlugin', RootController);

test/interpreter_functional/plugins/kbn_tp_run_pipeline/public/components/main.js

@@ -0,0 +1,104 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import React from 'react';
+import {
+  EuiPage,
+  EuiPageBody,
+  EuiPageContent,
+  EuiPageContentHeader,
+} from '@elastic/eui';
+
+class Main extends React.Component {
+
+  chartDiv = React.createRef();
+  exprDiv = React.createRef();
+
+  constructor(props) {
+    super(props);
+
+    this.state = {
+      expression: '',
+    };
+
+    window.runPipeline = async (expression, context = {}, initialContext = {}) => {
+      this.setState({ expression });
+      const adapters = {
+        requests: new props.RequestAdapter(),
+        data: new props.DataAdapter(),
+      };
+      return await props.runPipeline(expression, context, {
+        inspectorAdapters: adapters,
+        getInitialContext: () => initialContext,
+      });
+    };
+
+    const handlers = {
+      onDestroy: () => { return; },
+    };
+
+    window.renderPipelineResponse = async (context = {}) => {
+      return new Promise(resolve => {
+        if (context.type !== 'render') {
+          this.setState({ expression: 'Expression did not return render type!\n\n' + JSON.stringify(context) });
+          return resolve();
+        }
+        const renderer = props.registries.renderers.get(context.as);
+        if (!renderer) {
+          this.setState({ expression: 'Renderer was not found in registry!\n\n' + JSON.stringify(context) });
+          return resolve();
+        }
+        props.visualizationLoader.destroy(this.chartDiv);
+        const renderCompleteHandler = () => {
+          resolve('render complete');
+          this.chartDiv.removeEventListener('renderComplete', renderCompleteHandler);
+        };
+        this.chartDiv.addEventListener('renderComplete', renderCompleteHandler);
+        renderer.render(this.chartDiv, context.value, handlers);
+      });
+
+    };
+
+  }
+
+
+  render() {
+    const pStyle = {
+      display: 'flex',
+      width: '100%',
+      height: '300px'
+    };
+
+    return (
+      <EuiPage>
+        <EuiPageBody>
+          <EuiPageContent data-test-subj="pluginContent">
+            <EuiPageContentHeader>
+              runPipeline tests are running ...
+            </EuiPageContentHeader>
+            <div data-test-subj="pluginChart" ref={ref => this.chartDiv = ref} style={pStyle}/>
+            <div>{this.state.expression}</div>
+          </EuiPageContent>
+        </EuiPageBody>
+      </EuiPage>
+    );
+  }
+}
+
+export { Main };

test/interpreter_functional/snapshots/baseline/combined_test0.json

@@ -0,0 +1 @@
+{"type":"kibana_context"}

test/interpreter_functional/snapshots/baseline/combined_test1.json

@@ -0,0 +1 @@
+{"filters":[],"query":[],"type":"kibana_context"}

test/interpreter_functional/snapshots/baseline/combined_test2.json

@@ -0,0 +1 @@
+{"columns":[{"id":"col-0-2","name":"response.raw: Descending"},{"id":"col-1-1","name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"kibana_datatable"}

test/interpreter_functional/snapshots/baseline/combined_test3.json

@@ -0,0 +1 @@
+{"as":"visualization","type":"render","value":{"params":{"listenOnChange":true},"visConfig":{"dimensions":{"bucket":{"accessor":0},"metrics":[{"accessor":1,"format":{"id":"number"},"params":{}}]}},"visData":{"columns":[{"id":"col-0-2","name":"response.raw: Descending"},{"id":"col-1-1","name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"kibana_datatable"},"visType":"metric"}}

test/interpreter_functional/snapshots/baseline/final_output_test.json

@@ -0,0 +1 @@
+{"as":"visualization","type":"render","value":{"params":{"listenOnChange":true},"visConfig":{"dimensions":{"bucket":{"accessor":0},"metrics":[{"accessor":1,"format":{"id":"number"},"params":{}}]}},"visData":{"columns":[{"id":"col-0-2","name":"response.raw: Descending"},{"id":"col-1-1","name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"kibana_datatable"},"visType":"metric"}}

test/interpreter_functional/snapshots/baseline/step_output_test0.json

@@ -0,0 +1 @@
+{"type":"kibana_context"}

test/interpreter_functional/snapshots/baseline/step_output_test1.json

@@ -0,0 +1 @@
+{"filters":[],"query":[],"type":"kibana_context"}

test/interpreter_functional/snapshots/baseline/step_output_test2.json

@@ -0,0 +1 @@
+{"columns":[{"id":"col-0-2","name":"response.raw: Descending"},{"id":"col-1-1","name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"kibana_datatable"}

test/interpreter_functional/snapshots/baseline/step_output_test3.json

@@ -0,0 +1 @@
+{"as":"visualization","type":"render","value":{"params":{"listenOnChange":true},"visConfig":{"dimensions":{"bucket":{"accessor":0},"metrics":[{"accessor":1,"format":{"id":"number"},"params":{}}]}},"visData":{"columns":[{"id":"col-0-2","name":"response.raw: Descending"},{"id":"col-1-1","name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"kibana_datatable"},"visType":"metric"}}

test/interpreter_functional/test_suites/run_pipeline/basic.js

@@ -0,0 +1,112 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import expect from 'expect.js';
+import { expectExpressionProvider } from './helpers';
+
+// this file showcases how to use testing utilities defined in helpers.js together with the kbn_tp_run_pipeline
+// test plugin to write autmated tests for interprete
+export default function ({ getService, updateBaselines }) {
+
+  let expectExpression;
+  describe('basic visualize loader pipeline expression tests', () => {
+    before(() => {
+      expectExpression = expectExpressionProvider({ getService, updateBaselines });
+    });
+
+    // we should not use this for tests like the ones below. this should be unit tested.
+    // - tests against a single function could easily be written as unit tests (and should be)
+    describe('kibana function', () => {
+      it('returns kibana_context', async () => {
+        const result = await expectExpression('returns_kibana_context', 'kibana').getResponse();
+        expect(result).to.have.property('type', 'kibana_context');
+      });
+
+      it('correctly sets timeRange', async () => {
+        const result = await expectExpression('correctly_sets_timerange', 'kibana', {}, { timeRange: 'test' }).getResponse();
+        expect(result).to.have.property('timeRange', 'test');
+      });
+    });
+
+    // rather we want to use this to do integration tests.
+    describe('full expression', () => {
+      const expression = `kibana | kibana_context | esaggs index='logstash-*' aggConfigs='[
+          {"id":"1","enabled":true,"type":"count","schema":"metric","params":{}},
+          {"id":"2","enabled":true,"type":"terms","schema":"segment","params":
+            {"field":"response.raw","size":4,"order":"desc","orderBy":"1"}
+          }]'  | 
+        kibana_metric 
+          visConfig='{"dimensions":{"metrics":[{"accessor":1,"format":{"id":"number"},"params":{}}],"bucket":{"accessor":0}}}'
+      `;
+
+      // we can execute an expression and validate the result manually:
+      // const response = await expectExpression(expression).getResponse();
+      // expect(response)....
+
+      // we can also do snapshot comparison of result of our expression
+      // to update the snapshots run the tests with --updateBaselines
+      it ('runs the expression and compares final output', async () => {
+        await expectExpression('final_output_test', expression).toMatchSnapshot();
+      });
+
+      // its also possible to check snapshot at every step of expression (after execution of each function)
+      it ('runs the expression and compares output at every step', async () => {
+        await expectExpression('step_output_test', expression).steps.toMatchSnapshot();
+      });
+
+      // and we can do screenshot comparison of the rendered output of expression (if expression returns renderable)
+      it ('runs the expression and compares screenshots', async () => {
+        await expectExpression('final_screenshot_test', expression).toMatchScreenshot();
+      });
+
+      // it is also possible to combine different checks
+      it('runs the expression and combines different checks', async () => {
+        await (await expectExpression('combined_test', expression).steps.toMatchSnapshot()).toMatchScreenshot();
+      });
+    });
+
+    // if we want to do multiple different tests using the same data, or reusing a part of expression its
+    // possible to retrieve the intermediate result and reuse it in later expressions
+    describe('reusing partial results', () => {
+      it ('does some screenshot comparisons', async () => {
+        const expression = `kibana | kibana_context | esaggs index='logstash-*' aggConfigs='[
+          {"id":"1","enabled":true,"type":"count","schema":"metric","params":{}},
+          {"id":"2","enabled":true,"type":"terms","schema":"segment","params":
+            {"field":"response.raw","size":4,"order":"desc","orderBy":"1"}
+          }]'`;
+        // we execute the part of expression that fetches the data and store its response
+        const context = await expectExpression('partial_test', expression).getResponse();
+
+        // we reuse that response to render 3 different charts and compare screenshots with baselines
+        const tagCloudExpr =
+          `tagcloud visConfig='{"metric":{"accessor":1,"format":{"id":"number"}},"bucket":{"accessor":0}}'`;
+        await expectExpression('partial_test_1', tagCloudExpr, context).toMatchScreenshot();
+
+        const metricExpr =
+          `kibana_metric 
+          visConfig='{"dimensions":{"metrics":[{"accessor":1,"format":{"id":"number"}}],"bucket":{"accessor":0}}}'`;
+        await expectExpression('partial_test_2', metricExpr, context).toMatchScreenshot();
+
+        const regionMapExpr =
+          `regionmap visConfig='{"metric":{"accessor":1,"format":{"id":"number"}},"bucket":{"accessor":0}}'`;
+        await expectExpression('partial_test_3', regionMapExpr, context).toMatchScreenshot();
+      });
+    });
+  });
+}

test/interpreter_functional/test_suites/run_pipeline/helpers.js

@@ -0,0 +1,127 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import expect from 'expect.js';
+
+// helper for testing interpreter expressions
+export const expectExpressionProvider = ({ getService, updateBaselines }) => {
+  const browser = getService('browser');
+  const screenshot = getService('screenshots');
+  const snapshots = getService('snapshots');
+  const log = getService('log');
+  const testSubjects = getService('testSubjects');
+  /**
+   * returns a handler object to test a given expression
+   * @name: name of the test
+   * @expression: expression to execute
+   * @context: context provided to the expression
+   * @initialContext: initialContext provided to the expression
+   * @returns handler object
+   */
+  return (name, expression, context = {}, initialContext = {}) => {
+    log.debug(`executing expression ${expression}`);
+    const steps = expression.split('|'); // todo: we should actually use interpreter parser and get the ast
+    let responsePromise;
+
+    const handler = {
+      /**
+       * checks if provided object matches expression result
+       * @param result: expected expression result
+       * @returns {Promise<void>}
+       */
+      toReturn: async result => {
+        const pipelineResponse = await handler.getResponse();
+        expect(pipelineResponse).to.eql(result);
+      },
+      /**
+       * returns expression response
+       * @returns {*}
+       */
+      getResponse: () => {
+        if (!responsePromise) responsePromise = handler.runExpression();
+        return responsePromise;
+      },
+      /**
+       * runs the expression and returns the result
+       * @param step: expression to execute
+       * @param stepContext: context to provide to expression
+       * @returns {Promise<*>} result of running expression
+       */
+      runExpression: async (step, stepContext) => {
+        log.debug(`running expression ${step || expression}`);
+        const promise = browser.executeAsync((expression, context, initialContext, done) => {
+          window.runPipeline(expression, context, initialContext).then(result => {
+            done(result);
+          });
+        }, step || expression, stepContext || context, initialContext);
+        return await promise;
+      },
+      steps: {
+        /**
+         * does a snapshot comparison between result of every function in the expression and the baseline
+         * @returns {Promise<void>}
+         */
+        toMatchSnapshot: async () => {
+          let lastResponse;
+          for (let i = 0; i < steps.length; i++) {
+            const step = steps[i];
+            lastResponse = await handler.runExpression(step, lastResponse);
+            const diff = await snapshots.compareAgainstBaseline(name + i, lastResponse, updateBaselines);
+            expect(diff).to.be.lessThan(0.05);
+          }
+          if (!responsePromise) {
+            responsePromise = new Promise(resolve => {
+              resolve(lastResponse);
+            });
+          }
+          return handler;
+        },
+      },
+      /**
+       * does a snapshot comparison between result of running the expression and baseline
+       * @returns {Promise<void>}
+       */
+      toMatchSnapshot: async () => {
+        const pipelineResponse = await handler.getResponse();
+        await snapshots.compareAgainstBaseline(name, pipelineResponse, updateBaselines);
+        return handler;
+      },
+      /**
+       * does a screenshot comparison between result of rendering expression and baseline
+       * @returns {Promise<void>}
+       */
+      toMatchScreenshot: async () => {
+        const pipelineResponse = await handler.getResponse();
+        const result = await browser.executeAsync((context, done) => {
+          window.renderPipelineResponse(context).then(result => {
+            done(result);
+          });
+        }, pipelineResponse);
+        log.debug('response of rendering: ', result);
+
+        const chartEl = await testSubjects.find('pluginChart');
+        const percentDifference = await screenshot.compareAgainstBaseline(name, updateBaselines, chartEl);
+        expect(percentDifference).to.be.lessThan(0.1);
+        return handler;
+      }
+    };
+
+    return handler;
+  };
+};

test/interpreter_functional/test_suites/run_pipeline/index.js

@@ -0,0 +1,41 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export default function ({ getService, getPageObjects, loadTestFile }) {
+  const browser = getService('browser');
+  const esArchiver = getService('esArchiver');
+  const kibanaServer = getService('kibanaServer');
+  const appsMenu = getService('appsMenu');
+  const testSubjects = getService('testSubjects');
+  const PageObjects = getPageObjects(['common', 'header']);
+
+  describe('runPipeline', function () {
+    before(async () => {
+      await esArchiver.loadIfNeeded('../functional/fixtures/es_archiver/logstash_functional');
+      await esArchiver.load('../functional/fixtures/es_archiver/visualize_embedding');
+      await kibanaServer.uiSettings.replace({ 'dateFormat:tz': 'Australia/North', 'defaultIndex': 'logstash-*' });
+      await browser.setWindowSize(1300, 900);
+      await PageObjects.common.navigateToApp('settings');
+      await appsMenu.clickLink('Run Pipeline');
+      await testSubjects.find('pluginContent');
+    });
+
+    loadTestFile(require.resolve('./basic'));
+  });
+}

test/scripts/jenkins_ci_group.sh

@@ -23,4 +23,5 @@ export TEST_BROWSER_HEADLESS=1
 
 if [ "$CI_GROUP" == "1" ]; then
   "$(FORCE_COLOR=0 yarn bin)/grunt" run:pluginFunctionalTestsRelease;
+  "$(FORCE_COLOR=0 yarn bin)/grunt" run:interpreterFunctionalTestsRelease;
 fi

yarn.lock

@@ -18121,6 +18121,11 @@ pngjs@3.3.1, pngjs@^3.0.0:
   resolved "https://registry.yarnpkg.com/pngjs/-/pngjs-3.3.1.tgz#8e14e6679ee7424b544334c3b2d21cea6d8c209a"
   integrity sha512-ggXCTsqHRIsGMkHlCEhbHhUmNTA2r1lpkE0NL4Q9S8spkXbm4vE9TVmPso2AGYn90Gltdz8W5CyzhcIGg2Gejg==
 
+pngjs@^3.4.0:
+  version "3.4.0"
+  resolved "https://registry.yarnpkg.com/pngjs/-/pngjs-3.4.0.tgz#99ca7d725965fb655814eaf65f38f12bbdbf555f"
+  integrity sha512-NCrCHhWmnQklfH4MtJMRjZ2a8c80qXeMlQMv2uVp9ISJMTt562SbGd6n2oq0PaPgKm7Z6pL9E2UlLIhC+SHL3w==
+
 podium@3.x.x:
   version "3.1.2"
   resolved "https://registry.yarnpkg.com/podium/-/podium-3.1.2.tgz#b701429739cf6bdde6b3015ae6b48d400817ce9e"