Math Aggregation to support Sibling Aggs for TSVB (#13681)

* Math Aggregation to support Sibling Aggs

* Fixing tests

* Deal with ResultSets

* including all var under params; updating docs

* Fixing bugs asscociated with blank Math aggs

* Adding aria props to elements

* Fixing aria labels; changing calculation label to bucket sript

* changing input to textarea

* Add compatability with percentiles; fix vars to be compatible with percentiles

* Adding comments

* Fixing typo in note

package.json

@@ -150,6 +150,7 @@
     "lodash": "3.10.1",
     "lru-cache": "4.1.1",
     "markdown-it": "8.3.2",
+    "mathjs": "3.16.2",
     "minimatch": "2.0.10",
     "mkdirp": "0.5.1",
     "moment": "2.13.0",

src/core_plugins/metrics/common/__tests__/agg_lookup.js

@@ -16,7 +16,7 @@ describe('aggLookup', () => {
 
     it('returns options for all aggs', () => {
       const options = createOptions();
-      expect(options).to.have.length(28);
+      expect(options).to.have.length(29);
       options.forEach((option) => {
         expect(option).to.have.property('label');
         expect(option).to.have.property('value');
@@ -32,13 +32,13 @@ describe('aggLookup', () => {
 
     it('returns options for pipeline', () => {
       const options = createOptions('pipeline');
-      expect(options).to.have.length(14);
+      expect(options).to.have.length(15);
       expect(options.every(opt => !isBasicAgg({ type: opt.value }))).to.equal(true);
     });
 
     it('returns options for all if given unknown key', () => {
       const options = createOptions('foo');
-      expect(options).to.have.length(28);
+      expect(options).to.have.length(29);
     });
 
   });

src/core_plugins/metrics/common/__tests__/calculate_label.js

@@ -15,7 +15,7 @@ describe('calculateLabel(metric, metrics)', () => {
   });
 
   it('returns "Calcuation" for a bucket script metric', () => {
-    expect(calculateLabel({ type: 'calculation' })).to.equal('Calculation');
+    expect(calculateLabel({ type: 'calculation' })).to.equal('Bucket Script');
   });
 
   it('returns formated label for series_agg', () => {

src/core_plugins/metrics/common/agg_lookup.js

@@ -1,7 +1,7 @@
 import _ from 'lodash';
 const lookup = {
   'count': 'Count',
-  'calculation': 'Calculation',
+  'calculation': 'Bucket Script',
   'std_deviation': 'Std. Deviation',
   'variance': 'Variance',
   'sum_of_squares': 'Sum of Sq.',
@@ -24,6 +24,7 @@ const lookup = {
   'sum_of_squares_bucket': 'Overall Sum of Sq.',
   'std_deviation_bucket': 'Overall Std. Deviation',
   'series_agg': 'Series Agg',
+  'math': 'Math',
   'serial_diff': 'Serial Difference',
   'filter_ratio': 'Filter Ratio',
   'positive_only': 'Positive Only',
@@ -43,6 +44,7 @@ const pipeline = [
   'sum_of_squares_bucket',
   'std_deviation_bucket',
   'series_agg',
+  'math',
   'serial_diff',
   'positive_only'
 ];

src/core_plugins/metrics/common/calculate_label.js

@@ -19,7 +19,8 @@ export default function calculateLabel(metric, metrics) {
   if (metric.alias) return metric.alias;
 
   if (metric.type === 'count') return 'Count';
-  if (metric.type === 'calculation') return 'Calculation';
+  if (metric.type === 'calculation') return 'Bucket Script';
+  if (metric.type === 'math') return 'Math';
   if (metric.type === 'series_agg') return `Series Agg (${metric.function})`;
   if (metric.type === 'filter_ratio') return 'Filter Ratio';
   if (metric.type === 'static') return `Static Value of ${metric.value}`;

src/core_plugins/metrics/public/components/aggs/agg_select.js

@@ -20,13 +20,12 @@ const metricAggs = [
 ];
 
 const pipelineAggs = [
-  { label: 'Calculation', value: 'calculation' },
+  { label: 'Bucket Script', value: 'calculation' },
   { label: 'Cumulative Sum', value: 'cumulative_sum' },
   { label: 'Derivative', value: 'derivative' },
   { label: 'Moving Average', value: 'moving_average' },
   { label: 'Positive Only', value: 'positive_only' },
   { label: 'Serial Difference', value: 'serial_diff' },
-  { label: 'Series Agg', value: 'series_agg' }
 ];
 
 const siblingAggs = [
@@ -39,6 +38,11 @@ const siblingAggs = [
   { label: 'Overall Variance', value: 'variance_bucket' }
 ];
 
+const specialAggs = [
+  { label: 'Series Agg', value: 'series_agg' },
+  { label: 'Math', value: 'math' }
+];
+
 class AggSelectOption extends Component {
 
   constructor(props) {
@@ -144,7 +148,9 @@ function AggSelect(props) {
       { label: 'Parent Pipeline Aggregations', value: null, pipeline: true, heading: true, disabled: true },
       ...pipelineAggs.filter(filterByPanelType(panelType)).map(agg => ({ ...agg, disabled: !enablePipelines })),
       { label: 'Sibling Pipeline Aggregations', value: null, pipeline: true, heading: true, disabled: true },
-      ...siblingAggs.map(agg => ({ ...agg, disabled: !enablePipelines }))
+      ...siblingAggs.map(agg => ({ ...agg, disabled: !enablePipelines })),
+      { label: 'Special Aggregations', value: null, pipeline: true, heading: true, disabled: true },
+      ...specialAggs.map(agg => ({ ...agg, disabled: !enablePipelines }))
     ];
   }
 

src/core_plugins/metrics/public/components/aggs/math.js

@@ -0,0 +1,96 @@
+import React, { Component, PropTypes } from 'react';
+import _ from 'lodash';
+import uuid from 'uuid';
+import AggRow from './agg_row';
+import AggSelect from './agg_select';
+
+import createChangeHandler from '../lib/create_change_handler';
+import createSelectHandler from '../lib/create_select_handler';
+import createTextHandler from '../lib/create_text_handler';
+import Vars from './vars';
+
+class MathAgg extends Component {
+
+  componentWillMount() {
+    if (!this.props.model.variables) {
+      this.props.onChange(_.assign({}, this.props.model, {
+        variables: [{ id: uuid.v1() }]
+      }));
+    }
+  }
+
+  render() {
+    const { siblings } = this.props;
+
+    const defaults = { script: '' };
+    const model = { ...defaults, ...this.props.model };
+
+    const handleChange = createChangeHandler(this.props.onChange, model);
+    const handleSelectChange = createSelectHandler(handleChange);
+    const handleTextChange = createTextHandler(handleChange);
+
+    return (
+      <AggRow
+        disableDelete={this.props.disableDelete}
+        model={this.props.model}
+        onAdd={this.props.onAdd}
+        onDelete={this.props.onDelete}
+        siblings={this.props.siblings}
+      >
+        <div className="vis_editor__row_item">
+          <div>
+            <div className="vis_editor__label">Aggregation</div>
+            <AggSelect
+              siblings={this.props.siblings}
+              value={model.type}
+              onChange={handleSelectChange('type')}
+            />
+            <div className="vis_editor__variables">
+              <div className="vis_editor__label">Variables</div>
+              <Vars
+                metrics={siblings}
+                onChange={handleChange}
+                name="variables"
+                model={model}
+                includeSiblings={true}
+              />
+            </div>
+            <div className="vis_editor__row_item">
+              <label className="vis_editor__label" htmlFor="mathExpressionInput">Expression</label>
+              <textarea
+                id="mathExpressionInput"
+                aria-describedby="mathExpressionDescription"
+                className="vis_editor__input-grows-100"
+                onChange={handleTextChange('script')}
+              >
+                {model.script}
+              </textarea>
+              <div className="vis_editor__note" id="mathExpressionDescription">
+                This field uses basic math expresions (see <a href="http://mathjs.org/docs/expressions/syntax.html" target="_blank">MathJS</a>) - Variables
+                are keys on the <code>params</code> object, i.e. <code>params.&lt;name&gt;</code> To access all the data use
+                <code>params._all.&lt;name&gt;.values</code> for an array of the values and <code>params._all.&lt;name&gt;.timestamps</code>
+                for an array of the timestamps.  <code>params._timestamp</code> is available for the current bucket&apos;s timestamp
+                and <code>params._index</code> is available for the current bucket&apos;s index.
+              </div>
+            </div>
+          </div>
+        </div>
+      </AggRow>
+    );
+  }
+
+}
+
+MathAgg.propTypes = {
+  disableDelete: PropTypes.bool,
+  fields: PropTypes.object,
+  model: PropTypes.object,
+  onAdd: PropTypes.func,
+  onChange: PropTypes.func,
+  onDelete: PropTypes.func,
+  panel: PropTypes.object,
+  series: PropTypes.object,
+  siblings: PropTypes.array,
+};
+
+export default MathAgg;

src/core_plugins/metrics/public/components/aggs/metric_select.js

@@ -20,10 +20,14 @@ function createTypeFilter(restrict, exclude) {
 
 
 // This filters out sibling aggs, percentiles, and special aggs (like Series Agg)
-export function filterRows(row) {
-  return  !/_bucket$/.test(row.type)
-    && !/^series/.test(row.type)
-    && !/^percentile/.test(row.type);
+export function filterRows(includeSiblings) {
+  return row => {
+    if (includeSiblings) return !/^series/.test(row.type) && !/^percentile/.test(row.type) && row.type !== 'math';
+    return  !/_bucket$/.test(row.type)
+      && !/^series/.test(row.type)
+      && !/^percentile/.test(row.type)
+      && row.type !== 'math';
+  };
 }
 
 function MetricSelect(props) {
@@ -32,7 +36,8 @@ function MetricSelect(props) {
     metric,
     onChange,
     value,
-    exclude
+    exclude,
+    includeSiblings
   } = props;
 
   const metrics = props.metrics
@@ -58,7 +63,7 @@ function MetricSelect(props) {
 
 
   const options = siblings
-    .filter(filterRows)
+    .filter(filterRows(includeSiblings))
     .map(row => {
       const label = calculateLabel(row, metrics);
       return { value: row.id, label };
@@ -80,6 +85,7 @@ MetricSelect.defaultProps = {
   exclude: [],
   metric: {},
   restrict: 'none',
+  includeSiblings: false
 };
 
 MetricSelect.propTypes = {
@@ -88,7 +94,8 @@ MetricSelect.propTypes = {
   metric: PropTypes.object,
   onChange: PropTypes.func,
   restrict: PropTypes.string,
-  value: PropTypes.string
+  value: PropTypes.string,
+  includeSiblings: PropTypes.bool
 };
 
 export default MetricSelect;

src/core_plugins/metrics/public/components/aggs/vars.js

@@ -39,10 +39,10 @@ class CalculationVars extends Component {
         <div className="vis_editor__calc_vars-var">
           <MetricSelect
             onChange={this.handleChange(row, 'field')}
-            exclude={['percentile']}
             metrics={this.props.metrics}
             metric={this.props.model}
             value={row.field}
+            includeSiblings={this.props.includeSiblings}
           />
         </div>
         <div className="vis_editor__calc_vars-control">
@@ -70,14 +70,16 @@ class CalculationVars extends Component {
 }
 
 CalculationVars.defaultProps = {
-  name: 'variables'
+  name: 'variables',
+  includeSiblings: false
 };
 
 CalculationVars.propTypes = {
   metrics: PropTypes.array,
   model: PropTypes.object,
   name: PropTypes.string,
-  onChange: PropTypes.func
+  onChange: PropTypes.func,
+  includeSiblings: PropTypes.bool
 };
 
 export default CalculationVars;

src/core_plugins/metrics/public/components/lib/agg_to_component.js

@@ -12,6 +12,7 @@ import { PositiveOnlyAgg } from '../aggs/positive_only';
 import { FilterRatioAgg } from '../aggs/filter_ratio';
 import { PercentileRankAgg } from '../aggs/percentile_rank';
 import { Static } from '../aggs/static';
+import MathAgg from '../aggs/math';
 export default {
   count: StdAgg,
   avg: StdAgg,
@@ -40,7 +41,8 @@ export default {
   serial_diff: SerialDiffAgg,
   filter_ratio: FilterRatioAgg,
   positive_only: PositiveOnlyAgg,
-  static: Static
+  static: Static,
+  math: MathAgg
 };
 
 

src/core_plugins/metrics/public/less/editor.less

@@ -58,6 +58,11 @@
     margin: 0 10px 0 0;
   }
 }
+
+.vis_editor__note {
+  .vis_editor__label;
+  font-style: italic;
+}
 .vis_editor__input {
   padding: 8px 10px;
   border-radius: @borderRadius;
@@ -322,6 +327,7 @@
   margin-bottom: 2px;
   padding: 10px;
   align-items: center;
+  .vis_editor__note,
   .vis_editor__label {
     margin-bottom: 5px;
     font-size: 12px;

src/core_plugins/metrics/server/lib/vis_data/response_processors/series/index.js

@@ -6,6 +6,7 @@ import stdMetric from './std_metric';
 import stdSibling from './std_sibling';
 import timeShift from './time_shift';
 import { dropLastBucket } from './drop_last_bucket';
+import { mathAgg } from './math';
 
 export default [
   percentile,
@@ -13,6 +14,7 @@ export default [
   stdDeviationSibling,
   stdMetric,
   stdSibling,
+  mathAgg,
   seriesAgg,
   timeShift,
   dropLastBucket

src/core_plugins/metrics/server/lib/vis_data/response_processors/series/math.js

@@ -0,0 +1,102 @@
+const percentileValueMatch = /\[([0-9\.]+)\]$/;
+import { startsWith, flatten, values, first, last } from 'lodash';
+import getDefaultDecoration from '../../helpers/get_default_decoration';
+import getSiblingAggValue from '../../helpers/get_sibling_agg_value';
+import getSplits from '../../helpers/get_splits';
+import mapBucket from '../../helpers/map_bucket';
+import mathjs from 'mathjs';
+
+const limitedEval = mathjs.eval;
+mathjs.import({
+  'import':     function () { throw new Error('Function import is not allowed in your expression.'); },
+  'createUnit': function () { throw new Error('Function createUnit is not allowed in your expression.'); },
+  'eval':       function () { throw new Error('Function eval is not allowed in your expression.'); },
+  'parse':      function () { throw new Error('Function parse is not allowed in your expression.'); },
+  'simplify':   function () { throw new Error('Function simplify is not allowed in your expression.'); },
+  'derivative': function () { throw new Error('Function derivative is not allowed in your expression.'); }
+}, { override: true });
+
+export function mathAgg(resp, panel, series) {
+  return next => results => {
+    const mathMetric = last(series.metrics);
+    if (mathMetric.type !== 'math') return next(results);
+    // Filter the results down to only the ones that match the series.id. Sometimes
+    // there will be data from other series mixed in.
+    results = results.filter(s => {
+      if (s.id.split(/:/)[0] === series.id) {
+        return false;
+      }
+      return true;
+    });
+    const decoration = getDefaultDecoration(series);
+    const splits = getSplits(resp, panel, series);
+    const mathSeries = splits.map((split) => {
+      if (mathMetric.variables.length) {
+        // Gather the data for the splits. The data will either be a sibling agg or
+        // a standard metric/pipeline agg
+        const splitData = mathMetric.variables.reduce((acc, v) => {
+          const metric = series.metrics.find(m => startsWith(v.field, m.id));
+          if (!metric) return acc;
+          if (/_bucket$/.test(metric.type)) {
+            acc[v.name] = split.timeseries.buckets.map(bucket => {
+              return [bucket.key, getSiblingAggValue(split, metric)];
+            });
+          } else {
+            const percentileMatch = v.field.match(percentileValueMatch);
+            const m = percentileMatch ? { ...metric, percent: percentileMatch[1] } : { ...metric };
+            acc[v.name] = split.timeseries.buckets.map(mapBucket(m));
+          }
+          return acc;
+        }, {});
+        // Create an params._all so the users can access the entire series of data
+        // in the Math.js equation
+        const all = Object.keys(splitData).reduce((acc, key) => {
+          acc[key] = {
+            values: splitData[key].map(x => x[1]),
+            timestamps: splitData[key].map(x => x[0])
+          };
+          return acc;
+        }, {});
+        // Get the first var and check that it shows up in the split data otherwise
+        // we need to return an empty array for the data since we can't opperate
+        // without the first varaible
+        const firstVar = first(mathMetric.variables);
+        if (!splitData[firstVar.name]) {
+          return {
+            id: split.id,
+            label: split.label,
+            color: split.color,
+            data: [],
+            ...decoration
+          };
+        }
+        // Use the first var to collect all the timestamps
+        const timestamps = splitData[firstVar.name].map(r => first(r));
+        // Map the timestamps to actual data
+        const data = timestamps.map((ts, index) => {
+          const params = mathMetric.variables.reduce((acc, v) => {
+            acc[v.name] = last(splitData[v.name].find(row => row[0] === ts));
+            return acc;
+          }, {});
+          // If some of the values are null, return the timestamp and null, this is
+          // a safety check for the user
+          const someNull = values(params).some(v => v == null);
+          if (someNull) return [ts, null];
+          // calculate the result based on the user's script and return the value
+          const result = limitedEval(mathMetric.script, { params: { ...params, _index: index, _timestamp: ts, _all: all } });
+          // if the result is an object (usually when the user is working with maps and functions) flatten the results and return the last value.
+          if (typeof result === 'object') return [ts, last(flatten(result.valueOf()))];
+          return [ts, result];
+        });
+        return {
+          id: split.id,
+          label: split.label,
+          color: split.color,
+          data,
+          ...decoration
+        };
+      }
+    });
+    return next(results.concat(mathSeries));
+  };
+}

tasks/config/licenses.js

@@ -34,6 +34,7 @@ module.exports = function () {
         'Public domain',
         'Unlicense',
         'WTFPL OR ISC',
+        'MIT OR GPL-2.0',
         'WTFPL',
       ],
       overrides: {