github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 17:59:23 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Maps] add Top term aggregation (#57875)

Browse source 
* [Maps] add Top term aggregation

* update pew-pew source to handle terms agg

* make helper function for pulling values from bucket

* update terms source

* better join labels

* categoricla meta

* remove unused constant

* remove unused changes

* remove unused constant METRIC_SCHEMA_CONFIG

* update jest expect

* fix auto complete suggestions for top term

* get category autocomplete working with style props from joins

* pluck categorical style meta with real field name

* mock MetricsEditor to fix jest test

* review feedback

* es_agg_utils.js to es_agg_utils.ts

* typing updates

* use composit agg to avoid search.buckets limit

* i18n update and functional test fix

* stop paging through results when request is aborted

* remove unused file

* do not use composite agg when no terms sub-aggregations

* clean up

* pass indexPattern to getValueAggsDsl

* review feedback

* more review feedback

* ts-ignore for untyped imports in tests

* more review feedback

* add bucket.hasOwnProperty check

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
This commit is contained in:
Nathan Reese 2020-02-26 07:50:11 -07:00 • committed by GitHub
parent 2ea4bdfe0d
commit 3212754e62
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
35 changed files with 729 additions and 419 deletions
Download patch file Download diff file Expand all files Collapse all files

4
x-pack/legacy/plugins/maps/common/constants.ts
View file

@ -117,16 +117,16 @@ export const DRAW_TYPE = {
POLYGON: 'POLYGON',
};
export const METRIC_TYPE = {
export const AGG_TYPE = {
AVG: 'avg',
COUNT: 'count',
MAX: 'max',
MIN: 'min',
SUM: 'sum',
TERMS: 'terms',
UNIQUE_COUNT: 'cardinality',
};
export const COUNT_AGG_TYPE = METRIC_TYPE.COUNT;
export const COUNT_PROP_LABEL = i18n.translate('xpack.maps.aggs.defaultCountLabel', {
defaultMessage: 'count',
});

22
x-pack/legacy/plugins/maps/public/actions/map_actions.js
View file

@ -18,6 +18,7 @@ import {
getTransientLayerId,
getOpenTooltips,
getQuery,
getDataRequestDescriptor,
} from '../selectors/map_selectors';
import { FLYOUT_STATE } from '../reducers/ui';
import {
@ -76,7 +77,7 @@ export const HIDE_LAYER_CONTROL = 'HIDE_LAYER_CONTROL';
export const HIDE_VIEW_CONTROL = 'HIDE_VIEW_CONTROL';
export const SET_WAITING_FOR_READY_HIDDEN_LAYERS = 'SET_WAITING_FOR_READY_HIDDEN_LAYERS';
function getLayerLoadingCallbacks(dispatch, layerId) {
function getLayerLoadingCallbacks(dispatch, getState, layerId) {
return {
startLoading: (dataId, requestToken, meta) =>
dispatch(startDataLoad(layerId, dataId, requestToken, meta)),
@ -87,6 +88,13 @@ function getLayerLoadingCallbacks(dispatch, layerId) {
updateSourceData: newData => {
dispatch(updateSourceDataRequest(layerId, newData));
},
isRequestStillActive: (dataId, requestToken) => {
const dataRequest = getDataRequestDescriptor(getState(), layerId, dataId);
if (!dataRequest) {
return false;
}
return dataRequest.dataRequestToken === requestToken;
},
registerCancelCallback: (requestToken, callback) =>
dispatch(registerCancelCallback(requestToken, callback)),
};
@ -98,11 +106,11 @@ function getLayerById(layerId, state) {
});
}
async function syncDataForAllLayers(getState, dispatch, dataFilters) {
async function syncDataForAllLayers(dispatch, getState, dataFilters) {
const state = getState();
const layerList = getLayerList(state);
const syncs = layerList.map(layer => {
const loadingFunctions = getLayerLoadingCallbacks(dispatch, layer.getId());
const loadingFunctions = getLayerLoadingCallbacks(dispatch, getState, layer.getId());
return layer.syncData({ ...loadingFunctions, dataFilters });
});
await Promise.all(syncs);
@ -412,7 +420,7 @@ export function mapExtentChanged(newMapConstants) {
},
});
const newDataFilters = { ...dataFilters, ...newMapConstants };
await syncDataForAllLayers(getState, dispatch, newDataFilters);
await syncDataForAllLayers(dispatch, getState, newDataFilters);
};
}
@ -653,7 +661,7 @@ export function syncDataForLayer(layerId) {
const targetLayer = getLayerById(layerId, getState());
if (targetLayer) {
const dataFilters = getDataFilters(getState());
const loadingFunctions = getLayerLoadingCallbacks(dispatch, layerId);
const loadingFunctions = getLayerLoadingCallbacks(dispatch, getState, layerId);
await targetLayer.syncData({
...loadingFunctions,
dataFilters,
@ -773,7 +781,7 @@ export function setQuery({ query, timeFilters, filters = [], refresh = false })
});
const dataFilters = getDataFilters(getState());
await syncDataForAllLayers(getState, dispatch, dataFilters);
await syncDataForAllLayers(dispatch, getState, dataFilters);
};
}
@ -792,7 +800,7 @@ export function triggerRefreshTimer() {
});
const dataFilters = getDataFilters(getState());
await syncDataForAllLayers(getState, dispatch, dataFilters);
await syncDataForAllLayers(dispatch, getState, dataFilters);
};
}

13
x-pack/legacy/plugins/maps/public/components/metric_editor.js
View file

@ -12,17 +12,16 @@ import { EuiFieldText, EuiFormRow } from '@elastic/eui';
import { MetricSelect, METRIC_AGGREGATION_VALUES } from './metric_select';
import { SingleFieldSelect } from './single_field_select';
import { METRIC_TYPE } from '../../common/constants';
import { AGG_TYPE } from '../../common/constants';
import { getTermsFields } from '../index_pattern_util';
function filterFieldsForAgg(fields, aggType) {
if (!fields) {
return [];
}
if (aggType === METRIC_TYPE.UNIQUE_COUNT) {
return fields.filter(field => {
return field.aggregatable;
});
if (aggType === AGG_TYPE.UNIQUE_COUNT || aggType === AGG_TYPE.TERMS) {
return getTermsFields(fields);
}
return fields.filter(field => {
@ -38,7 +37,7 @@ export function MetricEditor({ fields, metricsFilter, metric, onChange, removeBu
};
// unset field when new agg type does not support currently selected field.
if (metric.field && metricAggregationType !== METRIC_TYPE.COUNT) {
if (metric.field && metricAggregationType !== AGG_TYPE.COUNT) {
const fieldsForNewAggType = filterFieldsForAgg(fields, metricAggregationType);
const found = fieldsForNewAggType.find(field => {
return field.name === metric.field;
@ -64,7 +63,7 @@ export function MetricEditor({ fields, metricsFilter, metric, onChange, removeBu
};
let fieldSelect;
if (metric.type && metric.type !== METRIC_TYPE.COUNT) {
if (metric.type && metric.type !== AGG_TYPE.COUNT) {
fieldSelect = (
<EuiFormRow
label={i18n.translate('xpack.maps.metricsEditor.selectFieldLabel', {

20
x-pack/legacy/plugins/maps/public/components/metric_select.js
View file

@ -8,44 +8,50 @@ import React from 'react';
import PropTypes from 'prop-types';
import { i18n } from '@kbn/i18n';
import { EuiComboBox } from '@elastic/eui';
import { METRIC_TYPE } from '../../common/constants';
import { AGG_TYPE } from '../../common/constants';
const AGG_OPTIONS = [
{
label: i18n.translate('xpack.maps.metricSelect.averageDropDownOptionLabel', {
defaultMessage: 'Average',
}),
value: METRIC_TYPE.AVG,
value: AGG_TYPE.AVG,
},
{
label: i18n.translate('xpack.maps.metricSelect.countDropDownOptionLabel', {
defaultMessage: 'Count',
}),
value: METRIC_TYPE.COUNT,
value: AGG_TYPE.COUNT,
},
{
label: i18n.translate('xpack.maps.metricSelect.maxDropDownOptionLabel', {
defaultMessage: 'Max',
}),
value: METRIC_TYPE.MAX,
value: AGG_TYPE.MAX,
},
{
label: i18n.translate('xpack.maps.metricSelect.minDropDownOptionLabel', {
defaultMessage: 'Min',
}),
value: METRIC_TYPE.MIN,
value: AGG_TYPE.MIN,
},
{
label: i18n.translate('xpack.maps.metricSelect.sumDropDownOptionLabel', {
defaultMessage: 'Sum',
}),
value: METRIC_TYPE.SUM,
value: AGG_TYPE.SUM,
},
{
label: i18n.translate('xpack.maps.metricSelect.termsDropDownOptionLabel', {
defaultMessage: 'Top term',
}),
value: AGG_TYPE.TERMS,
},
{
label: i18n.translate('xpack.maps.metricSelect.cardinalityDropDownOptionLabel', {
defaultMessage: 'Unique count',
}),
value: METRIC_TYPE.UNIQUE_COUNT,
value: AGG_TYPE.UNIQUE_COUNT,
},
];

4
x-pack/legacy/plugins/maps/public/components/metrics_editor.js
View file

@ -10,7 +10,7 @@ import { i18n } from '@kbn/i18n';
import { FormattedMessage } from '@kbn/i18n/react';
import { EuiButtonEmpty, EuiSpacer, EuiTextAlign } from '@elastic/eui';
import { MetricEditor } from './metric_editor';
import { METRIC_TYPE } from '../../common/constants';
import { AGG_TYPE } from '../../common/constants';
export function MetricsEditor({ fields, metrics, onChange, allowMultipleMetrics, metricsFilter }) {
function renderMetrics() {
@ -100,6 +100,6 @@ MetricsEditor.propTypes = {
};
MetricsEditor.defaultProps = {
metrics: [{ type: METRIC_TYPE.COUNT }],
metrics: [{ type: AGG_TYPE.COUNT }],
allowMultipleMetrics: true,
};

8
x-pack/legacy/plugins/maps/public/connected_components/layer_panel/join_editor/resources/metrics_expression.js
View file

@ -16,7 +16,7 @@ import {
} from '@elastic/eui';
import { MetricsEditor } from '../../../../components/metrics_editor';
import { FormattedMessage } from '@kbn/i18n/react';
import { METRIC_TYPE } from '../../../../../common/constants';
import { AGG_TYPE } from '../../../../../common/constants';
export class MetricsExpression extends Component {
state = {
@ -59,7 +59,7 @@ export class MetricsExpression extends Component {
render() {
const metricExpressions = this.props.metrics
.filter(({ type, field }) => {
if (type === METRIC_TYPE.COUNT) {
if (type === AGG_TYPE.COUNT) {
return true;
}
@ -70,7 +70,7 @@ export class MetricsExpression extends Component {
})
.map(({ type, field }) => {
// do not use metric label so field and aggregation are not obscured.
if (type === METRIC_TYPE.COUNT) {
if (type === AGG_TYPE.COUNT) {
return 'count';
}
@ -130,5 +130,5 @@ MetricsExpression.propTypes = {
};
MetricsExpression.defaultProps = {
metrics: [{ type: METRIC_TYPE.COUNT }],
metrics: [{ type: AGG_TYPE.COUNT }],
};

6
x-pack/legacy/plugins/maps/public/connected_components/layer_panel/join_editor/resources/metrics_expression.test.js
View file

@ -4,6 +4,12 @@
* you may not use this file except in compliance with the Elastic License.
*/
jest.mock('../../../../components/metrics_editor', () => ({
MetricsEditor: () => {
return <div>mockMetricsEditor</div>;
},
}));
import React from 'react';
import { shallow } from 'enzyme';
import { MetricsExpression } from './metrics_expression';

15
x-pack/legacy/plugins/maps/public/connected_components/map/mb/view.js
View file

@ -23,6 +23,7 @@ import sprites1 from '@elastic/maki/dist/sprite@1.png';
import sprites2 from '@elastic/maki/dist/sprite@2.png';
import { DrawControl } from './draw_control';
import { TooltipControl } from './tooltip_control';
import { clampToLatBounds, clampToLonBounds } from '../../../elasticsearch_geo_utils';
mapboxgl.workerUrl = mbWorkerUrl;
mapboxgl.setRTLTextPlugin(mbRtlPlugin);
@ -234,12 +235,12 @@ export class MBMapContainer extends React.Component {
//clamping ot -89/89 latitudes since Mapboxgl does not seem to handle bounds that contain the poles (logs errors to the console when using -90/90)
const lnLatBounds = new mapboxgl.LngLatBounds(
new mapboxgl.LngLat(
clamp(goto.bounds.min_lon, -180, 180),
clamp(goto.bounds.min_lat, -89, 89)
clampToLonBounds(goto.bounds.min_lon),
clampToLatBounds(goto.bounds.min_lat)
),
new mapboxgl.LngLat(
clamp(goto.bounds.max_lon, -180, 180),
clamp(goto.bounds.max_lat, -89, 89)
clampToLonBounds(goto.bounds.max_lon),
clampToLatBounds(goto.bounds.max_lat)
)
);
//maxZoom ensure we're not zooming in too far on single points or small shapes
@ -306,9 +307,3 @@ export class MBMapContainer extends React.Component {
);
}
}
function clamp(val, min, max) {
if (val > max) val = max;
else if (val < min) val = min;
return val;
}

18
x-pack/legacy/plugins/maps/public/elasticsearch_geo_utils.js
View file

@ -433,3 +433,21 @@ export function convertMapExtentToPolygon({ maxLat, maxLon, minLat, minLon }) {
return formatEnvelopeAsPolygon({ maxLat, maxLon, minLat, minLon });
}
export function clampToLatBounds(lat) {
return clamp(lat, -89, 89);
}
export function clampToLonBounds(lon) {
return clamp(lon, -180, 180);
}
export function clamp(val, min, max) {
if (val > max) {
return max;
} else if (val < min) {
return min;
} else {
return val;
}
}

33
x-pack/legacy/plugins/maps/public/layers/fields/es_agg_field.js
View file

@ -5,9 +5,10 @@
*/
import { AbstractField } from './field';
import { COUNT_AGG_TYPE } from '../../../common/constants';
import { AGG_TYPE } from '../../../common/constants';
import { isMetricCountable } from '../util/is_metric_countable';
import { ESAggMetricTooltipProperty } from '../tooltips/es_aggmetric_tooltip_property';
import { getField, addFieldToDSL } from '../util/es_agg_utils';
export class ESAggMetricField extends AbstractField {
static type = 'ES_AGG';
@ -34,12 +35,11 @@ export class ESAggMetricField extends AbstractField {
}
isValid() {
return this.getAggType() === COUNT_AGG_TYPE ? true : !!this._esDocField;
return this.getAggType() === AGG_TYPE.COUNT ? true : !!this._esDocField;
}
async getDataType() {
// aggregations only provide numerical data
return 'number';
return this.getAggType() === AGG_TYPE.TERMS ? 'string' : 'number';
}
getESDocFieldName() {
@ -47,9 +47,9 @@ export class ESAggMetricField extends AbstractField {
}
getRequestDescription() {
return this.getAggType() !== COUNT_AGG_TYPE
return this.getAggType() !== AGG_TYPE.COUNT
? `${this.getAggType()} ${this.getESDocFieldName()}`
: COUNT_AGG_TYPE;
: AGG_TYPE.COUNT;
}
async createTooltipProperty(value) {
@ -63,18 +63,13 @@ export class ESAggMetricField extends AbstractField {
);
}
makeMetricAggConfig() {
const metricAggConfig = {
id: this.getName(),
enabled: true,
type: this.getAggType(),
schema: 'metric',
params: {},
getValueAggDsl(indexPattern) {
const field = getField(indexPattern, this.getESDocFieldName());
const aggType = this.getAggType();
const aggBody = aggType === AGG_TYPE.TERMS ? { size: 1, shard_size: 1 } : {};
return {
[aggType]: addFieldToDSL(aggBody, field),
};
if (this.getAggType() !== COUNT_AGG_TYPE) {
metricAggConfig.params = { field: this.getESDocFieldName() };
}
return metricAggConfig;
}
supportsFieldMeta() {
@ -85,4 +80,8 @@ export class ESAggMetricField extends AbstractField {
async getOrdinalFieldMetaRequest(config) {
return this._esDocField.getOrdinalFieldMetaRequest(config);
}
async getCategoricalFieldMetaRequest() {
return this._esDocField.getCategoricalFieldMetaRequest();
}
}

14
x-pack/legacy/plugins/maps/public/layers/fields/es_agg_field.test.js
View file

@ -5,24 +5,24 @@
*/
import { ESAggMetricField } from './es_agg_field';
import { METRIC_TYPE } from '../../../common/constants';
import { AGG_TYPE } from '../../../common/constants';
describe('supportsFieldMeta', () => {
test('Non-counting aggregations should support field meta', () => {
const avgMetric = new ESAggMetricField({ aggType: METRIC_TYPE.AVG });
const avgMetric = new ESAggMetricField({ aggType: AGG_TYPE.AVG });
expect(avgMetric.supportsFieldMeta()).toBe(true);
const maxMetric = new ESAggMetricField({ aggType: METRIC_TYPE.MAX });
const maxMetric = new ESAggMetricField({ aggType: AGG_TYPE.MAX });
expect(maxMetric.supportsFieldMeta()).toBe(true);
const minMetric = new ESAggMetricField({ aggType: METRIC_TYPE.MIN });
const minMetric = new ESAggMetricField({ aggType: AGG_TYPE.MIN });
expect(minMetric.supportsFieldMeta()).toBe(true);
});
test('Counting aggregations should not support field meta', () => {
const countMetric = new ESAggMetricField({ aggType: METRIC_TYPE.COUNT });
const countMetric = new ESAggMetricField({ aggType: AGG_TYPE.COUNT });
expect(countMetric.supportsFieldMeta()).toBe(false);
const sumMetric = new ESAggMetricField({ aggType: METRIC_TYPE.SUM });
const sumMetric = new ESAggMetricField({ aggType: AGG_TYPE.SUM });
expect(sumMetric.supportsFieldMeta()).toBe(false);
const uniqueCountMetric = new ESAggMetricField({ aggType: METRIC_TYPE.UNIQUE_COUNT });
const uniqueCountMetric = new ESAggMetricField({ aggType: AGG_TYPE.UNIQUE_COUNT });
expect(uniqueCountMetric.supportsFieldMeta()).toBe(false);
});
});

38
x-pack/legacy/plugins/maps/public/layers/sources/es_agg_source.js
View file

@ -8,8 +8,7 @@ import { AbstractESSource } from './es_source';
import { ESAggMetricField } from '../fields/es_agg_field';
import { ESDocField } from '../fields/es_doc_field';
import {
METRIC_TYPE,
COUNT_AGG_TYPE,
AGG_TYPE,
COUNT_PROP_LABEL,
COUNT_PROP_NAME,
FIELD_ORIGIN,
@ -18,23 +17,6 @@ import {
export const AGG_DELIMITER = '_of_';
export class AbstractESAggSource extends AbstractESSource {
static METRIC_SCHEMA_CONFIG = {
group: 'metrics',
name: 'metric',
title: 'Value',
min: 1,
max: Infinity,
aggFilter: [
METRIC_TYPE.AVG,
METRIC_TYPE.COUNT,
METRIC_TYPE.MAX,
METRIC_TYPE.MIN,
METRIC_TYPE.SUM,
METRIC_TYPE.UNIQUE_COUNT,
],
defaults: [{ schema: 'metric', type: METRIC_TYPE.COUNT }],
};
constructor(descriptor, inspectorAdapters) {
super(descriptor, inspectorAdapters);
this._metricFields = this._descriptor.metrics
@ -81,7 +63,7 @@ export class AbstractESAggSource extends AbstractESSource {
if (metrics.length === 0) {
metrics.push(
new ESAggMetricField({
aggType: COUNT_AGG_TYPE,
aggType: AGG_TYPE.COUNT,
source: this,
origin: this.getOriginForField(),
})
@ -91,15 +73,23 @@ export class AbstractESAggSource extends AbstractESSource {
}
formatMetricKey(aggType, fieldName) {
return aggType !== COUNT_AGG_TYPE ? `${aggType}${AGG_DELIMITER}${fieldName}` : COUNT_PROP_NAME;
return aggType !== AGG_TYPE.COUNT ? `${aggType}${AGG_DELIMITER}${fieldName}` : COUNT_PROP_NAME;
}
formatMetricLabel(aggType, fieldName) {
return aggType !== COUNT_AGG_TYPE ? `${aggType} of ${fieldName}` : COUNT_PROP_LABEL;
return aggType !== AGG_TYPE.COUNT ? `${aggType} of ${fieldName}` : COUNT_PROP_LABEL;
}
createMetricAggConfigs() {
return this.getMetricFields().map(esAggMetric => esAggMetric.makeMetricAggConfig());
getValueAggsDsl(indexPattern) {
const valueAggsDsl = {};
this.getMetricFields()
.filter(esAggMetric => {
return esAggMetric.getAggType() !== AGG_TYPE.COUNT;
})
.forEach(esAggMetric => {
valueAggsDsl[esAggMetric.getName()] = esAggMetric.getValueAggDsl(indexPattern);
});
return valueAggsDsl;
}
async getNumberFields() {

95
x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.js
View file

@ -4,68 +4,63 @@
* you may not use this file except in compliance with the Elastic License.
*/
import _ from 'lodash';
import { RENDER_AS } from './render_as';
import { getTileBoundingBox } from './geo_tile_utils';
import { EMPTY_FEATURE_COLLECTION } from '../../../../common/constants';
import { extractPropertiesFromBucket } from '../../util/es_agg_utils';
import { clamp } from '../../../elasticsearch_geo_utils';
export function convertToGeoJson({ table, renderAs }) {
if (!table || !table.rows) {
return EMPTY_FEATURE_COLLECTION;
}
const GRID_BUCKET_KEYS_TO_IGNORE = ['key', 'gridCentroid'];
const geoGridColumn = table.columns.find(
column => column.aggConfig.type.dslName === 'geotile_grid'
);
if (!geoGridColumn) {
return EMPTY_FEATURE_COLLECTION;
}
const metricColumns = table.columns.filter(column => {
return (
column.aggConfig.type.type === 'metrics' && column.aggConfig.type.dslName !== 'geo_centroid'
);
});
const geocentroidColumn = table.columns.find(
column => column.aggConfig.type.dslName === 'geo_centroid'
);
if (!geocentroidColumn) {
return EMPTY_FEATURE_COLLECTION;
}
const features = [];
table.rows.forEach(row => {
const gridKey = row[geoGridColumn.id];
if (!gridKey) {
return;
export function convertCompositeRespToGeoJson(esResponse, renderAs) {
return convertToGeoJson(
esResponse,
renderAs,
esResponse => {
return _.get(esResponse, 'aggregations.compositeSplit.buckets', []);
},
gridBucket => {
return gridBucket.key.gridSplit;
}
);
}
const properties = {};
metricColumns.forEach(metricColumn => {
properties[metricColumn.aggConfig.id] = row[metricColumn.id];
});
export function convertRegularRespToGeoJson(esResponse, renderAs) {
return convertToGeoJson(
esResponse,
renderAs,
esResponse => {
return _.get(esResponse, 'aggregations.gridSplit.buckets', []);
},
gridBucket => {
return gridBucket.key;
}
);
}
function convertToGeoJson(esResponse, renderAs, pluckGridBuckets, pluckGridKey) {
const features = [];
const gridBuckets = pluckGridBuckets(esResponse);
for (let i = 0; i < gridBuckets.length; i++) {
const gridBucket = gridBuckets[i];
const gridKey = pluckGridKey(gridBucket);
features.push({
type: 'Feature',
geometry: rowToGeometry({
row,
gridKey,
geocentroidColumn,
gridCentroid: gridBucket.gridCentroid,
renderAs,
}),
id: gridKey,
properties,
properties: extractPropertiesFromBucket(gridBucket, GRID_BUCKET_KEYS_TO_IGNORE),
});
});
}
return {
featureCollection: {
type: 'FeatureCollection',
features: features,
},
};
return features;
}
function rowToGeometry({ row, gridKey, geocentroidColumn, renderAs }) {
function rowToGeometry({ gridKey, gridCentroid, renderAs }) {
const { top, bottom, right, left } = getTileBoundingBox(gridKey);
if (renderAs === RENDER_AS.GRID) {
@ -83,10 +78,10 @@ function rowToGeometry({ row, gridKey, geocentroidColumn, renderAs }) {
};
}
// see https://github.com/elastic/elasticsearch/issues/24694 for why clampGrid is used
// see https://github.com/elastic/elasticsearch/issues/24694 for why clamp is used
const pointCoordinates = [
clampGrid(row[geocentroidColumn.id].lon, left, right),
clampGrid(row[geocentroidColumn.id].lat, bottom, top),
clamp(gridCentroid.location.lon, left, right),
clamp(gridCentroid.location.lat, bottom, top),
];
return {
@ -94,9 +89,3 @@ function rowToGeometry({ row, gridKey, geocentroidColumn, renderAs }) {
coordinates: pointCoordinates,
};
}
function clampGrid(val, min, max) {
if (val > max) val = max;
else if (val < min) val = min;
return val;
}

159
x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.test.ts Normal file
View file

@ -0,0 +1,159 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
jest.mock('../../../kibana_services', () => {});
// @ts-ignore
import { convertCompositeRespToGeoJson, convertRegularRespToGeoJson } from './convert_to_geojson';
// @ts-ignore
import { RENDER_AS } from './render_as';
describe('convertCompositeRespToGeoJson', () => {
const esResponse = {
aggregations: {
compositeSplit: {
after_key: {
gridSplit: '10/327/460',
},
buckets: [
{
key: { gridSplit: '4/4/6' },
doc_count: 65,
avg_of_bytes: { value: 5359.2307692307695 },
'terms_of_machine.os.keyword': {
buckets: [
{
key: 'win xp',
doc_count: 16,
},
],
},
gridCentroid: {
location: { lat: 36.62813963153614, lon: -81.94552666092149 },
count: 65,
},
},
],
},
},
};
it('Should convert elasticsearch aggregation response into feature collection of points', () => {
const features = convertCompositeRespToGeoJson(esResponse, RENDER_AS.POINT);
expect(features.length).toBe(1);
expect(features[0]).toEqual({
geometry: {
coordinates: [-81.94552666092149, 36.62813963153614],
type: 'Point',
},
id: '4/4/6',
properties: {
avg_of_bytes: 5359.2307692307695,
doc_count: 65,
'terms_of_machine.os.keyword': 'win xp',
},
type: 'Feature',
});
});
it('Should convert elasticsearch aggregation response into feature collection of Polygons', () => {
const features = convertCompositeRespToGeoJson(esResponse, RENDER_AS.GRID);
expect(features.length).toBe(1);
expect(features[0]).toEqual({
geometry: {
coordinates: [
[
[-67.5, 40.9799],
[-90, 40.9799],
[-90, 21.94305],
[-67.5, 21.94305],
[-67.5, 40.9799],
],
],
type: 'Polygon',
},
id: '4/4/6',
properties: {
avg_of_bytes: 5359.2307692307695,
doc_count: 65,
'terms_of_machine.os.keyword': 'win xp',
},
type: 'Feature',
});
});
});
describe('convertRegularRespToGeoJson', () => {
const esResponse = {
aggregations: {
gridSplit: {
buckets: [
{
key: '4/4/6',
doc_count: 65,
avg_of_bytes: { value: 5359.2307692307695 },
'terms_of_machine.os.keyword': {
buckets: [
{
key: 'win xp',
doc_count: 16,
},
],
},
gridCentroid: {
location: { lat: 36.62813963153614, lon: -81.94552666092149 },
count: 65,
},
},
],
},
},
};
it('Should convert elasticsearch aggregation response into feature collection of points', () => {
const features = convertRegularRespToGeoJson(esResponse, RENDER_AS.POINT);
expect(features.length).toBe(1);
expect(features[0]).toEqual({
geometry: {
coordinates: [-81.94552666092149, 36.62813963153614],
type: 'Point',
},
id: '4/4/6',
properties: {
avg_of_bytes: 5359.2307692307695,
doc_count: 65,
'terms_of_machine.os.keyword': 'win xp',
},
type: 'Feature',
});
});
it('Should convert elasticsearch aggregation response into feature collection of Polygons', () => {
const features = convertRegularRespToGeoJson(esResponse, RENDER_AS.GRID);
expect(features.length).toBe(1);
expect(features[0]).toEqual({
geometry: {
coordinates: [
[
[-67.5, 40.9799],
[-90, 40.9799],
[-90, 21.94305],
[-67.5, 21.94305],
[-67.5, 40.9799],
],
],
type: 'Polygon',
},
id: '4/4/6',
properties: {
avg_of_bytes: 5359.2307692307695,
doc_count: 65,
'terms_of_machine.os.keyword': 'win xp',
},
type: 'Feature',
});
});
});

201
x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/es_geo_grid_source.js
View file

@ -10,9 +10,7 @@ import uuid from 'uuid/v4';
import { VECTOR_SHAPE_TYPES } from '../vector_feature_types';
import { HeatmapLayer } from '../../heatmap_layer';
import { VectorLayer } from '../../vector_layer';
import { AggConfigs, Schemas } from 'ui/agg_types';
import { tabifyAggResponse } from '../../../../../../../../src/legacy/core_plugins/data/public';
import { convertToGeoJson } from './convert_to_geojson';
import { convertCompositeRespToGeoJson, convertRegularRespToGeoJson } from './convert_to_geojson';
import { VectorStyle } from '../../styles/vector/vector_style';
import {
getDefaultDynamicProperties,
@ -24,6 +22,8 @@ import { CreateSourceEditor } from './create_source_editor';
import { UpdateSourceEditor } from './update_source_editor';
import { GRID_RESOLUTION } from '../../grid_resolution';
import {
AGG_TYPE,
DEFAULT_MAX_BUCKETS_LIMIT,
SOURCE_DATA_ID_ORIGIN,
ES_GEO_GRID,
COUNT_PROP_NAME,
@ -34,21 +34,10 @@ import { getDataSourceLabel } from '../../../../common/i18n_getters';
import { AbstractESAggSource } from '../es_agg_source';
import { DynamicStyleProperty } from '../../styles/vector/properties/dynamic_style_property';
import { StaticStyleProperty } from '../../styles/vector/properties/static_style_property';
import { DataRequestAbortError } from '../../util/data_request';
const MAX_GEOTILE_LEVEL = 29;
const aggSchemas = new Schemas([
AbstractESAggSource.METRIC_SCHEMA_CONFIG,
{
group: 'buckets',
name: 'segment',
title: 'Geo Grid',
aggFilter: 'geotile_grid',
min: 1,
max: 1,
},
]);
export class ESGeoGridSource extends AbstractESAggSource {
static type = ES_GEO_GRID;
static title = i18n.translate('xpack.maps.source.esGridTitle', {
@ -175,15 +164,120 @@ export class ESGeoGridSource extends AbstractESAggSource {
);
}
async getGeoJsonWithMeta(layerName, searchFilters, registerCancelCallback) {
const indexPattern = await this.getIndexPattern();
const searchSource = await this._makeSearchSource(searchFilters, 0);
const aggConfigs = new AggConfigs(
indexPattern,
this._makeAggConfigs(searchFilters.geogridPrecision),
aggSchemas.all
);
searchSource.setField('aggs', aggConfigs.toDsl());
async _compositeAggRequest({
searchSource,
indexPattern,
precision,
layerName,
registerCancelCallback,
bucketsPerGrid,
isRequestStillActive,
}) {
const gridsPerRequest = Math.floor(DEFAULT_MAX_BUCKETS_LIMIT / bucketsPerGrid);
const aggs = {
compositeSplit: {
composite: {
size: gridsPerRequest,
sources: [
{
gridSplit: {
geotile_grid: {
field: this._descriptor.geoField,
precision,
},
},
},
],
},
aggs: {
gridCentroid: {
geo_centroid: {
field: this._descriptor.geoField,
},
},
...this.getValueAggsDsl(indexPattern),
},
},
};
const features = [];
let requestCount = 0;
let afterKey = null;
while (true) {
if (!isRequestStillActive()) {
// Stop paging through results if request is obsolete
throw new DataRequestAbortError();
}
requestCount++;
// circuit breaker to ensure reasonable number of requests
if (requestCount > 5) {
throw new Error(
i18n.translate('xpack.maps.source.esGrid.compositePaginationErrorMessage', {
defaultMessage: `{layerName} is causing too many requests. Reduce "Grid resolution" and/or reduce the number of top term "Metrics".`,
values: { layerName },
})
);
}
if (afterKey) {
aggs.compositeSplit.composite.after = afterKey;
}
searchSource.setField('aggs', aggs);
const requestId = afterKey ? `${this.getId()} afterKey ${afterKey.geoSplit}` : this.getId();
const esResponse = await this._runEsQuery({
requestId,
requestName: `${layerName} (${requestCount})`,
searchSource,
registerCancelCallback,
requestDescription: i18n.translate(
'xpack.maps.source.esGrid.compositeInspectorDescription',
{
defaultMessage: 'Elasticsearch geo grid aggregation request: {requestId}',
values: { requestId },
}
),
});
features.push(...convertCompositeRespToGeoJson(esResponse, this._descriptor.requestType));
afterKey = esResponse.aggregations.compositeSplit.after_key;
if (esResponse.aggregations.compositeSplit.buckets.length < gridsPerRequest) {
// Finished because request did not get full resultset back
break;
}
}
return features;
}
// Do not use composite aggregation when there are no terms sub-aggregations
// see https://github.com/elastic/kibana/pull/57875#issuecomment-590515482 for explanation on using separate code paths
async _nonCompositeAggRequest({
searchSource,
indexPattern,
precision,
layerName,
registerCancelCallback,
}) {
searchSource.setField('aggs', {
gridSplit: {
geotile_grid: {
field: this._descriptor.geoField,
precision,
},
aggs: {
gridCentroid: {
geo_centroid: {
field: this._descriptor.geoField,
},
},
...this.getValueAggsDsl(indexPattern),
},
},
});
const esResponse = await this._runEsQuery({
requestId: this.getId(),
requestName: layerName,
@ -194,14 +288,45 @@ export class ESGeoGridSource extends AbstractESAggSource {
}),
});
const tabifiedResp = tabifyAggResponse(aggConfigs, esResponse);
const { featureCollection } = convertToGeoJson({
table: tabifiedResp,
renderAs: this._descriptor.requestType,
return convertRegularRespToGeoJson(esResponse, this._descriptor.requestType);
}
async getGeoJsonWithMeta(layerName, searchFilters, registerCancelCallback, isRequestStillActive) {
const indexPattern = await this.getIndexPattern();
const searchSource = await this._makeSearchSource(searchFilters, 0);
let bucketsPerGrid = 1;
this.getMetricFields().forEach(metricField => {
if (metricField.getAggType() === AGG_TYPE.TERMS) {
// each terms aggregation increases the overall number of buckets per grid
bucketsPerGrid++;
}
});
const features =
bucketsPerGrid === 1
? await this._nonCompositeAggRequest({
searchSource,
indexPattern,
precision: searchFilters.geogridPrecision,
layerName,
registerCancelCallback,
})
: await this._compositeAggRequest({
searchSource,
indexPattern,
precision: searchFilters.geogridPrecision,
layerName,
registerCancelCallback,
bucketsPerGrid,
isRequestStillActive,
});
return {
data: featureCollection,
data: {
type: 'FeatureCollection',
features: features,
},
meta: {
areResultsTrimmed: false,
},
@ -212,24 +337,6 @@ export class ESGeoGridSource extends AbstractESAggSource {
return true;
}
_makeAggConfigs(precision) {
const metricAggConfigs = this.createMetricAggConfigs();
return [
...metricAggConfigs,
{
id: 'grid',
enabled: true,
type: 'geotile_grid',
schema: 'segment',
params: {
field: this._descriptor.geoField,
useGeocentroid: true,
precision: precision,
},
},
];
}
_createHeatmapLayerDescriptor(options) {
return HeatmapLayer.createDescriptor({
sourceDescriptor: this._descriptor,

3
x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/geo_tile_utils.js
View file

@ -6,6 +6,7 @@
import _ from 'lodash';
import { DECIMAL_DEGREES_PRECISION } from '../../../../common/constants';
import { clampToLatBounds } from '../../../elasticsearch_geo_utils';
const ZOOM_TILE_KEY_INDEX = 0;
const X_TILE_KEY_INDEX = 1;
@ -87,7 +88,7 @@ function sec(value) {
}
function latitudeToTile(lat, tileCount) {
const radians = (lat * Math.PI) / 180;
const radians = (clampToLatBounds(lat) * Math.PI) / 180;
const y = ((1 - Math.log(Math.tan(radians) + sec(radians)) / Math.PI) / 2) * tileCount;
return Math.floor(y);
}

2
x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/geo_tile_utils.test.js
View file

@ -4,6 +4,8 @@
* you may not use this file except in compliance with the Elastic License.
*/
jest.mock('../../../kibana_services', () => {});
import { parseTileKey, getTileBoundingBox, expandToTileBoundaries } from './geo_tile_utils';
it('Should parse tile key', () => {

19
x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/convert_to_lines.js
View file

@ -5,9 +5,11 @@
*/
import _ from 'lodash';
import { extractPropertiesFromBucket } from '../../util/es_agg_utils';
const LAT_INDEX = 0;
const LON_INDEX = 1;
const PEW_PEW_BUCKET_KEYS_TO_IGNORE = ['key', 'sourceCentroid'];
function parsePointFromKey(key) {
const split = key.split(',');
@ -25,25 +27,16 @@ export function convertToLines(esResponse) {
const dest = parsePointFromKey(destBucket.key);
const sourceBuckets = _.get(destBucket, 'sourceGrid.buckets', []);
for (let j = 0; j < sourceBuckets.length; j++) {
const { key, sourceCentroid, ...rest } = sourceBuckets[j];
// flatten metrics
Object.keys(rest).forEach(key => {
if (_.has(rest[key], 'value')) {
rest[key] = rest[key].value;
}
});
const sourceBucket = sourceBuckets[j];
const sourceCentroid = sourceBucket.sourceCentroid;
lineFeatures.push({
type: 'Feature',
geometry: {
type: 'LineString',
coordinates: [[sourceCentroid.location.lon, sourceCentroid.location.lat], dest],
},
id: `${dest.join()},${key}`,
properties: {
...rest,
},
id: `${dest.join()},${sourceBucket.key}`,
properties: extractPropertiesFromBucket(sourceBucket, PEW_PEW_BUCKET_KEYS_TO_IGNORE),
});
}
}

68
x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/convert_to_lines.test.ts Normal file
View file

@ -0,0 +1,68 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
// @ts-ignore
import { convertToLines } from './convert_to_lines';
const esResponse = {
aggregations: {
destSplit: {
buckets: [
{
key: '43.68389896117151, 10.39269994944334',
doc_count: 2,
sourceGrid: {
buckets: [
{
key: '4/9/3',
doc_count: 1,
terms_of_Carrier: {
buckets: [
{
key: 'ES-Air',
doc_count: 1,
},
],
},
sourceCentroid: {
location: {
lat: 68.15180202014744,
lon: 33.46390150487423,
},
count: 1,
},
avg_of_FlightDelayMin: {
value: 3,
},
},
],
},
},
],
},
},
};
it('Should convert elasticsearch aggregation response into feature collection of lines', () => {
const geoJson = convertToLines(esResponse);
expect(geoJson.featureCollection.features.length).toBe(1);
expect(geoJson.featureCollection.features[0]).toEqual({
geometry: {
coordinates: [
[33.46390150487423, 68.15180202014744],
[10.39269994944334, 43.68389896117151],
],
type: 'LineString',
},
id: '10.39269994944334,43.68389896117151,4/9/3',
properties: {
avg_of_FlightDelayMin: 3,
doc_count: 1,
terms_of_Carrier: 'ES-Air',
},
type: 'Feature',
});
});

8
x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/es_pew_pew_source.js
View file

@ -20,7 +20,6 @@ import { i18n } from '@kbn/i18n';
import { SOURCE_DATA_ID_ORIGIN, ES_PEW_PEW, COUNT_PROP_NAME } from '../../../../common/constants';
import { getDataSourceLabel } from '../../../../common/i18n_getters';
import { convertToLines } from './convert_to_lines';
import { AggConfigs, Schemas } from 'ui/agg_types';
import { AbstractESAggSource } from '../es_agg_source';
import { DynamicStyleProperty } from '../../styles/vector/properties/dynamic_style_property';
import { COLOR_GRADIENTS } from '../../styles/color_utils';
@ -28,8 +27,6 @@ import { indexPatterns } from '../../../../../../../../src/plugins/data/public';
const MAX_GEOTILE_LEVEL = 29;
const aggSchemas = new Schemas([AbstractESAggSource.METRIC_SCHEMA_CONFIG]);
export class ESPewPewSource extends AbstractESAggSource {
static type = ES_PEW_PEW;
static title = i18n.translate('xpack.maps.source.pewPewTitle', {
@ -170,9 +167,6 @@ export class ESPewPewSource extends AbstractESAggSource {
async getGeoJsonWithMeta(layerName, searchFilters, registerCancelCallback) {
const indexPattern = await this.getIndexPattern();
const metricAggConfigs = this.createMetricAggConfigs();
const aggConfigs = new AggConfigs(indexPattern, metricAggConfigs, aggSchemas.all);
const searchSource = await this._makeSearchSource(searchFilters, 0);
searchSource.setField('aggs', {
destSplit: {
@ -199,7 +193,7 @@ export class ESPewPewSource extends AbstractESAggSource {
field: this._descriptor.sourceGeoField,
},
},
...aggConfigs.toDsl(),
...this.getValueAggsDsl(indexPattern),
},
},
},

26
x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js
View file

@ -28,31 +28,7 @@ import { loadIndexSettings } from './load_index_settings';
import { DEFAULT_FILTER_BY_MAP_BOUNDS } from './constants';
import { ESDocField } from '../../fields/es_doc_field';
function getField(indexPattern, fieldName) {
const field = indexPattern.fields.getByName(fieldName);
if (!field) {
throw new Error(
i18n.translate('xpack.maps.source.esSearch.fieldNotFoundMsg', {
defaultMessage: `Unable to find '{fieldName}' in index-pattern '{indexPatternTitle}'.`,
values: { fieldName, indexPatternTitle: indexPattern.title },
})
);
}
return field;
}
function addFieldToDSL(dsl, field) {
return !field.scripted
? { ...dsl, field: field.name }
: {
...dsl,
script: {
source: field.script,
lang: field.lang,
},
};
}
import { getField, addFieldToDSL } from '../../util/es_agg_utils';
function getDocValueAndSourceFields(indexPattern, fieldNames) {
const docValueFields = [];

11
x-pack/legacy/plugins/maps/public/layers/sources/es_source.js
View file

@ -18,7 +18,7 @@ import { AggConfigs } from 'ui/agg_types';
import { i18n } from '@kbn/i18n';
import uuid from 'uuid/v4';
import { copyPersistentState } from '../../reducers/util';
import { ES_GEO_FIELD_TYPE, METRIC_TYPE } from '../../../common/constants';
import { ES_GEO_FIELD_TYPE, AGG_TYPE } from '../../../common/constants';
import { DataRequestAbortError } from '../util/data_request';
import { expandToTileBoundaries } from './es_geo_grid_source/geo_tile_utils';
@ -270,7 +270,7 @@ export class AbstractESSource extends AbstractVectorSource {
// Do not use field formatters for counting metrics
if (
metricField &&
(metricField.type === METRIC_TYPE.COUNT || metricField.type === METRIC_TYPE.UNIQUE_COUNT)
(metricField.type === AGG_TYPE.COUNT || metricField.type === AGG_TYPE.UNIQUE_COUNT)
) {
return null;
}
@ -347,13 +347,16 @@ export class AbstractESSource extends AbstractVectorSource {
}
getValueSuggestions = async (fieldName, query) => {
if (!fieldName) {
// fieldName could be an aggregation so it needs to be unpacked to expose raw field.
const metricField = this.getMetricFields().find(field => field.getName() === fieldName);
const realFieldName = metricField ? metricField.getESDocFieldName() : fieldName;
if (!realFieldName) {
return [];
}
try {
const indexPattern = await this.getIndexPattern();
const field = indexPattern.fields.getByName(fieldName);
const field = indexPattern.fields.getByName(realFieldName);
return await autocompleteService.getValueSuggestions({
indexPattern,
field,

91
x-pack/legacy/plugins/maps/public/layers/sources/es_term_source.js
View file

@ -6,46 +6,25 @@
import _ from 'lodash';
import { AggConfigs, Schemas } from 'ui/agg_types';
import { i18n } from '@kbn/i18n';
import {
COUNT_PROP_LABEL,
DEFAULT_MAX_BUCKETS_LIMIT,
FIELD_ORIGIN,
METRIC_TYPE,
} from '../../../common/constants';
import { DEFAULT_MAX_BUCKETS_LIMIT, FIELD_ORIGIN, AGG_TYPE } from '../../../common/constants';
import { ESDocField } from '../fields/es_doc_field';
import { AbstractESAggSource, AGG_DELIMITER } from './es_agg_source';
import { getField, addFieldToDSL, extractPropertiesFromBucket } from '../util/es_agg_utils';
const TERMS_AGG_NAME = 'join';
const FIELD_NAME_PREFIX = '__kbnjoin__';
const GROUP_BY_DELIMITER = '_groupby_';
const TERMS_BUCKET_KEYS_TO_IGNORE = ['key', 'doc_count'];
const aggSchemas = new Schemas([
AbstractESAggSource.METRIC_SCHEMA_CONFIG,
{
group: 'buckets',
name: 'segment',
title: 'Terms',
aggFilter: 'terms',
min: 1,
max: 1,
},
]);
export function extractPropertiesMap(rawEsData, propertyNames, countPropertyName) {
export function extractPropertiesMap(rawEsData, countPropertyName) {
const propertiesMap = new Map();
_.get(rawEsData, ['aggregations', TERMS_AGG_NAME, 'buckets'], []).forEach(termBucket => {
const properties = {};
const properties = extractPropertiesFromBucket(termBucket, TERMS_BUCKET_KEYS_TO_IGNORE);
if (countPropertyName) {
properties[countPropertyName] = termBucket.doc_count;
}
propertyNames.forEach(propertyName => {
if (_.has(termBucket, [propertyName, 'value'])) {
properties[propertyName] = _.get(termBucket, [propertyName, 'value']);
}
});
propertiesMap.set(termBucket.key.toString(), properties);
});
return propertiesMap;
@ -90,15 +69,27 @@ export class ESTermSource extends AbstractESAggSource {
formatMetricKey(aggType, fieldName) {
const metricKey =
aggType !== METRIC_TYPE.COUNT ? `${aggType}${AGG_DELIMITER}${fieldName}` : aggType;
aggType !== AGG_TYPE.COUNT ? `${aggType}${AGG_DELIMITER}${fieldName}` : aggType;
return `${FIELD_NAME_PREFIX}${metricKey}${GROUP_BY_DELIMITER}${
this._descriptor.indexPatternTitle
}.${this._termField.getName()}`;
}
formatMetricLabel(type, fieldName) {
const metricLabel = type !== METRIC_TYPE.COUNT ? `${type} ${fieldName}` : COUNT_PROP_LABEL;
return `${metricLabel} of ${this._descriptor.indexPatternTitle}:${this._termField.getName()}`;
switch (type) {
case AGG_TYPE.COUNT:
return i18n.translate('xpack.maps.source.esJoin.countLabel', {
defaultMessage: `Count of {indexPatternTitle}`,
values: { indexPatternTitle: this._descriptor.indexPatternTitle },
});
case AGG_TYPE.TERMS:
return i18n.translate('xpack.maps.source.esJoin.topTermLabel', {
defaultMessage: `Top {fieldName}`,
values: { fieldName },
});
default:
return `${type} ${fieldName}`;
}
}
async getPropertiesMap(searchFilters, leftSourceName, leftFieldName, registerCancelCallback) {
@ -108,9 +99,14 @@ export class ESTermSource extends AbstractESAggSource {
const indexPattern = await this.getIndexPattern();
const searchSource = await this._makeSearchSource(searchFilters, 0);
const configStates = this._makeAggConfigs();
const aggConfigs = new AggConfigs(indexPattern, configStates, aggSchemas.all);
searchSource.setField('aggs', aggConfigs.toDsl());
const termsField = getField(indexPattern, this._termField.getName());
const termsAgg = { size: DEFAULT_MAX_BUCKETS_LIMIT };
searchSource.setField('aggs', {
[TERMS_AGG_NAME]: {
terms: addFieldToDSL(termsAgg, termsField),
aggs: { ...this.getValueAggsDsl(indexPattern) },
},
});
const rawEsData = await this._runEsQuery({
requestId: this.getId(),
@ -120,19 +116,9 @@ export class ESTermSource extends AbstractESAggSource {
requestDescription: this._getRequestDescription(leftSourceName, leftFieldName),
});
const metricPropertyNames = configStates
.filter(configState => {
return configState.schema === 'metric' && configState.type !== METRIC_TYPE.COUNT;
})
.map(configState => {
return configState.id;
});
const countConfigState = configStates.find(configState => {
return configState.type === METRIC_TYPE.COUNT;
});
const countPropertyName = _.get(countConfigState, 'id');
const countPropertyName = this.formatMetricKey(AGG_TYPE.COUNT);
return {
propertiesMap: extractPropertiesMap(rawEsData, metricPropertyNames, countPropertyName),
propertiesMap: extractPropertiesMap(rawEsData, countPropertyName),
};
}
@ -164,23 +150,6 @@ export class ESTermSource extends AbstractESAggSource {
});
}
_makeAggConfigs() {
const metricAggConfigs = this.createMetricAggConfigs();
return [
...metricAggConfigs,
{
id: TERMS_AGG_NAME,
enabled: true,
type: 'terms',
schema: 'segment',
params: {
field: this._termField.getName(),
size: DEFAULT_MAX_BUCKETS_LIMIT,
},
},
];
}
async getDisplayName() {
//no need to localize. this is never rendered.
return `es_table ${this._descriptor.indexPatternId}`;

91
x-pack/legacy/plugins/maps/public/layers/sources/es_term_source.test.js
View file

@ -8,9 +8,6 @@ import { ESTermSource, extractPropertiesMap } from './es_term_source';
jest.mock('ui/new_platform');
jest.mock('../vector_layer', () => {});
jest.mock('ui/agg_types', () => ({
Schemas: function() {},
}));
jest.mock('ui/timefilter', () => {});
const indexPatternTitle = 'myIndex';
@ -44,7 +41,7 @@ describe('getMetricFields', () => {
expect(metrics[0].getAggType()).toEqual('count');
expect(metrics[0].getName()).toEqual('__kbnjoin__count_groupby_myIndex.myTermField');
expect(await metrics[0].getLabel()).toEqual('count of myIndex:myTermField');
expect(await metrics[0].getLabel()).toEqual('Count of myIndex');
});
it('should remove incomplete metric configurations', async () => {
@ -65,84 +62,13 @@ describe('getMetricFields', () => {
expect(metrics[1].getAggType()).toEqual('count');
expect(metrics[1].getName()).toEqual('__kbnjoin__count_groupby_myIndex.myTermField');
expect(await metrics[1].getLabel()).toEqual('count of myIndex:myTermField');
});
});
describe('_makeAggConfigs', () => {
describe('no metrics', () => {
let aggConfigs;
beforeAll(() => {
const source = new ESTermSource({
indexPatternTitle: indexPatternTitle,
term: termFieldName,
});
aggConfigs = source._makeAggConfigs();
});
it('should make default "count" metric agg config', () => {
expect(aggConfigs.length).toBe(2);
expect(aggConfigs[0]).toEqual({
id: '__kbnjoin__count_groupby_myIndex.myTermField',
enabled: true,
type: 'count',
schema: 'metric',
params: {},
});
});
it('should make "terms" buckets agg config', () => {
expect(aggConfigs.length).toBe(2);
expect(aggConfigs[1]).toEqual({
id: 'join',
enabled: true,
type: 'terms',
schema: 'segment',
params: {
field: termFieldName,
size: 10000,
},
});
});
});
describe('metrics', () => {
let aggConfigs;
beforeAll(() => {
const source = new ESTermSource({
indexPatternTitle: indexPatternTitle,
term: 'myTermField',
metrics: metricExamples,
});
aggConfigs = source._makeAggConfigs();
});
it('should ignore invalid metrics configs', () => {
expect(aggConfigs.length).toBe(3);
});
it('should make agg config for each valid metric', () => {
expect(aggConfigs[0]).toEqual({
id: '__kbnjoin__sum_of_myFieldGettingSummed_groupby_myIndex.myTermField',
enabled: true,
type: 'sum',
schema: 'metric',
params: {
field: sumFieldName,
},
});
expect(aggConfigs[1]).toEqual({
id: '__kbnjoin__count_groupby_myIndex.myTermField',
enabled: true,
type: 'count',
schema: 'metric',
params: {},
});
});
expect(await metrics[1].getLabel()).toEqual('Count of myIndex');
});
});
describe('extractPropertiesMap', () => {
const minPropName =
'__kbnjoin__min_of_avlAirTemp_groupby_kibana_sample_data_ky_avl.kytcCountyNmbr';
const responseWithNumberTypes = {
aggregations: {
join: {
@ -150,14 +76,14 @@ describe('extractPropertiesMap', () => {
{
key: 109,
doc_count: 1130,
'__kbnjoin__min_of_avlAirTemp_groupby_kibana_sample_data_ky_avl.kytcCountyNmbr': {
[minPropName]: {
value: 36,
},
},
{
key: 62,
doc_count: 448,
'__kbnjoin__min_of_avlAirTemp_groupby_kibana_sample_data_ky_avl.kytcCountyNmbr': {
[minPropName]: {
value: 0,
},
},
@ -166,11 +92,10 @@ describe('extractPropertiesMap', () => {
},
};
const countPropName = '__kbnjoin__count_groupby_kibana_sample_data_ky_avl.kytcCountyNmbr';
const minPropName =
'__kbnjoin__min_of_avlAirTemp_groupby_kibana_sample_data_ky_avl.kytcCountyNmbr';
let propertiesMap;
beforeAll(() => {
propertiesMap = extractPropertiesMap(responseWithNumberTypes, [minPropName], countPropName);
propertiesMap = extractPropertiesMap(responseWithNumberTypes, countPropName);
});
it('should create key for each join term', () => {

12
x-pack/legacy/plugins/maps/public/layers/styles/vector/properties/dynamic_size_property.js
View file

@ -43,16 +43,8 @@ function getSymbolSizeIcons() {
}
export class DynamicSizeProperty extends DynamicStyleProperty {
constructor(
options,
styleName,
field,
getFieldMeta,
getFieldFormatter,
getValueSuggestions,
isSymbolizedAsIcon
) {
super(options, styleName, field, getFieldMeta, getFieldFormatter, getValueSuggestions);
constructor(options, styleName, field, getFieldMeta, getFieldFormatter, isSymbolizedAsIcon) {
super(options, styleName, field, getFieldMeta, getFieldFormatter);
this._isSymbolizedAsIcon = isSymbolizedAsIcon;
}

27
x-pack/legacy/plugins/maps/public/layers/styles/vector/properties/dynamic_style_property.js
View file

@ -13,21 +13,22 @@ import React from 'react';
import { OrdinalLegend } from './components/ordinal_legend';
import { CategoricalLegend } from './components/categorical_legend';
import { OrdinalFieldMetaOptionsPopover } from '../components/ordinal_field_meta_options_popover';
import { ESAggMetricField } from '../../../fields/es_agg_field';
export class DynamicStyleProperty extends AbstractStyleProperty {
static type = STYLE_TYPE.DYNAMIC;
constructor(options, styleName, field, getFieldMeta, getFieldFormatter, source) {
constructor(options, styleName, field, getFieldMeta, getFieldFormatter) {
super(options, styleName);
this._field = field;
this._getFieldMeta = getFieldMeta;
this._getFieldFormatter = getFieldFormatter;
this._source = source;
}
getValueSuggestions = query => {
const fieldName = this.getFieldName();
return this._source && fieldName ? this._source.getValueSuggestions(fieldName, query) : [];
const fieldSource = this.getFieldSource();
return fieldSource && fieldName ? fieldSource.getValueSuggestions(fieldName, query) : [];
};
getFieldMeta() {
@ -38,6 +39,10 @@ export class DynamicStyleProperty extends AbstractStyleProperty {
return this._field;
}
getFieldSource() {
return this._field ? this._field.getSource() : null;
}
getFieldName() {
return this._field ? this._field.getName() : '';
}
@ -180,9 +185,10 @@ export class DynamicStyleProperty extends AbstractStyleProperty {
}
_pluckOrdinalStyleMetaFromFieldMetaData(fieldMetaData) {
const realFieldName = this._field.getESDocFieldName
? this._field.getESDocFieldName()
: this._field.getName();
const realFieldName =
this._field instanceof ESAggMetricField
? this._field.getESDocFieldName()
: this._field.getName();
const stats = fieldMetaData[realFieldName];
if (!stats) {
return null;
@ -203,12 +209,15 @@ export class DynamicStyleProperty extends AbstractStyleProperty {
}
_pluckCategoricalStyleMetaFromFieldMetaData(fieldMetaData) {
const name = this.getField().getName();
if (!fieldMetaData[name] || !fieldMetaData[name].buckets) {
const realFieldName =
this._field instanceof ESAggMetricField
? this._field.getESDocFieldName()
: this._field.getName();
if (!fieldMetaData[realFieldName] || !fieldMetaData[realFieldName].buckets) {
return null;
}
const ordered = fieldMetaData[name].buckets.map(bucket => {
const ordered = fieldMetaData[realFieldName].buckets.map(bucket => {
return {
key: bucket.key,
count: bucket.doc_count,

10
x-pack/legacy/plugins/maps/public/layers/styles/vector/vector_style.js
View file

@ -625,7 +625,6 @@ export class VectorStyle extends AbstractStyle {
field,
this._getFieldMeta,
this._getFieldFormatter,
this._source,
isSymbolizedAsIcon
);
} else {
@ -645,8 +644,7 @@ export class VectorStyle extends AbstractStyle {
styleName,
field,
this._getFieldMeta,
this._getFieldFormatter,
this._source
this._getFieldFormatter
);
} else {
throw new Error(`${descriptor} not implemented`);
@ -678,8 +676,7 @@ export class VectorStyle extends AbstractStyle {
VECTOR_STYLES.LABEL_TEXT,
field,
this._getFieldMeta,
this._getFieldFormatter,
this._source
this._getFieldFormatter
);
} else {
throw new Error(`${descriptor} not implemented`);
@ -698,8 +695,7 @@ export class VectorStyle extends AbstractStyle {
VECTOR_STYLES.ICON,
field,
this._getFieldMeta,
this._getFieldFormatter,
this._source
this._getFieldFormatter
);
} else {
throw new Error(`${descriptor} not implemented`);

6
x-pack/legacy/plugins/maps/public/layers/tooltips/es_aggmetric_tooltip_property.js
View file

@ -5,7 +5,7 @@
*/
import { ESTooltipProperty } from './es_tooltip_property';
import { METRIC_TYPE } from '../../../common/constants';
import { AGG_TYPE } from '../../../common/constants';
export class ESAggMetricTooltipProperty extends ESTooltipProperty {
constructor(propertyKey, propertyName, rawValue, indexPattern, metricField) {
@ -22,8 +22,8 @@ export class ESAggMetricTooltipProperty extends ESTooltipProperty {
return '-';
}
if (
this._metricField.getAggType() === METRIC_TYPE.COUNT ||
this._metricField.getAggType() === METRIC_TYPE.UNIQUE_COUNT
this._metricField.getAggType() === AGG_TYPE.COUNT ||
this._metricField.getAggType() === AGG_TYPE.UNIQUE_COUNT
) {
return this._rawValue;
}

37
x-pack/legacy/plugins/maps/public/layers/util/es_agg_utils.test.ts Normal file
View file

@ -0,0 +1,37 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { extractPropertiesFromBucket } from './es_agg_utils';
describe('extractPropertiesFromBucket', () => {
test('Should ignore specified keys', () => {
const properties = extractPropertiesFromBucket({ key: '4/4/6' }, ['key']);
expect(properties).toEqual({});
});
test('Should extract metric aggregation values', () => {
const properties = extractPropertiesFromBucket({ avg_of_bytes: { value: 5359 } });
expect(properties).toEqual({
avg_of_bytes: 5359,
});
});
test('Should extract bucket aggregation values', () => {
const properties = extractPropertiesFromBucket({
'terms_of_machine.os.keyword': {
buckets: [
{
key: 'win xp',
doc_count: 16,
},
],
},
});
expect(properties).toEqual({
'terms_of_machine.os.keyword': 'win xp',
});
});
});

51
x-pack/legacy/plugins/maps/public/layers/util/es_agg_utils.ts Normal file
View file

@ -0,0 +1,51 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { i18n } from '@kbn/i18n';
import _ from 'lodash';
import { IndexPattern, IFieldType } from '../../../../../../../src/plugins/data/public';
export function getField(indexPattern: IndexPattern, fieldName: string) {
const field = indexPattern.fields.getByName(fieldName);
if (!field) {
throw new Error(
i18n.translate('xpack.maps.source.esSearch.fieldNotFoundMsg', {
defaultMessage: `Unable to find '{fieldName}' in index-pattern '{indexPatternTitle}'.`,
values: { fieldName, indexPatternTitle: indexPattern.title },
})
);
}
return field;
}
export function addFieldToDSL(dsl: object, field: IFieldType) {
return !field.scripted
? { ...dsl, field: field.name }
: {
...dsl,
script: {
source: field.script,
lang: field.lang,
},
};
}
export function extractPropertiesFromBucket(bucket: any, ignoreKeys: string[] = []) {
const properties: Record<string | number, unknown> = {};
for (const key in bucket) {
if (ignoreKeys.includes(key) || !bucket.hasOwnProperty(key)) {
continue;
}
if (_.has(bucket[key], 'value')) {
properties[key] = bucket[key].value;
} else if (_.has(bucket[key], 'buckets')) {
properties[key] = _.get(bucket[key], 'buckets[0].key');
} else {
properties[key] = bucket[key];
}
}
return properties;
}

4
x-pack/legacy/plugins/maps/public/layers/util/is_metric_countable.js
View file

@ -4,8 +4,8 @@
* you may not use this file except in compliance with the Elastic License.
*/
import { METRIC_TYPE } from '../../../common/constants';
import { AGG_TYPE } from '../../../common/constants';
export function isMetricCountable(aggType) {
return [METRIC_TYPE.COUNT, METRIC_TYPE.SUM, METRIC_TYPE.UNIQUE_COUNT].includes(aggType);
return [AGG_TYPE.COUNT, AGG_TYPE.SUM, AGG_TYPE.UNIQUE_COUNT].includes(aggType);
}

15
x-pack/legacy/plugins/maps/public/layers/vector_layer.js
View file

@ -365,8 +365,10 @@ export class VectorLayer extends AbstractLayer {
onLoadError,
registerCancelCallback,
dataFilters,
isRequestStillActive,
}) {
const requestToken = Symbol(`layer-${this.getId()}-${SOURCE_DATA_ID_ORIGIN}`);
const dataRequestId = SOURCE_DATA_ID_ORIGIN;
const requestToken = Symbol(`layer-${this.getId()}-${dataRequestId}`);
const searchFilters = this._getSearchFilters(dataFilters);
const prevDataRequest = this.getSourceDataRequest();
const canSkipFetch = await canSkipSourceUpdate({
@ -382,22 +384,25 @@ export class VectorLayer extends AbstractLayer {
}
try {
startLoading(SOURCE_DATA_ID_ORIGIN, requestToken, searchFilters);
startLoading(dataRequestId, requestToken, searchFilters);
const layerName = await this.getDisplayName();
const { data: sourceFeatureCollection, meta } = await this._source.getGeoJsonWithMeta(
layerName,
searchFilters,
registerCancelCallback.bind(null, requestToken)
registerCancelCallback.bind(null, requestToken),
() => {
return isRequestStillActive(dataRequestId, requestToken);
}
);
const layerFeatureCollection = assignFeatureIds(sourceFeatureCollection);
stopLoading(SOURCE_DATA_ID_ORIGIN, requestToken, layerFeatureCollection, meta);
stopLoading(dataRequestId, requestToken, layerFeatureCollection, meta);
return {
refreshed: true,
featureCollection: layerFeatureCollection,
};
} catch (error) {
if (!(error instanceof DataRequestAbortError)) {
onLoadError(SOURCE_DATA_ID_ORIGIN, requestToken, error.message);
onLoadError(dataRequestId, requestToken, error.message);
}
return {
refreshed: false,

15
x-pack/legacy/plugins/maps/public/selectors/map_selectors.js
View file

@ -125,6 +125,21 @@ export const getRefreshConfig = ({ map }) => {
export const getRefreshTimerLastTriggeredAt = ({ map }) => map.mapState.refreshTimerLastTriggeredAt;
function getLayerDescriptor(state = {}, layerId) {
const layerListRaw = getLayerListRaw(state);
return layerListRaw.find(layer => layer.id === layerId);
}
export function getDataRequestDescriptor(state = {}, layerId, dataId) {
const layerDescriptor = getLayerDescriptor(state, layerId);
if (!layerDescriptor || !layerDescriptor.__dataRequests) {
return;
}
return _.get(layerDescriptor, '__dataRequests', []).find(dataRequest => {
return dataRequest.dataId === dataId;
});
}
export const getDataFilters = createSelector(
getMapExtent,
getMapBuffer,

1
x-pack/plugins/translations/translations/ja-JP.json
View file

@ -7263,7 +7263,6 @@
"xpack.maps.source.esGrid.finestDropdownOption": "最も細かい",
"xpack.maps.source.esGrid.geospatialFieldLabel": "地理空間フィールド",
"xpack.maps.source.esGrid.indexPatternLabel": "インデックスパターン",
"xpack.maps.source.esGrid.inspectorDescription": "Elasticsearch ジオグリッド集約リクエスト",
"xpack.maps.source.esGrid.metricsLabel": "メトリック",
"xpack.maps.source.esGrid.noIndexPatternErrorMessage": "インデックスパターン {id} が見つかりません",
"xpack.maps.source.esGrid.resolutionParamErrorMessage": "グリッド解像度パラメーターが認識されません: {resolution}",

1
x-pack/plugins/translations/translations/zh-CN.json
View file

@ -7263,7 +7263,6 @@
"xpack.maps.source.esGrid.finestDropdownOption": "最精致化",
"xpack.maps.source.esGrid.geospatialFieldLabel": "地理空间字段",
"xpack.maps.source.esGrid.indexPatternLabel": "索引模式",
"xpack.maps.source.esGrid.inspectorDescription": "Elasticsearch 地理网格聚合请求",
"xpack.maps.source.esGrid.metricsLabel": "指标",
"xpack.maps.source.esGrid.noIndexPatternErrorMessage": "找不到索引模式 {id}",
"xpack.maps.source.esGrid.resolutionParamErrorMessage": "无法识别网格分辨率参数：{resolution}",