[maps] ES|QL source (#173481)

Closes https://github.com/elastic/kibana/issues/167648

PR adds "ES|QL" card to "Add Layer" interface. Creates a layer renders
an ES|QL statement on the map

<img width="250" alt="Screenshot 2023-12-16 at 2 03 04 PM"
src="4d1e24f6-405b-4016-8e6f-4736742c6166">

<img width="800" alt="Screenshot 2023-12-16 at 1 54 24 PM"
src="8387551f-c3b5-4b15-84eb-aef18254d371">

### Known limitations
This PR is intended to be a first start and does not cover all
functionality. The following list identifies known limitations that will
have to be resolved in future work.
1. tooltips - Existing documents source supports lazy loading tooltips
to avoid pulling unused data on map render. How would this look for
ES|QL? Should tooltips only support local data?
2. ES|QL layer does not surface data view to unified search bar so
search type-ahead and filter bar will not show index-pattern fields from
ES|QL layer.
3. ES|QL layer does not surface geoField. This affects control for
drawing filters on map.
4. ES|QL layer does not support pulling field meta from Elasticsearch.
Instead, data-driven styling uses ranges from local data set. This will
be tricky as we can't just pull field ranges from index-pattern. Also
need to account for WHERE clause and other edge cases.
5. fit to bounds

---------

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Nick Peihl <nickpeihl@gmail.com>
This commit is contained in:
Nathan Reese 2024-01-03 08:42:52 -07:00 committed by GitHub
parent cb641a0b07
commit 9d66265931
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
53 changed files with 1398 additions and 64 deletions

View file

@ -1111,6 +1111,7 @@
"vega-spec-injector": "^0.0.2",
"vega-tooltip": "^0.28.0",
"vinyl": "^2.2.0",
"wellknown": "^0.5.0",
"whatwg-fetch": "^3.0.0",
"xml2js": "^0.5.0",
"xstate": "^4.38.2",

View file

@ -55,6 +55,7 @@ export {
getAggregateQueryMode,
getIndexPatternFromSQLQuery,
getIndexPatternFromESQLQuery,
getLimitFromESQLQuery,
getLanguageDisplayName,
cleanupESQLQueryForLensSuggestions,
} from './src/es_query';

View file

@ -12,6 +12,7 @@ import {
getAggregateQueryMode,
getIndexPatternFromSQLQuery,
getIndexPatternFromESQLQuery,
getLimitFromESQLQuery,
cleanupESQLQueryForLensSuggestions,
} from './es_aggregate_query';
@ -117,6 +118,33 @@ describe('sql query helpers', () => {
});
});
describe('getLimitFromESQLQuery', () => {
it('should return default limit when ES|QL query is empty', () => {
const limit = getLimitFromESQLQuery('');
expect(limit).toBe(500);
});
it('should return default limit when ES|QL query does not contain LIMIT command', () => {
const limit = getLimitFromESQLQuery('FROM foo');
expect(limit).toBe(500);
});
it('should return default limit when ES|QL query contains invalid LIMIT command', () => {
const limit = getLimitFromESQLQuery('FROM foo | LIMIT iAmNotANumber');
expect(limit).toBe(500);
});
it('should return limit when ES|QL query contains LIMIT command', () => {
const limit = getLimitFromESQLQuery('FROM foo | LIMIT 10000 | KEEP myField');
expect(limit).toBe(10000);
});
it('should return last limit when ES|QL query contains multiple LIMIT command', () => {
const limit = getLimitFromESQLQuery('FROM foo | LIMIT 200 | LIMIT 0');
expect(limit).toBe(0);
});
});
describe('cleanupESQLQueryForLensSuggestions', () => {
it('should not remove anything if a drop command is not present', () => {
expect(cleanupESQLQueryForLensSuggestions('from a | eval b = 1')).toBe('from a | eval b = 1');

View file

@ -5,10 +5,13 @@
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import type { Query, AggregateQuery } from '../filters';
type Language = keyof AggregateQuery;
const DEFAULT_ESQL_LIMIT = 500;
// Checks if the query is of type Query
export function isOfQueryType(arg?: Query | AggregateQuery): arg is Query {
return Boolean(arg && 'query' in arg);
@ -67,6 +70,17 @@ export function getIndexPatternFromESQLQuery(esql?: string): string {
return '';
}
export function getLimitFromESQLQuery(esql: string): number {
const limitCommands = esql.match(new RegExp(/LIMIT\s[0-9]+/, 'ig'));
if (!limitCommands) {
return DEFAULT_ESQL_LIMIT;
}
const lastIndex = limitCommands.length - 1;
const split = limitCommands[lastIndex].split(' ');
return parseInt(split[1], 10);
}
export function cleanupESQLQueryForLensSuggestions(esql?: string): string {
const pipes = (esql || '').split('|');
return pipes.filter((statement) => !/DROP\s/i.test(statement)).join('|');

View file

@ -20,6 +20,7 @@ export {
getIndexPatternFromSQLQuery,
getLanguageDisplayName,
getIndexPatternFromESQLQuery,
getLimitFromESQLQuery,
cleanupESQLQueryForLensSuggestions,
} from './es_aggregate_query';
export { fromCombinedFilter } from './from_combined_filter';

View file

@ -19,4 +19,7 @@ export type {
ESFilter,
MaybeReadonlyArray,
ClusterDetails,
ESQLColumn,
ESQLRow,
ESQLSearchReponse,
} from './src';

View file

@ -12,6 +12,9 @@ import {
AggregateOfMap as AggregationResultOfMap,
SearchHit,
ClusterDetails,
ESQLColumn,
ESQLRow,
ESQLSearchReponse,
} from './search';
export type ESFilter = estypes.QueryDslQueryContainer;
@ -41,4 +44,7 @@ export type {
AggregationResultOfMap,
SearchHit,
ClusterDetails,
ESQLColumn,
ESQLRow,
ESQLSearchReponse,
};

View file

@ -653,3 +653,15 @@ export interface ClusterDetails {
_shards?: estypes.ShardStatistics;
failures?: estypes.ShardFailure[];
}
export interface ESQLColumn {
name: string;
type: string;
}
export type ESQLRow = unknown[];
export interface ESQLSearchReponse {
columns: ESQLColumn[];
values: ESQLRow[];
}

View file

@ -200,6 +200,7 @@ interface EditorFooterProps {
disableSubmitAction?: boolean;
editorIsInline?: boolean;
isSpaceReduced?: boolean;
isLoading?: boolean;
}
export const EditorFooter = memo(function EditorFooter({
@ -214,6 +215,7 @@ export const EditorFooter = memo(function EditorFooter({
disableSubmitAction,
editorIsInline,
isSpaceReduced,
isLoading,
}: EditorFooterProps) {
const { euiTheme } = useEuiTheme();
const [isErrorPopoverOpen, setIsErrorPopoverOpen] = useState(false);
@ -331,6 +333,7 @@ export const EditorFooter = memo(function EditorFooter({
size="s"
fill
onClick={runQuery}
isLoading={isLoading}
isDisabled={Boolean(disableSubmitAction)}
data-test-subj="TextBasedLangEditor-run-query-button"
minWidth={isSpaceReduced ? false : undefined}

View file

@ -86,6 +86,8 @@ export interface TextBasedLanguagesEditorProps {
errors?: Error[];
/** Warning string as it comes from ES */
warning?: string;
/** Disables the editor and displays loading icon in run button */
isLoading?: boolean;
/** Disables the editor */
isDisabled?: boolean;
/** Indicator if the editor is on dark mode */
@ -149,6 +151,7 @@ export const TextBasedLanguagesEditor = memo(function TextBasedLanguagesEditor({
detectTimestamp = false,
errors: serverErrors,
warning: serverWarning,
isLoading,
isDisabled,
isDarkMode,
hideMinimizeButton,
@ -540,7 +543,9 @@ export const TextBasedLanguagesEditor = memo(function TextBasedLanguagesEditor({
},
overviewRulerBorder: false,
readOnly:
isDisabled || Boolean(!isCompactFocused && codeOneLiner && codeOneLiner.includes('...')),
isLoading ||
isDisabled ||
Boolean(!isCompactFocused && codeOneLiner && codeOneLiner.includes('...')),
};
if (isCompactFocused) {
@ -836,6 +841,7 @@ export const TextBasedLanguagesEditor = memo(function TextBasedLanguagesEditor({
disableSubmitAction={disableSubmitAction}
hideRunQueryText={hideRunQueryText}
isSpaceReduced={isSpaceReduced}
isLoading={isLoading}
/>
)}
</div>
@ -925,6 +931,7 @@ export const TextBasedLanguagesEditor = memo(function TextBasedLanguagesEditor({
editorIsInline={editorIsInline}
disableSubmitAction={disableSubmitAction}
isSpaceReduced={isSpaceReduced}
isLoading={isLoading}
{...editorMessages}
/>
)}

View file

@ -20,6 +20,7 @@ import { zipObject } from 'lodash';
import { Observable, defer, throwError } from 'rxjs';
import { catchError, map, switchMap, tap } from 'rxjs/operators';
import { buildEsQuery } from '@kbn/es-query';
import type { ESQLSearchReponse } from '@kbn/es-types';
import { getEsQueryConfig } from '../../es_query';
import { getTime } from '../../query';
import { ESQL_SEARCH_STRATEGY, IKibanaSearchRequest, ISearchGeneric, KibanaContext } from '..';
@ -90,14 +91,6 @@ interface ESQLSearchParams {
locale?: string;
}
interface ESQLSearchReponse {
columns?: Array<{
name: string;
type: string;
}>;
values: unknown[][];
}
export const getEsqlFn = ({ getStartDependencies }: EsqlFnArguments) => {
const essql: EsqlExpressionFunctionDefinition = {
name: 'esql',

View file

@ -15,9 +15,3 @@ export type IndexAsString<Map> = {
} & Map;
export type Omit<T, K extends keyof T> = Pick<T, Exclude<keyof T, K>>;
export interface BoolQuery {
must_not: Array<Record<string, any>>;
should: Array<Record<string, any>>;
filter: Array<Record<string, any>>;
}

View file

@ -50,7 +50,8 @@
"@kbn/search-errors",
"@kbn/search-response-warnings",
"@kbn/shared-ux-link-redirect-app",
"@kbn/bfetch-error"
"@kbn/bfetch-error",
"@kbn/es-types"
],
"exclude": [
"target/**/*",

View file

@ -69,6 +69,7 @@ export enum SOURCE_TYPES {
ES_SEARCH = 'ES_SEARCH',
ES_PEW_PEW = 'ES_PEW_PEW',
ES_ML_ANOMALIES = 'ML_ANOMALIES',
ESQL = 'ESQL',
EMS_XYZ = 'EMS_XYZ', // identifies a custom TMS source. EMS-prefix in the name is a little unfortunate :(
WMS = 'WMS',
KIBANA_TILEMAP = 'KIBANA_TILEMAP',
@ -327,6 +328,7 @@ export enum WIZARD_ID {
POINT_2_POINT = 'point2Point',
ES_DOCUMENT = 'esDocument',
ES_TOP_HITS = 'esTopHits',
ESQL = 'ESQL',
KIBANA_BASEMAP = 'kibanaBasemap',
MVT_VECTOR = 'mvtVector',
WMS_LAYER = 'wmsLayer',

View file

@ -9,6 +9,7 @@
import { FeatureCollection } from 'geojson';
import type { Query } from '@kbn/es-query';
import type { ESQLColumn } from '@kbn/es-types';
import { SortDirection } from '@kbn/data-plugin/common/search';
import {
AGG_TYPE,
@ -37,6 +38,20 @@ export type EMSFileSourceDescriptor = AbstractSourceDescriptor & {
tooltipProperties: string[];
};
export type ESQLSourceDescriptor = AbstractSourceDescriptor & {
// id: UUID
id: string;
esql: string;
columns: ESQLColumn[];
/*
* Date field used to narrow ES|QL requests by global time range
*/
dateField?: string;
narrowByGlobalSearch: boolean;
narrowByMapBounds: boolean;
applyForceRefresh: boolean;
};
export type AbstractESSourceDescriptor = AbstractSourceDescriptor & {
// id: UUID
id: string;

View file

@ -214,6 +214,10 @@ function getLayerKey(layerDescriptor: LayerDescriptor): LAYER_KEYS | null {
return LAYER_KEYS.ES_ML_ANOMALIES;
}
if (layerDescriptor.sourceDescriptor.type === SOURCE_TYPES.ESQL) {
return LAYER_KEYS.ESQL;
}
if (layerDescriptor.sourceDescriptor.type === SOURCE_TYPES.ES_SEARCH) {
const sourceDescriptor = layerDescriptor.sourceDescriptor as ESSearchSourceDescriptor;

View file

@ -27,6 +27,7 @@ export enum LAYER_KEYS {
ES_AGG_HEXAGONS = 'es_agg_hexagons',
ES_AGG_HEATMAP = 'es_agg_heatmap',
ES_ML_ANOMALIES = 'es_ml_anomalies',
ESQL = 'esql',
EMS_REGION = 'ems_region',
EMS_BASEMAP = 'ems_basemap',
KBN_TMS_RASTER = 'kbn_tms_raster',

View file

@ -49,7 +49,8 @@
"kibanaUtils",
"usageCollection",
"unifiedSearch",
"fieldFormats"
"fieldFormats",
"textBasedLanguages"
],
"extraPublicDirs": [
"common"

View file

@ -74,7 +74,7 @@ import { IVectorStyle } from '../classes/styles/vector/vector_style';
import { notifyLicensedFeatureUsage } from '../licensed_features';
import { IESAggField } from '../classes/fields/agg';
import { IField } from '../classes/fields/field';
import type { IESSource } from '../classes/sources/es_source';
import type { IVectorSource } from '../classes/sources/vector_source';
import { getDrawMode, getOpenTOCDetails } from '../selectors/ui_selectors';
import { isLayerGroup, LayerGroup } from '../classes/layers/layer_group';
import { isSpatialJoin } from '../classes/joins/is_spatial_join';
@ -849,7 +849,7 @@ export function setTileState(
}
function clearInspectorAdapters(layer: ILayer, adapters: Adapters) {
if (isLayerGroup(layer) || !layer.getSource().isESSource()) {
if (isLayerGroup(layer)) {
return;
}
@ -857,10 +857,15 @@ function clearInspectorAdapters(layer: ILayer, adapters: Adapters) {
adapters.vectorTiles.removeLayer(layer.getId());
}
const source = layer.getSource();
if ('getInspectorRequestIds' in source) {
(source as IVectorSource).getInspectorRequestIds().forEach((id) => {
adapters.requests!.resetRequest(id);
});
}
if (adapters.requests && 'getValidJoins' in layer) {
const vectorLayer = layer as IVectorLayer;
adapters.requests!.resetRequest((layer.getSource() as IESSource).getId());
vectorLayer.getValidJoins().forEach((join) => {
(layer as IVectorLayer).getValidJoins().forEach((join) => {
adapters.requests!.resetRequest(join.getRightJoinSource().getId());
});
}

View file

@ -26,7 +26,7 @@ export function buildVectorRequestMeta(
applyGlobalQuery: source.getApplyGlobalQuery(),
applyGlobalTime: source.getApplyGlobalTime(),
sourceMeta: source.getSyncMeta(dataFilters),
applyForceRefresh: source.isESSource() ? source.getApplyForceRefresh() : false,
applyForceRefresh: source.getApplyForceRefresh(),
isForceRefresh,
isFeatureEditorOpenForLayer,
};

View file

@ -82,7 +82,7 @@ export class RasterTileLayer extends AbstractLayer {
...dataFilters,
applyGlobalQuery: source.getApplyGlobalQuery(),
applyGlobalTime: source.getApplyGlobalTime(),
applyForceRefresh: source.isESSource() ? source.getApplyForceRefresh() : false,
applyForceRefresh: source.getApplyForceRefresh(),
sourceQuery: this.getQuery() || undefined,
isForceRefresh,
};

View file

@ -48,6 +48,7 @@ export type LayerWizard = {
export type RenderWizardArguments = {
previewLayers: (layerDescriptors: LayerDescriptor[]) => void;
mapColors: string[];
mostCommonDataViewId?: string;
// multi-step arguments for wizards that supply 'prerequisiteSteps'
currentStepId: string | null;
isOnFinalStep: boolean;

View file

@ -18,6 +18,7 @@ import {
} from '../../sources/es_geo_grid_source';
import { geoLineLayerWizardConfig } from '../../sources/es_geo_line_source';
import { point2PointLayerWizardConfig } from '../../sources/es_pew_pew_source/point_2_point_layer_wizard';
import { esqlLayerWizardConfig } from '../../sources/esql_source';
import { emsBoundariesLayerWizardConfig } from '../../sources/ems_file_source';
import { emsBaseMapLayerWizardConfig } from '../../sources/ems_tms_source';
import { kibanaBasemapLayerWizardConfig } from '../../sources/kibana_tilemap_source/kibana_base_map_layer_wizard';
@ -41,10 +42,10 @@ export function registerLayerWizards() {
registerLayerWizardInternal(layerGroupWizardConfig);
registerLayerWizardInternal(esDocumentsLayerWizardConfig);
registerLayerWizardInternal(choroplethLayerWizardConfig);
registerLayerWizardInternal(esqlLayerWizardConfig);
registerLayerWizardInternal(choroplethLayerWizardConfig);
registerLayerWizardInternal(spatialJoinWizardConfig);
registerLayerWizardInternal(point2PointLayerWizardConfig);
registerLayerWizardInternal(clustersLayerWizardConfig);
registerLayerWizardInternal(heatmapLayerWizardConfig);
@ -52,15 +53,16 @@ export function registerLayerWizards() {
registerLayerWizardInternal(esTopHitsLayerWizardConfig);
registerLayerWizardInternal(geoLineLayerWizardConfig);
registerLayerWizardInternal(point2PointLayerWizardConfig);
registerLayerWizardInternal(newVectorLayerWizardConfig);
registerLayerWizardInternal(emsBoundariesLayerWizardConfig);
registerLayerWizardInternal(emsBaseMapLayerWizardConfig);
registerLayerWizardInternal(newVectorLayerWizardConfig);
registerLayerWizardInternal(kibanaBasemapLayerWizardConfig);
registerLayerWizardInternal(tmsLayerWizardConfig);
registerLayerWizardInternal(wmsLayerWizardConfig);
registerLayerWizardInternal(kibanaBasemapLayerWizardConfig);
registerLayerWizardInternal(mvtVectorSourceWizardConfig);
registerLayerWizardInternal(ObservabilityLayerWizardConfig);
registerLayerWizardInternal(SecurityLayerWizardConfig);

View file

@ -230,6 +230,18 @@ export class ESGeoLineSource extends AbstractESAggSource {
);
}
getInspectorRequestIds(): string[] {
return [this._getTracksRequestId(), this._getEntitiesRequestId()];
}
private _getTracksRequestId() {
return `${this.getId()}_tracks`;
}
private _getEntitiesRequestId() {
return `${this.getId()}_entities`;
}
async _getGeoLineByTimeseries(
layerName: string,
requestMeta: VectorSourceRequestMeta,
@ -264,7 +276,7 @@ export class ESGeoLineSource extends AbstractESAggSource {
const warnings: SearchResponseWarning[] = [];
const resp = await this._runEsQuery({
requestId: `${this.getId()}_tracks`,
requestId: this._getTracksRequestId(),
requestName: getLayerFeaturesRequestName(layerName),
searchSource,
registerCancelCallback,
@ -356,7 +368,7 @@ export class ESGeoLineSource extends AbstractESAggSource {
}
const entityResp = await this._runEsQuery({
requestId: `${this.getId()}_entities`,
requestId: this._getEntitiesRequestId(),
requestName: i18n.translate('xpack.maps.source.esGeoLine.entityRequestName', {
defaultMessage: `load track entities ({layerName})`,
values: {
@ -431,7 +443,7 @@ export class ESGeoLineSource extends AbstractESAggSource {
},
});
const tracksResp = await this._runEsQuery({
requestId: `${this.getId()}_tracks`,
requestId: this._getTracksRequestId(),
requestName: getLayerFeaturesRequestName(layerName),
searchSource: tracksSearchSource,
registerCancelCallback,

View file

@ -568,6 +568,10 @@ export class ESSearchSource extends AbstractESSource implements IMvtVectorSource
return true;
}
getInspectorRequestIds(): string[] {
return [this.getId(), this._getFeaturesCountRequestId()];
}
async getGeoJsonWithMeta(
layerName: string,
requestMeta: VectorSourceRequestMeta,
@ -992,6 +996,10 @@ export class ESSearchSource extends AbstractESSource implements IMvtVectorSource
return !isWithin;
}
private _getFeaturesCountRequestId() {
return this.getId() + 'features_count';
}
async canLoadAllDocuments(
layerName: string,
requestMeta: VectorSourceRequestMeta,
@ -1003,7 +1011,7 @@ export class ESSearchSource extends AbstractESSource implements IMvtVectorSource
const searchSource = await this.makeSearchSource(requestMeta, 0);
searchSource.setField('trackTotalHits', maxResultWindow + 1);
const resp = await this._runEsQuery({
requestId: this.getId() + 'features_count',
requestId: this._getFeaturesCountRequestId(),
requestName: i18n.translate('xpack.maps.vectorSource.featuresCountRequestName', {
defaultMessage: 'load features count ({layerName})',
values: { layerName },

View file

@ -120,6 +120,10 @@ export class AbstractESSource extends AbstractVectorSource implements IESSource
return this._descriptor.id;
}
getInspectorRequestIds(): string[] {
return [this.getId()];
}
getApplyGlobalQuery(): boolean {
return this._descriptor.applyGlobalQuery;
}

View file

@ -0,0 +1,49 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { convertToGeoJson } from './convert_to_geojson';
describe('convertToGeoJson', () => {
test('should convert ES|QL response to feature collection', () => {
const resp = {
columns: [
{ name: 'location', type: 'geo_point' },
{ name: 'bytes', type: 'long' },
],
values: [
['POINT (-87.66208335757256 32.68147221766412)', 6901],
['POINT (-76.41376560553908 39.32566332165152)', 484],
],
};
const featureCollection = convertToGeoJson(resp);
expect(featureCollection).toEqual({
type: 'FeatureCollection',
features: [
{
geometry: {
coordinates: [-87.66208335757256, 32.68147221766412],
type: 'Point',
},
properties: {
bytes: 6901,
},
type: 'Feature',
},
{
geometry: {
coordinates: [-76.41376560553908, 39.32566332165152],
type: 'Point',
},
properties: {
bytes: 484,
},
type: 'Feature',
},
],
});
});
});

View file

@ -0,0 +1,47 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
// @ts-ignore
import { parse } from 'wellknown';
import { Feature, FeatureCollection, GeoJsonProperties } from 'geojson';
import type { ESQLSearchReponse } from '@kbn/es-types';
import { getGeometryColumnIndex } from './esql_utils';
export function convertToGeoJson(resp: ESQLSearchReponse): FeatureCollection {
const geometryIndex = getGeometryColumnIndex(resp.columns);
const features: Feature[] = [];
for (let i = 0; i < resp.values.length; i++) {
const hit = resp.values[i];
const wkt = hit[geometryIndex];
if (!wkt) {
continue;
}
try {
const geometry = parse(wkt);
const properties: GeoJsonProperties = {};
for (let j = 0; j < hit.length; j++) {
// do not store geometry in properties
if (j === geometryIndex) {
continue;
}
properties[resp.columns[j].name] = hit[j] as unknown;
}
features.push({
type: 'Feature',
geometry,
properties,
});
} catch (parseError) {
// TODO surface parse error in some kind of warning
}
}
return {
type: 'FeatureCollection',
features,
};
}

View file

@ -0,0 +1,119 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import React, { useEffect, useState } from 'react';
import { EuiSkeletonText } from '@elastic/eui';
import { ES_GEO_FIELD_TYPE } from '../../../../common/constants';
import type { ESQLSourceDescriptor } from '../../../../common/descriptor_types';
import { getIndexPatternService } from '../../../kibana_services';
import { ESQLEditor } from './esql_editor';
import { ESQL_GEO_POINT_TYPE } from './esql_utils';
interface Props {
mostCommonDataViewId?: string;
onSourceConfigChange: (sourceConfig: Partial<ESQLSourceDescriptor> | null) => void;
}
export function CreateSourceEditor(props: Props) {
const [isInitialized, setIsInitialized] = useState(false);
const [esql, setEsql] = useState('');
const [dateField, setDateField] = useState<string | undefined>();
useEffect(() => {
let ignore = false;
function getDataView() {
return props.mostCommonDataViewId
? getIndexPatternService().get(props.mostCommonDataViewId)
: getIndexPatternService().getDefaultDataView();
}
getDataView()
.then((dataView) => {
if (ignore) {
return;
}
if (dataView) {
let geoField: string | undefined;
const initialDateFields: string[] = [];
for (let i = 0; i < dataView.fields.length; i++) {
const field = dataView.fields[i];
if (!geoField && ES_GEO_FIELD_TYPE.GEO_POINT === field.type) {
geoField = field.name;
} else if ('date' === field.type) {
initialDateFields.push(field.name);
}
}
if (geoField) {
let initialDateField: string | undefined;
if (dataView.timeFieldName) {
initialDateField = dataView.timeFieldName;
} else if (initialDateFields.length) {
initialDateField = initialDateFields[0];
}
const initialEsql = `from ${dataView.getIndexPattern()} | keep ${geoField} | limit 10000`;
setDateField(initialDateField);
setEsql(initialEsql);
props.onSourceConfigChange({
columns: [
{
name: geoField,
type: ESQL_GEO_POINT_TYPE,
},
],
dateField: initialDateField,
esql: initialEsql,
});
}
}
setIsInitialized(true);
})
.catch((err) => {
if (ignore) {
return;
}
setIsInitialized(true);
});
return () => {
ignore = true;
};
// only run on mount
// eslint-disable-next-line react-hooks/exhaustive-deps
}, []);
return (
<EuiSkeletonText lines={3} isLoading={!isInitialized}>
<ESQLEditor
esql={esql}
onESQLChange={(change: {
columns: ESQLSourceDescriptor['columns'];
dateFields: string[];
esql: string;
}) => {
let nextDateField = dateField;
if (!dateField || !change.dateFields.includes(dateField)) {
nextDateField = change.dateFields.length ? change.dateFields[0] : undefined;
}
setDateField(nextDateField);
setEsql(change.esql);
const sourceConfig =
change.esql && change.esql.length
? {
columns: change.columns,
dateField: nextDateField,
esql: change.esql,
}
: null;
props.onSourceConfigChange(sourceConfig);
}}
/>
</EuiSkeletonText>
);
}

View file

@ -0,0 +1,106 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import React, { useState } from 'react';
import { i18n } from '@kbn/i18n';
import { isEqual } from 'lodash';
import useMountedState from 'react-use/lib/useMountedState';
import type { AggregateQuery } from '@kbn/es-query';
import type { ESQLColumn } from '@kbn/es-types';
import { TextBasedLangEditor } from '@kbn/text-based-languages/public';
import { getESQLMeta, verifyGeometryColumn } from './esql_utils';
interface Props {
esql: string;
onESQLChange: ({
columns,
dateFields,
esql,
}: {
columns: ESQLColumn[];
dateFields: string[];
esql: string;
}) => void;
}
export function ESQLEditor(props: Props) {
const isMounted = useMountedState();
const [error, setError] = useState<Error | undefined>();
const [warning, setWarning] = useState<string | undefined>();
const [isLoading, setIsLoading] = useState(false);
const [localQuery, setLocalQuery] = useState<AggregateQuery>({ esql: props.esql });
return (
<>
<TextBasedLangEditor
query={localQuery}
onTextLangQueryChange={setLocalQuery}
onTextLangQuerySubmit={async (query) => {
if (!query) {
return;
}
if (warning) {
setWarning(undefined);
}
if (error) {
setError(undefined);
}
setIsLoading(true);
try {
const esql = (query as { esql: string }).esql;
const esqlMeta = await getESQLMeta(esql);
if (!isMounted()) {
return;
}
verifyGeometryColumn(esqlMeta.columns);
if (esqlMeta.columns.length >= 6) {
setWarning(
i18n.translate('xpack.maps.esqlSource.tooManyColumnsWarning', {
defaultMessage: `ES|QL statement returns {count} columns. For faster maps, use 'DROP' or 'KEEP' to narrow columns.`,
values: {
count: esqlMeta.columns.length,
},
})
);
}
props.onESQLChange({
columns: esqlMeta.columns,
dateFields: esqlMeta.dateFields,
esql,
});
} catch (err) {
if (!isMounted()) {
return;
}
setError(err);
props.onESQLChange({
columns: [],
dateFields: [],
esql: '',
});
}
setIsLoading(false);
}}
errors={error ? [error] : undefined}
warning={warning}
expandCodeEditor={(status: boolean) => {
// never called because hideMinimizeButton hides UI
}}
isCodeEditorExpanded
hideMinimizeButton
editorIsInline
hideRunQueryText
isLoading={isLoading}
disableSubmitAction={isEqual(localQuery, props.esql)}
/>
</>
);
}

View file

@ -0,0 +1,47 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import React from 'react';
import { i18n } from '@kbn/i18n';
import { CreateSourceEditor } from './create_source_editor';
import { LayerWizard, RenderWizardArguments } from '../../layers';
import { sourceTitle, ESQLSource } from './esql_source';
import { LAYER_WIZARD_CATEGORY, WIZARD_ID } from '../../../../common/constants';
import type { ESQLSourceDescriptor } from '../../../../common/descriptor_types';
import { GeoJsonVectorLayer } from '../../layers/vector_layer';
import { DocumentsLayerIcon } from '../../layers/wizards/icons/documents_layer_icon';
export const esqlLayerWizardConfig: LayerWizard = {
id: WIZARD_ID.ESQL,
order: 10,
categories: [LAYER_WIZARD_CATEGORY.ELASTICSEARCH],
description: i18n.translate('xpack.maps.source.esqlDescription', {
defaultMessage: 'Create a map layer using the Elasticsearch Query Language',
}),
icon: DocumentsLayerIcon,
isBeta: true,
renderWizard: ({ previewLayers, mapColors, mostCommonDataViewId }: RenderWizardArguments) => {
const onSourceConfigChange = (sourceConfig: Partial<ESQLSourceDescriptor> | null) => {
if (!sourceConfig) {
previewLayers([]);
return;
}
const sourceDescriptor = ESQLSource.createDescriptor(sourceConfig);
const layerDescriptor = GeoJsonVectorLayer.createDescriptor({ sourceDescriptor }, mapColors);
previewLayers([layerDescriptor]);
};
return (
<CreateSourceEditor
mostCommonDataViewId={mostCommonDataViewId}
onSourceConfigChange={onSourceConfigChange}
/>
);
},
title: sourceTitle,
};

View file

@ -0,0 +1,292 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import React from 'react';
import { i18n } from '@kbn/i18n';
import { lastValueFrom } from 'rxjs';
import { tap } from 'rxjs/operators';
import { v4 as uuidv4 } from 'uuid';
import { Adapters } from '@kbn/inspector-plugin/common/adapters';
import { buildEsQuery, getIndexPatternFromESQLQuery, getLimitFromESQLQuery } from '@kbn/es-query';
import type { BoolQuery, Filter, Query } from '@kbn/es-query';
import type { ESQLSearchReponse } from '@kbn/es-types';
import { getEsQueryConfig } from '@kbn/data-service/src/es_query';
import { getTime } from '@kbn/data-plugin/public';
import { FIELD_ORIGIN, SOURCE_TYPES, VECTOR_SHAPE_TYPE } from '../../../../common/constants';
import type {
ESQLSourceDescriptor,
VectorSourceRequestMeta,
} from '../../../../common/descriptor_types';
import { createExtentFilter } from '../../../../common/elasticsearch_util';
import { DataRequest } from '../../util/data_request';
import { isValidStringConfig } from '../../util/valid_string_config';
import type { SourceEditorArgs } from '../source';
import { AbstractVectorSource, getLayerFeaturesRequestName } from '../vector_source';
import type { IVectorSource, GeoJsonWithMeta, SourceStatus } from '../vector_source';
import type { IField } from '../../fields/field';
import { InlineField } from '../../fields/inline_field';
import { getData, getUiSettings } from '../../../kibana_services';
import { convertToGeoJson } from './convert_to_geojson';
import { getFieldType, getGeometryColumnIndex } from './esql_utils';
import { UpdateSourceEditor } from './update_source_editor';
type ESQLSourceSyncMeta = Pick<
ESQLSourceDescriptor,
'columns' | 'dateField' | 'esql' | 'narrowByMapBounds'
>;
export const sourceTitle = i18n.translate('xpack.maps.source.esqlSearchTitle', {
defaultMessage: 'ES|QL',
});
export class ESQLSource extends AbstractVectorSource implements IVectorSource {
readonly _descriptor: ESQLSourceDescriptor;
static createDescriptor(descriptor: Partial<ESQLSourceDescriptor>): ESQLSourceDescriptor {
if (!isValidStringConfig(descriptor.esql)) {
throw new Error('Cannot create ESQLSourceDescriptor when esql is not provided');
}
return {
...descriptor,
id: isValidStringConfig(descriptor.id) ? descriptor.id! : uuidv4(),
type: SOURCE_TYPES.ESQL,
esql: descriptor.esql!,
columns: descriptor.columns ? descriptor.columns : [],
narrowByGlobalSearch:
typeof descriptor.narrowByGlobalSearch !== 'undefined'
? descriptor.narrowByGlobalSearch
: true,
narrowByMapBounds:
typeof descriptor.narrowByMapBounds !== 'undefined' ? descriptor.narrowByMapBounds : true,
applyForceRefresh:
typeof descriptor.applyForceRefresh !== 'undefined' ? descriptor.applyForceRefresh : true,
};
}
constructor(descriptor: ESQLSourceDescriptor) {
super(ESQLSource.createDescriptor(descriptor));
this._descriptor = descriptor;
}
private _getRequestId(): string {
return this._descriptor.id;
}
async getDisplayName() {
const pattern: string = getIndexPatternFromESQLQuery(this._descriptor.esql);
return pattern ? pattern : 'ES|QL';
}
async supportsFitToBounds(): Promise<boolean> {
return false;
}
getInspectorRequestIds() {
return [this._getRequestId()];
}
isQueryAware() {
return true;
}
getApplyGlobalQuery() {
return this._descriptor.narrowByGlobalSearch;
}
async isTimeAware() {
return !!this._descriptor.dateField;
}
getApplyGlobalTime() {
return !!this._descriptor.dateField;
}
getApplyForceRefresh() {
return this._descriptor.applyForceRefresh;
}
isFilterByMapBounds() {
return this._descriptor.narrowByMapBounds;
}
async getSupportedShapeTypes() {
return [VECTOR_SHAPE_TYPE.POINT];
}
supportsJoins() {
return false; // Joins will be part of ESQL statement and not client side join
}
async getGeoJsonWithMeta(
layerName: string,
requestMeta: VectorSourceRequestMeta,
registerCancelCallback: (callback: () => void) => void,
isRequestStillActive: () => boolean,
inspectorAdapters: Adapters
): Promise<GeoJsonWithMeta> {
const limit = getLimitFromESQLQuery(this._descriptor.esql);
const params: { query: string; filter?: { bool: BoolQuery } } = {
query: this._descriptor.esql,
};
const query: Query[] = [];
const filters: Filter[] = [];
if (this._descriptor.narrowByGlobalSearch) {
if (requestMeta.query) {
query.push(requestMeta.query);
}
if (requestMeta.embeddableSearchContext?.query) {
query.push(requestMeta.embeddableSearchContext.query);
}
filters.push(...requestMeta.filters);
if (requestMeta.embeddableSearchContext) {
filters.push(...requestMeta.embeddableSearchContext.filters);
}
}
if (this._descriptor.narrowByMapBounds && requestMeta.buffer) {
const geoField =
this._descriptor.columns[getGeometryColumnIndex(this._descriptor.columns)]?.name;
if (geoField) {
const extentFilter = createExtentFilter(requestMeta.buffer, [geoField]);
filters.push(extentFilter);
}
}
if (requestMeta.applyGlobalTime) {
const timeRange = requestMeta.timeslice
? {
from: new Date(requestMeta.timeslice.from).toISOString(),
to: new Date(requestMeta.timeslice.to).toISOString(),
mode: 'absolute' as 'absolute',
}
: requestMeta.timeFilters;
const timeFilter = getTime(undefined, timeRange, {
fieldName: this._descriptor.dateField,
});
if (timeFilter) {
filters.push(timeFilter);
}
}
params.filter = buildEsQuery(undefined, query, filters, getEsQueryConfig(getUiSettings()));
const requestResponder = inspectorAdapters.requests!.start(
getLayerFeaturesRequestName(layerName),
{
id: this._getRequestId(),
}
);
requestResponder.json(params);
const { rawResponse, requestParams } = await lastValueFrom(
getData()
.search.search(
{ params },
{
strategy: 'esql',
}
)
.pipe(
tap({
error(error) {
requestResponder.error({
json: 'attributes' in error ? error.attributes : { message: error.message },
});
},
})
)
);
requestResponder.ok({ json: rawResponse, requestParams });
const esqlSearchResponse = rawResponse as unknown as ESQLSearchReponse;
const resultsCount = esqlSearchResponse.values.length;
return {
data: convertToGeoJson(esqlSearchResponse),
meta: {
resultsCount,
areResultsTrimmed: resultsCount >= limit,
},
};
}
getSourceStatus(sourceDataRequest?: DataRequest): SourceStatus {
const meta = sourceDataRequest ? sourceDataRequest.getMeta() : null;
if (!meta) {
// no tooltip content needed when there is no feature collection or meta
return {
tooltipContent: null,
areResultsTrimmed: false,
};
}
if (meta.areResultsTrimmed) {
return {
tooltipContent: i18n.translate('xpack.maps.esqlSearch.resultsTrimmedMsg', {
defaultMessage: `Results limited to first {count} rows.`,
values: { count: meta.resultsCount?.toLocaleString() },
}),
areResultsTrimmed: true,
};
}
return {
tooltipContent: i18n.translate('xpack.maps.esqlSearch.rowCountMsg', {
defaultMessage: `Found {count} rows.`,
values: { count: meta.resultsCount?.toLocaleString() },
}),
areResultsTrimmed: false,
};
}
getFieldByName(fieldName: string): IField | null {
const column = this._descriptor.columns.find(({ name }) => {
return name === fieldName;
});
const fieldType = column ? getFieldType(column) : undefined;
return column && fieldType
? new InlineField({
fieldName: column.name,
source: this,
origin: FIELD_ORIGIN.SOURCE,
dataType: fieldType,
})
: null;
}
async getFields() {
const fields: IField[] = [];
this._descriptor.columns.forEach((column) => {
const fieldType = getFieldType(column);
if (fieldType) {
fields.push(
new InlineField({
fieldName: column.name,
source: this,
origin: FIELD_ORIGIN.SOURCE,
dataType: fieldType,
})
);
}
});
return fields;
}
renderSourceSettingsEditor({ onChange }: SourceEditorArgs) {
return <UpdateSourceEditor onChange={onChange} sourceDescriptor={this._descriptor} />;
}
getSyncMeta(): ESQLSourceSyncMeta {
return {
columns: this._descriptor.columns,
dateField: this._descriptor.dateField,
esql: this._descriptor.esql,
narrowByMapBounds: this._descriptor.narrowByMapBounds,
};
}
}

View file

@ -0,0 +1,130 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { i18n } from '@kbn/i18n';
import { lastValueFrom } from 'rxjs';
import { getIndexPatternFromESQLQuery } from '@kbn/es-query';
import type { ESQLColumn } from '@kbn/es-types';
import { getData, getIndexPatternService } from '../../../kibana_services';
export const ESQL_GEO_POINT_TYPE = 'geo_point';
const NO_GEOMETRY_COLUMN_ERROR_MSG = i18n.translate(
'xpack.maps.source.esql.noGeometryColumnErrorMsg',
{
defaultMessage: 'Elasticsearch ES|QL query does not have a geometry column.',
}
);
function isGeometryColumn(column: ESQLColumn) {
return column.type === ESQL_GEO_POINT_TYPE;
}
export function verifyGeometryColumn(columns: ESQLColumn[]) {
const geometryColumns = columns.filter(isGeometryColumn);
if (geometryColumns.length === 0) {
throw new Error(NO_GEOMETRY_COLUMN_ERROR_MSG);
}
if (geometryColumns.length > 1) {
throw new Error(
i18n.translate('xpack.maps.source.esql.multipleGeometryColumnErrorMsg', {
defaultMessage: `Elasticsearch ES|QL query has {count} geometry columns when only 1 is allowed. Use 'DROP' or 'KEEP' to narrow columns.`,
values: {
count: geometryColumns.length,
},
})
);
}
}
export function getGeometryColumnIndex(columns: ESQLColumn[]) {
const index = columns.findIndex(isGeometryColumn);
if (index === -1) {
throw new Error(NO_GEOMETRY_COLUMN_ERROR_MSG);
}
return index;
}
export async function getESQLMeta(esql: string) {
return {
columns: await getColumns(esql),
dateFields: await getDateFields(esql),
};
}
/*
* Map column.type to field type
* Supported column types https://www.elastic.co/guide/en/elasticsearch/reference/master/esql-limitations.html#_supported_types
*/
export function getFieldType(column: ESQLColumn) {
switch (column.type) {
case 'boolean':
case 'date':
case 'ip':
case 'keyword':
case 'text':
return 'string';
case 'double':
case 'int':
case 'long':
case 'unsigned_long':
return 'number';
default:
return undefined;
}
}
async function getColumns(esql: string) {
const params = {
query: esql + ' | limit 0',
};
try {
const resp = await lastValueFrom(
getData().search.search(
{ params },
{
strategy: 'esql',
}
)
);
return (resp.rawResponse as unknown as { columns: ESQLColumn[] }).columns;
} catch (error) {
throw new Error(
i18n.translate('xpack.maps.source.esql.getColumnsErrorMsg', {
defaultMessage: 'Unable to load columns. {errorMessage}',
values: { errorMessage: error.message },
})
);
}
}
export async function getDateFields(esql: string) {
const pattern: string = getIndexPatternFromESQLQuery(esql);
try {
// TODO pass field type filter to getFieldsForWildcard when field type filtering is supported
return (await getIndexPatternService().getFieldsForWildcard({ pattern }))
.filter((field) => {
return field.type === 'date';
})
.map((field) => {
return field.name;
});
} catch (error) {
throw new Error(
i18n.translate('xpack.maps.source.esql.getFieldsErrorMsg', {
defaultMessage: `Unable to load date fields from index pattern: {pattern}. {errorMessage}`,
values: {
errorMessage: error.message,
pattern,
},
})
);
}
}

View file

@ -0,0 +1,9 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export { ESQLSource } from './esql_source';
export { esqlLayerWizardConfig } from './esql_layer_wizard';

View file

@ -0,0 +1,205 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import React, { ChangeEvent, useEffect, useMemo, useState } from 'react';
import {
EuiFormRow,
EuiPanel,
EuiSelect,
EuiSkeletonText,
EuiSpacer,
EuiSwitch,
EuiSwitchEvent,
EuiTitle,
EuiToolTip,
} from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import { getIndexPatternFromESQLQuery } from '@kbn/es-query';
import type { ESQLSourceDescriptor } from '../../../../common/descriptor_types';
import type { OnSourceChangeArgs } from '../source';
import { ForceRefreshCheckbox } from '../../../components/force_refresh_checkbox';
import { ESQLEditor } from './esql_editor';
import { getDateFields } from './esql_utils';
interface Props {
onChange(...args: OnSourceChangeArgs[]): void;
sourceDescriptor: ESQLSourceDescriptor;
}
export function UpdateSourceEditor(props: Props) {
const [dateFields, setDateFields] = useState<string[]>([]);
const [isInitialized, setIsInitialized] = useState(false);
useEffect(() => {
let ignore = false;
getDateFields(props.sourceDescriptor.esql)
.then((initialDateFields) => {
if (ignore) {
return;
}
setDateFields(initialDateFields);
setIsInitialized(true);
})
.catch((err) => {
if (ignore) {
return;
}
setIsInitialized(true);
});
return () => {
ignore = true;
};
// only run on mount
// eslint-disable-next-line react-hooks/exhaustive-deps
}, []);
const dateSelectOptions = useMemo(() => {
return dateFields.map((dateField) => {
return {
value: dateField,
text: dateField,
};
});
}, [dateFields]);
const narrowByTimeInput = (
<EuiSwitch
label={i18n.translate('xpack.maps.esqlSource.narrowByGlobalTimeLabel', {
defaultMessage: `Narrow ES|QL statement by global time`,
})}
checked={dateFields.length === 0 ? false : !!props.sourceDescriptor.dateField}
onChange={(event: EuiSwitchEvent) => {
if (!event.target.checked) {
props.onChange({ propName: 'dateField', value: undefined });
return;
}
if (dateFields.length) {
props.onChange({ propName: 'dateField', value: dateFields[0] });
}
}}
disabled={dateFields.length === 0}
compressed
/>
);
return (
<>
<EuiPanel>
<EuiTitle size="xs">
<h5>
{i18n.translate('xpack.maps.esqlSearch.sourceEditorTitle', {
defaultMessage: 'ES|QL',
})}
</h5>
</EuiTitle>
<EuiSpacer size="m" />
<EuiSkeletonText lines={3} isLoading={!isInitialized}>
<ESQLEditor
esql={props.sourceDescriptor.esql}
onESQLChange={(change: {
columns: ESQLSourceDescriptor['columns'];
dateFields: string[];
esql: string;
}) => {
setDateFields(change.dateFields);
const changes: OnSourceChangeArgs[] = [
{ propName: 'columns', value: change.columns },
{ propName: 'esql', value: change.esql },
];
if (
props.sourceDescriptor.dateField &&
!change.dateFields.includes(props.sourceDescriptor.dateField)
) {
changes.push({
propName: 'dateField',
value: change.dateFields.length ? change.dateFields[0] : undefined,
});
}
props.onChange(...changes);
}}
/>
<EuiSpacer size="m" />
<EuiFormRow>
<EuiSwitch
label={i18n.translate('xpack.maps.esqlSource.narrowByMapExtentLabel', {
defaultMessage: 'Narrow ES|QL statement by visible map area',
})}
checked={props.sourceDescriptor.narrowByMapBounds}
onChange={(event: EuiSwitchEvent) => {
props.onChange({ propName: 'narrowByMapBounds', value: event.target.checked });
}}
compressed
/>
</EuiFormRow>
<EuiFormRow>
<EuiSwitch
label={i18n.translate('xpack.maps.esqlSource.narrowByGlobalSearchLabel', {
defaultMessage: `Narrow ES|QL statement by global search`,
})}
checked={props.sourceDescriptor.narrowByGlobalSearch}
onChange={(event: EuiSwitchEvent) => {
props.onChange({ propName: 'narrowByGlobalSearch', value: event.target.checked });
}}
compressed
/>
</EuiFormRow>
<EuiFormRow>
{dateFields.length === 0 ? (
<EuiToolTip
position="top"
content={i18n.translate('xpack.maps.esqlSource.noDateFieldsDisabledMsg', {
defaultMessage: `No date fields are available from index pattern: {pattern}.`,
values: {
pattern: getIndexPatternFromESQLQuery(props.sourceDescriptor.esql),
},
})}
>
{narrowByTimeInput}
</EuiToolTip>
) : (
narrowByTimeInput
)}
</EuiFormRow>
{props.sourceDescriptor.dateField && (
<EuiFormRow
label={i18n.translate('xpack.maps.source.esqlSource.dateFieldSelectLabel', {
defaultMessage: 'Date field',
})}
display="columnCompressed"
>
<EuiSelect
options={dateSelectOptions}
value={props.sourceDescriptor.dateField}
onChange={(e: ChangeEvent<HTMLSelectElement>) => {
props.onChange({ propName: 'dateField', value: e.target.value });
}}
compressed
/>
</EuiFormRow>
)}
<ForceRefreshCheckbox
applyForceRefresh={props.sourceDescriptor.applyForceRefresh}
setApplyForceRefresh={(applyForceRefresh: boolean) => {
props.onChange({ propName: 'applyForceRefresh', value: applyForceRefresh });
}}
/>
</EuiSkeletonText>
</EuiPanel>
<EuiSpacer size="s" />
</>
);
}

View file

@ -229,4 +229,8 @@ export class MVTSingleLayerVectorSource extends AbstractSource implements IMvtVe
// Its not possible to filter by geometry for vector tile sources since there is no way to get original geometry
return [];
}
getInspectorRequestIds(): string[] {
return [];
}
}

View file

@ -13,6 +13,7 @@ import { ESGeoGridSource } from './es_geo_grid_source';
import { ESGeoLineSource } from './es_geo_line_source';
import { ESPewPewSource } from './es_pew_pew_source';
import { ESSearchSource } from './es_search_source';
import { ESQLSource } from './esql_source';
import { GeoJsonFileSource } from './geojson_file_source';
import { KibanaTilemapSource } from './kibana_tilemap_source';
import { MVTSingleLayerVectorSource } from './mvt_single_layer_vector_source';
@ -56,6 +57,11 @@ export function setupSources() {
type: SOURCE_TYPES.ES_SEARCH,
});
registerSource({
ConstructorFunction: ESQLSource,
type: SOURCE_TYPES.ESQL,
});
registerSource({
ConstructorFunction: GeoJsonFileSource,
type: SOURCE_TYPES.GEOJSON_FILE,

View file

@ -59,6 +59,9 @@ export interface ISource {
isTimeAware(): Promise<boolean>;
getImmutableProperties(dataFilters: DataFilters): Promise<ImmutableSourceProperty[]>;
getAttributionProvider(): (() => Promise<Attribution[]>) | null;
/*
* Returns true when source implements IESSource interface
*/
isESSource(): boolean;
renderSourceSettingsEditor(sourceEditorArgs: SourceEditorArgs): ReactElement<any> | null;
supportsFitToBounds(): Promise<boolean>;

View file

@ -133,6 +133,11 @@ export interface IVectorSource extends ISource {
mbFeature,
onClose,
}: GetFeatureActionsArgs): TooltipFeatureAction[];
/*
* Provide unique ids for managing source requests in Inspector
*/
getInspectorRequestIds(): string[];
}
export class AbstractVectorSource extends AbstractSource implements IVectorSource {
@ -178,7 +183,7 @@ export class AbstractVectorSource extends AbstractSource implements IVectorSourc
isRequestStillActive: () => boolean,
inspectorAdapters: Adapters
): Promise<GeoJsonWithMeta> {
throw new Error('Should implement VectorSource#getGeoJson');
throw new Error('Should implement VectorSource#getGeoJsonWithMeta');
}
hasTooltipProperties() {
@ -285,4 +290,8 @@ export class AbstractVectorSource extends AbstractSource implements IVectorSourc
]
: [];
}
getInspectorRequestIds(): string[] {
return [];
}
}

View file

@ -24,12 +24,12 @@ export function ForceRefreshCheckbox({ applyForceRefresh, setApplyForceRefresh }
<EuiToolTip
position="top"
content={i18n.translate('xpack.maps.filterEditor.applyForceRefreshTooltip', {
defaultMessage: `When enabled, layer re-fetches data when automatic refresh fires and when "Refresh" is clicked.`,
defaultMessage: `When on, layer re-fetches data when automatic refresh fires and when "Refresh" is clicked.`,
})}
>
<EuiSwitch
label={i18n.translate('xpack.maps.filterEditor.applyForceRefreshLabel', {
defaultMessage: `Apply global refresh to layer data`,
defaultMessage: `Re-fetch layer data on refresh`,
})}
checked={applyForceRefresh}
onChange={onChange}

View file

@ -27,6 +27,7 @@ export const FlyoutBody = (props: Props) => {
const renderWizardArgs = {
previewLayers: props.previewLayers,
mapColors: props.mapColors,
mostCommonDataViewId: props.mostCommonDataViewId,
currentStepId: props.currentStepId,
isOnFinalStep: props.isOnFinalStep,
enableNextBtn: props.enableNextBtn,

View file

@ -8,11 +8,12 @@
import { connect } from 'react-redux';
import { FlyoutBody } from './flyout_body';
import { MapStoreState } from '../../../reducers/store';
import { getMapColors } from '../../../selectors/map_selectors';
import { getMapColors, getMostCommonDataViewId } from '../../../selectors/map_selectors';
function mapStateToProps(state: MapStoreState) {
return {
mapColors: getMapColors(state),
mostCommonDataViewId: getMostCommonDataViewId(state),
};
}

View file

@ -160,7 +160,13 @@ export class LayerWizardSelect extends Component<Props, State> {
<EuiCard
title={layerWizard.title}
titleSize="xs"
betaBadgeProps={{ label: layerWizard.isBeta ? 'beta' : '' }}
betaBadgeProps={{
label: layerWizard.isBeta
? i18n.translate('xpack.maps.layerWizardSelect.technicalPreviewLabel', {
defaultMessage: 'Technical preview',
})
: '',
}}
icon={icon}
onClick={onClick}
description={layerWizard.description}

View file

@ -435,6 +435,42 @@ export const getQueryableUniqueIndexPatternIds = createSelector(
}
);
export const getMostCommonDataViewId = createSelector(
getLayerList,
getWaitingForMapReadyLayerListRaw,
(layerList, waitingForMapReadyLayerList) => {
const counts: { [key: string]: number } = {};
function incrementCount(ids: string[]) {
ids.forEach((id) => {
const count = counts.hasOwnProperty(id) ? counts[id] : 0;
counts[id] = count + 1;
});
}
if (waitingForMapReadyLayerList.length) {
waitingForMapReadyLayerList.forEach((layerDescriptor) => {
const layer = createLayerInstance(layerDescriptor, []); // custom icons not needed, layer instance only used to get index pattern ids
incrementCount(layer.getIndexPatternIds());
});
} else {
layerList.forEach((layer) => {
incrementCount(layer.getIndexPatternIds());
});
}
let mostCommonId: string | undefined;
let mostCommonCount = 0;
Object.keys(counts).forEach((id) => {
if (counts[id] > mostCommonCount) {
mostCommonId = id;
mostCommonCount = counts[id];
}
});
return mostCommonId;
}
);
export const getGeoFieldNames = createSelector(
getLayerList,
getWaitingForMapReadyLayerListRaw,

View file

@ -120,6 +120,24 @@ export function registerMapsUsageCollector(usageCollection?: UsageCollectionSetu
_meta: { description: 'total number of es machine learning anomaly layers in cluster' },
},
},
esql: {
min: {
type: 'long',
_meta: { description: 'min number of ES|QL layers per map' },
},
max: {
type: 'long',
_meta: { description: 'max number of ES|QL layers per map' },
},
avg: {
type: 'float',
_meta: { description: 'avg number of ES|QL layers per map' },
},
total: {
type: 'long',
_meta: { description: 'total number of ES|QL layers in cluster' },
},
},
es_point_to_point: {
min: {
type: 'long',

View file

@ -78,6 +78,9 @@
"@kbn/search-response-warnings",
"@kbn/calculate-width-from-char-count",
"@kbn/content-management-table-list-view-common",
"@kbn/text-based-languages",
"@kbn/es-types",
"@kbn/data-service",
],
"exclude": [
"target/**/*",

View file

@ -388,4 +388,8 @@ export class AnomalySource implements IVectorSource {
async getDefaultFields(): Promise<Record<string, Record<string, string>>> {
return {};
}
getInspectorRequestIds() {
return [];
}
}

View file

@ -8222,6 +8222,34 @@
}
}
},
"esql": {
"properties": {
"min": {
"type": "long",
"_meta": {
"description": "min number of ES|QL layers per map"
}
},
"max": {
"type": "long",
"_meta": {
"description": "max number of ES|QL layers per map"
}
},
"avg": {
"type": "float",
"_meta": {
"description": "avg number of ES|QL layers per map"
}
},
"total": {
"type": "long",
"_meta": {
"description": "total number of ES|QL layers in cluster"
}
}
}
},
"es_point_to_point": {
"properties": {
"min": {

View file

@ -51,27 +51,28 @@ export default function ({ getService }: FtrProviderContext) {
delete mapUsage.timeCaptured;
expect(mapUsage).eql({
mapsTotalCount: 27,
mapsTotalCount: 28,
basemaps: {},
joins: { term: { min: 1, max: 1, total: 4, avg: 0.14814814814814814 } },
joins: { term: { min: 1, max: 1, total: 4, avg: 0.14285714285714285 } },
layerTypes: {
es_docs: { min: 1, max: 3, total: 20, avg: 0.7407407407407407 },
es_agg_grids: { min: 1, max: 1, total: 6, avg: 0.2222222222222222 },
es_point_to_point: { min: 1, max: 1, total: 1, avg: 0.037037037037037035 },
es_top_hits: { min: 1, max: 1, total: 2, avg: 0.07407407407407407 },
es_agg_heatmap: { min: 1, max: 1, total: 1, avg: 0.037037037037037035 },
kbn_tms_raster: { min: 1, max: 1, total: 1, avg: 0.037037037037037035 },
ems_basemap: { min: 1, max: 1, total: 1, avg: 0.037037037037037035 },
ems_region: { min: 1, max: 1, total: 1, avg: 0.037037037037037035 },
es_docs: { min: 1, max: 3, total: 20, avg: 0.7142857142857143 },
es_agg_grids: { min: 1, max: 1, total: 6, avg: 0.21428571428571427 },
es_point_to_point: { min: 1, max: 1, total: 1, avg: 0.03571428571428571 },
es_top_hits: { min: 1, max: 1, total: 2, avg: 0.07142857142857142 },
es_agg_heatmap: { min: 1, max: 1, total: 1, avg: 0.03571428571428571 },
esql: { min: 1, max: 1, total: 1, avg: 0.03571428571428571 },
kbn_tms_raster: { min: 1, max: 1, total: 1, avg: 0.03571428571428571 },
ems_basemap: { min: 1, max: 1, total: 1, avg: 0.03571428571428571 },
ems_region: { min: 1, max: 1, total: 1, avg: 0.03571428571428571 },
},
resolutions: {
coarse: { min: 1, max: 1, total: 4, avg: 0.14814814814814814 },
super_fine: { min: 1, max: 1, total: 3, avg: 0.1111111111111111 },
coarse: { min: 1, max: 1, total: 4, avg: 0.14285714285714285 },
super_fine: { min: 1, max: 1, total: 3, avg: 0.10714285714285714 },
},
scalingOptions: {
limit: { min: 1, max: 3, total: 15, avg: 0.5555555555555556 },
clusters: { min: 1, max: 1, total: 1, avg: 0.037037037037037035 },
mvt: { min: 1, max: 1, total: 4, avg: 0.14814814814814814 },
limit: { min: 1, max: 3, total: 15, avg: 0.5357142857142857 },
clusters: { min: 1, max: 1, total: 1, avg: 0.03571428571428571 },
mvt: { min: 1, max: 1, total: 4, avg: 0.14285714285714285 },
},
attributesPerMap: {
customIconsCount: {
@ -80,51 +81,51 @@ export default function ({ getService }: FtrProviderContext) {
min: 0,
},
dataSourcesCount: {
avg: 1.1851851851851851,
avg: 1.1785714285714286,
max: 6,
min: 1,
},
emsVectorLayersCount: {
idThatDoesNotExitForEMSFileSource: {
avg: 0.037037037037037035,
avg: 0.03571428571428571,
max: 1,
min: 1,
},
},
layerTypesCount: {
BLENDED_VECTOR: {
avg: 0.037037037037037035,
avg: 0.03571428571428571,
max: 1,
min: 1,
},
EMS_VECTOR_TILE: {
avg: 0.037037037037037035,
avg: 0.03571428571428571,
max: 1,
min: 1,
},
GEOJSON_VECTOR: {
avg: 0.8148148148148148,
avg: 0.8214285714285714,
max: 5,
min: 1,
},
HEATMAP: {
avg: 0.037037037037037035,
avg: 0.03571428571428571,
max: 1,
min: 1,
},
MVT_VECTOR: {
avg: 0.25925925925925924,
avg: 0.25,
max: 1,
min: 1,
},
RASTER_TILE: {
avg: 0.037037037037037035,
avg: 0.03571428571428571,
max: 1,
min: 1,
},
},
layersCount: {
avg: 1.2222222222222223,
avg: 1.2142857142857142,
max: 7,
min: 1,
},

View file

@ -0,0 +1,34 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ getPageObjects, getService }: FtrProviderContext) {
const PageObjects = getPageObjects(['maps']);
const security = getService('security');
describe('esql', () => {
before(async () => {
await security.testUser.setRoles(['global_maps_all', 'test_logstash_reader'], {
skipBrowserRefresh: true,
});
await PageObjects.maps.loadSavedMap('esql example');
});
after(async () => {
await security.testUser.restoreDefaults();
});
it('should display ES|QL statement results on map', async () => {
const tooltipText = await PageObjects.maps.getLayerTocTooltipMsg('logstash-*');
expect(tooltipText).to.equal(
'logstash-*\nFound 5 rows.\nResults narrowed by global time\nResults narrowed by visible map area'
);
});
});
}

View file

@ -58,6 +58,7 @@ export default function ({ loadTestFile, getService }) {
);
});
loadTestFile(require.resolve('./esql_source'));
loadTestFile(require.resolve('./documents_source'));
loadTestFile(require.resolve('./blended_vector_layer'));
loadTestFile(require.resolve('./saved_object_management'));

View file

@ -1168,3 +1168,25 @@
"updated_at": "2022-06-08T18:03:37.060Z",
"version": "WzE0MSwxXQ=="
}
{
"id": "f3bb9828-ad65-4feb-87d4-7a9f7deff8d5",
"type": "map",
"namespaces": [
"default"
],
"updated_at": "2023-12-17T15:28:47.759Z",
"created_at": "2023-12-17T15:28:47.759Z",
"version": "WzU0LDFd",
"attributes": {
"title": "esql example",
"description": "",
"mapStateJSON": "{\"adHocDataViews\":[],\"zoom\":4.1,\"center\":{\"lon\":-100.61091,\"lat\":33.23887},\"timeFilters\":{\"from\":\"2015-09-20T00:00:00.000Z\",\"to\":\"2015-09-20T01:00:00.000Z\"},\"refreshConfig\":{\"isPaused\":true,\"interval\":1000},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"settings\":{\"autoFitToDataBounds\":false,\"backgroundColor\":\"#ffffff\",\"customIcons\":[],\"disableInteractive\":false,\"disableTooltipControl\":false,\"hideToolbarOverlay\":false,\"hideLayerControl\":false,\"hideViewControl\":false,\"initialLocation\":\"LAST_SAVED_LOCATION\",\"fixedLocation\":{\"lat\":0,\"lon\":0,\"zoom\":2},\"browserLocation\":{\"zoom\":2},\"keydownScrollZoom\":false,\"maxZoom\":24,\"minZoom\":0,\"showScaleControl\":false,\"showSpatialFilters\":true,\"showTimesliderToggleButton\":true,\"spatialFiltersAlpa\":0.3,\"spatialFiltersFillColor\":\"#DA8B45\",\"spatialFiltersLineColor\":\"#DA8B45\"}}",
"layerListJSON": "[{\"sourceDescriptor\":{\"columns\":[{\"name\":\"geo.coordinates\",\"type\":\"geo_point\"}],\"dateField\":\"@timestamp\",\"esql\":\"from logstash-* | KEEP geo.coordinates | limit 10000\",\"id\":\"fad0e2eb-9278-415c-bdc8-1189a46eac0b\",\"type\":\"ESQL\",\"narrowByGlobalSearch\":true,\"narrowByMapBounds\":true,\"applyForceRefresh\":true},\"id\":\"59ca05b3-e3be-4fb4-ab4d-56c17b8bd589\",\"label\":null,\"minZoom\":0,\"maxZoom\":24,\"alpha\":0.75,\"visible\":true,\"style\":{\"type\":\"VECTOR\",\"properties\":{\"icon\":{\"type\":\"STATIC\",\"options\":{\"value\":\"marker\"}},\"fillColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#54B399\"}},\"lineColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#41937c\"}},\"lineWidth\":{\"type\":\"STATIC\",\"options\":{\"size\":1}},\"iconSize\":{\"type\":\"STATIC\",\"options\":{\"size\":6}},\"iconOrientation\":{\"type\":\"STATIC\",\"options\":{\"orientation\":0}},\"labelText\":{\"type\":\"STATIC\",\"options\":{\"value\":\"\"}},\"labelColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#000000\"}},\"labelSize\":{\"type\":\"STATIC\",\"options\":{\"size\":14}},\"labelZoomRange\":{\"options\":{\"useLayerZoomRange\":true,\"minZoom\":0,\"maxZoom\":24}},\"labelBorderColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#FFFFFF\"}},\"symbolizeAs\":{\"options\":{\"value\":\"circle\"}},\"labelBorderSize\":{\"options\":{\"size\":\"SMALL\"}},\"labelPosition\":{\"options\":{\"position\":\"CENTER\"}}},\"isTimeAware\":true},\"includeInFitToBounds\":true,\"type\":\"GEOJSON_VECTOR\",\"joins\":[],\"disableTooltips\":false}]",
"uiStateJSON": "{\"isLayerTOCOpen\":true,\"openTOCDetails\":[]}"
},
"references": [],
"managed": false,
"coreMigrationVersion": "8.8.0",
"typeMigrationVersion": "8.4.0"
}

View file

@ -13565,6 +13565,15 @@ concat-stream@^1.4.7, concat-stream@^1.5.0, concat-stream@~1.6.0:
readable-stream "^2.2.2"
typedarray "^0.0.6"
concat-stream@~1.5.0:
version "1.5.2"
resolved "https://registry.yarnpkg.com/concat-stream/-/concat-stream-1.5.2.tgz#708978624d856af41a5a741defdd261da752c266"
integrity sha1-cIl4Yk2FavQaWnQd790mHadSwmY=
dependencies:
inherits "~2.0.1"
readable-stream "~2.0.0"
typedarray "~0.0.5"
concaveman@*:
version "1.2.0"
resolved "https://registry.yarnpkg.com/concaveman/-/concaveman-1.2.0.tgz#4340f27c08a11bdc1d5fac13476862a2ab09b703"
@ -22286,7 +22295,7 @@ minimist-options@4.1.0:
is-plain-obj "^1.1.0"
kind-of "^6.0.3"
minimist@^1.1.1, minimist@^1.1.3, minimist@^1.2.0, minimist@^1.2.3, minimist@^1.2.5, minimist@^1.2.6, minimist@^1.2.8, minimist@~1.2.5:
minimist@^1.1.1, minimist@^1.1.3, minimist@^1.2.0, minimist@^1.2.3, minimist@^1.2.5, minimist@^1.2.6, minimist@^1.2.8, minimist@~1.2.0, minimist@~1.2.5:
version "1.2.8"
resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c"
integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==
@ -24806,6 +24815,11 @@ process-nextick-args@^2.0.0, process-nextick-args@~2.0.0:
resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2"
integrity sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==
process-nextick-args@~1.0.6:
version "1.0.7"
resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-1.0.7.tgz#150e20b756590ad3f91093f25a4f2ad8bff30ba3"
integrity sha1-FQ4gt1ZZCtP5EJPyWk8q2L/zC6M=
process-on-spawn@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/process-on-spawn/-/process-on-spawn-1.0.0.tgz#95b05a23073d30a17acfdc92a440efd2baefdc93"
@ -25998,6 +26012,18 @@ readable-stream@^4.0.0:
events "^3.3.0"
process "^0.11.10"
readable-stream@~2.0.0:
version "2.0.6"
resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.0.6.tgz#8f90341e68a53ccc928788dacfcd11b36eb9b78e"
integrity sha1-j5A0HmilPMySh4jaz80Rs265t44=
dependencies:
core-util-is "~1.0.0"
inherits "~2.0.1"
isarray "~1.0.0"
process-nextick-args "~1.0.6"
string_decoder "~0.10.x"
util-deprecate "~1.0.1"
readdir-glob@^1.0.0:
version "1.1.1"
resolved "https://registry.yarnpkg.com/readdir-glob/-/readdir-glob-1.1.1.tgz#f0e10bb7bf7bfa7e0add8baffdc54c3f7dbee6c4"
@ -29393,7 +29419,7 @@ typedarray-to-buffer@^3.1.5:
dependencies:
is-typedarray "^1.0.0"
typedarray@^0.0.6:
typedarray@^0.0.6, typedarray@~0.0.5:
version "0.0.6"
resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777"
integrity sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=
@ -30870,6 +30896,14 @@ websocket-extensions@>=0.1.1:
resolved "https://registry.yarnpkg.com/websocket-extensions/-/websocket-extensions-0.1.4.tgz#7f8473bc839dfd87608adb95d7eb075211578a42"
integrity sha512-OqedPIGOfsDlo31UNwYbCFMSaO9m9G/0faIHj5/dZFDMFqPTcx6UwqyOy3COEaEOg/9VsGIpdqn62W5KhoKSpg==
wellknown@^0.5.0:
version "0.5.0"
resolved "https://registry.yarnpkg.com/wellknown/-/wellknown-0.5.0.tgz#09ae9871fa826cf0a6ec1537ef00c379d78d7101"
integrity sha1-Ca6YcfqCbPCm7BU37wDDedeNcQE=
dependencies:
concat-stream "~1.5.0"
minimist "~1.2.0"
wgs84@0.0.0:
version "0.0.0"
resolved "https://registry.yarnpkg.com/wgs84/-/wgs84-0.0.0.tgz#34fdc555917b6e57cf2a282ed043710c049cdc76"