github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 09:19:04 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Canvas] Migrate the esdocs function to the SQL search strategy (#139921)

Browse source 
* Migrate `esdocs` expressions function to the data plugin SQL search strategy
* Remove `essql` search strategy from the canvas plugin
This commit is contained in:
Michael Dokolin 2022-09-08 09:10:14 +02:00 • committed by GitHub
parent d804a8421e
commit 04135d1f1f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 99 additions and 355 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/plugins/data/common/search/expressions/essql.ts
View file

@ -10,7 +10,7 @@ import type { KibanaRequest } from '@kbn/core/server';
import { buildEsQuery } from '@kbn/es-query';
import { castEsToKbnFieldTypeName, ES_FIELD_TYPES, KBN_FIELD_TYPES } from '@kbn/field-types';
import { i18n } from '@kbn/i18n';
import {
import type {
Datatable,
DatatableColumnType,
ExpressionFunctionDefinition,

154
x-pack/plugins/canvas/canvas_plugin_src/functions/browser/esdocs.ts
View file

@ -5,16 +5,24 @@
* 2.0.
*/
import { zipObject } from 'lodash';
import {
Datatable,
ExpressionFunctionDefinition,
ExpressionValueFilter,
} from '@kbn/expressions-plugin/common';
import { lastValueFrom } from 'rxjs';
import { Observable, catchError, from, map, switchMap, throwError } from 'rxjs';
import {
SqlRequestParams,
SqlSearchStrategyRequest,
SqlSearchStrategyResponse,
SQL_SEARCH_STRATEGY,
} from '@kbn/data-plugin/common';
import { searchService } from '../../../public/services';
import { ESSQL_SEARCH_STRATEGY } from '../../../common/lib/constants';
import { EssqlSearchStrategyRequest, EssqlSearchStrategyResponse } from '../../../types';
import { getFunctionHelp } from '../../../i18n';
import { buildBoolArray } from '../../../common/lib/request/build_bool_array';
import { normalizeType } from '../../../common/lib/request/normalize_type';
interface Arguments {
index: string;
@ -25,11 +33,15 @@ interface Arguments {
count: number;
}
function sanitize(value: string) {
return value.replace(/[\(\)]/g, '_');
}
export function esdocs(): ExpressionFunctionDefinition<
'esdocs',
ExpressionValueFilter,
Arguments,
any
Observable<Datatable>
> {
const { help, args: argHelp } = getFunctionHelp().esdocs;
@ -72,67 +84,97 @@ export function esdocs(): ExpressionFunctionDefinition<
help: argHelp.sort,
},
},
fn: async (input, args, handlers) => {
const { count, index, fields, sort } = args;
input.and = input.and.concat([
{
type: 'filter',
filterType: 'luceneQueryString',
query: args.query,
and: [],
},
]);
fn(input, args, { abortSignal }) {
// Load ad-hoc to avoid adding to the page load bundle size
const squel = await import('safe-squel');
return from(import('safe-squel')).pipe(
switchMap((squel) => {
const { count, index, fields, sort } = args;
let query = squel.select({
autoQuoteTableNames: true,
autoQuoteFieldNames: true,
autoQuoteAliasNames: true,
nameQuoteCharacter: '"',
});
let query = squel.select({
autoQuoteTableNames: true,
autoQuoteFieldNames: true,
autoQuoteAliasNames: true,
nameQuoteCharacter: '"',
});
if (index) {
query.from(index);
}
if (index) {
query.from(index);
}
if (fields) {
const allFields = fields.split(',').map((field) => field.trim());
allFields.forEach((field) => (query = query.field(field)));
}
if (fields) {
const allFields = fields.split(',').map((field) => field.trim());
allFields.forEach((field) => (query = query.field(field)));
}
if (sort) {
const [sortField, sortOrder] = sort.split(',').map((str) => str.trim());
if (sortField) {
query.order(`"${sortField}"`, sortOrder === 'asc');
}
}
if (sort) {
const [sortField, sortOrder] = sort.split(',').map((str) => str.trim());
if (sortField) {
query.order(`"${sortField}"`, sortOrder === 'asc');
}
}
const search = searchService.getService().search;
const params: SqlRequestParams = {
query: query.toString(),
fetch_size: count,
field_multi_value_leniency: true,
filter: {
bool: {
must: [
{ match_all: {} },
...buildBoolArray([
...input.and,
{
type: 'filter',
filterType: 'luceneQueryString',
query: args.query,
and: [],
},
]),
],
},
},
};
const req = {
count,
query: query.toString(),
filter: input.and,
};
const search = searchService.getService().search;
// We're requesting the data using the ESSQL strategy because
// the SQL routes return type information with the result set
return lastValueFrom(
search.search<EssqlSearchStrategyRequest, EssqlSearchStrategyResponse>(req, {
strategy: ESSQL_SEARCH_STRATEGY,
return search.search<SqlSearchStrategyRequest, SqlSearchStrategyResponse>(
{ params },
{ abortSignal, strategy: SQL_SEARCH_STRATEGY }
);
}),
catchError((error) => {
if (!error.err) {
error.message = `Unexpected error from Elasticsearch: ${error.message}`;
} else {
const { type, reason } = error.err.attributes;
error.message =
type === 'parsing_exception'
? `Couldn't parse Elasticsearch SQL query. You may need to add double quotes to names containing special characters. Check your query and try again. Error: ${reason}`
: `Unexpected error from Elasticsearch: ${type} - ${reason}`;
}
return throwError(() => error);
}),
map(({ rawResponse: body }) => {
const columns =
body.columns?.map(({ name, type }) => ({
id: sanitize(name),
name: sanitize(name),
meta: { type: normalizeType(type) },
})) ?? [];
const columnNames = columns.map(({ name }) => name);
const rows = body.rows.map((row) => zipObject(columnNames, row));
return {
type: 'datatable',
meta: {
type: 'essql',
},
columns,
rows,
} as Datatable;
})
).then((resp: EssqlSearchStrategyResponse) => {
return {
type: 'datatable',
meta: {
type: 'essql',
},
...resp,
};
});
);
},
};
}

1
x-pack/plugins/canvas/common/lib/constants.ts
View file

@ -49,5 +49,4 @@ export const API_ROUTE_SHAREABLE_RUNTIME_DOWNLOAD = `/public/canvas/${SHAREABLE_
export const CANVAS_EMBEDDABLE_CLASSNAME = `canvasEmbeddable`;
export const CONTEXT_MENU_TOP_BORDER_CLASSNAME = 'canvasContextMenu--topBorder';
export const API_ROUTE_FUNCTIONS = `${API_ROUTE}/fns`;
export const ESSQL_SEARCH_STRATEGY = 'essql';
export const HEADER_BANNER_HEIGHT = 32; // This value is also declared in `/src/core/public/_variables.scss`

18
x-pack/plugins/canvas/common/lib/request/sanitize_name.ts
View file

@ -1,18 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export function sanitizeName(name: string) {
// invalid characters
const invalid = ['(', ')'];
const pattern = invalid.map((v) => escapeRegExp(v)).join('|');
const regex = new RegExp(pattern, 'g');
return name.replace(regex, '_');
}
function escapeRegExp(string: string) {
return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
}

168
x-pack/plugins/canvas/server/lib/essql_strategy.test.ts
View file

@ -1,168 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { essqlSearchStrategyProvider } from './essql_strategy';
import { EssqlSearchStrategyRequest } from '../../types';
import { zipObject } from 'lodash';
import { lastValueFrom } from 'rxjs';
const getMockEssqlResponse = () => ({
body: {
columns: [
{ name: 'One', type: 'keyword' },
{ name: 'Two', type: 'keyword' },
],
rows: [
['foo', 'bar'],
['buz', 'baz'],
['beep', 'boop'],
],
cursor: 'cursor-value',
},
statusCode: 200,
});
const basicReq: EssqlSearchStrategyRequest = {
query: 'SELECT * FROM my_index;',
count: 3,
params: ['my_var'],
filter: [
{
type: 'filter',
filterType: 'exactly',
value: 'Test Value',
column: 'One',
and: [],
},
],
timezone: 'UTC',
};
describe('ESSQL search strategy', () => {
describe('strategy interface', () => {
it('returns a strategy with a `search` function', async () => {
const essqlSearch = await essqlSearchStrategyProvider();
expect(typeof essqlSearch.search).toBe('function');
});
});
describe('search()', () => {
let mockQuery: jest.Mock;
let mockClearCursor: jest.Mock;
let mockDeps: any;
beforeEach(() => {
mockQuery = jest.fn().mockResolvedValueOnce(getMockEssqlResponse());
mockClearCursor = jest.fn();
mockDeps = {
esClient: {
asCurrentUser: {
sql: {
query: mockQuery,
clearCursor: mockClearCursor,
},
},
},
} as unknown as any;
});
describe('query functionality', () => {
it('performs a simple query', async () => {
const sqlSearch = await essqlSearchStrategyProvider();
const result = await lastValueFrom(sqlSearch.search(basicReq, {}, mockDeps));
const [[request]] = mockQuery.mock.calls;
expect(request.format).toEqual('json');
expect(request.body).toEqual(
expect.objectContaining({
query: basicReq.query,
client_id: 'canvas',
fetch_size: basicReq.count,
time_zone: basicReq.timezone,
field_multi_value_leniency: true,
params: ['my_var'],
})
);
const expectedColumns = getMockEssqlResponse().body.columns.map((c) => ({
id: c.name,
name: c.name,
meta: { type: 'string' },
}));
const columnNames = expectedColumns.map((c) => c.name);
const expectedRows = getMockEssqlResponse().body.rows.map((r) => zipObject(columnNames, r));
expect(result.columns).toEqual(expectedColumns);
expect(result.rows).toEqual(expectedRows);
});
it('iterates over cursor to retrieve for records query', async () => {
const pageOne = {
body: {
columns: [
{ name: 'One', type: 'keyword' },
{ name: 'Two', type: 'keyword' },
],
rows: [['foo', 'bar']],
cursor: 'cursor-value',
},
};
const pageTwo = {
body: {
rows: [['buz', 'baz']],
},
};
mockQuery.mockReset().mockReturnValueOnce(pageOne).mockReturnValueOnce(pageTwo);
const sqlSearch = await essqlSearchStrategyProvider();
const result = await lastValueFrom(
sqlSearch.search({ ...basicReq, count: 2 }, {}, mockDeps)
);
expect(result.rows).toHaveLength(2);
});
it('closes any cursors that remain open', async () => {
const sqlSearch = await essqlSearchStrategyProvider();
await sqlSearch.search(basicReq, {}, mockDeps).toPromise();
const [[cursorReq]] = mockClearCursor.mock.calls;
expect(cursorReq.body.cursor).toEqual('cursor-value');
});
it('emits an error if the client throws', async () => {
const req: EssqlSearchStrategyRequest = {
query: 'SELECT * FROM my_index;',
count: 1,
params: [],
filter: [
{
type: 'filter',
filterType: 'exactly',
value: 'Test Value',
column: 'category.keyword',
and: [],
},
],
timezone: 'UTC',
};
expect.assertions(1);
mockQuery.mockReset().mockRejectedValueOnce(new Error('client error'));
const eqlSearch = await essqlSearchStrategyProvider();
eqlSearch.search(req, {}, mockDeps).subscribe(
() => {},
(err) => {
expect(err).toEqual(new Error('client error'));
}
);
});
});
});
});

104
x-pack/plugins/canvas/server/lib/essql_strategy.ts
View file

@ -1,104 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { from } from 'rxjs';
import { map, zipObject } from 'lodash';
import { ISearchStrategy } from '@kbn/data-plugin/server';
import { getKbnServerError } from '@kbn/kibana-utils-plugin/server';
import { EssqlSearchStrategyRequest, EssqlSearchStrategyResponse } from '../../types';
import { buildBoolArray } from '../../common/lib/request/build_bool_array';
import { sanitizeName } from '../../common/lib/request/sanitize_name';
import { normalizeType } from '../../common/lib/request/normalize_type';
export const essqlSearchStrategyProvider = (): ISearchStrategy<
EssqlSearchStrategyRequest,
EssqlSearchStrategyResponse
> => {
return {
search: (request, options, { esClient }) => {
const { count, query, filter, timezone, params } = request;
const searchUntilEnd = async () => {
try {
let response = await esClient.asCurrentUser.sql.query(
{
format: 'json',
body: {
query,
params,
field_multi_value_leniency: true,
time_zone: timezone,
fetch_size: count,
// @ts-expect-error `client_id` missing from `QuerySqlRequest` type
client_id: 'canvas',
filter: {
bool: {
must: [{ match_all: {} }, ...buildBoolArray(filter)],
},
},
},
},
{ meta: true }
);
let body = response.body;
const columns = body.columns!.map(({ name, type }) => {
return {
id: sanitizeName(name),
name: sanitizeName(name),
meta: { type: normalizeType(type) },
};
});
const columnNames = map(columns, 'name');
let rows = body.rows.map((row) => zipObject(columnNames, row));
// If we still have rows to retrieve, continue requesting data
// using the cursor until we have everything
while (rows.length < count && body.cursor !== undefined) {
// @ts-expect-error previous ts-ignore mess with the signature override
response = await esClient.asCurrentUser.sql.query(
{
format: 'json',
body: {
cursor: body.cursor,
},
},
{ meta: true }
);
body = response.body;
rows = [...rows, ...body.rows.map((row) => zipObject(columnNames, row))];
}
// If we used a cursor, clean it up
if (body.cursor !== undefined) {
await esClient.asCurrentUser.sql.clearCursor({
body: {
cursor: body.cursor,
},
});
}
return {
columns,
rows,
rawResponse: response,
};
} catch (e) {
throw getKbnServerError(e);
}
};
return from(searchUntilEnd());
},
};
};

7
x-pack/plugins/canvas/server/plugin.ts
View file

@ -17,7 +17,6 @@ import { HomeServerPluginSetup } from '@kbn/home-plugin/server';
import { EmbeddableSetup } from '@kbn/embeddable-plugin/server';
import { ReportingSetup } from '@kbn/reporting-plugin/server';
import { PluginSetupContract as FeaturesPluginSetup } from '@kbn/features-plugin/server';
import { ESSQL_SEARCH_STRATEGY } from '../common/lib/constants';
import { getCanvasFeature } from './feature';
import { initRoutes } from './routes';
import { registerCanvasUsageCollector } from './collectors';
@ -26,7 +25,6 @@ import { setupInterpreter } from './setup_interpreter';
import { customElementType, workpadTypeFactory, workpadTemplateType } from './saved_objects';
import type { CanvasSavedObjectTypeMigrationsDeps } from './saved_objects/migrations';
import { initializeTemplates } from './templates';
import { essqlSearchStrategyProvider } from './lib/essql_strategy';
import { getUISettings } from './ui_settings';
import { CanvasRouteHandlerContext, createWorkpadRouteContext } from './workpad_route_context';
@ -94,11 +92,6 @@ export class CanvasPlugin implements Plugin {
// we need the kibana index for the Canvas usage collector
const kibanaIndex = coreSetup.savedObjects.getKibanaIndex();
registerCanvasUsageCollector(plugins.usageCollection, kibanaIndex);
coreSetup.getStartServices().then(([_, depsStart]) => {
const strategy = essqlSearchStrategyProvider();
plugins.data.search.registerSearchStrategy(ESSQL_SEARCH_STRATEGY, strategy);
});
}
public start(coreStart: CoreStart) {