[Search] Use filter to bulk find (#85551)

* Use filter to bulk find

* Update x-pack/plugins/data_enhanced/server/search/session/session_service.ts

Co-authored-by: Lukas Olson <olson.lukas@gmail.com>

* Dashboard in space test

* Add warning on update failure

* fix merge

* Added functional test for sessions in space

* snapshot

* test cleanup

* Update src/plugins/data/common/es_query/kuery/node_types/node_builder.ts

Co-authored-by: Lukas Olson <olson.lukas@gmail.com>

* Revert "Update src/plugins/data/common/es_query/kuery/node_types/node_builder.ts"

This reverts commit 4b7e781fe6.

Co-authored-by: Lukas Olson <olson.lukas@gmail.com>
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Liza Katz 2020-12-17 12:17:02 +02:00 committed by GitHub
parent 2066b3d7ca
commit 5be169a4fc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 771 additions and 45 deletions

View file

@ -18,7 +18,7 @@
*/
export { KQLSyntaxError } from './kuery_syntax_error';
export { nodeTypes } from './node_types';
export { nodeTypes, nodeBuilder } from './node_types';
export * from './ast';
export * from './types';

View file

@ -24,6 +24,7 @@ import * as wildcard from './wildcard';
import { NodeTypes } from './types';
export { NodeTypes };
export { nodeBuilder } from './node_builder';
export const nodeTypes: NodeTypes = {
// This requires better typing of the different typings and their return types.

View file

@ -0,0 +1,38 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { KueryNode, nodeTypes } from '../types';
export const nodeBuilder = {
is: (fieldName: string, value: string | KueryNode) => {
return nodeTypes.function.buildNodeWithArgumentNodes('is', [
nodeTypes.literal.buildNode(fieldName),
typeof value === 'string' ? nodeTypes.literal.buildNode(value) : value,
nodeTypes.literal.buildNode(false),
]);
},
or: ([first, ...args]: KueryNode[]): KueryNode => {
return args.length ? nodeTypes.function.buildNode('or', [first, nodeBuilder.or(args)]) : first;
},
and: ([first, ...args]: KueryNode[]): KueryNode => {
return args.length
? nodeTypes.function.buildNode('and', [first, nodeBuilder.and(args)])
: first;
},
};

View file

@ -45,7 +45,7 @@ import { TaskManagerStartContract } from '../../../task_manager/server';
import { taskInstanceToAlertTaskInstance } from '../task_runner/alert_task_instance';
import { deleteTaskIfItExists } from '../lib/delete_task_if_it_exists';
import { RegistryAlertType } from '../alert_type_registry';
import { AlertsAuthorization, WriteOperations, ReadOperations, and } from '../authorization';
import { AlertsAuthorization, WriteOperations, ReadOperations } from '../authorization';
import { IEventLogClient } from '../../../../plugins/event_log/server';
import { parseIsoOrRelativeDate } from '../lib/iso_or_relative_date';
import { alertInstanceSummaryFromEventLog } from '../lib/alert_instance_summary_from_event_log';
@ -56,6 +56,7 @@ import { retryIfConflicts } from '../lib/retry_if_conflicts';
import { partiallyUpdateAlert } from '../saved_objects';
import { markApiKeyForInvalidation } from '../invalidate_pending_api_keys/mark_api_key_for_invalidation';
import { alertAuditEvent, AlertAuditAction } from './audit_events';
import { nodeBuilder } from '../../../../../src/plugins/data/common';
export interface RegistryAlertTypeWithAuth extends RegistryAlertType {
authorizedConsumers: string[];
@ -455,7 +456,7 @@ export class AlertsClient {
...options,
filter:
(authorizationFilter && options.filter
? and([esKuery.fromKueryExpression(options.filter), authorizationFilter])
? nodeBuilder.and([esKuery.fromKueryExpression(options.filter), authorizationFilter])
: authorizationFilter) ?? options.filter,
fields: fields ? this.includeFieldsRequiredForAuthentication(fields) : fields,
type: 'alert',
@ -517,7 +518,7 @@ export class AlertsClient {
...options,
filter:
(authorizationFilter && filter
? and([esKuery.fromKueryExpression(filter), authorizationFilter])
? nodeBuilder.and([esKuery.fromKueryExpression(filter), authorizationFilter])
: authorizationFilter) ?? filter,
page: 1,
perPage: 0,

View file

@ -5,36 +5,23 @@
*/
import { remove } from 'lodash';
import { nodeTypes } from '../../../../../src/plugins/data/common';
import { nodeBuilder } from '../../../../../src/plugins/data/common';
import { KueryNode } from '../../../../../src/plugins/data/server';
import { RegistryAlertTypeWithAuth } from './alerts_authorization';
export const is = (fieldName: string, value: string | KueryNode) =>
nodeTypes.function.buildNodeWithArgumentNodes('is', [
nodeTypes.literal.buildNode(fieldName),
typeof value === 'string' ? nodeTypes.literal.buildNode(value) : value,
nodeTypes.literal.buildNode(false),
]);
export const or = ([first, ...args]: KueryNode[]): KueryNode =>
args.length ? nodeTypes.function.buildNode('or', [first, or(args)]) : first;
export const and = ([first, ...args]: KueryNode[]): KueryNode =>
args.length ? nodeTypes.function.buildNode('and', [first, and(args)]) : first;
export function asFiltersByAlertTypeAndConsumer(
alertTypes: Set<RegistryAlertTypeWithAuth>
): KueryNode {
return or(
return nodeBuilder.or(
Array.from(alertTypes).reduce<KueryNode[]>((filters, { id, authorizedConsumers }) => {
ensureFieldIsSafeForQuery('alertTypeId', id);
filters.push(
and([
is(`alert.attributes.alertTypeId`, id),
or(
nodeBuilder.and([
nodeBuilder.is(`alert.attributes.alertTypeId`, id),
nodeBuilder.or(
Object.keys(authorizedConsumers).map((consumer) => {
ensureFieldIsSafeForQuery('consumer', consumer);
return is(`alert.attributes.consumer`, consumer);
return nodeBuilder.is(`alert.attributes.consumer`, consumer);
})
),
])

View file

@ -14,11 +14,14 @@ import {
SavedObjectsClientContract,
Logger,
SavedObject,
SavedObjectsBulkUpdateObject,
} from '../../../../../../src/core/server';
import {
IKibanaSearchRequest,
IKibanaSearchResponse,
ISearchOptions,
KueryNode,
nodeBuilder,
tapFirst,
} from '../../../../../../src/plugins/data/common';
import {
@ -81,6 +84,18 @@ export class BackgroundSessionService implements ISessionService {
}
};
/**
* Compiles a KQL Query to fetch sessions by ID.
* Done as a performance optimization workaround.
*/
private sessionIdsAsFilters(sessionIds: string[]): KueryNode {
return nodeBuilder.or(
sessionIds.map((id) => {
return nodeBuilder.is(`${BACKGROUND_SESSION_TYPE}.attributes.sessionId`, id);
})
);
}
/**
* Gets all {@link SessionSavedObjectAttributes | Background Searches} that
* currently being tracked by the service.
@ -90,17 +105,14 @@ export class BackgroundSessionService implements ISessionService {
* context of a user's session.
*/
private async getAllMappedSavedObjects() {
const activeMappingIds = Array.from(this.sessionSearchMap.keys())
.map((sessionId) => `"${sessionId}"`)
.join(' | ');
const filter = this.sessionIdsAsFilters(Array.from(this.sessionSearchMap.keys()));
const res = await this.internalSavedObjectsClient.find<BackgroundSessionSavedObjectAttributes>({
perPage: INMEM_MAX_SESSIONS, // If there are more sessions in memory, they will be synced when some items are cleared out.
type: BACKGROUND_SESSION_TYPE,
search: activeMappingIds,
searchFields: ['sessionId'],
filter,
namespaces: ['*'],
});
this.logger.debug(`getAllMappedSavedObjects | Got ${res.saved_objects.length} items`);
this.logger.warn(`getAllMappedSavedObjects | Got ${res.saved_objects.length} items`);
return res.saved_objects;
}
@ -135,6 +147,9 @@ export class BackgroundSessionService implements ISessionService {
updatedSessions.forEach((updatedSavedObject) => {
const sessionInfo = this.sessionSearchMap.get(updatedSavedObject.id)!;
if (updatedSavedObject.error) {
this.logger.warn(
`monitorMappedIds | update error ${JSON.stringify(updatedSavedObject.error) || ''}`
);
// Retry next time
sessionInfo.retryCount++;
} else if (updatedSavedObject.attributes.idMapping) {
@ -164,7 +179,9 @@ export class BackgroundSessionService implements ISessionService {
if (!activeMappingObjects.length) return [];
this.logger.debug(`updateAllSavedObjects | Updating ${activeMappingObjects.length} items`);
const updatedSessions = activeMappingObjects
const updatedSessions: Array<
SavedObjectsBulkUpdateObject<BackgroundSessionSavedObjectAttributes>
> = activeMappingObjects
.filter((so) => !so.error)
.map((sessionSavedObject) => {
const sessionInfo = this.sessionSearchMap.get(sessionSavedObject.id);
@ -173,7 +190,10 @@ export class BackgroundSessionService implements ISessionService {
...sessionSavedObject.attributes.idMapping,
...idMapping,
};
return sessionSavedObject;
return {
...sessionSavedObject,
namespace: sessionSavedObject.namespaces?.[0],
};
});
const updateResults = await this.internalSavedObjectsClient.bulkUpdate<BackgroundSessionSavedObjectAttributes>(

View file

@ -83,6 +83,7 @@ Array [
"lens",
"map",
"tag",
"background-session",
],
},
"ui": Array [
@ -204,6 +205,7 @@ Array [
"search",
"query",
"index-pattern",
"background-session",
"url",
],
"read": Array [],
@ -565,6 +567,7 @@ Array [
"lens",
"map",
"tag",
"background-session",
],
},
"ui": Array [
@ -686,6 +689,7 @@ Array [
"search",
"query",
"index-pattern",
"background-session",
"url",
],
"read": Array [],

View file

@ -28,7 +28,7 @@ export const buildOSSFeatures = ({ savedObjectTypes, includeTimelion }: BuildOSS
app: ['discover', 'kibana'],
catalogue: ['discover'],
savedObject: {
all: ['search', 'query', 'index-pattern'],
all: ['search', 'query', 'index-pattern', 'background-session'],
read: [],
},
ui: ['show', 'save', 'saveQuery'],
@ -156,6 +156,7 @@ export const buildOSSFeatures = ({ savedObjectTypes, includeTimelion }: BuildOSS
'lens',
'map',
'tag',
'background-session',
],
},
ui: ['createNew', 'show', 'showWriteControls', 'saveQuery'],

File diff suppressed because one or more lines are too long

View file

@ -0,0 +1,244 @@
{
"type": "index",
"value": {
"index": ".kibana",
"mappings": {
"properties": {
"config": {
"dynamic": "true",
"properties": {
"buildNum": {
"type": "keyword"
}
}
},
"dashboard": {
"dynamic": "strict",
"properties": {
"description": {
"type": "text"
},
"hits": {
"type": "integer"
},
"kibanaSavedObjectMeta": {
"properties": {
"searchSourceJSON": {
"type": "text"
}
}
},
"optionsJSON": {
"type": "text"
},
"panelsJSON": {
"type": "text"
},
"refreshInterval": {
"properties": {
"display": {
"type": "keyword"
},
"pause": {
"type": "boolean"
},
"section": {
"type": "integer"
},
"value": {
"type": "integer"
}
}
},
"timeFrom": {
"type": "keyword"
},
"timeRestore": {
"type": "boolean"
},
"timeTo": {
"type": "keyword"
},
"title": {
"type": "text"
},
"uiStateJSON": {
"type": "text"
},
"version": {
"type": "integer"
}
}
},
"index-pattern": {
"dynamic": "strict",
"properties": {
"fieldFormatMap": {
"type": "text"
},
"fields": {
"type": "text"
},
"intervalName": {
"type": "keyword"
},
"notExpandable": {
"type": "boolean"
},
"sourceFilters": {
"type": "text"
},
"timeFieldName": {
"type": "keyword"
},
"title": {
"type": "text"
}
}
},
"search": {
"dynamic": "strict",
"properties": {
"columns": {
"type": "keyword"
},
"description": {
"type": "text"
},
"hits": {
"type": "integer"
},
"kibanaSavedObjectMeta": {
"properties": {
"searchSourceJSON": {
"type": "text"
}
}
},
"sort": {
"type": "keyword"
},
"title": {
"type": "text"
},
"version": {
"type": "integer"
}
}
},
"server": {
"dynamic": "strict",
"properties": {
"uuid": {
"type": "keyword"
}
}
},
"timelion-sheet": {
"dynamic": "strict",
"properties": {
"description": {
"type": "text"
},
"hits": {
"type": "integer"
},
"kibanaSavedObjectMeta": {
"properties": {
"searchSourceJSON": {
"type": "text"
}
}
},
"timelion_chart_height": {
"type": "integer"
},
"timelion_columns": {
"type": "integer"
},
"timelion_interval": {
"type": "keyword"
},
"timelion_other_interval": {
"type": "keyword"
},
"timelion_rows": {
"type": "integer"
},
"timelion_sheet": {
"type": "text"
},
"title": {
"type": "text"
},
"version": {
"type": "integer"
}
}
},
"type": {
"type": "keyword"
},
"url": {
"dynamic": "strict",
"properties": {
"accessCount": {
"type": "long"
},
"accessDate": {
"type": "date"
},
"createDate": {
"type": "date"
},
"url": {
"fields": {
"keyword": {
"ignore_above": 2048,
"type": "keyword"
}
},
"type": "text"
}
}
},
"visualization": {
"dynamic": "strict",
"properties": {
"description": {
"type": "text"
},
"kibanaSavedObjectMeta": {
"properties": {
"searchSourceJSON": {
"type": "text"
}
}
},
"savedSearchId": {
"type": "keyword"
},
"title": {
"type": "text"
},
"uiStateJSON": {
"type": "text"
},
"version": {
"type": "integer"
},
"visState": {
"type": "text"
}
}
}
}
},
"settings": {
"index": {
"number_of_replicas": "1",
"number_of_shards": "1"
}
}
}
}

View file

@ -6,14 +6,14 @@
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../../../ftr_provider_context';
import { getSearchSessionIdByPanelProvider } from './get_search_session_id_by_panel';
export default function ({ getService, getPageObjects }: FtrProviderContext) {
const es = getService('es');
const testSubjects = getService('testSubjects');
const log = getService('log');
const PageObjects = getPageObjects(['common', 'header', 'dashboard', 'visChart']);
const dashboardPanelActions = getService('dashboardPanelActions');
const inspector = getService('inspector');
const getSearchSessionIdByPanel = getSearchSessionIdByPanelProvider(getService);
const queryBar = getService('queryBar');
const browser = getService('browser');
const sendToBackground = getService('sendToBackground');
@ -133,15 +133,4 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
});
});
});
// HELPERS
async function getSearchSessionIdByPanel(panelTitle: string) {
await dashboardPanelActions.openInspectorByTitle(panelTitle);
await inspector.openInspectorRequestsView();
const searchSessionId = await (
await testSubjects.find('inspectorRequestSearchSessionId')
).getAttribute('data-search-session-id');
await inspector.close();
return searchSessionId;
}
}

View file

@ -0,0 +1,22 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
// HELPERS
export function getSearchSessionIdByPanelProvider(getService: any) {
const dashboardPanelActions = getService('dashboardPanelActions');
const inspector = getService('inspector');
const testSubjects = getService('testSubjects');
return async function getSearchSessionIdByPanel(panelTitle: string) {
await dashboardPanelActions.openInspectorByTitle(panelTitle);
await inspector.openInspectorRequestsView();
const searchSessionId = await (
await testSubjects.find('inspectorRequestSearchSessionId')
).getAttribute('data-search-session-id');
await inspector.close();
return searchSessionId;
};
}

View file

@ -24,5 +24,6 @@ export default function ({ loadTestFile, getService }: FtrProviderContext) {
});
loadTestFile(require.resolve('./async_search'));
loadTestFile(require.resolve('./sessions_in_space'));
});
}

View file

@ -0,0 +1,90 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { FtrProviderContext } from '../../../../ftr_provider_context';
import { getSearchSessionIdByPanelProvider } from './get_search_session_id_by_panel';
export default function ({ getService, getPageObjects }: FtrProviderContext) {
const testSubjects = getService('testSubjects');
const esArchiver = getService('esArchiver');
const security = getService('security');
const PageObjects = getPageObjects([
'common',
'header',
'dashboard',
'visChart',
'security',
'timePicker',
]);
const getSearchSessionIdByPanel = getSearchSessionIdByPanelProvider(getService);
const browser = getService('browser');
const sendToBackground = getService('sendToBackground');
describe('dashboard in space', () => {
describe('Send to background in space', () => {
before(async () => {
await esArchiver.load('dashboard/session_in_space');
await security.role.create('data_analyst', {
elasticsearch: {
indices: [{ names: ['logstash-*'], privileges: ['all'] }],
},
kibana: [
{
base: ['all'],
spaces: ['another-space'],
},
],
});
await security.user.create('analyst', {
password: 'analyst-password',
roles: ['data_analyst'],
full_name: 'test user',
});
await PageObjects.security.forceLogout();
await PageObjects.security.login('analyst', 'analyst-password', {
expectSpaceSelector: false,
});
});
after(async () => {
await esArchiver.unload('dashboard/session_in_space');
await PageObjects.security.forceLogout();
});
it('Saves and restores a session', async () => {
await PageObjects.common.navigateToApp('dashboard', { basePath: 's/another-space' });
await PageObjects.dashboard.loadSavedDashboard('A Dashboard in another space');
await PageObjects.timePicker.setAbsoluteRange(
'Sep 1, 2015 @ 00:00:00.000',
'Oct 1, 2015 @ 00:00:00.000'
);
await PageObjects.dashboard.waitForRenderComplete();
await sendToBackground.expectState('completed');
await sendToBackground.save();
await sendToBackground.expectState('backgroundCompleted');
const savedSessionId = await getSearchSessionIdByPanel('A Pie in another space');
// load URL to restore a saved session
const url = await browser.getCurrentUrl();
const savedSessionURL = `${url}&searchSessionId=${savedSessionId}`;
await browser.get(savedSessionURL);
await PageObjects.header.waitUntilLoadingHasFinished();
await PageObjects.dashboard.waitForRenderComplete();
// Check that session is restored
await sendToBackground.expectState('restored');
await testSubjects.missingOrFail('embeddableErrorLabel');
});
});
});
}