github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 17:59:23 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Lens] give data plugin control of filter extraction, injection, and migrations (#120305)

Browse source
This commit is contained in:
Andrew Tate 2022-01-13 11:12:16 -06:00 • committed by GitHub
parent 1aadcd34c6
commit efc07eed86
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
42 changed files with 1092 additions and 378 deletions
Download patch file Download diff file Expand all files Collapse all files

24
x-pack/plugins/lens/server/embeddable/lens_embeddable_factory.test.ts
View file

@ -1,24 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import semverGte from 'semver/functions/gte';
import { lensEmbeddableFactory } from './lens_embeddable_factory';
import { migrations } from '../migrations/saved_object_migrations';
describe('saved object migrations and embeddable migrations', () => {
test('should have same versions registered (>7.13.0)', () => {
const savedObjectMigrationVersions = Object.keys(migrations).filter((version) => {
return semverGte(version, '7.13.1');
});
const embeddableMigrationVersions = lensEmbeddableFactory()?.migrations;
if (embeddableMigrationVersions) {
expect(savedObjectMigrationVersions.sort()).toEqual(
Object.keys(embeddableMigrationVersions).sort()
);
}
});
});

67
x-pack/plugins/lens/server/embeddable/lens_embeddable_factory.ts
View file

@ -1,67 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { EmbeddableRegistryDefinition } from 'src/plugins/embeddable/server';
import type { SerializableRecord } from '@kbn/utility-types';
import { DOC_TYPE } from '../../common';
import {
commonMakeReversePaletteAsCustom,
commonRemoveTimezoneDateHistogramParam,
commonRenameOperationsForFormula,
commonUpdateVisLayerType,
} from '../migrations/common_migrations';
import {
LensDocShape713,
LensDocShape715,
LensDocShapePre712,
VisState716,
VisStatePre715,
} from '../migrations/types';
import { extract, inject } from '../../common/embeddable_factory';
export const lensEmbeddableFactory = (): EmbeddableRegistryDefinition => {
return {
id: DOC_TYPE,
migrations: {
// This migration is run in 7.13.1 for `by value` panels because the 7.13 release window was missed.
'7.13.1': (state) => {
const lensState = state as unknown as { attributes: LensDocShapePre712 };
const migratedLensState = commonRenameOperationsForFormula(lensState.attributes);
return {
...lensState,
attributes: migratedLensState,
} as unknown as SerializableRecord;
},
'7.14.0': (state) => {
const lensState = state as unknown as { attributes: LensDocShape713 };
const migratedLensState = commonRemoveTimezoneDateHistogramParam(lensState.attributes);
return {
...lensState,
attributes: migratedLensState,
} as unknown as SerializableRecord;
},
'7.15.0': (state) => {
const lensState = state as unknown as { attributes: LensDocShape715<VisStatePre715> };
const migratedLensState = commonUpdateVisLayerType(lensState.attributes);
return {
...lensState,
attributes: migratedLensState,
} as unknown as SerializableRecord;
},
'7.16.0': (state) => {
const lensState = state as unknown as { attributes: LensDocShape715<VisState716> };
const migratedLensState = commonMakeReversePaletteAsCustom(lensState.attributes);
return {
...lensState,
attributes: migratedLensState,
} as unknown as SerializableRecord;
},
},
extract,
inject,
};
};

74
x-pack/plugins/lens/server/embeddable/make_lens_embeddable_factory.test.ts Normal file
View file

@ -0,0 +1,74 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import semverGte from 'semver/functions/gte';
import { makeLensEmbeddableFactory } from './make_lens_embeddable_factory';
import { getAllMigrations } from '../migrations/saved_object_migrations';
import { Filter } from '@kbn/es-query';
describe('embeddable migrations', () => {
test('should have all saved object migrations versions (>7.13.0)', () => {
const savedObjectMigrationVersions = Object.keys(getAllMigrations({})).filter((version) => {
return semverGte(version, '7.13.1');
});
const embeddableMigrationVersions = makeLensEmbeddableFactory({})()?.migrations;
if (embeddableMigrationVersions) {
expect(savedObjectMigrationVersions.sort()).toEqual(
Object.keys(embeddableMigrationVersions).sort()
);
}
});
test('should properly apply a filter migration within a lens visualization', () => {
const migrationVersion = 'some-version';
const lensVisualizationDoc = {
attributes: {
state: {
filters: [
{
filter: 1,
migrated: false,
},
{
filter: 2,
migrated: false,
},
],
},
},
};
const embeddableMigrationVersions = makeLensEmbeddableFactory({
[migrationVersion]: (filters: Filter[]) => {
return filters.map((filterState) => ({
...filterState,
migrated: true,
}));
},
})()?.migrations;
const migratedLensDoc = embeddableMigrationVersions?.[migrationVersion](lensVisualizationDoc);
expect(migratedLensDoc).toEqual({
attributes: {
state: {
filters: [
{
filter: 1,
migrated: true,
},
{
filter: 2,
migrated: true,
},
],
},
},
});
});
});

82
x-pack/plugins/lens/server/embeddable/make_lens_embeddable_factory.ts Normal file
View file

@ -0,0 +1,82 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { EmbeddableRegistryDefinition } from 'src/plugins/embeddable/server';
import type { SerializableRecord } from '@kbn/utility-types';
import {
mergeMigrationFunctionMaps,
MigrateFunctionsObject,
} from '../../../../../src/plugins/kibana_utils/common';
import { DOC_TYPE } from '../../common';
import {
commonMakeReversePaletteAsCustom,
commonRemoveTimezoneDateHistogramParam,
commonRenameFilterReferences,
commonRenameOperationsForFormula,
commonUpdateVisLayerType,
getLensFilterMigrations,
} from '../migrations/common_migrations';
import {
LensDocShape713,
LensDocShape715,
LensDocShapePre712,
VisState716,
VisStatePre715,
} from '../migrations/types';
import { extract, inject } from '../../common/embeddable_factory';
export const makeLensEmbeddableFactory =
(filterMigrations: MigrateFunctionsObject) => (): EmbeddableRegistryDefinition => {
return {
id: DOC_TYPE,
migrations: mergeMigrationFunctionMaps(getLensFilterMigrations(filterMigrations), {
// This migration is run in 7.13.1 for `by value` panels because the 7.13 release window was missed.
'7.13.1': (state) => {
const lensState = state as unknown as { attributes: LensDocShapePre712 };
const migratedLensState = commonRenameOperationsForFormula(lensState.attributes);
return {
...lensState,
attributes: migratedLensState,
} as unknown as SerializableRecord;
},
'7.14.0': (state) => {
const lensState = state as unknown as { attributes: LensDocShape713 };
const migratedLensState = commonRemoveTimezoneDateHistogramParam(lensState.attributes);
return {
...lensState,
attributes: migratedLensState,
} as unknown as SerializableRecord;
},
'7.15.0': (state) => {
const lensState = state as unknown as { attributes: LensDocShape715<VisStatePre715> };
const migratedLensState = commonUpdateVisLayerType(lensState.attributes);
return {
...lensState,
attributes: migratedLensState,
} as unknown as SerializableRecord;
},
'7.16.0': (state) => {
const lensState = state as unknown as { attributes: LensDocShape715<VisState716> };
const migratedLensState = commonMakeReversePaletteAsCustom(lensState.attributes);
return {
...lensState,
attributes: migratedLensState,
} as unknown as SerializableRecord;
},
'8.1.0': (state) => {
const lensState = state as unknown as { attributes: LensDocShape715<VisState716> };
const migratedLensState = commonRenameFilterReferences(lensState.attributes);
return {
...lensState,
attributes: migratedLensState,
} as unknown as SerializableRecord;
},
}),
extract,
inject,
};
};

44
x-pack/plugins/lens/server/migrations/common_migrations.test.ts Normal file
View file

@ -0,0 +1,44 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { Filter } from '@kbn/es-query';
import { getLensFilterMigrations } from './common_migrations';
describe('Lens migrations', () => {
describe('applying filter migrations', () => {
it('creates a filter migrations map that works on a lens visualization', () => {
const filterMigrations = {
'1.1': (filters: Filter[]) => filters.map((filter) => ({ ...filter, version: '1.1' })),
'2.2': (filters: Filter[]) => filters.map((filter) => ({ ...filter, version: '2.2' })),
'3.3': (filters: Filter[]) => filters.map((filter) => ({ ...filter, version: '3.3' })),
};
const lensVisualizationSavedObject = {
attributes: {
state: {
filters: [{}, {}],
},
},
};
const migrationMap = getLensFilterMigrations(filterMigrations);
expect(migrationMap['1.1'](lensVisualizationSavedObject).attributes.state.filters).toEqual([
{ version: '1.1' },
{ version: '1.1' },
]);
expect(migrationMap['2.2'](lensVisualizationSavedObject).attributes.state.filters).toEqual([
{ version: '2.2' },
{ version: '2.2' },
]);
expect(migrationMap['3.3'](lensVisualizationSavedObject).attributes.state.filters).toEqual([
{ version: '3.3' },
{ version: '3.3' },
]);
});
});
});

43
x-pack/plugins/lens/server/migrations/common_migrations.ts
View file

@ -7,6 +7,11 @@
import { cloneDeep } from 'lodash';
import { PaletteOutput } from 'src/plugins/charts/common';
import { Filter } from '@kbn/es-query';
import {
MigrateFunction,
MigrateFunctionsObject,
} from '../../../../../src/plugins/kibana_utils/common';
import {
LensDocShapePre712,
OperationTypePre712,
@ -19,6 +24,7 @@ import {
VisState716,
} from './types';
import { CustomPaletteParams, layerTypes } from '../../common';
import { LensDocShape } from './saved_object_migrations';
export const commonRenameOperationsForFormula = (
attributes: LensDocShapePre712
@ -155,3 +161,40 @@ export const commonMakeReversePaletteAsCustom = (
}
return newAttributes;
};
export const commonRenameFilterReferences = (attributes: LensDocShape715<VisState716>) => {
const newAttributes = cloneDeep(attributes);
for (const filter of newAttributes.state.filters) {
filter.meta.index = filter.meta.indexRefName;
delete filter.meta.indexRefName;
}
return newAttributes;
};
const getApplyFilterMigrationToLens = (filterMigration: MigrateFunction<Filter[]>) => {
return (savedObject: { attributes: LensDocShape }) => {
return {
...savedObject,
attributes: {
...savedObject.attributes,
state: {
...savedObject.attributes.state,
filters: filterMigration(savedObject.attributes.state.filters as unknown as Filter[]),
},
},
};
};
};
/**
* This creates a migration map that applies filter migrations to Lens visualizations
*/
export const getLensFilterMigrations = (filterMigrations: MigrateFunctionsObject) => {
const migrationMap: MigrateFunctionsObject = {};
for (const version in filterMigrations) {
if (filterMigrations.hasOwnProperty(version)) {
migrationMap[version] = getApplyFilterMigrationToLens(filterMigrations[version]);
}
}
return migrationMap;
};

162
x-pack/plugins/lens/server/migrations/saved_object_migrations.test.ts
View file

@ -6,7 +6,7 @@
*/
import { cloneDeep } from 'lodash';
import { migrations, LensDocShape } from './saved_object_migrations';
import { getAllMigrations, LensDocShape } from './saved_object_migrations';
import {
SavedObjectMigrationContext,
SavedObjectMigrationFn,
@ -15,8 +15,10 @@ import {
import { LensDocShape715, VisState716, VisStatePost715, VisStatePre715 } from './types';
import { CustomPaletteParams, layerTypes } from '../../common';
import { PaletteOutput } from 'src/plugins/charts/common';
import { Filter } from '@kbn/es-query';
describe('Lens migrations', () => {
const migrations = getAllMigrations({});
describe('7.7.0 missing dimensions in XY', () => {
const context = {} as SavedObjectMigrationContext;
@ -1404,4 +1406,162 @@ describe('Lens migrations', () => {
]);
});
});
describe('8.1.0 update filter reference schema', () => {
const context = { log: { warning: () => {} } } as unknown as SavedObjectMigrationContext;
const example = {
type: 'lens',
id: 'mocked-saved-object-id',
attributes: {
savedObjectId: '1',
title: 'MyRenamedOps',
description: '',
visualizationType: null,
state: {
datasourceMetaData: {
filterableIndexPatterns: [],
},
datasourceStates: {
indexpattern: {
currentIndexPatternId: 'logstash-*',
layers: {
'2': {
columns: {
'3': {
label: '@timestamp',
dataType: 'date',
operationType: 'date_histogram',
sourceField: '@timestamp',
isBucketed: true,
scale: 'interval',
params: { interval: 'auto', timeZone: 'Europe/Berlin' },
},
'4': {
label: '@timestamp',
dataType: 'date',
operationType: 'date_histogram',
sourceField: '@timestamp',
isBucketed: true,
scale: 'interval',
params: { interval: 'auto' },
},
'5': {
label: '@timestamp',
dataType: 'date',
operationType: 'my_unexpected_operation',
isBucketed: true,
scale: 'interval',
params: { timeZone: 'do not delete' },
},
},
columnOrder: ['3', '4', '5'],
},
},
},
},
visualization: {},
query: { query: '', language: 'kuery' },
filters: [
{
meta: {
alias: null,
negate: false,
disabled: false,
type: 'phrase',
key: 'geo.src',
params: { query: 'US' },
indexRefName: 'filter-index-pattern-0',
},
query: { match_phrase: { 'geo.src': 'US' } },
$state: { store: 'appState' },
},
{
meta: {
alias: null,
negate: false,
disabled: false,
type: 'phrase',
key: 'client_ip',
params: { query: '1234.5344.2243.3245' },
indexRefName: 'filter-index-pattern-2',
},
query: { match_phrase: { client_ip: '1234.5344.2243.3245' } },
$state: { store: 'appState' },
},
],
},
},
} as unknown as SavedObjectUnsanitizedDoc<LensDocShape715<VisState716>>;
it('should rename indexRefName to index in filters metadata', () => {
const expectedFilters = example.attributes.state.filters.map((filter) => {
return {
...filter,
meta: {
...filter.meta,
index: filter.meta.indexRefName,
indexRefName: undefined,
},
};
});
const result = migrations['8.1.0'](example, context) as ReturnType<
SavedObjectMigrationFn<LensDocShape, LensDocShape>
>;
expect(result.attributes.state.filters).toEqual(expectedFilters);
});
});
test('should properly apply a filter migration within a lens visualization', () => {
const migrationVersion = 'some-version';
const lensVisualizationDoc = {
attributes: {
state: {
filters: [
{
filter: 1,
migrated: false,
},
{
filter: 2,
migrated: false,
},
],
},
},
};
const migrationFunctionsObject = getAllMigrations({
[migrationVersion]: (filters: Filter[]) => {
return filters.map((filterState) => ({
...filterState,
migrated: true,
}));
},
});
const migratedLensDoc = migrationFunctionsObject[migrationVersion](
lensVisualizationDoc as SavedObjectUnsanitizedDoc,
{} as SavedObjectMigrationContext
);
expect(migratedLensDoc).toEqual({
attributes: {
state: {
filters: [
{
filter: 1,
migrated: true,
},
{
filter: 2,
migrated: true,
},
],
},
},
});
});
});

37
x-pack/plugins/lens/server/migrations/saved_object_migrations.ts
View file

@ -5,16 +5,18 @@
* 2.0.
*/
import { cloneDeep } from 'lodash';
import { cloneDeep, mergeWith } from 'lodash';
import { fromExpression, toExpression, Ast, ExpressionFunctionAST } from '@kbn/interpreter';
import {
SavedObjectMigrationMap,
SavedObjectMigrationFn,
SavedObjectReference,
SavedObjectUnsanitizedDoc,
SavedObjectMigrationContext,
} from 'src/core/server';
import { Filter } from '@kbn/es-query';
import { Query } from 'src/plugins/data/public';
import { MigrateFunctionsObject } from '../../../../../src/plugins/kibana_utils/common';
import { PersistableFilter } from '../../common';
import {
LensDocShapePost712,
@ -31,6 +33,8 @@ import {
commonRemoveTimezoneDateHistogramParam,
commonUpdateVisLayerType,
commonMakeReversePaletteAsCustom,
commonRenameFilterReferences,
getLensFilterMigrations,
} from './common_migrations';
interface LensDocShapePre710<VisualizationState = unknown> {
@ -440,7 +444,15 @@ const moveDefaultReversedPaletteToCustom: SavedObjectMigrationFn<
return { ...newDoc, attributes: commonMakeReversePaletteAsCustom(newDoc.attributes) };
};
export const migrations: SavedObjectMigrationMap = {
const renameFilterReferences: SavedObjectMigrationFn<
LensDocShape715<VisState716>,
LensDocShape715<VisState716>
> = (doc) => {
const newDoc = cloneDeep(doc);
return { ...newDoc, attributes: commonRenameFilterReferences(newDoc.attributes) };
};
const lensMigrations: SavedObjectMigrationMap = {
'7.7.0': removeInvalidAccessors,
// The order of these migrations matter, since the timefield migration relies on the aggConfigs
// sitting directly on the esaggs as an argument and not a nested function (which lens_auto_date was).
@ -453,4 +465,25 @@ export const migrations: SavedObjectMigrationMap = {
'7.14.0': removeTimezoneDateHistogramParam,
'7.15.0': addLayerTypeToVisualization,
'7.16.0': moveDefaultReversedPaletteToCustom,
'8.1.0': renameFilterReferences,
};
export const mergeSavedObjectMigrationMaps = (
obj1: SavedObjectMigrationMap,
obj2: SavedObjectMigrationMap
): SavedObjectMigrationMap => {
const customizer = (objValue: SavedObjectMigrationFn, srcValue: SavedObjectMigrationFn) => {
if (!srcValue || !objValue) {
return srcValue || objValue;
}
return (state: SavedObjectUnsanitizedDoc, context: SavedObjectMigrationContext) =>
objValue(srcValue(state, context), context);
};
return mergeWith({ ...obj1 }, obj2, customizer);
};
export const getAllMigrations = (
filterMigrations: MigrateFunctionsObject
): SavedObjectMigrationMap =>
mergeSavedObjectMigrationMaps(lensMigrations, getLensFilterMigrations(filterMigrations));

11
x-pack/plugins/lens/server/migrations/types.ts
View file

@ -8,7 +8,7 @@
import type { PaletteOutput } from 'src/plugins/charts/common';
import { Filter } from '@kbn/es-query';
import { Query } from 'src/plugins/data/public';
import type { CustomPaletteParams, LayerType } from '../../common';
import type { CustomPaletteParams, LayerType, PersistableFilter } from '../../common';
export type OperationTypePre712 =
| 'avg'
@ -191,10 +191,17 @@ export interface LensDocShape715<VisualizationState = unknown> {
};
visualization: VisualizationState;
query: Query;
filters: Filter[];
filters: PersistableFilter[];
};
}
export type LensDocShape810<VisualizationState = unknown> = Omit<
LensDocShape715<VisualizationState>,
'filters'
> & {
filters: Filter[];
};
export type VisState716 =
// Datatable
| {

14
x-pack/plugins/lens/server/plugin.tsx
View file

@ -7,7 +7,10 @@
import { Plugin, CoreSetup, CoreStart, PluginInitializerContext, Logger } from 'src/core/server';
import { UsageCollectionSetup } from 'src/plugins/usage_collection/server';
import { PluginStart as DataPluginStart } from 'src/plugins/data/server';
import {
PluginStart as DataPluginStart,
PluginSetup as DataPluginSetup,
} from 'src/plugins/data/server';
import { ExpressionsServerSetup } from 'src/plugins/expressions/server';
import { FieldFormatsStart } from 'src/plugins/field_formats/server';
import { TaskManagerSetupContract, TaskManagerStartContract } from '../../task_manager/server';
@ -19,14 +22,15 @@ import {
} from './usage';
import { setupSavedObjects } from './saved_objects';
import { EmbeddableSetup } from '../../../../src/plugins/embeddable/server';
import { lensEmbeddableFactory } from './embeddable/lens_embeddable_factory';
import { setupExpressions } from './expressions';
import { makeLensEmbeddableFactory } from './embeddable/make_lens_embeddable_factory';
export interface PluginSetupContract {
usageCollection?: UsageCollectionSetup;
taskManager?: TaskManagerSetupContract;
embeddable: EmbeddableSetup;
expressions: ExpressionsServerSetup;
data: DataPluginSetup;
}
export interface PluginStartContract {
@ -36,7 +40,7 @@ export interface PluginStartContract {
}
export interface LensServerPluginSetup {
lensEmbeddableFactory: typeof lensEmbeddableFactory;
lensEmbeddableFactory: ReturnType<typeof makeLensEmbeddableFactory>;
}
export class LensServerPlugin implements Plugin<LensServerPluginSetup, {}, {}, {}> {
@ -47,7 +51,8 @@ export class LensServerPlugin implements Plugin<LensServerPluginSetup, {}, {}, {
}
setup(core: CoreSetup<PluginStartContract>, plugins: PluginSetupContract) {
setupSavedObjects(core);
const filterMigrations = plugins.data.query.filterManager.getAllMigrations();
setupSavedObjects(core, filterMigrations);
setupRoutes(core, this.initializerContext.logger.get());
setupExpressions(core, plugins.expressions);
@ -61,6 +66,7 @@ export class LensServerPlugin implements Plugin<LensServerPluginSetup, {}, {}, {
initializeLensTelemetry(this.telemetryLogger, core, plugins.taskManager);
}
const lensEmbeddableFactory = makeLensEmbeddableFactory(filterMigrations);
plugins.embeddable.registerEmbeddableFactory(lensEmbeddableFactory());
return {
lensEmbeddableFactory,

7
x-pack/plugins/lens/server/saved_objects.ts
View file

@ -6,10 +6,11 @@
*/
import { CoreSetup } from 'kibana/server';
import { MigrateFunctionsObject } from '../../../../src/plugins/kibana_utils/common';
import { getEditPath } from '../common';
import { migrations } from './migrations/saved_object_migrations';
import { getAllMigrations } from './migrations/saved_object_migrations';
export function setupSavedObjects(core: CoreSetup) {
export function setupSavedObjects(core: CoreSetup, filterMigrations: MigrateFunctionsObject) {
core.savedObjects.registerType({
name: 'lens',
hidden: false,
@ -25,7 +26,7 @@ export function setupSavedObjects(core: CoreSetup) {
uiCapabilitiesPath: 'visualize.show',
}),
},
migrations,
migrations: getAllMigrations(filterMigrations),
mappings: {
properties: {
title: {