[ML] Adding dashboard custom url to lens created jobs (#142139)

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
James Gowdy 2022-10-12 20:52:04 +01:00 committed by GitHub
parent 839f8f6db4
commit aa710cb9c8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 362 additions and 182 deletions

View file

@ -21,8 +21,12 @@ import {
getLatestDataOrBucketTimestamp,
getEarliestDatafeedStartTime,
resolveMaxTimeInterval,
getFiltersForDSLQuery,
isKnownEmptyQuery,
} from './job_utils';
import { CombinedJob, Job } from '../types/anomaly_detection_jobs';
import { FilterStateStore } from '@kbn/es-query';
import moment from 'moment';
describe('ML - job utils', () => {
@ -613,3 +617,178 @@ describe('ML - job utils', () => {
});
});
});
describe('getFiltersForDSLQuery', () => {
describe('when DSL query contains match_all', () => {
test('returns empty array when query contains a must clause that contains match_all', () => {
const actual = getFiltersForDSLQuery(
{ bool: { must: [{ match_all: {} }] } },
'dataview-id',
'test-alias'
);
expect(actual).toEqual([]);
});
});
describe('when DSL query is valid', () => {
const query = {
bool: {
must: [],
filter: [
{
range: {
'@timestamp': {
format: 'strict_date_optional_time',
gte: '2007-09-29T15:05:14.509Z',
lte: '2022-09-29T15:05:14.509Z',
},
},
},
{
match_phrase: {
response_code: '200',
},
},
],
should: [],
must_not: [],
},
};
test('returns filters with alias', () => {
const actual = getFiltersForDSLQuery(query, 'dataview-id', 'test-alias');
expect(actual).toEqual([
{
$state: { store: 'appState' },
meta: {
alias: 'test-alias',
disabled: false,
index: 'dataview-id',
negate: false,
type: 'custom',
value:
'{"bool":{"must":[],"filter":[{"range":{"@timestamp":{"format":"strict_date_optional_time","gte":"2007-09-29T15:05:14.509Z","lte":"2022-09-29T15:05:14.509Z"}}},{"match_phrase":{"response_code":"200"}}],"should":[],"must_not":[]}}',
},
query,
},
]);
});
test('returns filter with no alias if alias is not provided', () => {
const actual = getFiltersForDSLQuery(query, 'dataview-id');
expect(actual).toEqual([
{
$state: { store: 'appState' },
meta: {
disabled: false,
index: 'dataview-id',
negate: false,
type: 'custom',
value:
'{"bool":{"must":[],"filter":[{"range":{"@timestamp":{"format":"strict_date_optional_time","gte":"2007-09-29T15:05:14.509Z","lte":"2022-09-29T15:05:14.509Z"}}},{"match_phrase":{"response_code":"200"}}],"should":[],"must_not":[]}}',
},
query,
},
]);
});
test('returns global state filter when GLOBAL_STATE is specified', () => {
const actual = getFiltersForDSLQuery(
query,
'dataview-id',
undefined,
FilterStateStore.GLOBAL_STATE
);
expect(actual).toEqual([
{
$state: { store: 'globalState' },
meta: {
disabled: false,
index: 'dataview-id',
negate: false,
type: 'custom',
value:
'{"bool":{"must":[],"filter":[{"range":{"@timestamp":{"format":"strict_date_optional_time","gte":"2007-09-29T15:05:14.509Z","lte":"2022-09-29T15:05:14.509Z"}}},{"match_phrase":{"response_code":"200"}}],"should":[],"must_not":[]}}',
},
query,
},
]);
});
});
describe('isKnownEmptyQuery', () => {
test('returns true for default lens created query', () => {
const result = isKnownEmptyQuery({
bool: {
filter: [],
must: [
{
match_all: {},
},
],
must_not: [],
},
});
expect(result).toBe(true);
});
test('returns true for default lens created query variation 1', () => {
const result = isKnownEmptyQuery({
bool: {
must: [
{
match_all: {},
},
],
must_not: [],
},
});
expect(result).toBe(true);
});
test('returns true for default lens created query variation 2', () => {
const result = isKnownEmptyQuery({
bool: {
must: [
{
match_all: {},
},
],
},
});
expect(result).toBe(true);
});
test('returns true for QA framework created query4', () => {
const result = isKnownEmptyQuery({
match_all: {},
});
expect(result).toBe(true);
});
test('returns false for query with match_phrase', () => {
const result = isKnownEmptyQuery({
match_phrase: {
region: 'us-east-1',
},
});
expect(result).toBe(false);
});
test('returns false for query with match_phrase in should', () => {
const result = isKnownEmptyQuery({
bool: {
should: [
{
match_phrase: {
region: 'us-east-1',
},
},
],
minimum_should_match: 1,
},
});
expect(result).toBe(false);
});
});
});

View file

@ -11,15 +11,25 @@ import moment, { Duration } from 'moment';
import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import numeral from '@elastic/numeral';
import { i18n } from '@kbn/i18n';
import type { Filter } from '@kbn/es-query';
import { isPopulatedObject } from '@kbn/ml-is-populated-object';
import type { SerializableRecord } from '@kbn/utility-types';
import { FilterStateStore } from '@kbn/es-query';
import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import { ALLOWED_DATA_UNITS, JOB_ID_MAX_LENGTH } from '../constants/validation';
import { parseInterval } from './parse_interval';
import { maxLengthValidator } from './validators';
import { CREATED_BY_LABEL } from '../constants/new_job';
import { CombinedJob, CustomSettings, Datafeed, Job, JobId } from '../types/anomaly_detection_jobs';
import { EntityField } from './anomaly_utils';
import { MlServerLimits } from '../types/ml_server_info';
import { JobValidationMessage, JobValidationMessageId } from '../constants/messages';
import type {
CombinedJob,
CustomSettings,
Datafeed,
Job,
JobId,
} from '../types/anomaly_detection_jobs';
import type { EntityField } from './anomaly_utils';
import type { MlServerLimits } from '../types/ml_server_info';
import type { JobValidationMessage, JobValidationMessageId } from '../constants/messages';
import { ES_AGGREGATION, ML_JOB_AGGREGATION } from '../constants/aggregation_types';
import { MLCATEGORY } from '../constants/field_types';
import { getAggregations, getDatafeedAggregations } from './datafeed_utils';
@ -866,3 +876,51 @@ export function resolveMaxTimeInterval(timeIntervals: string[]): number | undefi
return Number.isFinite(result) ? result : undefined;
}
export function getFiltersForDSLQuery(
datafeedQuery: QueryDslQueryContainer,
dataViewId: string | undefined,
alias?: string,
store = FilterStateStore.APP_STATE
): Filter[] {
if (isKnownEmptyQuery(datafeedQuery)) {
return [];
}
return [
{
meta: {
...(dataViewId !== undefined ? { index: dataViewId } : {}),
...(alias !== undefined ? { alias } : {}),
negate: false,
disabled: false,
type: 'custom',
value: JSON.stringify(datafeedQuery),
},
query: datafeedQuery as SerializableRecord,
$state: {
store,
},
},
];
}
// check to see if the query is a known "empty" shape
export function isKnownEmptyQuery(query: QueryDslQueryContainer) {
const queries = [
// the default query used by the job wizards
{ bool: { must: [{ match_all: {} }] } },
// the default query used created by lens created jobs
{ bool: { filter: [], must: [{ match_all: {} }], must_not: [] } },
// variations on the two previous queries
{ bool: { filter: [], must: [{ match_all: {} }] } },
{ bool: { must: [{ match_all: {} }], must_not: [] } },
// the query generated by QA Framework created jobs
{ match_all: {} },
];
if (queries.some((q) => isEqual(q, query))) {
return true;
}
return false;
}

View file

@ -145,6 +145,7 @@ export const renderApp = (
maps: deps.maps,
dataVisualizer: deps.dataVisualizer,
dataViews: deps.data.dataViews,
share: deps.share,
});
appMountParams.onAppLeave((actions) => actions.default());

View file

@ -1,94 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { getFiltersForDSLQuery } from './get_filters_for_datafeed_query';
describe('getFiltersForDSLQuery', () => {
describe('when DSL query contains match_all', () => {
test('returns empty array when query contains a must clause that contains match_all', () => {
const actual = getFiltersForDSLQuery(
{ bool: { must: [{ match_all: {} }] } },
'dataview-id',
'test-alias'
);
expect(actual).toEqual([]);
});
test('returns empty array when query contains match_all', () => {
const actual = getFiltersForDSLQuery({ match_all: {} }, 'dataview-id', 'test-alias');
expect(actual).toEqual([]);
});
});
describe('when DSL query is valid', () => {
const query = {
bool: {
must: [],
filter: [
{
range: {
'@timestamp': {
format: 'strict_date_optional_time',
gte: '2007-09-29T15:05:14.509Z',
lte: '2022-09-29T15:05:14.509Z',
},
},
},
{
match_phrase: {
response_code: '200',
},
},
],
should: [],
must_not: [],
},
};
test('returns filters with alias', () => {
const actual = getFiltersForDSLQuery(query, 'dataview-id', 'test-alias');
expect(actual).toEqual([
{
$state: { store: 'appState' },
meta: {
alias: 'test-alias',
disabled: false,
index: 'dataview-id',
negate: false,
type: 'custom',
value:
'{"bool":{"must":[],"filter":[{"range":{"@timestamp":{"format":"strict_date_optional_time","gte":"2007-09-29T15:05:14.509Z","lte":"2022-09-29T15:05:14.509Z"}}},{"match_phrase":{"response_code":"200"}}],"should":[],"must_not":[]}}',
},
query,
},
]);
});
test('returns empty array when dataViewId is invalid', () => {
const actual = getFiltersForDSLQuery(query, null, 'test-alias');
expect(actual).toEqual([]);
});
test('returns filter with no alias if alias is not provided', () => {
const actual = getFiltersForDSLQuery(query, 'dataview-id');
expect(actual).toEqual([
{
$state: { store: 'appState' },
meta: {
disabled: false,
index: 'dataview-id',
negate: false,
type: 'custom',
value:
'{"bool":{"must":[],"filter":[{"range":{"@timestamp":{"format":"strict_date_optional_time","gte":"2007-09-29T15:05:14.509Z","lte":"2022-09-29T15:05:14.509Z"}}},{"match_phrase":{"response_code":"200"}}],"should":[],"must_not":[]}}',
},
query,
},
]);
});
});
});

View file

@ -1,45 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { isPopulatedObject } from '@kbn/ml-is-populated-object';
import type { SerializableRecord } from '@kbn/utility-types';
import { FilterStateStore } from '@kbn/es-query';
import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import { isEqual } from 'lodash';
const defaultEmptyQuery = { bool: { must: [{ match_all: {} }] } };
export const getFiltersForDSLQuery = (
datafeedQuery: QueryDslQueryContainer,
dataViewId: string | null,
alias?: string
) => {
if (
datafeedQuery &&
!isPopulatedObject(datafeedQuery, ['match_all']) &&
!isEqual(datafeedQuery, defaultEmptyQuery) &&
dataViewId !== null
) {
return [
{
meta: {
index: dataViewId,
...(!!alias ? { alias } : {}),
negate: false,
disabled: false,
type: 'custom',
value: JSON.stringify(datafeedQuery),
},
query: datafeedQuery as SerializableRecord,
$state: {
store: FilterStateStore.APP_STATE,
},
},
];
}
return [];
};

View file

@ -53,7 +53,7 @@ import { useMlKibana } from '../../contexts/kibana';
import { getFieldTypeFromMapping } from '../../services/mapping_service';
import type { AnomaliesTableRecord } from '../../../../common/types/anomalies';
import { getQueryStringForInfluencers } from './get_query_string_for_influencers';
import { getFiltersForDSLQuery } from './get_filters_for_datafeed_query';
import { getFiltersForDSLQuery } from '../../../../common/util/job_utils';
interface LinksMenuProps {
anomaly: AnomaliesTableRecord;
bounds: TimeRangeBounds;
@ -112,7 +112,10 @@ export const LinksMenuUI = (props: LinksMenuProps) => {
},
}
: {}),
filters: getFiltersForDSLQuery(job.datafeed_config.query, dataViewId, job.job_id),
filters:
dataViewId === null
? []
: getFiltersForDSLQuery(job.datafeed_config.query, dataViewId, job.job_id),
});
return location;
};
@ -150,11 +153,10 @@ export const LinksMenuUI = (props: LinksMenuProps) => {
);
const locator = share.url.locators.get(MAPS_APP_LOCATOR);
const filtersFromDatafeedQuery = getFiltersForDSLQuery(
job.datafeed_config.query,
dataViewId,
job.job_id
);
const filtersFromDatafeedQuery =
dataViewId === null
? []
: getFiltersForDSLQuery(job.datafeed_config.query, dataViewId, job.job_id);
const location = await locator?.getLocation({
initialLayers,
timeRange,
@ -265,7 +267,10 @@ export const LinksMenuUI = (props: LinksMenuProps) => {
language: 'kuery',
query: kqlQuery,
},
filters: getFiltersForDSLQuery(job.datafeed_config.query, dataViewId, job.job_id),
filters:
dataViewId === null
? []
: getFiltersForDSLQuery(job.datafeed_config.query, dataViewId, job.job_id),
sort: [['timestamp, asc']],
});

View file

@ -10,7 +10,10 @@ import { TIME_RANGE_TYPE, URL_TYPE } from './constants';
import rison from 'rison-node';
import url from 'url';
import { getPartitioningFieldNames } from '../../../../../common/util/job_utils';
import {
getPartitioningFieldNames,
getFiltersForDSLQuery,
} from '../../../../../common/util/job_utils';
import { parseInterval } from '../../../../../common/util/parse_interval';
import { replaceTokensInUrlValue, isValidLabel } from '../../../util/custom_url_utils';
import { ml } from '../../../services/ml_api_service';
@ -20,7 +23,6 @@ import { getSavedObjectsClient, getDashboard } from '../../../util/dependency_ca
import { setStateToKbnUrl } from '@kbn/kibana-utils-plugin/public';
import { cleanEmptyKeys } from '@kbn/dashboard-plugin/public';
import { isFilterPinned } from '@kbn/es-query';
import { getFiltersForDSLQuery } from '../../../components/anomalies_table/get_filters_for_datafeed_query';
export function getNewCustomUrlDefaults(job, dashboards, dataViews) {
// Returns the settings object in the format used by the custom URL editor
@ -51,11 +53,10 @@ export function getNewCustomUrlDefaults(job, dashboards, dataViews) {
const indicesName = datafeedConfig.indices.join();
const defaultDataViewId = dataViews.find((dv) => dv.title === indicesName)?.id;
kibanaSettings.discoverIndexPatternId = defaultDataViewId;
kibanaSettings.filters = getFiltersForDSLQuery(
job.datafeed_config.query,
defaultDataViewId,
job.job_id
);
kibanaSettings.filters =
defaultDataViewId === null
? []
: getFiltersForDSLQuery(job.datafeed_config.query, defaultDataViewId, job.job_id);
}
return {

View file

@ -5,18 +5,21 @@
* 2.0.
*/
import { i18n } from '@kbn/i18n';
import { mergeWith, uniqBy, isEqual } from 'lodash';
import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import type { Embeddable } from '@kbn/lens-plugin/public';
import type {
Embeddable,
LensSavedObjectAttributes,
XYDataLayerConfig,
} from '@kbn/lens-plugin/public';
import type { IUiSettingsClient } from '@kbn/core/public';
import type { DataViewsContract } from '@kbn/data-views-plugin/public';
import type { TimefilterContract } from '@kbn/data-plugin/public';
import { Filter, Query, DataViewBase } from '@kbn/es-query';
import type { LensSavedObjectAttributes, XYDataLayerConfig } from '@kbn/lens-plugin/public';
import { i18n } from '@kbn/i18n';
import type { SharePluginStart } from '@kbn/share-plugin/public';
import type { DashboardAppLocatorParams } from '@kbn/dashboard-plugin/public';
import type { Filter, Query, DataViewBase } from '@kbn/es-query';
import { FilterStateStore } from '@kbn/es-query';
import type { JobCreatorType } from '../common/job_creator';
import { createEmptyJob, createEmptyDatafeed } from '../common/job_creator/util/default_configs';
@ -24,6 +27,7 @@ import { stashJobForCloning } from '../common/job_creator/util/general';
import type { ErrorType } from '../../../../../common/util/errors';
import { createDatafeedId } from '../../../../../common/util/job_utils';
import type { MlApiServices } from '../../../services/ml_api_service';
import { getFiltersForDSLQuery } from '../../../../../common/util/job_utils';
import {
CREATED_BY_LABEL,
DEFAULT_BUCKET_SPAN,
@ -33,6 +37,8 @@ import { createQueries } from '../utils/new_job_utils';
import { isCompatibleLayer, createDetectors, getJobsItemsFromEmbeddable } from './utils';
import { VisualizationExtractor } from './visualization_extractor';
type Dashboard = Embeddable['parent'];
interface CreationState {
success: boolean;
error?: ErrorType;
@ -47,10 +53,11 @@ interface CreateState {
export class QuickJobCreator {
constructor(
private dataViewClient: DataViewsContract,
private kibanaConfig: IUiSettingsClient,
private timeFilter: TimefilterContract,
private mlApiServices: MlApiServices
private readonly dataViewClient: DataViewsContract,
private readonly kibanaConfig: IUiSettingsClient,
private readonly timeFilter: TimefilterContract,
private readonly share: SharePluginStart,
private readonly mlApiServices: MlApiServices
) {}
public async createAndSaveJob(
@ -61,7 +68,7 @@ export class QuickJobCreator {
runInRealTime: boolean,
layerIndex: number
): Promise<CreateState> {
const { query, filters, to, from, vis } = getJobsItemsFromEmbeddable(embeddable);
const { query, filters, to, from, vis, dashboard } = getJobsItemsFromEmbeddable(embeddable);
if (query === undefined || filters === undefined) {
throw new Error('Cannot create job, query and filters are undefined');
}
@ -73,10 +80,13 @@ export class QuickJobCreator {
query,
filters,
bucketSpan,
layerIndex
);
const job = {
const datafeedId = createDatafeedId(jobId);
const datafeed = { ...datafeedConfig, job_id: jobId, datafeed_id: datafeedId };
const job: estypes.MlJob = {
...jobConfig,
job_id: jobId,
custom_settings: {
@ -84,12 +94,10 @@ export class QuickJobCreator {
jobType === JOB_TYPE.SINGLE_METRIC
? CREATED_BY_LABEL.SINGLE_METRIC_FROM_LENS
: CREATED_BY_LABEL.MULTI_METRIC_FROM_LENS,
...(await this.getCustomUrls(dashboard, datafeed)),
},
};
const datafeedId = createDatafeedId(jobId);
const datafeed = { ...datafeedConfig, job_id: jobId, datafeed_id: datafeedId };
const result: CreateState = {
jobCreated: { success: false },
datafeedCreated: { success: false },
@ -330,4 +338,49 @@ export class QuickJobCreator {
return mergedQueries;
}
private async createDashboardLink(dashboard: Dashboard, datafeedConfig: estypes.MlDatafeed) {
if (dashboard === undefined) {
// embeddable may have not been in a dashboard
return null;
}
const params: DashboardAppLocatorParams = {
dashboardId: dashboard.id,
timeRange: {
from: '$earliest$',
to: '$latest$',
mode: 'absolute',
},
filters: getFiltersForDSLQuery(
datafeedConfig.query,
undefined,
datafeedConfig.job_id,
FilterStateStore.GLOBAL_STATE
),
};
const dashboardLocator = this.share.url.locators.get('DASHBOARD_APP_LOCATOR');
const encodedUrl = dashboardLocator ? await dashboardLocator.getUrl(params) : '';
const url = decodeURIComponent(encodedUrl).replace(/^.+dashboards/, 'dashboards');
const dashboardName = dashboard.getOutput().title;
const urlName =
dashboardName === undefined
? i18n.translate('xpack.ml.newJob.fromLens.createJob.defaultUrlDashboard', {
defaultMessage: 'Original dashboard',
})
: i18n.translate('xpack.ml.newJob.fromLens.createJob.namedUrlDashboard', {
defaultMessage: 'Open {dashboardName}',
values: { dashboardName },
});
return { url_name: urlName, url_value: url, time_range: 'auto' };
}
private async getCustomUrls(dashboard: Dashboard, datafeedConfig: estypes.MlDatafeed) {
return dashboard !== undefined
? { custom_urls: [await this.createDashboardLink(dashboard, datafeedConfig)] }
: {};
}
}

View file

@ -17,6 +17,7 @@ import {
getDataViews,
getSavedObjectsClient,
getTimefilter,
getShare,
} from '../../../util/dependency_cache';
import { getDefaultQuery } from '../utils/new_job_utils';
@ -70,7 +71,13 @@ export async function resolver(
layerIndex = undefined;
}
const jobCreator = new QuickJobCreator(getDataViews(), getUiSettings(), getTimefilter(), ml);
const jobCreator = new QuickJobCreator(
getDataViews(),
getUiSettings(),
getTimefilter(),
getShare(),
ml
);
await jobCreator.createAndStashADJob(vis, from, to, query, filters, layerIndex);
}

View file

@ -92,12 +92,15 @@ export function getJobsItemsFromEmbeddable(embeddable: Embeddable) {
);
}
const dashboard = embeddable.parent?.type === 'dashboard' ? embeddable.parent : undefined;
return {
vis,
from,
to,
query,
filters,
dashboard,
};
}

View file

@ -27,6 +27,7 @@ import type { DataViewsContract } from '@kbn/data-views-plugin/public';
import type { SecurityPluginStart } from '@kbn/security-plugin/public';
import type { MapsStartApi } from '@kbn/maps-plugin/public';
import type { DataVisualizerPluginStart } from '@kbn/data-visualizer-plugin/public';
import type { SharePluginStart } from '@kbn/share-plugin/public';
export interface DependencyCache {
timefilter: DataPublicPluginSetup['query']['timefilter'] | null;
@ -49,6 +50,7 @@ export interface DependencyCache {
maps: MapsStartApi | null;
dataVisualizer: DataVisualizerPluginStart | null;
dataViews: DataViewsContract | null;
share: SharePluginStart | null;
}
const cache: DependencyCache = {
@ -72,6 +74,7 @@ const cache: DependencyCache = {
maps: null,
dataVisualizer: null,
dataViews: null,
share: null,
};
export function setDependencyCache(deps: Partial<DependencyCache>) {
@ -94,6 +97,7 @@ export function setDependencyCache(deps: Partial<DependencyCache>) {
cache.dashboard = deps.dashboard || null;
cache.dataVisualizer = deps.dataVisualizer || null;
cache.dataViews = deps.dataViews || null;
cache.share = deps.share || null;
}
export function getTimefilter() {
@ -228,15 +232,22 @@ export function getDataViews() {
return cache.dataViews;
}
export function clearCache() {
Object.keys(cache).forEach((k) => {
cache[k as keyof DependencyCache] = null;
});
}
export function getFileDataVisualizer() {
if (cache.dataVisualizer === null) {
throw new Error("dataVisualizer hasn't been initialized");
}
return cache.dataVisualizer;
}
export function getShare() {
if (cache.share === null) {
throw new Error("share hasn't been initialized");
}
return cache.share;
}
export function clearCache() {
Object.keys(cache).forEach((k) => {
cache[k as keyof DependencyCache] = null;
});
}

View file

@ -82,6 +82,7 @@ export const CompatibleLayer: FC<Props> = ({ layer, layerIndex, embeddable }) =>
data.dataViews,
uiSettings,
data.query.timefilter.timefilter,
share,
mlApiServices
),
// eslint-disable-next-line react-hooks/exhaustive-deps