[ML] Data Frame Analytics results view: add link to custom visualizations for viewing scatterplot charts (#149647)

## Summary

This PR adds a link to the custom visualization UI from scatterplot
matrix charts in the data frame analytics job wizard and results views.
This allows you to view the data and edit the visualization inside the
custom visualization editor, and then save it to a dashboard to explore
the data alongside other contextual information.

<img width="884" alt="image"
src="https://user-images.githubusercontent.com/7405507/215125288-9d5527fa-6074-42bd-98f1-2c04cd025432.png">

Related meta issue: https://github.com/elastic/kibana/issues/131551

### Checklist

Delete any items that are not applicable to this PR.

- [ ] Any text added follows [EUI's writing
guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses
sentence case text and includes [i18n
support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md)
- [ ]
[Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html)
was added for features that require explanation or tutorials
- [ ] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [ ] Any UI touched in this PR is usable by keyboard only (learn more
about [keyboard accessibility](https://webaim.org/techniques/keyboard/))
- [ ] Any UI touched in this PR does not create any new axe failures
(run axe in browser:
[FF](https://addons.mozilla.org/en-US/firefox/addon/axe-devtools/),
[Chrome](https://chrome.google.com/webstore/detail/axe-web-accessibility-tes/lhdoppojpmngadmnindnejefpokejbdd?hl=en-US))
- [ ] If a plugin configuration key changed, check if it needs to be
allowlisted in the cloud and added to the [docker
list](https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker)
- [ ] This renders correctly on smaller devices using a responsive
layout. (You can test this [in your
browser](https://www.browserstack.com/guide/responsive-testing-on-local-server))
- [ ] This was checked for [cross-browser
compatibility](https://www.elastic.co/support/matrix#matrix_browsers)

---------

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Melissa Alvarez 2023-01-31 12:16:44 -07:00 committed by GitHub
parent a32b5a450a
commit e534d87843
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 203 additions and 23 deletions

View file

@ -12,8 +12,12 @@ import { __IntlProvider as IntlProvider } from '@kbn/i18n-react';
import { euiLightVars as euiThemeLight } from '@kbn/ui-theme';
import { createFilterManagerMock } from '@kbn/data-plugin/public/query/filter_manager/filter_manager.mock';
import { ScatterplotMatrix } from './scatterplot_matrix';
const mockFilterManager = createFilterManagerMock();
const mockEsSearch = jest.fn((body) => ({
hits: { hits: [{ fields: { x: [1], y: [2] } }, { fields: { x: [2], y: [3] } }] },
}));
@ -21,6 +25,26 @@ jest.mock('../../contexts/kibana', () => ({
useMlApiContext: () => ({
esSearch: mockEsSearch,
}),
useMlKibana: () => ({
services: {
application: {
navigateToApp: jest.fn(),
},
data: {
query: {
filterManager: mockFilterManager,
timefilter: {
timefilter: {
getTime: jest.fn(() => {
return { from: '', to: '' };
}),
getRefreshInterval: jest.fn(),
},
},
},
},
},
}),
}));
const mockEuiTheme = euiThemeLight;

View file

@ -5,9 +5,9 @@
* 2.0.
*/
import React, { useMemo, useEffect, useState, FC } from 'react';
import React, { useMemo, useEffect, useState, FC, useCallback } from 'react';
import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import rison from '@kbn/rison';
import {
EuiCallOut,
@ -17,12 +17,14 @@ import {
EuiFlexItem,
EuiFormRow,
EuiIconTip,
EuiLink,
EuiSelect,
EuiSpacer,
EuiSwitch,
} from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import { Query } from '@kbn/data-plugin/common/query';
import { DataView } from '@kbn/data-views-plugin/public';
import { stringHash } from '@kbn/ml-string-hash';
@ -31,7 +33,7 @@ import { isRuntimeMappings } from '../../../../common/util/runtime_field_utils';
import { RuntimeMappings } from '../../../../common/types/fields';
import { getCombinedRuntimeMappings } from '../data_grid';
import { useMlApiContext } from '../../contexts/kibana';
import { useMlApiContext, useMlKibana } from '../../contexts/kibana';
import { getProcessedFields } from '../data_grid';
import { useCurrentEuiTheme } from '../color_range_legend';
@ -101,6 +103,7 @@ export interface ScatterplotMatrixProps {
searchQuery?: estypes.QueryDslQueryContainer;
runtimeMappings?: RuntimeMappings;
indexPattern?: DataView;
query?: Query;
}
export const ScatterplotMatrix: FC<ScatterplotMatrixProps> = ({
@ -112,9 +115,13 @@ export const ScatterplotMatrix: FC<ScatterplotMatrixProps> = ({
searchQuery,
runtimeMappings,
indexPattern,
query,
}) => {
const { esSearch } = useMlApiContext();
const kibana = useMlKibana();
const {
services: { application, data },
} = kibana;
// dynamicSize is optionally used for outlier charts where the scatterplot marks
// are sized according to outlier_score
const [dynamicSize, setDynamicSize] = useState<boolean>(false);
@ -142,6 +149,8 @@ export const ScatterplotMatrix: FC<ScatterplotMatrixProps> = ({
{ items: any[]; backgroundItems: any[]; columns: string[]; messages: string[] } | undefined
>();
const { euiTheme } = useCurrentEuiTheme();
// formats the array of field names for EuiComboBox
const fieldOptions = useMemo(
() =>
@ -172,7 +181,77 @@ export const ScatterplotMatrix: FC<ScatterplotMatrixProps> = ({
setDynamicSize(!dynamicSize);
};
const { euiTheme } = useCurrentEuiTheme();
const getCustomVisualizationLink = useCallback(() => {
const { columns } = splom!;
const outlierScoreField =
resultsField !== undefined ? `${resultsField}.${OUTLIER_SCORE_FIELD}` : undefined;
const vegaSpec = getScatterplotMatrixVegaLiteSpec(
true,
[],
[],
columns,
euiTheme,
resultsField,
color,
legendType,
dynamicSize
);
vegaSpec.$schema = 'https://vega.github.io/schema/vega-lite/v5.json';
vegaSpec.title = `Scatterplot matrix for ${index}`;
const fieldsToFetch = [
...columns,
// Add outlier_score field in fetch if it's available so custom visualization can use it
...(outlierScoreField ? [outlierScoreField] : []),
// Add field to color code by in fetch so custom visualization can use it - usually for classfication jobs
...(color ? [color] : []),
];
vegaSpec.data = {
url: {
'%context%': true,
...(indexPattern?.timeFieldName
? { ['%timefield%']: `${indexPattern?.timeFieldName}` }
: {}),
index,
body: {
fields: fieldsToFetch,
size: fetchSize,
_source: false,
},
},
format: { property: 'hits.hits' },
};
const globalState = encodeURIComponent(
rison.encode({
filters: data.query.filterManager.getFilters(),
refreshInterval: data.query.timefilter.timefilter.getRefreshInterval(),
time: data.query.timefilter.timefilter.getTime(),
})
);
const appState = encodeURIComponent(
rison.encode({
filters: [],
linked: false,
query,
uiState: {},
vis: {
aggs: [],
params: {
spec: JSON.stringify(vegaSpec, null, 2),
},
},
})
);
const basePath = `/create?type=vega&_g=${globalState}&_a=${appState}`;
return { path: basePath };
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [splom]);
useEffect(() => {
if (fields.length === 0) {
@ -316,6 +395,7 @@ export const ScatterplotMatrix: FC<ScatterplotMatrixProps> = ({
const { items, backgroundItems, columns } = splom;
return getScatterplotMatrixVegaLiteSpec(
false,
items,
backgroundItems,
columns,
@ -442,6 +522,29 @@ export const ScatterplotMatrix: FC<ScatterplotMatrixProps> = ({
</EuiFormRow>
</EuiFlexItem>
)}
{splom ? (
<EuiFlexItem grow={false}>
<EuiLink
onClick={async () => {
const customVisLink = getCustomVisualizationLink();
await application.navigateToApp('visualize#', {
path: customVisLink.path,
openInNewTab: false,
});
}}
data-test-subj="mlSplomoExploreInCustomVisualizationLink"
>
<EuiIconTip
content={i18n.translate('xpack.ml.splom.exploreInCustomVisualizationLabel', {
defaultMessage:
'Explore scatterplot charts in Vega based custom visualization',
})}
type="visVega"
size="l"
/>
</EuiLink>
</EuiFlexItem>
) : null}
</EuiFlexGroup>
{splom.messages.length > 0 && (

View file

@ -26,7 +26,7 @@ import {
describe('getColorSpec()', () => {
it('should return only user selection conditions and the default color for non-outlier specs', () => {
const colorSpec = getColorSpec(euiThemeLight);
const colorSpec = getColorSpec(false, euiThemeLight);
expect(colorSpec).toEqual({
condition: [{ selection: USER_SELECTION }, { selection: SINGLE_POINT_CLICK }],
@ -35,7 +35,7 @@ describe('getColorSpec()', () => {
});
it('should return user selection condition and conditional spec for outliers', () => {
const colorSpec = getColorSpec(euiThemeLight, 'outlier_score');
const colorSpec = getColorSpec(false, euiThemeLight, 'outlier_score');
expect(colorSpec).toEqual({
condition: {
@ -53,7 +53,13 @@ describe('getColorSpec()', () => {
it('should return user selection condition and a field based spec for non-outlier specs with legendType supplied', () => {
const colorName = 'the-color-field';
const colorSpec = getColorSpec(euiThemeLight, undefined, colorName, LEGEND_TYPES.NOMINAL);
const colorSpec = getColorSpec(
false,
euiThemeLight,
undefined,
colorName,
LEGEND_TYPES.NOMINAL
);
expect(colorSpec).toEqual({
condition: {
@ -70,10 +76,18 @@ describe('getColorSpec()', () => {
});
describe('getScatterplotMatrixVegaLiteSpec()', () => {
const forCustomLink = false;
it('should return the default spec for non-outliers without a legend', () => {
const data = [{ x: 1, y: 1 }];
const vegaLiteSpec = getScatterplotMatrixVegaLiteSpec(data, [], ['x', 'y'], euiThemeLight);
const vegaLiteSpec = getScatterplotMatrixVegaLiteSpec(
forCustomLink,
data,
[],
['x', 'y'],
euiThemeLight
);
const specForegroundLayer = vegaLiteSpec.spec.layer[0];
// A valid Vega Lite spec shouldn't throw an error when compiled.
@ -103,6 +117,7 @@ describe('getScatterplotMatrixVegaLiteSpec()', () => {
const data = [{ x: 1, y: 1 }];
const vegaLiteSpec = getScatterplotMatrixVegaLiteSpec(
forCustomLink,
data,
[],
['x', 'y'],
@ -151,6 +166,7 @@ describe('getScatterplotMatrixVegaLiteSpec()', () => {
const data = [{ x: 1, y: 1 }];
const vegaLiteSpec = getScatterplotMatrixVegaLiteSpec(
forCustomLink,
data,
[],
['x', 'y'],
@ -196,6 +212,7 @@ describe('getScatterplotMatrixVegaLiteSpec()', () => {
const data = [{ ['x.a']: 1, ['y[a]']: 1 }];
const vegaLiteSpec = getScatterplotMatrixVegaLiteSpec(
forCustomLink,
data,
[],
['x.a', 'y[a]'],

View file

@ -29,8 +29,10 @@ export const COLOR_SELECTION = euiPaletteColorBlind()[2];
export const COLOR_RANGE_OUTLIER = [euiPaletteColorBlind()[1], euiPaletteColorBlind()[2]];
export const COLOR_RANGE_NOMINAL = euiPaletteColorBlind({ rotations: 2 });
export const COLOR_RANGE_QUANTITATIVE = euiPalettePositive(5);
const CUSTOM_VIS_FIELDS_PATH = 'fields';
export const getColorSpec = (
forCustomVisLink: boolean,
euiTheme: typeof euiThemeLight,
escapedOutlierScoreField?: string,
color?: string,
@ -58,7 +60,10 @@ export const getColorSpec = (
return {
condition: {
selection: USER_SELECTION,
field: getEscapedVegaFieldName(color ?? '00FF00'),
field: `${forCustomVisLink ? `${CUSTOM_VIS_FIELDS_PATH}.` : ''}${getEscapedVegaFieldName(
color ?? '00FF00'
// When creating the custom link - this field is returned in an array so we need to access it
)}${forCustomVisLink ? '[0]' : ''}`,
type: legendType,
scale: {
range:
@ -76,6 +81,7 @@ export const getColorSpec = (
};
const getVegaSpecLayer = (
forCustomVisLink: boolean,
isBackground: boolean,
values: VegaValue[],
colorSpec: any,
@ -117,7 +123,8 @@ const getVegaSpecLayer = (
};
return {
data: { values: [...values] },
// Don't need to add static data for custom vis links
...(forCustomVisLink ? {} : { data: { values: [...values] } }),
mark: {
...(outliers && dynamicSize
? {
@ -133,7 +140,9 @@ const getVegaSpecLayer = (
? {
transform: [
{
calculate: `datum['${escapedOutlierScoreField}'] >= mlOutlierScoreThreshold.cutoff`,
calculate: `datum${
forCustomVisLink ? `.${CUSTOM_VIS_FIELDS_PATH}` : ''
}['${escapedOutlierScoreField}'] >= mlOutlierScoreThreshold.cutoff`,
as: 'is_outlier',
},
],
@ -154,7 +163,9 @@ const getVegaSpecLayer = (
opacity: {
condition: {
value: 1,
test: `(datum['${escapedOutlierScoreField}'] >= mlOutlierScoreThreshold.cutoff)`,
test: `(datum${
forCustomVisLink ? `.${CUSTOM_VIS_FIELDS_PATH}` : ''
}['${escapedOutlierScoreField}'] >= mlOutlierScoreThreshold.cutoff)`,
},
value: 0.5,
},
@ -162,19 +173,27 @@ const getVegaSpecLayer = (
: {}),
...(outliers
? {
order: { field: escapedOutlierScoreField },
order: {
field: `${
forCustomVisLink ? `${CUSTOM_VIS_FIELDS_PATH}.` : ''
}${escapedOutlierScoreField}`,
},
size: {
...(!dynamicSize
? {
condition: {
value: 40,
test: `(datum['${escapedOutlierScoreField}'] >= mlOutlierScoreThreshold.cutoff)`,
test: `(datum${
forCustomVisLink ? `.${CUSTOM_VIS_FIELDS_PATH}` : ''
}['${escapedOutlierScoreField}'] >= mlOutlierScoreThreshold.cutoff)`,
},
value: 8,
}
: {
type: LEGEND_TYPES.QUANTITATIVE,
field: escapedOutlierScoreField,
field: `${
forCustomVisLink ? `${CUSTOM_VIS_FIELDS_PATH}.` : ''
}${escapedOutlierScoreField}`,
scale: {
type: 'linear',
range: [8, 200],
@ -197,7 +216,14 @@ const getVegaSpecLayer = (
tooltip: [
...(color !== undefined
? // @ts-ignore
[{ type: colorSpec.condition.type, field: getEscapedVegaFieldName(color) }]
[
{
type: colorSpec.condition.type,
field: `${
forCustomVisLink ? `${CUSTOM_VIS_FIELDS_PATH}.` : ''
}${getEscapedVegaFieldName(color)}`,
},
]
: []),
...vegaColumns.map((d) => ({
type: LEGEND_TYPES.QUANTITATIVE,
@ -207,7 +233,9 @@ const getVegaSpecLayer = (
? [
{
type: LEGEND_TYPES.QUANTITATIVE,
field: escapedOutlierScoreField,
field: `${
forCustomVisLink ? `${CUSTOM_VIS_FIELDS_PATH}.` : ''
}${escapedOutlierScoreField}`,
format: '.3f',
},
]
@ -215,21 +243,22 @@ const getVegaSpecLayer = (
],
},
...(isBackground ? {} : selection),
width: SCATTERPLOT_SIZE,
height: SCATTERPLOT_SIZE,
...(forCustomVisLink ? {} : { width: SCATTERPLOT_SIZE }),
...(forCustomVisLink ? {} : { height: SCATTERPLOT_SIZE }),
};
};
// Escapes the characters .[] in field names with double backslashes
// since VEGA treats dots/brackets in field names as nested values.
// See https://vega.github.io/vega-lite/docs/field.html for details.
function getEscapedVegaFieldName(fieldName: string) {
return fieldName.replace(/([\.|\[|\]])/g, '\\$1');
function getEscapedVegaFieldName(fieldName: string, prependString: string = '') {
return `${prependString}${fieldName.replace(/([\.|\[|\]])/g, '\\$1')}`;
}
type VegaValue = Record<string, string | number>;
export const getScatterplotMatrixVegaLiteSpec = (
forCustomVisLink: boolean,
values: VegaValue[],
backgroundValues: VegaValue[],
columns: string[],
@ -240,12 +269,15 @@ export const getScatterplotMatrixVegaLiteSpec = (
dynamicSize?: boolean
): TopLevelSpec => {
const vegaValues = values;
const vegaColumns = columns.map(getEscapedVegaFieldName);
const vegaColumns = columns.map((column) =>
getEscapedVegaFieldName(column, forCustomVisLink ? 'fields.' : '')
);
const outliers = resultsField !== undefined;
const escapedOutlierScoreField = `${resultsField}\\.${OUTLIER_SCORE_FIELD}`;
const colorSpec = getColorSpec(
forCustomVisLink,
euiTheme,
resultsField && escapedOutlierScoreField,
color,
@ -282,6 +314,7 @@ export const getScatterplotMatrixVegaLiteSpec = (
spec: {
layer: [
getVegaSpecLayer(
forCustomVisLink,
false,
vegaValues,
colorSpec,
@ -298,6 +331,7 @@ export const getScatterplotMatrixVegaLiteSpec = (
if (backgroundValues.length) {
schema.spec.layer.unshift(
getVegaSpecLayer(
forCustomVisLink,
true,
backgroundValues,
colorSpec,

View file

@ -225,6 +225,7 @@ export const ExplorationPageWrapper: FC<Props> = ({
<ExpandableSectionSplom
fields={scatterplotFieldOptions}
index={jobConfig?.dest.index}
indexPattern={indexPattern}
color={
jobType === ANALYSIS_CONFIG_TYPE.REGRESSION ||
jobType === ANALYSIS_CONFIG_TYPE.CLASSIFICATION

View file

@ -145,6 +145,7 @@ export const OutlierExploration: FC<ExplorationProps> = React.memo(({ jobId }) =
index={jobConfig?.dest.index}
resultsField={jobConfig?.dest.results_field}
searchQuery={searchQuery}
query={query}
/>
)}
{showLegacyFeatureInfluenceFormatCallout && (