github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 17:59:23 -04:00
Code Issues Projects Releases Packages Wiki Activity

[ML] Data Frame Analytics: Adds scatterplot matrix to regression/classification results pages. (#88353)

Browse source 
- Adds support for scatterplot matrices to regression/classification results pages
- Lazy loads the scatterplot matrix including Vega code using Suspense. The approach is taken from the Kibana Vega plugin, creating this separate bundle means you'll load the 600kb+ Vega code only on pages where actually needed and not e.g. already on the analytics job list. Note for reviews: The file scatterplot_matrix_view.tsx did not change besides the default export, it just shows up as a new file because of the refactoring to support lazy loading.
- Adds support for analytics configuration that use the excludes instead of includes field list.
- Adds the field used for color legends to tooltips.
This commit is contained in:
Walter Rafelsberger 2021-02-01 12:30:58 +01:00 • committed by GitHub
parent 1b8c3c1dcc
commit fb19aab307
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
23 changed files with 629 additions and 345 deletions
Download patch file Download diff file Expand all files Collapse all files

2
x-pack/plugins/ml/public/application/components/scatterplot_matrix/index.ts
View file

@ -4,5 +4,7 @@
* you may not use this file except in compliance with the Elastic License.
*/
export { useScatterplotFieldOptions } from './use_scatterplot_field_options';
export { LEGEND_TYPES } from './scatterplot_matrix_vega_lite_spec';
export { ScatterplotMatrix } from './scatterplot_matrix';
export type { ScatterplotMatrixViewProps as ScatterplotMatrixProps } from './scatterplot_matrix_view';

319
x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix.tsx
View file

@ -4,316 +4,15 @@
* you may not use this file except in compliance with the Elastic License.
*/
import React, { useMemo, useEffect, useState, FC } from 'react';
import React, { FC, Suspense } from 'react';
// There is still an issue with Vega Lite's typings with the strict mode Kibana is using.
// @ts-ignore
import { compile } from 'vega-lite/build-es5/vega-lite';
import { parse, View, Warn } from 'vega';
import { Handler } from 'vega-tooltip';
import type { ScatterplotMatrixViewProps } from './scatterplot_matrix_view';
import { ScatterplotMatrixLoading } from './scatterplot_matrix_loading';
import {
htmlIdGenerator,
EuiComboBox,
EuiComboBoxOptionOption,
EuiFlexGroup,
EuiFlexItem,
EuiFormRow,
EuiLoadingSpinner,
EuiSelect,
EuiSpacer,
EuiSwitch,
EuiText,
} from '@elastic/eui';
const ScatterplotMatrixLazy = React.lazy(() => import('./scatterplot_matrix_view'));
import { i18n } from '@kbn/i18n';
import type { SearchResponse7 } from '../../../../common/types/es_client';
import { useMlApiContext } from '../../contexts/kibana';
import { getProcessedFields } from '../data_grid';
import { useCurrentEuiTheme } from '../color_range_legend';
import {
getScatterplotMatrixVegaLiteSpec,
LegendType,
OUTLIER_SCORE_FIELD,
} from './scatterplot_matrix_vega_lite_spec';
import './scatterplot_matrix.scss';
const SCATTERPLOT_MATRIX_DEFAULT_FIELDS = 4;
const SCATTERPLOT_MATRIX_DEFAULT_FETCH_SIZE = 1000;
const SCATTERPLOT_MATRIX_DEFAULT_FETCH_MIN_SIZE = 1;
const SCATTERPLOT_MATRIX_DEFAULT_FETCH_MAX_SIZE = 10000;
const TOGGLE_ON = i18n.translate('xpack.ml.splom.toggleOn', {
defaultMessage: 'On',
});
const TOGGLE_OFF = i18n.translate('xpack.ml.splom.toggleOff', {
defaultMessage: 'Off',
});
const sampleSizeOptions = [100, 1000, 10000].map((d) => ({ value: d, text: '' + d }));
interface ScatterplotMatrixProps {
fields: string[];
index: string;
resultsField?: string;
color?: string;
legendType?: LegendType;
}
export const ScatterplotMatrix: FC<ScatterplotMatrixProps> = ({
fields: allFields,
index,
resultsField,
color,
legendType,
}) => {
const { esSearch } = useMlApiContext();
// dynamicSize is optionally used for outlier charts where the scatterplot marks
// are sized according to outlier_score
const [dynamicSize, setDynamicSize] = useState<boolean>(false);
// used to give the use the option to customize the fields used for the matrix axes
const [fields, setFields] = useState<string[]>([]);
useEffect(() => {
const defaultFields =
allFields.length > SCATTERPLOT_MATRIX_DEFAULT_FIELDS
? allFields.slice(0, SCATTERPLOT_MATRIX_DEFAULT_FIELDS)
: allFields;
setFields(defaultFields);
}, [allFields]);
// the amount of documents to be fetched
const [fetchSize, setFetchSize] = useState<number>(SCATTERPLOT_MATRIX_DEFAULT_FETCH_SIZE);
// flag to add a random score to the ES query to fetch documents
const [randomizeQuery, setRandomizeQuery] = useState<boolean>(false);
const [isLoading, setIsLoading] = useState<boolean>(false);
// contains the fetched documents and columns to be passed on to the Vega spec.
const [splom, setSplom] = useState<{ items: any[]; columns: string[] } | undefined>();
// formats the array of field names for EuiComboBox
const fieldOptions = useMemo(
() =>
allFields.map((d) => ({
label: d,
})),
[allFields]
);
const fieldsOnChange = (newFields: EuiComboBoxOptionOption[]) => {
setFields(newFields.map((d) => d.label));
};
const fetchSizeOnChange = (e: React.ChangeEvent<HTMLSelectElement>) => {
setFetchSize(
Math.min(
Math.max(parseInt(e.target.value, 10), SCATTERPLOT_MATRIX_DEFAULT_FETCH_MIN_SIZE),
SCATTERPLOT_MATRIX_DEFAULT_FETCH_MAX_SIZE
)
);
};
const randomizeQueryOnChange = () => {
setRandomizeQuery(!randomizeQuery);
};
const dynamicSizeOnChange = () => {
setDynamicSize(!dynamicSize);
};
const { euiTheme } = useCurrentEuiTheme();
useEffect(() => {
async function fetchSplom(options: { didCancel: boolean }) {
setIsLoading(true);
try {
const queryFields = [
...fields,
...(color !== undefined ? [color] : []),
...(legendType !== undefined ? [] : [`${resultsField}.${OUTLIER_SCORE_FIELD}`]),
];
const query = randomizeQuery
? {
function_score: {
random_score: { seed: 10, field: '_seq_no' },
},
}
: { match_all: {} };
const resp: SearchResponse7 = await esSearch({
index,
body: {
fields: queryFields,
_source: false,
query,
from: 0,
size: fetchSize,
},
});
if (!options.didCancel) {
const items = resp.hits.hits.map((d) =>
getProcessedFields(d.fields, (key: string) =>
key.startsWith(`${resultsField}.feature_importance`)
)
);
setSplom({ columns: fields, items });
setIsLoading(false);
}
} catch (e) {
// TODO error handling
setIsLoading(false);
}
}
const options = { didCancel: false };
fetchSplom(options);
return () => {
options.didCancel = true;
};
// stringify the fields array, otherwise the comparator will trigger on new but identical instances.
}, [fetchSize, JSON.stringify(fields), index, randomizeQuery, resultsField]);
const htmlId = useMemo(() => htmlIdGenerator()(), []);
useEffect(() => {
if (splom === undefined) {
return;
}
const { items, columns } = splom;
const values =
resultsField !== undefined
? items
: items.map((d) => {
d[`${resultsField}.${OUTLIER_SCORE_FIELD}`] = 0;
return d;
});
const vegaSpec = getScatterplotMatrixVegaLiteSpec(
values,
columns,
euiTheme,
resultsField,
color,
legendType,
dynamicSize
);
const vgSpec = compile(vegaSpec).spec;
const view = new View(parse(vgSpec))
.logLevel(Warn)
.renderer('canvas')
.tooltip(new Handler().call)
.initialize(`#${htmlId}`);
view.runAsync(); // evaluate and render the view
}, [resultsField, splom, color, legendType, dynamicSize]);
return (
<>
{splom === undefined ? (
<EuiText textAlign="center">
<EuiSpacer size="l" />
<EuiLoadingSpinner size="l" />
<EuiSpacer size="l" />
</EuiText>
) : (
<>
<EuiFlexGroup>
<EuiFlexItem>
<EuiFormRow
label={i18n.translate('xpack.ml.splom.fieldSelectionLabel', {
defaultMessage: 'Fields',
})}
display="rowCompressed"
fullWidth
>
<EuiComboBox
compressed
fullWidth
placeholder={i18n.translate('xpack.ml.splom.fieldSelectionPlaceholder', {
defaultMessage: 'Select fields',
})}
options={fieldOptions}
selectedOptions={fields.map((d) => ({
label: d,
}))}
onChange={fieldsOnChange}
isClearable={true}
data-test-subj="mlScatterplotMatrixFieldsComboBox"
/>
</EuiFormRow>
</EuiFlexItem>
<EuiFlexItem style={{ width: '200px' }} grow={false}>
<EuiFormRow
label={i18n.translate('xpack.ml.splom.SampleSizeLabel', {
defaultMessage: 'Sample size',
})}
display="rowCompressed"
fullWidth
>
<EuiSelect
compressed
options={sampleSizeOptions}
value={fetchSize}
onChange={fetchSizeOnChange}
/>
</EuiFormRow>
</EuiFlexItem>
<EuiFlexItem style={{ width: '120px' }} grow={false}>
<EuiFormRow
label={i18n.translate('xpack.ml.splom.RandomScoringLabel', {
defaultMessage: 'Random scoring',
})}
display="rowCompressed"
fullWidth
>
<EuiSwitch
name="mlScatterplotMatrixRandomizeQuery"
label={randomizeQuery ? TOGGLE_ON : TOGGLE_OFF}
checked={randomizeQuery}
onChange={randomizeQueryOnChange}
disabled={isLoading}
/>
</EuiFormRow>
</EuiFlexItem>
{resultsField !== undefined && legendType === undefined && (
<EuiFlexItem style={{ width: '120px' }} grow={false}>
<EuiFormRow
label={i18n.translate('xpack.ml.splom.dynamicSizeLabel', {
defaultMessage: 'Dynamic size',
})}
display="rowCompressed"
fullWidth
>
<EuiSwitch
name="mlScatterplotMatrixDynamicSize"
label={dynamicSize ? TOGGLE_ON : TOGGLE_OFF}
checked={dynamicSize}
onChange={dynamicSizeOnChange}
disabled={isLoading}
/>
</EuiFormRow>
</EuiFlexItem>
)}
</EuiFlexGroup>
<div id={htmlId} className="mlScatterplotMatrix" />
</>
)}
</>
);
};
export const ScatterplotMatrix: FC<ScatterplotMatrixViewProps> = (props) => (
<Suspense fallback={<ScatterplotMatrixLoading />}>
<ScatterplotMatrixLazy {...props} />
</Suspense>
);

19
x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix_loading.tsx Normal file
View file

@ -0,0 +1,19 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import React from 'react';
import { EuiLoadingSpinner, EuiSpacer, EuiText } from '@elastic/eui';
export const ScatterplotMatrixLoading = () => {
return (
<EuiText textAlign="center">
<EuiSpacer size="l" />
<EuiLoadingSpinner size="l" />
<EuiSpacer size="l" />
</EuiText>
);
};

1
x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix_vega_lite_spec.test.ts
View file

@ -163,6 +163,7 @@ describe('getScatterplotMatrixVegaLiteSpec()', () => {
type: 'nominal',
});
expect(vegaLiteSpec.spec.encoding.tooltip).toEqual([
{ field: 'the-color-field', type: 'nominal' },
{ field: 'x', type: 'quantitative' },
{ field: 'y', type: 'quantitative' },
]);

11
x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix_vega_lite_spec.ts
View file

@ -35,6 +35,8 @@ export const getColorSpec = (
color?: string,
legendType?: LegendType
) => {
// For outlier detection result pages coloring is done based on a threshold.
// This returns a Vega spec using a conditional to return the color.
if (outliers) {
return {
condition: {
@ -45,6 +47,8 @@ export const getColorSpec = (
};
}
// Based on the type of the color field,
// this returns either a continuous or categorical color spec.
if (color !== undefined && legendType !== undefined) {
return {
field: color,
@ -80,6 +84,8 @@ export const getScatterplotMatrixVegaLiteSpec = (
});
}
const colorSpec = getColorSpec(euiTheme, outliers, color, legendType);
return {
$schema: 'https://vega.github.io/schema/vega-lite/v4.17.0.json',
background: 'transparent',
@ -115,10 +121,10 @@ export const getScatterplotMatrixVegaLiteSpec = (
: { type: 'circle', opacity: 0.75, size: 8 }),
},
encoding: {
color: getColorSpec(euiTheme, outliers, color, legendType),
color: colorSpec,
...(dynamicSize
? {
stroke: getColorSpec(euiTheme, outliers, color, legendType),
stroke: colorSpec,
opacity: {
condition: {
value: 1,
@ -163,6 +169,7 @@ export const getScatterplotMatrixVegaLiteSpec = (
scale: { zero: false },
},
tooltip: [
...(color !== undefined ? [{ type: colorSpec.type, field: color }] : []),
...columns.map((d) => ({ type: LEGEND_TYPES.QUANTITATIVE, field: d })),
...(outliers
? [{ type: LEGEND_TYPES.QUANTITATIVE, field: OUTLIER_SCORE_FIELD, format: '.3f' }]

0
x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix.scss → x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix_view.scss
View file

323
x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix_view.tsx Normal file
View file

@ -0,0 +1,323 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import React, { useMemo, useEffect, useState, FC } from 'react';
// There is still an issue with Vega Lite's typings with the strict mode Kibana is using.
// @ts-ignore
import { compile } from 'vega-lite/build-es5/vega-lite';
import { parse, View, Warn } from 'vega';
import { Handler } from 'vega-tooltip';
import {
htmlIdGenerator,
EuiComboBox,
EuiComboBoxOptionOption,
EuiFlexGroup,
EuiFlexItem,
EuiFormRow,
EuiSelect,
EuiSwitch,
} from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import type { SearchResponse7 } from '../../../../common/types/es_client';
import type { ResultsSearchQuery } from '../../data_frame_analytics/common/analytics';
import { useMlApiContext } from '../../contexts/kibana';
import { getProcessedFields } from '../data_grid';
import { useCurrentEuiTheme } from '../color_range_legend';
import { ScatterplotMatrixLoading } from './scatterplot_matrix_loading';
import {
getScatterplotMatrixVegaLiteSpec,
LegendType,
OUTLIER_SCORE_FIELD,
} from './scatterplot_matrix_vega_lite_spec';
import './scatterplot_matrix_view.scss';
const SCATTERPLOT_MATRIX_DEFAULT_FIELDS = 4;
const SCATTERPLOT_MATRIX_DEFAULT_FETCH_SIZE = 1000;
const SCATTERPLOT_MATRIX_DEFAULT_FETCH_MIN_SIZE = 1;
const SCATTERPLOT_MATRIX_DEFAULT_FETCH_MAX_SIZE = 10000;
const TOGGLE_ON = i18n.translate('xpack.ml.splom.toggleOn', {
defaultMessage: 'On',
});
const TOGGLE_OFF = i18n.translate('xpack.ml.splom.toggleOff', {
defaultMessage: 'Off',
});
const sampleSizeOptions = [100, 1000, 10000].map((d) => ({ value: d, text: '' + d }));
export interface ScatterplotMatrixViewProps {
fields: string[];
index: string;
resultsField?: string;
color?: string;
legendType?: LegendType;
searchQuery?: ResultsSearchQuery;
}
export const ScatterplotMatrixView: FC<ScatterplotMatrixViewProps> = ({
fields: allFields,
index,
resultsField,
color,
legendType,
searchQuery,
}) => {
const { esSearch } = useMlApiContext();
// dynamicSize is optionally used for outlier charts where the scatterplot marks
// are sized according to outlier_score
const [dynamicSize, setDynamicSize] = useState<boolean>(false);
// used to give the use the option to customize the fields used for the matrix axes
const [fields, setFields] = useState<string[]>([]);
useEffect(() => {
const defaultFields =
allFields.length > SCATTERPLOT_MATRIX_DEFAULT_FIELDS
? allFields.slice(0, SCATTERPLOT_MATRIX_DEFAULT_FIELDS)
: allFields;
setFields(defaultFields);
}, [allFields]);
// the amount of documents to be fetched
const [fetchSize, setFetchSize] = useState<number>(SCATTERPLOT_MATRIX_DEFAULT_FETCH_SIZE);
// flag to add a random score to the ES query to fetch documents
const [randomizeQuery, setRandomizeQuery] = useState<boolean>(false);
const [isLoading, setIsLoading] = useState<boolean>(false);
// contains the fetched documents and columns to be passed on to the Vega spec.
const [splom, setSplom] = useState<{ items: any[]; columns: string[] } | undefined>();
// formats the array of field names for EuiComboBox
const fieldOptions = useMemo(
() =>
allFields.map((d) => ({
label: d,
})),
[allFields]
);
const fieldsOnChange = (newFields: EuiComboBoxOptionOption[]) => {
setFields(newFields.map((d) => d.label));
};
const fetchSizeOnChange = (e: React.ChangeEvent<HTMLSelectElement>) => {
setFetchSize(
Math.min(
Math.max(parseInt(e.target.value, 10), SCATTERPLOT_MATRIX_DEFAULT_FETCH_MIN_SIZE),
SCATTERPLOT_MATRIX_DEFAULT_FETCH_MAX_SIZE
)
);
};
const randomizeQueryOnChange = () => {
setRandomizeQuery(!randomizeQuery);
};
const dynamicSizeOnChange = () => {
setDynamicSize(!dynamicSize);
};
const { euiTheme } = useCurrentEuiTheme();
useEffect(() => {
async function fetchSplom(options: { didCancel: boolean }) {
setIsLoading(true);
try {
const queryFields = [
...fields,
...(color !== undefined ? [color] : []),
...(legendType !== undefined ? [] : [`${resultsField}.${OUTLIER_SCORE_FIELD}`]),
];
const queryFallback = searchQuery !== undefined ? searchQuery : { match_all: {} };
const query = randomizeQuery
? {
function_score: {
query: queryFallback,
random_score: { seed: 10, field: '_seq_no' },
},
}
: queryFallback;
const resp: SearchResponse7 = await esSearch({
index,
body: {
fields: queryFields,
_source: false,
query,
from: 0,
size: fetchSize,
},
});
if (!options.didCancel) {
const items = resp.hits.hits.map((d) =>
getProcessedFields(d.fields, (key: string) =>
key.startsWith(`${resultsField}.feature_importance`)
)
);
setSplom({ columns: fields, items });
setIsLoading(false);
}
} catch (e) {
// TODO error handling
setIsLoading(false);
}
}
const options = { didCancel: false };
fetchSplom(options);
return () => {
options.didCancel = true;
};
// stringify the fields array and search, otherwise the comparator will trigger on new but identical instances.
}, [fetchSize, JSON.stringify({ fields, searchQuery }), index, randomizeQuery, resultsField]);
const htmlId = useMemo(() => htmlIdGenerator()(), []);
useEffect(() => {
if (splom === undefined) {
return;
}
const { items, columns } = splom;
const values =
resultsField !== undefined
? items
: items.map((d) => {
d[`${resultsField}.${OUTLIER_SCORE_FIELD}`] = 0;
return d;
});
const vegaSpec = getScatterplotMatrixVegaLiteSpec(
values,
columns,
euiTheme,
resultsField,
color,
legendType,
dynamicSize
);
const vgSpec = compile(vegaSpec).spec;
const view = new View(parse(vgSpec))
.logLevel(Warn)
.renderer('canvas')
.tooltip(new Handler().call)
.initialize(`#${htmlId}`);
view.runAsync(); // evaluate and render the view
}, [resultsField, splom, color, legendType, dynamicSize]);
return (
<>
{splom === undefined ? (
<ScatterplotMatrixLoading />
) : (
<>
<EuiFlexGroup>
<EuiFlexItem>
<EuiFormRow
label={i18n.translate('xpack.ml.splom.fieldSelectionLabel', {
defaultMessage: 'Fields',
})}
display="rowCompressed"
fullWidth
>
<EuiComboBox
compressed
fullWidth
placeholder={i18n.translate('xpack.ml.splom.fieldSelectionPlaceholder', {
defaultMessage: 'Select fields',
})}
options={fieldOptions}
selectedOptions={fields.map((d) => ({
label: d,
}))}
onChange={fieldsOnChange}
isClearable={true}
data-test-subj="mlScatterplotMatrixFieldsComboBox"
/>
</EuiFormRow>
</EuiFlexItem>
<EuiFlexItem style={{ width: '200px' }} grow={false}>
<EuiFormRow
label={i18n.translate('xpack.ml.splom.sampleSizeLabel', {
defaultMessage: 'Sample size',
})}
display="rowCompressed"
fullWidth
>
<EuiSelect
compressed
options={sampleSizeOptions}
value={fetchSize}
onChange={fetchSizeOnChange}
/>
</EuiFormRow>
</EuiFlexItem>
<EuiFlexItem style={{ width: '120px' }} grow={false}>
<EuiFormRow
label={i18n.translate('xpack.ml.splom.randomScoringLabel', {
defaultMessage: 'Random scoring',
})}
display="rowCompressed"
fullWidth
>
<EuiSwitch
name="mlScatterplotMatrixRandomizeQuery"
label={randomizeQuery ? TOGGLE_ON : TOGGLE_OFF}
checked={randomizeQuery}
onChange={randomizeQueryOnChange}
disabled={isLoading}
/>
</EuiFormRow>
</EuiFlexItem>
{resultsField !== undefined && legendType === undefined && (
<EuiFlexItem style={{ width: '120px' }} grow={false}>
<EuiFormRow
label={i18n.translate('xpack.ml.splom.dynamicSizeLabel', {
defaultMessage: 'Dynamic size',
})}
display="rowCompressed"
fullWidth
>
<EuiSwitch
name="mlScatterplotMatrixDynamicSize"
label={dynamicSize ? TOGGLE_ON : TOGGLE_OFF}
checked={dynamicSize}
onChange={dynamicSizeOnChange}
disabled={isLoading}
/>
</EuiFormRow>
</EuiFlexItem>
)}
</EuiFlexGroup>
<div id={htmlId} className="mlScatterplotMatrix" data-test-subj="mlScatterplotMatrix" />
</>
)}
</>
);
};
// required for dynamic import using React.lazy()
// eslint-disable-next-line import/no-default-export
export default ScatterplotMatrixView;

50
x-pack/plugins/ml/public/application/components/scatterplot_matrix/use_scatterplot_field_options.ts Normal file
View file

@ -0,0 +1,50 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { useMemo } from 'react';
import type { IndexPattern } from '../../../../../../../src/plugins/data/public';
import { ML__INCREMENTAL_ID } from '../../data_frame_analytics/common/fields';
export const useScatterplotFieldOptions = (
indexPattern?: IndexPattern,
includes?: string[],
excludes?: string[],
resultsField = ''
): string[] => {
return useMemo(() => {
const fields: string[] = [];
if (indexPattern === undefined || includes === undefined) {
return fields;
}
if (includes.length > 1) {
fields.push(
...includes.filter((d) =>
indexPattern.fields.some((f) => f.name === d && f.type === 'number')
)
);
} else {
fields.push(
...indexPattern.fields
.filter(
(f) =>
f.type === 'number' &&
!indexPattern.metaFields.includes(f.name) &&
!f.name.startsWith(`${resultsField}.`) &&
f.name !== ML__INCREMENTAL_ID
)
.map((f) => f.name)
);
}
return Array.isArray(excludes) && excludes.length > 0
? fields.filter((f) => !excludes.includes(f))
: fields;
}, [indexPattern, includes, excludes]);
};

22
x-pack/plugins/ml/public/application/data_frame_analytics/common/get_scatterplot_matrix_legend_type.ts Normal file
View file

@ -0,0 +1,22 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { ANALYSIS_CONFIG_TYPE } from './analytics';
import { AnalyticsJobType } from '../pages/analytics_management/hooks/use_create_analytics_form/state';
import { LEGEND_TYPES } from '../../components/scatterplot_matrix/scatterplot_matrix_vega_lite_spec';
export const getScatterplotMatrixLegendType = (jobType: AnalyticsJobType | 'unknown') => {
switch (jobType) {
case ANALYSIS_CONFIG_TYPE.CLASSIFICATION:
return LEGEND_TYPES.NOMINAL;
case ANALYSIS_CONFIG_TYPE.REGRESSION:
return LEGEND_TYPES.QUANTITATIVE;
default:
return undefined;
}
};

1
x-pack/plugins/ml/public/application/data_frame_analytics/common/index.ts
View file

@ -41,6 +41,7 @@ export {
export { getIndexData } from './get_index_data';
export { getIndexFields } from './get_index_fields';
export { getScatterplotMatrixLegendType } from './get_scatterplot_matrix_legend_type';
export { useResultsViewConfig } from './use_results_view_config';
export { DataFrameAnalyticsConfig } from '../../../../common/types/data_frame_analytics';

6
x-pack/plugins/ml/public/application/data_frame_analytics/common/use_results_view_config.ts
View file

@ -102,6 +102,12 @@ export const useResultsViewConfig = (jobId: string) => {
try {
indexP = await mlContext.indexPatterns.get(destIndexPatternId);
// Force refreshing the fields list here because a user directly coming
// from the job creation wizard might land on the page without the
// index pattern being fully initialized because it was created
// before the analytics job populated the destination index.
await mlContext.indexPatterns.refreshFields(indexP);
} catch (e) {
indexP = undefined;
}

16
x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step_form.tsx
View file

@ -27,10 +27,10 @@ import {
TRAINING_PERCENT_MAX,
FieldSelectionItem,
} from '../../../../common/analytics';
import { getScatterplotMatrixLegendType } from '../../../../common/get_scatterplot_matrix_legend_type';
import { CreateAnalyticsStepProps } from '../../../analytics_management/hooks/use_create_analytics_form';
import { Messages } from '../shared';
import {
AnalyticsJobType,
DEFAULT_MODEL_MEMORY_LIMIT,
State,
} from '../../../analytics_management/hooks/use_create_analytics_form/state';
@ -51,18 +51,7 @@ import { SEARCH_QUERY_LANGUAGE } from '../../../../../../../common/constants/sea
import { ExplorationQueryBarProps } from '../../../analytics_exploration/components/exploration_query_bar/exploration_query_bar';
import { Query } from '../../../../../../../../../../src/plugins/data/common/query';
import { LEGEND_TYPES, ScatterplotMatrix } from '../../../../../components/scatterplot_matrix';
const getScatterplotMatrixLegendType = (jobType: AnalyticsJobType) => {
switch (jobType) {
case ANALYSIS_CONFIG_TYPE.CLASSIFICATION:
return LEGEND_TYPES.NOMINAL;
case ANALYSIS_CONFIG_TYPE.REGRESSION:
return LEGEND_TYPES.QUANTITATIVE;
default:
return undefined;
}
};
import { ScatterplotMatrix } from '../../../../../components/scatterplot_matrix';
const requiredFieldsErrorText = i18n.translate(
'xpack.ml.dataframe.analytics.createWizard.requiredFieldsErrorMessage',
@ -498,6 +487,7 @@ export const ConfigurationStepForm: FC<CreateAnalyticsStepProps> = ({
: undefined
}
legendType={getScatterplotMatrixLegendType(jobType)}
searchQuery={jobConfigQuery}
/>
</EuiPanel>
<EuiSpacer />

13
x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/expandable_section/expandable_section_splom.tsx
View file

@ -12,17 +12,14 @@ import { FormattedMessage } from '@kbn/i18n/react';
import { EuiHorizontalRule, EuiSpacer } from '@elastic/eui';
import { ScatterplotMatrix } from '../../../../../components/scatterplot_matrix';
import {
ScatterplotMatrix,
ScatterplotMatrixProps,
} from '../../../../../components/scatterplot_matrix';
import { ExpandableSection } from './expandable_section';
interface ExpandableSectionSplomProps {
fields: string[];
index: string;
resultsField?: string;
}
export const ExpandableSectionSplom: FC<ExpandableSectionSplomProps> = (props) => {
export const ExpandableSectionSplom: FC<ScatterplotMatrixProps> = (props) => {
const splomSectionHeaderItems = undefined;
const splomSectionContent = (
<>

41
x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_page_wrapper/exploration_page_wrapper.tsx
View file

@ -9,16 +9,21 @@ import React, { FC, useCallback, useState } from 'react';
import { EuiCallOut, EuiFlexGroup, EuiFlexItem, EuiPanel, EuiSpacer, EuiText } from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import { getAnalysisType, getDependentVar } from '../../../../../../../common/util/analytics_utils';
import { useScatterplotFieldOptions } from '../../../../../components/scatterplot_matrix';
import {
defaultSearchQuery,
getScatterplotMatrixLegendType,
useResultsViewConfig,
DataFrameAnalyticsConfig,
} from '../../../../common';
import { ResultsSearchQuery } from '../../../../common/analytics';
import { ResultsSearchQuery, ANALYSIS_CONFIG_TYPE } from '../../../../common/analytics';
import { DataFrameTaskStateType } from '../../../analytics_management/components/analytics_list/common';
import { ExpandableSectionAnalytics } from '../expandable_section';
import { ExpandableSectionAnalytics, ExpandableSectionSplom } from '../expandable_section';
import { ExplorationResultsTable } from '../exploration_results_table';
import { ExplorationQueryBar } from '../exploration_query_bar';
import { JobConfigErrorCallout } from '../job_config_error_callout';
@ -99,6 +104,14 @@ export const ExplorationPageWrapper: FC<Props> = ({
language: pageUrlState.queryLanguage,
};
const resultsField = jobConfig?.dest.results_field ?? '';
const scatterplotFieldOptions = useScatterplotFieldOptions(
indexPattern,
jobConfig?.analyzed_fields.includes,
jobConfig?.analyzed_fields.excludes,
resultsField
);
if (indexPatternErrorMessage !== undefined) {
return (
<EuiPanel grow={false}>
@ -125,6 +138,9 @@ export const ExplorationPageWrapper: FC<Props> = ({
);
}
const jobType =
jobConfig && jobConfig.analysis ? getAnalysisType(jobConfig?.analysis) : undefined;
return (
<>
{typeof jobConfig?.description !== 'undefined' && (
@ -179,6 +195,27 @@ export const ExplorationPageWrapper: FC<Props> = ({
<EvaluatePanel jobConfig={jobConfig} jobStatus={jobStatus} searchQuery={searchQuery} />
)}
{isLoadingJobConfig === true && jobConfig === undefined && <LoadingPanel />}
{isLoadingJobConfig === false &&
jobConfig !== undefined &&
isInitialized === true &&
typeof jobConfig?.id === 'string' &&
scatterplotFieldOptions.length > 1 &&
typeof jobConfig?.analysis !== 'undefined' && (
<ExpandableSectionSplom
fields={scatterplotFieldOptions}
index={jobConfig?.dest.index}
color={
jobType === ANALYSIS_CONFIG_TYPE.REGRESSION ||
jobType === ANALYSIS_CONFIG_TYPE.CLASSIFICATION
? getDependentVar(jobConfig.analysis)
: undefined
}
legendType={getScatterplotMatrixLegendType(jobType)}
searchQuery={searchQuery}
/>
)}
{isLoadingJobConfig === true && jobConfig === undefined && <LoadingPanel />}
{isLoadingJobConfig === false &&
jobConfig !== undefined &&

15
x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/outlier_exploration.tsx
View file

@ -4,7 +4,7 @@
* you may not use this file except in compliance with the Elastic License.
*/
import React, { useState, FC, useCallback } from 'react';
import React, { useCallback, useState, FC } from 'react';
import { EuiCallOut, EuiPanel, EuiSpacer, EuiText } from '@elastic/eui';
@ -15,6 +15,7 @@ import {
COLOR_RANGE,
COLOR_RANGE_SCALE,
} from '../../../../../components/color_range_legend';
import { useScatterplotFieldOptions } from '../../../../../components/scatterplot_matrix';
import { SavedSearchQuery } from '../../../../../contexts/ml';
import { defaultSearchQuery, isOutlierAnalysis, useResultsViewConfig } from '../../../../common';
@ -90,6 +91,13 @@ export const OutlierExploration: FC<ExplorationProps> = React.memo(({ jobId }) =
(d) => d.id === `${resultsField}.${FEATURE_INFLUENCE}.feature_name`
) === -1;
const scatterplotFieldOptions = useScatterplotFieldOptions(
indexPattern,
jobConfig?.analyzed_fields.includes,
jobConfig?.analyzed_fields.excludes,
resultsField
);
if (indexPatternErrorMessage !== undefined) {
return (
<EuiPanel grow={false}>
@ -126,11 +134,12 @@ export const OutlierExploration: FC<ExplorationProps> = React.memo(({ jobId }) =
</>
)}
{typeof jobConfig?.id === 'string' && <ExpandableSectionAnalytics jobId={jobConfig?.id} />}
{typeof jobConfig?.id === 'string' && jobConfig?.analyzed_fields.includes.length > 1 && (
{typeof jobConfig?.id === 'string' && scatterplotFieldOptions.length > 1 && (
<ExpandableSectionSplom
fields={jobConfig?.analyzed_fields.includes}
fields={scatterplotFieldOptions}
index={jobConfig?.dest.index}
resultsField={jobConfig?.dest.results_field}
searchQuery={searchQuery}
/>
)}
{showLegacyFeatureInfluenceFormatCallout && (

4
x-pack/plugins/translations/translations/ja-JP.json
View file

@ -14209,8 +14209,8 @@
"xpack.ml.splom.dynamicSizeLabel": "動的サイズ",
"xpack.ml.splom.fieldSelectionLabel": "フィールド",
"xpack.ml.splom.fieldSelectionPlaceholder": "フィールドを選択",
"xpack.ml.splom.RandomScoringLabel": "ランダムスコアリング",
"xpack.ml.splom.SampleSizeLabel": "サンプルサイズ",
"xpack.ml.splom.randomScoringLabel": "ランダムスコアリング",
"xpack.ml.splom.sampleSizeLabel": "サンプルサイズ",
"xpack.ml.splom.toggleOff": "オフ",
"xpack.ml.splom.toggleOn": "オン",
"xpack.ml.splomSpec.outlierScoreThresholdName": "異常スコアしきい値：",

4
x-pack/plugins/translations/translations/zh-CN.json
View file

@ -14248,8 +14248,8 @@
"xpack.ml.splom.dynamicSizeLabel": "动态大小",
"xpack.ml.splom.fieldSelectionLabel": "字段",
"xpack.ml.splom.fieldSelectionPlaceholder": "选择字段",
"xpack.ml.splom.RandomScoringLabel": "随机评分",
"xpack.ml.splom.SampleSizeLabel": "样例大小",
"xpack.ml.splom.randomScoringLabel": "随机评分",
"xpack.ml.splom.sampleSizeLabel": "样例大小",
"xpack.ml.splom.toggleOff": "关闭",
"xpack.ml.splom.toggleOn": "开启",
"xpack.ml.splomSpec.outlierScoreThresholdName": "离群值分数阈值：",

21
x-pack/test/functional/apps/ml/data_frame_analytics/classification_creation.ts
View file

@ -40,6 +40,17 @@ export default function ({ getService }: FtrProviderContext) {
modelMemory: '60mb',
createIndexPattern: true,
expected: {
scatterplotMatrixColorStats: [
// background
{ key: '#000000', value: 94 },
// tick/grid/axis
{ key: '#DDDDDD', value: 1 },
{ key: '#D3DAE6', value: 1 },
{ key: '#F5F7FA', value: 1 },
// scatterplot circles
{ key: '#6A717D', value: 1 },
{ key: '#54B39A', value: 1 },
],
row: {
type: 'classification',
status: 'stopped',
@ -89,6 +100,12 @@ export default function ({ getService }: FtrProviderContext) {
await ml.testExecution.logTestStep('displays the include fields selection');
await ml.dataFrameAnalyticsCreation.assertIncludeFieldsSelectionExists();
await ml.testExecution.logTestStep('displays the scatterplot matrix');
await ml.dataFrameAnalyticsScatterplot.assertScatterplotMatrix(
'mlAnalyticsCreateJobWizardScatterplotMatrixFormRow',
testData.expected.scatterplotMatrixColorStats
);
await ml.testExecution.logTestStep('continues to the additional options step');
await ml.dataFrameAnalyticsCreation.continueToAdditionalOptionsStep();
@ -207,6 +224,10 @@ export default function ({ getService }: FtrProviderContext) {
await ml.dataFrameAnalyticsResults.assertResultsTableExists();
await ml.dataFrameAnalyticsResults.assertResultsTableTrainingFiltersExist();
await ml.dataFrameAnalyticsResults.assertResultsTableNotEmpty();
await ml.dataFrameAnalyticsScatterplot.assertScatterplotMatrix(
'mlDFExpandableSection-splom',
testData.expected.scatterplotMatrixColorStats
);
});
it('displays the analytics job in the map view', async () => {

31
x-pack/test/functional/apps/ml/data_frame_analytics/outlier_detection_creation.ts
View file

@ -49,6 +49,27 @@ export default function ({ getService }: FtrProviderContext) {
{ chartAvailable: true, id: 'Exterior2nd', legend: '3 categories' },
{ chartAvailable: true, id: 'Fireplaces', legend: '0 - 3' },
],
scatterplotMatrixColorStatsWizard: [
// background
{ key: '#000000', value: 91 },
// tick/grid/axis
{ key: '#6A717D', value: 2 },
{ key: '#F5F7FA', value: 2 },
{ key: '#D3DAE6', value: 1 },
// scatterplot circles
{ key: '#54B399', value: 1 },
{ key: '#54B39A', value: 1 },
],
scatterplotMatrixColorStatsResults: [
// background
{ key: '#000000', value: 91 },
// tick/grid/axis, grey markers
// the red outlier color is not above the 1% threshold.
{ key: '#6A717D', value: 2 },
{ key: '#98A2B3', value: 1 },
{ key: '#F5F7FA', value: 2 },
{ key: '#D3DAE6', value: 1 },
],
row: {
type: 'outlier_detection',
status: 'stopped',
@ -105,6 +126,12 @@ export default function ({ getService }: FtrProviderContext) {
await ml.testExecution.logTestStep('displays the include fields selection');
await ml.dataFrameAnalyticsCreation.assertIncludeFieldsSelectionExists();
await ml.testExecution.logTestStep('displays the scatterplot matrix');
await ml.dataFrameAnalyticsScatterplot.assertScatterplotMatrix(
'mlAnalyticsCreateJobWizardScatterplotMatrixFormRow',
testData.expected.scatterplotMatrixColorStatsWizard
);
await ml.testExecution.logTestStep('continues to the additional options step');
await ml.dataFrameAnalyticsCreation.continueToAdditionalOptionsStep();
@ -221,6 +248,10 @@ export default function ({ getService }: FtrProviderContext) {
await ml.dataFrameAnalyticsResults.assertOutlierTablePanelExists();
await ml.dataFrameAnalyticsResults.assertResultsTableExists();
await ml.dataFrameAnalyticsResults.assertResultsTableNotEmpty();
await ml.dataFrameAnalyticsScatterplot.assertScatterplotMatrix(
'mlDFExpandableSection-splom',
testData.expected.scatterplotMatrixColorStatsResults
);
});
it('displays the analytics job in the map view', async () => {

20
x-pack/test/functional/apps/ml/data_frame_analytics/regression_creation.ts
View file

@ -39,6 +39,16 @@ export default function ({ getService }: FtrProviderContext) {
modelMemory: '20mb',
createIndexPattern: true,
expected: {
scatterplotMatrixColorStats: [
// background
{ key: '#000000', value: 80 },
// tick/grid/axis
{ key: '#6A717D', value: 1 },
{ key: '#F5F7FA', value: 2 },
{ key: '#D3DAE6', value: 1 },
// because a continuous color scale is used for the scatterplot circles,
// none of the generated colors is above the 1% threshold.
],
row: {
type: 'regression',
status: 'stopped',
@ -89,6 +99,12 @@ export default function ({ getService }: FtrProviderContext) {
await ml.testExecution.logTestStep('displays the include fields selection');
await ml.dataFrameAnalyticsCreation.assertIncludeFieldsSelectionExists();
await ml.testExecution.logTestStep('displays the scatterplot matrix');
await ml.dataFrameAnalyticsScatterplot.assertScatterplotMatrix(
'mlAnalyticsCreateJobWizardScatterplotMatrixFormRow',
testData.expected.scatterplotMatrixColorStats
);
await ml.testExecution.logTestStep('continues to the additional options step');
await ml.dataFrameAnalyticsCreation.continueToAdditionalOptionsStep();
@ -207,6 +223,10 @@ export default function ({ getService }: FtrProviderContext) {
await ml.dataFrameAnalyticsResults.assertResultsTableExists();
await ml.dataFrameAnalyticsResults.assertResultsTableTrainingFiltersExist();
await ml.dataFrameAnalyticsResults.assertResultsTableNotEmpty();
await ml.dataFrameAnalyticsScatterplot.assertScatterplotMatrix(
'mlDFExpandableSection-splom',
testData.expected.scatterplotMatrixColorStats
);
});
it('displays the analytics job in the map view', async () => {

10
x-pack/test/functional/services/canvas_element.ts
View file

@ -43,9 +43,13 @@ export async function CanvasElementProvider({ getService }: FtrProviderContext)
public async getImageData(selector: string): Promise<number[]> {
return await driver.executeScript(
`
const el = document.querySelector('${selector}');
const ctx = el.getContext('2d');
return ctx.getImageData(0, 0, el.width, el.height).data;
try {
const el = document.querySelector('${selector}');
const ctx = el.getContext('2d');
return ctx.getImageData(0, 0, el.width, el.height).data;
} catch(e) {
return [];
}
`
);
}

40
x-pack/test/functional/services/ml/data_frame_analytics_scatterplot.ts Normal file
View file

@ -0,0 +1,40 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../ftr_provider_context';
export function MachineLearningDataFrameAnalyticsScatterplotProvider({
getService,
}: FtrProviderContext) {
const canvasElement = getService('canvasElement');
const testSubjects = getService('testSubjects');
return new (class AnalyticsScatterplot {
public async assertScatterplotMatrix(
dataTestSubj: string,
expectedColorStats: Array<{
key: string;
value: number;
}>
) {
await testSubjects.existOrFail(dataTestSubj);
await testSubjects.existOrFail('mlScatterplotMatrix');
const actualColorStats = await canvasElement.getColorStats(
`[data-test-subj="mlScatterplotMatrix"] canvas`,
expectedColorStats,
1
);
expect(actualColorStats.every((d) => d.withinTolerance)).to.eql(
true,
`Color stats for scatterplot matrix should be within tolerance. Expected: '${JSON.stringify(
expectedColorStats
)}' (got '${JSON.stringify(actualColorStats)}')`
);
}
})();
}

5
x-pack/test/functional/services/ml/index.ts
View file

@ -17,6 +17,7 @@ import { MachineLearningDataFrameAnalyticsProvider } from './data_frame_analytic
import { MachineLearningDataFrameAnalyticsCreationProvider } from './data_frame_analytics_creation';
import { MachineLearningDataFrameAnalyticsEditProvider } from './data_frame_analytics_edit';
import { MachineLearningDataFrameAnalyticsResultsProvider } from './data_frame_analytics_results';
import { MachineLearningDataFrameAnalyticsScatterplotProvider } from './data_frame_analytics_scatterplot';
import { MachineLearningDataFrameAnalyticsMapProvider } from './data_frame_analytics_map';
import { MachineLearningDataFrameAnalyticsTableProvider } from './data_frame_analytics_table';
import { MachineLearningDataVisualizerProvider } from './data_visualizer';
@ -63,6 +64,9 @@ export function MachineLearningProvider(context: FtrProviderContext) {
const dataFrameAnalyticsResults = MachineLearningDataFrameAnalyticsResultsProvider(context);
const dataFrameAnalyticsMap = MachineLearningDataFrameAnalyticsMapProvider(context);
const dataFrameAnalyticsTable = MachineLearningDataFrameAnalyticsTableProvider(context);
const dataFrameAnalyticsScatterplot = MachineLearningDataFrameAnalyticsScatterplotProvider(
context
);
const dataVisualizer = MachineLearningDataVisualizerProvider(context);
const dataVisualizerTable = MachineLearningDataVisualizerTableProvider(context, commonUI);
@ -105,6 +109,7 @@ export function MachineLearningProvider(context: FtrProviderContext) {
dataFrameAnalyticsResults,
dataFrameAnalyticsMap,
dataFrameAnalyticsTable,
dataFrameAnalyticsScatterplot,
dataVisualizer,
dataVisualizerFileBased,
dataVisualizerIndexBased,