github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 01:38:56 -04:00
Code Issues Projects Releases Packages Wiki Activity

[ML] DF Analytics Regression exploration: replace table with data grid (#63650) (#63739)

Browse source 
* add feature_importance column correctly

* wip: switch regression table to datagrid

* add search bar to regression view

* ensure feature importance fields show up correctly

* wip: filter by training/testing

* remove separate testing/training filter

* make error more clear

* handle lucene queries

* remove unnecessary comment

* ensure boolean shows up correctly.no sorting by feature importance

* remove unused translations
This commit is contained in:
Melissa Alvarez 2020-04-16 15:36:03 -04:00 • committed by GitHub
parent 5a88920c04
commit eb56bfe603
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 550 additions and 603 deletions
Download patch file Download diff file Expand all files Collapse all files

48
x-pack/plugins/ml/public/application/data_frame_analytics/common/analytics.ts
View file

@ -13,7 +13,6 @@ import { ml } from '../../services/ml_api_service';
import { Dictionary } from '../../../../common/types/common';
import { getErrorMessage } from '../../../../common/util/errors';
import { SavedSearchQuery } from '../../contexts/ml';
import { SortDirection } from '../../components/ml_in_memory_table';
export type IndexName = string;
export type IndexPattern = string;
@ -53,13 +52,9 @@ export interface ClassificationAnalysis {
classification: Classification;
}
export interface LoadExploreDataArg {
field: string;
direction: SortDirection;
export interface LoadRegressionExploreDataArg {
filterByIsTraining?: boolean;
searchQuery: SavedSearchQuery;
requiresKeyword?: boolean;
pageIndex?: number;
pageSize?: number;
}
export const SEARCH_SIZE = 1000;
@ -272,6 +267,11 @@ export const isResultsSearchBoolQuery = (arg: any): arg is ResultsSearchBoolQuer
return keys.length === 1 && keys[0] === 'bool';
};
export const isQueryStringQuery = (arg: any): arg is QueryStringQuery => {
const keys = Object.keys(arg);
return keys.length === 1 && keys[0] === 'query_string';
};
export const isRegressionEvaluateResponse = (arg: any): arg is RegressionEvaluateResponse => {
const keys = Object.keys(arg);
return (
@ -396,6 +396,10 @@ interface ResultsSearchTermQuery {
term: Dictionary<any>;
}
interface QueryStringQuery {
query_string: Dictionary<any>;
}
export type ResultsSearchQuery = ResultsSearchBoolQuery | ResultsSearchTermQuery | SavedSearchQuery;
export function getEvalQueryBody({
@ -409,16 +413,34 @@ export function getEvalQueryBody({
searchQuery?: ResultsSearchQuery;
ignoreDefaultQuery?: boolean;
}) {
let query: ResultsSearchQuery = {
let query;
const trainingQuery: ResultsSearchQuery = {
term: { [`${resultsField}.is_training`]: { value: isTraining } },
};
if (searchQuery !== undefined && ignoreDefaultQuery === true) {
query = searchQuery;
} else if (searchQuery !== undefined && isResultsSearchBoolQuery(searchQuery)) {
const searchQueryClone = cloneDeep(searchQuery);
searchQueryClone.bool.must.push(query);
const searchQueryClone = cloneDeep(searchQuery);
if (isResultsSearchBoolQuery(searchQueryClone)) {
if (searchQueryClone.bool.must === undefined) {
searchQueryClone.bool.must = [];
}
searchQueryClone.bool.must.push(trainingQuery);
query = searchQueryClone;
} else if (isQueryStringQuery(searchQueryClone)) {
query = {
bool: {
must: [searchQueryClone, trainingQuery],
},
};
} else {
// Not a bool or string query so we need to create it so can add the trainingQuery
query = {
bool: {
must: [trainingQuery],
},
};
}
return query;
}

58
x-pack/plugins/ml/public/application/data_frame_analytics/common/fields.ts
View file

@ -246,8 +246,15 @@ export const getDefaultFieldsFromJobCaps = (
fields: Field[],
jobConfig: DataFrameAnalyticsConfig,
needsDestIndexFields: boolean
): { selectedFields: Field[]; docFields: Field[]; depVarType?: ES_FIELD_TYPES } => {
const fieldsObj = { selectedFields: [], docFields: [] };
): {
selectedFields: Field[];
docFields: Field[];
depVarType?: ES_FIELD_TYPES;
} => {
const fieldsObj = {
selectedFields: [],
docFields: [],
};
if (fields.length === 0) {
return fieldsObj;
}
@ -267,38 +274,37 @@ export const getDefaultFieldsFromJobCaps = (
const featureImportanceFields = [];
if ((numTopFeatureImportanceValues ?? 0) > 0) {
featureImportanceFields.push(
...fields.map(d => ({
id: `${resultsField}.feature_importance.${d.id}`,
name: `${resultsField}.feature_importance.${d.name}`,
type: KBN_FIELD_TYPES.NUMBER,
}))
featureImportanceFields.push({
id: `${resultsField}.feature_importance`,
name: `${resultsField}.feature_importance`,
type: KBN_FIELD_TYPES.NUMBER,
});
}
let allFields: any = [];
// Only need to add these fields if we didn't use dest index pattern to get the fields
if (needsDestIndexFields === true) {
allFields.push(
{
id: `${resultsField}.is_training`,
name: `${resultsField}.is_training`,
type: ES_FIELD_TYPES.BOOLEAN,
},
{ id: predictedField, name: predictedField, type }
);
}
// Only need to add these fields if we didn't use dest index pattern to get the fields
const allFields: any =
needsDestIndexFields === true
? [
{
id: `${resultsField}.is_training`,
name: `${resultsField}.is_training`,
type: ES_FIELD_TYPES.BOOLEAN,
},
{ id: predictedField, name: predictedField, type },
...featureImportanceFields,
]
: [];
allFields.push(...fields);
allFields.push(...fields, ...featureImportanceFields);
allFields.sort(({ name: a }: { name: string }, { name: b }: { name: string }) =>
sortRegressionResultsFields(a, b, jobConfig)
);
// Remove feature_importance fields provided by dest index since feature_importance is an array the path is not valid
if (needsDestIndexFields === false) {
allFields = allFields.filter((field: any) => !field.name.includes('.feature_importance.'));
}
let selectedFields = allFields.filter(
(field: any) =>
field.name === predictedField ||
(!field.name.includes('.keyword') && !field.name.includes('.feature_importance.'))
(field: any) => field.name === predictedField || !field.name.includes('.keyword')
);
if (selectedFields.length > DEFAULT_REGRESSION_COLUMNS) {

16
x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/classification_exploration/use_explore_data.ts
View file

@ -22,7 +22,6 @@ import { newJobCapsService } from '../../../../../services/new_job_capabilities_
import { Field } from '../../../../../../../common/types/fields';
import { ES_FIELD_TYPES } from '../../../../../../../../../../src/plugins/data/public';
import {
LoadExploreDataArg,
defaultSearchQuery,
ResultsSearchQuery,
isResultsSearchBoolQuery,
@ -37,12 +36,23 @@ import {
SEARCH_SIZE,
SearchQuery,
} from '../../../../common';
import { SavedSearchQuery } from '../../../../../contexts/ml';
interface LoadClassificationExploreDataArg {
direction: SortDirection;
filterByIsTraining?: boolean;
field: string;
searchQuery: SavedSearchQuery;
requiresKeyword?: boolean;
pageIndex?: number;
pageSize?: number;
}
export type TableItem = Record<string, any>;
export interface UseExploreDataReturnType {
errorMessage: string;
loadExploreData: (arg: LoadExploreDataArg) => void;
loadExploreData: (arg: LoadClassificationExploreDataArg) => void;
sortField: EsFieldName;
sortDirection: SortDirection;
status: INDEX_STATUS;
@ -84,7 +94,7 @@ export const useExploreData = (
direction,
searchQuery,
requiresKeyword,
}: LoadExploreDataArg) => {
}: LoadClassificationExploreDataArg) => {
if (jobConfig !== undefined) {
setErrorMessage('');
setStatus(INDEX_STATUS.LOADING);

206
x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/evaluate_panel.tsx
View file

@ -4,7 +4,7 @@
* you may not use this file except in compliance with the Elastic License.
*/
import React, { FC, Fragment, useEffect, useState } from 'react';
import React, { FC, useEffect, useState } from 'react';
import { i18n } from '@kbn/i18n';
import { FormattedMessage } from '@kbn/i18n/react';
import {
@ -17,7 +17,7 @@ import {
EuiTitle,
} from '@elastic/eui';
import { useMlKibana } from '../../../../../contexts/kibana';
import { ErrorCallout } from '../error_callout';
import { SavedSearchQuery } from '../../../../../contexts/ml';
import {
getValuesFromResponse,
getDependentVar,
@ -33,14 +33,13 @@ import { EvaluateStat } from './evaluate_stat';
import {
isResultsSearchBoolQuery,
isRegressionEvaluateResponse,
ResultsSearchQuery,
ANALYSIS_CONFIG_TYPE,
} from '../../../../common/analytics';
interface Props {
jobConfig: DataFrameAnalyticsConfig;
jobStatus?: DATA_FRAME_TASK_STATE;
searchQuery: ResultsSearchQuery;
searchQuery: SavedSearchQuery;
}
const defaultEval: Eval = { meanSquaredError: '', rSquared: '', error: null };
@ -54,6 +53,7 @@ export const EvaluatePanel: FC<Props> = ({ jobConfig, jobStatus, searchQuery })
const [generalizationEval, setGeneralizationEval] = useState<Eval>(defaultEval);
const [isLoadingTraining, setIsLoadingTraining] = useState<boolean>(false);
const [isLoadingGeneralization, setIsLoadingGeneralization] = useState<boolean>(false);
const [isTrainingFilter, setIsTrainingFilter] = useState<boolean | undefined>(undefined);
const [trainingDocsCount, setTrainingDocsCount] = useState<null | number>(null);
const [generalizationDocsCount, setGeneralizationDocsCount] = useState<null | number>(null);
@ -92,8 +92,8 @@ export const EvaluatePanel: FC<Props> = ({ jobConfig, jobStatus, searchQuery })
} else {
setIsLoadingGeneralization(false);
setGeneralizationEval({
meanSquaredError: '',
rSquared: '',
meanSquaredError: '--',
rSquared: '--',
error: genErrorEval.error,
});
}
@ -128,108 +128,78 @@ export const EvaluatePanel: FC<Props> = ({ jobConfig, jobStatus, searchQuery })
} else {
setIsLoadingTraining(false);
setTrainingEval({
meanSquaredError: '',
rSquared: '',
meanSquaredError: '--',
rSquared: '--',
error: trainingErrorEval.error,
});
}
};
const loadData = async ({
isTrainingClause,
}: {
isTrainingClause?: { query: string; operator: string };
}) => {
// searchBar query is filtering for testing data
if (isTrainingClause !== undefined && isTrainingClause.query === 'false') {
loadGeneralizationData();
const docsCountResp = await loadDocsCount({
isTraining: false,
searchQuery,
resultsField,
destIndex: jobConfig.dest.index,
});
if (docsCountResp.success === true) {
setGeneralizationDocsCount(docsCountResp.docsCount);
} else {
setGeneralizationDocsCount(null);
}
setTrainingDocsCount(0);
setTrainingEval({
meanSquaredError: '--',
rSquared: '--',
error: null,
});
} else if (isTrainingClause !== undefined && isTrainingClause.query === 'true') {
// searchBar query is filtering for training data
loadTrainingData();
const docsCountResp = await loadDocsCount({
isTraining: true,
searchQuery,
resultsField,
destIndex: jobConfig.dest.index,
});
if (docsCountResp.success === true) {
setTrainingDocsCount(docsCountResp.docsCount);
} else {
setTrainingDocsCount(null);
}
setGeneralizationDocsCount(0);
setGeneralizationEval({
meanSquaredError: '--',
rSquared: '--',
error: null,
});
const loadData = async () => {
loadGeneralizationData(false);
const genDocsCountResp = await loadDocsCount({
ignoreDefaultQuery: false,
isTraining: false,
searchQuery,
resultsField,
destIndex: jobConfig.dest.index,
});
if (genDocsCountResp.success === true) {
setGeneralizationDocsCount(genDocsCountResp.docsCount);
} else {
// No is_training clause/filter from search bar so load both
loadGeneralizationData(false);
const genDocsCountResp = await loadDocsCount({
ignoreDefaultQuery: false,
isTraining: false,
searchQuery,
resultsField,
destIndex: jobConfig.dest.index,
});
if (genDocsCountResp.success === true) {
setGeneralizationDocsCount(genDocsCountResp.docsCount);
} else {
setGeneralizationDocsCount(null);
}
setGeneralizationDocsCount(null);
}
loadTrainingData(false);
const trainDocsCountResp = await loadDocsCount({
ignoreDefaultQuery: false,
isTraining: true,
searchQuery,
resultsField,
destIndex: jobConfig.dest.index,
});
if (trainDocsCountResp.success === true) {
setTrainingDocsCount(trainDocsCountResp.docsCount);
} else {
setTrainingDocsCount(null);
}
loadTrainingData(false);
const trainDocsCountResp = await loadDocsCount({
ignoreDefaultQuery: false,
isTraining: true,
searchQuery,
resultsField,
destIndex: jobConfig.dest.index,
});
if (trainDocsCountResp.success === true) {
setTrainingDocsCount(trainDocsCountResp.docsCount);
} else {
setTrainingDocsCount(null);
}
};
useEffect(() => {
const hasIsTrainingClause =
isResultsSearchBoolQuery(searchQuery) &&
searchQuery.bool.must.filter(
(clause: any) => clause.match && clause.match[`${resultsField}.is_training`] !== undefined
);
const isTrainingClause =
hasIsTrainingClause &&
hasIsTrainingClause[0] &&
hasIsTrainingClause[0].match[`${resultsField}.is_training`];
let isTraining: boolean | undefined;
const query =
isResultsSearchBoolQuery(searchQuery) && (searchQuery.bool.should || searchQuery.bool.filter);
loadData({ isTrainingClause });
if (query !== undefined && query !== false) {
for (let i = 0; i < query.length; i++) {
const clause = query[i];
if (clause.match && clause.match[`${resultsField}.is_training`] !== undefined) {
isTraining = clause.match[`${resultsField}.is_training`];
break;
} else if (
clause.bool &&
(clause.bool.should !== undefined || clause.bool.filter !== undefined)
) {
const innerQuery = clause.bool.should || clause.bool.filter;
if (innerQuery !== undefined) {
for (let j = 0; j < innerQuery.length; j++) {
const innerClause = innerQuery[j];
if (
innerClause.match &&
innerClause.match[`${resultsField}.is_training`] !== undefined
) {
isTraining = innerClause.match[`${resultsField}.is_training`];
break;
}
}
}
}
}
}
setIsTrainingFilter(isTraining);
loadData();
}, [JSON.stringify(searchQuery)]);
return (
@ -293,13 +263,18 @@ export const EvaluatePanel: FC<Props> = ({ jobConfig, jobStatus, searchQuery })
defaultMessage="{docsCount, plural, one {# doc} other {# docs}} evaluated"
values={{ docsCount: generalizationDocsCount }}
/>
{isTrainingFilter === true && generalizationDocsCount === 0 && (
<FormattedMessage
id="xpack.ml.dataframe.analytics.regressionExploration.generalizationFilterText"
defaultMessage=". Filtering for training data."
/>
)}
</EuiText>
)}
<EuiSpacer />
<EuiFlexGroup>
{generalizationEval.error !== null && <ErrorCallout error={generalizationEval.error} />}
{generalizationEval.error === null && (
<Fragment>
<EuiFlexGroup direction="column" gutterSize="none">
<EuiFlexItem>
<EuiFlexGroup>
<EuiFlexItem>
<EvaluateStat
dataTestSubj={'mlDFAnalyticsRegressionGenMSEstat'}
@ -316,7 +291,14 @@ export const EvaluatePanel: FC<Props> = ({ jobConfig, jobStatus, searchQuery })
isMSE={false}
/>
</EuiFlexItem>
</Fragment>
</EuiFlexGroup>
</EuiFlexItem>
{generalizationEval.error !== null && (
<EuiFlexItem grow={false}>
<EuiText size="xs" color="danger">
{generalizationEval.error}
</EuiText>
</EuiFlexItem>
)}
</EuiFlexGroup>
</EuiFlexItem>
@ -338,13 +320,18 @@ export const EvaluatePanel: FC<Props> = ({ jobConfig, jobStatus, searchQuery })
defaultMessage="{docsCount, plural, one {# doc} other {# docs}} evaluated"
values={{ docsCount: trainingDocsCount }}
/>
{isTrainingFilter === false && trainingDocsCount === 0 && (
<FormattedMessage
id="xpack.ml.dataframe.analytics.regressionExploration.trainingFilterText"
defaultMessage=". Filtering for testing data."
/>
)}
</EuiText>
)}
<EuiSpacer />
<EuiFlexGroup>
{trainingEval.error !== null && <ErrorCallout error={trainingEval.error} />}
{trainingEval.error === null && (
<Fragment>
<EuiFlexGroup direction="column" gutterSize="none">
<EuiFlexItem>
<EuiFlexGroup>
<EuiFlexItem>
<EvaluateStat
dataTestSubj={'mlDFAnalyticsRegressionTrainingMSEstat'}
@ -361,7 +348,14 @@ export const EvaluatePanel: FC<Props> = ({ jobConfig, jobStatus, searchQuery })
isMSE={false}
/>
</EuiFlexItem>
</Fragment>
</EuiFlexGroup>
</EuiFlexItem>
{trainingEval.error !== null && (
<EuiFlexItem grow={false}>
<EuiText size="xs" color="danger">
{trainingEval.error}
</EuiText>
</EuiFlexItem>
)}
</EuiFlexGroup>
</EuiFlexItem>

135
x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/regression_exploration_data_grid.tsx Normal file
View file

@ -0,0 +1,135 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import React, { Dispatch, FC, SetStateAction, useCallback, useMemo } from 'react';
import { i18n } from '@kbn/i18n';
import { EuiDataGrid, EuiDataGridPaginationProps, EuiDataGridSorting } from '@elastic/eui';
import { euiDataGridStyle, euiDataGridToolbarSettings } from '../../../../common';
import { mlFieldFormatService } from '../../../../../services/field_format_service';
import { IndexPattern } from '../../../../../../../../../../src/plugins/data/public';
const PAGE_SIZE_OPTIONS = [5, 10, 25, 50];
type Pagination = Pick<EuiDataGridPaginationProps, 'pageIndex' | 'pageSize'>;
type TableItem = Record<string, any>;
interface ExplorationDataGridProps {
colorRange?: (d: number) => string;
columns: any[];
indexPattern: IndexPattern;
pagination: Pagination;
resultsField: string;
rowCount: number;
selectedFields: string[];
setPagination: Dispatch<SetStateAction<Pagination>>;
setSelectedFields: Dispatch<SetStateAction<string[]>>;
setSortingColumns: Dispatch<SetStateAction<EuiDataGridSorting['columns']>>;
sortingColumns: EuiDataGridSorting['columns'];
tableItems: TableItem[];
}
export const RegressionExplorationDataGrid: FC<ExplorationDataGridProps> = ({
columns,
indexPattern,
pagination,
resultsField,
rowCount,
selectedFields,
setPagination,
setSelectedFields,
setSortingColumns,
sortingColumns,
tableItems,
}) => {
const renderCellValue = useMemo(() => {
return ({ rowIndex, columnId }: { rowIndex: number; columnId: string; setCellProps: any }) => {
const adjustedRowIndex = rowIndex - pagination.pageIndex * pagination.pageSize;
const fullItem = tableItems[adjustedRowIndex];
if (fullItem === undefined) {
return null;
}
let format: any;
if (indexPattern !== undefined) {
format = mlFieldFormatService.getFieldFormatFromIndexPattern(indexPattern, columnId, '');
}
const cellValue =
fullItem.hasOwnProperty(columnId) && fullItem[columnId] !== undefined
? fullItem[columnId]
: null;
if (format !== undefined) {
return format.convert(cellValue, 'text');
}
if (typeof cellValue === 'string' || cellValue === null) {
return cellValue;
}
if (typeof cellValue === 'boolean') {
return cellValue ? 'true' : 'false';
}
if (typeof cellValue === 'object' && cellValue !== null) {
return JSON.stringify(cellValue);
}
return cellValue;
};
}, [resultsField, rowCount, tableItems, pagination.pageIndex, pagination.pageSize]);
const onChangeItemsPerPage = useCallback(
pageSize => {
setPagination(p => {
const pageIndex = Math.floor((p.pageSize * p.pageIndex) / pageSize);
return { pageIndex, pageSize };
});
},
[setPagination]
);
const onChangePage = useCallback(pageIndex => setPagination(p => ({ ...p, pageIndex })), [
setPagination,
]);
const onSort = useCallback(sc => setSortingColumns(sc), [setSortingColumns]);
return (
<EuiDataGrid
aria-label={i18n.translate(
'xpack.ml.dataframe.analytics.regressionExploration.dataGridAriaLabel',
{
defaultMessage: 'Regression results table',
}
)}
columns={columns}
columnVisibility={{
visibleColumns: selectedFields,
setVisibleColumns: setSelectedFields,
}}
gridStyle={euiDataGridStyle}
rowCount={rowCount}
renderCellValue={renderCellValue}
sorting={{ columns: sortingColumns, onSort }}
toolbarVisibility={euiDataGridToolbarSettings}
pagination={{
...pagination,
pageSizeOptions: PAGE_SIZE_OPTIONS,
onChangeItemsPerPage,
onChangePage,
}}
/>
);
};

475
x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/results_table.tsx
View file

@ -4,72 +4,41 @@
* you may not use this file except in compliance with the Elastic License.
*/
import React, { Fragment, FC, useEffect, useState } from 'react';
import moment from 'moment-timezone';
import React, { Fragment, FC, useEffect } from 'react';
import { i18n } from '@kbn/i18n';
import {
EuiBadge,
EuiButtonIcon,
EuiCallOut,
EuiCheckbox,
EuiFlexGroup,
EuiFlexItem,
EuiFormRow,
EuiPanel,
EuiPopover,
EuiPopoverTitle,
EuiProgress,
EuiSpacer,
EuiText,
EuiToolTip,
Query,
} from '@elastic/eui';
import { Query as QueryType } from '../../../analytics_management/components/analytics_list/common';
import { ES_FIELD_TYPES } from '../../../../../../../../../../src/plugins/data/public';
import { mlFieldFormatService } from '../../../../../services/field_format_service';
import {
ColumnType,
mlInMemoryTableBasicFactory,
OnTableChangeArg,
SortingPropType,
SORT_DIRECTION,
} from '../../../../../components/ml_in_memory_table';
import { formatHumanReadableDateTimeSeconds } from '../../../../../util/date_utils';
import { Field } from '../../../../../../../common/types/fields';
import { SavedSearchQuery } from '../../../../../contexts/ml';
import {
BASIC_NUMERICAL_TYPES,
EXTENDED_NUMERICAL_TYPES,
toggleSelectedField,
isKeywordAndTextType,
sortRegressionResultsFields,
} from '../../../../common/fields';
import {
DataFrameAnalyticsConfig,
EsFieldName,
EsDoc,
MAX_COLUMNS,
getPredictedFieldName,
INDEX_STATUS,
SEARCH_SIZE,
defaultSearchQuery,
getDependentVar,
} from '../../../../common';
import { getTaskStateBadge } from '../../../analytics_management/components/analytics_list/columns';
import { DATA_FRAME_TASK_STATE } from '../../../analytics_management/components/analytics_list/common';
import { IndexPattern } from '../../../../../../../../../../src/plugins/data/public';
import { useExploreData, TableItem } from './use_explore_data';
import { useExploreData } from './use_explore_data';
import { ExplorationTitle } from './regression_exploration';
const PAGE_SIZE_OPTIONS = [5, 10, 25, 50];
const MlInMemoryTableBasic = mlInMemoryTableBasicFactory<TableItem>();
import { RegressionExplorationDataGrid } from './regression_exploration_data_grid';
import { ExplorationQueryBar } from '../exploration_query_bar';
const showingDocs = i18n.translate(
'xpack.ml.dataframe.analytics.regressionExploration.documentsShownHelpText',
@ -95,308 +64,65 @@ interface Props {
export const ResultsTable: FC<Props> = React.memo(
({ indexPattern, jobConfig, jobStatus, setEvaluateSearchQuery }) => {
const [pageIndex, setPageIndex] = useState(0);
const [pageSize, setPageSize] = useState(25);
const [selectedFields, setSelectedFields] = useState([] as Field[]);
const [docFields, setDocFields] = useState([] as Field[]);
const [depVarType, setDepVarType] = useState<ES_FIELD_TYPES | undefined>(undefined);
const [isColumnsPopoverVisible, setColumnsPopoverVisible] = useState(false);
const [searchQuery, setSearchQuery] = useState<SavedSearchQuery>(defaultSearchQuery);
const [searchError, setSearchError] = useState<any>(undefined);
const [searchString, setSearchString] = useState<string | undefined>(undefined);
const predictedFieldName = getPredictedFieldName(
jobConfig.dest.results_field,
jobConfig.analysis
);
const dependentVariable = getDependentVar(jobConfig.analysis);
function toggleColumnsPopover() {
setColumnsPopoverVisible(!isColumnsPopoverVisible);
}
function closeColumnsPopover() {
setColumnsPopoverVisible(false);
}
function toggleColumn(column: EsFieldName) {
if (tableItems.length > 0 && jobConfig !== undefined) {
// spread to a new array otherwise the component wouldn't re-render
setSelectedFields([
...toggleSelectedField(selectedFields, column, jobConfig.dest.results_field, depVarType),
]);
}
}
const needsDestIndexFields = indexPattern && indexPattern.title === jobConfig.source.index[0];
const resultsField = jobConfig.dest.results_field;
const {
errorMessage,
loadExploreData,
sortField,
sortDirection,
status,
tableItems,
} = useExploreData(
jobConfig,
needsDestIndexFields,
fieldTypes,
pagination,
searchQuery,
selectedFields,
rowCount,
setPagination,
setSearchQuery,
setSelectedFields,
setDocFields,
setDepVarType
);
setSortingColumns,
sortingColumns,
status,
tableFields,
tableItems,
} = useExploreData(jobConfig, needsDestIndexFields);
const columns: Array<ColumnType<TableItem>> = selectedFields
.sort(({ name: a }, { name: b }) => sortRegressionResultsFields(a, b, jobConfig))
.map(field => {
const { type } = field;
let format: any;
if (indexPattern !== undefined) {
format = mlFieldFormatService.getFieldFormatFromIndexPattern(indexPattern, field.id, '');
}
useEffect(() => {
setEvaluateSearchQuery(searchQuery);
}, [JSON.stringify(searchQuery)]);
const columns = tableFields
.sort((a: any, b: any) => sortRegressionResultsFields(a, b, jobConfig))
.map((field: any) => {
// Built-in values are ['boolean', 'currency', 'datetime', 'numeric', 'json']
// To fall back to the default string schema it needs to be undefined.
let schema;
let isSortable = true;
const type = fieldTypes[field];
const isNumber =
type !== undefined &&
(BASIC_NUMERICAL_TYPES.has(type) || EXTENDED_NUMERICAL_TYPES.has(type));
const column: ColumnType<TableItem> = {
field: field.name,
name: field.name,
sortable: true,
truncateText: true,
};
const render = (d: any, fullItem: EsDoc) => {
if (format !== undefined) {
d = format.convert(d, 'text');
return d;
}
if (Array.isArray(d) && d.every(item => typeof item === 'string')) {
// If the cells data is an array of strings, return as a comma separated list.
// The list will get limited to 5 items with `…` at the end if there's more in the original array.
return `${d.slice(0, 5).join(', ')}${d.length > 5 ? ', …' : ''}`;
} else if (Array.isArray(d)) {
// If the cells data is an array of e.g. objects, display a 'array' badge with a
// tooltip that explains that this type of field is not supported in this table.
return (
<EuiToolTip
content={i18n.translate(
'xpack.ml.dataframe.analytics.regressionExploration.indexArrayToolTipContent',
{
defaultMessage:
'The full content of this array based column cannot be displayed.',
}
)}
>
<EuiBadge>
{i18n.translate(
'xpack.ml.dataframe.analytics.regressionExploration.indexArrayBadgeContent',
{
defaultMessage: 'array',
}
)}
</EuiBadge>
</EuiToolTip>
);
}
return d;
};
if (isNumber) {
column.dataType = 'number';
column.render = render;
} else if (typeof type !== 'undefined') {
switch (type) {
case ES_FIELD_TYPES.BOOLEAN:
column.dataType = ES_FIELD_TYPES.BOOLEAN;
column.render = d => (d ? 'true' : 'false');
break;
case ES_FIELD_TYPES.DATE:
column.align = 'right';
if (format !== undefined) {
column.render = render;
} else {
column.render = (d: any) => {
if (d !== undefined) {
return formatHumanReadableDateTimeSeconds(moment(d).unix() * 1000);
}
return d;
};
}
break;
default:
column.render = render;
break;
}
} else {
column.render = render;
schema = 'numeric';
}
return column;
switch (type) {
case 'date':
schema = 'datetime';
break;
case 'geo_point':
schema = 'json';
break;
case 'boolean':
schema = 'boolean';
break;
}
if (field === `${resultsField}.feature_importance`) {
isSortable = false;
}
return { id: field, schema, isSortable };
});
const docFieldsCount = docFields.length;
useEffect(() => {
if (
jobConfig !== undefined &&
columns.length > 0 &&
selectedFields.length > 0 &&
sortField !== undefined &&
sortDirection !== undefined &&
selectedFields.some(field => field.name === sortField)
) {
let field = sortField;
// If sorting by predictedField use dependentVar type
if (predictedFieldName === sortField) {
field = dependentVariable;
}
const requiresKeyword = isKeywordAndTextType(field);
loadExploreData({
field: sortField,
direction: sortDirection,
searchQuery,
requiresKeyword,
});
}
}, [JSON.stringify(searchQuery)]);
useEffect(() => {
// By default set sorting to descending on the prediction field (`<dependent_varible or prediction_field_name>_prediction`).
// if that's not available sort ascending on the first column. Check if the current sorting field is still available.
if (
jobConfig !== undefined &&
columns.length > 0 &&
selectedFields.length > 0 &&
!selectedFields.some(field => field.name === sortField)
) {
const predictedFieldSelected = selectedFields.some(
field => field.name === predictedFieldName
);
// CHECK IF keyword suffix is needed (if predicted field is selected we have to check the dependent variable type)
let sortByField = predictedFieldSelected ? dependentVariable : selectedFields[0].name;
const requiresKeyword = isKeywordAndTextType(sortByField);
sortByField = predictedFieldSelected ? predictedFieldName : sortByField;
const direction = predictedFieldSelected ? SORT_DIRECTION.DESC : SORT_DIRECTION.ASC;
loadExploreData({ field: sortByField, direction, searchQuery, requiresKeyword });
}
}, [
jobConfig,
columns.length,
selectedFields.length,
sortField,
sortDirection,
tableItems.length,
]);
let sorting: SortingPropType = false;
let onTableChange;
if (columns.length > 0 && sortField !== '' && sortField !== undefined) {
sorting = {
sort: {
field: sortField,
direction: sortDirection,
},
};
onTableChange = ({
page = { index: 0, size: 10 },
sort = { field: sortField, direction: sortDirection },
}: OnTableChangeArg) => {
const { index, size } = page;
setPageIndex(index);
setPageSize(size);
if (sort.field !== sortField || sort.direction !== sortDirection) {
let field = sort.field;
// If sorting by predictedField use depVar for type check
if (predictedFieldName === sort.field) {
field = dependentVariable;
}
loadExploreData({
...sort,
searchQuery,
requiresKeyword: isKeywordAndTextType(field),
});
}
};
}
const pagination = {
initialPageIndex: pageIndex,
initialPageSize: pageSize,
totalItemCount: tableItems.length,
pageSizeOptions: PAGE_SIZE_OPTIONS,
hidePerPageOptions: false,
};
const onQueryChange = ({ query, error }: { query: QueryType; error: any }) => {
if (error) {
setSearchError(error.message);
} else {
try {
const esQueryDsl = Query.toESQuery(query);
setSearchQuery(esQueryDsl);
setSearchString(query.text);
setSearchError(undefined);
// set query for use in evaluate panel
setEvaluateSearchQuery(esQueryDsl);
} catch (e) {
setSearchError(e.toString());
}
}
};
const search = {
onChange: onQueryChange,
defaultQuery: searchString,
box: {
incremental: false,
placeholder: i18n.translate(
'xpack.ml.dataframe.analytics.regressionExploration.searchBoxPlaceholder',
{
defaultMessage: 'E.g. avg>0.5',
}
),
},
filters: [
{
type: 'field_value_toggle_group',
field: `${jobConfig.dest.results_field}.is_training`,
items: [
{
value: false,
name: i18n.translate(
'xpack.ml.dataframe.analytics.regressionExploration.isTestingLabel',
{
defaultMessage: 'Testing',
}
),
},
{
value: true,
name: i18n.translate(
'xpack.ml.dataframe.analytics.regressionExploration.isTrainingLabel',
{
defaultMessage: 'Training',
}
),
},
],
},
],
};
const docFieldsCount = tableFields.length;
if (jobConfig === undefined) {
return null;
@ -428,11 +154,6 @@ export const ResultsTable: FC<Props> = React.memo(
);
}
const tableError =
status === INDEX_STATUS.ERROR && errorMessage.includes('parsing_exception')
? errorMessage
: searchError;
return (
<EuiPanel grow={false} data-test-subj="mlDFAnalyticsRegressionExplorationTablePanel">
<EuiFlexGroup alignItems="center" justifyContent="spaceBetween" responsive={false}>
@ -464,52 +185,6 @@ export const ResultsTable: FC<Props> = React.memo(
</EuiText>
)}
</EuiFlexItem>
<EuiFlexItem grow={false}>
<EuiText size="s">
<EuiPopover
id="popover"
button={
<EuiButtonIcon
iconType="gear"
onClick={toggleColumnsPopover}
aria-label={i18n.translate(
'xpack.ml.dataframe.analytics.regressionExploration.selectColumnsAriaLabel',
{
defaultMessage: 'Select columns',
}
)}
/>
}
isOpen={isColumnsPopoverVisible}
closePopover={closeColumnsPopover}
ownFocus
>
<EuiPopoverTitle>
{i18n.translate(
'xpack.ml.dataframe.analytics.regressionExploration.selectFieldsPopoverTitle',
{
defaultMessage: 'Select fields',
}
)}
</EuiPopoverTitle>
<div style={{ maxHeight: '400px', overflowY: 'scroll' }}>
{docFields.map(({ name }) => (
<EuiCheckbox
id={name}
key={name}
label={name}
checked={selectedFields.some(field => field.name === name)}
onChange={() => toggleColumn(name)}
disabled={
selectedFields.some(field => field.name === name) &&
selectedFields.length === 1
}
/>
))}
</div>
</EuiPopover>
</EuiText>
</EuiFlexItem>
</EuiFlexGroup>
</EuiFlexItem>
</EuiFlexGroup>
@ -518,29 +193,39 @@ export const ResultsTable: FC<Props> = React.memo(
<EuiProgress size="xs" color="accent" max={1} value={0} />
)}
{(columns.length > 0 || searchQuery !== defaultSearchQuery) && (
<Fragment>
<EuiFormRow
helpText={tableItems.length === SEARCH_SIZE ? showingFirstDocs : showingDocs}
>
<Fragment />
</EuiFormRow>
<EuiSpacer />
<MlInMemoryTableBasic
allowNeutralSort={false}
columns={columns}
compressed
hasActions={false}
isSelectable={false}
items={tableItems}
onTableChange={onTableChange}
pagination={pagination}
responsive={false}
search={search}
error={tableError}
sorting={sorting}
/>
</Fragment>
<EuiFlexGroup direction="column">
<EuiFlexItem grow={false}>
<EuiSpacer size="s" />
<EuiFlexGroup justifyContent="spaceBetween">
<EuiFlexItem>
<ExplorationQueryBar
indexPattern={indexPattern}
setSearchQuery={setSearchQuery}
/>
</EuiFlexItem>
</EuiFlexGroup>
<EuiFormRow
helpText={tableItems.length === SEARCH_SIZE ? showingFirstDocs : showingDocs}
>
<Fragment />
</EuiFormRow>
</EuiFlexItem>
<EuiFlexItem grow={false}>
<RegressionExplorationDataGrid
columns={columns}
indexPattern={indexPattern}
pagination={pagination}
resultsField={jobConfig.dest.results_field}
rowCount={rowCount}
selectedFields={selectedFields}
setPagination={setPagination}
setSelectedFields={setSelectedFields}
setSortingColumns={setSortingColumns}
sortingColumns={sortingColumns}
tableItems={tableItems}
/>
</EuiFlexItem>
</EuiFlexGroup>
)}
</EuiPanel>
);

211
x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/use_explore_data.ts
View file

@ -4,97 +4,156 @@
* you may not use this file except in compliance with the Elastic License.
*/
import React, { useEffect, useState } from 'react';
import { useEffect, useState, Dispatch, SetStateAction } from 'react';
import { EuiDataGridPaginationProps, EuiDataGridSorting } from '@elastic/eui';
import { SearchResponse } from 'elasticsearch';
import { cloneDeep } from 'lodash';
import { SortDirection, SORT_DIRECTION } from '../../../../../components/ml_in_memory_table';
import { ml } from '../../../../../services/ml_api_service';
import { getNestedProperty } from '../../../../../util/object_utils';
import { newJobCapsService } from '../../../../../services/new_job_capabilities_service';
import { SORT_DIRECTION } from '../../../../../components/ml_in_memory_table';
import { SavedSearchQuery } from '../../../../../contexts/ml';
import {
getDefaultFieldsFromJobCaps,
getDependentVar,
getFlattenedFields,
getPredictedFieldName,
DataFrameAnalyticsConfig,
EsFieldName,
INDEX_STATUS,
SEARCH_SIZE,
SearchQuery,
} from '../../../../common';
import { Field } from '../../../../../../../common/types/fields';
import { Dictionary } from '../../../../../../../common/types/common';
import { isKeywordAndTextType } from '../../../../common/fields';
import { ES_FIELD_TYPES } from '../../../../../../../../../../src/plugins/data/public';
import {
LoadExploreDataArg,
LoadRegressionExploreDataArg,
defaultSearchQuery,
ResultsSearchQuery,
isResultsSearchBoolQuery,
} from '../../../../common/analytics';
export type TableItem = Record<string, any>;
type Pagination = Pick<EuiDataGridPaginationProps, 'pageIndex' | 'pageSize'>;
export interface UseExploreDataReturnType {
errorMessage: string;
loadExploreData: (arg: LoadExploreDataArg) => void;
sortField: EsFieldName;
sortDirection: SortDirection;
fieldTypes: { [key: string]: ES_FIELD_TYPES };
pagination: Pagination;
rowCount: number;
searchQuery: SavedSearchQuery;
selectedFields: EsFieldName[];
setFilterByIsTraining: Dispatch<SetStateAction<undefined | boolean>>;
setPagination: Dispatch<SetStateAction<Pagination>>;
setSearchQuery: Dispatch<SetStateAction<SavedSearchQuery>>;
setSelectedFields: Dispatch<SetStateAction<EsFieldName[]>>;
setSortingColumns: Dispatch<SetStateAction<EuiDataGridSorting['columns']>>;
sortingColumns: EuiDataGridSorting['columns'];
status: INDEX_STATUS;
tableFields: string[];
tableItems: TableItem[];
}
type EsSorting = Dictionary<{
order: 'asc' | 'desc';
}>;
// The types specified in `@types/elasticsearch` are out of date and still have `total: number`.
interface SearchResponse7 extends SearchResponse<any> {
hits: SearchResponse<any>['hits'] & {
total: {
value: number;
relation: string;
};
};
}
export const useExploreData = (
jobConfig: DataFrameAnalyticsConfig | undefined,
needsDestIndexFields: boolean,
selectedFields: Field[],
setSelectedFields: React.Dispatch<React.SetStateAction<Field[]>>,
setDocFields: React.Dispatch<React.SetStateAction<Field[]>>,
setDepVarType: React.Dispatch<React.SetStateAction<ES_FIELD_TYPES | undefined>>
jobConfig: DataFrameAnalyticsConfig,
needsDestIndexFields: boolean
): UseExploreDataReturnType => {
const [errorMessage, setErrorMessage] = useState('');
const [status, setStatus] = useState(INDEX_STATUS.UNUSED);
const [selectedFields, setSelectedFields] = useState([] as EsFieldName[]);
const [tableFields, setTableFields] = useState<string[]>([]);
const [tableItems, setTableItems] = useState<TableItem[]>([]);
const [sortField, setSortField] = useState<string>('');
const [sortDirection, setSortDirection] = useState<SortDirection>(SORT_DIRECTION.ASC);
const [fieldTypes, setFieldTypes] = useState<{ [key: string]: ES_FIELD_TYPES }>({});
const [rowCount, setRowCount] = useState(0);
const [pagination, setPagination] = useState({ pageIndex: 0, pageSize: 25 });
const [searchQuery, setSearchQuery] = useState<SavedSearchQuery>(defaultSearchQuery);
const [filterByIsTraining, setFilterByIsTraining] = useState<undefined | boolean>(undefined);
const [sortingColumns, setSortingColumns] = useState<EuiDataGridSorting['columns']>([]);
const predictedFieldName = getPredictedFieldName(
jobConfig.dest.results_field,
jobConfig.analysis
);
const dependentVariable = getDependentVar(jobConfig.analysis);
const getDefaultSelectedFields = () => {
const { fields } = newJobCapsService;
if (selectedFields.length === 0 && jobConfig !== undefined) {
const {
selectedFields: defaultSelected,
docFields,
depVarType,
} = getDefaultFieldsFromJobCaps(fields, jobConfig, needsDestIndexFields);
const { selectedFields: defaultSelected, docFields } = getDefaultFieldsFromJobCaps(
fields,
jobConfig,
needsDestIndexFields
);
setDepVarType(depVarType);
setSelectedFields(defaultSelected);
setDocFields(docFields);
const types: { [key: string]: ES_FIELD_TYPES } = {};
const allFields: string[] = [];
docFields.forEach(field => {
types[field.id] = field.type;
allFields.push(field.id);
});
setFieldTypes(types);
setSelectedFields(defaultSelected.map(field => field.id));
setTableFields(allFields);
}
};
const loadExploreData = async ({
field,
direction,
searchQuery,
requiresKeyword,
}: LoadExploreDataArg) => {
filterByIsTraining: isTraining,
searchQuery: incomingQuery,
}: LoadRegressionExploreDataArg) => {
if (jobConfig !== undefined) {
setErrorMessage('');
setStatus(INDEX_STATUS.LOADING);
try {
const resultsField = jobConfig.dest.results_field;
const searchQueryClone: ResultsSearchQuery = cloneDeep(searchQuery);
const searchQueryClone: ResultsSearchQuery = cloneDeep(incomingQuery);
let query: ResultsSearchQuery;
const { pageIndex, pageSize } = pagination;
// If filterByIsTraining is defined - add that in to the final query
const trainingQuery =
isTraining !== undefined
? {
term: { [`${resultsField}.is_training`]: { value: isTraining } },
}
: undefined;
if (JSON.stringify(searchQuery) === JSON.stringify(defaultSearchQuery)) {
query = {
if (JSON.stringify(incomingQuery) === JSON.stringify(defaultSearchQuery)) {
const existsQuery = {
exists: {
field: resultsField,
},
};
query = {
bool: {
must: [existsQuery],
},
};
if (trainingQuery !== undefined && isResultsSearchBoolQuery(query)) {
query.bool.must.push(trainingQuery);
}
} else if (isResultsSearchBoolQuery(searchQueryClone)) {
if (searchQueryClone.bool.must === undefined) {
searchQueryClone.bool.must = [];
@ -106,32 +165,37 @@ export const useExploreData = (
},
});
if (trainingQuery !== undefined) {
searchQueryClone.bool.must.push(trainingQuery);
}
query = searchQueryClone;
} else {
query = searchQueryClone;
}
const body: SearchQuery = {
query,
};
if (field !== undefined) {
body.sort = [
{
[`${field}${requiresKeyword ? '.keyword' : ''}`]: {
order: direction,
},
},
];
}
const sort: EsSorting = sortingColumns
.map(column => {
const { id } = column;
column.id = isKeywordAndTextType(id) ? `${id}.keyword` : id;
return column;
})
.reduce((s, column) => {
s[column.id] = { order: column.direction };
return s;
}, {} as EsSorting);
const resp: SearchResponse<any> = await ml.esSearch({
const resp: SearchResponse7 = await ml.esSearch({
index: jobConfig.dest.index,
size: SEARCH_SIZE,
body,
body: {
query,
from: pageIndex * pageSize,
size: pageSize,
...(Object.keys(sort).length > 0 ? { sort } : {}),
},
});
setSortField(field);
setSortDirection(direction);
setRowCount(resp.hits.total.value);
const docs = resp.hits.hits;
@ -183,10 +247,45 @@ export const useExploreData = (
};
useEffect(() => {
if (jobConfig !== undefined) {
getDefaultSelectedFields();
}
getDefaultSelectedFields();
}, [jobConfig && jobConfig.id]);
return { errorMessage, loadExploreData, sortField, sortDirection, status, tableItems };
// By default set sorting to descending on the prediction field (`<dependent_varible or prediction_field_name>_prediction`).
useEffect(() => {
const sortByField = isKeywordAndTextType(dependentVariable)
? `${predictedFieldName}.keyword`
: predictedFieldName;
const direction = SORT_DIRECTION.DESC;
setSortingColumns([{ id: sortByField, direction }]);
}, [jobConfig && jobConfig.id]);
useEffect(() => {
loadExploreData({ filterByIsTraining, searchQuery });
}, [
filterByIsTraining,
jobConfig && jobConfig.id,
pagination,
searchQuery,
selectedFields,
sortingColumns,
]);
return {
errorMessage,
fieldTypes,
pagination,
searchQuery,
selectedFields,
rowCount,
setFilterByIsTraining,
setPagination,
setSelectedFields,
setSortingColumns,
setSearchQuery,
sortingColumns,
status,
tableItems,
tableFields,
};
};

2
x-pack/plugins/translations/translations/ja-JP.json
View file

@ -9587,8 +9587,6 @@
"xpack.ml.dataframe.analytics.regressionExploration.generalError": "データの読み込み中にエラーが発生しました。",
"xpack.ml.dataframe.analytics.regressionExploration.generalizationDocsCount": "{docsCount, plural, one {# doc} other {# docs}}が評価されました",
"xpack.ml.dataframe.analytics.regressionExploration.generalizationErrorTitle": "一般化エラー",
"xpack.ml.dataframe.analytics.regressionExploration.indexArrayBadgeContent": "配列",
"xpack.ml.dataframe.analytics.regressionExploration.indexArrayToolTipContent": "この配列ベースの列の完全なコンテンツは表示できません。",
"xpack.ml.dataframe.analytics.regressionExploration.indexError": "インデックスデータの読み込み中にエラーが発生しました。",
"xpack.ml.dataframe.analytics.regressionExploration.isTestingLabel": "テスト",
"xpack.ml.dataframe.analytics.regressionExploration.isTrainingLabel": "トレーニング",

2
x-pack/plugins/translations/translations/zh-CN.json
View file

@ -9590,8 +9590,6 @@
"xpack.ml.dataframe.analytics.regressionExploration.generalError": "加载数据时出错。",
"xpack.ml.dataframe.analytics.regressionExploration.generalizationDocsCount": "{docsCount, plural, one {# 个文档} other {# 个文档}}已评估",
"xpack.ml.dataframe.analytics.regressionExploration.generalizationErrorTitle": "泛化误差",
"xpack.ml.dataframe.analytics.regressionExploration.indexArrayBadgeContent": "数组",
"xpack.ml.dataframe.analytics.regressionExploration.indexArrayToolTipContent": "此基于数组的列的完整内容无法显示。",
"xpack.ml.dataframe.analytics.regressionExploration.indexError": "加载索引数据时出错。",
"xpack.ml.dataframe.analytics.regressionExploration.isTestingLabel": "测试",
"xpack.ml.dataframe.analytics.regressionExploration.isTrainingLabel": "培训",