github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 09:48:58 -04:00
Code Issues Projects Releases Packages Wiki Activity

[ML] Data Frame Analytics: Ensure creation and results views display nested fields correctly (#96905)

Browse source 
* create analytics field service

* remove unnecessary field filter. update types

* create common base class for newJobCapabilites classes for AD and DFA

* fix column schema for histogram

* update endpoint to be consistent with AD job caps

* add unit test for removeNestedFieldChildren helper function

* removes obsolete const
This commit is contained in:
Melissa Alvarez 2021-04-15 12:52:09 -04:00 • committed by GitHub
parent e11ac98b3a
commit 53e2d5d725
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
38 changed files with 1044 additions and 191 deletions
Download patch file Download diff file Expand all files Collapse all files

4
x-pack/plugins/ml/common/types/fields.ts
View file

@ -50,6 +50,10 @@ export interface NewJobCaps {
aggs: Aggregation[];
}
export interface NewJobCapsResponse {
[indexPattern: string]: NewJobCaps;
}
export interface AggFieldPair {
agg: Aggregation;
field: Field;

6
x-pack/plugins/ml/public/application/components/data_grid/common.ts
View file

@ -103,7 +103,7 @@ export function getCombinedRuntimeMappings(
): RuntimeMappings | undefined {
let combinedRuntimeMappings = {};
// And runtime field mappings defined by index pattern
// Add runtime field mappings defined by index pattern
if (indexPattern) {
const computedFields = indexPattern?.getComputedFields();
if (computedFields?.runtimeFields !== undefined) {
@ -147,6 +147,7 @@ export const getDataGridSchemasFromFieldTypes = (fieldTypes: FieldTypes, results
case 'date':
schema = 'datetime';
break;
case 'nested':
case 'geo_point':
schema = 'json';
break;
@ -238,6 +239,9 @@ export const getDataGridSchemaFromKibanaFieldType = (
case KBN_FIELD_TYPES.NUMBER:
schema = 'numeric';
break;
case KBN_FIELD_TYPES.NESTED:
schema = 'json';
break;
}
if (schema === undefined && field?.aggregatable === false) {

58
x-pack/plugins/ml/public/application/data_frame_analytics/common/fields.ts
View file

@ -16,7 +16,7 @@ import {
isRegressionAnalysis,
} from '../../../../common/util/analytics_utils';
import { ES_FIELD_TYPES, KBN_FIELD_TYPES } from '../../../../../../../src/plugins/data/public';
import { newJobCapsService } from '../../services/new_job_capabilities_service';
import { newJobCapsServiceAnalytics } from '../../services/new_job_capabilities/new_job_capabilities_service_analytics';
import { FEATURE_IMPORTANCE, FEATURE_INFLUENCE, OUTLIER_SCORE, TOP_CLASSES } from './constants';
import { DataFrameAnalyticsConfig } from '../../../../common/types/data_frame_analytics';
@ -54,18 +54,18 @@ export const ML__ID_COPY = 'ml__id_copy';
export const ML__INCREMENTAL_ID = 'ml__incremental_id';
export const isKeywordAndTextType = (fieldName: string): boolean => {
const { fields } = newJobCapsService;
const { fields } = newJobCapsServiceAnalytics;
const fieldType = fields.find((field) => field.name === fieldName)?.type;
let isBothTypes = false;
// If it's a keyword type - check if it has a corresponding text type
if (fieldType !== undefined && fieldType === ES_FIELD_TYPES.KEYWORD) {
const field = newJobCapsService.getFieldById(fieldName.replace(/\.keyword$/, ''));
const field = newJobCapsServiceAnalytics.getFieldById(fieldName.replace(/\.keyword$/, ''));
isBothTypes = field !== null && field.type === ES_FIELD_TYPES.TEXT;
} else if (fieldType !== undefined && fieldType === ES_FIELD_TYPES.TEXT) {
// If text, check if has corresponding keyword type
const field = newJobCapsService.getFieldById(`${fieldName}.keyword`);
const field = newJobCapsServiceAnalytics.getFieldById(`${fieldName}.keyword`);
isBothTypes = field !== null && field.type === ES_FIELD_TYPES.KEYWORD;
}
@ -180,24 +180,22 @@ export const getDefaultFieldsFromJobCaps = (
// default is 'ml'
const resultsField = jobConfig.dest.results_field;
const featureImportanceFields = [];
const topClassesFields = [];
const allFields: any = [];
let type: ES_FIELD_TYPES | undefined;
let predictedField: string | undefined;
if (isOutlierAnalysis(jobConfig.analysis)) {
if (jobConfig.analysis.outlier_detection.compute_feature_influence) {
featureImportanceFields.push({
id: `${resultsField}.${FEATURE_INFLUENCE}`,
name: `${resultsField}.${FEATURE_INFLUENCE}`,
type: KBN_FIELD_TYPES.UNKNOWN,
});
if (!jobConfig.analysis.outlier_detection.compute_feature_influence) {
// remove all feature influence fields
fields = fields.filter(
(field) => !field.name.includes(`${resultsField}.${FEATURE_INFLUENCE}`)
);
} else {
// remove flattened feature influence fields
fields = fields.filter(
(field: any) => !field.name.includes(`${resultsField}.${FEATURE_INFLUENCE}.`)
);
}
// remove flattened feature influence fields
fields = fields.filter(
(field) => !field.name.includes(`${resultsField}.${FEATURE_INFLUENCE}.`)
);
// Only need to add these fields if we didn't use dest index pattern to get the fields
if (needsDestIndexFields === true) {
@ -211,7 +209,7 @@ export const getDefaultFieldsFromJobCaps = (
if (isClassificationAnalysis(jobConfig.analysis) || isRegressionAnalysis(jobConfig.analysis)) {
const dependentVariable = getDependentVar(jobConfig.analysis);
type = newJobCapsService.getFieldById(dependentVariable)?.type;
type = newJobCapsServiceAnalytics.getFieldById(dependentVariable)?.type;
const predictionFieldName = getPredictionFieldName(jobConfig.analysis);
const numTopFeatureImportanceValues = getNumTopFeatureImportanceValues(jobConfig.analysis);
const numTopClasses = getNumTopClasses(jobConfig.analysis);
@ -221,24 +219,24 @@ export const getDefaultFieldsFromJobCaps = (
predictionFieldName ? predictionFieldName : defaultPredictionField
}`;
if ((numTopFeatureImportanceValues ?? 0) > 0) {
featureImportanceFields.push({
id: `${resultsField}.${FEATURE_IMPORTANCE}`,
name: `${resultsField}.${FEATURE_IMPORTANCE}`,
type: KBN_FIELD_TYPES.UNKNOWN,
});
if ((numTopFeatureImportanceValues ?? 0) === 0) {
// remove all feature importance fields
fields = fields.filter(
(field: any) => !field.name.includes(`${resultsField}.${FEATURE_IMPORTANCE}`)
);
} else {
// remove flattened feature importance fields
fields = fields.filter(
(field: any) => !field.name.includes(`${resultsField}.${FEATURE_IMPORTANCE}.`)
);
}
if ((numTopClasses ?? 0) > 0) {
topClassesFields.push({
id: `${resultsField}.${TOP_CLASSES}`,
name: `${resultsField}.${TOP_CLASSES}`,
type: KBN_FIELD_TYPES.UNKNOWN,
});
if ((numTopClasses ?? 0) === 0) {
// remove all top classes fields
fields = fields.filter(
(field: any) => !field.name.includes(`${resultsField}.${TOP_CLASSES}`)
);
} else {
// remove flattened top classes fields
fields = fields.filter(
(field: any) => !field.name.includes(`${resultsField}.${TOP_CLASSES}.`)
@ -258,7 +256,7 @@ export const getDefaultFieldsFromJobCaps = (
}
}
allFields.push(...fields, ...featureImportanceFields, ...topClassesFields);
allFields.push(...fields);
allFields.sort(({ name: a }: { name: string }, { name: b }: { name: string }) =>
sortExplorationResultsFields(a, b, jobConfig)
);

4
x-pack/plugins/ml/public/application/data_frame_analytics/common/get_index_fields.ts
View file

@ -7,7 +7,7 @@
import { ES_FIELD_TYPES } from '../../../../../../../src/plugins/data/public';
import { newJobCapsService } from '../../services/new_job_capabilities_service';
import { newJobCapsServiceAnalytics } from '../../services/new_job_capabilities/new_job_capabilities_service_analytics';
import { getDefaultFieldsFromJobCaps, DataFrameAnalyticsConfig } from '../common';
@ -19,7 +19,7 @@ export const getIndexFields = (
jobConfig: DataFrameAnalyticsConfig | undefined,
needsDestIndexFields: boolean
) => {
const { fields } = newJobCapsService;
const { fields } = newJobCapsServiceAnalytics;
if (jobConfig !== undefined) {
const { selectedFields: defaultSelected, docFields } = getDefaultFieldsFromJobCaps(
fields,

4
x-pack/plugins/ml/public/application/data_frame_analytics/common/use_results_view_config.ts
View file

@ -15,7 +15,7 @@ import { extractErrorMessage } from '../../../../common/util/errors';
import { getIndexPatternIdFromName } from '../../util/index_utils';
import { ml } from '../../services/ml_api_service';
import { newJobCapsService } from '../../services/new_job_capabilities_service';
import { newJobCapsServiceAnalytics } from '../../services/new_job_capabilities/new_job_capabilities_service_analytics';
import { useMlContext } from '../../contexts/ml';
import { DataFrameAnalyticsConfig } from '../common';
@ -125,7 +125,7 @@ export const useResultsViewConfig = (jobId: string) => {
}
if (indexP !== undefined) {
await newJobCapsService.initializeFromIndexPattern(indexP, false, false);
await newJobCapsServiceAnalytics.initializeFromIndexPattern(indexP);
setJobConfig(analyticsConfigs.data_frame_analytics[0]);
setIndexPattern(indexP);
setIsInitialized(true);

4
x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step_form.tsx
View file

@ -19,7 +19,7 @@ import {
import { i18n } from '@kbn/i18n';
import { debounce, cloneDeep } from 'lodash';
import { newJobCapsService } from '../../../../../services/new_job_capabilities_service';
import { newJobCapsServiceAnalytics } from '../../../../../services/new_job_capabilities/new_job_capabilities_service_analytics';
import { useMlContext } from '../../../../../contexts/ml';
import { getCombinedRuntimeMappings } from '../../../../../components/data_grid/common';
@ -196,7 +196,7 @@ export const ConfigurationStepForm: FC<ConfigurationStepProps> = ({
const depVarOptions = [];
let depVarUpdate = formState.dependentVariable;
// Get fields and filter for supported types for job type
const { fields } = newJobCapsService;
const { fields } = newJobCapsServiceAnalytics;
let resetDependentVariable = true;
for (const field of fields) {

10
x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/supported_fields_message.tsx
View file

@ -16,7 +16,7 @@ import { OMIT_FIELDS } from '../../../../../../../common/constants/field_types';
import { BASIC_NUMERICAL_TYPES, EXTENDED_NUMERICAL_TYPES } from '../../../../common/fields';
import { CATEGORICAL_TYPES } from './form_options_validation';
import { ES_FIELD_TYPES } from '../../../../../../../../../../src/plugins/data/public';
import { newJobCapsService } from '../../../../../services/new_job_capabilities_service';
import { newJobCapsServiceAnalytics } from '../../../../../services/new_job_capabilities/new_job_capabilities_service_analytics';
import { DataFrameAnalysisConfigType } from '../../../../../../../common/types/data_frame_analytics';
const containsClassificationFieldsCb = ({ name, type }: Field) =>
@ -32,7 +32,9 @@ const containsRegressionFieldsCb = ({ name, type }: Field) =>
(BASIC_NUMERICAL_TYPES.has(type) || EXTENDED_NUMERICAL_TYPES.has(type));
const containsOutlierFieldsCb = ({ name, type }: Field) =>
!OMIT_FIELDS.includes(name) && name !== EVENT_RATE_FIELD_ID && BASIC_NUMERICAL_TYPES.has(type);
!OMIT_FIELDS.includes(name) &&
name !== EVENT_RATE_FIELD_ID &&
(BASIC_NUMERICAL_TYPES.has(type) || EXTENDED_NUMERICAL_TYPES.has(type));
const callbacks: Record<DataFrameAnalysisConfigType, (f: Field) => boolean> = {
[ANALYSIS_CONFIG_TYPE.CLASSIFICATION]: containsClassificationFieldsCb,
@ -71,7 +73,7 @@ export const SupportedFieldsMessage: FC<Props> = ({ jobType }) => {
setSourceIndexContainsSupportedFields,
] = useState<boolean>(true);
const [sourceIndexFieldsCheckFailed, setSourceIndexFieldsCheckFailed] = useState<boolean>(false);
const { fields } = newJobCapsService;
const { fields } = newJobCapsServiceAnalytics;
// Find out if index pattern contains supported fields for job type. Provides a hint in the form
// that job may not run correctly if no supported fields are found.
@ -90,8 +92,6 @@ export const SupportedFieldsMessage: FC<Props> = ({ jobType }) => {
useEffect(() => {
if (jobType !== undefined) {
setSourceIndexContainsSupportedFields(true);
setSourceIndexFieldsCheckFailed(false);
validateFields();
}
}, [jobType]);

67
x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/hooks/use_index_data.ts
View file

@ -15,6 +15,7 @@ import { IndexPattern } from '../../../../../../../../../src/plugins/data/public
import { isRuntimeMappings } from '../../../../../../common/util/runtime_field_utils';
import { RuntimeMappings } from '../../../../../../common/types/fields';
import { DEFAULT_SAMPLER_SHARD_SIZE } from '../../../../../../common/constants/field_histograms';
import { newJobCapsServiceAnalytics } from '../../../../services/new_job_capabilities/new_job_capabilities_service_analytics';
import { DataLoader } from '../../../../datavisualizer/index_based/data_loader';
@ -49,6 +50,41 @@ function getRuntimeFieldColumns(runtimeMappings: RuntimeMappings) {
});
}
function getInitialColumns(indexPattern: IndexPattern) {
const { fields } = newJobCapsServiceAnalytics;
const columns = fields.map((field: any) => {
const schema =
getDataGridSchemaFromESFieldType(field.type) || getDataGridSchemaFromKibanaFieldType(field);
return {
id: field.name,
schema,
isExpandable: schema !== 'boolean',
isRuntimeFieldColumn: false,
};
});
// Add runtime fields defined in index pattern to columns
if (indexPattern) {
const computedFields = indexPattern?.getComputedFields();
if (isRuntimeMappings(computedFields.runtimeFields)) {
Object.keys(computedFields.runtimeFields).forEach((runtimeField) => {
const schema = getDataGridSchemaFromESFieldType(
computedFields.runtimeFields[runtimeField].type
);
columns.push({
id: runtimeField,
schema,
isExpandable: schema !== 'boolean',
isRuntimeFieldColumn: true,
});
});
}
}
return columns;
}
export const useIndexData = (
indexPattern: IndexPattern,
query: Record<string, any> | undefined,
@ -58,23 +94,7 @@ export const useIndexData = (
const indexPatternFields = useMemo(() => getFieldsFromKibanaIndexPattern(indexPattern), [
indexPattern,
]);
const [columns, setColumns] = useState<MLEuiDataGridColumn[]>([
...indexPatternFields.map((id) => {
const field = indexPattern.fields.getByName(id);
const isRuntimeFieldColumn = field?.runtimeField !== undefined;
const schema = isRuntimeFieldColumn
? getDataGridSchemaFromESFieldType(field?.type as estypes.RuntimeField['type'])
: getDataGridSchemaFromKibanaFieldType(field);
return {
id,
schema,
isExpandable: schema !== 'boolean',
isRuntimeFieldColumn,
};
}),
]);
const [columns, setColumns] = useState<MLEuiDataGridColumn[]>(getInitialColumns(indexPattern));
const dataGrid = useDataGrid(columns);
const {
@ -131,18 +151,7 @@ export const useIndexData = (
...(combinedRuntimeMappings ? getRuntimeFieldColumns(combinedRuntimeMappings) : []),
]);
} else {
setColumns([
...indexPatternFields.map((id) => {
const field = indexPattern.fields.getByName(id);
const schema = getDataGridSchemaFromKibanaFieldType(field);
return {
id,
schema,
isExpandable: schema !== 'boolean',
isRuntimeFieldColumn: field?.runtimeField !== undefined,
};
}),
]);
setColumns(getInitialColumns(indexPattern));
}
setRowCount(typeof resp.hits.total === 'number' ? resp.hits.total : resp.hits.total.value);
setRowCountRelation(

2
x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/util/general.ts
View file

@ -8,7 +8,7 @@
import { i18n } from '@kbn/i18n';
import { Job, Datafeed, Detector } from '../../../../../../../common/types/anomaly_detection_jobs';
import { newJobCapsService } from '../../../../../services/new_job_capabilities_service';
import { newJobCapsService } from '../../../../../services/new_job_capabilities/new_job_capabilities_service';
import { NavigateToPath } from '../../../../../contexts/kibana';
import {
ML_JOB_AGGREGATION,

2
x-pack/plugins/ml/public/application/jobs/new_job/pages/components/datafeed_step/components/time_field/time_field.tsx
View file

@ -9,7 +9,7 @@ import React, { FC, useContext, useEffect, useState } from 'react';
import { TimeFieldSelect } from './time_field_select';
import { JobCreatorContext } from '../../../job_creator_context';
import { newJobCapsService } from '../../../../../../../services/new_job_capabilities_service';
import { newJobCapsService } from '../../../../../../../services/new_job_capabilities/new_job_capabilities_service';
import { AdvancedJobCreator } from '../../../../../common/job_creator';
import { Description } from './description';

2
x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/advanced_view/metric_selection.tsx
View file

@ -9,7 +9,7 @@ import React, { Fragment, FC, useContext, useState } from 'react';
import { JobCreatorContext } from '../../../job_creator_context';
import { AdvancedJobCreator } from '../../../../../common/job_creator';
import { newJobCapsService } from '../../../../../../../services/new_job_capabilities_service';
import { newJobCapsService } from '../../../../../../../services/new_job_capabilities/new_job_capabilities_service';
import { Aggregation, Field } from '../../../../../../../../../common/types/fields';
import { MetricSelector } from './metric_selector';
import { RichDetector } from '../../../../../common/job_creator/advanced_job_creator';

2
x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_field/categorization_field.tsx
View file

@ -9,7 +9,7 @@ import React, { FC, useContext, useEffect, useState } from 'react';
import { CategorizationFieldSelect } from './categorization_field_select';
import { JobCreatorContext } from '../../../job_creator_context';
import { newJobCapsService } from '../../../../../../../services/new_job_capabilities_service';
import { newJobCapsService } from '../../../../../../../services/new_job_capabilities/new_job_capabilities_service';
import {
AdvancedJobCreator,
CategorizationJobCreator,

2
x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_partition_field/categorization_per_partition_dropdown.tsx
View file

@ -10,7 +10,7 @@ import { EuiFormRow } from '@elastic/eui';
import { FormattedMessage } from '@kbn/i18n/react';
import { JobCreatorContext } from '../../../job_creator_context';
import { CategorizationJobCreator } from '../../../../../common/job_creator';
import { newJobCapsService } from '../../../../../../../services/new_job_capabilities_service';
import { newJobCapsService } from '../../../../../../../services/new_job_capabilities/new_job_capabilities_service';
import { CategorizationPerPartitionFieldSelect } from './categorization_per_partition_input';
export const CategorizationPerPartitionFieldDropdown = ({

2
x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/influencers/influencers.tsx
View file

@ -9,7 +9,7 @@ import React, { FC, useContext, useEffect, useState } from 'react';
import { InfluencersSelect } from './influencers_select';
import { JobCreatorContext } from '../../../job_creator_context';
import { newJobCapsService } from '../../../../../../../services/new_job_capabilities_service';
import { newJobCapsService } from '../../../../../../../services/new_job_capabilities/new_job_capabilities_service';
import {
MultiMetricJobCreator,
PopulationJobCreator,

2
x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/multi_metric_view/metric_selection.tsx
View file

@ -11,7 +11,7 @@ import { JobCreatorContext } from '../../../job_creator_context';
import { MultiMetricJobCreator } from '../../../../../common/job_creator';
import { LineChartData } from '../../../../../common/chart_loader';
import { DropDownLabel, DropDownProps } from '../agg_select';
import { newJobCapsService } from '../../../../../../../services/new_job_capabilities_service';
import { newJobCapsService } from '../../../../../../../services/new_job_capabilities/new_job_capabilities_service';
import { AggFieldPair } from '../../../../../../../../../common/types/fields';
import { sortFields } from '../../../../../../../../../common/util/fields_utils';
import { getChartSettings, defaultChartSettings } from '../../../charts/common/settings';

2
x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/population_view/metric_selection.tsx
View file

@ -12,7 +12,7 @@ import { JobCreatorContext } from '../../../job_creator_context';
import { PopulationJobCreator } from '../../../../../common/job_creator';
import { LineChartData } from '../../../../../common/chart_loader';
import { DropDownLabel, DropDownProps } from '../agg_select';
import { newJobCapsService } from '../../../../../../../services/new_job_capabilities_service';
import { newJobCapsService } from '../../../../../../../services/new_job_capabilities/new_job_capabilities_service';
import { Field, AggFieldPair } from '../../../../../../../../../common/types/fields';
import { sortFields } from '../../../../../../../../../common/util/fields_utils';
import { getChartSettings, defaultChartSettings } from '../../../charts/common/settings';

2
x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/single_metric_view/metric_selection.tsx
View file

@ -10,7 +10,7 @@ import { JobCreatorContext } from '../../../job_creator_context';
import { SingleMetricJobCreator } from '../../../../../common/job_creator';
import { LineChartData } from '../../../../../common/chart_loader';
import { AggSelect, DropDownLabel, DropDownProps, createLabel } from '../agg_select';
import { newJobCapsService } from '../../../../../../../services/new_job_capabilities_service';
import { newJobCapsService } from '../../../../../../../services/new_job_capabilities/new_job_capabilities_service';
import { AggFieldPair } from '../../../../../../../../../common/types/fields';
import { sortFields } from '../../../../../../../../../common/util/fields_utils';
import { AnomalyChart, CHART_TYPE } from '../../../charts/anomaly_chart';

2
x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/split_field/by_field.tsx
View file

@ -14,7 +14,7 @@ import { Field } from '../../../../../../../../../common/types/fields';
import {
newJobCapsService,
filterCategoryFields,
} from '../../../../../../../services/new_job_capabilities_service';
} from '../../../../../../../services/new_job_capabilities/new_job_capabilities_service';
import { MultiMetricJobCreator, PopulationJobCreator } from '../../../../../common/job_creator';
interface Props {

2
x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/split_field/split_field.tsx
View file

@ -12,7 +12,7 @@ import { JobCreatorContext } from '../../../job_creator_context';
import {
newJobCapsService,
filterCategoryFields,
} from '../../../../../../../services/new_job_capabilities_service';
} from '../../../../../../../services/new_job_capabilities/new_job_capabilities_service';
import { Description } from './description';
import {
MultiMetricJobCreator,

2
x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/summary_count_field/summary_count_field.tsx
View file

@ -9,7 +9,7 @@ import React, { FC, useContext, useEffect, useState } from 'react';
import { SummaryCountFieldSelect } from './summary_count_field_select';
import { JobCreatorContext } from '../../../job_creator_context';
import { newJobCapsService } from '../../../../../../../services/new_job_capabilities_service';
import { newJobCapsService } from '../../../../../../../services/new_job_capabilities/new_job_capabilities_service';
import {
MultiMetricJobCreator,
PopulationJobCreator,

2
x-pack/plugins/ml/public/application/jobs/new_job/pages/new_job/page.tsx
View file

@ -37,7 +37,7 @@ import { useMlContext } from '../../../../contexts/ml';
import { getTimeFilterRange } from '../../../../components/full_time_range_selector';
import { getTimeBucketsFromCache } from '../../../../util/time_buckets';
import { ExistingJobsAndGroups, mlJobService } from '../../../../services/job_service';
import { newJobCapsService } from '../../../../services/new_job_capabilities_service';
import { newJobCapsService } from '../../../../services/new_job_capabilities/new_job_capabilities_service';
import { EVENT_RATE_FIELD_ID } from '../../../../../../common/types/fields';
import { getNewJobDefaults } from '../../../../services/ml_server_info';
import { useToastNotificationService } from '../../../../services/toast_notification_service';

2
x-pack/plugins/ml/public/application/jobs/new_job/pages/new_job/wizard.tsx
View file

@ -19,7 +19,7 @@ import { JobCreatorType } from '../../common/job_creator';
import { ChartLoader } from '../../common/chart_loader';
import { ResultsLoader } from '../../common/results_loader';
import { JobValidator } from '../../common/job_validator';
import { newJobCapsService } from '../../../../services/new_job_capabilities_service';
import { newJobCapsService } from '../../../../services/new_job_capabilities/new_job_capabilities_service';
import { WizardSteps } from './wizard_steps';
import { WizardHorizontalSteps } from './wizard_horizontal_steps';
import { JOB_TYPE } from '../../../../../../common/constants/new_job';

8
x-pack/plugins/ml/public/application/routing/routes/data_frame_analytics/analytics_job_creation.tsx
View file

@ -17,7 +17,10 @@ import { useResolver } from '../../use_resolver';
import { basicResolvers } from '../../resolvers';
import { Page } from '../../../data_frame_analytics/pages/analytics_creation';
import { breadcrumbOnClickFactory, getBreadcrumbWithUrlForApp } from '../../breadcrumbs';
import { loadNewJobCapabilities } from '../../../services/new_job_capabilities_service';
import {
loadNewJobCapabilities,
DATA_FRAME_ANALYTICS,
} from '../../../services/new_job_capabilities/load_new_job_capabilities';
export const analyticsJobsCreationRouteFactory = (
navigateToPath: NavigateToPath,
@ -43,7 +46,8 @@ const PageWrapper: FC<PageProps> = ({ location, deps }) => {
const { context } = useResolver(index, savedSearchId, deps.config, {
...basicResolvers(deps),
jobCaps: () => loadNewJobCapabilities(index, savedSearchId, deps.indexPatterns),
analyticsFields: () =>
loadNewJobCapabilities(index, savedSearchId, deps.indexPatterns, DATA_FRAME_ANALYTICS),
});
return (

8
x-pack/plugins/ml/public/application/routing/routes/new_job/wizard.tsx
View file

@ -17,7 +17,10 @@ import { useResolver } from '../../use_resolver';
import { Page } from '../../../jobs/new_job/pages/new_job';
import { JOB_TYPE } from '../../../../../common/constants/new_job';
import { mlJobService } from '../../../services/job_service';
import { loadNewJobCapabilities } from '../../../services/new_job_capabilities_service';
import {
loadNewJobCapabilities,
ANOMALY_DETECTOR,
} from '../../../services/new_job_capabilities/load_new_job_capabilities';
import { checkCreateJobsCapabilitiesResolver } from '../../../capabilities/check_capabilities';
import { getBreadcrumbWithUrlForApp } from '../../breadcrumbs';
import { useCreateAndNavigateToMlLink } from '../../../contexts/kibana/use_create_url';
@ -137,7 +140,8 @@ const PageWrapper: FC<WizardPageProps> = ({ location, jobType, deps }) => {
const { context, results } = useResolver(index, savedSearchId, deps.config, {
...basicResolvers(deps),
privileges: () => checkCreateJobsCapabilitiesResolver(redirectToJobsManagementPage),
jobCaps: () => loadNewJobCapabilities(index, savedSearchId, deps.indexPatterns),
jobCaps: () =>
loadNewJobCapabilities(index, savedSearchId, deps.indexPatterns, ANOMALY_DETECTOR),
existingJobsAndGroups: mlJobService.getJobAndGroupIds,
});

611
x-pack/plugins/ml/public/application/services/__mocks__/nested_field_index_response.json Normal file
View file

@ -0,0 +1,611 @@
{
"nested-field-index":{
"aggs":[
{
"id":"count",
"title":"Count",
"kibanaName":"count",
"dslName":"count",
"type":"metrics",
"mlModelPlotAgg":{
"max":"max",
"min":"min"
}
},
{
"id":"high_count",
"title":"High count",
"kibanaName":"count",
"dslName":"count",
"type":"metrics",
"mlModelPlotAgg":{
"max":"max",
"min":"min"
}
},
{
"id":"low_count",
"title":"Low count",
"kibanaName":"count",
"dslName":"count",
"type":"metrics",
"mlModelPlotAgg":{
"max":"max",
"min":"min"
}
},
{
"id":"mean",
"title":"Mean",
"kibanaName":"avg",
"dslName":"avg",
"type":"metrics",
"mlModelPlotAgg":{
"max":"avg",
"min":"avg"
},
"fieldIds":[
"time_spent"
]
},
{
"id":"high_mean",
"title":"High mean",
"kibanaName":"avg",
"dslName":"avg",
"type":"metrics",
"mlModelPlotAgg":{
"max":"avg",
"min":"avg"
},
"fieldIds":[
"time_spent"
]
},
{
"id":"low_mean",
"title":"Low mean",
"kibanaName":"avg",
"dslName":"avg",
"type":"metrics",
"mlModelPlotAgg":{
"max":"avg",
"min":"avg"
},
"fieldIds":[
"time_spent"
]
},
{
"id":"sum",
"title":"Sum",
"kibanaName":"sum",
"dslName":"sum",
"type":"metrics",
"mlModelPlotAgg":{
"max":"sum",
"min":"sum"
},
"fieldIds":[
"time_spent"
]
},
{
"id":"high_sum",
"title":"High sum",
"kibanaName":"sum",
"dslName":"sum",
"type":"metrics",
"mlModelPlotAgg":{
"max":"sum",
"min":"sum"
},
"fieldIds":[
"time_spent"
]
},
{
"id":"low_sum",
"title":"Low sum",
"kibanaName":"sum",
"dslName":"sum",
"type":"metrics",
"mlModelPlotAgg":{
"max":"sum",
"min":"sum"
},
"fieldIds":[
"time_spent"
]
},
{
"id":"median",
"title":"Median",
"kibanaName":"median",
"dslName":"percentiles",
"type":"metrics",
"mlModelPlotAgg":{
"max":"max",
"min":"min"
},
"fieldIds": [
"time_spent"
]
},
{
"id":"high_median",
"title":"High median",
"kibanaName":"median",
"dslName":"percentiles",
"type":"metrics",
"mlModelPlotAgg":{
"max":"max",
"min":"min"
},
"fieldIds":[
"time_spent"
]
},
{
"id":"low_median",
"title":"Low median",
"kibanaName":"median",
"dslName":"percentiles",
"type":"metrics",
"mlModelPlotAgg":{
"max":"max",
"min":"min"
},
"fieldIds":[
"time_spent"
]
},
{
"id":"min",
"title":"Min",
"kibanaName":"min",
"dslName":"min",
"type":"metrics",
"mlModelPlotAgg":{
"max":"min",
"min":"min"
},
"fieldIds":[
"time_spent"
]
},
{
"id":"max",
"title":"Max",
"kibanaName":"max",
"dslName":"max",
"type":"metrics",
"mlModelPlotAgg":{
"max":"max",
"min":"max"
},
"fieldIds":[
"time_spent"
]
},
{
"id":"distinct_count",
"title":"Distinct count",
"kibanaName":"cardinality",
"dslName":"cardinality",
"type":"metrics",
"mlModelPlotAgg":{
"max":"max",
"min":"min"
},
"fieldIds":[
"group.keyword",
"user.first.keyword",
"user.last.keyword",
"time_spent"
]
},
{
"id":"non_zero_count",
"title":"Non zero count",
"kibanaName":null,
"dslName":null,
"type":"metrics",
"mlModelPlotAgg":{
"max":"max",
"min":"min"
}
},
{
"id":"high_non_zero_count",
"title":"High non zero count",
"kibanaName":null,
"dslName":null,
"type":"metrics",
"mlModelPlotAgg":{
"max":"max",
"min":"min"
}
},
{
"id":"low_non_zero_count",
"title":"Low non zero count",
"kibanaName":null,
"dslName":null,
"type":"metrics",
"mlModelPlotAgg":{
"max":"max",
"min":"min"
}
},
{
"id":"high_distinct_count",
"title":"High distinct count",
"kibanaName":null,
"dslName":null,
"type":"metrics",
"mlModelPlotAgg":{
"max":"max",
"min":"min"
},
"fieldIds":[
"group.keyword",
"user.first.keyword",
"user.last.keyword",
"time_spent"
]
},
{
"id":"low_distinct_count",
"title":"Low distinct count",
"kibanaName":null,
"dslName":null,
"type":"metrics",
"mlModelPlotAgg":{
"max":"max",
"min":"min"
},
"fieldIds":[
"group.keyword",
"user.first.keyword",
"user.last.keyword",
"time_spent"
]
},
{
"id":"metric",
"title":"Metric",
"kibanaName":null,
"dslName":null,
"type":"metrics",
"mlModelPlotAgg":{
"max":"max",
"min":"min"
},
"fieldIds":[
"time_spent"
]
},
{
"id":"varp",
"title":"varp",
"kibanaName":null,
"dslName":null,
"type":"metrics",
"mlModelPlotAgg":{
"max":"max",
"min":"min"
},
"fieldIds":[
"time_spent"
]
},
{
"id":"high_varp",
"title":"High varp",
"kibanaName":null,
"dslName":null,
"type":"metrics",
"mlModelPlotAgg":{
"max":"max",
"min":"min"
},
"fieldIds":[
"time_spent"
]
},
{
"id":"low_varp",
"title":"Low varp",
"kibanaName":null,
"dslName":null,
"type":"metrics",
"mlModelPlotAgg":{
"max":"max",
"min":"min"
},
"fieldIds":[
"time_spent"
]
},
{
"id":"non_null_sum",
"title":"Non null sum",
"kibanaName":null,
"dslName":null,
"type":"metrics",
"mlModelPlotAgg":{
"max":"max",
"min":"min"
},
"fieldIds":[
"time_spent"
]
},
{
"id":"high_non_null_sum",
"title":"High non null sum",
"kibanaName":null,
"dslName":null,
"type":"metrics",
"mlModelPlotAgg":{
"max":"max",
"min":"min"
},
"fieldIds":[
"time_spent"
]
},
{
"id":"low_non_null_sum",
"title":"Low non null sum",
"kibanaName":null,
"dslName":null,
"type":"metrics",
"mlModelPlotAgg":{
"max":"max",
"min":"min"
},
"fieldIds":[
"time_spent"
]
},
{
"id":"rare",
"title":"Rare",
"kibanaName":null,
"dslName":null,
"type":"metrics",
"mlModelPlotAgg":{
"max":"max",
"min":"min"
}
},
{
"id":"freq_rare",
"title":"Freq rare",
"kibanaName":null,
"dslName":null,
"type":"metrics",
"mlModelPlotAgg":{
"max":"max",
"min":"min"
}
},
{
"id":"info_content",
"title":"Info content",
"kibanaName":null,
"dslName":null,
"type":"metrics",
"mlModelPlotAgg":{
"max":"max",
"min":"min"
},
"fieldIds":[
"group",
"user.first",
"user.last",
"group.keyword",
"user.first.keyword",
"user.last.keyword",
"time_spent"
]
},
{
"id":"high_info_content",
"title":"High info content",
"kibanaName":null,
"dslName":null,
"type":"metrics",
"mlModelPlotAgg":{
"max":"max",
"min":"min"
},
"fieldIds":[
"group",
"user.first",
"user.last",
"group.keyword",
"usr.first.keyword",
"user.last.keyword",
"time_spent"
]
},
{
"id":"low_info_content",
"title":"Low info content",
"kibanaName":null,
"dslName":null,
"type":"metrics",
"mlModelPlotAgg":{
"max":"max",
"min":"min"
},
"fieldIds":[
"group",
"user.first",
"user.last",
"group.keyword",
"user.first.keyword",
"user.last.keyword",
"time_spent"
]
},
{
"id":"time_of_day",
"title":"Time of day",
"kibanaName":null,
"dslName":null,
"type":"metrics",
"mlModelPlotAgg":{
"max":"max",
"min":"min"
}
},
{
"id":"time_of_week",
"title":"Time of week",
"kibanaName":null,
"dslName":null,
"type":"metrics",
"mlModelPlotAgg":{
"max":"max",
"min":"min"
}
},
{
"id":"lat_long",
"title":"Lat long",
"kibanaName":null,
"dslName":null,
"type":"metrics",
"mlModelPlotAgg":{
"max":"max",
"min":"min"
},
"fieldIds":[
]
}
],
"fields":[
{
"id":"group",
"name":"group",
"type":"text",
"aggregatable":false,
"aggIds":[
"info_content",
"high_info_content",
"low_info_content"
]
},
{
"id":"group.keyword",
"name":"group.keyword",
"type":"keyword",
"aggregatable":true,
"aggIds":[
"distinct_count",
"high_distinct_count",
"low_distinct_count",
"info_content",
"high_info_content",
"low_info_content"
]
},
{
"id":"time_spent",
"name":"time_spent",
"type":"long",
"aggregatable":true,
"aggIds":[
"mean",
"high_mean",
"low_mean",
"sum",
"high_sum",
"low_sum",
"median",
"high_median",
"low_median",
"min",
"max",
"distinct_count",
"high_distinct_count",
"low_distinct_count",
"metric",
"varp",
"high_varp",
"low_varp",
"non_null_sum",
"high_non_null_sum",
"low_non_null_sum",
"info_content",
"high_info_content",
"low_info_content"
]
},
{
"id":"user",
"name":"user",
"type":"nested",
"aggregatable":false,
"aggIds":[
]
},
{
"id":"user.first",
"name":"user.first",
"type":"text",
"aggregatable":false,
"aggIds":[
"info_content",
"high_info_content",
"low_info_content"
]
},
{
"id":"user.first.keyword",
"name":"user.first.keyword",
"type":"keyword",
"aggregatable":true,
"aggIds":[
"distinct_count",
"high_distinct_count",
"low_distinct_count",
"info_content",
"high_info_content",
"low_info_content"
]
},
{
"id":"user.last",
"name":"user.last",
"type":"text",
"aggregatable":false,
"aggIds":[
"info_content",
"high_info_content",
"low_info_content"
]
},
{
"id":"user.last.keyword",
"name":"user.last.keyword",
"type":"keyword",
"aggregatable":true,
"aggIds":[
"distinct_count",
"high_distinct_count",
"low_distinct_count",
"info_content",
"high_info_content",
"low_info_content"
]
}
]
}
}

9
x-pack/plugins/ml/public/application/services/ml_api_service/data_frame_analytics.ts
View file

@ -15,6 +15,7 @@ import {
UpdateDataFrameAnalyticsConfig,
} from '../../data_frame_analytics/common';
import { DeepPartial } from '../../../../common/types/common';
import { NewJobCapsResponse } from '../../../../common/types/fields';
import {
DeleteDataFrameAnalyticsWithIndexStatus,
AnalyticsMapReturnType,
@ -175,4 +176,12 @@ export const dataFrameAnalytics = {
body,
});
},
newJobCapsAnalytics(indexPatternTitle: string, isRollup: boolean = false) {
const query = isRollup === true ? { rollup: true } : {};
return http<NewJobCapsResponse>({
path: `${basePath()}/data_frame/analytics/new_job_caps/${indexPatternTitle}`,
method: 'GET',
query,
});
},
};

8
x-pack/plugins/ml/public/application/services/new_job_capabilities/index.ts Normal file
View file

@ -0,0 +1,8 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export { NewJobCapabilitiesServiceBase, processTextAndKeywordFields } from './new_job_capabilities';

49
x-pack/plugins/ml/public/application/services/new_job_capabilities/load_new_job_capabilities.ts Normal file
View file

@ -0,0 +1,49 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { IIndexPattern, IndexPatternsContract } from '../../../../../../../src/plugins/data/public';
import { getIndexPatternAndSavedSearch } from '../../util/index_utils';
import { JobType } from '../../../../common/types/saved_objects';
import { newJobCapsServiceAnalytics } from '../new_job_capabilities/new_job_capabilities_service_analytics';
import { newJobCapsService } from '../new_job_capabilities/new_job_capabilities_service';
export const ANOMALY_DETECTOR = 'anomaly-detector';
export const DATA_FRAME_ANALYTICS = 'data-frame-analytics';
// called in the routing resolve block to initialize the NewJobCapabilites
// service for the corresponding job type with the currently selected index pattern
export function loadNewJobCapabilities(
indexPatternId: string,
savedSearchId: string,
indexPatterns: IndexPatternsContract,
jobType: JobType
) {
return new Promise(async (resolve, reject) => {
const serviceToUse =
jobType === ANOMALY_DETECTOR ? newJobCapsService : newJobCapsServiceAnalytics;
if (indexPatternId !== undefined) {
// index pattern is being used
const indexPattern: IIndexPattern = await indexPatterns.get(indexPatternId);
await serviceToUse.initializeFromIndexPattern(indexPattern);
resolve(serviceToUse.newJobCaps);
} else if (savedSearchId !== undefined) {
// saved search is being used
// load the index pattern from the saved search
const { indexPattern } = await getIndexPatternAndSavedSearch(savedSearchId);
if (indexPattern === null) {
// eslint-disable-next-line no-console
console.error('Cannot retrieve index pattern from saved search');
reject();
return;
}
await serviceToUse.initializeFromIndexPattern(indexPattern);
resolve(serviceToUse.newJobCaps);
} else {
reject();
}
});
}

6
x-pack/plugins/ml/public/application/services/new_job_capabilities._service.test.ts → x-pack/plugins/ml/public/application/services/new_job_capabilities/new_job_capabilities._service.test.ts
View file

@ -6,13 +6,13 @@
*/
import { newJobCapsService } from './new_job_capabilities_service';
import { IndexPattern } from '../../../../../../src/plugins/data/public';
import { IndexPattern } from '../../../../../../../src/plugins/data/public';
// there is magic happening here. starting the include name with `mock..`
// ensures it can be lazily loaded by the jest.mock function below.
import mockCloudwatchResponse from './__mocks__/cloudwatch_job_caps_response.json';
import mockCloudwatchResponse from '../__mocks__/cloudwatch_job_caps_response.json';
jest.mock('./ml_api_service', () => ({
jest.mock('../ml_api_service', () => ({
ml: {
jobs: {
newJobCaps: jest.fn(() => Promise.resolve(mockCloudwatchResponse)),

65
x-pack/plugins/ml/public/application/services/new_job_capabilities/new_job_capabilities.ts Normal file
View file

@ -0,0 +1,65 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { Aggregation, Field, NewJobCaps } from '../../../../common/types/fields';
import { ES_FIELD_TYPES } from '../../../../../../../src/plugins/data/public';
// create two lists, one removing text fields if there are keyword equivalents and vice versa
export function processTextAndKeywordFields(fields: Field[]) {
const keywordIds = fields.filter((f) => f.type === ES_FIELD_TYPES.KEYWORD).map((f) => f.id);
const textIds = fields.filter((f) => f.type === ES_FIELD_TYPES.TEXT).map((f) => f.id);
const fieldsPreferringKeyword = fields.filter(
(f) =>
f.type !== ES_FIELD_TYPES.TEXT ||
(f.type === ES_FIELD_TYPES.TEXT && keywordIds.includes(`${f.id}.keyword`) === false)
);
const fieldsPreferringText = fields.filter(
(f) =>
f.type !== ES_FIELD_TYPES.KEYWORD ||
(f.type === ES_FIELD_TYPES.KEYWORD &&
textIds.includes(f.id.replace(/\.keyword$/, '')) === false)
);
return { fieldsPreferringKeyword, fieldsPreferringText };
}
export class NewJobCapabilitiesServiceBase {
protected _fields: Field[];
protected _aggs: Aggregation[];
constructor() {
this._fields = [];
this._aggs = [];
}
public get fields(): Field[] {
return this._fields;
}
public get aggs(): Aggregation[] {
return this._aggs;
}
public get newJobCaps(): NewJobCaps {
return {
fields: this._fields,
aggs: this._aggs,
};
}
public getFieldById(id: string): Field | null {
const field = this._fields.find((f) => f.id === id);
return field === undefined ? null : field;
}
public getAggById(id: string): Aggregation | null {
const agg = this._aggs.find((f) => f.id === id);
return agg === undefined ? null : agg;
}
}

95
x-pack/plugins/ml/public/application/services/new_job_capabilities_service.ts → x-pack/plugins/ml/public/application/services/new_job_capabilities/new_job_capabilities_service.ts
View file

@ -4,68 +4,25 @@
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import {
Field,
Aggregation,
AggId,
FieldId,
NewJobCaps,
EVENT_RATE_FIELD_ID,
} from '../../../common/types/fields';
import {
ES_FIELD_TYPES,
IIndexPattern,
IndexPatternsContract,
} from '../../../../../../src/plugins/data/public';
import { ml } from './ml_api_service';
import { getIndexPatternAndSavedSearch } from '../util/index_utils';
// called in the routing resolve block to initialize the
// newJobCapsService with the currently selected index pattern
export function loadNewJobCapabilities(
indexPatternId: string,
savedSearchId: string,
indexPatterns: IndexPatternsContract
) {
return new Promise(async (resolve, reject) => {
if (indexPatternId !== undefined) {
// index pattern is being used
const indexPattern: IIndexPattern = await indexPatterns.get(indexPatternId);
await newJobCapsService.initializeFromIndexPattern(indexPattern);
resolve(newJobCapsService.newJobCaps);
} else if (savedSearchId !== undefined) {
// saved search is being used
// load the index pattern from the saved search
const { indexPattern } = await getIndexPatternAndSavedSearch(savedSearchId);
if (indexPattern === null) {
// eslint-disable-next-line no-console
console.error('Cannot retrieve index pattern from saved search');
reject();
return;
}
await newJobCapsService.initializeFromIndexPattern(indexPattern);
resolve(newJobCapsService.newJobCaps);
} else {
reject();
}
});
}
} from '../../../../common/types/fields';
import { ES_FIELD_TYPES, IIndexPattern } from '../../../../../../../src/plugins/data/public';
import { ml } from '../ml_api_service';
import { processTextAndKeywordFields, NewJobCapabilitiesServiceBase } from './new_job_capabilities';
const categoryFieldTypes = [ES_FIELD_TYPES.TEXT, ES_FIELD_TYPES.KEYWORD, ES_FIELD_TYPES.IP];
class NewJobCapsService {
private _fields: Field[] = [];
class NewJobCapsService extends NewJobCapabilitiesServiceBase {
private _catFields: Field[] = [];
private _dateFields: Field[] = [];
private _aggs: Aggregation[] = [];
private _includeEventRateField: boolean = true;
private _removeTextFields: boolean = true;
public get fields(): Field[] {
return this._fields;
}
public get catFields(): Field[] {
return this._catFields;
}
@ -74,17 +31,6 @@ class NewJobCapsService {
return this._dateFields;
}
public get aggs(): Aggregation[] {
return this._aggs;
}
public get newJobCaps(): NewJobCaps {
return {
fields: this._fields,
aggs: this._aggs,
};
}
public get categoryFields(): Field[] {
return filterCategoryFields(this._fields);
}
@ -126,16 +72,6 @@ class NewJobCapsService {
console.error('Unable to load new job capabilities', error); // eslint-disable-line no-console
}
}
public getFieldById(id: string): Field | null {
const field = this._fields.find((f) => f.id === id);
return field === undefined ? null : field;
}
public getAggById(id: string): Aggregation | null {
const agg = this._aggs.find((f) => f.id === id);
return agg === undefined ? null : agg;
}
}
// using the response from the endpoint, create the field and aggs objects
@ -231,27 +167,6 @@ function addEventRateField(aggs: Aggregation[], fields: Field[]) {
fields.splice(0, 0, eventRateField);
}
// create two lists, one removing text fields if there are keyword equivalents and vice versa
function processTextAndKeywordFields(fields: Field[]) {
const keywordIds = fields.filter((f) => f.type === ES_FIELD_TYPES.KEYWORD).map((f) => f.id);
const textIds = fields.filter((f) => f.type === ES_FIELD_TYPES.TEXT).map((f) => f.id);
const fieldsPreferringKeyword = fields.filter(
(f) =>
f.type !== ES_FIELD_TYPES.TEXT ||
(f.type === ES_FIELD_TYPES.TEXT && keywordIds.includes(`${f.id}.keyword`) === false)
);
const fieldsPreferringText = fields.filter(
(f) =>
f.type !== ES_FIELD_TYPES.KEYWORD ||
(f.type === ES_FIELD_TYPES.KEYWORD &&
textIds.includes(f.id.replace(/\.keyword$/, '')) === false)
);
return { fieldsPreferringKeyword, fieldsPreferringText };
}
export function filterCategoryFields(fields: Field[]) {
return fields.filter((f) => categoryFieldTypes.includes(f.type));
}

66
x-pack/plugins/ml/public/application/services/new_job_capabilities/new_job_capabilities_service_analytics.ts Normal file
View file

@ -0,0 +1,66 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { Field, NewJobCapsResponse } from '../../../../common/types/fields';
import { ES_FIELD_TYPES, IIndexPattern } from '../../../../../../../src/plugins/data/public';
import { processTextAndKeywordFields, NewJobCapabilitiesServiceBase } from './new_job_capabilities';
import { ml } from '../ml_api_service';
// Keep top nested field and remove all <nested_field>.* fields
export function removeNestedFieldChildren(resp: NewJobCapsResponse, indexPatternTitle: string) {
const results = resp[indexPatternTitle];
const fields: Field[] = [];
const nestedFields: Record<string, boolean> = {};
if (results !== undefined) {
results.fields.forEach((field: Field) => {
if (field.type === ES_FIELD_TYPES.NESTED && nestedFields[field.name] === undefined) {
nestedFields[field.name] = true;
fields.push(field);
}
});
if (Object.keys(nestedFields).length > 0) {
results.fields.forEach((field: Field) => {
if (field.type !== ES_FIELD_TYPES.NESTED) {
const fieldNameParts = field.name.split('.');
const rootOfField = fieldNameParts.shift();
if (rootOfField && nestedFields[rootOfField] === undefined) {
fields.push(field);
}
}
});
} else {
fields.push(...results.fields);
}
}
return fields;
}
class NewJobCapsServiceAnalytics extends NewJobCapabilitiesServiceBase {
public async initializeFromIndexPattern(indexPattern: IIndexPattern) {
try {
const resp: NewJobCapsResponse = await ml.dataFrameAnalytics.newJobCapsAnalytics(
indexPattern.title,
indexPattern.type === 'rollup'
);
const allFields = removeNestedFieldChildren(resp, indexPattern.title);
const { fieldsPreferringKeyword } = processTextAndKeywordFields(allFields);
// set the main fields list to contain fields which have been filtered to prefer
// keyword fields over text fields.
// e.g. if foo.keyword and foo exist, don't add foo to the list.
this._fields = fieldsPreferringKeyword;
} catch (error) {
console.error('Unable to load analytics index fields', error); // eslint-disable-line no-console
}
}
}
export const newJobCapsServiceAnalytics = new NewJobCapsServiceAnalytics();

33
x-pack/plugins/ml/public/application/services/new_job_capabilities/remove_nested_field_children.test.ts Normal file
View file

@ -0,0 +1,33 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { removeNestedFieldChildren } from './new_job_capabilities_service_analytics';
import { IndexPattern } from '../../../../../../../src/plugins/data/public';
// there is magic happening here. starting the include name with `mock..`
// ensures it can be lazily loaded by the jest.mock function below.
import nestedFieldIndexResponse from '../__mocks__/nested_field_index_response.json';
const indexPattern = ({
id: 'nested-field-index',
title: 'nested-field-index',
} as unknown) as IndexPattern;
describe('removeNestedFieldChildren', () => {
describe('cloudwatch newJobCapsAnalytics()', () => {
it('can get job caps fields from endpoint json', async () => {
// @ts-ignore
const fields = removeNestedFieldChildren(nestedFieldIndexResponse, indexPattern.title);
const nestedField = fields.find(({ type }) => type === 'nested');
const nestedFieldRoot = nestedField?.name;
const regex = new RegExp(`^${nestedFieldRoot}\\.`, 'i');
expect(fields).toHaveLength(4);
expect(fields.some((field) => field.name.match(regex))).toBe(false);
});
});
});

11
x-pack/plugins/ml/server/models/job_service/new_job_caps/field_service.ts
View file

@ -70,7 +70,7 @@ class FieldsService {
}
// create field object from the results from _field_caps
private async createFields(): Promise<Field[]> {
private async createFields(includeNested: boolean = false): Promise<Field[]> {
const fieldCaps = await this.loadFieldCaps();
const fields: Field[] = [];
if (fieldCaps && fieldCaps.fields) {
@ -80,7 +80,10 @@ class FieldsService {
if (firstKey !== undefined) {
const field = fc[firstKey];
// add to the list of fields if the field type can be used by ML
if (supportedTypes.includes(field.type) === true && field.metadata_field !== true) {
if (
(supportedTypes.includes(field.type) === true && field.metadata_field !== true) ||
(includeNested && field.type === ES_FIELD_TYPES.NESTED)
) {
fields.push({
id: k,
name: k,
@ -101,7 +104,7 @@ class FieldsService {
// based on what is available in the rollup job
// the _indexPattern will be replaced with a comma separated list
// of index patterns from all of the rollup jobs
public async getData(): Promise<NewJobCaps> {
public async getData(includeNested: boolean = false): Promise<NewJobCaps> {
let rollupFields: RollupFields = {};
if (this._isRollup) {
@ -128,7 +131,7 @@ class FieldsService {
}
const aggs = cloneDeep([...aggregations, ...mlOnlyAggregations]);
const fields: Field[] = await this.createFields();
const fields: Field[] = await this.createFields(includeNested);
return combineFieldsAndAggs(fields, aggs, rollupFields);
}

6
x-pack/plugins/ml/server/models/job_service/new_job_caps/new_job_caps.ts
View file

@ -7,13 +7,9 @@
import { IScopedClusterClient, SavedObjectsClientContract } from 'kibana/server';
import { _DOC_COUNT } from '../../../../common/constants/field_types';
import { Aggregation, Field, NewJobCaps } from '../../../../common/types/fields';
import { Aggregation, Field, NewJobCapsResponse } from '../../../../common/types/fields';
import { fieldServiceProvider } from './field_service';
export interface NewJobCapsResponse {
[indexPattern: string]: NewJobCaps;
}
export function newJobCapsProvider(client: IScopedClusterClient) {
async function newJobCaps(
indexPattern: string,

1
x-pack/plugins/ml/server/routes/apidoc.json
View file

@ -18,6 +18,7 @@
"DeleteDataFrameAnalytics",
"JobsExist",
"GetDataFrameAnalyticsIdMap",
"AnalyticsNewJobCaps",
"ValidateDataFrameAnalytics",
"DataVisualizer",

68
x-pack/plugins/ml/server/routes/data_frame_analytics.ts
View file

@ -10,6 +10,7 @@ import { wrapError } from '../client/error_wrapper';
import { analyticsAuditMessagesProvider } from '../models/data_frame_analytics/analytics_audit_messages';
import { RouteInitialization } from '../types';
import { JOB_MAP_NODE_TYPES } from '../../common/constants/data_frame_analytics';
import { Field, Aggregation } from '../../common/types/fields';
import {
dataAnalyticsJobConfigSchema,
dataAnalyticsJobUpdateSchema,
@ -21,11 +22,14 @@ import {
deleteDataFrameAnalyticsJobSchema,
jobsExistSchema,
analyticsQuerySchema,
analyticsNewJobCapsParamsSchema,
analyticsNewJobCapsQuerySchema,
} from './schemas/data_analytics_schema';
import { GetAnalyticsMapArgs, ExtendAnalyticsMapArgs } from '../models/data_frame_analytics/types';
import { IndexPatternHandler } from '../models/data_frame_analytics/index_patterns';
import { AnalyticsManager } from '../models/data_frame_analytics/analytics_manager';
import { validateAnalyticsJob } from '../models/data_frame_analytics/validation';
import { fieldServiceProvider } from '../models/job_service/new_job_caps/field_service';
import { DeleteDataFrameAnalyticsWithIndexStatus } from '../../common/types/data_frame_analytics';
import { getAuthorizationHeader } from '../lib/request_authorization';
import type { MlClient } from '../lib/ml_client';
@ -58,6 +62,24 @@ function getExtendedMap(
return analytics.extendAnalyticsMapForAnalyticsJob(idOptions);
}
// replace the recursive field and agg references with a
// map of ids to allow it to be stringified for transportation
// over the network.
function convertForStringify(aggs: Aggregation[], fields: Field[]): void {
fields.forEach((f) => {
f.aggIds = f.aggs ? f.aggs.map((a) => a.id) : [];
delete f.aggs;
});
aggs.forEach((a) => {
if (a.fields !== undefined) {
// if the aggregation supports fields, i.e. it's fields list isn't undefined,
// create a list of field ids
a.fieldIds = a.fields.map((f) => f.id);
}
delete a.fields;
});
}
/**
* Routes for the data frame analytics
*/
@ -671,6 +693,52 @@ export function dataFrameAnalyticsRoutes({ router, mlLicense, routeGuard }: Rout
})
);
/**
* @apiGroup DataFrameAnalytics
*
* @api {get} api/data_frame/analytics/fields/:indexPattern Get index pattern fields for analytics
* @apiName AnalyticsNewJobCaps
* @apiDescription Retrieve the index fields for analytics
*/
router.get(
{
path: '/api/ml/data_frame/analytics/new_job_caps/{indexPattern}',
validate: {
params: analyticsNewJobCapsParamsSchema,
query: analyticsNewJobCapsQuerySchema,
},
options: {
tags: ['access:ml:canGetJobs'],
},
},
routeGuard.fullLicenseAPIGuard(async ({ client, request, response, context }) => {
try {
const { indexPattern } = request.params;
const isRollup = request.query.rollup === 'true';
const savedObjectsClient = context.core.savedObjects.client;
const fieldService = fieldServiceProvider(
indexPattern,
isRollup,
client,
savedObjectsClient
);
const { fields, aggs } = await fieldService.getData(true);
convertForStringify(aggs, fields);
return response.ok({
body: {
[indexPattern]: {
aggs,
fields,
},
},
});
} catch (e) {
return response.customError(wrapError(e));
}
})
);
/**
* @apiGroup DataFrameAnalytics
*

6
x-pack/plugins/ml/server/routes/schemas/data_analytics_schema.ts
View file

@ -102,3 +102,9 @@ export const jobsExistSchema = schema.object({
export const analyticsMapQuerySchema = schema.maybe(
schema.object({ treatAsRoot: schema.maybe(schema.any()), type: schema.maybe(schema.string()) })
);
export const analyticsNewJobCapsParamsSchema = schema.object({ indexPattern: schema.string() });
export const analyticsNewJobCapsQuerySchema = schema.maybe(
schema.object({ rollup: schema.maybe(schema.string()) })
);