github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 01:13:23 -04:00
Code Issues Projects Releases Packages Wiki Activity

[ML] Job import/export calendar and filter warnings (#107416)

Browse source 
* [ML] Job import/export calendar and filter warnings

* fixing translation id

* adding export callout

* fixing translation id

* translation ids

* bug in global calendar check

* code clean up based on review

* updating text

* updatiung text

* updating apidoc

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
James Gowdy 2021-08-11 18:13:48 +01:00 • committed by GitHub
parent 0d55d30c97
commit 6710a0643d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 390 additions and 35 deletions
Download patch file Download diff file Expand all files Collapse all files

24
x-pack/plugins/ml/common/types/filters.ts Normal file
View file

@ -0,0 +1,24 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export interface Filter {
filter_id: string;
description?: string;
items: string[];
}
interface FilterUsage {
jobs: string[];
detectors: string[];
}
export interface FilterStats {
filter_id: string;
description?: string;
item_count: number;
used_by?: FilterUsage;
}

187
x-pack/plugins/ml/public/application/components/import_export_jobs/export_jobs_flyout/export_job_warning_callout.tsx Normal file
View file

@ -0,0 +1,187 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import React, { FC } from 'react';
import { FormattedMessage } from '@kbn/i18n/react';
import { i18n } from '@kbn/i18n';
import { EuiCallOut, EuiText, EuiAccordion, EuiSpacer } from '@elastic/eui';
import type { JobDependencies } from './jobs_export_service';
interface Props {
jobs: JobDependencies;
}
export const ExportJobDependenciesWarningCallout: FC<Props> = ({ jobs: allJobs }) => {
const [jobs, jobsWithCalendars, jobsWithFilters] = filterJobs(allJobs);
const usingCalendars = jobsWithCalendars.length > 0;
const usingFilters = jobsWithFilters.length > 0;
if (usingCalendars === false && usingFilters === false) {
return null;
}
return (
<>
<EuiCallOut
title={getTitle(jobs, jobsWithCalendars.length, jobsWithFilters.length)}
color="warning"
>
<FormattedMessage
id="xpack.ml.importExport.exportFlyout.exportJobDependenciesWarningCallout.calendarDependencies"
defaultMessage="When you export jobs, calendars and filter lists are not included. You must create the filter lists before you import jobs; otherwise, the import fails. If you want the new jobs to continue to ignore scheduled events, you must create the calendars."
/>
<EuiSpacer />
{usingCalendars && (
<EuiAccordion
id="advancedOptions"
paddingSize="s"
aria-label={i18n.translate(
'xpack.ml.importExport.exportFlyout.exportJobDependenciesWarningCallout.jobUsingCalendarsAria',
{
defaultMessage: 'Jobs using calendars',
}
)}
buttonContent={
<FormattedMessage
id="xpack.ml.importExport.exportFlyout.exportJobDependenciesWarningCallout.jobUsingCalendarsButton"
defaultMessage="Jobs using calendars"
/>
}
>
<CalendarJobList jobs={jobsWithCalendars} />
</EuiAccordion>
)}
{usingFilters && (
<EuiAccordion
id="advancedOptions"
paddingSize="s"
aria-label={i18n.translate(
'xpack.ml.importExport.exportFlyout.exportJobDependenciesWarningCallout.jobUsingFiltersAria',
{
defaultMessage: 'Jobs using filter lists',
}
)}
buttonContent={
<FormattedMessage
id="xpack.ml.importExport.exportFlyout.exportJobDependenciesWarningCallout.jobUsingFiltersButton"
defaultMessage="Jobs using filter lists"
/>
}
>
<FilterJobList jobs={jobsWithFilters} />
</EuiAccordion>
)}
</EuiCallOut>
<EuiSpacer size="m" />
</>
);
};
const CalendarJobList: FC<{ jobs: JobDependencies }> = ({ jobs }) => (
<>
{jobs.length > 0 && (
<>
{jobs.map(({ jobId, calendarIds }) => (
<>
<EuiText size="s">
<h5>{jobId}</h5>
{calendarIds.length > 0 && (
<FormattedMessage
id="xpack.ml.importExport.exportFlyout.exportJobDependenciesWarningCallout.calendarList"
defaultMessage="{num, plural, one {calendar} other {calendars}}: {calendars}"
values={{ num: calendarIds.length, calendars: calendarIds.join(', ') }}
/>
)}
</EuiText>
<EuiSpacer size="s" />
</>
))}
</>
)}
</>
);
const FilterJobList: FC<{ jobs: JobDependencies }> = ({ jobs }) => (
<>
{jobs.length > 0 && (
<>
{jobs.map(({ jobId, filterIds }) => (
<>
<EuiText size="s">
<h5>{jobId}</h5>
{filterIds.length > 0 && (
<FormattedMessage
id="xpack.ml.importExport.exportFlyout.exportJobDependenciesWarningCallout.filterList"
defaultMessage="Filter {num, plural, one {list} other {lists}}: {filters}"
values={{ num: filterIds.length, filters: filterIds.join(', ') }}
/>
)}
</EuiText>
<EuiSpacer size="s" />
</>
))}
</>
)}
</>
);
function getTitle(jobs: JobDependencies, calendarCount: number, filterCount: number) {
if (calendarCount > 0 && filterCount === 0) {
return i18n.translate(
'xpack.ml.importExport.exportFlyout.exportJobDependenciesWarningCallout.calendarOnlyTitle',
{
defaultMessage:
'{jobCount, plural, one {# job uses} other {# jobs use}} {calendarCount, plural, one {a calendar} other {calendars}}',
values: { jobCount: jobs.length, calendarCount },
}
);
}
if (calendarCount === 0 && filterCount > 0) {
return i18n.translate(
'xpack.ml.importExport.exportFlyout.exportJobDependenciesWarningCallout.filterOnlyTitle',
{
defaultMessage:
'{jobCount, plural, one {# job uses} other {# jobs use}} {filterCount, plural, one {a filter list} other {filter lists}}',
values: { jobCount: jobs.length, filterCount },
}
);
}
return i18n.translate(
'xpack.ml.importExport.exportFlyout.exportJobDependenciesWarningCallout.filterAndCalendarTitle',
{
defaultMessage:
'{jobCount, plural, one {# job uses} other {# jobs use}} filter lists and calendars',
values: { jobCount: jobs.length },
}
);
}
function filterJobs(jobs: JobDependencies) {
return jobs.reduce(
(acc, job) => {
const usingCalendars = job.calendarIds.length > 0;
const usingFilters = job.filterIds.length > 0;
if (usingCalendars || usingFilters) {
acc[0].push(job);
if (usingCalendars) {
acc[1].push(job);
}
if (usingFilters) {
acc[2].push(job);
}
}
return acc;
},
[[], [], []] as JobDependencies[]
);
}

29
x-pack/plugins/ml/public/application/components/import_export_jobs/export_jobs_flyout/export_jobs_flyout.tsx
View file

@ -27,7 +27,9 @@ import {
} from '@elastic/eui';
import { useMlApiContext, useMlKibana } from '../../../contexts/kibana';
import { ExportJobDependenciesWarningCallout } from './export_job_warning_callout';
import { JobsExportService } from './jobs_export_service';
import type { JobDependencies } from './jobs_export_service';
import { toastNotificationServiceProvider } from '../../../services/toast_notification_service';
import type { JobType } from '../../../../../common/types/saved_objects';
@ -66,6 +68,9 @@ export const ExportJobsFlyout: FC<Props> = ({ isDisabled, currentTab }) => {
[toasts]
);
const [jobDependencies, setJobDependencies] = useState<JobDependencies>([]);
const [selectedJobDependencies, setSelectedJobDependencies] = useState<JobDependencies>([]);
useEffect(
function onFlyoutChange() {
setLoadingADJobs(true);
@ -81,6 +86,22 @@ export const ExportJobsFlyout: FC<Props> = ({ isDisabled, currentTab }) => {
.then(({ jobs }) => {
setLoadingADJobs(false);
setAdJobIds(jobs.map((j) => j.job_id));
jobsExportService
.getJobDependencies(jobs)
.then((jobDeps) => {
setJobDependencies(jobDeps);
setLoadingADJobs(false);
})
.catch((error) => {
const errorTitle = i18n.translate(
'xpack.ml.importExport.exportFlyout.calendarsError',
{
defaultMessage: 'Could not load calendars',
}
);
displayErrorToast(error, errorTitle);
});
})
.catch((error) => {
const errorTitle = i18n.translate('xpack.ml.importExport.exportFlyout.adJobsError', {
@ -88,6 +109,7 @@ export const ExportJobsFlyout: FC<Props> = ({ isDisabled, currentTab }) => {
});
displayErrorToast(error, errorTitle);
});
getDataFrameAnalytics()
.then(({ data_frame_analytics: dataFrameAnalytics }) => {
setLoadingDFAJobs(false);
@ -159,6 +181,12 @@ export const ExportJobsFlyout: FC<Props> = ({ isDisabled, currentTab }) => {
switchTab();
}, [selectedJobIds]);
useEffect(() => {
setSelectedJobDependencies(
jobDependencies.filter(({ jobId }) => selectedJobIds.includes(jobId))
);
}, [selectedJobIds]);
function switchTab() {
const jobType =
selectedJobType === 'anomaly-detector' ? 'data-frame-analytics' : 'anomaly-detector';
@ -195,6 +223,7 @@ export const ExportJobsFlyout: FC<Props> = ({ isDisabled, currentTab }) => {
</EuiTitle>
</EuiFlyoutHeader>
<EuiFlyoutBody>
<ExportJobDependenciesWarningCallout jobs={selectedJobDependencies} />
<EuiTabs size="s">
<EuiTab
isSelected={selectedJobType === 'anomaly-detector'}

82
x-pack/plugins/ml/public/application/components/import_export_jobs/export_jobs_flyout/jobs_export_service.ts
View file

@ -11,6 +11,10 @@ import type { MlApiServices } from '../../../services/ml_api_service';
import type { JobType } from '../../../../../common/types/saved_objects';
import type { Job, Datafeed } from '../../../../../common/types/anomaly_detection_jobs';
import type { DataFrameAnalyticsConfig } from '../../../../../common/types/data_frame_analytics';
import { GLOBAL_CALENDAR } from '../../../../../common/constants/calendars';
export type JobDependencies = Array<{ jobId: string; calendarIds: string[]; filterIds: string[] }>;
export type FiltersPerJob = Array<{ jobId: string; filterIds: string[] }>;
type ExportableConfigs =
| Array<
@ -51,4 +55,82 @@ export class JobsExportService {
(jobType === 'anomaly-detector' ? 'anomaly_detection' : 'data_frame_analytics') + '_jobs.json'
);
}
public async getJobDependencies(jobs: Job[]): Promise<JobDependencies> {
const calendars = await this._mlApiServices.calendars();
// create a map of all jobs in groups
const groups = jobs.reduce((acc, cur) => {
if (Array.isArray(cur.groups)) {
cur.groups.forEach((g) => {
if (acc[g] === undefined) {
acc[g] = [];
}
acc[g].push(cur.job_id);
});
}
return acc;
}, {} as Record<string, string[]>);
const isGroup = (id: string) => groups[id] !== undefined;
// create a map of all calendars in jobs
const calendarsPerJob = calendars.reduce((acc, cur) => {
cur.job_ids.forEach((jId) => {
if (jId === GLOBAL_CALENDAR) {
// add the calendar to all jobs
jobs.forEach((j) => {
if (acc[j.job_id] === undefined) {
acc[j.job_id] = [];
}
acc[j.job_id].push(cur.calendar_id);
});
} else if (isGroup(jId)) {
// add the calendar to every job in this group
groups[jId].forEach((jId2) => {
if (acc[jId2] === undefined) {
acc[jId2] = [];
}
acc[jId2].push(cur.calendar_id);
});
} else {
// add the calendar to just this job
if (acc[jId] === undefined) {
acc[jId] = [];
}
acc[jId].push(cur.calendar_id);
}
});
return acc;
}, {} as Record<string, string[]>);
// create a map of all filters in jobs,
// by extracting the filters from the job's detectors
const filtersPerJob = jobs.reduce((acc, cur) => {
if (acc[cur.job_id] === undefined) {
acc[cur.job_id] = [];
}
cur.analysis_config.detectors.forEach((d) => {
if (d.custom_rules !== undefined) {
d.custom_rules.forEach((r) => {
if (r.scope !== undefined) {
Object.values(r.scope).forEach((scope) => {
acc[cur.job_id].push(scope.filter_id);
});
}
});
}
});
return acc;
}, {} as Record<string, string[]>);
return jobs.map((j) => {
const jobId = j.job_id;
return {
jobId,
calendarIds: [...new Set(calendarsPerJob[jobId])] ?? [],
filterIds: [...new Set(filtersPerJob[jobId])] ?? [],
};
});
}
}

21
x-pack/plugins/ml/public/application/components/import_export_jobs/import_jobs_flyout/cannot_import_jobs_callout.tsx
View file

@ -64,14 +64,23 @@ const SkippedJobList: FC<{ jobs: SkippedJobs[] }> = ({ jobs }) => (
<>
{jobs.length > 0 && (
<>
{jobs.map(({ jobId, missingIndices }) => (
{jobs.map(({ jobId, missingIndices, missingFilters }) => (
<EuiText size="s">
<h5>{jobId}</h5>
<FormattedMessage
id="xpack.ml.importExport.importFlyout.cannotImportJobCallout.missingIndex"
defaultMessage="Missing index {num, plural, one {pattern} other {patterns}}: {indices}"
values={{ num: missingIndices.length, indices: missingIndices.join(',') }}
/>
{missingIndices.length > 0 && (
<FormattedMessage
id="xpack.ml.importExport.importFlyout.cannotImportJobCallout.missingIndex"
defaultMessage="Missing index {num, plural, one {pattern} other {patterns}}: {indices}"
values={{ num: missingIndices.length, indices: missingIndices.join(',') }}
/>
)}
{missingFilters.length > 0 && (
<FormattedMessage
id="xpack.ml.importExport.importFlyout.cannotImportJobCallout.missingFilters"
defaultMessage="Missing filter {num, plural, one {list} other {lists}}: {filters}"
values={{ num: missingFilters.length, filters: missingFilters.join(',') }}
/>
)}
</EuiText>
))}
</>

4
x-pack/plugins/ml/public/application/components/import_export_jobs/import_jobs_flyout/import_jobs_flyout.tsx
View file

@ -47,6 +47,7 @@ export const ImportJobsFlyout: FC<Props> = ({ isDisabled }) => {
const {
jobs: { bulkCreateJobs },
dataFrameAnalytics: { createDataFrameAnalytics },
filters: { filters: getFilters },
} = useMlApiContext();
const {
services: {
@ -130,7 +131,8 @@ export const ImportJobsFlyout: FC<Props> = ({ isDisabled }) => {
const validatedJobs = await jobImportService.validateJobs(
loadedFile.jobs,
loadedFile.jobType,
getIndexPatternTitles
getIndexPatternTitles,
getFilters
);
if (loadedFile.jobType === 'anomaly-detector') {

46
x-pack/plugins/ml/public/application/components/import_export_jobs/import_jobs_flyout/jobs_import_service.ts
View file

@ -7,6 +7,7 @@
import type { JobType } from '../../../../../common/types/saved_objects';
import type { Job, Datafeed } from '../../../../../common/types/anomaly_detection_jobs';
import type { Filter } from '../../../../../common/types/filters';
import type { DataFrameAnalyticsConfig } from '../../../data_frame_analytics/common';
export interface ImportedAdJob {
@ -33,6 +34,7 @@ export interface JobIdObject {
export interface SkippedJobs {
jobId: string;
missingIndices: string[];
missingFilters: string[];
}
function isImportedAdJobs(obj: any): obj is ImportedAdJob[] {
@ -127,17 +129,25 @@ export class JobImportService {
public async validateJobs(
jobs: ImportedAdJob[] | DataFrameAnalyticsConfig[],
type: JobType,
getIndexPatternTitles: (refresh?: boolean) => Promise<string[]>
getIndexPatternTitles: (refresh?: boolean) => Promise<string[]>,
getFilters: () => Promise<Filter[]>
) {
const existingIndexPatterns = new Set(await getIndexPatternTitles());
const existingFilters = new Set((await getFilters()).map((f) => f.filter_id));
const tempJobs: Array<{ jobId: string; destIndex?: string }> = [];
const tempSkippedJobIds: SkippedJobs[] = [];
const skippedJobs: SkippedJobs[] = [];
const commonJobs: Array<{ jobId: string; indices: string[]; destIndex?: string }> =
const commonJobs: Array<{
jobId: string;
indices: string[];
filters?: string[];
destIndex?: string;
}> =
type === 'anomaly-detector'
? (jobs as ImportedAdJob[]).map((j) => ({
jobId: j.job.job_id,
indices: j.datafeed.indices,
filters: getFilterIdsFromJob(j.job),
}))
: (jobs as DataFrameAnalyticsConfig[]).map((j) => ({
jobId: j.id,
@ -145,24 +155,46 @@ export class JobImportService {
indices: Array.isArray(j.source.index) ? j.source.index : [j.source.index],
}));
commonJobs.forEach(({ jobId, indices, destIndex }) => {
commonJobs.forEach(({ jobId, indices, filters = [], destIndex }) => {
const missingIndices = indices.filter((i) => existingIndexPatterns.has(i) === false);
if (missingIndices.length === 0) {
const missingFilters = filters.filter((i) => existingFilters.has(i) === false);
if (missingIndices.length === 0 && missingFilters.length === 0) {
tempJobs.push({
jobId,
...(type === 'data-frame-analytics' ? { destIndex } : {}),
});
} else {
tempSkippedJobIds.push({
skippedJobs.push({
jobId,
missingIndices,
missingFilters,
});
}
});
return {
jobs: tempJobs,
skippedJobs: tempSkippedJobIds,
skippedJobs,
};
}
}
function getFilterIdsFromJob(job: Job) {
const filters = new Set<string>();
job.analysis_config.detectors.forEach((d) => {
if (d.custom_rules === undefined) {
return;
}
d.custom_rules.forEach((r) => {
if (r.scope === undefined) {
return;
}
Object.values(r.scope).forEach((s) => {
filters.add(s.filter_id);
});
});
});
return [...filters];
}

11
x-pack/plugins/ml/public/application/services/ml_api_service/filters.ts
View file

@ -11,18 +11,19 @@
import { http } from '../http_service';
import { basePath } from './index';
import type { Filter, FilterStats } from '../../../../common/types/filters';
export const filters = {
filters(obj?: { filterId?: string }) {
const filterId = obj && obj.filterId ? `/${obj.filterId}` : '';
return http<any>({
return http<Filter[]>({
path: `${basePath()}/filters${filterId}`,
method: 'GET',
});
},
filtersStats() {
return http<any>({
return http<FilterStats[]>({
path: `${basePath()}/filters/_stats`,
method: 'GET',
});
@ -34,7 +35,7 @@ export const filters = {
description,
items,
});
return http<any>({
return http<Filter>({
path: `${basePath()}/filters`,
method: 'PUT',
body,
@ -48,7 +49,7 @@ export const filters = {
...(removeItems !== undefined ? { removeItems } : {}),
});
return http<any>({
return http<Filter>({
path: `${basePath()}/filters/${filterId}`,
method: 'PUT',
body,
@ -56,7 +57,7 @@ export const filters = {
},
deleteFilter(filterId: string) {
return http<any>({
return http<{ acknowledged: boolean }>({
path: `${basePath()}/filters/${filterId}`,
method: 'DELETE',
});

17
x-pack/plugins/ml/server/models/filter/filter_manager.ts
View file

@ -9,14 +9,8 @@ import { estypes } from '@elastic/elasticsearch';
import Boom from '@hapi/boom';
import type { MlClient } from '../../lib/ml_client';
// import { DetectorRule, DetectorRuleScope } from '../../../common/types/detector_rules';
import { Job } from '../../../common/types/anomaly_detection_jobs';
export interface Filter {
filter_id: string;
description?: string;
items: string[];
}
import type { Job } from '../../../common/types/anomaly_detection_jobs';
import type { Filter, FilterStats } from '../../../common/types/filters';
export interface FormFilter {
filterId: string;
@ -43,13 +37,6 @@ interface FilterUsage {
detectors: string[];
}
interface FilterStats {
filter_id: string;
description?: string;
item_count: number;
used_by: FilterUsage;
}
interface FiltersInUse {
[id: string]: FilterUsage;
}

2
x-pack/plugins/ml/server/models/filter/index.ts
View file

@ -5,4 +5,4 @@
* 2.0.
*/
export { FilterManager, Filter, FormFilter, UpdateFilter } from './filter_manager';
export { FilterManager, FormFilter, UpdateFilter } from './filter_manager';

2
x-pack/plugins/ml/server/routes/apidoc.json
View file

@ -32,6 +32,7 @@
"GetAnomalyDetectorsStats",
"GetAnomalyDetectorsStatsById",
"CloseAnomalyDetectorsJob",
"ResetAnomalyDetectorsJob",
"ValidateAnomalyDetector",
"ForecastAnomalyDetector",
"GetRecords",
@ -71,6 +72,7 @@
"ForceStartDatafeeds",
"StopDatafeeds",
"CloseJobs",
"ResetJobs",
"JobsSummary",
"JobsWithTimeRange",
"GetJobForCloning",