github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 01:38:56 -04:00
Code Issues Projects Releases Packages Wiki Activity

[ML] Refactoring anomaly detector job types (#59556) (#59655)

Browse source 
* [ML] Refactoring anomaly detector job types

* removing calendar type

* update validateCardinality

* fixing test

* updating  types in functional tests

* using state constants
This commit is contained in:
James Gowdy 2020-03-09 17:12:59 +00:00 • committed by GitHub
parent 06aed736c2
commit 331c68e049
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
63 changed files with 536 additions and 330 deletions
Download patch file Download diff file Expand all files Collapse all files

8
x-pack/legacy/plugins/ml/common/types/jobs.test.js → x-pack/legacy/plugins/ml/common/types/anomaly_detection_jobs/combined_job.test.ts
View file

@ -4,12 +4,12 @@
* you may not use this file except in compliance with the Elastic License.
*/
import jobConfigFarequote from './__mocks__/job_config_farequote';
import { isMlJob, isMlJobs } from './jobs';
// @ts-ignore importing JSON file
import jobConfigFarequote from '../__mocks__/job_config_farequote';
import { isCombinedJobWithStats } from './combined_job';
describe('Types: Jobs', () => {
test('Minimal integrity check.', () => {
expect(isMlJob(jobConfigFarequote)).toBe(true);
expect(isMlJobs([jobConfigFarequote])).toBe(true);
expect(isCombinedJobWithStats(jobConfigFarequote)).toBe(true);
});
});

15
x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/configs/combined_job.ts → x-pack/legacy/plugins/ml/common/types/anomaly_detection_jobs/combined_job.ts
View file

@ -6,14 +6,25 @@
import { cloneDeep } from 'lodash';
import { Datafeed } from './datafeed';
import { DatafeedStats } from './datafeed_stats';
import { Job } from './job';
import { JobStats } from './job_stats';
export type JobWithStats = Job & JobStats;
export type DatafeedWithStats = Datafeed & DatafeedStats;
// in older implementations of the job config, the datafeed was placed inside the job
// for convenience.
export interface CombinedJob extends Job {
calendars?: string[];
datafeed_config: Datafeed;
}
export interface CombinedJobWithStats extends JobWithStats {
calendars?: string[];
datafeed_config: DatafeedWithStats;
}
export function expandCombinedJobConfig(combinedJob: CombinedJob) {
const combinedJobClone = cloneDeep(combinedJob);
const job = combinedJobClone;
@ -22,3 +33,7 @@ export function expandCombinedJobConfig(combinedJob: CombinedJob) {
return { job, datafeed };
}
export function isCombinedJobWithStats(arg: any): arg is CombinedJobWithStats {
return typeof arg.job_id === 'string';
}

9
x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/configs/datafeed.ts → x-pack/legacy/plugins/ml/common/types/anomaly_detection_jobs/datafeed.ts
View file

@ -4,7 +4,7 @@
* you may not use this file except in compliance with the Elastic License.
*/
import { IndexPatternTitle } from '../../../../../../../common/types/kibana';
import { IndexPatternTitle } from '../kibana';
import { JobId } from './job';
export type DatafeedId = string;
@ -15,11 +15,8 @@ export interface Datafeed {
chunking_config?: ChunkingConfig;
frequency?: string;
indices: IndexPatternTitle[];
/**
* The datafeed can contain indexes and indices
*/
indexes?: IndexPatternTitle[];
job_id?: JobId;
indexes?: IndexPatternTitle[]; // The datafeed can contain indexes and indices
job_id: JobId;
query: object;
query_delay?: string;
script_fields?: object;

25
x-pack/legacy/plugins/ml/common/types/anomaly_detection_jobs/datafeed_stats.ts Normal file
View file

@ -0,0 +1,25 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { Node } from './job_stats';
import { DATAFEED_STATE } from '../../constants/states';
export interface DatafeedStats {
datafeed_id: string;
state: DATAFEED_STATE;
node: Node;
assignment_explanation: string;
timing_stats: TimingStats;
}
interface TimingStats {
job_id: string;
search_count: number;
bucket_count: number;
total_search_time_ms: number;
average_search_time_per_bucket_ms: number;
exponential_average_search_time_per_hour_ms: number;
}

3
x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/configs/index.ts → x-pack/legacy/plugins/ml/common/types/anomaly_detection_jobs/index.ts
View file

@ -5,5 +5,8 @@
*/
export * from './job';
export * from './job_stats';
export * from './datafeed';
export * from './datafeed_stats';
export * from './combined_job';
export * from './summary_job';

12
x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/configs/job.ts → x-pack/legacy/plugins/ml/common/types/anomaly_detection_jobs/job.ts
View file

@ -4,8 +4,8 @@
* you may not use this file except in compliance with the Elastic License.
*/
import { UrlConfig } from '../../../../../../../common/types/custom_urls';
import { CREATED_BY_LABEL } from '../../../../../../../common/constants/new_job';
import { UrlConfig } from '../custom_urls';
import { CREATED_BY_LABEL } from '../../constants/new_job';
export type JobId = string;
export type BucketSpan = string;
@ -29,6 +29,14 @@ export interface Job {
renormalization_window_days?: number;
results_index_name?: string;
results_retention_days?: number;
// optional properties added when the job has been created
create_time?: number;
finished_time?: number;
job_type?: 'anomaly_detector';
job_version?: string;
model_snapshot_id?: string;
deleting?: boolean;
}
export interface AnalysisConfig {

95
x-pack/legacy/plugins/ml/common/types/anomaly_detection_jobs/job_stats.ts Normal file
View file

@ -0,0 +1,95 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { JOB_STATE } from '../../constants/states';
export interface JobStats {
job_id: string;
data_counts: DataCounts;
model_size_stats: ModelSizeStats;
forecasts_stats: ForecastsStats;
state: JOB_STATE;
node: Node;
assignment_explanation: string;
open_time: string;
timing_stats: TimingStats;
}
export interface DataCounts {
job_id: string;
processed_record_count: number;
processed_field_count: number;
input_bytes: number;
input_field_count: number;
invalid_date_count: number;
missing_field_count: number;
out_of_order_timestamp_count: number;
empty_bucket_count: number;
sparse_bucket_count: number;
bucket_count: number;
earliest_record_timestamp: number;
latest_record_timestamp: number;
last_data_time: number;
input_record_count: number;
latest_empty_bucket_timestamp: number;
latest_sparse_bucket_timestamp: number;
latest_bucket_timestamp?: number; // stat added by the UI
}
export interface ModelSizeStats {
job_id: string;
result_type: string;
model_bytes: number;
model_bytes_exceeded: number;
model_bytes_memory_limit: number;
total_by_field_count: number;
total_over_field_count: number;
total_partition_field_count: number;
bucket_allocation_failures_count: number;
memory_status: 'ok' | 'soft_limit' | 'hard_limit';
categorized_doc_count: number;
total_category_count: number;
frequent_category_count: number;
rare_category_count: number;
dead_category_count: number;
categorization_status: 'ok' | 'warn';
log_time: number;
timestamp: number;
}
export interface ForecastsStats {
total: number;
forecasted_jobs: number;
memory_bytes?: any;
records?: any;
processing_time_ms?: any;
status?: any;
}
export interface Node {
id: string;
name: string;
ephemeral_id: string;
transport_address: string;
attributes: {
'transform.remote_connect'?: boolean;
'ml.machine_memory'?: number;
'xpack.installed'?: boolean;
'transform.node'?: boolean;
'ml.max_open_jobs'?: number;
};
}
interface TimingStats {
job_id: string;
bucket_count: number;
total_bucket_processing_time_ms: number;
minimum_bucket_processing_time_ms: number;
maximum_bucket_processing_time_ms: number;
average_bucket_processing_time_ms: number;
exponential_average_bucket_processing_time_ms: number;
exponential_average_bucket_processing_time_per_hour_ms: number;
}

57
x-pack/legacy/plugins/ml/common/types/anomaly_detection_jobs/summary_job.ts Normal file
View file

@ -0,0 +1,57 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { Moment } from 'moment';
import { CombinedJob, CombinedJobWithStats } from './combined_job';
export { Datafeed } from './datafeed';
export { DatafeedStats } from './datafeed_stats';
export interface MlSummaryJob {
id: string;
description: string;
groups: string[];
processed_record_count?: number;
memory_status?: string;
jobState: string;
datafeedIndices: string[];
hasDatafeed: boolean;
datafeedId: string;
datafeedState: string;
latestTimestampMs?: number;
earliestTimestampMs?: number;
latestResultsTimestampMs?: number;
fullJob?: CombinedJob;
nodeName?: string;
auditMessage?: Partial<AuditMessage>;
isSingleMetricViewerJob: boolean;
deleting?: boolean;
latestTimestampSortValue?: number;
}
export interface AuditMessage {
job_id: string;
msgTime: number;
level: number;
highestLevel: number;
highestLevelText: string;
text: string;
}
export type MlSummaryJobs = MlSummaryJob[];
export interface MlJobWithTimeRange extends CombinedJobWithStats {
timeRange: {
from: number;
to: number;
fromPx: number;
toPx: number;
fromMoment: Moment;
toMoment: Moment;
widthPx: number;
label: string;
};
}

97
x-pack/legacy/plugins/ml/common/types/jobs.ts
View file

@ -1,97 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { Moment } from 'moment';
// TS TODO: This is not yet a fully fledged representation of the job data structure,
// but it fulfills some basic TypeScript related needs.
export interface MlJob {
analysis_config: {
bucket_span: string;
detectors: object[];
influencers: string[];
};
analysis_limits: {
categorization_examples_limit: number;
model_memory_limit: string;
};
create_time: number;
custom_settings: object;
data_counts: {
earliest_record_timestamp: number;
latest_record_timestamp: number;
};
data_description: {
time_field: string;
time_format: string;
};
datafeed_config: object;
description: string;
established_model_memory: number;
finished_time: number;
job_id: string;
job_type: string;
job_version: string;
model_plot_config: object;
model_size_stats: object;
model_snapshot_id: string;
model_snapshot_min_version: string;
model_snapshot_retention_days: number;
results_index_name: string;
state: string;
}
export interface MlSummaryJob {
id: string;
description: string;
groups: string[];
processed_record_count: number;
memory_status?: string;
jobState: string;
hasDatafeed: boolean;
datafeedId?: string;
datafeedIndices: any[];
datafeedState?: string;
latestTimestampMs: number;
earliestTimestampMs?: number;
latestResultsTimestampMs: number;
isSingleMetricViewerJob: boolean;
nodeName?: string;
deleting?: boolean;
fullJob?: any;
auditMessage?: any;
latestTimestampSortValue?: number;
}
export type MlSummaryJobs = MlSummaryJob[];
export interface MlJobWithTimeRange extends MlJob {
groups: string[];
timeRange: {
from: number;
to: number;
fromPx: number;
toPx: number;
fromMoment: Moment;
toMoment: Moment;
widthPx: number;
label: string;
};
}
export function isMlJob(arg: any): arg is MlJob {
return typeof arg.job_id === 'string';
}
// eslint-disable-next-line @typescript-eslint/no-empty-interface
export interface MlJobs extends Array<MlJob> {}
export function isMlJobs(arg: any): arg is MlJobs {
if (Array.isArray(arg) === false) {
return false;
}
return arg.every((d: MlJob) => isMlJob(d));
}

2
x-pack/legacy/plugins/ml/common/types/modules.ts
View file

@ -4,7 +4,7 @@
* you may not use this file except in compliance with the Elastic License.
*/
import { SavedObjectAttributes } from 'src/core/public';
import { Datafeed, Job } from '../../public/application/jobs/new_job/common/job_creator/configs';
import { Datafeed, Job } from '../types/anomaly_detection_jobs';
export interface ModuleJob {
id: string;

2
x-pack/legacy/plugins/ml/common/util/job_utils.d.ts vendored
View file

@ -4,7 +4,7 @@
* you may not use this file except in compliance with the Elastic License.
*/
import { Job } from '../../public/application/jobs/new_job/common/job_creator/configs';
import { Job } from '../types/anomaly_detection_jobs';
export interface ValidationMessage {
id: string;

2
x-pack/legacy/plugins/ml/public/application/components/job_selector/job_select_service_utils.ts
View file

@ -9,7 +9,7 @@ import moment from 'moment';
import d3 from 'd3';
import { Dictionary } from '../../../../common/types/common';
import { MlJobWithTimeRange } from '../../../../common/types/jobs';
import { MlJobWithTimeRange } from '../../../../common/types/anomaly_detection_jobs';
export function getGroupsFromJobs(jobs: MlJobWithTimeRange[]) {
const groups: Dictionary<any> = {};

2
x-pack/legacy/plugins/ml/public/application/components/job_selector/job_selector.tsx
View file

@ -24,7 +24,7 @@ import { i18n } from '@kbn/i18n';
import { useMlKibana } from '../../contexts/kibana';
import { Dictionary } from '../../../../common/types/common';
import { MlJobWithTimeRange } from '../../../../common/types/jobs';
import { MlJobWithTimeRange } from '../../../../common/types/anomaly_detection_jobs';
import { ml } from '../../services/ml_api_service';
import { useUrlState } from '../../util/url_state';
// @ts-ignore

2
x-pack/legacy/plugins/ml/public/application/components/job_selector/use_job_selection.ts
View file

@ -10,7 +10,7 @@ import { useEffect } from 'react';
import { i18n } from '@kbn/i18n';
import { getToastNotifications } from '../../util/dependency_cache';
import { MlJobWithTimeRange } from '../../../../common/types/jobs';
import { MlJobWithTimeRange } from '../../../../common/types/anomaly_detection_jobs';
import { useUrlState } from '../../util/url_state';

2
x-pack/legacy/plugins/ml/public/application/explorer/explorer_utils.d.ts vendored
View file

@ -6,7 +6,7 @@
import { Moment } from 'moment';
import { CombinedJob } from '../jobs/new_job/common/job_creator/configs';
import { CombinedJob } from '../../../common/types/anomaly_detection_jobs';
import { TimeBucketsInterval } from '../util/time_buckets';

3
x-pack/legacy/plugins/ml/public/application/jobs/components/custom_url_editor/list.test.tsx
View file

@ -6,7 +6,8 @@
import React from 'react';
import { shallow } from 'enzyme';
import { Job } from '../../new_job/common/job_creator/configs';
import { Job } from '../../../../../common/types/anomaly_detection_jobs';
import { CustomUrlList, CustomUrlListProps } from './list';
function prepareTest(setCustomUrlsFn: jest.Mock) {

2
x-pack/legacy/plugins/ml/public/application/jobs/components/custom_url_editor/list.tsx
View file

@ -25,7 +25,7 @@ import { getTestUrl } from './utils';
import { parseInterval } from '../../../../../common/util/parse_interval';
import { TIME_RANGE_TYPE } from './constants';
import { UrlConfig, KibanaUrlConfig } from '../../../../../common/types/custom_urls';
import { Job } from '../../new_job/common/job_creator/configs';
import { Job } from '../../../../../common/types/anomaly_detection_jobs';
function isValidTimeRange(timeRange: KibanaUrlConfig['time_range']): boolean {
// Allow empty timeRange string, which gives the 'auto' behaviour.

2
x-pack/legacy/plugins/ml/public/application/jobs/components/custom_url_editor/utils.d.ts vendored
View file

@ -6,7 +6,7 @@
import { IIndexPattern } from 'src/plugins/data/common';
import { UrlConfig } from '../../../../../common/types/custom_urls';
import { Job } from '../../new_job/common/job_creator/configs';
import { Job } from '../../../../../common/types/anomaly_detection_jobs';
import { TimeRangeType } from './constants';
export interface TimeRange {

2
x-pack/legacy/plugins/ml/public/application/jobs/jobs_list/components/edit_job_flyout/tabs/custom_urls.tsx
View file

@ -35,7 +35,7 @@ import {
import { withKibana } from '../../../../../../../../../../../src/plugins/kibana_react/public';
import { loadSavedDashboards, loadIndexPatterns } from '../edit_utils';
import { openCustomUrlWindow } from '../../../../../util/custom_url_utils';
import { Job } from '../../../../new_job/common/job_creator/configs';
import { Job } from '../../../../../../../common/types/anomaly_detection_jobs';
import { UrlConfig } from '../../../../../../../common/types/custom_urls';
import { IIndexPattern } from '../../../../../../../../../../../src/plugins/data/common/index_patterns';
import { MlKibanaReactContextValue } from '../../../../../contexts/kibana';

7
x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/advanced_job_creator.ts
View file

@ -8,7 +8,12 @@ import { SavedSearchSavedObject } from '../../../../../../common/types/kibana';
import { JobCreator } from './job_creator';
import { Field, Aggregation, SplitField } from '../../../../../../common/types/fields';
import { Job, Datafeed, Detector, CustomRule } from './configs';
import {
Job,
Datafeed,
Detector,
CustomRule,
} from '../../../../../../common/types/anomaly_detection_jobs';
import { createBasicDetector } from './util/default_configs';
import { JOB_TYPE } from '../../../../../../common/constants/new_job';
import { getRichDetectors } from './util/general';

2
x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts
View file

@ -9,7 +9,7 @@ import { IndexPattern } from '../../../../../../../../../../src/plugins/data/pub
import { SavedSearchSavedObject } from '../../../../../../common/types/kibana';
import { JobCreator } from './job_creator';
import { Field, Aggregation, mlCategory } from '../../../../../../common/types/fields';
import { Job, Datafeed, Detector } from './configs';
import { Job, Datafeed, Detector } from '../../../../../../common/types/anomaly_detection_jobs';
import { createBasicDetector } from './util/default_configs';
import {
JOB_TYPE,

10
x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/job_creator.ts
View file

@ -9,7 +9,15 @@ import { UrlConfig } from '../../../../../../common/types/custom_urls';
import { IndexPatternTitle } from '../../../../../../common/types/kibana';
import { ML_JOB_AGGREGATION } from '../../../../../../common/constants/aggregation_types';
import { ES_FIELD_TYPES } from '../../../../../../../../../../src/plugins/data/public';
import { Job, Datafeed, Detector, JobId, DatafeedId, BucketSpan, CustomSettings } from './configs';
import {
Job,
Datafeed,
Detector,
JobId,
DatafeedId,
BucketSpan,
CustomSettings,
} from '../../../../../../common/types/anomaly_detection_jobs';
import { Aggregation, Field } from '../../../../../../common/types/fields';
import { createEmptyJob, createEmptyDatafeed } from './util/default_configs';
import { mlJobService } from '../../../../services/job_service';

2
x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/multi_metric_job_creator.ts
View file

@ -12,7 +12,7 @@ import {
SplitField,
AggFieldPair,
} from '../../../../../../common/types/fields';
import { Job, Datafeed, Detector } from './configs';
import { Job, Datafeed, Detector } from '../../../../../../common/types/anomaly_detection_jobs';
import { createBasicDetector } from './util/default_configs';
import {
JOB_TYPE,

2
x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/population_job_creator.ts
View file

@ -12,7 +12,7 @@ import {
SplitField,
AggFieldPair,
} from '../../../../../../common/types/fields';
import { Job, Datafeed, Detector } from './configs';
import { Job, Datafeed, Detector } from '../../../../../../common/types/anomaly_detection_jobs';
import { createBasicDetector } from './util/default_configs';
import { JOB_TYPE, CREATED_BY_LABEL } from '../../../../../../common/constants/new_job';
import { getRichDetectors } from './util/general';

7
x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/single_metric_job_creator.ts
View file

@ -8,7 +8,12 @@ import { SavedSearchSavedObject } from '../../../../../../common/types/kibana';
import { parseInterval } from '../../../../../../common/util/parse_interval';
import { JobCreator } from './job_creator';
import { Field, Aggregation, AggFieldPair } from '../../../../../../common/types/fields';
import { Job, Datafeed, Detector, BucketSpan } from './configs';
import {
Job,
Datafeed,
Detector,
BucketSpan,
} from '../../../../../../common/types/anomaly_detection_jobs';
import { createBasicDetector } from './util/default_configs';
import {
ML_JOB_AGGREGATION,

4
x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/util/default_configs.ts
View file

@ -4,10 +4,9 @@
* you may not use this file except in compliance with the Elastic License.
*/
import { Job, Datafeed } from '../configs';
import { IndexPatternTitle } from '../../../../../../../common/types/kibana';
import { Field, Aggregation, EVENT_RATE_FIELD_ID } from '../../../../../../../common/types/fields';
import { Detector } from '../configs';
import { Job, Datafeed, Detector } from '../../../../../../../common/types/anomaly_detection_jobs';
export function createEmptyJob(): Job {
return {
@ -28,6 +27,7 @@ export function createEmptyJob(): Job {
export function createEmptyDatafeed(indexPatternTitle: IndexPatternTitle): Datafeed {
return {
datafeed_id: '',
job_id: '',
indices: [indexPatternTitle],
query: {},
};

2
x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/util/general.ts
View file

@ -5,7 +5,7 @@
*/
import { i18n } from '@kbn/i18n';
import { Job, Datafeed, Detector } from '../configs';
import { Job, Datafeed, Detector } from '../../../../../../../common/types/anomaly_detection_jobs';
import { newJobCapsService } from '../../../../../services/new_job_capabilities_service';
import {
ML_JOB_AGGREGATION,

2
x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_runner/job_runner.ts
View file

@ -8,7 +8,7 @@ import { BehaviorSubject } from 'rxjs';
import { ml } from '../../../../services/ml_api_service';
import { mlJobService } from '../../../../services/job_service';
import { JobCreator } from '../job_creator';
import { DatafeedId, JobId } from '../job_creator/configs';
import { DatafeedId, JobId } from '../../../../../../common/types/anomaly_detection_jobs';
import { DATAFEED_STATE } from '../../../../../../common/constants/states';
const REFRESH_INTERVAL_MS = 100;

2
x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_validator/util.ts
View file

@ -6,7 +6,7 @@
import { i18n } from '@kbn/i18n';
import { BasicValidations } from './job_validator';
import { Job, Datafeed } from '../job_creator/configs';
import { Job, Datafeed } from '../../../../../../common/types/anomaly_detection_jobs';
import {
ALLOWED_DATA_UNITS,
JOB_ID_MAX_LENGTH,

2
x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/common/datafeed_preview_flyout/datafeed_preview_flyout.tsx
View file

@ -18,7 +18,7 @@ import {
EuiSpacer,
EuiLoadingSpinner,
} from '@elastic/eui';
import { CombinedJob } from '../../../../common/job_creator/configs';
import { CombinedJob } from '../../../../../../../../common/types/anomaly_detection_jobs';
import { MLJobEditor } from '../../../../../jobs_list/components/ml_job_editor';
import { JobCreatorContext } from '../../job_creator_context';
import { mlJobService } from '../../../../../../services/job_service';

2
x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/common/json_editor_flyout/json_editor_flyout.tsx
View file

@ -19,7 +19,7 @@ import {
EuiSpacer,
} from '@elastic/eui';
import { collapseLiteralStrings } from '../../../../../../../../shared_imports';
import { Datafeed } from '../../../../common/job_creator/configs';
import { Datafeed } from '../../../../../../../../common/types/anomaly_detection_jobs';
import { ML_EDITOR_MODE, MLJobEditor } from '../../../../../jobs_list/components/ml_job_editor';
import { isValidJson } from '../../../../../../../../common/util/validation_utils';
import { JobCreatorContext } from '../../job_creator_context';

2
x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/job_details_step/components/additional_section/components/custom_urls/custom_urls_selection.tsx
View file

@ -9,7 +9,7 @@ import { CustomUrls } from '../../../../../../../../jobs_list/components/edit_jo
import { UrlConfig } from '../../../../../../../../../../../common/types/custom_urls';
import { JobCreatorContext } from '../../../../../job_creator_context';
import { Description } from './description';
import { CombinedJob } from '../../../../../../../common/job_creator/configs';
import { CombinedJob } from '../../../../../../../../../../../common/types/anomaly_detection_jobs';
export const CustomUrlsSelection: FC = () => {
const { jobCreator, jobCreatorUpdate } = useContext(JobCreatorContext);

2
x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/advanced_view/detector_list.tsx
View file

@ -25,7 +25,7 @@ import { JobCreatorContext } from '../../../job_creator_context';
import { AdvancedJobCreator } from '../../../../../common/job_creator';
import { Validation } from '../../../../../common/job_validator';
import { detectorToString } from '../../../../../../../util/string_utils';
import { Detector } from '../../../../../common/job_creator/configs';
import { Detector } from '../../../../../../../../../common/types/anomaly_detection_jobs';
interface Props {
isActive: boolean;

2
x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/index_or_search/preconfigured_job_redirect.ts
View file

@ -7,7 +7,7 @@
import { IndexPatternsContract } from '../../../../../../../../../../src/plugins/data/public';
import { mlJobService } from '../../../../services/job_service';
import { loadIndexPatterns, getIndexPatternIdFromName } from '../../../../util/index_utils';
import { CombinedJob } from '../../common/job_creator/configs';
import { CombinedJob } from '../../../../../../common/types/anomaly_detection_jobs';
import { CREATED_BY_LABEL, JOB_TYPE } from '../../../../../../common/constants/new_job';
export async function preConfiguredJobRedirect(indexPatterns: IndexPatternsContract) {

2
x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/new_job/page.tsx
View file

@ -37,7 +37,7 @@ import { useMlContext } from '../../../../contexts/ml';
import { getTimeFilterRange } from '../../../../components/full_time_range_selector';
import { TimeBuckets } from '../../../../util/time_buckets';
import { ExistingJobsAndGroups, mlJobService } from '../../../../services/job_service';
import { expandCombinedJobConfig } from '../../common/job_creator/configs';
import { expandCombinedJobConfig } from '../../../../../../common/types/anomaly_detection_jobs';
import { newJobCapsService } from '../../../../services/new_job_capabilities_service';
import { EVENT_RATE_FIELD_ID } from '../../../../../../common/types/fields';
import { getNewJobDefaults } from '../../../../services/ml_server_info';

2
x-pack/legacy/plugins/ml/public/application/jobs/new_job/recognize/page.tsx
View file

@ -41,7 +41,7 @@ import { ModuleJobs } from './components/module_jobs';
import { checkForSavedObjects } from './resolvers';
import { JobSettingsForm, JobSettingsFormValues } from './components/job_settings_form';
import { TimeRange } from '../common/components';
import { JobId } from '../common/job_creator/configs';
import { JobId } from '../../../../../common/types/anomaly_detection_jobs';
export interface ModuleJobUI extends ModuleJob {
datafeedResult?: DatafeedResponse;

2
x-pack/legacy/plugins/ml/public/application/overview/components/anomaly_detection_panel/actions.tsx
View file

@ -9,7 +9,7 @@ import { EuiToolTip, EuiButtonEmpty } from '@elastic/eui';
import { i18n } from '@kbn/i18n';
// @ts-ignore no module file
import { getLink } from '../../../jobs/jobs_list/components/job_actions/results';
import { MlSummaryJobs } from '../../../../../common/types/jobs';
import { MlSummaryJobs } from '../../../../../common/types/anomaly_detection_jobs';
interface Props {
jobsList: MlSummaryJobs;

2
x-pack/legacy/plugins/ml/public/application/overview/components/anomaly_detection_panel/anomaly_detection_panel.tsx
View file

@ -21,7 +21,7 @@ import { AnomalyDetectionTable } from './table';
import { ml } from '../../../services/ml_api_service';
import { getGroupsFromJobs, getStatsBarData, getJobsWithTimerange } from './utils';
import { Dictionary } from '../../../../../common/types/common';
import { MlSummaryJobs, MlSummaryJob } from '../../../../../common/types/jobs';
import { MlSummaryJobs, MlSummaryJob } from '../../../../../common/types/anomaly_detection_jobs';
export type GroupsDictionary = Dictionary<Group>;

2
x-pack/legacy/plugins/ml/public/application/overview/components/anomaly_detection_panel/table.tsx
View file

@ -27,7 +27,7 @@ import { formatHumanReadableDateTimeSeconds } from '../../../util/date_utils';
import { ExplorerLink } from './actions';
import { getJobsFromGroup } from './utils';
import { GroupsDictionary, Group } from './anomaly_detection_panel';
import { MlSummaryJobs } from '../../../../../common/types/jobs';
import { MlSummaryJobs } from '../../../../../common/types/anomaly_detection_jobs';
import { StatsBar, JobStatsBarStats } from '../../../components/stats_bar';
// @ts-ignore
import { JobSelectorBadge } from '../../../components/job_selector/job_selector_badge/index';

6
x-pack/legacy/plugins/ml/public/application/overview/components/anomaly_detection_panel/utils.ts
View file

@ -7,7 +7,7 @@
import { i18n } from '@kbn/i18n';
import { JOB_STATE, DATAFEED_STATE } from '../../../../../common/constants/states';
import { Group, GroupsDictionary } from './anomaly_detection_panel';
import { MlSummaryJobs, MlSummaryJob } from '../../../../../common/types/jobs';
import { MlSummaryJobs, MlSummaryJob } from '../../../../../common/types/anomaly_detection_jobs';
export function getGroupsFromJobs(
jobs: MlSummaryJobs
@ -43,7 +43,7 @@ export function getGroupsFromJobs(
// if incoming job latest timestamp is greater than the last saved one, replace it
if (groups[g].latest_timestamp === undefined) {
groups[g].latest_timestamp = job.latestTimestampMs;
} else if (job.latestTimestampMs > groups[g].latest_timestamp) {
} else if (job.latestTimestampMs && job.latestTimestampMs > groups[g].latest_timestamp) {
groups[g].latest_timestamp = job.latestTimestampMs;
}
}
@ -53,7 +53,7 @@ export function getGroupsFromJobs(
groups.ungrouped.docs_processed += job.processed_record_count;
groups.ungrouped.jobs_in_group++;
// if incoming job latest timestamp is greater than the last saved one, replace it
if (job.latestTimestampMs > groups.ungrouped.latest_timestamp) {
if (job.latestTimestampMs && job.latestTimestampMs > groups.ungrouped.latest_timestamp) {
groups.ungrouped.latest_timestamp = job.latestTimestampMs;
}
}

2
x-pack/legacy/plugins/ml/public/application/routing/routes/explorer.tsx
View file

@ -9,7 +9,7 @@ import useObservable from 'react-use/lib/useObservable';
import { i18n } from '@kbn/i18n';
import { MlJobWithTimeRange } from '../../../../common/types/jobs';
import { MlJobWithTimeRange } from '../../../../common/types/anomaly_detection_jobs';
import { MlRoute, PageLoader, PageProps } from '../router';
import { useRefresh } from '../use_refresh';

2
x-pack/legacy/plugins/ml/public/application/routing/routes/timeseriesexplorer.tsx
View file

@ -11,7 +11,7 @@ import moment from 'moment';
import { i18n } from '@kbn/i18n';
import { MlJobWithTimeRange } from '../../../../common/types/jobs';
import { MlJobWithTimeRange } from '../../../../common/types/anomaly_detection_jobs';
import { TimeSeriesExplorer } from '../../timeseriesexplorer';
import { getDateFormatTz, TimeRangeBounds } from '../../explorer/explorer_utils';

2
x-pack/legacy/plugins/ml/public/application/services/calendar_service.ts
View file

@ -7,7 +7,7 @@
import { i18n } from '@kbn/i18n';
import { ml } from './ml_api_service';
import { Calendar, CalendarId } from '../../../common/types/calendars';
import { JobId } from '../jobs/new_job/common/job_creator/configs';
import { JobId } from '../../../common/types/anomaly_detection_jobs';
class CalendarService {
/**
* Assigns a job id to the calendar.

2
x-pack/legacy/plugins/ml/public/application/services/forecast_service.d.ts vendored
View file

@ -5,7 +5,7 @@
*/
import { Observable } from 'rxjs';
import { Job } from '../jobs/new_job/common/job_creator/configs';
import { Job } from '../../../common/types/anomaly_detection_jobs';
export interface ForecastData {
success: boolean;

2
x-pack/legacy/plugins/ml/public/application/services/job_service.d.ts vendored
View file

@ -5,7 +5,7 @@
*/
import { SearchResponse } from 'elasticsearch';
import { CombinedJob } from '../jobs/new_job/common/job_creator/configs';
import { CombinedJob } from '../../../common/types/anomaly_detection_jobs';
import { Calendar } from '../../../common/types/calendars';
export interface ExistingJobsAndGroups {

8
x-pack/legacy/plugins/ml/public/application/services/ml_api_service/index.d.ts vendored
View file

@ -11,7 +11,6 @@ import { AggFieldNamePair } from '../../../../common/types/fields';
import { Category } from '../../../../common/types/categories';
import { ExistingJobsAndGroups } from '../job_service';
import { PrivilegesResponse } from '../../../../common/types/privileges';
import { MlJobWithTimeRange, MlSummaryJobs } from '../../../../common/types/jobs';
import { MlServerDefaults, MlServerLimits } from '../ml_server_info';
import { ES_AGGREGATION } from '../../../../common/constants/aggregation_types';
import { DataFrameAnalyticsStats } from '../../data_frame_analytics/pages/analytics_management/components/analytics_list/common';
@ -21,7 +20,12 @@ import { DeepPartial } from '../../../../common/types/common';
import { PartitionFieldsDefinition } from '../results_service/result_service_rx';
import { annotations } from './annotations';
import { Calendar, CalendarId, UpdateCalendar } from '../../../../common/types/calendars';
import { CombinedJob, JobId } from '../../jobs/new_job/common/job_creator/configs';
import {
MlJobWithTimeRange,
MlSummaryJobs,
CombinedJob,
JobId,
} from '../../../../common/types/anomaly_detection_jobs';
import {
CategorizationAnalyzer,
CategoryFieldExample,

2
x-pack/legacy/plugins/ml/public/application/services/results_service/result_service_rx.ts
View file

@ -16,7 +16,7 @@ import { map } from 'rxjs/operators';
import _ from 'lodash';
import { Dictionary } from '../../../../common/types/common';
import { ML_MEDIAN_PERCENTS } from '../../../../common/util/job_utils';
import { JobId } from '../../jobs/new_job/common/job_creator/configs';
import { JobId } from '../../../../common/types/anomaly_detection_jobs';
import { ml } from '../ml_api_service';
import { ML_RESULTS_INDEX_PATTERN } from '../../../../common/constants/index_patterns';
import { CriteriaField } from './index';

4
x-pack/legacy/plugins/ml/public/application/timeseriesexplorer/components/timeseries_chart/timeseries_chart.d.ts vendored
View file

@ -7,10 +7,10 @@
import d3 from 'd3';
import { Annotation } from '../../../../../common/types/annotations';
import { MlJob } from '../../../../../common/types/jobs';
import { CombinedJob } from '../../../../../common/types/anomaly_detection_jobs';
interface Props {
selectedJob: MlJob;
selectedJob: CombinedJob;
}
interface State {

2
x-pack/legacy/plugins/ml/public/application/timeseriesexplorer/timeseries_search_service.ts
View file

@ -14,7 +14,7 @@ import { isModelPlotEnabled } from '../../../common/util/job_utils';
import { buildConfigFromDetector } from '../util/chart_config_builder';
import { mlResultsService } from '../services/results_service';
import { ModelPlotOutput } from '../services/results_service/result_service_rx';
import { Job } from '../jobs/new_job/common/job_creator/configs';
import { Job } from '../../../common/types/anomaly_detection_jobs';
function getMetricData(
job: Job,

2
x-pack/legacy/plugins/ml/public/application/timeseriesexplorer/timeseriesexplorer_utils/get_focus_data.ts
View file

@ -13,7 +13,7 @@ import {
} from '../../../../common/constants/search';
import { mlTimeSeriesSearchService } from '../timeseries_search_service';
import { mlResultsService, CriteriaField } from '../../services/results_service';
import { Job } from '../../jobs/new_job/common/job_creator/configs';
import { Job } from '../../../../common/types/anomaly_detection_jobs';
import { MAX_SCHEDULED_EVENTS, TIME_FIELD_NAME } from '../timeseriesexplorer_constants';
import {
processDataForFocusAnomalies,

2
x-pack/legacy/plugins/ml/public/application/timeseriesexplorer/timeseriesexplorer_utils/validate_job_selection.ts
View file

@ -10,7 +10,7 @@ import { i18n } from '@kbn/i18n';
import { getToastNotifications } from '../../util/dependency_cache';
import { MlJobWithTimeRange } from '../../../../common/types/jobs';
import { MlJobWithTimeRange } from '../../../../common/types/anomaly_detection_jobs';
import { getTimeRangeFromSelection } from '../../components/job_selector/job_select_service_utils';
import { mlJobService } from '../../services/job_service';

5
x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts
View file

@ -10,7 +10,7 @@ import numeral from '@elastic/numeral';
import { CallAPIOptions, RequestHandlerContext, SavedObjectsClientContract } from 'kibana/server';
import { IndexPatternAttributes } from 'src/plugins/data/server';
import { merge } from 'lodash';
import { MlJob } from '../../../../../legacy/plugins/ml/common/types/jobs';
import { CombinedJobWithStats } from '../../../../../legacy/plugins/ml/common/types/anomaly_detection_jobs';
import {
KibanaObjects,
ModuleDataFeed,
@ -29,7 +29,6 @@ import {
prefixDatafeedId,
} from '../../../../../legacy/plugins/ml/common/util/job_utils';
import { mlLog } from '../../client/log';
// @ts-ignore
import { jobServiceProvider } from '../job_service';
import { resultsServiceProvider } from '../results_service';
@ -61,7 +60,7 @@ interface RawModuleConfig {
}
interface MlJobStats {
jobs: MlJob[];
jobs: CombinedJobWithStats[];
}
interface Config {

77
x-pack/plugins/ml/server/models/job_service/datafeeds.js → x-pack/plugins/ml/server/models/job_service/datafeeds.ts
View file

@ -4,24 +4,45 @@
* you may not use this file except in compliance with the Elastic License.
*/
import { APICaller } from 'src/core/server';
import { i18n } from '@kbn/i18n';
import {
JOB_STATE,
DATAFEED_STATE,
} from '../../../../../legacy/plugins/ml/common/constants/states';
import { fillResultsWithTimeouts, isRequestTimeout } from './error_utils';
import {
Datafeed,
DatafeedStats,
} from '../../../../../legacy/plugins/ml/common/types/anomaly_detection_jobs';
export function datafeedsProvider(callWithRequest) {
async function forceStartDatafeeds(datafeedIds, start, end) {
export interface MlDatafeedsResponse {
datafeeds: Datafeed[];
count: number;
}
export interface MlDatafeedsStatsResponse {
datafeeds: DatafeedStats[];
count: number;
}
interface Results {
[id: string]: {
started: boolean;
error?: any;
};
}
export function datafeedsProvider(callAsCurrentUser: APICaller) {
async function forceStartDatafeeds(datafeedIds: string[], start: number, end: number) {
const jobIds = await getJobIdsByDatafeedId();
const doStartsCalled = datafeedIds.reduce((p, c) => {
p[c] = false;
return p;
}, {});
const doStartsCalled = datafeedIds.reduce((acc, cur) => {
acc[cur] = false;
return acc;
}, {} as { [id: string]: boolean });
const results = {};
const results: Results = {};
async function doStart(datafeedId) {
async function doStart(datafeedId: string): Promise<{ started: boolean; error?: string }> {
if (doStartsCalled[datafeedId] === false) {
doStartsCalled[datafeedId] = true;
try {
@ -30,6 +51,8 @@ export function datafeedsProvider(callWithRequest) {
} catch (error) {
return { started: false, error };
}
} else {
return { started: true };
}
}
@ -64,10 +87,10 @@ export function datafeedsProvider(callWithRequest) {
return results;
}
async function openJob(jobId) {
async function openJob(jobId: string) {
let opened = false;
try {
const resp = await callWithRequest('ml.openJob', { jobId });
const resp = await callAsCurrentUser('ml.openJob', { jobId });
opened = resp.opened;
} catch (error) {
if (error.statusCode === 409) {
@ -79,16 +102,16 @@ export function datafeedsProvider(callWithRequest) {
return opened;
}
async function startDatafeed(datafeedId, start, end) {
return callWithRequest('ml.startDatafeed', { datafeedId, start, end });
async function startDatafeed(datafeedId: string, start: number, end: number) {
return callAsCurrentUser('ml.startDatafeed', { datafeedId, start, end });
}
async function stopDatafeeds(datafeedIds) {
const results = {};
async function stopDatafeeds(datafeedIds: string[]) {
const results: Results = {};
for (const datafeedId of datafeedIds) {
try {
results[datafeedId] = await callWithRequest('ml.stopDatafeed', { datafeedId });
results[datafeedId] = await callAsCurrentUser('ml.stopDatafeed', { datafeedId });
} catch (error) {
if (isRequestTimeout(error)) {
return fillResultsWithTimeouts(results, datafeedId, datafeedIds, DATAFEED_STATE.STOPPED);
@ -99,24 +122,24 @@ export function datafeedsProvider(callWithRequest) {
return results;
}
async function forceDeleteDatafeed(datafeedId) {
return callWithRequest('ml.deleteDatafeed', { datafeedId, force: true });
async function forceDeleteDatafeed(datafeedId: string) {
return callAsCurrentUser('ml.deleteDatafeed', { datafeedId, force: true });
}
async function getDatafeedIdsByJobId() {
const datafeeds = await callWithRequest('ml.datafeeds');
return datafeeds.datafeeds.reduce((p, c) => {
p[c.job_id] = c.datafeed_id;
return p;
}, {});
const { datafeeds } = await callAsCurrentUser<MlDatafeedsResponse>('ml.datafeeds');
return datafeeds.reduce((acc, cur) => {
acc[cur.job_id] = cur.datafeed_id;
return acc;
}, {} as { [id: string]: string });
}
async function getJobIdsByDatafeedId() {
const datafeeds = await callWithRequest('ml.datafeeds');
return datafeeds.datafeeds.reduce((p, c) => {
p[c.datafeed_id] = c.job_id;
return p;
}, {});
const { datafeeds } = await callAsCurrentUser<MlDatafeedsResponse>('ml.datafeeds');
return datafeeds.reduce((acc, cur) => {
acc[cur.datafeed_id] = cur.job_id;
return acc;
}, {} as { [id: string]: string });
}
return {

31
x-pack/plugins/ml/server/models/job_service/error_utils.js → x-pack/plugins/ml/server/models/job_service/error_utils.ts
View file

@ -11,14 +11,27 @@ import {
} from '../../../../../legacy/plugins/ml/common/constants/states';
const REQUEST_TIMEOUT = 'RequestTimeout';
type ACTION_STATE = DATAFEED_STATE | JOB_STATE;
export function isRequestTimeout(error) {
export function isRequestTimeout(error: { displayName: string }) {
return error.displayName === REQUEST_TIMEOUT;
}
interface Results {
[id: string]: {
[status: string]: any;
error?: any;
};
}
// populate a results object with timeout errors
// for the ids which haven't already been set
export function fillResultsWithTimeouts(results, id, ids, status) {
export function fillResultsWithTimeouts(
results: Results,
id: string,
ids: string[],
status: ACTION_STATE
) {
const action = getAction(status);
const extra =
ids.length - Object.keys(results).length > 1
@ -49,20 +62,20 @@ export function fillResultsWithTimeouts(results, id, ids, status) {
},
};
return ids.reduce((p, c) => {
if (results[c] === undefined) {
p[c] = {
return ids.reduce((acc, cur) => {
if (results[cur] === undefined) {
acc[cur] = {
[status]: false,
error,
};
} else {
p[c] = results[c];
acc[cur] = results[cur];
}
return p;
}, {});
return acc;
}, {} as Results);
}
function getAction(status) {
function getAction(status: ACTION_STATE) {
let action = '';
if (status === DATAFEED_STATE.STARTED) {
action = 'start';

32
x-pack/plugins/ml/server/models/job_service/groups.js → x-pack/plugins/ml/server/models/job_service/groups.ts
View file

@ -4,17 +4,33 @@
* you may not use this file except in compliance with the Elastic License.
*/
import { APICaller } from 'src/core/server';
import { CalendarManager } from '../calendar';
import { GLOBAL_CALENDAR } from '../../../../../legacy/plugins/ml/common/constants/calendars';
import { Job } from '../../../../../legacy/plugins/ml/common/types/anomaly_detection_jobs';
import { MlJobsResponse } from './jobs';
export function groupsProvider(callWithRequest) {
const calMngr = new CalendarManager(callWithRequest);
interface Group {
id: string;
jobIds: string[];
calendarIds: string[];
}
interface Results {
[id: string]: {
success: boolean;
error?: any;
};
}
export function groupsProvider(callAsCurrentUser: APICaller) {
const calMngr = new CalendarManager(callAsCurrentUser);
async function getAllGroups() {
const groups = {};
const jobIds = {};
const groups: { [id: string]: Group } = {};
const jobIds: { [id: string]: undefined | null } = {};
const [{ jobs }, calendars] = await Promise.all([
callWithRequest('ml.jobs'),
callAsCurrentUser<MlJobsResponse>('ml.jobs'),
calMngr.getAllCalendars(),
]);
@ -58,12 +74,12 @@ export function groupsProvider(callWithRequest) {
return Object.keys(groups).map(g => groups[g]);
}
async function updateGroups(jobs) {
const results = {};
async function updateGroups(jobs: Job[]) {
const results: Results = {};
for (const job of jobs) {
const { job_id: jobId, groups } = job;
try {
await callWithRequest('ml.updateJob', { jobId, body: { groups } });
await callAsCurrentUser('ml.updateJob', { jobId, body: { groups } });
results[jobId] = { success: true };
} catch (error) {
results[jobId] = { success: false, error };

3
x-pack/plugins/ml/server/models/job_service/index.js → x-pack/plugins/ml/server/models/job_service/index.ts
View file

@ -4,13 +4,14 @@
* you may not use this file except in compliance with the Elastic License.
*/
import { APICaller } from 'src/core/server';
import { datafeedsProvider } from './datafeeds';
import { jobsProvider } from './jobs';
import { groupsProvider } from './groups';
import { newJobCapsProvider } from './new_job_caps';
import { newJobChartsProvider, topCategoriesProvider } from './new_job';
export function jobServiceProvider(callAsCurrentUser) {
export function jobServiceProvider(callAsCurrentUser: APICaller) {
return {
...datafeedsProvider(callAsCurrentUser),
...jobsProvider(callAsCurrentUser),

247
x-pack/plugins/ml/server/models/job_service/jobs.js → x-pack/plugins/ml/server/models/job_service/jobs.ts
View file

@ -5,34 +5,60 @@
*/
import { i18n } from '@kbn/i18n';
import { uniq } from 'lodash';
import { APICaller } from 'src/core/server';
import {
JOB_STATE,
DATAFEED_STATE,
} from '../../../../../legacy/plugins/ml/common/constants/states';
import { datafeedsProvider } from './datafeeds';
import {
MlSummaryJob,
AuditMessage,
Job,
JobStats,
DatafeedWithStats,
CombinedJobWithStats,
} from '../../../../../legacy/plugins/ml/common/types/anomaly_detection_jobs';
import { datafeedsProvider, MlDatafeedsResponse, MlDatafeedsStatsResponse } from './datafeeds';
import { jobAuditMessagesProvider } from '../job_audit_messages';
import { resultsServiceProvider } from '../results_service';
import { CalendarManager } from '../calendar';
import { CalendarManager, Calendar } from '../calendar';
import { fillResultsWithTimeouts, isRequestTimeout } from './error_utils';
import {
getLatestDataOrBucketTimestamp,
isTimeSeriesViewJob,
} from '../../../../../legacy/plugins/ml/common/util/job_utils';
import { groupsProvider } from './groups';
import { uniq } from 'lodash';
export function jobsProvider(callWithRequest) {
const { forceDeleteDatafeed, getDatafeedIdsByJobId } = datafeedsProvider(callWithRequest);
const { getAuditMessagesSummary } = jobAuditMessagesProvider(callWithRequest);
const { getLatestBucketTimestampByJob } = resultsServiceProvider(callWithRequest);
const calMngr = new CalendarManager(callWithRequest);
export interface MlJobsResponse {
jobs: Job[];
count: number;
}
async function forceDeleteJob(jobId) {
return callWithRequest('ml.deleteJob', { jobId, force: true });
export interface MlJobsStatsResponse {
jobs: JobStats[];
count: number;
}
interface Results {
[id: string]: {
[status: string]: boolean;
error?: any;
};
}
export function jobsProvider(callAsCurrentUser: APICaller) {
const { forceDeleteDatafeed, getDatafeedIdsByJobId } = datafeedsProvider(callAsCurrentUser);
const { getAuditMessagesSummary } = jobAuditMessagesProvider(callAsCurrentUser);
const { getLatestBucketTimestampByJob } = resultsServiceProvider(callAsCurrentUser);
const calMngr = new CalendarManager(callAsCurrentUser);
async function forceDeleteJob(jobId: string) {
return callAsCurrentUser('ml.deleteJob', { jobId, force: true });
}
async function deleteJobs(jobIds) {
const results = {};
async function deleteJobs(jobIds: string[]) {
const results: Results = {};
const datafeedIds = await getDatafeedIdsByJobId();
for (const jobId of jobIds) {
@ -68,11 +94,11 @@ export function jobsProvider(callWithRequest) {
return results;
}
async function closeJobs(jobIds) {
const results = {};
async function closeJobs(jobIds: string[]) {
const results: Results = {};
for (const jobId of jobIds) {
try {
await callWithRequest('ml.closeJob', { jobId });
await callAsCurrentUser('ml.closeJob', { jobId });
results[jobId] = { closed: true };
} catch (error) {
if (isRequestTimeout(error)) {
@ -88,7 +114,7 @@ export function jobsProvider(callWithRequest) {
// if the job has failed we want to attempt a force close.
// however, if we received a 409 due to the datafeed being started we should not attempt a force close.
try {
await callWithRequest('ml.closeJob', { jobId, force: true });
await callAsCurrentUser('ml.closeJob', { jobId, force: true });
results[jobId] = { closed: true };
} catch (error2) {
if (isRequestTimeout(error)) {
@ -104,14 +130,14 @@ export function jobsProvider(callWithRequest) {
return results;
}
async function jobsSummary(jobIds = []) {
const fullJobsList = await createFullJobsList();
async function jobsSummary(jobIds: string[] = []) {
const fullJobsList: CombinedJobWithStats[] = await createFullJobsList();
const fullJobsIds = fullJobsList.map(job => job.job_id);
const auditMessages = await getAuditMessagesSummary(fullJobsIds);
const auditMessagesByJob = auditMessages.reduce((p, c) => {
p[c.job_id] = c;
return p;
}, {});
const auditMessages: AuditMessage[] = await getAuditMessagesSummary(fullJobsIds);
const auditMessagesByJob = auditMessages.reduce((acc, cur) => {
acc[cur.job_id] = cur;
return acc;
}, {} as { [id: string]: AuditMessage });
const deletingStr = i18n.translate('xpack.ml.models.jobService.deletingJob', {
defaultMessage: 'deleting',
@ -122,11 +148,11 @@ export function jobsProvider(callWithRequest) {
typeof job.datafeed_config === 'object' && Object.keys(job.datafeed_config).length > 0;
const dataCounts = job.data_counts;
const tempJob = {
const tempJob: MlSummaryJob = {
id: job.job_id,
description: job.description || '',
groups: Array.isArray(job.groups) ? job.groups.sort() : [],
processed_record_count: job.data_counts.processed_record_count,
processed_record_count: job.data_counts?.processed_record_count,
memory_status: job.model_size_stats ? job.model_size_stats.memory_status : '',
jobState: job.deleting === true ? deletingStr : job.state,
hasDatafeed,
@ -135,11 +161,11 @@ export function jobsProvider(callWithRequest) {
datafeedIndices:
hasDatafeed && job.datafeed_config.indices ? job.datafeed_config.indices : [],
datafeedState: hasDatafeed && job.datafeed_config.state ? job.datafeed_config.state : '',
latestTimestampMs: dataCounts.latest_record_timestamp,
earliestTimestampMs: dataCounts.earliest_record_timestamp,
latestTimestampMs: dataCounts?.latest_record_timestamp,
earliestTimestampMs: dataCounts?.earliest_record_timestamp,
latestResultsTimestampMs: getLatestDataOrBucketTimestamp(
dataCounts.latest_record_timestamp,
dataCounts.latest_bucket_timestamp
dataCounts?.latest_record_timestamp as number,
dataCounts?.latest_bucket_timestamp as number
),
isSingleMetricViewerJob: isTimeSeriesViewJob(job),
nodeName: job.node ? job.node.name : undefined,
@ -149,7 +175,11 @@ export function jobsProvider(callWithRequest) {
tempJob.fullJob = job;
}
const auditMessage = auditMessagesByJob[tempJob.id];
if (auditMessage !== undefined && job.create_time <= auditMessage.msgTime) {
if (
auditMessage !== undefined &&
job.create_time !== undefined &&
job.create_time <= auditMessage.msgTime
) {
tempJob.auditMessage = {
level: auditMessage.highestLevel,
text: auditMessage.highestLevelText,
@ -163,19 +193,19 @@ export function jobsProvider(callWithRequest) {
async function jobsWithTimerange() {
const fullJobsList = await createFullJobsList();
const jobsMap = {};
const jobsMap: { [id: string]: string[] } = {};
const jobs = fullJobsList.map(job => {
jobsMap[job.job_id] = job.groups || [];
const hasDatafeed =
typeof job.datafeed_config === 'object' && Object.keys(job.datafeed_config).length > 0;
const timeRange = {};
const timeRange: { to?: number; from?: number } = {};
const dataCounts = job.data_counts;
if (dataCounts !== undefined) {
timeRange.to = getLatestDataOrBucketTimestamp(
dataCounts.latest_record_timestamp,
dataCounts.latest_bucket_timestamp
dataCounts.latest_record_timestamp as number,
dataCounts.latest_bucket_timestamp as number
);
timeRange.from = dataCounts.earliest_record_timestamp;
}
@ -195,56 +225,65 @@ export function jobsProvider(callWithRequest) {
return { jobs, jobsMap };
}
async function createFullJobsList(jobIds = []) {
const [JOBS, JOB_STATS, DATAFEEDS, DATAFEED_STATS, CALENDARS, BUCKET_TIMESTAMPS] = [
0,
1,
2,
3,
4,
5,
async function createFullJobsList(jobIds: string[] = []) {
const jobs: CombinedJobWithStats[] = [];
const groups: { [jobId: string]: string[] } = {};
const datafeeds: { [id: string]: DatafeedWithStats } = {};
const calendarsByJobId: { [jobId: string]: string[] } = {};
const requests: [
Promise<MlJobsResponse>,
Promise<MlJobsStatsResponse>,
Promise<MlDatafeedsResponse>,
Promise<MlDatafeedsStatsResponse>,
Promise<Calendar[]>,
Promise<{ [id: string]: number | undefined }>
] = [
jobIds.length > 0
? callAsCurrentUser<MlJobsResponse>('ml.jobs', { jobId: jobIds }) // move length check in side call
: callAsCurrentUser<MlJobsResponse>('ml.jobs'),
jobIds.length > 0
? callAsCurrentUser<MlJobsStatsResponse>('ml.jobStats', { jobId: jobIds })
: callAsCurrentUser<MlJobsStatsResponse>('ml.jobStats'),
callAsCurrentUser<MlDatafeedsResponse>('ml.datafeeds'),
callAsCurrentUser<MlDatafeedsStatsResponse>('ml.datafeedStats'),
calMngr.getAllCalendars(),
getLatestBucketTimestampByJob(),
];
const jobs = [];
const groups = {};
const datafeeds = {};
const calendarsByJobId = {};
const requests =
jobIds.length > 0
? [
callWithRequest('ml.jobs', { jobId: jobIds }),
callWithRequest('ml.jobStats', { jobId: jobIds }),
]
: [callWithRequest('ml.jobs'), callWithRequest('ml.jobStats')];
requests.push(
callWithRequest('ml.datafeeds'),
callWithRequest('ml.datafeedStats'),
calMngr.getAllCalendars(),
getLatestBucketTimestampByJob()
);
const [
jobResults,
jobStatsResults,
datafeedResults,
datafeedStatsResults,
calendarResults,
latestBucketTimestampByJob,
] = await Promise.all<
MlJobsResponse,
MlJobsStatsResponse,
MlDatafeedsResponse,
MlDatafeedsStatsResponse,
Calendar[],
{ [id: string]: number | undefined }
>(requests);
const results = await Promise.all(requests);
if (results[DATAFEEDS] && results[DATAFEEDS].datafeeds) {
results[DATAFEEDS].datafeeds.forEach(datafeed => {
if (results[DATAFEED_STATS] && results[DATAFEED_STATS].datafeeds) {
const datafeedStats = results[DATAFEED_STATS].datafeeds.find(
if (datafeedResults && datafeedResults.datafeeds) {
datafeedResults.datafeeds.forEach(datafeed => {
if (datafeedStatsResults && datafeedStatsResults.datafeeds) {
const datafeedStats = datafeedStatsResults.datafeeds.find(
ds => ds.datafeed_id === datafeed.datafeed_id
);
if (datafeedStats) {
datafeed.state = datafeedStats.state;
datafeed.timing_stats = datafeedStats.timing_stats;
datafeeds[datafeed.job_id] = { ...datafeed, ...datafeedStats };
}
}
datafeeds[datafeed.job_id] = datafeed;
});
}
// create list of jobs per group.
// used for assigning calendars to jobs when a calendar has
// only been attached to a group
if (results[JOBS] && results[JOBS].jobs) {
results[JOBS].jobs.forEach(job => {
if (jobResults && jobResults.jobs) {
jobResults.jobs.forEach(job => {
calendarsByJobId[job.job_id] = [];
if (job.groups !== undefined) {
@ -259,8 +298,8 @@ export function jobsProvider(callWithRequest) {
}
// assign calendars to jobs
if (results[CALENDARS]) {
results[CALENDARS].forEach(cal => {
if (calendarResults) {
calendarResults.forEach(cal => {
cal.job_ids.forEach(id => {
if (groups[id]) {
groups[id].forEach(jId => {
@ -285,42 +324,42 @@ export function jobsProvider(callWithRequest) {
}
// create jobs objects containing job stats, datafeeds, datafeed stats and calendars
if (results[JOBS] && results[JOBS].jobs) {
results[JOBS].jobs.forEach(job => {
job.data_counts = {};
job.model_size_stats = {};
job.datafeed_config = {};
if (jobResults && jobResults.jobs) {
jobResults.jobs.forEach(job => {
const tempJob = job as CombinedJobWithStats;
if (calendarsByJobId[job.job_id].length) {
job.calendars = calendarsByJobId[job.job_id];
if (calendarsByJobId[tempJob.job_id].length) {
tempJob.calendars = calendarsByJobId[tempJob.job_id];
}
if (results[JOB_STATS] && results[JOB_STATS].jobs) {
const jobStats = results[JOB_STATS].jobs.find(js => js.job_id === job.job_id);
if (jobStatsResults && jobStatsResults.jobs) {
const jobStats = jobStatsResults.jobs.find(js => js.job_id === tempJob.job_id);
if (jobStats !== undefined) {
job.state = jobStats.state;
job.data_counts = jobStats.data_counts;
job.model_size_stats = jobStats.model_size_stats;
tempJob.state = jobStats.state;
tempJob.data_counts = jobStats.data_counts;
tempJob.model_size_stats = jobStats.model_size_stats;
if (jobStats.node) {
job.node = jobStats.node;
tempJob.node = jobStats.node;
}
if (jobStats.open_time) {
job.open_time = jobStats.open_time;
tempJob.open_time = jobStats.open_time;
}
// Add in the timestamp of the last bucket processed for each job if available.
if (results[BUCKET_TIMESTAMPS] && results[BUCKET_TIMESTAMPS][job.job_id]) {
job.data_counts.latest_bucket_timestamp = results[BUCKET_TIMESTAMPS][job.job_id];
const latestBucketTimestamp =
latestBucketTimestampByJob && latestBucketTimestampByJob[tempJob.job_id];
if (latestBucketTimestamp) {
tempJob.data_counts.latest_bucket_timestamp = latestBucketTimestamp;
}
}
}
const datafeed = datafeeds[job.job_id];
const datafeed = datafeeds[tempJob.job_id];
if (datafeed !== undefined) {
job.datafeed_config = datafeed;
tempJob.datafeed_config = datafeed;
}
jobs.push(job);
jobs.push(tempJob);
});
}
return jobs;
@ -331,7 +370,7 @@ export function jobsProvider(callWithRequest) {
const detailed = true;
const jobIds = [];
try {
const tasksList = await callWithRequest('tasks.list', { actions, detailed });
const tasksList = await callAsCurrentUser('tasks.list', { actions, detailed });
Object.keys(tasksList.nodes).forEach(nodeId => {
const tasks = tasksList.nodes[nodeId].tasks;
Object.keys(tasks).forEach(taskId => {
@ -341,7 +380,7 @@ export function jobsProvider(callWithRequest) {
} catch (e) {
// if the user doesn't have permission to load the task list,
// use the jobs list to get the ids of deleting jobs
const { jobs } = await callWithRequest('ml.jobs');
const { jobs } = await callAsCurrentUser<MlJobsResponse>('ml.jobs');
jobIds.push(...jobs.filter(j => j.deleting === true).map(j => j.job_id));
}
return { jobIds };
@ -350,11 +389,13 @@ export function jobsProvider(callWithRequest) {
// Checks if each of the jobs in the specified list of IDs exist.
// Job IDs in supplied array may contain wildcard '*' characters
// e.g. *_low_request_rate_ecs
async function jobsExist(jobIds = []) {
async function jobsExist(jobIds: string[] = []) {
// Get the list of job IDs.
const jobsInfo = await callWithRequest('ml.jobs', { jobId: jobIds });
const jobsInfo = await callAsCurrentUser<MlJobsResponse>('ml.jobs', {
jobId: jobIds,
});
const results = {};
const results: { [id: string]: boolean } = {};
if (jobsInfo.count > 0) {
const allJobIds = jobsInfo.jobs.map(job => job.job_id);
@ -375,8 +416,8 @@ export function jobsProvider(callWithRequest) {
}
async function getAllJobAndGroupIds() {
const { getAllGroups } = groupsProvider(callWithRequest);
const jobs = await callWithRequest('ml.jobs');
const { getAllGroups } = groupsProvider(callAsCurrentUser);
const jobs = await callAsCurrentUser<MlJobsResponse>('ml.jobs');
const jobIds = jobs.jobs.map(job => job.job_id);
const groups = await getAllGroups();
const groupIds = groups.map(group => group.id);
@ -387,10 +428,10 @@ export function jobsProvider(callWithRequest) {
};
}
async function getLookBackProgress(jobId, start, end) {
async function getLookBackProgress(jobId: string, start: number, end: number) {
const datafeedId = `datafeed-${jobId}`;
const [jobStats, isRunning] = await Promise.all([
callWithRequest('ml.jobStats', { jobId: [jobId] }),
callAsCurrentUser<MlJobsStatsResponse>('ml.jobStats', { jobId: [jobId] }),
isDatafeedRunning(datafeedId),
]);
@ -408,8 +449,10 @@ export function jobsProvider(callWithRequest) {
return { progress: 0, isRunning: false, isJobClosed: true };
}
async function isDatafeedRunning(datafeedId) {
const stats = await callWithRequest('ml.datafeedStats', { datafeedId: [datafeedId] });
async function isDatafeedRunning(datafeedId: string) {
const stats = await callAsCurrentUser<MlDatafeedsStatsResponse>('ml.datafeedStats', {
datafeedId: [datafeedId],
});
if (stats.datafeeds.length) {
const state = stats.datafeeds[0].state;
return (

14
x-pack/plugins/ml/server/models/job_validation/validate_cardinality.d.ts vendored
View file

@ -5,16 +5,6 @@
*/
import { APICaller } from 'src/core/server';
import {
Job,
Datafeed,
} from '../../../../../legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/configs';
import { CombinedJob } from '../../../../../legacy/plugins/ml/common/types/anomaly_detection_jobs';
interface ValidateCardinalityConfig extends Job {
datafeed_config?: Datafeed;
}
export function validateCardinality(
callAsCurrentUser: APICaller,
job: ValidateCardinalityConfig
): any[];
export function validateCardinality(callAsCurrentUser: APICaller, job: CombinedJob): any[];

2
x-pack/plugins/ml/server/models/job_validation/validate_time_range.ts
View file

@ -7,7 +7,7 @@
import { APICaller } from 'src/core/server';
import { ES_FIELD_TYPES } from '../../../../../../src/plugins/data/server';
import { parseInterval } from '../../../../../legacy/plugins/ml/common/util/parse_interval';
import { CombinedJob } from '../../../../../legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/configs';
import { CombinedJob } from '../../../../../legacy/plugins/ml/common/types/anomaly_detection_jobs';
// @ts-ignore
import { validateJobObject } from './validate_job_object';

10
x-pack/plugins/ml/server/routes/job_service.ts
View file

@ -20,7 +20,7 @@ import {
topCategoriesSchema,
updateGroupsSchema,
} from './schemas/job_service_schema';
// @ts-ignore no declaration module
import { jobServiceProvider } from '../models/job_service';
import { categorizationExamplesProvider } from '../models/job_service/new_job';
@ -209,8 +209,7 @@ export function jobServiceRoutes({ router, mlLicense }: RouteInitialization) {
mlLicense.fullLicenseAPIGuard(async (context, request, response) => {
try {
const { jobsWithTimerange } = jobServiceProvider(context.ml!.mlClient.callAsCurrentUser);
const { dateFormatTz } = request.body;
const resp = await jobsWithTimerange(dateFormatTz);
const resp = await jobsWithTimerange();
return response.ok({
body: resp,
@ -420,10 +419,7 @@ export function jobServiceRoutes({ router, mlLicense }: RouteInitialization) {
splitFieldValue,
} = request.body;
const { newJobLineChart } = jobServiceProvider(
context.ml!.mlClient.callAsCurrentUser,
request
);
const { newJobLineChart } = jobServiceProvider(context.ml!.mlClient.callAsCurrentUser);
const resp = await newJobLineChart(
indexPatternTitle,
timeField,

2
x-pack/test/functional/apps/machine_learning/anomaly_detection/anomaly_explorer.ts
View file

@ -9,7 +9,7 @@ import { FtrProviderContext } from '../../../ftr_provider_context';
import {
Job,
Datafeed,
} from '../../../../..//legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/configs';
} from '../../../../..//legacy/plugins/ml/common/types/anomaly_detection_jobs';
const JOB_CONFIG: Job = {
job_id: `fq_multi_1_ae`,

2
x-pack/test/functional/apps/machine_learning/anomaly_detection/single_metric_viewer.ts
View file

@ -9,7 +9,7 @@ import { FtrProviderContext } from '../../../ftr_provider_context';
import {
Job,
Datafeed,
} from '../../../../..//legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/configs';
} from '../../../../..//legacy/plugins/ml/common/types/anomaly_detection_jobs';
const JOB_CONFIG: Job = {
job_id: `fq_single_1_smv`,

3
x-pack/test/functional/services/machine_learning/api.ts
View file

@ -10,8 +10,7 @@ import { FtrProviderContext } from '../../ftr_provider_context';
import { JOB_STATE, DATAFEED_STATE } from '../../../../legacy/plugins/ml/common/constants/states';
import { DATA_FRAME_TASK_STATE } from '../../../../legacy/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/analytics_list/common';
import { Job } from '../../../..//legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/configs/job';
import { Datafeed } from '../../../..//legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/configs/datafeed';
import { Job, Datafeed } from '../../../..//legacy/plugins/ml/common/types/anomaly_detection_jobs';
export type MlApi = ProvidedType<typeof MachineLearningAPIProvider>;