[6.x] [ML] Removing angular services refactor (#18654)

This commit is contained in:
James Gowdy 2018-04-30 15:39:00 +01:00 committed by GitHub
parent 641f9da5af
commit 6f30c7b905
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
67 changed files with 3568 additions and 3595 deletions

View file

@ -17,11 +17,7 @@ import 'plugins/ml/factories/listener_factory';
import 'plugins/ml/factories/state_factory';
import 'plugins/ml/lib/angular_bootstrap_patch';
import 'plugins/ml/jobs';
import 'plugins/ml/services/ml_clipboard_service';
import 'plugins/ml/services/job_service';
import 'plugins/ml/services/calendar_service';
import 'plugins/ml/services/ml_api_service';
import 'plugins/ml/services/results_service';
import 'plugins/ml/components/messagebar';
import 'plugins/ml/datavisualizer';
import 'plugins/ml/explorer';

View file

@ -27,14 +27,16 @@ import {
showTypicalForFunction,
getSeverity
} from 'plugins/ml/util/anomaly_utils';
import { getFieldTypeFromMapping } from 'plugins/ml/services/mapping_service';
import { ResultsServiceProvider } from 'plugins/ml/services/results_service';
import { JobServiceProvider } from 'plugins/ml/services/job_service';
import { FieldFormatServiceProvider } from 'plugins/ml/services/field_format_service';
import template from './anomalies_table.html';
import 'plugins/ml/components/controls';
import 'plugins/ml/components/paginated_table';
import 'plugins/ml/filters/format_value';
import 'plugins/ml/filters/metric_change_description';
import 'plugins/ml/services/job_service';
import 'plugins/ml/services/mapping_service';
import './expanded_row/expanded_row_directive';
import './influencers_cell/influencers_cell_directive';
@ -48,13 +50,10 @@ module.directive('mlAnomaliesTable', function (
$window,
$route,
timefilter,
mlJobService,
mlESMappingService,
mlResultsService,
Private,
mlAnomaliesTableService,
mlSelectIntervalService,
mlSelectSeverityService,
mlFieldFormatService,
formatValueFilter) {
return {
@ -77,6 +76,9 @@ module.directive('mlAnomaliesTable', function (
// just remove these resets.
mlSelectIntervalService.state.reset().changed();
mlSelectSeverityService.state.reset().changed();
const mlResultsService = Private(ResultsServiceProvider);
const mlJobService = Private(JobServiceProvider);
const mlFieldFormatService = Private(FieldFormatServiceProvider);
scope.momentInterval = 'second';
@ -195,7 +197,7 @@ module.directive('mlAnomaliesTable', function (
findFieldType(datafeedIndices[i]);
function findFieldType(index) {
mlESMappingService.getFieldTypeFromMapping(index, categorizationFieldName)
getFieldTypeFromMapping(index, categorizationFieldName)
.then((resp) => {
if (resp !== '') {
createAndOpenUrl(index, resp);

View file

@ -16,28 +16,28 @@ const module = uiModules.get('apps/ml');
module.service('mlConfirmModalService', function ($modal, $q) {
this.open = function (options) {
const deferred = $q.defer();
$modal.open({
template,
controller: 'MlConfirmModal',
backdrop: 'static',
keyboard: false,
size: (options.size === undefined) ? 'sm' : options.size,
resolve: {
params: function () {
return {
message: options.message,
title: options.title,
okLabel: options.okLabel,
cancelLabel: options.cancelLabel,
hideCancel: options.hideCancel,
ok: deferred.resolve,
cancel: deferred.reject,
};
return $q((resolve, reject) => {
$modal.open({
template,
controller: 'MlConfirmModal',
backdrop: 'static',
keyboard: false,
size: (options.size === undefined) ? 'sm' : options.size,
resolve: {
params: function () {
return {
message: options.message,
title: options.title,
okLabel: options.okLabel,
cancelLabel: options.cancelLabel,
hideCancel: options.hideCancel,
ok: resolve,
cancel: reject,
};
}
}
}
});
});
return deferred.promise;
};
});

View file

@ -11,7 +11,9 @@ import PropTypes from 'prop-types';
import React, { Component } from 'react';
import { RecognizedResult } from './recognized_result';
export function dataRecognizerProvider(ml) {
import { ml } from 'plugins/ml/services/ml_api_service';
export function dataRecognizerProvider() {
class DataRecognizer extends Component {
constructor(props) {

View file

@ -15,7 +15,6 @@ import _ from 'lodash';
import d3 from 'd3';
import moment from 'moment';
import 'plugins/ml/services/results_service';
import { parseInterval } from 'ui/utils/parse_interval';
import { numTicksForDateFormat } from 'plugins/ml/util/chart_utils';
import { calculateTextWidth } from 'plugins/ml/util/string_utils';

View file

@ -6,15 +6,14 @@
import moment from 'moment';
import template from './full_time_range_selector.html';
import { FieldsServiceProvider } from 'plugins/ml/services/fields_service';
import { FullTimeRangeSelectorServiceProvider } from 'plugins/ml/components/full_time_range_selector/full_time_range_selector_service';
import { uiModules } from 'ui/modules';
const module = uiModules.get('apps/ml');
module.directive('mlFullTimeRangeSelector', function (mlFullTimeRangeSelectorService) {
module.directive('mlFullTimeRangeSelector', function (Private) {
return {
restrict: 'E',
replace: true,
@ -25,33 +24,11 @@ module.directive('mlFullTimeRangeSelector', function (mlFullTimeRangeSelectorSer
query: '='
},
controller: function ($scope) {
const mlFullTimeRangeSelectorService = Private(FullTimeRangeSelectorServiceProvider);
$scope.setFullTimeRange = function () {
mlFullTimeRangeSelectorService.setFullTimeRange($scope.indexPattern, $scope.query);
};
}
};
})
.service('mlFullTimeRangeSelectorService', function (
timefilter,
Notifier,
Private) {
const notify = new Notifier();
const fieldsService = Private(FieldsServiceProvider);
// called on button press
this.setFullTimeRange = function (indexPattern, query) {
// load the earliest and latest time stamps for the index
fieldsService.getTimeFieldRange(
indexPattern.title,
indexPattern.timeFieldName,
query)
.then((resp) => {
timefilter.time.from = moment(resp.start.epoch).toISOString();
timefilter.time.to = moment(resp.end.epoch).toISOString();
})
.catch((resp) => {
notify.error(resp);
});
};
});
});

View file

@ -0,0 +1,36 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import moment from 'moment';
import { ml } from 'plugins/ml/services/ml_api_service';
export function FullTimeRangeSelectorServiceProvider(timefilter, Notifier, $q) {
const notify = new Notifier();
function setFullTimeRange(indexPattern, query) {
// load the earliest and latest time stamps for the index
$q.when(ml.getTimeFieldRange({
index: indexPattern.title,
timeFieldName: indexPattern.timeFieldName,
query
}))
.then((resp) => {
timefilter.time.from = moment(resp.start.epoch).toISOString();
timefilter.time.to = moment(resp.end.epoch).toISOString();
})
.catch((resp) => {
notify.error(resp);
});
}
return {
setFullTimeRange
};
}

View file

@ -10,11 +10,13 @@ import _ from 'lodash';
import template from './job_group_select.html';
import { JobServiceProvider } from 'plugins/ml/services/job_service';
import { CalendarServiceProvider } from 'plugins/ml/services/calendar_service';
import { InitAfterBindingsWorkaround } from 'ui/compat';
import { uiModules } from 'ui/modules';
const module = uiModules.get('apps/ml');
module.directive('mlJobGroupSelect', function (es, ml, $timeout, mlJobService, mlCalendarService) {
module.directive('mlJobGroupSelect', function (es, $timeout, Private) {
return {
restrict: 'E',
template,
@ -26,7 +28,10 @@ module.directive('mlJobGroupSelect', function (es, ml, $timeout, mlJobService, m
controllerAs: 'mlGroupSelect',
bindToController: true,
controller: class MlGroupSelectController extends InitAfterBindingsWorkaround {
initAfterBindings($scope) {
const mlJobService = Private(JobServiceProvider);
const mlCalendarService = Private(CalendarServiceProvider);
this.$scope = $scope;
this.selectedGroups = [];
this.groups = [];

View file

@ -18,17 +18,19 @@ import d3 from 'd3';
import template from './job_select_list.html';
import { isTimeSeriesViewJob } from 'plugins/ml/../common/util/job_utils';
import { JobServiceProvider } from 'plugins/ml/services/job_service';
import { uiModules } from 'ui/modules';
const module = uiModules.get('apps/ml');
module.directive('mlJobSelectList', function (mlJobService, mlJobSelectService, timefilter) {
module.directive('mlJobSelectList', function (Private, mlJobSelectService, timefilter) {
return {
restrict: 'AE',
replace: true,
transclude: true,
template,
controller: function ($scope) {
const mlJobService = Private(JobServiceProvider);
$scope.jobs = [];
$scope.groups = [];
$scope.homelessJobs = [];

View file

@ -11,10 +11,13 @@
import _ from 'lodash';
import { notify } from 'ui/notify';
import { JobServiceProvider } from 'plugins/ml/services/job_service';
import { uiModules } from 'ui/modules';
const module = uiModules.get('apps/ml');
module.service('mlJobSelectService', function ($rootScope, mlJobService, globalState) {
module.service('mlJobSelectService', function ($rootScope, Private, globalState) {
const mlJobService = Private(JobServiceProvider);
const self = this;

View file

@ -9,13 +9,15 @@
import 'ngreact';
import { JobServiceProvider } from 'plugins/ml/services/job_service';
import { uiModules } from 'ui/modules';
const module = uiModules.get('apps/ml', ['react']);
import { ValidateJob } from './validate_job_view';
module.directive('mlValidateJob', function ($injector) {
const mlJobService = $injector.get('mlJobService');
const Private = $injector.get('Private');
const mlJobService = Private(JobServiceProvider);
const reactDirective = $injector.get('reactDirective');
return reactDirective(

View file

@ -32,6 +32,7 @@ import { checkGetJobsPrivilege } from 'plugins/ml/privilege/check_privilege';
import { createSearchItems } from 'plugins/ml/jobs/new_job/utils/new_job_utils';
import { getIndexPatternWithRoute, getSavedSearchWithRoute, timeBasedIndexCheck } from 'plugins/ml/util/index_utils';
import { checkMlNodesAvailable } from 'plugins/ml/ml_nodes_check/check_ml_nodes';
import { ml } from 'plugins/ml/services/ml_api_service';
import template from './datavisualizer.html';
uiRoutes
@ -55,10 +56,10 @@ module
$route,
$timeout,
$window,
$q,
Private,
timefilter,
AppState,
ml) {
AppState) {
timefilter.enableTimeRangeSelector();
timefilter.enableAutoRefreshSelector();
@ -461,7 +462,7 @@ module
buckets.setBarTarget(BAR_TARGET);
const aggInterval = buckets.getInterval();
ml.getVisualizerFieldStats({
$q.when(ml.getVisualizerFieldStats({
indexPatternTitle: indexPattern.title,
query: $scope.searchQuery,
timeFieldName: indexPattern.timeFieldName,
@ -470,7 +471,7 @@ module
samplerShardSize: $scope.samplerShardSize,
interval: aggInterval.expression,
fields: numberFields
})
}))
.then((resp) => {
// Add the metric stats to the existing stats in the corresponding card.
_.each($scope.metricCards, (card) => {
@ -520,7 +521,7 @@ module
if (fields.length > 0) {
ml.getVisualizerFieldStats({
$q.when(ml.getVisualizerFieldStats({
indexPatternTitle: indexPattern.title,
query: $scope.searchQuery,
fields: fields,
@ -529,7 +530,7 @@ module
latest: $scope.latest,
samplerShardSize: $scope.samplerShardSize,
maxExamples: 10
})
}))
.then((resp) => {
// Add the metric stats to the existing stats in the corresponding card.
_.each($scope.fieldCards, (card) => {
@ -575,7 +576,7 @@ module
// 2. List of aggregatable fields that do not exist in docs
// 3. List of non-aggregatable fields that do exist in docs.
// 4. List of non-aggregatable fields that do not exist in docs.
ml.getVisualizerOverallStats({
$q.when(ml.getVisualizerOverallStats({
indexPatternTitle: indexPattern.title,
query: $scope.searchQuery,
timeFieldName: indexPattern.timeFieldName,
@ -584,7 +585,7 @@ module
latest: $scope.latest,
aggregatableFields: aggregatableFields,
nonAggregatableFields: nonAggregatableFields
})
}))
.then((resp) => {
$scope.overallStats = resp;
createMetricCards();

View file

@ -16,8 +16,10 @@ import _ from 'lodash';
import { parseInterval } from 'ui/utils/parse_interval';
import { buildConfigFromDetector } from 'plugins/ml/util/chart_config_builder';
import { mlEscape } from 'plugins/ml/util/string_utils';
import { JobServiceProvider } from 'plugins/ml/services/job_service';
export function explorerChartConfigBuilder(mlJobService) {
export function explorerChartConfigBuilder(Private) {
const mlJobService = Private(JobServiceProvider);
const compiledTooltip = _.template(
'<div class="explorer-chart-info-tooltip">job ID: <%= jobId %><br/>' +

View file

@ -24,18 +24,19 @@ import { TimeBucketsProvider } from 'ui/time_buckets';
import 'plugins/ml/filters/format_value';
import loadingIndicatorWrapperTemplate from 'plugins/ml/components/loading_indicator/loading_indicator_wrapper.html';
import { mlEscape } from 'plugins/ml/util/string_utils';
import { FieldFormatServiceProvider } from 'plugins/ml/services/field_format_service';
import { uiModules } from 'ui/modules';
const module = uiModules.get('apps/ml');
module.directive('mlExplorerChart', function (
Private,
formatValueFilter,
mlChartTooltipService,
mlSelectSeverityService,
mlFieldFormatService) {
Private,
mlSelectSeverityService) {
function link(scope, element) {
const mlFieldFormatService = Private(FieldFormatServiceProvider);
console.log('ml-explorer-chart directive link series config:', scope.seriesConfig);
if (typeof scope.seriesConfig === 'undefined') {
// just return so the empty directive renders without an error later on

View file

@ -21,15 +21,16 @@ const module = uiModules.get('apps/ml');
import { explorerChartConfigBuilder } from './explorer_chart_config_builder';
import { chartLimits } from 'plugins/ml/util/chart_utils';
import { isTimeSeriesViewDetector } from 'plugins/ml/../common/util/job_utils';
import 'plugins/ml/services/results_service';
import { ResultsServiceProvider } from 'plugins/ml/services/results_service';
import { JobServiceProvider } from 'plugins/ml/services/job_service';
module.controller('MlExplorerChartsContainerController', function ($scope, $injector) {
const Private = $injector.get('Private');
const mlJobService = $injector.get('mlJobService');
const mlExplorerDashboardService = $injector.get('mlExplorerDashboardService');
const mlResultsService = $injector.get('mlResultsService');
const mlSelectSeverityService = $injector.get('mlSelectSeverityService');
const $q = $injector.get('$q');
const mlResultsService = Private(ResultsServiceProvider);
const mlJobService = Private(JobServiceProvider);
$scope.seriesToPlot = [];

View file

@ -20,9 +20,6 @@ import moment from 'moment';
import 'plugins/ml/components/anomalies_table';
import 'plugins/ml/components/influencers_list';
import 'plugins/ml/components/job_select_list';
import 'plugins/ml/services/field_format_service';
import 'plugins/ml/services/job_service';
import 'plugins/ml/services/results_service';
import { FilterBarQueryFilterProvider } from 'ui/filter_bar/query_filter';
import { parseInterval } from 'ui/utils/parse_interval';
@ -34,6 +31,9 @@ import { checkGetJobsPrivilege } from 'plugins/ml/privilege/check_privilege';
import { getIndexPatterns } from 'plugins/ml/util/index_utils';
import { refreshIntervalWatcher } from 'plugins/ml/util/refresh_interval_watcher';
import { IntervalHelperProvider, getBoundsRoundedToInterval } from 'plugins/ml/util/ml_time_buckets';
import { ResultsServiceProvider } from 'plugins/ml/services/results_service';
import { JobServiceProvider } from 'plugins/ml/services/job_service';
import { FieldFormatServiceProvider } from 'plugins/ml/services/field_format_service';
uiRoutes
.when('/explorer/?', {
@ -56,9 +56,6 @@ module.controller('MlExplorerController', function (
Private,
timefilter,
mlCheckboxShowChartsService,
mlFieldFormatService,
mlJobService,
mlResultsService,
mlJobSelectService,
mlExplorerDashboardService,
mlSelectLimitService,
@ -72,6 +69,9 @@ module.controller('MlExplorerController', function (
const TimeBuckets = Private(IntervalHelperProvider);
const queryFilter = Private(FilterBarQueryFilterProvider);
const mlResultsService = Private(ResultsServiceProvider);
const mlJobService = Private(JobServiceProvider);
const mlFieldFormatService = Private(FieldFormatServiceProvider);
let resizeTimeout = null;

View file

@ -12,8 +12,10 @@ import { parseInterval } from 'ui/utils/parse_interval';
import { ML_RESULTS_INDEX_PATTERN } from 'plugins/ml/constants/index_patterns';
import { replaceTokensInUrlValue } from 'plugins/ml/util/custom_url_utils';
import { JobServiceProvider } from 'plugins/ml/services/job_service';
export function CustomUrlEditorServiceProvider(es, mlJobService, $q) {
export function CustomUrlEditorServiceProvider(es, Private, $q) {
const mlJobService = Private(JobServiceProvider);
// Builds the full URL for testing out a custom URL configuration, which
// may contain dollar delimited partition / influencer entity tokens and

View file

@ -5,11 +5,13 @@
*/
import { CreateWatchServiceProvider } from 'plugins/ml/jobs/new_job/simple/components/watcher/create_watch_service';
import { uiModules } from 'ui/modules';
const module = uiModules.get('apps/ml');
module.controller('MlCreateWatchModal', function ($scope, $modalInstance, params, mlMessageBarService, mlCreateWatchService) {
module.controller('MlCreateWatchModal', function ($scope, $modalInstance, params, mlMessageBarService, Private) {
const mlCreateWatchService = Private(CreateWatchServiceProvider);
const msgs = mlMessageBarService; // set a reference to the message bar service
msgs.clear();

View file

@ -17,6 +17,7 @@ import { parseInterval } from 'plugins/ml/../common/util/parse_interval';
import { CustomUrlEditorServiceProvider } from 'plugins/ml/jobs/components/custom_url_editor/custom_url_editor_service';
import { isWebUrl } from 'plugins/ml/util/string_utils';
import { newJobLimits } from 'plugins/ml/jobs/new_job/utils/new_job_defaults';
import { JobServiceProvider } from 'plugins/ml/services/job_service';
import { uiModules } from 'ui/modules';
const module = uiModules.get('apps/ml');
@ -28,10 +29,10 @@ module.controller('MlEditJobModal', function (
$window,
params,
Private,
mlJobService,
mlMessageBarService) {
const msgs = mlMessageBarService;
msgs.clear();
const mlJobService = Private(JobServiceProvider);
$scope.saveLock = false;
const refreshJob = params.pscope.refreshJob;

View file

@ -7,9 +7,11 @@
import _ from 'lodash';
import moment from 'moment';
import { toLocaleString, detectorToString } from 'plugins/ml/util/string_utils';
import { copyTextToClipboard } from 'plugins/ml/util/clipboard_utils';
import { JOB_STATE, DATAFEED_STATE } from 'plugins/ml/../common/constants/states';
import { ML_DATA_PREVIEW_COUNT } from 'plugins/ml/../common/util/job_utils';
import { checkPermission } from 'plugins/ml/privilege/check_privilege';
import { JobServiceProvider } from 'plugins/ml/services/job_service';
import numeral from '@elastic/numeral';
import chrome from 'ui/chrome';
import angular from 'angular';
@ -18,7 +20,7 @@ import template from './expanded_row.html';
import { uiModules } from 'ui/modules';
const module = uiModules.get('apps/ml');
module.directive('mlJobListExpandedRow', function ($location, mlMessageBarService, mlJobService, mlClipboardService) {
module.directive('mlJobListExpandedRow', function ($location, mlMessageBarService, Private) {
return {
restrict: 'AE',
replace: false,
@ -34,6 +36,7 @@ module.directive('mlJobListExpandedRow', function ($location, mlMessageBarServic
template,
link: function ($scope, $element) {
const msgs = mlMessageBarService; // set a reference to the message bar service
const mlJobService = Private(JobServiceProvider);
const TIME_FORMAT = 'YYYY-MM-DD HH:mm:ss';
const DATA_FORMAT = '0.0 b';
@ -169,7 +172,7 @@ module.directive('mlJobListExpandedRow', function ($location, mlMessageBarServic
$scope.copyToClipboard = function (job) {
const newJob = angular.copy(job);
const success = mlClipboardService.copy(angular.toJson(newJob));
const success = copyTextToClipboard(angular.toJson(newJob));
if (success) {
// flash the background color of the json box
// to show the contents has been copied.

View file

@ -8,7 +8,8 @@
import 'ngreact';
import 'plugins/ml/services/forecast_service';
import { ForecastServiceProvider } from 'plugins/ml/services/forecast_service';
import { uiModules } from 'ui/modules';
const module = uiModules.get('apps/ml', ['react']);
@ -16,7 +17,8 @@ const module = uiModules.get('apps/ml', ['react']);
import { ForecastsTable } from './forecasts_table';
module.directive('mlForecastsTable', function ($injector) {
const mlForecastService = $injector.get('mlForecastService');
const Private = $injector.get('Private');
const mlForecastService = Private(ForecastServiceProvider);
const reactDirective = $injector.get('reactDirective');
return reactDirective(

View file

@ -16,4 +16,3 @@ import './expanded_row';
import 'ui/directives/confirm_click';
import 'plugins/ml/components/paginated_table';
import 'plugins/ml/components/validate_job';
import 'plugins/ml/services/notification_service';

View file

@ -9,6 +9,9 @@
import moment from 'moment';
import angular from 'angular';
import { JobServiceProvider } from 'plugins/ml/services/job_service';
import { CreateWatchServiceProvider } from 'plugins/ml/jobs/new_job/simple/components/watcher/create_watch_service';
import { uiModules } from 'ui/modules';
const module = uiModules.get('apps/ml');
@ -17,10 +20,11 @@ module.controller('MlJobTimepickerModal', function (
$rootScope,
$modalInstance,
params,
mlJobService,
mlCreateWatchService,
Private,
mlMessageBarService) {
const msgs = mlMessageBarService;
const mlJobService = Private(JobServiceProvider);
const mlCreateWatchService = Private(CreateWatchServiceProvider);
$scope.saveLock = false;
$scope.watcherEnabled = mlCreateWatchService.isWatcherEnabled();

View file

@ -16,6 +16,7 @@ import jobsListControlsHtml from './jobs_list_controls.html';
import jobsListArrow from 'plugins/ml/components/paginated_table/open.html';
import { isTimeSeriesViewJob } from 'plugins/ml/../common/util/job_utils';
import { toLocaleString, mlEscape } from 'plugins/ml/util/string_utils';
import { copyTextToClipboard } from 'plugins/ml/util/clipboard_utils';
import uiRoutes from 'ui/routes';
import { checkLicense } from 'plugins/ml/license/check_license';
@ -31,6 +32,9 @@ import createWatchTemplate from 'plugins/ml/jobs/jobs_list/create_watch_modal/cr
import { buttonsEnabledChecks } from 'plugins/ml/jobs/jobs_list/buttons_enabled_checks';
import { cloudServiceProvider } from 'plugins/ml/services/cloud_service';
import { loadNewJobDefaults } from 'plugins/ml/jobs/new_job/utils/new_job_defaults';
import { JobServiceProvider } from 'plugins/ml/services/job_service';
import { CalendarServiceProvider } from 'plugins/ml/services/calendar_service';
import { JobMessagesServiceProvider } from 'plugins/ml/services/job_messages_service';
uiRoutes
.when('/jobs/?', {
@ -60,11 +64,7 @@ module.controller('MlJobsList',
kbnUrl,
Private,
mlMessageBarService,
mlClipboardService,
mlJobService,
mlCalendarService,
mlDatafeedService,
mlNotificationService) {
mlDatafeedService) {
timefilter.disableTimeRangeSelector(); // remove time picker from top of page
timefilter.disableAutoRefreshSelector(); // remove time picker from top of page
@ -85,6 +85,9 @@ module.controller('MlJobsList',
$scope.mlNodesAvailable = mlNodesAvailable();
$scope.permissionToViewMlNodeCount = permissionToViewMlNodeCount();
const mlJobService = Private(JobServiceProvider);
const mlCalendarService = Private(CalendarServiceProvider);
const jobMessagesService = Private(JobMessagesServiceProvider);
const { isRunningOnCloud, getCloudId } = Private(cloudServiceProvider);
$scope.isCloud = isRunningOnCloud();
$scope.cloudId = getCloudId();
@ -169,7 +172,7 @@ module.controller('MlJobsList',
};
$scope.copyToClipboard = function (job) {
const success = mlClipboardService.copy(angular.toJson(job));
const success = copyTextToClipboard(angular.toJson(job));
if (success) {
msgs.clear();
msgs.info(job.job_id + ' JSON copied to clipboard');
@ -436,7 +439,7 @@ module.controller('MlJobsList',
}
}
return mlNotificationService.getJobAuditMessages(fromRange, jobId)
return jobMessagesService.getJobAuditMessages(fromRange, jobId)
.then((resp) => {
const messages = resp.messages;
_.each(messages, (msg) => {
@ -488,7 +491,7 @@ module.controller('MlJobsList',
createTimes[job.job_id] = moment(job.create_time).valueOf();
});
mlNotificationService.getAuditMessagesSummary()
jobMessagesService.getAuditMessagesSummary()
.then((resp) => {
const messagesPerJob = resp.messagesPerJob;
_.each(messagesPerJob, (job) => {

View file

@ -12,7 +12,7 @@ import angular from 'angular';
import { uiModules } from 'ui/modules';
const module = uiModules.get('apps/ml');
module.controller('MlDetectorFilterModal', function ($scope, $modalInstance, params, mlJobService, mlMessageBarService) {
module.controller('MlDetectorFilterModal', function ($scope, $modalInstance, params, mlMessageBarService) {
const msgs = mlMessageBarService;
msgs.clear();
$scope.title = 'Add new filter';

View file

@ -16,11 +16,12 @@ import { detectorToString } from 'plugins/ml/util/string_utils';
import template from './detectors_list.html';
import detectorModalTemplate from 'plugins/ml/jobs/new_job/advanced/detector_modal/detector_modal.html';
import detectorFilterModalTemplate from 'plugins/ml/jobs/new_job/advanced/detector_filter_modal/detector_filter_modal.html';
import { JobServiceProvider } from 'plugins/ml/services/job_service';
import { uiModules } from 'ui/modules';
const module = uiModules.get('apps/ml');
module.directive('mlJobDetectorsList', function ($modal, $q, mlJobService) {
module.directive('mlJobDetectorsList', function ($modal, $q, Private) {
return {
restrict: 'AE',
replace: true,
@ -33,6 +34,7 @@ module.directive('mlJobDetectorsList', function ($modal, $q, mlJobService) {
},
template,
controller: function ($scope) {
const mlJobService = Private(JobServiceProvider);
$scope.addDetector = function (dtr, index) {
if (dtr !== undefined) {

View file

@ -18,7 +18,7 @@ import { checkLicense } from 'plugins/ml/license/check_license';
import { checkCreateJobsPrivilege } from 'plugins/ml/privilege/check_privilege';
import template from './new_job.html';
import saveStatusTemplate from 'plugins/ml/jobs/new_job/advanced/save_status_modal/save_status_modal.html';
import { createSearchItems } from 'plugins/ml/jobs/new_job/utils/new_job_utils';
import { createSearchItems, createJobForSaving } from 'plugins/ml/jobs/new_job/utils/new_job_utils';
import { getIndexPatterns, getIndexPatternWithRoute, getSavedSearchWithRoute, timeBasedIndexCheck } from 'plugins/ml/util/index_utils';
import { ML_JOB_FIELD_TYPES, ES_FIELD_TYPES } from 'plugins/ml/../common/constants/field_types';
import { checkMlNodesAvailable } from 'plugins/ml/ml_nodes_check/check_ml_nodes';
@ -28,6 +28,8 @@ import {
ML_DATA_PREVIEW_COUNT,
basicJobValidation
} from 'plugins/ml/../common/util/job_utils';
import { JobServiceProvider } from 'plugins/ml/services/job_service';
import { ml } from 'plugins/ml/services/ml_api_service';
uiRoutes
.when('/jobs/new_job/advanced', {
@ -65,19 +67,15 @@ module.controller('MlNewJob',
$location,
$modal,
$q,
$timeout,
courier,
es,
ml,
Private,
timefilter,
esServerUrl,
mlJobService,
mlMessageBarService,
mlDatafeedService,
mlConfirmModalService) {
const mlJobService = Private(JobServiceProvider);
timefilter.disableTimeRangeSelector(); // remove time picker from top of page
timefilter.disableAutoRefreshSelector(); // remove time picker from top of page
const MODE = {
@ -433,12 +431,6 @@ module.controller('MlNewJob',
}
}
function createJobForSaving(job) {
const newJob = angular.copy(job);
delete newJob.datafeed_config;
return newJob;
}
function saveFunc() {
if ($scope.ui.useDedicatedIndex) {

View file

@ -9,13 +9,12 @@
import template from './bucket_span_estimator.html';
import { getQueryFromSavedSearch } from 'plugins/ml/jobs/new_job/utils/new_job_utils';
import { EVENT_RATE_COUNT_FIELD } from 'plugins/ml/jobs/new_job/simple/components/constants/general';
import { ml } from 'plugins/ml/services/ml_api_service';
import { uiModules } from 'ui/modules';
const module = uiModules.get('apps/ml');
module.directive('mlBucketSpanEstimator', function ($injector) {
const ml = $injector.get('ml');
module.directive('mlBucketSpanEstimator', function ($q) {
return {
restrict: 'AE',
replace: false,
@ -79,7 +78,7 @@ module.directive('mlBucketSpanEstimator', function ($injector) {
});
}
ml.estimateBucketSpan(data)
$q.when(ml.estimateBucketSpan(data))
.then((interval) => {
if (interval.error) {
errorHandler(interval.message);

View file

@ -6,12 +6,14 @@
import { PostSaveServiceProvider } from './post_save_service';
import { CreateWatchServiceProvider } from 'plugins/ml/jobs/new_job/simple/components/watcher/create_watch_service';
import template from './post_save_options.html';
import { uiModules } from 'ui/modules';
const module = uiModules.get('apps/ml');
module.directive('mlPostSaveOptions', function (mlPostSaveService, mlCreateWatchService) {
module.directive('mlPostSaveOptions', function (Private) {
return {
restrict: 'AE',
replace: false,
@ -23,13 +25,16 @@ module.directive('mlPostSaveOptions', function (mlPostSaveService, mlCreateWatch
template,
link: function ($scope) {
$scope.watcherEnabled = mlCreateWatchService.isWatcherEnabled();
$scope.status = mlPostSaveService.status;
$scope.STATUS = mlPostSaveService.STATUS;
const postSaveService = Private(PostSaveServiceProvider);
const createWatchService = Private(CreateWatchServiceProvider);
mlCreateWatchService.reset();
$scope.watcherEnabled = createWatchService.isWatcherEnabled();
$scope.status = postSaveService.status;
$scope.STATUS = postSaveService.STATUS;
mlCreateWatchService.config.includeInfluencers = $scope.includeInfluencers;
createWatchService.reset();
createWatchService.config.includeInfluencers = $scope.includeInfluencers;
$scope.runInRealtime = false;
$scope.createWatch = false;
$scope.embedded = true;
@ -39,55 +44,8 @@ module.directive('mlPostSaveOptions', function (mlPostSaveService, mlCreateWatch
};
$scope.apply = function () {
mlPostSaveService.apply($scope.jobId, $scope.runInRealtime, $scope.createWatch);
postSaveService.apply($scope.jobId, $scope.runInRealtime, $scope.createWatch);
};
}
};
}).service('mlPostSaveService', function (mlJobService, mlMessageBarService, $q, mlCreateWatchService) {
const msgs = mlMessageBarService;
this.STATUS = {
SAVE_FAILED: -1,
SAVING: 0,
SAVED: 1,
};
this.status = {
realtimeJob: null,
watch: null
};
mlCreateWatchService.status = this.status;
this.externalCreateWatch;
this.startRealtimeJob = function (jobId) {
const deferred = $q.defer();
this.status.realtimeJob = this.STATUS.SAVING;
const datafeedId = mlJobService.getDatafeedId(jobId);
mlJobService.openJob(jobId)
.finally(() => {
mlJobService.startDatafeed(datafeedId, jobId, 0, undefined)
.then(() => {
this.status.realtimeJob = this.STATUS.SAVED;
deferred.resolve();
}).catch((resp) => {
msgs.error('Could not start datafeed: ', resp);
this.status.realtimeJob = this.STATUS.SAVE_FAILED;
deferred.reject();
});
});
return deferred.promise;
};
this.apply = function (jobId, runInRealtime, createWatch) {
if (runInRealtime) {
this.startRealtimeJob(jobId)
.then(() => {
if (createWatch) {
mlCreateWatchService.createNewWatch(jobId);
}
});
}
};
});

View file

@ -0,0 +1,70 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { JobServiceProvider } from 'plugins/ml/services/job_service';
import { CreateWatchServiceProvider } from 'plugins/ml/jobs/new_job/simple/components/watcher/create_watch_service';
export function PostSaveServiceProvider(Private, mlMessageBarService, $q) {
const msgs = mlMessageBarService;
const mlJobService = Private(JobServiceProvider);
const createWatchService = Private(CreateWatchServiceProvider);
class PostSaveService {
constructor() {
this.STATUS = {
SAVE_FAILED: -1,
SAVING: 0,
SAVED: 1,
};
this.status = {
realtimeJob: null,
watch: null
};
createWatchService.status = this.status;
this.externalCreateWatch;
}
startRealtimeJob(jobId) {
return $q((resolve, reject) => {
this.status.realtimeJob = this.STATUS.SAVING;
const datafeedId = mlJobService.getDatafeedId(jobId);
mlJobService.openJob(jobId)
.finally(() => {
mlJobService.startDatafeed(datafeedId, jobId, 0, undefined)
.then(() => {
this.status.realtimeJob = this.STATUS.SAVED;
resolve();
}).catch((resp) => {
msgs.error('Could not start datafeed: ', resp);
this.status.realtimeJob = this.STATUS.SAVE_FAILED;
reject();
});
});
});
}
apply(jobId, runInRealtime, createWatch) {
if (runInRealtime) {
this.startRealtimeJob(jobId)
.then(() => {
if (createWatch) {
createWatchService.createNewWatch(jobId);
}
});
}
}
}
return new PostSaveService();
}

View file

@ -10,7 +10,9 @@
// based on the cardinality of the field being used to split the data.
// the limit should be 10MB plus 20kB per series, rounded up to the nearest MB.
export function CalculateModelMemoryLimitProvider(ml) {
import { ml } from 'plugins/ml/services/ml_api_service';
export function CalculateModelMemoryLimitProvider() {
return function calculateModelMemoryLimit(
indexPattern,
splitFieldName,

View file

@ -11,9 +11,13 @@
import _ from 'lodash';
import { IntervalHelperProvider } from 'plugins/ml/util/ml_time_buckets';
import { calculateTextWidth } from 'plugins/ml/util/string_utils';
import { ResultsServiceProvider } from 'plugins/ml/services/results_service';
import { SimpleJobSearchServiceProvider } from 'plugins/ml/jobs/new_job/simple/components/utils/search_service';
export function ChartDataUtilsProvider($q, Private, timefilter, mlSimpleJobSearchService, mlResultsService) {
export function ChartDataUtilsProvider($q, Private, timefilter) {
const TimeBuckets = Private(IntervalHelperProvider);
const mlResultsService = Private(ResultsServiceProvider);
const mlSimpleJobSearchService = Private(SimpleJobSearchServiceProvider);
function loadDocCountData(formConfig, chartData) {
return $q((resolve, reject) => {

View file

@ -11,85 +11,83 @@ import _ from 'lodash';
import { ML_RESULTS_INDEX_PATTERN } from 'plugins/ml/constants/index_patterns';
import { escapeForElasticsearchQuery } from 'plugins/ml/util/string_utils';
import { uiModules } from 'ui/modules';
const module = uiModules.get('apps/ml');
module.service('mlSimpleJobSearchService', function ($q, es) {
export function SimpleJobSearchServiceProvider($q, es) {
// detector swimlane search
this.getScoresByRecord = function (jobId, earliestMs, latestMs, interval, firstSplitField) {
const deferred = $q.defer();
const obj = {
success: true,
results: {}
};
function getScoresByRecord(jobId, earliestMs, latestMs, interval, firstSplitField) {
return $q((resolve, reject) => {
const obj = {
success: true,
results: {}
};
let jobIdFilterStr = 'job_id: ' + jobId;
if (firstSplitField && firstSplitField.value !== undefined) {
// Escape any reserved characters for the query_string query,
// wrapping the value in quotes to do a phrase match.
// Backslash is a special character in JSON strings, so doubly escape
// any backslash characters which exist in the field value.
jobIdFilterStr += ` AND ${escapeForElasticsearchQuery(firstSplitField.name)}:`;
jobIdFilterStr += `"${String(firstSplitField.value).replace(/\\/g, '\\\\')}"`;
}
let jobIdFilterStr = 'job_id: ' + jobId;
if (firstSplitField && firstSplitField.value !== undefined) {
// Escape any reserved characters for the query_string query,
// wrapping the value in quotes to do a phrase match.
// Backslash is a special character in JSON strings, so doubly escape
// any backslash characters which exist in the field value.
jobIdFilterStr += ` AND ${escapeForElasticsearchQuery(firstSplitField.name)}:`;
jobIdFilterStr += `"${String(firstSplitField.value).replace(/\\/g, '\\\\')}"`;
}
es.search({
index: ML_RESULTS_INDEX_PATTERN,
size: 0,
body: {
query: {
bool: {
filter: [{
query_string: {
query: 'result_type:record'
}
}, {
bool: {
must: [{
range: {
timestamp: {
gte: earliestMs,
lte: latestMs,
format: 'epoch_millis'
es.search({
index: ML_RESULTS_INDEX_PATTERN,
size: 0,
body: {
query: {
bool: {
filter: [{
query_string: {
query: 'result_type:record'
}
}, {
bool: {
must: [{
range: {
timestamp: {
gte: earliestMs,
lte: latestMs,
format: 'epoch_millis'
}
}
}
}, {
query_string: {
query: jobIdFilterStr
}
}]
}
}]
}
},
aggs: {
detector_index: {
terms: {
field: 'detector_index',
order: {
recordScore: 'desc'
}
},
aggs: {
recordScore: {
max: {
field: 'record_score'
}, {
query_string: {
query: jobIdFilterStr
}
}]
}
}]
}
},
aggs: {
detector_index: {
terms: {
field: 'detector_index',
order: {
recordScore: 'desc'
}
},
byTime: {
date_histogram: {
field: 'timestamp',
interval: interval,
min_doc_count: 1,
extended_bounds: {
min: earliestMs,
max: latestMs
aggs: {
recordScore: {
max: {
field: 'record_score'
}
},
aggs: {
recordScore: {
max: {
field: 'record_score'
byTime: {
date_histogram: {
field: 'timestamp',
interval: interval,
min_doc_count: 1,
extended_bounds: {
min: earliestMs,
max: latestMs
}
},
aggs: {
recordScore: {
max: {
field: 'record_score'
}
}
}
}
@ -97,69 +95,73 @@ module.service('mlSimpleJobSearchService', function ($q, es) {
}
}
}
}
})
.then((resp) => {
const detectorsByIndex = _.get(resp, ['aggregations', 'detector_index', 'buckets'], []);
_.each(detectorsByIndex, (dtr) => {
const dtrResults = {};
const dtrIndex = +dtr.key;
const buckets = _.get(dtr, ['byTime', 'buckets'], []);
for (let j = 0; j < buckets.length; j++) {
const bkt = buckets[j];
const time = bkt.key;
dtrResults[time] = {
recordScore: _.get(bkt, ['recordScore', 'value']),
};
}
obj.results[dtrIndex] = dtrResults;
});
deferred.resolve(obj);
})
.catch((resp) => {
deferred.reject(resp);
});
return deferred.promise;
};
.then((resp) => {
const detectorsByIndex = _.get(resp, ['aggregations', 'detector_index', 'buckets'], []);
_.each(detectorsByIndex, (dtr) => {
const dtrResults = {};
const dtrIndex = +dtr.key;
this.getCategoryFields = function (index, field, size, query) {
const deferred = $q.defer();
const obj = {
success: true,
results: {}
};
const buckets = _.get(dtr, ['byTime', 'buckets'], []);
for (let j = 0; j < buckets.length; j++) {
const bkt = buckets[j];
const time = bkt.key;
dtrResults[time] = {
recordScore: _.get(bkt, ['recordScore', 'value']),
};
}
obj.results[dtrIndex] = dtrResults;
});
es.search({
index,
size: 0,
body: {
query: query,
aggs: {
catFields: {
terms: {
field: field,
size: size
resolve(obj);
})
.catch((resp) => {
reject(resp);
});
});
}
function getCategoryFields(index, field, size, query) {
return $q((resolve, reject) => {
const obj = {
success: true,
results: {}
};
es.search({
index,
size: 0,
body: {
query: query,
aggs: {
catFields: {
terms: {
field: field,
size: size
}
}
}
}
}
})
.then((resp) => {
obj.results.values = [];
const catFields = _.get(resp, ['aggregations', 'catFields', 'buckets'], []);
_.each(catFields, (f) => {
obj.results.values.push(f.key);
})
.then((resp) => {
obj.results.values = [];
const catFields = _.get(resp, ['aggregations', 'catFields', 'buckets'], []);
_.each(catFields, (f) => {
obj.results.values.push(f.key);
});
resolve(obj);
})
.catch((resp) => {
reject(resp);
});
deferred.resolve(obj);
})
.catch((resp) => {
deferred.reject(resp);
});
});
}
return deferred.promise;
return {
getScoresByRecord,
getCategoryFields
};
});
}

View file

@ -8,13 +8,15 @@
import _ from 'lodash';
import { parseInterval } from 'ui/utils/parse_interval';
import { CreateWatchServiceProvider } from 'plugins/ml/jobs/new_job/simple/components/watcher/create_watch_service';
import template from './create_watch.html';
import { ml } from 'plugins/ml/services/ml_api_service';
import { uiModules } from 'ui/modules';
const module = uiModules.get('apps/ml');
module.directive('mlCreateWatch', function (es, ml, mlCreateWatchService) {
module.directive('mlCreateWatch', function (es, $q, Private) {
return {
restrict: 'AE',
replace: false,
@ -25,7 +27,7 @@ module.directive('mlCreateWatch', function (es, ml, mlCreateWatchService) {
},
template,
link: function ($scope) {
const mlCreateWatchService = Private(CreateWatchServiceProvider);
$scope.config = mlCreateWatchService.config;
$scope.status = mlCreateWatchService.status;
$scope.STATUS = mlCreateWatchService.STATUS;
@ -56,7 +58,7 @@ module.directive('mlCreateWatch', function (es, ml, mlCreateWatchService) {
}
// load elasticsearch settings to see if email has been configured
ml.getNotificationSettings().then((resp) => {
$q.when(ml.getNotificationSettings()).then((resp) => {
if (_.has(resp, 'defaults.xpack.notification.email')) {
$scope.ui.emailEnabled = true;
}

View file

@ -14,38 +14,10 @@ import emailBody from './email.html';
import emailInfluencersBody from './email-influencers.html';
import { watch } from './watch.js';
import { uiModules } from 'ui/modules';
const module = uiModules.get('apps/ml');
module.service('mlCreateWatchService', function ($http, $q, Private) {
export function CreateWatchServiceProvider($http, $q, Private) {
const xpackInfo = Private(XPackInfoProvider);
this.config = {};
this.STATUS = {
SAVE_FAILED: -1,
SAVING: 0,
SAVED: 1,
};
this.status = {
realtimeJob: null,
watch: null
};
this.reset = function () {
this.status.realtimeJob = null;
this.status.watch = null;
this.config.id = '';
this.config.includeEmail = false;
this.config.email = '';
this.config.interval = '20m';
this.config.watcherEditURL = '';
this.config.includeInfluencers = false;
this.config.threshold = { display: 'critical', val: 75 };
};
const compiledEmailBody = _.template(emailBody);
const emailSection = {
@ -67,67 +39,6 @@ module.service('mlCreateWatchService', function ($http, $q, Private) {
return Math.floor(Math.random() * (max - min + 1) + min);
}
this.createNewWatch = function (jobId) {
const deferred = $q.defer();
this.status.watch = this.STATUS.SAVING;
if (jobId !== undefined) {
const id = `ml-${jobId}`;
this.config.id = id;
// set specific properties of the the watch
watch.input.search.request.body.query.bool.filter[0].term.job_id = jobId;
watch.input.search.request.body.query.bool.filter[1].range.timestamp.gte = `now-${this.config.interval}`;
watch.input.search.request.body.aggs.bucket_results.filter.range.anomaly_score.gte = this.config.threshold.val;
if (this.config.includeEmail && this.config.email !== '') {
const emails = this.config.email.split(',');
emailSection.send_email.email.to = emails;
// create the html by adding the variables to the compiled email body.
emailSection.send_email.email.body.html = compiledEmailBody({
serverAddress: chrome.getAppUrl(),
influencersSection: ((this.config.includeInfluencers === true) ? emailInfluencersBody : '')
});
// add email section to watch
watch.actions.send_email = emailSection.send_email;
}
// set the trigger interval to be a random number between 60 and 120 seconds
// this is to avoid all watches firing at once if the server restarts
// and the watches synchronise
const triggerInterval = randomNumber(60, 120);
watch.trigger.schedule.interval = `${triggerInterval}s`;
const watchModel = {
id,
upstreamJSON: {
id,
type: 'json',
watch
}
};
if (id !== '') {
saveWatch(watchModel)
.then(() => {
this.status.watch = this.STATUS.SAVED;
this.config.watcherEditURL =
`${chrome.getBasePath()}/app/kibana#/management/elasticsearch/watcher/watches/watch/${id}/edit?_g=()`;
deferred.resolve();
})
.catch((resp) => {
this.status.watch = this.STATUS.SAVE_FAILED;
deferred.reject(resp);
});
}
} else {
this.status.watch = this.STATUS.SAVE_FAILED;
deferred.reject();
}
return deferred.promise;
};
function saveWatch(watchModel) {
const basePath = chrome.addBasePath('/api/watcher');
const url = `${basePath}/watch/${watchModel.id}`;
@ -138,19 +49,111 @@ module.service('mlCreateWatchService', function ($http, $q, Private) {
});
}
this.isWatcherEnabled = function () {
return xpackInfo.get('features.watcher.isAvailable', false);
};
this.loadWatch = function (jobId) {
const id = `ml-${jobId}`;
const basePath = chrome.addBasePath('/api/watcher');
const url = `${basePath}/watch/${id}`;
return $http.get(url)
.catch(e => {
throw e.data.message;
class CreateWatchService {
constructor() {
this.config = {};
this.STATUS = {
SAVE_FAILED: -1,
SAVING: 0,
SAVED: 1,
};
this.status = {
realtimeJob: null,
watch: null
};
}
reset() {
this.status.realtimeJob = null;
this.status.watch = null;
this.config.id = '';
this.config.includeEmail = false;
this.config.email = '';
this.config.interval = '20m';
this.config.watcherEditURL = '';
this.config.includeInfluencers = false;
this.config.threshold = { display: 'critical', val: 75 };
}
createNewWatch = function (jobId) {
return $q((resolve, reject) => {
this.status.watch = this.STATUS.SAVING;
if (jobId !== undefined) {
const id = `ml-${jobId}`;
this.config.id = id;
// set specific properties of the the watch
watch.input.search.request.body.query.bool.filter[0].term.job_id = jobId;
watch.input.search.request.body.query.bool.filter[1].range.timestamp.gte = `now-${this.config.interval}`;
watch.input.search.request.body.aggs.bucket_results.filter.range.anomaly_score.gte = this.config.threshold.val;
if (this.config.includeEmail && this.config.email !== '') {
const emails = this.config.email.split(',');
emailSection.send_email.email.to = emails;
// create the html by adding the variables to the compiled email body.
emailSection.send_email.email.body.html = compiledEmailBody({
serverAddress: chrome.getAppUrl(),
influencersSection: ((this.config.includeInfluencers === true) ? emailInfluencersBody : '')
});
// add email section to watch
watch.actions.send_email = emailSection.send_email;
}
// set the trigger interval to be a random number between 60 and 120 seconds
// this is to avoid all watches firing at once if the server restarts
// and the watches synchronize
const triggerInterval = randomNumber(60, 120);
watch.trigger.schedule.interval = `${triggerInterval}s`;
const watchModel = {
id,
upstreamJSON: {
id,
type: 'json',
watch
}
};
if (id !== '') {
saveWatch(watchModel)
.then(() => {
this.status.watch = this.STATUS.SAVED;
this.config.watcherEditURL =
`${chrome.getBasePath()}/app/kibana#/management/elasticsearch/watcher/watches/watch/${id}/edit?_g=()`;
resolve();
})
.catch((resp) => {
this.status.watch = this.STATUS.SAVE_FAILED;
reject(resp);
});
}
} else {
this.status.watch = this.STATUS.SAVE_FAILED;
reject();
}
});
};
}
isWatcherEnabled() {
return xpackInfo.get('features.watcher.isAvailable', false);
}
});
loadWatch(jobId) {
const id = `ml-${jobId}`;
const basePath = chrome.addBasePath('/api/watcher');
const url = `${basePath}/watch/${id}`;
return $http.get(url)
.catch(e => {
throw e.data.message;
});
}
}
return new CreateWatchService();
}

View file

@ -37,6 +37,9 @@ import {
createResultsUrl,
addNewJobToRecentlyAccessed,
moveToAdvancedJobCreationProvider } from 'plugins/ml/jobs/new_job/utils/new_job_utils';
import { JobServiceProvider } from 'plugins/ml/services/job_service';
import { MultiMetricJobServiceProvider } from './create_job_service';
import { FullTimeRangeSelectorServiceProvider } from 'plugins/ml/components/full_time_range_selector/full_time_range_selector_service';
import template from './create_job.html';
uiRoutes
@ -61,10 +64,7 @@ module
$route,
timefilter,
Private,
mlJobService,
mlMultiMetricJobService,
mlMessageBarService,
mlFullTimeRangeSelectorService,
AppState) {
timefilter.enableTimeRangeSelector();
@ -74,6 +74,9 @@ module
const moveToAdvancedJobCreation = Private(moveToAdvancedJobCreationProvider);
const calculateModelMemoryLimit = Private(CalculateModelMemoryLimitProvider);
const chartDataUtils = Private(ChartDataUtilsProvider);
const mlJobService = Private(JobServiceProvider);
const mlMultiMetricJobService = Private(MultiMetricJobServiceProvider);
const mlFullTimeRangeSelectorService = Private(FullTimeRangeSelectorServiceProvider);
$scope.addNewJobToRecentlyAccessed = addNewJobToRecentlyAccessed;
const stateDefaults = {

View file

@ -7,340 +7,339 @@
import _ from 'lodash';
import angular from 'angular';
import { EVENT_RATE_COUNT_FIELD } from 'plugins/ml/jobs/new_job/simple/components/constants/general';
import { ML_MEDIAN_PERCENTS } from 'plugins/ml/../common/util/job_utils';
import { FieldFormatServiceProvider } from 'plugins/ml/services/field_format_service';
import { JobServiceProvider } from 'plugins/ml/services/job_service';
import { createJobForSaving } from 'plugins/ml/jobs/new_job/utils/new_job_utils';
import { uiModules } from 'ui/modules';
const module = uiModules.get('apps/ml');
module.service('mlMultiMetricJobService', function (
export function MultiMetricJobServiceProvider(
$q,
es,
timefilter,
Private,
mlFieldFormatService,
mlJobService) {
Private) {
this.chartData = {
job: {
swimlane: [],
line: [],
bars: [],
earliestTime: Number.MAX_SAFE_INTEGER
},
detectors: {},
percentComplete: 0,
loadingDifference: 0,
lastLoadTimestamp: null,
eventRateHighestValue: 0,
chartTicksMargin: { width: 30 },
totalResults: 0
};
this.job = {};
const mlJobService = Private(JobServiceProvider);
const fieldFormatService = Private(FieldFormatServiceProvider);
this.clearChartData = function () {
this.chartData.job.swimlane = [];
this.chartData.job.line = [];
this.chartData.job.bars = [];
this.chartData.detectors = {};
this.chartData.percentComplete = 0;
this.chartData.loadingDifference = 0;
this.chartData.eventRateHighestValue = 0;
this.chartData.totalResults = 0;
class MultiMetricJobService {
this.job = {};
};
this.getLineChartResults = function (formConfig, thisLoadTimestamp) {
const deferred = $q.defer();
const fieldIds = Object.keys(formConfig.fields).sort();
this.chartData.job.earliestTime = formConfig.start;
// move event rate field to the front of the list
const idx = _.findIndex(fieldIds, (id) => id === EVENT_RATE_COUNT_FIELD);
if(idx !== -1) {
fieldIds.splice(idx, 1);
fieldIds.splice(0, 0, EVENT_RATE_COUNT_FIELD);
constructor() {
this.chartData = {
job: {
swimlane: [],
line: [],
bars: [],
earliestTime: Number.MAX_SAFE_INTEGER
},
detectors: {},
percentComplete: 0,
loadingDifference: 0,
lastLoadTimestamp: null,
eventRateHighestValue: 0,
chartTicksMargin: { width: 30 },
totalResults: 0
};
this.job = {};
}
_.each(fieldIds, (fieldId) => {
this.chartData.detectors[fieldId] = {
line: [],
swimlane: [],
highestValue: 0
};
});
clearChartData() {
this.chartData.job.swimlane = [];
this.chartData.job.line = [];
this.chartData.job.bars = [];
this.chartData.detectors = {};
this.chartData.percentComplete = 0;
this.chartData.loadingDifference = 0;
this.chartData.eventRateHighestValue = 0;
this.chartData.totalResults = 0;
this.job = {};
}
const searchJson = getSearchJsonFromConfig(formConfig);
getLineChartResults(formConfig, thisLoadTimestamp) {
return $q((resolve, reject) => {
es.search(searchJson)
.then((resp) => {
// if this is the last chart load, wipe all previous chart data
if (thisLoadTimestamp === this.chartData.lastLoadTimestamp) {
_.each(fieldIds, (fieldId) => {
this.chartData.detectors[fieldId] = {
line: [],
swimlane: [],
highestValue: 0
};
const fieldIds = Object.keys(formConfig.fields).sort();
if (fieldId !== EVENT_RATE_COUNT_FIELD) {
const field = formConfig.fields[fieldId];
const aggType = field.agg.type.dslName;
this.chartData.detectors[fieldId].fieldFormat = mlFieldFormatService.getFieldFormatFromIndexPattern(
formConfig.indexPattern,
fieldId,
aggType);
}
this.chartData.job.earliestTime = formConfig.start;
});
} else {
deferred.resolve(this.chartData);
// move event rate field to the front of the list
const idx = _.findIndex(fieldIds, (id) => id === EVENT_RATE_COUNT_FIELD);
if(idx !== -1) {
fieldIds.splice(idx, 1);
fieldIds.splice(0, 0, EVENT_RATE_COUNT_FIELD);
}
const aggregationsByTime = _.get(resp, ['aggregations', 'times', 'buckets'], []);
_.each(aggregationsByTime, (dataForTime) => {
const time = +dataForTime.key;
const date = new Date(time);
const docCount = +dataForTime.doc_count;
_.each(fieldIds, (fieldId) => {
this.chartData.detectors[fieldId] = {
line: [],
swimlane: [],
highestValue: 0
};
});
this.chartData.job.swimlane.push({
date: date,
time: time,
value: 0,
color: '',
percentComplete: 0
});
this.chartData.job.earliestTime = (time < this.chartData.job.earliestTime) ? time : this.chartData.job.earliestTime;
const searchJson = getSearchJsonFromConfig(formConfig);
// used to draw the x axis labels on first render
this.chartData.job.line.push({
date: date,
time: time,
value: null,
});
es.search(searchJson)
.then((resp) => {
// if this is the last chart load, wipe all previous chart data
if (thisLoadTimestamp === this.chartData.lastLoadTimestamp) {
_.each(fieldIds, (fieldId) => {
this.chartData.detectors[fieldId] = {
line: [],
swimlane: [],
highestValue: 0
};
_.each(fieldIds, (fieldId) => {
let value;
if (fieldId === EVENT_RATE_COUNT_FIELD) {
value = docCount;
} else if (typeof dataForTime[fieldId].value !== 'undefined') {
value = dataForTime[fieldId].value;
} else if (typeof dataForTime[fieldId].values !== 'undefined') {
value = dataForTime[fieldId].values[ML_MEDIAN_PERCENTS];
}
if (fieldId !== EVENT_RATE_COUNT_FIELD) {
const field = formConfig.fields[fieldId];
const aggType = field.agg.type.dslName;
this.chartData.detectors[fieldId].fieldFormat = fieldFormatService.getFieldFormatFromIndexPattern(
formConfig.indexPattern,
fieldId,
aggType);
}
if (!isFinite(value) || docCount === 0) {
value = null;
}
if (this.chartData.detectors[fieldId]) {
this.chartData.detectors[fieldId].line.push({
date,
time,
value,
});
} else {
resolve(this.chartData);
}
const aggregationsByTime = _.get(resp, ['aggregations', 'times', 'buckets'], []);
// init swimlane
this.chartData.detectors[fieldId].swimlane.push({
date,
time,
_.each(aggregationsByTime, (dataForTime) => {
const time = +dataForTime.key;
const date = new Date(time);
const docCount = +dataForTime.doc_count;
this.chartData.job.swimlane.push({
date: date,
time: time,
value: 0,
color: '',
percentComplete: 0
});
this.chartData.job.earliestTime = (time < this.chartData.job.earliestTime) ? time : this.chartData.job.earliestTime;
if (value !== null) {
this.chartData.detectors[fieldId].highestValue =
Math.ceil(Math.max(this.chartData.detectors[fieldId].highestValue, Math.abs(value)));
}
// used to draw the x axis labels on first render
this.chartData.job.line.push({
date: date,
time: time,
value: null,
});
}
_.each(fieldIds, (fieldId) => {
let value;
if (fieldId === EVENT_RATE_COUNT_FIELD) {
value = docCount;
} else if (typeof dataForTime[fieldId].value !== 'undefined') {
value = dataForTime[fieldId].value;
} else if (typeof dataForTime[fieldId].values !== 'undefined') {
value = dataForTime[fieldId].values[ML_MEDIAN_PERCENTS];
}
if (!isFinite(value) || docCount === 0) {
value = null;
}
if (this.chartData.detectors[fieldId]) {
this.chartData.detectors[fieldId].line.push({
date,
time,
value,
});
// init swimlane
this.chartData.detectors[fieldId].swimlane.push({
date,
time,
value: 0,
color: '',
percentComplete: 0
});
if (value !== null) {
this.chartData.detectors[fieldId].highestValue =
Math.ceil(Math.max(this.chartData.detectors[fieldId].highestValue, Math.abs(value)));
}
}
});
});
resolve(this.chartData);
})
.catch((resp) => {
reject(resp);
});
});
deferred.resolve(this.chartData);
})
.catch((resp) => {
deferred.reject(resp);
});
return deferred.promise;
};
function getSearchJsonFromConfig(formConfig) {
const interval = formConfig.chartInterval.getInterval().asMilliseconds() + 'ms';
// clone the query as we're modifying it
const query = _.cloneDeep(formConfig.combinedQuery);
const json = {
'index': formConfig.indexPattern.title,
'size': 0,
'body': {
'query': {},
'aggs': {
'times': {
'date_histogram': {
'field': formConfig.timeField,
'interval': interval,
'min_doc_count': 0,
'extended_bounds': {
'min': formConfig.start,
'max': formConfig.end,
}
}
}
}
}
};
query.bool.must.push({
'range': {
[formConfig.timeField]: {
'gte': formConfig.start,
'lte': formConfig.end,
'format': formConfig.format
}
}
});
// if the data is partitioned, add an additional search term
if (formConfig.firstSplitFieldName !== undefined) {
query.bool.must.push({
term: {
[formConfig.splitField.name]: formConfig.firstSplitFieldName
}
});
}
json.body.query = query;
getJobFromConfig(formConfig) {
const job = mlJobService.getBlankJob();
job.data_description.time_field = formConfig.timeField;
if (Object.keys(formConfig.fields).length) {
json.body.aggs.times.aggs = {};
_.each(formConfig.fields, (field) => {
if (field.id !== EVENT_RATE_COUNT_FIELD) {
json.body.aggs.times.aggs[field.id] = {
[field.agg.type.dslName]: { field: field.name }
};
if (field.agg.type.dslName === 'percentiles') {
json.body.aggs.times.aggs[field.id][field.agg.type.dslName].percents = [ML_MEDIAN_PERCENTS];
_.each(formConfig.fields, (field, key) => {
let func = field.agg.type.mlName;
if (formConfig.isSparseData) {
if (field.agg.type.dslName === 'count') {
func = func.replace(/count/, 'non_zero_count');
} else if(field.agg.type.dslName === 'sum') {
func = func.replace(/sum/, 'non_null_sum');
}
}
});
}
const dtr = {
function: func
};
return json;
}
dtr.detector_description = func;
function createJobForSaving(job) {
const newJob = angular.copy(job);
delete newJob.datafeed_config;
return newJob;
}
this.getJobFromConfig = function (formConfig) {
const job = mlJobService.getBlankJob();
job.data_description.time_field = formConfig.timeField;
_.each(formConfig.fields, (field, key) => {
let func = field.agg.type.mlName;
if (formConfig.isSparseData) {
if (field.agg.type.dslName === 'count') {
func = func.replace(/count/, 'non_zero_count');
} else if(field.agg.type.dslName === 'sum') {
func = func.replace(/sum/, 'non_null_sum');
if (key !== EVENT_RATE_COUNT_FIELD) {
dtr.field_name = field.name;
dtr.detector_description += `(${field.name})`;
}
if (formConfig.splitField !== undefined) {
dtr.partition_field_name = formConfig.splitField.name;
}
job.analysis_config.detectors.push(dtr);
});
const influencerFields = formConfig.influencerFields.map(f => f.name);
if (influencerFields && influencerFields.length) {
job.analysis_config.influencers = influencerFields;
}
const dtr = {
function: func
let query = {
match_all: {}
};
if (formConfig.query.query_string.query !== '*' || formConfig.filters.length) {
query = formConfig.combinedQuery;
}
job.analysis_config.bucket_span = formConfig.bucketSpan;
job.analysis_limits = {
model_memory_limit: formConfig.modelMemoryLimit
};
dtr.detector_description = func;
delete job.data_description.field_delimiter;
delete job.data_description.quote_character;
delete job.data_description.time_format;
delete job.data_description.format;
if (key !== EVENT_RATE_COUNT_FIELD) {
dtr.field_name = field.name;
dtr.detector_description += `(${field.name})`;
const indices = formConfig.indexPattern.title.split(',').map(i => i.trim());
job.datafeed_config = {
query,
indices
};
job.job_id = formConfig.jobId;
job.description = formConfig.description;
job.groups = formConfig.jobGroups;
if (formConfig.useDedicatedIndex) {
job.results_index_name = job.job_id;
}
if (formConfig.splitField !== undefined) {
dtr.partition_field_name = formConfig.splitField.name;
}
job.analysis_config.detectors.push(dtr);
});
const influencerFields = formConfig.influencerFields.map(f => f.name);
if (influencerFields && influencerFields.length) {
job.analysis_config.influencers = influencerFields;
return job;
}
let query = {
match_all: {}
};
if (formConfig.query.query_string.query !== '*' || formConfig.filters.length) {
query = formConfig.combinedQuery;
}
createJob(formConfig) {
return $q((resolve, reject) => {
job.analysis_config.bucket_span = formConfig.bucketSpan;
this.job = this.getJobFromConfig(formConfig);
const job = createJobForSaving(this.job);
job.analysis_limits = {
model_memory_limit: formConfig.modelMemoryLimit
};
// DO THE SAVE
mlJobService.saveNewJob(job)
.then((resp) => {
if (resp.success) {
resolve(this.job);
} else {
reject(resp);
}
});
delete job.data_description.field_delimiter;
delete job.data_description.quote_character;
delete job.data_description.time_format;
delete job.data_description.format;
const indices = formConfig.indexPattern.title.split(',').map(i => i.trim());
job.datafeed_config = {
query,
indices
};
job.job_id = formConfig.jobId;
job.description = formConfig.description;
job.groups = formConfig.jobGroups;
if (formConfig.useDedicatedIndex) {
job.results_index_name = job.job_id;
}
return job;
};
this.createJob = function (formConfig) {
const deferred = $q.defer();
this.job = this.getJobFromConfig(formConfig);
const job = createJobForSaving(this.job);
// DO THE SAVE
mlJobService.saveNewJob(job)
.then((resp) => {
if (resp.success) {
deferred.resolve(this.job);
} else {
deferred.reject(resp);
}
});
}
return deferred.promise;
startDatafeed(formConfig) {
const datafeedId = mlJobService.getDatafeedId(formConfig.jobId);
return mlJobService.startDatafeed(datafeedId, formConfig.jobId, formConfig.start, formConfig.end);
}
stopDatafeed(formConfig) {
const datafeedId = mlJobService.getDatafeedId(formConfig.jobId);
return mlJobService.stopDatafeed(datafeedId, formConfig.jobId);
}
}
return new MultiMetricJobService();
}
function getSearchJsonFromConfig(formConfig) {
const interval = formConfig.chartInterval.getInterval().asMilliseconds() + 'ms';
// clone the query as we're modifying it
const query = _.cloneDeep(formConfig.combinedQuery);
const json = {
index: formConfig.indexPattern.title,
size: 0,
body: {
query: {},
aggs: {
times: {
date_histogram: {
field: formConfig.timeField,
interval: interval,
min_doc_count: 0,
extended_bounds: {
min: formConfig.start,
max: formConfig.end,
}
}
}
}
}
};
this.startDatafeed = function (formConfig) {
const datafeedId = mlJobService.getDatafeedId(formConfig.jobId);
return mlJobService.startDatafeed(datafeedId, formConfig.jobId, formConfig.start, formConfig.end);
};
query.bool.must.push({
range: {
[formConfig.timeField]: {
gte: formConfig.start,
lte: formConfig.end,
format: formConfig.format
}
}
});
this.stopDatafeed = function (formConfig) {
const datafeedId = mlJobService.getDatafeedId(formConfig.jobId);
return mlJobService.stopDatafeed(datafeedId, formConfig.jobId);
};
// if the data is partitioned, add an additional search term
if (formConfig.firstSplitFieldName !== undefined) {
query.bool.must.push({
term: {
[formConfig.splitField.name]: formConfig.firstSplitFieldName
}
});
}
});
json.body.query = query;
if (Object.keys(formConfig.fields).length) {
json.body.aggs.times.aggs = {};
_.each(formConfig.fields, (field) => {
if (field.id !== EVENT_RATE_COUNT_FIELD) {
json.body.aggs.times.aggs[field.id] = {
[field.agg.type.dslName]: { field: field.name }
};
if (field.agg.type.dslName === 'percentiles') {
json.body.aggs.times.aggs[field.id][field.agg.type.dslName].percents = [ML_MEDIAN_PERCENTS];
}
}
});
}
return json;
}

View file

@ -36,6 +36,9 @@ import {
createResultsUrl,
addNewJobToRecentlyAccessed,
moveToAdvancedJobCreationProvider } from 'plugins/ml/jobs/new_job/utils/new_job_utils';
import { JobServiceProvider } from 'plugins/ml/services/job_service';
import { PopulationJobServiceProvider } from './create_job_service';
import { FullTimeRangeSelectorServiceProvider } from 'plugins/ml/components/full_time_range_selector/full_time_range_selector_service';
import template from './create_job.html';
uiRoutes
@ -62,10 +65,7 @@ module
$q,
timefilter,
Private,
mlJobService,
mlPopulationJobService,
mlMessageBarService,
mlFullTimeRangeSelectorService,
AppState) {
timefilter.enableTimeRangeSelector();
@ -74,6 +74,9 @@ module
const MlTimeBuckets = Private(IntervalHelperProvider);
const moveToAdvancedJobCreation = Private(moveToAdvancedJobCreationProvider);
const chartDataUtils = Private(ChartDataUtilsProvider);
const mlJobService = Private(JobServiceProvider);
const mlPopulationJobService = Private(PopulationJobServiceProvider);
const mlFullTimeRangeSelectorService = Private(FullTimeRangeSelectorServiceProvider);
$scope.addNewJobToRecentlyAccessed = addNewJobToRecentlyAccessed;
const stateDefaults = {

View file

@ -7,193 +7,304 @@
import _ from 'lodash';
import angular from 'angular';
import { EVENT_RATE_COUNT_FIELD } from 'plugins/ml/jobs/new_job/simple/components/constants/general';
import { ML_MEDIAN_PERCENTS } from 'plugins/ml/../common/util/job_utils';
import { IntervalHelperProvider } from 'plugins/ml/util/ml_time_buckets';
import { FieldFormatServiceProvider } from 'plugins/ml/services/field_format_service';
import { JobServiceProvider } from 'plugins/ml/services/job_service';
import { createJobForSaving } from 'plugins/ml/jobs/new_job/utils/new_job_utils';
import { uiModules } from 'ui/modules';
const module = uiModules.get('apps/ml');
module.service('mlPopulationJobService', function (
export function PopulationJobServiceProvider(
$q,
es,
timefilter,
Private,
mlFieldFormatService,
mlJobService) {
Private) {
const mlJobService = Private(JobServiceProvider);
const TimeBuckets = Private(IntervalHelperProvider);
const fieldFormatService = Private(FieldFormatServiceProvider);
const OVER_FIELD_EXAMPLES_COUNT = 40;
this.chartData = {
job: {
swimlane: [],
line: [],
bars: [],
earliestTime: Number.MAX_SAFE_INTEGER
},
detectors: {},
percentComplete: 0,
loadingDifference: 0,
lastLoadTimestamp: null,
eventRateHighestValue: 0,
chartTicksMargin: { width: 30 },
totalResults: 0
};
this.job = {};
class PopulationJobService {
this.clearChartData = function () {
this.chartData.job.swimlane = [];
this.chartData.job.line = [];
this.chartData.job.bars = [];
this.chartData.detectors = {};
this.chartData.percentComplete = 0;
this.chartData.loadingDifference = 0;
this.chartData.eventRateHighestValue = 0;
this.chartData.totalResults = 0;
this.job = {};
};
this.getLineChartResults = function (formConfig, thisLoadTimestamp) {
const deferred = $q.defer();
const fieldIds = formConfig.fields.map(f => f.id);
this.chartData.job.earliestTime = formConfig.start;
// move event rate field to the front of the list
const idx = _.findIndex(fieldIds, (id) => id === EVENT_RATE_COUNT_FIELD);
if(idx !== -1) {
fieldIds.splice(idx, 1);
fieldIds.splice(0, 0, EVENT_RATE_COUNT_FIELD);
constructor() {
this.chartData = {
job: {
swimlane: [],
line: [],
bars: [],
earliestTime: Number.MAX_SAFE_INTEGER
},
detectors: {},
percentComplete: 0,
loadingDifference: 0,
lastLoadTimestamp: null,
eventRateHighestValue: 0,
chartTicksMargin: { width: 30 },
totalResults: 0
};
this.job = {};
}
fieldIds.forEach((fieldId, i) => {
this.chartData.detectors[i] = {
line: [],
swimlane: [],
highestValue: 0
};
});
clearChartData() {
this.chartData.job.swimlane = [];
this.chartData.job.line = [];
this.chartData.job.bars = [];
this.chartData.detectors = {};
this.chartData.percentComplete = 0;
this.chartData.loadingDifference = 0;
this.chartData.eventRateHighestValue = 0;
this.chartData.totalResults = 0;
this.job = {};
}
const searchJson = getSearchJsonFromConfig(formConfig);
getLineChartResults(formConfig, thisLoadTimestamp) {
return $q((resolve, reject) => {
es.search(searchJson)
.then((resp) => {
// if this is the last chart load, wipe all previous chart data
if (thisLoadTimestamp === this.chartData.lastLoadTimestamp) {
const fieldIds = formConfig.fields.map(f => f.id);
fieldIds.forEach((fieldId, i) => {
this.chartData.detectors[i] = {
line: [],
swimlane: [],
highestValue: 0
};
this.chartData.job.earliestTime = formConfig.start;
if (fieldId !== EVENT_RATE_COUNT_FIELD) {
const field = formConfig.fields[i];
const aggType = field.agg.type.dslName;
this.chartData.detectors[i].fieldFormat = mlFieldFormatService.getFieldFormatFromIndexPattern(
formConfig.indexPattern,
fieldId,
aggType);
}
});
} else {
deferred.resolve(this.chartData);
// move event rate field to the front of the list
const idx = _.findIndex(fieldIds, (id) => id === EVENT_RATE_COUNT_FIELD);
if(idx !== -1) {
fieldIds.splice(idx, 1);
fieldIds.splice(0, 0, EVENT_RATE_COUNT_FIELD);
}
const aggregationsByTime = _.get(resp, ['aggregations', 'times', 'buckets'], []);
_.each(aggregationsByTime, (dataForTime) => {
const time = +dataForTime.key;
const date = new Date(time);
fieldIds.forEach((fieldId, i) => {
this.chartData.detectors[i] = {
line: [],
swimlane: [],
highestValue: 0
};
});
this.chartData.job.swimlane.push({
date: date,
time: time,
value: 0,
color: '',
percentComplete: 0
});
const searchJson = getSearchJsonFromConfig(formConfig, timefilter, TimeBuckets);
this.chartData.job.earliestTime = (time < this.chartData.job.earliestTime) ? time : this.chartData.job.earliestTime;
es.search(searchJson)
.then((resp) => {
// if this is the last chart load, wipe all previous chart data
if (thisLoadTimestamp === this.chartData.lastLoadTimestamp) {
this.chartData.job.line.push({
date: date,
time: time,
value: null,
});
fieldIds.forEach((fieldId, i) => {
this.chartData.detectors[i] = {
line: [],
swimlane: [],
highestValue: 0
};
fieldIds.forEach((fieldId, i) => {
const populationBuckets = _.get(dataForTime, ['population', 'buckets'], []);
const values = [];
if (fieldId === EVENT_RATE_COUNT_FIELD) {
populationBuckets.forEach(b => {
// check to see if the data is split.
if (b[i] === undefined) {
values.push({ label: b.key, value: b.doc_count });
} else {
// a split is being used, so an additional filter was added to the search
values.push({ label: b.key, value: b[i].doc_count });
if (fieldId !== EVENT_RATE_COUNT_FIELD) {
const field = formConfig.fields[i];
const aggType = field.agg.type.dslName;
this.chartData.detectors[i].fieldFormat = fieldFormatService.getFieldFormatFromIndexPattern(
formConfig.indexPattern,
fieldId,
aggType);
}
});
} else if (typeof dataForTime.population !== 'undefined') {
populationBuckets.forEach(b => {
const tempBucket = b[i];
let value = null;
// check to see if the data is split
// if the field has been split, an additional filter and aggregation
// has been added to the search in the form of splitValue
const tempValue = (tempBucket.value === undefined && tempBucket.splitValue !== undefined) ?
tempBucket.splitValue : tempBucket;
// check to see if values is exists rather than value.
// if values exists, the aggregation was median
if (tempValue.value === undefined && tempValue.values !== undefined) {
value = tempValue.values[ML_MEDIAN_PERCENTS];
} else {
value = tempValue.value;
}
values.push({ label: b.key, value: (isFinite(value) ? value : null) });
});
} else {
resolve(this.chartData);
}
const aggregationsByTime = _.get(resp, ['aggregations', 'times', 'buckets'], []);
const highestValueField = _.reduce(values, (p, c) => (c.value > p.value) ? c : p, { value: 0 });
_.each(aggregationsByTime, (dataForTime) => {
const time = +dataForTime.key;
const date = new Date(time);
if (this.chartData.detectors[i]) {
this.chartData.detectors[i].line.push({
date,
time,
values,
});
// init swimlane
this.chartData.detectors[i].swimlane.push({
date,
time,
this.chartData.job.swimlane.push({
date: date,
time: time,
value: 0,
color: '',
percentComplete: 0
});
this.chartData.detectors[i].highestValue =
Math.ceil(Math.max(this.chartData.detectors[i].highestValue, Math.abs(highestValueField.value)));
}
});
});
this.chartData.job.earliestTime = (time < this.chartData.job.earliestTime) ? time : this.chartData.job.earliestTime;
deferred.resolve(this.chartData);
})
.catch((resp) => {
deferred.reject(resp);
this.chartData.job.line.push({
date: date,
time: time,
value: null,
});
fieldIds.forEach((fieldId, i) => {
const populationBuckets = _.get(dataForTime, ['population', 'buckets'], []);
const values = [];
if (fieldId === EVENT_RATE_COUNT_FIELD) {
populationBuckets.forEach(b => {
// check to see if the data is split.
if (b[i] === undefined) {
values.push({ label: b.key, value: b.doc_count });
} else {
// a split is being used, so an additional filter was added to the search
values.push({ label: b.key, value: b[i].doc_count });
}
});
} else if (typeof dataForTime.population !== 'undefined') {
populationBuckets.forEach(b => {
const tempBucket = b[i];
let value = null;
// check to see if the data is split
// if the field has been split, an additional filter and aggregation
// has been added to the search in the form of splitValue
const tempValue = (tempBucket.value === undefined && tempBucket.splitValue !== undefined) ?
tempBucket.splitValue : tempBucket;
// check to see if values is exists rather than value.
// if values exists, the aggregation was median
if (tempValue.value === undefined && tempValue.values !== undefined) {
value = tempValue.values[ML_MEDIAN_PERCENTS];
} else {
value = tempValue.value;
}
values.push({ label: b.key, value: (isFinite(value) ? value : null) });
});
}
const highestValueField = _.reduce(values, (p, c) => (c.value > p.value) ? c : p, { value: 0 });
if (this.chartData.detectors[i]) {
this.chartData.detectors[i].line.push({
date,
time,
values,
});
// init swimlane
this.chartData.detectors[i].swimlane.push({
date,
time,
value: 0,
color: '',
percentComplete: 0
});
this.chartData.detectors[i].highestValue =
Math.ceil(Math.max(this.chartData.detectors[i].highestValue, Math.abs(highestValueField.value)));
}
});
});
resolve(this.chartData);
})
.catch((resp) => {
reject(resp);
});
});
}
getJobFromConfig(formConfig) {
const job = mlJobService.getBlankJob();
job.data_description.time_field = formConfig.timeField;
formConfig.fields.forEach(field => {
let func = field.agg.type.mlName;
if (formConfig.isSparseData) {
if (field.agg.type.dslName === 'count') {
func = func.replace(/count/, 'non_zero_count');
} else if(field.agg.type.dslName === 'sum') {
func = func.replace(/sum/, 'non_null_sum');
}
}
const dtr = {
function: func
};
dtr.detector_description = func;
if (field.id !== EVENT_RATE_COUNT_FIELD) {
dtr.field_name = field.name;
dtr.detector_description += `(${field.name})`;
}
if (field.splitField !== undefined) {
dtr.by_field_name = field.splitField.name;
dtr.detector_description += ` by ${dtr.by_field_name}`;
}
if (formConfig.overField !== undefined) {
dtr.over_field_name = formConfig.overField.name;
dtr.detector_description += ` over ${dtr.over_field_name}`;
}
// if (formConfig.splitField !== undefined) {
// dtr.partition_field_name = formConfig.splitField;
// }
job.analysis_config.detectors.push(dtr);
});
return deferred.promise;
};
const influencerFields = formConfig.influencerFields.map(f => f.name);
if (influencerFields && influencerFields.length) {
job.analysis_config.influencers = influencerFields;
}
let query = {
match_all: {}
};
if (formConfig.query.query_string.query !== '*' || formConfig.filters.length) {
query = formConfig.combinedQuery;
}
job.analysis_config.bucket_span = formConfig.bucketSpan;
job.analysis_limits = {
model_memory_limit: formConfig.modelMemoryLimit
};
delete job.data_description.field_delimiter;
delete job.data_description.quote_character;
delete job.data_description.time_format;
delete job.data_description.format;
const indices = formConfig.indexPattern.title.split(',').map(i => i.trim());
job.datafeed_config = {
query,
indices,
};
job.job_id = formConfig.jobId;
job.description = formConfig.description;
job.groups = formConfig.jobGroups;
if (formConfig.useDedicatedIndex) {
job.results_index_name = job.job_id;
}
return job;
}
createJob(formConfig) {
return $q((resolve, reject) => {
this.job = this.getJobFromConfig(formConfig);
const job = createJobForSaving(this.job);
// DO THE SAVE
mlJobService.saveNewJob(job)
.then((resp) => {
if (resp.success) {
resolve(this.job);
} else {
reject(resp);
}
});
});
}
startDatafeed(formConfig) {
const datafeedId = mlJobService.getDatafeedId(formConfig.jobId);
return mlJobService.startDatafeed(datafeedId, formConfig.jobId, formConfig.start, formConfig.end);
}
stopDatafeed(formConfig) {
const datafeedId = mlJobService.getDatafeedId(formConfig.jobId);
return mlJobService.stopDatafeed(datafeedId, formConfig.jobId);
}
}
function getSearchJsonFromConfig(formConfig) {
const bounds = timefilter.getActiveBounds();
@ -207,19 +318,19 @@ module.service('mlPopulationJobService', function (
const query = _.cloneDeep(formConfig.combinedQuery);
const json = {
'index': formConfig.indexPattern.title,
'size': 0,
'body': {
'query': {},
'aggs': {
'times': {
'date_histogram': {
'field': formConfig.timeField,
'interval': interval,
'min_doc_count': 0,
'extended_bounds': {
'min': formConfig.start,
'max': formConfig.end,
index: formConfig.indexPattern.title,
size: 0,
body: {
query: {},
aggs: {
times: {
date_histogram: {
field: formConfig.timeField,
interval: interval,
min_doc_count: 0,
extended_bounds: {
min: formConfig.start,
max: formConfig.end,
}
}
}
@ -228,11 +339,11 @@ module.service('mlPopulationJobService', function (
};
query.bool.must.push({
'range': {
range: {
[formConfig.timeField]: {
'gte': formConfig.start,
'lte': formConfig.end,
'format': formConfig.format
gte: formConfig.start,
lte: formConfig.end,
format: formConfig.format
}
}
});
@ -319,118 +430,5 @@ module.service('mlPopulationJobService', function (
return json;
}
function createJobForSaving(job) {
const newJob = angular.copy(job);
delete newJob.datafeed_config;
return newJob;
}
this.getJobFromConfig = function (formConfig) {
const job = mlJobService.getBlankJob();
job.data_description.time_field = formConfig.timeField;
formConfig.fields.forEach(field => {
let func = field.agg.type.mlName;
if (formConfig.isSparseData) {
if (field.agg.type.dslName === 'count') {
func = func.replace(/count/, 'non_zero_count');
} else if(field.agg.type.dslName === 'sum') {
func = func.replace(/sum/, 'non_null_sum');
}
}
const dtr = {
function: func
};
dtr.detector_description = func;
if (field.id !== EVENT_RATE_COUNT_FIELD) {
dtr.field_name = field.name;
dtr.detector_description += `(${field.name})`;
}
if (field.splitField !== undefined) {
dtr.by_field_name = field.splitField.name;
dtr.detector_description += ` by ${dtr.by_field_name}`;
}
if (formConfig.overField !== undefined) {
dtr.over_field_name = formConfig.overField.name;
dtr.detector_description += ` over ${dtr.over_field_name}`;
}
// if (formConfig.splitField !== undefined) {
// dtr.partition_field_name = formConfig.splitField;
// }
job.analysis_config.detectors.push(dtr);
});
const influencerFields = formConfig.influencerFields.map(f => f.name);
if (influencerFields && influencerFields.length) {
job.analysis_config.influencers = influencerFields;
}
let query = {
match_all: {}
};
if (formConfig.query.query_string.query !== '*' || formConfig.filters.length) {
query = formConfig.combinedQuery;
}
job.analysis_config.bucket_span = formConfig.bucketSpan;
job.analysis_limits = {
model_memory_limit: formConfig.modelMemoryLimit
};
delete job.data_description.field_delimiter;
delete job.data_description.quote_character;
delete job.data_description.time_format;
delete job.data_description.format;
const indices = formConfig.indexPattern.title.split(',').map(i => i.trim());
job.datafeed_config = {
query,
indices
};
job.job_id = formConfig.jobId;
job.description = formConfig.description;
job.groups = formConfig.jobGroups;
if (formConfig.useDedicatedIndex) {
job.results_index_name = job.job_id;
}
return job;
};
this.createJob = function (formConfig) {
const deferred = $q.defer();
this.job = this.getJobFromConfig(formConfig);
const job = createJobForSaving(this.job);
// DO THE SAVE
mlJobService.saveNewJob(job)
.then((resp) => {
if (resp.success) {
deferred.resolve(this.job);
} else {
deferred.reject(resp);
}
});
return deferred.promise;
};
this.startDatafeed = function (formConfig) {
const datafeedId = mlJobService.getDatafeedId(formConfig.jobId);
return mlJobService.startDatafeed(datafeedId, formConfig.jobId, formConfig.start, formConfig.end);
};
this.stopDatafeed = function (formConfig) {
const datafeedId = mlJobService.getDatafeedId(formConfig.jobId);
return mlJobService.stopDatafeed(datafeedId, formConfig.jobId);
};
});
return new PopulationJobService();
}

View file

@ -19,6 +19,9 @@ import { checkLicenseExpired } from 'plugins/ml/license/check_license';
import { checkCreateJobsPrivilege } from 'plugins/ml/privilege/check_privilege';
import { getIndexPatternWithRoute, getSavedSearchWithRoute } from 'plugins/ml/util/index_utils';
import { checkMlNodesAvailable } from 'plugins/ml/ml_nodes_check/check_ml_nodes';
import { JobServiceProvider } from 'plugins/ml/services/job_service';
import { CreateRecognizerJobsServiceProvider } from './create_job_service';
import { ml } from 'plugins/ml/services/ml_api_service';
import template from './create_job.html';
uiRoutes
@ -42,13 +45,12 @@ module
$window,
$route,
$q,
ml,
timefilter,
Private,
mlCreateRecognizerJobsService,
mlJobService,
mlMessageBarService) {
const mlJobService = Private(JobServiceProvider);
const mlCreateRecognizerJobsService = Private(CreateRecognizerJobsServiceProvider);
timefilter.disableTimeRangeSelector();
timefilter.disableAutoRefreshSelector();
$scope.tt = timefilter;
@ -143,9 +145,9 @@ module
};
function loadJobConfigs() {
// load the job and datafeed configs as well as the kibana saved objects
// from the recognizer endpoint
ml.getDataRecognizerModule({ moduleId })
// load the job and datafeed configs as well as the kibana saved objects
// from the recognizer endpoint
$q.when(ml.getDataRecognizerModule({ moduleId }))
.then(resp => {
// populate the jobs and datafeeds
if (resp.jobs && resp.jobs.length) {
@ -259,7 +261,7 @@ module
const tempQuery = (savedSearch.id === undefined) ?
undefined : combinedQuery;
ml.setupDataRecognizerConfig({ moduleId, prefix, groups, query: tempQuery, indexPatternName })
$q.when(ml.setupDataRecognizerConfig({ moduleId, prefix, groups, query: tempQuery, indexPatternName }))
.then((resp) => {
if (resp.jobs) {
$scope.formConfig.jobs.forEach((job) => {

View file

@ -6,126 +6,49 @@
import angular from 'angular';
import { getQueryFromSavedSearch } from 'plugins/ml/jobs/new_job/utils/new_job_utils';
import { SavedObjectsClientProvider } from 'ui/saved_objects';
import { JobServiceProvider } from 'plugins/ml/services/job_service';
import { ml } from 'plugins/ml/services/ml_api_service';
import { uiModules } from 'ui/modules';
const module = uiModules.get('apps/ml');
module.service('mlCreateRecognizerJobsService', function (
es,
Private,
$http,
$q,
chrome,
mlJobService) {
export function CreateRecognizerJobsServiceProvider(Private, $q) {
const mlJobService = Private(JobServiceProvider);
const savedObjectsClient = Private(SavedObjectsClientProvider);
class CreateRecognizerJobsService {
this.createJob = function (job, formConfig) {
return $q((resolve, reject) => {
const newJob = angular.copy(job.jobConfig);
const jobId = formConfig.jobLabel + job.id;
newJob.job_id = jobId;
newJob.groups = formConfig.jobGroups;
constructor() {}
mlJobService.saveNewJob(newJob)
.then((resp) => {
if (resp.success) {
createDatafeed(job, formConfig) {
return $q((resolve, reject) => {
const jobId = formConfig.jobLabel + job.id;
mlJobService.saveNewDatafeed(job.datafeedConfig, jobId)
.then((resp) => {
resolve(resp);
} else {
})
.catch((resp) => {
reject(resp);
}
});
});
};
this.createDatafeed = function (job, formConfig) {
return $q((resolve, reject) => {
const jobId = formConfig.jobLabel + job.id;
mlJobService.saveNewDatafeed(job.datafeedConfig, jobId)
.then((resp) => {
resolve(resp);
})
.catch((resp) => {
reject(resp);
});
});
};
this.startDatafeed = function (datafeedId, jobId, start, end) {
return mlJobService.startDatafeed(datafeedId, jobId, start, end);
};
this.stopDatafeed = function (formConfig) {
const datafeedId = mlJobService.getDatafeedId(formConfig.jobId);
return mlJobService.stopDatafeed(datafeedId, formConfig.jobId);
};
this.checkDatafeedStatus = function (formConfig) {
return mlJobService.updateSingleJobDatafeedState(formConfig.jobId);
};
this.loadExistingSavedObjects = function (type) {
return savedObjectsClient.find({ type, perPage: 1000 });
};
this.createSavedObject = function (type, obj) {
return savedObjectsClient.create(type, obj);
};
this.createSavedObjectWithId = function (type, id, obj) {
const basePath = chrome.addBasePath('/api/saved_objects');
const url = `${basePath}/${type}/${id}`;
return $http.post(url, { attributes: obj })
.catch(e => {
throw e.data.message;
});
});
};
}
this.indexTimeRange = function (indexPattern, formConfig) {
return $q((resolve, reject) => {
const obj = { success: true, start: { epoch: 0, string: '' }, end: { epoch: 0, string: '' } };
startDatafeed(datafeedId, jobId, start, end) {
return mlJobService.startDatafeed(datafeedId, jobId, start, end);
}
loadExistingSavedObjects(type) {
return savedObjectsClient.find({ type, perPage: 1000 });
}
indexTimeRange(indexPattern, formConfig) {
const query = getQueryFromSavedSearch(formConfig);
es.search({
return ml.getTimeFieldRange({
index: indexPattern.title,
size: 0,
body: {
query,
aggs: {
earliest: {
min: {
field: indexPattern.timeFieldName
}
},
latest: {
max: {
field: indexPattern.timeFieldName
}
}
}
}
})
.then((resp) => {
if (resp.aggregations && resp.aggregations.earliest && resp.aggregations.latest) {
obj.start.epoch = resp.aggregations.earliest.value;
obj.start.string = resp.aggregations.earliest.value_as_string;
obj.end.epoch = resp.aggregations.latest.value;
obj.end.string = resp.aggregations.latest.value_as_string;
}
resolve(obj);
})
.catch((resp) => {
reject(resp);
});
});
};
});
timeFieldName: indexPattern.timeFieldName,
query
});
}
}
return new CreateRecognizerJobsService();
}

View file

@ -36,6 +36,9 @@ import {
createResultsUrl,
addNewJobToRecentlyAccessed,
moveToAdvancedJobCreationProvider } from 'plugins/ml/jobs/new_job/utils/new_job_utils';
import { JobServiceProvider } from 'plugins/ml/services/job_service';
import { SingleMetricJobServiceProvider } from './create_job_service';
import { FullTimeRangeSelectorServiceProvider } from 'plugins/ml/components/full_time_range_selector/full_time_range_selector_service';
import template from './create_job.html';
@ -63,10 +66,7 @@ module
$q,
timefilter,
Private,
mlJobService,
mlSingleMetricJobService,
mlMessageBarService,
mlFullTimeRangeSelectorService,
AppState) {
timefilter.enableTimeRangeSelector();
@ -74,6 +74,9 @@ module
const msgs = mlMessageBarService;
const MlTimeBuckets = Private(IntervalHelperProvider);
const moveToAdvancedJobCreation = Private(moveToAdvancedJobCreationProvider);
const mlJobService = Private(JobServiceProvider);
const mlSingleMetricJobService = Private(SingleMetricJobServiceProvider);
const mlFullTimeRangeSelectorService = Private(FullTimeRangeSelectorServiceProvider);
const stateDefaults = {
mlJobSettings: {}

View file

@ -7,471 +7,468 @@
import _ from 'lodash';
import angular from 'angular';
import 'ui/timefilter';
import { parseInterval } from 'ui/utils/parse_interval';
import { ML_MEDIAN_PERCENTS } from 'plugins/ml/../common/util/job_utils';
import { calculateTextWidth } from 'plugins/ml/util/string_utils';
import { FieldFormatServiceProvider } from 'plugins/ml/services/field_format_service';
import { JobServiceProvider } from 'plugins/ml/services/job_service';
import { ResultsServiceProvider } from 'plugins/ml/services/results_service';
import { createJobForSaving } from 'plugins/ml/jobs/new_job/utils/new_job_utils';
import { uiModules } from 'ui/modules';
const module = uiModules.get('apps/ml');
module.service('mlSingleMetricJobService', function (
export function SingleMetricJobServiceProvider(
$q,
es,
timefilter,
Private,
mlFieldFormatService,
mlJobService,
mlResultsService) {
Private) {
this.chartData = {
line: [],
model: [],
swimlane: [],
hasBounds: false,
percentComplete: 0,
highestValue: 0,
chartTicksMargin: { width: 30 },
totalResults: 0
};
this.job = {};
const mlJobService = Private(JobServiceProvider);
const mlResultsService = Private(ResultsServiceProvider);
const fieldFormatService = Private(FieldFormatServiceProvider);
this.getLineChartResults = function (formConfig) {
const deferred = $q.defer();
class SingleMetricJobService {
this.chartData.line = [];
this.chartData.model = [];
this.chartData.swimlane = [];
this.chartData.hasBounds = false;
this.chartData.percentComplete = 0;
this.chartData.loadingDifference = 0;
this.chartData.eventRateHighestValue = 0;
this.chartData.totalResults = 0;
const aggType = formConfig.agg.type.dslName;
if (formConfig.field && formConfig.field.id) {
this.chartData.fieldFormat = mlFieldFormatService.getFieldFormatFromIndexPattern(
formConfig.indexPattern,
formConfig.field.id,
aggType);
} else {
delete this.chartData.fieldFormat;
}
const obj = {
success: true,
results: {}
};
const searchJson = getSearchJsonFromConfig(formConfig);
es.search(searchJson)
.then((resp) => {
const aggregationsByTime = _.get(resp, ['aggregations', 'times', 'buckets'], []);
let highestValue = 0;
_.each(aggregationsByTime, (dataForTime) => {
const time = dataForTime.key;
let value = _.get(dataForTime, ['field_value', 'value']);
if (value === undefined && formConfig.field !== null) {
value = _.get(dataForTime, ['field_value', 'values', ML_MEDIAN_PERCENTS]);
}
if (value === undefined && formConfig.field === null) {
value = dataForTime.doc_count;
}
if (!isFinite(value) || dataForTime.doc_count === 0) {
value = null;
}
if (value > highestValue) {
highestValue = value;
}
obj.results[time] = {
actual: value,
};
});
this.chartData.totalResults = resp.hits.total;
this.chartData.line = processLineChartResults(obj.results);
this.chartData.highestValue = Math.ceil(highestValue);
// Append extra 10px to width of tick label for highest axis value to allow for tick padding.
if (this.chartData.fieldFormat !== undefined) {
const highValueFormatted = this.chartData.fieldFormat.convert(this.chartData.highestValue, 'text');
this.chartData.chartTicksMargin.width = calculateTextWidth(highValueFormatted, false) + 10;
} else {
this.chartData.chartTicksMargin.width = calculateTextWidth(this.chartData.highestValue, true) + 10;
}
deferred.resolve(this.chartData);
})
.catch((resp) => {
deferred.reject(resp);
});
return deferred.promise;
};
function processLineChartResults(data, scale = 1) {
const lineData = [];
_.each(data, (dataForTime, t) => {
const time = +t;
const date = new Date(time);
lineData.push({
date: date,
time: time,
lower: (dataForTime.modelLower * scale),
value: dataForTime.actual,
upper: (dataForTime.modelUpper * scale)
});
});
return _.sortBy(lineData, 'time');
}
function processSwimlaneResults(bucketScoreData, init) {
// create a dataset in format used by the model plot chart.
// create empty swimlane dataset
// i.e. array of Objects with keys date (JavaScript date), value, lower and upper.
const swimlaneData = [];
_.each(bucketScoreData, (value, t) => {
const time = +t;
const date = new Date(time);
value = init ? 0 : value;
swimlaneData.push({
date,
time,
value,
color: ''
});
});
return swimlaneData;
}
function getSearchJsonFromConfig(formConfig) {
const interval = formConfig.chartInterval.getInterval().asMilliseconds() + 'ms';
// clone the query as we're modifying it
const query = _.cloneDeep(formConfig.combinedQuery);
const json = {
'index': formConfig.indexPattern.title,
'size': 0,
'body': {
'query': {},
'aggs': {
'times': {
'date_histogram': {
'field': formConfig.timeField,
'interval': interval,
'min_doc_count': 0,
'extended_bounds': {
'min': formConfig.start,
'max': formConfig.end,
}
}
}
}
}
};
query.bool.must.push({
'range': {
[formConfig.timeField]: {
'gte': formConfig.start,
'lte': formConfig.end,
'format': formConfig.format
}
}
});
json.body.query = query;
if (formConfig.field !== null) {
json.body.aggs.times.aggs = {
'field_value': {
[formConfig.agg.type.dslName]: { field: formConfig.field.name }
}
constructor() {
this.chartData = {
line: [],
model: [],
swimlane: [],
hasBounds: false,
percentComplete: 0,
highestValue: 0,
chartTicksMargin: { width: 30 },
totalResults: 0
};
this.job = {};
}
return json;
}
getLineChartResults(formConfig) {
return $q((resolve, reject) => {
function createJobForSaving(job) {
const newJob = angular.copy(job);
delete newJob.datafeed_config;
return newJob;
}
this.chartData.line = [];
this.chartData.model = [];
this.chartData.swimlane = [];
this.chartData.hasBounds = false;
this.chartData.percentComplete = 0;
this.chartData.loadingDifference = 0;
this.chartData.eventRateHighestValue = 0;
this.chartData.totalResults = 0;
this.getJobFromConfig = function (formConfig) {
const job = mlJobService.getBlankJob();
job.data_description.time_field = formConfig.timeField;
let func = formConfig.agg.type.mlName;
if (formConfig.isSparseData) {
if (formConfig.agg.type.dslName === 'count') {
func = func.replace(/count/, 'non_zero_count');
} else if(formConfig.agg.type.dslName === 'sum') {
func = func.replace(/sum/, 'non_null_sum');
}
}
const dtr = {
function: func
};
let query = {
match_all: {}
};
if (formConfig.query.query_string.query !== '*' || formConfig.filters.length) {
query = formConfig.combinedQuery;
}
if (formConfig.field && formConfig.field.id) {
dtr.field_name = formConfig.field.id;
}
job.analysis_config.detectors.push(dtr);
job.analysis_config.bucket_span = formConfig.bucketSpan;
job.analysis_limits = {
model_memory_limit: formConfig.modelMemoryLimit
};
delete job.data_description.field_delimiter;
delete job.data_description.quote_character;
delete job.data_description.time_format;
delete job.data_description.format;
const bucketSpanSeconds = parseInterval(formConfig.bucketSpan).asSeconds();
const indices = formConfig.indexPattern.title.split(',').map(i => i.trim());
job.datafeed_config = {
query,
indices,
};
job.job_id = formConfig.jobId;
job.description = formConfig.description;
job.groups = formConfig.jobGroups;
job.model_plot_config = {
enabled: true
};
if (formConfig.useDedicatedIndex) {
job.results_index_name = job.job_id;
}
// Use the original es agg type rather than the ML version
// e.g. count rather than high_count
const aggType = formConfig.agg.type.dslName;
const interval = bucketSpanSeconds * 1000;
switch (aggType) {
case 'count':
job.analysis_config.summary_count_field_name = 'doc_count';
job.datafeed_config.aggregations = {
buckets: {
date_histogram: {
field: formConfig.timeField,
interval: interval
},
aggregations: {
[formConfig.timeField]: {
max: {
field: formConfig.timeField
}
}
}
}
};
break;
case 'avg':
case 'median':
case 'sum':
case 'min':
case 'max':
job.analysis_config.summary_count_field_name = 'doc_count';
job.datafeed_config.aggregations = {
buckets: {
date_histogram: {
field: formConfig.timeField,
interval: ((interval / 100) * 10) // use 10% of bucketSpan to allow for better sampling
},
aggregations: {
[dtr.field_name]: {
[aggType]: {
field: formConfig.field.name
}
},
[formConfig.timeField]: {
max: {
field: formConfig.timeField
}
}
}
}
};
break;
case 'cardinality':
job.analysis_config.summary_count_field_name = 'dc_' + dtr.field_name;
job.datafeed_config.aggregations = {
buckets: {
date_histogram: {
field: formConfig.timeField,
interval: interval
},
aggregations: {
[formConfig.timeField]: {
max: {
field: formConfig.timeField
}
},
[job.analysis_config.summary_count_field_name]: {
[aggType]: {
field: formConfig.field.name
}
}
}
}
};
// finally, modify the detector before saving
dtr.function = 'non_zero_count';
// add a description using the original function name rather 'non_zero_count'
// as the user may not be aware it's been changed
dtr.detector_description = `${func} (${dtr.field_name})`;
delete dtr.field_name;
break;
default:
break;
}
console.log('auto created job: ', job);
return job;
};
this.createJob = function (formConfig) {
const deferred = $q.defer();
this.job = this.getJobFromConfig(formConfig);
const job = createJobForSaving(this.job);
// DO THE SAVE
mlJobService.saveNewJob(job)
.then((resp) => {
if (resp.success) {
deferred.resolve(this.job);
const aggType = formConfig.agg.type.dslName;
if (formConfig.field && formConfig.field.id) {
this.chartData.fieldFormat = fieldFormatService.getFieldFormatFromIndexPattern(
formConfig.indexPattern,
formConfig.field.id,
aggType);
} else {
deferred.reject(resp);
delete this.chartData.fieldFormat;
}
const obj = {
success: true,
results: {}
};
const searchJson = getSearchJsonFromConfig(formConfig);
es.search(searchJson)
.then((resp) => {
const aggregationsByTime = _.get(resp, ['aggregations', 'times', 'buckets'], []);
let highestValue = 0;
_.each(aggregationsByTime, (dataForTime) => {
const time = dataForTime.key;
let value = _.get(dataForTime, ['field_value', 'value']);
if (value === undefined && formConfig.field !== null) {
value = _.get(dataForTime, ['field_value', 'values', ML_MEDIAN_PERCENTS]);
}
if (value === undefined && formConfig.field === null) {
value = dataForTime.doc_count;
}
if (!isFinite(value) || dataForTime.doc_count === 0) {
value = null;
}
if (value > highestValue) {
highestValue = value;
}
obj.results[time] = {
actual: value,
};
});
this.chartData.totalResults = resp.hits.total;
this.chartData.line = processLineChartResults(obj.results);
this.chartData.highestValue = Math.ceil(highestValue);
// Append extra 10px to width of tick label for highest axis value to allow for tick padding.
if (this.chartData.fieldFormat !== undefined) {
const highValueFormatted = this.chartData.fieldFormat.convert(this.chartData.highestValue, 'text');
this.chartData.chartTicksMargin.width = calculateTextWidth(highValueFormatted, false) + 10;
} else {
this.chartData.chartTicksMargin.width = calculateTextWidth(this.chartData.highestValue, true) + 10;
}
resolve(this.chartData);
})
.catch((resp) => {
reject(resp);
});
});
return deferred.promise;
};
this.startDatafeed = function (formConfig) {
const datafeedId = mlJobService.getDatafeedId(formConfig.jobId);
return mlJobService.startDatafeed(datafeedId, formConfig.jobId, formConfig.start, formConfig.end);
};
this.stopDatafeed = function (formConfig) {
const datafeedId = mlJobService.getDatafeedId(formConfig.jobId);
return mlJobService.stopDatafeed(datafeedId, formConfig.jobId);
};
this.checkDatafeedState = function (formConfig) {
return mlJobService.updateSingleJobDatafeedState(formConfig.jobId);
};
this.loadModelData = function (formConfig) {
const deferred = $q.defer();
let start = formConfig.start;
if (this.chartData.model.length > 5) {
// only load the model since the end of the last time we checked
// but discard the last 5 buckets in case the model has changed
start = this.chartData.model[this.chartData.model.length - 5].time;
for (let i = 0; i < 5; i++) {
this.chartData.model.pop();
}
}
// Obtain the model plot data, passing 0 for the detectorIndex and empty list of partitioning fields.
mlResultsService.getModelPlotOutput(
formConfig.jobId,
0,
[],
start,
formConfig.end,
formConfig.resultsIntervalSeconds + 's',
formConfig.agg.type.mlModelPlotAgg
)
.then(data => {
// for count, scale the model upper and lower by the
// ratio of chart interval to bucketspan.
// this will force the model bounds to be drawn in the correct location
let scale = 1;
if (formConfig &&
(formConfig.agg.type.mlName === 'count' ||
formConfig.agg.type.mlName === 'high_count' ||
formConfig.agg.type.mlName === 'low_count' ||
formConfig.agg.type.mlName === 'distinct_count')) {
const chartIntervalSeconds = formConfig.chartInterval.getInterval().asSeconds();
const bucketSpan = parseInterval(formConfig.bucketSpan);
if (bucketSpan !== null) {
scale = chartIntervalSeconds / bucketSpan.asSeconds();
getJobFromConfig(formConfig) {
const job = mlJobService.getBlankJob();
job.data_description.time_field = formConfig.timeField;
let func = formConfig.agg.type.mlName;
if (formConfig.isSparseData) {
if (formConfig.agg.type.dslName === 'count') {
func = func.replace(/count/, 'non_zero_count');
} else if(formConfig.agg.type.dslName === 'sum') {
func = func.replace(/sum/, 'non_null_sum');
}
}
const dtr = {
function: func
};
let query = {
match_all: {}
};
if (formConfig.query.query_string.query !== '*' || formConfig.filters.length) {
query = formConfig.combinedQuery;
}
if (formConfig.field && formConfig.field.id) {
dtr.field_name = formConfig.field.id;
}
job.analysis_config.detectors.push(dtr);
job.analysis_config.bucket_span = formConfig.bucketSpan;
job.analysis_limits = {
model_memory_limit: formConfig.modelMemoryLimit
};
delete job.data_description.field_delimiter;
delete job.data_description.quote_character;
delete job.data_description.time_format;
delete job.data_description.format;
const bucketSpanSeconds = parseInterval(formConfig.bucketSpan).asSeconds();
const indices = formConfig.indexPattern.title.split(',').map(i => i.trim());
job.datafeed_config = {
query,
indices,
};
job.job_id = formConfig.jobId;
job.description = formConfig.description;
job.groups = formConfig.jobGroups;
job.model_plot_config = {
enabled: true
};
if (formConfig.useDedicatedIndex) {
job.results_index_name = job.job_id;
}
// Use the original es agg type rather than the ML version
// e.g. count rather than high_count
const aggType = formConfig.agg.type.dslName;
const interval = bucketSpanSeconds * 1000;
switch (aggType) {
case 'count':
job.analysis_config.summary_count_field_name = 'doc_count';
job.datafeed_config.aggregations = {
buckets: {
date_histogram: {
field: formConfig.timeField,
interval: interval
},
aggregations: {
[formConfig.timeField]: {
max: {
field: formConfig.timeField
}
}
}
}
};
break;
case 'avg':
case 'median':
case 'sum':
case 'min':
case 'max':
job.analysis_config.summary_count_field_name = 'doc_count';
job.datafeed_config.aggregations = {
buckets: {
date_histogram: {
field: formConfig.timeField,
interval: ((interval / 100) * 10) // use 10% of bucketSpan to allow for better sampling
},
aggregations: {
[dtr.field_name]: {
[aggType]: {
field: formConfig.field.name
}
},
[formConfig.timeField]: {
max: {
field: formConfig.timeField
}
}
}
}
};
break;
case 'cardinality':
job.analysis_config.summary_count_field_name = 'dc_' + dtr.field_name;
job.datafeed_config.aggregations = {
buckets: {
date_histogram: {
field: formConfig.timeField,
interval: interval
},
aggregations: {
[formConfig.timeField]: {
max: {
field: formConfig.timeField
}
},
[job.analysis_config.summary_count_field_name]: {
[aggType]: {
field: formConfig.field.name
}
}
}
}
};
// finally, modify the detector before saving
dtr.function = 'non_zero_count';
// add a description using the original function name rather 'non_zero_count'
// as the user may not be aware it's been changed
dtr.detector_description = `${func} (${dtr.field_name})`;
delete dtr.field_name;
break;
default:
break;
}
return job;
}
createJob(formConfig) {
return $q((resolve, reject) => {
this.job = this.getJobFromConfig(formConfig);
const job = createJobForSaving(this.job);
// DO THE SAVE
mlJobService.saveNewJob(job)
.then((resp) => {
if (resp.success) {
resolve(this.job);
} else {
reject(resp);
}
});
});
}
startDatafeed(formConfig) {
const datafeedId = mlJobService.getDatafeedId(formConfig.jobId);
return mlJobService.startDatafeed(datafeedId, formConfig.jobId, formConfig.start, formConfig.end);
}
stopDatafeed(formConfig) {
const datafeedId = mlJobService.getDatafeedId(formConfig.jobId);
return mlJobService.stopDatafeed(datafeedId, formConfig.jobId);
}
checkDatafeedState(formConfig) {
return mlJobService.updateSingleJobDatafeedState(formConfig.jobId);
}
loadModelData(formConfig) {
return $q((resolve, reject) => {
let start = formConfig.start;
if (this.chartData.model.length > 5) {
// only load the model since the end of the last time we checked
// but discard the last 5 buckets in case the model has changed
start = this.chartData.model[this.chartData.model.length - 5].time;
for (let i = 0; i < 5; i++) {
this.chartData.model.pop();
}
}
this.chartData.model = this.chartData.model.concat(processLineChartResults(data.results, scale));
// Obtain the model plot data, passing 0 for the detectorIndex and empty list of partitioning fields.
mlResultsService.getModelPlotOutput(
formConfig.jobId,
0,
[],
start,
formConfig.end,
formConfig.resultsIntervalSeconds + 's',
formConfig.agg.type.mlModelPlotAgg
)
.then(data => {
// for count, scale the model upper and lower by the
// ratio of chart interval to bucketspan.
// this will force the model bounds to be drawn in the correct location
let scale = 1;
if (formConfig &&
(formConfig.agg.type.mlName === 'count' ||
formConfig.agg.type.mlName === 'high_count' ||
formConfig.agg.type.mlName === 'low_count' ||
formConfig.agg.type.mlName === 'distinct_count')) {
const chartIntervalSeconds = formConfig.chartInterval.getInterval().asSeconds();
const bucketSpan = parseInterval(formConfig.bucketSpan);
if (bucketSpan !== null) {
scale = chartIntervalSeconds / bucketSpan.asSeconds();
}
}
const lastBucket = this.chartData.model[this.chartData.model.length - 1];
const time = (lastBucket !== undefined) ? lastBucket.time : formConfig.start;
this.chartData.model = this.chartData.model.concat(processLineChartResults(data.results, scale));
const pcnt = ((time - formConfig.start + formConfig.resultsIntervalSeconds) / (formConfig.end - formConfig.start) * 100);
this.chartData.percentComplete = Math.round(pcnt);
const lastBucket = this.chartData.model[this.chartData.model.length - 1];
const time = (lastBucket !== undefined) ? lastBucket.time : formConfig.start;
const pcnt = ((time - formConfig.start + formConfig.resultsIntervalSeconds) / (formConfig.end - formConfig.start) * 100);
this.chartData.percentComplete = Math.round(pcnt);
resolve(this.chartData);
})
.catch(() => {
reject(this.chartData);
});
deferred.resolve(this.chartData);
})
.catch(() => {
deferred.reject(this.chartData);
});
}
return deferred.promise;
loadSwimlaneData(formConfig) {
return $q((resolve) => {
mlResultsService.getScoresByBucket(
[formConfig.jobId],
formConfig.start,
formConfig.end,
formConfig.resultsIntervalSeconds + 's',
1
)
.then((data) => {
const jobResults = data.results[formConfig.jobId];
this.chartData.swimlane = processSwimlaneResults(jobResults);
this.chartData.swimlaneInterval = formConfig.resultsIntervalSeconds * 1000;
resolve(this.chartData);
})
.catch(() => {
resolve(this.chartData);
});
});
}
}
return new SingleMetricJobService();
}
function processLineChartResults(data, scale = 1) {
const lineData = [];
_.each(data, (dataForTime, t) => {
const time = +t;
const date = new Date(time);
lineData.push({
date: date,
time: time,
lower: (dataForTime.modelLower * scale),
value: dataForTime.actual,
upper: (dataForTime.modelUpper * scale)
});
});
return _.sortBy(lineData, 'time');
}
function processSwimlaneResults(bucketScoreData, init) {
// create a dataset in format used by the model plot chart.
// create empty swimlane dataset
// i.e. array of Objects with keys date (JavaScript date), value, lower and upper.
const swimlaneData = [];
_.each(bucketScoreData, (value, t) => {
const time = +t;
const date = new Date(time);
value = init ? 0 : value;
swimlaneData.push({
date,
time,
value,
color: ''
});
});
return swimlaneData;
}
function getSearchJsonFromConfig(formConfig) {
const interval = formConfig.chartInterval.getInterval().asMilliseconds() + 'ms';
// clone the query as we're modifying it
const query = _.cloneDeep(formConfig.combinedQuery);
const json = {
index: formConfig.indexPattern.title,
size: 0,
body: {
query: {},
aggs: {
times: {
date_histogram: {
field: formConfig.timeField,
interval: interval,
min_doc_count: 0,
extended_bounds: {
min: formConfig.start,
max: formConfig.end,
}
}
}
}
}
};
this.loadSwimlaneData = function (formConfig) {
const deferred = $q.defer();
query.bool.must.push({
range: {
[formConfig.timeField]: {
gte: formConfig.start,
lte: formConfig.end,
format: formConfig.format
}
}
});
mlResultsService.getScoresByBucket(
[formConfig.jobId],
formConfig.start,
formConfig.end,
formConfig.resultsIntervalSeconds + 's',
1
)
.then((data) => {
const jobResults = data.results[formConfig.jobId];
this.chartData.swimlane = processSwimlaneResults(jobResults);
this.chartData.swimlaneInterval = formConfig.resultsIntervalSeconds * 1000;
deferred.resolve(this.chartData);
})
.catch(() => {
deferred.resolve(this.chartData);
});
json.body.query = query;
return deferred.promise;
};
});
if (formConfig.field !== null) {
json.body.aggs.times.aggs = {
field_value: {
[formConfig.agg.type.dslName]: { field: formConfig.field.name }
}
};
}
return json;
}

View file

@ -4,7 +4,7 @@
* you may not use this file except in compliance with the Elastic License.
*/
import { ml } from 'plugins/ml/services/ml_api_service';
let defaults = {
anomaly_detectors: {},
@ -12,7 +12,7 @@ let defaults = {
};
let limits = {};
export function loadNewJobDefaults(ml) {
export function loadNewJobDefaults() {
return new Promise((resolve) => {
ml.mlInfo()
.then((resp) => {

View file

@ -10,6 +10,7 @@ import _ from 'lodash';
import moment from 'moment';
import { migrateFilter } from 'ui/courier/data_source/_migrate_filter.js';
import { addItemToRecentlyAccessed } from 'plugins/ml/util/recently_accessed';
import { JobServiceProvider } from 'plugins/ml/services/job_service';
export function getQueryFromSavedSearch(formConfig) {
const must = [];
@ -103,12 +104,19 @@ export function createResultsUrl(jobIds, start, end, resultsPage) {
return path;
}
export function createJobForSaving(job) {
const newJob = _.cloneDeep(job);
delete newJob.datafeed_config;
return newJob;
}
export function addNewJobToRecentlyAccessed(jobId, resultsUrl) {
const urlParts = resultsUrl.match(/ml#\/(.+?)(\?.+)/);
addItemToRecentlyAccessed(urlParts[1], jobId, urlParts[2]);
}
export function moveToAdvancedJobCreationProvider(mlJobService, $location) {
export function moveToAdvancedJobCreationProvider(Private, $location) {
const mlJobService = Private(JobServiceProvider);
return function moveToAdvancedJobCreation(job) {
mlJobService.currentJob = job;
$location.path('jobs/new_job/advanced');

View file

@ -4,13 +4,13 @@
* you may not use this file except in compliance with the Elastic License.
*/
import { ml } from 'plugins/ml/services/ml_api_service';
let mlNodeCount = 0;
let userHasPermissionToViewMlNodeCount = false;
export function checkMlNodesAvailable(ml, kbnUrl) {
getMlNodeCount(ml).then((nodes) => {
export function checkMlNodesAvailable(kbnUrl) {
getMlNodeCount().then((nodes) => {
if (nodes.count !== undefined && nodes.count > 0) {
Promise.resolve();
} else {
@ -20,7 +20,7 @@ export function checkMlNodesAvailable(ml, kbnUrl) {
});
}
export function getMlNodeCount(ml) {
export function getMlNodeCount() {
return new Promise((resolve) => {
ml.mlNodeCount()
.then((nodes) => {

View file

@ -5,8 +5,9 @@
*/
import { ml } from 'plugins/ml/services/ml_api_service';
export function privilegesProvider(Promise, ml) {
export function privilegesProvider() {
function getPrivileges() {
const privileges = {

View file

@ -8,79 +8,92 @@
import _ from 'lodash';
import { uiModules } from 'ui/modules';
const module = uiModules.get('apps/ml');
import { ml } from 'plugins/ml/services/ml_api_service';
import { JobServiceProvider } from 'plugins/ml/services/job_service';
module.service('mlCalendarService', function ($q, ml, mlJobService, mlMessageBarService) {
let calendarService = undefined;
export function CalendarServiceProvider($q, Private, mlMessageBarService) {
const msgs = mlMessageBarService;
this.calendars = [];
// list of calendar ids per job id
this.jobCalendars = {};
// list of calendar ids per group id
this.groupCalendars = {};
const mlJobService = Private(JobServiceProvider);
this.loadCalendars = function (jobs) {
return $q((resolve, reject) => {
let calendars = [];
jobs.forEach((j) => {
this.jobCalendars[j.job_id] = [];
});
const groups = {};
mlJobService.getJobGroups().forEach((g) => {
groups[g.id] = g;
});
class CalendarService {
constructor() {
this.calendars = [];
// list of calendar ids per job id
this.jobCalendars = {};
// list of calendar ids per group id
this.groupCalendars = {};
}
ml.calendars()
.then((resp) => {
calendars = resp;
// loop through calendars and their job_ids and create jobCalendars
// if a group is found, expand it out to its member jobs
calendars.forEach((cal) => {
cal.job_ids.forEach((id) => {
let isGroup = false;
// the job_id could be either a job id or a group id
if (this.jobCalendars[id] !== undefined) {
this.jobCalendars[id].push(cal.calendar_id);
} else if (groups[id] !== undefined) {
isGroup = true;
// expand out the group into its jobs and add each job
groups[id].jobs.forEach((j) => {
this.jobCalendars[j.job_id].push(cal.calendar_id);
});
} else {
// not a known job or a known group. assume it's a unused group
isGroup = true;
}
if (isGroup) {
// keep track of calendars per group
if (this.groupCalendars[id] === undefined) {
this.groupCalendars[id] = [cal.calendar_id];
} else {
this.groupCalendars[id].push(cal.calendar_id);
}
}
});
});
// deduplicate as group expansion may have added dupes.
_.each(this.jobCalendars, (cal, id) => {
this.jobCalendars[id] = _.uniq(cal);
});
this.calendars = calendars;
resolve({ calendars });
})
.catch((err) => {
msgs.error('Calendars list could not be retrieved');
msgs.error('', err);
reject({ calendars, err });
loadCalendars(jobs) {
return $q((resolve, reject) => {
let calendars = [];
jobs.forEach((j) => {
this.jobCalendars[j.job_id] = [];
});
const groups = {};
mlJobService.getJobGroups().forEach((g) => {
groups[g.id] = g;
});
});
};
// get the list of calendar groups
this.getCalendarGroups = function () {
return Object.keys(this.groupCalendars).map(gId => ({ id: gId }));
};
});
ml.calendars()
.then((resp) => {
calendars = resp;
// loop through calendars and their job_ids and create jobCalendars
// if a group is found, expand it out to its member jobs
calendars.forEach((cal) => {
cal.job_ids.forEach((id) => {
let isGroup = false;
// the job_id could be either a job id or a group id
if (this.jobCalendars[id] !== undefined) {
this.jobCalendars[id].push(cal.calendar_id);
} else if (groups[id] !== undefined) {
isGroup = true;
// expand out the group into its jobs and add each job
groups[id].jobs.forEach((j) => {
this.jobCalendars[j.job_id].push(cal.calendar_id);
});
} else {
// not a known job or a known group. assume it's a unused group
isGroup = true;
}
if (isGroup) {
// keep track of calendars per group
if (this.groupCalendars[id] === undefined) {
this.groupCalendars[id] = [cal.calendar_id];
} else {
this.groupCalendars[id].push(cal.calendar_id);
}
}
});
});
// deduplicate as group expansion may have added dupes.
_.each(this.jobCalendars, (cal, id) => {
this.jobCalendars[id] = _.uniq(cal);
});
this.calendars = calendars;
resolve({ calendars });
})
.catch((err) => {
msgs.error('Calendars list could not be retrieved');
msgs.error('', err);
reject({ calendars, err });
});
});
}
// get the list of calendar groups
getCalendarGroups() {
return Object.keys(this.groupCalendars).map(id => ({ id }));
}
}
if (calendarService === undefined) {
calendarService = new CalendarService();
}
return calendarService;
}

View file

@ -11,124 +11,122 @@ import _ from 'lodash';
import 'ui/courier';
import { mlFunctionToESAggregation } from 'plugins/ml/../common/util/job_utils';
import { getIndexPatternProvider } from 'plugins/ml/util/index_utils';
import { uiModules } from 'ui/modules';
const module = uiModules.get('apps/ml');
import { JobServiceProvider } from 'plugins/ml/services/job_service';
// Service for accessing FieldFormat objects configured for a Kibana index pattern
// for use in formatting the actual and typical values from anomalies.
module.service('mlFieldFormatService', function (
export function FieldFormatServiceProvider(
$q,
courier,
Private,
mlJobService) {
const indexPatternIdsByJob = {};
const formatsByJob = {};
Private) {
const mlJobService = Private(JobServiceProvider);
const getIndexPattern = Private(getIndexPatternProvider);
// Populate the service with the FieldFormats for the list of jobs with the
// specified IDs. List of Kibana index patterns is passed, with a title
// attribute set in each pattern which will be compared to the index pattern
// configured in the datafeed of each job.
// Builds a map of Kibana FieldFormats (ui/field_formats/field_format.js)
// against detector index by job ID.
this.populateFormats = function (jobIds, indexPatterns) {
const deferred = $q.defer();
// Populate a map of index pattern IDs against job ID, by finding the ID of the index
// pattern with a title attribute which matches the index configured in the datafeed.
// If a Kibana index pattern has not been created
// for this index, then no custom field formatting will occur.
_.each(jobIds, (jobId) => {
const jobObj = mlJobService.getJob(jobId);
const datafeedIndices = jobObj.datafeed_config.indices;
const indexPattern = _.find(indexPatterns, (index) => {
return _.find(datafeedIndices, (datafeedIndex) => {
return index.get('title') === datafeedIndex;
});
});
// Check if index pattern has been configured to match the index in datafeed.
if (indexPattern !== undefined) {
indexPatternIdsByJob[jobId] = indexPattern.id;
}
});
const promises = jobIds.map(jobId => $q.all([
getFormatsForJob(jobId)
]));
$q.all(promises).then((fmtsByJobByDetector) => {
_.each(fmtsByJobByDetector, (formatsByDetector, index) => {
formatsByJob[jobIds[index]] = formatsByDetector[0];
});
deferred.resolve(formatsByJob);
}).catch(err => {
console.log('mlFieldFormatService error populating formats:', err);
deferred.reject({ formats: {}, err });
});
return deferred.promise;
};
// Return the FieldFormat to use for formatting values from
// the detector from the job with the specified ID.
this.getFieldFormat = function (jobId, detectorIndex) {
return _.get(formatsByJob, [jobId, detectorIndex]);
};
// Utility for returning the FieldFormat from a full populated Kibana index pattern object
// containing the list of fields by name with their formats.
this.getFieldFormatFromIndexPattern = function (fullIndexPattern, fieldName, esAggName) {
// Don't use the field formatter for distinct count detectors as
// e.g. distinct_count(clientip) should be formatted as a count, not as an IP address.
let fieldFormat = undefined;
if (esAggName !== 'cardinality') {
const indexPatternFields = _.get(fullIndexPattern, 'fields.byName', []);
fieldFormat = _.get(indexPatternFields, [fieldName, 'format']);
class FieldFormatService {
constructor() {
this.indexPatternIdsByJob = {};
this.formatsByJob = {};
}
return fieldFormat;
};
function getFormatsForJob(jobId) {
const deferred = $q.defer();
const jobObj = mlJobService.getJob(jobId);
const detectors = jobObj.analysis_config.detectors || [];
const formatsByDetector = {};
const indexPatternId = indexPatternIdsByJob[jobId];
if (indexPatternId !== undefined) {
// Load the full index pattern configuration to obtain the formats of each field.
getIndexPattern(indexPatternId)
.then((indexPatternData) => {
// Store the FieldFormat for each job by detector_index.
const fieldsByName = _.get(indexPatternData, 'fields.byName', []);
_.each(detectors, (dtr) => {
const esAgg = mlFunctionToESAggregation(dtr.function);
// distinct_count detectors should fall back to the default
// formatter as the values are just counts.
if (dtr.field_name !== undefined && esAgg !== 'cardinality') {
formatsByDetector[dtr.detector_index] = _.get(fieldsByName, [dtr.field_name, 'format']);
}
// Populate the service with the FieldFormats for the list of jobs with the
// specified IDs. List of Kibana index patterns is passed, with a title
// attribute set in each pattern which will be compared to the index pattern
// configured in the datafeed of each job.
// Builds a map of Kibana FieldFormats (ui/field_formats/field_format.js)
// against detector index by job ID.
populateFormats(jobIds, indexPatterns) {
return $q((resolve, reject) => {
// Populate a map of index pattern IDs against job ID, by finding the ID of the index
// pattern with a title attribute which matches the index configured in the datafeed.
// If a Kibana index pattern has not been created
// for this index, then no custom field formatting will occur.
_.each(jobIds, (jobId) => {
const jobObj = mlJobService.getJob(jobId);
const datafeedIndices = jobObj.datafeed_config.indices;
const indexPattern = _.find(indexPatterns, (index) => {
return _.find(datafeedIndices, (datafeedIndex) => {
return index.get('title') === datafeedIndex;
});
});
deferred.resolve(formatsByDetector);
}).catch(err => {
deferred.reject(err);
// Check if index pattern has been configured to match the index in datafeed.
if (indexPattern !== undefined) {
this.indexPatternIdsByJob[jobId] = indexPattern.id;
}
});
return deferred.promise;
} else {
deferred.resolve(formatsByDetector);
const promises = jobIds.map(jobId => $q.all([
this.getFormatsForJob(jobId)
]));
$q.all(promises).then((fmtsByJobByDetector) => {
_.each(fmtsByJobByDetector, (formatsByDetector, index) => {
this.formatsByJob[jobIds[index]] = formatsByDetector[0];
});
resolve(this.formatsByJob);
}).catch(err => {
console.log('fieldFormatService error populating formats:', err);
reject({ formats: {}, err });
});
});
}
// Return the FieldFormat to use for formatting values from
// the detector from the job with the specified ID.
getFieldFormat(jobId, detectorIndex) {
return _.get(this.formatsByJob, [jobId, detectorIndex]);
}
// Utility for returning the FieldFormat from a full populated Kibana index pattern object
// containing the list of fields by name with their formats.
getFieldFormatFromIndexPattern(fullIndexPattern, fieldName, esAggName) {
// Don't use the field formatter for distinct count detectors as
// e.g. distinct_count(clientip) should be formatted as a count, not as an IP address.
let fieldFormat = undefined;
if (esAggName !== 'cardinality') {
const indexPatternFields = _.get(fullIndexPattern, 'fields.byName', []);
fieldFormat = _.get(indexPatternFields, [fieldName, 'format']);
}
return fieldFormat;
}
getFormatsForJob(jobId) {
return $q((resolve, reject) => {
const jobObj = mlJobService.getJob(jobId);
const detectors = jobObj.analysis_config.detectors || [];
const formatsByDetector = {};
const indexPatternId = this.indexPatternIdsByJob[jobId];
if (indexPatternId !== undefined) {
// Load the full index pattern configuration to obtain the formats of each field.
getIndexPattern(indexPatternId)
.then((indexPatternData) => {
// Store the FieldFormat for each job by detector_index.
const fieldsByName = _.get(indexPatternData, 'fields.byName', []);
_.each(detectors, (dtr) => {
const esAgg = mlFunctionToESAggregation(dtr.function);
// distinct_count detectors should fall back to the default
// formatter as the values are just counts.
if (dtr.field_name !== undefined && esAgg !== 'cardinality') {
formatsByDetector[dtr.detector_index] = _.get(fieldsByName, [dtr.field_name, 'format']);
}
});
resolve(formatsByDetector);
}).catch(err => {
reject(err);
});
} else {
resolve(formatsByDetector);
}
});
}
}
});
return new FieldFormatService();
}

View file

@ -1,57 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
// Service for carrying out queries to obtain data
// specific to fields in Elasticsearch indices.
export function FieldsServiceProvider(es, ml) {
// Obtains the cardinality of one or more fields.
// Returns an Object whose keys are the names of the fields,
// with values equal to the cardinality of the field.
function getCardinalityOfFields(
index,
types,
fieldNames,
query,
timeFieldName,
earliestMs,
latestMs) {
return ml.getCardinalityOfFields({
index,
types,
fieldNames,
query,
timeFieldName,
earliestMs,
latestMs
});
}
// Returns the range of the specified time field.
// Returns an Object containing start and end properties,
// holding the value as an epoch (ms since the Unix epoch)
// and as a formatted string.
function getTimeFieldRange(
index,
timeFieldName,
query) {
return ml.getTimeFieldRange({
index,
timeFieldName,
query
});
}
return {
getCardinalityOfFields,
getTimeFieldRange
};
}

View file

@ -11,155 +11,150 @@
import _ from 'lodash';
import { ML_RESULTS_INDEX_PATTERN } from 'plugins/ml/constants/index_patterns';
import { ml } from 'plugins/ml/services/ml_api_service';
import { uiModules } from 'ui/modules';
const module = uiModules.get('apps/ml');
module.service('mlForecastService', function ($q, es, ml) {
export function ForecastServiceProvider(es, $q) {
// Gets a basic summary of the most recently run forecasts for the specified
// job, with results at or later than the supplied timestamp.
// Extra query object can be supplied, or pass null if no additional query.
// Returned response contains a forecasts property, which is an array of objects
// containing id, earliest and latest keys.
this.getForecastsSummary = function (
function getForecastsSummary(
job,
query,
earliestMs,
maxResults
) {
const deferred = $q.defer();
const obj = {
success: true,
forecasts: []
};
return $q((resolve, reject) => {
const obj = {
success: true,
forecasts: []
};
// Build the criteria to use in the bool filter part of the request.
// Add criteria for the job ID, result type and earliest time, plus
// the additional query if supplied.
const filterCriteria = [
{
term: { result_type: 'model_forecast_request_stats' }
},
{
term: { job_id: job.job_id }
},
{
range: {
timestamp: {
gte: earliestMs,
format: 'epoch_millis'
}
}
}
];
if (query) {
filterCriteria.push(query);
}
es.search({
index: ML_RESULTS_INDEX_PATTERN,
size: maxResults,
body: {
query: {
bool: {
filter: filterCriteria
}
// Build the criteria to use in the bool filter part of the request.
// Add criteria for the job ID, result type and earliest time, plus
// the additional query if supplied.
const filterCriteria = [
{
term: { result_type: 'model_forecast_request_stats' }
},
sort: [
{ forecast_create_timestamp: { 'order': 'desc' } }
]
}
})
.then((resp) => {
if (resp.hits.total !== 0) {
_.each(resp.hits.hits, (hit) => {
obj.forecasts.push(hit._source);
});
{
term: { job_id: job.job_id }
},
{
range: {
timestamp: {
gte: earliestMs,
format: 'epoch_millis'
}
}
}
];
deferred.resolve(obj);
if (query) {
filterCriteria.push(query);
}
es.search({
index: ML_RESULTS_INDEX_PATTERN,
size: maxResults,
body: {
query: {
bool: {
filter: filterCriteria
}
},
sort: [
{ forecast_create_timestamp: { 'order': 'desc' } }
]
}
})
.catch((resp) => {
deferred.reject(resp);
});
.then((resp) => {
if (resp.hits.total !== 0) {
obj.forecasts = resp.hits.hits.map(hit => hit._source);
}
return deferred.promise;
};
resolve(obj);
})
.catch((resp) => {
reject(resp);
});
});
}
// Obtains the earliest and latest timestamps for the forecast data from
// the forecast with the specified ID.
// Returned response contains earliest and latest properties which are the
// timestamps of the first and last model_forecast results.
this.getForecastDateRange = function (job, forecastId) {
function getForecastDateRange(job, forecastId) {
const deferred = $q.defer();
const obj = {
success: true,
earliest: null,
latest: null
};
return $q((resolve, reject) => {
const obj = {
success: true,
earliest: null,
latest: null
};
// Build the criteria to use in the bool filter part of the request.
// Add criteria for the job ID, forecast ID, result type and time range.
const filterCriteria = [{
query_string: {
query: 'result_type:model_forecast',
analyze_wildcard: true
}
},
{
term: { job_id: job.job_id }
},
{
term: { forecast_id: forecastId }
}];
// Build the criteria to use in the bool filter part of the request.
// Add criteria for the job ID, forecast ID, result type and time range.
const filterCriteria = [{
query_string: {
query: 'result_type:model_forecast',
analyze_wildcard: true
}
},
{
term: { job_id: job.job_id }
},
{
term: { forecast_id: forecastId }
}];
// TODO - add in criteria for detector index and entity fields (by, over, partition)
// once forecasting with these parameters is supported.
// TODO - add in criteria for detector index and entity fields (by, over, partition)
// once forecasting with these parameters is supported.
es.search({
index: ML_RESULTS_INDEX_PATTERN,
size: 0,
body: {
query: {
bool: {
filter: filterCriteria
}
},
aggs: {
earliest: {
min: {
field: 'timestamp'
es.search({
index: ML_RESULTS_INDEX_PATTERN,
size: 0,
body: {
query: {
bool: {
filter: filterCriteria
}
},
latest: {
max: {
field: 'timestamp'
aggs: {
earliest: {
min: {
field: 'timestamp'
}
},
latest: {
max: {
field: 'timestamp'
}
}
}
}
}
})
.then((resp) => {
obj.earliest = _.get(resp, 'aggregations.earliest.value', null);
obj.latest = _.get(resp, 'aggregations.latest.value', null);
if (obj.earliest === null || obj.latest === null) {
deferred.reject(resp);
} else {
deferred.resolve(obj);
}
})
.catch((resp) => {
deferred.reject(resp);
});
.then((resp) => {
obj.earliest = _.get(resp, 'aggregations.earliest.value', null);
obj.latest = _.get(resp, 'aggregations.latest.value', null);
if (obj.earliest === null || obj.latest === null) {
reject(resp);
} else {
resolve(obj);
}
})
.catch((resp) => {
reject(resp);
});
return deferred.promise;
};
});
}
// Obtains the requested forecast model data for the forecast with the specified ID.
this.getForecastData = function (
function getForecastData(
job,
detectorIndex,
forecastId,
@ -198,184 +193,192 @@ module.service('mlForecastService', function ($q, es, ml) {
}
}
const deferred = $q.defer();
const obj = {
success: true,
results: {}
};
// Build the criteria to use in the bool filter part of the request.
// Add criteria for the job ID, forecast ID, detector index, result type and time range.
const filterCriteria = [{
query_string: {
query: 'result_type:model_forecast',
analyze_wildcard: true
}
},
{
term: { job_id: job.job_id }
},
{
term: { forecast_id: forecastId }
},
{
term: { detector_index: detectorIndex }
},
{
range: {
timestamp: {
gte: earliestMs,
lte: latestMs,
format: 'epoch_millis'
}
}
}];
// Add in term queries for each of the specified criteria.
_.each(criteriaFields, (criteria) => {
filterCriteria.push({
term: {
[criteria.fieldName]: criteria.fieldValue
}
});
});
// If an aggType object has been passed in, use it.
// Otherwise default to avg, min and max aggs for the
// forecast prediction, upper and lower
const forecastAggs = (aggType === undefined) ?
{ avg: 'avg', max: 'max', min: 'min' } :
{
avg: aggType.avg,
max: aggType.max,
min: aggType.min
return $q((resolve, reject) => {
const obj = {
success: true,
results: {}
};
es.search({
index: ML_RESULTS_INDEX_PATTERN,
size: 0,
body: {
query: {
bool: {
filter: filterCriteria
// Build the criteria to use in the bool filter part of the request.
// Add criteria for the job ID, forecast ID, detector index, result type and time range.
const filterCriteria = [{
query_string: {
query: 'result_type:model_forecast',
analyze_wildcard: true
}
},
{
term: { job_id: job.job_id }
},
{
term: { forecast_id: forecastId }
},
{
term: { detector_index: detectorIndex }
},
{
range: {
timestamp: {
gte: earliestMs,
lte: latestMs,
format: 'epoch_millis'
}
},
aggs: {
times: {
date_histogram: {
field: 'timestamp',
interval: interval,
min_doc_count: 1
},
aggs: {
prediction: {
[forecastAggs.avg]: {
field: 'forecast_prediction'
}
}
}];
// Add in term queries for each of the specified criteria.
_.each(criteriaFields, (criteria) => {
filterCriteria.push({
term: {
[criteria.fieldName]: criteria.fieldValue
}
});
});
// If an aggType object has been passed in, use it.
// Otherwise default to avg, min and max aggs for the
// forecast prediction, upper and lower
const forecastAggs = (aggType === undefined) ?
{ avg: 'avg', max: 'max', min: 'min' } :
{
avg: aggType.avg,
max: aggType.max,
min: aggType.min
};
es.search({
index: ML_RESULTS_INDEX_PATTERN,
size: 0,
body: {
query: {
bool: {
filter: filterCriteria
}
},
aggs: {
times: {
date_histogram: {
field: 'timestamp',
interval: interval,
min_doc_count: 1
},
forecastUpper: {
[forecastAggs.max]: {
field: 'forecast_upper'
}
},
forecastLower: {
[forecastAggs.min]: {
field: 'forecast_lower'
aggs: {
prediction: {
[forecastAggs.avg]: {
field: 'forecast_prediction'
}
},
forecastUpper: {
[forecastAggs.max]: {
field: 'forecast_upper'
}
},
forecastLower: {
[forecastAggs.min]: {
field: 'forecast_lower'
}
}
}
}
}
}
}
})
.then((resp) => {
const aggregationsByTime = _.get(resp, ['aggregations', 'times', 'buckets'], []);
_.each(aggregationsByTime, (dataForTime) => {
const time = dataForTime.key;
obj.results[time] = {
prediction: _.get(dataForTime, ['prediction', 'value']),
forecastUpper: _.get(dataForTime, ['forecastUpper', 'value']),
forecastLower: _.get(dataForTime, ['forecastLower', 'value'])
};
})
.then((resp) => {
const aggregationsByTime = _.get(resp, ['aggregations', 'times', 'buckets'], []);
_.each(aggregationsByTime, (dataForTime) => {
const time = dataForTime.key;
obj.results[time] = {
prediction: _.get(dataForTime, ['prediction', 'value']),
forecastUpper: _.get(dataForTime, ['forecastUpper', 'value']),
forecastLower: _.get(dataForTime, ['forecastLower', 'value'])
};
});
resolve(obj);
})
.catch((resp) => {
reject(resp);
});
deferred.resolve(obj);
})
.catch((resp) => {
deferred.reject(resp);
});
return deferred.promise;
};
});
}
// Runs a forecast
this.runForecast = function (jobId, duration) {
function runForecast(jobId, duration) {
console.log('ML forecast service run forecast with duration:', duration);
const deferred = $q.defer();
return $q((resolve, reject) => {
ml.forecast({
jobId,
duration
})
.then((resp) => {
deferred.resolve(resp);
}).catch((err) => {
deferred.reject(err);
});
return deferred.promise;
};
ml.forecast({
jobId,
duration
})
.then((resp) => {
resolve(resp);
}).catch((err) => {
reject(err);
});
});
}
// Gets stats for a forecast that has been run on the specified job.
// Returned response contains a stats property, including
// forecast_progress (a value from 0 to 1),
// and forecast_status ('finished' when complete) properties.
this.getForecastRequestStats = function (job, forecastId) {
const deferred = $q.defer();
const obj = {
success: true,
stats: {}
};
function getForecastRequestStats(job, forecastId) {
return $q((resolve, reject) => {
const obj = {
success: true,
stats: {}
};
// Build the criteria to use in the bool filter part of the request.
// Add criteria for the job ID, result type and earliest time.
const filterCriteria = [{
query_string: {
query: 'result_type:model_forecast_request_stats',
analyze_wildcard: true
}
},
{
term: { job_id: job.job_id }
},
{
term: { forecast_id: forecastId }
}];
// Build the criteria to use in the bool filter part of the request.
// Add criteria for the job ID, result type and earliest time.
const filterCriteria = [{
query_string: {
query: 'result_type:model_forecast_request_stats',
analyze_wildcard: true
}
},
{
term: { job_id: job.job_id }
},
{
term: { forecast_id: forecastId }
}];
es.search({
index: ML_RESULTS_INDEX_PATTERN,
size: 1,
body: {
query: {
bool: {
filter: filterCriteria
es.search({
index: ML_RESULTS_INDEX_PATTERN,
size: 1,
body: {
query: {
bool: {
filter: filterCriteria
}
}
}
}
})
.then((resp) => {
if (resp.hits.total !== 0) {
obj.stats = _.first(resp.hits.hits)._source;
}
deferred.resolve(obj);
})
.catch((resp) => {
deferred.reject(resp);
});
.then((resp) => {
if (resp.hits.total !== 0) {
obj.stats = _.first(resp.hits.hits)._source;
}
resolve(obj);
})
.catch((resp) => {
reject(resp);
});
return deferred.promise;
});
}
return {
getForecastsSummary,
getForecastDateRange,
getForecastData,
runForecast,
getForecastRequestStats
};
});
}

View file

@ -8,43 +8,39 @@
// service for interacting with the server
import { uiModules } from 'ui/modules';
const module = uiModules.get('apps/ml');
import chrome from 'ui/chrome';
import 'isomorphic-fetch';
import { addSystemApiHeader } from 'ui/system_api';
module.service('prlHttpService', function ($http, $q) {
// request function returns a promise
// once resolved, just the data response is returned
this.request = function (options) {
export function http(options) {
return new Promise((resolve, reject) => {
if(options && options.url) {
let url = '';
url = url + (options.url || '');
const headers = addSystemApiHeader({});
const allHeaders = (options.headers === undefined) ?
headers :
Object.assign(options.headers, headers);
const headers = addSystemApiHeader({
'Content-Type': 'application/json',
'kbn-version': chrome.getXsrfToken(),
...options.headers
});
const allHeaders = (options.headers === undefined) ? headers : { ...options.headers, ...headers };
const body = (options.data === undefined) ? null : JSON.stringify(options.data);
const deferred = $q.defer();
$http({
url: url,
fetch(url, {
method: (options.method || 'GET'),
headers: (allHeaders),
params: (options.params || {}),
data: (options.data || null)
credentials: 'same-origin',
body,
})
.then(function successCallback(response) {
deferred.resolve(response.data);
}, function errorCallback(response) {
deferred.reject(response.data);
.then((resp) => {
resolve(resp.json());
})
.catch((resp) => {
reject(resp);
});
return deferred.promise;
} else {
reject();
}
};
});
});
}

View file

@ -0,0 +1,170 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
// Service for carrying out Elasticsearch queries to obtain data for the
// Ml Results dashboards.
import { ML_NOTIFICATION_INDEX_PATTERN } from 'plugins/ml/constants/index_patterns';
export function JobMessagesServiceProvider(es, $q) {
// search for audit messages, jobId is optional.
// without it, all jobs will be listed.
// fromRange should be a string formatted in ES time units. e.g. 12h, 1d, 7d
function getJobAuditMessages(fromRange, jobId) {
return $q((resolve, reject) => {
let jobFilter = {};
// if no jobId specified, load all of the messages
if (jobId !== undefined) {
jobFilter = {
bool: {
should: [
{
term: {
job_id: '' // catch system messages
}
},
{
term: {
job_id: jobId // messages for specified jobId
}
}
]
}
};
}
let timeFilter = {};
if (fromRange !== undefined && fromRange !== '') {
timeFilter = {
range: {
timestamp: {
gte: `now-${fromRange}`,
lte: 'now'
}
}
};
}
es.search({
index: ML_NOTIFICATION_INDEX_PATTERN,
ignore_unavailable: true,
size: 1000,
body:
{
sort: [
{ timestamp: { order: 'asc' } },
{ job_id: { order: 'asc' } }
],
query: {
bool: {
filter: [
{
bool: {
must_not: {
term: {
level: 'activity'
}
}
}
},
jobFilter,
timeFilter
]
}
}
}
})
.then((resp) => {
let messages = [];
if (resp.hits.total !== 0) {
messages = resp.hits.hits.map(hit => hit._source);
}
resolve({ messages });
})
.catch((resp) => {
reject(resp);
});
});
}
// search highest, most recent audit messages for all jobs for the last 24hrs.
function getAuditMessagesSummary() {
return $q((resolve, reject) => {
es.search({
index: ML_NOTIFICATION_INDEX_PATTERN,
ignore_unavailable: true,
size: 0,
body: {
query: {
bool: {
filter: {
range: {
timestamp: {
gte: 'now-1d'
}
}
}
}
},
aggs: {
levelsPerJob: {
terms: {
field: 'job_id',
},
aggs: {
levels: {
terms: {
field: 'level',
},
aggs: {
latestMessage: {
terms: {
field: 'message.raw',
size: 1,
order: {
latestMessage: 'desc'
}
},
aggs: {
latestMessage: {
max: {
field: 'timestamp'
}
}
}
}
}
}
}
}
}
}
})
.then((resp) => {
let messagesPerJob = [];
if (resp.hits.total !== 0 &&
resp.aggregations &&
resp.aggregations.levelsPerJob &&
resp.aggregations.levelsPerJob.buckets &&
resp.aggregations.levelsPerJob.buckets.length) {
messagesPerJob = resp.aggregations.levelsPerJob.buckets;
}
resolve({ messagesPerJob });
})
.catch((resp) => {
reject(resp);
});
});
}
return {
getJobAuditMessages,
getAuditMessagesSummary
};
}

File diff suppressed because it is too large Load diff

View file

@ -8,34 +8,30 @@
import _ from 'lodash';
import { uiModules } from 'ui/modules';
const module = uiModules.get('apps/ml');
import { ml } from 'plugins/ml/services/ml_api_service';
module.service('mlESMappingService', function ($q, ml) {
// Returns the mapping type of the specified field.
// Accepts fieldName containing dots representing a nested sub-field.
this.getFieldTypeFromMapping = function (index, fieldName) {
return $q((resolve, reject) => {
if (index !== '') {
ml.getFieldCaps({ index, fields: [fieldName] })
.then((resp) => {
let fieldType = '';
_.each(resp.fields, (field) => {
_.each(field, (type) => {
if (fieldType === '') {
fieldType = type.type;
}
});
// Returns the mapping type of the specified field.
// Accepts fieldName containing dots representing a nested sub-field.
export function getFieldTypeFromMapping(index, fieldName) {
return new Promise((resolve, reject) => {
if (index !== '') {
ml.getFieldCaps({ index, fields: [fieldName] })
.then((resp) => {
let fieldType = '';
_.each(resp.fields, (field) => {
_.each(field, (type) => {
if (fieldType === '') {
fieldType = type.type;
}
});
resolve(fieldType);
})
.catch((error) => {
reject(error);
});
} else {
reject();
}
});
};
});
resolve(fieldType);
})
.catch((error) => {
reject(error);
});
} else {
reject();
}
});
}

View file

@ -7,142 +7,139 @@
import { pick } from 'lodash';
import './http_service';
import chrome from 'ui/chrome';
import { uiModules } from 'ui/modules';
const module = uiModules.get('apps/ml');
import { http } from './http_service';
module.service('ml', function (prlHttpService) {
const http = prlHttpService;
const basePath = chrome.addBasePath('/api/ml');
const basePath = chrome.addBasePath('/api/ml');
this.jobs = function (obj) {
export const ml = {
jobs(obj) {
const jobId = (obj && obj.jobId) ? `/${obj.jobId}` : '';
return http.request({
return http({
url: `${basePath}/anomaly_detectors${jobId}`,
});
};
},
this.jobStats = function (obj) {
jobStats(obj) {
const jobId = (obj && obj.jobId) ? `/${obj.jobId}` : '';
return http.request({
return http({
url: `${basePath}/anomaly_detectors${jobId}/_stats`,
});
};
},
this.addJob = function (obj) {
return http.request({
addJob(obj) {
return http({
url: `${basePath}/anomaly_detectors/${obj.jobId}`,
method: 'PUT',
data: obj.job
});
};
},
this.openJob = function (obj) {
return http.request({
openJob(obj) {
return http({
url: `${basePath}/anomaly_detectors/${obj.jobId}/_open`,
method: 'POST'
});
};
},
this.closeJob = function (obj) {
return http.request({
closeJob(obj) {
return http({
url: `${basePath}/anomaly_detectors/${obj.jobId}/_close`,
method: 'POST'
});
};
},
this.forceCloseJob = function (obj) {
return http.request({
forceCloseJob(obj) {
return http({
url: `${basePath}/anomaly_detectors/${obj.jobId}/_close?force=true`,
method: 'POST'
});
};
},
this.deleteJob = function (obj) {
return http.request({
deleteJob(obj) {
return http({
url: `${basePath}/anomaly_detectors/${obj.jobId}`,
method: 'DELETE'
});
};
},
this.forceDeleteJob = function (obj) {
return http.request({
forceDeleteJob(obj) {
return http({
url: `${basePath}/anomaly_detectors/${obj.jobId}?force=true`,
method: 'DELETE'
});
};
},
this.updateJob = function (obj) {
return http.request({
updateJob(obj) {
return http({
url: `${basePath}/anomaly_detectors/${obj.jobId}/_update`,
method: 'POST',
data: obj.job
});
};
},
this.estimateBucketSpan = function (obj) {
return http.request({
estimateBucketSpan(obj) {
return http({
url: `${basePath}/validate/estimate_bucket_span`,
method: 'POST',
data: obj
});
};
},
this.validateJob = function (obj) {
return http.request({
validateJob(obj) {
return http({
url: `${basePath}/validate/job`,
method: 'POST',
data: obj
});
};
},
this.datafeeds = function (obj) {
datafeeds(obj) {
const datafeedId = (obj && obj.datafeedId) ? `/${obj.datafeedId}` : '';
return http.request({
return http({
url: `${basePath}/datafeeds${datafeedId}`,
});
};
},
this.datafeedStats = function (obj) {
datafeedStats(obj) {
const datafeedId = (obj && obj.datafeedId) ? `/${obj.datafeedId}` : '';
return http.request({
return http({
url: `${basePath}/datafeeds${datafeedId}/_stats`,
});
};
},
this.addDatafeed = function (obj) {
return http.request({
addDatafeed(obj) {
return http({
url: `${basePath}/datafeeds/${obj.datafeedId}`,
method: 'PUT',
data: obj.datafeedConfig
});
};
},
this.updateDatafeed = function (obj) {
return http.request({
updateDatafeed(obj) {
return http({
url: `${basePath}/datafeeds/${obj.datafeedId}/_update`,
method: 'POST',
data: obj.datafeedConfig
});
};
},
this.deleteDatafeed = function (obj) {
return http.request({
deleteDatafeed(obj) {
return http({
url: `${basePath}/datafeeds/${obj.datafeedId}`,
method: 'DELETE'
});
};
},
this.forceDeleteDatafeed = function (obj) {
return http.request({
forceDeleteDatafeed(obj) {
return http({
url: `${basePath}/datafeeds/${obj.datafeedId}?force=true`,
method: 'DELETE'
});
};
},
this.startDatafeed = function (obj) {
startDatafeed(obj) {
const data = {};
if(obj.start !== undefined) {
data.start = obj.start;
@ -150,78 +147,78 @@ module.service('ml', function (prlHttpService) {
if(obj.end !== undefined) {
data.end = obj.end;
}
return http.request({
return http({
url: `${basePath}/datafeeds/${obj.datafeedId}/_start`,
method: 'POST',
data
});
};
},
this.stopDatafeed = function (obj) {
return http.request({
stopDatafeed(obj) {
return http({
url: `${basePath}/datafeeds/${obj.datafeedId}/_stop`,
method: 'POST'
});
};
},
this.datafeedPreview = function (obj) {
return http.request({
datafeedPreview(obj) {
return http({
url: `${basePath}/datafeeds/${obj.datafeedId}/_preview`,
method: 'GET'
});
};
},
this.validateDetector = function (obj) {
return http.request({
validateDetector(obj) {
return http({
url: `${basePath}/anomaly_detectors/_validate/detector`,
method: 'POST',
data: obj.detector
});
};
},
this.forecast = function (obj) {
forecast(obj) {
const data = {};
if(obj.duration !== undefined) {
data.duration = obj.duration;
}
return http.request({
return http({
url: `${basePath}/anomaly_detectors/${obj.jobId}/_forecast`,
method: 'POST',
data
});
};
},
this.overallBuckets = function (obj) {
overallBuckets(obj) {
const data = pick(obj, [
'topN',
'bucketSpan',
'start',
'end'
]);
return http.request({
return http({
url: `${basePath}/anomaly_detectors/${obj.jobId}/results/overall_buckets`,
method: 'POST',
data
});
};
},
this.checkPrivilege = function (obj) {
return http.request({
checkPrivilege(obj) {
return http({
url: `${basePath}/_has_privileges`,
method: 'POST',
data: obj
});
};
},
this.getNotificationSettings = function () {
return http.request({
getNotificationSettings() {
return http({
url: `${basePath}/notification_settings`,
method: 'GET'
});
};
},
this.getFieldCaps = function (obj) {
getFieldCaps(obj) {
const data = {};
if(obj.index !== undefined) {
data.index = obj.index;
@ -229,28 +226,28 @@ module.service('ml', function (prlHttpService) {
if(obj.fields !== undefined) {
data.fields = obj.fields;
}
return http.request({
return http({
url: `${basePath}/indices/field_caps`,
method: 'POST',
data
});
};
},
this.recognizeIndex = function (obj) {
return http.request({
recognizeIndex(obj) {
return http({
url: `${basePath}/modules/recognize/${obj.indexPatternTitle}`,
method: 'GET'
});
};
},
this.getDataRecognizerModule = function (obj) {
return http.request({
getDataRecognizerModule(obj) {
return http({
url: `${basePath}/modules/get_module/${obj.moduleId}`,
method: 'GET'
});
};
},
this.setupDataRecognizerConfig = function (obj) {
setupDataRecognizerConfig(obj) {
const data = pick(obj, [
'prefix',
'groups',
@ -258,14 +255,14 @@ module.service('ml', function (prlHttpService) {
'query'
]);
return http.request({
return http({
url: `${basePath}/modules/setup/${obj.moduleId}`,
method: 'POST',
data
});
};
},
this.getVisualizerFieldStats = function (obj) {
getVisualizerFieldStats(obj) {
const data = pick(obj, [
'query',
'timeFieldName',
@ -277,14 +274,14 @@ module.service('ml', function (prlHttpService) {
'maxExamples'
]);
return http.request({
return http({
url: `${basePath}/data_visualizer/get_field_stats/${obj.indexPatternTitle}`,
method: 'POST',
data
});
};
},
this.getVisualizerOverallStats = function (obj) {
getVisualizerOverallStats(obj) {
const data = pick(obj, [
'query',
'timeFieldName',
@ -295,61 +292,60 @@ module.service('ml', function (prlHttpService) {
'nonAggregatableFields'
]);
return http.request({
return http({
url: `${basePath}/data_visualizer/get_overall_stats/${obj.indexPatternTitle}`,
method: 'POST',
data
});
};
},
this.calendars = function (obj) {
calendars(obj) {
const calendarId = (obj && obj.calendarId) ? `/${obj.calendarId}` : '';
return http.request({
return http({
url: `${basePath}/calendars${calendarId}`,
method: 'GET'
});
};
},
this.addCalendar = function (obj) {
return http.request({
addCalendar(obj) {
return http({
url: `${basePath}/calendars`,
method: 'PUT',
data: obj
});
};
},
this.updateCalendar = function (obj) {
updateCalendar(obj) {
const calendarId = (obj && obj.calendarId) ? `/${obj.calendarId}` : '';
return http.request({
return http({
url: `${basePath}/calendars${calendarId}`,
method: 'PUT',
data: obj
});
};
},
this.deleteCalendar = function (obj) {
return http.request({
deleteCalendar(obj) {
return http({
url: `${basePath}/calendars/${obj.calendarId}`,
method: 'DELETE'
});
};
},
this.mlNodeCount = function () {
return http.request({
mlNodeCount() {
return http({
url: `${basePath}/ml_node_count`,
method: 'GET'
});
};
},
this.mlInfo = function () {
return http.request({
mlInfo() {
return http({
url: `${basePath}/info`,
method: 'GET'
});
};
},
this.calculateModelMemoryLimit = function (obj) {
calculateModelMemoryLimit(obj) {
const data = pick(obj, [
'indexPattern',
'splitFieldName',
@ -361,14 +357,14 @@ module.service('ml', function (prlHttpService) {
'latestMs'
]);
return http.request({
return http({
url: `${basePath}/validate/calculate_model_memory_limit`,
method: 'POST',
data
});
};
},
this.getCardinalityOfFields = function (obj) {
getCardinalityOfFields(obj) {
const data = pick(obj, [
'index',
'types',
@ -379,25 +375,24 @@ module.service('ml', function (prlHttpService) {
'latestMs'
]);
return http.request({
return http({
url: `${basePath}/fields_service/field_cardinality`,
method: 'POST',
data
});
};
},
this.getTimeFieldRange = function (obj) {
getTimeFieldRange(obj) {
const data = pick(obj, [
'index',
'timeFieldName',
'query'
]);
return http.request({
return http({
url: `${basePath}/fields_service/time_field_range`,
method: 'POST',
data
});
};
});
}
};

View file

@ -1,52 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
// service for copying text to the users clipboard
// can only work when triggered via a user event, as part of an onclick or ng-click
// returns success
// e.g. mlClipboardService.copy("this could be abused!");
import { uiModules } from 'ui/modules';
const module = uiModules.get('apps/ml');
module.service('mlClipboardService', function () {
function copyTextToClipboard(text) {
const textArea = document.createElement('textarea');
textArea.style.position = 'fixed';
textArea.style.top = 0;
textArea.style.left = 0;
textArea.style.width = '2em';
textArea.style.height = '2em';
textArea.style.padding = 0;
textArea.style.border = 'none';
textArea.style.outline = 'none';
textArea.style.boxShadow = 'none';
textArea.style.background = 'transparent';
textArea.value = text;
document.body.appendChild(textArea);
textArea.select();
let successful = false;
try {
successful = document.execCommand('copy');
const msg = successful ? 'successful' : 'unsuccessful';
console.log('Copying text command was ' + msg);
} catch (err) {
console.log('Oops, unable to copy');
}
document.body.removeChild(textArea);
return successful;
}
this.copy = copyTextToClipboard;
});

View file

@ -1,176 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
// Service for carrying out Elasticsearch queries to obtain data for the
// Ml Results dashboards.
import _ from 'lodash';
import { ML_NOTIFICATION_INDEX_PATTERN } from 'plugins/ml/constants/index_patterns';
import { uiModules } from 'ui/modules';
const module = uiModules.get('apps/ml');
module.service('mlNotificationService', function ($q, es) {
// search for audit messages, jobId is optional.
// without it, all jobs will be listed.
// fromRange should be a string formatted in ES time units. e.g. 12h, 1d, 7d
this.getJobAuditMessages = function (fromRange, jobId) {
const deferred = $q.defer();
const messages = [];
let jobFilter = {};
// if no jobId specified, load all of the messages
if (jobId !== undefined) {
jobFilter = {
'bool': {
'should': [
{
'term': {
'job_id': '' // catch system messages
}
},
{
'term': {
'job_id': jobId // messages for specified jobId
}
}
]
}
};
}
let timeFilter = {};
if (fromRange !== undefined && fromRange !== '') {
timeFilter = {
'range': {
'timestamp': {
'gte': 'now-' + fromRange,
'lte': 'now'
}
}
};
}
es.search({
index: ML_NOTIFICATION_INDEX_PATTERN,
ignore_unavailable: true,
size: 1000,
body:
{
sort: [
{ 'timestamp': { 'order': 'asc' } },
{ 'job_id': { 'order': 'asc' } }
],
'query': {
'bool': {
'filter': [
{
'bool': {
'must_not': {
'term': {
'level': 'activity'
}
}
}
},
jobFilter,
timeFilter
]
}
}
}
})
.then((resp) => {
if (resp.hits.total !== 0) {
_.each(resp.hits.hits, (hit) => {
messages.push(hit._source);
});
}
deferred.resolve({ messages });
})
.catch((resp) => {
deferred.reject(resp);
});
return deferred.promise;
};
// search highest, most recent audit messages for all jobs for the last 24hrs.
this.getAuditMessagesSummary = function () {
const deferred = $q.defer();
const aggs = [];
es.search({
index: ML_NOTIFICATION_INDEX_PATTERN,
ignore_unavailable: true,
size: 0,
body: {
'query': {
'bool': {
'filter': {
'range': {
'timestamp': {
'gte': 'now-1d'
}
}
}
}
},
'aggs': {
'levelsPerJob': {
'terms': {
'field': 'job_id',
},
'aggs': {
'levels': {
'terms': {
'field': 'level',
},
'aggs': {
'latestMessage': {
'terms': {
'field': 'message.raw',
'size': 1,
'order': {
'latestMessage': 'desc'
}
},
'aggs': {
'latestMessage': {
'max': {
'field': 'timestamp'
}
}
}
}
}
}
}
}
}
}
})
.then((resp) => {
if (resp.hits.total !== 0 &&
resp.aggregations &&
resp.aggregations.levelsPerJob &&
resp.aggregations.levelsPerJob.buckets &&
resp.aggregations.levelsPerJob.buckets.length) {
_.each(resp.aggregations.levelsPerJob.buckets, (agg) => {
aggs.push(agg);
});
}
deferred.resolve({ messagesPerJob: aggs });
})
.catch((resp) => {
deferred.reject(resp);
});
return deferred.promise;
};
});

View file

@ -14,16 +14,14 @@ import { ML_MEDIAN_PERCENTS } from 'plugins/ml/../common/util/job_utils';
import { escapeForElasticsearchQuery } from 'plugins/ml/util/string_utils';
import { ML_RESULTS_INDEX_PATTERN } from 'plugins/ml/constants/index_patterns';
import { uiModules } from 'ui/modules';
const module = uiModules.get('apps/ml');
import { ml } from 'plugins/ml/services/ml_api_service';
module.service('mlResultsService', function ($q, es, ml) {
// Obtains the maximum bucket anomaly scores by job ID and time.
// Pass an empty array or ['*'] to search over all job IDs.
// Returned response contains a results property, with a key for job
// which has results for the specified time range.
this.getScoresByBucket = function (jobIds, earliestMs, latestMs, interval, maxResults) {
export function ResultsServiceProvider($q, es) {
// Obtains the maximum bucket anomaly scores by job ID and time.
// Pass an empty array or ['*'] to search over all job IDs.
// Returned response contains a results property, with a key for job
// which has results for the specified time range.
function getScoresByBucket(jobIds, earliestMs, latestMs, interval, maxResults) {
return $q((resolve, reject) => {
const obj = {
success: true,
@ -141,13 +139,13 @@ module.service('mlResultsService', function ($q, es, ml) {
reject(resp);
});
});
};
}
// Obtains a list of scheduled events by job ID and time.
// Pass an empty array or ['*'] to search over all job IDs.
// Returned response contains a events property, which will only
// contains keys for jobs which have scheduled events for the specified time range.
this.getScheduledEventsByBucket = function (
function getScheduledEventsByBucket(
jobIds,
earliestMs,
latestMs,
@ -256,14 +254,14 @@ module.service('mlResultsService', function ($q, es, ml) {
reject(resp);
});
});
};
}
// Obtains the top influencers, by maximum influencer score, for the specified index, time range and job ID(s).
// Pass an empty array or ['*'] to search over all job IDs.
// Returned response contains an influencers property, with a key for each of the influencer field names,
// whose value is an array of objects containing influencerFieldValue, maxAnomalyScore and sumAnomalyScore keys.
this.getTopInfluencers = function (jobIds, earliestMs, latestMs, maxFieldNames, maxFieldValues) {
function getTopInfluencers(jobIds, earliestMs, latestMs, maxFieldNames, maxFieldValues) {
return $q((resolve, reject) => {
const obj = { success: true, influencers: {} };
@ -385,21 +383,20 @@ module.service('mlResultsService', function ($q, es, ml) {
reject(resp);
});
});
};
}
// Obtains the top influencer field values, by maximum anomaly score, for a
// particular index, field name and job ID(s).
// Pass an empty array or ['*'] to search over all job IDs.
// Returned response contains a results property, which is an array of objects
// containing influencerFieldValue, maxAnomalyScore and sumAnomalyScore keys.
this.getTopInfluencerValues = function (jobIds, influencerFieldName, earliestMs, latestMs, maxResults) {
function getTopInfluencerValues(jobIds, influencerFieldName, earliestMs, latestMs, maxResults) {
return $q((resolve, reject) => {
const obj = { success: true, results: [] };
// Build the criteria to use in the bool filter part of the request.
// Adds criteria for the time range plus any specified job IDs.
const boolCriteria = [];
boolCriteria.push({
const boolCriteria = [{
range: {
timestamp: {
gte: earliestMs,
@ -407,7 +404,8 @@ module.service('mlResultsService', function ($q, es, ml) {
format: 'epoch_millis'
}
}
});
}];
if (jobIds && jobIds.length > 0 && !(jobIds.length === 1 && jobIds[0] === '*')) {
let jobIdFilterStr = '';
_.each(jobIds, (jobId, i) => {
@ -487,12 +485,12 @@ module.service('mlResultsService', function ($q, es, ml) {
reject(resp);
});
});
};
}
// Obtains the overall bucket scores for the specified job ID(s).
// Pass ['*'] to search over all job IDs.
// Returned response contains a results property as an object of max score by time.
this.getOverallBucketScores = function (jobIds, topN, earliestMs, latestMs, interval) {
function getOverallBucketScores(jobIds, topN, earliestMs, latestMs, interval) {
return $q((resolve, reject) => {
const obj = { success: true, results: {} };
@ -518,14 +516,14 @@ module.service('mlResultsService', function ($q, es, ml) {
reject(resp);
});
});
};
}
// Obtains the maximum score by influencer_field_value and by time for the specified job ID(s)
// (pass an empty array or ['*'] to search over all job IDs), and specified influencer field
// values (pass an empty array to search over all field values).
// Returned response contains a results property with influencer field values keyed
// against max score by time.
this.getInfluencerValueMaxScoreByTime = function (
function getInfluencerValueMaxScoreByTime(
jobIds,
influencerFieldName,
influencerFieldValues,
@ -666,13 +664,13 @@ module.service('mlResultsService', function ($q, es, ml) {
reject(resp);
});
});
};
}
// Obtains the definition of the category with the specified ID and job ID.
// Returned response contains four properties - categoryId, regex, examples
// and terms (space delimited String of the common tokens matched in values of the category).
this.getCategoryDefinition = function (jobId, categoryId) {
function getCategoryDefinition(jobId, categoryId) {
return $q((resolve, reject) => {
const obj = { success: true, categoryId: categoryId, terms: null, regex: null, examples: [] };
@ -704,14 +702,14 @@ module.service('mlResultsService', function ($q, es, ml) {
reject(resp);
});
});
};
}
// Obtains the categorization examples for the categories with the specified IDs
// from the given index and job ID.
// Returned response contains two properties - jobId and
// examplesByCategoryId (list of examples against categoryId).
this.getCategoryExamples = function (jobId, categoryIds, maxExamples) {
function getCategoryExamples(jobId, categoryIds, maxExamples) {
return $q((resolve, reject) => {
const obj = { success: true, jobId: jobId, examplesByCategoryId: {} };
@ -747,7 +745,7 @@ module.service('mlResultsService', function ($q, es, ml) {
reject(resp);
});
});
};
}
// Queries Elasticsearch to obtain record level results containing the influencers
@ -755,7 +753,7 @@ module.service('mlResultsService', function ($q, es, ml) {
// Pass an empty array or ['*'] to search over all job IDs.
// Returned response contains a records property, with each record containing
// only the fields job_id, detector_index, record_score and influencers.
this.getRecordInfluencers = function (jobIds, threshold, earliestMs, latestMs, maxResults) {
function getRecordInfluencers(jobIds, threshold, earliestMs, latestMs, maxResults) {
return $q((resolve, reject) => {
const obj = { success: true, records: [] };
@ -851,7 +849,7 @@ module.service('mlResultsService', function ($q, es, ml) {
reject(resp);
});
});
};
}
// Queries Elasticsearch to obtain the record level results containing the specified influencer(s),
@ -860,7 +858,7 @@ module.service('mlResultsService', function ($q, es, ml) {
// 'fieldValue' properties. The influencer array uses 'should' for the nested bool query,
// so this returns record level results which have at least one of the influencers.
// Pass an empty array or ['*'] to search over all job IDs.
this.getRecordsForInfluencer = function (jobIds, influencers, threshold, earliestMs, latestMs, maxResults) {
function getRecordsForInfluencer(jobIds, influencers, threshold, earliestMs, latestMs, maxResults) {
return $q((resolve, reject) => {
const obj = { success: true, records: [] };
@ -957,7 +955,7 @@ module.service('mlResultsService', function ($q, es, ml) {
},
sort: [
{ record_score: { order: 'desc' } }
],
]
}
})
.then((resp) => {
@ -972,13 +970,13 @@ module.service('mlResultsService', function ($q, es, ml) {
reject(resp);
});
});
};
}
// Queries Elasticsearch to obtain the record level results for the specified job and detector,
// time range, record score threshold, and whether to only return results containing influencers.
// An additional, optional influencer field name and value may also be provided.
this.getRecordsForDetector = function (
function getRecordsForDetector(
jobId,
detectorIndex,
checkForInfluencers,
@ -1098,22 +1096,22 @@ module.service('mlResultsService', function ($q, es, ml) {
reject(resp);
});
});
};
}
// Queries Elasticsearch to obtain all the record level results for the specified job(s), time range,
// and record score threshold.
// Pass an empty array or ['*'] to search over all job IDs.
// Returned response contains a records property, which is an array of the matching results.
this.getRecords = function (jobIds, threshold, earliestMs, latestMs, maxResults) {
function getRecords(jobIds, threshold, earliestMs, latestMs, maxResults) {
return this.getRecordsForInfluencer(jobIds, [], threshold, earliestMs, latestMs, maxResults);
};
}
// Queries Elasticsearch to obtain the record level results matching the given criteria,
// for the specified job(s), time range, and record score threshold.
// criteriaFields parameter must be an array, with each object in the array having 'fieldName'
// 'fieldValue' properties.
// Pass an empty array or ['*'] to search over all job IDs.
this.getRecordsForCriteria = function (jobIds, criteriaFields, threshold, earliestMs, latestMs, maxResults) {
function getRecordsForCriteria(jobIds, criteriaFields, threshold, earliestMs, latestMs, maxResults) {
return $q((resolve, reject) => {
const obj = { success: true, records: [] };
@ -1202,7 +1200,7 @@ module.service('mlResultsService', function ($q, es, ml) {
reject(resp);
});
});
};
}
// Queries Elasticsearch to obtain metric aggregation results.
@ -1213,7 +1211,7 @@ module.service('mlResultsService', function ($q, es, ml) {
// Extra query object can be supplied, or pass null if no additional query
// to that built from the supplied entity fields.
// Returned response contains a results property containing the requested aggregation.
this.getMetricData = function (
function getMetricData(
index,
types,
entityFields,
@ -1364,7 +1362,7 @@ module.service('mlResultsService', function ($q, es, ml) {
reject(resp);
});
});
};
}
// Queries Elasticsearch to obtain event rate data i.e. the count
// of documents over time.
@ -1372,7 +1370,7 @@ module.service('mlResultsService', function ($q, es, ml) {
// Extra query object can be supplied, or pass null if no additional query.
// Returned response contains a results property, which is an object
// of document counts against time (epoch millis).
this.getEventRateData = function (
function getEventRateData(
index,
query,
timeFieldName,
@ -1440,9 +1438,9 @@ module.service('mlResultsService', function ($q, es, ml) {
reject(resp);
});
});
};
}
this.getModelPlotOutput = function (
function getModelPlotOutput(
jobId,
detectorIndex,
criteriaFields,
@ -1584,13 +1582,13 @@ module.service('mlResultsService', function ($q, es, ml) {
reject(resp);
});
});
};
}
// Queries Elasticsearch to obtain the max record score over time for the specified job,
// criteria, time range, and aggregation interval.
// criteriaFields parameter must be an array, with each object in the array having 'fieldName'
// 'fieldValue' properties.
this.getRecordMaxScoreByTime = function (jobId, criteriaFields, earliestMs, latestMs, interval) {
function getRecordMaxScoreByTime(jobId, criteriaFields, earliestMs, latestMs, interval) {
return $q((resolve, reject) => {
const obj = {
success: true,
@ -1698,6 +1696,26 @@ module.service('mlResultsService', function ($q, es, ml) {
reject(resp);
});
});
}
return {
getScoresByBucket,
getScheduledEventsByBucket,
getTopInfluencers,
getTopInfluencerValues,
getOverallBucketScores,
getInfluencerValueMaxScoreByTime,
getCategoryDefinition,
getCategoryExamples,
getRecordInfluencers,
getRecordsForInfluencer,
getRecordsForDetector,
getRecords,
getRecordsForCriteria,
getMetricData,
getEventRateData,
getModelPlotOutput,
getRecordMaxScoreByTime
};
});
}

View file

@ -17,6 +17,7 @@ import { checkLicense } from 'plugins/ml/license/check_license';
import { checkGetJobsPrivilege, checkPermission } from 'plugins/ml/privilege/check_privilege';
import { getMlNodeCount, mlNodesAvailable } from 'plugins/ml/ml_nodes_check/check_ml_nodes';
import { buttonsEnabledChecks } from 'plugins/ml/settings/scheduled_events/calendars_list/buttons_enabled_checks';
import { ml } from 'plugins/ml/services/ml_api_service';
import template from './calendars_list.html';
@ -39,9 +40,9 @@ module.controller('MlCalendarsList',
$filter,
$route,
$location,
$q,
pagerFactory,
Private,
ml,
timefilter,
mlConfirmModalService) {
@ -116,7 +117,7 @@ module.controller('MlCalendarsList',
title: `Delete calendar`
})
.then(() => {
ml.deleteCalendar({ calendarId })
$q.when(ml.deleteCalendar({ calendarId }))
.then(loadCalendars)
.catch((error) => {
console.log(error);
@ -126,7 +127,7 @@ module.controller('MlCalendarsList',
};
function loadCalendars() {
ml.calendars()
$q.when(ml.calendars())
.then((resp) => {
calendars = resp;
$scope.pager = pagerFactory.create(calendars.length, PAGE_SIZE, 1);

View file

@ -18,6 +18,9 @@ import uiRoutes from 'ui/routes';
import { checkLicense } from 'plugins/ml/license/check_license';
import { checkGetJobsPrivilege } from 'plugins/ml/privilege/check_privilege';
import { checkMlNodesAvailable } from 'plugins/ml/ml_nodes_check/check_ml_nodes';
import { JobServiceProvider } from 'plugins/ml/services/job_service';
import { CalendarServiceProvider } from 'plugins/ml/services/calendar_service';
import { ml } from 'plugins/ml/services/ml_api_service';
import template from './create_calendar.html';
@ -47,13 +50,14 @@ module.controller('MlCreateCalendar',
$scope,
$route,
$location,
ml,
$q,
timefilter,
mlMessageBarService,
mlJobService,
mlCalendarService) {
Private) {
const msgs = mlMessageBarService;
msgs.clear();
const mlJobService = Private(JobServiceProvider);
const mlCalendarService = Private(CalendarServiceProvider);
const calendarId = $route.current.params.calendarId;
$scope.isNewCalendar = (calendarId === undefined);
@ -137,8 +141,8 @@ module.controller('MlCreateCalendar',
if (validateCalendarId(calendar.calendarId, $scope.validation.checks)) {
$scope.saveLock = true;
const saveFunc = $scope.isNewCalendar ? ml.addCalendar : ml.updateCalendar;
saveFunc(calendar)
const saveFunc = $scope.isNewCalendar ? (c => ml.addCalendar(c)) : (c => ml.updateCalendar(c));
$q.when(saveFunc(calendar))
.then(() => {
$location.path('settings/calendars_list');
$scope.saveLock = false;

View file

@ -32,6 +32,7 @@ import { isJobVersionGte } from '../../../common/util/job_utils';
import { parseInterval } from '../../../common/util/parse_interval';
import { Modal } from './modal';
import { PROGRESS_STATES } from './progress_states';
import { ml } from 'plugins/ml/services/ml_api_service';
const FORECAST_JOB_MIN_VERSION = '6.1.0'; // Forecasting only allowed for jobs created >= 6.1.0.
const FORECASTS_VIEW_MAX = 5; // Display links to a maximum of 5 forecasts.
@ -122,7 +123,7 @@ class ForecastingModal extends Component {
jobOpeningState: PROGRESS_STATES.WAITING
});
this.props.jobService.openJob(this.props.job.job_id)
this.props.mlJobService.openJob(this.props.job.job_id)
.then(() => {
// If open was successful run the forecast, then close the job again.
this.setState({
@ -159,7 +160,7 @@ class ForecastingModal extends Component {
// formats accepted by Kibana (w, M, y) are not valid formats in Elasticsearch.
const durationInSeconds = parseInterval(this.state.newForecastDuration).asSeconds();
this.props.forecastService.runForecast(this.props.job.job_id, `${durationInSeconds}s`)
this.props.mlForecastService.runForecast(this.props.job.job_id, `${durationInSeconds}s`)
.then((resp) => {
// Endpoint will return { acknowledged:true, id: <now timestamp> } before forecast is complete.
// So wait for results and then refresh the dashboard to the end of the forecast.
@ -179,7 +180,7 @@ class ForecastingModal extends Component {
let previousProgress = 0;
let noProgressMs = 0;
this.forecastChecker = setInterval(() => {
this.props.forecastService.getForecastRequestStats(this.props.job, forecastId)
this.props.mlForecastService.getForecastRequestStats(this.props.job, forecastId)
.then((resp) => {
// Get the progress (stats value is between 0 and 1).
const progress = _.get(resp, ['stats', 'forecast_progress'], previousProgress);
@ -197,7 +198,7 @@ class ForecastingModal extends Component {
if (closeJobAfterRunning === true) {
this.setState({ jobClosingState: PROGRESS_STATES.WAITING });
this.props.jobService.closeJob(this.props.job.job_id)
this.props.mlJobService.closeJob(this.props.job.job_id)
.then(() => {
this.setState({
jobClosingState: PROGRESS_STATES.DONE
@ -262,7 +263,7 @@ class ForecastingModal extends Component {
forecast_status: FORECAST_REQUEST_STATE.FINISHED
}
};
this.props.forecastService.getForecastsSummary(
this.props.mlForecastService.getForecastsSummary(
job,
statusFinishedQuery,
bounds.min.valueOf(),
@ -281,14 +282,15 @@ class ForecastingModal extends Component {
// of partitioning fields.
const entityFieldNames = this.props.entities.map(entity => entity.fieldName);
if (entityFieldNames.length > 0) {
this.props.fieldsService.getCardinalityOfFields(
job.datafeed_config.indices,
job.datafeed_config.types,
entityFieldNames,
job.datafeed_config.query,
job.data_description.time_field,
job.data_counts.earliest_record_timestamp,
job.data_counts.latest_record_timestamp)
ml.getCardinalityOfFields({
index: job.datafeed_config.indices,
types: job.datafeed_config.types,
fieldNames: entityFieldNames,
query: job.datafeed_config.query,
timeFieldName: job.data_description.time_field,
earliestMs: job.data_counts.earliest_record_timestamp,
latestMs: job.data_counts.latest_record_timestamp
})
.then((results) => {
let numPartitions = 1;
Object.values(results).forEach((cardinality) => {
@ -397,9 +399,8 @@ ForecastingModal.propTypes = {
job: PropTypes.object,
detectorIndex: PropTypes.number,
entities: PropTypes.array,
forecastService: PropTypes.object.isRequired,
jobService: PropTypes.object.isRequired,
fieldsService: PropTypes.object.isRequired,
mlForecastService: PropTypes.object.isRequired,
mlJobService: PropTypes.object.isRequired,
loadForForecastId: PropTypes.func,
};

View file

@ -10,24 +10,23 @@ import 'ngreact';
import { uiModules } from 'ui/modules';
const module = uiModules.get('apps/ml', ['react']);
import { FieldsServiceProvider } from 'plugins/ml/services/fields_service';
import { JobServiceProvider } from 'plugins/ml/services/job_service';
import { ForecastServiceProvider } from 'plugins/ml/services/forecast_service';
import { ForecastingModal } from './forecasting_modal';
module.directive('mlForecastingModal', function ($injector, Private) {
const forecastService = $injector.get('mlForecastService');
const jobService = $injector.get('mlJobService');
const fieldsService = Private(FieldsServiceProvider);
module.directive('mlForecastingModal', function ($injector) {
const Private = $injector.get('Private');
const mlJobService = Private(JobServiceProvider);
const mlForecastService = Private(ForecastServiceProvider);
const timefilter = $injector.get('timefilter');
const reactDirective = $injector.get('reactDirective');
return reactDirective(
ForecastingModal,
undefined,
{ restrict: 'E' },
{
forecastService,
jobService,
fieldsService,
mlForecastService,
mlJobService,
timefilter
}
);

View file

@ -31,6 +31,7 @@ import ContextChartMask from 'plugins/ml/timeseriesexplorer/context_chart_mask';
import { findNearestChartPointToTime } from 'plugins/ml/timeseriesexplorer/timeseriesexplorer_utils';
import 'plugins/ml/filters/format_value';
import { mlEscape } from 'plugins/ml/util/string_utils';
import { FieldFormatServiceProvider } from 'plugins/ml/services/field_format_service';
import { uiModules } from 'ui/modules';
const module = uiModules.get('apps/ml');
@ -38,15 +39,15 @@ const module = uiModules.get('apps/ml');
module.directive('mlTimeseriesChart', function (
$compile,
$timeout,
Private,
timefilter,
mlAnomaliesTableService,
mlFieldFormatService,
formatValueFilter,
Private,
mlChartTooltipService) {
function link(scope, element) {
const mlFieldFormatService = Private(FieldFormatServiceProvider);
// Key dimensions for the viz and constituent charts.
let svgWidth = angular.element('.results-container').width();
const focusZoomPanelHeight = 25;

View file

@ -8,19 +8,19 @@
import _ from 'lodash';
import { FieldsServiceProvider } from 'plugins/ml/services/fields_service';
import { ml } from 'plugins/ml/services/ml_api_service';
import { isModelPlotEnabled } from 'plugins/ml/../common/util/job_utils';
import { buildConfigFromDetector } from 'plugins/ml/util/chart_config_builder';
import { ResultsServiceProvider } from 'plugins/ml/services/results_service';
import { uiModules } from 'ui/modules';
const module = uiModules.get('apps/ml');
module.service('mlTimeSeriesSearchService', function (
$q,
$timeout,
Private,
es,
mlResultsService) {
Private) {
const mlResultsService = Private(ResultsServiceProvider);
this.getMetricData = function (job, detectorIndex, entityFields, earliestMs, latestMs, interval) {
if (isModelPlotEnabled(job, detectorIndex, entityFields)) {
@ -63,40 +63,40 @@ module.service('mlTimeSeriesSearchService', function (
interval
);
} else {
const deferred = $q.defer();
const obj = {
success: true,
results: {}
};
return $q((resolve, reject) => {
const obj = {
success: true,
results: {}
};
const chartConfig = buildConfigFromDetector(job, detectorIndex);
const chartConfig = buildConfigFromDetector(job, detectorIndex);
mlResultsService.getMetricData(
chartConfig.datafeedConfig.indices,
chartConfig.datafeedConfig.types,
entityFields,
chartConfig.datafeedConfig.query,
chartConfig.metricFunction,
chartConfig.metricFieldName,
chartConfig.timeField,
earliestMs,
latestMs,
interval
)
.then((resp) => {
_.each(resp.results, (value, time) => {
obj.results[time] = {
'actual': value
};
mlResultsService.getMetricData(
chartConfig.datafeedConfig.indices,
chartConfig.datafeedConfig.types,
entityFields,
chartConfig.datafeedConfig.query,
chartConfig.metricFunction,
chartConfig.metricFieldName,
chartConfig.timeField,
earliestMs,
latestMs,
interval
)
.then((resp) => {
_.each(resp.results, (value, time) => {
obj.results[time] = {
'actual': value
};
});
resolve(obj);
})
.catch((resp) => {
reject(resp);
});
deferred.resolve(obj);
})
.catch((resp) => {
deferred.reject(resp);
});
return deferred.promise;
});
}
};
@ -106,54 +106,54 @@ module.service('mlTimeSeriesSearchService', function (
// Queries Elasticsearch if necessary to obtain the distinct count of entities
// for which data is being plotted.
this.getChartDetails = function (job, detectorIndex, entityFields, earliestMs, latestMs) {
const deferred = $q.defer();
const obj = { success: true, results: { functionLabel: '', entityData: { entities: [] } } };
return $q((resolve, reject) => {
const obj = { success: true, results: { functionLabel: '', entityData: { entities: [] } } };
const chartConfig = buildConfigFromDetector(job, detectorIndex);
let functionLabel = chartConfig.metricFunction;
if (chartConfig.metricFieldName !== undefined) {
functionLabel += ' ';
functionLabel += chartConfig.metricFieldName;
}
obj.results.functionLabel = functionLabel;
const chartConfig = buildConfigFromDetector(job, detectorIndex);
let functionLabel = chartConfig.metricFunction;
if (chartConfig.metricFieldName !== undefined) {
functionLabel += ' ';
functionLabel += chartConfig.metricFieldName;
}
obj.results.functionLabel = functionLabel;
const blankEntityFields = _.filter(entityFields, (entity) => {
return entity.fieldValue.length === 0;
});
const blankEntityFields = _.filter(entityFields, (entity) => {
return entity.fieldValue.length === 0;
});
// Look to see if any of the entity fields have defined values
// (i.e. blank input), and if so obtain the cardinality.
if (blankEntityFields.length === 0) {
obj.results.entityData.count = 1;
obj.results.entityData.entities = entityFields;
deferred.resolve(obj);
} else {
const entityFieldNames = _.map(blankEntityFields, 'fieldName');
const fieldsService = Private(FieldsServiceProvider);
fieldsService.getCardinalityOfFields(
chartConfig.datafeedConfig.indices,
chartConfig.datafeedConfig.types,
entityFieldNames,
chartConfig.datafeedConfig.query,
chartConfig.timeField,
earliestMs,
latestMs)
.then((results) => {
_.each(blankEntityFields, (field) => {
obj.results.entityData.entities.push({
fieldName: field.fieldName,
cardinality: _.get(results, field.fieldName, 0)
});
});
deferred.resolve(obj);
// Look to see if any of the entity fields have defined values
// (i.e. blank input), and if so obtain the cardinality.
if (blankEntityFields.length === 0) {
obj.results.entityData.count = 1;
obj.results.entityData.entities = entityFields;
resolve(obj);
} else {
const entityFieldNames = _.map(blankEntityFields, 'fieldName');
ml.getCardinalityOfFields({
index: chartConfig.datafeedConfig.indices,
types: chartConfig.datafeedConfig.types,
fieldNames: entityFieldNames,
query: chartConfig.datafeedConfig.query,
timeFieldName: chartConfig.timeField,
earliestMs,
latestMs
})
.catch((resp) => {
deferred.reject(resp);
});
}
.then((results) => {
_.each(blankEntityFields, (field) => {
obj.results.entityData.entities.push({
fieldName: field.fieldName,
cardinality: _.get(results, field.fieldName, 0)
});
});
return deferred.promise;
resolve(obj);
})
.catch((resp) => {
reject(resp);
});
}
});
};
});

View file

@ -16,10 +16,6 @@ import _ from 'lodash';
import moment from 'moment';
import 'plugins/ml/components/anomalies_table';
import 'plugins/ml/services/field_format_service';
import 'plugins/ml/services/forecast_service';
import 'plugins/ml/services/job_service';
import 'plugins/ml/services/results_service';
import { notify } from 'ui/notify';
import uiRoutes from 'ui/routes';
@ -42,8 +38,12 @@ import {
processScheduledEventsForChart } from 'plugins/ml/timeseriesexplorer/timeseriesexplorer_utils';
import { refreshIntervalWatcher } from 'plugins/ml/util/refresh_interval_watcher';
import { IntervalHelperProvider, getBoundsRoundedToInterval } from 'plugins/ml/util/ml_time_buckets';
import { ResultsServiceProvider } from 'plugins/ml/services/results_service';
import template from './timeseriesexplorer.html';
import { getMlNodeCount } from 'plugins/ml/ml_nodes_check/check_ml_nodes';
import { JobServiceProvider } from 'plugins/ml/services/job_service';
import { FieldFormatServiceProvider } from 'plugins/ml/services/field_format_service';
import { ForecastServiceProvider } from 'plugins/ml/services/forecast_service';
uiRoutes
.when('/timeseriesexplorer/?', {
@ -63,20 +63,12 @@ module.controller('MlTimeSeriesExplorerController', function (
$scope,
$route,
$timeout,
$compile,
$modal,
Private,
$q,
es,
timefilter,
AppState,
mlJobService,
mlResultsService,
mlJobSelectService,
mlTimeSeriesSearchService,
mlForecastService,
mlAnomaliesTableService,
mlFieldFormatService) {
mlAnomaliesTableService) {
$scope.timeFieldName = 'timestamp';
timefilter.enableTimeRangeSelector();
@ -86,6 +78,10 @@ module.controller('MlTimeSeriesExplorerController', function (
const ANOMALIES_MAX_RESULTS = 500;
const MAX_SCHEDULED_EVENTS = 10; // Max number of scheduled events displayed per bucket.
const TimeBuckets = Private(IntervalHelperProvider);
const mlResultsService = Private(ResultsServiceProvider);
const mlJobService = Private(JobServiceProvider);
const mlFieldFormatService = Private(FieldFormatServiceProvider);
const mlForecastService = Private(ForecastServiceProvider);
$scope.jobPickerSelections = [];
$scope.selectedJob;

View file

@ -0,0 +1,44 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
// service for copying text to the users clipboard
// can only work when triggered via a user event, as part of an onclick or ng-click
// returns success
// e.g. mlClipboardService.copy("this could be abused!");
export function copyTextToClipboard(text) {
const textArea = document.createElement('textarea');
textArea.style.position = 'fixed';
textArea.style.top = 0;
textArea.style.left = 0;
textArea.style.width = '2em';
textArea.style.height = '2em';
textArea.style.padding = 0;
textArea.style.border = 'none';
textArea.style.outline = 'none';
textArea.style.boxShadow = 'none';
textArea.style.background = 'transparent';
textArea.value = text;
document.body.appendChild(textArea);
textArea.select();
let successful = false;
try {
successful = document.execCommand('copy');
const msg = successful ? 'successful' : 'unsuccessful';
console.log('Copying text command was ' + msg);
} catch (err) {
console.log('Oops, unable to copy');
}
document.body.removeChild(textArea);
return successful;
}