diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java index 6575a4aacb71..872a69b1b95c 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java @@ -121,7 +121,10 @@ final class MLRequestConverters { RequestConverters.Params params = new RequestConverters.Params(); if (getJobRequest.getAllowNoMatch() != null) { - params.putParam("allow_no_match", Boolean.toString(getJobRequest.getAllowNoMatch())); + params.putParam(GetJobRequest.ALLOW_NO_MATCH.getPreferredName(), Boolean.toString(getJobRequest.getAllowNoMatch())); + } + if (getJobRequest.getForExport() != null) { + params.putParam(GetJobRequest.FOR_EXPORT, Boolean.toString(getJobRequest.getForExport())); } request.addParameters(params.asMap()); return request; @@ -270,6 +273,9 @@ final class MLRequestConverters { params.putParam(GetDatafeedRequest.ALLOW_NO_MATCH.getPreferredName(), Boolean.toString(getDatafeedRequest.getAllowNoMatch())); } + if (getDatafeedRequest.getForExport() != null) { + params.putParam(GetDatafeedRequest.FOR_EXPORT, Boolean.toString(getDatafeedRequest.getForExport())); + } request.addParameters(params.asMap()); return request; } @@ -645,7 +651,10 @@ final class MLRequestConverters { } } if (getRequest.getAllowNoMatch() != null) { - params.putParam(GetDataFrameAnalyticsRequest.ALLOW_NO_MATCH.getPreferredName(), Boolean.toString(getRequest.getAllowNoMatch())); + params.putParam(GetDataFrameAnalyticsRequest.ALLOW_NO_MATCH, Boolean.toString(getRequest.getAllowNoMatch())); + } + if (getRequest.getForExport() != null) { + params.putParam(GetDataFrameAnalyticsRequest.FOR_EXPORT, Boolean.toString(getRequest.getForExport())); } request.addParameters(params.asMap()); return request; diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsRequest.java index 40698c4b528f..bddc8b4c3e1b 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsRequest.java @@ -23,7 +23,6 @@ import org.elasticsearch.client.Validatable; import org.elasticsearch.client.ValidationException; import org.elasticsearch.client.core.PageParams; import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.ParseField; import java.util.Arrays; import java.util.List; @@ -32,11 +31,13 @@ import java.util.Optional; public class GetDataFrameAnalyticsRequest implements Validatable { - public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match"); + public static final String ALLOW_NO_MATCH = "allow_no_match"; + public static final String FOR_EXPORT = "for_export"; private final List ids; private Boolean allowNoMatch; private PageParams pageParams; + private Boolean forExport; /** * Helper method to create a request that will get ALL Data Frame Analytics @@ -58,6 +59,22 @@ public class GetDataFrameAnalyticsRequest implements Validatable { return allowNoMatch; } + /** + * Setting this flag to `true` removes certain fields from the configuration on retrieval. + * + * This is useful when getting the configuration and wanting to put it in another cluster. + * + * Default value is false. + * @param forExport Boolean value indicating if certain fields should be removed + */ + public void setForExport(boolean forExport) { + this.forExport = forExport; + } + + public Boolean getForExport() { + return forExport; + } + /** * Whether to ignore if a wildcard expression matches no data frame analytics. * @@ -94,11 +111,12 @@ public class GetDataFrameAnalyticsRequest implements Validatable { GetDataFrameAnalyticsRequest other = (GetDataFrameAnalyticsRequest) o; return Objects.equals(ids, other.ids) && Objects.equals(allowNoMatch, other.allowNoMatch) + && Objects.equals(forExport, other.forExport) && Objects.equals(pageParams, other.pageParams); } @Override public int hashCode() { - return Objects.hash(ids, allowNoMatch, pageParams); + return Objects.hash(ids, allowNoMatch, forExport, pageParams); } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDatafeedRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDatafeedRequest.java index 3408550c406c..38e0b86ffc04 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDatafeedRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDatafeedRequest.java @@ -41,10 +41,12 @@ public class GetDatafeedRequest implements Validatable, ToXContentObject { public static final ParseField DATAFEED_IDS = new ParseField("datafeed_ids"); public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match"); + public static final String FOR_EXPORT = "for_export"; private static final String ALL_DATAFEEDS = "_all"; private final List datafeedIds; private Boolean allowNoMatch; + private Boolean forExport; @SuppressWarnings("unchecked") public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( @@ -100,9 +102,25 @@ public class GetDatafeedRequest implements Validatable, ToXContentObject { return allowNoMatch; } + /** + * Setting this flag to `true` removes certain fields from the configuration on retrieval. + * + * This is useful when getting the configuration and wanting to put it in another cluster. + * + * Default value is false. + * @param forExport Boolean value indicating if certain fields should be removed + */ + public void setForExport(boolean forExport) { + this.forExport = forExport; + } + + public Boolean getForExport() { + return forExport; + } + @Override public int hashCode() { - return Objects.hash(datafeedIds, allowNoMatch); + return Objects.hash(datafeedIds, forExport, allowNoMatch); } @Override @@ -117,7 +135,8 @@ public class GetDatafeedRequest implements Validatable, ToXContentObject { GetDatafeedRequest that = (GetDatafeedRequest) other; return Objects.equals(datafeedIds, that.datafeedIds) && - Objects.equals(allowNoMatch, that.allowNoMatch); + Objects.equals(allowNoMatch, that.allowNoMatch) && + Objects.equals(forExport, that.forExport); } @Override diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetJobRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetJobRequest.java index d3079fb7360c..6ad09de62fda 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetJobRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetJobRequest.java @@ -42,10 +42,12 @@ public class GetJobRequest implements Validatable, ToXContentObject { public static final ParseField JOB_IDS = new ParseField("job_ids"); public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match"); + public static final String FOR_EXPORT = "for_export"; private static final String ALL_JOBS = "_all"; private final List jobIds; private Boolean allowNoMatch; + private Boolean forExport; @SuppressWarnings("unchecked") public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( @@ -100,9 +102,25 @@ public class GetJobRequest implements Validatable, ToXContentObject { return allowNoMatch; } + /** + * Setting this flag to `true` removes certain fields from the configuration on retrieval. + * + * This is useful when getting the configuration and wanting to put it in another cluster. + * + * Default value is false. + * @param forExport Boolean value indicating if certain fields should be removed + */ + public void setForExport(boolean forExport) { + this.forExport = forExport; + } + + public Boolean getForExport() { + return forExport; + } + @Override public int hashCode() { - return Objects.hash(jobIds, allowNoMatch); + return Objects.hash(jobIds, forExport, allowNoMatch); } @Override @@ -117,6 +135,7 @@ public class GetJobRequest implements Validatable, ToXContentObject { GetJobRequest that = (GetJobRequest) other; return Objects.equals(jobIds, that.jobIds) && + Objects.equals(forExport, that.forExport) && Objects.equals(allowNoMatch, that.allowNoMatch); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java index 287eda10d123..823a4a026f39 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java @@ -340,6 +340,7 @@ public class MlClientDocumentationIT extends ESRestHighLevelClientTestCase { // tag::get-job-request GetJobRequest request = new GetJobRequest("get-machine-learning-job1", "get-machine-learning-job*"); // <1> request.setAllowNoMatch(true); // <2> + request.setForExport(false); // <3> // end::get-job-request // tag::get-job-execute @@ -836,6 +837,7 @@ public class MlClientDocumentationIT extends ESRestHighLevelClientTestCase { // tag::get-datafeed-request GetDatafeedRequest request = new GetDatafeedRequest(datafeedId); // <1> request.setAllowNoMatch(true); // <2> + request.setForExport(false); // <3> // end::get-datafeed-request // tag::get-datafeed-execute @@ -2861,6 +2863,7 @@ public class MlClientDocumentationIT extends ESRestHighLevelClientTestCase { { // tag::get-data-frame-analytics-request GetDataFrameAnalyticsRequest request = new GetDataFrameAnalyticsRequest("my-analytics-config"); // <1> + request.setForExport(false); // <2> // end::get-data-frame-analytics-request // tag::get-data-frame-analytics-execute diff --git a/docs/java-rest/high-level/ml/get-data-frame-analytics.asciidoc b/docs/java-rest/high-level/ml/get-data-frame-analytics.asciidoc index 2e956d43074c..6ddadea25afd 100644 --- a/docs/java-rest/high-level/ml/get-data-frame-analytics.asciidoc +++ b/docs/java-rest/high-level/ml/get-data-frame-analytics.asciidoc @@ -21,6 +21,9 @@ IDs, or the special wildcard `_all` to get all {dfanalytics-jobs}. include-tagged::{doc-tests-file}[{api}-request] -------------------------------------------------- <1> Constructing a new GET request referencing an existing {dfanalytics-job} +<2> Optional boolean value for requesting the {dfanalytics-job} in a format that can +then be put into another cluster. Certain fields that can only be set when +the {dfanalytics-job} is created are removed. include::../execution.asciidoc[] diff --git a/docs/java-rest/high-level/ml/get-datafeed.asciidoc b/docs/java-rest/high-level/ml/get-datafeed.asciidoc index 0aa5ec2ec0dc..25eda75d15c3 100644 --- a/docs/java-rest/high-level/ml/get-datafeed.asciidoc +++ b/docs/java-rest/high-level/ml/get-datafeed.asciidoc @@ -25,6 +25,9 @@ include-tagged::{doc-tests-file}[{api}-request] contain wildcards. <2> Whether to ignore if a wildcard expression matches no datafeeds. (This includes `_all` string or when no datafeeds have been specified). +<3> Optional boolean value for requesting the datafeed in a format that can +then be put into another cluster. Certain fields that can only be set when +the datafeed is created are removed. [id="{upid}-{api}-response"] ==== Get datafeed response diff --git a/docs/java-rest/high-level/ml/get-job.asciidoc b/docs/java-rest/high-level/ml/get-job.asciidoc index 3fde9b98f310..e42995edcf8d 100644 --- a/docs/java-rest/high-level/ml/get-job.asciidoc +++ b/docs/java-rest/high-level/ml/get-job.asciidoc @@ -25,6 +25,9 @@ include-tagged::{doc-tests-file}[{api}-request] wildcards. <2> Whether to ignore if a wildcard expression matches no {anomaly-jobs}. (This includes `_all` string or when no jobs have been specified). +<3> Optional boolean value for requesting the job in a format that can +then be put into another cluster. Certain fields that can only be set when +the job is created are removed. [id="{upid}-{api}-response"] ==== Get {anomaly-jobs} response diff --git a/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc index 57b3d9d51b8f..ed6442c66ec6 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc @@ -57,6 +57,10 @@ all {dfeeds}. (Optional, boolean) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-datafeeds] +`for_export`:: +(Optional, boolean) +include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=for-export] + [[ml-get-datafeed-results]] == {api-response-body-title} diff --git a/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc index af3bd514b91f..668c95c9891a 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc @@ -50,6 +50,9 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection-defaul (Optional, boolean) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-jobs] +`for_export`:: +(Optional, boolean) +include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=for-export] [[ml-get-job-results]] == {api-response-body-title} diff --git a/docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc index aa3d66764b55..a6adac16dbef 100644 --- a/docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc +++ b/docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc @@ -71,6 +71,10 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=from] (Optional, integer) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=size] +`for_export`:: +(Optional, boolean) +include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=for-export] + [role="child_attributes"] [[ml-get-dfanalytics-results]] == {api-response-body-title} diff --git a/docs/reference/ml/df-analytics/apis/get-trained-models.asciidoc b/docs/reference/ml/df-analytics/apis/get-trained-models.asciidoc index 0700b92027e8..79ed1325e1ba 100644 --- a/docs/reference/ml/df-analytics/apis/get-trained-models.asciidoc +++ b/docs/reference/ml/df-analytics/apis/get-trained-models.asciidoc @@ -67,9 +67,7 @@ Specifies whether the included model definition should be returned as a JSON map `for_export`:: (Optional, boolean) -Indicates if certain fields should be removed from the model configuration on -retrieval. This allows the model to be in an acceptable format to be retrieved -and then added to another cluster. Default is false. +include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=for-export] `from`:: (Optional, integer) diff --git a/docs/reference/ml/ml-shared.asciidoc b/docs/reference/ml/ml-shared.asciidoc index 3dd1c37a168d..685546d87bb5 100644 --- a/docs/reference/ml/ml-shared.asciidoc +++ b/docs/reference/ml/ml-shared.asciidoc @@ -678,6 +678,12 @@ The number of individual forecasts currently available for the job. A value of `1` or more indicates that forecasts exist. end::forecast-total[] +tag::for-export[] +Indicates if certain fields should be removed from the configuration on +retrieval. This allows the configuration to be in an acceptable format to be retrieved +and then added to another cluster. Default is false. +end::for-export[] + tag::frequency[] The interval at which scheduled queries are made while the {dfeed} runs in real time. The default value is either the bucket span for short bucket spans, or, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java index 2cac811dcd5f..e711ba2a71e6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java @@ -11,6 +11,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.AbstractDiffable; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; @@ -49,6 +50,8 @@ import java.util.Objects; import java.util.Random; import java.util.concurrent.TimeUnit; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT; + /** * Datafeed configuration options. Describes where to proactively pull input * data from. @@ -65,6 +68,9 @@ public class DatafeedConfig extends AbstractDiffable implements private static final int TWO_MINS_SECONDS = 2 * SECONDS_IN_MINUTE; private static final int TWENTY_MINS_SECONDS = 20 * SECONDS_IN_MINUTE; private static final int HALF_DAY_SECONDS = 12 * 60 * SECONDS_IN_MINUTE; + public static final int DEFAULT_AGGREGATION_CHUNKING_BUCKETS = 1000; + private static final TimeValue MIN_DEFAULT_QUERY_DELAY = TimeValue.timeValueMinutes(1); + private static final TimeValue MAX_DEFAULT_QUERY_DELAY = TimeValue.timeValueMinutes(2); private static final Logger logger = LogManager.getLogger(DatafeedConfig.class); @@ -451,17 +457,43 @@ public class DatafeedConfig extends AbstractDiffable implements @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(ID.getPreferredName(), id); - builder.field(Job.ID.getPreferredName(), jobId); - if (params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) { - builder.field(CONFIG_TYPE.getPreferredName(), TYPE); + if (params.paramAsBoolean(FOR_EXPORT, false) == false) { + builder.field(ID.getPreferredName(), id); + // We don't include the job_id in export as we assume the PUT will be referring to a new job as well + builder.field(Job.ID.getPreferredName(), jobId); + if (params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) { + builder.field(CONFIG_TYPE.getPreferredName(), TYPE); + } + if (headers.isEmpty() == false && params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) { + builder.field(HEADERS.getPreferredName(), headers); + } + builder.field(QUERY_DELAY.getPreferredName(), queryDelay.getStringRep()); + if (chunkingConfig != null) { + builder.field(CHUNKING_CONFIG.getPreferredName(), chunkingConfig); + } + builder.startObject(INDICES_OPTIONS.getPreferredName()); + indicesOptions.toXContent(builder, params); + builder.endObject(); + } else { // Don't include random defaults or unnecessary defaults in export + if (queryDelay.equals(defaultRandomQueryDelay(jobId)) == false) { + builder.field(QUERY_DELAY.getPreferredName(), queryDelay.getStringRep()); + } + // Indices options are a pretty advanced feature, better to not include them if they are just the default ones + if (indicesOptions.equals(SearchRequest.DEFAULT_INDICES_OPTIONS) == false) { + builder.startObject(INDICES_OPTIONS.getPreferredName()); + indicesOptions.toXContent(builder, params); + builder.endObject(); + } + // Removing the default chunking config as it is determined by OTHER fields + if (chunkingConfig != null && chunkingConfig.equals(defaultChunkingConfig(aggProvider)) == false) { + builder.field(CHUNKING_CONFIG.getPreferredName(), chunkingConfig); + } } - builder.field(QUERY_DELAY.getPreferredName(), queryDelay.getStringRep()); + builder.field(QUERY.getPreferredName(), queryProvider.getQuery()); if (frequency != null) { builder.field(FREQUENCY.getPreferredName(), frequency.getStringRep()); } builder.field(INDICES.getPreferredName(), indices); - builder.field(QUERY.getPreferredName(), queryProvider.getQuery()); if (aggProvider != null) { builder.field(AGGREGATIONS.getPreferredName(), aggProvider.getAggs()); } @@ -473,26 +505,34 @@ public class DatafeedConfig extends AbstractDiffable implements builder.endObject(); } builder.field(SCROLL_SIZE.getPreferredName(), scrollSize); - if (chunkingConfig != null) { - builder.field(CHUNKING_CONFIG.getPreferredName(), chunkingConfig); - } - if (headers.isEmpty() == false && params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) { - builder.field(HEADERS.getPreferredName(), headers); - } if (delayedDataCheckConfig != null) { builder.field(DELAYED_DATA_CHECK_CONFIG.getPreferredName(), delayedDataCheckConfig); } if (maxEmptySearches != null) { builder.field(MAX_EMPTY_SEARCHES.getPreferredName(), maxEmptySearches); } - builder.startObject(INDICES_OPTIONS.getPreferredName()); - indicesOptions.toXContent(builder, params); - builder.endObject(); - builder.endObject(); return builder; } + private static TimeValue defaultRandomQueryDelay(String jobId) { + Random random = new Random(jobId.hashCode()); + long delayMillis = random.longs(MIN_DEFAULT_QUERY_DELAY.millis(), MAX_DEFAULT_QUERY_DELAY.millis()).findFirst().getAsLong(); + return TimeValue.timeValueMillis(delayMillis); + } + + private static ChunkingConfig defaultChunkingConfig(@Nullable AggProvider aggProvider) { + if (aggProvider == null || aggProvider.getParsedAggs() == null) { + return ChunkingConfig.newAuto(); + } else { + long histogramIntervalMillis = ExtractorUtils.getHistogramIntervalMillis(aggProvider.getParsedAggs()); + if (histogramIntervalMillis <= 0) { + throw ExceptionsHelper.badRequestException(Messages.DATAFEED_AGGREGATIONS_INTERVAL_MUST_BE_GREATER_THAN_ZERO); + } + return ChunkingConfig.newManual(TimeValue.timeValueMillis(DEFAULT_AGGREGATION_CHUNKING_BUCKETS * histogramIntervalMillis)); + } + } + /** * The lists of indices and types are compared for equality but they are not * sorted first so this test could fail simply because the indices and types @@ -586,10 +626,6 @@ public class DatafeedConfig extends AbstractDiffable implements public static class Builder { - public static final int DEFAULT_AGGREGATION_CHUNKING_BUCKETS = 1000; - private static final TimeValue MIN_DEFAULT_QUERY_DELAY = TimeValue.timeValueMinutes(1); - private static final TimeValue MAX_DEFAULT_QUERY_DELAY = TimeValue.timeValueMinutes(2); - private String id; private String jobId; private TimeValue queryDelay; @@ -826,25 +862,13 @@ public class DatafeedConfig extends AbstractDiffable implements private void setDefaultChunkingConfig() { if (chunkingConfig == null) { - if (aggProvider == null || aggProvider.getParsedAggs() == null) { - chunkingConfig = ChunkingConfig.newAuto(); - } else { - long histogramIntervalMillis = ExtractorUtils.getHistogramIntervalMillis(aggProvider.getParsedAggs()); - if (histogramIntervalMillis <= 0) { - throw ExceptionsHelper.badRequestException(Messages.DATAFEED_AGGREGATIONS_INTERVAL_MUST_BE_GREATER_THAN_ZERO); - } - chunkingConfig = ChunkingConfig.newManual(TimeValue.timeValueMillis( - DEFAULT_AGGREGATION_CHUNKING_BUCKETS * histogramIntervalMillis)); - } + chunkingConfig = defaultChunkingConfig(aggProvider); } } private void setDefaultQueryDelay() { if (queryDelay == null) { - Random random = new Random(jobId.hashCode()); - long delayMillis = random.longs(MIN_DEFAULT_QUERY_DELAY.millis(), MAX_DEFAULT_QUERY_DELAY.millis()) - .findFirst().getAsLong(); - queryDelay = TimeValue.timeValueMillis(delayMillis); + queryDelay = defaultRandomQueryDelay(jobId); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java index a72dc95b9668..f904280ce1fc 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java @@ -35,6 +35,7 @@ import java.util.Objects; import static org.elasticsearch.common.xcontent.ObjectParser.ValueType.OBJECT_ARRAY_BOOLEAN_OR_STRING; import static org.elasticsearch.common.xcontent.ObjectParser.ValueType.VALUE; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT; public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable { @@ -227,34 +228,34 @@ public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(ID.getPreferredName(), id); + if (params.paramAsBoolean(FOR_EXPORT, false) == false) { + builder.field(ID.getPreferredName(), id); + if (createTime != null) { + builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + "_string", createTime.toEpochMilli()); + } + if (version != null) { + builder.field(VERSION.getPreferredName(), version); + } + if (headers.isEmpty() == false && params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) { + builder.field(HEADERS.getPreferredName(), headers); + } + if (params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) { + builder.field(CONFIG_TYPE.getPreferredName(), TYPE); + } + } if (description != null) { builder.field(DESCRIPTION.getPreferredName(), description); } builder.field(SOURCE.getPreferredName(), source); builder.field(DEST.getPreferredName(), dest); - builder.startObject(ANALYSIS.getPreferredName()); builder.field(analysis.getWriteableName(), analysis, new MapParams(Collections.singletonMap(VERSION.getPreferredName(), version == null ? null : version.toString()))); builder.endObject(); - - if (params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) { - builder.field(CONFIG_TYPE.getPreferredName(), TYPE); - } if (analyzedFields != null) { builder.field(ANALYZED_FIELDS.getPreferredName(), analyzedFields); } builder.field(MODEL_MEMORY_LIMIT.getPreferredName(), getModelMemoryLimit().getStringRep()); - if (headers.isEmpty() == false && params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) { - builder.field(HEADERS.getPreferredName(), headers); - } - if (createTime != null) { - builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + "_string", createTime.toEpochMilli()); - } - if (version != null) { - builder.field(VERSION.getPreferredName(), version); - } builder.field(ALLOW_LAZY_START.getPreferredName(), allowLazyStart); builder.field(MAX_NUM_THREADS.getPreferredName(), maxNumThreads); builder.endObject(); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java index 9ae67880215a..fbea2045a9e2 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java @@ -45,6 +45,7 @@ import java.util.stream.Collectors; import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.xpack.core.ml.utils.NamedXContentObjectHelper.writeNamedObject; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT; public class TrainedModelConfig implements ToXContentObject, Writeable { @@ -52,7 +53,6 @@ public class TrainedModelConfig implements ToXContentObject, Writeable { public static final String NAME = "trained_model_config"; public static final int CURRENT_DEFINITION_COMPRESSION_VERSION = 1; public static final String DECOMPRESS_DEFINITION = "decompress_definition"; - public static final String FOR_EXPORT = "for_export"; public static final String TOTAL_FEATURE_IMPORTANCE = "total_feature_importance"; private static final Set RESERVED_METADATA_FIELDS = Collections.singleton(TOTAL_FEATURE_IMPORTANCE); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java index 69dc8ef88089..78cab7051fe9 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java @@ -33,6 +33,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Date; +import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; @@ -42,6 +43,8 @@ import java.util.Set; import java.util.TreeSet; import java.util.concurrent.TimeUnit; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT; + /** * This class represents a configured and created Job. The creation time is set * to the time the object was constructed and the finished time and last @@ -513,11 +516,35 @@ public class Job extends AbstractDiffable implements Writeable, ToXContentO public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { final String humanReadableSuffix = "_string"; - - builder.field(ID.getPreferredName(), jobId); - builder.field(JOB_TYPE.getPreferredName(), jobType); - if (jobVersion != null) { - builder.field(JOB_VERSION.getPreferredName(), jobVersion); + if (params.paramAsBoolean(FOR_EXPORT, false) == false) { + builder.field(ID.getPreferredName(), jobId); + builder.field(JOB_TYPE.getPreferredName(), jobType); + if (jobVersion != null) { + builder.field(JOB_VERSION.getPreferredName(), jobVersion); + } + builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + humanReadableSuffix, createTime.getTime()); + if (finishedTime != null) { + builder.timeField(FINISHED_TIME.getPreferredName(), FINISHED_TIME.getPreferredName() + humanReadableSuffix, + finishedTime.getTime()); + } + if (modelSnapshotId != null) { + builder.field(MODEL_SNAPSHOT_ID.getPreferredName(), modelSnapshotId); + } + if (deleting) { + builder.field(DELETING.getPreferredName(), deleting); + } + if (modelSnapshotMinVersion != null) { + builder.field(MODEL_SNAPSHOT_MIN_VERSION.getPreferredName(), modelSnapshotMinVersion); + } + if (customSettings != null) { + builder.field(CUSTOM_SETTINGS.getPreferredName(), customSettings); + } + } else { + if (customSettings != null) { + HashMap newCustomSettings = new HashMap<>(customSettings); + newCustomSettings.remove("created_by"); + builder.field(CUSTOM_SETTINGS.getPreferredName(), newCustomSettings); + } } if (groups.isEmpty() == false) { builder.field(GROUPS.getPreferredName(), groups); @@ -525,11 +552,6 @@ public class Job extends AbstractDiffable implements Writeable, ToXContentO if (description != null) { builder.field(DESCRIPTION.getPreferredName(), description); } - builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + humanReadableSuffix, createTime.getTime()); - if (finishedTime != null) { - builder.timeField(FINISHED_TIME.getPreferredName(), FINISHED_TIME.getPreferredName() + humanReadableSuffix, - finishedTime.getTime()); - } builder.field(ANALYSIS_CONFIG.getPreferredName(), analysisConfig, params); if (analysisLimits != null) { builder.field(ANALYSIS_LIMITS.getPreferredName(), analysisLimits, params); @@ -555,19 +577,7 @@ public class Job extends AbstractDiffable implements Writeable, ToXContentO if (resultsRetentionDays != null) { builder.field(RESULTS_RETENTION_DAYS.getPreferredName(), resultsRetentionDays); } - if (customSettings != null) { - builder.field(CUSTOM_SETTINGS.getPreferredName(), customSettings); - } - if (modelSnapshotId != null) { - builder.field(MODEL_SNAPSHOT_ID.getPreferredName(), modelSnapshotId); - } - if (modelSnapshotMinVersion != null) { - builder.field(MODEL_SNAPSHOT_MIN_VERSION.getPreferredName(), modelSnapshotMinVersion); - } builder.field(RESULTS_INDEX_NAME.getPreferredName(), resultsIndexName); - if (deleting) { - builder.field(DELETING.getPreferredName(), deleting); - } builder.field(ALLOW_LAZY_OPEN.getPreferredName(), allowLazyOpen); return builder; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ToXContentParams.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ToXContentParams.java index 84a4c26d4393..b755c13ce18f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ToXContentParams.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ToXContentParams.java @@ -18,6 +18,14 @@ public final class ToXContentParams { */ public static final String FOR_INTERNAL_STORAGE = "for_internal_storage"; + /** + * Parameter to indicate if this XContent serialization should only include fields that are allowed to be used + * on PUT + * + * This helps to GET a configuration, copy it, and then PUT it directly without removing or changing any fields in between + */ + public static final String FOR_EXPORT = "for_export"; + /** * When serialising POJOs to X Content this indicates whether the calculated (i.e. not stored) fields * should be included or not diff --git a/x-pack/plugin/ml/qa/basic-multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlBasicMultiNodeIT.java b/x-pack/plugin/ml/qa/basic-multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlBasicMultiNodeIT.java index dcfd97ff4fc5..2acb421c6e9d 100644 --- a/x-pack/plugin/ml/qa/basic-multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlBasicMultiNodeIT.java +++ b/x-pack/plugin/ml/qa/basic-multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlBasicMultiNodeIT.java @@ -18,6 +18,7 @@ import org.yaml.snakeyaml.util.UriEncoder; import java.io.IOException; import java.io.UncheckedIOException; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -106,31 +107,7 @@ public class MlBasicMultiNodeIT extends ESRestTestCase { } public void testMiniFarequoteWithDatafeeder() throws Exception { - boolean datesHaveNanoSecondResolution = randomBoolean(); - String dateMappingType = datesHaveNanoSecondResolution ? "date_nanos" : "date"; - String dateFormat = datesHaveNanoSecondResolution ? "strict_date_optional_time_nanos" : "strict_date_optional_time"; - String randomNanos = datesHaveNanoSecondResolution ? "," + randomIntBetween(100000000, 999999999) : ""; - Request createAirlineDataRequest = new Request("PUT", "/airline-data"); - createAirlineDataRequest.setJsonEntity("{" - + " \"mappings\": {" - + " \"properties\": {" - + " \"time\": { \"type\":\"" + dateMappingType + "\", \"format\":\"" + dateFormat + "\"}," - + " \"airline\": { \"type\":\"keyword\"}," - + " \"responsetime\": { \"type\":\"float\"}" - + " }" - + " }" - + "}"); - client().performRequest(createAirlineDataRequest); - Request airlineData1 = new Request("PUT", "/airline-data/_doc/1"); - airlineData1.setJsonEntity("{\"time\":\"2016-06-01T00:00:00" + randomNanos + "Z\",\"airline\":\"AAA\",\"responsetime\":135.22}"); - client().performRequest(airlineData1); - Request airlineData2 = new Request("PUT", "/airline-data/_doc/2"); - airlineData2.setJsonEntity("{\"time\":\"2016-06-01T01:59:00" + randomNanos + "Z\",\"airline\":\"AAA\",\"responsetime\":541.76}"); - client().performRequest(airlineData2); - - // Ensure all data is searchable - refreshAllIndices(); - + createAndIndexFarequote(); String jobId = "mini-farequote-with-data-feeder-job"; createFarequoteJob(jobId); String datafeedId = "bar"; @@ -265,6 +242,211 @@ public class MlBasicMultiNodeIT extends ESRestTestCase { client().performRequest(new Request("DELETE", BASE_PATH + "anomaly_detectors/" + jobId)); } + @SuppressWarnings("unchecked") + public void testExportAndPutJob() throws Exception { + String jobId = "test-export-import-job"; + createFarequoteJob(jobId); + Response jobResponse = client().performRequest( + new Request("GET", BASE_PATH + "anomaly_detectors/" + jobId + "?for_export=true")); + Map originalJobBody = (Map)((List) entityAsMap(jobResponse).get("jobs")).get(0); + + XContentBuilder xContentBuilder = jsonBuilder().map(originalJobBody); + Request request = new Request("PUT", BASE_PATH + "anomaly_detectors/" + jobId + "-import"); + request.setJsonEntity(Strings.toString(xContentBuilder)); + client().performRequest(request); + + Response importedJobResponse = client().performRequest( + new Request("GET", BASE_PATH + "anomaly_detectors/" + jobId + "-import" + "?for_export=true")); + Map importedJobBody = (Map)((List) entityAsMap(importedJobResponse).get("jobs")).get(0); + assertThat(originalJobBody, equalTo(importedJobBody)); + } + + @SuppressWarnings("unchecked") + public void testExportAndPutDatafeed() throws Exception { + createAndIndexFarequote(); + String jobId = "test-export-import-datafeed"; + createFarequoteJob(jobId); + String datafeedId = jobId + "-datafeed"; + createDatafeed(datafeedId, jobId); + + Response dfResponse = client().performRequest( + new Request("GET", BASE_PATH + "datafeeds/" + datafeedId + "?for_export=true")); + Map originalDfBody = (Map)((List) entityAsMap(dfResponse).get("datafeeds")).get(0); + + //Delete this so we can PUT another datafeed for the same job + client().performRequest(new Request("DELETE", BASE_PATH + "datafeeds/" + datafeedId)); + + Map toPut = new HashMap<>(originalDfBody); + toPut.put("job_id", jobId); + XContentBuilder xContentBuilder = jsonBuilder().map(toPut); + Request request = new Request("PUT", BASE_PATH + "datafeeds/" + datafeedId + "-import"); + request.setJsonEntity(Strings.toString(xContentBuilder)); + client().performRequest(request); + + Response importedDfResponse = client().performRequest( + new Request("GET", BASE_PATH + "datafeeds/" + datafeedId + "-import" + "?for_export=true")); + Map importedDfBody = (Map)((List) entityAsMap(importedDfResponse).get("datafeeds")).get(0); + assertThat(originalDfBody, equalTo(importedDfBody)); + } + + @SuppressWarnings("unchecked") + public void testExportAndPutDataFrameAnalytics_OutlierDetection() throws Exception { + createAndIndexFarequote(); + String analyticsId = "outlier-export-import"; + XContentBuilder xContentBuilder = jsonBuilder(); + xContentBuilder.startObject(); + { + xContentBuilder.field("description", "outlier analytics"); + + xContentBuilder.startObject("source"); + { + xContentBuilder.field("index", "airline-data"); + } + xContentBuilder.endObject(); + xContentBuilder.startObject("dest"); + { + xContentBuilder.field("index", "outliers-airline-data"); + } + xContentBuilder.endObject(); + xContentBuilder.startObject("analysis"); + { + xContentBuilder.startObject("outlier_detection"); + { + xContentBuilder.field("compute_feature_influence", false); + } + xContentBuilder.endObject(); + } + xContentBuilder.endObject(); + } + xContentBuilder.endObject(); + + Request request = new Request("PUT", BASE_PATH + "data_frame/analytics/" + analyticsId); + request.setJsonEntity(Strings.toString(xContentBuilder)); + client().performRequest(request); + + Response jobResponse = client().performRequest( + new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "?for_export=true")); + Map originalJobBody = (Map)((List) entityAsMap(jobResponse).get("data_frame_analytics")).get(0); + + XContentBuilder newBuilder = jsonBuilder().map(originalJobBody); + request = new Request("PUT", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import"); + request.setJsonEntity(Strings.toString(newBuilder)); + client().performRequest(request); + + Response importedJobResponse = client().performRequest( + new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import" + "?for_export=true")); + Map importedJobBody = (Map)((List) entityAsMap(importedJobResponse) + .get("data_frame_analytics")) + .get(0); + assertThat(originalJobBody, equalTo(importedJobBody)); + } + + @SuppressWarnings("unchecked") + public void testExportAndPutDataFrameAnalytics_Regression() throws Exception { + createAndIndexFarequote(); + String analyticsId = "regression-export-import"; + XContentBuilder xContentBuilder = jsonBuilder(); + xContentBuilder.startObject(); + { + xContentBuilder.field("description", "regression analytics"); + + xContentBuilder.startObject("source"); + { + xContentBuilder.field("index", "airline-data"); + } + xContentBuilder.endObject(); + xContentBuilder.startObject("dest"); + { + xContentBuilder.field("index", "regression-airline-data"); + } + xContentBuilder.endObject(); + xContentBuilder.startObject("analysis"); + { + xContentBuilder.startObject("regression"); + { + xContentBuilder.field("dependent_variable", "responsetime"); + xContentBuilder.field("training_percent", 50); + } + xContentBuilder.endObject(); + } + xContentBuilder.endObject(); + } + xContentBuilder.endObject(); + + Request request = new Request("PUT", BASE_PATH + "data_frame/analytics/" + analyticsId); + request.setJsonEntity(Strings.toString(xContentBuilder)); + client().performRequest(request); + + Response jobResponse = client().performRequest( + new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "?for_export=true")); + Map originalJobBody = (Map)((List) entityAsMap(jobResponse).get("data_frame_analytics")).get(0); + + XContentBuilder newBuilder = jsonBuilder().map(originalJobBody); + request = new Request("PUT", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import"); + request.setJsonEntity(Strings.toString(newBuilder)); + client().performRequest(request); + + Response importedJobResponse = client().performRequest( + new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import" + "?for_export=true")); + Map importedJobBody = (Map)((List) entityAsMap(importedJobResponse) + .get("data_frame_analytics")) + .get(0); + assertThat(originalJobBody, equalTo(importedJobBody)); + } + + @SuppressWarnings("unchecked") + public void testExportAndPutDataFrameAnalytics_Classification() throws Exception { + createAndIndexFarequote(); + String analyticsId = "classification-export-import"; + XContentBuilder xContentBuilder = jsonBuilder(); + xContentBuilder.startObject(); + { + xContentBuilder.field("description", "classification analytics"); + + xContentBuilder.startObject("source"); + { + xContentBuilder.field("index", "airline-data"); + } + xContentBuilder.endObject(); + xContentBuilder.startObject("dest"); + { + xContentBuilder.field("index", "classification-airline-data"); + } + xContentBuilder.endObject(); + xContentBuilder.startObject("analysis"); + { + xContentBuilder.startObject("classification"); + { + xContentBuilder.field("dependent_variable", "airline"); + xContentBuilder.field("training_percent", 60); + } + xContentBuilder.endObject(); + } + xContentBuilder.endObject(); + } + xContentBuilder.endObject(); + + Request request = new Request("PUT", BASE_PATH + "data_frame/analytics/" + analyticsId); + request.setJsonEntity(Strings.toString(xContentBuilder)); + client().performRequest(request); + + Response jobResponse = client().performRequest( + new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "?for_export=true")); + Map originalJobBody = (Map)((List) entityAsMap(jobResponse).get("data_frame_analytics")).get(0); + + XContentBuilder newBuilder = jsonBuilder().map(originalJobBody); + request = new Request("PUT", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import"); + request.setJsonEntity(Strings.toString(newBuilder)); + client().performRequest(request); + + Response importedJobResponse = client().performRequest( + new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import" + "?for_export=true")); + Map importedJobBody = (Map)((List) entityAsMap(importedJobResponse) + .get("data_frame_analytics")) + .get(0); + assertThat(originalJobBody, equalTo(importedJobBody)); + } + private Response createDatafeed(String datafeedId, String jobId) throws Exception { XContentBuilder xContentBuilder = jsonBuilder(); xContentBuilder.startObject(); @@ -322,4 +504,31 @@ public class MlBasicMultiNodeIT extends ESRestTestCase { assertThat(asMap.get("flushed"), is(true)); assertThat(asMap.get("last_finalized_bucket_end"), equalTo(expectedLastFinalizedBucketEnd)); } + + private void createAndIndexFarequote() throws Exception { + boolean datesHaveNanoSecondResolution = randomBoolean(); + String dateMappingType = datesHaveNanoSecondResolution ? "date_nanos" : "date"; + String dateFormat = datesHaveNanoSecondResolution ? "strict_date_optional_time_nanos" : "strict_date_optional_time"; + String randomNanos = datesHaveNanoSecondResolution ? "," + randomIntBetween(100000000, 999999999) : ""; + Request createAirlineDataRequest = new Request("PUT", "/airline-data"); + createAirlineDataRequest.setJsonEntity("{" + + " \"mappings\": {" + + " \"properties\": {" + + " \"time\": { \"type\":\"" + dateMappingType + "\", \"format\":\"" + dateFormat + "\"}," + + " \"airline\": { \"type\":\"keyword\"}," + + " \"responsetime\": { \"type\":\"float\"}" + + " }" + + " }" + + "}"); + client().performRequest(createAirlineDataRequest); + Request airlineData1 = new Request("PUT", "/airline-data/_doc/1"); + airlineData1.setJsonEntity("{\"time\":\"2016-06-01T00:00:00" + randomNanos + "Z\",\"airline\":\"AAA\",\"responsetime\":135.22}"); + client().performRequest(airlineData1); + Request airlineData2 = new Request("PUT", "/airline-data/_doc/2"); + airlineData2.setJsonEntity("{\"time\":\"2016-06-01T01:59:00" + randomNanos + "Z\",\"airline\":\"AAA\",\"responsetime\":541.76}"); + client().performRequest(airlineData2); + + // Ensure all data is searchable + refreshAllIndices(); + } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractor.java index 16447f2d97ec..94b0a7277a88 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractor.java @@ -320,11 +320,11 @@ public class ChunkedDataExtractor implements DataExtractor { } /** - * This heuristic is a direct copy of the manual chunking config auto-creation done in {@link DatafeedConfig.Builder} + * This heuristic is a direct copy of the manual chunking config auto-creation done in {@link DatafeedConfig} */ @Override public long estimateChunk() { - return DatafeedConfig.Builder.DEFAULT_AGGREGATION_CHUNKING_BUCKETS * histogramIntervalMillis; + return DatafeedConfig.DEFAULT_AGGREGATION_CHUNKING_BUCKETS * histogramIntervalMillis; } @Override diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedsAction.java index 6eeb50a8c2be..35ca39a4e05c 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedsAction.java @@ -16,9 +16,12 @@ import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.ml.MachineLearning; import java.io.IOException; +import java.util.Collections; import java.util.List; +import java.util.Set; import static org.elasticsearch.rest.RestRequest.Method.GET; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT; public class RestGetDatafeedsAction extends BaseRestHandler { @@ -51,4 +54,9 @@ public class RestGetDatafeedsAction extends BaseRestHandler { restRequest.paramAsBoolean(Request.ALLOW_NO_DATAFEEDS, request.allowNoMatch()))); return channel -> client.execute(GetDatafeedsAction.INSTANCE, request, new RestToXContentListener<>(channel)); } + + @Override + protected Set responseParams() { + return Collections.singleton(FOR_EXPORT); + } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestGetDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestGetDataFrameAnalyticsAction.java index 6d6c8f367655..140b00f0a0bd 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestGetDataFrameAnalyticsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestGetDataFrameAnalyticsAction.java @@ -16,9 +16,12 @@ import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig; import org.elasticsearch.xpack.ml.MachineLearning; import java.io.IOException; +import java.util.Collections; import java.util.List; +import java.util.Set; import static org.elasticsearch.rest.RestRequest.Method.GET; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT; public class RestGetDataFrameAnalyticsAction extends BaseRestHandler { @@ -49,4 +52,9 @@ public class RestGetDataFrameAnalyticsAction extends BaseRestHandler { request.isAllowNoResources())); return channel -> client.execute(GetDataFrameAnalyticsAction.INSTANCE, request, new RestToXContentListener<>(channel)); } + + @Override + protected Set responseParams() { + return Collections.singleton(FOR_EXPORT); + } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestGetTrainedModelsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestGetTrainedModelsAction.java index c729c4d5f1cd..bc563c823fa0 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestGetTrainedModelsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestGetTrainedModelsAction.java @@ -34,6 +34,7 @@ import java.util.Set; import static java.util.Arrays.asList; import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.xpack.core.ml.action.GetTrainedModelsAction.Request.ALLOW_NO_MATCH; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT; public class RestGetTrainedModelsAction extends BaseRestHandler { @@ -98,7 +99,7 @@ public class RestGetTrainedModelsAction extends BaseRestHandler { @Override protected Set responseParams() { - return Set.of(TrainedModelConfig.DECOMPRESS_DEFINITION, TrainedModelConfig.FOR_EXPORT); + return Set.of(TrainedModelConfig.DECOMPRESS_DEFINITION, FOR_EXPORT); } private static class RestToXContentListenerWithDefaultValues extends RestToXContentListener { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobsAction.java index 2ee8605dc6e9..a90df97bddd4 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobsAction.java @@ -18,9 +18,12 @@ import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.ml.MachineLearning; import java.io.IOException; +import java.util.Collections; import java.util.List; +import java.util.Set; import static org.elasticsearch.rest.RestRequest.Method.GET; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT; public class RestGetJobsAction extends BaseRestHandler { @@ -53,4 +56,9 @@ public class RestGetJobsAction extends BaseRestHandler { restRequest.paramAsBoolean(Request.ALLOW_NO_JOBS, request.allowNoMatch()))); return channel -> client.execute(GetJobsAction.INSTANCE, request, new RestToXContentListener<>(channel)); } + + @Override + protected Set responseParams() { + return Collections.singleton(FOR_EXPORT); + } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_data_frame_analytics.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_data_frame_analytics.json index 80b4649453f7..5bd3cd20b413 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_data_frame_analytics.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_data_frame_analytics.json @@ -43,6 +43,12 @@ "type":"int", "description":"specifies a max number of analytics to get", "default":100 + }, + "for_export": { + "required": false, + "type": "boolean", + "default": false, + "description": "Omits fields that are illegal to set on data frame analytics PUT" } } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_datafeeds.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_datafeeds.json index 56f5ea49f852..b7a3760e56a3 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_datafeeds.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_datafeeds.json @@ -38,6 +38,12 @@ "required":false, "description":"Whether to ignore if a wildcard expression matches no datafeeds. (This includes `_all` string or when no datafeeds have been specified)", "deprecated":true + }, + "for_export": { + "required": false, + "type": "boolean", + "default": false, + "description": "Omits fields that are illegal to set on datafeed PUT" } } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_jobs.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_jobs.json index 7a1ebaed08ce..ae3f0299747a 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_jobs.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_jobs.json @@ -38,6 +38,12 @@ "required":false, "description":"Whether to ignore if a wildcard expression matches no jobs. (This includes `_all` string or when no jobs have been specified)", "deprecated":true + }, + "for_export": { + "required": false, + "type": "boolean", + "default": false, + "description": "Omits fields that are illegal to set on job PUT" } } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml index b5816f80829a..722f7d6512e3 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml @@ -2168,3 +2168,37 @@ setup: { "description": "blah" } + +--- +"Test GET config for export": + + - do: + ml.put_data_frame_analytics: + id: "simple-outlier-detection" + body: > + { + "source": { + "index": "index-source" + }, + "dest": { + "index": "index-dest" + }, + "analysis": {"outlier_detection":{}} + } + - do: + ml.get_data_frame_analytics: + id: "simple-outlier-detection" + for_export: true + - match: { data_frame_analytics.0.source.index.0: "index-source" } + - match: { data_frame_analytics.0.source.query: {"match_all" : {} } } + - match: { data_frame_analytics.0.dest.index: "index-dest" } + - match: { data_frame_analytics.0.analysis: { + "outlier_detection":{ + "compute_feature_influence": true, + "outlier_fraction": 0.05, + "standardization_enabled": true + } + }} + - is_false: data_frame_analytics.0.create_time + - is_false: data_frame_analytics.0.version + - is_false: data_frame_analytics.0.id diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/datafeeds_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/datafeeds_crud.yml index b105576312e2..9b740c0c9a03 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/datafeeds_crud.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/datafeeds_crud.yml @@ -506,3 +506,23 @@ setup: - match: { datafeeds.0.indices_options.ignore_throttled: false } - match: { datafeeds.0.indices_options.allow_no_indices: false } +--- +"Test get datafeed for export": + - do: + ml.put_datafeed: + datafeed_id: test-for-export + body: > + { + "job_id":"datafeeds-crud-1", + "indexes":["index-foo"] + } + - do: + ml.get_datafeeds: + datafeed_id: test-for-export + for_export: true + - match: { datafeeds.0.indices.0: "index-foo"} + - is_false: datafeeds.0.datafeed_id + - is_false: datafeeds.0.job_id + - is_false: datafeeds.0.create_time + - is_false: datafeeds.0.query_delay + - is_false: datafeeds.0.chunking_config diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get.yml index 1ef10bfec147..b305e73069ef 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get.yml @@ -86,3 +86,19 @@ setup: - match: { jobs.0.description: "Job 1"} - match: { jobs.1.job_id: "jobs-get-2"} - match: { jobs.1.description: "Job 2"} +--- +"Test get job for export": + + - do: + ml.get_jobs: + job_id: jobs-get-1 + for_export: true + - match: { jobs.0.description: "Job 1"} + - is_false: job_id + - is_false: job_type + - is_false: job_version + - is_false: create_time + - is_false: finished_time + - is_false: model_snapshot_id + - is_false: model_snapshot_min_version + - is_false: deleting