From 8af319481ed8862bc4bdd9d909de9eb2bb5b2d0e Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Wed, 21 Aug 2019 19:58:09 +0300 Subject: [PATCH] [ML] Add description to DF analytics (#45774) --- .../dataframe/DataFrameAnalyticsConfig.java | 30 +++++++++++++--- .../client/MachineLearningIT.java | 2 ++ .../MlClientDocumentationIT.java | 1 + .../DataFrameAnalyticsConfigTests.java | 3 ++ .../ml/put-data-frame-analytics.asciidoc | 1 + .../apis/dfanalyticsresources.asciidoc | 3 ++ .../apis/put-dfanalytics.asciidoc | 7 +++- .../dataframe/DataFrameAnalyticsConfig.java | 35 +++++++++++++++++-- .../ml/job/results/ReservedFieldNames.java | 1 + .../DataFrameAnalyticsConfigTests.java | 3 ++ .../test/ml/data_frame_analytics_crud.yml | 33 +++++++++++++++++ 11 files changed, 110 insertions(+), 9 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsConfig.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsConfig.java index e7eb2967d815..cdba0e5d5c72 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsConfig.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsConfig.java @@ -48,6 +48,7 @@ public class DataFrameAnalyticsConfig implements ToXContentObject { } private static final ParseField ID = new ParseField("id"); + private static final ParseField DESCRIPTION = new ParseField("description"); private static final ParseField SOURCE = new ParseField("source"); private static final ParseField DEST = new ParseField("dest"); private static final ParseField ANALYSIS = new ParseField("analysis"); @@ -60,6 +61,7 @@ public class DataFrameAnalyticsConfig implements ToXContentObject { static { PARSER.declareString(Builder::setId, ID); + PARSER.declareString(Builder::setDescription, DESCRIPTION); PARSER.declareObject(Builder::setSource, (p, c) -> DataFrameAnalyticsSource.fromXContent(p), SOURCE); PARSER.declareObject(Builder::setDest, (p, c) -> DataFrameAnalyticsDest.fromXContent(p), DEST); PARSER.declareObject(Builder::setAnalysis, (p, c) -> parseAnalysis(p), ANALYSIS); @@ -95,6 +97,7 @@ public class DataFrameAnalyticsConfig implements ToXContentObject { } private final String id; + private final String description; private final DataFrameAnalyticsSource source; private final DataFrameAnalyticsDest dest; private final DataFrameAnalysis analysis; @@ -103,10 +106,12 @@ public class DataFrameAnalyticsConfig implements ToXContentObject { private final Instant createTime; private final Version version; - private DataFrameAnalyticsConfig(@Nullable String id, @Nullable DataFrameAnalyticsSource source, @Nullable DataFrameAnalyticsDest dest, - @Nullable DataFrameAnalysis analysis, @Nullable FetchSourceContext analyzedFields, - @Nullable ByteSizeValue modelMemoryLimit, @Nullable Instant createTime, @Nullable Version version) { + private DataFrameAnalyticsConfig(@Nullable String id, @Nullable String description, @Nullable DataFrameAnalyticsSource source, + @Nullable DataFrameAnalyticsDest dest, @Nullable DataFrameAnalysis analysis, + @Nullable FetchSourceContext analyzedFields, @Nullable ByteSizeValue modelMemoryLimit, + @Nullable Instant createTime, @Nullable Version version) { this.id = id; + this.description = description; this.source = source; this.dest = dest; this.analysis = analysis; @@ -120,6 +125,10 @@ public class DataFrameAnalyticsConfig implements ToXContentObject { return id; } + public String getDescription() { + return description; + } + public DataFrameAnalyticsSource getSource() { return source; } @@ -154,6 +163,9 @@ public class DataFrameAnalyticsConfig implements ToXContentObject { if (id != null) { builder.field(ID.getPreferredName(), id); } + if (description != null) { + builder.field(DESCRIPTION.getPreferredName(), description); + } if (source != null) { builder.field(SOURCE.getPreferredName(), source); } @@ -189,6 +201,7 @@ public class DataFrameAnalyticsConfig implements ToXContentObject { DataFrameAnalyticsConfig other = (DataFrameAnalyticsConfig) o; return Objects.equals(id, other.id) + && Objects.equals(description, other.description) && Objects.equals(source, other.source) && Objects.equals(dest, other.dest) && Objects.equals(analysis, other.analysis) @@ -200,7 +213,7 @@ public class DataFrameAnalyticsConfig implements ToXContentObject { @Override public int hashCode() { - return Objects.hash(id, source, dest, analysis, analyzedFields, modelMemoryLimit, createTime, version); + return Objects.hash(id, description, source, dest, analysis, analyzedFields, modelMemoryLimit, createTime, version); } @Override @@ -211,6 +224,7 @@ public class DataFrameAnalyticsConfig implements ToXContentObject { public static class Builder { private String id; + private String description; private DataFrameAnalyticsSource source; private DataFrameAnalyticsDest dest; private DataFrameAnalysis analysis; @@ -226,6 +240,11 @@ public class DataFrameAnalyticsConfig implements ToXContentObject { return this; } + public Builder setDescription(String description) { + this.description = description; + return this; + } + public Builder setSource(DataFrameAnalyticsSource source) { this.source = Objects.requireNonNull(source); return this; @@ -262,7 +281,8 @@ public class DataFrameAnalyticsConfig implements ToXContentObject { } public DataFrameAnalyticsConfig build() { - return new DataFrameAnalyticsConfig(id, source, dest, analysis, analyzedFields, modelMemoryLimit, createTime, version); + return new DataFrameAnalyticsConfig(id, description, source, dest, analysis, analyzedFields, modelMemoryLimit, createTime, + version); } } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java index 2111b314c9b4..9d7f1d9352ca 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java @@ -1225,6 +1225,7 @@ public class MachineLearningIT extends ESRestHighLevelClientTestCase { .setIndex("put-test-dest-index") .build()) .setAnalysis(OutlierDetection.createDefault()) + .setDescription("some description") .build(); createIndex("put-test-source-index", defaultMappingForTest()); @@ -1241,6 +1242,7 @@ public class MachineLearningIT extends ESRestHighLevelClientTestCase { assertThat(createdConfig.getAnalysis(), equalTo(config.getAnalysis())); assertThat(createdConfig.getAnalyzedFields(), equalTo(config.getAnalyzedFields())); assertThat(createdConfig.getModelMemoryLimit(), equalTo(ByteSizeValue.parseBytesSizeValue("1gb", ""))); // default value + assertThat(createdConfig.getDescription(), equalTo("some description")); } public void testGetDataFrameAnalyticsConfig_SingleConfig() throws Exception { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java index 809985a9bb30..a581e6f39bcb 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java @@ -2951,6 +2951,7 @@ public class MlClientDocumentationIT extends ESRestHighLevelClientTestCase { .setAnalysis(outlierDetection) // <4> .setAnalyzedFields(analyzedFields) // <5> .setModelMemoryLimit(new ByteSizeValue(5, ByteSizeUnit.MB)) // <6> + .setDescription("this is an example description") // <7> .build(); // end::put-data-frame-analytics-config diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsConfigTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsConfigTests.java index 609bb543da59..13f277ade995 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsConfigTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsConfigTests.java @@ -49,6 +49,9 @@ public class DataFrameAnalyticsConfigTests extends AbstractXContentTestCase The analysis to be performed <5> The fields to be included in / excluded from the analysis <6> The memory limit for the model created as part of the analysis process +<7> Optionally, a human-readable description [id="{upid}-{api}-query-config"] diff --git a/docs/reference/ml/df-analytics/apis/dfanalyticsresources.asciidoc b/docs/reference/ml/df-analytics/apis/dfanalyticsresources.asciidoc index db5e932ab116..34535ba274bd 100644 --- a/docs/reference/ml/df-analytics/apis/dfanalyticsresources.asciidoc +++ b/docs/reference/ml/df-analytics/apis/dfanalyticsresources.asciidoc @@ -42,6 +42,9 @@ PUT _ml/data_frame/analytics/loganalytics // CONSOLE // TEST[setup:setup_logdata] +`description`:: + (Optional, string) A description of the job. + `dest`:: (object) The destination configuration of the analysis. The `index` property (string) is the name of the index in which to store the results of the diff --git a/docs/reference/ml/df-analytics/apis/put-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/put-dfanalytics.asciidoc index e89711d23c2e..d695876c9f0f 100644 --- a/docs/reference/ml/df-analytics/apis/put-dfanalytics.asciidoc +++ b/docs/reference/ml/df-analytics/apis/put-dfanalytics.asciidoc @@ -65,7 +65,10 @@ and mappings. (Optional, object) You can specify both `includes` and/or `excludes` patterns. If `analyzed_fields` is not set, only the relevant fields will be included. For example, all the numeric fields for {oldetection}. - + +`description`:: + (Optional, string) A description of the job. + `dest`:: (Required, object) The destination configuration, consisting of `index` and optionally `results_field` (`ml` by default). See @@ -94,6 +97,7 @@ type is `outlier_detection`: -------------------------------------------------- PUT _ml/data_frame/analytics/loganalytics { + "description": "Outlier detection on log data", "source": { "index": "logdata" }, @@ -115,6 +119,7 @@ The API returns the following result: ---- { "id": "loganalytics", + "description": "Outlier detection on log data", "source": { "index": ["logdata"], "query": { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java index f194d108ad0b..2d65013e31e1 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java @@ -44,6 +44,7 @@ public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable { public static final ByteSizeValue PROCESS_MEMORY_OVERHEAD = new ByteSizeValue(20, ByteSizeUnit.MB); public static final ParseField ID = new ParseField("id"); + public static final ParseField DESCRIPTION = new ParseField("description"); public static final ParseField SOURCE = new ParseField("source"); public static final ParseField DEST = new ParseField("dest"); public static final ParseField ANALYSIS = new ParseField("analysis"); @@ -62,6 +63,7 @@ public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable { parser.declareString((c, s) -> {}, CONFIG_TYPE); parser.declareString(Builder::setId, ID); + parser.declareString(Builder::setDescription, DESCRIPTION); parser.declareObject(Builder::setSource, DataFrameAnalyticsSource.createParser(ignoreUnknownFields), SOURCE); parser.declareObject(Builder::setDest, DataFrameAnalyticsDest.createParser(ignoreUnknownFields), DEST); parser.declareObject(Builder::setAnalysis, (p, c) -> parseAnalysis(p, ignoreUnknownFields), ANALYSIS); @@ -100,6 +102,7 @@ public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable { } private final String id; + private final String description; private final DataFrameAnalyticsSource source; private final DataFrameAnalyticsDest dest; private final DataFrameAnalysis analysis; @@ -117,10 +120,11 @@ public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable { private final Instant createTime; private final Version version; - public DataFrameAnalyticsConfig(String id, DataFrameAnalyticsSource source, DataFrameAnalyticsDest dest, + public DataFrameAnalyticsConfig(String id, String description, DataFrameAnalyticsSource source, DataFrameAnalyticsDest dest, DataFrameAnalysis analysis, Map headers, ByteSizeValue modelMemoryLimit, FetchSourceContext analyzedFields, Instant createTime, Version version) { this.id = ExceptionsHelper.requireNonNull(id, ID); + this.description = description; this.source = ExceptionsHelper.requireNonNull(source, SOURCE); this.dest = ExceptionsHelper.requireNonNull(dest, DEST); this.analysis = ExceptionsHelper.requireNonNull(analysis, ANALYSIS); @@ -133,6 +137,11 @@ public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable { public DataFrameAnalyticsConfig(StreamInput in) throws IOException { this.id = in.readString(); + if (in.getVersion().onOrAfter(Version.V_7_4_0)) { + description = in.readOptionalString(); + } else { + description = null; + } this.source = new DataFrameAnalyticsSource(in); this.dest = new DataFrameAnalyticsDest(in); this.analysis = in.readNamedWriteable(DataFrameAnalysis.class); @@ -152,6 +161,10 @@ public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable { return id; } + public String getDescription() { + return description; + } + public DataFrameAnalyticsSource getSource() { return source; } @@ -188,6 +201,9 @@ public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field(ID.getPreferredName(), id); + if (description != null) { + builder.field(DESCRIPTION.getPreferredName(), description); + } builder.field(SOURCE.getPreferredName(), source); builder.field(DEST.getPreferredName(), dest); @@ -218,6 +234,9 @@ public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable { @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(id); + if (out.getVersion().onOrAfter(Version.V_7_4_0)) { + out.writeOptionalString(description); + } source.writeTo(out); dest.writeTo(out); out.writeNamedWriteable(analysis); @@ -242,6 +261,7 @@ public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable { DataFrameAnalyticsConfig other = (DataFrameAnalyticsConfig) o; return Objects.equals(id, other.id) + && Objects.equals(description, other.description) && Objects.equals(source, other.source) && Objects.equals(dest, other.dest) && Objects.equals(analysis, other.analysis) @@ -254,7 +274,7 @@ public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable { @Override public int hashCode() { - return Objects.hash(id, source, dest, analysis, headers, getModelMemoryLimit(), analyzedFields, createTime, version); + return Objects.hash(id, description, source, dest, analysis, headers, getModelMemoryLimit(), analyzedFields, createTime, version); } @Override @@ -269,6 +289,7 @@ public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable { public static class Builder { private String id; + private String description; private DataFrameAnalyticsSource source; private DataFrameAnalyticsDest dest; private DataFrameAnalysis analysis; @@ -287,6 +308,7 @@ public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable { public Builder(DataFrameAnalyticsConfig config, ByteSizeValue maxModelMemoryLimit) { this.id = config.id; + this.description = config.description; this.source = new DataFrameAnalyticsSource(config.source); this.dest = new DataFrameAnalyticsDest(config.dest); this.analysis = config.analysis; @@ -304,6 +326,11 @@ public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable { return id; } + public Builder setDescription(String description) { + this.description = description; + return this; + } + public Builder setId(String id) { this.id = ExceptionsHelper.requireNonNull(id, ID); return this; @@ -354,7 +381,8 @@ public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable { */ public DataFrameAnalyticsConfig build() { applyMaxModelMemoryLimit(); - return new DataFrameAnalyticsConfig(id, source, dest, analysis, headers, modelMemoryLimit, analyzedFields, createTime, version); + return new DataFrameAnalyticsConfig(id, description, source, dest, analysis, headers, modelMemoryLimit, analyzedFields, + createTime, version); } /** @@ -365,6 +393,7 @@ public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable { public DataFrameAnalyticsConfig buildForMemoryEstimation() { return new DataFrameAnalyticsConfig( id != null ? id : "dummy", + description, source, dest != null ? dest : new DataFrameAnalyticsDest("dummy", null), analysis, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java index 92583693af2e..91aa424a2483 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java @@ -286,6 +286,7 @@ public final class ReservedFieldNames { ChunkingConfig.TIME_SPAN_FIELD.getPreferredName(), DataFrameAnalyticsConfig.ID.getPreferredName(), + DataFrameAnalyticsConfig.DESCRIPTION.getPreferredName(), DataFrameAnalyticsConfig.SOURCE.getPreferredName(), DataFrameAnalyticsConfig.DEST.getPreferredName(), DataFrameAnalyticsConfig.ANALYSIS.getPreferredName(), diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigTests.java index f71857661152..e9d338eb4d18 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigTests.java @@ -113,6 +113,9 @@ public class DataFrameAnalyticsConfigTests extends AbstractSerializingTestCase + { + "description": "This is a described config", + "source": { + "index": "index-source" + }, + "dest": { + "index": "index-dest" + }, + "analysis": { + "regression": { + "dependent_variable": "foo" + } + } + } + - match: { id: "with-description" } + - match: { description: "This is a described config" } + - match: { source.index: ["index-source"] } + - match: { dest.index: "index-dest" } + - match: { analysis: { + "regression":{ + "dependent_variable": "foo" + } + }} + - is_true: create_time + - is_true: version