[Transform] Allow specifying destination index aliases in the Transform's dest config (#94943)

This commit is contained in:
Przemysław Witek 2023-04-17 15:08:43 +02:00 committed by GitHub
parent 32c17d79c5
commit 2b70165ffd
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
41 changed files with 1076 additions and 299 deletions

View file

@ -0,0 +1,5 @@
pr: 94943
summary: Allow specifying destination index aliases in the Transform's `dest` config
area: Transform
type: enhancement
issues: []

View file

@ -193,6 +193,25 @@ mappings for the destination index are undesirable, use the
<<indices-create-index,Create index API>> prior to starting the {transform}. <<indices-create-index,Create index API>> prior to starting the {transform}.
end::dest-index[] end::dest-index[]
tag::dest-aliases[]
The aliases that the destination index for the {transform} should have.
Aliases are manipulated using the stored credentials of the transform, which means the secondary credentials supplied
at creation time (if both primary and secondary credentials are specified).
The destination index is added to the aliases regardless whether the destination
index was created by the transform or pre-created by the user.
end::dest-aliases[]
tag::dest-aliases-alias[]
The name of the alias.
end::dest-aliases-alias[]
tag::dest-aliases-move-on-creation[]
Whether or not the destination index should be the **only** index in this alias.
If `true`, all the other indices will be removed from this alias before adding the destination index to this alias.
Defaults to `false`.
end::dest-aliases-move-on-creation[]
tag::dest-pipeline[] tag::dest-pipeline[]
The unique identifier for an <<ingest,ingest pipeline>>. The unique identifier for an <<ingest,ingest pipeline>>.
end::dest-pipeline[] end::dest-pipeline[]

View file

@ -111,6 +111,26 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=dest]
(Required, string) (Required, string)
include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=dest-index] include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=dest-index]
//Begin aliases
`aliases`:::
(Optional, array of objects)
include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=dest-aliases]
+
.Properties of `aliases`
[%collapsible%open]
=====
`alias`::::
(Required, string)
include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=dest-aliases-alias]
`move_on_creation`::::
(Optional, boolean)
include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=dest-aliases-move-on-creation]
=====
//End aliases
`pipeline`::: `pipeline`:::
(Optional, string) (Optional, string)
include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=dest-pipeline] include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=dest-pipeline]

View file

@ -92,6 +92,26 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=dest]
(Required, string) (Required, string)
include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=dest-index] include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=dest-index]
//Begin aliases
`aliases`:::
(Optional, array of objects)
include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=dest-aliases]
+
.Properties of `aliases`
[%collapsible%open]
=====
`alias`::::
(Required, string)
include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=dest-aliases-alias]
`move_on_creation`::::
(Optional, boolean)
include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=dest-aliases-move-on-creation]
=====
//End aliases
`pipeline`::: `pipeline`:::
(Optional, string) (Optional, string)
include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=dest-pipeline] include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=dest-pipeline]

View file

@ -40,6 +40,8 @@ public class TransformMessages {
+ "Use force stop and then restart the transform once error is resolved."; + "Use force stop and then restart the transform once error is resolved.";
public static final String FAILED_TO_CREATE_DESTINATION_INDEX = "Could not create destination index [{0}] for transform [{1}]"; public static final String FAILED_TO_CREATE_DESTINATION_INDEX = "Could not create destination index [{0}] for transform [{1}]";
public static final String FAILED_TO_SET_UP_DESTINATION_ALIASES =
"Could not set up aliases for destination index [{0}] for transform [{1}]";
public static final String FAILED_TO_RELOAD_TRANSFORM_CONFIGURATION = "Failed to reload transform configuration for transform [{0}]"; public static final String FAILED_TO_RELOAD_TRANSFORM_CONFIGURATION = "Failed to reload transform configuration for transform [{0}]";
public static final String FAILED_TO_LOAD_TRANSFORM_CONFIGURATION = "Failed to load transform configuration for transform [{0}]"; public static final String FAILED_TO_LOAD_TRANSFORM_CONFIGURATION = "Failed to load transform configuration for transform [{0}]";
public static final String FAILED_TO_PARSE_TRANSFORM_CONFIGURATION = "Failed to parse transform configuration for transform [{0}]"; public static final String FAILED_TO_PARSE_TRANSFORM_CONFIGURATION = "Failed to parse transform configuration for transform [{0}]";

View file

@ -0,0 +1,111 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
package org.elasticsearch.xpack.core.transform.transforms;
import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.xcontent.ConstructingObjectParser;
import org.elasticsearch.xcontent.ParseField;
import org.elasticsearch.xcontent.ToXContentObject;
import org.elasticsearch.xcontent.XContentBuilder;
import org.elasticsearch.xcontent.XContentParser;
import org.elasticsearch.xpack.core.transform.utils.ExceptionsHelper;
import java.io.IOException;
import java.util.Objects;
import static org.elasticsearch.action.ValidateActions.addValidationError;
import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
public class DestAlias implements Writeable, ToXContentObject {
public static final ParseField ALIAS = new ParseField("alias");
public static final ParseField MOVE_ON_CREATION = new ParseField("move_on_creation");
public static final ConstructingObjectParser<DestAlias, Void> STRICT_PARSER = createParser(false);
public static final ConstructingObjectParser<DestAlias, Void> LENIENT_PARSER = createParser(true);
private static ConstructingObjectParser<DestAlias, Void> createParser(boolean lenient) {
ConstructingObjectParser<DestAlias, Void> parser = new ConstructingObjectParser<>(
"data_frame_config_dest_alias",
lenient,
args -> new DestAlias((String) args[0], (Boolean) args[1])
);
parser.declareString(constructorArg(), ALIAS);
parser.declareBoolean(optionalConstructorArg(), MOVE_ON_CREATION);
return parser;
}
private final String alias;
private final boolean moveOnCreation;
public DestAlias(String alias, Boolean moveOnCreation) {
this.alias = ExceptionsHelper.requireNonNull(alias, ALIAS.getPreferredName());
this.moveOnCreation = moveOnCreation != null ? moveOnCreation : false;
}
public DestAlias(final StreamInput in) throws IOException {
alias = in.readString();
moveOnCreation = in.readBoolean();
}
public String getAlias() {
return alias;
}
public boolean isMoveOnCreation() {
return moveOnCreation;
}
public ActionRequestValidationException validate(ActionRequestValidationException validationException) {
if (alias.isEmpty()) {
validationException = addValidationError("dest.aliases.alias must not be empty", validationException);
}
return validationException;
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeString(alias);
out.writeBoolean(moveOnCreation);
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
builder.field(ALIAS.getPreferredName(), alias);
builder.field(MOVE_ON_CREATION.getPreferredName(), moveOnCreation);
builder.endObject();
return builder;
}
@Override
public boolean equals(Object other) {
if (other == this) {
return true;
}
if (other == null || other.getClass() != getClass()) {
return false;
}
DestAlias that = (DestAlias) other;
return Objects.equals(alias, that.alias) && moveOnCreation == that.moveOnCreation;
}
@Override
public int hashCode() {
return Objects.hash(alias, moveOnCreation);
}
public static DestAlias fromXContent(final XContentParser parser, boolean lenient) throws IOException {
return lenient ? LENIENT_PARSER.apply(parser, null) : STRICT_PARSER.apply(parser, null);
}
}

View file

@ -7,6 +7,7 @@
package org.elasticsearch.xpack.core.transform.transforms; package org.elasticsearch.xpack.core.transform.transforms;
import org.elasticsearch.TransportVersion;
import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.StreamOutput;
@ -21,6 +22,7 @@ import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
import org.elasticsearch.xpack.core.transform.utils.ExceptionsHelper; import org.elasticsearch.xpack.core.transform.utils.ExceptionsHelper;
import java.io.IOException; import java.io.IOException;
import java.util.List;
import java.util.Objects; import java.util.Objects;
import java.util.function.Consumer; import java.util.function.Consumer;
@ -31,32 +33,42 @@ import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstr
public class DestConfig implements Writeable, ToXContentObject { public class DestConfig implements Writeable, ToXContentObject {
public static final ParseField INDEX = new ParseField("index"); public static final ParseField INDEX = new ParseField("index");
public static final ParseField ALIASES = new ParseField("aliases");
public static final ParseField PIPELINE = new ParseField("pipeline"); public static final ParseField PIPELINE = new ParseField("pipeline");
public static final ConstructingObjectParser<DestConfig, Void> STRICT_PARSER = createParser(false); public static final ConstructingObjectParser<DestConfig, Void> STRICT_PARSER = createParser(false);
public static final ConstructingObjectParser<DestConfig, Void> LENIENT_PARSER = createParser(true); public static final ConstructingObjectParser<DestConfig, Void> LENIENT_PARSER = createParser(true);
@SuppressWarnings("unchecked")
private static ConstructingObjectParser<DestConfig, Void> createParser(boolean lenient) { private static ConstructingObjectParser<DestConfig, Void> createParser(boolean lenient) {
ConstructingObjectParser<DestConfig, Void> parser = new ConstructingObjectParser<>( ConstructingObjectParser<DestConfig, Void> parser = new ConstructingObjectParser<>(
"data_frame_config_dest", "data_frame_config_dest",
lenient, lenient,
args -> new DestConfig((String) args[0], (String) args[1]) args -> new DestConfig((String) args[0], (List<DestAlias>) args[1], (String) args[2])
); );
parser.declareString(constructorArg(), INDEX); parser.declareString(constructorArg(), INDEX);
parser.declareObjectArray(optionalConstructorArg(), lenient ? DestAlias.LENIENT_PARSER : DestAlias.STRICT_PARSER, ALIASES);
parser.declareString(optionalConstructorArg(), PIPELINE); parser.declareString(optionalConstructorArg(), PIPELINE);
return parser; return parser;
} }
private final String index; private final String index;
private final List<DestAlias> aliases;
private final String pipeline; private final String pipeline;
public DestConfig(String index, String pipeline) { public DestConfig(String index, List<DestAlias> aliases, String pipeline) {
this.index = ExceptionsHelper.requireNonNull(index, INDEX.getPreferredName()); this.index = ExceptionsHelper.requireNonNull(index, INDEX.getPreferredName());
this.aliases = aliases;
this.pipeline = pipeline; this.pipeline = pipeline;
} }
public DestConfig(final StreamInput in) throws IOException { public DestConfig(final StreamInput in) throws IOException {
index = in.readString(); index = in.readString();
if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_8_0)) {
aliases = in.readOptionalList(DestAlias::new);
} else {
aliases = null;
}
pipeline = in.readOptionalString(); pipeline = in.readOptionalString();
} }
@ -64,6 +76,10 @@ public class DestConfig implements Writeable, ToXContentObject {
return index; return index;
} }
public List<DestAlias> getAliases() {
return aliases != null ? aliases : List.of();
}
public String getPipeline() { public String getPipeline() {
return pipeline; return pipeline;
} }
@ -80,6 +96,9 @@ public class DestConfig implements Writeable, ToXContentObject {
@Override @Override
public void writeTo(StreamOutput out) throws IOException { public void writeTo(StreamOutput out) throws IOException {
out.writeString(index); out.writeString(index);
if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_8_0)) {
out.writeOptionalCollection(aliases);
}
out.writeOptionalString(pipeline); out.writeOptionalString(pipeline);
} }
@ -87,6 +106,9 @@ public class DestConfig implements Writeable, ToXContentObject {
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(); builder.startObject();
builder.field(INDEX.getPreferredName(), index); builder.field(INDEX.getPreferredName(), index);
if (aliases != null) {
builder.field(ALIASES.getPreferredName(), aliases);
}
if (pipeline != null) { if (pipeline != null) {
builder.field(PIPELINE.getPreferredName(), pipeline); builder.field(PIPELINE.getPreferredName(), pipeline);
} }
@ -104,12 +126,12 @@ public class DestConfig implements Writeable, ToXContentObject {
} }
DestConfig that = (DestConfig) other; DestConfig that = (DestConfig) other;
return Objects.equals(index, that.index) && Objects.equals(pipeline, that.pipeline); return Objects.equals(index, that.index) && Objects.equals(aliases, that.aliases) && Objects.equals(pipeline, that.pipeline);
} }
@Override @Override
public int hashCode() { public int hashCode() {
return Objects.hash(index, pipeline); return Objects.hash(index, aliases, pipeline);
} }
public static DestConfig fromXContent(final XContentParser parser, boolean lenient) throws IOException { public static DestConfig fromXContent(final XContentParser parser, boolean lenient) throws IOException {

View file

@ -55,7 +55,7 @@ public class PreviewTransformActionRequestTests extends AbstractSerializingTrans
TransformConfig config = new TransformConfig( TransformConfig config = new TransformConfig(
"transform-preview", "transform-preview",
randomSourceConfig(), randomSourceConfig(),
new DestConfig("unused-transform-preview-index", null), new DestConfig("unused-transform-preview-index", null, null),
null, null,
randomBoolean() ? TransformConfigTests.randomSyncConfig() : null, randomBoolean() ? TransformConfigTests.randomSyncConfig() : null,
null, null,

View file

@ -0,0 +1,73 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
package org.elasticsearch.xpack.core.transform.transforms;
import org.elasticsearch.common.ValidationException;
import org.elasticsearch.common.io.stream.Writeable.Reader;
import org.elasticsearch.xcontent.XContentParser;
import org.elasticsearch.xcontent.json.JsonXContent;
import org.elasticsearch.xpack.core.transform.AbstractSerializingTransformTestCase;
import org.junit.Before;
import java.io.IOException;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.emptyString;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.notNullValue;
public class DestAliasTests extends AbstractSerializingTransformTestCase<DestAlias> {
private boolean lenient;
public static DestAlias randomDestAlias() {
return new DestAlias(randomAlphaOfLength(10), randomBoolean());
}
@Before
public void setRandomFeatures() {
lenient = randomBoolean();
}
@Override
protected DestAlias doParseInstance(XContentParser parser) throws IOException {
return DestAlias.fromXContent(parser, lenient);
}
@Override
protected boolean supportsUnknownFields() {
return lenient;
}
@Override
protected DestAlias createTestInstance() {
return randomDestAlias();
}
@Override
protected DestAlias mutateInstance(DestAlias instance) {
return new DestAlias(instance.getAlias() + "-x", instance.isMoveOnCreation() == false);
}
@Override
protected Reader<DestAlias> instanceReader() {
return DestAlias::new;
}
public void testFailOnEmptyAlias() throws IOException {
boolean lenient2 = randomBoolean();
String json = "{ \"alias\": \"\" }";
try (XContentParser parser = createParser(JsonXContent.jsonXContent, json)) {
DestAlias destAlias = DestAlias.fromXContent(parser, lenient2);
assertThat(destAlias.getAlias(), is(emptyString()));
ValidationException validationException = destAlias.validate(null);
assertThat(validationException, is(notNullValue()));
assertThat(validationException.getMessage(), containsString("dest.aliases.alias must not be empty"));
}
}
}

View file

@ -26,7 +26,11 @@ public class DestConfigTests extends AbstractSerializingTransformTestCase<DestCo
private boolean lenient; private boolean lenient;
public static DestConfig randomDestConfig() { public static DestConfig randomDestConfig() {
return new DestConfig(randomAlphaOfLength(10), randomBoolean() ? null : randomAlphaOfLength(10)); return new DestConfig(
randomAlphaOfLength(10),
randomBoolean() ? null : randomList(5, DestAliasTests::randomDestAlias),
randomBoolean() ? null : randomAlphaOfLength(10)
);
} }
@Before @Before

View file

@ -21,6 +21,7 @@ import org.elasticsearch.xpack.core.transform.transforms.pivot.PivotConfigTests;
import java.io.IOException; import java.io.IOException;
import java.time.Instant; import java.time.Instant;
import java.util.Collections; import java.util.Collections;
import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Optional; import java.util.Optional;
@ -146,7 +147,7 @@ public class TransformConfigUpdateTests extends AbstractWireSerializingTransform
assertThat(config, equalTo(update.apply(config))); assertThat(config, equalTo(update.apply(config)));
SourceConfig sourceConfig = new SourceConfig("the_new_index"); SourceConfig sourceConfig = new SourceConfig("the_new_index");
DestConfig destConfig = new DestConfig("the_new_dest", "my_new_pipeline"); DestConfig destConfig = new DestConfig("the_new_dest", List.of(new DestAlias("my_new_alias", false)), "my_new_pipeline");
TimeValue frequency = TimeValue.timeValueSeconds(10); TimeValue frequency = TimeValue.timeValueSeconds(10);
SyncConfig syncConfig = new TimeSyncConfig("time_field", TimeValue.timeValueSeconds(30)); SyncConfig syncConfig = new TimeSyncConfig("time_field", TimeValue.timeValueSeconds(30));
String newDescription = "new description"; String newDescription = "new description";

View file

@ -48,7 +48,8 @@ testClusters.matching { it.name == 'javaRestTest' }.configureEach {
rolesFile file('roles.yml') rolesFile file('roles.yml')
user username: "x_pack_rest_user", password: "x-pack-test-password" user username: "x_pack_rest_user", password: "x-pack-test-password"
user username: "john_junior", password: "x-pack-test-password", role: "transform_admin" user username: "john_junior", password: "x-pack-test-password", role: "transform_admin"
user username: "bill_senior", password: "x-pack-test-password", role: "transform_admin,source_index_access" user username: "bill_senior", password: "x-pack-test-password", role: "transform_admin,source_index_access,dest_index_access"
user username: "not_a_transform_admin", password: "x-pack-test-password", role: "source_index_access" user username: "source_and_dest_index_access_only", password: "x-pack-test-password", role: "source_index_access,dest_index_access"
user username: "fleet_access", password: "x-pack-test-password", role: "transform_admin,source_index_access,fleet_index_access" user username: "transform_user_but_not_admin", password: "x-pack-test-password", role: "transform_user,source_index_access,dest_index_access"
user username: "fleet_access", password: "x-pack-test-password", role: "transform_admin,fleet_index_access,dest_index_access"
} }

View file

@ -4,9 +4,21 @@ source_index_access:
- cluster:monitor/main - cluster:monitor/main
- cluster:monitor/tasks/lists - cluster:monitor/tasks/lists
indices: indices:
# Give access to the source and destination indices because the transform roles alone do not provide access to # Give access to the source indices because the transform roles alone do not provide access to non-transform indices
# non-transform indices - names: [ 'transform-permissions-*-index' ]
- names: [ 'transform-permissions-*' ] privileges:
- create_index
- indices:admin/refresh
- read
- write
- view_index_metadata
- indices:data/write/bulk
- indices:data/write/index
dest_index_access:
indices:
# Give access to the destination indices
- names: [ 'transform-permissions-*-dest' ]
privileges: privileges:
- create_index - create_index
- indices:admin/refresh - indices:admin/refresh

View file

@ -16,8 +16,11 @@ import org.elasticsearch.core.TimeValue;
import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.AggregatorFactories;
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
import org.elasticsearch.xpack.core.transform.transforms.DestAlias;
import org.elasticsearch.xpack.core.transform.transforms.DestConfig;
import org.elasticsearch.xpack.core.transform.transforms.QueryConfig; import org.elasticsearch.xpack.core.transform.transforms.QueryConfig;
import org.elasticsearch.xpack.core.transform.transforms.SettingsConfig; import org.elasticsearch.xpack.core.transform.transforms.SettingsConfig;
import org.elasticsearch.xpack.core.transform.transforms.SourceConfig;
import org.elasticsearch.xpack.core.transform.transforms.TimeSyncConfig; import org.elasticsearch.xpack.core.transform.transforms.TimeSyncConfig;
import org.elasticsearch.xpack.core.transform.transforms.TransformConfig; import org.elasticsearch.xpack.core.transform.transforms.TransformConfig;
import org.elasticsearch.xpack.core.transform.transforms.pivot.SingleGroupSource; import org.elasticsearch.xpack.core.transform.transforms.pivot.SingleGroupSource;
@ -25,10 +28,12 @@ import org.elasticsearch.xpack.core.transform.transforms.pivot.TermsGroupSource;
import org.junit.After; import org.junit.After;
import java.io.IOException; import java.io.IOException;
import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import static java.util.stream.Collectors.toSet; import static java.util.stream.Collectors.toSet;
import static org.elasticsearch.common.xcontent.support.XContentMapValues.extractValue; import static org.elasticsearch.common.xcontent.support.XContentMapValues.extractValue;
@ -42,16 +47,24 @@ import static org.hamcrest.Matchers.nullValue;
public class TransformInsufficientPermissionsIT extends TransformRestTestCase { public class TransformInsufficientPermissionsIT extends TransformRestTestCase {
private static final String TEST_ADMIN_USERNAME = "x_pack_rest_user"; private enum Users {
private static final String TEST_ADMIN_HEADER = basicAuthHeaderValue(TEST_ADMIN_USERNAME, TEST_PASSWORD_SECURE_STRING); TEST_ADMIN("x_pack_rest_user", List.of()),
private static final String JUNIOR_USERNAME = "john_junior"; JUNIOR("john_junior", List.of("transform_admin")),
private static final String JUNIOR_HEADER = basicAuthHeaderValue(JUNIOR_USERNAME, TEST_PASSWORD_SECURE_STRING); SENIOR("bill_senior", List.of("transform_admin", "source_index_access", "dest_index_access")),
private static final String SENIOR_USERNAME = "bill_senior"; SOURCE_AND_DEST_INDEX_ACCESS_ONLY("source_and_dest_index_access_only", List.of("source_index_access", "dest_index_access")),
private static final String SENIOR_HEADER = basicAuthHeaderValue(SENIOR_USERNAME, TEST_PASSWORD_SECURE_STRING); TRANSFORM_USER_BUT_NOT_ADMIN("transform_user_but_not_admin", List.of("transform_user", "source_index_access", "dest_index_access")),
private static final String NOT_A_TRANSFORM_ADMIN = "not_a_transform_admin"; FLEET_ACCESS("fleet_access", List.of("transform_admin", "fleet_index_access", "dest_index_access"));
private static final String NOT_A_TRANSFORM_ADMIN_HEADER = basicAuthHeaderValue(NOT_A_TRANSFORM_ADMIN, TEST_PASSWORD_SECURE_STRING);
private static final String FLEET_ACCESS_USERNAME = "fleet_access"; private final String username;
private static final String FLEET_ACCESS_HEADER = basicAuthHeaderValue(FLEET_ACCESS_USERNAME, TEST_PASSWORD_SECURE_STRING); private final String effectiveRoles;
private final String header;
Users(String username, List<String> effectiveRoles) {
this.username = username;
this.effectiveRoles = effectiveRoles.stream().sorted().collect(Collectors.joining(","));
this.header = basicAuthHeaderValue(username, TEST_PASSWORD_SECURE_STRING);
}
}
private static final int NUM_USERS = 28; private static final int NUM_USERS = 28;
@ -85,10 +98,10 @@ public class TransformInsufficientPermissionsIT extends TransformRestTestCase {
private void testTransformPermissionsNoDefer(boolean unattended) throws Exception { private void testTransformPermissionsNoDefer(boolean unattended) throws Exception {
String transformId = "transform-permissions-nodefer-" + (unattended ? 1 : 0); String transformId = "transform-permissions-nodefer-" + (unattended ? 1 : 0);
String sourceIndexName = transformId + "-index"; String sourceIndexName = transformId + "-index";
String destIndexName = sourceIndexName + "-dest"; String destIndexName = transformId + "-dest";
createReviewsIndex(sourceIndexName, 10, NUM_USERS, TransformIT::getUserIdForRow, TransformIT::getDateStringForRow); createReviewsIndex(sourceIndexName, 10, NUM_USERS, TransformIT::getUserIdForRow, TransformIT::getDateStringForRow);
TransformConfig config = createConfig(transformId, sourceIndexName, destIndexName, unattended); TransformConfig config = createConfig(transformId, sourceIndexName, destIndexName, null, unattended);
ResponseException e = expectThrows( ResponseException e = expectThrows(
ResponseException.class, ResponseException.class,
@ -96,7 +109,7 @@ public class TransformInsufficientPermissionsIT extends TransformRestTestCase {
transformId, transformId,
Strings.toString(config), Strings.toString(config),
RequestOptions.DEFAULT.toBuilder() RequestOptions.DEFAULT.toBuilder()
.addHeader(AUTH_KEY, JUNIOR_HEADER) .addHeader(AUTH_KEY, Users.JUNIOR.header)
.addParameter("defer_validation", String.valueOf(false)) .addParameter("defer_validation", String.valueOf(false))
.build() .build()
) )
@ -107,11 +120,11 @@ public class TransformInsufficientPermissionsIT extends TransformRestTestCase {
containsString( containsString(
Strings.format( Strings.format(
"Cannot create transform [%s] because user %s lacks the required permissions " "Cannot create transform [%s] because user %s lacks the required permissions "
+ "[%s:[read, view_index_metadata], %s:[create_index, index, read]]", + "[%s:[create_index, index, read], %s:[read, view_index_metadata]]",
transformId, transformId,
JUNIOR_USERNAME, Users.JUNIOR.username,
sourceIndexName, destIndexName,
destIndexName sourceIndexName
) )
) )
); );
@ -120,7 +133,7 @@ public class TransformInsufficientPermissionsIT extends TransformRestTestCase {
transformId, transformId,
Strings.toString(config), Strings.toString(config),
RequestOptions.DEFAULT.toBuilder() RequestOptions.DEFAULT.toBuilder()
.addHeader(AUTH_KEY, SENIOR_HEADER) .addHeader(AUTH_KEY, Users.SENIOR.header)
.addParameter("defer_validation", String.valueOf(false)) .addParameter("defer_validation", String.valueOf(false))
.build() .build()
); );
@ -128,6 +141,44 @@ public class TransformInsufficientPermissionsIT extends TransformRestTestCase {
assertGreen(transformId); assertGreen(transformId);
} }
public void testTransformWithDestinationAlias() throws Exception {
String transformId = "transform-permissions-with-destination-alias";
String sourceIndexName = transformId + "-index";
String destIndexName = transformId + "-dest";
String destAliasName = transformId + "-alias";
createReviewsIndex(sourceIndexName, 10, NUM_USERS, TransformIT::getUserIdForRow, TransformIT::getDateStringForRow);
TransformConfig config = createConfig(
transformId,
sourceIndexName,
destIndexName,
List.of(new DestAlias(destAliasName, false)),
false
);
ResponseException e = expectThrows(
ResponseException.class,
() -> putTransform(
transformId,
Strings.toString(config),
RequestOptions.DEFAULT.toBuilder().addHeader(AUTH_KEY, Users.SENIOR.header).build()
)
);
assertThat(e.getResponse().getStatusLine().getStatusCode(), is(equalTo(403)));
assertThat(
e.getMessage(),
containsString(
Strings.format(
"Cannot create transform [%s] because user %s lacks the required permissions [%s:[manage], %s:[manage], %s:[]]",
transformId,
Users.SENIOR.username,
destAliasName,
destIndexName,
sourceIndexName
)
)
);
}
/** /**
* defer_validation = true * defer_validation = true
* unattended = false * unattended = false
@ -136,10 +187,10 @@ public class TransformInsufficientPermissionsIT extends TransformRestTestCase {
public void testTransformPermissionsDeferNoUnattendedNoDest() throws Exception { public void testTransformPermissionsDeferNoUnattendedNoDest() throws Exception {
String transformId = "transform-permissions-defer-nounattended"; String transformId = "transform-permissions-defer-nounattended";
String sourceIndexName = transformId + "-index"; String sourceIndexName = transformId + "-index";
String destIndexName = sourceIndexName + "-dest"; String destIndexName = transformId + "-dest";
createReviewsIndex(sourceIndexName, 10, NUM_USERS, TransformIT::getUserIdForRow, TransformIT::getDateStringForRow); createReviewsIndex(sourceIndexName, 10, NUM_USERS, TransformIT::getUserIdForRow, TransformIT::getDateStringForRow);
TransformConfig config = createConfig(transformId, sourceIndexName, destIndexName, false); TransformConfig config = createConfig(transformId, sourceIndexName, destIndexName, null, false);
putTransform( putTransform(
transformId, transformId,
Strings.toString(config), Strings.toString(config),
@ -147,17 +198,17 @@ public class TransformInsufficientPermissionsIT extends TransformRestTestCase {
); );
String authIssue = Strings.format( String authIssue = Strings.format(
"Cannot create transform [%s] because user %s lacks the required permissions " "Cannot create transform [%s] because user %s lacks the required permissions "
+ "[%s:[read, view_index_metadata], %s:[create_index, index, read]]", + "[%s:[create_index, index, read], %s:[read, view_index_metadata]]",
transformId, transformId,
JUNIOR_USERNAME, Users.JUNIOR.username,
sourceIndexName, destIndexName,
destIndexName sourceIndexName
); );
assertRed(transformId, authIssue); assertRed(transformId, authIssue);
ResponseException e = expectThrows( ResponseException e = expectThrows(
ResponseException.class, ResponseException.class,
() -> startTransform(config.getId(), RequestOptions.DEFAULT.toBuilder().addHeader(AUTH_KEY, JUNIOR_HEADER).build()) () -> startTransform(config.getId(), RequestOptions.DEFAULT.toBuilder().addHeader(AUTH_KEY, Users.JUNIOR.header).build())
); );
assertThat(e.getResponse().getStatusLine().getStatusCode(), is(equalTo(403))); assertThat(e.getResponse().getStatusLine().getStatusCode(), is(equalTo(403)));
assertThat(e.getMessage(), containsString(authIssue)); assertThat(e.getMessage(), containsString(authIssue));
@ -166,7 +217,7 @@ public class TransformInsufficientPermissionsIT extends TransformRestTestCase {
e = expectThrows( e = expectThrows(
ResponseException.class, ResponseException.class,
() -> startTransform(config.getId(), RequestOptions.DEFAULT.toBuilder().addHeader(AUTH_KEY, SENIOR_HEADER).build()) () -> startTransform(config.getId(), RequestOptions.DEFAULT.toBuilder().addHeader(AUTH_KEY, Users.SENIOR.header).build())
); );
assertThat(e.getResponse().getStatusLine().getStatusCode(), is(equalTo(403))); assertThat(e.getResponse().getStatusLine().getStatusCode(), is(equalTo(403)));
assertThat(e.getMessage(), containsString(authIssue)); assertThat(e.getMessage(), containsString(authIssue));
@ -174,7 +225,7 @@ public class TransformInsufficientPermissionsIT extends TransformRestTestCase {
assertRed(transformId, authIssue); assertRed(transformId, authIssue);
// update transform's credentials so that the transform has permission to access source/dest indices // update transform's credentials so that the transform has permission to access source/dest indices
updateConfig(transformId, "{}", RequestOptions.DEFAULT.toBuilder().addHeader(AUTH_KEY, SENIOR_HEADER).build()); updateConfig(transformId, "{}", RequestOptions.DEFAULT.toBuilder().addHeader(AUTH_KEY, Users.SENIOR.header).build());
assertGreen(transformId); assertGreen(transformId);
@ -193,12 +244,12 @@ public class TransformInsufficientPermissionsIT extends TransformRestTestCase {
public void testTransformPermissionsDeferNoUnattendedDest() throws Exception { public void testTransformPermissionsDeferNoUnattendedDest() throws Exception {
String transformId = "transform-permissions-defer-nounattended-dest-exists"; String transformId = "transform-permissions-defer-nounattended-dest-exists";
String sourceIndexName = transformId + "-index"; String sourceIndexName = transformId + "-index";
String destIndexName = sourceIndexName + "-dest"; String destIndexName = transformId + "-dest";
createReviewsIndex(sourceIndexName, 10, NUM_USERS, TransformIT::getUserIdForRow, TransformIT::getDateStringForRow); createReviewsIndex(sourceIndexName, 10, NUM_USERS, TransformIT::getUserIdForRow, TransformIT::getDateStringForRow);
createIndex(adminClient(), destIndexName, Settings.EMPTY); createIndex(adminClient(), destIndexName, Settings.EMPTY);
TransformConfig config = createConfig(transformId, sourceIndexName, destIndexName, false); TransformConfig config = createConfig(transformId, sourceIndexName, destIndexName, null, false);
putTransform( putTransform(
transformId, transformId,
Strings.toString(config), Strings.toString(config),
@ -206,17 +257,17 @@ public class TransformInsufficientPermissionsIT extends TransformRestTestCase {
); );
String authIssue = Strings.format( String authIssue = Strings.format(
"Cannot create transform [%s] because user %s lacks the required permissions " "Cannot create transform [%s] because user %s lacks the required permissions "
+ "[%s:[read, view_index_metadata], %s:[index, read]]", + "[%s:[index, read], %s:[read, view_index_metadata]]",
transformId, transformId,
JUNIOR_USERNAME, Users.JUNIOR.username,
sourceIndexName, destIndexName,
destIndexName sourceIndexName
); );
assertRed(transformId, authIssue); assertRed(transformId, authIssue);
ResponseException e = expectThrows( ResponseException e = expectThrows(
ResponseException.class, ResponseException.class,
() -> startTransform(config.getId(), RequestOptions.DEFAULT.toBuilder().addHeader(AUTH_KEY, JUNIOR_HEADER).build()) () -> startTransform(config.getId(), RequestOptions.DEFAULT.toBuilder().addHeader(AUTH_KEY, Users.JUNIOR.header).build())
); );
assertThat(e.getResponse().getStatusLine().getStatusCode(), is(equalTo(403))); assertThat(e.getResponse().getStatusLine().getStatusCode(), is(equalTo(403)));
assertThat(e.getMessage(), containsString(authIssue)); assertThat(e.getMessage(), containsString(authIssue));
@ -225,7 +276,7 @@ public class TransformInsufficientPermissionsIT extends TransformRestTestCase {
e = expectThrows( e = expectThrows(
ResponseException.class, ResponseException.class,
() -> startTransform(config.getId(), RequestOptions.DEFAULT.toBuilder().addHeader(AUTH_KEY, SENIOR_HEADER).build()) () -> startTransform(config.getId(), RequestOptions.DEFAULT.toBuilder().addHeader(AUTH_KEY, Users.SENIOR.header).build())
); );
assertThat(e.getResponse().getStatusLine().getStatusCode(), is(equalTo(403))); assertThat(e.getResponse().getStatusLine().getStatusCode(), is(equalTo(403)));
assertThat(e.getMessage(), containsString(authIssue)); assertThat(e.getMessage(), containsString(authIssue));
@ -233,7 +284,7 @@ public class TransformInsufficientPermissionsIT extends TransformRestTestCase {
assertRed(transformId, authIssue); assertRed(transformId, authIssue);
// update transform's credentials so that the transform has permission to access source/dest indices // update transform's credentials so that the transform has permission to access source/dest indices
updateConfig(transformId, "{}", RequestOptions.DEFAULT.toBuilder().addHeader(AUTH_KEY, SENIOR_HEADER).build()); updateConfig(transformId, "{}", RequestOptions.DEFAULT.toBuilder().addHeader(AUTH_KEY, Users.SENIOR.header).build());
assertGreen(transformId); assertGreen(transformId);
@ -244,37 +295,101 @@ public class TransformInsufficientPermissionsIT extends TransformRestTestCase {
assertGreen(transformId); assertGreen(transformId);
} }
/**
* defer_validation = false
* unattended = false
*/
public void testNoTransformAdminRoleNoDeferNoUnattended() throws Exception {
testNoTransformAdminRole(false, false);
}
/**
* defer_validation = false
* unattended = true
*/
public void testNoTransformAdminRoleNoDeferUnattended() throws Exception {
testNoTransformAdminRole(false, true);
}
/**
* defer_validation = true
* unattended = false
*/
public void testNoTransformAdminRoleDeferNoUnattended() throws Exception {
testNoTransformAdminRole(true, false);
}
/**
* defer_validation = true
* unattended = true
*/
public void testNoTransformAdminRoleDeferUnattended() throws Exception {
testNoTransformAdminRole(true, true);
}
private void testNoTransformAdminRole(boolean deferValidation, boolean unattended) throws Exception {
Users user = randomFrom(Users.SOURCE_AND_DEST_INDEX_ACCESS_ONLY, Users.TRANSFORM_USER_BUT_NOT_ADMIN);
String transformId = "transform-permissions-no-transform-role";
String sourceIndexName = transformId + "-index";
String destIndexName = transformId + "-dest";
createReviewsIndex(sourceIndexName, 10, NUM_USERS, TransformIT::getUserIdForRow, TransformIT::getDateStringForRow);
TransformConfig config = createConfig(transformId, sourceIndexName, destIndexName, null, unattended);
ResponseException e = expectThrows(
ResponseException.class,
() -> putTransform(
transformId,
Strings.toString(config),
RequestOptions.DEFAULT.toBuilder()
.addHeader(AUTH_KEY, user.header)
.addParameter("defer_validation", String.valueOf(deferValidation))
.build()
)
);
assertThat(e.getResponse().getStatusLine().getStatusCode(), is(equalTo(403)));
assertThat(
e.getMessage(),
containsString(
Strings.format(
"action [cluster:admin/transform/put] is unauthorized for user [%s] with effective roles [%s], "
+ "this action is granted by the cluster privileges [manage_data_frame_transforms,manage_transform,manage,all]",
user.username,
user.effectiveRoles
)
)
);
}
/** /**
* defer_validation = true * defer_validation = true
* unattended = false * unattended = false
*/ */
public void testNoTransformAdminRoleInSecondaryAuth() throws Exception { public void testNoTransformAdminRoleInSecondaryAuth() throws Exception {
String transformId = "transform-permissions-no-admin-role"; Users user = randomFrom(Users.SOURCE_AND_DEST_INDEX_ACCESS_ONLY, Users.TRANSFORM_USER_BUT_NOT_ADMIN);
String transformId = "transform-permissions-no-transform-role-in-sec-auth";
String sourceIndexName = transformId + "-index"; String sourceIndexName = transformId + "-index";
String destIndexName = sourceIndexName + "-dest"; String destIndexName = transformId + "-dest";
createReviewsIndex(sourceIndexName, 10, NUM_USERS, TransformIT::getUserIdForRow, TransformIT::getDateStringForRow); createReviewsIndex(sourceIndexName, 10, NUM_USERS, TransformIT::getUserIdForRow, TransformIT::getDateStringForRow);
TransformConfig config = createConfig(transformId, sourceIndexName, destIndexName, false); TransformConfig config = createConfig(transformId, sourceIndexName, destIndexName, null, false);
// PUT with defer_validation should work even though the secondary auth does not have transform_admin role // PUT with defer_validation should work even though the secondary auth does not have transform_admin nor transform_user role
putTransform( putTransform(
transformId, transformId,
Strings.toString(config), Strings.toString(config),
RequestOptions.DEFAULT.toBuilder() RequestOptions.DEFAULT.toBuilder()
.addHeader(SECONDARY_AUTH_KEY, NOT_A_TRANSFORM_ADMIN_HEADER) .addHeader(SECONDARY_AUTH_KEY, user.header)
.addParameter("defer_validation", String.valueOf(true)) .addParameter("defer_validation", String.valueOf(true))
.build() .build()
); );
// _update should work even though the secondary auth does not have transform_admin role // _update should work even though the secondary auth does not have transform_admin nor transform_user role
updateConfig( updateConfig(transformId, "{}", RequestOptions.DEFAULT.toBuilder().addHeader(SECONDARY_AUTH_KEY, user.header).build());
transformId,
"{}",
RequestOptions.DEFAULT.toBuilder().addHeader(SECONDARY_AUTH_KEY, NOT_A_TRANSFORM_ADMIN_HEADER).build()
);
// _start works because user not_a_transform_admin has data access // _start works because user source_and_dest_index_access_only does have data access
startTransform(config.getId(), RequestOptions.DEFAULT); startTransform(config.getId(), RequestOptions.DEFAULT);
waitUntilCheckpoint(transformId, 1);
} }
/** /**
@ -285,10 +400,10 @@ public class TransformInsufficientPermissionsIT extends TransformRestTestCase {
public void testTransformPermissionsDeferUnattendedNoDest() throws Exception { public void testTransformPermissionsDeferUnattendedNoDest() throws Exception {
String transformId = "transform-permissions-defer-unattended"; String transformId = "transform-permissions-defer-unattended";
String sourceIndexName = transformId + "-index"; String sourceIndexName = transformId + "-index";
String destIndexName = sourceIndexName + "-dest"; String destIndexName = transformId + "-dest";
createReviewsIndex(sourceIndexName, 10, NUM_USERS, TransformIT::getUserIdForRow, TransformIT::getDateStringForRow); createReviewsIndex(sourceIndexName, 10, NUM_USERS, TransformIT::getUserIdForRow, TransformIT::getDateStringForRow);
TransformConfig config = createConfig(transformId, sourceIndexName, destIndexName, true); TransformConfig config = createConfig(transformId, sourceIndexName, destIndexName, null, true);
putTransform( putTransform(
transformId, transformId,
Strings.toString(config), Strings.toString(config),
@ -296,11 +411,11 @@ public class TransformInsufficientPermissionsIT extends TransformRestTestCase {
); );
String authIssue = Strings.format( String authIssue = Strings.format(
"Cannot create transform [%s] because user %s lacks the required permissions " "Cannot create transform [%s] because user %s lacks the required permissions "
+ "[%s:[read, view_index_metadata], %s:[create_index, index, read]]", + "[%s:[create_index, index, read], %s:[read, view_index_metadata]]",
transformId, transformId,
JUNIOR_USERNAME, Users.JUNIOR.username,
sourceIndexName, destIndexName,
destIndexName sourceIndexName
); );
assertRed(transformId, authIssue); assertRed(transformId, authIssue);
@ -311,7 +426,7 @@ public class TransformInsufficientPermissionsIT extends TransformRestTestCase {
assertBusy(() -> assertRed(transformId, authIssue, noSuchIndexIssue), 10, TimeUnit.SECONDS); assertBusy(() -> assertRed(transformId, authIssue, noSuchIndexIssue), 10, TimeUnit.SECONDS);
// update transform's credentials so that the transform has permission to access source/dest indices // update transform's credentials so that the transform has permission to access source/dest indices
updateConfig(transformId, "{}", RequestOptions.DEFAULT.toBuilder().addHeader(AUTH_KEY, SENIOR_HEADER).build()); updateConfig(transformId, "{}", RequestOptions.DEFAULT.toBuilder().addHeader(AUTH_KEY, Users.SENIOR.header).build());
waitUntilCheckpoint(transformId, 1); waitUntilCheckpoint(transformId, 1);
// transform is green again // transform is green again
@ -326,12 +441,12 @@ public class TransformInsufficientPermissionsIT extends TransformRestTestCase {
public void testTransformPermissionsDeferUnattendedDest() throws Exception { public void testTransformPermissionsDeferUnattendedDest() throws Exception {
String transformId = "transform-permissions-defer-unattended-dest-exists"; String transformId = "transform-permissions-defer-unattended-dest-exists";
String sourceIndexName = transformId + "-index"; String sourceIndexName = transformId + "-index";
String destIndexName = sourceIndexName + "-dest"; String destIndexName = transformId + "-dest";
createReviewsIndex(sourceIndexName, 10, NUM_USERS, TransformIT::getUserIdForRow, TransformIT::getDateStringForRow); createReviewsIndex(sourceIndexName, 10, NUM_USERS, TransformIT::getUserIdForRow, TransformIT::getDateStringForRow);
createIndex(adminClient(), destIndexName, Settings.EMPTY); createIndex(adminClient(), destIndexName, Settings.EMPTY);
TransformConfig config = createConfig(transformId, sourceIndexName, destIndexName, true); TransformConfig config = createConfig(transformId, sourceIndexName, destIndexName, null, true);
putTransform( putTransform(
transformId, transformId,
Strings.toString(config), Strings.toString(config),
@ -339,11 +454,11 @@ public class TransformInsufficientPermissionsIT extends TransformRestTestCase {
); );
String authIssue = Strings.format( String authIssue = Strings.format(
"Cannot create transform [%s] because user %s lacks the required permissions " "Cannot create transform [%s] because user %s lacks the required permissions "
+ "[%s:[read, view_index_metadata], %s:[index, read]]", + "[%s:[index, read], %s:[read, view_index_metadata]]",
transformId, transformId,
JUNIOR_USERNAME, Users.JUNIOR.username,
sourceIndexName, destIndexName,
destIndexName sourceIndexName
); );
assertRed(transformId, authIssue); assertRed(transformId, authIssue);
@ -353,7 +468,7 @@ public class TransformInsufficientPermissionsIT extends TransformRestTestCase {
assertRed(transformId, authIssue); assertRed(transformId, authIssue);
// update transform's credentials so that the transform has permission to access source/dest indices // update transform's credentials so that the transform has permission to access source/dest indices
updateConfig(transformId, "{}", RequestOptions.DEFAULT.toBuilder().addHeader(AUTH_KEY, SENIOR_HEADER).build()); updateConfig(transformId, "{}", RequestOptions.DEFAULT.toBuilder().addHeader(AUTH_KEY, Users.SENIOR.header).build());
waitUntilCheckpoint(transformId, 1); waitUntilCheckpoint(transformId, 1);
// transform is green again // transform is green again
@ -363,14 +478,17 @@ public class TransformInsufficientPermissionsIT extends TransformRestTestCase {
public void testPreviewRequestFailsPermissionsCheck() throws Exception { public void testPreviewRequestFailsPermissionsCheck() throws Exception {
String transformId = "transform-permissions-preview"; String transformId = "transform-permissions-preview";
String sourceIndexName = transformId + "-index"; String sourceIndexName = transformId + "-index";
String destIndexName = sourceIndexName + "-dest"; String destIndexName = transformId + "-dest";
createReviewsIndex(sourceIndexName, 10, NUM_USERS, TransformIT::getUserIdForRow, TransformIT::getDateStringForRow); createReviewsIndex(sourceIndexName, 10, NUM_USERS, TransformIT::getUserIdForRow, TransformIT::getDateStringForRow);
TransformConfig config = createConfig(transformId, sourceIndexName, destIndexName, false); TransformConfig config = createConfig(transformId, sourceIndexName, destIndexName, null, false);
ResponseException e = expectThrows( ResponseException e = expectThrows(
ResponseException.class, ResponseException.class,
() -> previewTransform(Strings.toString(config), RequestOptions.DEFAULT.toBuilder().addHeader(AUTH_KEY, JUNIOR_HEADER).build()) () -> previewTransform(
Strings.toString(config),
RequestOptions.DEFAULT.toBuilder().addHeader(AUTH_KEY, Users.JUNIOR.header).build()
)
); );
assertThat(e.getResponse().getStatusLine().getStatusCode(), is(equalTo(403))); assertThat(e.getResponse().getStatusLine().getStatusCode(), is(equalTo(403)));
assertThat( assertThat(
@ -378,16 +496,16 @@ public class TransformInsufficientPermissionsIT extends TransformRestTestCase {
containsString( containsString(
Strings.format( Strings.format(
"Cannot preview transform [%s] because user %s lacks the required permissions " "Cannot preview transform [%s] because user %s lacks the required permissions "
+ "[%s:[read, view_index_metadata], %s:[create_index, index, read]]", + "[%s:[create_index, index, read], %s:[read, view_index_metadata]]",
transformId, transformId,
JUNIOR_USERNAME, Users.JUNIOR.username,
sourceIndexName, destIndexName,
destIndexName sourceIndexName
) )
) )
); );
previewTransform(Strings.toString(config), RequestOptions.DEFAULT.toBuilder().addHeader(AUTH_KEY, SENIOR_HEADER).build()); previewTransform(Strings.toString(config), RequestOptions.DEFAULT.toBuilder().addHeader(AUTH_KEY, Users.SENIOR.header).build());
} }
public void testFleetIndicesAccess() throws Exception { public void testFleetIndicesAccess() throws Exception {
@ -395,33 +513,38 @@ public class TransformInsufficientPermissionsIT extends TransformRestTestCase {
String sourceIndexPattern = ".fleet-agents*"; String sourceIndexPattern = ".fleet-agents*";
String destIndexName = transformId + "-dest"; String destIndexName = transformId + "-dest";
TransformConfig config = createConfig(transformId, sourceIndexPattern, destIndexName, false); TransformConfig config = createConfig(transformId, sourceIndexPattern, destIndexName, null, false);
ResponseException e = expectThrows( ResponseException e = expectThrows(
ResponseException.class, ResponseException.class,
() -> previewTransform( () -> previewTransform(
Strings.toString(config), Strings.toString(config),
RequestOptions.DEFAULT.toBuilder().addHeader(AUTH_KEY, FLEET_ACCESS_HEADER).build() RequestOptions.DEFAULT.toBuilder().addHeader(AUTH_KEY, Users.FLEET_ACCESS.header).build()
) )
); );
// The _preview request got past the authorization step (which is what interests us in this test) but failed because the referenced // The _preview request got past the authorization step (which is what interests us in this test) but failed because the referenced
// source indices do not exist. // source indices do not exist.
assertThat(e.getResponse().getStatusLine().getStatusCode(), is(equalTo(400))); assertThat("Error was: " + e.getMessage(), e.getResponse().getStatusLine().getStatusCode(), is(equalTo(400)));
assertThat(e.getMessage(), containsString("Source indices have been deleted or closed.")); assertThat(e.getMessage(), containsString("Source indices have been deleted or closed."));
} }
@Override @Override
protected Settings restAdminSettings() { protected Settings restAdminSettings() {
return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", TEST_ADMIN_HEADER).build(); return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", Users.TEST_ADMIN.header).build();
} }
@Override @Override
protected Settings restClientSettings() { protected Settings restClientSettings() {
return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", JUNIOR_HEADER).build(); return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", Users.JUNIOR.header).build();
} }
private TransformConfig createConfig(String transformId, String sourceIndexName, String destIndexName, boolean unattended) private TransformConfig createConfig(
throws Exception { String transformId,
String sourceIndexName,
String destIndexName,
List<DestAlias> aliases,
boolean unattended
) throws Exception {
Map<String, SingleGroupSource> groups = Map.of( Map<String, SingleGroupSource> groups = Map.of(
"by-day", "by-day",
createDateHistogramGroupSourceWithCalendarInterval("timestamp", DateHistogramInterval.DAY, null), createDateHistogramGroupSourceWithCalendarInterval("timestamp", DateHistogramInterval.DAY, null),
@ -435,7 +558,12 @@ public class TransformInsufficientPermissionsIT extends TransformRestTestCase {
.addAggregator(AggregationBuilders.avg("review_score").field("stars")) .addAggregator(AggregationBuilders.avg("review_score").field("stars"))
.addAggregator(AggregationBuilders.max("timestamp").field("timestamp")); .addAggregator(AggregationBuilders.max("timestamp").field("timestamp"));
TransformConfig config = createTransformConfigBuilder(transformId, destIndexName, QueryConfig.matchAll(), sourceIndexName) TransformConfig config = TransformConfig.builder()
.setId(transformId)
.setSource(new SourceConfig(new String[] { sourceIndexName }, QueryConfig.matchAll(), Collections.emptyMap()))
.setDest(new DestConfig(destIndexName, aliases, null))
.setFrequency(TimeValue.timeValueSeconds(10))
.setDescription("Test transform config id: " + transformId)
.setPivotConfig(createPivotConfig(groups, aggs)) .setPivotConfig(createPivotConfig(groups, aggs))
.setSyncConfig(new TimeSyncConfig("timestamp", TimeValue.timeValueSeconds(1))) .setSyncConfig(new TimeSyncConfig("timestamp", TimeValue.timeValueSeconds(1)))
.setSettings(new SettingsConfig.Builder().setAlignCheckpoints(false).setUnattended(unattended).build()) .setSettings(new SettingsConfig.Builder().setAlignCheckpoints(false).setUnattended(unattended).build())

View file

@ -391,7 +391,7 @@ public abstract class TransformRestTestCase extends ESRestTestCase {
return TransformConfig.builder() return TransformConfig.builder()
.setId(id) .setId(id)
.setSource(new SourceConfig(sourceIndices, queryConfig, Collections.emptyMap())) .setSource(new SourceConfig(sourceIndices, queryConfig, Collections.emptyMap()))
.setDest(new DestConfig(destinationIndex, null)) .setDest(new DestConfig(destinationIndex, null, null))
.setFrequency(TimeValue.timeValueSeconds(10)) .setFrequency(TimeValue.timeValueSeconds(10))
.setDescription("Test transform config id: " + id); .setDescription("Test transform config id: " + id);
} }

View file

@ -56,7 +56,7 @@ public class DateHistogramGroupByIT extends ContinuousTestCase {
} }
transformConfigBuilder.setSource(new SourceConfig(CONTINUOUS_EVENTS_SOURCE_INDEX)); transformConfigBuilder.setSource(new SourceConfig(CONTINUOUS_EVENTS_SOURCE_INDEX));
transformConfigBuilder.setDest(new DestConfig(NAME, INGEST_PIPELINE)); transformConfigBuilder.setDest(new DestConfig(NAME, null, INGEST_PIPELINE));
transformConfigBuilder.setId(NAME); transformConfigBuilder.setId(NAME);
var groupConfig = TransformRestTestCase.createGroupConfig( var groupConfig = TransformRestTestCase.createGroupConfig(

View file

@ -62,7 +62,7 @@ public class DateHistogramGroupByOtherTimeFieldIT extends ContinuousTestCase {
transformConfigBuilder.setSettings(addCommonSettings(new SettingsConfig.Builder()).setDatesAsEpochMillis(true).build()); transformConfigBuilder.setSettings(addCommonSettings(new SettingsConfig.Builder()).setDatesAsEpochMillis(true).build());
} }
transformConfigBuilder.setSource(new SourceConfig(CONTINUOUS_EVENTS_SOURCE_INDEX)); transformConfigBuilder.setSource(new SourceConfig(CONTINUOUS_EVENTS_SOURCE_INDEX));
transformConfigBuilder.setDest(new DestConfig(NAME, INGEST_PIPELINE)); transformConfigBuilder.setDest(new DestConfig(NAME, null, INGEST_PIPELINE));
transformConfigBuilder.setId(NAME); transformConfigBuilder.setId(NAME);
Map<String, SingleGroupSource> groupSource = new HashMap<>(); Map<String, SingleGroupSource> groupSource = new HashMap<>();

View file

@ -47,7 +47,7 @@ public class HistogramGroupByIT extends ContinuousTestCase {
TransformConfig.Builder transformConfigBuilder = new TransformConfig.Builder(); TransformConfig.Builder transformConfigBuilder = new TransformConfig.Builder();
addCommonBuilderParameters(transformConfigBuilder); addCommonBuilderParameters(transformConfigBuilder);
transformConfigBuilder.setSource(new SourceConfig(CONTINUOUS_EVENTS_SOURCE_INDEX)); transformConfigBuilder.setSource(new SourceConfig(CONTINUOUS_EVENTS_SOURCE_INDEX));
transformConfigBuilder.setDest(new DestConfig(NAME, INGEST_PIPELINE)); transformConfigBuilder.setDest(new DestConfig(NAME, null, INGEST_PIPELINE));
transformConfigBuilder.setId(NAME); transformConfigBuilder.setId(NAME);
var groupConfig = TransformRestTestCase.createGroupConfig( var groupConfig = TransformRestTestCase.createGroupConfig(

View file

@ -58,7 +58,7 @@ public class LatestContinuousIT extends ContinuousTestCase {
public TransformConfig createConfig() { public TransformConfig createConfig() {
TransformConfig.Builder transformConfigBuilder = new TransformConfig.Builder().setId(NAME) TransformConfig.Builder transformConfigBuilder = new TransformConfig.Builder().setId(NAME)
.setSource(new SourceConfig(new String[] { CONTINUOUS_EVENTS_SOURCE_INDEX }, QueryConfig.matchAll(), RUNTIME_MAPPINGS)) .setSource(new SourceConfig(new String[] { CONTINUOUS_EVENTS_SOURCE_INDEX }, QueryConfig.matchAll(), RUNTIME_MAPPINGS))
.setDest(new DestConfig(NAME, INGEST_PIPELINE)) .setDest(new DestConfig(NAME, null, INGEST_PIPELINE))
.setLatestConfig(new LatestConfig(List.of(eventField), timestampField)); .setLatestConfig(new LatestConfig(List.of(eventField), timestampField));
addCommonBuilderParameters(transformConfigBuilder); addCommonBuilderParameters(transformConfigBuilder);
return transformConfigBuilder.build(); return transformConfigBuilder.build();

View file

@ -45,7 +45,7 @@ public class TermsGroupByIT extends ContinuousTestCase {
TransformConfig.Builder transformConfigBuilder = new TransformConfig.Builder(); TransformConfig.Builder transformConfigBuilder = new TransformConfig.Builder();
addCommonBuilderParameters(transformConfigBuilder); addCommonBuilderParameters(transformConfigBuilder);
transformConfigBuilder.setSource(new SourceConfig(CONTINUOUS_EVENTS_SOURCE_INDEX)); transformConfigBuilder.setSource(new SourceConfig(CONTINUOUS_EVENTS_SOURCE_INDEX));
transformConfigBuilder.setDest(new DestConfig(NAME, INGEST_PIPELINE)); transformConfigBuilder.setDest(new DestConfig(NAME, null, INGEST_PIPELINE));
transformConfigBuilder.setId(NAME); transformConfigBuilder.setId(NAME);
var groupConfig = TransformRestTestCase.createGroupConfig( var groupConfig = TransformRestTestCase.createGroupConfig(

View file

@ -54,7 +54,7 @@ public class TermsOnDateGroupByIT extends ContinuousTestCase {
TransformConfig.Builder transformConfigBuilder = new TransformConfig.Builder(); TransformConfig.Builder transformConfigBuilder = new TransformConfig.Builder();
addCommonBuilderParameters(transformConfigBuilder); addCommonBuilderParameters(transformConfigBuilder);
transformConfigBuilder.setSource(new SourceConfig(CONTINUOUS_EVENTS_SOURCE_INDEX)); transformConfigBuilder.setSource(new SourceConfig(CONTINUOUS_EVENTS_SOURCE_INDEX));
transformConfigBuilder.setDest(new DestConfig(NAME, INGEST_PIPELINE)); transformConfigBuilder.setDest(new DestConfig(NAME, null, INGEST_PIPELINE));
transformConfigBuilder.setId(NAME); transformConfigBuilder.setId(NAME);
var groupConfig = TransformRestTestCase.createGroupConfig( var groupConfig = TransformRestTestCase.createGroupConfig(

View file

@ -12,12 +12,12 @@ import org.elasticsearch.client.Request;
import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.ResponseException;
import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder; import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.junit.Before; import org.junit.Before;
import java.io.IOException; import java.io.IOException;
import java.util.Arrays; import java.util.Arrays;
import java.util.Locale;
import java.util.Map; import java.util.Map;
import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.containsString;
@ -72,59 +72,74 @@ public class TransformDeleteIT extends TransformRestTestCase {
public void testDeleteDoesNotDeleteDestinationIndexByDefault() throws Exception { public void testDeleteDoesNotDeleteDestinationIndexByDefault() throws Exception {
String transformId = "transform-1"; String transformId = "transform-1";
String transformDest = transformId + "_idx"; String transformDest = transformId + "_idx";
setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME, transformDest); String transformDestAlias = transformId + "_alias";
setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME, transformDest, transformDestAlias);
createTransform(transformId, transformDest); createTransform(transformId, transformDest, transformDestAlias);
assertFalse(indexExists(transformDest)); assertFalse(indexExists(transformDest));
assertFalse(aliasExists(transformDestAlias));
startTransform(transformId); startTransform(transformId);
waitForTransformCheckpoint(transformId, 1); waitForTransformCheckpoint(transformId, 1);
stopTransform(transformId, false); stopTransform(transformId, false);
assertTrue(indexExists(transformDest)); assertTrue(indexExists(transformDest));
assertTrue(aliasExists(transformDestAlias));
deleteTransform(transformId); deleteTransform(transformId);
assertTrue(indexExists(transformDest)); assertTrue(indexExists(transformDest));
assertTrue(aliasExists(transformDestAlias));
} }
public void testDeleteWithParamDeletesAutoCreatedDestinationIndex() throws Exception { public void testDeleteWithParamDeletesAutoCreatedDestinationIndex() throws Exception {
String transformId = "transform-2"; String transformId = "transform-2";
String transformDest = transformId + "_idx"; String transformDest = transformId + "_idx";
setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME, transformDest); String transformDestAlias = transformId + "_alias";
setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME, transformDest, transformDestAlias);
createTransform(transformId, transformDest); createTransform(transformId, transformDest, transformDestAlias);
assertFalse(indexExists(transformDest)); assertFalse(indexExists(transformDest));
assertFalse(aliasExists(transformDestAlias));
startTransform(transformId); startTransform(transformId);
waitForTransformCheckpoint(transformId, 1); waitForTransformCheckpoint(transformId, 1);
stopTransform(transformId, false); stopTransform(transformId, false);
assertTrue(indexExists(transformDest)); assertTrue(indexExists(transformDest));
assertTrue(aliasExists(transformDestAlias));
deleteTransform(transformId, true); deleteTransform(transformId, true);
assertFalse(indexExists(transformDest)); assertFalse(indexExists(transformDest));
assertFalse(aliasExists(transformDest));
} }
public void testDeleteWithParamDeletesManuallyCreatedDestinationIndex() throws Exception { public void testDeleteWithParamDeletesManuallyCreatedDestinationIndex() throws Exception {
String transformId = "transform-3"; String transformId = "transform-3";
String transformDest = transformId + "_idx"; String transformDest = transformId + "_idx";
setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME, transformDest); String transformDestAlias = transformId + "_alias";
setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME, transformDest, transformDestAlias);
createIndex(transformDest); createIndex(transformDest);
assertTrue(indexExists(transformDest)); assertTrue(indexExists(transformDest));
// The alias does not exist yet, it will be created when the transform starts
assertFalse(aliasExists(transformDestAlias));
createTransform(transformId, transformDest); createTransform(transformId, transformDest, transformDestAlias);
assertFalse(aliasExists(transformDestAlias));
startTransform(transformId); startTransform(transformId);
waitForTransformCheckpoint(transformId, 1); waitForTransformCheckpoint(transformId, 1);
stopTransform(transformId, false); stopTransform(transformId, false);
assertTrue(indexExists(transformDest)); assertTrue(indexExists(transformDest));
assertTrue(aliasExists(transformDestAlias));
deleteTransform(transformId, true); deleteTransform(transformId, true);
assertFalse(indexExists(transformDest)); assertFalse(indexExists(transformDest));
assertFalse(aliasExists(transformDestAlias));
} }
public void testDeleteWithParamDoesNotDeleteAlias() throws Exception { public void testDeleteWithParamDoesNotDeleteManuallySetUpAlias() throws Exception {
String transformId = "transform-4"; String transformId = "transform-4";
String transformDest = transformId + "_idx"; String transformDest = transformId + "_idx";
String transformDestAlias = transformId + "_alias"; String transformDestAlias = transformId + "_alias";
@ -132,22 +147,22 @@ public class TransformDeleteIT extends TransformRestTestCase {
createIndex(transformDest, null, null, "\"" + transformDestAlias + "\": { \"is_write_index\": true }"); createIndex(transformDest, null, null, "\"" + transformDestAlias + "\": { \"is_write_index\": true }");
assertTrue(indexExists(transformDest)); assertTrue(indexExists(transformDest));
assertTrue(indexExists(transformDestAlias)); assertTrue(aliasExists(transformDestAlias));
createTransform(transformId, transformDestAlias); createTransform(transformId, transformDestAlias, null);
startTransform(transformId); startTransform(transformId);
waitForTransformCheckpoint(transformId, 1); waitForTransformCheckpoint(transformId, 1);
stopTransform(transformId, false); stopTransform(transformId, false);
assertTrue(indexExists(transformDest)); assertTrue(indexExists(transformDest));
assertTrue(aliasExists(transformDestAlias));
ResponseException e = expectThrows(ResponseException.class, () -> deleteTransform(transformId, true)); ResponseException e = expectThrows(ResponseException.class, () -> deleteTransform(transformId, true));
assertThat( assertThat(
e.getMessage(), e.getMessage(),
containsString( containsString(
String.format( Strings.format(
Locale.ROOT,
"The provided expression [%s] matches an alias, specify the corresponding concrete indices instead.", "The provided expression [%s] matches an alias, specify the corresponding concrete indices instead.",
transformDestAlias transformDestAlias
) )
@ -155,16 +170,20 @@ public class TransformDeleteIT extends TransformRestTestCase {
); );
} }
private void createTransform(String transformId, String destIndex) throws IOException { private void createTransform(String transformId, String destIndex, String destAlias) throws IOException {
final Request createTransformRequest = createRequestWithAuth( final Request createTransformRequest = createRequestWithAuth(
"PUT", "PUT",
getTransformEndpoint() + transformId, getTransformEndpoint() + transformId,
BASIC_AUTH_VALUE_TRANSFORM_ADMIN_1 BASIC_AUTH_VALUE_TRANSFORM_ADMIN_1
); );
String config = String.format(Locale.ROOT, """ String destAliases = destAlias != null ? Strings.format("""
, "aliases": [{"alias": "%s"}]
""", destAlias) : "";
String config = Strings.format("""
{ {
"dest": { "dest": {
"index": "%s" "index": "%s"
%s
}, },
"source": { "source": {
"index": "%s" "index": "%s"
@ -185,7 +204,7 @@ public class TransformDeleteIT extends TransformRestTestCase {
} }
} }
} }
}""", destIndex, REVIEWS_INDEX_NAME); }""", destIndex, destAliases, REVIEWS_INDEX_NAME);
createTransformRequest.setJsonEntity(config); createTransformRequest.setJsonEntity(config);
Map<String, Object> createTransformResponse = entityAsMap(client().performRequest(createTransformRequest)); Map<String, Object> createTransformResponse = entityAsMap(client().performRequest(createTransformRequest));
assertThat(createTransformResponse.get("acknowledged"), equalTo(Boolean.TRUE)); assertThat(createTransformResponse.get("acknowledged"), equalTo(Boolean.TRUE));

View file

@ -0,0 +1,155 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
package org.elasticsearch.xpack.transform.integration;
import org.elasticsearch.Version;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.Response;
import org.elasticsearch.common.xcontent.support.XContentMapValues;
import org.elasticsearch.xpack.core.transform.transforms.DestAlias;
import org.elasticsearch.xpack.core.transform.transforms.SettingsConfig;
import org.junit.Before;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
public class TransformDestIndexIT extends TransformRestTestCase {
private static boolean indicesCreated = false;
// preserve indices in order to reuse source indices in several test cases
@Override
protected boolean preserveIndicesUponCompletion() {
return true;
}
@Before
public void createIndexes() throws IOException {
// it's not possible to run it as @BeforeClass as clients aren't initialized then, so we need this little hack
if (indicesCreated) {
return;
}
createReviewsIndex();
indicesCreated = true;
}
public void testTransformDestIndexMetadata() throws Exception {
long testStarted = System.currentTimeMillis();
String transformId = "test_meta";
createPivotReviewsTransform(transformId, "pivot_reviews", null);
startAndWaitForTransform(transformId, "pivot_reviews");
Response mappingResponse = client().performRequest(new Request("GET", "pivot_reviews/_mapping"));
Map<?, ?> mappingAsMap = entityAsMap(mappingResponse);
assertEquals(
Version.CURRENT.toString(),
XContentMapValues.extractValue("pivot_reviews.mappings._meta._transform.version.created", mappingAsMap)
);
assertTrue(
(Long) XContentMapValues.extractValue("pivot_reviews.mappings._meta._transform.creation_date_in_millis", mappingAsMap) < System
.currentTimeMillis()
);
assertTrue(
(Long) XContentMapValues.extractValue(
"pivot_reviews.mappings._meta._transform.creation_date_in_millis",
mappingAsMap
) > testStarted
);
assertEquals(transformId, XContentMapValues.extractValue("pivot_reviews.mappings._meta._transform.transform", mappingAsMap));
assertEquals("transform", XContentMapValues.extractValue("pivot_reviews.mappings._meta.created_by", mappingAsMap));
Response aliasesResponse = client().performRequest(new Request("GET", "pivot_reviews/_alias"));
Map<?, ?> aliasesAsMap = entityAsMap(aliasesResponse);
assertEquals(Map.of(), XContentMapValues.extractValue("pivot_reviews.aliases", aliasesAsMap));
}
public void testTransformDestIndexAliases() throws Exception {
String transformId = "test_aliases";
String destIndex1 = transformId + ".1";
String destIndex2 = transformId + ".2";
String destAliasAll = transformId + ".all";
String destAliasLatest = transformId + ".latest";
List<DestAlias> destAliases = List.of(new DestAlias(destAliasAll, false), new DestAlias(destAliasLatest, true));
// Create the transform
createPivotReviewsTransform(transformId, destIndex1, null, null, destAliases, null, null, null, REVIEWS_INDEX_NAME);
startAndWaitForTransform(transformId, destIndex1);
// Verify that both aliases are configured on the dest index
assertAliases(destIndex1, destAliasAll, destAliasLatest);
// Verify that the search results are the same, regardless whether we use index or alias
assertHitsAreTheSame(destIndex1, destAliasAll, destAliasLatest);
// Delete the transform
deleteTransform(transformId);
assertAliases(destIndex1, destAliasAll, destAliasLatest);
// Create the transform again (this time with a different destination index)
createPivotReviewsTransform(transformId, destIndex2, null, null, destAliases, null, null, null, REVIEWS_INDEX_NAME);
startAndWaitForTransform(transformId, destIndex2);
// Verify that destAliasLatest no longer points at destIndex1 but it now points at destIndex2
assertAliases(destIndex1, destAliasAll);
assertAliases(destIndex2, destAliasAll, destAliasLatest);
}
public void testTransformDestIndexCreatedDuringUpdate() throws Exception {
String transformId = "test_dest_index_on_update";
String destIndex = transformId + "-dest";
assertFalse(indexExists(destIndex));
// Create and start the unattended transform
createPivotReviewsTransform(
transformId,
destIndex,
null,
null,
null,
new SettingsConfig.Builder().setUnattended(true).build(),
null,
null,
REVIEWS_INDEX_NAME
);
startTransform(transformId);
// Verify that the destination index does not exist
assertFalse(indexExists(destIndex));
// Update the unattended transform. This will trigger destination index creation.
// The update has to change something in the config (here, max_page_search_size). Otherwise it would have been optimized away.
updateTransform(transformId, """
{ "settings": { "max_page_search_size": 123 } }""");
// Verify that the destination index now exists
assertTrue(indexExists(destIndex));
}
private static void assertAliases(String index, String... aliases) throws IOException {
Map<String, Map<?, ?>> expectedAliases = Arrays.stream(aliases).collect(Collectors.toMap(a -> a, a -> Map.of()));
Response aliasesResponse = client().performRequest(new Request("GET", index + "/_alias"));
assertEquals(expectedAliases, XContentMapValues.extractValue(index + ".aliases", entityAsMap(aliasesResponse)));
}
private static void assertHitsAreTheSame(String index, String... aliases) throws IOException {
Response indexSearchResponse = client().performRequest(new Request("GET", index + "/_search"));
Object indexSearchHits = XContentMapValues.extractValue("hits", entityAsMap(indexSearchResponse));
for (String alias : aliases) {
Response aliasSearchResponse = client().performRequest(new Request("GET", alias + "/_search"));
Object aliasSearchHits = XContentMapValues.extractValue("hits", entityAsMap(aliasSearchResponse));
assertEquals("index = " + index + ", alias = " + alias, indexSearchHits, aliasSearchHits);
}
}
}

View file

@ -1,67 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
package org.elasticsearch.xpack.transform.integration;
import org.elasticsearch.Version;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.Response;
import org.elasticsearch.common.xcontent.support.XContentMapValues;
import org.junit.Before;
import java.io.IOException;
import java.util.Map;
public class TransformDestIndexMetadataIT extends TransformRestTestCase {
private boolean indicesCreated = false;
// preserve indices in order to reuse source indices in several test cases
@Override
protected boolean preserveIndicesUponCompletion() {
return true;
}
@Before
public void createIndexes() throws IOException {
// it's not possible to run it as @BeforeClass as clients aren't initialized then, so we need this little hack
if (indicesCreated) {
return;
}
createReviewsIndex();
indicesCreated = true;
}
public void testTransformDestIndexMetadata() throws Exception {
long testStarted = System.currentTimeMillis();
createPivotReviewsTransform("test_meta", "pivot_reviews", null);
startAndWaitForTransform("test_meta", "pivot_reviews");
Response mappingResponse = client().performRequest(new Request("GET", "pivot_reviews/_mapping"));
Map<?, ?> mappingAsMap = entityAsMap(mappingResponse);
assertEquals(
Version.CURRENT.toString(),
XContentMapValues.extractValue("pivot_reviews.mappings._meta._transform.version.created", mappingAsMap)
);
assertTrue(
(Long) XContentMapValues.extractValue("pivot_reviews.mappings._meta._transform.creation_date_in_millis", mappingAsMap) < System
.currentTimeMillis()
);
assertTrue(
(Long) XContentMapValues.extractValue(
"pivot_reviews.mappings._meta._transform.creation_date_in_millis",
mappingAsMap
) > testStarted
);
assertEquals("test_meta", XContentMapValues.extractValue("pivot_reviews.mappings._meta._transform.transform", mappingAsMap));
assertEquals("transform", XContentMapValues.extractValue("pivot_reviews.mappings._meta.created_by", mappingAsMap));
}
}

View file

@ -109,6 +109,8 @@ public class TransformPivotRestIT extends TransformRestTestCase {
transformIndex, transformIndex,
null, null,
null, null,
null,
null,
BASIC_AUTH_VALUE_NO_ACCESS, BASIC_AUTH_VALUE_NO_ACCESS,
BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS, BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS,
REVIEWS_INDEX_NAME REVIEWS_INDEX_NAME
@ -138,6 +140,9 @@ public class TransformPivotRestIT extends TransformRestTestCase {
transformIndex, transformIndex,
null, null,
null, null,
null,
null,
null,
BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS, BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS,
indexName indexName
); );

View file

@ -22,6 +22,8 @@ import org.elasticsearch.common.xcontent.support.XContentMapValues;
import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.test.rest.ESRestTestCase;
import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentBuilder;
import org.elasticsearch.xpack.core.transform.TransformField; import org.elasticsearch.xpack.core.transform.TransformField;
import org.elasticsearch.xpack.core.transform.transforms.DestAlias;
import org.elasticsearch.xpack.core.transform.transforms.SettingsConfig;
import org.elasticsearch.xpack.core.transform.transforms.persistence.TransformInternalIndexConstants; import org.elasticsearch.xpack.core.transform.transforms.persistence.TransformInternalIndexConstants;
import org.junit.After; import org.junit.After;
import org.junit.AfterClass; import org.junit.AfterClass;
@ -237,12 +239,7 @@ public abstract class TransformRestTestCase extends ESRestTestCase {
} }
protected void createPivotReviewsTransform(String transformId, String transformIndex, String query) throws IOException { protected void createPivotReviewsTransform(String transformId, String transformIndex, String query) throws IOException {
createPivotReviewsTransform(transformId, transformIndex, query, null); createPivotReviewsTransform(transformId, transformIndex, query, null, null);
}
protected void createPivotReviewsTransform(String transformId, String transformIndex, String query, String pipeline)
throws IOException {
createPivotReviewsTransform(transformId, transformIndex, query, pipeline, null);
} }
protected void createReviewsIndexNano() throws IOException { protected void createReviewsIndexNano() throws IOException {
@ -293,30 +290,29 @@ public abstract class TransformRestTestCase extends ESRestTestCase {
String transformIndex, String transformIndex,
String query, String query,
String pipeline, String pipeline,
String authHeader, List<DestAlias> destAliases,
String sourceIndex SettingsConfig settings,
) throws IOException {
createPivotReviewsTransform(transformId, transformIndex, query, pipeline, authHeader, null, sourceIndex);
}
protected void createPivotReviewsTransform(
String transformId,
String transformIndex,
String query,
String pipeline,
String authHeader, String authHeader,
String secondaryAuthHeader, String secondaryAuthHeader,
String sourceIndex String sourceIndex
) throws IOException { ) throws IOException {
String config = "{"; String config = "{";
String destConfig = Strings.format("""
"dest": {"index":"%s"
""", transformIndex);
if (pipeline != null) { if (pipeline != null) {
config += Strings.format(""" destConfig += Strings.format("""
"dest": {"index":"%s", "pipeline":"%s"},""", transformIndex, pipeline); , "pipeline":"%s"
} else { """, pipeline);
config += Strings.format("""
"dest": {"index":"%s"},""", transformIndex);
} }
if (destAliases != null && destAliases.isEmpty() == false) {
destConfig += ", \"aliases\":[";
destConfig += String.join(",", destAliases.stream().map(Strings::toString).toList());
destConfig += "]";
}
destConfig += "},";
config += destConfig;
if (query != null) { if (query != null) {
config += Strings.format(""" config += Strings.format("""
@ -326,6 +322,10 @@ public abstract class TransformRestTestCase extends ESRestTestCase {
"source": {"index":"%s"},""", sourceIndex); "source": {"index":"%s"},""", sourceIndex);
} }
if (settings != null) {
config += "\"settings\": " + Strings.toString(settings) + ",";
}
config += """ config += """
"pivot": { "pivot": {
"group_by": { "group_by": {
@ -355,7 +355,6 @@ public abstract class TransformRestTestCase extends ESRestTestCase {
}, },
"frequency": "1s" "frequency": "1s"
}"""; }""";
createReviewsTransform(transformId, authHeader, secondaryAuthHeader, config); createReviewsTransform(transformId, authHeader, secondaryAuthHeader, config);
} }
@ -396,7 +395,19 @@ public abstract class TransformRestTestCase extends ESRestTestCase {
protected void createPivotReviewsTransform(String transformId, String transformIndex, String query, String pipeline, String authHeader) protected void createPivotReviewsTransform(String transformId, String transformIndex, String query, String pipeline, String authHeader)
throws IOException { throws IOException {
createPivotReviewsTransform(transformId, transformIndex, query, pipeline, authHeader, null, REVIEWS_INDEX_NAME); createPivotReviewsTransform(transformId, transformIndex, query, pipeline, null, null, authHeader, null, REVIEWS_INDEX_NAME);
}
protected void updateTransform(String transformId, String update) throws IOException {
final Request updateTransformRequest = createRequestWithSecondaryAuth(
"POST",
getTransformEndpoint() + transformId + "/_update",
null,
null
);
updateTransformRequest.setJsonEntity(update);
client().performRequest(updateTransformRequest);
} }
protected void startTransform(String transformId) throws IOException { protected void startTransform(String transformId) throws IOException {
@ -639,7 +650,7 @@ public abstract class TransformRestTestCase extends ESRestTestCase {
"indices": [ "indices": [
{ {
"names": [ %s ], "names": [ %s ],
"privileges": [ "create_index", "delete_index", "read", "write", "view_index_metadata" ] "privileges": [ "create_index", "delete_index", "read", "write", "view_index_metadata", "manage" ]
} }
] ]
}""", indicesStr)); }""", indicesStr));

View file

@ -148,7 +148,7 @@ public class TransformNoRemoteClusterClientNodeIT extends TransformSingleNodeTes
private static TransformConfig randomConfig(String transformId, String sourceIndex) { private static TransformConfig randomConfig(String transformId, String sourceIndex) {
return new TransformConfig.Builder().setId(transformId) return new TransformConfig.Builder().setId(transformId)
.setSource(new SourceConfig(sourceIndex)) .setSource(new SourceConfig(sourceIndex))
.setDest(new DestConfig("my-dest-index", null)) .setDest(new DestConfig("my-dest-index", null, null))
.setPivotConfig(PivotConfigTests.randomPivotConfig()) .setPivotConfig(PivotConfigTests.randomPivotConfig())
.build(); .build();
} }

View file

@ -154,7 +154,7 @@ public class TransformNoTransformNodeIT extends TransformSingleNodeTestCase {
private static TransformConfig randomConfig(String transformId) { private static TransformConfig randomConfig(String transformId) {
return new TransformConfig.Builder().setId(transformId) return new TransformConfig.Builder().setId(transformId)
.setSource(new SourceConfig("my-index")) .setSource(new SourceConfig("my-index"))
.setDest(new DestConfig("my-dest-index", null)) .setDest(new DestConfig("my-dest-index", null, null))
.setPivotConfig(PivotConfigTests.randomPivotConfig()) .setPivotConfig(PivotConfigTests.randomPivotConfig())
.build(); .build();
} }

View file

@ -135,7 +135,7 @@ public class TransformProgressIT extends TransformSingleNodeTestCase {
boolean missingBucket = userWithMissingBuckets > 0; boolean missingBucket = userWithMissingBuckets > 0;
createReviewsIndex(userWithMissingBuckets); createReviewsIndex(userWithMissingBuckets);
SourceConfig sourceConfig = new SourceConfig(REVIEWS_INDEX_NAME); SourceConfig sourceConfig = new SourceConfig(REVIEWS_INDEX_NAME);
DestConfig destConfig = new DestConfig("unnecessary", null); DestConfig destConfig = new DestConfig("unnecessary", null, null);
GroupConfig histgramGroupConfig = new GroupConfig( GroupConfig histgramGroupConfig = new GroupConfig(
Collections.emptyMap(), Collections.emptyMap(),
Collections.singletonMap("every_50", new HistogramGroupSource("count", null, missingBucket, 50.0)) Collections.singletonMap("every_50", new HistogramGroupSource("count", null, missingBucket, 50.0))

View file

@ -22,6 +22,7 @@ import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesRequest;
import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesResponse; import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesResponse;
import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
import org.elasticsearch.xpack.core.security.support.Exceptions; import org.elasticsearch.xpack.core.security.support.Exceptions;
import org.elasticsearch.xpack.core.transform.transforms.DestAlias;
import org.elasticsearch.xpack.core.transform.transforms.NullRetentionPolicyConfig; import org.elasticsearch.xpack.core.transform.transforms.NullRetentionPolicyConfig;
import org.elasticsearch.xpack.core.transform.transforms.TransformConfig; import org.elasticsearch.xpack.core.transform.transforms.TransformConfig;
@ -119,6 +120,17 @@ final class TransformPrivilegeChecker {
&& config.getRetentionPolicyConfig() instanceof NullRetentionPolicyConfig == false) { && config.getRetentionPolicyConfig() instanceof NullRetentionPolicyConfig == false) {
destPrivileges.add("delete"); destPrivileges.add("delete");
} }
if (config.getDestination().getAliases() != null && config.getDestination().getAliases().isEmpty() == false) {
destPrivileges.add("manage");
RoleDescriptor.IndicesPrivileges destAliasPrivileges = RoleDescriptor.IndicesPrivileges.builder()
.indices(config.getDestination().getAliases().stream().map(DestAlias::getAlias).toList())
.privileges("manage")
.allowRestrictedIndices(true)
.build();
indicesPrivileges.add(destAliasPrivileges);
}
RoleDescriptor.IndicesPrivileges destIndexPrivileges = RoleDescriptor.IndicesPrivileges.builder() RoleDescriptor.IndicesPrivileges destIndexPrivileges = RoleDescriptor.IndicesPrivileges.builder()
.indices(destIndex) .indices(destIndex)
.privileges(destPrivileges) .privileges(destPrivileges)

View file

@ -29,14 +29,13 @@ import org.elasticsearch.xpack.core.transform.transforms.AuthorizationState;
import org.elasticsearch.xpack.core.transform.transforms.TransformCheckpoint; import org.elasticsearch.xpack.core.transform.transforms.TransformCheckpoint;
import org.elasticsearch.xpack.core.transform.transforms.TransformConfig; import org.elasticsearch.xpack.core.transform.transforms.TransformConfig;
import org.elasticsearch.xpack.core.transform.transforms.TransformConfigUpdate; import org.elasticsearch.xpack.core.transform.transforms.TransformConfigUpdate;
import org.elasticsearch.xpack.core.transform.transforms.TransformDestIndexSettings;
import org.elasticsearch.xpack.core.transform.transforms.TransformStoredDoc; import org.elasticsearch.xpack.core.transform.transforms.TransformStoredDoc;
import org.elasticsearch.xpack.core.transform.transforms.persistence.TransformInternalIndexConstants; import org.elasticsearch.xpack.core.transform.transforms.persistence.TransformInternalIndexConstants;
import org.elasticsearch.xpack.transform.notifications.TransformAuditor;
import org.elasticsearch.xpack.transform.persistence.SeqNoPrimaryTermAndIndex; import org.elasticsearch.xpack.transform.persistence.SeqNoPrimaryTermAndIndex;
import org.elasticsearch.xpack.transform.persistence.TransformConfigManager; import org.elasticsearch.xpack.transform.persistence.TransformConfigManager;
import org.elasticsearch.xpack.transform.persistence.TransformIndex; import org.elasticsearch.xpack.transform.persistence.TransformIndex;
import java.time.Clock;
import java.util.Map; import java.util.Map;
/** /**
@ -118,6 +117,7 @@ public class TransformUpdater {
Settings settings, Settings settings,
Client client, Client client,
TransformConfigManager transformConfigManager, TransformConfigManager transformConfigManager,
TransformAuditor auditor,
final TransformConfig config, final TransformConfig config,
final TransformConfigUpdate update, final TransformConfigUpdate update,
final SeqNoPrimaryTermAndIndex seqNoPrimaryTermAndIndex, final SeqNoPrimaryTermAndIndex seqNoPrimaryTermAndIndex,
@ -179,6 +179,7 @@ public class TransformUpdater {
updateTransformConfiguration( updateTransformConfiguration(
client, client,
transformConfigManager, transformConfigManager,
auditor,
indexNameExpressionResolver, indexNameExpressionResolver,
updatedConfig, updatedConfig,
destIndexMappings, destIndexMappings,
@ -293,6 +294,7 @@ public class TransformUpdater {
private static void updateTransformConfiguration( private static void updateTransformConfiguration(
Client client, Client client,
TransformConfigManager transformConfigManager, TransformConfigManager transformConfigManager,
TransformAuditor auditor,
IndexNameExpressionResolver indexNameExpressionResolver, IndexNameExpressionResolver indexNameExpressionResolver,
TransformConfig config, TransformConfig config,
Map<String, String> mappings, Map<String, String> mappings,
@ -328,11 +330,9 @@ public class TransformUpdater {
); );
// <1> Create destination index if necessary // <1> Create destination index if necessary
String[] dest = indexNameExpressionResolver.concreteIndexNames( final String destinationIndex = config.getDestination().getIndex();
clusterState, String[] dest = indexNameExpressionResolver.concreteIndexNames(clusterState, IndicesOptions.lenientExpandOpen(), destinationIndex);
IndicesOptions.lenientExpandOpen(),
config.getDestination().getIndex()
);
String[] src = indexNameExpressionResolver.concreteIndexNames( String[] src = indexNameExpressionResolver.concreteIndexNames(
clusterState, clusterState,
IndicesOptions.lenientExpandOpen(), IndicesOptions.lenientExpandOpen(),
@ -345,26 +345,19 @@ public class TransformUpdater {
// we allow source indices to disappear. If the source and destination indices do not exist, don't do anything // we allow source indices to disappear. If the source and destination indices do not exist, don't do anything
// the transform will just have to dynamically create the destination index without special mapping. // the transform will just have to dynamically create the destination index without special mapping.
&& src.length > 0) { && src.length > 0) {
createDestinationIndex(client, config, mappings, createDestinationListener); TransformIndex.createDestinationIndex(
client,
auditor,
indexNameExpressionResolver,
clusterState,
config,
mappings,
createDestinationListener
);
} else { } else {
createDestinationListener.onResponse(null); createDestinationListener.onResponse(null);
} }
} }
private static void createDestinationIndex(
Client client,
TransformConfig config,
Map<String, String> mappings,
ActionListener<Boolean> listener
) {
TransformDestIndexSettings generatedDestIndexSettings = TransformIndex.createTransformDestIndexSettings(
mappings,
config.getId(),
Clock.systemUTC()
);
TransformIndex.createDestinationIndex(client, config, generatedDestIndexSettings, listener);
}
private TransformUpdater() {} private TransformUpdater() {}
} }

View file

@ -45,6 +45,7 @@ import org.elasticsearch.xpack.core.transform.TransformField;
import org.elasticsearch.xpack.core.transform.action.PreviewTransformAction; import org.elasticsearch.xpack.core.transform.action.PreviewTransformAction;
import org.elasticsearch.xpack.core.transform.action.PreviewTransformAction.Request; import org.elasticsearch.xpack.core.transform.action.PreviewTransformAction.Request;
import org.elasticsearch.xpack.core.transform.action.PreviewTransformAction.Response; import org.elasticsearch.xpack.core.transform.action.PreviewTransformAction.Response;
import org.elasticsearch.xpack.core.transform.transforms.DestAlias;
import org.elasticsearch.xpack.core.transform.transforms.SourceConfig; import org.elasticsearch.xpack.core.transform.transforms.SourceConfig;
import org.elasticsearch.xpack.core.transform.transforms.SyncConfig; import org.elasticsearch.xpack.core.transform.transforms.SyncConfig;
import org.elasticsearch.xpack.core.transform.transforms.TransformConfig; import org.elasticsearch.xpack.core.transform.transforms.TransformConfig;
@ -145,6 +146,7 @@ public class TransportPreviewTransformAction extends HandledTransportAction<Requ
config.getSource(), config.getSource(),
config.getDestination().getPipeline(), config.getDestination().getPipeline(),
config.getDestination().getIndex(), config.getDestination().getIndex(),
config.getDestination().getAliases(),
config.getSyncConfig(), config.getSyncConfig(),
listener listener
), ),
@ -199,6 +201,7 @@ public class TransportPreviewTransformAction extends HandledTransportAction<Requ
SourceConfig source, SourceConfig source,
String pipeline, String pipeline,
String dest, String dest,
List<DestAlias> aliases,
SyncConfig syncConfig, SyncConfig syncConfig,
ActionListener<Response> listener ActionListener<Response> listener
) { ) {

View file

@ -123,6 +123,7 @@ public class TransportResetTransformAction extends AcknowledgedTransportMasterNo
settings, settings,
client, client,
transformConfigManager, transformConfigManager,
auditor,
transformConfigAndVersion.v1(), transformConfigAndVersion.v1(),
TransformConfigUpdate.EMPTY, TransformConfigUpdate.EMPTY,
transformConfigAndVersion.v2(), transformConfigAndVersion.v2(),

View file

@ -14,9 +14,7 @@ import org.elasticsearch.ElasticsearchSecurityException;
import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.ElasticsearchStatusException;
import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.ActionFilters;
import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.action.support.master.TransportMasterNodeAction; import org.elasticsearch.action.support.master.TransportMasterNodeAction;
import org.elasticsearch.client.internal.Client; import org.elasticsearch.client.internal.Client;
import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterState;
@ -39,7 +37,6 @@ import org.elasticsearch.xpack.core.transform.action.StartTransformAction;
import org.elasticsearch.xpack.core.transform.action.ValidateTransformAction; import org.elasticsearch.xpack.core.transform.action.ValidateTransformAction;
import org.elasticsearch.xpack.core.transform.transforms.AuthorizationState; import org.elasticsearch.xpack.core.transform.transforms.AuthorizationState;
import org.elasticsearch.xpack.core.transform.transforms.TransformConfig; import org.elasticsearch.xpack.core.transform.transforms.TransformConfig;
import org.elasticsearch.xpack.core.transform.transforms.TransformDestIndexSettings;
import org.elasticsearch.xpack.core.transform.transforms.TransformState; import org.elasticsearch.xpack.core.transform.transforms.TransformState;
import org.elasticsearch.xpack.core.transform.transforms.TransformTaskParams; import org.elasticsearch.xpack.core.transform.transforms.TransformTaskParams;
import org.elasticsearch.xpack.core.transform.transforms.TransformTaskState; import org.elasticsearch.xpack.core.transform.transforms.TransformTaskState;
@ -51,8 +48,6 @@ import org.elasticsearch.xpack.transform.persistence.TransformIndex;
import org.elasticsearch.xpack.transform.transforms.TransformNodes; import org.elasticsearch.xpack.transform.transforms.TransformNodes;
import org.elasticsearch.xpack.transform.transforms.TransformTask; import org.elasticsearch.xpack.transform.transforms.TransformTask;
import java.time.Clock;
import java.util.Map;
import java.util.function.Consumer; import java.util.function.Consumer;
import java.util.function.Predicate; import java.util.function.Predicate;
@ -183,7 +178,22 @@ public class TransportStartTransformAction extends TransportMasterNodeAction<Sta
// <3> If the destination index exists, start the task, otherwise deduce our mappings for the destination index and create it // <3> If the destination index exists, start the task, otherwise deduce our mappings for the destination index and create it
ActionListener<ValidateTransformAction.Response> validationListener = ActionListener.wrap(validationResponse -> { ActionListener<ValidateTransformAction.Response> validationListener = ActionListener.wrap(validationResponse -> {
createDestinationIndex(state, transformConfigHolder.get(), validationResponse.getDestIndexMappings(), createOrGetIndexListener); if (Boolean.TRUE.equals(transformConfigHolder.get().getSettings().getUnattended())) {
logger.debug(
() -> format("[%s] Skip dest index creation as this is an unattended transform", transformConfigHolder.get().getId())
);
createOrGetIndexListener.onResponse(true);
return;
}
TransformIndex.createDestinationIndex(
client,
auditor,
indexNameExpressionResolver,
state,
transformConfigHolder.get(),
validationResponse.getDestIndexMappings(),
createOrGetIndexListener
);
}, e -> { }, e -> {
if (Boolean.TRUE.equals(transformConfigHolder.get().getSettings().getUnattended())) { if (Boolean.TRUE.equals(transformConfigHolder.get().getSettings().getUnattended())) {
logger.debug( logger.debug(
@ -261,60 +271,6 @@ public class TransportStartTransformAction extends TransportMasterNodeAction<Sta
transformConfigManager.getTransformConfiguration(request.getId(), getTransformListener); transformConfigManager.getTransformConfiguration(request.getId(), getTransformListener);
} }
private void createDestinationIndex(
ClusterState state,
TransformConfig config,
Map<String, String> destIndexMappings,
ActionListener<Boolean> listener
) {
if (Boolean.TRUE.equals(config.getSettings().getUnattended())) {
logger.debug(() -> format("[%s] Skip dest index creation as this is an unattended transform", config.getId()));
listener.onResponse(true);
return;
}
final String destinationIndex = config.getDestination().getIndex();
String[] dest = indexNameExpressionResolver.concreteIndexNames(state, IndicesOptions.lenientExpandOpen(), destinationIndex);
if (dest.length == 0) {
TransformDestIndexSettings generatedDestIndexSettings = TransformIndex.createTransformDestIndexSettings(
destIndexMappings,
config.getId(),
Clock.systemUTC()
);
TransformIndex.createDestinationIndex(client, config, generatedDestIndexSettings, ActionListener.wrap(r -> {
String message = Boolean.FALSE.equals(config.getSettings().getDeduceMappings())
? "Created destination index [" + destinationIndex + "]."
: "Created destination index [" + destinationIndex + "] with deduced mappings.";
auditor.info(config.getId(), message);
listener.onResponse(r);
}, listener::onFailure));
} else {
auditor.info(config.getId(), "Using existing destination index [" + destinationIndex + "].");
ClientHelper.executeAsyncWithOrigin(
client.threadPool().getThreadContext(),
ClientHelper.TRANSFORM_ORIGIN,
client.admin().indices().prepareStats(dest).clear().setDocs(true).request(),
ActionListener.<IndicesStatsResponse>wrap(r -> {
long docTotal = r.getTotal().docs.getCount();
if (docTotal > 0L) {
auditor.warning(
config.getId(),
"Non-empty destination index [" + destinationIndex + "]. " + "Contains [" + docTotal + "] total documents."
);
}
listener.onResponse(true);
}, e -> {
String msg = "Unable to determine destination index stats, error: " + e.getMessage();
logger.warn(msg, e);
auditor.warning(config.getId(), msg);
listener.onResponse(true);
}),
client.admin().indices()::stats
);
}
}
@Override @Override
protected ClusterBlockException checkBlock(StartTransformAction.Request request, ClusterState state) { protected ClusterBlockException checkBlock(StartTransformAction.Request request, ClusterState state) {
return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);

View file

@ -133,6 +133,7 @@ public class TransportUpdateTransformAction extends TransportTasksAction<Transfo
settings, settings,
client, client,
transformConfigManager, transformConfigManager,
auditor,
configAndVersion.v1(), configAndVersion.v1(),
update, update,
configAndVersion.v2(), configAndVersion.v2(),

View file

@ -158,6 +158,7 @@ public class TransportUpgradeTransformsAction extends TransportMasterNodeAction<
settings, settings,
client, client,
transformConfigManager, transformConfigManager,
auditor,
config, config,
update, update,
configAndVersion.v2(), configAndVersion.v2(),

View file

@ -12,20 +12,28 @@ import org.apache.logging.log4j.Logger;
import org.elasticsearch.Version; import org.elasticsearch.Version;
import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.admin.indices.alias.Alias;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesAction;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
import org.elasticsearch.action.admin.indices.create.CreateIndexAction; import org.elasticsearch.action.admin.indices.create.CreateIndexAction;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
import org.elasticsearch.action.admin.indices.get.GetIndexAction; import org.elasticsearch.action.admin.indices.get.GetIndexAction;
import org.elasticsearch.action.admin.indices.get.GetIndexRequest; import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.client.internal.Client; import org.elasticsearch.client.internal.Client;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.cluster.metadata.MappingMetadata;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.core.ClientHelper;
import org.elasticsearch.xpack.core.transform.TransformField; import org.elasticsearch.xpack.core.transform.TransformField;
import org.elasticsearch.xpack.core.transform.TransformMessages; import org.elasticsearch.xpack.core.transform.TransformMessages;
import org.elasticsearch.xpack.core.transform.transforms.DestAlias;
import org.elasticsearch.xpack.core.transform.transforms.TransformConfig; import org.elasticsearch.xpack.core.transform.transforms.TransformConfig;
import org.elasticsearch.xpack.core.transform.transforms.TransformDestIndexSettings; import org.elasticsearch.xpack.core.transform.transforms.TransformDestIndexSettings;
import org.elasticsearch.xpack.transform.notifications.TransformAuditor;
import java.time.Clock; import java.time.Clock;
import java.util.HashMap; import java.util.HashMap;
@ -33,6 +41,7 @@ import java.util.Map;
import java.util.Set; import java.util.Set;
import static java.util.Collections.singletonMap; import static java.util.Collections.singletonMap;
import static java.util.stream.Collectors.joining;
import static java.util.stream.Collectors.toMap; import static java.util.stream.Collectors.toMap;
import static org.elasticsearch.xpack.core.ClientHelper.TRANSFORM_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.TRANSFORM_ORIGIN;
import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;
@ -92,12 +101,82 @@ public final class TransformIndex {
public static void createDestinationIndex( public static void createDestinationIndex(
Client client, Client client,
TransformConfig transformConfig, TransformAuditor auditor,
IndexNameExpressionResolver indexNameExpressionResolver,
ClusterState clusterState,
TransformConfig config,
Map<String, String> destIndexMappings,
ActionListener<Boolean> listener
) {
final String destinationIndex = config.getDestination().getIndex();
String[] dest = indexNameExpressionResolver.concreteIndexNames(clusterState, IndicesOptions.lenientExpandOpen(), destinationIndex);
// <3> Final listener
ActionListener<Boolean> setUpDestinationAliasesListener = ActionListener.wrap(r -> {
String message = "Set up aliases ["
+ config.getDestination().getAliases().stream().map(DestAlias::getAlias).collect(joining(", "))
+ "] for destination index ["
+ destinationIndex
+ "].";
auditor.info(config.getId(), message);
listener.onResponse(r);
}, listener::onFailure);
// <2> Set up destination index aliases, regardless whether the destination index was created by the transform or by the user
ActionListener<Boolean> createDestinationIndexListener = ActionListener.wrap(createdDestinationIndex -> {
if (createdDestinationIndex) {
String message = Boolean.FALSE.equals(config.getSettings().getDeduceMappings())
? "Created destination index [" + destinationIndex + "]."
: "Created destination index [" + destinationIndex + "] with deduced mappings.";
auditor.info(config.getId(), message);
}
setUpDestinationAliases(client, config, setUpDestinationAliasesListener);
}, listener::onFailure);
if (dest.length == 0) {
TransformDestIndexSettings generatedDestIndexSettings = createTransformDestIndexSettings(
destIndexMappings,
config.getId(),
Clock.systemUTC()
);
// <1> Create destination index
createDestinationIndex(client, config, generatedDestIndexSettings, createDestinationIndexListener);
} else {
auditor.info(config.getId(), "Using existing destination index [" + destinationIndex + "].");
// <1'> Audit existing destination index' stats
ClientHelper.executeAsyncWithOrigin(
client.threadPool().getThreadContext(),
ClientHelper.TRANSFORM_ORIGIN,
client.admin().indices().prepareStats(dest).clear().setDocs(true).request(),
ActionListener.<IndicesStatsResponse>wrap(r -> {
long docTotal = r.getTotal().docs.getCount();
if (docTotal > 0L) {
auditor.warning(
config.getId(),
"Non-empty destination index [" + destinationIndex + "]. " + "Contains [" + docTotal + "] total documents."
);
}
createDestinationIndexListener.onResponse(false);
}, e -> {
String msg = "Unable to determine destination index stats, error: " + e.getMessage();
logger.warn(msg, e);
auditor.warning(config.getId(), msg);
createDestinationIndexListener.onResponse(false);
}),
client.admin().indices()::stats
);
}
}
static void createDestinationIndex(
Client client,
TransformConfig config,
TransformDestIndexSettings destIndexSettings, TransformDestIndexSettings destIndexSettings,
ActionListener<Boolean> listener ActionListener<Boolean> listener
) { ) {
CreateIndexRequest request = new CreateIndexRequest(transformConfig.getDestination().getIndex()); CreateIndexRequest request = new CreateIndexRequest(config.getDestination().getIndex());
request.settings(destIndexSettings.getSettings()); request.settings(destIndexSettings.getSettings());
request.mapping(destIndexSettings.getMappings()); request.mapping(destIndexSettings.getMappings());
for (Alias alias : destIndexSettings.getAliases()) { for (Alias alias : destIndexSettings.getAliases()) {
@ -105,7 +184,7 @@ public final class TransformIndex {
} }
ClientHelper.executeWithHeadersAsync( ClientHelper.executeWithHeadersAsync(
transformConfig.getHeaders(), config.getHeaders(),
TRANSFORM_ORIGIN, TRANSFORM_ORIGIN,
client, client,
CreateIndexAction.INSTANCE, CreateIndexAction.INSTANCE,
@ -115,8 +194,47 @@ public final class TransformIndex {
}, e -> { }, e -> {
String message = TransformMessages.getMessage( String message = TransformMessages.getMessage(
TransformMessages.FAILED_TO_CREATE_DESTINATION_INDEX, TransformMessages.FAILED_TO_CREATE_DESTINATION_INDEX,
transformConfig.getDestination().getIndex(), config.getDestination().getIndex(),
transformConfig.getId() config.getId()
);
logger.error(message);
listener.onFailure(new RuntimeException(message, e));
})
);
}
static void setUpDestinationAliases(Client client, TransformConfig config, ActionListener<Boolean> listener) {
// No aliases configured to be set up, just move on
if (config.getDestination().getAliases() == null || config.getDestination().getAliases().isEmpty()) {
listener.onResponse(true);
return;
}
IndicesAliasesRequest request = new IndicesAliasesRequest();
for (DestAlias destAlias : config.getDestination().getAliases()) {
if (destAlias.isMoveOnCreation()) {
request.addAliasAction(IndicesAliasesRequest.AliasActions.remove().alias(destAlias.getAlias()).index("*"));
}
}
for (DestAlias destAlias : config.getDestination().getAliases()) {
request.addAliasAction(
IndicesAliasesRequest.AliasActions.add().alias(destAlias.getAlias()).index(config.getDestination().getIndex())
);
}
ClientHelper.executeWithHeadersAsync(
config.getHeaders(),
TRANSFORM_ORIGIN,
client,
IndicesAliasesAction.INSTANCE,
request,
ActionListener.wrap(aliasesResponse -> {
listener.onResponse(true);
}, e -> {
String message = TransformMessages.getMessage(
TransformMessages.FAILED_TO_SET_UP_DESTINATION_ALIASES,
config.getDestination().getIndex(),
config.getId()
); );
logger.error(message); logger.error(message);
listener.onFailure(new RuntimeException(message, e)); listener.onFailure(new RuntimeException(message, e));

View file

@ -30,6 +30,7 @@ import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesResponse;
import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
import org.elasticsearch.xpack.core.security.authz.permission.ResourcePrivileges; import org.elasticsearch.xpack.core.security.authz.permission.ResourcePrivileges;
import org.elasticsearch.xpack.core.security.user.User; import org.elasticsearch.xpack.core.security.user.User;
import org.elasticsearch.xpack.core.transform.transforms.DestAlias;
import org.elasticsearch.xpack.core.transform.transforms.DestConfig; import org.elasticsearch.xpack.core.transform.transforms.DestConfig;
import org.elasticsearch.xpack.core.transform.transforms.SourceConfig; import org.elasticsearch.xpack.core.transform.transforms.SourceConfig;
import org.elasticsearch.xpack.core.transform.transforms.TimeRetentionPolicyConfig; import org.elasticsearch.xpack.core.transform.transforms.TimeRetentionPolicyConfig;
@ -76,9 +77,10 @@ public class TransformPrivilegeCheckerTests extends ESTestCase {
.addPrivilege("read", false) .addPrivilege("read", false)
.addPrivilege("delete", false) .addPrivilege("delete", false)
.build(); .build();
private static final String DEST_ALIAS_NAME = "some-dest-alias";
private static final TransformConfig TRANSFORM_CONFIG = new TransformConfig.Builder().setId(TRANSFORM_ID) private static final TransformConfig TRANSFORM_CONFIG = new TransformConfig.Builder().setId(TRANSFORM_ID)
.setSource(new SourceConfig(SOURCE_INDEX_NAME)) .setSource(new SourceConfig(SOURCE_INDEX_NAME))
.setDest(new DestConfig(DEST_INDEX_NAME, null)) .setDest(new DestConfig(DEST_INDEX_NAME, null, null))
.build(); .build();
private ThreadPool threadPool; private ThreadPool threadPool;
private SecurityContext securityContext; private SecurityContext securityContext;
@ -282,6 +284,50 @@ public class TransformPrivilegeCheckerTests extends ESTestCase {
); );
} }
public void testCheckPrivileges_CheckDestIndexPrivileges_DestAliasExists() {
ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
.metadata(
Metadata.builder()
.put(IndexMetadata.builder(DEST_INDEX_NAME).settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0))
)
.build();
TransformConfig config = new TransformConfig.Builder(TRANSFORM_CONFIG).setDest(
new DestConfig(DEST_INDEX_NAME, List.of(new DestAlias(DEST_ALIAS_NAME, randomBoolean())), null)
).build();
TransformPrivilegeChecker.checkPrivileges(
OPERATION_NAME,
SETTINGS,
securityContext,
indexNameExpressionResolver,
clusterState,
client,
config,
true,
ActionListener.wrap(aVoid -> {
HasPrivilegesRequest request = client.lastHasPrivilegesRequest;
assertThat(request.username(), is(equalTo(USER_NAME)));
assertThat(request.applicationPrivileges(), is(emptyArray()));
assertThat(request.clusterPrivileges(), is(emptyArray()));
assertThat(request.indexPrivileges(), is(arrayWithSize(3)));
RoleDescriptor.IndicesPrivileges sourceIndicesPrivileges = request.indexPrivileges()[0];
assertThat(sourceIndicesPrivileges.getIndices(), is(arrayContaining(SOURCE_INDEX_NAME)));
assertThat(sourceIndicesPrivileges.getPrivileges(), is(arrayContaining("read", "view_index_metadata")));
assertThat(sourceIndicesPrivileges.allowRestrictedIndices(), is(true));
RoleDescriptor.IndicesPrivileges destIndicesPrivileges = request.indexPrivileges()[1];
assertThat(destIndicesPrivileges.getIndices(), is(arrayContaining(DEST_ALIAS_NAME)));
assertThat(destIndicesPrivileges.getPrivileges(), is(arrayContaining("manage")));
assertThat(destIndicesPrivileges.allowRestrictedIndices(), is(true));
RoleDescriptor.IndicesPrivileges destAliasesPrivileges = request.indexPrivileges()[2];
assertThat(destAliasesPrivileges.getIndices(), is(arrayContaining(DEST_INDEX_NAME)));
assertThat(destAliasesPrivileges.getPrivileges(), is(arrayContaining("read", "index", "manage")));
assertThat(destAliasesPrivileges.allowRestrictedIndices(), is(true));
}, e -> fail(e.getMessage()))
);
}
public void testCheckPrivileges_MissingPrivileges() { public void testCheckPrivileges_MissingPrivileges() {
ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT) ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
.metadata( .metadata(
@ -290,7 +336,7 @@ public class TransformPrivilegeCheckerTests extends ESTestCase {
) )
.build(); .build();
TransformConfig config = new TransformConfig.Builder(TRANSFORM_CONFIG).setSource(new SourceConfig(SOURCE_INDEX_NAME_2)) TransformConfig config = new TransformConfig.Builder(TRANSFORM_CONFIG).setSource(new SourceConfig(SOURCE_INDEX_NAME_2))
.setDest(new DestConfig(DEST_INDEX_NAME_2, null)) .setDest(new DestConfig(DEST_INDEX_NAME_2, null, null))
.build(); .build();
client.nextHasPrivilegesResponse = new HasPrivilegesResponse( client.nextHasPrivilegesResponse = new HasPrivilegesResponse(
USER_NAME, USER_NAME,

View file

@ -18,6 +18,7 @@ import org.elasticsearch.action.support.master.AcknowledgedRequest;
import org.elasticsearch.client.internal.Client; import org.elasticsearch.client.internal.Client;
import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.core.Tuple; import org.elasticsearch.core.Tuple;
@ -42,6 +43,8 @@ import org.elasticsearch.xpack.core.transform.transforms.TransformState;
import org.elasticsearch.xpack.core.transform.transforms.TransformStoredDoc; import org.elasticsearch.xpack.core.transform.transforms.TransformStoredDoc;
import org.elasticsearch.xpack.core.transform.transforms.TransformTaskState; import org.elasticsearch.xpack.core.transform.transforms.TransformTaskState;
import org.elasticsearch.xpack.transform.action.TransformUpdater.UpdateResult; import org.elasticsearch.xpack.transform.action.TransformUpdater.UpdateResult;
import org.elasticsearch.xpack.transform.notifications.MockTransformAuditor;
import org.elasticsearch.xpack.transform.notifications.TransformAuditor;
import org.elasticsearch.xpack.transform.persistence.InMemoryTransformConfigManager; import org.elasticsearch.xpack.transform.persistence.InMemoryTransformConfigManager;
import org.elasticsearch.xpack.transform.persistence.SeqNoPrimaryTermAndIndex; import org.elasticsearch.xpack.transform.persistence.SeqNoPrimaryTermAndIndex;
import org.elasticsearch.xpack.transform.persistence.TransformConfigManager; import org.elasticsearch.xpack.transform.persistence.TransformConfigManager;
@ -59,6 +62,7 @@ import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.not;
import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.notNullValue;
import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.nullValue;
import static org.mockito.Mockito.mock;
public class TransformUpdaterTests extends ESTestCase { public class TransformUpdaterTests extends ESTestCase {
@ -68,6 +72,8 @@ public class TransformUpdaterTests extends ESTestCase {
private final SecurityContext johnSecurityContext = newSecurityContextFor(JOHN); private final SecurityContext johnSecurityContext = newSecurityContextFor(JOHN);
private final IndexNameExpressionResolver indexNameExpressionResolver = TestIndexNameExpressionResolver.newInstance(); private final IndexNameExpressionResolver indexNameExpressionResolver = TestIndexNameExpressionResolver.newInstance();
private Client client; private Client client;
private ClusterService clusterService = mock(ClusterService.class);
private TransformAuditor auditor = new MockTransformAuditor(clusterService);
private final Settings settings = Settings.builder().put(XPackSettings.SECURITY_ENABLED.getKey(), true).build(); private final Settings settings = Settings.builder().put(XPackSettings.SECURITY_ENABLED.getKey(), true).build();
private static class MyMockClient extends NoOpClient { private static class MyMockClient extends NoOpClient {
@ -111,6 +117,8 @@ public class TransformUpdaterTests extends ESTestCase {
client.close(); client.close();
} }
client = new MyMockClient(getTestName()); client = new MyMockClient(getTestName());
clusterService = mock(ClusterService.class);
auditor = new MockTransformAuditor(clusterService);
} }
@After @After
@ -140,6 +148,7 @@ public class TransformUpdaterTests extends ESTestCase {
settings, settings,
client, client,
transformConfigManager, transformConfigManager,
auditor,
maxCompatibleConfig, maxCompatibleConfig,
update, update,
null, // seqNoPrimaryTermAndIndex null, // seqNoPrimaryTermAndIndex
@ -174,6 +183,7 @@ public class TransformUpdaterTests extends ESTestCase {
settings, settings,
client, client,
transformConfigManager, transformConfigManager,
auditor,
minCompatibleConfig, minCompatibleConfig,
update, update,
null, // seqNoPrimaryTermAndIndex null, // seqNoPrimaryTermAndIndex
@ -245,6 +255,7 @@ public class TransformUpdaterTests extends ESTestCase {
settings, settings,
client, client,
transformConfigManager, transformConfigManager,
auditor,
oldConfig, oldConfig,
update, update,
null, // seqNoPrimaryTermAndIndex null, // seqNoPrimaryTermAndIndex
@ -311,6 +322,7 @@ public class TransformUpdaterTests extends ESTestCase {
settings, settings,
client, client,
transformConfigManager, transformConfigManager,
auditor,
oldConfigForDryRunUpdate, oldConfigForDryRunUpdate,
update, update,
null, // seqNoPrimaryTermAndIndex null, // seqNoPrimaryTermAndIndex
@ -357,6 +369,7 @@ public class TransformUpdaterTests extends ESTestCase {
settings, settings,
client, client,
transformConfigManager, transformConfigManager,
auditor,
oldConfig, oldConfig,
TransformConfigUpdate.EMPTY, TransformConfigUpdate.EMPTY,
null, // seqNoPrimaryTermAndIndex null, // seqNoPrimaryTermAndIndex
@ -398,6 +411,7 @@ public class TransformUpdaterTests extends ESTestCase {
settings, settings,
client, client,
transformConfigManager, transformConfigManager,
auditor,
oldConfig, oldConfig,
TransformConfigUpdate.EMPTY, TransformConfigUpdate.EMPTY,
null, // seqNoPrimaryTermAndIndex null, // seqNoPrimaryTermAndIndex
@ -431,6 +445,7 @@ public class TransformUpdaterTests extends ESTestCase {
settings, settings,
client, client,
transformConfigManager, transformConfigManager,
auditor,
oldConfig, oldConfig,
TransformConfigUpdate.EMPTY, TransformConfigUpdate.EMPTY,
null, // seqNoPrimaryTermAndIndex null, // seqNoPrimaryTermAndIndex

View file

@ -8,6 +8,8 @@ package org.elasticsearch.xpack.transform.persistence;
import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.LatchedActionListener; import org.elasticsearch.action.LatchedActionListener;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesAction;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
import org.elasticsearch.action.admin.indices.create.CreateIndexAction; import org.elasticsearch.action.admin.indices.create.CreateIndexAction;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
import org.elasticsearch.action.admin.indices.get.GetIndexAction; import org.elasticsearch.action.admin.indices.get.GetIndexAction;
@ -21,6 +23,10 @@ import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentParser;
import org.elasticsearch.xcontent.json.JsonXContent; import org.elasticsearch.xcontent.json.JsonXContent;
import org.elasticsearch.xpack.core.transform.transforms.DestAlias;
import org.elasticsearch.xpack.core.transform.transforms.DestConfig;
import org.elasticsearch.xpack.core.transform.transforms.SourceConfig;
import org.elasticsearch.xpack.core.transform.transforms.TransformConfig;
import org.elasticsearch.xpack.core.transform.transforms.TransformConfigTests; import org.elasticsearch.xpack.core.transform.transforms.TransformConfigTests;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
@ -32,6 +38,7 @@ import java.time.Clock;
import java.time.Instant; import java.time.Instant;
import java.time.ZoneId; import java.time.ZoneId;
import java.util.HashMap; import java.util.HashMap;
import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.concurrent.CountDownLatch; import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
@ -40,6 +47,8 @@ import static java.util.Collections.emptyMap;
import static java.util.Collections.singletonMap; import static java.util.Collections.singletonMap;
import static org.elasticsearch.common.xcontent.support.XContentMapValues.extractValue; import static org.elasticsearch.common.xcontent.support.XContentMapValues.extractValue;
import static org.hamcrest.Matchers.anEmptyMap; import static org.hamcrest.Matchers.anEmptyMap;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.empty;
import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.is;
import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.any;
@ -152,6 +161,60 @@ public class TransformIndexTests extends ESTestCase {
assertThat(extractValue("_doc._meta._transform.creation_date_in_millis", map), equalTo(CURRENT_TIME_MILLIS)); assertThat(extractValue("_doc._meta._transform.creation_date_in_millis", map), equalTo(CURRENT_TIME_MILLIS));
assertThat(extractValue("_doc._meta.created_by", map), equalTo(CREATED_BY)); assertThat(extractValue("_doc._meta.created_by", map), equalTo(CREATED_BY));
} }
assertThat(createIndexRequest.aliases(), is(empty()));
}
public void testSetUpDestinationAliases_NullAliases() {
doAnswer(withResponse(null)).when(client).execute(any(), any(), any());
TransformConfig config = new TransformConfig.Builder().setId("my-id")
.setSource(new SourceConfig("my-source"))
.setDest(new DestConfig("my-dest", null, null))
.build();
TransformIndex.setUpDestinationAliases(client, config, ActionListener.wrap(Assert::assertTrue, e -> fail(e.getMessage())));
verifyNoMoreInteractions(client);
}
public void testSetUpDestinationAliases_EmptyAliases() {
doAnswer(withResponse(null)).when(client).execute(any(), any(), any());
TransformConfig config = new TransformConfig.Builder().setId("my-id")
.setSource(new SourceConfig("my-source"))
.setDest(new DestConfig("my-dest", List.of(), null))
.build();
TransformIndex.setUpDestinationAliases(client, config, ActionListener.wrap(Assert::assertTrue, e -> fail(e.getMessage())));
verifyNoMoreInteractions(client);
}
public void testSetUpDestinationAliases() {
doAnswer(withResponse(null)).when(client).execute(any(), any(), any());
String destIndex = "my-dest";
TransformConfig config = new TransformConfig.Builder().setId("my-id")
.setSource(new SourceConfig("my-source"))
.setDest(new DestConfig(destIndex, List.of(new DestAlias(".all", false), new DestAlias(".latest", true)), null))
.build();
TransformIndex.setUpDestinationAliases(client, config, ActionListener.wrap(Assert::assertTrue, e -> fail(e.getMessage())));
ArgumentCaptor<IndicesAliasesRequest> indicesAliasesRequestCaptor = ArgumentCaptor.forClass(IndicesAliasesRequest.class);
verify(client).execute(eq(IndicesAliasesAction.INSTANCE), indicesAliasesRequestCaptor.capture(), any());
verify(client, atLeastOnce()).threadPool();
verifyNoMoreInteractions(client);
IndicesAliasesRequest indicesAliasesRequest = indicesAliasesRequestCaptor.getValue();
assertThat(
indicesAliasesRequest.getAliasActions(),
contains(
IndicesAliasesRequest.AliasActions.remove().alias(".latest").index("*"),
IndicesAliasesRequest.AliasActions.add().alias(".all").index(destIndex),
IndicesAliasesRequest.AliasActions.add().alias(".latest").index(destIndex)
)
);
} }
public void testCreateMappingsFromStringMap() { public void testCreateMappingsFromStringMap() {

View file

@ -139,6 +139,33 @@
"type": "string", "type": "string",
"pattern": "^.*$" "pattern": "^.*$"
}, },
"aliases": {
"$id": "#root/dest/aliases",
"title": "Aliases",
"type": "array",
"items": [
{
"type": "object",
"description": "describes a single alias",
"additionalProperties": false,
"required": [
"alias"
],
"properties": {
"alias": {
"type": "string",
"description": "single alias name"
},
"move_on_creation": {
"type": "boolean",
"description": "remove this alias from all the indices it points to and assign it to the dest index only",
"default": false
}
}
}
],
"pattern": "^.*$"
},
"pipeline": { "pipeline": {
"$id": "#root/dest/pipeline", "$id": "#root/dest/pipeline",
"title": "Pipeline", "title": "Pipeline",