diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/GetAutoFollowPatternResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/GetAutoFollowPatternResponse.java index 2a9a9b4e9bc7..bb293ff19dba 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/GetAutoFollowPatternResponse.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/GetAutoFollowPatternResponse.java @@ -82,11 +82,19 @@ public final class GetAutoFollowPatternResponse { @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( - "pattern", true, args -> new Pattern((String) args[0], (List) args[1], (String) args[2])); + "pattern", + true, + args -> new Pattern((String) args[0], + (List) args[1], + args[2] == null ? Collections.emptyList() : (List) args[2], + (String) args[3]) + ); static { PARSER.declareString(ConstructingObjectParser.constructorArg(), PutFollowRequest.REMOTE_CLUSTER_FIELD); PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), PutAutoFollowPatternRequest.LEADER_PATTERNS_FIELD); + PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), + PutAutoFollowPatternRequest.LEADER_EXCLUSION_PATTERNS_FIELD); PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), PutAutoFollowPatternRequest.FOLLOW_PATTERN_FIELD); PARSER.declareObject(Pattern::setSettings, (p, c) -> Settings.fromXContent(p), PutAutoFollowPatternRequest.SETTINGS); PARSER.declareInt(Pattern::setMaxReadRequestOperationCount, FollowConfig.MAX_READ_REQUEST_OPERATION_COUNT); @@ -123,11 +131,16 @@ public final class GetAutoFollowPatternResponse { private final String remoteCluster; private final List leaderIndexPatterns; + private final List leaderIndexExclusionPatterns; private final String followIndexNamePattern; - Pattern(String remoteCluster, List leaderIndexPatterns, String followIndexNamePattern) { + Pattern(String remoteCluster, + List leaderIndexPatterns, + List leaderIndexExclusionPatterns, + String followIndexNamePattern) { this.remoteCluster = remoteCluster; this.leaderIndexPatterns = leaderIndexPatterns; + this.leaderIndexExclusionPatterns = leaderIndexExclusionPatterns; this.followIndexNamePattern = followIndexNamePattern; } @@ -139,6 +152,10 @@ public final class GetAutoFollowPatternResponse { return leaderIndexPatterns; } + public List getLeaderIndexExclusionPatterns() { + return leaderIndexExclusionPatterns; + } + public String getFollowIndexNamePattern() { return followIndexNamePattern; } @@ -151,6 +168,7 @@ public final class GetAutoFollowPatternResponse { Pattern pattern = (Pattern) o; return Objects.equals(remoteCluster, pattern.remoteCluster) && Objects.equals(leaderIndexPatterns, pattern.leaderIndexPatterns) && + Objects.equals(leaderIndexExclusionPatterns, pattern.leaderIndexExclusionPatterns) && Objects.equals(followIndexNamePattern, pattern.followIndexNamePattern); } @@ -160,6 +178,7 @@ public final class GetAutoFollowPatternResponse { super.hashCode(), remoteCluster, leaderIndexPatterns, + leaderIndexExclusionPatterns, followIndexNamePattern ); } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/PutAutoFollowPatternRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/PutAutoFollowPatternRequest.java index ecad88856612..7cc8b6a0fa5c 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/PutAutoFollowPatternRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/PutAutoFollowPatternRequest.java @@ -14,23 +14,36 @@ import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import java.io.IOException; +import java.util.Collections; import java.util.List; import java.util.Objects; public final class PutAutoFollowPatternRequest extends FollowConfig implements Validatable, ToXContentObject { static final ParseField LEADER_PATTERNS_FIELD = new ParseField("leader_index_patterns"); + static final ParseField LEADER_EXCLUSION_PATTERNS_FIELD = new ParseField("leader_index_exclusion_patterns"); static final ParseField FOLLOW_PATTERN_FIELD = new ParseField("follow_index_pattern"); private final String name; private final String remoteCluster; private final List leaderIndexPatterns; + private final List leaderIndexExclusionPatterns; private String followIndexNamePattern; - public PutAutoFollowPatternRequest(String name, String remoteCluster, List leaderIndexPatterns) { + public PutAutoFollowPatternRequest(String name, + String remoteCluster, + List leaderIndexPatterns) { + this(name, remoteCluster, leaderIndexPatterns, Collections.emptyList()); + } + + public PutAutoFollowPatternRequest(String name, + String remoteCluster, + List leaderIndexPatterns, + List leaderIndexExclusionPatterns) { this.name = Objects.requireNonNull(name); this.remoteCluster = Objects.requireNonNull(remoteCluster); this.leaderIndexPatterns = Objects.requireNonNull(leaderIndexPatterns); + this.leaderIndexExclusionPatterns = Objects.requireNonNull(leaderIndexExclusionPatterns); } public String getName() { @@ -45,6 +58,10 @@ public final class PutAutoFollowPatternRequest extends FollowConfig implements V return leaderIndexPatterns; } + public List getLeaderIndexExclusionPatterns() { + return leaderIndexExclusionPatterns; + } + public String getFollowIndexNamePattern() { return followIndexNamePattern; } @@ -58,6 +75,9 @@ public final class PutAutoFollowPatternRequest extends FollowConfig implements V builder.startObject(); builder.field(PutFollowRequest.REMOTE_CLUSTER_FIELD.getPreferredName(), remoteCluster); builder.field(LEADER_PATTERNS_FIELD.getPreferredName(), leaderIndexPatterns); + if (leaderIndexExclusionPatterns.isEmpty() == false) { + builder.field(LEADER_EXCLUSION_PATTERNS_FIELD.getPreferredName(), leaderIndexExclusionPatterns); + } if (followIndexNamePattern != null) { builder.field(FOLLOW_PATTERN_FIELD.getPreferredName(), followIndexNamePattern); } @@ -75,6 +95,7 @@ public final class PutAutoFollowPatternRequest extends FollowConfig implements V return Objects.equals(name, that.name) && Objects.equals(remoteCluster, that.remoteCluster) && Objects.equals(leaderIndexPatterns, that.leaderIndexPatterns) && + Objects.equals(leaderIndexExclusionPatterns, that.leaderIndexExclusionPatterns) && Objects.equals(followIndexNamePattern, that.followIndexNamePattern); } @@ -85,6 +106,7 @@ public final class PutAutoFollowPatternRequest extends FollowConfig implements V name, remoteCluster, leaderIndexPatterns, + leaderIndexExclusionPatterns, followIndexNamePattern ); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/CCRIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/CCRIT.java index 593712802ff2..2459fe30b587 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/CCRIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/CCRIT.java @@ -58,6 +58,7 @@ import static org.hamcrest.Matchers.hasEntry; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; public class CCRIT extends ESRestHighLevelClientTestCase { @@ -244,8 +245,10 @@ public class CCRIT extends ESRestHighLevelClientTestCase { public void testAutoFollowing() throws Exception { CcrClient ccrClient = highLevelClient().ccr(); - PutAutoFollowPatternRequest putAutoFollowPatternRequest = - new PutAutoFollowPatternRequest("pattern1", "local_cluster", Collections.singletonList("logs-*")); + PutAutoFollowPatternRequest putAutoFollowPatternRequest = new PutAutoFollowPatternRequest("pattern1", + "local_cluster", + Collections.singletonList("logs-*"), + Collections.singletonList("logs-excluded")); putAutoFollowPatternRequest.setFollowIndexNamePattern("copy-{{leader_index}}"); final int followerNumberOfReplicas = randomIntBetween(0, 4); final Settings autoFollowerPatternSettings = @@ -255,6 +258,11 @@ public class CCRIT extends ESRestHighLevelClientTestCase { execute(putAutoFollowPatternRequest, ccrClient::putAutoFollowPattern, ccrClient::putAutoFollowPatternAsync); assertThat(putAutoFollowPatternResponse.isAcknowledged(), is(true)); + CreateIndexRequest createExcludedIndexRequest = new CreateIndexRequest("logs-excluded"); + CreateIndexResponse createExcludedIndexResponse = + highLevelClient().indices().create(createExcludedIndexRequest, RequestOptions.DEFAULT); + assertThat(createExcludedIndexResponse.isAcknowledged(), is(true)); + CreateIndexRequest createIndexRequest = new CreateIndexRequest("logs-20200101"); CreateIndexResponse response = highLevelClient().indices().create(createIndexRequest, RequestOptions.DEFAULT); assertThat(response.isAcknowledged(), is(true)); @@ -264,11 +272,13 @@ public class CCRIT extends ESRestHighLevelClientTestCase { CcrStatsResponse ccrStatsResponse = execute(ccrStatsRequest, ccrClient::getCcrStats, ccrClient::getCcrStatsAsync); assertThat(ccrStatsResponse.getAutoFollowStats().getNumberOfSuccessfulFollowIndices(), equalTo(1L)); assertThat(ccrStatsResponse.getIndicesFollowStats().getShardFollowStats("copy-logs-20200101"), notNullValue()); + assertThat(ccrStatsResponse.getIndicesFollowStats().getShardFollowStats("copy-logs-excluded"), nullValue()); }); assertThat(indexExists("copy-logs-20200101"), is(true)); assertThat( getIndexSettingsAsMap("copy-logs-20200101"), hasEntry("index.number_of_replicas", Integer.toString(followerNumberOfReplicas))); + assertThat(indexExists("copy-logs-excluded"), is(false)); GetAutoFollowPatternRequest getAutoFollowPatternRequest = randomBoolean() ? new GetAutoFollowPatternRequest("pattern1") : new GetAutoFollowPatternRequest(); @@ -279,6 +289,7 @@ public class CCRIT extends ESRestHighLevelClientTestCase { assertThat(pattern, notNullValue()); assertThat(pattern.getRemoteCluster(), equalTo(putAutoFollowPatternRequest.getRemoteCluster())); assertThat(pattern.getLeaderIndexPatterns(), equalTo(putAutoFollowPatternRequest.getLeaderIndexPatterns())); + assertThat(pattern.getLeaderIndexExclusionPatterns(), equalTo(putAutoFollowPatternRequest.getLeaderIndexExclusionPatterns())); assertThat(pattern.getFollowIndexNamePattern(), equalTo(putAutoFollowPatternRequest.getFollowIndexNamePattern())); assertThat(pattern.getSettings(), equalTo(autoFollowerPatternSettings)); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/CcrRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/CcrRequestConvertersTests.java index 970cd4b03b99..f0ea0b58c255 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/CcrRequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/CcrRequestConvertersTests.java @@ -101,7 +101,10 @@ public class CcrRequestConvertersTests extends ESTestCase { public void testPutAutofollowPattern() throws Exception { PutAutoFollowPatternRequest putAutoFollowPatternRequest = new PutAutoFollowPatternRequest(randomAlphaOfLength(4), - randomAlphaOfLength(4), Arrays.asList(generateRandomStringArray(4, 4, false))); + randomAlphaOfLength(4), + Arrays.asList(generateRandomStringArray(4, 4, false)), + Arrays.asList(generateRandomStringArray(4, 4, false)) + ); if (randomBoolean()) { putAutoFollowPatternRequest.setFollowIndexNamePattern(randomAlphaOfLength(4)); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ccr/GetAutoFollowPatternResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ccr/GetAutoFollowPatternResponseTests.java index 23e52a8c32ed..abd785841f0a 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ccr/GetAutoFollowPatternResponseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ccr/GetAutoFollowPatternResponseTests.java @@ -39,6 +39,7 @@ public class GetAutoFollowPatternResponseTests extends AbstractResponseTestCase< for (int i = 0; i < numPatterns; i++) { String remoteCluster = randomAlphaOfLength(4); List leaderIndexPatterns = Collections.singletonList(randomAlphaOfLength(4)); + List leaderIndexExclusionsPatterns = randomList(0, randomIntBetween(1, 10), () -> randomAlphaOfLength(4)); String followIndexNamePattern = randomAlphaOfLength(4); final Settings settings = Settings.builder().put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), randomIntBetween(0, 4)).build(); @@ -89,6 +90,7 @@ public class GetAutoFollowPatternResponseTests extends AbstractResponseTestCase< new AutoFollowMetadata.AutoFollowPattern( remoteCluster, leaderIndexPatterns, + leaderIndexExclusionsPatterns, followIndexNamePattern, settings, active, @@ -124,6 +126,7 @@ public class GetAutoFollowPatternResponseTests extends AbstractResponseTestCase< assertThat(serverPattern.getRemoteCluster(), equalTo(clientPattern.getRemoteCluster())); assertThat(serverPattern.getLeaderIndexPatterns(), equalTo(clientPattern.getLeaderIndexPatterns())); assertThat(serverPattern.getFollowIndexPattern(), equalTo(clientPattern.getFollowIndexNamePattern())); + assertThat(serverPattern.getLeaderIndexExclusionPatterns(), equalTo(clientPattern.getLeaderIndexExclusionPatterns())); assertThat(serverPattern.getSettings(), equalTo(clientPattern.getSettings())); assertThat(serverPattern.getMaxOutstandingReadRequests(), equalTo(clientPattern.getMaxOutstandingReadRequests())); assertThat(serverPattern.getMaxOutstandingWriteRequests(), equalTo(clientPattern.getMaxOutstandingWriteRequests())); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ccr/PutAutoFollowPatternRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ccr/PutAutoFollowPatternRequestTests.java index 466a12e94791..5f80c95728a1 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ccr/PutAutoFollowPatternRequestTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ccr/PutAutoFollowPatternRequestTests.java @@ -28,7 +28,10 @@ public class PutAutoFollowPatternRequestTests extends AbstractRequestTestCase< protected PutAutoFollowPatternRequest createClientTestInstance() { // Name isn't serialized, because it specified in url path, so no need to randomly generate it here. PutAutoFollowPatternRequest putAutoFollowPatternRequest = new PutAutoFollowPatternRequest("name", - randomAlphaOfLength(4), Arrays.asList(generateRandomStringArray(4, 4, false))); + randomAlphaOfLength(4), + Arrays.asList(generateRandomStringArray(4, 4, false)), + Arrays.asList(generateRandomStringArray(4, 4, false)) + ); if (randomBoolean()) { putAutoFollowPatternRequest.setFollowIndexNamePattern(randomAlphaOfLength(4)); } @@ -75,6 +78,7 @@ public class PutAutoFollowPatternRequestTests extends AbstractRequestTestCase< assertThat(serverInstance.getName(), equalTo(clientTestInstance.getName())); assertThat(serverInstance.getRemoteCluster(), equalTo(clientTestInstance.getRemoteCluster())); assertThat(serverInstance.getLeaderIndexPatterns(), equalTo(clientTestInstance.getLeaderIndexPatterns())); + assertThat(serverInstance.getLeaderIndexExclusionPatterns(), equalTo(clientTestInstance.getLeaderIndexExclusionPatterns())); assertThat(serverInstance.getFollowIndexNamePattern(), equalTo(clientTestInstance.getFollowIndexNamePattern())); assertFollowConfig(serverInstance.getParameters(), clientTestInstance); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CCRDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CCRDocumentationIT.java index d16f23006787..36d193f1a960 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CCRDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CCRDocumentationIT.java @@ -474,12 +474,13 @@ public class CCRDocumentationIT extends ESRestHighLevelClientTestCase { new PutAutoFollowPatternRequest( "my_pattern", // <1> "local", // <2> - Arrays.asList("logs-*", "metrics-*") // <3> + Arrays.asList("logs-*", "metrics-*"), // <3> + Arrays.asList("logs-excluded", "metrics-excluded") // <4> ); - request.setFollowIndexNamePattern("copy-{{leader_index}}"); // <4> + request.setFollowIndexNamePattern("copy-{{leader_index}}"); // <5> Settings settings = Settings.builder().put("index.number_of_replicas", 0L).build(); - request.setSettings(settings); // <5> + request.setSettings(settings); // <6> // end::ccr-put-auto-follow-pattern-request // tag::ccr-put-auto-follow-pattern-execute diff --git a/docs/java-rest/high-level/ccr/put_auto_follow_pattern.asciidoc b/docs/java-rest/high-level/ccr/put_auto_follow_pattern.asciidoc index a57b26738a4e..e9ebbe7c86fe 100644 --- a/docs/java-rest/high-level/ccr/put_auto_follow_pattern.asciidoc +++ b/docs/java-rest/high-level/ccr/put_auto_follow_pattern.asciidoc @@ -21,8 +21,9 @@ include-tagged::{doc-tests-file}[{api}-request] <1> The name of the auto follow pattern. <2> The name of the remote cluster. <3> The leader index patterns. -<4> The pattern used to create the follower index -<5> The settings overrides for the follower index +<4> The leader index exclusion patterns. +<5> The pattern used to create the follower index. +<6> The settings overrides for the follower index. [id="{upid}-{api}-response"] ==== Response diff --git a/docs/reference/ccr/apis/auto-follow/get-auto-follow-pattern.asciidoc b/docs/reference/ccr/apis/auto-follow/get-auto-follow-pattern.asciidoc index 5ea23782e196..1d4a6d97bdd9 100644 --- a/docs/reference/ccr/apis/auto-follow/get-auto-follow-pattern.asciidoc +++ b/docs/reference/ccr/apis/auto-follow/get-auto-follow-pattern.asciidoc @@ -22,6 +22,10 @@ PUT /_ccr/auto_follow/my_auto_follow_pattern [ "leader_index*" ], + "leader_index_exclusion_patterns": + [ + "leader_index_001" + ], "follow_index_pattern" : "{{leader_index}}-follower" } -------------------------------------------------- @@ -96,6 +100,10 @@ The API returns the following result: [ "leader_index*" ], + "leader_index_exclusion_patterns": + [ + "leader_index_001" + ], "follow_index_pattern" : "{{leader_index}}-follower" } } diff --git a/docs/reference/ccr/apis/auto-follow/put-auto-follow-pattern.asciidoc b/docs/reference/ccr/apis/auto-follow/put-auto-follow-pattern.asciidoc index 1ef668fae9f7..b3e9a570f2bf 100644 --- a/docs/reference/ccr/apis/auto-follow/put-auto-follow-pattern.asciidoc +++ b/docs/reference/ccr/apis/auto-follow/put-auto-follow-pattern.asciidoc @@ -20,6 +20,10 @@ PUT /_ccr/auto_follow/ [ "" ], + "leader_index_exclusion_patterns": + [ + "" + ], "follow_index_pattern" : "" } -------------------------------------------------- @@ -27,6 +31,7 @@ PUT /_ccr/auto_follow/ // TEST[s//auto_follow_pattern_name/] // TEST[s//remote_cluster/] // TEST[s//leader_index*/] +// TEST[s///] // TEST[s//{{leader_index}}-follower/] ////////////////////////// @@ -54,7 +59,10 @@ This API creates a new named collection of <> against the remote cluster specified in the request body. Newly created indices on the remote cluster matching any of the specified patterns will be automatically configured as follower -indices. +indices. Additionally, this API can be used to update existing +<>. Note that follower indices that were configured automatically +before updating an auto-follow pattern will remain unchanged even if they don't match against +the new patterns. [[ccr-put-auto-follow-pattern-path-parms]] ==== {api-path-parms-title} @@ -72,6 +80,11 @@ indices. (Optional, array) An array of simple index patterns to match against indices in the remote cluster specified by the `remote_cluster` field. +`leader_index_exclusion_patterns`:: + (Optional, array) An array of simple index patterns that can be used to exclude indices + from being auto-followed. Indices in the remote cluster whose names are matching one or + more `leader_index_patterns` and one or more `leader_index_exclusion_patterns` won't be followed. + `follow_index_pattern`:: (Optional, string) The name of follower index. The template `{{leader_index}}` can be used to derive the name of the follower index from the name of the diff --git a/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java b/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java index af3497a1cdcb..8bef41ec1545 100644 --- a/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java +++ b/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java @@ -116,6 +116,7 @@ public class AutoFollowIT extends ESCCRRestTestCase { } final String autoFollowPatternName = getTestName().toLowerCase(Locale.ROOT); + final String excludedIndex = "metrics-20210102"; try { int initialNumberOfSuccessfulFollowedIndices = getNumberOfSuccessfulFollowedIndices(); Request request = new Request("PUT", "/_ccr/auto_follow/" + autoFollowPatternName); @@ -128,6 +129,11 @@ public class AutoFollowIT extends ESCCRRestTestCase { bodyBuilder.value("metrics-*"); } bodyBuilder.endArray(); + bodyBuilder.startArray("leader_index_exclusion_patterns"); + { + bodyBuilder.value(excludedIndex); + } + bodyBuilder.endArray(); bodyBuilder.field("remote_cluster", "leader_cluster"); if (overrideNumberOfReplicas) { bodyBuilder.startObject("settings"); @@ -142,6 +148,12 @@ public class AutoFollowIT extends ESCCRRestTestCase { } assertOK(client().performRequest(request)); + try (RestClient leaderClient = buildLeaderClient()) { + request = new Request("PUT", "/" + excludedIndex); + request.setJsonEntity("{\"mappings\": {\"properties\": {\"field\": {\"type\": \"keyword\"}}}}"); + assertOK(leaderClient.performRequest(request)); + } + try (RestClient leaderClient = buildLeaderClient()) { request = new Request("PUT", "/metrics-20210101"); request.setJsonEntity("{\"mappings\": {\"properties\": {\"field\": {\"type\": \"keyword\"}}}}"); @@ -162,6 +174,7 @@ public class AutoFollowIT extends ESCCRRestTestCase { } else { assertThat(getIndexSettingsAsMap("metrics-20210101"), hasEntry("index.number_of_replicas", "1")); } + assertThat(indexExists(excludedIndex), is(false)); }); assertBusy(() -> { @@ -170,6 +183,7 @@ public class AutoFollowIT extends ESCCRRestTestCase { }, 30, TimeUnit.SECONDS); } finally { + cleanUpLeader(List.of("metrics-20210101", excludedIndex), List.of(), List.of()); cleanUpFollower(List.of("metrics-20210101"), List.of(), List.of(autoFollowPatternName)); } } diff --git a/x-pack/plugin/ccr/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/ccr/auto_follow.yml b/x-pack/plugin/ccr/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/ccr/auto_follow.yml index db54e2c38c7c..fb094f1b9974 100644 --- a/x-pack/plugin/ccr/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/ccr/auto_follow.yml +++ b/x-pack/plugin/ccr/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/ccr/auto_follow.yml @@ -52,7 +52,127 @@ catch: missing ccr.get_auto_follow_pattern: name: my_pattern +--- +"Test put and delete auto follow pattern without specifying exclusion patterns": + - skip: + version: " - 7.13.99" + reason: "auto-follow exclusion patterns is supported since 7.14.0" + - do: + cluster.state: {} + + - set: {master_node: master} + + - do: + nodes.info: {} + + - set: {nodes.$master.transport_address: local_ip} + + - do: + cluster.put_settings: + body: + transient: + cluster.remote.local.seeds: $local_ip + flat_settings: true + + - match: {transient: {cluster.remote.local.seeds: $local_ip}} + + - do: + ccr.put_auto_follow_pattern: + name: my_pattern + body: + remote_cluster: local + leader_index_patterns: ['logs-*'] + max_outstanding_read_requests: 2 + - is_true: acknowledged + + - do: + ccr.get_auto_follow_pattern: + name: my_pattern + - match: { patterns.0.name: 'my_pattern' } + - match: { patterns.0.pattern.remote_cluster: 'local' } + - match: { patterns.0.pattern.leader_index_patterns: ['logs-*'] } + - match: { patterns.0.pattern.leader_index_exclusion_patterns: [] } + - match: { patterns.0.pattern.max_outstanding_read_requests: 2 } + + - do: + ccr.get_auto_follow_pattern: {} + - match: { patterns.0.name: 'my_pattern' } + - match: { patterns.0.pattern.remote_cluster: 'local' } + - match: { patterns.0.pattern.leader_index_patterns: ['logs-*'] } + - match: { patterns.0.pattern.leader_index_exclusion_patterns: [] } + - match: { patterns.0.pattern.max_outstanding_read_requests: 2 } + + - do: + ccr.delete_auto_follow_pattern: + name: my_pattern + - is_true: acknowledged + + - do: + catch: missing + ccr.get_auto_follow_pattern: + name: my_pattern +--- +"Test put and delete auto follow pattern with exclusion patterns": + - skip: + version: " - 7.13.99" + reason: "auto-follow exclusion patterns is supported since 7.14.0" + + - do: + cluster.state: {} + + - set: {master_node: master} + + - do: + nodes.info: {} + + - set: {nodes.$master.transport_address: local_ip} + + - do: + cluster.put_settings: + body: + transient: + cluster.remote.local.seeds: $local_ip + flat_settings: true + + - match: {transient: {cluster.remote.local.seeds: $local_ip}} + + - do: + ccr.put_auto_follow_pattern: + name: my_pattern + body: + remote_cluster: local + leader_index_patterns: ['logs-*'] + leader_index_exclusion_patterns: ['logs-excluded'] + max_outstanding_read_requests: 2 + - is_true: acknowledged + + - do: + ccr.get_auto_follow_pattern: + name: my_pattern + - match: { patterns.0.name: 'my_pattern' } + - match: { patterns.0.pattern.remote_cluster: 'local' } + - match: { patterns.0.pattern.leader_index_patterns: ['logs-*'] } + - match: { patterns.0.pattern.leader_index_exclusion_patterns: ['logs-excluded'] } + - match: { patterns.0.pattern.max_outstanding_read_requests: 2 } + + - do: + ccr.get_auto_follow_pattern: {} + - match: { patterns.0.name: 'my_pattern' } + - match: { patterns.0.pattern.remote_cluster: 'local' } + - match: { patterns.0.pattern.leader_index_patterns: ['logs-*'] } + - match: { patterns.0.pattern.leader_index_exclusion_patterns: ['logs-excluded'] } + - match: { patterns.0.pattern.max_outstanding_read_requests: 2 } + + - do: + ccr.delete_auto_follow_pattern: + name: my_pattern + - is_true: acknowledged + + - do: + catch: missing + ccr.get_auto_follow_pattern: + name: my_pattern --- "Test pause and resume auto follow pattern": - skip: diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java index cbdcabfe8990..8bd26cfcac66 100644 --- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java @@ -514,13 +514,45 @@ public class AutoFollowIT extends CcrIntegTestCase { assertThat(followerClient().admin().indices().prepareStats("copy-*").get().getIndices().size(), equalTo(leaderIndices.get())); } + public void testAutoFollowExclusion() throws Exception { + Settings leaderIndexSettings = Settings.builder() + .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1) + .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0) + .build(); + + putAutoFollowPatterns("my-pattern1", new String[] {"logs-*"}, Collections.singletonList("logs-2018*")); + + createLeaderIndex("logs-201801", leaderIndexSettings); + AutoFollowStats autoFollowStats = getAutoFollowStats(); + assertThat(autoFollowStats.getNumberOfSuccessfulFollowIndices(), equalTo(0L)); + assertThat(autoFollowStats.getNumberOfFailedFollowIndices(), equalTo(0L)); + assertThat(autoFollowStats.getNumberOfFailedRemoteClusterStateRequests(), equalTo(0L)); + assertFalse(ESIntegTestCase.indexExists("copy-logs-201801", followerClient())); + + createLeaderIndex("logs-201701", leaderIndexSettings); + assertLongBusy(() -> { + AutoFollowStats autoFollowStatsResponse = getAutoFollowStats(); + assertThat(autoFollowStatsResponse.getNumberOfSuccessfulFollowIndices(), equalTo(1L)); + assertThat(autoFollowStatsResponse.getNumberOfFailedFollowIndices(), greaterThanOrEqualTo(0L)); + assertThat(autoFollowStatsResponse.getNumberOfFailedRemoteClusterStateRequests(), equalTo(0L)); + }); + assertTrue(ESIntegTestCase.indexExists("copy-logs-201701", followerClient())); + assertFalse(ESIntegTestCase.indexExists("copy-logs-201801", followerClient())); + } + private void putAutoFollowPatterns(String name, String[] patterns) { + putAutoFollowPatterns(name, patterns, Collections.emptyList()); + } + + private void putAutoFollowPatterns(String name, String[] patterns, List exclusionPatterns) { PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(); request.setName(name); request.setRemoteCluster("leader_cluster"); request.setLeaderIndexPatterns(Arrays.asList(patterns)); + request.setLeaderIndexExclusionPatterns(exclusionPatterns); // Need to set this, because following an index in the same cluster request.setFollowIndexNamePattern("copy-{{leader_index}}"); + assertTrue(followerClient().execute(PutAutoFollowPatternAction.INSTANCE, request).actionGet().isAcknowledged()); } diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrLicenseIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrLicenseIT.java index 9a7bedfe867e..6c180ded5009 100644 --- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrLicenseIT.java +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrLicenseIT.java @@ -168,6 +168,7 @@ public class CcrLicenseIT extends CcrSingleNodeTestCase { AutoFollowPattern autoFollowPattern = new AutoFollowPattern( "test_alias", Collections.singletonList("logs-*"), + Collections.emptyList(), null, Settings.EMPTY, true, diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportActivateAutoFollowPatternAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportActivateAutoFollowPatternAction.java index 0559f79e5003..a79c63250ef4 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportActivateAutoFollowPatternAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportActivateAutoFollowPatternAction.java @@ -77,6 +77,7 @@ public class TransportActivateAutoFollowPatternAction extends AcknowledgedTransp new AutoFollowMetadata.AutoFollowPattern( previousAutoFollowPattern.getRemoteCluster(), previousAutoFollowPattern.getLeaderIndexPatterns(), + previousAutoFollowPattern.getLeaderIndexExclusionPatterns(), previousAutoFollowPattern.getFollowIndexPattern(), previousAutoFollowPattern.getSettings(), request.isActive(), diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternAction.java index 05b4ed2caf1b..933ff36c4cf7 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternAction.java @@ -44,7 +44,6 @@ import java.util.function.Consumer; import java.util.stream.Collectors; public class TransportPutAutoFollowPatternAction extends AcknowledgedTransportMasterNodeAction { - private final Client client; private final CcrLicenseChecker ccrLicenseChecker; @@ -142,10 +141,18 @@ public class TransportPutAutoFollowPatternAction extends AcknowledgedTransportMa followedLeaderIndices.put(request.getName(), followedIndexUUIDs); // Mark existing leader indices as already auto followed: if (previousPattern != null) { - markExistingIndicesAsAutoFollowedForNewPatterns(request.getLeaderIndexPatterns(), remoteClusterState.metadata(), - previousPattern, followedIndexUUIDs); + markExistingIndicesAsAutoFollowedForNewPatterns(request.getLeaderIndexPatterns(), + request.getLeaderIndexExclusionPatterns(), + remoteClusterState.metadata(), + previousPattern, + followedIndexUUIDs + ); } else { - markExistingIndicesAsAutoFollowed(request.getLeaderIndexPatterns(), remoteClusterState.metadata(), followedIndexUUIDs); + markExistingIndicesAsAutoFollowed(request.getLeaderIndexPatterns(), + request.getLeaderIndexExclusionPatterns(), + remoteClusterState.metadata(), + followedIndexUUIDs + ); } if (filteredHeaders != null) { @@ -155,6 +162,7 @@ public class TransportPutAutoFollowPatternAction extends AcknowledgedTransportMa AutoFollowPattern autoFollowPattern = new AutoFollowPattern( request.getRemoteCluster(), request.getLeaderIndexPatterns(), + request.getLeaderIndexExclusionPatterns(), request.getFollowIndexNamePattern(), request.getSettings(), true, @@ -178,6 +186,7 @@ public class TransportPutAutoFollowPatternAction extends AcknowledgedTransportMa private static void markExistingIndicesAsAutoFollowedForNewPatterns( List leaderIndexPatterns, + List leaderIndexExclusionPatterns, Metadata leaderMetadata, AutoFollowPattern previousPattern, List followedIndexUUIDS) { @@ -186,17 +195,18 @@ public class TransportPutAutoFollowPatternAction extends AcknowledgedTransportMa .stream() .filter(p -> previousPattern.getLeaderIndexPatterns().contains(p) == false) .collect(Collectors.toList()); - markExistingIndicesAsAutoFollowed(newPatterns, leaderMetadata, followedIndexUUIDS); + markExistingIndicesAsAutoFollowed(newPatterns, leaderIndexExclusionPatterns, leaderMetadata, followedIndexUUIDS); } private static void markExistingIndicesAsAutoFollowed( List patterns, + List exclusionPatterns, Metadata leaderMetadata, List followedIndexUUIDS) { for (final IndexMetadata indexMetadata : leaderMetadata) { IndexAbstraction indexAbstraction = leaderMetadata.getIndicesLookup().get(indexMetadata.getIndex().getName()); - if (AutoFollowPattern.match(patterns, indexAbstraction)) { + if (AutoFollowPattern.match(patterns, exclusionPatterns, indexAbstraction)) { followedIndexUUIDS.add(indexMetadata.getIndexUUID()); } } diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowMetadataTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowMetadataTests.java index 9c057e4e69fb..c60f65875fbe 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowMetadataTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowMetadataTests.java @@ -43,9 +43,11 @@ public class AutoFollowMetadataTests extends AbstractSerializingTestCase> headers = new HashMap<>(numEntries); for (int i = 0; i < numEntries; i++) { List leaderPatterns = Arrays.asList(generateRandomStringArray(4, 4, false)); + List leaderExclusionPatterns = Arrays.asList(generateRandomStringArray(4, 4, false)); AutoFollowMetadata.AutoFollowPattern autoFollowPattern = new AutoFollowMetadata.AutoFollowPattern( randomAlphaOfLength(4), leaderPatterns, + leaderExclusionPatterns, randomAlphaOfLength(4), Settings.builder().put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), randomIntBetween(0, 4)).build(), true, diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRInfoTransportActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRInfoTransportActionTests.java index ecc58df18d42..2e0e03e267e5 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRInfoTransportActionTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRInfoTransportActionTests.java @@ -100,6 +100,7 @@ public class CCRInfoTransportActionTests extends ESTestCase { AutoFollowMetadata.AutoFollowPattern pattern = new AutoFollowMetadata.AutoFollowPattern( "remote_cluser", Collections.singletonList("logs" + i + "*"), + Collections.emptyList(), null, Settings.EMPTY, true, diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinatorTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinatorTests.java index 92466a38294a..b7cb113a5493 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinatorTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinatorTests.java @@ -97,6 +97,7 @@ public class AutoFollowCoordinatorTests extends ESTestCase { AutoFollowPattern autoFollowPattern = new AutoFollowPattern( "remote", Collections.singletonList("logs-*"), + Collections.emptyList(), null, Settings.EMPTY, true, @@ -184,6 +185,7 @@ public class AutoFollowCoordinatorTests extends ESTestCase { AutoFollowPattern autoFollowPattern = new AutoFollowPattern( "remote", Collections.singletonList("logs-*"), + Collections.emptyList(), null, Settings.EMPTY, true, @@ -269,6 +271,7 @@ public class AutoFollowCoordinatorTests extends ESTestCase { AutoFollowPattern autoFollowPattern = new AutoFollowPattern( "remote", Collections.singletonList("logs-*"), + Collections.emptyList(), null, Settings.EMPTY, true, @@ -336,6 +339,7 @@ public class AutoFollowCoordinatorTests extends ESTestCase { AutoFollowPattern autoFollowPattern = new AutoFollowPattern( "remote", Collections.singletonList("logs-*"), + Collections.emptyList(), null, Settings.EMPTY, true, @@ -408,6 +412,7 @@ public class AutoFollowCoordinatorTests extends ESTestCase { autoFollowPatterns.put("pattern_1", new AutoFollowPattern( remoteCluster, List.of("logs-*", "test-*"), + Collections.emptyList(), "copy-", Settings.EMPTY, false, @@ -425,7 +430,9 @@ public class AutoFollowCoordinatorTests extends ESTestCase { ); autoFollowPatterns.put("pattern_2", new AutoFollowPattern( remoteCluster, - List.of("users-*"), "copy-", + List.of("users-*"), + Collections.emptyList(), + "copy-", Settings.EMPTY, false, null, @@ -660,6 +667,7 @@ public class AutoFollowCoordinatorTests extends ESTestCase { AutoFollowPattern autoFollowPattern = new AutoFollowPattern( "remote", Collections.singletonList("logs-*"), + Collections.emptyList(), null, Settings.EMPTY, true, null, @@ -734,6 +742,7 @@ public class AutoFollowCoordinatorTests extends ESTestCase { final AutoFollowPattern autoFollowPattern = new AutoFollowPattern( "remote", Collections.singletonList("metrics-*"), + Collections.emptyList(), null, Settings.EMPTY, true, @@ -804,6 +813,7 @@ public class AutoFollowCoordinatorTests extends ESTestCase { final AutoFollowPattern inactiveAutoFollowPattern = new AutoFollowPattern( "remote", Collections.singletonList("metrics-*"), + Collections.emptyList(), null, Settings.EMPTY, false, @@ -830,6 +840,7 @@ public class AutoFollowCoordinatorTests extends ESTestCase { AutoFollowPattern autoFollowPattern = new AutoFollowPattern( "remote", Collections.singletonList("*"), + Collections.emptyList(), null, Settings.EMPTY, true, @@ -885,6 +896,7 @@ public class AutoFollowCoordinatorTests extends ESTestCase { final AutoFollowPattern autoFollowPattern = new AutoFollowPattern( "remote", Collections.singletonList("*"), + Collections.emptyList(), null, Settings.EMPTY, true, @@ -1023,6 +1035,7 @@ public class AutoFollowCoordinatorTests extends ESTestCase { AutoFollowPattern autoFollowPattern = new AutoFollowPattern( "remote", Collections.singletonList("metrics-*"), + Collections.emptyList(), null, Settings.EMPTY, true, @@ -1042,6 +1055,7 @@ public class AutoFollowCoordinatorTests extends ESTestCase { autoFollowPattern = new AutoFollowPattern( "remote", Collections.singletonList("metrics-*"), + Collections.emptyList(), "eu-metrics-0", Settings.EMPTY, true, @@ -1061,6 +1075,7 @@ public class AutoFollowCoordinatorTests extends ESTestCase { autoFollowPattern = new AutoFollowPattern( "remote", Collections.singletonList("metrics-*"), + Collections.emptyList(), "eu-{{leader_index}}", Settings.EMPTY, true, @@ -1158,6 +1173,7 @@ public class AutoFollowCoordinatorTests extends ESTestCase { "pattern1", new AutoFollowPattern( "remote1", Collections.singletonList("logs-*"), + Collections.emptyList(), null, Settings.EMPTY, true, @@ -1178,6 +1194,7 @@ public class AutoFollowCoordinatorTests extends ESTestCase { new AutoFollowPattern( "remote2", Collections.singletonList("logs-*"), + Collections.emptyList(), null, Settings.EMPTY, true, @@ -1198,6 +1215,7 @@ public class AutoFollowCoordinatorTests extends ESTestCase { new AutoFollowPattern( "remote2", Collections.singletonList("metrics-*"), + Collections.emptyList(), null, Settings.EMPTY, true, @@ -1242,6 +1260,7 @@ public class AutoFollowCoordinatorTests extends ESTestCase { new AutoFollowPattern( "remote1", Collections.singletonList("metrics-*"), + Collections.emptyList(), null, Settings.EMPTY, true, @@ -1336,6 +1355,7 @@ public class AutoFollowCoordinatorTests extends ESTestCase { new AutoFollowPattern( "remote1", Collections.singletonList("logs-*"), + Collections.emptyList(), null, Settings.EMPTY, true, @@ -1355,6 +1375,7 @@ public class AutoFollowCoordinatorTests extends ESTestCase { "pattern2", new AutoFollowPattern("remote2", Collections.singletonList("logs-*"), + Collections.emptyList(), null, Settings.EMPTY, true, @@ -1374,6 +1395,7 @@ public class AutoFollowCoordinatorTests extends ESTestCase { "pattern3", new AutoFollowPattern("remote2", Collections.singletonList("metrics-*"), + Collections.emptyList(), null, Settings.EMPTY, true, @@ -1409,6 +1431,7 @@ public class AutoFollowCoordinatorTests extends ESTestCase { (name, pattern) -> new AutoFollowPattern( pattern.getRemoteCluster(), pattern.getLeaderIndexPatterns(), + pattern.getLeaderIndexExclusionPatterns(), pattern.getFollowIndexPattern(), Settings.EMPTY, false, @@ -1428,6 +1451,7 @@ public class AutoFollowCoordinatorTests extends ESTestCase { "pattern3", (name, pattern) -> new AutoFollowPattern(pattern.getRemoteCluster(), pattern.getLeaderIndexPatterns(), + pattern.getLeaderIndexExclusionPatterns(), pattern.getFollowIndexPattern(), Settings.EMPTY, false, @@ -1459,6 +1483,7 @@ public class AutoFollowCoordinatorTests extends ESTestCase { new AutoFollowPattern( "remote1", Collections.singletonList("metrics-*"), + Collections.emptyList(), null, Settings.EMPTY, true, @@ -1478,6 +1503,7 @@ public class AutoFollowCoordinatorTests extends ESTestCase { "pattern2", (name, pattern) -> new AutoFollowPattern(pattern.getRemoteCluster(), pattern.getLeaderIndexPatterns(), + pattern.getLeaderIndexExclusionPatterns(), pattern.getFollowIndexPattern(), Settings.EMPTY, false, @@ -1523,6 +1549,7 @@ public class AutoFollowCoordinatorTests extends ESTestCase { AutoFollowPattern autoFollowPattern = new AutoFollowPattern( "remote", Collections.singletonList("logs-*"), + Collections.emptyList(), null, Settings.EMPTY, true, @@ -1601,6 +1628,7 @@ public class AutoFollowCoordinatorTests extends ESTestCase { AutoFollowPattern autoFollowPattern = new AutoFollowPattern( "remote", Collections.singletonList("logs-*"), + Collections.emptyList(), null, Settings.EMPTY, true, @@ -1674,6 +1702,7 @@ public class AutoFollowCoordinatorTests extends ESTestCase { AutoFollowPattern autoFollowPattern = new AutoFollowPattern( "remote", Collections.singletonList("logs-*"), + Collections.emptyList(), null, Settings.EMPTY, true, @@ -1755,6 +1784,7 @@ public class AutoFollowCoordinatorTests extends ESTestCase { AutoFollowPattern autoFollowPattern = new AutoFollowPattern( "remote", Collections.singletonList("logs-*"), + Collections.emptyList(), null, Settings.EMPTY, true, @@ -1853,6 +1883,7 @@ public class AutoFollowCoordinatorTests extends ESTestCase { final AutoFollowPattern pattern = new AutoFollowPattern( "remote", List.of("*"), + Collections.emptyList(), "{}", Settings.EMPTY, true, @@ -1934,6 +1965,7 @@ public class AutoFollowCoordinatorTests extends ESTestCase { new AutoFollowPattern( "remote", List.of("docs-*"), + Collections.emptyList(), null, Settings.EMPTY, true, @@ -2019,6 +2051,109 @@ public class AutoFollowCoordinatorTests extends ESTestCase { } } + public void testExcludedPatternIndicesAreNotAutoFollowed() { + final Client client = mock(Client.class); + when(client.getRemoteClusterClient(anyString())).thenReturn(client); + + final String pattern = "pattern1"; + final ClusterState localState = ClusterState.builder(new ClusterName("local")) + .metadata(Metadata.builder() + .putCustom(AutoFollowMetadata.TYPE, + new AutoFollowMetadata( + Map.of( + pattern, + new AutoFollowPattern( + "remote", + List.of("docs-*"), + List.of("docs-excluded-*"), + null, + Settings.EMPTY, + true, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ) + ), + Map.of(pattern, List.of()), + Map.of(pattern, Map.of())))) + .build(); + + ClusterState remoteState = null; + final int nbLeaderIndices = randomIntBetween(0, 15); + for (int i = 0; i < nbLeaderIndices; i++) { + String indexName = "docs-" + i; + if (remoteState == null) { + remoteState = createRemoteClusterState(indexName, true); + } else { + remoteState = createRemoteClusterState(remoteState, indexName); + } + } + + final int nbLeaderExcludedIndices = randomIntBetween(1, 15); + for (int i = 0; i < nbLeaderExcludedIndices; i++) { + String indexName = "docs-excluded-" + i; + remoteState = createRemoteClusterState(remoteState, indexName); + } + + final ClusterState finalRemoteState = remoteState; + final AtomicReference lastModifiedClusterState = new AtomicReference<>(localState); + final List results = new ArrayList<>(); + final Set followedIndices = ConcurrentCollections.newConcurrentSet(); + final AutoFollower autoFollower = + new AutoFollower("remote", results::addAll, localClusterStateSupplier(localState), () -> 1L, Runnable::run) { + @Override + void getRemoteClusterState(String remoteCluster, + long metadataVersion, + BiConsumer handler) { + assertThat(remoteCluster, equalTo("remote")); + handler.accept(new ClusterStateResponse(new ClusterName("remote"), finalRemoteState, false), null); + } + + @Override + void createAndFollow(Map headers, + PutFollowAction.Request followRequest, + Runnable successHandler, + Consumer failureHandler) { + followedIndices.add(followRequest.getLeaderIndex()); + successHandler.run(); + } + + @Override + void updateAutoFollowMetadata(Function updateFunction, Consumer handler) { + lastModifiedClusterState.updateAndGet(updateFunction::apply); + handler.accept(null); + } + + @Override + void cleanFollowedRemoteIndices(ClusterState remoteClusterState, List patterns) { + // Ignore, to avoid invoking updateAutoFollowMetadata(...) twice + } + }; + autoFollower.start(); + + assertThat(results, notNullValue()); + assertThat(results.size(), equalTo(1)); + + AutoFollowMetadata autoFollowMetadata = lastModifiedClusterState.get().metadata().custom(AutoFollowMetadata.TYPE); + final List autoFollowedIndices = autoFollowMetadata.getFollowedLeaderIndexUUIDs().get(pattern); + assertThat(autoFollowedIndices.size(), equalTo(nbLeaderIndices)); + + for (ObjectObjectCursor index : remoteState.metadata().indices()) { + final Index remoteIndex = index.value.getIndex(); + boolean followed = remoteIndex.getName().startsWith("docs-excluded") == false; + assertThat(results.get(0).autoFollowExecutionResults.containsKey(index.value.getIndex()), is(followed)); + assertThat(followedIndices.contains(index.key), is(followed)); + assertThat(autoFollowedIndices.contains(remoteIndex.getUUID()), equalTo(followed)); + } + } + private static ClusterState createRemoteClusterState(String indexName, boolean enableSoftDeletes) { return createRemoteClusterState(indexName, enableSoftDeletes, 0L); } diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/GetAutoFollowPatternResponseTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/GetAutoFollowPatternResponseTests.java index 030955725e31..329e33c5584e 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/GetAutoFollowPatternResponseTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/GetAutoFollowPatternResponseTests.java @@ -35,6 +35,7 @@ public class GetAutoFollowPatternResponseTests extends AbstractWireSerializingTe AutoFollowPattern autoFollowPattern = new AutoFollowPattern( "remote", Collections.singletonList(randomAlphaOfLength(4)), + Collections.singletonList(randomAlphaOfLength(4)), randomAlphaOfLength(4), Settings.builder().put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), randomIntBetween(0, 4)).build(), true, diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutAutoFollowPatternRequestTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutAutoFollowPatternRequestTests.java index 3572ed53c8c2..340dcd651bf9 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutAutoFollowPatternRequestTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutAutoFollowPatternRequestTests.java @@ -49,6 +49,11 @@ public class PutAutoFollowPatternRequestTests extends AbstractSerializingTestCas if (randomBoolean()) { request.setFollowIndexNamePattern(randomAlphaOfLength(4)); } + if (randomBoolean()) { + request.setLeaderIndexExclusionPatterns( + Arrays.asList(generateRandomStringArray(randomIntBetween(1, 10), randomIntBetween(1, 20), false)) + ); + } ResumeFollowActionRequestTests.generateFollowParameters(request.getParameters()); return request; } @@ -63,6 +68,11 @@ public class PutAutoFollowPatternRequestTests extends AbstractSerializingTestCas if (randomBoolean()) { request.setFollowIndexNamePattern(randomAlphaOfLength(4)); } + if (randomBoolean()) { + request.setLeaderIndexExclusionPatterns( + Arrays.asList(generateRandomStringArray(randomIntBetween(1, 10), randomIntBetween(1, 20), false)) + ); + } ResumeFollowActionRequestTests.generateFollowParameters(request.getParameters()); return request; } diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportActivateAutoFollowPatternActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportActivateAutoFollowPatternActionTests.java index 26209a170bd6..557f0a8bd4ed 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportActivateAutoFollowPatternActionTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportActivateAutoFollowPatternActionTests.java @@ -20,6 +20,7 @@ import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata; import org.elasticsearch.xpack.core.ccr.action.ActivateAutoFollowPatternAction.Request; import java.util.Arrays; +import java.util.Collections; import static java.util.Collections.singletonMap; import static org.hamcrest.Matchers.equalTo; @@ -83,6 +84,7 @@ public class TransportActivateAutoFollowPatternActionTests extends ESTestCase { private static AutoFollowMetadata.AutoFollowPattern randomAutoFollowPattern() { return new AutoFollowMetadata.AutoFollowPattern(randomAlphaOfLength(5), randomSubsetOf(Arrays.asList("test-*", "user-*", "logs-*", "failures-*")), + Collections.emptyList(), randomFrom("{{leader_index}}", "{{leader_index}}-follower", "test"), Settings.builder().put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), randomIntBetween(0, 4)).build(), randomBoolean(), diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportDeleteAutoFollowPatternActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportDeleteAutoFollowPatternActionTests.java index 3c369eda3e38..3a6c9b467fb4 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportDeleteAutoFollowPatternActionTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportDeleteAutoFollowPatternActionTests.java @@ -39,6 +39,7 @@ public class TransportDeleteAutoFollowPatternActionTests extends ESTestCase { new AutoFollowPattern( "eu_cluster", existingPatterns, + Collections.emptyList(), null, Settings.EMPTY, true, null, @@ -67,6 +68,7 @@ public class TransportDeleteAutoFollowPatternActionTests extends ESTestCase { new AutoFollowPattern( "asia_cluster", existingPatterns, + Collections.emptyList(), null, Settings.EMPTY, true, @@ -118,6 +120,7 @@ public class TransportDeleteAutoFollowPatternActionTests extends ESTestCase { new AutoFollowPattern( "eu_cluster", existingPatterns, + Collections.emptyList(), null, Settings.EMPTY, true, diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportGetAutoFollowPatternActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportGetAutoFollowPatternActionTests.java index b8d9ea944b0d..04d700704fe0 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportGetAutoFollowPatternActionTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportGetAutoFollowPatternActionTests.java @@ -30,6 +30,7 @@ public class TransportGetAutoFollowPatternActionTests extends ESTestCase { new AutoFollowPattern( "test_alias1", Collections.singletonList("index-*"), + Collections.emptyList(), null, Settings.EMPTY, true, @@ -50,6 +51,7 @@ public class TransportGetAutoFollowPatternActionTests extends ESTestCase { new AutoFollowPattern( "test_alias1", Collections.singletonList("index-*"), + Collections.emptyList(), null, Settings.EMPTY, true, diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternActionTests.java index b1d7aea4b30d..2433f4fdcc9a 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternActionTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternActionTests.java @@ -35,6 +35,10 @@ public class TransportPutAutoFollowPatternActionTests extends ESTestCase { request.setName("name1"); request.setRemoteCluster("eu_cluster"); request.setLeaderIndexPatterns(Collections.singletonList("logs-*")); + final boolean withExclusionPatterns = randomBoolean(); + if (withExclusionPatterns) { + request.setLeaderIndexExclusionPatterns(Collections.singletonList("logs-excluded-*")); + } final int numberOfReplicas = randomIntBetween(0, 4); request.setSettings(Settings.builder().put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), numberOfReplicas).build()); @@ -53,6 +57,13 @@ public class TransportPutAutoFollowPatternActionTests extends ESTestCase { assertThat(autoFollowMetadata.getPatterns().get("name1").getRemoteCluster(), equalTo("eu_cluster")); assertThat(autoFollowMetadata.getPatterns().get("name1").getLeaderIndexPatterns().size(), equalTo(1)); assertThat(autoFollowMetadata.getPatterns().get("name1").getLeaderIndexPatterns().get(0), equalTo("logs-*")); + final List exclusionPatterns = autoFollowMetadata.getPatterns().get("name1").getLeaderIndexExclusionPatterns(); + if (withExclusionPatterns) { + assertThat(exclusionPatterns.size(), equalTo(1)); + assertThat(exclusionPatterns.get(0), equalTo("logs-excluded-*")); + } else { + assertThat(exclusionPatterns.size(), equalTo(0)); + } assertThat( autoFollowMetadata.getPatterns().get("name1").getSettings().keySet(), hasItem(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey()) @@ -69,6 +80,10 @@ public class TransportPutAutoFollowPatternActionTests extends ESTestCase { request.setName("name1"); request.setRemoteCluster("eu_cluster"); request.setLeaderIndexPatterns(Collections.singletonList("logs-*")); + boolean withExclusionPatterns = randomBoolean(); + if (withExclusionPatterns) { + request.setLeaderIndexExclusionPatterns(Collections.singletonList("logs-excluded-*")); + } ClusterState localState = ClusterState.builder(new ClusterName("us_cluster")) .metadata(Metadata.builder()) @@ -76,6 +91,7 @@ public class TransportPutAutoFollowPatternActionTests extends ESTestCase { int numLeaderIndices = randomIntBetween(1, 8); int numMatchingLeaderIndices = randomIntBetween(1, 8); + int numExcludedLeaderIndices = randomIntBetween(1, 8); Metadata.Builder mdBuilder = Metadata.builder(); for (int i = 0; i < numLeaderIndices; i++) { mdBuilder.put(IndexMetadata.builder("transactions-" + i) @@ -89,6 +105,12 @@ public class TransportPutAutoFollowPatternActionTests extends ESTestCase { .numberOfShards(1) .numberOfReplicas(0)); } + for (int i = 0; i < numExcludedLeaderIndices; i++) { + mdBuilder.put(IndexMetadata.builder("logs-excluded-" + i) + .settings(settings(Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(0)); + } ClusterState remoteState = ClusterState.builder(new ClusterName("eu_cluster")) .metadata(mdBuilder) @@ -101,8 +123,18 @@ public class TransportPutAutoFollowPatternActionTests extends ESTestCase { assertThat(autoFollowMetadata.getPatterns().get("name1").getRemoteCluster(), equalTo("eu_cluster")); assertThat(autoFollowMetadata.getPatterns().get("name1").getLeaderIndexPatterns().size(), equalTo(1)); assertThat(autoFollowMetadata.getPatterns().get("name1").getLeaderIndexPatterns().get(0), equalTo("logs-*")); + final List exclusionPatterns = autoFollowMetadata.getPatterns().get("name1").getLeaderIndexExclusionPatterns(); + if (withExclusionPatterns) { + assertThat(exclusionPatterns.size(), equalTo(1)); + assertThat(exclusionPatterns.get(0), equalTo("logs-excluded-*")); + } else { + assertThat(exclusionPatterns.size(), equalTo(0)); + } + + final int expectedAutoFollowIndexCount = + withExclusionPatterns ? numMatchingLeaderIndices : numMatchingLeaderIndices + numExcludedLeaderIndices; assertThat(autoFollowMetadata.getFollowedLeaderIndexUUIDs().size(), equalTo(1)); - assertThat(autoFollowMetadata.getFollowedLeaderIndexUUIDs().get("name1").size(), equalTo(numMatchingLeaderIndices)); + assertThat(autoFollowMetadata.getFollowedLeaderIndexUUIDs().get("name1").size(), equalTo(expectedAutoFollowIndexCount)); } public void testInnerPut_existingLeaderIndicesAndAutoFollowMetadata() { @@ -110,6 +142,10 @@ public class TransportPutAutoFollowPatternActionTests extends ESTestCase { request.setName("name1"); request.setRemoteCluster("eu_cluster"); request.setLeaderIndexPatterns(Arrays.asList("logs-*", "transactions-*")); + boolean withExclusionPatterns = randomBoolean(); + if (withExclusionPatterns) { + request.setLeaderIndexExclusionPatterns(Collections.singletonList("logs-excluded-*")); + } Map existingAutoFollowPatterns = new HashMap<>(); List existingPatterns = new ArrayList<>(); @@ -119,6 +155,7 @@ public class TransportPutAutoFollowPatternActionTests extends ESTestCase { new AutoFollowPattern( "eu_cluster", existingPatterns, + Collections.emptyList(), null, Settings.EMPTY, true, @@ -152,6 +189,13 @@ public class TransportPutAutoFollowPatternActionTests extends ESTestCase { .numberOfShards(1) .numberOfReplicas(0)); } + int numExcludedLeaderIndices = randomIntBetween(1, 8); + for (int i = 0; i < numExcludedLeaderIndices; i++) { + mdBuilder.put(IndexMetadata.builder("logs-excluded-" + i) + .settings(settings(Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(0)); + } ClusterState remoteState = ClusterState.builder(new ClusterName("eu_cluster")) .metadata(mdBuilder) @@ -165,10 +209,18 @@ public class TransportPutAutoFollowPatternActionTests extends ESTestCase { assertThat(autoFollowMetadata.getPatterns().get("name1").getLeaderIndexPatterns().size(), equalTo(2)); assertThat(autoFollowMetadata.getPatterns().get("name1").getLeaderIndexPatterns().get(0), equalTo("logs-*")); assertThat(autoFollowMetadata.getPatterns().get("name1").getLeaderIndexPatterns().get(1), equalTo("transactions-*")); + final List exclusionPatterns = autoFollowMetadata.getPatterns().get("name1").getLeaderIndexExclusionPatterns(); + if (withExclusionPatterns) { + assertThat(exclusionPatterns.size(), equalTo(1)); + assertThat(exclusionPatterns.get(0), equalTo("logs-excluded-*")); + } else { + assertThat(exclusionPatterns.size(), equalTo(0)); + } + final int expectedAutoFollowIndexCount = + withExclusionPatterns ? numLeaderIndices : numLeaderIndices + numExcludedLeaderIndices; assertThat(autoFollowMetadata.getFollowedLeaderIndexUUIDs().size(), equalTo(1)); - assertThat(autoFollowMetadata.getFollowedLeaderIndexUUIDs().get("name1").size(), equalTo(numLeaderIndices + 1)); + assertThat(autoFollowMetadata.getFollowedLeaderIndexUUIDs().get("name1").size(), equalTo(expectedAutoFollowIndexCount + 1)); assertThat(autoFollowMetadata.getHeaders().size(), equalTo(1)); assertThat(autoFollowMetadata.getHeaders().get("name1"), notNullValue()); } - } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/AutoFollowMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/AutoFollowMetadata.java index 495cfebb8287..b7ace5f6afc7 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/AutoFollowMetadata.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/AutoFollowMetadata.java @@ -181,6 +181,7 @@ public class AutoFollowMetadata extends AbstractNamedDiffable i public static final ParseField ACTIVE = new ParseField("active"); public static final ParseField REMOTE_CLUSTER_FIELD = new ParseField("remote_cluster"); public static final ParseField LEADER_PATTERNS_FIELD = new ParseField("leader_index_patterns"); + public static final ParseField LEADER_EXCLUSION_PATTERNS_FIELD = new ParseField("leader_index_exclusion_patterns"); public static final ParseField FOLLOW_PATTERN_FIELD = new ParseField("follow_index_pattern"); public static final ParseField SETTINGS_FIELD = new ParseField("settings"); @@ -190,24 +191,26 @@ public class AutoFollowMetadata extends AbstractNamedDiffable i args -> new AutoFollowPattern( (String) args[0], (List) args[1], - (String) args[2], - args[3] == null ? Settings.EMPTY : (Settings) args[3], - args[4] == null || (boolean) args[4], - (Integer) args[5], + args[2] == null ? Collections.emptyList() : (List) args[2], + (String) args[3], + args[4] == null ? Settings.EMPTY : (Settings) args[4], + args[5] == null || (boolean) args[5], (Integer) args[6], (Integer) args[7], (Integer) args[8], - (ByteSizeValue) args[9], + (Integer) args[9], (ByteSizeValue) args[10], - (Integer) args[11], - (ByteSizeValue) args[12], - (TimeValue) args[13], - (TimeValue) args[14]) + (ByteSizeValue) args[11], + (Integer) args[12], + (ByteSizeValue) args[13], + (TimeValue) args[14], + (TimeValue) args[15]) ); static { PARSER.declareString(ConstructingObjectParser.constructorArg(), REMOTE_CLUSTER_FIELD); PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), LEADER_PATTERNS_FIELD); + PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), LEADER_EXCLUSION_PATTERNS_FIELD); PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), FOLLOW_PATTERN_FIELD); PARSER.declareObject( ConstructingObjectParser.optionalConstructorArg(), @@ -220,6 +223,7 @@ public class AutoFollowMetadata extends AbstractNamedDiffable i private final String remoteCluster; private final List leaderIndexPatterns; + private final List leaderIndexExclusionPatterns; private final String followIndexPattern; private final Settings settings; private final boolean active; @@ -227,6 +231,7 @@ public class AutoFollowMetadata extends AbstractNamedDiffable i public AutoFollowPattern( String remoteCluster, List leaderIndexPatterns, + List leaderIndexExclusionPatterns, String followIndexPattern, Settings settings, boolean active, @@ -245,6 +250,7 @@ public class AutoFollowMetadata extends AbstractNamedDiffable i maxReadRequestSize, maxWriteRequestSize, maxWriteBufferCount, maxWriteBufferSize, maxRetryDelay, pollTimeout); this.remoteCluster = remoteCluster; this.leaderIndexPatterns = leaderIndexPatterns; + this.leaderIndexExclusionPatterns = Objects.requireNonNull(leaderIndexExclusionPatterns); this.followIndexPattern = followIndexPattern; this.settings = Objects.requireNonNull(settings); this.active = active; @@ -263,8 +269,11 @@ public class AutoFollowMetadata extends AbstractNamedDiffable i return new AutoFollowPattern(remoteCluster, leaderIndexPatterns, followIndexPattern, settings, in); } - private AutoFollowPattern(String remoteCluster, List leaderIndexPatterns, - String followIndexPattern, Settings settings, StreamInput in) throws IOException { + private AutoFollowPattern(String remoteCluster, + List leaderIndexPatterns, + String followIndexPattern, + Settings settings, + StreamInput in) throws IOException { super(in); this.remoteCluster = remoteCluster; this.leaderIndexPatterns = leaderIndexPatterns; @@ -275,18 +284,27 @@ public class AutoFollowMetadata extends AbstractNamedDiffable i } else { this.active = true; } + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + this.leaderIndexExclusionPatterns = in.readStringList(); + } else { + this.leaderIndexExclusionPatterns = Collections.emptyList(); + } } public boolean match(IndexAbstraction indexAbstraction) { - return match(leaderIndexPatterns, indexAbstraction); + return match(leaderIndexPatterns, leaderIndexExclusionPatterns, indexAbstraction); } - public static boolean match(List leaderIndexPatterns, IndexAbstraction indexAbstraction) { - boolean matches = Regex.simpleMatch(leaderIndexPatterns, indexAbstraction.getName()); + public static boolean match(List leaderIndexPatterns, + List leaderIndexExclusionPatterns, + IndexAbstraction indexAbstraction) { + boolean matches = Regex.simpleMatch(leaderIndexExclusionPatterns, indexAbstraction.getName()) == false && + Regex.simpleMatch(leaderIndexPatterns, indexAbstraction.getName()); if (matches) { return true; } else { return indexAbstraction.getParentDataStream() != null && + Regex.simpleMatch(leaderIndexExclusionPatterns, indexAbstraction.getParentDataStream().getName()) == false && Regex.simpleMatch(leaderIndexPatterns, indexAbstraction.getParentDataStream().getName()); } } @@ -299,6 +317,10 @@ public class AutoFollowMetadata extends AbstractNamedDiffable i return leaderIndexPatterns; } + public List getLeaderIndexExclusionPatterns() { + return leaderIndexExclusionPatterns; + } + public String getFollowIndexPattern() { return followIndexPattern; } @@ -323,6 +345,9 @@ public class AutoFollowMetadata extends AbstractNamedDiffable i if (out.getVersion().onOrAfter(Version.V_7_5_0)) { out.writeBoolean(active); } + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + out.writeStringCollection(leaderIndexExclusionPatterns); + } } @Override @@ -330,6 +355,7 @@ public class AutoFollowMetadata extends AbstractNamedDiffable i builder.field(ACTIVE.getPreferredName(), active); builder.field(REMOTE_CLUSTER_FIELD.getPreferredName(), remoteCluster); builder.array(LEADER_PATTERNS_FIELD.getPreferredName(), leaderIndexPatterns.toArray(new String[0])); + builder.array(LEADER_EXCLUSION_PATTERNS_FIELD.getPreferredName(), leaderIndexExclusionPatterns.toArray(new String[0])); if (followIndexPattern != null) { builder.field(FOLLOW_PATTERN_FIELD.getPreferredName(), followIndexPattern); } @@ -353,13 +379,21 @@ public class AutoFollowMetadata extends AbstractNamedDiffable i return active == pattern.active && remoteCluster.equals(pattern.remoteCluster) && leaderIndexPatterns.equals(pattern.leaderIndexPatterns) && + leaderIndexExclusionPatterns.equals(pattern.leaderIndexExclusionPatterns) && followIndexPattern.equals(pattern.followIndexPattern) && settings.equals(pattern.settings); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), remoteCluster, leaderIndexPatterns, followIndexPattern, settings, active); + return Objects.hash(super.hashCode(), + remoteCluster, + leaderIndexPatterns, + leaderIndexExclusionPatterns, + followIndexPattern, + settings, + active + ); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutAutoFollowPatternAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutAutoFollowPatternAction.java index df1266d67448..e5b154d72552 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutAutoFollowPatternAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutAutoFollowPatternAction.java @@ -22,6 +22,7 @@ import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata.AutoFollowPattern; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.Collections; import java.util.List; import java.util.Objects; @@ -52,6 +53,10 @@ public class PutAutoFollowPatternAction extends ActionType PARSER.declareStringArray((params, value) -> params.leaderIndexPatterns = value, AutoFollowPattern.LEADER_PATTERNS_FIELD); PARSER.declareString((params, value) -> params.followIndexNamePattern = value, AutoFollowPattern.FOLLOW_PATTERN_FIELD); PARSER.declareObject((params, value) -> params.settings = value, (p, c) -> Settings.fromXContent(p), SETTINGS_FIELD); + PARSER.declareStringArray( + (params, value) -> params.leaderIndexExclusionPatterns = value, + AutoFollowPattern.LEADER_EXCLUSION_PATTERNS_FIELD + ); FollowParameters.initParser(PARSER); } @@ -62,6 +67,9 @@ public class PutAutoFollowPatternAction extends ActionType request.setRemoteCluster(parameters.remoteCluster); request.setLeaderIndexPatterns(parameters.leaderIndexPatterns); request.setFollowIndexNamePattern(parameters.followIndexNamePattern); + if (parameters.leaderIndexExclusionPatterns != null) { + request.setLeaderIndexExclusionPatterns(parameters.leaderIndexExclusionPatterns); + } request.setSettings(parameters.settings); request.setParameters(parameters); return request; @@ -73,6 +81,7 @@ public class PutAutoFollowPatternAction extends ActionType private String followIndexNamePattern; private Settings settings = Settings.EMPTY; private FollowParameters parameters = new FollowParameters(); + private List leaderIndexExclusionPatterns = Collections.emptyList(); public Request() { } @@ -131,6 +140,14 @@ public class PutAutoFollowPatternAction extends ActionType this.leaderIndexPatterns = leaderIndexPatterns; } + public List getLeaderIndexExclusionPatterns() { + return leaderIndexExclusionPatterns; + } + + public void setLeaderIndexExclusionPatterns(List leaderIndexExclusionPatterns) { + this.leaderIndexExclusionPatterns = leaderIndexExclusionPatterns; + } + public String getFollowIndexNamePattern() { return followIndexNamePattern; } @@ -165,6 +182,9 @@ public class PutAutoFollowPatternAction extends ActionType settings = Settings.readSettingsFromStream(in); } parameters = new FollowParameters(in); + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + leaderIndexExclusionPatterns = in.readStringList(); + } } @Override @@ -178,6 +198,9 @@ public class PutAutoFollowPatternAction extends ActionType Settings.writeSettingsToStream(settings, out); } parameters.writeTo(out); + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + out.writeStringCollection(leaderIndexExclusionPatterns); + } } @Override @@ -186,6 +209,7 @@ public class PutAutoFollowPatternAction extends ActionType { builder.field(REMOTE_CLUSTER_FIELD.getPreferredName(), remoteCluster); builder.field(AutoFollowPattern.LEADER_PATTERNS_FIELD.getPreferredName(), leaderIndexPatterns); + builder.field(AutoFollowPattern.LEADER_EXCLUSION_PATTERNS_FIELD.getPreferredName(), leaderIndexExclusionPatterns); if (followIndexNamePattern != null) { builder.field(AutoFollowPattern.FOLLOW_PATTERN_FIELD.getPreferredName(), followIndexNamePattern); } @@ -210,13 +234,14 @@ public class PutAutoFollowPatternAction extends ActionType return Objects.equals(name, request.name) && Objects.equals(remoteCluster, request.remoteCluster) && Objects.equals(leaderIndexPatterns, request.leaderIndexPatterns) && + Objects.equals(leaderIndexExclusionPatterns, request.leaderIndexExclusionPatterns) && Objects.equals(followIndexNamePattern, request.followIndexNamePattern) && Objects.equals(parameters, request.parameters); } @Override public int hashCode() { - return Objects.hash(name, remoteCluster, leaderIndexPatterns, followIndexNamePattern, parameters); + return Objects.hash(name, remoteCluster, leaderIndexPatterns, leaderIndexExclusionPatterns, followIndexNamePattern, parameters); } // This class only exists for reuse of the FollowParameters class, see comment above the parser field. @@ -226,7 +251,7 @@ public class PutAutoFollowPatternAction extends ActionType private List leaderIndexPatterns; private String followIndexNamePattern; private Settings settings = Settings.EMPTY; - + private List leaderIndexExclusionPatterns; } }