Adding deprecation warnings for rank and sub_searches (#114854)

docs/changelog/114854.yaml

@@ -0,0 +1,10 @@
+pr: 114854
+summary: Adding deprecation warnings for rrf using rank and `sub_searches`
+area: Search
+type: deprecation
+issues: []
+deprecation:
+  title: Adding deprecation warnings for rrf using rank and `sub_searches`
+  area: REST API
+  details: Search API parameter `sub_searches` will no longer be a supported and will be removed in future releases. Similarly, `rrf` can only be used through the specified `retriever` and no longer though the `rank` parameter
+  impact: Requests specifying rrf through `rank` and/or `sub_searches` elements will be disallowed in a future version. Users should instead utilize the new `retriever` parameter.

modules/reindex/src/test/java/org/elasticsearch/reindex/BulkByScrollParallelizationHelperTests.java

@@ -15,8 +15,8 @@ import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
+import java.util.Collections;
 
-import static java.util.Collections.emptyList;
 import static org.elasticsearch.reindex.BulkByScrollParallelizationHelper.sliceIntoSubRequests;
 import static org.elasticsearch.search.RandomSearchRequestGenerator.randomSearchRequest;
 import static org.elasticsearch.search.RandomSearchRequestGenerator.randomSearchSourceBuilder;
@@ -24,7 +24,7 @@ import static org.elasticsearch.search.RandomSearchRequestGenerator.randomSearch
 public class BulkByScrollParallelizationHelperTests extends ESTestCase {
     public void testSliceIntoSubRequests() throws IOException {
         SearchRequest searchRequest = randomSearchRequest(
-            () -> randomSearchSourceBuilder(() -> null, () -> null, () -> null, () -> null, () -> emptyList(), () -> null, () -> null)
+            () -> randomSearchSourceBuilder(() -> null, () -> null, () -> null, Collections::emptyList, () -> null, () -> null)
         );
         if (searchRequest.source() != null) {
             // Clear the slice builder if there is one set. We can't call sliceIntoSubRequests if it is.

server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java

@@ -25,6 +25,7 @@ import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.Booleans;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.core.UpdateForV10;
 import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
@@ -98,10 +99,11 @@ public final class SearchSourceBuilder implements Writeable, ToXContentObject, R
     public static final ParseField TIMEOUT_FIELD = new ParseField("timeout");
     public static final ParseField TERMINATE_AFTER_FIELD = new ParseField("terminate_after");
     public static final ParseField QUERY_FIELD = new ParseField("query");
-    public static final ParseField SUB_SEARCHES_FIELD = new ParseField("sub_searches");
+    @UpdateForV10(owner = UpdateForV10.Owner.SEARCH_RELEVANCE) // remove [sub_searches] and [rank] support in 10.0
+    public static final ParseField SUB_SEARCHES_FIELD = new ParseField("sub_searches").withAllDeprecated("retriever");
+    public static final ParseField RANK_FIELD = new ParseField("rank").withAllDeprecated("retriever");
     public static final ParseField POST_FILTER_FIELD = new ParseField("post_filter");
     public static final ParseField KNN_FIELD = new ParseField("knn");
-    public static final ParseField RANK_FIELD = new ParseField("rank");
     public static final ParseField MIN_SCORE_FIELD = new ParseField("min_score");
     public static final ParseField VERSION_FIELD = new ParseField("version");
     public static final ParseField SEQ_NO_PRIMARY_TERM_FIELD = new ParseField("seq_no_primary_term");

server/src/test/java/org/elasticsearch/search/AbstractSearchTestCase.java

@@ -89,7 +89,6 @@ public abstract class AbstractSearchTestCase extends ESTestCase {
         return RandomSearchRequestGenerator.randomSearchSourceBuilder(
             HighlightBuilderTests::randomHighlighterBuilder,
             SuggestBuilderTests::randomSuggestBuilder,
-            TestRankBuilder::randomRankBuilder,
             QueryRescorerBuilderTests::randomRescoreBuilder,
             randomExtBuilders,
             CollapseBuilderTests::randomCollapseBuilder,

server/src/test/java/org/elasticsearch/search/vectors/KnnSearchRequestParserTests.java

@@ -74,7 +74,6 @@ public class KnnSearchRequestParserTests extends ESTestCase {
             () -> null,
             () -> null,
             () -> null,
-            () -> null,
             Collections::emptyList,
             () -> null,
             () -> null

test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java

@@ -23,13 +23,11 @@ import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
 import org.elasticsearch.search.builder.PointInTimeBuilder;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.elasticsearch.search.builder.SubSearchSourceBuilder;
 import org.elasticsearch.search.collapse.CollapseBuilder;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 import org.elasticsearch.search.fetch.subphase.FieldAndFormat;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
 import org.elasticsearch.search.internal.SearchContext;
-import org.elasticsearch.search.rank.RankBuilder;
 import org.elasticsearch.search.rescore.RescorerBuilder;
 import org.elasticsearch.search.searchafter.SearchAfterBuilder;
 import org.elasticsearch.search.slice.SliceBuilder;
@@ -122,7 +120,6 @@ public class RandomSearchRequestGenerator {
     public static SearchSourceBuilder randomSearchSourceBuilder(
         Supplier<HighlightBuilder> randomHighlightBuilder,
         Supplier<SuggestBuilder> randomSuggestBuilder,
-        Supplier<RankBuilder> rankContextBuilderSupplier,
         Supplier<RescorerBuilder<?>> randomRescoreBuilder,
         Supplier<List<SearchExtBuilder>> randomExtBuilders,
         Supplier<CollapseBuilder> randomCollapseBuilder,
@@ -250,17 +247,6 @@ public class RandomSearchRequestGenerator {
         }
         if (randomBoolean()) {
             builder.query(QueryBuilders.termQuery(randomAlphaOfLengthBetween(5, 20), randomAlphaOfLengthBetween(5, 20)));
-        } else if (randomBoolean()) {
-            builder.subSearches(
-                List.of(
-                    new SubSearchSourceBuilder(
-                        QueryBuilders.termQuery(randomAlphaOfLengthBetween(5, 20), randomAlphaOfLengthBetween(5, 20))
-                    ),
-                    new SubSearchSourceBuilder(
-                        QueryBuilders.termQuery(randomAlphaOfLengthBetween(5, 20), randomAlphaOfLengthBetween(5, 20))
-                    )
-                )
-            );
         }
         if (randomBoolean()) {
             builder.postFilter(QueryBuilders.termQuery(randomAlphaOfLengthBetween(5, 20), randomAlphaOfLengthBetween(5, 20)));
@@ -354,9 +340,6 @@ public class RandomSearchRequestGenerator {
         if (randomBoolean()) {
             builder.suggest(randomSuggestBuilder.get());
         }
-        if (randomBoolean()) {
-            builder.rankBuilder(rankContextBuilderSupplier.get());
-        }
         if (randomBoolean()) {
             int numRescores = randomIntBetween(1, 5);
             for (int i = 0; i < numRescores; i++) {

x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankBuilder.java

@@ -36,16 +36,17 @@ import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstr
 
 /**
  * The builder to support RRF. Adds user-defined parameters for window size and rank constant.
+ *
+ * @deprecated RRF support is provided through the retriever framework. Please use {@link RRFRetrieverBuilder instead}
  */
+@Deprecated
 public class RRFRankBuilder extends RankBuilder {
 
-    public static final int DEFAULT_RANK_CONSTANT = 60;
-
     public static final ParseField RANK_CONSTANT_FIELD = new ParseField("rank_constant");
 
     static final ConstructingObjectParser<RRFRankBuilder, Void> PARSER = new ConstructingObjectParser<>(RRFRankPlugin.NAME, args -> {
         int windowSize = args[0] == null ? DEFAULT_RANK_WINDOW_SIZE : (int) args[0];
-        int rankConstant = args[1] == null ? DEFAULT_RANK_CONSTANT : (int) args[1];
+        int rankConstant = args[1] == null ? RRFRetrieverBuilder.DEFAULT_RANK_CONSTANT : (int) args[1];
         return new RRFRankBuilder(windowSize, rankConstant);
     });
 

x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankDoc.java

@@ -19,7 +19,7 @@ import java.io.IOException;
 import java.util.Arrays;
 import java.util.Objects;
 
-import static org.elasticsearch.xpack.rank.rrf.RRFRankBuilder.DEFAULT_RANK_CONSTANT;
+import static org.elasticsearch.xpack.rank.rrf.RRFRetrieverBuilder.DEFAULT_RANK_CONSTANT;
 
 /**
  * {@code RRFRankDoc} supports additional ranking information

x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilder.java

@@ -12,6 +12,7 @@ import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.license.LicenseUtils;
+import org.elasticsearch.search.rank.RankBuilder;
 import org.elasticsearch.search.rank.RankDoc;
 import org.elasticsearch.search.retriever.CompoundRetrieverBuilder;
 import org.elasticsearch.search.retriever.RetrieverBuilder;
@@ -31,7 +32,6 @@ import java.util.Objects;
 
 import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
 import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
-import static org.elasticsearch.xpack.rank.rrf.RRFRankPlugin.NAME;
 
 /**
  * An rrf retriever is used to represent an rrf rank element, but
@@ -50,6 +50,7 @@ public final class RRFRetrieverBuilder extends CompoundRetrieverBuilder<RRFRetri
     public static final ParseField RANK_WINDOW_SIZE_FIELD = new ParseField("rank_window_size");
     public static final ParseField RANK_CONSTANT_FIELD = new ParseField("rank_constant");
 
+    public static final int DEFAULT_RANK_CONSTANT = 60;
     @SuppressWarnings("unchecked")
     static final ConstructingObjectParser<RRFRetrieverBuilder, RetrieverParserContext> PARSER = new ConstructingObjectParser<>(
         NAME,
@@ -57,8 +58,8 @@ public final class RRFRetrieverBuilder extends CompoundRetrieverBuilder<RRFRetri
         args -> {
             List<RetrieverBuilder> childRetrievers = (List<RetrieverBuilder>) args[0];
             List<RetrieverSource> innerRetrievers = childRetrievers.stream().map(r -> new RetrieverSource(r, null)).toList();
-            int rankWindowSize = args[1] == null ? RRFRankBuilder.DEFAULT_RANK_WINDOW_SIZE : (int) args[1];
+            int rankWindowSize = args[1] == null ? RankBuilder.DEFAULT_RANK_WINDOW_SIZE : (int) args[1];
-            int rankConstant = args[2] == null ? RRFRankBuilder.DEFAULT_RANK_CONSTANT : (int) args[2];
+            int rankConstant = args[2] == null ? DEFAULT_RANK_CONSTANT : (int) args[2];
             return new RRFRetrieverBuilder(innerRetrievers, rankWindowSize, rankConstant);
         }
     );

x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderParsingTests.java

@@ -41,7 +41,7 @@ public class RRFRetrieverBuilderParsingTests extends AbstractXContentTestCase<RR
         if (randomBoolean()) {
             rankWindowSize = randomIntBetween(1, 10000);
         }
-        int rankConstant = RRFRankBuilder.DEFAULT_RANK_CONSTANT;
+        int rankConstant = RRFRetrieverBuilder.DEFAULT_RANK_CONSTANT;
         if (randomBoolean()) {
             rankConstant = randomIntBetween(1, 1000000);
         }

x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/100_rank_rrf.yml

@@ -2,6 +2,8 @@ setup:
   - requires:
       cluster_features: "gte_v8.8.0"
       reason: 'rank added in 8.8'
+  - skip:
+      features: "warnings"
 
   - do:
       indices.create:
@@ -59,7 +61,14 @@ setup:
 ---
 "Simple rank with bm25 search and kNN search":
 
+  - requires:
+      cluster_features: ["gte_v8.16.0"]
+      reason: "deprecation added in 8.16"
+      test_runner_features: warnings
+
   - do:
+      warnings:
+        - "Deprecated field [rank] used, replaced by [retriever]"
       search:
         index: test
         body:
@@ -94,7 +103,15 @@ setup:
 ---
 "Simple rank with multiple bm25 sub searches":
 
+  - requires:
+      cluster_features: ["gte_v8.16.0"]
+      reason: "deprecation added in 8.16"
+      test_runner_features: warnings
+
   - do:
+      warnings:
+        - "Deprecated field [rank] used, replaced by [retriever]"
+        - Deprecated field [sub_searches] used, replaced by [retriever]
       search:
         index: test
         body:
@@ -135,7 +152,15 @@ setup:
 ---
 "Simple rank with multiple bm25 sub_searches and a knn search":
 
+  - requires:
+      cluster_features: ["gte_v8.16.0"]
+      reason: "deprecation added in 8.16"
+      test_runner_features: warnings
+
   - do:
+      warnings:
+        - "Deprecated field [rank] used, replaced by [retriever]"
+        - Deprecated field [sub_searches] used, replaced by [retriever]
       search:
         index: test
         body:

x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/150_rank_rrf_pagination.yml

@@ -63,7 +63,15 @@ setup:
 ---
 "Standard pagination within rank_window_size":
   # this test retrieves the same results from two queries, and applies a simple pagination skipping the first result
+  - requires:
+      cluster_features: [ "gte_v8.16.0" ]
+      reason: "deprecation added in 8.16"
+      test_runner_features: warnings
+
   - do:
+      warnings:
+        - "Deprecated field [rank] used, replaced by [retriever]"
+        - Deprecated field [sub_searches] used, replaced by [retriever]
       search:
         index: test
         body:
@@ -170,7 +178,15 @@ setup:
 ---
 "Standard pagination outside rank_window_size":
   # in this example, from starts *after* rank_window_size so, we expect 0 results to be returned
+  - requires:
+      cluster_features: [ "gte_v8.16.0" ]
+      reason: "deprecation added in 8.16"
+      test_runner_features: warnings
+
   - do:
+      warnings:
+        - "Deprecated field [rank] used, replaced by [retriever]"
+        - Deprecated field [sub_searches] used, replaced by [retriever]
       search:
         index: test
         body:
@@ -274,7 +290,15 @@ setup:
 ---
 "Standard pagination partially outside rank_window_size":
   # in this example we have that from starts *within* rank_window_size, but "from + size" goes over
+  - requires:
+      cluster_features: [ "gte_v8.16.0" ]
+      reason: "deprecation added in 8.16"
+      test_runner_features: warnings
+
   - do:
+      warnings:
+        - "Deprecated field [rank] used, replaced by [retriever]"
+        - Deprecated field [sub_searches] used, replaced by [retriever]
       search:
         index: test
         body:
@@ -384,7 +408,15 @@ setup:
   # queryA has a result set of [1, 2, 3, 4] and
   # queryB has a result set of [4, 3, 1, 2]
   # so for rank_constant=10, the expected order is [1, 4, 3, 2]
+  - requires:
+      cluster_features: ["gte_v8.16.0"]
+      reason: "deprecation added in 8.16"
+      test_runner_features: warnings
+
   - do:
+      warnings:
+        - "Deprecated field [rank] used, replaced by [retriever]"
+        - Deprecated field [sub_searches] used, replaced by [retriever]
       search:
         index: test
         body:
@@ -488,6 +520,9 @@ setup:
   - match: { hits.hits.1._id: "4" }
 
   - do:
+      warnings:
+        - "Deprecated field [rank] used, replaced by [retriever]"
+        - Deprecated field [sub_searches] used, replaced by [retriever]
       search:
         index: test
         body:
@@ -597,7 +632,15 @@ setup:
   # queryA has a result set of [5, 1] and
   # queryB has a result set of [4, 3, 1, 2]
   # so for rank_constant=10, the expected order is [1, 5, 4, 3, 2]
+  - requires:
+      cluster_features: ["gte_v8.16.0"]
+      reason: "deprecation added in 8.16"
+      test_runner_features: warnings
+
   - do:
+      warnings:
+        - "Deprecated field [rank] used, replaced by [retriever]"
+        - Deprecated field [sub_searches] used, replaced by [retriever]
       search:
         index: test
         body:
@@ -685,6 +728,9 @@ setup:
   - match: { hits.hits.1._id: "5" }
 
   - do:
+      warnings:
+        - "Deprecated field [rank] used, replaced by [retriever]"
+        - Deprecated field [sub_searches] used, replaced by [retriever]
       search:
         index: test
         body:
@@ -772,6 +818,9 @@ setup:
   - match: { hits.hits.1._id: "3" }
 
   - do:
+      warnings:
+        - "Deprecated field [rank] used, replaced by [retriever]"
+        - Deprecated field [sub_searches] used, replaced by [retriever]
       search:
         index: test
         body:
@@ -867,7 +916,15 @@ setup:
   # queryB has a result set of [4, 3]
   # so for rank_constant=10, the expected order is [5, 4, 1, 3],
   # and the rank_window_size-sized result set that we'd paginate over is [5, 4]
+  - requires:
+      cluster_features: ["gte_v8.16.0"]
+      reason: "deprecation added in 8.16"
+      test_runner_features: warnings
+
   - do:
+      warnings:
+        - "Deprecated field [rank] used, replaced by [retriever]"
+        - Deprecated field [sub_searches] used, replaced by [retriever]
       search:
         index: test
         body:
@@ -955,6 +1012,10 @@ setup:
   - match: { hits.hits.1._id: "4" }
 
   - do:
+      warnings:
+        - "Deprecated field [rank] used, replaced by [retriever]"
+        - Deprecated field [sub_searches] used, replaced by [retriever]
+
       search:
         index: test
         body:

x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/200_rank_rrf_script.yml

@@ -67,7 +67,14 @@ setup:
 ---
 "RRF using single knn and single BM25 with a scripted metric aggregation":
 
+  - requires:
+      cluster_features: ["gte_v8.16.0"]
+      reason: "deprecation added in 8.16"
+      test_runner_features: warnings
+
   - do:
+      warnings:
+        - "Deprecated field [rank] used, replaced by [retriever]"
       search:
         index: test
         body:
@@ -140,7 +147,14 @@ setup:
 ---
 "RRF using multi-knn only with a scripted metric aggregation":
 
+  - requires:
+      cluster_features: ["gte_v8.16.0"]
+      reason: "deprecation added in 8.16"
+      test_runner_features: warnings
+
   - do:
+      warnings:
+        - "Deprecated field [rank] used, replaced by [retriever]"
       search:
         index: test
         body:
@@ -195,7 +209,14 @@ setup:
 ---
 "RRF using multi-knn and single BM25 with a scripted metric aggregation":
 
+  - requires:
+      cluster_features: ["gte_v8.16.0"]
+      reason: "deprecation added in 8.16"
+      test_runner_features: warnings
+
   - do:
+      warnings:
+        - "Deprecated field [rank] used, replaced by [retriever]"
       search:
         index: test
         body:

x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/550_rrf_sub_searches_explain.yml

@@ -75,7 +75,14 @@ setup:
 ---
 "using a top level knn and query":
 
+  - requires:
+      cluster_features: ["gte_v8.16.0"]
+      reason: "deprecation added in 8.16"
+      test_runner_features: warnings
+
   - do:
+      warnings:
+        - "Deprecated field [rank] used, replaced by [retriever]"
       search:
         index: test
         body:
@@ -129,7 +136,15 @@ setup:
 ---
 "using sub_searches":
 
+  - requires:
+      cluster_features: [ "gte_v8.16.0" ]
+      reason: "deprecation added in 8.16"
+      test_runner_features: warnings
+
   - do:
+      warnings:
+        - "Deprecated field [rank] used, replaced by [retriever]"
+        - Deprecated field [sub_searches] used, replaced by [retriever]
       search:
         index: test
         body:
@@ -194,7 +209,14 @@ setup:
 ---
 "using named top level knn and query":
 
+  - requires:
+      cluster_features: ["gte_v8.16.0"]
+      reason: "deprecation added in 8.16"
+      test_runner_features: warnings
+
   - do:
+      warnings:
+        - "Deprecated field [rank] used, replaced by [retriever]"
       search:
         index: test
         body:
@@ -251,7 +273,15 @@ setup:
 ---
 "using named sub_searches":
 
+  - requires:
+      cluster_features: ["gte_v8.16.0"]
+      reason: "deprecation added in 8.16"
+      test_runner_features: warnings
+
   - do:
+      warnings:
+        - "Deprecated field [rank] used, replaced by [retriever]"
+        - Deprecated field [sub_searches] used, replaced by [retriever]
       search:
         index: test
         body:
@@ -320,7 +350,15 @@ setup:
 ---
 "using a mix of named and unnamed queries":
 
+  - requires:
+      cluster_features: ["gte_v8.16.0"]
+      reason: "deprecation added in 8.16"
+      test_runner_features: warnings
+
   - do:
+      warnings:
+        - "Deprecated field [rank] used, replaced by [retriever]"
+        - Deprecated field [sub_searches] used, replaced by [retriever]
       search:
         index: test
         body:

x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/600_rrf_retriever_profile.yml

@@ -172,7 +172,14 @@ setup:
 ---
 "using query and dfs knn search":
 
+  - requires:
+      cluster_features: ["gte_v8.16.0"]
+      reason: "deprecation added in 8.16"
+      test_runner_features: warnings
+
   - do:
+      warnings:
+        - "Deprecated field [rank] used, replaced by [retriever]"
       search:
         index: test
         body: