diff --git a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestRerankingServiceExtension.java b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestRerankingServiceExtension.java index 765c69e28a9a..8be2317a9ee6 100644 --- a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestRerankingServiceExtension.java +++ b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestRerankingServiceExtension.java @@ -36,6 +36,7 @@ import org.elasticsearch.xpack.core.inference.results.RankedDocsResults; import java.io.IOException; import java.util.ArrayList; +import java.util.Comparator; import java.util.EnumSet; import java.util.HashMap; import java.util.List; @@ -148,16 +149,28 @@ public class TestRerankingServiceExtension implements InferenceServiceExtension } private RankedDocsResults makeResults(List input) { - List results = new ArrayList<>(); int totalResults = input.size(); - float minScore = random.nextFloat(-1f, 1f); - float resultDiff = 0.2f; - for (int i = 0; i < input.size(); i++) { - results.add( - new RankedDocsResults.RankedDoc(totalResults - 1 - i, minScore + resultDiff * (totalResults - i), input.get(i)) - ); + try { + List results = new ArrayList<>(); + for (int i = 0; i < totalResults; i++) { + results.add(new RankedDocsResults.RankedDoc(i, Float.parseFloat(input.get(i)), input.get(i))); + } + return new RankedDocsResults(results.stream().sorted(Comparator.reverseOrder()).toList()); + } catch (NumberFormatException ex) { + List results = new ArrayList<>(); + float minScore = random.nextFloat(-1f, 1f); + float resultDiff = 0.2f; + for (int i = 0; i < input.size(); i++) { + results.add( + new RankedDocsResults.RankedDoc( + totalResults - 1 - i, + minScore + resultDiff * (totalResults - i), + input.get(totalResults - 1 - i) + ) + ); + } + return new RankedDocsResults(results); } - return new RankedDocsResults(results); } protected ServiceSettings getServiceSettingsFromMap(Map serviceSettingsMap) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java index e8fcb8dfe117..032817ce758a 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java @@ -27,6 +27,9 @@ public class InferenceFeatures implements FeatureSpecification { private static final NodeFeature SEMANTIC_TEXT_HIGHLIGHTER = new NodeFeature("semantic_text.highlighter"); private static final NodeFeature SEMANTIC_TEXT_HIGHLIGHTER_DEFAULT = new NodeFeature("semantic_text.highlighter.default"); + private static final NodeFeature TEST_RERANKING_SERVICE_PARSE_TEXT_AS_SCORE = new NodeFeature( + "test_reranking_service.parse_text_as_score" + ); @Override public Set getTestFeatures() { @@ -45,7 +48,8 @@ public class InferenceFeatures implements FeatureSpecification { TextSimilarityRankRetrieverBuilder.TEXT_SIMILARITY_RERANKER_ALIAS_HANDLING_FIX, SemanticInferenceMetadataFieldsMapper.INFERENCE_METADATA_FIELDS_ENABLED_BY_DEFAULT, SEMANTIC_TEXT_HIGHLIGHTER_DEFAULT, - SEMANTIC_KNN_FILTER_FIX + SEMANTIC_KNN_FILTER_FIX, + TEST_RERANKING_SERVICE_PARSE_TEXT_AS_SCORE ); } } diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/70_text_similarity_rank_retriever.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/70_text_similarity_rank_retriever.yml index 9a6ecffe29d4..6ed9cf22ad0e 100644 --- a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/70_text_similarity_rank_retriever.yml +++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/70_text_similarity_rank_retriever.yml @@ -2,7 +2,6 @@ setup: - skip: features: - close_to - - contains - requires: test_runner_features: "close_to" @@ -33,16 +32,8 @@ setup: type: keyword subtopic: type: keyword - - - do: - index: - index: test-index - id: doc_1 - body: - text: "As seen from Earth, a solar eclipse happens when the Moon is directly between the Earth and the Sun." - topic: [ "science" ] - subtopic: [ "technology" ] - refresh: true + inference_text_field: + type: text - do: index: @@ -52,6 +43,7 @@ setup: text: "The phases of the Moon come from the position of the Moon relative to the Earth and Sun." topic: [ "science" ] subtopic: [ "astronomy" ] + inference_text_field: "0" refresh: true - do: @@ -61,11 +53,27 @@ setup: body: text: "Sun Moon Lake is a lake in Nantou County, Taiwan. It is the largest lake in Taiwan." topic: [ "geography" ] + inference_text_field: "1" + refresh: true + + - do: + index: + index: test-index + id: doc_1 + body: + text: "As seen from Earth, a solar eclipse happens when the Moon is directly between the Earth and the Sun." + topic: [ "science" ] + subtopic: [ "technology" ] + inference_text_field: "-1" refresh: true --- "Simple text similarity rank retriever": + - requires: + cluster_features: "test_reranking_service.parse_text_as_score" + reason: test_reranking_service can now parse provided input as score to provide deterministic ranks + - do: search: index: test-index @@ -75,14 +83,37 @@ setup: retriever: text_similarity_reranker: retriever: + # this one returns docs 1 and 2 standard: query: - term: - topic: "science" + bool: { + should: [ + { + constant_score: { + filter: { + term: { + subtopic: "technology" + } + }, + boost: 10 + } + }, + { + constant_score: { + filter: { + term: { + subtopic: "astronomy" + } + }, + boost: 1 + } + } + ] + } rank_window_size: 10 inference_id: my-rerank-model inference_text: "How often does the moon hide the sun?" - field: text + field: inference_text_field size: 10 - match: { hits.total.value: 2 } @@ -94,6 +125,10 @@ setup: --- "Simple text similarity rank retriever and filtering": + - requires: + cluster_features: "test_reranking_service.parse_text_as_score" + reason: test_reranking_service can now parse provided input as score to provide deterministic ranks + - do: search: index: test-index @@ -103,6 +138,7 @@ setup: retriever: text_similarity_reranker: retriever: + # this one returns doc 1 standard: query: term: @@ -113,7 +149,7 @@ setup: rank_window_size: 10 inference_id: my-rerank-model inference_text: "How often does the moon hide the sun?" - field: text + field: inference_text_field size: 10 - match: { hits.total.value: 1 } @@ -143,7 +179,7 @@ setup: rank_window_size: 10 inference_id: i-dont-exist inference_text: "How often does the moon hide the sun?" - field: text + field: inference_text_field size: 10 --- @@ -169,13 +205,17 @@ setup: rank_window_size: 10 inference_id: i-dont-exist inference_text: "asdfasdf" - field: text + field: inference_text_field size: 10 --- "text similarity reranking with explain": + - requires: + cluster_features: "test_reranking_service.parse_text_as_score" + reason: test_reranking_service can now parse provided input as score to provide deterministic ranks + - do: search: index: test-index @@ -186,10 +226,32 @@ setup: text_similarity_reranker: { retriever: { + # this one returns doc 1 and 2 standard: { query: { - term: { - topic: "science" + bool: { + should: [ + { + constant_score: { + filter: { + term: { + subtopic: "technology" + } + }, + boost: 10 + } + }, + { + constant_score: { + filter: { + term: { + subtopic: "astronomy" + } + }, + boost: 1 + } + } + ] } } } @@ -197,17 +259,17 @@ setup: rank_window_size: 10, inference_id: my-rerank-model, inference_text: "How often does the moon hide the sun?", - field: text + field: inference_text_field } } size: 10 explain: true - - contains: { hits.hits: { _id: "doc_2" } } - - contains: { hits.hits: { _id: "doc_1" } } + - match: { hits.hits.0._id: "doc_2" } + - match: { hits.hits.1._id: "doc_1" } - - match: {hits.hits.0._explanation.description: "/text_similarity_reranker.match.using.inference.endpoint:.\\[my-rerank-model\\].on.document.field:.\\[text\\].*/" } - - match: {hits.hits.0._explanation.details.0.description: "/weight.*science.*/" } + - match: {hits.hits.0._explanation.description: "/text_similarity_reranker.match.using.inference.endpoint:.\\[my-rerank-model\\].on.document.field:.\\[inference_text_field\\].*/" } + - match: {hits.hits.0._explanation.details.0.details.0.description: "/subtopic.*astronomy.*/" } --- "text similarity reranker properly handles aliases": @@ -281,7 +343,7 @@ setup: rank_window_size: 10 inference_id: my-rerank-model inference_text: "How often does the moon hide the sun?" - field: text + field: inference_text_field size: 10 - match: { hits.total.value: 1 } diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/800_rrf_with_text_similarity_reranker_retriever.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/800_rrf_with_text_similarity_reranker_retriever.yml index 98c4ae9e642d..423fce15ff11 100644 --- a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/800_rrf_with_text_similarity_reranker_retriever.yml +++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/800_rrf_with_text_similarity_reranker_retriever.yml @@ -39,15 +39,9 @@ setup: type: keyword integer: type: integer + inference_text_field: + type: text - - do: - index: - index: test-index - id: doc_1 - body: - text: "Sun Moon Lake is a lake in Nantou County, Taiwan. It is the largest lake in Taiwan." - topic: [ "geography" ] - integer: 1 - do: index: @@ -58,6 +52,7 @@ setup: topic: [ "science" ] subtopic: [ "astronomy" ] integer: 2 + inference_text_field: "0" - do: index: @@ -68,6 +63,17 @@ setup: topic: [ "science" ] subtopic: [ "technology" ] integer: 3 + inference_text_field: "1" + + - do: + index: + index: test-index + id: doc_1 + body: + text: "Sun Moon Lake is a lake in Nantou County, Taiwan. It is the largest lake in Taiwan." + topic: [ "geography" ] + integer: 1 + inference_text_field: "-1" - do: indices.refresh: {} @@ -75,6 +81,10 @@ setup: --- "rrf retriever with a nested text similarity reranker": + - requires: + cluster_features: "test_reranking_service.parse_text_as_score" + reason: test_reranking_service can now parse provided input as score to provide deterministic ranks + - do: search: index: test-index @@ -86,6 +96,7 @@ setup: retrievers: [ { + # this one retrieves docs 1 and 2 standard: { query: { bool: { @@ -120,13 +131,14 @@ setup: } }, { + # this one retrieves doc 2 text_similarity_reranker: { retriever: { standard: { query: { term: { - topic: "science" + subtopic: "astronomy" } } } @@ -134,7 +146,7 @@ setup: rank_window_size: 10, inference_id: my-rerank-model, inference_text: "How often does the moon hide the sun?", - field: text + field: inference_text_field } } ], @@ -149,20 +161,21 @@ setup: field: topic size: 10 - - match: { hits.total.value: 3 } - - length: { hits.hits: 2 } + - match: { hits.total.value: 2 } + - length: { hits.hits: 1 } - - contains: { hits.hits: { _id: "doc_1" } } - - contains: { hits.hits: { _id: "doc_3" } } + - match: { hits.hits.0._id: "doc_1" } - - match: { aggregations.topics.buckets.0.key: "science" } - - match: { aggregations.topics.buckets.0.doc_count: 2 } - - match: { aggregations.topics.buckets.1.key: "geography" } - - match: { aggregations.topics.buckets.1.doc_count: 1 } + - contains: { aggregations.topics.buckets: { key: "geography", doc_count: 1 } } + - contains: { aggregations.topics.buckets: { key: "science", doc_count: 1 } } --- "Text similarity reranker on top of an RRF retriever": + - requires: + cluster_features: "test_reranking_service.parse_text_as_score" + reason: test_reranking_service can now parse provided input as score to provide deterministic ranks + - do: search: index: test-index @@ -174,10 +187,12 @@ setup: text_similarity_reranker: { retriever: { + # this one retrieves docs 1 and 3 so final ranking should be 3 and 1 based on reranking service rrf: { retrievers: [ { + # this one retrieves docs 1 and 3 standard: { query: { bool: { @@ -212,6 +227,7 @@ setup: } }, { + # this one retrieves doc 1 standard: { query: { term: { @@ -228,7 +244,7 @@ setup: rank_window_size: 10, inference_id: my-rerank-model, inference_text: "How often does the moon hide the sun?", - field: text + field: inference_text_field } } size: 10 @@ -253,6 +269,10 @@ setup: --- "explain using rrf retriever and text-similarity": + - requires: + cluster_features: "test_reranking_service.parse_text_as_score" + reason: test_reranking_service can now parse provided input as score to provide deterministic ranks + - do: search: index: test-index @@ -264,6 +284,7 @@ setup: retrievers: [ { + # this one retrieves docs 1 and 2 standard: { query: { bool: { @@ -298,6 +319,7 @@ setup: } }, { + # this one retrieves doc 2 text_similarity_reranker: { retriever: { @@ -312,7 +334,7 @@ setup: rank_window_size: 10, inference_id: my-rerank-model, inference_text: "How often does the moon hide the sun?", - field: text + field: inference_text_field } } ], @@ -332,5 +354,5 @@ setup: - match: {hits.hits.0._explanation.details.0.details.0.details.0.description: "/ConstantScore.*/" } - match: {hits.hits.0._explanation.details.1.value: 1} - match: {hits.hits.0._explanation.details.1.description: "/rrf.score:.\\[0.5\\].*/" } - - match: {hits.hits.0._explanation.details.1.details.0.description: "/text_similarity_reranker.match.using.inference.endpoint:.\\[my-rerank-model\\].on.document.field:.\\[text\\].*/" } + - match: {hits.hits.0._explanation.details.1.details.0.description: "/text_similarity_reranker.match.using.inference.endpoint:.\\[my-rerank-model\\].on.document.field:.\\[inference_text_field\\].*/" } - match: {hits.hits.0._explanation.details.1.details.0.details.0.description: "/weight.*astronomy.*/" }