github-mirrors/elasticsearch
1
0
Fork 0
mirror of https://github.com/elastic/elasticsearch.git synced 2025-04-24 23:27:25 -04:00
Code Issues Projects Releases Packages Wiki Activity

New vector_rescore parameter as a quantized index type option (#124581)

Browse source 
This adds a new parameter to the quantized index mapping that allows
default oversampling and rescoring to occur. 

This doesn't adjust any of the defaults. It allows it to be configured.
When the user provides `rescore_vector: {oversample: <number>}` in the
query it will overwrite it.

For example, here is how to use it with bbq:

```
PUT rescored_bbq
{
  "mappings": {
    "properties": {
      "vector": {
        "type": "dense_vector",
        "index_options": {
          "type": "bbq_hnsw",
          "rescore_vector": {"oversample": 3.0}
        }
      }
    }
  }
}
```

Then, when querying, it will auto oversample the `k` by `3x` and rerank
with the raw vectors.

```
POST _search
{
  "knn": {
    "query_vector": [...],
    "field": "vector"
  }
}
```
This commit is contained in:
Benjamin Trent 2025-03-13 09:40:08 -04:00 • committed by GitHub
parent cd25958700
commit b2c1c4e0f0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 866 additions and 58 deletions
Download patch file Download diff file Expand all files Collapse all files

90
rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_bbq_hnsw.yml
View file

@ -244,3 +244,93 @@ setup:
index: dynamic_dim_bbq_hnsw
body:
vector: [1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0]
---
"Test index configured rescore vector":
- requires:
cluster_features: ["mapper.dense_vector.rescore_vector"]
reason: Needs rescore_vector feature
- skip:
features: "headers"
- do:
indices.create:
index: bbq_rescore_hnsw
body:
settings:
index:
number_of_shards: 1
mappings:
properties:
vector:
type: dense_vector
dims: 64
index: true
similarity: max_inner_product
index_options:
type: bbq_hnsw
rescore_vector:
oversample: 1.5
- do:
bulk:
index: bbq_rescore_hnsw
refresh: true
body: |
{ "index": {"_id": "1"}}
{ "vector": [0.077, 0.32 , -0.205, 0.63 , 0.032, 0.201, 0.167, -0.313, 0.176, 0.531, -0.375, 0.334, -0.046, 0.078, -0.349, 0.272, 0.307, -0.083, 0.504, 0.255, -0.404, 0.289, -0.226, -0.132, -0.216, 0.49 , 0.039, 0.507, -0.307, 0.107, 0.09 , -0.265, -0.285, 0.336, -0.272, 0.369, -0.282, 0.086, -0.132, 0.475, -0.224, 0.203, 0.439, 0.064, 0.246, -0.396, 0.297, 0.242, -0.028, 0.321, -0.022, -0.009, -0.001 , 0.031, -0.533, 0.45, -0.683, 1.331, 0.194, -0.157, -0.1 , -0.279, -0.098, -0.176] }
{ "index": {"_id": "2"}}
{ "vector": [0.196, 0.514, 0.039, 0.555, -0.042, 0.242, 0.463, -0.348, -0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048, 0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, -0.438, -0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, -0.138, -0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429, -0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166, 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, -0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013] }
{ "index": {"_id": "3"}}
{ "vector": [0.196, 0.514, 0.039, 0.555, -0.042, 0.242, 0.463, -0.348, -0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048, 0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, -0.438, -0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, -0.138, -0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429, -0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166, 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, -0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013] }
- do:
headers:
Content-Type: application/json
search:
rest_total_hits_as_int: true
index: bbq_rescore_hnsw
body:
knn:
field: vector
query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393,
0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015,
0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259,
-0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 ,
-0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232,
-0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034,
-0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582,
-0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158]
k: 3
num_candidates: 3
- match: { hits.total: 3 }
- set: { hits.hits.0._score: rescore_score0 }
- set: { hits.hits.1._score: rescore_score1 }
- set: { hits.hits.2._score: rescore_score2 }
- do:
headers:
Content-Type: application/json
search:
rest_total_hits_as_int: true
index: bbq_rescore_hnsw
body:
query:
script_score:
query: {match_all: {} }
script:
source: "double similarity = dotProduct(params.query_vector, 'vector'); return similarity < 0 ? 1 / (1 + -1 * similarity) : similarity + 1"
params:
query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393,
0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015,
0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259,
-0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 ,
-0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232,
-0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034,
-0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582,
-0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158]
# Compare scores as hit IDs may change depending on how things are distributed
- match: { hits.total: 3 }
- match: { hits.hits.0._score: $rescore_score0 }
- match: { hits.hits.1._score: $rescore_score1 }
- match: { hits.hits.2._score: $rescore_score2 }

89
rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_byte_quantized.yml
View file

@ -611,3 +611,92 @@ setup:
- match: { hits.hits.0._id: "1"}
- match: { hits.hits.1._id: "2"}
- match: { hits.hits.2._id: "3"}
---
"Test index configured rescore vector":
- requires:
cluster_features: ["mapper.dense_vector.rescore_vector"]
reason: Needs rescore_vector feature
- skip:
features: "headers"
- do:
indices.create:
index: int8_rescore_hnsw
body:
settings:
index:
number_of_shards: 1
mappings:
properties:
vector:
type: dense_vector
dims: 64
index: true
similarity: max_inner_product
index_options:
type: int8_hnsw
rescore_vector:
oversample: 1.5
- do:
bulk:
index: int8_rescore_hnsw
refresh: true
body: |
{ "index": {"_id": "1"}}
{ "vector": [0.077, 0.32 , -0.205, 0.63 , 0.032, 0.201, 0.167, -0.313, 0.176, 0.531, -0.375, 0.334, -0.046, 0.078, -0.349, 0.272, 0.307, -0.083, 0.504, 0.255, -0.404, 0.289, -0.226, -0.132, -0.216, 0.49 , 0.039, 0.507, -0.307, 0.107, 0.09 , -0.265, -0.285, 0.336, -0.272, 0.369, -0.282, 0.086, -0.132, 0.475, -0.224, 0.203, 0.439, 0.064, 0.246, -0.396, 0.297, 0.242, -0.028, 0.321, -0.022, -0.009, -0.001 , 0.031, -0.533, 0.45, -0.683, 1.331, 0.194, -0.157, -0.1 , -0.279, -0.098, -0.176] }
{ "index": {"_id": "2"}}
{ "vector": [0.196, 0.514, 0.039, 0.555, -0.042, 0.242, 0.463, -0.348, -0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048, 0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, -0.438, -0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, -0.138, -0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429, -0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166, 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, -0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013] }
{ "index": {"_id": "3"}}
{ "vector": [0.196, 0.514, 0.039, 0.555, -0.042, 0.242, 0.463, -0.348, -0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048, 0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, -0.438, -0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, -0.138, -0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429, -0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166, 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, -0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013] }
- do:
headers:
Content-Type: application/json
search:
rest_total_hits_as_int: true
index: int8_rescore_hnsw
body:
knn:
field: vector
query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393,
0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015,
0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259,
-0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 ,
-0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232,
-0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034,
-0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582,
-0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158]
k: 3
num_candidates: 3
- match: { hits.total: 3 }
- set: { hits.hits.0._score: rescore_score0 }
- set: { hits.hits.1._score: rescore_score1 }
- set: { hits.hits.2._score: rescore_score2 }
- do:
headers:
Content-Type: application/json
search:
rest_total_hits_as_int: true
body:
query:
script_score:
query: {match_all: {} }
script:
source: "double similarity = dotProduct(params.query_vector, 'vector'); return similarity < 0 ? 1 / (1 + -1 * similarity) : similarity + 1"
params:
query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393,
0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015,
0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259,
-0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 ,
-0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232,
-0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034,
-0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582,
-0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158]
# Compare scores as hit IDs may change depending on how things are distributed
- match: { hits.total: 3 }
- match: { hits.hits.0._score: $rescore_score0 }
- match: { hits.hits.1._score: $rescore_score1 }
- match: { hits.hits.2._score: $rescore_score2 }

89
rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_half_byte_quantized.yml
View file

@ -642,3 +642,92 @@ setup:
index: dynamic_dim_hnsw_quantized
body:
vector: [1.0, 2.0, 3.0, 4.0, 5.0, 6.0]
---
"Test index configured rescore vector":
- requires:
cluster_features: ["mapper.dense_vector.rescore_vector"]
reason: Needs rescore_vector feature
- skip:
features: "headers"
- do:
indices.create:
index: int4_rescore_hnsw
body:
settings:
index:
number_of_shards: 1
mappings:
properties:
vector:
type: dense_vector
dims: 64
index: true
similarity: max_inner_product
index_options:
type: int4_hnsw
rescore_vector:
oversample: 1.5
- do:
bulk:
index: int4_rescore_hnsw
refresh: true
body: |
{ "index": {"_id": "1"}}
{ "vector": [0.077, 0.32 , -0.205, 0.63 , 0.032, 0.201, 0.167, -0.313, 0.176, 0.531, -0.375, 0.334, -0.046, 0.078, -0.349, 0.272, 0.307, -0.083, 0.504, 0.255, -0.404, 0.289, -0.226, -0.132, -0.216, 0.49 , 0.039, 0.507, -0.307, 0.107, 0.09 , -0.265, -0.285, 0.336, -0.272, 0.369, -0.282, 0.086, -0.132, 0.475, -0.224, 0.203, 0.439, 0.064, 0.246, -0.396, 0.297, 0.242, -0.028, 0.321, -0.022, -0.009, -0.001 , 0.031, -0.533, 0.45, -0.683, 1.331, 0.194, -0.157, -0.1 , -0.279, -0.098, -0.176] }
{ "index": {"_id": "2"}}
{ "vector": [0.196, 0.514, 0.039, 0.555, -0.042, 0.242, 0.463, -0.348, -0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048, 0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, -0.438, -0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, -0.138, -0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429, -0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166, 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, -0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013] }
{ "index": {"_id": "3"}}
{ "vector": [0.196, 0.514, 0.039, 0.555, -0.042, 0.242, 0.463, -0.348, -0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048, 0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, -0.438, -0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, -0.138, -0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429, -0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166, 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, -0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013] }
- do:
headers:
Content-Type: application/json
search:
rest_total_hits_as_int: true
index: int4_rescore_hnsw
body:
knn:
field: vector
query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393,
0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015,
0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259,
-0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 ,
-0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232,
-0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034,
-0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582,
-0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158]
k: 3
num_candidates: 3
- match: { hits.total: 3 }
- set: { hits.hits.0._score: rescore_score0 }
- set: { hits.hits.1._score: rescore_score1 }
- set: { hits.hits.2._score: rescore_score2 }
- do:
headers:
Content-Type: application/json
search:
rest_total_hits_as_int: true
body:
query:
script_score:
query: {match_all: {} }
script:
source: "double similarity = dotProduct(params.query_vector, 'vector'); return similarity < 0 ? 1 / (1 + -1 * similarity) : similarity + 1"
params:
query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393,
0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015,
0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259,
-0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 ,
-0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232,
-0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034,
-0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582,
-0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158]
# Compare scores as hit IDs may change depending on how things are distributed
- match: { hits.total: 3 }
- match: { hits.hits.0._score: $rescore_score0 }
- match: { hits.hits.1._score: $rescore_score1 }
- match: { hits.hits.2._score: $rescore_score2 }

90
rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_bbq_flat.yml
View file

@ -249,3 +249,93 @@ setup:
index: dynamic_dim_bbq_flat
body:
vector: [1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0]
---
"Test index configured rescore vector":
- requires:
cluster_features: ["mapper.dense_vector.rescore_vector"]
reason: Needs rescore_vector feature
- skip:
features: "headers"
- do:
indices.create:
index: bbq_rescore_flat
body:
settings:
index:
number_of_shards: 1
mappings:
properties:
vector:
type: dense_vector
dims: 64
index: true
similarity: max_inner_product
index_options:
type: bbq_flat
rescore_vector:
oversample: 1.5
- do:
bulk:
index: bbq_rescore_flat
refresh: true
body: |
{ "index": {"_id": "1"}}
{ "vector": [0.077, 0.32 , -0.205, 0.63 , 0.032, 0.201, 0.167, -0.313, 0.176, 0.531, -0.375, 0.334, -0.046, 0.078, -0.349, 0.272, 0.307, -0.083, 0.504, 0.255, -0.404, 0.289, -0.226, -0.132, -0.216, 0.49 , 0.039, 0.507, -0.307, 0.107, 0.09 , -0.265, -0.285, 0.336, -0.272, 0.369, -0.282, 0.086, -0.132, 0.475, -0.224, 0.203, 0.439, 0.064, 0.246, -0.396, 0.297, 0.242, -0.028, 0.321, -0.022, -0.009, -0.001 , 0.031, -0.533, 0.45, -0.683, 1.331, 0.194, -0.157, -0.1 , -0.279, -0.098, -0.176] }
{ "index": {"_id": "2"}}
{ "vector": [0.196, 0.514, 0.039, 0.555, -0.042, 0.242, 0.463, -0.348, -0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048, 0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, -0.438, -0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, -0.138, -0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429, -0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166, 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, -0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013] }
{ "index": {"_id": "3"}}
{ "vector": [0.196, 0.514, 0.039, 0.555, -0.042, 0.242, 0.463, -0.348, -0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048, 0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, -0.438, -0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, -0.138, -0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429, -0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166, 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, -0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013] }
- do:
headers:
Content-Type: application/json
search:
rest_total_hits_as_int: true
index: bbq_rescore_flat
body:
knn:
field: vector
query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393,
0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015,
0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259,
-0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 ,
-0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232,
-0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034,
-0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582,
-0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158]
k: 3
num_candidates: 3
- match: { hits.total: 3 }
- set: { hits.hits.0._score: rescore_score0 }
- set: { hits.hits.1._score: rescore_score1 }
- set: { hits.hits.2._score: rescore_score2 }
- do:
headers:
Content-Type: application/json
search:
rest_total_hits_as_int: true
index: bbq_rescore_flat
body:
query:
script_score:
query: {match_all: {} }
script:
source: "double similarity = dotProduct(params.query_vector, 'vector'); return similarity < 0 ? 1 / (1 + -1 * similarity) : similarity + 1"
params:
query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393,
0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015,
0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259,
-0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 ,
-0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232,
-0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034,
-0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582,
-0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158]
# Compare scores as hit IDs may change depending on how things are distributed
- match: { hits.total: 3 }
- match: { hits.hits.0._score: $rescore_score0 }
- match: { hits.hits.1._score: $rescore_score1 }
- match: { hits.hits.2._score: $rescore_score2 }

90
rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_int4_flat.yml
View file

@ -405,3 +405,93 @@ setup:
- match: { hits.hits.0._score: $rescore_score0 }
- match: { hits.hits.1._score: $rescore_score1 }
- match: { hits.hits.2._score: $rescore_score2 }
---
"Test index configured rescore vector":
- requires:
cluster_features: ["mapper.dense_vector.rescore_vector"]
reason: Needs rescore_vector feature
- skip:
features: "headers"
- do:
indices.create:
index: int4_rescore_flat
body:
settings:
index:
number_of_shards: 1
mappings:
properties:
vector:
type: dense_vector
dims: 64
index: true
similarity: max_inner_product
index_options:
type: int4_flat
rescore_vector:
oversample: 1.5
- do:
bulk:
index: int4_rescore_flat
refresh: true
body: |
{ "index": {"_id": "1"}}
{ "vector": [0.077, 0.32 , -0.205, 0.63 , 0.032, 0.201, 0.167, -0.313, 0.176, 0.531, -0.375, 0.334, -0.046, 0.078, -0.349, 0.272, 0.307, -0.083, 0.504, 0.255, -0.404, 0.289, -0.226, -0.132, -0.216, 0.49 , 0.039, 0.507, -0.307, 0.107, 0.09 , -0.265, -0.285, 0.336, -0.272, 0.369, -0.282, 0.086, -0.132, 0.475, -0.224, 0.203, 0.439, 0.064, 0.246, -0.396, 0.297, 0.242, -0.028, 0.321, -0.022, -0.009, -0.001 , 0.031, -0.533, 0.45, -0.683, 1.331, 0.194, -0.157, -0.1 , -0.279, -0.098, -0.176] }
{ "index": {"_id": "2"}}
{ "vector": [0.196, 0.514, 0.039, 0.555, -0.042, 0.242, 0.463, -0.348, -0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048, 0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, -0.438, -0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, -0.138, -0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429, -0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166, 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, -0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013] }
{ "index": {"_id": "3"}}
{ "vector": [0.196, 0.514, 0.039, 0.555, -0.042, 0.242, 0.463, -0.348, -0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048, 0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, -0.438, -0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, -0.138, -0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429, -0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166, 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, -0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013] }
- do:
headers:
Content-Type: application/json
search:
rest_total_hits_as_int: true
index: int4_rescore_flat
body:
knn:
field: vector
query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393,
0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015,
0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259,
-0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 ,
-0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232,
-0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034,
-0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582,
-0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158]
k: 3
num_candidates: 3
- match: { hits.total: 3 }
- set: { hits.hits.0._score: rescore_score0 }
- set: { hits.hits.1._score: rescore_score1 }
- set: { hits.hits.2._score: rescore_score2 }
- do:
headers:
Content-Type: application/json
search:
rest_total_hits_as_int: true
index: int4_rescore_flat
body:
query:
script_score:
query: {match_all: {} }
script:
source: "double similarity = dotProduct(params.query_vector, 'vector'); return similarity < 0 ? 1 / (1 + -1 * similarity) : similarity + 1"
params:
query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393,
0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015,
0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259,
-0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 ,
-0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232,
-0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034,
-0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582,
-0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158]
# Compare scores as hit IDs may change depending on how things are distributed
- match: { hits.total: 3 }
- match: { hits.hits.0._score: $rescore_score0 }
- match: { hits.hits.1._score: $rescore_score1 }
- match: { hits.hits.2._score: $rescore_score2 }

90
rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_int8_flat.yml
View file

@ -346,3 +346,93 @@ setup:
index: true
index_options:
type: int8_flat
---
"Test index configured rescore vector":
- requires:
cluster_features: ["mapper.dense_vector.rescore_vector"]
reason: Needs rescore_vector feature
- skip:
features: "headers"
- do:
indices.create:
index: int8_rescore_flat
body:
settings:
index:
number_of_shards: 1
mappings:
properties:
vector:
type: dense_vector
dims: 64
index: true
similarity: max_inner_product
index_options:
type: int8_flat
rescore_vector:
oversample: 1.5
- do:
bulk:
index: int8_rescore_flat
refresh: true
body: |
{ "index": {"_id": "1"}}
{ "vector": [0.077, 0.32 , -0.205, 0.63 , 0.032, 0.201, 0.167, -0.313, 0.176, 0.531, -0.375, 0.334, -0.046, 0.078, -0.349, 0.272, 0.307, -0.083, 0.504, 0.255, -0.404, 0.289, -0.226, -0.132, -0.216, 0.49 , 0.039, 0.507, -0.307, 0.107, 0.09 , -0.265, -0.285, 0.336, -0.272, 0.369, -0.282, 0.086, -0.132, 0.475, -0.224, 0.203, 0.439, 0.064, 0.246, -0.396, 0.297, 0.242, -0.028, 0.321, -0.022, -0.009, -0.001 , 0.031, -0.533, 0.45, -0.683, 1.331, 0.194, -0.157, -0.1 , -0.279, -0.098, -0.176] }
{ "index": {"_id": "2"}}
{ "vector": [0.196, 0.514, 0.039, 0.555, -0.042, 0.242, 0.463, -0.348, -0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048, 0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, -0.438, -0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, -0.138, -0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429, -0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166, 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, -0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013] }
{ "index": {"_id": "3"}}
{ "vector": [0.196, 0.514, 0.039, 0.555, -0.042, 0.242, 0.463, -0.348, -0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048, 0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, -0.438, -0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, -0.138, -0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429, -0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166, 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, -0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013] }
- do:
headers:
Content-Type: application/json
search:
rest_total_hits_as_int: true
index: int8_rescore_flat
body:
knn:
field: vector
query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393,
0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015,
0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259,
-0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 ,
-0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232,
-0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034,
-0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582,
-0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158]
k: 3
num_candidates: 3
- match: { hits.total: 3 }
- set: { hits.hits.0._score: rescore_score0 }
- set: { hits.hits.1._score: rescore_score1 }
- set: { hits.hits.2._score: rescore_score2 }
- do:
headers:
Content-Type: application/json
search:
rest_total_hits_as_int: true
index: int8_rescore_flat
body:
query:
script_score:
query: {match_all: {} }
script:
source: "double similarity = dotProduct(params.query_vector, 'vector'); return similarity < 0 ? 1 / (1 + -1 * similarity) : similarity + 1"
params:
query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393,
0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015,
0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259,
-0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 ,
-0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232,
-0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034,
-0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582,
-0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158]
# Compare scores as hit IDs may change depending on how things are distributed
- match: { hits.total: 3 }
- match: { hits.hits.0._score: $rescore_score0 }
- match: { hits.hits.1._score: $rescore_score1 }
- match: { hits.hits.2._score: $rescore_score2 }