Upgrade to lucene-9.5.0-snapshot-d19c3e2e0ed (#92957)

9.5 will include several changes related to vector search. An extensive list is available at https://github.com/apache/lucene/milestone/4 .

build-tools-internal/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 8.7.0
-lucene            = 9.4.2
+lucene            = 9.5.0-snapshot-d19c3e2e0ed
 
 bundled_jdk_vendor = openjdk
 bundled_jdk = 19.0.1+10@afdd2e245b014143b62ccb916125e3ce
@@ -21,12 +21,12 @@ netty             = 4.1.86.Final
 
 commons_lang3                   = 3.9
 
+antlr4            = 4.11.1
 # when updating this version, you need to ensure compatibility with:
 #  - modules/ingest-attachment (transitive dependency, check the upstream POM)
 #  - distribution/tools/plugin-cli
 #  - x-pack/plugin/security
 bouncycastle=1.64
-
 # used by security and idp (need to be in sync due to cross-dependency in testing)
 opensaml = 4.0.1
 

distribution/src/config/jvm.options

@@ -73,3 +73,6 @@
 
 ## GC logging
 -Xlog:gc*,gc+age=trace,safepoint:file=@loggc@:utctime,level,pid,tags:filecount=32,filesize=64m
+
+# temporarily disable the Panama-based MMapDirectory
+-Dorg.apache.lucene.store.MMapDirectory.enableMemorySegments=false

docs/Versions.asciidoc

@@ -1,8 +1,8 @@
 
 include::{docs-root}/shared/versions/stack/{source_branch}.asciidoc[]
 
-:lucene_version:        9.4.2
+:lucene_version:        9.5.0
-:lucene_version_path:   9_4_2
+:lucene_version_path:   9_5_0
 :jdk:                   11.0.2
 :jdk_major:             11
 :build_type:            tar

docs/changelog/92957.yaml

@@ -0,0 +1,5 @@
+pr: 92957
+summary: Upgrade to lucene-9.5.0-snapshot-d19c3e2e0ed
+area: Search
+type: upgrade
+issues: []

gradle/verification-metadata.xml

@@ -759,26 +759,36 @@
             <sha256 value="37f5216e14af2772930dff9b8734353f0a80e89ba3f33e065441de6537c5e842" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.ibm.icu" name="icu4j" version="61.1">
+       <component group="com.ibm.icu" name="icu4j" version="61.1">
-         <artifact name="icu4j-61.1.jar">
+           <artifact name="icu4j-61.1.jar">
-            <sha256 value="55c98eb1838b2a4bb9a07dc36bd378532d64d0cdcb7ceee914236866a7de4464" origin="Generated by Gradle"/>
+               <sha256 value="55c98eb1838b2a4bb9a07dc36bd378532d64d0cdcb7ceee914236866a7de4464"
-         </artifact>
+                       origin="Generated by Gradle"/>
-      </component>
+           </artifact>
-      <component group="com.ibm.icu" name="icu4j" version="68.2">
+       </component>
-         <artifact name="icu4j-68.2.jar">
+       <component group="com.ibm.icu" name="icu4j" version="68.2">
-            <sha256 value="9bd7bf869a44ba8aeb0cddd7e6616e88cd4795ba5bfce2230447cb0e185a646c" origin="Generated by Gradle"/>
+           <artifact name="icu4j-68.2.jar">
-         </artifact>
+               <sha256 value="9bd7bf869a44ba8aeb0cddd7e6616e88cd4795ba5bfce2230447cb0e185a646c"
-      </component>
+                       origin="Generated by Gradle"/>
-      <component group="com.jamesmurty.utils" name="java-xmlbuilder" version="0.4">
+           </artifact>
-         <artifact name="java-xmlbuilder-0.4.jar">
+       </component>
-            <sha256 value="681e53c4ffd59fa12068803b259e3a83d43f07a47c112e748a187dee179eb31f" origin="Generated by Gradle"/>
+       <component group="com.ibm.icu" name="icu4j" version="71.1">
-         </artifact>
+           <artifact name="icu4j-71.1.jar">
-      </component>
+               <sha256 value="91c4f8ebf0ceb489547098fe9d5c09a65eb419caea6ed714867f5280800bcf1a"
-      <component group="com.jcraft" name="jsch" version="0.1.54">
+                       origin="Generated by Gradle"/>
-         <artifact name="jsch-0.1.54.jar">
+           </artifact>
-            <sha256 value="92eb273a3316762478fdd4fe03a0ce1842c56f496c9c12fe1235db80450e1fdb" origin="Generated by Gradle"/>
+       </component>
-         </artifact>
+       <component group="com.jamesmurty.utils" name="java-xmlbuilder" version="0.4">
-      </component>
+           <artifact name="java-xmlbuilder-0.4.jar">
+               <sha256 value="681e53c4ffd59fa12068803b259e3a83d43f07a47c112e748a187dee179eb31f"
+                       origin="Generated by Gradle"/>
+           </artifact>
+       </component>
+       <component group="com.jcraft" name="jsch" version="0.1.54">
+           <artifact name="jsch-0.1.54.jar">
+               <sha256 value="92eb273a3316762478fdd4fe03a0ce1842c56f496c9c12fe1235db80450e1fdb"
+                       origin="Generated by Gradle"/>
+           </artifact>
+       </component>
       <component group="com.jcraft" name="jsch" version="0.1.55">
          <artifact name="jsch-0.1.55.jar">
             <sha256 value="d492b15a6d2ea3f1cc39c422c953c40c12289073dbe8360d98c0f6f9ec74fc44" origin="Generated by Gradle"/>
@@ -1678,49 +1688,82 @@
             <sha256 value="adb1a33c07b45c39b926bdeeadf800f701be9c3d04e0deb543069e5f09856185" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.abego.treelayout" name="org.abego.treelayout.core" version="1.0.3">
+       <component group="org.abego.treelayout" name="org.abego.treelayout.core" version="1.0.3">
-         <artifact name="org.abego.treelayout.core-1.0.3.jar">
+           <artifact name="org.abego.treelayout.core-1.0.3.jar">
-            <sha256 value="fa5e31395c39c2e7d46aca0f81f72060931607b2fa41bd36038eb2cb6fb93326" origin="Generated by Gradle"/>
+               <sha256 value="fa5e31395c39c2e7d46aca0f81f72060931607b2fa41bd36038eb2cb6fb93326"
-         </artifact>
+                       origin="Generated by Gradle"/>
-      </component>
+           </artifact>
-      <component group="org.antlr" name="ST4" version="4.3">
+       </component>
-         <artifact name="ST4-4.3.jar">
+       <component group="org.antlr" name="ST4" version="4.3">
-            <sha256 value="28547dba48cfceb77b6efbfe069aebe9ed3324ae60dbd52093d13a1d636ed069" origin="Generated by Gradle"/>
+           <artifact name="ST4-4.3.jar">
-         </artifact>
+               <sha256 value="28547dba48cfceb77b6efbfe069aebe9ed3324ae60dbd52093d13a1d636ed069"
-      </component>
+                       origin="Generated by Gradle"/>
-      <component group="org.antlr" name="antlr-runtime" version="3.4">
+           </artifact>
-         <artifact name="antlr-runtime-3.4.jar">
+       </component>
-            <sha256 value="5b7cf53b7b30b034023f58030c8147c433f2bee0fe7dec8fae6bebf3708c5a63" origin="Generated by Gradle"/>
+       <component group="org.antlr" name="ST4" version="4.3.4">
-         </artifact>
+           <artifact name="ST4-4.3.4.jar">
-      </component>
+               <sha256 value="f927ac384c46d749f8b5ec68972a53aed21e00313509299616edb73bfa15ff33"
-      <component group="org.antlr" name="antlr-runtime" version="3.5.2">
+                       origin="Generated by Gradle"/>
-         <artifact name="antlr-runtime-3.5.2.jar">
+           </artifact>
-            <sha256 value="ce3fc8ecb10f39e9a3cddcbb2ce350d272d9cd3d0b1e18e6fe73c3b9389c8734" origin="Generated by Gradle"/>
+       </component>
-         </artifact>
+       <component group="org.antlr" name="antlr-runtime" version="3.4">
-      </component>
+           <artifact name="antlr-runtime-3.4.jar">
-      <component group="org.antlr" name="antlr4" version="4.5.3">
+               <sha256 value="5b7cf53b7b30b034023f58030c8147c433f2bee0fe7dec8fae6bebf3708c5a63"
-         <artifact name="antlr4-4.5.3.jar">
+                       origin="Generated by Gradle"/>
-            <sha256 value="a32de739cfdf515774e696f91aa9697d2e7731e5cb5045ca8a4b657f8b1b4fb4" origin="Generated by Gradle"/>
+           </artifact>
-         </artifact>
+       </component>
-      </component>
+       <component group="org.antlr" name="antlr-runtime" version="3.5.2">
-      <component group="org.antlr" name="antlr4" version="4.9.2">
+           <artifact name="antlr-runtime-3.5.2.jar">
-         <artifact name="antlr4-4.9.2.jar">
+               <sha256 value="ce3fc8ecb10f39e9a3cddcbb2ce350d272d9cd3d0b1e18e6fe73c3b9389c8734"
-            <sha256 value="7d66253762da7c8c7ab6ac05da1471aeeb3cb8e92310ecfb08f939306b4c7dae" origin="Generated by Gradle"/>
+                       origin="Generated by Gradle"/>
-         </artifact>
+           </artifact>
-      </component>
+       </component>
-      <component group="org.antlr" name="antlr4-runtime" version="4.10.1">
+       <component group="org.antlr" name="antlr-runtime" version="3.5.3">
-         <artifact name="antlr4-runtime-4.10.1.jar">
+           <artifact name="antlr-runtime-3.5.3.jar">
-            <sha256 value="da66be0c98acfb29bc708300d05f1a3269c40f9984a4cb9251cf2ba1898d1334" origin="Generated by Gradle"/>
+               <sha256 value="68bf9f5a33dfcb34033495c587e6236bef4e37aa6612919f5b1e843b90669fb9"
-         </artifact>
+                       origin="Generated by Gradle"/>
-      </component>
+           </artifact>
-      <component group="org.antlr" name="antlr4-runtime" version="4.5.1-1">
+       </component>
-         <artifact name="antlr4-runtime-4.5.1-1.jar">
+       <component group="org.antlr" name="antlr4" version="4.11.1">
-            <sha256 value="ffca72bc2a25bb2b0c80a58cee60530a78be17da739bb6c91a8c2e3584ca099e" origin="Generated by Gradle"/>
+           <artifact name="antlr4-4.11.1.jar">
-         </artifact>
+               <sha256 value="e9686e8a663ca512afe3a2eeb6f6ad3f303abb46188991f19ebc6a0fd9c1c14f"
-      </component>
+                       origin="Generated by Gradle"/>
-      <component group="org.antlr" name="antlr4-runtime" version="4.5.3">
+           </artifact>
-         <artifact name="antlr4-runtime-4.5.3.jar">
+       </component>
-            <sha256 value="93bca08ec995caeaaf60bdf80035a0be8507fcdabd3c2618fd8c5aab4444a752" origin="Generated by Gradle"/>
+       <component group="org.antlr" name="antlr4" version="4.5.3">
+           <artifact name="antlr4-4.5.3.jar">
+               <sha256 value="a32de739cfdf515774e696f91aa9697d2e7731e5cb5045ca8a4b657f8b1b4fb4"
+                       origin="Generated by Gradle"/>
+           </artifact>
+       </component>
+       <component group="org.antlr" name="antlr4" version="4.9.2">
+           <artifact name="antlr4-4.9.2.jar">
+               <sha256 value="7d66253762da7c8c7ab6ac05da1471aeeb3cb8e92310ecfb08f939306b4c7dae"
+                       origin="Generated by Gradle"/>
+           </artifact>
+       </component>
+       <component group="org.antlr" name="antlr4-runtime" version="4.10.1">
+           <artifact name="antlr4-runtime-4.10.1.jar">
+               <sha256 value="da66be0c98acfb29bc708300d05f1a3269c40f9984a4cb9251cf2ba1898d1334"
+                       origin="Generated by Gradle"/>
+           </artifact>
+       </component>
+       <component group="org.antlr" name="antlr4-runtime" version="4.11.1">
+           <artifact name="antlr4-runtime-4.11.1.jar">
+               <sha256 value="e06c6553c1ccc14d36052ec4b0fc6f13b808cf957b5b1dc3f61bf401996ada59"
+                       origin="Generated by Gradle"/>
+           </artifact>
+       </component>
+       <component group="org.antlr" name="antlr4-runtime" version="4.5.1-1">
+           <artifact name="antlr4-runtime-4.5.1-1.jar">
+               <sha256 value="ffca72bc2a25bb2b0c80a58cee60530a78be17da739bb6c91a8c2e3584ca099e"
+                       origin="Generated by Gradle"/>
+           </artifact>
+       </component>
+       <component group="org.antlr" name="antlr4-runtime" version="4.5.3">
+           <artifact name="antlr4-runtime-4.5.3.jar">
+               <sha256 value="93bca08ec995caeaaf60bdf80035a0be8507fcdabd3c2618fd8c5aab4444a752"
+                       origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="org.antlr" name="antlr4-runtime" version="4.9.2">
@@ -2611,6 +2654,11 @@
             <sha256 value="cafa2c66ac81894c860e2213c6ba71d4c5a3ae6a26be698318e7de88efdd4db5" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.lucene" name="lucene-analysis-common" version="9.5.0-snapshot-d19c3e2e0ed">
+         <artifact name="lucene-analysis-common-9.5.0-snapshot-d19c3e2e0ed.jar">
+            <sha256 value="73f276384d65dc01697617112d960a0f01fcc197c4925f2144f386fcb21d9445" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.lucene" name="lucene-analysis-icu" version="9.4.1">
          <artifact name="lucene-analysis-icu-9.4.1.jar">
             <sha256 value="a5f35d7294ef475e9e3145e0a191551f5841da738212c4923b0a403a027db200" origin="Generated by Gradle"/>
@@ -2621,6 +2669,11 @@
             <sha256 value="6fc669da198bea83f2c0616a4eb765f30b7dcb729725aa098528151bb36be9d6" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.lucene" name="lucene-analysis-icu" version="9.5.0-snapshot-d19c3e2e0ed">
+         <artifact name="lucene-analysis-icu-9.5.0-snapshot-d19c3e2e0ed.jar">
+            <sha256 value="946c7756ee4320d09b5d55e7895b6285a5ad29e99c9de54d2aec620c4378fbac" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.lucene" name="lucene-analysis-kuromoji" version="9.4.1">
          <artifact name="lucene-analysis-kuromoji-9.4.1.jar">
             <sha256 value="daaf3c6939a9b234682d45102b8c26872f0b2d33631feefb4795a54a6dddc75a" origin="Generated by Gradle"/>
@@ -2631,6 +2684,11 @@
             <sha256 value="13928e760951491b66e1b37f3396a0ee4b150f02a4cbe57743b68dbd0d09ab61" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.lucene" name="lucene-analysis-kuromoji" version="9.5.0-snapshot-d19c3e2e0ed">
+         <artifact name="lucene-analysis-kuromoji-9.5.0-snapshot-d19c3e2e0ed.jar">
+            <sha256 value="0a1a38c2e357c67e3583ef8894f336b389a5a5337795b1ef698d7cb0b826d8e6" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.lucene" name="lucene-analysis-morfologik" version="9.4.1">
          <artifact name="lucene-analysis-morfologik-9.4.1.jar">
             <sha256 value="55cc0c58ef51d62ad0db64d130bbf163ec2bb9431acb898446e9039165c97f5b" origin="Generated by Gradle"/>
@@ -2641,6 +2699,11 @@
             <sha256 value="2835fb73f8d58ded27e9201a8d9546e0bdc5d028f5ae5f112f6f74cf71a291ee" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.lucene" name="lucene-analysis-morfologik" version="9.5.0-snapshot-d19c3e2e0ed">
+         <artifact name="lucene-analysis-morfologik-9.5.0-snapshot-d19c3e2e0ed.jar">
+            <sha256 value="7e2cf5326302d55962944d3ba7a18b8a76b9a49d87e15c4c1904a0b4ae6afdac" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.lucene" name="lucene-analysis-nori" version="9.4.1">
          <artifact name="lucene-analysis-nori-9.4.1.jar">
             <sha256 value="f3e3119787e002606181078e2d2d28715ea80222dd82d5f34013ab11a097e786" origin="Generated by Gradle"/>
@@ -2651,6 +2714,11 @@
             <sha256 value="316776f69a9e493c36c9a1b9b18b1f10de3914a45537e7f324784ab11d748267" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.lucene" name="lucene-analysis-nori" version="9.5.0-snapshot-d19c3e2e0ed">
+         <artifact name="lucene-analysis-nori-9.5.0-snapshot-d19c3e2e0ed.jar">
+            <sha256 value="4de35194eebdfb226cfe497058cec2dfd5834f7da78251bd5bc5362e0b36f807" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.lucene" name="lucene-analysis-phonetic" version="9.4.1">
          <artifact name="lucene-analysis-phonetic-9.4.1.jar">
             <sha256 value="57b21cf3c13a128d5817c369b43bb843a332ada8569bee99300b2439024daf1a" origin="Generated by Gradle"/>
@@ -2661,6 +2729,11 @@
             <sha256 value="30c2af3312daae2facbc43c49e4635b8f6426641d2870f7c1750ba2496c889fe" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.lucene" name="lucene-analysis-phonetic" version="9.5.0-snapshot-d19c3e2e0ed">
+         <artifact name="lucene-analysis-phonetic-9.5.0-snapshot-d19c3e2e0ed.jar">
+            <sha256 value="e65ce76f0ca6dd3ff3befde0a8a04951af46e195284ee6e75eab32ad25aabce3" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.lucene" name="lucene-analysis-smartcn" version="9.4.1">
          <artifact name="lucene-analysis-smartcn-9.4.1.jar">
             <sha256 value="ff65b08a942c4e9b4050f4d4a697cb550fe04e12464af14869c09f02fc75bab5" origin="Generated by Gradle"/>
@@ -2671,6 +2744,11 @@
             <sha256 value="ac101ee6815499d04a29693f089b141a2dd207a063e47968475bd829caa9bcf2" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.lucene" name="lucene-analysis-smartcn" version="9.5.0-snapshot-d19c3e2e0ed">
+         <artifact name="lucene-analysis-smartcn-9.5.0-snapshot-d19c3e2e0ed.jar">
+            <sha256 value="ca83467b4d20f00a4dbbf7c0ec35b977ed9dfc74498d563335e5a4d4c7d9474e" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.lucene" name="lucene-analysis-stempel" version="9.4.1">
          <artifact name="lucene-analysis-stempel-9.4.1.jar">
             <sha256 value="d32007019f35bef2f0ff47b192f4d2faac457bafa448336c905e0c1433e929e7" origin="Generated by Gradle"/>
@@ -2681,6 +2759,11 @@
             <sha256 value="7b8848fa29b4506f87e03c86699feb243cd258fe3ce92f0db8201417689e530a" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.lucene" name="lucene-analysis-stempel" version="9.5.0-snapshot-d19c3e2e0ed">
+         <artifact name="lucene-analysis-stempel-9.5.0-snapshot-d19c3e2e0ed.jar">
+            <sha256 value="d0ebadf4fee6e727197981f2295dc043ddd47077b58624e76af9d4b84e65464b" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.lucene" name="lucene-backward-codecs" version="9.4.1">
          <artifact name="lucene-backward-codecs-9.4.1.jar">
             <sha256 value="abdbf830fae812b27ddc98d8287e60582c8f9cffd8292a064f1f9aa8ab5e0ff0" origin="Generated by Gradle"/>
@@ -2691,6 +2774,11 @@
             <sha256 value="8a6dc9181693d61b25ddb057cec31abdb4d6573b12b4a096b9b5739093a45c91" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.lucene" name="lucene-backward-codecs" version="9.5.0-snapshot-d19c3e2e0ed">
+         <artifact name="lucene-backward-codecs-9.5.0-snapshot-d19c3e2e0ed.jar">
+            <sha256 value="e1ec8f11b9c83c0249eb145729921a08227bbcfe883c409a17309b5ab6a3824e" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.lucene" name="lucene-codecs" version="9.4.1">
          <artifact name="lucene-codecs-9.4.1.jar">
             <sha256 value="e69bcce29b43e9e09c85eb9e8a17128aa90a631c8a1b014b4efc7b8487222f4e" origin="Generated by Gradle"/>
@@ -2701,6 +2789,11 @@
             <sha256 value="a8bdb8e14bb6813837e5db84b4d163cd5bc6761bb370d5e6ade28561522eb587" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.lucene" name="lucene-codecs" version="9.5.0-snapshot-d19c3e2e0ed">
+         <artifact name="lucene-codecs-9.5.0-snapshot-d19c3e2e0ed.jar">
+            <sha256 value="3f57ada190b4561df53d0521df67a64204b315720ccc4cc04a5f621cdac21a42" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.lucene" name="lucene-core" version="9.4.1">
          <artifact name="lucene-core-9.4.1.jar">
             <sha256 value="992f847526c953f734a18295901b6fbb8977a1d99275e01ed219e7e57cc0ae34" origin="Generated by Gradle"/>
@@ -2711,6 +2804,11 @@
             <sha256 value="fd6db1bfe92c51439332230c1e95d2ac63633a9c3067d0355985594ff415431a" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.lucene" name="lucene-core" version="9.5.0-snapshot-d19c3e2e0ed">
+         <artifact name="lucene-core-9.5.0-snapshot-d19c3e2e0ed.jar">
+            <sha256 value="481c30915303e79477779b21fdf2769cd944dc5b5acef4ea2eac0bae5a5a59da" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.lucene" name="lucene-expressions" version="9.4.1">
          <artifact name="lucene-expressions-9.4.1.jar">
             <sha256 value="b037fbd8131443cf8a2f35f9c07eb68bf4d95cab01be67733a9225e7629a94e7" origin="Generated by Gradle"/>
@@ -2721,6 +2819,11 @@
             <sha256 value="ed0366731de0a57c77e34113df8ea34c365f93d1e60556ec20dc253a10f516f5" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.lucene" name="lucene-expressions" version="9.5.0-snapshot-d19c3e2e0ed">
+         <artifact name="lucene-expressions-9.5.0-snapshot-d19c3e2e0ed.jar">
+            <sha256 value="edc4e4d8398bf91bbda2717f835f0c94d816dcd26d7a7e2173a3563e8d25a719" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.lucene" name="lucene-grouping" version="9.4.1">
          <artifact name="lucene-grouping-9.4.1.jar">
             <sha256 value="dd88245285b240246280a4f922780a68831ed79df40a4884650d17902a33e361" origin="Generated by Gradle"/>
@@ -2731,6 +2834,11 @@
             <sha256 value="0312ba56249fb252ae8842fd148bc231bf4f3c20569a66f92105f1739815381d" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.lucene" name="lucene-grouping" version="9.5.0-snapshot-d19c3e2e0ed">
+         <artifact name="lucene-grouping-9.5.0-snapshot-d19c3e2e0ed.jar">
+            <sha256 value="91ac493aae11d211f2ffbe8457cf2f0c7aba68fcf2ddddb0d4da53b4d6b471cf" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.lucene" name="lucene-highlighter" version="9.4.1">
          <artifact name="lucene-highlighter-9.4.1.jar">
             <sha256 value="241af5dd27834d0a37e9ab2ef235aa125ca680181a78c51248511ee80c053240" origin="Generated by Gradle"/>
@@ -2741,6 +2849,11 @@
             <sha256 value="8512892073eb1c5f13a3fb1bf1b245cd66aeb1b45f7acfb47a83968a4141318d" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.lucene" name="lucene-highlighter" version="9.5.0-snapshot-d19c3e2e0ed">
+         <artifact name="lucene-highlighter-9.5.0-snapshot-d19c3e2e0ed.jar">
+            <sha256 value="ff7eda5d69ef6aaad97fa36472252fc992d0151a6c8c5aad920ac05540be8a19" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.lucene" name="lucene-join" version="9.4.1">
          <artifact name="lucene-join-9.4.1.jar">
             <sha256 value="1dfd06b72ce8ac45e1747a1781bf7f75681de366e17e88756c171e506f0719bb" origin="Generated by Gradle"/>
@@ -2751,6 +2864,11 @@
             <sha256 value="ae04eaa43bde57407a19aa9ab86038b8ee3b0d5c9d9333ecf63d283c306e6b0e" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.lucene" name="lucene-join" version="9.5.0-snapshot-d19c3e2e0ed">
+         <artifact name="lucene-join-9.5.0-snapshot-d19c3e2e0ed.jar">
+            <sha256 value="c29e7b0e2de53408c3d04b6562577679a3932a5f03c45b3d26ad22b1147f2132" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.lucene" name="lucene-memory" version="9.4.1">
          <artifact name="lucene-memory-9.4.1.jar">
             <sha256 value="2426d5a2475c9163dcc28b2e6f2624381abc2b83ede3f6140ab3833724ac60e6" origin="Generated by Gradle"/>
@@ -2761,6 +2879,11 @@
             <sha256 value="a0c9dee125bed499005893dcd9737a02b3aebe3cfa01ce6eb8d28a4e3b5dc5d4" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.lucene" name="lucene-memory" version="9.5.0-snapshot-d19c3e2e0ed">
+         <artifact name="lucene-memory-9.5.0-snapshot-d19c3e2e0ed.jar">
+            <sha256 value="f9ee01b34d8fa37e9b48426a00cf7e7fa947dd0356330ff9d14cefee345b2255" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.lucene" name="lucene-misc" version="9.4.1">
          <artifact name="lucene-misc-9.4.1.jar">
             <sha256 value="de8335139c1622234fc3c0d4973fcee9a7a7d53b26bc1b9ac72c3ba2c2f59b58" origin="Generated by Gradle"/>
@@ -2771,6 +2894,11 @@
             <sha256 value="c3a3f2a487cf62bad685527f9ace8d4728aa173ab56054c328e6e397584e1992" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.lucene" name="lucene-misc" version="9.5.0-snapshot-d19c3e2e0ed">
+         <artifact name="lucene-misc-9.5.0-snapshot-d19c3e2e0ed.jar">
+            <sha256 value="43c258c6016e776d70a248301f4586ab643491800ed1a887bacf1240aaad92a6" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.lucene" name="lucene-queries" version="9.4.1">
          <artifact name="lucene-queries-9.4.1.jar">
             <sha256 value="0fce03ae6070868fb7369763ea4a29aecdcc13c6342c7fc94c90d3ae29860a40" origin="Generated by Gradle"/>
@@ -2781,6 +2909,11 @@
             <sha256 value="1924525c4c706706baecbac7f9c0e434630b9ce595e55ae6aa63ca6d465fb453" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.lucene" name="lucene-queries" version="9.5.0-snapshot-d19c3e2e0ed">
+         <artifact name="lucene-queries-9.5.0-snapshot-d19c3e2e0ed.jar">
+            <sha256 value="7b1534514f1766fc6b6851fcd18e24cbece06373a03410cf71e2b76f23c99bc1" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.lucene" name="lucene-queryparser" version="9.4.1">
          <artifact name="lucene-queryparser-9.4.1.jar">
             <sha256 value="c7bf66dea0ae9e7dc840347fd7eaefa9250a91f018bc2503888afed08010adea" origin="Generated by Gradle"/>
@@ -2791,6 +2924,11 @@
             <sha256 value="dfc681a69bc60dd89e6bbf964a9b310b11e96de212da8597309b7caeb1cf591d" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.lucene" name="lucene-queryparser" version="9.5.0-snapshot-d19c3e2e0ed">
+         <artifact name="lucene-queryparser-9.5.0-snapshot-d19c3e2e0ed.jar">
+            <sha256 value="b4542f0109a7a53c366ff0f4b2b22c573ab4a2400e928d9f0d065c1f731d6e8a" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.lucene" name="lucene-sandbox" version="9.4.1">
          <artifact name="lucene-sandbox-9.4.1.jar">
             <sha256 value="ddc756a446986101ab17ac15aedfeb14588832afff38b86c07a13cd93e6e4890" origin="Generated by Gradle"/>
@@ -2801,6 +2939,11 @@
             <sha256 value="c381fe2797766d91b663a7cbee60e952a1c81dc37551445086f48f0dc5d47216" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.lucene" name="lucene-sandbox" version="9.5.0-snapshot-d19c3e2e0ed">
+         <artifact name="lucene-sandbox-9.5.0-snapshot-d19c3e2e0ed.jar">
+            <sha256 value="6c246106bd4ad6ac2a99ee8e87b76e25c7b17ffad333fbeed6206e3d23fba5ba" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.lucene" name="lucene-spatial-extras" version="9.4.1">
          <artifact name="lucene-spatial-extras-9.4.1.jar">
             <sha256 value="2fe68c8a9141af9f7a7862b576c88b74ad4509f5a2427cb423463d4233e76d4c" origin="Generated by Gradle"/>
@@ -2811,6 +2954,11 @@
             <sha256 value="c13713f828a03d47d3634e59abc7a53df13c23531d1b6b70c72d190ff64c5cd4" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.lucene" name="lucene-spatial-extras" version="9.5.0-snapshot-d19c3e2e0ed">
+         <artifact name="lucene-spatial-extras-9.5.0-snapshot-d19c3e2e0ed.jar">
+            <sha256 value="2ac4f5172e74a3772b68e86d5346f1edd88f4627d2fe3ce63e2b3d7717f36ee3" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.lucene" name="lucene-spatial3d" version="9.4.1">
          <artifact name="lucene-spatial3d-9.4.1.jar">
             <sha256 value="5911614baa3f6e75ee55d0ad63aa16286887766b2fda0654507ce6412d16353b" origin="Generated by Gradle"/>
@@ -2821,6 +2969,11 @@
             <sha256 value="c9688a9ab51d5a827260892de6435595a0d524ff9426288af806eb595a000dbb" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.lucene" name="lucene-spatial3d" version="9.5.0-snapshot-d19c3e2e0ed">
+         <artifact name="lucene-spatial3d-9.5.0-snapshot-d19c3e2e0ed.jar">
+            <sha256 value="4f4d96e385ddb5187714f5ab274d0e8cb74cab56dedbc5fa1363389c98a617a1" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.lucene" name="lucene-suggest" version="9.4.1">
          <artifact name="lucene-suggest-9.4.1.jar">
             <sha256 value="0bd1662bfaa60d6cba12e69b74f90e162900ed077320fe37fbf9e8f5cd78e59f" origin="Generated by Gradle"/>
@@ -2831,6 +2984,11 @@
             <sha256 value="0d56f92c5414149562ea50c6863ac50fcbbb51239d4a72c2bafbe852fb7e0a27" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.lucene" name="lucene-suggest" version="9.5.0-snapshot-d19c3e2e0ed">
+         <artifact name="lucene-suggest-9.5.0-snapshot-d19c3e2e0ed.jar">
+            <sha256 value="94878bf9c520702c0d5293922dd06866e25bedbd6f9bb823e28eb34dac907197" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.lucene" name="lucene-test-framework" version="9.4.1">
          <artifact name="lucene-test-framework-9.4.1.jar">
             <sha256 value="63850af6b753e0802cba7697db7821727b9b2e23eef11177c1ddbfb37ef3eaf2" origin="Generated by Gradle"/>
@@ -2841,6 +2999,11 @@
             <sha256 value="be318e426e3ce93d704fff41abc8ccd09bb1def69a40615ecc549c21895dd79d" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.lucene" name="lucene-test-framework" version="9.5.0-snapshot-d19c3e2e0ed">
+         <artifact name="lucene-test-framework-9.5.0-snapshot-d19c3e2e0ed.jar">
+            <sha256 value="561369612e4602cf99bcc53201b4bf9b95ed09096ba593e755416f1c96ff4bfb" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.maven" name="maven-model" version="3.6.2">
          <artifact name="maven-model-3.6.2.jar">
             <sha256 value="f4ada31d7217efc11d2264dec3716623cefa3440cfb2b6b1dcc640a825159a7d" origin="Generated by Gradle"/>
@@ -3329,24 +3492,34 @@
             <sha256 value="6cd91991323dd7b2fb28ca93d7ac12af5a86a2f53279e2b35827b30313fd0b9f" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.fusesource.leveldbjni" name="leveldbjni-all" version="1.8">
+       <component group="org.fusesource.leveldbjni" name="leveldbjni-all" version="1.8">
-         <artifact name="leveldbjni-all-1.8.jar">
+           <artifact name="leveldbjni-all-1.8.jar">
-            <sha256 value="c297213b0e6f9392305952753f3099a4c02e70b3656266fe01867e7b6c160ffe" origin="Generated by Gradle"/>
+               <sha256 value="c297213b0e6f9392305952753f3099a4c02e70b3656266fe01867e7b6c160ffe"
-         </artifact>
+                       origin="Generated by Gradle"/>
-      </component>
+           </artifact>
-      <component group="org.glassfish" name="javax.json" version="1.0.4">
+       </component>
-         <artifact name="javax.json-1.0.4.jar">
+       <component group="org.glassfish" name="javax.json" version="1.0.4">
-            <sha256 value="0e1dec40a1ede965941251eda968aeee052cc4f50378bc316cc48e8159bdbeb4" origin="Generated by Gradle"/>
+           <artifact name="javax.json-1.0.4.jar">
-         </artifact>
+               <sha256 value="0e1dec40a1ede965941251eda968aeee052cc4f50378bc316cc48e8159bdbeb4"
-      </component>
+                       origin="Generated by Gradle"/>
-      <component group="org.hamcrest" name="hamcrest" version="2.1">
+           </artifact>
-         <artifact name="hamcrest-2.1.jar">
+       </component>
-            <sha256 value="ba93b2e3a562322ba432f0a1b53addcc55cb188253319a020ed77f824e692050" origin="Generated by Gradle"/>
+       <component group="org.glassfish" name="javax.json" version="1.1.4">
-         </artifact>
+           <artifact name="javax.json-1.1.4.jar">
-      </component>
+               <sha256 value="17fdeb7e22375a7fb40bb0551306f6dcf2b5743078668adcdf6c642c9a9ec955"
-      <component group="org.hamcrest" name="hamcrest" version="2.2">
+                       origin="Generated by Gradle"/>
-         <artifact name="hamcrest-2.2.jar">
+           </artifact>
-            <sha256 value="5e62846a89f05cd78cd9c1a553f340d002458380c320455dd1f8fc5497a8a1c1" origin="Generated by Gradle"/>
+       </component>
+       <component group="org.hamcrest" name="hamcrest" version="2.1">
+           <artifact name="hamcrest-2.1.jar">
+               <sha256 value="ba93b2e3a562322ba432f0a1b53addcc55cb188253319a020ed77f824e692050"
+                       origin="Generated by Gradle"/>
+           </artifact>
+       </component>
+       <component group="org.hamcrest" name="hamcrest" version="2.2">
+           <artifact name="hamcrest-2.2.jar">
+               <sha256 value="5e62846a89f05cd78cd9c1a553f340d002458380c320455dd1f8fc5497a8a1c1"
+                       origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="org.hamcrest" name="hamcrest-core" version="1.3">

libs/h3/src/test/java/org/elasticsearch/h3/ParentChildNavigationTests.java

@@ -21,7 +21,6 @@ package org.elasticsearch.h3;
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
 
 import org.apache.lucene.geo.Point;
-import org.apache.lucene.spatial3d.geom.GeoArea;
 import org.apache.lucene.spatial3d.geom.GeoPoint;
 import org.apache.lucene.spatial3d.geom.GeoPolygon;
 import org.apache.lucene.spatial3d.geom.GeoPolygonFactory;
@@ -133,23 +132,13 @@ public class ParentChildNavigationTests extends ESTestCase {
     public void testNoChildrenIntersecting() {
         String[] h3Addresses = H3.getStringRes0Cells();
         String h3Address = RandomPicks.randomFrom(random(), h3Addresses);
-        // Once testIssue91915 is fixed, put upper limit of the loop to H3.MAX_H3_RES
+        for (int i = 1; i <= H3.MAX_H3_RES; i++) {
-        for (int i = 1; i <= 10; i++) {
             h3Addresses = H3.h3ToChildren(h3Address);
             assertIntersectingChildren(h3Address, h3Addresses);
             h3Address = RandomPicks.randomFrom(random(), h3Addresses);
         }
     }
 
-    public void testIssue91915() {
-        GeoPolygon polygon1 = getGeoPolygon("8ec82ea0650155f");
-        GeoPolygon polygon2 = getGeoPolygon("8ec82ea06501447");
-        // these polygons are disjoint but due to https://github.com/apache/lucene/issues/11883
-        // they are reported as intersects. Once this is fixed this test will fail, we should adjust
-        // testNoChildrenIntersecting
-        assertEquals("see https://github.com/elastic/elasticsearch/issues/91915", GeoArea.OVERLAPS, polygon1.getRelationship(polygon2));
-    }
-
     private void assertIntersectingChildren(String h3Address, String[] children) {
         int size = H3.h3ToNotIntersectingChildrenSize(h3Address);
         for (int i = 0; i < size; i++) {

libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentBuilder.java

@@ -435,6 +435,22 @@ public final class XContentBuilder implements Closeable, Flushable {
         return this;
     }
 
+    public XContentBuilder array(String name, byte[] values) throws IOException {
+        return field(name).values(values);
+    }
+
+    private XContentBuilder values(byte[] values) throws IOException {
+        if (values == null) {
+            return nullValue();
+        }
+        startArray();
+        for (byte b : values) {
+            value(b);
+        }
+        endArray();
+        return this;
+    }
+
     ////////////////////////////////////////////////////////////////////////////
     // Double
     //////////////////////////////////

modules/lang-expression/build.gradle

@@ -17,7 +17,7 @@ esplugin {
 dependencies {
   api "org.apache.lucene:lucene-expressions:${versions.lucene}"
   runtimeOnly "org.apache.lucene:lucene-codecs:${versions.lucene}"
-  runtimeOnly 'org.antlr:antlr4-runtime:4.5.1-1'
+  runtimeOnly "org.antlr:antlr4-runtime:${versions.antlr4}"
   runtimeOnly 'org.ow2.asm:asm:7.2'
   runtimeOnly 'org.ow2.asm:asm-commons:7.2'
   runtimeOnly 'org.ow2.asm:asm-tree:7.2'

modules/lang-painless/build.gradle

@@ -34,7 +34,7 @@ configurations {
 }
 
 dependencies {
-    api 'org.antlr:antlr4-runtime:4.5.3'
+    api "org.antlr:antlr4-runtime:${versions.antlr4}"
     api 'org.ow2.asm:asm-util:7.2'
     api 'org.ow2.asm:asm-tree:7.2'
     api 'org.ow2.asm:asm-commons:7.2'
@@ -174,7 +174,7 @@ configurations {
 }
 
 dependencies {
-    regenerate 'org.antlr:antlr4:4.5.3'
+  regenerate "org.antlr:antlr4:${versions.antlr4}"
 }
 
 String grammarPath = 'src/main/antlr'
@@ -274,7 +274,7 @@ configurations {
 }
 
 dependencies {
-    regenerate 'org.antlr:antlr4:4.5.3'
+  regenerate "org.antlr:antlr4:${versions.antlr4}"
 }
 
 String suggestGrammarPath = 'src/main/antlr'

modules/lang-painless/src/main/java/module-info.java

@@ -13,7 +13,7 @@ module org.elasticsearch.painless {
     requires org.elasticsearch.server;
     requires org.elasticsearch.xcontent;
 
-    requires antlr4.runtime;
+    requires org.antlr.antlr4.runtime;
     requires org.apache.lucene.core;
     requires org.objectweb.asm;
     requires org.objectweb.asm.commons;

modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java

@@ -11,6 +11,7 @@ import org.antlr.v4.runtime.tree.AbstractParseTreeVisitor;
  * @param <T> The return type of the visit operation. Use {@link Void} for
  * operations with no return type.
  */
+@SuppressWarnings("CheckReturnValue")
 class PainlessParserBaseVisitor<T> extends AbstractParseTreeVisitor<T> implements PainlessParserVisitor<T> {
     /**
      * {@inheritDoc}

server/src/main/java/org/elasticsearch/Version.java

@@ -131,7 +131,8 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_8_5_4 = new Version(8_05_04_99, TransportVersion.V_8_5_4, org.apache.lucene.util.Version.LUCENE_9_4_2);
     public static final Version V_8_6_0 = new Version(8_06_00_99, TransportVersion.V_8_6_0, org.apache.lucene.util.Version.LUCENE_9_4_2);
     public static final Version V_8_6_1 = new Version(8_06_01_99, TransportVersion.V_8_6_1, org.apache.lucene.util.Version.LUCENE_9_4_2);
-    public static final Version V_8_7_0 = new Version(8_07_00_99, TransportVersion.V_8_7_0, org.apache.lucene.util.Version.LUCENE_9_4_2);
+    public static final Version V_8_7_0 = new Version(8_07_00_99, TransportVersion.V_8_7_0, org.apache.lucene.util.Version.LUCENE_9_5_0);
+
     public static final Version CURRENT = V_8_7_0;
 
     private static final Map<Integer, Version> VERSION_IDS;

server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java

@@ -20,6 +20,7 @@ import org.apache.lucene.codecs.StoredFieldsReader;
 import org.apache.lucene.codecs.TermVectorsReader;
 import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat;
 import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.ByteVectorValues;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.FieldInfo;
@@ -137,7 +138,7 @@ final class IndexDiskUsageAnalyzer {
         final int skipMask = 0x1FF; // 511
         while (docID < reader.maxDoc()) {
             cancellationChecker.logEvent();
-            storedFieldsReader.visitDocument(docID, visitor);
+            storedFieldsReader.document(docID, visitor);
             // As we already estimate the size of stored fields, we can trade off the accuracy for the speed of the estimate.
             // Here we only visit 1/11 documents instead of all documents. Ideally, we should visit 1 doc then skip 10 docs
             // to avoid missing many skew documents. But, documents are stored in chunks in compressed format and a chunk can
@@ -525,23 +526,47 @@ final class IndexDiskUsageAnalyzer {
             cancellationChecker.checkForCancellation();
             directory.resetBytesRead();
             if (field.getVectorDimension() > 0) {
-                iterateDocValues(reader.maxDoc(), () -> vectorReader.getVectorValues(field.name), vectors -> {
+                switch (field.getVectorEncoding()) {
-                    cancellationChecker.logEvent();
+                    case BYTE -> {
-                    vectors.vectorValue();
+                        iterateDocValues(reader.maxDoc(), () -> vectorReader.getByteVectorValues(field.name), vectors -> {
-                });
+                            cancellationChecker.logEvent();
+                            vectors.vectorValue();
+                        });
 
-                // do a couple of randomized searches to figure out min and max offsets of index file
+                        // do a couple of randomized searches to figure out min and max offsets of index file
-                VectorValues vectorValues = vectorReader.getVectorValues(field.name);
+                        ByteVectorValues vectorValues = vectorReader.getByteVectorValues(field.name);
-                int numDocsToVisit = reader.maxDoc() < 10 ? reader.maxDoc() : 10 * (int) Math.log10(reader.maxDoc());
+                        int numDocsToVisit = reader.maxDoc() < 10 ? reader.maxDoc() : 10 * (int) Math.log10(reader.maxDoc());
-                int skipFactor = Math.max(reader.maxDoc() / numDocsToVisit, 1);
+                        int skipFactor = Math.max(reader.maxDoc() / numDocsToVisit, 1);
-                for (int i = 0; i < reader.maxDoc(); i += skipFactor) {
+                        for (int i = 0; i < reader.maxDoc(); i += skipFactor) {
-                    if ((i = vectorValues.advance(i)) == DocIdSetIterator.NO_MORE_DOCS) {
+                            if ((i = vectorValues.advance(i)) == DocIdSetIterator.NO_MORE_DOCS) {
-                        break;
+                                break;
+                            }
+                            cancellationChecker.checkForCancellation();
+                            vectorReader.search(field.name, vectorValues.vectorValue(), 100, null, Integer.MAX_VALUE);
+                        }
+                        stats.addKnnVectors(field.name, directory.getBytesRead());
+                    }
+                    case FLOAT32 -> {
+                        iterateDocValues(reader.maxDoc(), () -> vectorReader.getVectorValues(field.name), vectors -> {
+                            cancellationChecker.logEvent();
+                            vectors.vectorValue();
+                        });
+
+                        // do a couple of randomized searches to figure out min and max offsets of index file
+                        VectorValues vectorValues = vectorReader.getVectorValues(field.name);
+                        int numDocsToVisit = reader.maxDoc() < 10 ? reader.maxDoc() : 10 * (int) Math.log10(reader.maxDoc());
+                        int skipFactor = Math.max(reader.maxDoc() / numDocsToVisit, 1);
+                        for (int i = 0; i < reader.maxDoc(); i += skipFactor) {
+                            if ((i = vectorValues.advance(i)) == DocIdSetIterator.NO_MORE_DOCS) {
+                                break;
+                            }
+                            cancellationChecker.checkForCancellation();
+                            vectorReader.search(field.name, vectorValues.vectorValue(), 100, null, Integer.MAX_VALUE);
+                        }
+                        stats.addKnnVectors(field.name, directory.getBytesRead());
                     }
-                    cancellationChecker.checkForCancellation();
-                    vectorReader.search(field.name, vectorValues.vectorValue(), 100, null, Integer.MAX_VALUE);
                 }
-                stats.addKnnVectors(field.name, directory.getBytesRead());
+
             }
         }
     }

server/src/main/java/org/elasticsearch/common/lucene/Lucene.java

@@ -86,7 +86,7 @@ import java.util.List;
 import java.util.Map;
 
 public class Lucene {
-    public static final String LATEST_CODEC = "Lucene94";
+    public static final String LATEST_CODEC = "Lucene95";
 
     public static final String SOFT_DELETES_FIELD = "__soft_deletes";
 

server/src/main/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQuery.java

@@ -299,7 +299,29 @@ public class MultiPhrasePrefixQuery extends Query {
     @Override
     public void visit(QueryVisitor visitor) {
         if (visitor.acceptField(field)) {
-            visitor.visitLeaf(this);    // TODO implement term visiting
+            visitor = visitor.getSubVisitor(BooleanClause.Occur.MUST, this);
+            for (int i = 0; i < termArrays.size() - 1; i++) {
+                if (termArrays.get(i).length == 1) {
+                    visitor.consumeTerms(this, termArrays.get(i)[0]);
+                } else {
+                    QueryVisitor shouldVisitor = visitor.getSubVisitor(BooleanClause.Occur.SHOULD, this);
+                    shouldVisitor.consumeTerms(this, termArrays.get(i));
+                }
+            }
+            /* We don't report automata here because this breaks the unified highlighter,
+               which extracts automata separately from phrases. MPPQ gets rewritten to a
+               SpanMTQQuery by the PhraseHelper in any case, so highlighting is taken
+               care of there instead.  If we extract automata here then the trailing prefix
+               word will be highlighted wherever it appears in the document, instead of only
+               as part of a phrase. This can be re-instated once we switch to using Matches
+               to highlight.
+            for (Term prefixTerm : termArrays.get(termArrays.size() - 1)) {
+                visitor.consumeTermsMatching(this, field, () -> {
+                    CompiledAutomaton ca = new CompiledAutomaton(PrefixQuery.toAutomaton(prefixTerm.bytes()));
+                    return ca.runAutomaton;
+                });
+            }
+            */
         }
     }
 }

server/src/main/java/org/elasticsearch/index/codec/CodecService.java

@@ -9,7 +9,7 @@
 package org.elasticsearch.index.codec;
 
 import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.lucene94.Lucene94Codec;
+import org.apache.lucene.codecs.lucene95.Lucene95Codec;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.mapper.MapperService;
@@ -35,11 +35,11 @@ public class CodecService {
     public CodecService(@Nullable MapperService mapperService, BigArrays bigArrays) {
         final var codecs = new HashMap<String, Codec>();
         if (mapperService == null) {
-            codecs.put(DEFAULT_CODEC, new Lucene94Codec());
+            codecs.put(DEFAULT_CODEC, new Lucene95Codec());
-            codecs.put(BEST_COMPRESSION_CODEC, new Lucene94Codec(Lucene94Codec.Mode.BEST_COMPRESSION));
+            codecs.put(BEST_COMPRESSION_CODEC, new Lucene95Codec(Lucene95Codec.Mode.BEST_COMPRESSION));
         } else {
-            codecs.put(DEFAULT_CODEC, new PerFieldMapperCodec(Lucene94Codec.Mode.BEST_SPEED, mapperService, bigArrays));
+            codecs.put(DEFAULT_CODEC, new PerFieldMapperCodec(Lucene95Codec.Mode.BEST_SPEED, mapperService, bigArrays));
-            codecs.put(BEST_COMPRESSION_CODEC, new PerFieldMapperCodec(Lucene94Codec.Mode.BEST_COMPRESSION, mapperService, bigArrays));
+            codecs.put(BEST_COMPRESSION_CODEC, new PerFieldMapperCodec(Lucene95Codec.Mode.BEST_COMPRESSION, mapperService, bigArrays));
         }
         codecs.put(LUCENE_DEFAULT_CODEC, Codec.getDefault());
         for (String codec : Codec.availableCodecs()) {

server/src/main/java/org/elasticsearch/index/codec/PerFieldMapperCodec.java

@@ -13,7 +13,7 @@ import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.codecs.KnnVectorsFormat;
 import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
-import org.apache.lucene.codecs.lucene94.Lucene94Codec;
+import org.apache.lucene.codecs.lucene95.Lucene95Codec;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.index.IndexMode;
@@ -37,7 +37,7 @@ import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
  * per index in real time via the mapping API. If no specific postings format or vector format is
  * configured for a specific field the default postings or vector format is used.
  */
-public class PerFieldMapperCodec extends Lucene94Codec {
+public class PerFieldMapperCodec extends Lucene95Codec {
 
     private final MapperService mapperService;
     private final DocValuesFormat docValuesFormat = new Lucene90DocValuesFormat();

server/src/main/java/org/elasticsearch/index/engine/IdStoredFieldLoader.java

@@ -35,7 +35,7 @@ final class IdStoredFieldLoader {
 
     private static CheckedBiConsumer<Integer, StoredFieldVisitor, IOException> getStoredFieldsReader(LeafReader in) {
         if (in instanceof SequentialStoredFieldsLeafReader) {
-            return (((SequentialStoredFieldsLeafReader) in).getSequentialStoredFieldsReader())::visitDocument;
+            return (((SequentialStoredFieldsLeafReader) in).getSequentialStoredFieldsReader())::document;
         }
         throw new IllegalArgumentException("Requires a SequentialStoredFieldsReader, got " + in.getClass());
     }

server/src/main/java/org/elasticsearch/index/engine/LuceneChangesSnapshot.java

@@ -338,7 +338,7 @@ final class LuceneChangesSnapshot implements Translog.Snapshot {
             assert singleConsumer : "Sequential access optimization must not be enabled for multiple consumers";
             assert parallelArray.useSequentialStoredFieldsReader;
             assert storedFieldsReaderOrd == leaf.ord : storedFieldsReaderOrd + " != " + leaf.ord;
-            storedFieldsReader.visitDocument(segmentDocID, fields);
+            storedFieldsReader.document(segmentDocID, fields);
         } else {
             leaf.reader().document(segmentDocID, fields);
         }

server/src/main/java/org/elasticsearch/index/engine/RecoverySourcePruneMergePolicy.java

@@ -195,8 +195,8 @@ final class RecoverySourcePruneMergePolicy extends OneMergeWrappingMergePolicy {
             }
 
             @Override
-            public void visitDocument(int docID, StoredFieldVisitor visitor) throws IOException {
+            public void document(int docID, StoredFieldVisitor visitor) throws IOException {
-                in.visitDocument(docID, visitor);
+                in.document(docID, visitor);
             }
 
             @Override
@@ -220,11 +220,11 @@ final class RecoverySourcePruneMergePolicy extends OneMergeWrappingMergePolicy {
             }
 
             @Override
-            public void visitDocument(int docID, StoredFieldVisitor visitor) throws IOException {
+            public void document(int docID, StoredFieldVisitor visitor) throws IOException {
                 if (recoverySourceToKeep != null && recoverySourceToKeep.get(docID)) {
-                    super.visitDocument(docID, visitor);
+                    super.document(docID, visitor);
                 } else {
-                    super.visitDocument(docID, new FilterStoredFieldVisitor(visitor) {
+                    super.document(docID, new FilterStoredFieldVisitor(visitor) {
                         @Override
                         public Status needsField(FieldInfo fieldInfo) throws IOException {
                             if (recoverySourceField.equals(fieldInfo.name)) {

server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java

@@ -11,6 +11,7 @@ package org.elasticsearch.index.engine;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.index.BaseTermsEnum;
 import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.ByteVectorValues;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.FieldInfo;
@@ -30,6 +31,8 @@ import org.apache.lucene.index.SortedDocValues;
 import org.apache.lucene.index.SortedNumericDocValues;
 import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.index.StoredFieldVisitor;
+import org.apache.lucene.index.StoredFields;
+import org.apache.lucene.index.TermVectors;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.index.VectorEncoding;
@@ -349,11 +352,21 @@ final class TranslogDirectoryReader extends DirectoryReader {
             return getDelegate().getVectorValues(field);
         }
 
+        @Override
+        public ByteVectorValues getByteVectorValues(String field) throws IOException {
+            return getDelegate().getByteVectorValues(field);
+        }
+
         @Override
         public TopDocs searchNearestVectors(String field, float[] target, int k, Bits acceptDocs, int visitedLimit) throws IOException {
             return getDelegate().searchNearestVectors(field, target, k, acceptDocs, visitedLimit);
         }
 
+        @Override
+        public TopDocs searchNearestVectors(String field, BytesRef target, int k, Bits acceptDocs, int visitedLimit) throws IOException {
+            return getDelegate().searchNearestVectors(field, target, k, acceptDocs, visitedLimit);
+        }
+
         @Override
         public FieldInfos getFieldInfos() {
             return getDelegate().getFieldInfos();
@@ -382,6 +395,16 @@ final class TranslogDirectoryReader extends DirectoryReader {
             return getDelegate().getTermVectors(docID);
         }
 
+        @Override
+        public TermVectors termVectors() throws IOException {
+            return getDelegate().termVectors();
+        }
+
+        @Override
+        public StoredFields storedFields() throws IOException {
+            return getDelegate().storedFields();
+        }
+
         @Override
         public int numDocs() {
             return 1;

server/src/main/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoader.java

@@ -108,7 +108,7 @@ public abstract class StoredFieldLoader {
     private static CheckedBiConsumer<Integer, FieldsVisitor, IOException> sequentialReader(LeafReaderContext ctx) {
         LeafReader leafReader = ctx.reader();
         if (leafReader instanceof SequentialStoredFieldsLeafReader lf) {
-            return lf.getSequentialStoredFieldsReader()::visitDocument;
+            return lf.getSequentialStoredFieldsReader()::document;
         }
         return leafReader::document;
     }

server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java

@@ -17,8 +17,8 @@ import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.PointValues;
 import org.apache.lucene.index.SortedNumericDocValues;
-import org.apache.lucene.sandbox.search.IndexSortSortedNumericDocValuesRangeQuery;
 import org.apache.lucene.search.IndexOrDocValuesQuery;
+import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.Version;

server/src/main/java/org/elasticsearch/index/mapper/DocumentLeafReader.java

@@ -9,6 +9,7 @@
 package org.elasticsearch.index.mapper;
 
 import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.ByteVectorValues;
 import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
@@ -24,6 +25,8 @@ import org.apache.lucene.index.SortedDocValues;
 import org.apache.lucene.index.SortedNumericDocValues;
 import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.index.StoredFieldVisitor;
+import org.apache.lucene.index.StoredFields;
+import org.apache.lucene.index.TermVectors;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.VectorEncoding;
 import org.apache.lucene.index.VectorSimilarityFunction;
@@ -46,10 +49,9 @@ import java.util.function.Consumer;
 /**
  * A {@link LeafReader} over a lucene document that exposes doc values and stored fields.
  * Note that unlike lucene's {@link MemoryIndex} implementation, this holds no state and
- * does not attempt to do any analysis on text fields.  It also supports stored
+ * does not attempt to do any analysis on text fields.  It is used to back index-time
- * fields where MemoryIndex does not.  It is used to back index-time scripts that
+ * scripts that reference field data and stored fields from a document that has not yet
- * reference field data and stored fields from a document that has not yet been
+ * been indexed.
- * indexed.
  */
 class DocumentLeafReader extends LeafReader {
 
@@ -175,6 +177,11 @@ class DocumentLeafReader extends LeafReader {
         }
     }
 
+    @Override
+    public StoredFields storedFields() throws IOException {
+        throw new UnsupportedOperationException();
+    }
+
     @Override
     public CacheHelper getCoreCacheHelper() {
         throw new UnsupportedOperationException();
@@ -240,6 +247,21 @@ class DocumentLeafReader extends LeafReader {
         throw new UnsupportedOperationException();
     }
 
+    @Override
+    public ByteVectorValues getByteVectorValues(String field) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public TopDocs searchNearestVectors(String field, BytesRef target, int k, Bits acceptDocs, int visitedLimit) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public TermVectors termVectors() throws IOException {
+        throw new UnsupportedOperationException();
+    }
+
     @Override
     public CacheHelper getReaderCacheHelper() {
         throw new UnsupportedOperationException();

server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java

@@ -17,8 +17,8 @@ import org.apache.lucene.document.StoredField;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.sandbox.document.HalfFloatPoint;
-import org.apache.lucene.sandbox.search.IndexSortSortedNumericDocValuesRangeQuery;
 import org.apache.lucene.search.IndexOrDocValuesQuery;
+import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;

server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java

@@ -9,19 +9,21 @@
 package org.elasticsearch.index.mapper.vectors;
 
 import org.apache.lucene.codecs.KnnVectorsFormat;
-import org.apache.lucene.codecs.lucene94.Lucene94HnswVectorsFormat;
+import org.apache.lucene.codecs.lucene95.Lucene95HnswVectorsFormat;
 import org.apache.lucene.document.BinaryDocValuesField;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.KnnByteVectorField;
 import org.apache.lucene.document.KnnVectorField;
 import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.ByteVectorValues;
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.VectorSimilarityFunction;
 import org.apache.lucene.index.VectorValues;
 import org.apache.lucene.search.FieldExistsQuery;
+import org.apache.lucene.search.KnnByteVectorQuery;
 import org.apache.lucene.search.KnnVectorQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.VectorUtil;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.index.fielddata.FieldDataContext;
@@ -43,6 +45,7 @@ import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentParser.Token;
 
 import java.io.IOException;
@@ -51,6 +54,7 @@ import java.time.ZoneId;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
+import java.util.function.Function;
 import java.util.stream.Stream;
 
 import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
@@ -178,13 +182,18 @@ public class DenseVectorFieldMapper extends FieldMapper {
             }
 
             @Override
-            public float readValue(ByteBuffer byteBuffer) {
+            public void readAndWriteValue(ByteBuffer byteBuffer, XContentBuilder b) throws IOException {
-                return byteBuffer.get();
+                b.value(byteBuffer.get());
+            }
+
+            @Override
+            KnnByteVectorField createKnnVectorField(String name, BytesRef vector, VectorSimilarityFunction function) {
+                return new KnnByteVectorField(name, vector, function);
             }
 
             @Override
             KnnVectorField createKnnVectorField(String name, float[] vector, VectorSimilarityFunction function) {
-                return new KnnVectorField(name, VectorUtil.toBytesRef(vector), function);
+                throw new IllegalArgumentException("cannot create a float vector field from byte");
             }
 
             @Override
@@ -245,7 +254,11 @@ public class DenseVectorFieldMapper extends FieldMapper {
             }
 
             @Override
-            void checkVectorMagnitude(VectorSimilarity similarity, float[] vector, float squaredMagnitude) {
+            void checkVectorMagnitude(
+                VectorSimilarity similarity,
+                Function<StringBuilder, StringBuilder> appender,
+                float squaredMagnitude
+            ) {
                 StringBuilder errorBuilder = null;
 
                 if (similarity == VectorSimilarity.COSINE && Math.sqrt(squaredMagnitude) == 0.0f) {
@@ -255,9 +268,91 @@ public class DenseVectorFieldMapper extends FieldMapper {
                 }
 
                 if (errorBuilder != null) {
-                    throw new IllegalArgumentException(appendErrorElements(errorBuilder, vector).toString());
+                    throw new IllegalArgumentException(appender.apply(errorBuilder).toString());
                 }
             }
+
+            @Override
+            public Field parseKnnVector(DocumentParserContext context, DenseVectorFieldMapper fieldMapper) throws IOException {
+                int index = 0;
+                byte[] vector = new byte[fieldMapper.dims];
+                float squaredMagnitude = 0;
+                for (Token token = context.parser().nextToken(); token != Token.END_ARRAY; token = context.parser().nextToken()) {
+                    fieldMapper.checkDimensionExceeded(index, context);
+                    ensureExpectedToken(Token.VALUE_NUMBER, token, context.parser());
+                    final int value;
+                    if (context.parser().numberType() != XContentParser.NumberType.INT) {
+                        float floatValue = context.parser().floatValue(true);
+                        if (floatValue % 1.0f != 0.0f) {
+                            throw new IllegalArgumentException(
+                                "element_type ["
+                                    + this
+                                    + "] vectors only support non-decimal values but found decimal value ["
+                                    + floatValue
+                                    + "] at dim ["
+                                    + index
+                                    + "];"
+                            );
+                        }
+                        value = (int) floatValue;
+                    } else {
+                        value = context.parser().intValue(true);
+                    }
+                    if (value < Byte.MIN_VALUE || value > Byte.MAX_VALUE) {
+                        throw new IllegalArgumentException(
+                            "element_type ["
+                                + this
+                                + "] vectors only support integers between ["
+                                + Byte.MIN_VALUE
+                                + ", "
+                                + Byte.MAX_VALUE
+                                + "] but found ["
+                                + value
+                                + "] at dim ["
+                                + index
+                                + "];"
+                        );
+                    }
+                    vector[index++] = (byte) value;
+                    squaredMagnitude += value * value;
+                }
+                fieldMapper.checkDimensionMatches(index, context);
+                BytesRef bytesVector = new BytesRef(vector);
+                checkVectorMagnitude(fieldMapper.similarity, errorByteElementsAppender(bytesVector), squaredMagnitude);
+                return createKnnVectorField(fieldMapper.fieldType().name(), bytesVector, fieldMapper.similarity.function);
+            }
+
+            @Override
+            double parseKnnVectorToByteBuffer(DocumentParserContext context, DenseVectorFieldMapper fieldMapper, ByteBuffer byteBuffer)
+                throws IOException {
+                double dotProduct = 0f;
+                int index = 0;
+                for (Token token = context.parser().nextToken(); token != Token.END_ARRAY; token = context.parser().nextToken()) {
+                    fieldMapper.checkDimensionExceeded(index, context);
+                    ensureExpectedToken(Token.VALUE_NUMBER, token, context.parser());
+                    int value = context.parser().intValue(true);
+                    if (value < Byte.MIN_VALUE || value > Byte.MAX_VALUE) {
+                        throw new IllegalArgumentException(
+                            "element_type ["
+                                + this
+                                + "] vectors only support integers between ["
+                                + Byte.MIN_VALUE
+                                + ", "
+                                + Byte.MAX_VALUE
+                                + "] but found ["
+                                + value
+                                + "] at dim ["
+                                + index
+                                + "];"
+                        );
+                    }
+                    byteBuffer.put((byte) value);
+                    dotProduct += value * value;
+                    index++;
+                }
+                fieldMapper.checkDimensionMatches(index, context);
+                return dotProduct;
+            }
         },
 
         FLOAT(4) {
@@ -273,8 +368,8 @@ public class DenseVectorFieldMapper extends FieldMapper {
             }
 
             @Override
-            public float readValue(ByteBuffer byteBuffer) {
+            public void readAndWriteValue(ByteBuffer byteBuffer, XContentBuilder b) throws IOException {
-                return byteBuffer.getFloat();
+                b.value(byteBuffer.getFloat());
             }
 
             @Override
@@ -282,6 +377,11 @@ public class DenseVectorFieldMapper extends FieldMapper {
                 return new KnnVectorField(name, vector, function);
             }
 
+            @Override
+            KnnByteVectorField createKnnVectorField(String name, BytesRef vector, VectorSimilarityFunction function) {
+                throw new IllegalArgumentException("cannot create a byte vector field from float");
+            }
+
             @Override
             IndexFieldData.Builder fielddataBuilder(DenseVectorFieldType denseVectorFieldType, FieldDataContext fieldDataContext) {
                 return new VectorIndexFieldData.Builder(
@@ -300,7 +400,11 @@ public class DenseVectorFieldMapper extends FieldMapper {
             }
 
             @Override
-            void checkVectorMagnitude(VectorSimilarity similarity, float[] vector, float squaredMagnitude) {
+            void checkVectorMagnitude(
+                VectorSimilarity similarity,
+                Function<StringBuilder, StringBuilder> appender,
+                float squaredMagnitude
+            ) {
                 StringBuilder errorBuilder = null;
 
                 if (similarity == VectorSimilarity.DOT_PRODUCT && Math.abs(squaredMagnitude - 1.0f) > 1e-4f) {
@@ -314,9 +418,48 @@ public class DenseVectorFieldMapper extends FieldMapper {
                 }
 
                 if (errorBuilder != null) {
-                    throw new IllegalArgumentException(appendErrorElements(errorBuilder, vector).toString());
+                    throw new IllegalArgumentException(appender.apply(errorBuilder).toString());
                 }
             }
+
+            @Override
+            public Field parseKnnVector(DocumentParserContext context, DenseVectorFieldMapper fieldMapper) throws IOException {
+                int index = 0;
+                float[] vector = new float[fieldMapper.dims];
+                float squaredMagnitude = 0;
+                for (Token token = context.parser().nextToken(); token != Token.END_ARRAY; token = context.parser().nextToken()) {
+                    fieldMapper.checkDimensionExceeded(index, context);
+                    ensureExpectedToken(Token.VALUE_NUMBER, token, context.parser());
+
+                    float value = context.parser().floatValue(true);
+                    vector[index++] = value;
+                    squaredMagnitude += value * value;
+                }
+                fieldMapper.checkDimensionMatches(index, context);
+                checkVectorBounds(vector);
+                checkVectorMagnitude(fieldMapper.similarity, errorFloatElementsAppender(vector), squaredMagnitude);
+                return createKnnVectorField(fieldMapper.fieldType().name(), vector, fieldMapper.similarity.function);
+            }
+
+            @Override
+            double parseKnnVectorToByteBuffer(DocumentParserContext context, DenseVectorFieldMapper fieldMapper, ByteBuffer byteBuffer)
+                throws IOException {
+                double dotProduct = 0f;
+                int index = 0;
+                float[] vector = new float[fieldMapper.dims];
+                for (Token token = context.parser().nextToken(); token != Token.END_ARRAY; token = context.parser().nextToken()) {
+                    fieldMapper.checkDimensionExceeded(index, context);
+                    ensureExpectedToken(Token.VALUE_NUMBER, token, context.parser());
+                    float value = context.parser().floatValue(true);
+                    vector[index] = value;
+                    byteBuffer.putFloat(value);
+                    dotProduct += value * value;
+                    index++;
+                }
+                fieldMapper.checkDimensionMatches(index, context);
+                checkVectorBounds(vector);
+                return dotProduct;
+            }
         };
 
         final int elementBytes;
@@ -327,15 +470,26 @@ public class DenseVectorFieldMapper extends FieldMapper {
 
         public abstract void writeValue(ByteBuffer byteBuffer, float value);
 
-        public abstract float readValue(ByteBuffer byteBuffer);
+        public abstract void readAndWriteValue(ByteBuffer byteBuffer, XContentBuilder b) throws IOException;
 
         abstract KnnVectorField createKnnVectorField(String name, float[] vector, VectorSimilarityFunction function);
 
+        abstract KnnByteVectorField createKnnVectorField(String name, BytesRef vector, VectorSimilarityFunction function);
+
         abstract IndexFieldData.Builder fielddataBuilder(DenseVectorFieldType denseVectorFieldType, FieldDataContext fieldDataContext);
 
+        abstract Field parseKnnVector(DocumentParserContext context, DenseVectorFieldMapper fieldMapper) throws IOException;
+
+        abstract double parseKnnVectorToByteBuffer(DocumentParserContext context, DenseVectorFieldMapper fieldMapper, ByteBuffer byteBuffer)
+            throws IOException;
+
         public abstract void checkVectorBounds(float[] vector);
 
-        abstract void checkVectorMagnitude(VectorSimilarity similarity, float[] vector, float squaredMagnitude);
+        abstract void checkVectorMagnitude(
+            VectorSimilarity similarity,
+            Function<StringBuilder, StringBuilder> errorElementsAppender,
+            float squaredMagnitude
+        );
 
         void checkNanAndInfinite(float[] vector) {
             StringBuilder errorBuilder = null;
@@ -384,6 +538,30 @@ public class DenseVectorFieldMapper extends FieldMapper {
             errorBuilder.append("]");
             return errorBuilder;
         }
+
+        StringBuilder appendErrorElements(StringBuilder errorBuilder, BytesRef vector) {
+            // Include the first five elements of the invalid vector in the error message
+            errorBuilder.append(" Preview of invalid vector: [");
+            for (int i = vector.offset; i < vector.offset + Math.min(5, vector.length); i++) {
+                if (i > vector.offset) {
+                    errorBuilder.append(", ");
+                }
+                errorBuilder.append(vector.bytes[i]);
+            }
+            if (vector.length >= 5) {
+                errorBuilder.append(", ...");
+            }
+            errorBuilder.append("]");
+            return errorBuilder;
+        }
+
+        Function<StringBuilder, StringBuilder> errorFloatElementsAppender(float[] vector) {
+            return sb -> appendErrorElements(sb, vector);
+        }
+
+        Function<StringBuilder, StringBuilder> errorByteElementsAppender(BytesRef vector) {
+            return sb -> appendErrorElements(sb, vector);
+        }
     }
 
     static final Map<String, ElementType> namesToElementType = Map.of(
@@ -546,7 +724,7 @@ public class DenseVectorFieldMapper extends FieldMapper {
             throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support term queries");
         }
 
-        public KnnVectorQuery createKnnQuery(float[] queryVector, int numCands, Query filter) {
+        public Query createKnnQuery(BytesRef queryVector, int numCands, Query filter) {
             if (isIndexed() == false) {
                 throw new IllegalArgumentException(
                     "to perform knn search on field [" + name() + "], its mapping must have [index] set to [true]"
@@ -559,6 +737,35 @@ public class DenseVectorFieldMapper extends FieldMapper {
                 );
             }
 
+            if (elementType != ElementType.BYTE) {
+                throw new IllegalArgumentException(
+                    "only [" + ElementType.BYTE + "] elements are supported when querying field [" + name() + "]"
+                );
+            }
+
+            if (similarity == VectorSimilarity.DOT_PRODUCT || similarity == VectorSimilarity.COSINE) {
+                float squaredMagnitude = 0.0f;
+                for (int i = queryVector.offset; i < queryVector.offset + queryVector.length; i++) {
+                    squaredMagnitude += queryVector.bytes[i] * queryVector.bytes[i];
+                }
+                elementType.checkVectorMagnitude(similarity, elementType.errorByteElementsAppender(queryVector), squaredMagnitude);
+            }
+
+            return new KnnByteVectorQuery(name(), queryVector, numCands, filter);
+        }
+
+        public Query createKnnQuery(float[] queryVector, int numCands, Query filter) {
+            if (isIndexed() == false) {
+                throw new IllegalArgumentException(
+                    "to perform knn search on field [" + name() + "], its mapping must have [index] set to [true]"
+                );
+            }
+
+            if (queryVector.length != dims) {
+                throw new IllegalArgumentException(
+                    "the query vector has a different dimension [" + queryVector.length + "] than the index vectors [" + dims + "]"
+                );
+            }
             elementType.checkVectorBounds(queryVector);
 
             if (similarity == VectorSimilarity.DOT_PRODUCT || similarity == VectorSimilarity.COSINE) {
@@ -566,10 +773,18 @@ public class DenseVectorFieldMapper extends FieldMapper {
                 for (float e : queryVector) {
                     squaredMagnitude += e * e;
                 }
-                elementType.checkVectorMagnitude(similarity, queryVector, squaredMagnitude);
+                elementType.checkVectorMagnitude(similarity, elementType.errorFloatElementsAppender(queryVector), squaredMagnitude);
             }
-
+            return switch (elementType) {
-            return new KnnVectorQuery(name(), queryVector, numCands, filter);
+                case BYTE -> {
+                    byte[] bytes = new byte[queryVector.length];
+                    for (int i = 0; i < queryVector.length; i++) {
+                        bytes[i] = (byte) queryVector[i];
+                    }
+                    yield new KnnByteVectorQuery(name(), new BytesRef(bytes), numCands, filter);
+                }
+                case FLOAT -> new KnnVectorQuery(name(), queryVector, numCands, filter);
+            };
         }
     }
 
@@ -628,21 +843,7 @@ public class DenseVectorFieldMapper extends FieldMapper {
     }
 
     private Field parseKnnVector(DocumentParserContext context) throws IOException {
-        float[] vector = new float[dims];
+        return elementType.parseKnnVector(context, this);
-        float squaredMagnitude = 0.0f;
-        int index = 0;
-        for (Token token = context.parser().nextToken(); token != Token.END_ARRAY; token = context.parser().nextToken()) {
-            checkDimensionExceeded(index, context);
-            ensureExpectedToken(Token.VALUE_NUMBER, token, context.parser());
-
-            float value = context.parser().floatValue(true);
-            vector[index++] = value;
-            squaredMagnitude += value * value;
-        }
-        checkDimensionMatches(index, context);
-        elementType.checkVectorBounds(vector);
-        elementType.checkVectorMagnitude(similarity, vector, squaredMagnitude);
-        return elementType.createKnnVectorField(fieldType().name(), vector, similarity.function);
     }
 
     private Field parseBinaryDocValuesVector(DocumentParserContext context) throws IOException {
@@ -653,22 +854,7 @@ public class DenseVectorFieldMapper extends FieldMapper {
             : new byte[dims * elementType.elementBytes];
 
         ByteBuffer byteBuffer = ByteBuffer.wrap(bytes);
-        double dotProduct = 0f;
+        double dotProduct = elementType.parseKnnVectorToByteBuffer(context, this, byteBuffer);
-
-        int index = 0;
-        float[] vector = new float[dims];
-        for (Token token = context.parser().nextToken(); token != Token.END_ARRAY; token = context.parser().nextToken()) {
-            checkDimensionExceeded(index, context);
-            ensureExpectedToken(Token.VALUE_NUMBER, token, context.parser());
-            float value = context.parser().floatValue(true);
-            vector[index] = value;
-            elementType.writeValue(byteBuffer, value);
-            dotProduct += value * value;
-            index++;
-        }
-        checkDimensionMatches(index, context);
-        elementType.checkVectorBounds(vector);
-
         if (indexCreatedVersion.onOrAfter(Version.V_7_5_0)) {
             // encode vector magnitude at the end
             float vectorMagnitude = (float) Math.sqrt(dotProduct);
@@ -759,7 +945,7 @@ public class DenseVectorFieldMapper extends FieldMapper {
             return null; // use default format
         } else {
             HnswIndexOptions hnswIndexOptions = (HnswIndexOptions) indexOptions;
-            return new Lucene94HnswVectorsFormat(hnswIndexOptions.m, hnswIndexOptions.efConstruction);
+            return new Lucene95HnswVectorsFormat(hnswIndexOptions.m, hnswIndexOptions.efConstruction);
         }
     }
 
@@ -778,6 +964,7 @@ public class DenseVectorFieldMapper extends FieldMapper {
 
     private class IndexedSyntheticFieldLoader implements SourceLoader.SyntheticFieldLoader {
         private VectorValues values;
+        private ByteVectorValues byteVectorValues;
         private boolean hasValue;
 
         @Override
@@ -788,13 +975,20 @@ public class DenseVectorFieldMapper extends FieldMapper {
         @Override
         public DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf) throws IOException {
             values = leafReader.getVectorValues(name());
-            if (values == null) {
+            if (values != null) {
-                return null;
+                return docId -> {
+                    hasValue = docId == values.advance(docId);
+                    return hasValue;
+                };
             }
-            return docId -> {
+            byteVectorValues = leafReader.getByteVectorValues(name());
-                hasValue = docId == values.advance(docId);
+            if (byteVectorValues != null) {
-                return hasValue;
+                return docId -> {
-            };
+                    hasValue = docId == byteVectorValues.advance(docId);
+                    return hasValue;
+                };
+            }
+            return null;
         }
 
         @Override
@@ -808,8 +1002,15 @@ public class DenseVectorFieldMapper extends FieldMapper {
                 return;
             }
             b.startArray(simpleName());
-            for (float v : values.vectorValue()) {
+            if (values != null) {
-                b.value(v);
+                for (float v : values.vectorValue()) {
+                    b.value(v);
+                }
+            } else if (byteVectorValues != null) {
+                BytesRef vectorValue = byteVectorValues.vectorValue();
+                for (int i = vectorValue.offset; i < vectorValue.offset + vectorValue.length; i++) {
+                    b.value(vectorValue.bytes[i]);
+                }
             }
             b.endArray();
         }
@@ -850,7 +1051,7 @@ public class DenseVectorFieldMapper extends FieldMapper {
             BytesRef ref = values.binaryValue();
             ByteBuffer byteBuffer = ByteBuffer.wrap(ref.bytes, ref.offset, ref.length);
             for (int dim = 0; dim < dims; dim++) {
-                b.value(elementType.readValue(byteBuffer));
+                elementType.readAndWriteValue(byteBuffer, b);
             }
             b.endArray();
         }

server/src/main/java/org/elasticsearch/index/mapper/vectors/VectorDVLeafFieldData.java

@@ -11,7 +11,6 @@ package org.elasticsearch.index.mapper.vectors;
 import org.apache.lucene.index.BinaryDocValues;
 import org.apache.lucene.index.DocValues;
 import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.VectorValues;
 import org.elasticsearch.Version;
 import org.elasticsearch.index.fielddata.LeafFieldData;
 import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
@@ -56,15 +55,9 @@ final class VectorDVLeafFieldData implements LeafFieldData {
     public DocValuesScriptFieldFactory getScriptFieldFactory(String name) {
         try {
             if (indexed) {
-                VectorValues values = reader.getVectorValues(field);
-                if (values == VectorValues.EMPTY) {
-                    // There's no way for KnnDenseVectorDocValuesField to reliably differentiate between VectorValues.EMPTY and
-                    // values that can be iterated through. Since VectorValues.EMPTY throws on docID(), pass a null instead.
-                    values = null;
-                }
                 return switch (elementType) {
-                    case BYTE -> new ByteKnnDenseVectorDocValuesField(values, name, elementType, dims);
+                    case BYTE -> new ByteKnnDenseVectorDocValuesField(reader.getByteVectorValues(field), name, dims);
-                    case FLOAT -> new KnnDenseVectorDocValuesField(values, name, elementType, dims);
+                    case FLOAT -> new KnnDenseVectorDocValuesField(reader.getVectorValues(field), name, dims);
                 };
             } else {
                 BinaryDocValues values = DocValues.getBinary(reader, field);

server/src/main/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilder.java

@@ -407,11 +407,11 @@ public class CombinedFieldsQueryBuilder extends AbstractQueryBuilder<CombinedFie
         }
 
         @Override
-        protected Query newSynonymQuery(TermAndBoost[] terms) {
+        protected Query newSynonymQuery(String field, TermAndBoost[] terms) {
             CombinedFieldQuery.Builder query = new CombinedFieldQuery.Builder();
             for (TermAndBoost termAndBoost : terms) {
                 assert termAndBoost.boost == BoostAttribute.DEFAULT_BOOST;
-                BytesRef bytes = termAndBoost.term.bytes();
+                BytesRef bytes = termAndBoost.term;
                 query.addTerm(bytes);
             }
             for (FieldAndBoost fieldAndBoost : fields) {
@@ -424,8 +424,8 @@ public class CombinedFieldsQueryBuilder extends AbstractQueryBuilder<CombinedFie
 
         @Override
         protected Query newTermQuery(Term term, float boost) {
-            TermAndBoost termAndBoost = new TermAndBoost(term, boost);
+            TermAndBoost termAndBoost = new TermAndBoost(term.bytes(), boost);
-            return newSynonymQuery(new TermAndBoost[] { termAndBoost });
+            return newSynonymQuery(term.field(), new TermAndBoost[] { termAndBoost });
         }
 
         @Override

server/src/main/java/org/elasticsearch/index/search/MatchQueryParser.java

@@ -538,9 +538,9 @@ public class MatchQueryParser {
             } else {
                 // We don't apply prefix on synonyms
                 final TermAndBoost[] termAndBoosts = current.stream()
-                    .map(t -> new TermAndBoost(t, BoostAttribute.DEFAULT_BOOST))
+                    .map(t -> new TermAndBoost(t.bytes(), BoostAttribute.DEFAULT_BOOST))
                     .toArray(TermAndBoost[]::new);
-                q.add(newSynonymQuery(termAndBoosts), operator);
+                q.add(newSynonymQuery(field, termAndBoosts), operator);
             }
         }
 
@@ -648,9 +648,9 @@ public class MatchQueryParser {
                     } else {
                         // We don't apply prefix on synonyms
                         final TermAndBoost[] termAndBoosts = Arrays.stream(terms)
-                            .map(t -> new TermAndBoost(t, BoostAttribute.DEFAULT_BOOST))
+                            .map(t -> new TermAndBoost(t.bytes(), BoostAttribute.DEFAULT_BOOST))
                             .toArray(TermAndBoost[]::new);
-                        queryPos = newSynonymQuery(termAndBoosts);
+                        queryPos = newSynonymQuery(field, termAndBoosts);
                     }
                 }
                 if (queryPos != null) {

server/src/main/java/org/elasticsearch/index/search/MultiMatchQueryParser.java

@@ -196,10 +196,10 @@ public class MultiMatchQueryParser extends MatchQueryParser {
         }
 
         @Override
-        protected Query newSynonymQuery(TermAndBoost[] terms) {
+        protected Query newSynonymQuery(String field, TermAndBoost[] terms) {
             BytesRef[] values = new BytesRef[terms.length];
             for (int i = 0; i < terms.length; i++) {
-                values[i] = terms[i].term.bytes();
+                values[i] = terms[i].term;
             }
             return blendTerms(context, values, tieBreaker, lenient, blendedFields);
         }

server/src/main/java/org/elasticsearch/script/VectorScoreScriptUtils.java

@@ -52,18 +52,18 @@ public class VectorScoreScriptUtils {
         public ByteDenseVectorFunction(ScoreScript scoreScript, DenseVectorDocValuesField field, List<Number> queryVector) {
             super(scoreScript, field);
             DenseVector.checkDimensions(field.get().getDims(), queryVector.size());
-
-            float[] vector = new float[queryVector.size()];
             this.queryVector = new byte[queryVector.size()];
+            float[] validateValues = new float[queryVector.size()];
             int queryMagnitude = 0;
             for (int i = 0; i < queryVector.size(); i++) {
-                float value = queryVector.get(i).floatValue();
+                final Number number = queryVector.get(i);
-                vector[i] = value;
+                byte value = number.byteValue();
-                this.queryVector[i] = (byte) value;
+                this.queryVector[i] = value;
                 queryMagnitude += value * value;
+                validateValues[i] = number.floatValue();
             }
             this.qvMagnitude = (float) Math.sqrt(queryMagnitude);
-            field.getElementType().checkVectorBounds(vector);
+            field.getElementType().checkVectorBounds(validateValues);
         }
     }
 

server/src/main/java/org/elasticsearch/script/field/vectors/ByteKnnDenseVector.java

@@ -27,6 +27,7 @@ public class ByteKnnDenseVector implements DenseVector {
 
     @Override
     public float[] getVector() {
+        // TODO it would be really nice if we didn't transform the `byte[]` arrays to `float[]`
         if (floatDocVector == null) {
             floatDocVector = new float[docVector.length];
 

server/src/main/java/org/elasticsearch/script/field/vectors/ByteKnnDenseVectorDocValuesField.java

@@ -8,7 +8,7 @@
 
 package org.elasticsearch.script.field.vectors;
 
-import org.apache.lucene.index.VectorValues;
+import org.apache.lucene.index.ByteVectorValues;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.ElementType;
@@ -19,12 +19,12 @@ import java.io.IOException;
 import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
 
 public class ByteKnnDenseVectorDocValuesField extends DenseVectorDocValuesField {
-    protected VectorValues input; // null if no vectors
+    protected ByteVectorValues input; // null if no vectors
     protected BytesRef vector;
     protected final int dims;
 
-    public ByteKnnDenseVectorDocValuesField(@Nullable VectorValues input, String name, ElementType elementType, int dims) {
+    public ByteKnnDenseVectorDocValuesField(@Nullable ByteVectorValues input, String name, int dims) {
-        super(name, elementType);
+        super(name, ElementType.BYTE);
         this.dims = dims;
         this.input = input;
     }
@@ -38,11 +38,11 @@ public class ByteKnnDenseVectorDocValuesField extends DenseVectorDocValuesField
         if (currentDoc == NO_MORE_DOCS || docId < currentDoc) {
             vector = null;
         } else if (docId == currentDoc) {
-            vector = input.binaryValue();
+            vector = input.vectorValue();
         } else {
             currentDoc = input.advance(docId);
             if (currentDoc == docId) {
-                vector = input.binaryValue();
+                vector = input.vectorValue();
             } else {
                 vector = null;
             }

server/src/main/java/org/elasticsearch/script/field/vectors/KnnDenseVectorDocValuesField.java

@@ -22,8 +22,8 @@ public class KnnDenseVectorDocValuesField extends DenseVectorDocValuesField {
     protected float[] vector;
     protected final int dims;
 
-    public KnnDenseVectorDocValuesField(@Nullable VectorValues input, String name, ElementType elementType, int dims) {
+    public KnnDenseVectorDocValuesField(@Nullable VectorValues input, String name, int dims) {
-        super(name, elementType);
+        super(name, ElementType.FLOAT);
         this.dims = dims;
         this.input = input;
     }

server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/QueryToFilterAdapter.java

@@ -9,13 +9,13 @@
 package org.elasticsearch.search.aggregations.bucket.filter;
 
 import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.sandbox.search.IndexSortSortedNumericDocValuesRangeQuery;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.BulkScorer;
 import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.IndexOrDocValuesQuery;
 import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery;
 import org.apache.lucene.search.LeafCollector;
 import org.apache.lucene.search.PointRangeQuery;
 import org.apache.lucene.search.Query;

server/src/main/java/org/elasticsearch/search/internal/ExitableDirectoryReader.java

@@ -9,6 +9,7 @@
 package org.elasticsearch.search.internal;
 
 import org.apache.lucene.codecs.StoredFieldsReader;
+import org.apache.lucene.index.ByteVectorValues;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.FilterDirectoryReader;
 import org.apache.lucene.index.FilterLeafReader;
@@ -126,6 +127,45 @@ class ExitableDirectoryReader extends FilterDirectoryReader {
             return reader;
         }
 
+        @Override
+        public ByteVectorValues getByteVectorValues(String field) throws IOException {
+            ByteVectorValues vectorValues = in.getByteVectorValues(field);
+            if (vectorValues == null) {
+                return null;
+            }
+            return queryCancellation.isEnabled() ? new ExitableByteVectorValues(queryCancellation, vectorValues) : vectorValues;
+        }
+
+        @Override
+        public TopDocs searchNearestVectors(String field, BytesRef target, int k, Bits acceptDocs, int visitedLimit) throws IOException {
+            if (queryCancellation.isEnabled() == false) {
+                return in.searchNearestVectors(field, target, k, acceptDocs, visitedLimit);
+            }
+            // when acceptDocs is null due to no doc deleted, we will instantiate a new one that would
+            // match all docs to allow timeout checking.
+            final Bits updatedAcceptDocs = acceptDocs == null ? new Bits.MatchAllBits(maxDoc()) : acceptDocs;
+            Bits timeoutCheckingAcceptDocs = new Bits() {
+                private static final int MAX_CALLS_BEFORE_QUERY_TIMEOUT_CHECK = 10;
+                private int calls;
+
+                @Override
+                public boolean get(int index) {
+                    if (calls++ % MAX_CALLS_BEFORE_QUERY_TIMEOUT_CHECK == 0) {
+                        queryCancellation.checkCancelled();
+                    }
+
+                    return updatedAcceptDocs.get(index);
+                }
+
+                @Override
+                public int length() {
+                    return updatedAcceptDocs.length();
+                }
+            };
+
+            return in.searchNearestVectors(field, target, k, timeoutCheckingAcceptDocs, visitedLimit);
+        }
+
         @Override
         public VectorValues getVectorValues(String field) throws IOException {
             VectorValues vectorValues = in.getVectorValues(field);
@@ -433,6 +473,57 @@ class ExitableDirectoryReader extends FilterDirectoryReader {
         }
     }
 
+    private static class ExitableByteVectorValues extends ByteVectorValues {
+        private int calls;
+        private final QueryCancellation queryCancellation;
+        private final ByteVectorValues in;
+
+        private ExitableByteVectorValues(QueryCancellation queryCancellation, ByteVectorValues in) {
+            this.queryCancellation = queryCancellation;
+            this.in = in;
+        }
+
+        @Override
+        public int dimension() {
+            return in.dimension();
+        }
+
+        @Override
+        public int size() {
+            return in.size();
+        }
+
+        @Override
+        public BytesRef vectorValue() throws IOException {
+            return in.vectorValue();
+        }
+
+        @Override
+        public int docID() {
+            return in.docID();
+        }
+
+        @Override
+        public int nextDoc() throws IOException {
+            final int nextDoc = in.nextDoc();
+            checkAndThrowWithSampling();
+            return nextDoc;
+        }
+
+        @Override
+        public int advance(int target) throws IOException {
+            final int advance = in.advance(target);
+            checkAndThrowWithSampling();
+            return advance;
+        }
+
+        private void checkAndThrowWithSampling() {
+            if ((calls++ & ExitableIntersectVisitor.MAX_CALLS_BEFORE_QUERY_TIMEOUT_CHECK) == 0) {
+                this.queryCancellation.checkCancelled();
+            }
+        }
+    }
+
     private static class ExitableVectorValues extends FilterVectorValues {
         private int calls;
         private final QueryCancellation queryCancellation;

server/src/main/java/org/elasticsearch/search/internal/FieldUsageTrackingDirectoryReader.java

@@ -10,6 +10,7 @@ package org.elasticsearch.search.internal;
 
 import org.apache.lucene.codecs.StoredFieldsReader;
 import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.ByteVectorValues;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.Fields;
@@ -23,6 +24,8 @@ import org.apache.lucene.index.SortedDocValues;
 import org.apache.lucene.index.SortedNumericDocValues;
 import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.index.StoredFieldVisitor;
+import org.apache.lucene.index.StoredFields;
+import org.apache.lucene.index.TermVectors;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.index.VectorValues;
@@ -108,6 +111,21 @@ public class FieldUsageTrackingDirectoryReader extends FilterDirectoryReader {
             return f;
         }
 
+        @Override
+        public TermVectors termVectors() throws IOException {
+            TermVectors termVectors = super.termVectors();
+            return new TermVectors() {
+                @Override
+                public Fields get(int doc) throws IOException {
+                    Fields f = termVectors.get(doc);
+                    if (f != null) {
+                        f = new FieldUsageTrackingTermVectorFields(f);
+                    }
+                    return f;
+                }
+            };
+        }
+
         @Override
         public PointValues getPointValues(String field) throws IOException {
             PointValues pointValues = super.getPointValues(field);
@@ -126,6 +144,21 @@ public class FieldUsageTrackingDirectoryReader extends FilterDirectoryReader {
             }
         }
 
+        @Override
+        public StoredFields storedFields() throws IOException {
+            StoredFields storedFields = super.storedFields();
+            return new StoredFields() {
+                @Override
+                public void document(int docID, StoredFieldVisitor visitor) throws IOException {
+                    if (visitor instanceof FieldNamesProvidingStoredFieldsVisitor) {
+                        storedFields.document(docID, new FieldUsageFieldsVisitor((FieldNamesProvidingStoredFieldsVisitor) visitor));
+                    } else {
+                        storedFields.document(docID, new FieldUsageStoredFieldVisitor(visitor));
+                    }
+                }
+            };
+        }
+
         @Override
         public Terms terms(String field) throws IOException {
             Terms terms = super.terms(field);
@@ -195,6 +228,24 @@ public class FieldUsageTrackingDirectoryReader extends FilterDirectoryReader {
             return vectorValues;
         }
 
+        @Override
+        public ByteVectorValues getByteVectorValues(String field) throws IOException {
+            ByteVectorValues vectorValues = super.getByteVectorValues(field);
+            if (vectorValues != null) {
+                notifier.onKnnVectorsUsed(field);
+            }
+            return vectorValues;
+        }
+
+        @Override
+        public TopDocs searchNearestVectors(String field, BytesRef target, int k, Bits acceptDocs, int visitedLimit) throws IOException {
+            TopDocs topDocs = super.searchNearestVectors(field, target, k, acceptDocs, visitedLimit);
+            if (topDocs != null) {
+                notifier.onKnnVectorsUsed(field);
+            }
+            return topDocs;
+        }
+
         @Override
         public TopDocs searchNearestVectors(String field, float[] target, int k, Bits acceptDocs, int visitedLimit) throws IOException {
             TopDocs topDocs = super.searchNearestVectors(field, target, k, acceptDocs, visitedLimit);
@@ -223,8 +274,8 @@ public class FieldUsageTrackingDirectoryReader extends FilterDirectoryReader {
             }
 
             @Override
-            public void visitDocument(int docID, StoredFieldVisitor visitor) throws IOException {
+            public void document(int docID, StoredFieldVisitor visitor) throws IOException {
-                reader.visitDocument(docID, new FieldUsageStoredFieldVisitor(visitor));
+                reader.document(docID, new FieldUsageStoredFieldVisitor(visitor));
             }
 
             @Override

server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java

@@ -44,6 +44,7 @@ public class KnnSearchBuilder implements Writeable, ToXContentFragment, Rewritea
 
     private static final ConstructingObjectParser<KnnSearchBuilder, Void> PARSER = new ConstructingObjectParser<>("knn", args -> {
         @SuppressWarnings("unchecked")
+        // TODO optimize parsing for when BYTE values are provided
         List<Float> vector = (List<Float>) args[1];
         float[] vectorArray = new float[vector.size()];
         for (int i = 0; i < vector.size(); i++) {

server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java

@@ -12,9 +12,11 @@ import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.KnnVectorQuery;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
 import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.DenseVectorFieldType;
@@ -41,12 +43,22 @@ public class KnnVectorQueryBuilder extends AbstractQueryBuilder<KnnVectorQueryBu
 
     private final String fieldName;
     private final float[] queryVector;
+    private final byte[] byteQueryVector;
     private final int numCands;
     private final List<QueryBuilder> filterQueries;
 
     public KnnVectorQueryBuilder(String fieldName, float[] queryVector, int numCands) {
         this.fieldName = fieldName;
-        this.queryVector = queryVector;
+        this.queryVector = Objects.requireNonNull(queryVector);
+        this.byteQueryVector = null;
+        this.numCands = numCands;
+        this.filterQueries = new ArrayList<>();
+    }
+
+    public KnnVectorQueryBuilder(String fieldName, byte[] queryVector, int numCands) {
+        this.fieldName = fieldName;
+        this.queryVector = null;
+        this.byteQueryVector = Objects.requireNonNull(queryVector);
         this.numCands = numCands;
         this.filterQueries = new ArrayList<>();
     }
@@ -55,7 +67,13 @@ public class KnnVectorQueryBuilder extends AbstractQueryBuilder<KnnVectorQueryBu
         super(in);
         this.fieldName = in.readString();
         this.numCands = in.readVInt();
-        this.queryVector = in.readFloatArray();
+        if (in.getVersion().before(Version.V_8_7_0)) {
+            this.queryVector = in.readFloatArray();
+            this.byteQueryVector = null;
+        } else {
+            this.queryVector = in.readBoolean() ? in.readFloatArray() : null;
+            this.byteQueryVector = in.readBoolean() ? in.readByteArray() : null;
+        }
         if (in.getVersion().before(Version.V_8_2_0)) {
             this.filterQueries = new ArrayList<>();
         } else {
@@ -67,10 +85,16 @@ public class KnnVectorQueryBuilder extends AbstractQueryBuilder<KnnVectorQueryBu
         return fieldName;
     }
 
+    @Nullable
     public float[] queryVector() {
         return queryVector;
     }
 
+    @Nullable
+    public byte[] getByteQueryVector() {
+        return byteQueryVector;
+    }
+
     public int numCands() {
         return numCands;
     }
@@ -95,7 +119,29 @@ public class KnnVectorQueryBuilder extends AbstractQueryBuilder<KnnVectorQueryBu
     protected void doWriteTo(StreamOutput out) throws IOException {
         out.writeString(fieldName);
         out.writeVInt(numCands);
-        out.writeFloatArray(queryVector);
+        if (out.getVersion().onOrAfter(Version.V_8_7_0)) {
+            boolean queryVectorNotNull = queryVector != null;
+            out.writeBoolean(queryVectorNotNull);
+            if (queryVectorNotNull) {
+                out.writeFloatArray(queryVector);
+            }
+            boolean byteVectorNotNull = byteQueryVector != null;
+            out.writeBoolean(byteVectorNotNull);
+            if (byteVectorNotNull) {
+                out.writeByteArray(byteQueryVector);
+            }
+        } else {
+            final float[] f;
+            if (queryVector != null) {
+                f = queryVector;
+            } else {
+                f = new float[byteQueryVector.length];
+                for (int i = 0; i < byteQueryVector.length; i++) {
+                    f[i] = byteQueryVector[i];
+                }
+            }
+            out.writeFloatArray(f);
+        }
         if (out.getVersion().onOrAfter(Version.V_8_2_0)) {
             writeQueries(out, filterQueries);
         }
@@ -103,7 +149,10 @@ public class KnnVectorQueryBuilder extends AbstractQueryBuilder<KnnVectorQueryBu
 
     @Override
     protected void doXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject(NAME).field("field", fieldName).field("vector", queryVector).field("num_candidates", numCands);
+        builder.startObject(NAME)
+            .field("field", fieldName)
+            .field("vector", queryVector != null ? queryVector : byteQueryVector)
+            .field("num_candidates", numCands);
         if (filterQueries.isEmpty() == false) {
             builder.startArray("filters");
             for (QueryBuilder filterQuery : filterQueries) {
@@ -135,7 +184,9 @@ public class KnnVectorQueryBuilder extends AbstractQueryBuilder<KnnVectorQueryBu
             rewrittenQueries.add(rewrittenQuery);
         }
         if (changed) {
-            return new KnnVectorQueryBuilder(fieldName, queryVector, numCands).addFilterQueries(rewrittenQueries);
+            return byteQueryVector != null
+                ? new KnnVectorQueryBuilder(fieldName, byteQueryVector, numCands).addFilterQueries(rewrittenQueries)
+                : new KnnVectorQueryBuilder(fieldName, queryVector, numCands).addFilterQueries(rewrittenQueries);
         }
         return this;
     }
@@ -161,18 +212,21 @@ public class KnnVectorQueryBuilder extends AbstractQueryBuilder<KnnVectorQueryBu
         Query filterQuery = booleanQuery.clauses().isEmpty() ? null : booleanQuery;
 
         DenseVectorFieldType vectorFieldType = (DenseVectorFieldType) fieldType;
-        return vectorFieldType.createKnnQuery(queryVector, numCands, filterQuery);
+        return queryVector != null
+            ? vectorFieldType.createKnnQuery(queryVector, numCands, filterQuery)
+            : vectorFieldType.createKnnQuery(new BytesRef(byteQueryVector), numCands, filterQuery);
     }
 
     @Override
     protected int doHashCode() {
-        return Objects.hash(fieldName, Arrays.hashCode(queryVector), numCands, filterQueries);
+        return Objects.hash(fieldName, Arrays.hashCode(queryVector), Arrays.hashCode(byteQueryVector), numCands, filterQueries);
     }
 
     @Override
     protected boolean doEquals(KnnVectorQueryBuilder other) {
         return Objects.equals(fieldName, other.fieldName)
             && Arrays.equals(queryVector, other.queryVector)
+            && Arrays.equals(byteQueryVector, other.byteQueryVector)
             && numCands == other.numCands
             && Objects.equals(filterQueries, other.filterQueries);
     }

server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java

@@ -13,8 +13,8 @@ import org.apache.lucene.codecs.KnnVectorsFormat;
 import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
 import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat;
-import org.apache.lucene.codecs.lucene94.Lucene94Codec;
+import org.apache.lucene.codecs.lucene95.Lucene95Codec;
-import org.apache.lucene.codecs.lucene94.Lucene94HnswVectorsFormat;
+import org.apache.lucene.codecs.lucene95.Lucene95HnswVectorsFormat;
 import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
 import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;
 import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
@@ -80,6 +80,7 @@ import java.util.function.Consumer;
 
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
@@ -261,9 +262,11 @@ public class IndexDiskUsageAnalyzerTests extends ESTestCase {
             final IndexDiskUsageStats stats = IndexDiskUsageAnalyzer.analyze(testShardId(), lastCommit(dir), () -> {});
             logger.info("--> stats {}", stats);
 
-            int dataBytes = numDocs * dimension * Float.BYTES; // size of flat vector data
+            long dataBytes = (long) numDocs * dimension * Float.BYTES; // size of flat vector data
-            int indexBytesEstimate = numDocs * Integer.BYTES * Lucene94HnswVectorsFormat.DEFAULT_MAX_CONN * 2; // rough size of HNSW graph
+            long indexBytesEstimate = (long) numDocs * (Lucene95HnswVectorsFormat.DEFAULT_MAX_CONN / 2); // rough size of HNSW graph
-            assertTrue(stats.total().getKnnVectorsBytes() > dataBytes + indexBytesEstimate);
+            assertThat(stats.total().getKnnVectorsBytes(), greaterThan(dataBytes));
+            long connectionOverhead = stats.total().getKnnVectorsBytes() - dataBytes;
+            assertThat(connectionOverhead, greaterThan(indexBytesEstimate));
         }
     }
 
@@ -323,7 +326,7 @@ public class IndexDiskUsageAnalyzerTests extends ESTestCase {
     public void testCompletionField() throws Exception {
         IndexWriterConfig config = new IndexWriterConfig().setCommitOnClose(true)
             .setUseCompoundFile(false)
-            .setCodec(new Lucene94Codec(Lucene94Codec.Mode.BEST_SPEED) {
+            .setCodec(new Lucene95Codec(Lucene95Codec.Mode.BEST_SPEED) {
                 @Override
                 public PostingsFormat getPostingsFormatForField(String field) {
                     if (field.startsWith("suggest_")) {
@@ -410,25 +413,25 @@ public class IndexDiskUsageAnalyzerTests extends ESTestCase {
     enum CodecMode {
         BEST_SPEED {
             @Override
-            Lucene94Codec.Mode mode() {
+            Lucene95Codec.Mode mode() {
-                return Lucene94Codec.Mode.BEST_SPEED;
+                return Lucene95Codec.Mode.BEST_SPEED;
             }
         },
 
         BEST_COMPRESSION {
             @Override
-            Lucene94Codec.Mode mode() {
+            Lucene95Codec.Mode mode() {
-                return Lucene94Codec.Mode.BEST_COMPRESSION;
+                return Lucene95Codec.Mode.BEST_COMPRESSION;
             }
         };
 
-        abstract Lucene94Codec.Mode mode();
+        abstract Lucene95Codec.Mode mode();
     }
 
     static void indexRandomly(Directory directory, CodecMode codecMode, int numDocs, Consumer<Document> addFields) throws IOException {
         IndexWriterConfig config = new IndexWriterConfig().setCommitOnClose(true)
             .setUseCompoundFile(randomBoolean())
-            .setCodec(new Lucene94Codec(codecMode.mode()));
+            .setCodec(new Lucene95Codec(codecMode.mode()));
         try (IndexWriter writer = new IndexWriter(directory, config)) {
             for (int i = 0; i < numDocs; i++) {
                 final Document doc = new Document();
@@ -636,7 +639,7 @@ public class IndexDiskUsageAnalyzerTests extends ESTestCase {
         try (DirectoryReader reader = DirectoryReader.open(source)) {
             IndexWriterConfig config = new IndexWriterConfig().setSoftDeletesField(Lucene.SOFT_DELETES_FIELD)
                 .setUseCompoundFile(randomBoolean())
-                .setCodec(new Lucene94Codec(mode.mode()) {
+                .setCodec(new Lucene95Codec(mode.mode()) {
                     @Override
                     public PostingsFormat getPostingsFormatForField(String field) {
                         return new Lucene90PostingsFormat();
@@ -649,7 +652,7 @@ public class IndexDiskUsageAnalyzerTests extends ESTestCase {
 
                     @Override
                     public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
-                        return new Lucene94HnswVectorsFormat();
+                        return new Lucene95HnswVectorsFormat();
                     }
 
                     @Override

server/src/test/java/org/elasticsearch/index/codec/CodecTests.java

@@ -10,7 +10,7 @@ package org.elasticsearch.index.codec;
 
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat;
-import org.apache.lucene.codecs.lucene94.Lucene94Codec;
+import org.apache.lucene.codecs.lucene95.Lucene95Codec;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexWriter;
@@ -42,21 +42,21 @@ public class CodecTests extends ESTestCase {
     public void testResolveDefaultCodecs() throws Exception {
         CodecService codecService = createCodecService();
         assertThat(codecService.codec("default"), instanceOf(PerFieldMapperCodec.class));
-        assertThat(codecService.codec("default"), instanceOf(Lucene94Codec.class));
+        assertThat(codecService.codec("default"), instanceOf(Lucene95Codec.class));
     }
 
     public void testDefault() throws Exception {
         Codec codec = createCodecService().codec("default");
-        assertStoredFieldsCompressionEquals(Lucene94Codec.Mode.BEST_SPEED, codec);
+        assertStoredFieldsCompressionEquals(Lucene95Codec.Mode.BEST_SPEED, codec);
     }
 
     public void testBestCompression() throws Exception {
         Codec codec = createCodecService().codec("best_compression");
-        assertStoredFieldsCompressionEquals(Lucene94Codec.Mode.BEST_COMPRESSION, codec);
+        assertStoredFieldsCompressionEquals(Lucene95Codec.Mode.BEST_COMPRESSION, codec);
     }
 
     // write some docs with it, inspect .si to see this was the used compression
-    private void assertStoredFieldsCompressionEquals(Lucene94Codec.Mode expected, Codec actual) throws Exception {
+    private void assertStoredFieldsCompressionEquals(Lucene95Codec.Mode expected, Codec actual) throws Exception {
         Directory dir = newDirectory();
         IndexWriterConfig iwc = newIndexWriterConfig(null);
         iwc.setCodec(actual);
@@ -68,7 +68,7 @@ public class CodecTests extends ESTestCase {
         SegmentReader sr = (SegmentReader) ir.leaves().get(0).reader();
         String v = sr.getSegmentInfo().info.getAttribute(Lucene90StoredFieldsFormat.MODE_KEY);
         assertNotNull(v);
-        assertEquals(expected, Lucene94Codec.Mode.valueOf(v));
+        assertEquals(expected, Lucene95Codec.Mode.valueOf(v));
         ir.close();
         dir.close();
     }

server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java

@@ -8,7 +8,7 @@
 
 package org.elasticsearch.index.codec;
 
-import org.apache.lucene.codecs.lucene94.Lucene94Codec;
+import org.apache.lucene.codecs.lucene95.Lucene95Codec;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
@@ -74,7 +74,7 @@ public class PerFieldMapperCodecTests extends ESTestCase {
                 """;
             mapperService.merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE);
         }
-        return new PerFieldMapperCodec(Lucene94Codec.Mode.BEST_SPEED, mapperService, BigArrays.NON_RECYCLING_INSTANCE);
+        return new PerFieldMapperCodec(Lucene95Codec.Mode.BEST_SPEED, mapperService, BigArrays.NON_RECYCLING_INSTANCE);
     }
 
 }

server/src/test/java/org/elasticsearch/index/engine/CompletionStatsCacheTests.java

@@ -8,7 +8,7 @@
 package org.elasticsearch.index.engine;
 
 import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.lucene94.Lucene94Codec;
+import org.apache.lucene.codecs.lucene95.Lucene95Codec;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexWriter;
@@ -45,7 +45,7 @@ public class CompletionStatsCacheTests extends ESTestCase {
     public void testCompletionStatsCache() throws IOException, InterruptedException {
         final IndexWriterConfig indexWriterConfig = newIndexWriterConfig();
         final PostingsFormat postingsFormat = new Completion90PostingsFormat();
-        indexWriterConfig.setCodec(new Lucene94Codec() {
+        indexWriterConfig.setCodec(new Lucene95Codec() {
             @Override
             public PostingsFormat getPostingsFormatForField(String field) {
                 return postingsFormat; // all fields are suggest fields

server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java

@@ -16,9 +16,9 @@ import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.MultiReader;
 import org.apache.lucene.index.SortedNumericDocValues;
-import org.apache.lucene.sandbox.search.IndexSortSortedNumericDocValuesRangeQuery;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.IndexOrDocValuesQuery;
+import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.store.Directory;
 import org.elasticsearch.Version;

server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java

@@ -20,9 +20,9 @@ import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.sandbox.document.HalfFloatPoint;
-import org.apache.lucene.sandbox.search.IndexSortSortedNumericDocValuesRangeQuery;
 import org.apache.lucene.search.IndexOrDocValuesQuery;
 import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.Sort;

server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java

@@ -12,8 +12,9 @@ import com.carrotsearch.randomizedtesting.generators.RandomPicks;
 
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.KnnVectorsFormat;
-import org.apache.lucene.codecs.lucene94.Lucene94HnswVectorsFormat;
+import org.apache.lucene.codecs.lucene95.Lucene95HnswVectorsFormat;
 import org.apache.lucene.document.BinaryDocValuesField;
+import org.apache.lucene.document.KnnByteVectorField;
 import org.apache.lucene.document.KnnVectorField;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.search.FieldExistsQuery;
@@ -42,8 +43,8 @@ import java.nio.ByteBuffer;
 import java.util.Arrays;
 import java.util.List;
 
-import static org.apache.lucene.codecs.lucene94.Lucene94HnswVectorsFormat.DEFAULT_BEAM_WIDTH;
+import static org.apache.lucene.codecs.lucene95.Lucene95HnswVectorsFormat.DEFAULT_BEAM_WIDTH;
-import static org.apache.lucene.codecs.lucene94.Lucene94HnswVectorsFormat.DEFAULT_MAX_CONN;
+import static org.apache.lucene.codecs.lucene95.Lucene95HnswVectorsFormat.DEFAULT_MAX_CONN;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
@@ -250,19 +251,19 @@ public class DenseVectorFieldMapperTests extends MapperTestCase {
             )
         );
 
-        float[] vector = { (byte) -1, (byte) 1, (byte) 127 };
+        byte[] vector = { (byte) -1, (byte) 1, (byte) 127 };
         ParsedDocument doc1 = mapper.parse(source(b -> b.array("field", vector)));
 
         IndexableField[] fields = doc1.rootDoc().getFields("field");
         assertEquals(1, fields.length);
-        assertThat(fields[0], instanceOf(KnnVectorField.class));
+        assertThat(fields[0], instanceOf(KnnByteVectorField.class));
 
-        KnnVectorField vectorField = (KnnVectorField) fields[0];
+        KnnByteVectorField vectorField = (KnnByteVectorField) fields[0];
-        vectorField.binaryValue();
+        vectorField.vectorValue();
         assertEquals(
             "Parsed vector is not equal to original.",
             new BytesRef(new byte[] { (byte) -1, (byte) 1, (byte) 127 }),
-            vectorField.binaryValue()
+            vectorField.vectorValue()
         );
         assertEquals(similarity.function, vectorField.fieldType().vectorSimilarityFunction());
     }
@@ -332,9 +333,7 @@ public class DenseVectorFieldMapperTests extends MapperTestCase {
         assertNotNull(e.getCause());
         assertThat(
             e.getCause().getMessage(),
-            containsString(
+            containsString("The [cosine] similarity does not support vectors with zero magnitude. Preview of invalid vector: [0, 0, 0]")
-                "The [cosine] similarity does not support vectors with zero magnitude. Preview of invalid vector: [-0.0, 0.0, 0.0]"
-            )
         );
     }
 
@@ -544,7 +543,13 @@ public class DenseVectorFieldMapperTests extends MapperTestCase {
         );
         assertThat(
             e.getCause().getMessage(),
-            containsString("element_type [byte] vectors only support integers between [-128, 127] but found [128.0] at dim [0];")
+            containsString("element_type [byte] vectors only support integers between [-128, 127] but found [128] at dim [0];")
+        );
+
+        e = expectThrows(MapperParsingException.class, () -> mapper.parse(source(b -> b.array("field", new float[] { 18.2f, 0, 0 }))));
+        assertThat(
+            e.getCause().getMessage(),
+            containsString("element_type [byte] vectors only support non-decimal values but found decimal value [18.2] at dim [0];")
         );
 
         e = expectThrows(
@@ -553,7 +558,7 @@ public class DenseVectorFieldMapperTests extends MapperTestCase {
         );
         assertThat(
             e.getCause().getMessage(),
-            containsString("element_type [byte] vectors only support integers between [-128, 127] but found [-129.0] at dim [2];")
+            containsString("element_type [byte] vectors only support integers between [-128, 127] but found [-129] at dim [2];")
         );
     }
 
@@ -693,8 +698,8 @@ public class DenseVectorFieldMapperTests extends MapperTestCase {
         Codec codec = codecService.codec("default");
         assertThat(codec, instanceOf(PerFieldMapperCodec.class));
         KnnVectorsFormat knnVectorsFormat = ((PerFieldMapperCodec) codec).getKnnVectorsFormatForField("field");
-        assertThat(knnVectorsFormat, instanceOf(Lucene94HnswVectorsFormat.class));
+        assertThat(knnVectorsFormat, instanceOf(Lucene95HnswVectorsFormat.class));
-        String expectedString = "Lucene94HnswVectorsFormat(name=Lucene94HnswVectorsFormat, maxConn="
+        String expectedString = "Lucene95HnswVectorsFormat(name=Lucene95HnswVectorsFormat, maxConn="
             + m
             + ", beamWidth="
             + efConstruction
@@ -725,14 +730,12 @@ public class DenseVectorFieldMapperTests extends MapperTestCase {
 
         @Override
         public SyntheticSourceExample example(int maxValues) throws IOException {
-            List<Float> value = randomList(dims, dims, this::randomValue);
+            Object value = elementType == ElementType.BYTE
+                ? randomList(dims, dims, ESTestCase::randomByte)
+                : randomList(dims, dims, ESTestCase::randomFloat);
             return new SyntheticSourceExample(value, value, this::mapping);
         }
 
-        private float randomValue() {
-            return elementType == ElementType.BYTE ? ESTestCase.randomByte() : ESTestCase.randomFloat();
-        }
-
         private void mapping(XContentBuilder b) throws IOException {
             b.field("type", "dense_vector");
             b.field("dims", dims);
@@ -753,7 +756,7 @@ public class DenseVectorFieldMapperTests extends MapperTestCase {
         }
 
         @Override
-        public List<SyntheticSourceInvalidExample> invalidExample() throws IOException {
+        public List<SyntheticSourceInvalidExample> invalidExample() {
             return List.of();
         }
     }

server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java

@@ -8,6 +8,7 @@
 
 package org.elasticsearch.index.mapper.vectors;
 
+import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.Version;
 import org.elasticsearch.index.fielddata.FieldDataContext;
 import org.elasticsearch.index.mapper.FieldTypeTestCase;
@@ -174,5 +175,8 @@ public class DenseVectorFieldTypeTests extends FieldTypeTestCase {
         );
         e = expectThrows(IllegalArgumentException.class, () -> cosineField.createKnnQuery(new float[] { 0.0f, 0.0f, 0.0f }, 10, null));
         assertThat(e.getMessage(), containsString("The [cosine] similarity does not support vectors with zero magnitude."));
+
+        e = expectThrows(IllegalArgumentException.class, () -> cosineField.createKnnQuery(new BytesRef(new byte[] { 0, 0, 0 }), 10, null));
+        assertThat(e.getMessage(), containsString("The [cosine] similarity does not support vectors with zero magnitude."));
     }
 }

server/src/test/java/org/elasticsearch/index/mapper/vectors/KnnDenseVectorScriptDocValuesTests.java

@@ -8,9 +8,9 @@
 
 package org.elasticsearch.index.mapper.vectors;
 
+import org.apache.lucene.index.ByteVectorValues;
 import org.apache.lucene.index.VectorValues;
 import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.VectorUtil;
 import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.ElementType;
 import org.elasticsearch.script.field.vectors.ByteKnnDenseVectorDocValuesField;
 import org.elasticsearch.script.field.vectors.DenseVector;
@@ -30,12 +30,7 @@ public class KnnDenseVectorScriptDocValuesTests extends ESTestCase {
         float[][] vectors = { { 1, 1, 1 }, { 1, 1, 2 }, { 1, 1, 3 } };
         float[] expectedMagnitudes = { 1.7320f, 2.4495f, 3.3166f };
 
-        DenseVectorDocValuesField field = new KnnDenseVectorDocValuesField(
+        DenseVectorDocValuesField field = new KnnDenseVectorDocValuesField(wrap(vectors), "test", dims);
-            wrap(vectors, ElementType.FLOAT),
-            "test",
-            ElementType.FLOAT,
-            dims
-        );
         DenseVectorScriptDocValues scriptDocValues = field.toScriptDocValues();
         for (int i = 0; i < vectors.length; i++) {
             field.setNextDocId(i);
@@ -51,12 +46,7 @@ public class KnnDenseVectorScriptDocValuesTests extends ESTestCase {
         float[][] vectors = { { 1, 1, 1 }, { 1, 1, 2 }, { 1, 1, 3 } };
         float[] expectedMagnitudes = { 1.7320f, 2.4495f, 3.3166f };
 
-        DenseVectorDocValuesField field = new ByteKnnDenseVectorDocValuesField(
+        DenseVectorDocValuesField field = new ByteKnnDenseVectorDocValuesField(wrapBytes(vectors), "test", dims);
-            wrap(vectors, ElementType.BYTE),
-            "test",
-            ElementType.BYTE,
-            dims
-        );
         DenseVectorScriptDocValues scriptDocValues = field.toScriptDocValues();
         for (int i = 0; i < vectors.length; i++) {
             field.setNextDocId(i);
@@ -70,12 +60,7 @@ public class KnnDenseVectorScriptDocValuesTests extends ESTestCase {
     public void testFloatMetadataAndIterator() throws IOException {
         int dims = 3;
         float[][] vectors = fill(new float[randomIntBetween(1, 5)][dims], ElementType.FLOAT);
-        DenseVectorDocValuesField field = new KnnDenseVectorDocValuesField(
+        DenseVectorDocValuesField field = new KnnDenseVectorDocValuesField(wrap(vectors), "test", dims);
-            wrap(vectors, ElementType.FLOAT),
-            "test",
-            ElementType.FLOAT,
-            dims
-        );
         for (int i = 0; i < vectors.length; i++) {
             field.setNextDocId(i);
             DenseVector dv = field.get();
@@ -94,12 +79,7 @@ public class KnnDenseVectorScriptDocValuesTests extends ESTestCase {
     public void testByteMetadataAndIterator() throws IOException {
         int dims = 3;
         float[][] vectors = fill(new float[randomIntBetween(1, 5)][dims], ElementType.BYTE);
-        DenseVectorDocValuesField field = new ByteKnnDenseVectorDocValuesField(
+        DenseVectorDocValuesField field = new ByteKnnDenseVectorDocValuesField(wrapBytes(vectors), "test", dims);
-            wrap(vectors, ElementType.BYTE),
-            "test",
-            ElementType.BYTE,
-            dims
-        );
         for (int i = 0; i < vectors.length; i++) {
             field.setNextDocId(i);
             DenseVector dv = field.get();
@@ -127,12 +107,7 @@ public class KnnDenseVectorScriptDocValuesTests extends ESTestCase {
     public void testFloatMissingValues() throws IOException {
         int dims = 3;
         float[][] vectors = { { 1, 1, 1 }, { 1, 1, 2 }, { 1, 1, 3 } };
-        DenseVectorDocValuesField field = new KnnDenseVectorDocValuesField(
+        DenseVectorDocValuesField field = new KnnDenseVectorDocValuesField(wrap(vectors), "test", dims);
-            wrap(vectors, ElementType.FLOAT),
-            "test",
-            ElementType.FLOAT,
-            dims
-        );
         DenseVectorScriptDocValues scriptDocValues = field.toScriptDocValues();
 
         field.setNextDocId(3);
@@ -146,12 +121,7 @@ public class KnnDenseVectorScriptDocValuesTests extends ESTestCase {
     public void testByteMissingValues() throws IOException {
         int dims = 3;
         float[][] vectors = { { 1, 1, 1 }, { 1, 1, 2 }, { 1, 1, 3 } };
-        DenseVectorDocValuesField field = new ByteKnnDenseVectorDocValuesField(
+        DenseVectorDocValuesField field = new ByteKnnDenseVectorDocValuesField(wrapBytes(vectors), "test", dims);
-            wrap(vectors, ElementType.BYTE),
-            "test",
-            ElementType.BYTE,
-            dims
-        );
         DenseVectorScriptDocValues scriptDocValues = field.toScriptDocValues();
 
         field.setNextDocId(3);
@@ -165,12 +135,7 @@ public class KnnDenseVectorScriptDocValuesTests extends ESTestCase {
     public void testFloatGetFunctionIsNotAccessible() throws IOException {
         int dims = 3;
         float[][] vectors = { { 1, 1, 1 }, { 1, 1, 2 }, { 1, 1, 3 } };
-        DenseVectorDocValuesField field = new KnnDenseVectorDocValuesField(
+        DenseVectorDocValuesField field = new KnnDenseVectorDocValuesField(wrap(vectors), "test", dims);
-            wrap(vectors, ElementType.FLOAT),
-            "test",
-            ElementType.FLOAT,
-            dims
-        );
         DenseVectorScriptDocValues scriptDocValues = field.toScriptDocValues();
 
         field.setNextDocId(0);
@@ -186,12 +151,7 @@ public class KnnDenseVectorScriptDocValuesTests extends ESTestCase {
     public void testByteGetFunctionIsNotAccessible() throws IOException {
         int dims = 3;
         float[][] vectors = { { 1, 1, 1 }, { 1, 1, 2 }, { 1, 1, 3 } };
-        DenseVectorDocValuesField field = new ByteKnnDenseVectorDocValuesField(
+        DenseVectorDocValuesField field = new ByteKnnDenseVectorDocValuesField(wrapBytes(vectors), "test", dims);
-            wrap(vectors, ElementType.BYTE),
-            "test",
-            ElementType.BYTE,
-            dims
-        );
         DenseVectorScriptDocValues scriptDocValues = field.toScriptDocValues();
 
         field.setNextDocId(0);
@@ -206,7 +166,7 @@ public class KnnDenseVectorScriptDocValuesTests extends ESTestCase {
 
     public void testFloatMissingVectorValues() throws IOException {
         int dims = 7;
-        DenseVectorDocValuesField emptyKnn = new KnnDenseVectorDocValuesField(null, "test", ElementType.FLOAT, dims);
+        DenseVectorDocValuesField emptyKnn = new KnnDenseVectorDocValuesField(null, "test", dims);
 
         emptyKnn.setNextDocId(0);
         assertEquals(0, emptyKnn.toScriptDocValues().size());
@@ -220,7 +180,7 @@ public class KnnDenseVectorScriptDocValuesTests extends ESTestCase {
 
     public void testByteMissingVectorValues() throws IOException {
         int dims = 7;
-        DenseVectorDocValuesField emptyKnn = new ByteKnnDenseVectorDocValuesField(null, "test", ElementType.BYTE, dims);
+        DenseVectorDocValuesField emptyKnn = new ByteKnnDenseVectorDocValuesField(null, "test", dims);
 
         emptyKnn.setNextDocId(0);
         assertEquals(0, emptyKnn.toScriptDocValues().size());
@@ -232,7 +192,50 @@ public class KnnDenseVectorScriptDocValuesTests extends ESTestCase {
         assertEquals("Cannot iterate over single valued dense_vector field, use get() instead", e.getMessage());
     }
 
-    public static VectorValues wrap(float[][] vectors, ElementType elementType) {
+    public static ByteVectorValues wrapBytes(float[][] vectors) {
+        return new ByteVectorValues() {
+            int index = 0;
+            byte[] byteVector = new byte[vectors[0].length];
+
+            @Override
+            public int dimension() {
+                return 0;
+            }
+
+            @Override
+            public int size() {
+                return vectors.length;
+            }
+
+            @Override
+            public BytesRef vectorValue() {
+                for (int i = 0; i < byteVector.length; i++) {
+                    byteVector[i] = (byte) vectors[index][i];
+                }
+                return new BytesRef(byteVector);
+            }
+
+            @Override
+            public int docID() {
+                return index;
+            }
+
+            @Override
+            public int nextDoc() {
+                throw new UnsupportedOperationException();
+            }
+
+            @Override
+            public int advance(int target) {
+                if (target >= size()) {
+                    return NO_MORE_DOCS;
+                }
+                return index = target;
+            }
+        };
+    }
+
+    public static VectorValues wrap(float[][] vectors) {
         return new VectorValues() {
             int index = 0;
 
@@ -253,17 +256,11 @@ public class KnnDenseVectorScriptDocValuesTests extends ESTestCase {
 
             @Override
             public BytesRef binaryValue() {
-                if (elementType == ElementType.FLOAT) {
+                ByteBuffer byteBuffer = ByteBuffer.allocate(ElementType.FLOAT.elementBytes * vectors[index].length);
-                    ByteBuffer byteBuffer = ByteBuffer.allocate(elementType.elementBytes * vectors[index].length);
+                for (float value : vectors[index]) {
-                    for (float value : vectors[index]) {
+                    ElementType.FLOAT.writeValue(byteBuffer, value);
-                        elementType.writeValue(byteBuffer, value);
-                    }
-                    return new BytesRef(byteBuffer.array());
-                } else if (elementType == ElementType.BYTE) {
-                    return VectorUtil.toBytesRef(vectors[index]);
-                } else {
-                    throw new IllegalStateException("unknown element_type [" + elementType + "]");
                 }
+                return new BytesRef(byteBuffer.array());
             }
 
             @Override
@@ -283,11 +280,6 @@ public class KnnDenseVectorScriptDocValuesTests extends ESTestCase {
                 }
                 return index = target;
             }
-
-            @Override
-            public long cost() {
-                return size();
-            }
         };
     }
 }

server/src/test/java/org/elasticsearch/script/VectorScoreScriptUtilsTests.java

@@ -55,12 +55,7 @@ public class VectorScoreScriptUtilsTests extends ESTestCase {
                 dims,
                 Version.CURRENT
             ),
-            new KnnDenseVectorDocValuesField(
+            new KnnDenseVectorDocValuesField(KnnDenseVectorScriptDocValuesTests.wrap(new float[][] { docVector }), "test", dims)
-                KnnDenseVectorScriptDocValuesTests.wrap(new float[][] { docVector }, ElementType.FLOAT),
-                "test",
-                ElementType.FLOAT,
-                dims
-            )
         );
         for (DenseVectorDocValuesField field : fields) {
             field.setNextDocId(0);
@@ -140,12 +135,7 @@ public class VectorScoreScriptUtilsTests extends ESTestCase {
                 ElementType.BYTE,
                 dims
             ),
-            new ByteKnnDenseVectorDocValuesField(
+            new ByteKnnDenseVectorDocValuesField(KnnDenseVectorScriptDocValuesTests.wrapBytes(new float[][] { docVector }), "test", dims)
-                KnnDenseVectorScriptDocValuesTests.wrap(new float[][] { docVector }, ElementType.BYTE),
-                "test",
-                ElementType.BYTE,
-                dims
-            )
         );
         for (DenseVectorDocValuesField field : fields) {
             field.setNextDocId(0);
@@ -233,24 +223,14 @@ public class VectorScoreScriptUtilsTests extends ESTestCase {
                 dims,
                 Version.CURRENT
             ),
-            new KnnDenseVectorDocValuesField(
+            new KnnDenseVectorDocValuesField(KnnDenseVectorScriptDocValuesTests.wrap(new float[][] { docVector }), "field2", dims),
-                KnnDenseVectorScriptDocValuesTests.wrap(new float[][] { docVector }, ElementType.FLOAT),
-                "field2",
-                ElementType.FLOAT,
-                dims
-            ),
             new ByteBinaryDenseVectorDocValuesField(
                 BinaryDenseVectorScriptDocValuesTests.wrap(new float[][] { docVector }, ElementType.BYTE, Version.CURRENT),
                 "field3",
                 ElementType.BYTE,
                 dims
             ),
-            new ByteKnnDenseVectorDocValuesField(
+            new ByteKnnDenseVectorDocValuesField(KnnDenseVectorScriptDocValuesTests.wrapBytes(new float[][] { docVector }), "field4", dims)
-                KnnDenseVectorScriptDocValuesTests.wrap(new float[][] { docVector }, ElementType.BYTE),
-                "field4",
-                ElementType.BYTE,
-                dims
-            )
         );
         for (DenseVectorDocValuesField field : fields) {
             field.setNextDocId(0);
@@ -388,12 +368,7 @@ public class VectorScoreScriptUtilsTests extends ESTestCase {
                 ElementType.BYTE,
                 dims
             ),
-            new ByteKnnDenseVectorDocValuesField(
+            new ByteKnnDenseVectorDocValuesField(KnnDenseVectorScriptDocValuesTests.wrapBytes(new float[][] { docVector }), "test", dims)
-                KnnDenseVectorScriptDocValuesTests.wrap(new float[][] { docVector }, ElementType.BYTE),
-                "test",
-                ElementType.BYTE,
-                dims
-            )
         );
 
         for (DenseVectorDocValuesField field : fields) {

server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java

@@ -0,0 +1,245 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.search.vectors;
+
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.KnnByteVectorQuery;
+import org.apache.lucene.search.KnnVectorQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.Version;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.core.Tuple;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
+import org.elasticsearch.index.query.MatchNoneQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.query.SearchExecutionContext;
+import org.elasticsearch.index.query.TermQueryBuilder;
+import org.elasticsearch.test.AbstractBuilderTestCase;
+import org.elasticsearch.test.AbstractQueryTestCase;
+import org.elasticsearch.test.VersionUtils;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.instanceOf;
+
+abstract class AbstractKnnVectorQueryBuilderTestCase extends AbstractQueryTestCase<KnnVectorQueryBuilder> {
+    private static final String VECTOR_FIELD = "vector";
+    private static final String VECTOR_ALIAS_FIELD = "vector_alias";
+    private static final int VECTOR_DIMENSION = 3;
+
+    abstract DenseVectorFieldMapper.ElementType elementType();
+
+    @Override
+    protected void initializeAdditionalMappings(MapperService mapperService) throws IOException {
+        XContentBuilder builder = XContentFactory.jsonBuilder()
+            .startObject()
+            .startObject("properties")
+            .startObject(VECTOR_FIELD)
+            .field("type", "dense_vector")
+            .field("dims", VECTOR_DIMENSION)
+            .field("index", true)
+            .field("similarity", "l2_norm")
+            .field("element_type", elementType())
+            .endObject()
+            .startObject(VECTOR_ALIAS_FIELD)
+            .field("type", "alias")
+            .field("path", VECTOR_FIELD)
+            .endObject()
+            .endObject()
+            .endObject();
+        mapperService.merge(
+            MapperService.SINGLE_MAPPING_NAME,
+            new CompressedXContent(Strings.toString(builder)),
+            MapperService.MergeReason.MAPPING_UPDATE
+        );
+    }
+
+    @Override
+    protected KnnVectorQueryBuilder doCreateTestQueryBuilder() {
+        String fieldName = randomBoolean() ? VECTOR_FIELD : VECTOR_ALIAS_FIELD;
+        byte[] byteVector = new byte[VECTOR_DIMENSION];
+        float[] vector = new float[VECTOR_DIMENSION];
+        for (int i = 0; i < vector.length; i++) {
+            vector[i] = randomFloat();
+            byteVector[i] = randomByte();
+        }
+        int numCands = randomIntBetween(1, 1000);
+
+        KnnVectorQueryBuilder queryBuilder = switch (elementType()) {
+            case BYTE -> new KnnVectorQueryBuilder(fieldName, byteVector, numCands);
+            case FLOAT -> new KnnVectorQueryBuilder(fieldName, vector, numCands);
+        };
+
+        if (randomBoolean()) {
+            List<QueryBuilder> filters = new ArrayList<>();
+            int numFilters = randomIntBetween(1, 5);
+            for (int i = 0; i < numFilters; i++) {
+                String filterFieldName = randomBoolean() ? KEYWORD_FIELD_NAME : TEXT_FIELD_NAME;
+                filters.add(QueryBuilders.termQuery(filterFieldName, randomAlphaOfLength(10)));
+            }
+            queryBuilder.addFilterQueries(filters);
+        }
+        return queryBuilder;
+    }
+
+    @Override
+    protected void doAssertLuceneQuery(KnnVectorQueryBuilder queryBuilder, Query query, SearchExecutionContext context) throws IOException {
+        switch (elementType()) {
+            case FLOAT -> assertTrue(query instanceof KnnVectorQuery);
+            case BYTE -> assertTrue(query instanceof KnnByteVectorQuery);
+        }
+
+        BooleanQuery.Builder builder = new BooleanQuery.Builder();
+        for (QueryBuilder qb : queryBuilder.filterQueries()) {
+            builder.add(qb.toQuery(context), BooleanClause.Occur.FILTER);
+        }
+        BooleanQuery booleanQuery = builder.build();
+        Query filterQuery = booleanQuery.clauses().isEmpty() ? null : booleanQuery;
+        // The field should always be resolved to the concrete field
+        Query knnVectorQueryBuilt = switch (elementType()) {
+            case BYTE -> new KnnByteVectorQuery(
+                VECTOR_FIELD,
+                new BytesRef(queryBuilder.getByteQueryVector()),
+                queryBuilder.numCands(),
+                filterQuery
+            );
+            case FLOAT -> new KnnVectorQuery(VECTOR_FIELD, queryBuilder.queryVector(), queryBuilder.numCands(), filterQuery);
+        };
+        assertEquals(query, knnVectorQueryBuilt);
+    }
+
+    public void testWrongDimension() {
+        SearchExecutionContext context = createSearchExecutionContext();
+        KnnVectorQueryBuilder query = new KnnVectorQueryBuilder(VECTOR_FIELD, new float[] { 1.0f, 2.0f }, 10);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> query.doToQuery(context));
+        assertThat(e.getMessage(), containsString("the query vector has a different dimension [2] than the index vectors [3]"));
+    }
+
+    public void testNonexistentField() {
+        SearchExecutionContext context = createSearchExecutionContext();
+        KnnVectorQueryBuilder query = new KnnVectorQueryBuilder("nonexistent", new float[] { 1.0f, 1.0f, 1.0f }, 10);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> query.doToQuery(context));
+        assertThat(e.getMessage(), containsString("field [nonexistent] does not exist in the mapping"));
+    }
+
+    public void testWrongFieldType() {
+        SearchExecutionContext context = createSearchExecutionContext();
+        KnnVectorQueryBuilder query = new KnnVectorQueryBuilder(
+            AbstractBuilderTestCase.KEYWORD_FIELD_NAME,
+            new float[] { 1.0f, 1.0f, 1.0f },
+            10
+        );
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> query.doToQuery(context));
+        assertThat(e.getMessage(), containsString("[knn] queries are only supported on [dense_vector] fields"));
+    }
+
+    @Override
+    public void testValidOutput() {
+        KnnVectorQueryBuilder query = new KnnVectorQueryBuilder(VECTOR_FIELD, new float[] { 1.0f, 2.0f, 3.0f }, 10);
+        String expected = """
+            {
+              "knn" : {
+                "field" : "vector",
+                "vector" : [
+                  1.0,
+                  2.0,
+                  3.0
+                ],
+                "num_candidates" : 10
+              }
+            }""";
+        assertEquals(expected, query.toString());
+    }
+
+    @Override
+    public void testMustRewrite() throws IOException {
+        SearchExecutionContext context = createSearchExecutionContext();
+        context.setAllowUnmappedFields(true);
+        TermQueryBuilder termQuery = new TermQueryBuilder("unmapped_field", 42);
+        KnnVectorQueryBuilder query = new KnnVectorQueryBuilder(VECTOR_FIELD, new float[] { 1.0f, 2.0f, 3.0f }, VECTOR_DIMENSION);
+        query.addFilterQuery(termQuery);
+
+        IllegalStateException e = expectThrows(IllegalStateException.class, () -> query.toQuery(context));
+        assertEquals("Rewrite first", e.getMessage());
+
+        QueryBuilder rewrittenQuery = query.rewrite(context);
+        assertThat(rewrittenQuery, instanceOf(MatchNoneQueryBuilder.class));
+    }
+
+    public void testBWCVersionSerialization() throws IOException {
+        float[] bwcFloat = new float[VECTOR_DIMENSION];
+        KnnVectorQueryBuilder query = createTestQueryBuilder();
+        if (query.queryVector() != null) {
+            bwcFloat = query.queryVector();
+        } else {
+            for (int i = 0; i < query.getByteQueryVector().length; i++) {
+                bwcFloat[i] = query.getByteQueryVector()[i];
+            }
+        }
+        KnnVectorQueryBuilder queryWithNoFilters = new KnnVectorQueryBuilder(query.getFieldName(), bwcFloat, query.numCands()).queryName(
+            query.queryName()
+        ).boost(query.boost());
+
+        KnnVectorQueryBuilder queryNoByteQuery = new KnnVectorQueryBuilder(query.getFieldName(), bwcFloat, query.numCands()).queryName(
+            query.queryName()
+        ).boost(query.boost()).addFilterQueries(query.filterQueries());
+
+        Version newVersion = VersionUtils.randomVersionBetween(random(), Version.V_8_7_0, Version.CURRENT);
+        Version beforeByteQueryVersion = VersionUtils.randomVersionBetween(random(), Version.V_8_2_0, Version.V_8_6_0);
+        Version beforeFilterVersion = VersionUtils.randomVersionBetween(random(), Version.V_8_0_0, Version.V_8_1_0);
+
+        assertSerialization(query, newVersion);
+        assertSerialization(queryNoByteQuery, beforeByteQueryVersion);
+        assertSerialization(queryWithNoFilters, beforeFilterVersion);
+
+        for (var tuple : List.of(
+            Tuple.tuple(beforeByteQueryVersion, queryNoByteQuery),
+            Tuple.tuple(beforeFilterVersion, queryWithNoFilters)
+        )) {
+            try (BytesStreamOutput output = new BytesStreamOutput()) {
+                output.setVersion(tuple.v1());
+                output.writeNamedWriteable(query);
+                try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry())) {
+                    in.setVersion(tuple.v1());
+                    KnnVectorQueryBuilder deserializedQuery = (KnnVectorQueryBuilder) in.readNamedWriteable(QueryBuilder.class);
+                    assertEquals(tuple.v2(), deserializedQuery);
+                    assertEquals(tuple.v2().hashCode(), deserializedQuery.hashCode());
+                }
+            }
+        }
+    }
+
+    @Override
+    public void testUnknownObjectException() {
+        assumeTrue("Test isn't relevant, since query is never parsed from xContent", false);
+    }
+
+    @Override
+    public void testFromXContent() {
+        assumeTrue("Test isn't relevant, since query is never parsed from xContent", false);
+    }
+
+    @Override
+    public void testUnknownField() {
+        assumeTrue("Test isn't relevant, since query is never parsed from xContent", false);
+    }
+}

server/src/test/java/org/elasticsearch/search/vectors/KnnByteVectorQueryBuilderTests.java

@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.search.vectors;
+
+import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
+
+public class KnnByteVectorQueryBuilderTests extends AbstractKnnVectorQueryBuilderTestCase {
+    @Override
+    DenseVectorFieldMapper.ElementType elementType() {
+        return DenseVectorFieldMapper.ElementType.BYTE;
+    }
+}

server/src/test/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilderTests.java

@@ -8,196 +8,11 @@
 
 package org.elasticsearch.search.vectors;
 
-import org.apache.lucene.search.BooleanClause;
+import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.KnnVectorQuery;
-import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.compress.CompressedXContent;
-import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.query.MatchNoneQueryBuilder;
-import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.index.query.SearchExecutionContext;
-import org.elasticsearch.index.query.TermQueryBuilder;
-import org.elasticsearch.test.AbstractBuilderTestCase;
-import org.elasticsearch.test.AbstractQueryTestCase;
-import org.elasticsearch.test.VersionUtils;
-import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xcontent.XContentFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.instanceOf;
-
-public class KnnVectorQueryBuilderTests extends AbstractQueryTestCase<KnnVectorQueryBuilder> {
-    private static final String VECTOR_FIELD = "vector";
-    private static final String VECTOR_ALIAS_FIELD = "vector_alias";
-    private static final int VECTOR_DIMENSION = 3;
 
+public class KnnVectorQueryBuilderTests extends AbstractKnnVectorQueryBuilderTestCase {
     @Override
-    protected void initializeAdditionalMappings(MapperService mapperService) throws IOException {
+    DenseVectorFieldMapper.ElementType elementType() {
-        XContentBuilder builder = XContentFactory.jsonBuilder()
+        return DenseVectorFieldMapper.ElementType.FLOAT;
-            .startObject()
-            .startObject("properties")
-            .startObject(VECTOR_FIELD)
-            .field("type", "dense_vector")
-            .field("dims", VECTOR_DIMENSION)
-            .field("index", true)
-            .field("similarity", "l2_norm")
-            .endObject()
-            .startObject(VECTOR_ALIAS_FIELD)
-            .field("type", "alias")
-            .field("path", VECTOR_FIELD)
-            .endObject()
-            .endObject()
-            .endObject();
-        mapperService.merge(
-            MapperService.SINGLE_MAPPING_NAME,
-            new CompressedXContent(Strings.toString(builder)),
-            MapperService.MergeReason.MAPPING_UPDATE
-        );
-    }
-
-    @Override
-    protected KnnVectorQueryBuilder doCreateTestQueryBuilder() {
-        String fieldName = randomBoolean() ? VECTOR_FIELD : VECTOR_ALIAS_FIELD;
-        float[] vector = new float[VECTOR_DIMENSION];
-        for (int i = 0; i < vector.length; i++) {
-            vector[i] = randomFloat();
-        }
-        int numCands = randomIntBetween(1, 1000);
-
-        KnnVectorQueryBuilder queryBuilder = new KnnVectorQueryBuilder(fieldName, vector, numCands);
-
-        if (randomBoolean()) {
-            List<QueryBuilder> filters = new ArrayList<>();
-            int numFilters = randomIntBetween(1, 5);
-            for (int i = 0; i < numFilters; i++) {
-                String filterFieldName = randomBoolean() ? KEYWORD_FIELD_NAME : TEXT_FIELD_NAME;
-                filters.add(QueryBuilders.termQuery(filterFieldName, randomAlphaOfLength(10)));
-            }
-            queryBuilder.addFilterQueries(filters);
-        }
-        return queryBuilder;
-    }
-
-    @Override
-    protected void doAssertLuceneQuery(KnnVectorQueryBuilder queryBuilder, Query query, SearchExecutionContext context) throws IOException {
-        assertTrue(query instanceof KnnVectorQuery);
-        KnnVectorQuery knnVectorQuery = (KnnVectorQuery) query;
-
-        BooleanQuery.Builder builder = new BooleanQuery.Builder();
-        for (QueryBuilder qb : queryBuilder.filterQueries()) {
-            builder.add(qb.toQuery(context), BooleanClause.Occur.FILTER);
-        }
-        BooleanQuery booleanQuery = builder.build();
-        Query filterQuery = booleanQuery.clauses().isEmpty() ? null : booleanQuery;
-        // The field should always be resolved to the concrete field
-        Query knnVectorQueryBuilt = new KnnVectorQuery(VECTOR_FIELD, queryBuilder.queryVector(), queryBuilder.numCands(), filterQuery);
-        assertEquals(knnVectorQuery, knnVectorQueryBuilt);
-    }
-
-    public void testWrongDimension() {
-        SearchExecutionContext context = createSearchExecutionContext();
-        KnnVectorQueryBuilder query = new KnnVectorQueryBuilder(VECTOR_FIELD, new float[] { 1.0f, 2.0f }, 10);
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> query.doToQuery(context));
-        assertThat(e.getMessage(), containsString("the query vector has a different dimension [2] than the index vectors [3]"));
-    }
-
-    public void testNonexistentField() {
-        SearchExecutionContext context = createSearchExecutionContext();
-        KnnVectorQueryBuilder query = new KnnVectorQueryBuilder("nonexistent", new float[] { 1.0f, 1.0f, 1.0f }, 10);
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> query.doToQuery(context));
-        assertThat(e.getMessage(), containsString("field [nonexistent] does not exist in the mapping"));
-    }
-
-    public void testWrongFieldType() {
-        SearchExecutionContext context = createSearchExecutionContext();
-        KnnVectorQueryBuilder query = new KnnVectorQueryBuilder(
-            AbstractBuilderTestCase.KEYWORD_FIELD_NAME,
-            new float[] { 1.0f, 1.0f, 1.0f },
-            10
-        );
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> query.doToQuery(context));
-        assertThat(e.getMessage(), containsString("[knn] queries are only supported on [dense_vector] fields"));
-    }
-
-    @Override
-    public void testValidOutput() {
-        KnnVectorQueryBuilder query = new KnnVectorQueryBuilder(VECTOR_FIELD, new float[] { 1.0f, 2.0f, 3.0f }, 10);
-        String expected = """
-            {
-              "knn" : {
-                "field" : "vector",
-                "vector" : [
-                  1.0,
-                  2.0,
-                  3.0
-                ],
-                "num_candidates" : 10
-              }
-            }""";
-        assertEquals(expected, query.toString());
-    }
-
-    @Override
-    public void testMustRewrite() throws IOException {
-        SearchExecutionContext context = createSearchExecutionContext();
-        context.setAllowUnmappedFields(true);
-        TermQueryBuilder termQuery = new TermQueryBuilder("unmapped_field", 42);
-        KnnVectorQueryBuilder query = new KnnVectorQueryBuilder(VECTOR_FIELD, new float[] { 1.0f, 2.0f, 3.0f }, VECTOR_DIMENSION);
-        query.addFilterQuery(termQuery);
-
-        IllegalStateException e = expectThrows(IllegalStateException.class, () -> query.toQuery(context));
-        assertEquals("Rewrite first", e.getMessage());
-
-        QueryBuilder rewrittenQuery = query.rewrite(context);
-        assertThat(rewrittenQuery, instanceOf(MatchNoneQueryBuilder.class));
-    }
-
-    public void testOldVersionSerialization() throws IOException {
-        KnnVectorQueryBuilder query = createTestQueryBuilder();
-        KnnVectorQueryBuilder queryWithNoFilters = new KnnVectorQueryBuilder(query.getFieldName(), query.queryVector(), query.numCands());
-        queryWithNoFilters.queryName(query.queryName()).boost(query.boost());
-
-        Version newVersion = VersionUtils.randomVersionBetween(random(), Version.V_8_2_0, Version.CURRENT);
-        Version oldVersion = VersionUtils.randomVersionBetween(random(), Version.V_8_0_0, Version.V_8_1_0);
-
-        assertSerialization(query, newVersion);
-        assertSerialization(queryWithNoFilters, oldVersion);
-
-        try (BytesStreamOutput output = new BytesStreamOutput()) {
-            output.setVersion(newVersion);
-            output.writeNamedWriteable(query);
-            try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry())) {
-                in.setVersion(oldVersion);
-                KnnVectorQueryBuilder deserializedQuery = (KnnVectorQueryBuilder) in.readNamedWriteable(QueryBuilder.class);
-                assertEquals(queryWithNoFilters, deserializedQuery);
-                assertEquals(queryWithNoFilters.hashCode(), deserializedQuery.hashCode());
-            }
-        }
-    }
-
-    @Override
-    public void testUnknownObjectException() throws IOException {
-        // Test isn't relevant, since query is never parsed from xContent
-    }
-
-    @Override
-    public void testFromXContent() throws IOException {
-        // Test isn't relevant, since query is never parsed from xContent
-    }
-
-    @Override
-    public void testUnknownField() throws IOException {
-        // Test isn't relevant, since query is never parsed from xContent
     }
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/frozen/RewriteCachingDirectoryReader.java

@@ -7,6 +7,7 @@
 package org.elasticsearch.index.engine.frozen;
 
 import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.ByteVectorValues;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
@@ -22,11 +23,14 @@ import org.apache.lucene.index.SortedDocValues;
 import org.apache.lucene.index.SortedNumericDocValues;
 import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.index.StoredFieldVisitor;
+import org.apache.lucene.index.StoredFields;
+import org.apache.lucene.index.TermVectors;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.VectorValues;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
 
 import java.io.IOException;
 import java.util.Comparator;
@@ -216,11 +220,21 @@ final class RewriteCachingDirectoryReader extends DirectoryReader {
             throw new UnsupportedOperationException();
         }
 
+        @Override
+        public ByteVectorValues getByteVectorValues(String field) throws IOException {
+            throw new UnsupportedOperationException();
+        }
+
         @Override
         public TopDocs searchNearestVectors(String field, float[] target, int k, Bits acceptDocs, int visitedLimit) throws IOException {
             throw new UnsupportedOperationException();
         }
 
+        @Override
+        public TopDocs searchNearestVectors(String field, BytesRef target, int k, Bits acceptDocs, int visitedLimit) throws IOException {
+            throw new UnsupportedOperationException();
+        }
+
         @Override
         public FieldInfos getFieldInfos() {
             return fieldInfos;
@@ -249,6 +263,16 @@ final class RewriteCachingDirectoryReader extends DirectoryReader {
             throw new UnsupportedOperationException();
         }
 
+        @Override
+        public TermVectors termVectors() throws IOException {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public StoredFields storedFields() throws IOException {
+            throw new UnsupportedOperationException();
+        }
+
         @Override
         public int numDocs() {
             return numDocs;

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java

@@ -8,6 +8,7 @@ package org.elasticsearch.xpack.core.security.authz.accesscontrol;
 
 import org.apache.lucene.codecs.StoredFieldsReader;
 import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.ByteVectorValues;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
@@ -21,7 +22,9 @@ import org.apache.lucene.index.SortedDocValues;
 import org.apache.lucene.index.SortedNumericDocValues;
 import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.index.StoredFieldVisitor;
+import org.apache.lucene.index.StoredFields;
 import org.apache.lucene.index.TermState;
+import org.apache.lucene.index.TermVectors;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.index.VectorValues;
@@ -162,6 +165,34 @@ public final class FieldSubsetReader extends SequentialStoredFieldsLeafReader {
         return f.iterator().hasNext() ? f : null;
     }
 
+    @Override
+    public TermVectors termVectors() throws IOException {
+        TermVectors termVectors = super.termVectors();
+        return new TermVectors() {
+            @Override
+            public Fields get(int doc) throws IOException {
+                Fields f = termVectors.get(doc);
+                if (f == null) {
+                    return null;
+                }
+                f = new FieldFilterFields(f);
+                // we need to check for emptyness, so we can return null:
+                return f.iterator().hasNext() ? f : null;
+            }
+        };
+    }
+
+    @Override
+    public StoredFields storedFields() throws IOException {
+        StoredFields storedFields = super.storedFields();
+        return new StoredFields() {
+            @Override
+            public void document(int docID, StoredFieldVisitor visitor) throws IOException {
+                storedFields.document(docID, new FieldSubsetStoredFieldVisitor(visitor));
+            }
+        };
+    }
+
     /** Filter a map by a {@link CharacterRunAutomaton} that defines the fields to retain. */
     @SuppressWarnings("unchecked")
     static Map<String, Object> filter(Map<String, ?> map, CharacterRunAutomaton includeAutomaton, int initialState) {
@@ -287,6 +318,16 @@ public final class FieldSubsetReader extends SequentialStoredFieldsLeafReader {
         return hasField(field) ? super.searchNearestVectors(field, target, k, acceptDocs, visitedLimit) : null;
     }
 
+    @Override
+    public ByteVectorValues getByteVectorValues(String field) throws IOException {
+        return hasField(field) ? super.getByteVectorValues(field) : null;
+    }
+
+    @Override
+    public TopDocs searchNearestVectors(String field, BytesRef target, int k, Bits acceptDocs, int visitedLimit) throws IOException {
+        return hasField(field) ? super.searchNearestVectors(field, target, k, acceptDocs, visitedLimit) : null;
+    }
+
     // we share core cache keys (for e.g. fielddata)
 
     @Override
@@ -310,8 +351,8 @@ public final class FieldSubsetReader extends SequentialStoredFieldsLeafReader {
         }
 
         @Override
-        public void visitDocument(int docID, StoredFieldVisitor visitor) throws IOException {
+        public void document(int docID, StoredFieldVisitor visitor) throws IOException {
-            reader.visitDocument(docID, new FieldSubsetStoredFieldVisitor(visitor));
+            reader.document(docID, new FieldSubsetStoredFieldVisitor(visitor));
         }
 
         @Override

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java

@@ -11,6 +11,7 @@ import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.IntPoint;
+import org.apache.lucene.document.KnnByteVectorField;
 import org.apache.lucene.document.KnnVectorField;
 import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.document.SortedDocValuesField;
@@ -20,6 +21,7 @@ import org.apache.lucene.document.StoredField;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.ByteVectorValues;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
@@ -214,6 +216,39 @@ public class FieldSubsetReaderTests extends ESTestCase {
         IOUtils.close(ir, iw, dir);
     }
 
+    public void testKnnByteVectors() throws Exception {
+        Directory dir = newDirectory();
+        IndexWriterConfig iwc = new IndexWriterConfig(null);
+        IndexWriter iw = new IndexWriter(dir, iwc);
+
+        Document doc = new Document();
+        doc.add(new KnnByteVectorField("fieldA", new BytesRef(new byte[] { 1, 2, 3 })));
+        doc.add(new KnnByteVectorField("fieldB", new BytesRef(new byte[] { 3, 2, 1 })));
+        iw.addDocument(doc);
+
+        DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")));
+        LeafReader leafReader = ir.leaves().get(0).reader();
+
+        // Check that fieldA behaves as normal
+        ByteVectorValues vectorValues = leafReader.getByteVectorValues("fieldA");
+        assertEquals(3, vectorValues.dimension());
+        assertEquals(1, vectorValues.size());
+        assertEquals(0, vectorValues.nextDoc());
+        assertNotNull(vectorValues.binaryValue());
+        assertNotNull(vectorValues.vectorValue());
+
+        TopDocs topDocs = leafReader.searchNearestVectors("fieldA", new BytesRef(new byte[] { 1, 1, 1 }), 5, null, Integer.MAX_VALUE);
+        assertNotNull(topDocs);
+        assertEquals(1, topDocs.scoreDocs.length);
+
+        // Check that we can't see fieldB
+        assertNull(leafReader.getByteVectorValues("fieldB"));
+        assertNull(leafReader.searchNearestVectors("fieldB", new BytesRef(new byte[] { 1, 1, 1 }), 5, null, Integer.MAX_VALUE));
+
+        TestUtil.checkReader(ir);
+        IOUtils.close(ir, iw, dir);
+    }
+
     /**
      * test filtering two stored fields (string)
      */
@@ -232,10 +267,16 @@ public class FieldSubsetReaderTests extends ESTestCase {
         DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")));
 
         // see only one field
-        Document d2 = ir.document(0);
+        {
-        assertEquals(1, d2.getFields().size());
+            Document d2 = ir.document(0);
-        assertEquals("testA", d2.get("fieldA"));
+            assertEquals(1, d2.getFields().size());
-
+            assertEquals("testA", d2.get("fieldA"));
+        }
+        {
+            Document d2 = ir.storedFields().document(0);
+            assertEquals(1, d2.getFields().size());
+            assertEquals("testA", d2.get("fieldA"));
+        }
         TestUtil.checkReader(ir);
         IOUtils.close(ir, iw, dir);
     }
@@ -258,10 +299,16 @@ public class FieldSubsetReaderTests extends ESTestCase {
         DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")));
 
         // see only one field
-        Document d2 = ir.document(0);
+        {
-        assertEquals(1, d2.getFields().size());
+            Document d2 = ir.document(0);
-        assertEquals(new BytesRef("testA"), d2.getBinaryValue("fieldA"));
+            assertEquals(1, d2.getFields().size());
-
+            assertEquals(new BytesRef("testA"), d2.getBinaryValue("fieldA"));
+        }
+        {
+            Document d2 = ir.storedFields().document(0);
+            assertEquals(1, d2.getFields().size());
+            assertEquals(new BytesRef("testA"), d2.getBinaryValue("fieldA"));
+        }
         TestUtil.checkReader(ir);
         IOUtils.close(ir, iw, dir);
     }
@@ -284,10 +331,16 @@ public class FieldSubsetReaderTests extends ESTestCase {
         DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")));
 
         // see only one field
-        Document d2 = ir.document(0);
+        {
-        assertEquals(1, d2.getFields().size());
+            Document d2 = ir.document(0);
-        assertEquals(1, d2.getField("fieldA").numericValue());
+            assertEquals(1, d2.getFields().size());
-
+            assertEquals(1, d2.getField("fieldA").numericValue());
+        }
+        {
+            Document d2 = ir.storedFields().document(0);
+            assertEquals(1, d2.getFields().size());
+            assertEquals(1, d2.getField("fieldA").numericValue());
+        }
         TestUtil.checkReader(ir);
         IOUtils.close(ir, iw, dir);
     }
@@ -310,10 +363,16 @@ public class FieldSubsetReaderTests extends ESTestCase {
         DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")));
 
         // see only one field
-        Document d2 = ir.document(0);
+        {
-        assertEquals(1, d2.getFields().size());
+            Document d2 = ir.document(0);
-        assertEquals(1L, d2.getField("fieldA").numericValue());
+            assertEquals(1, d2.getFields().size());
-
+            assertEquals(1L, d2.getField("fieldA").numericValue());
+        }
+        {
+            Document d2 = ir.storedFields().document(0);
+            assertEquals(1, d2.getFields().size());
+            assertEquals(1L, d2.getField("fieldA").numericValue());
+        }
         TestUtil.checkReader(ir);
         IOUtils.close(ir, iw, dir);
     }
@@ -336,10 +395,16 @@ public class FieldSubsetReaderTests extends ESTestCase {
         DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")));
 
         // see only one field
-        Document d2 = ir.document(0);
+        {
-        assertEquals(1, d2.getFields().size());
+            Document d2 = ir.document(0);
-        assertEquals(1F, d2.getField("fieldA").numericValue());
+            assertEquals(1, d2.getFields().size());
-
+            assertEquals(1F, d2.getField("fieldA").numericValue());
+        }
+        {
+            Document d2 = ir.storedFields().document(0);
+            assertEquals(1, d2.getFields().size());
+            assertEquals(1F, d2.getField("fieldA").numericValue());
+        }
         TestUtil.checkReader(ir);
         IOUtils.close(ir, iw, dir);
     }
@@ -362,10 +427,16 @@ public class FieldSubsetReaderTests extends ESTestCase {
         DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")));
 
         // see only one field
-        Document d2 = ir.document(0);
+        {
-        assertEquals(1, d2.getFields().size());
+            Document d2 = ir.document(0);
-        assertEquals(1D, d2.getField("fieldA").numericValue());
+            assertEquals(1, d2.getFields().size());
-
+            assertEquals(1D, d2.getField("fieldA").numericValue());
+        }
+        {
+            Document d2 = ir.storedFields().document(0);
+            assertEquals(1, d2.getFields().size());
+            assertEquals(1D, d2.getField("fieldA").numericValue());
+        }
         TestUtil.checkReader(ir);
         IOUtils.close(ir, iw, dir);
     }
@@ -624,10 +695,16 @@ public class FieldSubsetReaderTests extends ESTestCase {
         DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(automaton));
 
         // see only one field
-        Document d2 = ir.document(0);
+        {
-        assertEquals(1, d2.getFields().size());
+            Document d2 = ir.document(0);
-        assertEquals("{\"fieldA\":\"testA\"}", d2.getBinaryValue(SourceFieldMapper.NAME).utf8ToString());
+            assertEquals(1, d2.getFields().size());
-
+            assertEquals("{\"fieldA\":\"testA\"}", d2.getBinaryValue(SourceFieldMapper.NAME).utf8ToString());
+        }
+        {
+            Document d2 = ir.storedFields().document(0);
+            assertEquals(1, d2.getFields().size());
+            assertEquals("{\"fieldA\":\"testA\"}", d2.getBinaryValue(SourceFieldMapper.NAME).utf8ToString());
+        }
         TestUtil.checkReader(ir);
         IOUtils.close(ir, iw, dir);
     }
@@ -1025,11 +1102,17 @@ public class FieldSubsetReaderTests extends ESTestCase {
 
         // see no vectors
         assertNull(segmentReader.getTermVectors(0));
+        assertNull(segmentReader.termVectors().get(0));
 
         // see no stored fields
-        Document document = segmentReader.document(0);
+        {
-        assertEquals(0, document.getFields().size());
+            Document document = segmentReader.document(0);
-
+            assertEquals(0, document.getFields().size());
+        }
+        {
+            Document document = segmentReader.storedFields().document(0);
+            assertEquals(0, document.getFields().size());
+        }
         TestUtil.checkReader(ir);
         IOUtils.close(ir, iw, dir);
     }

x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java

@@ -11,8 +11,8 @@ import org.apache.lucene.document.Field;
 import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.document.StoredField;
-import org.apache.lucene.sandbox.search.IndexSortSortedNumericDocValuesRangeQuery;
 import org.apache.lucene.search.IndexOrDocValuesQuery;
+import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;