github-mirrors/elasticsearch
1
0
Fork 0
mirror of https://github.com/elastic/elasticsearch.git synced 2025-04-25 15:47:23 -04:00
Code Issues Projects Releases Packages Wiki Activity

Add an optimised int8 vector distance function for aarch64. (#106133)

Browse source 
This commit adds an optimised int8 vector distance implementation for aarch64. Additional platforms like, say, x64, will be added as a follow-up.

The vector distance implementation outperforms Lucene's Pamana Vector implementation for binary comparisons by approx 5x (depending on the number of dimensions). It does so by means of compiler intrinsics built into a separate native library and link by Panama's FFI. Comparisons are performed on off-heap mmap'ed vector data.

The implementation is currently only used during merging of scalar quantized segments, through a custom format ES814HnswScalarQuantizedVectorsFormat, but its usage will likely be expanded over time.

Co-authored-by: Benjamin Trent <ben.w.trent@gmail.com>
Co-authored-by: Lorenzo Dematté <lorenzo.dematte@elastic.co>
Co-authored-by: Mark Vieira <portugee@gmail.com>
Co-authored-by: Ryan Ernst <ryan@iernst.net>
This commit is contained in:
Chris Hegarty 2024-04-12 08:44:21 +01:00 • committed by GitHub
parent fb1bc58664
commit 6b52d7837b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
63 changed files with 4812 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

4
server/src/main/java/module-info.java
View file

@ -32,6 +32,7 @@ module org.elasticsearch.server {
requires org.elasticsearch.plugin.analysis;
requires org.elasticsearch.grok;
requires org.elasticsearch.tdigest;
requires org.elasticsearch.vec;
requires com.sun.jna;
requires hppc;
@ -437,7 +438,8 @@ module org.elasticsearch.server {
provides org.apache.lucene.codecs.KnnVectorsFormat
with
org.elasticsearch.index.codec.vectors.ES813FlatVectorFormat,
org.elasticsearch.index.codec.vectors.ES813Int8FlatVectorFormat;
org.elasticsearch.index.codec.vectors.ES813Int8FlatVectorFormat,
org.elasticsearch.index.codec.vectors.ES814HnswScalarQuantizedVectorsFormat;
provides org.apache.lucene.codecs.Codec with Elasticsearch814Codec;
provides org.apache.logging.log4j.core.util.ContextDataProvider with org.elasticsearch.common.logging.DynamicContextDataProvider;