diff --git a/docs/changelog/113607.yaml b/docs/changelog/113607.yaml new file mode 100644 index 000000000000..eb25d2600a55 --- /dev/null +++ b/docs/changelog/113607.yaml @@ -0,0 +1,5 @@ +pr: 113607 +summary: Add more `dense_vector` details for cluster stats field stats +area: Search +type: enhancement +issues: [] diff --git a/docs/reference/cluster/stats.asciidoc b/docs/reference/cluster/stats.asciidoc index 8e4f630ef7da..5dd84abc96e1 100644 --- a/docs/reference/cluster/stats.asciidoc +++ b/docs/reference/cluster/stats.asciidoc @@ -432,6 +432,15 @@ To get information on segment files, use the < vectorIndexTypeCount; // count of mappings by index type + Map vectorSimilarityTypeCount; // count of mappings by similarity + Map vectorElementTypeCount; // count of mappings by element type int indexedVectorCount; // number of times vectors with index:true are used in mappings of this cluster int indexedVectorDimMin; // minimum dimension of indexed vectors in this cluster int indexedVectorDimMax; // maximum dimension of indexed vectors in this cluster @@ -31,21 +35,14 @@ public final class DenseVectorFieldStats extends FieldStats { indexedVectorCount = 0; indexedVectorDimMin = UNSET; indexedVectorDimMax = UNSET; - } - - DenseVectorFieldStats(StreamInput in) throws IOException { - super(in); - indexedVectorCount = in.readVInt(); - indexedVectorDimMin = in.readVInt(); - indexedVectorDimMax = in.readVInt(); + vectorIndexTypeCount = new HashMap<>(); + vectorSimilarityTypeCount = new HashMap<>(); + vectorElementTypeCount = new HashMap<>(); } @Override public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - out.writeVInt(indexedVectorCount); - out.writeVInt(indexedVectorDimMin); - out.writeVInt(indexedVectorDimMax); + assert false : "writeTo should not be called on DenseVectorFieldStats"; } @Override @@ -53,6 +50,21 @@ public final class DenseVectorFieldStats extends FieldStats { builder.field("indexed_vector_count", indexedVectorCount); builder.field("indexed_vector_dim_min", indexedVectorDimMin); builder.field("indexed_vector_dim_max", indexedVectorDimMax); + if (vectorIndexTypeCount.isEmpty() == false) { + builder.startObject("vector_index_type_count"); + builder.mapContents(vectorIndexTypeCount); + builder.endObject(); + } + if (vectorSimilarityTypeCount.isEmpty() == false) { + builder.startObject("vector_similarity_type_count"); + builder.mapContents(vectorSimilarityTypeCount); + builder.endObject(); + } + if (vectorElementTypeCount.isEmpty() == false) { + builder.startObject("vector_element_type_count"); + builder.mapContents(vectorElementTypeCount); + builder.endObject(); + } } @Override @@ -69,11 +81,53 @@ public final class DenseVectorFieldStats extends FieldStats { DenseVectorFieldStats that = (DenseVectorFieldStats) o; return indexedVectorCount == that.indexedVectorCount && indexedVectorDimMin == that.indexedVectorDimMin - && indexedVectorDimMax == that.indexedVectorDimMax; + && indexedVectorDimMax == that.indexedVectorDimMax + && Objects.equals(vectorIndexTypeCount, that.vectorIndexTypeCount) + && Objects.equals(vectorSimilarityTypeCount, that.vectorSimilarityTypeCount) + && Objects.equals(vectorElementTypeCount, that.vectorElementTypeCount); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), indexedVectorCount, indexedVectorDimMin, indexedVectorDimMax); + return Objects.hash( + super.hashCode(), + indexedVectorCount, + indexedVectorDimMin, + indexedVectorDimMax, + vectorIndexTypeCount, + vectorSimilarityTypeCount, + vectorElementTypeCount + ); + } + + @Override + public String toString() { + return "DenseVectorFieldStats{" + + "vectorIndexTypeCount=" + + vectorIndexTypeCount + + ", vectorSimilarityTypeCount=" + + vectorSimilarityTypeCount + + ", vectorElementTypeCount=" + + vectorElementTypeCount + + ", indexedVectorCount=" + + indexedVectorCount + + ", indexedVectorDimMin=" + + indexedVectorDimMin + + ", indexedVectorDimMax=" + + indexedVectorDimMax + + ", scriptCount=" + + scriptCount + + ", scriptLangs=" + + scriptLangs + + ", fieldScriptStats=" + + fieldScriptStats + + ", name='" + + name + + '\'' + + ", count=" + + count + + ", indexCount=" + + indexCount + + '}'; } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingStats.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingStats.java index f1391cca82c5..d2e597316991 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingStats.java @@ -86,9 +86,30 @@ public final class MappingStats implements ToXContentFragment, Writeable { FieldStats stats; if (type.equals("dense_vector")) { stats = fieldTypes.computeIfAbsent(type, DenseVectorFieldStats::new); - boolean indexed = fieldMapping.containsKey("index") ? (boolean) fieldMapping.get("index") : false; + DenseVectorFieldStats vStats = (DenseVectorFieldStats) stats; + if (fieldMapping.containsKey("similarity")) { + Object similarity = fieldMapping.get("similarity"); + vStats.vectorSimilarityTypeCount.compute(similarity.toString(), (t, c) -> c == null ? count : c + count); + } + String elementTypeStr = "float"; + if (fieldMapping.containsKey("element_type")) { + Object elementType = fieldMapping.get("element_type"); + elementTypeStr = elementType.toString(); + } + vStats.vectorElementTypeCount.compute(elementTypeStr, (t, c) -> c == null ? count : c + count); + boolean indexed = fieldMapping.containsKey("index") && (boolean) fieldMapping.get("index"); if (indexed) { - DenseVectorFieldStats vStats = (DenseVectorFieldStats) stats; + Object indexOptions = fieldMapping.get("index_options"); + // NOTE, while the default for `float` is now `int8_hnsw`, that is actually added to the mapping + // if the value is truly missing & we are indexed, we default to hnsw. + String indexTypeStr = "hnsw"; + if (indexOptions instanceof Map indexOptionsMap) { + Object indexType = indexOptionsMap.get("type"); + if (indexType != null) { + indexTypeStr = indexType.toString(); + } + } + vStats.vectorIndexTypeCount.compute(indexTypeStr, (t, c) -> c == null ? count : c + count); vStats.indexedVectorCount += count; Object obj = fieldMapping.get("dims"); if (obj != null) { @@ -100,6 +121,8 @@ public final class MappingStats implements ToXContentFragment, Writeable { vStats.indexedVectorDimMax = dims; } } + } else { + vStats.vectorIndexTypeCount.compute(DenseVectorFieldStats.NOT_INDEXED, (t, c) -> c == null ? 1 : c + 1); } } else { stats = fieldTypes.computeIfAbsent(type, FieldStats::new); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStatsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStatsAction.java index 449f5d5b770c..014360477bee 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStatsAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStatsAction.java @@ -31,7 +31,10 @@ import static org.elasticsearch.rest.RestUtils.getTimeout; @ServerlessScope(Scope.INTERNAL) public class RestClusterStatsAction extends BaseRestHandler { - private static final Set SUPPORTED_CAPABILITIES = Set.of("human-readable-total-docs-size"); + private static final Set SUPPORTED_CAPABILITIES = Set.of( + "human-readable-total-docs-size", + "verbose-dense-vector-mapping-stats" + ); private static final Set SUPPORTED_CAPABILITIES_CCS_STATS = Sets.union(SUPPORTED_CAPABILITIES, Set.of("ccs-stats")); public static final FeatureFlag CCS_TELEMETRY_FEATURE_FLAG = new FeatureFlag("ccs_telemetry"); diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/MappingStatsTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/MappingStatsTests.java index 8d212a2e0109..2c374c7d26de 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/MappingStatsTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/MappingStatsTests.java @@ -114,7 +114,16 @@ public class MappingStatsTests extends AbstractWireSerializingTestCase