diff --git a/docs/reference/searchable-snapshots/apis/node-cache-stats.asciidoc b/docs/reference/searchable-snapshots/apis/node-cache-stats.asciidoc new file mode 100644 index 000000000000..49a6ce99ed53 --- /dev/null +++ b/docs/reference/searchable-snapshots/apis/node-cache-stats.asciidoc @@ -0,0 +1,136 @@ +[role="xpack"] +[testenv="enterprise"] +[[searchable-snapshots-api-cache-stats]] +=== Cache stats API +++++ +Cache stats +++++ + +Provide statistics about the searchable snapshots <>. + +[[searchable-snapshots-api-cache-stats-request]] +==== {api-request-title} + +`GET /_searchable_snapshots/cache/stats` + + +`GET /_searchable_snapshots//cache/stats` + +[[searchable-snapshots-api-cache-stats-prereqs]] +==== {api-prereq-title} + +If the {es} {security-features} are enabled, you must have the +`manage` cluster privilege to use this API. +For more information, see <>. + +[[searchable-snapshots-api-cache-stats-desc]] +==== {api-description-title} + +You can use the Cache Stats API to retrieve statistics about the +usage of the <> on nodes in a cluster. + +[[searchable-snapshots-api-cache-stats-path-params]] +==== {api-path-parms-title} + +``:: + (Optional, string) The names of particular nodes in the cluster to target. + For example, `nodeId1,nodeId2`. For node selection options, see + <>. + +[[searchable-snapshots-api-cache-stats-query-params]] +==== {api-query-parms-title} + +include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] + +[role="child_attributes"] +[[searchable-snapshots-api-cache-stats-response-body]] +==== {api-response-body-title} + +`nodes`:: +(object) +Contains statistics for the nodes selected by the request. ++ +.Properties of `nodes` +[%collapsible%open] +==== +``:: +(object) +Contains statistics for the node with the given identifier. ++ +.Properties of `` +[%collapsible%open] +===== +`shared_cache`:: +(object) +Contains statistics about the shared cache file. ++ +.Properties of `shared_cache` +[%collapsible%open] +====== +`reads`:: +(long) Number of times the shared cache is used to read data from. + +`bytes_read_in_bytes`:: +(long) The total of bytes read from the shared cache. + +`writes`:: +(long) Number of times data from the blob store repository is written in the shared cache. + +`bytes_written_in_bytes`:: +(long) The total of bytes written in the shared cache. + +`evictions`:: +(long) Number of regions evicted from the shared cache file. + +`num_regions`:: +(integer) Number of regions in the shared cache file. + +`size_in_bytes`:: +(long) The total size in bytes of the shared cache file. + +`region_size_in_bytes`:: +(long) The size in bytes of a region in the shared cache file. +====== +===== +==== + + +[[searchable-snapshots-api-cache-stats-example]] +==== {api-examples-title} + +Retrieves the searchable snapshots shared cache file statistics for all data nodes: + +[source,console] +-------------------------------------------------- +GET /_searchable_snapshots/cache/stats +-------------------------------------------------- +// TEST[setup:node] + +The API returns the following response: + +[source,console-result] +---- +{ + "nodes" : { + "eerrtBMtQEisohZzxBLUSw" : { + "shared_cache" : { + "reads" : 6051, + "bytes_read_in_bytes" : 5448829, + "writes" : 37, + "bytes_written_in_bytes" : 1208320, + "evictions" : 5, + "num_regions" : 65536, + "size_in_bytes" : 1099511627776, + "region_size_in_bytes" : 16777216 + } + } + } +} +---- +// TESTRESPONSE[s/"reads" : 6051/"reads" : 0/] +// TESTRESPONSE[s/"bytes_read_in_bytes" : 5448829/"bytes_read_in_bytes" : 0/] +// TESTRESPONSE[s/"writes" : 37/"writes" : 0/] +// TESTRESPONSE[s/"bytes_written_in_bytes" : 1208320/"bytes_written_in_bytes" : 0/] +// TESTRESPONSE[s/"evictions" : 5/"evictions" : 0/] +// TESTRESPONSE[s/"num_regions" : 65536/"num_regions" : 0/] +// TESTRESPONSE[s/"size_in_bytes" : 1099511627776/"size_in_bytes" : 0/] +// TESTRESPONSE[s/"eerrtBMtQEisohZzxBLUSw"/\$node_name/] diff --git a/docs/reference/searchable-snapshots/apis/searchable-snapshots-apis.asciidoc b/docs/reference/searchable-snapshots/apis/searchable-snapshots-apis.asciidoc index 24e35db88c2f..f19c7415d00f 100644 --- a/docs/reference/searchable-snapshots/apis/searchable-snapshots-apis.asciidoc +++ b/docs/reference/searchable-snapshots/apis/searchable-snapshots-apis.asciidoc @@ -6,5 +6,7 @@ You can use the following APIs to perform searchable snapshots operations. * <> +* <> include::mount-snapshot.asciidoc[] +include::node-cache-stats.asciidoc[] diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/searchable_snapshots.cache_stats.json b/rest-api-spec/src/main/resources/rest-api-spec/api/searchable_snapshots.cache_stats.json new file mode 100644 index 000000000000..3cfb71bb11e1 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/searchable_snapshots.cache_stats.json @@ -0,0 +1,35 @@ +{ + "searchable_snapshots.cache_stats": { + "documentation": { + "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/searchable-snapshots-apis.html", + "description": "Retrieve node-level cache statistics about searchable snapshots." + }, + "stability": "experimental", + "visibility":"public", + "headers":{ + "accept": [ "application/json"] + }, + "url": { + "paths": [ + { + "path": "/_searchable_snapshots/cache/stats", + "methods": [ + "GET" + ] + }, + { + "path": "/_searchable_snapshots/{node_id}/cache/stats", + "methods": [ + "GET" + ], + "parts":{ + "node_id":{ + "type":"list", + "description":"A comma-separated list of node IDs or names to limit the returned information; use `_local` to return information from the node you're connecting to, leave empty to get information from all nodes" + } + } + } + ] + } + } +} diff --git a/x-pack/plugin/searchable-snapshots/qa/rest/build.gradle b/x-pack/plugin/searchable-snapshots/qa/rest/build.gradle index c80fbe9dee4e..3f115bf8f9f9 100644 --- a/x-pack/plugin/searchable-snapshots/qa/rest/build.gradle +++ b/x-pack/plugin/searchable-snapshots/qa/rest/build.gradle @@ -10,7 +10,7 @@ final File repoDir = file("$buildDir/testclusters/repo") restResources { restApi { - include 'indices', 'search', 'bulk', 'snapshot', 'nodes', '_common', 'searchable_snapshots' + include 'indices', 'search', 'bulk', 'snapshot', 'nodes', '_common', 'searchable_snapshots', 'cluster' } } diff --git a/x-pack/plugin/searchable-snapshots/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/shared_cache_stats.yml b/x-pack/plugin/searchable-snapshots/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/shared_cache_stats.yml new file mode 100644 index 000000000000..34f6452e11a0 --- /dev/null +++ b/x-pack/plugin/searchable-snapshots/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/shared_cache_stats.yml @@ -0,0 +1,108 @@ +--- +setup: + - skip: + version: " - 7.99.99" + reason: node-level cache statistics added in 8.0.0 + + - do: + indices.create: + index: docs + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + + - do: + bulk: + body: + - index: + _index: docs + _id: 1 + - field: foo + - index: + _index: docs + _id: 2 + - field: bar + - index: + _index: docs + _id: 3 + - field: baz + + - do: + snapshot.create_repository: + repository: repository-fs + body: + type: fs + settings: + location: "repository-fs" + + # Remove the snapshot if a previous test failed to delete it. + # Useful for third party tests that runs the test against a real external service. + - do: + snapshot.delete: + repository: repository-fs + snapshot: snapshot + ignore: 404 + + - do: + snapshot.create: + repository: repository-fs + snapshot: snapshot + wait_for_completion: true + + - do: + indices.delete: + index: docs +--- +teardown: + + - do: + snapshot.delete: + repository: repository-fs + snapshot: snapshot + ignore: 404 + + - do: + snapshot.delete_repository: + repository: repository-fs + +--- +"Node Cache Stats API with Frozen Indices": + + - do: + cluster.state: {} + - set: { master_node: node_id } + + - do: + searchable_snapshots.mount: + repository: repository-fs + snapshot: snapshot + wait_for_completion: true + storage: shared_cache + body: + index: docs + renamed_index: frozen-docs + + - match: { snapshot.snapshot: snapshot } + - match: { snapshot.shards.failed: 0 } + - match: { snapshot.shards.successful: 1 } + + - do: + searchable_snapshots.cache_stats: + human: true + + - is_true: nodes.$node_id + - is_true: nodes.$node_id.shared_cache + - match: { nodes.$node_id.shared_cache.reads: 0 } + - match: { nodes.$node_id.shared_cache.bytes_read: "0b" } + - match: { nodes.$node_id.shared_cache.bytes_read_in_bytes: 0 } + - match: { nodes.$node_id.shared_cache.writes: 0 } + - match: { nodes.$node_id.shared_cache.bytes_written: "0b" } + - match: { nodes.$node_id.shared_cache.bytes_written_in_bytes: 0 } + - match: { nodes.$node_id.shared_cache.evictions: 0 } + - match: { nodes.$node_id.shared_cache.num_regions: 64 } + - match: { nodes.$node_id.shared_cache.size: "16mb" } + - match: { nodes.$node_id.shared_cache.size_in_bytes: 16777216 } + - match: { nodes.$node_id.shared_cache.region_size: "256kb" } + - match: { nodes.$node_id.shared_cache.region_size_in_bytes: 262144 } + diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java index 611ccb95c747..ae46f9eca4ed 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java @@ -12,6 +12,7 @@ import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.xpack.searchablesnapshots.action.cache.TransportSearchableSnapshotsNodeCachesStatsAction; import org.elasticsearch.xpack.searchablesnapshots.allocation.decider.DedicatedFrozenNodeAllocationDecider; import org.elasticsearch.xpack.searchablesnapshots.cache.blob.BlobStoreCacheService; import org.elasticsearch.client.Client; @@ -43,6 +44,7 @@ import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.engine.EngineFactory; import org.elasticsearch.index.engine.FrozenEngine; import org.elasticsearch.index.engine.ReadOnlyEngine; +import org.elasticsearch.xpack.searchablesnapshots.rest.RestSearchableSnapshotsNodeCachesStatsAction; import org.elasticsearch.xpack.searchablesnapshots.store.SearchableSnapshotDirectory; import org.elasticsearch.index.store.Store; import org.elasticsearch.index.translog.Translog; @@ -343,6 +345,7 @@ public class SearchableSnapshots extends Plugin implements IndexStorePlugin, Eng PersistentCache.cleanUp(settings, nodeEnvironment); } this.allocator.set(new SearchableSnapshotAllocator(client, clusterService.getRerouteService(), frozenCacheInfoService)); + components.add(new FrozenCacheServiceSupplier(frozenCacheService.get())); components.add(new CacheServiceSupplier(cacheService.get())); return Collections.unmodifiableList(components); } @@ -479,7 +482,11 @@ public class SearchableSnapshots extends Plugin implements IndexStorePlugin, Eng new ActionHandler<>(XPackInfoFeatureAction.SEARCHABLE_SNAPSHOTS, SearchableSnapshotsInfoTransportAction.class), new ActionHandler<>(TransportSearchableSnapshotCacheStoresAction.TYPE, TransportSearchableSnapshotCacheStoresAction.class), new ActionHandler<>(FrozenCacheInfoAction.INSTANCE, FrozenCacheInfoAction.TransportAction.class), - new ActionHandler<>(FrozenCacheInfoNodeAction.INSTANCE, FrozenCacheInfoNodeAction.TransportAction.class) + new ActionHandler<>(FrozenCacheInfoNodeAction.INSTANCE, FrozenCacheInfoNodeAction.TransportAction.class), + new ActionHandler<>( + TransportSearchableSnapshotsNodeCachesStatsAction.TYPE, + TransportSearchableSnapshotsNodeCachesStatsAction.class + ) ); } @@ -495,7 +502,8 @@ public class SearchableSnapshots extends Plugin implements IndexStorePlugin, Eng return List.of( new RestSearchableSnapshotsStatsAction(), new RestClearSearchableSnapshotsCacheAction(), - new RestMountSearchableSnapshotAction() + new RestMountSearchableSnapshotAction(), + new RestSearchableSnapshotsNodeCachesStatsAction() ); } @@ -656,6 +664,9 @@ public class SearchableSnapshots extends Plugin implements IndexStorePlugin, Eng Releasables.close(frozenCacheService.get()); } + /** + * Allows to inject the {@link CacheService} instance to transport actions + */ public static final class CacheServiceSupplier implements Supplier { @Nullable @@ -670,4 +681,22 @@ public class SearchableSnapshots extends Plugin implements IndexStorePlugin, Eng return cacheService; } } + + /** + * Allows to inject the {@link FrozenCacheService} instance to transport actions + */ + public static final class FrozenCacheServiceSupplier implements Supplier { + + @Nullable + private final FrozenCacheService frozenCacheService; + + FrozenCacheServiceSupplier(@Nullable FrozenCacheService frozenCacheService) { + this.frozenCacheService = frozenCacheService; + } + + @Override + public FrozenCacheService get() { + return frozenCacheService; + } + } } diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/cache/TransportSearchableSnapshotsNodeCachesStatsAction.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/cache/TransportSearchableSnapshotsNodeCachesStatsAction.java new file mode 100644 index 000000000000..a5470e706cbc --- /dev/null +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/cache/TransportSearchableSnapshotsNodeCachesStatsAction.java @@ -0,0 +1,281 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.searchablesnapshots.action.cache; + +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.FailedNodeException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.nodes.BaseNodeResponse; +import org.elasticsearch.action.support.nodes.BaseNodesRequest; +import org.elasticsearch.action.support.nodes.BaseNodesResponse; +import org.elasticsearch.action.support.nodes.TransportNodesAction; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.xcontent.ToXContentFragment; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportRequest; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots; +import org.elasticsearch.xpack.searchablesnapshots.cache.shared.FrozenCacheService; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +/** + * Node level stats about searchable snapshots caches. + */ +public class TransportSearchableSnapshotsNodeCachesStatsAction extends TransportNodesAction< + TransportSearchableSnapshotsNodeCachesStatsAction.NodesRequest, + TransportSearchableSnapshotsNodeCachesStatsAction.NodesCachesStatsResponse, + TransportSearchableSnapshotsNodeCachesStatsAction.NodeRequest, + TransportSearchableSnapshotsNodeCachesStatsAction.NodeCachesStatsResponse> { + + public static final String ACTION_NAME = "cluster:admin/xpack/searchable_snapshots/cache/stats"; + + public static final ActionType TYPE = new ActionType<>(ACTION_NAME, NodesCachesStatsResponse::new); + + private final Supplier frozenCacheService; + + @Inject + public TransportSearchableSnapshotsNodeCachesStatsAction( + ThreadPool threadPool, + ClusterService clusterService, + TransportService transportService, + ActionFilters actionFilters, + SearchableSnapshots.FrozenCacheServiceSupplier frozenCacheService + ) { + super( + ACTION_NAME, + threadPool, + clusterService, + transportService, + actionFilters, + NodesRequest::new, + NodeRequest::new, + ThreadPool.Names.MANAGEMENT, + ThreadPool.Names.SAME, + NodeCachesStatsResponse.class + ); + this.frozenCacheService = frozenCacheService; + } + + @Override + protected NodesCachesStatsResponse newResponse( + NodesRequest request, + List responses, + List failures + ) { + return new NodesCachesStatsResponse(clusterService.getClusterName(), responses, failures); + } + + @Override + protected NodeRequest newNodeRequest(NodesRequest request) { + return new NodeRequest(); + } + + @Override + protected NodeCachesStatsResponse newNodeResponse(StreamInput in) throws IOException { + return new NodeCachesStatsResponse(in); + } + + @Override + protected void resolveRequest(NodesRequest request, ClusterState clusterState) { + final ImmutableOpenMap dataNodes = clusterState.getNodes().getDataNodes(); + + final DiscoveryNode[] resolvedNodes; + if (request.nodesIds() == null || request.nodesIds().length == 0) { + resolvedNodes = dataNodes.values().toArray(DiscoveryNode.class); + } else { + resolvedNodes = Arrays.stream(request.nodesIds()) + .filter(dataNodes::containsKey) + .map(dataNodes::get) + .collect(Collectors.toList()) + .toArray(DiscoveryNode[]::new); + } + request.setConcreteNodes(resolvedNodes); + } + + @Override + protected NodeCachesStatsResponse nodeOperation(NodeRequest request, Task task) { + final FrozenCacheService.Stats frozenCacheStats; + if (frozenCacheService.get() != null) { + frozenCacheStats = frozenCacheService.get().getStats(); + } else { + frozenCacheStats = FrozenCacheService.Stats.EMPTY; + } + return new NodeCachesStatsResponse( + clusterService.localNode(), + frozenCacheStats.getNumberOfRegions(), + frozenCacheStats.getSize(), + frozenCacheStats.getRegionSize(), + frozenCacheStats.getWriteCount(), + frozenCacheStats.getWriteBytes(), + frozenCacheStats.getReadCount(), + frozenCacheStats.getReadBytes(), + frozenCacheStats.getEvictCount() + ); + } + + public static final class NodeRequest extends TransportRequest { + + public NodeRequest() {} + + public NodeRequest(StreamInput in) throws IOException { + super(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + } + } + + public static final class NodesRequest extends BaseNodesRequest { + + public NodesRequest(String[] nodes) { + super(nodes); + } + + public NodesRequest(StreamInput in) throws IOException { + super(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + } + } + + public static class NodeCachesStatsResponse extends BaseNodeResponse implements ToXContentFragment { + + private final int numRegions; + private final long size; + private final long regionSize; + private final long writes; + private final long bytesWritten; + private final long reads; + private final long bytesRead; + private final long evictions; + + public NodeCachesStatsResponse( + DiscoveryNode node, + int numRegions, + long size, + long regionSize, + long writes, + long bytesWritten, + long reads, + long bytesRead, + long evictions + ) { + super(node); + this.numRegions = numRegions; + this.size = size; + this.regionSize = regionSize; + this.writes = writes; + this.bytesWritten = bytesWritten; + this.reads = reads; + this.bytesRead = bytesRead; + this.evictions = evictions; + } + + public NodeCachesStatsResponse(StreamInput in) throws IOException { + super(in); + this.numRegions = in.readVInt(); + this.size = in.readVLong(); + this.regionSize = in.readVLong(); + this.writes = in.readVLong(); + this.bytesWritten = in.readVLong(); + this.reads = in.readVLong(); + this.bytesRead = in.readVLong(); + this.evictions = in.readVLong(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeVInt(numRegions); + out.writeVLong(size); + out.writeVLong(regionSize); + out.writeVLong(writes); + out.writeVLong(bytesWritten); + out.writeVLong(reads); + out.writeVLong(bytesRead); + out.writeVLong(evictions); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(getNode().getId()); + { + builder.startObject("shared_cache"); + { + builder.field("reads", reads); + builder.humanReadableField("bytes_read_in_bytes", "bytes_read", ByteSizeValue.ofBytes(bytesRead)); + builder.field("writes", writes); + builder.humanReadableField("bytes_written_in_bytes", "bytes_written", ByteSizeValue.ofBytes(bytesWritten)); + builder.field("evictions", evictions); + builder.field("num_regions", numRegions); + builder.humanReadableField("size_in_bytes", "size", ByteSizeValue.ofBytes(size)); + builder.humanReadableField("region_size_in_bytes", "region_size", ByteSizeValue.ofBytes(regionSize)); + } + builder.endObject(); + } + builder.endObject(); + return builder; + } + } + + public static class NodesCachesStatsResponse extends BaseNodesResponse implements ToXContentObject { + + public NodesCachesStatsResponse(StreamInput in) throws IOException { + super(in); + } + + public NodesCachesStatsResponse(ClusterName clusterName, List nodes, List failures) { + super(clusterName, nodes, failures); + } + + @Override + protected List readNodesFrom(StreamInput in) throws IOException { + return in.readList(NodeCachesStatsResponse::new); + } + + @Override + protected void writeNodesTo(StreamOutput out, List nodes) throws IOException { + out.writeList(nodes); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + { + builder.startObject("nodes"); + for (NodeCachesStatsResponse node : getNodes()) { + node.toXContent(builder, params); + } + builder.endObject(); + } + builder.endObject(); + return builder; + } + + } +} diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/CacheService.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/CacheService.java index 5ab99250c48e..09a43ca8825e 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/CacheService.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/CacheService.java @@ -781,5 +781,4 @@ public class CacheService extends AbstractLifecycleComponent { return "cache file event [type=" + type + ", value=" + value + ']'; } } - } diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/PersistentCache.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/PersistentCache.java index bd3102bfe6b2..ec3948706c1b 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/PersistentCache.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/PersistentCache.java @@ -49,11 +49,11 @@ import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.Index; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.shard.ShardPath; -import org.elasticsearch.xpack.searchablesnapshots.cache.common.CacheFile; -import org.elasticsearch.xpack.searchablesnapshots.cache.common.CacheKey; import org.elasticsearch.repositories.IndexId; import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.xpack.searchablesnapshots.cache.common.ByteRange; +import org.elasticsearch.xpack.searchablesnapshots.cache.common.CacheFile; +import org.elasticsearch.xpack.searchablesnapshots.cache.common.CacheKey; import java.io.Closeable; import java.io.IOException; diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/shared/FrozenCacheService.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/shared/FrozenCacheService.java index 07248c9aee16..fe53bf7991d9 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/shared/FrozenCacheService.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/shared/FrozenCacheService.java @@ -28,12 +28,12 @@ import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.KeyedLock; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.xpack.searchablesnapshots.cache.common.CacheKey; -import org.elasticsearch.xpack.searchablesnapshots.cache.common.SparseFileTracker; import org.elasticsearch.node.NodeRoleSettings; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.DataTier; import org.elasticsearch.xpack.searchablesnapshots.cache.common.ByteRange; +import org.elasticsearch.xpack.searchablesnapshots.cache.common.CacheKey; +import org.elasticsearch.xpack.searchablesnapshots.cache.common.SparseFileTracker; import java.io.IOException; import java.io.UncheckedIOException; @@ -48,6 +48,7 @@ import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.atomic.LongAdder; import java.util.function.Consumer; import java.util.function.LongSupplier; import java.util.function.Predicate; @@ -169,10 +170,12 @@ public class FrozenCacheService implements Releasable { private final KeyedLock keyedLock = new KeyedLock<>(); private final SharedBytes sharedBytes; + private final long cacheSize; private final long regionSize; private final ByteSizeValue rangeSize; private final ByteSizeValue recoveryRangeSize; + private final int numRegions; private final ConcurrentLinkedQueue freeRegions = new ConcurrentLinkedQueue<>(); private final Entry[] freqs; private final int maxFreq; @@ -182,12 +185,20 @@ public class FrozenCacheService implements Releasable { private final CacheDecayTask decayTask; + private final LongAdder writeCount = new LongAdder(); + private final LongAdder writeBytes = new LongAdder(); + + private final LongAdder readCount = new LongAdder(); + private final LongAdder readBytes = new LongAdder(); + + private final LongAdder evictCount = new LongAdder(); + @SuppressWarnings({ "unchecked", "rawtypes" }) public FrozenCacheService(NodeEnvironment environment, Settings settings, ThreadPool threadPool) { this.currentTimeSupplier = threadPool::relativeTimeInMillis; - final long cacheSize = SNAPSHOT_CACHE_SIZE_SETTING.get(settings).getBytes(); + this.cacheSize = SNAPSHOT_CACHE_SIZE_SETTING.get(settings).getBytes(); final long regionSize = SNAPSHOT_CACHE_REGION_SIZE_SETTING.get(settings).getBytes(); - final int numRegions = Math.toIntExact(cacheSize / regionSize); + this.numRegions = Math.toIntExact(cacheSize / regionSize); keyMapping = new ConcurrentHashMap<>(); if (Assertions.ENABLED) { regionOwners = new AtomicReference[numRegions]; @@ -206,7 +217,7 @@ public class FrozenCacheService implements Releasable { this.minTimeDelta = SNAPSHOT_CACHE_MIN_TIME_DELTA_SETTING.get(settings).millis(); freqs = new Entry[maxFreq]; try { - sharedBytes = new SharedBytes(numRegions, regionSize, environment); + sharedBytes = new SharedBytes(numRegions, regionSize, environment, writeBytes::add, readBytes::add); } catch (IOException e) { throw new UncheckedIOException(e); } @@ -345,6 +356,19 @@ public class FrozenCacheService implements Releasable { return freeRegions.size(); } + public Stats getStats() { + return new Stats( + numRegions, + cacheSize, + regionSize, + evictCount.sum(), + writeCount.sum(), + writeBytes.sum(), + readCount.sum(), + readBytes.sum() + ); + } + private synchronized boolean invariant(final Entry e, boolean present) { boolean found = false; for (int i = 0; i < maxFreq; i++) { @@ -595,6 +619,7 @@ public class FrozenCacheService implements Releasable { public boolean tryEvict() { if (refCount() <= 1 && evicted.compareAndSet(false, true)) { logger.trace("evicted {} with channel offset {}", regionKey, physicalStartOffset()); + evictCount.increment(); decRef(); return true; } @@ -604,6 +629,7 @@ public class FrozenCacheService implements Releasable { public boolean forceEvict() { if (evicted.compareAndSet(false, true)) { logger.trace("force evicted {} with channel offset {}", regionKey, physicalStartOffset()); + evictCount.increment(); decRef(); return true; } @@ -614,10 +640,6 @@ public class FrozenCacheService implements Releasable { return evicted.get(); } - public boolean isReleased() { - return isEvicted() && refCount() == 0; - } - @Override protected void closeInternal() { // now actually free the region associated with this chunk @@ -676,6 +698,7 @@ public class FrozenCacheService implements Releasable { gap.end() - gap.start(), progress -> gap.onProgress(start + progress) ); + writeCount.increment(); } finally { decRef(); } @@ -717,6 +740,7 @@ public class FrozenCacheService implements Releasable { + '-' + rangeToRead.start() + ']'; + readCount.increment(); listener.onResponse(read); }, listener::onFailure); } @@ -827,4 +851,70 @@ public class FrozenCacheService implements Releasable { void fillCacheRange(SharedBytes.IO channel, long channelPos, long relativePos, long length, Consumer progressUpdater) throws IOException; } + + public static class Stats { + + public static final Stats EMPTY = new Stats(0, 0L, 0L, 0L, 0L, 0L, 0L, 0L); + + private final int numberOfRegions; + private final long size; + private final long regionSize; + private final long evictCount; + private final long writeCount; + private final long writeBytes; + private final long readCount; + private final long readBytes; + + private Stats( + int numberOfRegions, + long size, + long regionSize, + long evictCount, + long writeCount, + long writeBytes, + long readCount, + long readBytes + ) { + this.numberOfRegions = numberOfRegions; + this.size = size; + this.regionSize = regionSize; + this.evictCount = evictCount; + this.writeCount = writeCount; + this.writeBytes = writeBytes; + this.readCount = readCount; + this.readBytes = readBytes; + } + + public int getNumberOfRegions() { + return numberOfRegions; + } + + public long getSize() { + return size; + } + + public long getRegionSize() { + return regionSize; + } + + public long getEvictCount() { + return evictCount; + } + + public long getWriteCount() { + return writeCount; + } + + public long getWriteBytes() { + return writeBytes; + } + + public long getReadCount() { + return readCount; + } + + public long getReadBytes() { + return readBytes; + } + } } diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/shared/SharedBytes.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/shared/SharedBytes.java index 736be9861533..74bcc99766cf 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/shared/SharedBytes.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/shared/SharedBytes.java @@ -27,6 +27,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.util.Map; +import java.util.function.IntConsumer; public class SharedBytes extends AbstractRefCounted { @@ -47,10 +48,13 @@ public class SharedBytes extends AbstractRefCounted { // TODO: for systems like Windows without true p-write/read support we should split this up into multiple channels since positional // operations in #IO are not contention-free there (https://bugs.java.com/bugdatabase/view_bug.do?bug_id=6265734) private final FileChannel fileChannel; - private final Path path; - SharedBytes(int numRegions, long regionSize, NodeEnvironment environment) throws IOException { + private final IntConsumer writeBytes; + private final IntConsumer readBytes; + + SharedBytes(int numRegions, long regionSize, NodeEnvironment environment, IntConsumer writeBytes, IntConsumer readBytes) + throws IOException { super("shared-bytes"); this.numRegions = numRegions; this.regionSize = regionSize; @@ -90,6 +94,8 @@ public class SharedBytes extends AbstractRefCounted { } } this.path = cacheFile; + this.writeBytes = writeBytes; + this.readBytes = readBytes; } /** @@ -169,7 +175,9 @@ public class SharedBytes extends AbstractRefCounted { @SuppressForbidden(reason = "Use positional reads on purpose") public int read(ByteBuffer dst, long position) throws IOException { checkOffsets(position, dst.remaining()); - return fileChannel.read(dst, position); + final int bytesRead = fileChannel.read(dst, position); + readBytes.accept(bytesRead); + return bytesRead; } @SuppressForbidden(reason = "Use positional writes on purpose") @@ -178,7 +186,9 @@ public class SharedBytes extends AbstractRefCounted { assert position % PAGE_SIZE == 0; assert src.remaining() % PAGE_SIZE == 0; checkOffsets(position, src.remaining()); - return fileChannel.write(src, position); + final int bytesWritten = fileChannel.write(src, position); + writeBytes.accept(bytesWritten); + return bytesWritten; } private void checkOffsets(long position, long length) { diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/rest/RestSearchableSnapshotsNodeCachesStatsAction.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/rest/RestSearchableSnapshotsNodeCachesStatsAction.java new file mode 100644 index 000000000000..c7206c781715 --- /dev/null +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/rest/RestSearchableSnapshotsNodeCachesStatsAction.java @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.searchablesnapshots.rest; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.searchablesnapshots.action.cache.TransportSearchableSnapshotsNodeCachesStatsAction; + +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.GET; + +/** + * Node level stats for searchable snapshots caches. + */ +public class RestSearchableSnapshotsNodeCachesStatsAction extends BaseRestHandler { + + @Override + public List routes() { + return List.of( + new RestHandler.Route(GET, "/_searchable_snapshots/cache/stats"), + new RestHandler.Route(GET, "/_searchable_snapshots/{nodeId}/cache/stats") + ); + } + + @Override + public String getName() { + return "searchable_snapshots_cache_stats_action"; + } + + @Override + public BaseRestHandler.RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) { + final String[] nodesIds = Strings.splitStringByCommaToArray(request.param("nodeId")); + return channel -> client.execute( + TransportSearchableSnapshotsNodeCachesStatsAction.TYPE, + new TransportSearchableSnapshotsNodeCachesStatsAction.NodesRequest(nodesIds), + new RestToXContentListener<>(channel) + ); + } +} diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/FrozenIndexInput.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/FrozenIndexInput.java index aa01aef989a1..6a41c928fdfb 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/FrozenIndexInput.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/FrozenIndexInput.java @@ -159,7 +159,7 @@ public class FrozenIndexInput extends MetadataCachingIndexInput { final long streamStartPosition = rangeToWrite.start() + relativePos; try (InputStream input = openInputStreamFromBlobStore(streamStartPosition, len)) { - this.writeCacheFile(channel, input, channelPos, relativePos, len, progressUpdater, startTimeNanos); + writeCacheFile(channel, input, channelPos, relativePos, len, progressUpdater, startTimeNanos); } }, directory.cacheFetchAsyncExecutor() diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java index ceaddb16f053..b8b481ba6207 100644 --- a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java +++ b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java @@ -164,6 +164,7 @@ public class Constants { "cluster:admin/xpack/rollup/start", "cluster:admin/xpack/rollup/stop", "cluster:admin/xpack/searchable_snapshots/cache/clear", + "cluster:admin/xpack/searchable_snapshots/cache/stats", "cluster:admin/xpack/security/api_key/create", "cluster:admin/xpack/security/api_key/get", "cluster:admin/xpack/security/api_key/grant",