diff --git a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java
index 9f990fbd97cd..d64eb663b409 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java
@@ -36,7 +36,6 @@ import org.apache.lucene.util.BitSet;
 import org.apache.lucene.util.BitSetIterator;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.SparseFixedBitSet;
-import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.lucene.util.CombinedBitSet;
 import org.elasticsearch.search.dfs.AggregatedDfs;
@@ -53,7 +52,6 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 import java.util.PriorityQueue;
-import java.util.Set;
 import java.util.concurrent.Callable;
 import java.util.concurrent.Executor;
 import java.util.stream.Collectors;
@@ -82,7 +80,6 @@ public class ContextIndexSearcher extends IndexSearcher implements Releasable {
     // don't create slices with less than this number of docs
     private final int minimumDocsPerSlice;
 
-    private final Set<Thread> timeoutOverwrites = ConcurrentCollections.newConcurrentSet();
     private volatile boolean timeExceeded = false;
 
     /** constructor for non-concurrent search */
@@ -374,6 +371,8 @@ public class ContextIndexSearcher extends IndexSearcher implements Releasable {
         }
     }
 
+    private static final ThreadLocal<Boolean> timeoutOverwrites = ThreadLocal.withInitial(() -> false);
+
     /**
      * Similar to the lucene implementation, with the following changes made:
      * 1) postCollection is performed after each segment is collected. This is needed for aggregations, performed by search threads
@@ -397,12 +396,12 @@ public class ContextIndexSearcher extends IndexSearcher implements Releasable {
                 try {
                     // Search phase has finished, no longer need to check for timeout
                     // otherwise the aggregation post-collection phase might get cancelled.
-                    boolean added = timeoutOverwrites.add(Thread.currentThread());
-                    assert added;
+                    assert timeoutOverwrites.get() == false;
+                    timeoutOverwrites.set(true);
                     doAggregationPostCollection(collector);
                 } finally {
-                    boolean removed = timeoutOverwrites.remove(Thread.currentThread());
-                    assert removed;
+                    assert timeoutOverwrites.get();
+                    timeoutOverwrites.set(false);
                 }
             }
         }
@@ -420,7 +419,7 @@ public class ContextIndexSearcher extends IndexSearcher implements Releasable {
     }
 
     public void throwTimeExceededException() {
-        if (timeoutOverwrites.contains(Thread.currentThread()) == false) {
+        if (timeoutOverwrites.get() == false) {
             throw new TimeExceededException();
         }
     }