From fbbbdd7eec1e118cea7b2fc3e9ee51a87bbb3ef5 Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 7 Apr 2025 18:54:37 +0100 Subject: [PATCH] Allow overriding blob container path in tests (#126391) Some `AbstractBlobContainerRetriesTestCase#createBlobContainer` implementations choose a path for the container randomly, but we have a need for a test which re-creates the same container against a different `S3Service` and `BlobStore` and must therefore specify the same path each time. This commit exposes a parameter that lets callers specify a container path. --- ...CloudStorageBlobContainerRetriesTests.java | 22 ++++++----- .../s3/S3BlobContainerRetriesTests.java | 38 ++++++++++--------- .../url/URLBlobContainerRetriesTests.java | 6 ++- .../AbstractBlobContainerRetriesTestCase.java | 16 ++++---- 4 files changed, 47 insertions(+), 35 deletions(-) diff --git a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java index 1acde0be845d..2e801be13949 100644 --- a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java +++ b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java @@ -127,7 +127,8 @@ public class GoogleCloudStorageBlobContainerRetriesTests extends AbstractBlobCon final @Nullable TimeValue readTimeout, final @Nullable Boolean disableChunkedEncoding, final @Nullable ByteSizeValue bufferSize, - final @Nullable Integer maxBulkDeletes + final @Nullable Integer maxBulkDeletes, + final @Nullable BlobPath blobContainerPath ) { final Settings.Builder clientSettings = Settings.builder(); final String client = randomAlphaOfLength(5).toLowerCase(Locale.ROOT); @@ -207,7 +208,10 @@ public class GoogleCloudStorageBlobContainerRetriesTests extends AbstractBlobCon new GcsRepositoryStatsCollector() ); - return new GoogleCloudStorageBlobContainer(randomBoolean() ? BlobPath.EMPTY : BlobPath.EMPTY.add("foo"), blobStore); + return new GoogleCloudStorageBlobContainer( + Objects.requireNonNullElse(blobContainerPath, randomBoolean() ? BlobPath.EMPTY : BlobPath.EMPTY.add("foo")), + blobStore + ); } public void testShouldRetryOnConnectionRefused() { @@ -224,7 +228,7 @@ public class GoogleCloudStorageBlobContainerRetriesTests extends AbstractBlobCon private void executeListBlobsAndAssertRetries() { final int maxRetries = randomIntBetween(3, 5); - final BlobContainer blobContainer = createBlobContainer(maxRetries, null, null, null, null); + final BlobContainer blobContainer = createBlobContainer(maxRetries, null, null, null, null, null); expectThrows(StorageException.class, () -> blobContainer.listBlobs(randomPurpose())); assertEquals(maxRetries + 1, requestCounters.get("/storage/v1/b/bucket/o").get()); } @@ -233,7 +237,7 @@ public class GoogleCloudStorageBlobContainerRetriesTests extends AbstractBlobCon final int maxRetries = randomIntBetween(2, 10); final AtomicInteger countDown = new AtomicInteger(maxRetries); - final BlobContainer blobContainer = createBlobContainer(maxRetries, null, null, null, null); + final BlobContainer blobContainer = createBlobContainer(maxRetries, null, null, null, null, null); // SDK reads in 2 MB chunks so we use twice that to simulate 2 chunks final byte[] bytes = randomBytes(1 << 22); @@ -262,7 +266,7 @@ public class GoogleCloudStorageBlobContainerRetriesTests extends AbstractBlobCon final int maxRetries = randomIntBetween(2, 10); final CountDown countDown = new CountDown(maxRetries); - final BlobContainer blobContainer = createBlobContainer(maxRetries, null, null, null, null); + final BlobContainer blobContainer = createBlobContainer(maxRetries, null, null, null, null, null); final byte[] bytes = randomBlobContent(); httpServer.createContext("/upload/storage/v1/b/bucket/o", safeHandler(exchange -> { assertThat(exchange.getRequestURI().getQuery(), containsString("uploadType=multipart")); @@ -304,7 +308,7 @@ public class GoogleCloudStorageBlobContainerRetriesTests extends AbstractBlobCon public void testWriteBlobWithReadTimeouts() { final byte[] bytes = randomByteArrayOfLength(randomIntBetween(10, 128)); final TimeValue readTimeout = TimeValue.timeValueMillis(randomIntBetween(100, 500)); - final BlobContainer blobContainer = createBlobContainer(1, readTimeout, null, null, null); + final BlobContainer blobContainer = createBlobContainer(1, readTimeout, null, null, null, null); // HTTP server does not send a response httpServer.createContext("/upload/storage/v1/b/bucket/o", exchange -> { @@ -356,7 +360,7 @@ public class GoogleCloudStorageBlobContainerRetriesTests extends AbstractBlobCon logger.debug("starting with resumable upload id [{}]", sessionUploadId.get()); final TimeValue readTimeout = allowReadTimeout.get() ? TimeValue.timeValueSeconds(3) : null; - final BlobContainer blobContainer = createBlobContainer(nbErrors + 1, readTimeout, null, null, null); + final BlobContainer blobContainer = createBlobContainer(nbErrors + 1, readTimeout, null, null, null, null); httpServer.createContext("/upload/storage/v1/b/bucket/o", safeHandler(exchange -> { final BytesReference requestBody = Streams.readFully(exchange.getRequestBody()); @@ -503,7 +507,7 @@ public class GoogleCloudStorageBlobContainerRetriesTests extends AbstractBlobCon return Integer.toString(totalDeletesSent++); } }; - final BlobContainer blobContainer = createBlobContainer(1, null, null, null, null); + final BlobContainer blobContainer = createBlobContainer(1, null, null, null, null, null); httpServer.createContext("/batch/storage/v1", safeHandler(exchange -> { assert pendingDeletes.get() <= MAX_DELETES_PER_BATCH; @@ -539,7 +543,7 @@ public class GoogleCloudStorageBlobContainerRetriesTests extends AbstractBlobCon httpServer.createContext("/", new ResponseInjectingHttpHandler(requestHandlers, new GoogleCloudStorageHttpHandler("bucket"))); final int maxRetries = randomIntBetween(1, 3); - final BlobContainer container = createBlobContainer(maxRetries, null, null, null, null); + final BlobContainer container = createBlobContainer(maxRetries, null, null, null, null, null); final byte[] data = randomBytes(randomIntBetween(1, BlobContainerUtils.MAX_REGISTER_CONTENT_LENGTH)); final String key = randomIdentifier(); diff --git a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java index f1dc877f891d..7b3d96b878ac 100644 --- a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java +++ b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java @@ -164,7 +164,8 @@ public class S3BlobContainerRetriesTests extends AbstractBlobContainerRetriesTes final @Nullable TimeValue readTimeout, final @Nullable Boolean disableChunkedEncoding, final @Nullable ByteSizeValue bufferSize, - final @Nullable Integer maxBulkDeletes + final @Nullable Integer maxBulkDeletes, + final @Nullable BlobPath blobContainerPath ) { final Settings.Builder clientSettings = Settings.builder(); final String clientName = randomAlphaOfLength(5).toLowerCase(Locale.ROOT); @@ -216,7 +217,10 @@ public class S3BlobContainerRetriesTests extends AbstractBlobContainerRetriesTes new S3RepositoriesMetrics(new RepositoriesMetrics(recordingMeterRegistry)), BackoffPolicy.constantBackoff(TimeValue.timeValueMillis(1), MAX_NUMBER_SNAPSHOT_DELETE_RETRIES) ); - return new S3BlobContainer(randomBoolean() ? BlobPath.EMPTY : BlobPath.EMPTY.add("foo"), s3BlobStore) { + return new S3BlobContainer( + Objects.requireNonNullElse(blobContainerPath, randomBoolean() ? BlobPath.EMPTY : BlobPath.EMPTY.add("foo")), + s3BlobStore + ) { @Override public InputStream readBlob(OperationPurpose purpose, String blobName) throws IOException { return new AssertingInputStream(new S3RetryingInputStream(purpose, s3BlobStore, buildKey(blobName)) { @@ -261,7 +265,7 @@ public class S3BlobContainerRetriesTests extends AbstractBlobContainerRetriesTes final int maxRetries = randomInt(5); final CountDown countDown = new CountDown(maxRetries + 1); - final BlobContainer blobContainer = createBlobContainer(maxRetries, null, true, null, null); + final BlobContainer blobContainer = createBlobContainer(maxRetries, null, true, null, null, null); final byte[] bytes = randomBlobContent(); httpServer.createContext(downloadStorageEndpoint(blobContainer, "write_blob_max_retries"), exchange -> { @@ -309,7 +313,7 @@ public class S3BlobContainerRetriesTests extends AbstractBlobContainerRetriesTes public void testWriteBlobWithReadTimeouts() { final byte[] bytes = randomByteArrayOfLength(randomIntBetween(10, 128)); final TimeValue readTimeout = TimeValue.timeValueMillis(randomIntBetween(100, 500)); - final BlobContainer blobContainer = createBlobContainer(1, readTimeout, true, null, null); + final BlobContainer blobContainer = createBlobContainer(1, readTimeout, true, null, null, null); // HTTP server does not send a response httpServer.createContext(downloadStorageEndpoint(blobContainer, "write_blob_timeout"), exchange -> { @@ -343,7 +347,7 @@ public class S3BlobContainerRetriesTests extends AbstractBlobContainerRetriesTes final boolean useTimeout = rarely(); final TimeValue readTimeout = useTimeout ? TimeValue.timeValueMillis(randomIntBetween(100, 500)) : null; final ByteSizeValue bufferSize = ByteSizeValue.of(5, ByteSizeUnit.MB); - final BlobContainer blobContainer = createBlobContainer(null, readTimeout, true, bufferSize, null); + final BlobContainer blobContainer = createBlobContainer(null, readTimeout, true, bufferSize, null, null); final int parts = randomIntBetween(1, 5); final long lastPartSize = randomLongBetween(10, 512); @@ -439,7 +443,7 @@ public class S3BlobContainerRetriesTests extends AbstractBlobContainerRetriesTes final boolean useTimeout = rarely(); final TimeValue readTimeout = useTimeout ? TimeValue.timeValueMillis(randomIntBetween(100, 500)) : null; final ByteSizeValue bufferSize = ByteSizeValue.of(5, ByteSizeUnit.MB); - final BlobContainer blobContainer = createBlobContainer(null, readTimeout, true, bufferSize, null); + final BlobContainer blobContainer = createBlobContainer(null, readTimeout, true, bufferSize, null, null); final int parts = randomIntBetween(1, 5); final long lastPartSize = randomLongBetween(10, 512); @@ -548,7 +552,7 @@ public class S3BlobContainerRetriesTests extends AbstractBlobContainerRetriesTes 0, randomFrom(1000, Math.toIntExact(S3Repository.BUFFER_SIZE_SETTING.get(Settings.EMPTY).getBytes())) ); - final BlobContainer blobContainer = createBlobContainer(maxRetries, null, true, ByteSizeValue.ofBytes(bufferSizeBytes), null); + final BlobContainer blobContainer = createBlobContainer(maxRetries, null, true, ByteSizeValue.ofBytes(bufferSizeBytes), null, null); final int meaningfulProgressBytes = Math.max(1, bufferSizeBytes / 100); final byte[] bytes = randomBlobContent(); @@ -621,7 +625,7 @@ public class S3BlobContainerRetriesTests extends AbstractBlobContainerRetriesTes 0, randomFrom(1000, Math.toIntExact(S3Repository.BUFFER_SIZE_SETTING.get(Settings.EMPTY).getBytes())) ); - final BlobContainer blobContainer = createBlobContainer(maxRetries, null, true, ByteSizeValue.ofBytes(bufferSizeBytes), null); + final BlobContainer blobContainer = createBlobContainer(maxRetries, null, true, ByteSizeValue.ofBytes(bufferSizeBytes), null, null); final byte[] bytes = randomBlobContent(); @@ -659,7 +663,7 @@ public class S3BlobContainerRetriesTests extends AbstractBlobContainerRetriesTes 0, randomFrom(1000, Math.toIntExact(S3Repository.BUFFER_SIZE_SETTING.get(Settings.EMPTY).getBytes())) ); - final BlobContainer blobContainer = createBlobContainer(maxRetries, null, true, ByteSizeValue.ofBytes(bufferSizeBytes), null); + final BlobContainer blobContainer = createBlobContainer(maxRetries, null, true, ByteSizeValue.ofBytes(bufferSizeBytes), null, null); final int meaningfulProgressBytes = Math.max(1, bufferSizeBytes / 100); final byte[] bytes = randomBlobContent(512); @@ -752,7 +756,7 @@ public class S3BlobContainerRetriesTests extends AbstractBlobContainerRetriesTes public void testDoesNotRetryOnNotFound() { final int maxRetries = between(3, 5); - final BlobContainer blobContainer = createBlobContainer(maxRetries, null, true, null, null); + final BlobContainer blobContainer = createBlobContainer(maxRetries, null, true, null, null, null); final AtomicInteger numberOfReads = new AtomicInteger(0); @SuppressForbidden(reason = "use a http server") @@ -784,7 +788,7 @@ public class S3BlobContainerRetriesTests extends AbstractBlobContainerRetriesTes public void testSnapshotDeletesRetryOnThrottlingError() throws IOException { // disable AWS-client retries - final BlobContainer blobContainer = createBlobContainer(0, null, true, null, null); + final BlobContainer blobContainer = createBlobContainer(0, null, true, null, null, null); int numBlobsToDelete = randomIntBetween(500, 3000); List blobsToDelete = new ArrayList<>(); @@ -804,7 +808,7 @@ public class S3BlobContainerRetriesTests extends AbstractBlobContainerRetriesTes public void testSnapshotDeletesAbortRetriesWhenThreadIsInterrupted() { // disable AWS-client retries - final BlobContainer blobContainer = createBlobContainer(0, null, true, null, null); + final BlobContainer blobContainer = createBlobContainer(0, null, true, null, null, null); int numBlobsToDelete = randomIntBetween(500, 3000); List blobsToDelete = new ArrayList<>(); @@ -841,7 +845,7 @@ public class S3BlobContainerRetriesTests extends AbstractBlobContainerRetriesTes public void testNonSnapshotDeletesAreNotRetried() { // disable AWS-client retries - final BlobContainer blobContainer = createBlobContainer(0, null, true, null, null); + final BlobContainer blobContainer = createBlobContainer(0, null, true, null, null, null); int numBlobsToDelete = randomIntBetween(500, 3000); List blobsToDelete = new ArrayList<>(); @@ -870,7 +874,7 @@ public class S3BlobContainerRetriesTests extends AbstractBlobContainerRetriesTes public void testNonThrottlingErrorsAreNotRetried() { // disable AWS-client retries - final BlobContainer blobContainer = createBlobContainer(0, null, true, null, null); + final BlobContainer blobContainer = createBlobContainer(0, null, true, null, null, null); int numBlobsToDelete = randomIntBetween(500, 3000); List blobsToDelete = new ArrayList<>(); @@ -949,7 +953,7 @@ public class S3BlobContainerRetriesTests extends AbstractBlobContainerRetriesTes public void testGetRegisterRetries() { final var maxRetries = between(0, 3); - final BlobContainer blobContainer = createBlobContainer(maxRetries, null, null, null, null); + final BlobContainer blobContainer = createBlobContainer(maxRetries, null, null, null, null, null); interface FailingHandlerFactory { void addHandler(String blobName, Integer... responseCodes); @@ -1019,7 +1023,7 @@ public class S3BlobContainerRetriesTests extends AbstractBlobContainerRetriesTes public void testSuppressedDeletionErrorsAreCapped() { final TimeValue readTimeout = TimeValue.timeValueMillis(randomIntBetween(100, 500)); int maxBulkDeleteSize = randomIntBetween(1, 10); - final BlobContainer blobContainer = createBlobContainer(1, readTimeout, true, null, maxBulkDeleteSize); + final BlobContainer blobContainer = createBlobContainer(1, readTimeout, true, null, maxBulkDeleteSize, null); httpServer.createContext("/", exchange -> { if (isMultiDeleteRequest(exchange)) { exchange.sendResponseHeaders( @@ -1051,7 +1055,7 @@ public class S3BlobContainerRetriesTests extends AbstractBlobContainerRetriesTes public void testTrimmedLogAndCappedSuppressedErrorOnMultiObjectDeletionException() { final TimeValue readTimeout = TimeValue.timeValueMillis(randomIntBetween(100, 500)); int maxBulkDeleteSize = randomIntBetween(10, 30); - final BlobContainer blobContainer = createBlobContainer(1, readTimeout, true, null, maxBulkDeleteSize); + final BlobContainer blobContainer = createBlobContainer(1, readTimeout, true, null, maxBulkDeleteSize, null); final Pattern pattern = Pattern.compile("(.+?)"); httpServer.createContext("/", exchange -> { diff --git a/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/URLBlobContainerRetriesTests.java b/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/URLBlobContainerRetriesTests.java index a1996664be51..4b0125d35194 100644 --- a/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/URLBlobContainerRetriesTests.java +++ b/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/URLBlobContainerRetriesTests.java @@ -29,6 +29,7 @@ import java.net.InetSocketAddress; import java.net.MalformedURLException; import java.net.SocketTimeoutException; import java.net.URL; +import java.util.Objects; import static org.hamcrest.Matchers.either; import static org.hamcrest.Matchers.instanceOf; @@ -77,7 +78,8 @@ public class URLBlobContainerRetriesTests extends AbstractBlobContainerRetriesTe TimeValue readTimeout, Boolean disableChunkedEncoding, ByteSizeValue bufferSize, - Integer maxBulkDeletes + Integer maxBulkDeletes, + BlobPath blobContainerPath ) { Settings.Builder settingsBuilder = Settings.builder(); @@ -98,7 +100,7 @@ public class URLBlobContainerRetriesTests extends AbstractBlobContainerRetriesTe factory.create(httpClientSettings), httpClientSettings ); - return urlBlobStore.blobContainer(BlobPath.EMPTY); + return urlBlobStore.blobContainer(Objects.requireNonNullElse(blobContainerPath, BlobPath.EMPTY)); } catch (MalformedURLException e) { throw new RuntimeException("Unable to create URLBlobStore", e); } diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/AbstractBlobContainerRetriesTestCase.java b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/AbstractBlobContainerRetriesTestCase.java index 16a152b8539c..6d9c41362e2e 100644 --- a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/AbstractBlobContainerRetriesTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/AbstractBlobContainerRetriesTestCase.java @@ -15,6 +15,7 @@ import com.sun.net.httpserver.HttpServer; import org.apache.http.ConnectionClosedException; import org.apache.http.HttpStatus; import org.elasticsearch.common.blobstore.BlobContainer; +import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.blobstore.OperationPurpose; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.Streams; @@ -83,7 +84,8 @@ public abstract class AbstractBlobContainerRetriesTestCase extends ESTestCase { @Nullable TimeValue readTimeout, @Nullable Boolean disableChunkedEncoding, @Nullable ByteSizeValue bufferSize, - @Nullable Integer maxBulkDeletes + @Nullable Integer maxBulkDeletes, + @Nullable BlobPath blobContainerPath ); protected org.hamcrest.Matcher readTimeoutExceptionMatcher() { @@ -92,7 +94,7 @@ public abstract class AbstractBlobContainerRetriesTestCase extends ESTestCase { } public void testReadNonexistentBlobThrowsNoSuchFileException() { - final BlobContainer blobContainer = createBlobContainer(between(1, 5), null, null, null, null); + final BlobContainer blobContainer = createBlobContainer(between(1, 5), null, null, null, null, null); final long position = randomLongBetween(0, MAX_RANGE_VAL); final int length = randomIntBetween(1, Math.toIntExact(Math.min(Integer.MAX_VALUE, MAX_RANGE_VAL - position))); final Exception exception = expectThrows(NoSuchFileException.class, () -> { @@ -119,7 +121,7 @@ public abstract class AbstractBlobContainerRetriesTestCase extends ESTestCase { final byte[] bytes = randomBlobContent(); final TimeValue readTimeout = TimeValue.timeValueSeconds(between(1, 3)); - final BlobContainer blobContainer = createBlobContainer(maxRetries, readTimeout, null, null, null); + final BlobContainer blobContainer = createBlobContainer(maxRetries, readTimeout, null, null, null, null); httpServer.createContext(downloadStorageEndpoint(blobContainer, "read_blob_max_retries"), exchange -> { Streams.readFully(exchange.getRequestBody()); if (countDown.countDown()) { @@ -176,7 +178,7 @@ public abstract class AbstractBlobContainerRetriesTestCase extends ESTestCase { final CountDown countDown = new CountDown(maxRetries + 1); final TimeValue readTimeout = TimeValue.timeValueSeconds(between(5, 10)); - final BlobContainer blobContainer = createBlobContainer(maxRetries, readTimeout, null, null, null); + final BlobContainer blobContainer = createBlobContainer(maxRetries, readTimeout, null, null, null, null); final byte[] bytes = randomBlobContent(); httpServer.createContext(downloadStorageEndpoint(blobContainer, "read_range_blob_max_retries"), exchange -> { Streams.readFully(exchange.getRequestBody()); @@ -248,7 +250,7 @@ public abstract class AbstractBlobContainerRetriesTestCase extends ESTestCase { public void testReadBlobWithReadTimeouts() { final int maxRetries = randomInt(5); final TimeValue readTimeout = TimeValue.timeValueMillis(between(100, 200)); - final BlobContainer blobContainer = createBlobContainer(maxRetries, readTimeout, null, null, null); + final BlobContainer blobContainer = createBlobContainer(maxRetries, readTimeout, null, null, null, null); // HTTP server does not send a response httpServer.createContext(downloadStorageEndpoint(blobContainer, "read_blob_unresponsive"), exchange -> {}); @@ -305,7 +307,7 @@ public abstract class AbstractBlobContainerRetriesTestCase extends ESTestCase { public void testReadBlobWithNoHttpResponse() { final TimeValue readTimeout = TimeValue.timeValueMillis(between(100, 200)); - final BlobContainer blobContainer = createBlobContainer(randomInt(5), readTimeout, null, null, null); + final BlobContainer blobContainer = createBlobContainer(randomInt(5), readTimeout, null, null, null, null); // HTTP server closes connection immediately httpServer.createContext(downloadStorageEndpoint(blobContainer, "read_blob_no_response"), HttpExchange::close); @@ -325,7 +327,7 @@ public abstract class AbstractBlobContainerRetriesTestCase extends ESTestCase { public void testReadBlobWithPrematureConnectionClose() { final int maxRetries = randomInt(20); - final BlobContainer blobContainer = createBlobContainer(maxRetries, null, null, null, null); + final BlobContainer blobContainer = createBlobContainer(maxRetries, null, null, null, null, null); final boolean alwaysFlushBody = randomBoolean();