diff --git a/docs/changelog/126087.yaml b/docs/changelog/126087.yaml
new file mode 100644
index 000000000000..94d5c5e0c28b
--- /dev/null
+++ b/docs/changelog/126087.yaml
@@ -0,0 +1,5 @@
+pr: 126087
+summary: Upgrade to repository-gcs to use com.google.cloud:google-cloud-storage-bom:2.50.0
+area: Snapshot/Restore
+type: upgrade
+issues: []
diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml
index f6f986327c6e..9c39b0ec396f 100644
--- a/gradle/verification-metadata.xml
+++ b/gradle/verification-metadata.xml
@@ -499,14 +499,14 @@
-
-
-
+
+
+
-
-
-
+
+
+
@@ -514,6 +514,11 @@
+
+
+
+
+
@@ -524,11 +529,31 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -539,9 +564,9 @@
-
-
-
+
+
+
@@ -549,24 +574,44 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
-
+
+
+
@@ -589,6 +634,11 @@
+
+
+
+
+
@@ -649,6 +699,11 @@
+
+
+
+
+
@@ -684,6 +739,11 @@
+
+
+
+
+
@@ -699,11 +759,21 @@
+
+
+
+
+
+
+
+
+
+
@@ -714,6 +784,11 @@
+
+
+
+
+
@@ -724,6 +799,11 @@
+
+
+
+
+
@@ -1369,6 +1449,11 @@
+
+
+
+
+
@@ -1494,11 +1579,26 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -3412,6 +3512,11 @@
+
+
+
+
+
@@ -4627,9 +4732,9 @@
-
-
-
+
+
+
diff --git a/modules/repository-gcs/build.gradle b/modules/repository-gcs/build.gradle
index d23a0f4a7e44..6b3e3d451993 100644
--- a/modules/repository-gcs/build.gradle
+++ b/modules/repository-gcs/build.gradle
@@ -18,40 +18,44 @@ apply plugin: 'elasticsearch.internal-cluster-test'
esplugin {
description = 'The GCS repository plugin adds Google Cloud Storage support for repositories.'
- classname ='org.elasticsearch.repositories.gcs.GoogleCloudStoragePlugin'
+ classname = 'org.elasticsearch.repositories.gcs.GoogleCloudStoragePlugin'
}
dependencies {
- api 'com.google.cloud:google-cloud-storage:2.13.1'
- api 'com.google.cloud:google-cloud-core:2.8.28'
- api 'com.google.cloud:google-cloud-core-http:2.8.28'
- runtimeOnly 'com.google.guava:guava:32.0.1-jre'
- runtimeOnly 'com.google.guava:failureaccess:1.0.1'
- api "commons-logging:commons-logging:${versions.commonslogging}"
- api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
- api "commons-codec:commons-codec:${versions.commonscodec}"
- api 'com.google.api:api-common:2.3.1'
- api 'com.google.api:gax:2.20.1'
- api 'org.threeten:threetenbp:1.6.5'
- api "com.google.protobuf:protobuf-java-util:${versions.protobuf}"
- api "com.google.protobuf:protobuf-java:${versions.protobuf}"
- api 'com.google.code.gson:gson:2.10'
- api 'com.google.api.grpc:proto-google-common-protos:2.9.6'
- api 'com.google.api.grpc:proto-google-iam-v1:1.6.2'
- api 'com.google.auth:google-auth-library-credentials:1.11.0'
- api 'com.google.auth:google-auth-library-oauth2-http:1.11.0'
- api "com.google.oauth-client:google-oauth-client:${versions.google_oauth_client}"
- api 'com.google.api-client:google-api-client:2.1.1'
- api 'com.google.http-client:google-http-client:1.42.3'
- api 'com.google.http-client:google-http-client-gson:1.42.3'
- api 'com.google.http-client:google-http-client-appengine:1.42.3'
- api 'com.google.http-client:google-http-client-jackson2:1.42.3'
- api "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
- api 'com.google.api:gax-httpjson:0.105.1'
- api 'io.grpc:grpc-context:1.49.2'
- api 'io.opencensus:opencensus-api:0.31.1'
- api 'io.opencensus:opencensus-contrib-http-util:0.31.1'
- api 'com.google.apis:google-api-services-storage:v1-rev20220705-2.0.0'
+ // dependencies consistent with 'com.google.cloud:google-cloud-storage-bom:2.50.0'
+ implementation 'com.google.cloud:google-cloud-storage:2.50.0'
+ implementation 'com.google.cloud:google-cloud-core:2.53.1'
+ implementation 'com.google.cloud:google-cloud-core-http:2.53.1'
+ runtimeOnly 'com.google.guava:guava:33.4.0-jre'
+ runtimeOnly 'com.google.guava:failureaccess:1.0.2'
+ runtimeOnly "org.slf4j:slf4j-api:${versions.slf4j}" // 2.0.16 in bom
+ runtimeOnly "commons-codec:commons-codec:${versions.commonscodec}" // 1.18.0 in bom
+ implementation 'com.google.api:api-common:2.46.1'
+ implementation 'com.google.api:gax:2.63.1'
+ implementation 'org.threeten:threetenbp:1.7.0'
+ runtimeOnly "com.google.protobuf:protobuf-java-util:${versions.protobuf}" // 3.25.5 in bom
+ runtimeOnly "com.google.protobuf:protobuf-java:${versions.protobuf}"
+ runtimeOnly 'com.google.code.gson:gson:2.12.1'
+ runtimeOnly 'com.google.api.grpc:proto-google-common-protos:2.54.1'
+ runtimeOnly 'com.google.api.grpc:proto-google-iam-v1:1.49.1'
+ implementation 'com.google.auth:google-auth-library-credentials:1.33.1'
+ implementation 'com.google.auth:google-auth-library-oauth2-http:1.33.1'
+ runtimeOnly "com.google.oauth-client:google-oauth-client:${versions.google_oauth_client}" // 1.37.0 in bom
+ implementation 'com.google.api-client:google-api-client:2.7.2'
+ implementation 'com.google.http-client:google-http-client:1.46.3'
+ runtimeOnly 'com.google.http-client:google-http-client-gson:1.46.3'
+ runtimeOnly 'com.google.http-client:google-http-client-appengine:1.46.3'
+ runtimeOnly 'com.google.http-client:google-http-client-jackson2:1.46.3'
+ runtimeOnly "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" // 2.18.2 in bom
+ runtimeOnly 'com.google.api:gax-httpjson:2.63.1'
+ runtimeOnly 'io.opencensus:opencensus-api:0.31.1'
+ runtimeOnly 'io.opencensus:opencensus-contrib-http-util:0.31.1'
+ implementation 'com.google.apis:google-api-services-storage:v1-rev20250224-2.0.0'
+ implementation 'org.checkerframework:checker-qual:3.49.0'
+ runtimeOnly 'io.opentelemetry:opentelemetry-api:1.47.0'
+ runtimeOnly 'io.opentelemetry:opentelemetry-context:1.47.0'
+ runtimeOnly 'com.google.api.grpc:proto-google-cloud-storage-v2:2.50.0'
+ runtimeOnly 'io.grpc:grpc-api:1.70.0'
testImplementation "org.apache.httpcomponents:httpclient:${versions.httpclient}"
testImplementation "org.apache.httpcomponents:httpcore:${versions.httpcore}"
@@ -62,7 +66,7 @@ dependencies {
restResources {
restApi {
- include '_common', 'cluster', 'nodes', 'snapshot','indices', 'index', 'bulk', 'count'
+ include '_common', 'cluster', 'nodes', 'snapshot', 'indices', 'index', 'bulk', 'count'
}
}
@@ -123,11 +127,6 @@ tasks.named("thirdPartyAudit").configure {
'com.google.appengine.api.urlfetch.HTTPResponse',
'com.google.appengine.api.urlfetch.URLFetchService',
'com.google.appengine.api.urlfetch.URLFetchServiceFactory',
- // commons-logging optional dependencies
- 'org.apache.avalon.framework.logger.Logger',
- 'org.apache.log.Hierarchy',
- 'org.apache.log.Logger',
- 'javax.jms.Message',
// optional apache http client dependencies
'org.apache.http.ConnectionReuseStrategy',
@@ -171,19 +170,59 @@ tasks.named("thirdPartyAudit").configure {
'org.apache.http.protocol.HttpProcessor',
'org.apache.http.protocol.HttpRequestExecutor',
- // commons-logging provided dependencies
- 'javax.servlet.ServletContextEvent',
- 'javax.servlet.ServletContextListener'
+ // grpc/proto stuff
+ 'com.google.api.gax.grpc.GrpcCallContext',
+ 'com.google.api.gax.grpc.GrpcCallSettings',
+ 'com.google.api.gax.grpc.GrpcCallSettings$Builder',
+ 'com.google.api.gax.grpc.GrpcInterceptorProvider',
+ 'com.google.api.gax.grpc.GrpcStatusCode',
+ 'com.google.api.gax.grpc.GrpcStubCallableFactory',
+ 'com.google.api.gax.grpc.InstantiatingGrpcChannelProvider',
+ 'com.google.api.gax.grpc.InstantiatingGrpcChannelProvider$Builder',
+ 'com.google.cloud.grpc.GrpcTransportOptions',
+ 'com.google.cloud.grpc.GrpcTransportOptions$Builder',
+ 'com.google.cloud.opentelemetry.metric.GoogleCloudMetricExporter',
+ 'com.google.cloud.opentelemetry.metric.MetricConfiguration',
+ 'com.google.cloud.opentelemetry.metric.MetricConfiguration$Builder',
+ 'com.google.storage.v2.StorageClient',
+ 'com.google.storage.v2.StorageClient$ListBucketsPagedResponse',
+ 'com.google.storage.v2.StorageSettings',
+ 'com.google.storage.v2.StorageSettings$Builder',
+ 'com.google.storage.v2.stub.GrpcStorageStub',
+ 'com.google.storage.v2.stub.StorageStubSettings',
+ // opentelemetry implementation stuff
+ 'io.grpc.opentelemetry.GrpcOpenTelemetry',
+ 'io.grpc.opentelemetry.GrpcOpenTelemetry$Builder',
+ 'io.grpc.protobuf.ProtoUtils',
+ 'io.opentelemetry.contrib.gcp.resource.GCPResourceProvider',
+ 'io.opentelemetry.sdk.OpenTelemetrySdk',
+ 'io.opentelemetry.sdk.OpenTelemetrySdkBuilder',
+ 'io.opentelemetry.sdk.common.CompletableResultCode',
+ 'io.opentelemetry.sdk.common.export.MemoryMode',
+ 'io.opentelemetry.sdk.metrics.Aggregation',
+ 'io.opentelemetry.sdk.metrics.InstrumentSelector',
+ 'io.opentelemetry.sdk.metrics.InstrumentSelectorBuilder',
+ 'io.opentelemetry.sdk.metrics.InstrumentType',
+ 'io.opentelemetry.sdk.metrics.SdkMeterProvider',
+ 'io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder',
+ 'io.opentelemetry.sdk.metrics.View',
+ 'io.opentelemetry.sdk.metrics.ViewBuilder',
+ 'io.opentelemetry.sdk.metrics.data.AggregationTemporality',
+ 'io.opentelemetry.sdk.metrics.export.DefaultAggregationSelector',
+ 'io.opentelemetry.sdk.metrics.export.MetricExporter',
+ 'io.opentelemetry.sdk.metrics.export.PeriodicMetricReader',
+ 'io.opentelemetry.sdk.metrics.export.PeriodicMetricReaderBuilder',
+ 'io.opentelemetry.sdk.resources.Resource',
)
- if(buildParams.graalVmRuntime == false) {
+ if (buildParams.graalVmRuntime == false) {
ignoreMissingClasses(
- 'org.graalvm.nativeimage.hosted.Feature',
- 'org.graalvm.nativeimage.hosted.Feature$BeforeAnalysisAccess',
- 'org.graalvm.nativeimage.hosted.Feature$DuringAnalysisAccess',
- 'org.graalvm.nativeimage.hosted.Feature$FeatureAccess',
- 'org.graalvm.nativeimage.hosted.RuntimeReflection'
+ 'org.graalvm.nativeimage.hosted.Feature',
+ 'org.graalvm.nativeimage.hosted.Feature$BeforeAnalysisAccess',
+ 'org.graalvm.nativeimage.hosted.Feature$DuringAnalysisAccess',
+ 'org.graalvm.nativeimage.hosted.Feature$FeatureAccess',
+ 'org.graalvm.nativeimage.hosted.RuntimeReflection'
)
}
}
@@ -213,7 +252,7 @@ Map expansions = [
tasks.named("processYamlRestTestResources").configure {
inputs.properties(expansions)
- filter("tokens" : expansions, ReplaceTokens.class)
+ filter("tokens": expansions, ReplaceTokens.class)
}
tasks.named("internalClusterTest").configure {
diff --git a/modules/repository-gcs/licenses/checker-qual-LICENSE.txt b/modules/repository-gcs/licenses/checker-qual-LICENSE.txt
new file mode 100644
index 000000000000..9837c6b69fda
--- /dev/null
+++ b/modules/repository-gcs/licenses/checker-qual-LICENSE.txt
@@ -0,0 +1,22 @@
+Checker Framework qualifiers
+Copyright 2004-present by the Checker Framework developers
+
+MIT License:
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
diff --git a/modules/repository-gcs/licenses/grpc-context-NOTICE.txt b/modules/repository-gcs/licenses/checker-qual-NOTICE.txt
similarity index 100%
rename from modules/repository-gcs/licenses/grpc-context-NOTICE.txt
rename to modules/repository-gcs/licenses/checker-qual-NOTICE.txt
diff --git a/modules/repository-gcs/licenses/commons-logging-NOTICE.txt b/modules/repository-gcs/licenses/commons-logging-NOTICE.txt
deleted file mode 100644
index 72eb32a90245..000000000000
--- a/modules/repository-gcs/licenses/commons-logging-NOTICE.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-Apache Commons CLI
-Copyright 2001-2009 The Apache Software Foundation
-
-This product includes software developed by
-The Apache Software Foundation (http://www.apache.org/).
diff --git a/modules/repository-gcs/licenses/grpc-context-LICENSE.txt b/modules/repository-gcs/licenses/grpc-api-LICENSE.txt
similarity index 100%
rename from modules/repository-gcs/licenses/grpc-context-LICENSE.txt
rename to modules/repository-gcs/licenses/grpc-api-LICENSE.txt
diff --git a/modules/repository-gcs/licenses/grpc-api-NOTICE.txt b/modules/repository-gcs/licenses/grpc-api-NOTICE.txt
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/modules/repository-gcs/licenses/log4j-NOTICE.txt b/modules/repository-gcs/licenses/log4j-NOTICE.txt
deleted file mode 100644
index bbb5fb3f66e2..000000000000
--- a/modules/repository-gcs/licenses/log4j-NOTICE.txt
+++ /dev/null
@@ -1,20 +0,0 @@
-Apache Log4j
-Copyright 1999-2023 Apache Software Foundation
-
-This product includes software developed at
-The Apache Software Foundation (http://www.apache.org/).
-
-ResolverUtil.java
-Copyright 2005-2006 Tim Fennell
-
-Dumbster SMTP test server
-Copyright 2004 Jason Paul Kitchen
-
-TypeUtil.java
-Copyright 2002-2012 Ramnivas Laddad, Juergen Hoeller, Chris Beams
-
-picocli (http://picocli.info)
-Copyright 2017 Remko Popma
-
-TimeoutBlockingWaitStrategy.java and parts of Util.java
-Copyright 2011 LMAX Ltd.
diff --git a/modules/repository-gcs/licenses/commons-logging-LICENSE.txt b/modules/repository-gcs/licenses/opentelemetry-api-LICENSE.txt
similarity index 100%
rename from modules/repository-gcs/licenses/commons-logging-LICENSE.txt
rename to modules/repository-gcs/licenses/opentelemetry-api-LICENSE.txt
diff --git a/modules/repository-gcs/licenses/opentelemetry-api-NOTICE.txt b/modules/repository-gcs/licenses/opentelemetry-api-NOTICE.txt
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/modules/repository-gcs/licenses/log4j-LICENSE.txt b/modules/repository-gcs/licenses/opentelemetry-context-LICENSE.txt
similarity index 99%
rename from modules/repository-gcs/licenses/log4j-LICENSE.txt
rename to modules/repository-gcs/licenses/opentelemetry-context-LICENSE.txt
index 6279e5206de1..57bc88a15a0e 100644
--- a/modules/repository-gcs/licenses/log4j-LICENSE.txt
+++ b/modules/repository-gcs/licenses/opentelemetry-context-LICENSE.txt
@@ -1,4 +1,3 @@
-
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
@@ -187,7 +186,7 @@
same "printed page" as the copyright notice for easier
identification within third-party archives.
- Copyright 1999-2005 The Apache Software Foundation
+ Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@@ -200,3 +199,4 @@
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
+
diff --git a/modules/repository-gcs/licenses/opentelemetry-context-NOTICE.txt b/modules/repository-gcs/licenses/opentelemetry-context-NOTICE.txt
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/modules/repository-gcs/licenses/slf4j-api-LICENSE.txt b/modules/repository-gcs/licenses/slf4j-api-LICENSE.txt
new file mode 100644
index 000000000000..8fda22f4d72f
--- /dev/null
+++ b/modules/repository-gcs/licenses/slf4j-api-LICENSE.txt
@@ -0,0 +1,21 @@
+Copyright (c) 2004-2014 QOS.ch
+All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/modules/repository-gcs/licenses/slf4j-api-NOTICE.txt b/modules/repository-gcs/licenses/slf4j-api-NOTICE.txt
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStore.java b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStore.java
index 7c2b1d55ad73..a529d08ce47b 100644
--- a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStore.java
+++ b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStore.java
@@ -75,9 +75,9 @@ import static org.elasticsearch.core.Strings.format;
class GoogleCloudStorageBlobStore implements BlobStore {
/**
- * see com.google.cloud.BaseWriteChannel#DEFAULT_CHUNK_SIZE
+ * see {@link com.google.cloud.storage.BaseStorageWriteChannel#chunkSize}
*/
- static final int SDK_DEFAULT_CHUNK_SIZE = 60 * 256 * 1024;
+ static final int SDK_DEFAULT_CHUNK_SIZE = Math.toIntExact(ByteSizeValue.ofMb(16).getBytes());
private static final Logger logger = LogManager.getLogger(GoogleCloudStorageBlobStore.class);
@@ -652,7 +652,17 @@ class GoogleCloudStorageBlobStore implements BlobStore {
@SuppressForbidden(reason = "channel is based on a socket")
@Override
public int write(final ByteBuffer src) throws IOException {
- return SocketAccess.doPrivilegedIOException(() -> channel.write(src));
+ try {
+ return SocketAccess.doPrivilegedIOException(() -> channel.write(src));
+ } catch (IOException e) {
+ // BaseStorageWriteChannel#write wraps StorageException in an IOException, but BaseStorageWriteChannel#close
+ // does not, if we unwrap StorageExceptions here, it simplifies our retry-on-gone logic
+ final StorageException storageException = (StorageException) ExceptionsHelper.unwrap(e, StorageException.class);
+ if (storageException != null) {
+ throw storageException;
+ }
+ throw e;
+ }
}
@Override
diff --git a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java
index 626385377569..14c5be8c9ca9 100644
--- a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java
+++ b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java
@@ -41,6 +41,7 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.Proxy;
+import java.net.SocketException;
import java.net.URI;
import java.net.URL;
import java.net.UnknownHostException;
@@ -275,6 +276,10 @@ public class GoogleCloudStorageService {
if (ExceptionsHelper.unwrap(prevThrowable, UnknownHostException.class) != null) {
return true;
}
+ // Also retry on `SocketException`s
+ if (ExceptionsHelper.unwrap(prevThrowable, SocketException.class) != null) {
+ return true;
+ }
return delegate.shouldRetry(prevThrowable, prevResponse);
}
);
diff --git a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java
index 0d8933b44b7a..10a62842a766 100644
--- a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java
+++ b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java
@@ -21,6 +21,7 @@ import com.google.cloud.storage.StorageOptions;
import com.sun.net.httpserver.HttpHandler;
import org.apache.http.HttpStatus;
+import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.common.BackoffPolicy;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.UUIDs;
@@ -327,8 +328,7 @@ public class GoogleCloudStorageBlobContainerRetriesTests extends AbstractBlobCon
}
public void testWriteLargeBlob() throws IOException {
- // See {@link BaseWriteChannel#DEFAULT_CHUNK_SIZE}
- final int defaultChunkSize = 60 * 256 * 1024;
+ final int defaultChunkSize = GoogleCloudStorageBlobStore.SDK_DEFAULT_CHUNK_SIZE;
final int nbChunks = randomIntBetween(3, 5);
final int lastChunkSize = randomIntBetween(1, defaultChunkSize - 1);
final int totalChunks = nbChunks + 1;
@@ -412,13 +412,16 @@ public class GoogleCloudStorageBlobContainerRetriesTests extends AbstractBlobCon
}
}
- final String range = exchange.getRequestHeaders().getFirst("Content-Range");
- assertTrue(Strings.hasLength(range));
+ final String contentRangeHeaderValue = exchange.getRequestHeaders().getFirst("Content-Range");
+ final HttpHeaderParser.ContentRange contentRange = HttpHeaderParser.parseContentRangeHeader(contentRangeHeaderValue);
+ assertNotNull("Invalid content range header: " + contentRangeHeaderValue, contentRange);
- if (range.equals("bytes */*")) {
+ if (contentRange.hasRange() == false) {
+ // Content-Range: */... is a status check
+ // https://cloud.google.com/storage/docs/performing-resumable-uploads#status-check
final int receivedSoFar = bytesReceived.get();
if (receivedSoFar > 0) {
- exchange.getResponseHeaders().add("Range", Strings.format("bytes=0-%d", receivedSoFar));
+ exchange.getResponseHeaders().add("Range", Strings.format("bytes=0-%s", receivedSoFar));
}
exchange.getResponseHeaders().add("Content-Length", "0");
exchange.sendResponseHeaders(308 /* Resume Incomplete */, -1);
@@ -429,7 +432,6 @@ public class GoogleCloudStorageBlobContainerRetriesTests extends AbstractBlobCon
assertThat(Math.toIntExact(requestBody.length()), anyOf(equalTo(defaultChunkSize), equalTo(lastChunkSize)));
- final HttpHeaderParser.ContentRange contentRange = HttpHeaderParser.parseContentRangeHeader(range);
final int rangeStart = Math.toIntExact(contentRange.start());
final int rangeEnd = Math.toIntExact(contentRange.end());
assertThat(rangeEnd + 1 - rangeStart, equalTo(Math.toIntExact(requestBody.length())));
@@ -437,15 +439,20 @@ public class GoogleCloudStorageBlobContainerRetriesTests extends AbstractBlobCon
bytesReceived.updateAndGet(existing -> Math.max(existing, rangeEnd));
if (contentRange.size() != null) {
+ exchange.getResponseHeaders().add("x-goog-stored-content-length", String.valueOf(bytesReceived.get() + 1));
exchange.sendResponseHeaders(RestStatus.OK.getStatus(), -1);
return;
} else {
- exchange.getResponseHeaders().add("Range", Strings.format("bytes=%d/%d", rangeStart, rangeEnd));
+ exchange.getResponseHeaders().add("Range", Strings.format("bytes=%s-%s", rangeStart, rangeEnd));
exchange.getResponseHeaders().add("Content-Length", "0");
exchange.sendResponseHeaders(308 /* Resume Incomplete */, -1);
return;
}
}
+ } else {
+ ExceptionsHelper.maybeDieOnAnotherThread(
+ new AssertionError("Unexpected request" + exchange.getRequestMethod() + " " + exchange.getRequestURI())
+ );
}
if (randomBoolean()) {
diff --git a/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/GoogleCloudStorageHttpHandler.java b/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/GoogleCloudStorageHttpHandler.java
index 0adb4c600455..c9a52f3c2ffb 100644
--- a/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/GoogleCloudStorageHttpHandler.java
+++ b/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/GoogleCloudStorageHttpHandler.java
@@ -250,7 +250,7 @@ public class GoogleCloudStorageHttpHandler implements HttpHandler {
if (updateResponse.rangeHeader() != null) {
exchange.getResponseHeaders().add("Range", updateResponse.rangeHeader().headerString());
}
- exchange.getResponseHeaders().add("Content-Length", "0");
+ exchange.getResponseHeaders().add("x-goog-stored-content-length", String.valueOf(updateResponse.storedContentLength()));
exchange.sendResponseHeaders(updateResponse.statusCode(), -1);
} else {
exchange.sendResponseHeaders(RestStatus.NOT_FOUND.getStatus(), -1);
diff --git a/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/MockGcsBlobStore.java b/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/MockGcsBlobStore.java
index d0a33e3f4ac2..b6c4f4ec3d88 100644
--- a/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/MockGcsBlobStore.java
+++ b/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/MockGcsBlobStore.java
@@ -189,13 +189,13 @@ public class MockGcsBlobStore {
// Next we determine the response
if (valueToReturn.completed) {
- updateResponse.set(new UpdateResponse(RestStatus.OK.getStatus(), valueToReturn.getRange()));
+ updateResponse.set(new UpdateResponse(RestStatus.OK.getStatus(), valueToReturn.getRange(), valueToReturn.length()));
} else if (contentRange.hasSize() && contentRange.size() == valueToReturn.contents.length()) {
updateBlob(valueToReturn.path(), valueToReturn.ifGenerationMatch(), valueToReturn.contents);
valueToReturn = valueToReturn.complete();
- updateResponse.set(new UpdateResponse(RestStatus.OK.getStatus(), valueToReturn.getRange()));
+ updateResponse.set(new UpdateResponse(RestStatus.OK.getStatus(), valueToReturn.getRange(), valueToReturn.length()));
} else {
- updateResponse.set(new UpdateResponse(RESUME_INCOMPLETE, valueToReturn.getRange()));
+ updateResponse.set(new UpdateResponse(RESUME_INCOMPLETE, valueToReturn.getRange(), valueToReturn.length()));
}
return valueToReturn;
});
@@ -203,7 +203,7 @@ public class MockGcsBlobStore {
return updateResponse.get();
}
- record UpdateResponse(int statusCode, HttpHeaderParser.Range rangeHeader) {}
+ record UpdateResponse(int statusCode, HttpHeaderParser.Range rangeHeader, long storedContentLength) {}
void deleteBlob(String path) {
blobs.remove(path);