From 6bc1452b43f0646abf50e503eaea04e7581be233 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Wed, 28 May 2025 17:29:39 +1000 Subject: [PATCH] Make repositories project aware (#128285) Pass project-id explicitly to repository factory and make it part of the repository interface. Relates: ES-11839 --- .../repositories/azure/AzureRepository.java | 3 + .../azure/AzureRepositoryPlugin.java | 3 +- .../azure/AzureRepositorySettingsTests.java | 5 ++ ...eCloudStorageBlobStoreRepositoryTests.java | 3 +- .../gcs/GoogleCloudStoragePlugin.java | 3 +- .../gcs/GoogleCloudStorageRepository.java | 3 + .../gcs/GoogleCloudStoragePluginTests.java | 31 ++++++++ .../s3/S3RepositoryThirdPartyTests.java | 1 + .../s3/S3BlobStoreRepositoryTests.java | 13 ++- .../repositories/s3/S3BlobStore.java | 7 ++ .../repositories/s3/S3Repository.java | 4 + .../repositories/s3/S3RepositoryPlugin.java | 23 +++++- .../s3/S3BlobContainerRetriesTests.java | 1 + .../repositories/s3/S3RepositoryTests.java | 8 +- .../repository/url/URLRepositoryPlugin.java | 3 +- .../repositories/url/URLRepository.java | 4 +- .../repositories/url/URLRepositoryTests.java | 8 +- .../repositories/hdfs/HdfsPlugin.java | 10 ++- .../repositories/hdfs/HdfsRepository.java | 4 +- .../hdfs/HdfsBlobStoreContainerTests.java | 24 ++++++ .../plan/ShardSnapshotsServiceIT.java | 14 +++- .../repositories/InvalidRepositoryIT.java | 14 +++- ...BlobStoreRepositoryOperationPurposeIT.java | 14 +++- ...etadataLoadingDuringSnapshotRestoreIT.java | 14 +++- .../SnapshotsServiceDoubleFinalizationIT.java | 14 +++- .../repositories/FilterRepository.java | 6 ++ .../repositories/InvalidRepository.java | 10 ++- .../repositories/RepositoriesModule.java | 10 ++- .../repositories/RepositoriesService.java | 79 ++++++++++++++++--- .../repositories/Repository.java | 29 ++++++- .../repositories/UnknownTypeRepository.java | 10 ++- .../blobstore/BlobStoreRepository.java | 9 +++ .../blobstore/MeteredBlobStoreRepository.java | 4 +- .../repositories/fs/FsRepository.java | 4 +- .../ReservedRepositoryActionTests.java | 3 +- .../RepositoriesHealthTrackerTests.java | 6 +- .../index/shard/IndexShardTests.java | 2 +- .../repositories/InvalidRepositoryTests.java | 5 ++ .../RepositoriesServiceTests.java | 36 ++++++--- .../UnknownTypeRepositoryTests.java | 7 +- ...bStoreRepositoryDeleteThrottlingTests.java | 10 ++- .../BlobStoreRepositoryRestoreTests.java | 1 + .../blobstore/BlobStoreRepositoryTests.java | 1 + .../repositories/fs/FsRepositoryTests.java | 7 ++ .../snapshots/SnapshotResiliencyTests.java | 10 ++- ...ncySimulatingBlobStoreRepositoryTests.java | 3 +- .../LatencySimulatingBlobStoreRepository.java | 4 +- .../LatencySimulatingRepositoryPlugin.java | 3 +- .../index/shard/RestoreOnlyRepository.java | 10 ++- .../snapshots/mockstore/MockRepository.java | 22 +++++- .../java/org/elasticsearch/xpack/ccr/Ccr.java | 3 +- .../xpack/ccr/repository/CcrRepository.java | 17 +++- .../ShardFollowTaskReplicationTests.java | 2 +- .../engine/FollowEngineIndexShardTests.java | 2 +- .../CcrRepositoryRetentionLeaseTests.java | 6 +- .../SourceOnlySnapshotRepository.java | 8 +- .../SourceOnlySnapshotShardTests.java | 21 +++-- .../lucene/bwc/AbstractArchiveTestCase.java | 14 +++- ...chableSnapshotDiskThresholdIntegTests.java | 14 +++- ...archableSnapshotsPrewarmingIntegTests.java | 10 ++- ...SnapshotRecoveryStateIntegrationTests.java | 10 ++- .../SearchableSnapshotDirectoryTests.java | 1 + .../xpack/slm/SLMHealthBlockedSnapshotIT.java | 27 +++++-- .../xpack/slm/SLMStatDisruptionIT.java | 35 +++++--- .../SnapshotBasedIndexRecoveryIT.java | 40 ++++++++-- .../analyze/RepositoryAnalysisFailureIT.java | 7 +- .../analyze/RepositoryAnalysisSuccessIT.java | 7 +- .../votingonly/VotingOnlyNodePluginTests.java | 14 +++- 68 files changed, 642 insertions(+), 118 deletions(-) diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java index ff6bdfaa0072..dceafe753e60 100644 --- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java +++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java @@ -11,6 +11,7 @@ package org.elasticsearch.repositories.azure; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; @@ -110,6 +111,7 @@ public class AzureRepository extends MeteredBlobStoreRepository { private final RepositoriesMetrics repositoriesMetrics; public AzureRepository( + final ProjectId projectId, final RepositoryMetadata metadata, final NamedXContentRegistry namedXContentRegistry, final AzureStorageService storageService, @@ -119,6 +121,7 @@ public class AzureRepository extends MeteredBlobStoreRepository { final RepositoriesMetrics repositoriesMetrics ) { super( + projectId, metadata, namedXContentRegistry, clusterService, diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java index 058027bda1ec..62d45cb399bf 100644 --- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java +++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java @@ -62,10 +62,11 @@ public class AzureRepositoryPlugin extends Plugin implements RepositoryPlugin, R RecoverySettings recoverySettings, RepositoriesMetrics repositoriesMetrics ) { - return Collections.singletonMap(AzureRepository.TYPE, metadata -> { + return Collections.singletonMap(AzureRepository.TYPE, (projectId, metadata) -> { AzureStorageService storageService = azureStoreService.get(); assert storageService != null; return new AzureRepository( + projectId, metadata, namedXContentRegistry, storageService, diff --git a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java index b6b6c96f7aff..5f56e321d8f5 100644 --- a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java +++ b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java @@ -9,6 +9,7 @@ package org.elasticsearch.repositories.azure; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; @@ -23,6 +24,7 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.NamedXContentRegistry; import static org.elasticsearch.repositories.blobstore.BlobStoreRepository.READONLY_SETTING_KEY; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; import static org.mockito.Mockito.mock; @@ -35,7 +37,9 @@ public class AzureRepositorySettingsTests extends ESTestCase { .putList(Environment.PATH_DATA_SETTING.getKey(), tmpPaths()) .put(settings) .build(); + final ProjectId projectId = randomProjectIdOrDefault(); final AzureRepository azureRepository = new AzureRepository( + projectId, new RepositoryMetadata("foo", "azure", internalSettings), NamedXContentRegistry.EMPTY, mock(AzureStorageService.class), @@ -44,6 +48,7 @@ public class AzureRepositorySettingsTests extends ESTestCase { new RecoverySettings(settings, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)), RepositoriesMetrics.NOOP ); + assertThat(azureRepository.getProjectId(), equalTo(projectId)); assertThat(azureRepository.getBlobStore(), is(nullValue())); return azureRepository; } diff --git a/modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java b/modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java index 41f41981c09c..0ee1f8073f6c 100644 --- a/modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java +++ b/modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java @@ -276,7 +276,8 @@ public class GoogleCloudStorageBlobStoreRepositoryTests extends ESMockAPIBasedRe ) { return Collections.singletonMap( GoogleCloudStorageRepository.TYPE, - metadata -> new GoogleCloudStorageRepository( + (projectId, metadata) -> new GoogleCloudStorageRepository( + projectId, metadata, registry, this.storageService, diff --git a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStoragePlugin.java b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStoragePlugin.java index 3f7d3ae4825f..97b781255bbb 100644 --- a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStoragePlugin.java +++ b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStoragePlugin.java @@ -57,7 +57,8 @@ public class GoogleCloudStoragePlugin extends Plugin implements RepositoryPlugin ) { return Collections.singletonMap( GoogleCloudStorageRepository.TYPE, - metadata -> new GoogleCloudStorageRepository( + (projectId, metadata) -> new GoogleCloudStorageRepository( + projectId, metadata, namedXContentRegistry, this.storageService, diff --git a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java index 8eff8be0762e..91973186bbe3 100644 --- a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java +++ b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java @@ -11,6 +11,7 @@ package org.elasticsearch.repositories.gcs; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.BackoffPolicy; @@ -88,6 +89,7 @@ class GoogleCloudStorageRepository extends MeteredBlobStoreRepository { private final GcsRepositoryStatsCollector statsCollector; GoogleCloudStorageRepository( + final ProjectId projectId, final RepositoryMetadata metadata, final NamedXContentRegistry namedXContentRegistry, final GoogleCloudStorageService storageService, @@ -97,6 +99,7 @@ class GoogleCloudStorageRepository extends MeteredBlobStoreRepository { final GcsRepositoryStatsCollector statsCollector ) { super( + projectId, metadata, namedXContentRegistry, clusterService, diff --git a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStoragePluginTests.java b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStoragePluginTests.java index bd34018dd782..96750968d898 100644 --- a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStoragePluginTests.java +++ b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStoragePluginTests.java @@ -9,13 +9,22 @@ package org.elasticsearch.repositories.gcs; +import org.elasticsearch.cluster.metadata.RepositoryMetadata; +import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.MockBigArrays; +import org.elasticsearch.indices.recovery.RecoverySettings; +import org.elasticsearch.repositories.blobstore.BlobStoreTestUtil; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.NamedXContentRegistry; import org.junit.Assert; import java.util.List; +import static org.hamcrest.Matchers.equalTo; +import static org.mockito.Mockito.mock; + public class GoogleCloudStoragePluginTests extends ESTestCase { public void testExposedSettings() { @@ -37,4 +46,26 @@ public class GoogleCloudStoragePluginTests extends ESTestCase { settings.stream().map(Setting::getKey).toList() ); } + + public void testRepositoryProjectId() { + final var projectId = randomProjectIdOrDefault(); + final var repository = new GoogleCloudStorageRepository( + projectId, + new RepositoryMetadata( + randomIdentifier(), + GoogleCloudStorageRepository.TYPE, + Settings.builder() + .put(GoogleCloudStorageRepository.BUCKET.getKey(), randomIdentifier()) + .put(GoogleCloudStorageRepository.BASE_PATH.getKey(), randomIdentifier()) + .build() + ), + NamedXContentRegistry.EMPTY, + mock(GoogleCloudStorageService.class), + BlobStoreTestUtil.mockClusterService(), + MockBigArrays.NON_RECYCLING_INSTANCE, + new RecoverySettings(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)), + mock(GcsRepositoryStatsCollector.class) + ); + assertThat(repository.getProjectId(), equalTo(projectId)); + } } diff --git a/modules/repository-s3/qa/third-party/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java b/modules/repository-s3/qa/third-party/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java index 2963f0dbd857..cb7af68cd4b9 100644 --- a/modules/repository-s3/qa/third-party/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java +++ b/modules/repository-s3/qa/third-party/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java @@ -147,6 +147,7 @@ public class S3RepositoryThirdPartyTests extends AbstractThirdPartyRepositoryTes // construct our own repo instance so we can inject a threadpool that allows to control the passage of time try ( var repository = new S3Repository( + randomProjectIdOrDefault(), node().injector().getInstance(RepositoriesService.class).repository(TEST_REPO_NAME).getMetadata(), xContentRegistry(), node().injector().getInstance(PluginsService.class).filterPlugins(S3RepositoryPlugin.class).findFirst().get().getService(), diff --git a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java index e792cb784bfb..09b8f8179d3d 100644 --- a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java +++ b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java @@ -22,6 +22,7 @@ import org.apache.logging.log4j.Level; import org.apache.logging.log4j.core.LogEvent; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.action.support.broadcast.BroadcastResponse; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; @@ -598,6 +599,7 @@ public class S3BlobStoreRepositoryTests extends ESMockAPIBasedRepositoryIntegTes @Override protected S3Repository createRepository( + ProjectId projectId, RepositoryMetadata metadata, NamedXContentRegistry registry, ClusterService clusterService, @@ -605,7 +607,16 @@ public class S3BlobStoreRepositoryTests extends ESMockAPIBasedRepositoryIntegTes RecoverySettings recoverySettings, S3RepositoriesMetrics s3RepositoriesMetrics ) { - return new S3Repository(metadata, registry, getService(), clusterService, bigArrays, recoverySettings, s3RepositoriesMetrics) { + return new S3Repository( + projectId, + metadata, + registry, + getService(), + clusterService, + bigArrays, + recoverySettings, + s3RepositoriesMetrics + ) { @Override public BlobStore blobStore() { diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java index 17646591f623..be6ae9d84605 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java @@ -27,6 +27,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.common.BackoffPolicy; import org.elasticsearch.common.Strings; @@ -38,6 +39,7 @@ import org.elasticsearch.common.blobstore.BlobStoreException; import org.elasticsearch.common.blobstore.OperationPurpose; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.repositories.RepositoriesMetrics; import org.elasticsearch.rest.RestStatus; @@ -74,6 +76,8 @@ class S3BlobStore implements BlobStore { private static final Logger logger = LogManager.getLogger(S3BlobStore.class); + @Nullable // if the blobstore is at the cluster level + private final ProjectId projectId; private final S3Service service; private final BigArrays bigArrays; @@ -106,6 +110,7 @@ class S3BlobStore implements BlobStore { private final boolean addPurposeCustomQueryParameter; S3BlobStore( + @Nullable ProjectId projectId, S3Service service, String bucket, boolean serverSideEncryption, @@ -119,6 +124,7 @@ class S3BlobStore implements BlobStore { S3RepositoriesMetrics s3RepositoriesMetrics, BackoffPolicy retryThrottledDeleteBackoffPolicy ) { + this.projectId = projectId; this.service = service; this.bigArrays = bigArrays; this.bucket = bucket; @@ -257,6 +263,7 @@ class S3BlobStore implements BlobStore { } public AmazonS3Reference clientReference() { + // TODO: use service.client(ProjectId, RepositoryMetadata), see https://github.com/elastic/elasticsearch/pull/127631 return service.client(repositoryMetadata); } diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java index 0904f37e3974..92b0dcc2f1cd 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java @@ -14,6 +14,7 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.support.RefCountingRunnable; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.BackoffPolicy; @@ -275,6 +276,7 @@ class S3Repository extends MeteredBlobStoreRepository { * Constructs an s3 backed repository */ S3Repository( + final ProjectId projectId, final RepositoryMetadata metadata, final NamedXContentRegistry namedXContentRegistry, final S3Service service, @@ -284,6 +286,7 @@ class S3Repository extends MeteredBlobStoreRepository { final S3RepositoriesMetrics s3RepositoriesMetrics ) { super( + projectId, metadata, namedXContentRegistry, clusterService, @@ -468,6 +471,7 @@ class S3Repository extends MeteredBlobStoreRepository { @Override protected S3BlobStore createBlobStore() { return new S3BlobStore( + getProjectId(), service, bucket, serverSideEncryption, diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java index 6e7989dbaf00..9f5aedb0d144 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java @@ -13,6 +13,7 @@ import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain; import org.apache.lucene.util.SetOnce; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Setting; @@ -57,6 +58,7 @@ public class S3RepositoryPlugin extends Plugin implements RepositoryPlugin, Relo // proxy method for testing protected S3Repository createRepository( + final ProjectId projectId, final RepositoryMetadata metadata, final NamedXContentRegistry registry, final ClusterService clusterService, @@ -64,7 +66,16 @@ public class S3RepositoryPlugin extends Plugin implements RepositoryPlugin, Relo final RecoverySettings recoverySettings, final S3RepositoriesMetrics s3RepositoriesMetrics ) { - return new S3Repository(metadata, registry, service.get(), clusterService, bigArrays, recoverySettings, s3RepositoriesMetrics); + return new S3Repository( + projectId, + metadata, + registry, + service.get(), + clusterService, + bigArrays, + recoverySettings, + s3RepositoriesMetrics + ); } @Override @@ -99,7 +110,15 @@ public class S3RepositoryPlugin extends Plugin implements RepositoryPlugin, Relo final S3RepositoriesMetrics s3RepositoriesMetrics = new S3RepositoriesMetrics(repositoriesMetrics); return Collections.singletonMap( S3Repository.TYPE, - metadata -> createRepository(metadata, registry, clusterService, bigArrays, recoverySettings, s3RepositoriesMetrics) + (projectId, metadata) -> createRepository( + projectId, + metadata, + registry, + clusterService, + bigArrays, + recoverySettings, + s3RepositoriesMetrics + ) ); } diff --git a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java index 1fb929e654fc..1a6cb96bfc8e 100644 --- a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java +++ b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java @@ -231,6 +231,7 @@ public class S3BlobContainerRetriesTests extends AbstractBlobContainerRetriesTes final RepositoryMetadata repositoryMetadata = new RepositoryMetadata("repository", S3Repository.TYPE, repositorySettings.build()); final S3BlobStore s3BlobStore = new S3BlobStore( + randomProjectIdOrDefault(), service, "bucket", S3Repository.SERVER_SIDE_ENCRYPTION_SETTING.getDefault(Settings.EMPTY), diff --git a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java index 51362b943548..f5b56048d13f 100644 --- a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java +++ b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java @@ -12,6 +12,7 @@ package org.elasticsearch.repositories.s3; import software.amazon.awssdk.http.SdkHttpClient; import software.amazon.awssdk.services.s3.S3Client; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.common.ReferenceDocs; import org.elasticsearch.common.settings.ClusterSettings; @@ -32,6 +33,7 @@ import java.util.Map; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; @@ -152,7 +154,9 @@ public class S3RepositoryTests extends ESTestCase { } private S3Repository createS3Repo(RepositoryMetadata metadata) { - return new S3Repository( + final ProjectId projectId = randomProjectIdOrDefault(); + final S3Repository s3Repository = new S3Repository( + projectId, metadata, NamedXContentRegistry.EMPTY, new DummyS3Service(mock(Environment.class), mock(ResourceWatcherService.class)), @@ -161,6 +165,8 @@ public class S3RepositoryTests extends ESTestCase { new RecoverySettings(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)), S3RepositoriesMetrics.NOOP ); + assertThat(s3Repository.getProjectId(), equalTo(projectId)); + return s3Repository; } public void testAnalysisFailureDetail() { diff --git a/modules/repository-url/src/main/java/org/elasticsearch/plugin/repository/url/URLRepositoryPlugin.java b/modules/repository-url/src/main/java/org/elasticsearch/plugin/repository/url/URLRepositoryPlugin.java index 75da8aad02fa..821b761aa663 100644 --- a/modules/repository-url/src/main/java/org/elasticsearch/plugin/repository/url/URLRepositoryPlugin.java +++ b/modules/repository-url/src/main/java/org/elasticsearch/plugin/repository/url/URLRepositoryPlugin.java @@ -53,7 +53,8 @@ public class URLRepositoryPlugin extends Plugin implements RepositoryPlugin { ) { return Collections.singletonMap( URLRepository.TYPE, - metadata -> new URLRepository( + (projectId, metadata) -> new URLRepository( + projectId, metadata, env, namedXContentRegistry, diff --git a/modules/repository-url/src/main/java/org/elasticsearch/repositories/url/URLRepository.java b/modules/repository-url/src/main/java/org/elasticsearch/repositories/url/URLRepository.java index eca846f955bf..b2ba97cdd4a1 100644 --- a/modules/repository-url/src/main/java/org/elasticsearch/repositories/url/URLRepository.java +++ b/modules/repository-url/src/main/java/org/elasticsearch/repositories/url/URLRepository.java @@ -11,6 +11,7 @@ package org.elasticsearch.repositories.url; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.blobstore.BlobContainer; @@ -92,6 +93,7 @@ public class URLRepository extends BlobStoreRepository { * Constructs a read-only URL-based repository */ public URLRepository( + ProjectId projectId, RepositoryMetadata metadata, Environment environment, NamedXContentRegistry namedXContentRegistry, @@ -100,7 +102,7 @@ public class URLRepository extends BlobStoreRepository { RecoverySettings recoverySettings, URLHttpClient.Factory httpClientFactory ) { - super(metadata, namedXContentRegistry, clusterService, bigArrays, recoverySettings, BlobPath.EMPTY); + super(projectId, metadata, namedXContentRegistry, clusterService, bigArrays, recoverySettings, BlobPath.EMPTY); if (URL_SETTING.exists(metadata.settings()) == false && REPOSITORIES_URL_SETTING.exists(environment.settings()) == false) { throw new RepositoryException(metadata.name(), "missing url"); diff --git a/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java b/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java index 6243de077468..34c360550a11 100644 --- a/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java +++ b/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java @@ -9,6 +9,7 @@ package org.elasticsearch.repositories.url; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.common.blobstore.url.http.URLHttpClient; import org.elasticsearch.common.settings.ClusterSettings; @@ -26,6 +27,7 @@ import java.io.IOException; import java.nio.file.Path; import java.util.Collections; +import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.CoreMatchers.nullValue; @@ -34,7 +36,9 @@ import static org.mockito.Mockito.mock; public class URLRepositoryTests extends ESTestCase { private URLRepository createRepository(Settings baseSettings, RepositoryMetadata repositoryMetadata) { - return new URLRepository( + final ProjectId projectId = randomProjectIdOrDefault(); + final URLRepository repository = new URLRepository( + projectId, repositoryMetadata, TestEnvironment.newEnvironment(baseSettings), new NamedXContentRegistry(Collections.emptyList()), @@ -43,6 +47,8 @@ public class URLRepositoryTests extends ESTestCase { new RecoverySettings(baseSettings, new ClusterSettings(baseSettings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)), mock(URLHttpClient.Factory.class) ); + assertThat(repository.getProjectId(), equalTo(projectId)); + return repository; } public void testWhiteListingRepoURL() throws IOException { diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsPlugin.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsPlugin.java index 939317c432d2..9d80a270c724 100644 --- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsPlugin.java +++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsPlugin.java @@ -71,7 +71,15 @@ public final class HdfsPlugin extends Plugin implements RepositoryPlugin { ) { return Collections.singletonMap( "hdfs", - (metadata) -> new HdfsRepository(metadata, env, namedXContentRegistry, clusterService, bigArrays, recoverySettings) + (projectId, metadata) -> new HdfsRepository( + projectId, + metadata, + env, + namedXContentRegistry, + clusterService, + bigArrays, + recoverySettings + ) ); } } diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java index 8f250d00f8d9..de01da575b8b 100644 --- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java +++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java @@ -17,6 +17,7 @@ import org.apache.hadoop.io.retry.FailoverProxyProvider; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; @@ -56,6 +57,7 @@ public final class HdfsRepository extends BlobStoreRepository { private final String pathSetting; public HdfsRepository( + ProjectId projectId, RepositoryMetadata metadata, Environment environment, NamedXContentRegistry namedXContentRegistry, @@ -63,7 +65,7 @@ public final class HdfsRepository extends BlobStoreRepository { BigArrays bigArrays, RecoverySettings recoverySettings ) { - super(metadata, namedXContentRegistry, clusterService, bigArrays, recoverySettings, BlobPath.EMPTY); + super(projectId, metadata, namedXContentRegistry, clusterService, bigArrays, recoverySettings, BlobPath.EMPTY); this.environment = environment; this.chunkSize = metadata.settings().getAsBytesSize("chunk_size", null); diff --git a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java index 580805102619..d0b84021e704 100644 --- a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java +++ b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java @@ -19,13 +19,21 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.UnsupportedFileSystemException; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.util.Progressable; +import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.common.blobstore.BlobContainer; import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.MockBigArrays; import org.elasticsearch.core.Streams; import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.env.Environment; +import org.elasticsearch.indices.recovery.RecoverySettings; +import org.elasticsearch.repositories.blobstore.BlobStoreTestUtil; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.fixtures.hdfs.HdfsClientThreadLeakFilter; +import org.elasticsearch.xcontent.NamedXContentRegistry; import org.hamcrest.CoreMatchers; import org.mockito.AdditionalMatchers; import org.mockito.Mockito; @@ -49,6 +57,8 @@ import static org.elasticsearch.repositories.blobstore.BlobStoreTestUtil.randomP import static org.elasticsearch.repositories.blobstore.ESBlobStoreRepositoryIntegTestCase.randomBytes; import static org.elasticsearch.repositories.blobstore.ESBlobStoreRepositoryIntegTestCase.readBlobFully; import static org.elasticsearch.repositories.blobstore.ESBlobStoreRepositoryIntegTestCase.writeBlob; +import static org.hamcrest.Matchers.equalTo; +import static org.mockito.Mockito.mock; @ThreadLeakFilters(filters = { HdfsClientThreadLeakFilter.class }) public class HdfsBlobStoreContainerTests extends ESTestCase { @@ -107,6 +117,20 @@ public class HdfsBlobStoreContainerTests extends ESTestCase { }); } + public void testRepositoryProjectId() { + final var projectId = randomProjectIdOrDefault(); + final var repository = new HdfsRepository( + projectId, + new RepositoryMetadata(randomIdentifier(), "hdfs", Settings.builder().put("uri", "hdfs:///").put("path", "foo").build()), + mock(Environment.class), + NamedXContentRegistry.EMPTY, + BlobStoreTestUtil.mockClusterService(), + MockBigArrays.NON_RECYCLING_INSTANCE, + new RecoverySettings(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)) + ); + assertThat(repository.getProjectId(), equalTo(projectId)); + } + public void testReadOnly() throws Exception { FileContext fileContext = createTestContext(); // Constructor will not create dir if read only diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/plan/ShardSnapshotsServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/plan/ShardSnapshotsServiceIT.java index bf3665968d2a..06e4d06fcee0 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/plan/ShardSnapshotsServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/plan/ShardSnapshotsServiceIT.java @@ -13,6 +13,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.blobstore.BlobContainer; @@ -79,7 +80,15 @@ public class ShardSnapshotsServiceIT extends ESIntegTestCase { ) { return Collections.singletonMap( TYPE, - metadata -> new FailingRepo(metadata, env, namedXContentRegistry, clusterService, bigArrays, recoverySettings) + (projectId, metadata) -> new FailingRepo( + projectId, + metadata, + env, + namedXContentRegistry, + clusterService, + bigArrays, + recoverySettings + ) ); } } @@ -94,6 +103,7 @@ public class ShardSnapshotsServiceIT extends ESIntegTestCase { private final boolean failLoadShardSnapshots; public FailingRepo( + ProjectId projectId, RepositoryMetadata metadata, Environment environment, NamedXContentRegistry namedXContentRegistry, @@ -101,7 +111,7 @@ public class ShardSnapshotsServiceIT extends ESIntegTestCase { BigArrays bigArrays, RecoverySettings recoverySettings ) { - super(metadata, environment, namedXContentRegistry, clusterService, bigArrays, recoverySettings); + super(projectId, metadata, environment, namedXContentRegistry, clusterService, bigArrays, recoverySettings); this.failGetRepositoryData = metadata.settings().getAsBoolean(FAIL_GET_REPOSITORY_DATA_SETTING_KEY, false); this.failLoadShardSnapshot = metadata.settings().getAsBoolean(FAIL_LOAD_SHARD_SNAPSHOT_SETTING_KEY, false); this.failLoadShardSnapshots = metadata.settings().getAsBoolean(FAIL_LOAD_SHARD_SNAPSHOTS_SETTING_KEY, false); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/repositories/InvalidRepositoryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/repositories/InvalidRepositoryIT.java index fc817d02c3c0..62967fc2d035 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/repositories/InvalidRepositoryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/repositories/InvalidRepositoryIT.java @@ -11,6 +11,7 @@ package org.elasticsearch.repositories; import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesResponse; import org.elasticsearch.action.admin.cluster.repositories.verify.VerifyRepositoryResponse; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Setting; @@ -50,6 +51,7 @@ public class InvalidRepositoryIT extends ESIntegTestCase { ); public UnstableRepository( + ProjectId projectId, RepositoryMetadata metadata, Environment environment, NamedXContentRegistry namedXContentRegistry, @@ -57,7 +59,7 @@ public class InvalidRepositoryIT extends ESIntegTestCase { BigArrays bigArrays, RecoverySettings recoverySettings ) { - super(metadata, environment, namedXContentRegistry, clusterService, bigArrays, recoverySettings); + super(projectId, metadata, environment, namedXContentRegistry, clusterService, bigArrays, recoverySettings); List unstableNodes = UNSTABLE_NODES.get(metadata.settings()); if (unstableNodes.contains(clusterService.getNodeName())) { throw new RepositoryException(metadata.name(), "Failed to create repository: current node is not stable"); @@ -76,7 +78,15 @@ public class InvalidRepositoryIT extends ESIntegTestCase { ) { return Collections.singletonMap( TYPE, - (metadata) -> new UnstableRepository(metadata, env, namedXContentRegistry, clusterService, bigArrays, recoverySettings) + (projectId, metadata) -> new UnstableRepository( + projectId, + metadata, + env, + namedXContentRegistry, + clusterService, + bigArrays, + recoverySettings + ) ); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryOperationPurposeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryOperationPurposeIT.java index b2e02b2f4c27..fb119caa25cb 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryOperationPurposeIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryOperationPurposeIT.java @@ -9,6 +9,7 @@ package org.elasticsearch.repositories.blobstore; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.blobstore.BlobContainer; @@ -100,13 +101,22 @@ public class BlobStoreRepositoryOperationPurposeIT extends AbstractSnapshotInteg ) { return Map.of( ASSERTING_REPO_TYPE, - metadata -> new AssertingRepository(metadata, env, namedXContentRegistry, clusterService, bigArrays, recoverySettings) + (projectId, metadata) -> new AssertingRepository( + projectId, + metadata, + env, + namedXContentRegistry, + clusterService, + bigArrays, + recoverySettings + ) ); } } private static class AssertingRepository extends FsRepository { AssertingRepository( + ProjectId projectId, RepositoryMetadata metadata, Environment environment, NamedXContentRegistry namedXContentRegistry, @@ -114,7 +124,7 @@ public class BlobStoreRepositoryOperationPurposeIT extends AbstractSnapshotInteg BigArrays bigArrays, RecoverySettings recoverySettings ) { - super(metadata, environment, namedXContentRegistry, clusterService, bigArrays, recoverySettings); + super(projectId, metadata, environment, namedXContentRegistry, clusterService, bigArrays, recoverySettings); } @Override diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/MetadataLoadingDuringSnapshotRestoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/MetadataLoadingDuringSnapshotRestoreIT.java index d07e5a94bc40..89a8af49efde 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/MetadataLoadingDuringSnapshotRestoreIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/MetadataLoadingDuringSnapshotRestoreIT.java @@ -14,6 +14,7 @@ import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotR import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.util.BigArrays; @@ -176,6 +177,7 @@ public class MetadataLoadingDuringSnapshotRestoreIT extends AbstractSnapshotInte final Map indicesMetadata = new ConcurrentHashMap<>(); public CountingMockRepository( + final ProjectId projectId, final RepositoryMetadata metadata, final Environment environment, final NamedXContentRegistry namedXContentRegistry, @@ -183,7 +185,7 @@ public class MetadataLoadingDuringSnapshotRestoreIT extends AbstractSnapshotInte BigArrays bigArrays, RecoverySettings recoverySettings ) { - super(metadata, environment, namedXContentRegistry, clusterService, bigArrays, recoverySettings); + super(projectId, metadata, environment, namedXContentRegistry, clusterService, bigArrays, recoverySettings); } @Override @@ -216,7 +218,15 @@ public class MetadataLoadingDuringSnapshotRestoreIT extends AbstractSnapshotInte ) { return Collections.singletonMap( TYPE, - metadata -> new CountingMockRepository(metadata, env, namedXContentRegistry, clusterService, bigArrays, recoverySettings) + (projectId, metadata) -> new CountingMockRepository( + projectId, + metadata, + env, + namedXContentRegistry, + clusterService, + bigArrays, + recoverySettings + ) ); } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotsServiceDoubleFinalizationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotsServiceDoubleFinalizationIT.java index 616ad294cf86..7c00a93fa495 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotsServiceDoubleFinalizationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotsServiceDoubleFinalizationIT.java @@ -16,6 +16,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateObserver; import org.elasticsearch.cluster.SnapshotDeletionsInProgress; import org.elasticsearch.cluster.SnapshotsInProgress; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.blobstore.BlobContainer; @@ -212,7 +213,15 @@ public class SnapshotsServiceDoubleFinalizationIT extends AbstractSnapshotIntegT ) { return Map.of( REPO_TYPE, - metadata -> new TestRepository(metadata, env, namedXContentRegistry, clusterService, bigArrays, recoverySettings) + (projectId, metadata) -> new TestRepository( + projectId, + metadata, + env, + namedXContentRegistry, + clusterService, + bigArrays, + recoverySettings + ) ); } } @@ -223,6 +232,7 @@ public class SnapshotsServiceDoubleFinalizationIT extends AbstractSnapshotIntegT private final AtomicReference barrierRef = new AtomicReference<>(); public TestRepository( + ProjectId projectId, RepositoryMetadata metadata, Environment environment, NamedXContentRegistry namedXContentRegistry, @@ -230,7 +240,7 @@ public class SnapshotsServiceDoubleFinalizationIT extends AbstractSnapshotIntegT BigArrays bigArrays, RecoverySettings recoverySettings ) { - super(metadata, environment, namedXContentRegistry, clusterService, bigArrays, recoverySettings); + super(projectId, metadata, environment, namedXContentRegistry, clusterService, bigArrays, recoverySettings); } public CyclicBarrier blockOnceForListBlobs() { diff --git a/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java b/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java index 5d1b82da4bdc..34803a12d66c 100644 --- a/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.component.Lifecycle; @@ -44,6 +45,11 @@ public class FilterRepository implements Repository { return in; } + @Override + public ProjectId getProjectId() { + return in.getProjectId(); + } + @Override public RepositoryMetadata getMetadata() { return in.getMetadata(); diff --git a/server/src/main/java/org/elasticsearch/repositories/InvalidRepository.java b/server/src/main/java/org/elasticsearch/repositories/InvalidRepository.java index 28d83a1dc24b..f722bdd2b455 100644 --- a/server/src/main/java/org/elasticsearch/repositories/InvalidRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/InvalidRepository.java @@ -13,6 +13,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.component.AbstractLifecycleComponent; @@ -35,10 +36,12 @@ import java.util.function.BooleanSupplier; */ public class InvalidRepository extends AbstractLifecycleComponent implements Repository { + private final ProjectId projectId; private final RepositoryMetadata repositoryMetadata; private final RepositoryException creationException; - public InvalidRepository(RepositoryMetadata repositoryMetadata, RepositoryException creationException) { + public InvalidRepository(ProjectId projectId, RepositoryMetadata repositoryMetadata, RepositoryException creationException) { + this.projectId = projectId; this.repositoryMetadata = repositoryMetadata; this.creationException = creationException; } @@ -51,6 +54,11 @@ public class InvalidRepository extends AbstractLifecycleComponent implements Rep ); } + @Override + public ProjectId getProjectId() { + return projectId; + } + @Override public RepositoryMetadata getMetadata() { return repositoryMetadata; diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoriesModule.java b/server/src/main/java/org/elasticsearch/repositories/RepositoriesModule.java index b236b1fa730f..22adf929a3a0 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoriesModule.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoriesModule.java @@ -54,7 +54,15 @@ public final class RepositoriesModule { Map factories = new HashMap<>(); factories.put( FsRepository.TYPE, - metadata -> new FsRepository(metadata, env, namedXContentRegistry, clusterService, bigArrays, recoverySettings) + (projectId, metadata) -> new FsRepository( + projectId, + metadata, + env, + namedXContentRegistry, + clusterService, + bigArrays, + recoverySettings + ) ); for (RepositoryPlugin repoPlugin : repoPlugins) { diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java b/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java index f94e0a95fecd..83f71bf6e3da 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java @@ -30,6 +30,7 @@ import org.elasticsearch.cluster.SnapshotDeletionsInProgress; import org.elasticsearch.cluster.SnapshotsInProgress; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.cluster.metadata.RepositoriesMetadata; import org.elasticsearch.cluster.metadata.RepositoryMetadata; @@ -44,7 +45,9 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.ListenableFuture; +import org.elasticsearch.core.FixForMultiProject; import org.elasticsearch.core.IOUtils; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.Index; @@ -65,7 +68,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.function.BiConsumer; -import java.util.function.Function; +import java.util.function.BiFunction; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -653,7 +656,7 @@ public class RepositoriesService extends AbstractLifecycleComponent implements C // TODO: this catch is bogus, it means the old repo is already closed, // but we have nothing to replace it logger.warn(() -> "failed to change repository [" + repositoryMetadata.name() + "]", ex); - repository = new InvalidRepository(repositoryMetadata, ex); + repository = new InvalidRepository(state.metadata().getProject().id(), repositoryMetadata, ex); } } } else { @@ -661,7 +664,7 @@ public class RepositoriesService extends AbstractLifecycleComponent implements C repository = createRepository(repositoryMetadata, typesRegistry, RepositoriesService::createUnknownTypeRepository); } catch (RepositoryException ex) { logger.warn(() -> "failed to create repository [" + repositoryMetadata.name() + "]", ex); - repository = new InvalidRepository(repositoryMetadata, ex); + repository = new InvalidRepository(state.metadata().getProject().id(), repositoryMetadata, ex); } } assert repository != null : "repository should not be null here"; @@ -822,19 +825,33 @@ public class RepositoriesService extends AbstractLifecycleComponent implements C /** * Creates repository holder. This method starts the repository */ + @FixForMultiProject(description = "resolve the actual ProjectId") + @Deprecated(forRemoval = true) private static Repository createRepository( RepositoryMetadata repositoryMetadata, Map factories, - Function defaultFactory + BiFunction defaultFactory + ) { + return createRepository(ProjectId.DEFAULT, repositoryMetadata, factories, defaultFactory); + } + + /** + * Creates repository holder. This method starts the repository + */ + private static Repository createRepository( + @Nullable ProjectId projectId, + RepositoryMetadata repositoryMetadata, + Map factories, + BiFunction defaultFactory ) { logger.debug("creating repository [{}][{}]", repositoryMetadata.type(), repositoryMetadata.name()); Repository.Factory factory = factories.get(repositoryMetadata.type()); if (factory == null) { - return defaultFactory.apply(repositoryMetadata); + return defaultFactory.apply(projectId, repositoryMetadata); } Repository repository = null; try { - repository = factory.create(repositoryMetadata, factories::get); + repository = factory.create(projectId, repositoryMetadata, factories::get); repository.start(); return repository; } catch (Exception e) { @@ -859,21 +876,61 @@ public class RepositoriesService extends AbstractLifecycleComponent implements C * @return the started repository * @throws RepositoryException if repository type is not registered */ + @FixForMultiProject(description = "resolve the actual ProjectId") + @Deprecated(forRemoval = true) public Repository createRepository(RepositoryMetadata repositoryMetadata) { - return createRepository(repositoryMetadata, typesRegistry, RepositoriesService::throwRepositoryTypeDoesNotExists); + return createRepository(ProjectId.DEFAULT, repositoryMetadata); } - private static Repository throwRepositoryTypeDoesNotExists(RepositoryMetadata repositoryMetadata) { - throw new RepositoryException(repositoryMetadata.name(), "repository type [" + repositoryMetadata.type() + "] does not exist"); + /** + * Creates a repository holder. + * + *

WARNING: This method is intended for expert only usage mainly in plugins/modules. Please take note of the following:

+ * + *
    + *
  • This method does not register the repository (e.g., in the cluster state).
    • + *
  • This method starts the repository. The repository should be closed after use.
    • + *
  • The repository metadata should be associated to an already registered non-internal repository type and factory pair.
    • + *
+ * + * @param projectId the project that the repository is associated with + * @param repositoryMetadata the repository metadata + * @return the started repository + * @throws RepositoryException if repository type is not registered + */ + public Repository createRepository(ProjectId projectId, RepositoryMetadata repositoryMetadata) { + return createRepository( + Objects.requireNonNull(projectId), + repositoryMetadata, + typesRegistry, + RepositoriesService::throwRepositoryTypeDoesNotExists + ); } - private static Repository createUnknownTypeRepository(RepositoryMetadata repositoryMetadata) { + /** + * Similar to {@link #createRepository(ProjectId, RepositoryMetadata)}, but repository is not associated with a project, i.e. the + * repository is at the cluster level. + */ + public Repository createNonProjectRepository(RepositoryMetadata repositoryMetadata) { + assert DiscoveryNode.isStateless(clusterService.getSettings()) + : "outside stateless only project level repositories are allowed: " + repositoryMetadata; + return createRepository(null, repositoryMetadata, typesRegistry, RepositoriesService::throwRepositoryTypeDoesNotExists); + } + + private static Repository throwRepositoryTypeDoesNotExists(ProjectId projectId, RepositoryMetadata repositoryMetadata) { + throw new RepositoryException( + repositoryMetadata.name(), + "repository type [" + repositoryMetadata.type() + "] does not exist for project [" + projectId + "]" + ); + } + + private static Repository createUnknownTypeRepository(ProjectId projectId, RepositoryMetadata repositoryMetadata) { logger.warn( "[{}] repository type [{}] is unknown; ensure that all required plugins are installed on this node", repositoryMetadata.name(), repositoryMetadata.type() ); - return new UnknownTypeRepository(repositoryMetadata); + return new UnknownTypeRepository(projectId, repositoryMetadata); } public static void validateRepositoryName(final String repositoryName) { diff --git a/server/src/main/java/org/elasticsearch/repositories/Repository.java b/server/src/main/java/org/elasticsearch/repositories/Repository.java index 5f7971736fa4..74c5f3e0e9dc 100644 --- a/server/src/main/java/org/elasticsearch/repositories/Repository.java +++ b/server/src/main/java/org/elasticsearch/repositories/Repository.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.component.LifecycleComponent; @@ -57,15 +58,35 @@ public interface Repository extends LifecycleComponent { interface Factory { /** * Constructs a repository. - * @param metadata metadata for the repository including name and settings + * + * @param projectId the project-id for the repository or {@code null} if the repository is at the cluster level. + * @param metadata metadata for the repository including name and settings */ - Repository create(RepositoryMetadata metadata) throws Exception; + Repository create(@Nullable ProjectId projectId, RepositoryMetadata metadata) throws Exception; - default Repository create(RepositoryMetadata metadata, Function typeLookup) throws Exception { - return create(metadata); + /** + * Constructs a repository. + * @param projectId the project-id for the repository or {@code null} if the repository is at the cluster level. + * @param metadata metadata for the repository including name and settings + * @param typeLookup a function that returns the repository factory for the given repository type. + */ + default Repository create( + @Nullable ProjectId projectId, + RepositoryMetadata metadata, + Function typeLookup + ) throws Exception { + return create(projectId, metadata); } } + /** + * Get the project-id for the repository. + * + * @return the project-id, or {@code null} if the repository is at the cluster level. + */ + @Nullable + ProjectId getProjectId(); + /** * Returns metadata about this repository. */ diff --git a/server/src/main/java/org/elasticsearch/repositories/UnknownTypeRepository.java b/server/src/main/java/org/elasticsearch/repositories/UnknownTypeRepository.java index 066d8883bdc4..13d81d2d0d48 100644 --- a/server/src/main/java/org/elasticsearch/repositories/UnknownTypeRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/UnknownTypeRepository.java @@ -13,6 +13,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.component.AbstractLifecycleComponent; @@ -36,9 +37,11 @@ import java.util.function.BooleanSupplier; */ public class UnknownTypeRepository extends AbstractLifecycleComponent implements Repository { + private final ProjectId projectId; private final RepositoryMetadata repositoryMetadata; - public UnknownTypeRepository(RepositoryMetadata repositoryMetadata) { + public UnknownTypeRepository(ProjectId projectId, RepositoryMetadata repositoryMetadata) { + this.projectId = projectId; this.repositoryMetadata = repositoryMetadata; } @@ -49,6 +52,11 @@ public class UnknownTypeRepository extends AbstractLifecycleComponent implements ); } + @Override + public ProjectId getProjectId() { + return projectId; + } + @Override public RepositoryMetadata getMetadata() { return repositoryMetadata; diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index f571a1a78a64..d69b9b739f7c 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -40,6 +40,7 @@ import org.elasticsearch.cluster.SnapshotDeletionsInProgress; import org.elasticsearch.cluster.SnapshotsInProgress; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.RepositoriesMetadata; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -195,6 +196,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp private static final Logger shutdownLogger = LogManager.getLogger(ShutdownLogger.class); } + private final ProjectId projectId; protected volatile RepositoryMetadata metadata; protected final ThreadPool threadPool; @@ -485,6 +487,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp */ @SuppressWarnings("this-escape") protected BlobStoreRepository( + final ProjectId projectId, final RepositoryMetadata metadata, final NamedXContentRegistry namedXContentRegistry, final ClusterService clusterService, @@ -492,6 +495,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp final RecoverySettings recoverySettings, final BlobPath basePath ) { + this.projectId = projectId; this.metadata = metadata; this.threadPool = clusterService.getClusterApplierService().threadPool(); this.clusterService = clusterService; @@ -525,6 +529,11 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp ); } + @Override + public ProjectId getProjectId() { + return projectId; + } + @Override protected void doStart() { uncleanStart = metadata.pendingGeneration() > RepositoryData.EMPTY_REPO_GEN diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/MeteredBlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/MeteredBlobStoreRepository.java index 2dd69e918f9c..aa7fae749c32 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/MeteredBlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/MeteredBlobStoreRepository.java @@ -9,6 +9,7 @@ package org.elasticsearch.repositories.blobstore; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.UUIDs; @@ -26,6 +27,7 @@ public abstract class MeteredBlobStoreRepository extends BlobStoreRepository { private final RepositoryInfo repositoryInfo; public MeteredBlobStoreRepository( + ProjectId projectId, RepositoryMetadata metadata, NamedXContentRegistry namedXContentRegistry, ClusterService clusterService, @@ -34,7 +36,7 @@ public abstract class MeteredBlobStoreRepository extends BlobStoreRepository { BlobPath basePath, Map location ) { - super(metadata, namedXContentRegistry, clusterService, bigArrays, recoverySettings, basePath); + super(projectId, metadata, namedXContentRegistry, clusterService, bigArrays, recoverySettings, basePath); ThreadPool threadPool = clusterService.getClusterApplierService().threadPool(); this.repositoryInfo = new RepositoryInfo( UUIDs.randomBase64UUID(), diff --git a/server/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java b/server/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java index f284faf8f304..97e2c0b2de44 100644 --- a/server/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java @@ -11,6 +11,7 @@ package org.elasticsearch.repositories.fs; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.blobstore.BlobPath; @@ -75,6 +76,7 @@ public class FsRepository extends BlobStoreRepository { * Constructs a shared file system repository. */ public FsRepository( + ProjectId projectId, RepositoryMetadata metadata, Environment environment, NamedXContentRegistry namedXContentRegistry, @@ -82,7 +84,7 @@ public class FsRepository extends BlobStoreRepository { BigArrays bigArrays, RecoverySettings recoverySettings ) { - super(metadata, namedXContentRegistry, clusterService, bigArrays, recoverySettings, BlobPath.EMPTY); + super(projectId, metadata, namedXContentRegistry, clusterService, bigArrays, recoverySettings, BlobPath.EMPTY); this.environment = environment; String location = REPOSITORIES_LOCATION_SETTING.get(metadata.settings()); if (location.isEmpty()) { diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/repositories/reservedstate/ReservedRepositoryActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/repositories/reservedstate/ReservedRepositoryActionTests.java index 17ff2a85c11f..4d07c9fe897e 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/repositories/reservedstate/ReservedRepositoryActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/repositories/reservedstate/ReservedRepositoryActionTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequ import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; @@ -126,7 +127,7 @@ public class ReservedRepositoryActionTests extends ESTestCase { private RepositoriesService mockRepositoriesService() { var fsFactory = new Repository.Factory() { @Override - public Repository create(RepositoryMetadata metadata) { + public Repository create(ProjectId projectId, RepositoryMetadata metadata) { var repo = mock(Repository.class); doAnswer(invocation -> metadata).when(repo).getMetadata(); return repo; diff --git a/server/src/test/java/org/elasticsearch/health/node/tracker/RepositoriesHealthTrackerTests.java b/server/src/test/java/org/elasticsearch/health/node/tracker/RepositoriesHealthTrackerTests.java index 6dd07548b8b0..561c05165eab 100644 --- a/server/src/test/java/org/elasticsearch/health/node/tracker/RepositoriesHealthTrackerTests.java +++ b/server/src/test/java/org/elasticsearch/health/node/tracker/RepositoriesHealthTrackerTests.java @@ -68,7 +68,9 @@ public class RepositoriesHealthTrackerTests extends ESTestCase { public void testGetHealthUnknownType() { var repo = createRepositoryMetadata(); - when(repositoriesService.getRepositories()).thenReturn(Map.of(randomAlphaOfLength(10), new UnknownTypeRepository(repo))); + when(repositoriesService.getRepositories()).thenReturn( + Map.of(randomAlphaOfLength(10), new UnknownTypeRepository(randomProjectIdOrDefault(), repo)) + ); var health = repositoriesHealthTracker.determineCurrentHealth(); @@ -80,7 +82,7 @@ public class RepositoriesHealthTrackerTests extends ESTestCase { public void testGetHealthInvalid() { var repo = createRepositoryMetadata(); when(repositoriesService.getRepositories()).thenReturn( - Map.of(repo.name(), new InvalidRepository(repo, new RepositoryException(repo.name(), "Test"))) + Map.of(repo.name(), new InvalidRepository(randomProjectIdOrDefault(), repo, new RepositoryException(repo.name(), "Test"))) ); var health = repositoriesHealthTracker.determineCurrentHealth(); diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 1030e3d09b9a..10df837c8d4f 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -2842,7 +2842,7 @@ public class IndexShardTests extends IndexShardTestCase { DiscoveryNode localNode = DiscoveryNodeUtils.builder("foo").roles(emptySet()).build(); target.markAsRecovering("store", new RecoveryState(routing, localNode, null)); final PlainActionFuture future = new PlainActionFuture<>(); - target.restoreFromRepository(new RestoreOnlyRepository("test") { + target.restoreFromRepository(new RestoreOnlyRepository(randomProjectIdOrDefault(), "test") { @Override public void restoreShard( Store store, diff --git a/server/src/test/java/org/elasticsearch/repositories/InvalidRepositoryTests.java b/server/src/test/java/org/elasticsearch/repositories/InvalidRepositoryTests.java index f2097b37c03e..7a82ec02c956 100644 --- a/server/src/test/java/org/elasticsearch/repositories/InvalidRepositoryTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/InvalidRepositoryTests.java @@ -9,6 +9,7 @@ package org.elasticsearch.repositories; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.snapshots.SnapshotId; @@ -19,12 +20,16 @@ import static org.hamcrest.Matchers.isA; public class InvalidRepositoryTests extends ESTestCase { + private ProjectId projectId = randomProjectIdOrDefault(); + private InvalidRepository repository = new InvalidRepository( + projectId, new RepositoryMetadata("name", "type", Settings.EMPTY), new RepositoryException("name", "failed to create repository") ); public void testShouldThrowWhenGettingMetadata() { + assertThat(repository.getProjectId(), equalTo(projectId)); final var expectedException = expectThrows( RepositoryException.class, () -> repository.getSnapshotGlobalMetadata(new SnapshotId("name", "uuid")) diff --git a/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java b/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java index 291072ab2467..7a179a83af6b 100644 --- a/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateUpdateTask; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.RepositoriesMetadata; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -118,15 +119,15 @@ public class RepositoriesServiceTests extends ESTestCase { Map typesRegistry = Map.of( TestRepository.TYPE, - TestRepository::new, + (projectId, metadata1) -> new TestRepository(projectId, metadata1), UnstableRepository.TYPE, - UnstableRepository::new, + (projectId, metadata2) -> new UnstableRepository(projectId, metadata2), VerificationFailRepository.TYPE, - VerificationFailRepository::new, + (projectId, metadata3) -> new VerificationFailRepository(projectId, metadata3), MeteredRepositoryTypeA.TYPE, - metadata -> new MeteredRepositoryTypeA(metadata, clusterService), + (projectId, metadata) -> new MeteredRepositoryTypeA(projectId, metadata, clusterService), MeteredRepositoryTypeB.TYPE, - metadata -> new MeteredRepositoryTypeB(metadata, clusterService) + (projectId, metadata) -> new MeteredRepositoryTypeB(projectId, metadata, clusterService) ); repositoriesService = new RepositoriesService( Settings.EMPTY, @@ -304,7 +305,7 @@ public class RepositoriesServiceTests extends ESTestCase { @Override public void onFailure(Exception e) { assertThat(e, isA(RepositoryException.class)); - assertThat(e.getMessage(), equalTo("[" + repoName + "] repository type [unknown] does not exist")); + assertThat(e.getMessage(), equalTo("[" + repoName + "] repository type [unknown] does not exist for project [default]")); } }); } @@ -507,14 +508,21 @@ public class RepositoriesServiceTests extends ESTestCase { private static class TestRepository implements Repository { private static final String TYPE = "internal"; + private final ProjectId projectId; private RepositoryMetadata metadata; private boolean isClosed; private boolean isStarted; - private TestRepository(RepositoryMetadata metadata) { + private TestRepository(ProjectId projectId, RepositoryMetadata metadata) { + this.projectId = projectId; this.metadata = metadata; } + @Override + public ProjectId getProjectId() { + return projectId; + } + @Override public RepositoryMetadata getMetadata() { return metadata; @@ -662,8 +670,8 @@ public class RepositoriesServiceTests extends ESTestCase { private static class UnstableRepository extends TestRepository { private static final String TYPE = "unstable"; - private UnstableRepository(RepositoryMetadata metadata) { - super(metadata); + private UnstableRepository(ProjectId projectId, RepositoryMetadata metadata) { + super(projectId, metadata); throw new RepositoryException(TYPE, "failed to create unstable repository"); } } @@ -671,8 +679,8 @@ public class RepositoriesServiceTests extends ESTestCase { private static class VerificationFailRepository extends TestRepository { public static final String TYPE = "verify-fail"; - private VerificationFailRepository(RepositoryMetadata metadata) { - super(metadata); + private VerificationFailRepository(ProjectId projectId, RepositoryMetadata metadata) { + super(projectId, metadata); } @Override @@ -685,8 +693,9 @@ public class RepositoriesServiceTests extends ESTestCase { private static final String TYPE = "type-a"; private static final RepositoryStats STATS = new RepositoryStats(Map.of("GET", new BlobStoreActionStats(10, 13))); - private MeteredRepositoryTypeA(RepositoryMetadata metadata, ClusterService clusterService) { + private MeteredRepositoryTypeA(ProjectId projectId, RepositoryMetadata metadata, ClusterService clusterService) { super( + projectId, metadata, mock(NamedXContentRegistry.class), clusterService, @@ -712,8 +721,9 @@ public class RepositoriesServiceTests extends ESTestCase { private static final String TYPE = "type-b"; private static final RepositoryStats STATS = new RepositoryStats(Map.of("LIST", new BlobStoreActionStats(20, 25))); - private MeteredRepositoryTypeB(RepositoryMetadata metadata, ClusterService clusterService) { + private MeteredRepositoryTypeB(ProjectId projectId, RepositoryMetadata metadata, ClusterService clusterService) { super( + projectId, metadata, mock(NamedXContentRegistry.class), clusterService, diff --git a/server/src/test/java/org/elasticsearch/repositories/UnknownTypeRepositoryTests.java b/server/src/test/java/org/elasticsearch/repositories/UnknownTypeRepositoryTests.java index 022fa4df6a23..b7225546fd82 100644 --- a/server/src/test/java/org/elasticsearch/repositories/UnknownTypeRepositoryTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/UnknownTypeRepositoryTests.java @@ -9,16 +9,21 @@ package org.elasticsearch.repositories; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.test.ESTestCase; +import static org.hamcrest.Matchers.equalTo; + public class UnknownTypeRepositoryTests extends ESTestCase { - private UnknownTypeRepository repository = new UnknownTypeRepository(new RepositoryMetadata("name", "type", Settings.EMPTY)); + private ProjectId projectId = randomProjectIdOrDefault(); + private UnknownTypeRepository repository = new UnknownTypeRepository(projectId, new RepositoryMetadata("name", "type", Settings.EMPTY)); public void testShouldThrowWhenGettingMetadata() { + assertThat(repository.getProjectId(), equalTo(projectId)); expectThrows(RepositoryException.class, () -> repository.getSnapshotGlobalMetadata(new SnapshotId("name", "uuid"))); } diff --git a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryDeleteThrottlingTests.java b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryDeleteThrottlingTests.java index 4facaa391ec2..de1979cfcf5d 100644 --- a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryDeleteThrottlingTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryDeleteThrottlingTests.java @@ -75,7 +75,15 @@ public class BlobStoreRepositoryDeleteThrottlingTests extends ESSingleNodeTestCa ) { return Collections.singletonMap( TEST_REPO_TYPE, - (metadata) -> new FsRepository(metadata, env, namedXContentRegistry, clusterService, bigArrays, recoverySettings) { + (projectId, metadata) -> new FsRepository( + projectId, + metadata, + env, + namedXContentRegistry, + clusterService, + bigArrays, + recoverySettings + ) { @Override protected BlobStore createBlobStore() throws Exception { return new ConcurrencyLimitingBlobStore(super.createBlobStore()); diff --git a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java index 624a6322cd75..3666799ce096 100644 --- a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java @@ -219,6 +219,7 @@ public class BlobStoreRepositoryRestoreTests extends IndexShardTestCase { RepositoryMetadata repositoryMetadata = new RepositoryMetadata(randomAlphaOfLength(10), FsRepository.TYPE, settings); final ClusterService clusterService = BlobStoreTestUtil.mockClusterService(repositoryMetadata); final FsRepository repository = new FsRepository( + randomProjectIdOrDefault(), repositoryMetadata, createEnvironment(), xContentRegistry(), diff --git a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java index f9d689dc42a8..845dca770d7f 100644 --- a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java @@ -523,6 +523,7 @@ public class BlobStoreRepositoryTests extends ESSingleNodeTestCase { RepositoryMetadata repositoryMetadata = new RepositoryMetadata(randomAlphaOfLength(10), FsRepository.TYPE, settings); final ClusterService clusterService = BlobStoreTestUtil.mockClusterService(repositoryMetadata); final FsRepository repository = new FsRepository( + randomProjectIdOrDefault(), repositoryMetadata, createEnvironment(), xContentRegistry(), diff --git a/server/src/test/java/org/elasticsearch/repositories/fs/FsRepositoryTests.java b/server/src/test/java/org/elasticsearch/repositories/fs/FsRepositoryTests.java index 36cd84a0d32b..3b1eb8c1f39d 100644 --- a/server/src/test/java/org/elasticsearch/repositories/fs/FsRepositoryTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/fs/FsRepositoryTests.java @@ -26,6 +26,7 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.IOSupplier; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeUtils; @@ -101,7 +102,9 @@ public class FsRepositoryTests extends ESTestCase { int numDocs = indexDocs(directory); RepositoryMetadata metadata = new RepositoryMetadata("test", "fs", settings); + final ProjectId projectId = randomProjectIdOrDefault(); FsRepository repository = new FsRepository( + projectId, metadata, new Environment(settings, null), NamedXContentRegistry.EMPTY, @@ -109,6 +112,7 @@ public class FsRepositoryTests extends ESTestCase { MockBigArrays.NON_RECYCLING_INSTANCE, new RecoverySettings(settings, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)) ); + assertThat(repository.getProjectId(), equalTo(projectId)); repository.start(); final Settings indexSettings = Settings.builder().put(IndexMetadata.SETTING_INDEX_UUID, "myindexUUID").build(); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("myindex", indexSettings); @@ -227,7 +231,9 @@ public class FsRepositoryTests extends ESTestCase { final AtomicBoolean canErrorForWriteBlob = new AtomicBoolean(); final AtomicBoolean shouldErrorForWriteMetadataBlob = new AtomicBoolean(); final AtomicBoolean writeBlobErrored = new AtomicBoolean(false); + final ProjectId projectId = randomProjectIdOrDefault(); final var repository = new FsRepository( + projectId, metadata, new Environment(settings, null), NamedXContentRegistry.EMPTY, @@ -284,6 +290,7 @@ public class FsRepositoryTests extends ESTestCase { }; } }; + assertThat(repository.getProjectId(), equalTo(projectId)); repository.start(); final IndexSettings idxSettings = IndexSettingsModule.newIndexSettings( diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java index cae33abb126a..2daf1222748b 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java @@ -2169,7 +2169,15 @@ public class SnapshotResiliencyTests extends ESTestCase { clusterService, Collections.singletonMap( FsRepository.TYPE, - metadata -> new FsRepository(metadata, environment, xContentRegistry(), clusterService, bigArrays, recoverySettings) + (projectId, metadata) -> new FsRepository( + projectId, + metadata, + environment, + xContentRegistry(), + clusterService, + bigArrays, + recoverySettings + ) ), emptyMap(), threadPool, diff --git a/test/external-modules/latency-simulating-directory/src/internalClusterTest/java/org/elasticsearch/test/simulatedlatencyrepo/LatencySimulatingBlobStoreRepositoryTests.java b/test/external-modules/latency-simulating-directory/src/internalClusterTest/java/org/elasticsearch/test/simulatedlatencyrepo/LatencySimulatingBlobStoreRepositoryTests.java index 7f2cb85919d1..c553f0f10062 100644 --- a/test/external-modules/latency-simulating-directory/src/internalClusterTest/java/org/elasticsearch/test/simulatedlatencyrepo/LatencySimulatingBlobStoreRepositoryTests.java +++ b/test/external-modules/latency-simulating-directory/src/internalClusterTest/java/org/elasticsearch/test/simulatedlatencyrepo/LatencySimulatingBlobStoreRepositoryTests.java @@ -65,7 +65,8 @@ public class LatencySimulatingBlobStoreRepositoryTests extends AbstractSnapshotI ) { return Map.of( REPO_TYPE, - metadata -> new LatencySimulatingBlobStoreRepository( + (projectId, metadata) -> new LatencySimulatingBlobStoreRepository( + projectId, metadata, env, namedXContentRegistry, diff --git a/test/external-modules/latency-simulating-directory/src/main/java/org/elasticsearch/test/simulatedlatencyrepo/LatencySimulatingBlobStoreRepository.java b/test/external-modules/latency-simulating-directory/src/main/java/org/elasticsearch/test/simulatedlatencyrepo/LatencySimulatingBlobStoreRepository.java index cd2812a95cfa..a4126c41c092 100644 --- a/test/external-modules/latency-simulating-directory/src/main/java/org/elasticsearch/test/simulatedlatencyrepo/LatencySimulatingBlobStoreRepository.java +++ b/test/external-modules/latency-simulating-directory/src/main/java/org/elasticsearch/test/simulatedlatencyrepo/LatencySimulatingBlobStoreRepository.java @@ -9,6 +9,7 @@ package org.elasticsearch.test.simulatedlatencyrepo; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.blobstore.BlobContainer; @@ -30,6 +31,7 @@ class LatencySimulatingBlobStoreRepository extends FsRepository { private final Runnable simulator; protected LatencySimulatingBlobStoreRepository( + ProjectId projectId, RepositoryMetadata metadata, Environment env, NamedXContentRegistry namedXContentRegistry, @@ -38,7 +40,7 @@ class LatencySimulatingBlobStoreRepository extends FsRepository { RecoverySettings recoverySettings, Runnable simulator ) { - super(metadata, env, namedXContentRegistry, clusterService, bigArrays, recoverySettings); + super(projectId, metadata, env, namedXContentRegistry, clusterService, bigArrays, recoverySettings); this.simulator = simulator; } diff --git a/test/external-modules/latency-simulating-directory/src/main/java/org/elasticsearch/test/simulatedlatencyrepo/LatencySimulatingRepositoryPlugin.java b/test/external-modules/latency-simulating-directory/src/main/java/org/elasticsearch/test/simulatedlatencyrepo/LatencySimulatingRepositoryPlugin.java index f9582229dbb2..b60c965a6653 100644 --- a/test/external-modules/latency-simulating-directory/src/main/java/org/elasticsearch/test/simulatedlatencyrepo/LatencySimulatingRepositoryPlugin.java +++ b/test/external-modules/latency-simulating-directory/src/main/java/org/elasticsearch/test/simulatedlatencyrepo/LatencySimulatingRepositoryPlugin.java @@ -41,7 +41,8 @@ public class LatencySimulatingRepositoryPlugin extends Plugin implements Reposit ) { return Map.of( TYPE, - metadata -> new LatencySimulatingBlobStoreRepository( + (projectId, metadata) -> new LatencySimulatingBlobStoreRepository( + projectId, metadata, env, namedXContentRegistry, diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java b/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java index 3ec64ae315fd..67dd587258f2 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java +++ b/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.component.AbstractLifecycleComponent; @@ -42,9 +43,11 @@ import static org.elasticsearch.repositories.RepositoryData.MISSING_UUID; /** A dummy repository for testing which just needs restore overridden */ public abstract class RestoreOnlyRepository extends AbstractLifecycleComponent implements Repository { + private final ProjectId projectId; private final String indexName; - public RestoreOnlyRepository(String indexName) { + public RestoreOnlyRepository(ProjectId projectId, String indexName) { + this.projectId = projectId; this.indexName = indexName; } @@ -57,6 +60,11 @@ public abstract class RestoreOnlyRepository extends AbstractLifecycleComponent i @Override protected void doClose() {} + @Override + public ProjectId getProjectId() { + return projectId; + } + @Override public RepositoryMetadata getMetadata() { return null; diff --git a/test/framework/src/main/java/org/elasticsearch/snapshots/mockstore/MockRepository.java b/test/framework/src/main/java/org/elasticsearch/snapshots/mockstore/MockRepository.java index 89d02068b56d..252091bb45d8 100644 --- a/test/framework/src/main/java/org/elasticsearch/snapshots/mockstore/MockRepository.java +++ b/test/framework/src/main/java/org/elasticsearch/snapshots/mockstore/MockRepository.java @@ -15,6 +15,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.index.CorruptIndexException; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.blobstore.BlobContainer; @@ -89,7 +90,15 @@ public class MockRepository extends FsRepository { ) { return Collections.singletonMap( "mock", - (metadata) -> new MockRepository(metadata, env, namedXContentRegistry, clusterService, bigArrays, recoverySettings) + (projectId, metadata) -> new MockRepository( + projectId, + metadata, + env, + namedXContentRegistry, + clusterService, + bigArrays, + recoverySettings + ) ); } @@ -176,6 +185,7 @@ public class MockRepository extends FsRepository { private volatile boolean failOnDeleteContainer = false; public MockRepository( + ProjectId projectId, RepositoryMetadata metadata, Environment environment, NamedXContentRegistry namedXContentRegistry, @@ -183,7 +193,15 @@ public class MockRepository extends FsRepository { BigArrays bigArrays, RecoverySettings recoverySettings ) { - super(overrideSettings(metadata, environment), environment, namedXContentRegistry, clusterService, bigArrays, recoverySettings); + super( + projectId, + overrideSettings(metadata, environment), + environment, + namedXContentRegistry, + clusterService, + bigArrays, + recoverySettings + ); randomControlIOExceptionRate = metadata.settings().getAsDouble("random_control_io_exception_rate", 0.0); randomDataFileIOExceptionRate = metadata.settings().getAsDouble("random_data_file_io_exception_rate", 0.0); randomIOExceptionPattern = Pattern.compile(metadata.settings().get("random_io_exception_pattern", ".*")).asMatchPredicate(); diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java index bee4644f809d..bfc85f3efb0f 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java @@ -375,7 +375,8 @@ public class Ccr extends Plugin implements ActionPlugin, PersistentTaskPlugin, E ClusterService clusterService, RecoverySettings recoverySettings ) { - Repository.Factory repositoryFactory = (metadata) -> new CcrRepository( + Repository.Factory repositoryFactory = (projectId, metadata) -> new CcrRepository( + projectId, metadata, client, settings, diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java index 0bc72c8f7821..bd2c925d5234 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java @@ -37,6 +37,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.ShardRouting; @@ -138,6 +139,7 @@ public class CcrRepository extends AbstractLifecycleComponent implements Reposit private static final SnapshotId SNAPSHOT_ID = new SnapshotId(LATEST, LATEST); private static final String IN_SYNC_ALLOCATION_ID = "ccr_restore"; + private final ProjectId projectId; private final RepositoryMetadata metadata; private final CcrSettings ccrSettings; private final String localClusterName; @@ -151,7 +153,15 @@ public class CcrRepository extends AbstractLifecycleComponent implements Reposit private final SingleResultDeduplicator csDeduplicator; - public CcrRepository(RepositoryMetadata metadata, Client client, Settings settings, CcrSettings ccrSettings, ThreadPool threadPool) { + public CcrRepository( + ProjectId projectId, + RepositoryMetadata metadata, + Client client, + Settings settings, + CcrSettings ccrSettings, + ThreadPool threadPool + ) { + this.projectId = projectId; this.metadata = metadata; this.ccrSettings = ccrSettings; this.localClusterName = ClusterName.CLUSTER_NAME_SETTING.get(settings).value(); @@ -180,6 +190,11 @@ public class CcrRepository extends AbstractLifecycleComponent implements Reposit @Override protected void doClose() {} + @Override + public ProjectId getProjectId() { + return projectId; + } + @Override public RepositoryMetadata getMetadata() { return metadata; diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowTaskReplicationTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowTaskReplicationTests.java index 34ce43e58308..d0bf7f041db8 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowTaskReplicationTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowTaskReplicationTests.java @@ -529,7 +529,7 @@ public class ShardFollowTaskReplicationTests extends ESIndexLevelReplicationTest ); primaryShard.markAsRecovering("remote recovery from leader", new RecoveryState(routing, localNode, null)); final PlainActionFuture future = new PlainActionFuture<>(); - primaryShard.restoreFromRepository(new RestoreOnlyRepository(index.getName()) { + primaryShard.restoreFromRepository(new RestoreOnlyRepository(randomProjectIdOrDefault(), index.getName()) { @Override public void restoreShard( Store store, diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowEngineIndexShardTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowEngineIndexShardTests.java index 7d5d5cb5a40a..c83da1d5864f 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowEngineIndexShardTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowEngineIndexShardTests.java @@ -142,7 +142,7 @@ public class FollowEngineIndexShardTests extends IndexShardTestCase { DiscoveryNode localNode = DiscoveryNodeUtils.builder("foo").roles(emptySet()).build(); target.markAsRecovering("store", new RecoveryState(routing, localNode, null)); final PlainActionFuture future = new PlainActionFuture<>(); - target.restoreFromRepository(new RestoreOnlyRepository("test") { + target.restoreFromRepository(new RestoreOnlyRepository(randomProjectIdOrDefault(), "test") { @Override public void restoreShard( Store store, diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/repository/CcrRepositoryRetentionLeaseTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/repository/CcrRepositoryRetentionLeaseTests.java index 98ceeb96db07..4c8169cace7f 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/repository/CcrRepositoryRetentionLeaseTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/repository/CcrRepositoryRetentionLeaseTests.java @@ -110,13 +110,17 @@ public class CcrRepositoryRetentionLeaseTests extends ESTestCase { final ThreadPool threadPool = mock(ThreadPool.class); final ThreadContext threadContext = new ThreadContext(Settings.EMPTY); when(threadPool.getThreadContext()).thenReturn(threadContext); - return new CcrRepository( + final var projectId = randomProjectIdOrDefault(); + final CcrRepository ccrRepository = new CcrRepository( + projectId, repositoryMetadata, mock(Client.class), Settings.EMPTY, new CcrSettings(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, settings)), threadPool ); + assertThat(ccrRepository.getProjectId(), equalTo(projectId)); + return ccrRepository; } public void testWhenRetentionLeaseExpiresBeforeWeCanRenewIt() { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotRepository.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotRepository.java index 082d35ef54a5..b969e9d2dc4b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotRepository.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotRepository.java @@ -20,6 +20,7 @@ import org.apache.lucene.store.NIOFSDirectory; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.common.Strings; @@ -253,19 +254,20 @@ public final class SourceOnlySnapshotRepository extends FilterRepository { return new Repository.Factory() { @Override - public Repository create(RepositoryMetadata metadata) { + public Repository create(ProjectId projectId, RepositoryMetadata metadata) { throw new UnsupportedOperationException(); } @Override - public Repository create(RepositoryMetadata metadata, Function typeLookup) throws Exception { + public Repository create(ProjectId projectId, RepositoryMetadata metadata, Function typeLookup) + throws Exception { String delegateType = DELEGATE_TYPE.get(metadata.settings()); if (Strings.hasLength(delegateType) == false) { throw new IllegalArgumentException(DELEGATE_TYPE.getKey() + " must be set"); } Repository.Factory factory = typeLookup.apply(delegateType); return new SourceOnlySnapshotRepository( - factory.create(new RepositoryMetadata(metadata.name(), delegateType, metadata.settings()), typeLookup) + factory.create(projectId, new RepositoryMetadata(metadata.name(), delegateType, metadata.settings()), typeLookup) ); } }; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotShardTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotShardTests.java index 9503376b6120..7f57e9b494aa 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotShardTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotShardTests.java @@ -28,6 +28,7 @@ import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeUtils; @@ -91,6 +92,7 @@ import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; import static org.elasticsearch.cluster.routing.TestShardRouting.shardRoutingBuilder; +import static org.hamcrest.Matchers.equalTo; public class SourceOnlySnapshotShardTests extends IndexShardTestCase { @@ -120,7 +122,9 @@ public class SourceOnlySnapshotShardTests extends IndexShardTestCase { } SnapshotId snapshotId = new SnapshotId("test", "test"); IndexId indexId = new IndexId(shard.shardId().getIndexName(), shard.shardId().getIndex().getUUID()); - SourceOnlySnapshotRepository repository = new SourceOnlySnapshotRepository(createRepository()); + final var projectId = randomProjectIdOrDefault(); + SourceOnlySnapshotRepository repository = new SourceOnlySnapshotRepository(createRepository(projectId)); + assertThat(repository.getProjectId(), equalTo(projectId)); repository.start(); try (Engine.IndexCommitRef snapshotRef = shard.acquireLastIndexCommit(true)) { IndexShardSnapshotStatus indexShardSnapshotStatus = IndexShardSnapshotStatus.newInitializing(new ShardGeneration(-1L)); @@ -169,7 +173,9 @@ public class SourceOnlySnapshotShardTests extends IndexShardTestCase { recoverShardFromStore(shard); SnapshotId snapshotId = new SnapshotId("test", "test"); IndexId indexId = new IndexId(shard.shardId().getIndexName(), shard.shardId().getIndex().getUUID()); - SourceOnlySnapshotRepository repository = new SourceOnlySnapshotRepository(createRepository()); + final var projectId = randomProjectIdOrDefault(); + SourceOnlySnapshotRepository repository = new SourceOnlySnapshotRepository(createRepository(projectId)); + assertThat(repository.getProjectId(), equalTo(projectId)); repository.start(); try (Engine.IndexCommitRef snapshotRef = shard.acquireLastIndexCommit(true)) { IndexShardSnapshotStatus indexShardSnapshotStatus = IndexShardSnapshotStatus.newInitializing(new ShardGeneration(-1L)); @@ -208,7 +214,9 @@ public class SourceOnlySnapshotShardTests extends IndexShardTestCase { } IndexId indexId = new IndexId(shard.shardId().getIndexName(), shard.shardId().getIndex().getUUID()); - SourceOnlySnapshotRepository repository = new SourceOnlySnapshotRepository(createRepository()); + final var projectId = randomProjectIdOrDefault(); + SourceOnlySnapshotRepository repository = new SourceOnlySnapshotRepository(createRepository(projectId)); + assertThat(repository.getProjectId(), equalTo(projectId)); repository.start(); int totalFileCount; ShardGeneration shardGeneration; @@ -335,7 +343,9 @@ public class SourceOnlySnapshotShardTests extends IndexShardTestCase { } SnapshotId snapshotId = new SnapshotId("test", "test"); IndexId indexId = new IndexId(shard.shardId().getIndexName(), shard.shardId().getIndex().getUUID()); - SourceOnlySnapshotRepository repository = new SourceOnlySnapshotRepository(createRepository()); + final var projectId = randomProjectIdOrDefault(); + SourceOnlySnapshotRepository repository = new SourceOnlySnapshotRepository(createRepository(projectId)); + assertThat(repository.getProjectId(), equalTo(projectId)); repository.start(); try (Engine.IndexCommitRef snapshotRef = shard.acquireLastIndexCommit(true)) { IndexShardSnapshotStatus indexShardSnapshotStatus = IndexShardSnapshotStatus.newInitializing(null); @@ -557,11 +567,12 @@ public class SourceOnlySnapshotShardTests extends IndexShardTestCase { } /** Create a {@link Repository} with a random name **/ - private Repository createRepository() { + private Repository createRepository(ProjectId projectId) { Settings settings = Settings.builder().put("location", randomAlphaOfLength(10)).build(); RepositoryMetadata repositoryMetadata = new RepositoryMetadata(randomAlphaOfLength(10), FsRepository.TYPE, settings); final ClusterService clusterService = BlobStoreTestUtil.mockClusterService(repositoryMetadata); final Repository repository = new FsRepository( + projectId, repositoryMetadata, createEnvironment(), xContentRegistry(), diff --git a/x-pack/plugin/old-lucene-versions/src/internalClusterTest/java/org/elasticsearch/xpack/lucene/bwc/AbstractArchiveTestCase.java b/x-pack/plugin/old-lucene-versions/src/internalClusterTest/java/org/elasticsearch/xpack/lucene/bwc/AbstractArchiveTestCase.java index a24c673c1aef..614036d3792c 100644 --- a/x-pack/plugin/old-lucene-versions/src/internalClusterTest/java/org/elasticsearch/xpack/lucene/bwc/AbstractArchiveTestCase.java +++ b/x-pack/plugin/old-lucene-versions/src/internalClusterTest/java/org/elasticsearch/xpack/lucene/bwc/AbstractArchiveTestCase.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.lucene.bwc; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; @@ -66,13 +67,22 @@ public abstract class AbstractArchiveTestCase extends AbstractSnapshotIntegTestC ) { return Map.of( FAKE_VERSIONS_TYPE, - metadata -> new FakeVersionsRepo(metadata, env, namedXContentRegistry, clusterService, bigArrays, recoverySettings) + (projectId, metadata) -> new FakeVersionsRepo( + projectId, + metadata, + env, + namedXContentRegistry, + clusterService, + bigArrays, + recoverySettings + ) ); } // fakes an old index version format to activate license checks private static class FakeVersionsRepo extends FsRepository { FakeVersionsRepo( + ProjectId projectId, RepositoryMetadata metadata, Environment env, NamedXContentRegistry namedXContentRegistry, @@ -80,7 +90,7 @@ public abstract class AbstractArchiveTestCase extends AbstractSnapshotIntegTestC BigArrays bigArrays, RecoverySettings recoverySettings ) { - super(metadata, env, namedXContentRegistry, clusterService, bigArrays, recoverySettings); + super(projectId, metadata, env, namedXContentRegistry, clusterService, bigArrays, recoverySettings); } @Override diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/allocation/SearchableSnapshotDiskThresholdIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/allocation/SearchableSnapshotDiskThresholdIntegTests.java index 6650f264b7b8..518ff2354f49 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/allocation/SearchableSnapshotDiskThresholdIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/allocation/SearchableSnapshotDiskThresholdIntegTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.cluster.ClusterInfoServiceUtils; import org.elasticsearch.cluster.DiskUsage; import org.elasticsearch.cluster.DiskUsageIntegTestCase; import org.elasticsearch.cluster.InternalClusterInfoService; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.routing.ShardRoutingState; @@ -380,7 +381,15 @@ public class SearchableSnapshotDiskThresholdIntegTests extends DiskUsageIntegTes ) { return Collections.singletonMap( TYPE, - metadata -> new CustomMockRepository(metadata, env, namedXContentRegistry, clusterService, bigArrays, recoverySettings) + (projectId, metadata) -> new CustomMockRepository( + projectId, + metadata, + env, + namedXContentRegistry, + clusterService, + bigArrays, + recoverySettings + ) ); } } @@ -390,6 +399,7 @@ public class SearchableSnapshotDiskThresholdIntegTests extends DiskUsageIntegTes private static final CountDownLatch RESTORE_SHARD_LATCH = new CountDownLatch(1); public CustomMockRepository( + ProjectId projectId, RepositoryMetadata metadata, Environment environment, NamedXContentRegistry namedXContentRegistry, @@ -397,7 +407,7 @@ public class SearchableSnapshotDiskThresholdIntegTests extends DiskUsageIntegTes BigArrays bigArrays, RecoverySettings recoverySettings ) { - super(metadata, environment, namedXContentRegistry, clusterService, bigArrays, recoverySettings); + super(projectId, metadata, environment, namedXContentRegistry, clusterService, bigArrays, recoverySettings); } private void unlockRestore() { diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/SearchableSnapshotsPrewarmingIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/SearchableSnapshotsPrewarmingIntegTests.java index c91133dae8a8..dcf77d06e323 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/SearchableSnapshotsPrewarmingIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/SearchableSnapshotsPrewarmingIntegTests.java @@ -454,7 +454,15 @@ public class SearchableSnapshotsPrewarmingIntegTests extends ESSingleNodeTestCas ) { return Collections.singletonMap( "tracking", - (metadata) -> new FsRepository(metadata, env, namedXContentRegistry, clusterService, bigArrays, recoverySettings) { + (projectId, metadata) -> new FsRepository( + projectId, + metadata, + env, + namedXContentRegistry, + clusterService, + bigArrays, + recoverySettings + ) { @Override protected BlobStore createBlobStore() throws Exception { diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/recovery/SearchableSnapshotRecoveryStateIntegrationTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/recovery/SearchableSnapshotRecoveryStateIntegrationTests.java index 1397ce64327c..e0674c0151f6 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/recovery/SearchableSnapshotRecoveryStateIntegrationTests.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/recovery/SearchableSnapshotRecoveryStateIntegrationTests.java @@ -248,7 +248,15 @@ public class SearchableSnapshotRecoveryStateIntegrationTests extends BaseSearcha ) { return Collections.singletonMap( "test-fs", - (metadata) -> new FsRepository(metadata, env, namedXContentRegistry, clusterService, bigArrays, recoverySettings) + (projectId, metadata) -> new FsRepository( + projectId, + metadata, + env, + namedXContentRegistry, + clusterService, + bigArrays, + recoverySettings + ) ); } } diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/SearchableSnapshotDirectoryTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/SearchableSnapshotDirectoryTests.java index 98df96eca777..c090c715cafd 100644 --- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/SearchableSnapshotDirectoryTests.java +++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/SearchableSnapshotDirectoryTests.java @@ -593,6 +593,7 @@ public class SearchableSnapshotDirectoryTests extends AbstractSearchableSnapshot ); final BlobStoreRepository repository = new FsRepository( + randomProjectIdOrDefault(), repositoryMetadata, new Environment( Settings.builder() diff --git a/x-pack/plugin/slm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SLMHealthBlockedSnapshotIT.java b/x-pack/plugin/slm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SLMHealthBlockedSnapshotIT.java index 08eb3f2140ed..219dd20f4e62 100644 --- a/x-pack/plugin/slm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SLMHealthBlockedSnapshotIT.java +++ b/x-pack/plugin/slm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SLMHealthBlockedSnapshotIT.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusRe import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.cluster.SnapshotsInProgress; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; @@ -119,15 +120,24 @@ public class SLMHealthBlockedSnapshotIT extends AbstractSnapshotIntegTestCase { ) { return Map.of( TestDelayedRepo.TYPE, - metadata -> new TestDelayedRepo(metadata, env, namedXContentRegistry, clusterService, bigArrays, recoverySettings, () -> { - if (doDelay.get()) { - try { - assertTrue(delayedRepoLatch.await(1, TimeUnit.MINUTES)); - } catch (InterruptedException e) { - throw new RuntimeException(e); + (projectId, metadata) -> new TestDelayedRepo( + projectId, + metadata, + env, + namedXContentRegistry, + clusterService, + bigArrays, + recoverySettings, + () -> { + if (doDelay.get()) { + try { + assertTrue(delayedRepoLatch.await(1, TimeUnit.MINUTES)); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } } } - }) + ) ); } } @@ -137,6 +147,7 @@ public class SLMHealthBlockedSnapshotIT extends AbstractSnapshotIntegTestCase { private final Runnable delayFn; protected TestDelayedRepo( + ProjectId projectId, RepositoryMetadata metadata, Environment env, NamedXContentRegistry namedXContentRegistry, @@ -145,7 +156,7 @@ public class SLMHealthBlockedSnapshotIT extends AbstractSnapshotIntegTestCase { RecoverySettings recoverySettings, Runnable delayFn ) { - super(metadata, env, namedXContentRegistry, clusterService, bigArrays, recoverySettings); + super(projectId, metadata, env, namedXContentRegistry, clusterService, bigArrays, recoverySettings); this.delayFn = delayFn; } diff --git a/x-pack/plugin/slm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SLMStatDisruptionIT.java b/x-pack/plugin/slm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SLMStatDisruptionIT.java index aec02e6dced3..d3eff9eb8585 100644 --- a/x-pack/plugin/slm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SLMStatDisruptionIT.java +++ b/x-pack/plugin/slm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SLMStatDisruptionIT.java @@ -19,6 +19,7 @@ import org.elasticsearch.cluster.coordination.Coordinator; import org.elasticsearch.cluster.coordination.FollowersChecker; import org.elasticsearch.cluster.coordination.LagDetector; import org.elasticsearch.cluster.coordination.LeaderChecker; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; @@ -137,16 +138,25 @@ public class SLMStatDisruptionIT extends AbstractSnapshotIntegTestCase { ) { return Map.of( TestDelayedRepo.TYPE, - metadata -> new TestDelayedRepo(metadata, env, namedXContentRegistry, clusterService, bigArrays, recoverySettings, () -> { - // Only delay the first request - if (doDelay.getAndSet(false)) { - try { - assertTrue(delayedRepoLatch.await(1, TimeUnit.MINUTES)); - } catch (InterruptedException e) { - throw new RuntimeException(e); + (projectId, metadata) -> new TestDelayedRepo( + projectId, + metadata, + env, + namedXContentRegistry, + clusterService, + bigArrays, + recoverySettings, + () -> { + // Only delay the first request + if (doDelay.getAndSet(false)) { + try { + assertTrue(delayedRepoLatch.await(1, TimeUnit.MINUTES)); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } } } - }) + ) ); } } @@ -156,6 +166,7 @@ public class SLMStatDisruptionIT extends AbstractSnapshotIntegTestCase { private final Runnable delayFn; protected TestDelayedRepo( + ProjectId projectId, RepositoryMetadata metadata, Environment env, NamedXContentRegistry namedXContentRegistry, @@ -164,7 +175,7 @@ public class SLMStatDisruptionIT extends AbstractSnapshotIntegTestCase { RecoverySettings recoverySettings, Runnable delayFn ) { - super(metadata, env, namedXContentRegistry, clusterService, bigArrays, recoverySettings); + super(projectId, metadata, env, namedXContentRegistry, clusterService, bigArrays, recoverySettings); this.delayFn = delayFn; } @@ -199,7 +210,8 @@ public class SLMStatDisruptionIT extends AbstractSnapshotIntegTestCase { ) { return Map.of( TestRestartBeforeListenersRepo.TYPE, - metadata -> new TestRestartBeforeListenersRepo( + (projectId, metadata) -> new TestRestartBeforeListenersRepo( + projectId, metadata, env, namedXContentRegistry, @@ -223,6 +235,7 @@ public class SLMStatDisruptionIT extends AbstractSnapshotIntegTestCase { private final Runnable beforeResponseRunnable; protected TestRestartBeforeListenersRepo( + ProjectId projectId, RepositoryMetadata metadata, Environment env, NamedXContentRegistry namedXContentRegistry, @@ -231,7 +244,7 @@ public class SLMStatDisruptionIT extends AbstractSnapshotIntegTestCase { RecoverySettings recoverySettings, Runnable beforeResponseRunnable ) { - super(metadata, env, namedXContentRegistry, clusterService, bigArrays, recoverySettings); + super(projectId, metadata, env, namedXContentRegistry, clusterService, bigArrays, recoverySettings); this.beforeResponseRunnable = beforeResponseRunnable; } diff --git a/x-pack/plugin/snapshot-based-recoveries/src/internalClusterTest/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/SnapshotBasedIndexRecoveryIT.java b/x-pack/plugin/snapshot-based-recoveries/src/internalClusterTest/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/SnapshotBasedIndexRecoveryIT.java index 7c596bff281e..6f73efb2fa09 100644 --- a/x-pack/plugin/snapshot-based-recoveries/src/internalClusterTest/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/SnapshotBasedIndexRecoveryIT.java +++ b/x-pack/plugin/snapshot-based-recoveries/src/internalClusterTest/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/SnapshotBasedIndexRecoveryIT.java @@ -20,6 +20,7 @@ import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; import org.elasticsearch.cluster.routing.RecoverySource; @@ -151,11 +152,35 @@ public class SnapshotBasedIndexRecoveryIT extends AbstractSnapshotIntegTestCase ) { return Map.of( FAULTY_TYPE, - metadata -> new FaultyRepository(metadata, env, namedXContentRegistry, clusterService, bigArrays, recoverySettings), + (projectId, metadata) -> new FaultyRepository( + projectId, + metadata, + env, + namedXContentRegistry, + clusterService, + bigArrays, + recoverySettings + ), INSTRUMENTED_TYPE, - metadata -> new InstrumentedRepo(metadata, env, namedXContentRegistry, clusterService, bigArrays, recoverySettings), + (projectId, metadata) -> new InstrumentedRepo( + projectId, + metadata, + env, + namedXContentRegistry, + clusterService, + bigArrays, + recoverySettings + ), FILTER_TYPE, - metadata -> new FilterFsRepository(metadata, env, namedXContentRegistry, clusterService, bigArrays, recoverySettings) + (projectId, metadata) -> new FilterFsRepository( + projectId, + metadata, + env, + namedXContentRegistry, + clusterService, + bigArrays, + recoverySettings + ) ); } } @@ -164,6 +189,7 @@ public class SnapshotBasedIndexRecoveryIT extends AbstractSnapshotIntegTestCase AtomicLong totalBytesRead = new AtomicLong(); public InstrumentedRepo( + ProjectId projectId, RepositoryMetadata metadata, Environment environment, NamedXContentRegistry namedXContentRegistry, @@ -171,7 +197,7 @@ public class SnapshotBasedIndexRecoveryIT extends AbstractSnapshotIntegTestCase BigArrays bigArrays, RecoverySettings recoverySettings ) { - super(metadata, environment, namedXContentRegistry, clusterService, bigArrays, recoverySettings); + super(projectId, metadata, environment, namedXContentRegistry, clusterService, bigArrays, recoverySettings); } @Override @@ -206,6 +232,7 @@ public class SnapshotBasedIndexRecoveryIT extends AbstractSnapshotIntegTestCase public static class FaultyRepository extends FsRepository { public FaultyRepository( + ProjectId projectId, RepositoryMetadata metadata, Environment environment, NamedXContentRegistry namedXContentRegistry, @@ -213,7 +240,7 @@ public class SnapshotBasedIndexRecoveryIT extends AbstractSnapshotIntegTestCase BigArrays bigArrays, RecoverySettings recoverySettings ) { - super(metadata, environment, namedXContentRegistry, clusterService, bigArrays, recoverySettings); + super(projectId, metadata, environment, namedXContentRegistry, clusterService, bigArrays, recoverySettings); } @Override @@ -261,6 +288,7 @@ public class SnapshotBasedIndexRecoveryIT extends AbstractSnapshotIntegTestCase static final AtomicReference> delegateSupplierRef = new AtomicReference<>(IDENTITY); public FilterFsRepository( + ProjectId projectId, RepositoryMetadata metadata, Environment environment, NamedXContentRegistry namedXContentRegistry, @@ -268,7 +296,7 @@ public class SnapshotBasedIndexRecoveryIT extends AbstractSnapshotIntegTestCase BigArrays bigArrays, RecoverySettings recoverySettings ) { - super(metadata, environment, namedXContentRegistry, clusterService, bigArrays, recoverySettings); + super(projectId, metadata, environment, namedXContentRegistry, clusterService, bigArrays, recoverySettings); } static void wrapReadBlobMethod(BiFunction delegate) { diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RepositoryAnalysisFailureIT.java b/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RepositoryAnalysisFailureIT.java index 7a539c8f5194..399a9eee0d75 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RepositoryAnalysisFailureIT.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RepositoryAnalysisFailureIT.java @@ -9,6 +9,7 @@ package org.elasticsearch.repositories.blobstore.testkit.analyze; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.ReferenceDocs; @@ -545,7 +546,8 @@ public class RepositoryAnalysisFailureIT extends AbstractSnapshotIntegTestCase { ) { return Map.of( DISRUPTABLE_REPO_TYPE, - metadata -> new DisruptableRepository( + (projectId, metadata) -> new DisruptableRepository( + projectId, metadata, namedXContentRegistry, clusterService, @@ -562,6 +564,7 @@ public class RepositoryAnalysisFailureIT extends AbstractSnapshotIntegTestCase { private final AtomicReference blobStoreRef = new AtomicReference<>(); DisruptableRepository( + ProjectId projectId, RepositoryMetadata metadata, NamedXContentRegistry namedXContentRegistry, ClusterService clusterService, @@ -569,7 +572,7 @@ public class RepositoryAnalysisFailureIT extends AbstractSnapshotIntegTestCase { RecoverySettings recoverySettings, BlobPath basePath ) { - super(metadata, namedXContentRegistry, clusterService, bigArrays, recoverySettings, basePath); + super(projectId, metadata, namedXContentRegistry, clusterService, bigArrays, recoverySettings, basePath); } void setBlobStore(BlobStore blobStore) { diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RepositoryAnalysisSuccessIT.java b/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RepositoryAnalysisSuccessIT.java index 2af5c009d4ea..a65716a63f6d 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RepositoryAnalysisSuccessIT.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RepositoryAnalysisSuccessIT.java @@ -8,6 +8,7 @@ package org.elasticsearch.repositories.blobstore.testkit.analyze; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; @@ -187,7 +188,8 @@ public class RepositoryAnalysisSuccessIT extends AbstractSnapshotIntegTestCase { ) { return Map.of( ASSERTING_REPO_TYPE, - metadata -> new AssertingRepository( + (projectId, metadata) -> new AssertingRepository( + projectId, metadata, namedXContentRegistry, clusterService, @@ -217,6 +219,7 @@ public class RepositoryAnalysisSuccessIT extends AbstractSnapshotIntegTestCase { private final AtomicReference blobStoreRef = new AtomicReference<>(); AssertingRepository( + ProjectId projectId, RepositoryMetadata metadata, NamedXContentRegistry namedXContentRegistry, ClusterService clusterService, @@ -224,7 +227,7 @@ public class RepositoryAnalysisSuccessIT extends AbstractSnapshotIntegTestCase { RecoverySettings recoverySettings, BlobPath basePath ) { - super(metadata, namedXContentRegistry, clusterService, bigArrays, recoverySettings, basePath); + super(projectId, metadata, namedXContentRegistry, clusterService, bigArrays, recoverySettings, basePath); } void setBlobStore(BlobStore blobStore) { diff --git a/x-pack/plugin/voting-only-node/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/votingonly/VotingOnlyNodePluginTests.java b/x-pack/plugin/voting-only-node/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/votingonly/VotingOnlyNodePluginTests.java index b1df355417ac..bff42d689fb6 100644 --- a/x-pack/plugin/voting-only-node/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/votingonly/VotingOnlyNodePluginTests.java +++ b/x-pack/plugin/voting-only-node/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/votingonly/VotingOnlyNodePluginTests.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.admin.cluster.repositories.verify.VerifyReposito import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeRole; @@ -277,7 +278,15 @@ public class VotingOnlyNodePluginTests extends ESIntegTestCase { ) { return Collections.singletonMap( "verifyaccess-fs", - (metadata) -> new AccessVerifyingRepo(metadata, env, namedXContentRegistry, clusterService, bigArrays, recoverySettings) + (projectId, metadata) -> new AccessVerifyingRepo( + projectId, + metadata, + env, + namedXContentRegistry, + clusterService, + bigArrays, + recoverySettings + ) ); } @@ -286,6 +295,7 @@ public class VotingOnlyNodePluginTests extends ESIntegTestCase { private final ClusterService clusterService; private AccessVerifyingRepo( + ProjectId projectId, RepositoryMetadata metadata, Environment environment, NamedXContentRegistry namedXContentRegistry, @@ -293,7 +303,7 @@ public class VotingOnlyNodePluginTests extends ESIntegTestCase { BigArrays bigArrays, RecoverySettings recoverySettings ) { - super(metadata, environment, namedXContentRegistry, clusterService, bigArrays, recoverySettings); + super(projectId, metadata, environment, namedXContentRegistry, clusterService, bigArrays, recoverySettings); this.clusterService = clusterService; }