Skip to content

Commit a9b6d7a

Browse files
msfrohmsfroh
authored
Add option to enable remote store for segments only (#18773)
Currently, the remote store implementation is all or nothing. If you want anything stored in the remote store, you pretty much need to store everything in the remote store. This change adds an explicit setting so expert users can say, "No, thanks, I don't want any of this remote cluster state, remote translog stuff. I just want segments replicated to a remote store." --------- Signed-off-by: Michael Froh <[email protected]>
1 parent e22b412 commit a9b6d7a

File tree

12 files changed

+81
-32
lines changed

12 files changed

+81
-32
lines changed

‎server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java‎

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -227,9 +227,10 @@ public MetadataCreateIndexService(
227227
// Task is onboarded for throttling, it will get retried from associated TransportClusterManagerNodeAction.
228228
createIndexTaskKey = clusterService.registerClusterManagerTask(CREATE_INDEX, true);
229229
Supplier<Version> minNodeVersionSupplier = () -> clusterService.state().nodes().getMinNodeVersion();
230-
remoteStoreCustomMetadataResolver = isRemoteDataAttributePresent(settings)
231-
? new RemoteStoreCustomMetadataResolver(remoteStoreSettings, minNodeVersionSupplier, repositoriesServiceSupplier, settings)
232-
: null;
230+
remoteStoreCustomMetadataResolver = RemoteStoreNodeAttribute.isSegmentRepoConfigured(settings)
231+
&& RemoteStoreNodeAttribute.isTranslogRepoConfigured(settings)
232+
? new RemoteStoreCustomMetadataResolver(remoteStoreSettings, minNodeVersionSupplier, repositoriesServiceSupplier, settings)
233+
: null;
233234
}
234235

235236
public IndexScopedSettings getIndexScopedSettings() {
@@ -1178,10 +1179,19 @@ public static void updateRemoteStoreSettings(
11781179
if (remoteNode.isPresent()) {
11791180
translogRepo = RemoteStoreNodeAttribute.getTranslogRepoName(remoteNode.get().getAttributes());
11801181
segmentRepo = RemoteStoreNodeAttribute.getSegmentRepoName(remoteNode.get().getAttributes());
1181-
if (segmentRepo != null && translogRepo != null) {
1182-
settingsBuilder.put(SETTING_REMOTE_STORE_ENABLED, true)
1183-
.put(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, segmentRepo)
1184-
.put(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, translogRepo);
1182+
if (segmentRepo != null) {
1183+
settingsBuilder.put(SETTING_REMOTE_STORE_ENABLED, true).put(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, segmentRepo);
1184+
if (translogRepo != null) {
1185+
settingsBuilder.put(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, translogRepo);
1186+
} else if (isMigratingToRemoteStore(clusterSettings)) {
1187+
ValidationException validationException = new ValidationException();
1188+
validationException.addValidationErrors(
1189+
Collections.singletonList(
1190+
"Cluster is migrating to remote store but remote translog is not configured, failing index creation"
1191+
)
1192+
);
1193+
throw new IndexCreationException(indexName, validationException);
1194+
}
11851195
} else {
11861196
ValidationException validationException = new ValidationException();
11871197
validationException.addValidationErrors(

‎server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -506,6 +506,14 @@ public boolean isRemoteStoreNode() {
506506
return isClusterStateRepoConfigured(this.getAttributes()) && RemoteStoreNodeAttribute.isSegmentRepoConfigured(this.getAttributes());
507507
}
508508

509+
/**
510+
* Returns whether the node is a remote segment store node.
511+
* @return true if the node contains remote segment store node attributes, false otherwise
512+
*/
513+
public boolean isRemoteSegmentStoreNode() {
514+
return RemoteStoreNodeAttribute.isSegmentRepoConfigured(this.getAttributes());
515+
}
516+
509517
/**
510518
* Returns whether settings required for remote cluster state publication is configured
511519
* @return true if the node contains remote cluster state node attribute and remote routing table node attribute

‎server/src/main/java/org/opensearch/index/IndexService.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -679,7 +679,7 @@ public synchronized IndexShard createShard(
679679
Store remoteStore = null;
680680
Directory remoteDirectory = null;
681681
boolean seedRemote = false;
682-
if (targetNode.isRemoteStoreNode()) {
682+
if (targetNode.isRemoteSegmentStoreNode()) {
683683
if (this.indexSettings.isRemoteStoreEnabled()) {
684684
remoteDirectory = remoteDirectoryFactory.newDirectory(this.indexSettings, path);
685685
} else {

‎server/src/main/java/org/opensearch/index/IndexSettings.java‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1412,9 +1412,7 @@ public boolean isAssignedOnRemoteNode() {
14121412
* Returns if remote translog store is enabled for this index.
14131413
*/
14141414
public boolean isRemoteTranslogStoreEnabled() {
1415-
// Today enabling remote store automatically enables remote translog as well.
1416-
// which is why isRemoteStoreEnabled is used to represent isRemoteTranslogStoreEnabled
1417-
return isRemoteStoreEnabled;
1415+
return remoteStoreTranslogRepository != null && remoteStoreTranslogRepository.isEmpty() == false;
14181416
}
14191417

14201418
/**

‎server/src/main/java/org/opensearch/index/shard/IndexShard.java‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2821,14 +2821,16 @@ private void innerOpenEngineAndTranslog(LongSupplier globalCheckpointSupplier, b
28212821
syncSegmentsFromRemoteSegmentStore(false);
28222822
}
28232823
if (shardRouting.primary()) {
2824-
if (syncFromRemote) {
2825-
syncRemoteTranslogAndUpdateGlobalCheckpoint();
2826-
} else if (isSnapshotV2Restore() == false) {
2827-
// we will enter this block when we do not want to recover from remote translog.
2828-
// currently only during snapshot restore, we are coming into this block.
2829-
// here, as while initiliazing remote translog we cannot skip downloading translog files,
2830-
// so before that step, we are deleting the translog files present in remote store.
2831-
deleteTranslogFilesFromRemoteTranslog();
2824+
if (indexSettings.isRemoteTranslogStoreEnabled()) {
2825+
if (syncFromRemote) {
2826+
syncRemoteTranslogAndUpdateGlobalCheckpoint();
2827+
} else if (isSnapshotV2Restore() == false) {
2828+
// we will enter this block when we do not want to recover from remote translog.
2829+
// currently only during snapshot restore, we are coming into this block.
2830+
// here, as while initiliazing remote translog we cannot skip downloading translog files,
2831+
// so before that step, we are deleting the translog files present in remote store.
2832+
deleteTranslogFilesFromRemoteTranslog();
2833+
}
28322834
}
28332835
} else if (syncFromRemote) {
28342836
// For replicas, when we download segments from remote segment store, we need to make sure that local

‎server/src/main/java/org/opensearch/indices/IndicesService.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -217,7 +217,6 @@
217217
import static org.opensearch.index.TieredMergePolicyProvider.MIN_DEFAULT_MAX_MERGE_AT_ONCE;
218218
import static org.opensearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder;
219219
import static org.opensearch.indices.IndicesRequestCache.INDICES_REQUEST_CACHE_MAX_SIZE_ALLOWED_IN_CACHE_SETTING;
220-
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteDataAttributePresent;
221220
import static org.opensearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES;
222221

223222
/**
@@ -707,7 +706,7 @@ private static BiFunction<IndexSettings, ShardRouting, TranslogFactory> getTrans
707706
remoteStoreStatsTrackerFactory.getRemoteTranslogTransferTracker(shardRouting.shardId()),
708707
remoteStoreSettings
709708
);
710-
} else if (isRemoteDataAttributePresent(settings) && shardRouting.primary()) {
709+
} else if (RemoteStoreNodeAttribute.isTranslogRepoConfigured(settings) && shardRouting.primary()) {
711710
return new RemoteBlobStoreInternalTranslogFactory(
712711
repositoriesServiceSupplier,
713712
threadPool,

‎server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java‎

Lines changed: 34 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,8 @@ public class RemoteStoreNodeAttribute {
8181
REMOTE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEYS
8282
);
8383

84+
public static final String REMOTE_STORE_MODE_KEY = "remote_store.mode";
85+
8486
/**
8587
* Creates a new {@link RemoteStoreNodeAttribute}
8688
*/
@@ -190,17 +192,42 @@ private static Tuple<String, String> getValue(Map<String, String> attributes, Li
190192
return null;
191193
}
192194

195+
private enum RemoteStoreMode {
196+
SEGMENTS_ONLY,
197+
DEFAULT
198+
}
199+
193200
private Map<String, String> getValidatedRepositoryNames(DiscoveryNode node) {
194201
Set<Tuple<String, String>> repositoryNames = new HashSet<>();
195-
if (containsKey(node.getAttributes(), REMOTE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEYS)
196-
|| containsKey(node.getAttributes(), REMOTE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEYS)) {
197-
repositoryNames.add(validateAttributeNonNull(node, REMOTE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEYS));
198-
repositoryNames.add(validateAttributeNonNull(node, REMOTE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEYS));
199-
repositoryNames.add(validateAttributeNonNull(node, REMOTE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEYS));
200-
} else if (containsKey(node.getAttributes(), REMOTE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEYS)) {
201-
repositoryNames.add(validateAttributeNonNull(node, REMOTE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEYS));
202+
RemoteStoreMode remoteStoreMode = RemoteStoreMode.DEFAULT;
203+
if (containsKey(node.getAttributes(), List.of(REMOTE_STORE_MODE_KEY))) {
204+
String mode = node.getAttributes().get(REMOTE_STORE_MODE_KEY);
205+
if (mode != null && mode.equalsIgnoreCase(RemoteStoreMode.SEGMENTS_ONLY.name())) {
206+
remoteStoreMode = RemoteStoreMode.SEGMENTS_ONLY;
207+
} else if (mode != null && mode.equalsIgnoreCase(RemoteStoreMode.DEFAULT.name()) == false) {
208+
throw new IllegalStateException("Unknown remote store mode [" + mode + "] for node [" + node + "]");
209+
}
202210
}
211+
if (remoteStoreMode == RemoteStoreMode.SEGMENTS_ONLY) {
212+
repositoryNames.add(validateAttributeNonNull(node, REMOTE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEYS));
213+
} else if (containsKey(node.getAttributes(), REMOTE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEYS)
214+
|| containsKey(node.getAttributes(), REMOTE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEYS)) {
215+
repositoryNames.add(validateAttributeNonNull(node, REMOTE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEYS));
216+
repositoryNames.add(validateAttributeNonNull(node, REMOTE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEYS));
217+
repositoryNames.add(validateAttributeNonNull(node, REMOTE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEYS));
218+
} else if (containsKey(node.getAttributes(), REMOTE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEYS)) {
219+
repositoryNames.add(validateAttributeNonNull(node, REMOTE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEYS));
220+
}
203221
if (containsKey(node.getAttributes(), REMOTE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEYS)) {
222+
if (remoteStoreMode == RemoteStoreMode.SEGMENTS_ONLY) {
223+
throw new IllegalStateException(
224+
"Cannot set "
225+
+ REMOTE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEYS
226+
+ " attributes when remote store mode is set to segments only for node ["
227+
+ node
228+
+ "]"
229+
);
230+
}
204231
repositoryNames.add(validateAttributeNonNull(node, REMOTE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEYS));
205232
}
206233

‎server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -662,7 +662,7 @@ public void testJoinRemotePubClusterWithRemoteStoreNodes() {
662662
IllegalStateException.class,
663663
() -> JoinTaskExecutor.ensureNodesCompatibility(joiningNode, currentState.getNodes(), currentState.metadata())
664664
);
665-
assertTrue(e.getMessage().equals("a remote store node [" + joiningNode + "] is trying to join a non remote store cluster"));
665+
assertEquals("a remote store node [" + joiningNode + "] is trying to join a non remote store cluster", e.getMessage());
666666
}
667667

668668
public void testPreventJoinRemotePublicationClusterWithIncompatibleAttributes() {
@@ -719,7 +719,7 @@ public void testPreventJoinClusterWithRemoteStateNodeJoiningRemoteStoreCluster()
719719
IllegalStateException.class,
720720
() -> JoinTaskExecutor.ensureNodesCompatibility(joiningNode, currentState.getNodes(), currentState.metadata())
721721
);
722-
assertTrue(e.getMessage().equals("a non remote store node [" + joiningNode + "] is trying to join a remote store cluster"));
722+
assertEquals("a non remote store node [" + joiningNode + "] is trying to join a remote store cluster", e.getMessage());
723723
}
724724

725725
public void testPreventJoinClusterWithRemoteStoreNodeJoiningRemoteStateCluster() {
@@ -739,7 +739,7 @@ public void testPreventJoinClusterWithRemoteStoreNodeJoiningRemoteStateCluster()
739739
IllegalStateException.class,
740740
() -> JoinTaskExecutor.ensureNodesCompatibility(joiningNode, currentState.getNodes(), currentState.metadata())
741741
);
742-
assertTrue(e.getMessage().equals("a remote store node [" + joiningNode + "] is trying to join a non remote store cluster"));
742+
assertEquals("a remote store node [" + joiningNode + "] is trying to join a non remote store cluster", e.getMessage());
743743
}
744744

745745
public void testUpdatesClusterStateWithSingleNodeCluster() throws Exception {

‎server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1702,7 +1702,7 @@ public void testNewIndexIsRemoteStoreBackedForRemoteStoreDirectionAndMixedMode()
17021702
assertEquals(error.getMessage(), "failed to create index [test-index]");
17031703
assertThat(
17041704
error.getCause().getMessage(),
1705-
containsString("Cluster is migrating to remote store but no remote node found, failing index creation")
1705+
containsString("Cluster is migrating to remote store but remote translog is not configured, failing index creation")
17061706
);
17071707
}
17081708

@@ -1771,6 +1771,8 @@ private IndexMetadata testRemoteCustomData(boolean remoteStoreEnabled, PathType
17711771
Settings.Builder settingsBuilder = Settings.builder();
17721772
if (remoteStoreEnabled) {
17731773
settingsBuilder.put(NODE_ATTRIBUTES.getKey() + REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, "test");
1774+
settingsBuilder.put(NODE_ATTRIBUTES.getKey() + REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, "test");
1775+
settingsBuilder.put(NODE_ATTRIBUTES.getKey() + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, "test");
17741776
}
17751777
settingsBuilder.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.getKey(), pathType.toString());
17761778
Settings settings = settingsBuilder.build();

‎server/src/test/java/org/opensearch/index/seqno/ReplicationTrackerTests.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1570,6 +1570,7 @@ public void testInSyncIdsAreIgnoredIfNotValidatedByClusterManagerWithRemoteTrans
15701570
Settings settings = Settings.builder()
15711571
.put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT)
15721572
.put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, "true")
1573+
.put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "translog-repo")
15731574
.build();
15741575
final ReplicationTracker tracker = newTracker(primaryId, settings);
15751576
tracker.updateFromClusterManager(randomNonNegativeLong(), ids(active.keySet()), routingTable(initializing.keySet(), primaryId));

0 commit comments

Comments
 (0)