Incorporate feedback from @shwetathareja

Signed-off-by: Michael Froh <[email protected]>

Author: msfroh

server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java

@@ -227,7 +227,7 @@ public MetadataCreateIndexService(
         // Task is onboarded for throttling, it will get retried from associated TransportClusterManagerNodeAction.
         createIndexTaskKey = clusterService.registerClusterManagerTask(CREATE_INDEX, true);
         Supplier<Version> minNodeVersionSupplier = () -> clusterService.state().nodes().getMinNodeVersion();
-        remoteStoreCustomMetadataResolver = RemoteStoreNodeAttribute.isRemoteClusterStateConfigured(settings)
+        remoteStoreCustomMetadataResolver = RemoteStoreNodeAttribute.isSegmentRepoConfigured(settings)
             ? new RemoteStoreCustomMetadataResolver(remoteStoreSettings, minNodeVersionSupplier, repositoriesServiceSupplier, settings)
             : null;
     }

server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java

@@ -503,6 +503,14 @@ public boolean isSearchNode() {
      * @return true if the node contains remote store node attributes, false otherwise
      */
     public boolean isRemoteStoreNode() {
+        return isClusterStateRepoConfigured(this.getAttributes()) && RemoteStoreNodeAttribute.isSegmentRepoConfigured(this.getAttributes());
+    }
+
+    /**
+     * Returns whether the node is a remote segment store node.
+     * @return true if the node contains remote segment store node attributes, false otherwise
+     */
+    public boolean isRemoteSegmentStoreNode() {
         return RemoteStoreNodeAttribute.isSegmentRepoConfigured(this.getAttributes());
     }
 

server/src/main/java/org/opensearch/index/IndexService.java

@@ -673,7 +673,7 @@ public synchronized IndexShard createShard(
             Store remoteStore = null;
             Directory remoteDirectory = null;
             boolean seedRemote = false;
-            if (targetNode.isRemoteStoreNode()) {
+            if (targetNode.isRemoteSegmentStoreNode()) {
                 if (this.indexSettings.isRemoteStoreEnabled()) {
                     remoteDirectory = remoteDirectoryFactory.newDirectory(this.indexSettings, path);
                 } else {

server/src/main/java/org/opensearch/index/shard/IndexShard.java

@@ -587,7 +587,7 @@ public boolean shouldSeedRemoteStore() {
     public Function<String, Boolean> isShardOnRemoteEnabledNode = nodeId -> {
         DiscoveryNode node = discoveryNodes.get(nodeId);
         if (node != null) {
-            return node.isRemoteStoreNode();
+            return node.isRemoteSegmentStoreNode();
         }
         return false;
     };

server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java

@@ -81,7 +81,7 @@ public class RemoteStoreNodeAttribute {
         REMOTE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEYS
     );
 
-    public static final String REMOTE_STORE_SEGMENTS_ONLY_ATTRIBUTE_KEY = "remote_store.segments_only";
+    public static final String REMOTE_STORE_MODE_KEY = "remote_store.mode";
 
     /**
      * Creates a new {@link RemoteStoreNodeAttribute}
@@ -192,11 +192,23 @@ private static Tuple<String, String> getValue(Map<String, String> attributes, Li
         return null;
     }
 
+    private enum RemoteStoreMode {
+        SEGMENTS_ONLY,
+        DEFAULT
+    }
+
     private Map<String, String> getValidatedRepositoryNames(DiscoveryNode node) {
         Set<Tuple<String, String>> repositoryNames = new HashSet<>();
-        if (containsKey(node.getAttributes(), REMOTE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEYS)
-            && containsKey(node.getAttributes(), List.of(REMOTE_STORE_SEGMENTS_ONLY_ATTRIBUTE_KEY))
-            && "true".equals(node.getAttributes().get(REMOTE_STORE_SEGMENTS_ONLY_ATTRIBUTE_KEY).toLowerCase(Locale.ROOT))) {
+        RemoteStoreMode remoteStoreMode = RemoteStoreMode.DEFAULT;
+        if (containsKey(node.getAttributes(), List.of(REMOTE_STORE_MODE_KEY))) {
+            String mode = node.getAttributes().get(REMOTE_STORE_MODE_KEY);
+            if (mode != null && mode.equalsIgnoreCase(RemoteStoreMode.SEGMENTS_ONLY.name())) {
+                remoteStoreMode = RemoteStoreMode.SEGMENTS_ONLY;
+            } else if (mode != null && mode.equalsIgnoreCase(RemoteStoreMode.DEFAULT.name()) == false) {
+                throw new IllegalStateException("Unknown remote store mode [" + mode + "] for node [" + node + "]");
+            }
+        }
+        if (remoteStoreMode == RemoteStoreMode.SEGMENTS_ONLY) {
             repositoryNames.add(validateAttributeNonNull(node, REMOTE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEYS));
         } else if (containsKey(node.getAttributes(), REMOTE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEYS)
             || containsKey(node.getAttributes(), REMOTE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEYS)) {
@@ -207,6 +219,15 @@ && containsKey(node.getAttributes(), List.of(REMOTE_STORE_SEGMENTS_ONLY_ATTRIBUT
                 repositoryNames.add(validateAttributeNonNull(node, REMOTE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEYS));
             }
         if (containsKey(node.getAttributes(), REMOTE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEYS)) {
+            if (remoteStoreMode == RemoteStoreMode.SEGMENTS_ONLY) {
+                throw new IllegalStateException(
+                    "Cannot set "
+                        + REMOTE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEYS
+                        + " attributes when remote store mode is set to segments only for node ["
+                        + node
+                        + "]"
+                );
+            }
             repositoryNames.add(validateAttributeNonNull(node, REMOTE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEYS));
         }