Use reassign node api in state storage self heal (#21557)

Author: Evgenik2

ydb/core/blobstorage/nodewarden/distconf_invoke_state_storage.cpp

@@ -112,8 +112,13 @@ namespace NKikimr::NStorage {
             Self->Send(new IEventHandle(TEvents::TSystem::Poison, 0, Self->StateStorageSelfHealActor.value(), Self->SelfId(), nullptr, 0));
             Self->StateStorageSelfHealActor.reset();
         }
-
-        auto needReconfig = [&](auto name, auto clearFunc, auto ssMutableFunc, auto buildFunc) {
+        enum ReconfigType {
+            NONE,
+            ONE_NODE,
+            FULL
+        };
+        std::unordered_map<ui32, ui32> nodesToReplace;
+        auto needReconfig = [&](auto clearFunc, auto ssMutableFunc, auto buildFunc) {
             auto copyCurrentConfig = currentConfig;
             auto ss = *(copyCurrentConfig.*ssMutableFunc)();
             if (ss.RingGroupsSize() == 0) {
@@ -127,26 +132,127 @@ namespace NKikimr::NStorage {
             TIntrusivePtr<TStateStorageInfo> oldSSInfo;
             oldSSInfo = (*buildFunc)(ss);
             newSSInfo = (*buildFunc)(*(targetConfig.*ssMutableFunc)());
-            STLOG(PRI_DEBUG, BS_NODE, NW52, "Need to reconfig " << name << " " << (oldSSInfo->RingGroups != newSSInfo->RingGroups));
             if (oldSSInfo->RingGroups == newSSInfo->RingGroups) {
                 (targetConfig.*clearFunc)();
-                return false;
+                return ReconfigType::NONE;
             }
 
-            return true;
+            if (oldSSInfo->RingGroups.size() != newSSInfo->RingGroups.size()) {
+                return ReconfigType::FULL;
+            }
+
+            bool hasBadNodes = false;
+            for (ui32 ringGroupIdx : xrange(oldSSInfo->RingGroups.size())) {
+                auto& oldRg = oldSSInfo->RingGroups[ringGroupIdx];
+                auto& newRg = newSSInfo->RingGroups[ringGroupIdx];
+                if (oldRg.NToSelect != newRg.NToSelect || oldRg.Rings.size() != newRg.Rings.size()) {
+                    return ReconfigType::FULL;
+                }
+                for (ui32 j : xrange(oldRg.Rings.size())) {
+                    auto& oldRing = oldRg.Rings[j];
+                    auto& newRing = newRg.Rings[j];
+                    if (oldRing.IsDisabled != newRing.IsDisabled
+                        || oldRing.UseRingSpecificNodeSelection != newRing.UseRingSpecificNodeSelection
+                        || oldRing.Replicas.size() != newRing.Replicas.size()) {
+                        return ReconfigType::FULL;
+                    }
+                    for (auto& actorId : oldRing.Replicas) {
+                        if (!Self->SelfHealNodesState.contains(actorId.NodeId()) || Self->SelfHealNodesState.at(actorId.NodeId()) > 0) {
+                            hasBadNodes = true;
+                        }
+                    }
+                }
+            }
+            if (!hasBadNodes) {
+                return ReconfigType::NONE; // Current config is optimal and all nodes are good
+            }
+
+            // Check can be node replacement applyed
+            for (ui32 ringGroupIdx : xrange(oldSSInfo->RingGroups.size())) {
+                auto& oldRg = oldSSInfo->RingGroups[ringGroupIdx];
+                auto& newRg = newSSInfo->RingGroups[ringGroupIdx];
+
+                // Find not changed rings and place them on previous position
+                auto equalRingsByNodes = [](auto& ring1, auto& ring2) {
+                    if (ring1.Replicas.size() != ring2.Replicas.size()) {
+                        return false;
+                    }
+                    for(ui32 replicaPos : xrange(ring1.Replicas.size())) {
+                        if (ring1.Replicas[replicaPos].NodeId() != ring2.Replicas[replicaPos].NodeId()) {
+                            return false;
+                        }
+                    }
+                    return true;
+                };
+                for (ui32 oldRingIdx : xrange(oldRg.Rings.size())) {
+                    for (ui32 newRingIdx : xrange(newRg.Rings.size())) {
+                        if (newRingIdx != oldRingIdx && equalRingsByNodes(oldRg.Rings[oldRingIdx], newRg.Rings[newRingIdx])) {
+                            std::swap(newRg.Rings[newRingIdx], newRg.Rings[oldRingIdx]);
+                            break;
+                        }
+                    }
+                }
+                for (ui32 j : xrange(oldRg.Rings.size())) {
+                    auto& oldRing = oldRg.Rings[j];
+                    auto& newRing = newRg.Rings[j];
+                    if (oldRing == newRing) {
+                        continue;
+                    }
+                    // Place replicas in ring on previous position
+                    for (ui32 oldReplicaPos : xrange(oldRing.Replicas.size())) {
+                        for (ui32 newReplicaPos : xrange(newRing.Replicas.size())) {
+                            if (newReplicaPos != oldReplicaPos && oldRing.Replicas[oldReplicaPos].NodeId() == newRing.Replicas[newReplicaPos].NodeId()) {
+                                std::swap(newRing.Replicas[newReplicaPos], newRing.Replicas[oldReplicaPos]);
+                                break;
+                            }
+                        }
+                    }
+
+                    for (ui32 k : xrange(oldRing.Replicas.size())) {
+                        auto oldRep = oldRing.Replicas[k].NodeId();
+                        auto newRep = newRing.Replicas[k].NodeId();
+                        if (oldRep == newRep) {
+                            continue;
+                        }
+                        if (auto it = nodesToReplace.find(oldRep); it != nodesToReplace.end() && it->second != newRep) {
+                            return ReconfigType::FULL;
+                        }
+                        nodesToReplace[oldRep] = newRep;
+                    }
+                }
+            }
+            if (nodesToReplace.size() == 1) {
+                return ReconfigType::ONE_NODE;
+            }
+            return nodesToReplace.empty() ? ReconfigType::NONE : ReconfigType::FULL;
         };
-        #define NEED_RECONFIG(NAME) needReconfig(#NAME, &NKikimrBlobStorage::TStateStorageConfig::Clear##NAME##Config, &NKikimrBlobStorage::TStateStorageConfig::Mutable##NAME##Config, &NKikimr::Build##NAME##Info)
+        #define NEED_RECONFIG(NAME) needReconfig(&NKikimrBlobStorage::TStateStorageConfig::Clear##NAME##Config, &NKikimrBlobStorage::TStateStorageConfig::Mutable##NAME##Config, &NKikimr::Build##NAME##Info)
         auto needReconfigSS = NEED_RECONFIG(StateStorage);
         auto needReconfigSSB = NEED_RECONFIG(StateStorageBoard);
         auto needReconfigSB = NEED_RECONFIG(SchemeBoard);
+        #undef NEED_RECONFIG
 
-        if (!needReconfigSS && !needReconfigSSB && !needReconfigSB) {
+        if (needReconfigSS == ReconfigType::NONE && needReconfigSSB == ReconfigType::NONE && needReconfigSB == ReconfigType::NONE) {
             throw TExError() << "Current configuration is recommended. Nothing to self-heal.";
         }
-        #undef NEED_RECONFIG
+        if (nodesToReplace.size() == 1 && needReconfigSS != ReconfigType::FULL && needReconfigSSB != ReconfigType::FULL && needReconfigSB != ReconfigType::FULL) {
+            STLOG(PRI_DEBUG, BS_NODE, NW52, "Need to reconfig one node " << nodesToReplace.begin()->first << " to " << nodesToReplace.begin()->second
+                , (CurrentConfig, currentConfig), (TargetConfig, targetConfig));
+
+            TQuery::TReassignStateStorageNode cmd;
+            cmd.SetFrom(nodesToReplace.begin()->first);
+            cmd.SetTo(nodesToReplace.begin()->second);
+            cmd.SetStateStorage(needReconfigSS == ReconfigType::ONE_NODE);
+            cmd.SetStateStorageBoard(needReconfigSSB == ReconfigType::ONE_NODE);
+            cmd.SetSchemeBoard(needReconfigSB == ReconfigType::ONE_NODE);
+            ReassignStateStorageNode(cmd);
+            return;
+        }
 
         AdjustRingGroupActorIdOffsetInRecommendedStateStorageConfig(&targetConfig);
 
+        STLOG(PRI_DEBUG, BS_NODE, NW52, "Need to reconfig, starting StateStorageSelfHealActor", (CurrentConfig, currentConfig), (TargetConfig, targetConfig));
+
         Self->StateStorageSelfHealActor = Register(new TStateStorageSelfhealActor(Sender, Cookie,
             TDuration::Seconds(waitForConfigStep), std::move(currentConfig), std::move(targetConfig)));
         auto ev = PrepareResult(TResult::OK, std::nullopt);

ydb/tests/functional/config/test_distconf_sentinel_node_status.py

@@ -143,12 +143,20 @@ def do_test(self, configName):
         assert_eq(len(rg["Ring"]), 9)
         self.validate_contains_nodes(rg, [3])
         self.cluster.nodes[3].stop()
-        time.sleep(25)
+        for i in range(15):
+            time.sleep(2)
+            cfg = self.do_request_config()[f"{configName}Config"]
+            assert_eq(len(cfg["RingGroups"]), 1)
+
         rg2 = get_ring_group(self.do_request_config(), configName)
         assert_eq(rg["NToSelect"], 9)
         assert_eq(len(rg["Ring"]), 9)
         self.validate_not_contains_nodes(rg2, [3])
         assert_that(rg != rg2)
+        self.cluster.nodes[3].start()
+        time.sleep(25)
+        rg3 = get_ring_group(self.do_request_config(), configName)
+        assert_that(rg3 == rg2)  # Current config has no bad nodes and should not run self-heal
 
 
 class TestKiKiMRDistConfSelfHeal2NodesDisconnected(KiKiMRDistConfNodeStatusTest):